@based/db 0.0.70 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (263) hide show
  1. package/LICENSE +201 -0
  2. package/README.md +105 -77
  3. package/dist/lib/darwin_aarch64/include/libdeflate.h +23 -0
  4. package/dist/lib/darwin_aarch64/include/selva/db.h +0 -41
  5. package/dist/lib/darwin_aarch64/include/selva/fast_linear_search.h +6 -2
  6. package/dist/lib/darwin_aarch64/include/selva/fields.h +52 -37
  7. package/dist/lib/darwin_aarch64/include/selva/gmtime.h +137 -0
  8. package/dist/lib/darwin_aarch64/include/selva/hll.h +1 -1
  9. package/dist/lib/darwin_aarch64/include/selva/mblen.h +40 -0
  10. package/dist/lib/darwin_aarch64/include/selva/selva_string.h +8 -5
  11. package/dist/lib/darwin_aarch64/include/selva/thread.h +37 -0
  12. package/dist/lib/darwin_aarch64/include/selva/types.h +18 -4
  13. package/dist/lib/darwin_aarch64/libdeflate.dylib +0 -0
  14. package/dist/lib/darwin_aarch64/libjemalloc_selva.2.dylib +0 -0
  15. package/dist/lib/darwin_aarch64/libnode-v22.node +0 -0
  16. package/dist/lib/darwin_aarch64/libnode-v23.node +0 -0
  17. package/dist/lib/darwin_aarch64/libnode-v24.node +0 -0
  18. package/dist/lib/darwin_aarch64/libselva.dylib +0 -0
  19. package/dist/lib/darwin_aarch64/libxxhash.dylib +0 -0
  20. package/dist/lib/linux_aarch64/include/libdeflate.h +23 -0
  21. package/dist/lib/linux_aarch64/include/selva/db.h +0 -41
  22. package/dist/lib/linux_aarch64/include/selva/fast_linear_search.h +6 -2
  23. package/dist/lib/linux_aarch64/include/selva/fields.h +52 -37
  24. package/dist/lib/linux_aarch64/include/selva/gmtime.h +137 -0
  25. package/dist/lib/linux_aarch64/include/selva/hll.h +1 -1
  26. package/dist/lib/linux_aarch64/include/selva/mblen.h +40 -0
  27. package/dist/lib/linux_aarch64/include/selva/selva_string.h +8 -5
  28. package/dist/lib/linux_aarch64/include/selva/thread.h +37 -0
  29. package/dist/lib/linux_aarch64/include/selva/types.h +18 -4
  30. package/dist/lib/linux_aarch64/libdeflate.so +0 -0
  31. package/dist/lib/linux_aarch64/libnode-v22.node +0 -0
  32. package/dist/lib/linux_aarch64/libnode-v23.node +0 -0
  33. package/dist/lib/linux_aarch64/libnode-v24.node +0 -0
  34. package/dist/lib/linux_aarch64/libselva.so +0 -0
  35. package/dist/lib/linux_x86_64/include/libdeflate.h +23 -0
  36. package/dist/lib/linux_x86_64/include/selva/db.h +0 -41
  37. package/dist/lib/linux_x86_64/include/selva/fast_linear_search.h +6 -2
  38. package/dist/lib/linux_x86_64/include/selva/fields.h +52 -37
  39. package/dist/lib/linux_x86_64/include/selva/gmtime.h +137 -0
  40. package/dist/lib/linux_x86_64/include/selva/hll.h +1 -1
  41. package/dist/lib/linux_x86_64/include/selva/mblen.h +40 -0
  42. package/dist/lib/linux_x86_64/include/selva/selva_string.h +8 -5
  43. package/dist/lib/linux_x86_64/include/selva/thread.h +37 -0
  44. package/dist/lib/linux_x86_64/include/selva/types.h +18 -4
  45. package/dist/lib/linux_x86_64/libdeflate.so +0 -0
  46. package/dist/lib/linux_x86_64/libnode-v22.node +0 -0
  47. package/dist/lib/linux_x86_64/libnode-v23.node +0 -0
  48. package/dist/lib/linux_x86_64/libnode-v24.node +0 -0
  49. package/dist/lib/linux_x86_64/libselva.so +0 -0
  50. package/dist/src/client/index.d.ts +16 -21
  51. package/dist/src/client/index.js +48 -59
  52. package/dist/src/client/modify/Ctx.d.ts +38 -0
  53. package/dist/src/client/modify/Ctx.js +30 -0
  54. package/dist/src/client/modify/Tmp.d.ts +19 -0
  55. package/dist/src/client/modify/Tmp.js +67 -0
  56. package/dist/src/client/modify/create/index.d.ts +6 -0
  57. package/dist/src/client/modify/create/index.js +191 -0
  58. package/dist/src/client/modify/create/mark.d.ts +7 -0
  59. package/dist/src/client/modify/create/mark.js +42 -0
  60. package/dist/src/client/modify/cursor.d.ts +10 -0
  61. package/dist/src/client/modify/cursor.js +48 -0
  62. package/dist/src/client/modify/delete/index.d.ts +2 -0
  63. package/dist/src/client/modify/delete/index.js +39 -0
  64. package/dist/src/client/modify/drain.d.ts +7 -0
  65. package/dist/src/client/modify/drain.js +77 -0
  66. package/dist/src/client/modify/edges/binary.d.ts +3 -0
  67. package/dist/src/client/modify/edges/binary.js +28 -0
  68. package/dist/src/client/modify/edges/cardinality.d.ts +3 -0
  69. package/dist/src/client/modify/edges/cardinality.js +15 -0
  70. package/dist/src/client/modify/edges/header.d.ts +5 -0
  71. package/dist/src/client/modify/edges/header.js +21 -0
  72. package/dist/src/client/modify/edges/index.d.ts +3 -0
  73. package/dist/src/client/modify/edges/index.js +128 -0
  74. package/dist/src/client/modify/edges/reference.d.ts +3 -0
  75. package/dist/src/client/modify/edges/reference.js +31 -0
  76. package/dist/src/client/modify/edges/references.d.ts +3 -0
  77. package/dist/src/client/modify/edges/references.js +38 -0
  78. package/dist/src/client/modify/edges/separate.d.ts +3 -0
  79. package/dist/src/client/modify/edges/separate.js +24 -0
  80. package/dist/src/client/modify/edges/string.d.ts +3 -0
  81. package/dist/src/client/modify/edges/string.js +30 -0
  82. package/dist/src/client/modify/error.d.ts +7 -0
  83. package/dist/src/client/modify/error.js +68 -0
  84. package/dist/src/client/modify/expire/index.d.ts +2 -0
  85. package/dist/src/client/modify/expire/index.js +31 -0
  86. package/dist/src/client/modify/props/alias.d.ts +3 -0
  87. package/dist/src/client/modify/props/alias.js +26 -0
  88. package/dist/src/client/modify/props/binary.d.ts +5 -0
  89. package/dist/src/client/modify/props/binary.js +52 -0
  90. package/dist/src/client/modify/props/cardinality.d.ts +4 -0
  91. package/dist/src/client/modify/props/cardinality.js +46 -0
  92. package/dist/src/client/modify/props/delete.d.ts +3 -0
  93. package/dist/src/client/modify/props/delete.js +13 -0
  94. package/dist/src/client/modify/props/fixed.d.ts +4 -0
  95. package/dist/src/client/modify/props/fixed.js +105 -0
  96. package/dist/src/client/modify/props/increment.d.ts +3 -0
  97. package/dist/src/client/modify/props/increment.js +28 -0
  98. package/dist/src/client/modify/props/json.d.ts +3 -0
  99. package/dist/src/client/modify/props/json.js +5 -0
  100. package/dist/src/client/modify/props/main.d.ts +4 -0
  101. package/dist/src/client/modify/props/main.js +22 -0
  102. package/dist/src/client/modify/props/object.d.ts +3 -0
  103. package/dist/src/client/modify/props/object.js +37 -0
  104. package/dist/src/client/modify/props/reference.d.ts +6 -0
  105. package/dist/src/client/modify/props/reference.js +92 -0
  106. package/dist/src/client/modify/props/references.d.ts +3 -0
  107. package/dist/src/client/modify/props/references.js +231 -0
  108. package/dist/src/client/modify/props/separate.d.ts +3 -0
  109. package/dist/src/client/modify/props/separate.js +43 -0
  110. package/dist/src/client/modify/props/string.d.ts +5 -0
  111. package/dist/src/client/modify/props/string.js +51 -0
  112. package/dist/src/client/modify/props/text.d.ts +3 -0
  113. package/dist/src/client/modify/props/text.js +35 -0
  114. package/dist/src/client/modify/props/vector.d.ts +3 -0
  115. package/dist/src/client/modify/props/vector.js +30 -0
  116. package/dist/src/client/modify/resize.d.ts +3 -0
  117. package/dist/src/client/modify/resize.js +20 -0
  118. package/dist/src/client/modify/types.d.ts +3 -4
  119. package/dist/src/client/modify/types.js +2 -1
  120. package/dist/src/client/modify/uint.d.ts +7 -0
  121. package/dist/src/client/modify/uint.js +27 -0
  122. package/dist/src/client/modify/update/index.d.ts +3 -0
  123. package/dist/src/client/modify/update/index.js +75 -0
  124. package/dist/src/client/modify/upsert/index.d.ts +3 -0
  125. package/dist/src/client/modify/{upsert.js → upsert/index.js} +4 -4
  126. package/dist/src/client/modify/validate.d.ts +6 -0
  127. package/dist/src/client/modify/validate.js +24 -0
  128. package/dist/src/client/query/BasedDbQuery.d.ts +18 -15
  129. package/dist/src/client/query/BasedDbQuery.js +93 -96
  130. package/dist/src/client/query/{BasedIterable.d.ts → BasedQueryResponse.d.ts} +5 -4
  131. package/dist/src/client/query/{BasedIterable.js → BasedQueryResponse.js} +15 -10
  132. package/dist/src/client/query/aggregates/aggregation.d.ts +5 -3
  133. package/dist/src/client/query/aggregates/aggregation.js +105 -48
  134. package/dist/src/client/query/aggregates/types.d.ts +34 -12
  135. package/dist/src/client/query/aggregates/types.js +30 -12
  136. package/dist/src/client/query/debug.js +1 -1
  137. package/dist/src/client/query/display.d.ts +1 -1
  138. package/dist/src/client/query/display.js +21 -4
  139. package/dist/src/client/query/filter/FilterBranch.js +2 -2
  140. package/dist/src/client/query/filter/convertFilter.d.ts +3 -2
  141. package/dist/src/client/query/filter/convertFilter.js +13 -1
  142. package/dist/src/client/query/filter/createFixedFilterBuffer.js +16 -30
  143. package/dist/src/client/query/filter/createReferenceFilter.js +6 -20
  144. package/dist/src/client/query/filter/createVariableFilterBuffer.js +15 -32
  145. package/dist/src/client/query/filter/parseFilterValue.js +3 -11
  146. package/dist/src/client/query/filter/primitiveFilter.js +1 -1
  147. package/dist/src/client/query/filter/toBuffer.js +9 -15
  148. package/dist/src/client/query/filter/types.d.ts +6 -6
  149. package/dist/src/client/query/filter/types.js +8 -8
  150. package/dist/src/client/query/include/include.d.ts +3 -0
  151. package/dist/src/client/query/include/include.js +58 -0
  152. package/dist/src/client/query/include/props.d.ts +7 -7
  153. package/dist/src/client/query/include/props.js +26 -21
  154. package/dist/src/client/query/include/toByteCode.js +188 -0
  155. package/dist/src/client/query/include/utils.d.ts +3 -1
  156. package/dist/src/client/query/include/utils.js +13 -0
  157. package/dist/src/client/query/include/walk.d.ts +2 -2
  158. package/dist/src/client/query/include/walk.js +38 -23
  159. package/dist/src/client/query/query.d.ts +2 -3
  160. package/dist/src/client/query/query.js +2 -3
  161. package/dist/src/client/query/queryDef.d.ts +4 -2
  162. package/dist/src/client/query/queryDef.js +3 -5
  163. package/dist/src/client/query/queryDefToReadSchema.d.ts +3 -0
  164. package/dist/src/client/query/queryDefToReadSchema.js +126 -0
  165. package/dist/src/client/query/registerQuery.js +15 -4
  166. package/dist/src/client/query/search/index.d.ts +2 -1
  167. package/dist/src/client/query/search/index.js +9 -2
  168. package/dist/src/client/query/sort.js +3 -0
  169. package/dist/src/client/query/subscription/index.d.ts +1 -1
  170. package/dist/src/client/query/subscription/index.js +2 -2
  171. package/dist/src/client/query/subscription/types.d.ts +1 -1
  172. package/dist/src/client/query/thresholds.d.ts +1 -1
  173. package/dist/src/client/query/thresholds.js +1 -1
  174. package/dist/src/client/query/toByteCode/{toBuffer.js → toByteCode.js} +8 -8
  175. package/dist/src/client/query/types.d.ts +40 -25
  176. package/dist/src/client/query/types.js +2 -6
  177. package/dist/src/client/query/validation.d.ts +6 -0
  178. package/dist/src/client/query/validation.js +32 -8
  179. package/dist/src/client/setLocalClientSchema.js +6 -10
  180. package/dist/src/client/string.d.ts +3 -7
  181. package/dist/src/client/string.js +18 -47
  182. package/dist/src/hooks.d.ts +2 -3
  183. package/dist/src/hooks.js +14 -9
  184. package/dist/src/index.d.ts +3 -5
  185. package/dist/src/index.js +4 -6
  186. package/dist/src/native.d.ts +3 -2
  187. package/dist/src/native.js +9 -8
  188. package/dist/src/server/IoWorker.js +2 -3
  189. package/dist/src/server/QueryWorker.js +1 -1
  190. package/dist/src/server/blocks.js +5 -4
  191. package/dist/src/server/dbHash.js +1 -1
  192. package/dist/src/server/index.d.ts +4 -6
  193. package/dist/src/server/index.js +63 -96
  194. package/dist/src/server/migrate/index.d.ts +2 -4
  195. package/dist/src/server/migrate/index.js +31 -16
  196. package/dist/src/server/migrate/worker.js +2 -27
  197. package/dist/src/server/save.js +10 -8
  198. package/dist/src/server/schema.js +17 -25
  199. package/dist/src/server/start.d.ts +1 -0
  200. package/dist/src/server/start.js +18 -10
  201. package/dist/src/server/tree.d.ts +20 -0
  202. package/dist/src/server/tree.js +17 -2
  203. package/dist/src/server/workers/DbWorker.d.ts +1 -0
  204. package/dist/src/server/workers/DbWorker.js +8 -1
  205. package/dist/src/server/workers/io_worker.js +1 -1
  206. package/dist/src/server/workers/worker.js +9 -4
  207. package/dist/src/table.d.ts +4 -0
  208. package/dist/src/table.js +97 -0
  209. package/dist/src/types.d.ts +1 -1
  210. package/package.json +18 -17
  211. package/dist/lib/darwin_aarch64/include/selva/worker_ctx.h +0 -29
  212. package/dist/lib/linux_aarch64/include/selva/worker_ctx.h +0 -29
  213. package/dist/lib/linux_x86_64/include/selva/worker_ctx.h +0 -29
  214. package/dist/src/client/flushModify.d.ts +0 -36
  215. package/dist/src/client/flushModify.js +0 -197
  216. package/dist/src/client/modify/ModifyRes.d.ts +0 -23
  217. package/dist/src/client/modify/ModifyRes.js +0 -101
  218. package/dist/src/client/modify/alias.d.ts +0 -4
  219. package/dist/src/client/modify/alias.js +0 -57
  220. package/dist/src/client/modify/binary.d.ts +0 -6
  221. package/dist/src/client/modify/binary.js +0 -80
  222. package/dist/src/client/modify/cardinality.d.ts +0 -6
  223. package/dist/src/client/modify/cardinality.js +0 -63
  224. package/dist/src/client/modify/create.d.ts +0 -5
  225. package/dist/src/client/modify/create.js +0 -228
  226. package/dist/src/client/modify/delete.d.ts +0 -4
  227. package/dist/src/client/modify/delete.js +0 -47
  228. package/dist/src/client/modify/expire.d.ts +0 -3
  229. package/dist/src/client/modify/expire.js +0 -25
  230. package/dist/src/client/modify/fixed.d.ts +0 -5
  231. package/dist/src/client/modify/fixed.js +0 -235
  232. package/dist/src/client/modify/index.d.ts +0 -4
  233. package/dist/src/client/modify/index.js +0 -4
  234. package/dist/src/client/modify/json.d.ts +0 -4
  235. package/dist/src/client/modify/json.js +0 -26
  236. package/dist/src/client/modify/modify.d.ts +0 -5
  237. package/dist/src/client/modify/modify.js +0 -134
  238. package/dist/src/client/modify/references/appendEdgeRefs.d.ts +0 -4
  239. package/dist/src/client/modify/references/appendEdgeRefs.js +0 -33
  240. package/dist/src/client/modify/references/edge.d.ts +0 -5
  241. package/dist/src/client/modify/references/edge.js +0 -322
  242. package/dist/src/client/modify/references/getEdgeSize.d.ts +0 -3
  243. package/dist/src/client/modify/references/getEdgeSize.js +0 -27
  244. package/dist/src/client/modify/references/reference.d.ts +0 -11
  245. package/dist/src/client/modify/references/reference.js +0 -112
  246. package/dist/src/client/modify/references/references.d.ts +0 -16
  247. package/dist/src/client/modify/references/references.js +0 -308
  248. package/dist/src/client/modify/setCursor.d.ts +0 -5
  249. package/dist/src/client/modify/setCursor.js +0 -33
  250. package/dist/src/client/modify/string.d.ts +0 -5
  251. package/dist/src/client/modify/string.js +0 -71
  252. package/dist/src/client/modify/text.d.ts +0 -8
  253. package/dist/src/client/modify/text.js +0 -69
  254. package/dist/src/client/modify/update.d.ts +0 -6
  255. package/dist/src/client/modify/update.js +0 -90
  256. package/dist/src/client/modify/upsert.d.ts +0 -3
  257. package/dist/src/client/modify/vector.d.ts +0 -4
  258. package/dist/src/client/modify/vector.js +0 -53
  259. package/dist/src/client/query/include/toBuffer.js +0 -123
  260. package/dist/src/client/query/read/read.d.ts +0 -9
  261. package/dist/src/client/query/read/read.js +0 -483
  262. /package/dist/src/client/query/include/{toBuffer.d.ts → toByteCode.d.ts} +0 -0
  263. /package/dist/src/client/query/toByteCode/{toBuffer.d.ts → toByteCode.d.ts} +0 -0
@@ -1,80 +1,51 @@
1
1
  import native from '../native.js';
2
- import { readUint32, makeTmpBuffer, DECODER, ENCODER } from '@saulx/utils';
2
+ import { makeTmpBuffer, ENCODER, writeUint32 } from '@based/utils';
3
+ import { COMPRESSED, NOT_COMPRESSED } from '@based/protocol/db-read';
4
+ import { resize } from './modify/resize.js';
3
5
  const { getUint8Array: getTmpBuffer } = makeTmpBuffer(4096); // the usual page size?
4
- export const COMPRESSED = 1;
5
- export const NOT_COMPRESSED = 0;
6
- export const write = (buf, value, offset, noCompression, lang) => {
6
+ export const write = (ctx, value, offset, noCompression, lang) => {
7
+ const buf = ctx.array;
7
8
  value = value.normalize('NFKD');
8
9
  buf[offset] = lang || 0;
9
10
  const { written: l } = ENCODER.encodeInto(value, buf.subarray(offset + 2));
10
11
  let crc = native.crc32(buf.subarray(offset + 2, offset + 2 + l));
11
12
  // 50 len maybe if lvl 1
12
13
  if (value.length > 200 && !noCompression) {
13
- buf.copyWithin(offset + 6 + l, offset + 2, offset + 2 + l);
14
+ const insertPos = offset + 6 + l;
15
+ const startPos = offset + 2;
16
+ const endPos = offset + 2 + l;
17
+ const willEnd = insertPos + l;
18
+ resize(ctx, willEnd);
19
+ buf.copyWithin(insertPos, startPos, endPos);
14
20
  const size = native.compress(buf, offset + 6, l);
15
21
  if (size === 0) {
22
+ resize(ctx, l + 6);
16
23
  buf[offset + 1] = NOT_COMPRESSED;
17
24
  ENCODER.encodeInto(value, buf.subarray(offset + 2));
18
- buf[offset + l + 2] = crc;
19
- buf[offset + l + 3] = crc >>>= 8;
20
- buf[offset + l + 4] = crc >>>= 8;
21
- buf[offset + l + 5] = crc >>>= 8;
25
+ writeUint32(buf, crc, offset + l + 2);
22
26
  return l + 6;
23
27
  }
24
28
  else {
29
+ resize(ctx, size + 10);
25
30
  let len = l;
26
31
  buf[offset + 1] = COMPRESSED;
27
- buf[offset + 2] = len;
28
- buf[offset + 3] = len >>>= 8;
29
- buf[offset + 4] = len >>>= 8;
30
- buf[offset + 5] = len >>>= 8;
31
- buf[offset + size + 6] = crc;
32
- buf[offset + size + 7] = crc >>>= 8;
33
- buf[offset + size + 8] = crc >>>= 8;
34
- buf[offset + size + 9] = crc >>>= 8;
32
+ writeUint32(buf, len, offset + 2);
33
+ writeUint32(buf, crc, offset + size + 6);
35
34
  return size + 10;
36
35
  }
37
36
  }
38
37
  else {
39
38
  buf[offset + 1] = NOT_COMPRESSED;
40
- buf[offset + l + 2] = crc;
41
- buf[offset + l + 3] = crc >>>= 8;
42
- buf[offset + l + 4] = crc >>>= 8;
43
- buf[offset + l + 5] = crc >>>= 8;
39
+ writeUint32(buf, crc, offset + l + 2);
44
40
  return l + 6;
45
41
  }
46
42
  };
47
43
  export const compress = (str) => {
48
44
  const len = ENCODER.encode(str).byteLength;
49
45
  const tmpCompressBlock = getTmpBuffer(len * 3);
50
- const l = write(tmpCompressBlock, str, 0, false);
46
+ const l = write({ array: tmpCompressBlock }, str, 0, false);
51
47
  const nBuffer = new Uint8Array(l);
52
48
  nBuffer.set(tmpCompressBlock.subarray(0, l));
53
49
  return nBuffer;
54
50
  };
55
- export const decompress = (val) => {
56
- return read(val, 0, val.length, false);
57
- };
58
- export const read = (val, offset, len, stripCrc32) => {
59
- const type = val[offset + 1];
60
- if (type == COMPRESSED) {
61
- const origSize = readUint32(val, offset + 2);
62
- const newBuffer = getTmpBuffer(origSize);
63
- // Browser fallback required for this
64
- native.decompress(val, newBuffer, offset + 6, stripCrc32 ? len - 2 : len - 6);
65
- return DECODER.decode(newBuffer);
66
- }
67
- else if (type == NOT_COMPRESSED) {
68
- if (stripCrc32) {
69
- return DECODER.decode(val.subarray(offset + 2, len + offset));
70
- }
71
- else {
72
- return DECODER.decode(val.subarray(offset + 2, len + offset - 4));
73
- }
74
- }
75
- return '';
76
- };
77
- export const readUtf8 = (val, offset, len) => {
78
- return DECODER.decode(val.subarray(offset, len + offset));
79
- };
80
51
  //# sourceMappingURL=string.js.map
@@ -1,11 +1,10 @@
1
- import { StrictSchema } from '@based/schema';
1
+ import { StrictSchema, MigrateFns } from '@based/schema';
2
2
  import { BasedDbQuery } from './client/query/BasedDbQuery.js';
3
3
  import { OnClose, OnData, OnError } from './client/query/subscription/types.js';
4
4
  import { DbServer } from './server/index.js';
5
- import { TransformFns } from './server/migrate/index.js';
6
5
  import { DbSchema, SchemaChecksum } from './schema.js';
7
6
  export type DbClientHooks = {
8
- setSchema(schema: StrictSchema, transformFns?: TransformFns): Promise<SchemaChecksum>;
7
+ setSchema(schema: StrictSchema, transformFns?: MigrateFns): Promise<SchemaChecksum>;
9
8
  flushModify(buf: Uint8Array): Promise<{
10
9
  offsets: Record<number, number>;
11
10
  dbWriteTime?: number;
package/dist/src/hooks.js CHANGED
@@ -31,8 +31,8 @@ export const getDefaultHooks = (server, subInterval = 200) => {
31
31
  killed = true;
32
32
  };
33
33
  },
34
- setSchema(schema) {
35
- return server.setSchema(schema);
34
+ setSchema(schema, transformFns) {
35
+ return server.setSchema(schema, transformFns);
36
36
  },
37
37
  subscribeSchema(setSchema) {
38
38
  if (server.schema) {
@@ -43,13 +43,18 @@ export const getDefaultHooks = (server, subInterval = 200) => {
43
43
  });
44
44
  },
45
45
  flushModify(buf) {
46
- const d = performance.now();
47
- const offsets = server.modify(buf);
48
- const dbWriteTime = performance.now() - d;
49
- return Promise.resolve({
50
- offsets,
51
- dbWriteTime,
52
- });
46
+ try {
47
+ const d = performance.now();
48
+ const offsets = server.modify(buf);
49
+ const dbWriteTime = performance.now() - d;
50
+ return Promise.resolve({
51
+ offsets,
52
+ dbWriteTime,
53
+ });
54
+ }
55
+ catch (e) {
56
+ return Promise.reject(e);
57
+ }
53
58
  },
54
59
  getQueryBuf(buf) {
55
60
  return Promise.resolve(server.getQueryBuf(buf));
@@ -1,13 +1,11 @@
1
- import { compress, decompress } from './client/string.js';
2
- import { ModifyCtx } from './client/flushModify.js';
1
+ import { compress } from './client/string.js';
3
2
  import { DbServer } from './server/index.js';
4
3
  import { DbClient } from './client/index.js';
5
4
  import { getDefaultHooks } from './hooks.js';
6
5
  import { Emitter } from './shared/Emitter.js';
7
6
  import { BasedDbOpts } from './types.js';
8
- export * from './client/modify/modify.js';
7
+ import { decompress } from '@based/protocol/db-read';
9
8
  export { compress, decompress };
10
- export { ModifyCtx };
11
9
  export { DbClient, DbServer };
12
10
  export { xxHash64 } from './client/xxHash64.js';
13
11
  export { crc32 } from './client/crc32.js';
@@ -15,7 +13,7 @@ export { default as createHash } from './server/dbHash.js';
15
13
  export * from './utils.js';
16
14
  export * from './client/query/query.js';
17
15
  export * from './client/query/BasedDbQuery.js';
18
- export * from './client/query/BasedIterable.js';
16
+ export * from './client/query/BasedQueryResponse.js';
19
17
  export * from './server/save.js';
20
18
  export * from './hooks.js';
21
19
  export { getDefaultHooks, BasedDbOpts };
package/dist/src/index.js CHANGED
@@ -1,14 +1,12 @@
1
- import { compress, decompress } from './client/string.js';
2
- import { ModifyCtx } from './client/flushModify.js';
1
+ import { compress } from './client/string.js';
3
2
  import { DbServer } from './server/index.js';
4
3
  import { DbClient } from './client/index.js';
5
- import { wait } from '@saulx/utils';
4
+ import { wait } from '@based/utils';
6
5
  import { debugMode, debugServer } from './utils.js';
7
6
  import { getDefaultHooks } from './hooks.js';
8
7
  import { Emitter } from './shared/Emitter.js';
9
- export * from './client/modify/modify.js';
8
+ import { decompress } from '@based/protocol/db-read';
10
9
  export { compress, decompress };
11
- export { ModifyCtx };
12
10
  export { DbClient, DbServer };
13
11
  export { xxHash64 } from './client/xxHash64.js';
14
12
  export { crc32 } from './client/crc32.js';
@@ -16,7 +14,7 @@ export { default as createHash } from './server/dbHash.js';
16
14
  export * from './utils.js';
17
15
  export * from './client/query/query.js';
18
16
  export * from './client/query/BasedDbQuery.js';
19
- export * from './client/query/BasedIterable.js';
17
+ export * from './client/query/BasedQueryResponse.js';
20
18
  export * from './server/save.js';
21
19
  export * from './hooks.js';
22
20
  export { getDefaultHooks };
@@ -1,6 +1,7 @@
1
1
  declare const native: {
2
- threadCtx: any;
3
- workerCtxInit: () => void;
2
+ getThreadId: () => BigInt;
3
+ createThreadCtx: (dbCtx: any, threadId: BigInt) => void;
4
+ destroyThreadCtx: (dbCtx: any, threadId: BigInt) => void;
4
5
  externalFromInt(address: BigInt): any;
5
6
  intFromExternal(external: any): BigInt;
6
7
  modify: (data: Uint8Array, types: Uint8Array, dbCtx: any, dirtyBlocksOut: Float64Array) => any;
@@ -4,21 +4,22 @@ import db from '../../basedDbNative.cjs';
4
4
  const DECODER = new TextDecoder('utf-8');
5
5
  const ENCODER = new TextEncoder();
6
6
  const selvaIoErrlog = new Uint8Array(256);
7
- var compressor = db.createCompressor(); // put on threadCtx
7
+ var compressor = db.createCompressor();
8
8
  var decompressor = db.createDecompressor();
9
9
  function SelvaIoErrlogToString(buf) {
10
10
  let i;
11
11
  let len = (i = buf.indexOf(0)) >= 0 ? i : buf.byteLength;
12
12
  return DECODER.decode(selvaIoErrlog.slice(0, len));
13
13
  }
14
- // add worker CTX HERE
15
- // then add it to every function
16
- // worker should allways be here
17
- // then add ThreadCtx to modify ctx and query ctx
18
14
  const native = {
19
- threadCtx: null, // add compressors here as well!
20
- workerCtxInit: () => {
21
- return db.workerCtxInit();
15
+ getThreadId: () => {
16
+ return db.getThreadId();
17
+ },
18
+ createThreadCtx: (dbCtx, threadId) => {
19
+ db.createThreadCtx(dbCtx, threadId);
20
+ },
21
+ destroyThreadCtx: (dbCtx, threadId) => {
22
+ db.destroyThreadCtx(dbCtx, threadId);
22
23
  },
23
24
  externalFromInt(address) {
24
25
  return db.externalFromInt(address);
@@ -1,5 +1,5 @@
1
1
  import { DbWorker } from './workers/DbWorker.js';
2
- import { DECODER, readInt32 } from '@saulx/utils';
2
+ import { DECODER, readInt32 } from '@based/utils';
3
3
  import native from '../native.js';
4
4
  export class IoWorker extends DbWorker {
5
5
  constructor(address, db) {
@@ -8,8 +8,7 @@ export class IoWorker extends DbWorker {
8
8
  };
9
9
  super(address, db, onExit, 'io_worker.js');
10
10
  }
11
- handleMsg(_buf) {
12
- }
11
+ handleMsg(_buf) { }
13
12
  cb = (resolve) => {
14
13
  this.db.activeReaders++;
15
14
  this.resolvers.push((r) => {
@@ -1,5 +1,5 @@
1
1
  import { DbWorker } from './workers/DbWorker.js';
2
- import { readUint64 } from '@saulx/utils';
2
+ import { readUint64 } from '@based/utils';
3
3
  export class QueryWorker extends DbWorker {
4
4
  constructor(address, db, workerIndex) {
5
5
  const onExit = (_code) => {
@@ -1,7 +1,7 @@
1
1
  import native from '../native.js';
2
2
  import { join } from 'node:path';
3
- import { equals, readInt32 } from '@saulx/utils';
4
- import { VerifTree, destructureTreeKey, makeTreeKey, } from './tree.js';
3
+ import { equals, readInt32 } from '@based/utils';
4
+ import { VerifTree, destructureTreeKey, makeTreeKey } from './tree.js';
5
5
  const SELVA_ENOENT = -8;
6
6
  /**
7
7
  * Save a block.
@@ -64,7 +64,7 @@ export async function loadBlock(db, def, start) {
64
64
  const p = db.ioWorker.loadBlock(join(db.fileSystemPath, filename));
65
65
  block.loadPromise = p;
66
66
  await p;
67
- delete block.loadPromise;
67
+ block.loadPromise = null;
68
68
  // Update and verify the hash
69
69
  const hash = new Uint8Array(16);
70
70
  const end = start + def.blockCapacity - 1;
@@ -106,7 +106,8 @@ export async function unloadBlock(db, def, start) {
106
106
  */
107
107
  export function foreachBlock(db, def, cb, includeEmptyBlocks = false) {
108
108
  const step = def.blockCapacity;
109
- for (let start = 1; start <= def.lastId; start += step) {
109
+ const lastId = def.lastId || native.getTypeInfo(def.id, db.dbCtxExternal)[1];
110
+ for (let start = 1; start <= lastId; start += step) {
110
111
  const end = start + step - 1;
111
112
  const hash = new Uint8Array(16);
112
113
  const res = native.getNodeRangeHash(def.id, start, end, hash, db.dbCtxExternal);
@@ -1,6 +1,6 @@
1
1
  // @ts-ignore
2
2
  import db from '../../../basedDbNative.cjs';
3
- import { bufToHex } from '@saulx/utils';
3
+ import { bufToHex } from '@based/utils';
4
4
  export default function createHash() {
5
5
  const state = db.hashCreate();
6
6
  const hash = {
@@ -1,7 +1,6 @@
1
- import { LangName, StrictSchema } from '@based/schema';
1
+ import { LangName, MigrateFns, StrictSchema } from '@based/schema';
2
2
  import { StartOpts } from './start.js';
3
3
  import { VerifTree } from './tree.js';
4
- import { TransformFns } from './migrate/index.js';
5
4
  import exitHook from 'exit-hook';
6
5
  import { SchemaChecksum } from '../schema.js';
7
6
  import { IoWorker } from './IoWorker.js';
@@ -17,7 +16,6 @@ export declare class DbServer extends DbShared {
17
16
  #private;
18
17
  modifyDirtyRanges: Float64Array;
19
18
  dbCtxExternal: any;
20
- threadCtxExternal: any;
21
19
  migrating: number;
22
20
  saveInProgress: boolean;
23
21
  fileSystemPath: string;
@@ -60,11 +58,11 @@ export declare class DbServer extends DbShared {
60
58
  destroySortIndex(type: string, field: string, lang?: LangName): any;
61
59
  getSortIndex(typeId: number, field: number, start: number, lang: number): SortIndex;
62
60
  createSortIndexBuffer(typeId: number, field: number, start: number, lang: number): SortIndex;
63
- setSchema(strictSchema: StrictSchema, transformFns?: TransformFns): Promise<SchemaChecksum>;
64
- modify(bufWithHash: Uint8Array): Record<number, number> | null;
61
+ setSchema(strictSchema: StrictSchema, transformFns?: MigrateFns): Promise<SchemaChecksum>;
62
+ modify(payload: Uint8Array, skipParse?: boolean): Record<number, number> | null;
65
63
  addToQueryQueue(resolve: any, buf: any): Promise<Uint8Array<ArrayBuffer>>;
66
64
  getQueryBuf(buf: Uint8Array, fromQueue?: boolean): Promise<Uint8Array>;
67
- onQueryEnd(): any;
65
+ onQueryEnd(): void;
68
66
  stop(noSave?: boolean): Promise<void>;
69
67
  destroy(): Promise<void>;
70
68
  }
@@ -1,13 +1,14 @@
1
1
  import native from '../native.js';
2
2
  import { rm } from 'node:fs/promises';
3
3
  import { langCodesMap } from '@based/schema';
4
+ import { ID_FIELD_DEF } from '@based/schema/def';
4
5
  import { start } from './start.js';
5
6
  import { destructureTreeKey, makeTreeKeyFromNodeId } from './tree.js';
6
7
  import { save } from './save.js';
7
8
  import { setTimeout } from 'node:timers/promises';
8
9
  import { migrate } from './migrate/index.js';
9
10
  import { debugServer } from '../utils.js';
10
- import { readUint16, readUint32, readUint64 } from '@saulx/utils';
11
+ import { readUint16, readUint32, readUint64, writeUint32 } from '@based/utils';
11
12
  import { QueryType } from '../client/query/types.js';
12
13
  import { strictSchemaToDbSchema } from './schema.js';
13
14
  import { DbShared } from '../shared/DbBase.js';
@@ -27,7 +28,6 @@ class SortIndex {
27
28
  export class DbServer extends DbShared {
28
29
  modifyDirtyRanges;
29
30
  dbCtxExternal; // pointer to zig dbCtx
30
- threadCtxExternal; // pointer to zig dbCtx
31
31
  migrating = null;
32
32
  saveInProgress = false;
33
33
  fileSystemPath;
@@ -49,6 +49,10 @@ export class DbServer extends DbShared {
49
49
  this.fileSystemPath = path;
50
50
  this.sortIndexes = {};
51
51
  this.saveIntervalInSeconds = saveIntervalInSeconds;
52
+ if (process.stderr.isTTY) {
53
+ //this.on('info', (v) => console.error('Info:', v))
54
+ this.on('error', (v) => console.error('Error:', v));
55
+ }
52
56
  if (debug) {
53
57
  debugServer(this);
54
58
  }
@@ -202,17 +206,18 @@ export class DbServer extends DbShared {
202
206
  buf[4] = start >>> 8;
203
207
  let typeDef;
204
208
  let prop;
205
- for (const t in this.schemaTypesParsed) {
206
- typeDef = this.schemaTypesParsed[t];
207
- if (typeDef.id == typeId) {
208
- for (const p in typeDef.props) {
209
- const propDef = typeDef.props[p];
210
- if (propDef.prop == field && propDef.start == start) {
211
- prop = propDef;
212
- break;
213
- }
209
+ if (field === 255) {
210
+ prop = ID_FIELD_DEF;
211
+ typeDef = this.schemaTypesParsedById[typeId];
212
+ }
213
+ else {
214
+ typeDef = this.schemaTypesParsedById[typeId];
215
+ for (const p in typeDef.props) {
216
+ const propDef = typeDef.props[p];
217
+ if (propDef.prop == field && propDef.start == start) {
218
+ prop = propDef;
219
+ break;
214
220
  }
215
- break;
216
221
  }
217
222
  }
218
223
  if (!typeDef) {
@@ -234,6 +239,9 @@ export class DbServer extends DbShared {
234
239
  return sortIndex;
235
240
  }
236
241
  async setSchema(strictSchema, transformFns) {
242
+ if (this.stopped || !this.dbCtxExternal) {
243
+ throw new Error('Db is stopped');
244
+ }
237
245
  const schema = strictSchemaToDbSchema(strictSchema);
238
246
  if (schema.hash === this.schema?.hash) {
239
247
  // Todo something for sending back to actual client
@@ -262,89 +270,59 @@ export class DbServer extends DbShared {
262
270
  });
263
271
  return schema.hash;
264
272
  }
265
- modify(bufWithHash) {
266
- const schemaHash = readUint64(bufWithHash, 0);
267
- if (schemaHash !== this.schema?.hash) {
268
- this.emit('info', 'Schema mismatch in modify');
273
+ modify(payload, skipParse) {
274
+ const hash = readUint64(payload, 0);
275
+ const contentEnd = readUint32(payload, payload.byteLength - 4);
276
+ let result;
277
+ if (this.schema?.hash !== hash) {
278
+ this.emit('info', 'Schema mismatch in write');
269
279
  return null;
270
280
  }
271
- const buf = bufWithHash.subarray(8);
272
- const offsets = {};
273
- const dataLen = readUint32(buf, buf.length - 4);
274
- let typesSize = readUint16(buf, dataLen);
275
- let i = dataLen + 2;
276
- while (typesSize--) {
277
- const typeId = readUint16(buf, i);
278
- i += 2;
279
- const startId = readUint32(buf, i);
280
- const def = this.schemaTypesParsedById[typeId];
281
- if (!def) {
282
- console.error(`Wrong cannot get def in modify ${typeId} ${schemaHash} ${this.schema?.hash}!}`);
283
- return null;
284
- }
285
- let offset = def.lastId - startId;
286
- if (offset < 0) {
287
- offset = 0;
281
+ if (!skipParse) {
282
+ result = {};
283
+ let i = payload.byteLength - 4;
284
+ while (i > contentEnd) {
285
+ const typeId = readUint16(payload, i - 6);
286
+ const count = readUint32(payload, i - 4);
287
+ const typeDef = this.schemaTypesParsedById[typeId];
288
+ if (!typeDef) {
289
+ console.error('Missing typeDef, cancel write', { typeId, count });
290
+ this.emit('info', 'Missing typeDef, cancel write');
291
+ return null;
292
+ }
293
+ const lastId = typeDef.lastId ||
294
+ native.getTypeInfo(typeDef.id, this.dbCtxExternal)[1];
295
+ // TODO replace this with Ctx.created
296
+ const offset = lastId;
297
+ // write the offset into payload for zig to use
298
+ writeUint32(payload, offset, i - 4);
299
+ result[typeId] = offset;
300
+ typeDef.lastId = lastId + count;
301
+ i -= 6;
288
302
  }
289
- buf[i++] = offset;
290
- buf[i++] = offset >>> 8;
291
- buf[i++] = offset >>> 16;
292
- buf[i++] = offset >>> 24;
293
- const lastId = readUint32(buf, i);
294
- i += 4;
295
- def.lastId = lastId + offset;
296
- offsets[typeId] = offset;
297
303
  }
304
+ const content = payload.subarray(8, contentEnd);
305
+ const offsets = payload.subarray(contentEnd, payload.byteLength - 4);
298
306
  if (this.activeReaders) {
299
- this.modifyQueue.push(new Uint8Array(bufWithHash));
307
+ this.modifyQueue.push(new Uint8Array(payload));
300
308
  }
301
309
  else {
302
- this.#modify(buf);
303
- }
304
- return offsets;
305
- }
306
- #modify(buf) {
307
- if (this.stopped) {
308
- console.error('Db is stopped - trying to modify');
309
- return;
310
- }
311
- const end = buf.length - 4;
312
- const dataLen = readUint32(buf, end);
313
- let typesSize = readUint16(buf, dataLen);
314
- const typesLen = typesSize * 10;
315
- const types = buf.subarray(dataLen + 2, dataLen + typesLen + 2);
316
- const data = buf.subarray(0, dataLen);
317
- let i = dataLen + 2;
318
- while (typesSize--) {
319
- const typeId = readUint16(buf, i);
320
- const def = this.schemaTypesParsedById[typeId];
321
- const key = makeTreeKeyFromNodeId(def.id, def.blockCapacity, def.lastId);
322
- this.dirtyRanges.add(key);
323
- i += 10;
324
- }
325
- const view = new DataView(buf.buffer, buf.byteOffset);
326
- while (i < end) {
327
- // const key = view.getFloat64(i, true)
328
- // These node ranges may not actually exist
329
- // this.dirtyRanges.add(key)
330
- i += 8;
331
- }
332
- resizeModifyDirtyRanges(this);
333
- native.modify(data, types, this.dbCtxExternal, this.modifyDirtyRanges);
334
- for (let key of this.modifyDirtyRanges) {
335
- if (key === 0) {
336
- break;
310
+ resizeModifyDirtyRanges(this);
311
+ native.modify(content, offsets, this.dbCtxExternal, this.modifyDirtyRanges);
312
+ for (const key of this.modifyDirtyRanges) {
313
+ if (key === 0)
314
+ break;
315
+ this.dirtyRanges.add(key);
337
316
  }
338
- this.dirtyRanges.add(key);
339
317
  }
318
+ return result;
340
319
  }
341
320
  #expire() {
342
321
  resizeModifyDirtyRanges(this);
343
322
  native.modify(emptyUint8Array, emptyUint8Array, this.dbCtxExternal, this.modifyDirtyRanges);
344
- for (let key of this.modifyDirtyRanges) {
345
- if (key === 0) {
323
+ for (const key of this.modifyDirtyRanges) {
324
+ if (key === 0)
346
325
  break;
347
- }
348
326
  this.dirtyRanges.add(key);
349
327
  }
350
328
  }
@@ -411,14 +389,8 @@ export class DbServer extends DbShared {
411
389
  if (this.modifyQueue.length) {
412
390
  const modifyQueue = this.modifyQueue;
413
391
  this.modifyQueue = [];
414
- for (const bufWithHash of modifyQueue) {
415
- const schemaHash = readUint64(bufWithHash, 0);
416
- if (schemaHash !== this.schema?.hash) {
417
- this.emit('info', 'Schema mismatch in modify');
418
- return null;
419
- }
420
- const buf = bufWithHash.subarray(8);
421
- this.#modify(buf);
392
+ for (const payload of modifyQueue) {
393
+ this.modify(payload, true);
422
394
  }
423
395
  }
424
396
  if (this.queryQueue.size) {
@@ -454,6 +426,7 @@ export class DbServer extends DbShared {
454
426
  await Promise.all(this.workers.map((worker) => worker.terminate()));
455
427
  this.workers = [];
456
428
  native.stop(this.dbCtxExternal);
429
+ this.dbCtxExternal = null;
457
430
  await setTimeout(100);
458
431
  }
459
432
  catch (e) {
@@ -464,13 +437,7 @@ export class DbServer extends DbShared {
464
437
  async destroy() {
465
438
  await this.stop(true);
466
439
  if (this.fileSystemPath) {
467
- await rm(this.fileSystemPath, { recursive: true }).catch((err) => {
468
- // console.warn(
469
- // 'Error removing dump folder',
470
- // this.fileSystemPath,
471
- // err.message,
472
- // ),
473
- });
440
+ await rm(this.fileSystemPath, { recursive: true }).catch((err) => { });
474
441
  }
475
442
  }
476
443
  }
@@ -1,11 +1,9 @@
1
1
  import { DbServer } from '../index.js';
2
2
  import { DbSchema } from '../../schema.js';
3
+ import { MigrateFns } from '@based/schema';
3
4
  export type MigrateRange = {
4
5
  typeId: number;
5
6
  start: number;
6
7
  end: number;
7
8
  };
8
- type TransformFn = (node: Record<string, any>) => Record<string, any> | [string, Record<string, any>];
9
- export type TransformFns = Record<string, TransformFn>;
10
- export declare const migrate: (server: DbServer, fromSchema: DbSchema, toSchema: DbSchema, transform?: TransformFns) => Promise<void>;
11
- export {};
9
+ export declare const migrate: (server: DbServer, fromSchema: DbSchema, toSchema: DbSchema, transform?: MigrateFns) => Promise<void>;