@based/db 0.0.21 → 0.0.23

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. package/README.md +10 -10
  2. package/dist/lib/darwin_aarch64/include/selva/types.h +9 -18
  3. package/dist/lib/darwin_aarch64/libnode-v20.node +0 -0
  4. package/dist/lib/darwin_aarch64/libnode-v21.node +0 -0
  5. package/dist/lib/darwin_aarch64/libnode-v22.node +0 -0
  6. package/dist/lib/darwin_aarch64/libnode-v23.node +0 -0
  7. package/dist/lib/darwin_aarch64/libselva.dylib +0 -0
  8. package/dist/lib/linux_aarch64/include/selva/types.h +9 -18
  9. package/dist/lib/linux_aarch64/libjemalloc_selva.so.2 +0 -0
  10. package/dist/lib/linux_aarch64/libnode-v20.node +0 -0
  11. package/dist/lib/linux_aarch64/libnode-v21.node +0 -0
  12. package/dist/lib/linux_aarch64/libnode-v22.node +0 -0
  13. package/dist/lib/linux_aarch64/libnode-v23.node +0 -0
  14. package/dist/lib/linux_aarch64/libselva.so +0 -0
  15. package/dist/lib/linux_x86_64/include/selva/types.h +9 -18
  16. package/dist/lib/linux_x86_64/libjemalloc_selva.so.2 +0 -0
  17. package/dist/lib/linux_x86_64/libnode-v20.node +0 -0
  18. package/dist/lib/linux_x86_64/libnode-v21.node +0 -0
  19. package/dist/lib/linux_x86_64/libnode-v22.node +0 -0
  20. package/dist/lib/linux_x86_64/libnode-v23.node +0 -0
  21. package/dist/lib/linux_x86_64/libselva.so +0 -0
  22. package/dist/src/client/bitWise.d.ts +0 -1
  23. package/dist/src/client/bitWise.js +0 -10
  24. package/dist/src/client/{operations.d.ts → flushModify.d.ts} +4 -4
  25. package/dist/src/client/{operations.js → flushModify.js} +39 -15
  26. package/dist/src/client/index.d.ts +10 -6
  27. package/dist/src/client/index.js +12 -4
  28. package/dist/src/client/modify/ModifyRes.d.ts +2 -8
  29. package/dist/src/client/modify/ModifyRes.js +23 -29
  30. package/dist/src/client/modify/alias.d.ts +1 -1
  31. package/dist/src/client/modify/alias.js +5 -2
  32. package/dist/src/client/modify/binary.d.ts +1 -1
  33. package/dist/src/client/modify/binary.js +1 -1
  34. package/dist/src/client/modify/cardinality.d.ts +2 -2
  35. package/dist/src/client/modify/cardinality.js +7 -5
  36. package/dist/src/client/modify/create.js +10 -5
  37. package/dist/src/client/modify/delete.d.ts +3 -1
  38. package/dist/src/client/modify/delete.js +12 -5
  39. package/dist/src/client/modify/expire.js +1 -1
  40. package/dist/src/client/modify/fixed.js +11 -4
  41. package/dist/src/client/modify/references/appendRefs.d.ts +4 -0
  42. package/dist/src/client/modify/references/appendRefs.js +27 -0
  43. package/dist/src/client/modify/references/edge.d.ts +0 -1
  44. package/dist/src/client/modify/references/edge.js +191 -71
  45. package/dist/src/client/modify/references/getEdgeSize.d.ts +3 -0
  46. package/dist/src/client/modify/references/getEdgeSize.js +27 -0
  47. package/dist/src/client/modify/references/reference.js +40 -26
  48. package/dist/src/client/modify/references/references.js +18 -6
  49. package/dist/src/client/modify/string.js +2 -1
  50. package/dist/src/client/modify/types.d.ts +1 -0
  51. package/dist/src/client/modify/types.js +1 -0
  52. package/dist/src/client/modify/update.js +3 -4
  53. package/dist/src/client/modify/upsert.js +1 -0
  54. package/dist/src/client/modify/vector.js +0 -2
  55. package/dist/src/client/query/BasedDbQuery.d.ts +4 -3
  56. package/dist/src/client/query/BasedDbQuery.js +12 -3
  57. package/dist/src/client/query/aggregationFn.d.ts +3 -0
  58. package/dist/src/client/query/aggregationFn.js +9 -0
  59. package/dist/src/client/query/debug.js +2 -6
  60. package/dist/src/client/query/display.js +5 -2
  61. package/dist/src/client/query/filter/createFixedFilterBuffer.d.ts +2 -2
  62. package/dist/src/client/query/filter/createFixedFilterBuffer.js +34 -26
  63. package/dist/src/client/query/filter/createReferenceFilter.d.ts +1 -1
  64. package/dist/src/client/query/filter/createReferenceFilter.js +17 -7
  65. package/dist/src/client/query/filter/createVariableFilterBuffer.d.ts +1 -1
  66. package/dist/src/client/query/filter/createVariableFilterBuffer.js +45 -22
  67. package/dist/src/client/query/filter/filter.d.ts +1 -1
  68. package/dist/src/client/query/filter/filter.js +5 -0
  69. package/dist/src/client/query/filter/parseFilterValue.js +1 -1
  70. package/dist/src/client/query/filter/toBuffer.d.ts +2 -2
  71. package/dist/src/client/query/filter/toBuffer.js +16 -9
  72. package/dist/src/client/query/include/props.js +18 -1
  73. package/dist/src/client/query/include/toBuffer.d.ts +1 -1
  74. package/dist/src/client/query/include/toBuffer.js +22 -12
  75. package/dist/src/client/query/include/walk.js +2 -1
  76. package/dist/src/client/query/query.d.ts +1 -0
  77. package/dist/src/client/query/query.js +1 -0
  78. package/dist/src/client/query/queryDef.js +2 -1
  79. package/dist/src/client/query/read/read.d.ts +2 -1
  80. package/dist/src/client/query/read/read.js +95 -62
  81. package/dist/src/client/query/registerQuery.d.ts +1 -1
  82. package/dist/src/client/query/registerQuery.js +2 -1
  83. package/dist/src/client/query/search/index.d.ts +1 -1
  84. package/dist/src/client/query/search/index.js +56 -24
  85. package/dist/src/client/query/subscription/markers.js +2 -1
  86. package/dist/src/client/query/toBuffer.d.ts +1 -1
  87. package/dist/src/client/query/toBuffer.js +80 -32
  88. package/dist/src/client/query/types.d.ts +25 -3
  89. package/dist/src/client/query/types.js +6 -0
  90. package/dist/src/client/query/validation.js +1 -1
  91. package/dist/src/client/string.d.ts +1 -1
  92. package/dist/src/client/string.js +1 -2
  93. package/dist/src/client/xxHash64.d.ts +1 -1
  94. package/dist/src/index.d.ts +4 -3
  95. package/dist/src/index.js +20 -11
  96. package/dist/src/native.d.ts +8 -13
  97. package/dist/src/native.js +9 -31
  98. package/dist/src/server/dbHash.d.ts +5 -0
  99. package/dist/src/server/dbHash.js +27 -0
  100. package/dist/src/server/index.d.ts +10 -10
  101. package/dist/src/server/index.js +54 -35
  102. package/dist/src/server/migrate/index.js +1 -1
  103. package/dist/src/server/migrate/worker.js +2 -2
  104. package/dist/src/server/save.d.ts +1 -1
  105. package/dist/src/server/save.js +10 -10
  106. package/dist/src/server/start.js +4 -2
  107. package/dist/src/server/tree.d.ts +1 -1
  108. package/dist/src/server/tree.js +1 -1
  109. package/dist/src/utils.d.ts +3 -0
  110. package/dist/src/utils.js +23 -5
  111. package/package.json +3 -2
  112. package/dist/src/client/query/read/types.d.ts +0 -4
  113. package/dist/src/client/query/read/types.js +0 -5
@@ -1,21 +1,22 @@
1
1
  // @ts-ignore
2
2
  import db from '../../basedDbNative.cjs';
3
- import { bufToHex } from './utils.js';
3
+ // Can't import these from utils or it would be a cyclic import.
4
+ const DECODER = new TextDecoder('utf-8');
5
+ const ENCODER = new TextEncoder();
4
6
  const selvaIoErrlog = new Uint8Array(256);
5
- const textEncoder = new TextEncoder();
6
7
  var compressor = db.createCompressor();
7
8
  var decompressor = db.createDecompressor();
8
9
  function SelvaIoErrlogToString(buf) {
9
10
  let i;
10
11
  let len = (i = buf.indexOf(0)) >= 0 ? i : buf.byteLength;
11
- return new TextDecoder().decode(selvaIoErrlog.slice(0, len));
12
+ return DECODER.decode(selvaIoErrlog.slice(0, len));
12
13
  }
13
14
  export default {
14
15
  historyAppend(history, typeId, nodeId, dbCtx) {
15
16
  return db.historyAppend(history, typeId, nodeId, dbCtx);
16
17
  },
17
18
  historyCreate(pathname, mainLen) {
18
- const pathBuf = textEncoder.encode(pathname + '\0');
19
+ const pathBuf = ENCODER.encode(pathname + '\0');
19
20
  return db.historyCreate(pathBuf, mainLen + 16 - (mainLen % 16));
20
21
  },
21
22
  workerCtxInit: () => {
@@ -41,22 +42,22 @@ export default {
41
42
  return db.stop(dbCtx);
42
43
  },
43
44
  saveCommon: (path, dbCtx) => {
44
- const pathBuf = textEncoder.encode(path + '\0');
45
+ const pathBuf = ENCODER.encode(path + '\0');
45
46
  return db.saveCommon(pathBuf, dbCtx);
46
47
  },
47
48
  saveRange: (path, typeCode, start, end, dbCtx, hashOut) => {
48
- const pathBuf = textEncoder.encode(path + '\0');
49
+ const pathBuf = ENCODER.encode(path + '\0');
49
50
  return db.saveRange(pathBuf, typeCode, start, end, dbCtx, hashOut);
50
51
  },
51
52
  loadCommon: (path, dbCtx) => {
52
- const pathBuf = textEncoder.encode(path + '\0');
53
+ const pathBuf = ENCODER.encode(path + '\0');
53
54
  const err = db.loadCommon(pathBuf, dbCtx, selvaIoErrlog);
54
55
  if (err) {
55
56
  throw new Error(`Failed to load common. selvaError: ${err} cause:\n${SelvaIoErrlogToString(selvaIoErrlog)}`);
56
57
  }
57
58
  },
58
59
  loadRange: (path, dbCtx) => {
59
- const pathBuf = textEncoder.encode(path + '\0');
60
+ const pathBuf = ENCODER.encode(path + '\0');
60
61
  const err = db.loadRange(pathBuf, dbCtx, selvaIoErrlog);
61
62
  if (err) {
62
63
  throw new Error(`Failed to load a range. selvaError: ${err} cause:\n${SelvaIoErrlogToString(selvaIoErrlog)}`);
@@ -71,29 +72,6 @@ export default {
71
72
  getNodeRangeHash: (typeId, start, end, bufOut, dbCtx) => {
72
73
  return db.getNodeRangeHash(typeId, start, end, bufOut, dbCtx);
73
74
  },
74
- createHash: () => {
75
- const state = db.hashCreate();
76
- const hash = {
77
- update: (buf) => {
78
- db.hashUpdate(state, buf);
79
- return hash;
80
- },
81
- digest: (encoding) => {
82
- const buf = new Uint8Array(16);
83
- db.hashDigest(state, buf);
84
- if (encoding === 'hex') {
85
- return bufToHex(buf);
86
- }
87
- else {
88
- return buf;
89
- }
90
- },
91
- reset: () => {
92
- db.hashReset(state);
93
- },
94
- };
95
- return hash;
96
- },
97
75
  compress: (buf, offset, stringSize) => {
98
76
  return db.compress(compressor, buf, offset, stringSize);
99
77
  },
@@ -0,0 +1,5 @@
1
+ export default function createHash(): {
2
+ update: (buf: Uint8Array) => any;
3
+ digest: (encoding?: "hex") => Uint8Array | string;
4
+ reset: () => void;
5
+ };
@@ -0,0 +1,27 @@
1
+ // @ts-ignore
2
+ import db from '../../../basedDbNative.cjs';
3
+ import { bufToHex } from '../utils.js';
4
+ export default function createHash() {
5
+ const state = db.hashCreate();
6
+ const hash = {
7
+ update: (buf) => {
8
+ db.hashUpdate(state, buf);
9
+ return hash;
10
+ },
11
+ digest: (encoding) => {
12
+ const buf = new Uint8Array(16);
13
+ db.hashDigest(state, buf);
14
+ if (encoding === 'hex') {
15
+ return bufToHex(buf);
16
+ }
17
+ else {
18
+ return buf;
19
+ }
20
+ },
21
+ reset: () => {
22
+ db.hashReset(state);
23
+ },
24
+ };
25
+ return hash;
26
+ }
27
+ //# sourceMappingURL=dbHash.js.map
@@ -5,8 +5,8 @@ import { Worker, MessagePort } from 'node:worker_threads';
5
5
  import { TransformFns } from './migrate/index.js';
6
6
  import exitHook from 'exit-hook';
7
7
  declare class SortIndex {
8
- constructor(buf: Buffer, dbCtxExternal: any);
9
- buf: Buffer;
8
+ constructor(buf: Uint8Array, dbCtxExternal: any);
9
+ buf: Uint8Array;
10
10
  idx: any;
11
11
  cnt: number;
12
12
  }
@@ -18,7 +18,7 @@ export declare class DbWorker {
18
18
  resolvers: any[];
19
19
  callback: (resolve: any) => void;
20
20
  updateCtx(address: BigInt): Promise<void>;
21
- getQueryBuf(buf: Buffer): Promise<Uint8Array>;
21
+ getQueryBuf(buf: Uint8Array): Promise<Uint8Array>;
22
22
  }
23
23
  type OnSchemaChange = (schema: StrictSchema) => void;
24
24
  export declare class DbServer {
@@ -36,15 +36,15 @@ export declare class DbServer {
36
36
  merkleTree: ReturnType<typeof createTree>;
37
37
  dirtyRanges: Set<number>;
38
38
  csmtHashFun: {
39
- update: (buf: Buffer | Uint8Array) => any;
39
+ update: (buf: Uint8Array) => any;
40
40
  digest: (encoding?: "hex") => Uint8Array | string;
41
41
  reset: () => void;
42
42
  };
43
43
  workers: DbWorker[];
44
44
  availableWorkerIndex: number;
45
45
  processingQueries: number;
46
- modifyQueue: Buffer[];
47
- queryQueue: Map<Function, Buffer>;
46
+ modifyQueue: Uint8Array[];
47
+ queryQueue: Map<Function, Uint8Array>;
48
48
  stopped: boolean;
49
49
  onSchemaChange: OnSchemaChange;
50
50
  unlistenExit: ReturnType<typeof exitHook>;
@@ -59,7 +59,7 @@ export declare class DbServer {
59
59
  }): Promise<void>;
60
60
  save(): void | Promise<void>;
61
61
  createCsmtHashFun: () => {
62
- update: (buf: Buffer | Uint8Array) => any;
62
+ update: (buf: Uint8Array) => any;
63
63
  digest: (encoding?: "hex") => Uint8Array | string;
64
64
  reset: () => void;
65
65
  };
@@ -82,13 +82,13 @@ export declare class DbServer {
82
82
  }>;
83
83
  createSortIndexBuffer(typeId: number, field: number, start: number, lang: number): SortIndex;
84
84
  updateMerkleTree(): void;
85
- putSchema(strictSchema: StrictSchema, fromStart?: boolean, transformFns?: TransformFns): (StrictSchema & {
85
+ setSchema(strictSchema: StrictSchema, fromStart?: boolean, transformFns?: TransformFns): (StrictSchema & {
86
86
  lastId: number;
87
87
  }) | Promise<StrictSchema & {
88
88
  lastId: number;
89
89
  }>;
90
- modify(buf: Buffer): Record<number, number>;
91
- getQueryBuf(buf: Buffer): Promise<Uint8Array>;
90
+ modify(buf: Uint8Array): Record<number, number>;
91
+ getQueryBuf(buf: Uint8Array): Promise<Uint8Array>;
92
92
  onQueryEnd(): void;
93
93
  stop(noSave?: boolean): Promise<void>;
94
94
  destroy(): Promise<void>;
@@ -1,4 +1,5 @@
1
1
  import native from '../native.js';
2
+ import createDbHash from './dbHash.js';
2
3
  import { rm, writeFile } from 'node:fs/promises';
3
4
  import { dirname, join } from 'node:path';
4
5
  import { getPropType, langCodesMap, } from '@based/schema';
@@ -24,6 +25,12 @@ class SortIndex {
24
25
  idx;
25
26
  cnt = 0;
26
27
  }
28
+ function readUint16LE(buf, off) {
29
+ return buf[off] | (buf[off + 1] << 8);
30
+ }
31
+ function readUint32LE(buf, off) {
32
+ return (buf[off] | (buf[off + 1] << 8) | (buf[off + 2] << 16) | (buf[off + 3] << 24));
33
+ }
27
34
  export class DbWorker {
28
35
  constructor(address, db) {
29
36
  const { port1, port2 } = new MessageChannel();
@@ -73,7 +80,7 @@ export class DbServer {
73
80
  maxModifySize;
74
81
  merkleTree;
75
82
  dirtyRanges = new Set();
76
- csmtHashFun = native.createHash();
83
+ csmtHashFun = createDbHash();
77
84
  workers = [];
78
85
  availableWorkerIndex = -1;
79
86
  processingQueries = 0;
@@ -152,13 +159,16 @@ export class DbServer {
152
159
  if (sortIndex) {
153
160
  return sortIndex;
154
161
  }
155
- const buf = Buffer.allocUnsafe(8);
162
+ const buf = new Uint8Array(8);
156
163
  // size [2 type] [1 field] [2 start] [2 len] [propIndex] [lang]
157
164
  // call createSortBuf here
158
- buf.writeUint16LE(t.id, 0);
165
+ buf[0] = t.id;
166
+ buf[1] = t.id >>> 8;
159
167
  buf[2] = prop.prop;
160
- buf.writeUint16LE(prop.start, 3);
161
- buf.writeUint16LE(prop.len, 5);
168
+ buf[3] = prop.start;
169
+ buf[4] = prop.start >>> 8;
170
+ buf[5] = prop.len;
171
+ buf[6] = prop.len >>> 8;
162
172
  buf[7] = prop.typeIndex;
163
173
  buf[8] = langCode;
164
174
  sortIndex = new SortIndex(buf, this.dbCtxExternal);
@@ -179,10 +189,12 @@ export class DbServer {
179
189
  let sortIndex = fields[prop.start];
180
190
  if (sortIndex) {
181
191
  // [2 type] [1 field] [2 start] [1 lang]
182
- const buf = Buffer.allocUnsafe(6);
183
- buf.writeUint16LE(t.id, 0);
192
+ const buf = new Uint8Array(6);
193
+ buf[0] = t.id;
194
+ buf[1] = t.id >>> 8;
184
195
  buf[2] = prop.prop;
185
- buf.writeUint16LE(prop.start, 3);
196
+ buf[3] = prop.start;
197
+ buf[4] = prop.start >>> 8;
186
198
  buf[5] =
187
199
  langCodesMap.get(lang ?? Object.keys(this.schema?.locales ?? 'en')[0]) ?? 0;
188
200
  native.destroySortIndex(buf, this.dbCtxExternal);
@@ -208,10 +220,12 @@ export class DbServer {
208
220
  return migrate(this, schema, transform);
209
221
  }
210
222
  createSortIndexBuffer(typeId, field, start, lang) {
211
- const buf = Buffer.allocUnsafe(9);
212
- buf.writeUint16LE(typeId, 0);
223
+ const buf = new Uint8Array(9);
224
+ buf[0] = typeId;
225
+ buf[1] = typeId >>> 8;
213
226
  buf[2] = field;
214
- buf.writeUint16LE(start, 3);
227
+ buf[3] = start;
228
+ buf[4] = start >>> 8;
215
229
  let typeDef;
216
230
  let prop;
217
231
  for (const t in this.schemaTypesParsed) {
@@ -233,7 +247,8 @@ export class DbServer {
233
247
  if (!prop) {
234
248
  throw new Error(`Cannot find prop on db from query for sort ${field}`);
235
249
  }
236
- buf.writeUint16LE(prop.len, 5);
250
+ buf[5] = prop.len;
251
+ buf[6] = prop.len >>> 8;
237
252
  buf[7] = prop.typeIndex;
238
253
  buf[8] = lang;
239
254
  // put in modify stuff
@@ -247,7 +262,7 @@ export class DbServer {
247
262
  updateMerkleTree() {
248
263
  foreachDirtyBlock(this, (mtKey, typeId, start, end) => {
249
264
  const oldLeaf = this.merkleTree.search(mtKey);
250
- const hash = Buffer.allocUnsafe(16);
265
+ const hash = new Uint8Array(16);
251
266
  native.getNodeRangeHash(typeId, start, end, hash, this.dbCtxExternal);
252
267
  if (oldLeaf) {
253
268
  if (hashEq(oldLeaf.hash, hash)) {
@@ -267,7 +282,7 @@ export class DbServer {
267
282
  this.merkleTree.insert(mtKey, hash, data);
268
283
  });
269
284
  }
270
- putSchema(strictSchema, fromStart = false, transformFns) {
285
+ setSchema(strictSchema, fromStart = false, transformFns) {
271
286
  if (!fromStart && Object.keys(this.schema.types).length > 0) {
272
287
  return this.migrateSchema(strictSchema, transformFns);
273
288
  }
@@ -338,17 +353,18 @@ export class DbServer {
338
353
  }
339
354
  if (strictSchema.props) {
340
355
  // insert a root node
341
- const data = [2, 1, 0, 0, 0, 17, 9, 1, 0, 0, 0, 7, 1, 0, 1];
356
+ const data = [2, 1, 0, 0, 0, 1, 9, 1, 0, 0, 0, 7, 1, 0, 1];
342
357
  const blockKey = makeCsmtKey(1, 1);
343
- const buf = Buffer.alloc(data.length + 2 + 8 + 4);
358
+ const buf = new Uint8Array(data.length + 2 + 8 + 4);
359
+ const view = new DataView(buf.buffer, 0, buf.byteLength);
344
360
  // add content
345
361
  buf.set(data);
346
362
  // add typesLen
347
- buf.writeDoubleLE(0, data.length);
363
+ view.setFloat64(data.length, 0, true);
348
364
  // add dirty key
349
- buf.writeDoubleLE(blockKey, data.length + 2);
365
+ view.setFloat64(data.length + 2, blockKey, true);
350
366
  // add dataLen
351
- buf.writeUint32LE(data.length, buf.length - 4);
367
+ view.setUint32(buf.length - 4, data.length, true);
352
368
  this.modify(buf);
353
369
  }
354
370
  }
@@ -357,24 +373,26 @@ export class DbServer {
357
373
  }
358
374
  modify(buf) {
359
375
  const offsets = {};
360
- const dataLen = buf.readUint32LE(buf.length - 4);
361
- let typesSize = buf.readUint16LE(dataLen);
376
+ const dataLen = readUint32LE(buf, buf.length - 4);
377
+ let typesSize = readUint16LE(buf, dataLen);
362
378
  let i = dataLen + 2;
363
379
  while (typesSize--) {
364
- const typeId = buf.readUint16LE(i);
380
+ const typeId = readUint16LE(buf, i);
365
381
  i += 2;
366
- const startId = buf.readUint32LE(i);
382
+ const startId = readUint32LE(buf, i);
367
383
  const def = this.schemaTypesParsedById[typeId];
368
384
  const offset = def.lastId - startId;
369
- buf.writeUint32LE(offset, i);
370
- i += 4;
371
- const lastId = buf.readUint32LE(i);
385
+ buf[i++] = offset;
386
+ buf[i++] = offset >>> 8;
387
+ buf[i++] = offset >>> 16;
388
+ buf[i++] = offset >>> 24;
389
+ const lastId = readUint32LE(buf, i);
372
390
  i += 4;
373
391
  def.lastId = lastId + offset;
374
392
  offsets[typeId] = offset;
375
393
  }
376
394
  if (this.processingQueries) {
377
- this.modifyQueue.push(Buffer.from(buf));
395
+ this.modifyQueue.push(new Uint8Array(buf));
378
396
  }
379
397
  else {
380
398
  this.#modify(buf);
@@ -383,21 +401,22 @@ export class DbServer {
383
401
  }
384
402
  #modify(buf) {
385
403
  const end = buf.length - 4;
386
- const dataLen = buf.readUint32LE(end);
387
- let typesSize = buf.readUint16LE(dataLen);
404
+ const dataLen = readUint32LE(buf, end);
405
+ let typesSize = readUint16LE(buf, dataLen);
388
406
  const typesLen = typesSize * 10;
389
407
  const types = buf.subarray(dataLen + 2, dataLen + typesLen + 2);
390
408
  const data = buf.subarray(0, dataLen);
391
409
  let i = dataLen + 2;
392
410
  while (typesSize--) {
393
- const typeId = buf.readUint16LE(i);
411
+ const typeId = readUint16LE(buf, i);
394
412
  const def = this.schemaTypesParsedById[typeId];
395
413
  const key = makeCsmtKeyFromNodeId(def.id, def.blockCapacity, def.lastId);
396
414
  this.dirtyRanges.add(key);
397
415
  i += 10;
398
416
  }
417
+ const view = new DataView(buf.buffer, buf.byteOffset);
399
418
  while (i < end) {
400
- const key = buf.readDoubleLE(i);
419
+ const key = view.getFloat64(i, true);
401
420
  this.dirtyRanges.add(key);
402
421
  i += 8;
403
422
  }
@@ -412,13 +431,13 @@ export class DbServer {
412
431
  else {
413
432
  const queryType = buf[0];
414
433
  if (queryType == 2) {
415
- const s = 13 + buf.readUint16LE(11);
416
- const sortLen = buf.readUint16LE(s);
434
+ const s = 13 + readUint16LE(buf, 11);
435
+ const sortLen = readUint16LE(buf, s);
417
436
  if (sortLen) {
418
- const typeId = buf.readUint16LE(1);
437
+ const typeId = readUint16LE(buf, 1);
419
438
  const sort = buf.slice(s + 2, s + 2 + sortLen);
420
439
  const field = sort[1];
421
- const start = sort.readUint16LE(2 + 1);
440
+ const start = readUint16LE(sort, 3);
422
441
  let sortIndex = this.getSortIndex(typeId, field, start, 0);
423
442
  if (!sortIndex) {
424
443
  if (this.processingQueries) {
@@ -39,7 +39,7 @@ export const migrate = async (fromDbServer, toSchema, transform) => {
39
39
  await toDb.destroy();
40
40
  return fromDbServer.schema;
41
41
  }
42
- toSchema = await toDb.putSchema(toSchema);
42
+ toSchema = await toDb.setSchema(toSchema);
43
43
  const fromSchema = fromDbServer.schema;
44
44
  const fromCtx = fromDbServer.dbCtxExternal;
45
45
  const toCtx = toDb.server.dbCtxExternal;
@@ -14,8 +14,8 @@ else {
14
14
  const toDb = new BasedDb({ path });
15
15
  fromDb.server.dbCtxExternal = fromCtx;
16
16
  toDb.server.dbCtxExternal = toCtx;
17
- await fromDb.putSchema(fromSchema, true);
18
- await toDb.putSchema(toSchema, true);
17
+ await fromDb.setSchema(fromSchema, true);
18
+ await toDb.setSchema(toSchema, true);
19
19
  const map = {};
20
20
  for (const type in fromDb.client.schemaTypesParsed) {
21
21
  const { id, props } = fromDb.client.schemaTypesParsed[type];
@@ -1,2 +1,2 @@
1
1
  import { DbServer } from './index.js';
2
- export declare function save(db: DbServer, sync?: boolean): void | Promise<void>;
2
+ export declare function save<T extends boolean>(db: DbServer, sync?: T): T extends true ? void : Promise<void>;
@@ -42,25 +42,25 @@ export function save(db, sync = false) {
42
42
  });
43
43
  db.dirtyRanges.clear();
44
44
  const types = {};
45
+ const rangeDumps = {};
45
46
  for (const key in db.schemaTypesParsed) {
46
- const def = db.schemaTypesParsed[key];
47
- types[def.id] = { lastId: def.lastId, blockCapacity: def.blockCapacity };
48
- }
49
- const dumps = {};
50
- for (const key in db.schemaTypesParsed) {
51
- const def = db.schemaTypesParsed[key];
52
- dumps[def.id] = [];
47
+ const { id, lastId, blockCapacity } = db.schemaTypesParsed[key];
48
+ types[id] = { lastId, blockCapacity };
49
+ rangeDumps[id] = [];
53
50
  }
54
51
  db.merkleTree.visitLeafNodes((leaf) => {
55
- const [typeId] = destructureCsmtKey(leaf.key);
52
+ const [typeId, start] = destructureCsmtKey(leaf.key);
56
53
  const data = leaf.data;
57
- dumps[typeId].push({ ...data, hash: bufToHex(leaf.hash) });
54
+ if (start != data.start) {
55
+ console.error(`csmtKey start and range start mismatch: ${start} != ${data.start}`);
56
+ }
57
+ rangeDumps[typeId].push({ ...data, hash: bufToHex(leaf.hash) });
58
58
  });
59
59
  const data = {
60
60
  ts,
61
61
  types,
62
62
  commonDump: COMMON_SDB_FILE,
63
- rangeDumps: dumps,
63
+ rangeDumps,
64
64
  };
65
65
  const mtRoot = db.merkleTree.getRoot();
66
66
  if (mtRoot) {
@@ -52,7 +52,7 @@ export async function start(db, opts) {
52
52
  const schema = await readFile(join(path, SCHEMA_FILE));
53
53
  if (schema) {
54
54
  // Prop need to not call setting in selva
55
- db.putSchema(JSON.parse(schema.toString()), true);
55
+ db.setSchema(JSON.parse(schema.toString()), true);
56
56
  }
57
57
  }
58
58
  catch (err) {
@@ -65,7 +65,9 @@ export async function start(db, opts) {
65
65
  // FDN-791 CSMT is unstable (not history independent)
66
66
  // For now we just sort the types to ensure that we always
67
67
  // load in the same order.
68
- const types = Object.keys(db.schemaTypesParsed).sort((a, b) => db.schemaTypesParsed[a].id - db.schemaTypesParsed[b].id).reduce((obj, key) => {
68
+ const types = Object.keys(db.schemaTypesParsed)
69
+ .sort((a, b) => db.schemaTypesParsed[a].id - db.schemaTypesParsed[b].id)
70
+ .reduce((obj, key) => {
69
71
  obj[key] = db.schemaTypesParsed[key];
70
72
  return obj;
71
73
  }, {});
@@ -6,8 +6,8 @@ export type CsmtNodeRange = {
6
6
  start: number;
7
7
  end: number;
8
8
  };
9
- export declare const makeCsmtKey: (typeId: number, start: number) => number;
10
9
  export declare const destructureCsmtKey: (key: number) => number[];
10
+ export declare const makeCsmtKey: (typeId: number, start: number) => number;
11
11
  export declare const makeCsmtKeyFromNodeId: (typeId: number, blockCapacity: number, nodeId: number) => number;
12
12
  export declare function foreachBlock(db: DbServer, def: SchemaTypeDef, cb: (start: number, end: number, hash: Uint8Array) => void): Promise<void>;
13
13
  export declare function foreachDirtyBlock(db: DbServer, cb: (mtKey: number, typeId: number, start: number, end: number) => void): Promise<void>;
@@ -1,9 +1,9 @@
1
1
  import native from '../native.js';
2
- export const makeCsmtKey = (typeId, start) => typeId * 4294967296 + start;
3
2
  export const destructureCsmtKey = (key) => [
4
3
  (key / 4294967296) | 0, // typeId
5
4
  (key >>> 31) * 2147483648 + (key & 0x7fffffff), // start_node_id
6
5
  ];
6
+ export const makeCsmtKey = (typeId, start) => typeId * 4294967296 + start;
7
7
  export const makeCsmtKeyFromNodeId = (typeId, blockCapacity, nodeId) => {
8
8
  const tmp = nodeId - +!(nodeId % blockCapacity);
9
9
  return typeId * 4294967296 + ((tmp / blockCapacity) | 0) * blockCapacity + 1;
@@ -1,4 +1,7 @@
1
+ export declare const DECODER: TextDecoder;
2
+ export declare const ENCODER: TextEncoder;
1
3
  export declare const equals: (aB: Uint8Array, bB: Uint8Array) => boolean;
4
+ export declare function concatUint8Arr(bufs: Uint8Array[], totalByteLength?: number): Uint8Array;
2
5
  export declare const bufToHex: (a: Uint8Array) => string;
3
6
  export declare const hexToBuf: (s: string) => Uint8Array;
4
7
  export declare const base64encode: (a: Uint8Array, lineMax?: number) => string;
package/dist/src/utils.js CHANGED
@@ -1,11 +1,14 @@
1
- import native from './native.js';
1
+ import { encodeBase64 } from '@saulx/utils';
2
+ const native = (typeof window === 'undefined') ? (await import('./native.js')).default : null;
3
+ export const DECODER = new TextDecoder('utf-8');
4
+ export const ENCODER = new TextEncoder();
2
5
  export const equals = (aB, bB) => {
3
6
  const len = aB.byteLength;
4
7
  if (len != bB.byteLength) {
5
8
  return false;
6
9
  }
7
10
  let i = 0;
8
- if (len < 50) {
11
+ if (len < 50 || !native) {
9
12
  while (i < len) {
10
13
  if (aB[i] != bB[i]) {
11
14
  return false;
@@ -24,8 +27,18 @@ export const equals = (aB, bB) => {
24
27
  return native.equals(aB, bB);
25
28
  }
26
29
  };
27
- const charMap = new TextEncoder().encode("0123456789abcdef");
28
- const DECODER = new TextDecoder();
30
+ export function concatUint8Arr(bufs, totalByteLength) {
31
+ totalByteLength = totalByteLength ?? bufs.reduce((acc, cur) => acc + cur.byteLength, 0);
32
+ const res = new Uint8Array(totalByteLength);
33
+ let off = 0;
34
+ for (let i = 0; i < bufs.length; i++) {
35
+ const buf = bufs[i];
36
+ res.set(buf, off);
37
+ off += buf.byteLength;
38
+ }
39
+ return res;
40
+ }
41
+ const charMap = ENCODER.encode('0123456789abcdef');
29
42
  // Uint8Array.fromHex() and Uint8Array.toHex() are not available in V8
30
43
  // https://issues.chromium.org/issues/42204568
31
44
  export const bufToHex = (a) => {
@@ -79,6 +92,11 @@ function base64OutLen(n, lineMax) {
79
92
  export const base64encode = (a, lineMax = 72) => {
80
93
  // TODO Could fallback to @saulx/utils if native is not available
81
94
  const tmp = new Uint8Array(base64OutLen(a.byteLength, lineMax));
82
- return DECODER.decode(native.base64encode(tmp, a, lineMax));
95
+ if ((a.length < 10 && lineMax === 72) || !native) {
96
+ return encodeBase64(a);
97
+ }
98
+ else {
99
+ return DECODER.decode(native.base64encode(tmp, a, lineMax));
100
+ }
83
101
  };
84
102
  //# sourceMappingURL=utils.js.map
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@based/db",
3
- "version": "0.0.21",
3
+ "version": "0.0.23",
4
4
  "license": "MIT",
5
5
  "type": "module",
6
6
  "main": "./dist/src/index.js",
@@ -17,6 +17,7 @@
17
17
  "test": "npm run build && LOCPATH=../locale/locale-x86_64-gnu/locale ./scripts/lldb-node ./scripts/test.js",
18
18
  "test-gdb": "npm run build && LOCPATH=../locale/locale-x86_64-gnu/locale gdb -ex run --args node ./scripts/test.js",
19
19
  "test-fast": "LOCPATH=../locale/locale-x86_64-gnu/locale node ./scripts/test.js",
20
+ "test-fast-linux_aarch64": "podman run --rm -v \"$PWD/../..:/usr/src/based-db\" based-db-clibs-build-linux_aarch64 sh -c '\\. \"/usr/local/nvm/nvm.sh\"; cd /usr/src/based-db/packages/db; npm run test-fast'",
20
21
  "test-zig": "npm run build-zig -- debug && tsc && npm run test-fast",
21
22
  "test-ts": "tsc && node ./scripts/test.js",
22
23
  "perf": "npm run build && node benchmarks/references.js && node benchmarks/transfermarkt/transfermarkt-based.js"
@@ -31,7 +32,7 @@
31
32
  "basedDbNative.cjs"
32
33
  ],
33
34
  "dependencies": {
34
- "@based/schema": "5.0.0-alpha.5",
35
+ "@based/schema": "5.0.0-alpha.6",
35
36
  "@saulx/hash": "^3.0.0",
36
37
  "@saulx/utils": "^4.3.2",
37
38
  "exit-hook": "^4.0.0",
@@ -1,4 +0,0 @@
1
- export declare const READ_ID = 255;
2
- export declare const READ_EDGE = 252;
3
- export declare const READ_REFERENCES = 253;
4
- export declare const READ_REFERENCE = 254;
@@ -1,5 +0,0 @@
1
- export const READ_ID = 255;
2
- export const READ_EDGE = 252;
3
- export const READ_REFERENCES = 253;
4
- export const READ_REFERENCE = 254;
5
- //# sourceMappingURL=types.js.map