@based/db 0.0.28 → 0.0.29

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. package/README.md +1 -399
  2. package/dist/lib/darwin_aarch64/include/selva/db.h +8 -4
  3. package/dist/lib/darwin_aarch64/include/selva/selva_hash128.h +17 -7
  4. package/dist/lib/darwin_aarch64/include/selva/sort.h +21 -14
  5. package/dist/lib/darwin_aarch64/libdeflate.dylib +0 -0
  6. package/dist/lib/darwin_aarch64/libjemalloc_selva.2.dylib +0 -0
  7. package/dist/lib/darwin_aarch64/libnode-v20.node +0 -0
  8. package/dist/lib/darwin_aarch64/libnode-v21.node +0 -0
  9. package/dist/lib/darwin_aarch64/libnode-v22.node +0 -0
  10. package/dist/lib/darwin_aarch64/libnode-v23.node +0 -0
  11. package/dist/lib/darwin_aarch64/libselva.dylib +0 -0
  12. package/dist/lib/darwin_aarch64/libxxhash.dylib +0 -0
  13. package/dist/lib/linux_aarch64/include/selva/db.h +8 -4
  14. package/dist/lib/linux_aarch64/include/selva/selva_hash128.h +17 -7
  15. package/dist/lib/linux_aarch64/include/selva/sort.h +21 -14
  16. package/dist/lib/linux_aarch64/libdeflate.so +0 -0
  17. package/dist/lib/linux_aarch64/libjemalloc_selva.so.2 +0 -0
  18. package/dist/lib/linux_aarch64/libnode-v20.node +0 -0
  19. package/dist/lib/linux_aarch64/libnode-v21.node +0 -0
  20. package/dist/lib/linux_aarch64/libnode-v22.node +0 -0
  21. package/dist/lib/linux_aarch64/libnode-v23.node +0 -0
  22. package/dist/lib/linux_aarch64/libselva.so +0 -0
  23. package/dist/lib/linux_x86_64/include/selva/db.h +8 -4
  24. package/dist/lib/linux_x86_64/include/selva/selva_hash128.h +17 -7
  25. package/dist/lib/linux_x86_64/include/selva/sort.h +21 -14
  26. package/dist/lib/linux_x86_64/libjemalloc_selva.so.2 +0 -0
  27. package/dist/lib/linux_x86_64/libnode-v20.node +0 -0
  28. package/dist/lib/linux_x86_64/libnode-v21.node +0 -0
  29. package/dist/lib/linux_x86_64/libnode-v22.node +0 -0
  30. package/dist/lib/linux_x86_64/libnode-v23.node +0 -0
  31. package/dist/lib/linux_x86_64/libselva.so +0 -0
  32. package/dist/src/client/flushModify.d.ts +1 -1
  33. package/dist/src/client/flushModify.js +12 -17
  34. package/dist/src/client/index.d.ts +6 -4
  35. package/dist/src/client/index.js +19 -2
  36. package/dist/src/client/modify/ModifyRes.d.ts +1 -1
  37. package/dist/src/client/modify/ModifyRes.js +14 -18
  38. package/dist/src/client/modify/fixed.js +43 -8
  39. package/dist/src/client/modify/modify.js +0 -1
  40. package/dist/src/client/modify/references/{appendRefs.d.ts → appendEdgeRefs.d.ts} +1 -1
  41. package/dist/src/client/modify/references/{appendRefs.js → appendEdgeRefs.js} +5 -2
  42. package/dist/src/client/modify/references/edge.js +182 -175
  43. package/dist/src/client/modify/references/reference.js +4 -8
  44. package/dist/src/client/modify/references/references.js +18 -14
  45. package/dist/src/client/modify/string.js +0 -3
  46. package/dist/src/client/modify/text.js +11 -3
  47. package/dist/src/client/modify/types.d.ts +11 -0
  48. package/dist/src/client/modify/types.js +10 -0
  49. package/dist/src/client/modify/update.js +4 -1
  50. package/dist/src/client/modify/vector.js +13 -4
  51. package/dist/src/client/query/BasedDbQuery.d.ts +1 -1
  52. package/dist/src/client/query/BasedDbQuery.js +2 -2
  53. package/dist/src/client/query/BasedIterable.d.ts +1 -1
  54. package/dist/src/client/query/BasedIterable.js +7 -2
  55. package/dist/src/client/query/filter/createFixedFilterBuffer.d.ts +2 -1
  56. package/dist/src/client/query/filter/createFixedFilterBuffer.js +11 -28
  57. package/dist/src/client/query/filter/createReferenceFilter.d.ts +2 -1
  58. package/dist/src/client/query/filter/createReferenceFilter.js +10 -9
  59. package/dist/src/client/query/filter/createVariableFilterBuffer.d.ts +2 -1
  60. package/dist/src/client/query/filter/createVariableFilterBuffer.js +8 -10
  61. package/dist/src/client/query/filter/parseFilterValue.js +1 -1
  62. package/dist/src/client/query/filter/primitiveFilter.js +9 -9
  63. package/dist/src/client/query/filter/toBuffer.js +0 -15
  64. package/dist/src/client/query/filter/types.d.ts +1 -0
  65. package/dist/src/client/query/filter/types.js +1 -0
  66. package/dist/src/client/query/include/walk.js +0 -1
  67. package/dist/src/client/query/read/read.js +4 -4
  68. package/dist/src/client/query/search/index.js +11 -15
  69. package/dist/src/client/query/subscription/markers.js +1 -2
  70. package/dist/src/client/query/subscription/run.js +0 -2
  71. package/dist/src/client/query/thresholds.d.ts +0 -2
  72. package/dist/src/client/query/thresholds.js +0 -2
  73. package/dist/src/client/query/toBuffer.js +16 -42
  74. package/dist/src/client/query/types.d.ts +3 -2
  75. package/dist/src/client/query/validation.d.ts +1 -3
  76. package/dist/src/client/query/validation.js +6 -18
  77. package/dist/src/client/string.d.ts +2 -0
  78. package/dist/src/client/string.js +9 -13
  79. package/dist/src/index.d.ts +1 -1
  80. package/dist/src/index.js +7 -15
  81. package/dist/src/native.d.ts +1 -1
  82. package/dist/src/native.js +3 -3
  83. package/dist/src/server/csmt/draw-dot.js +2 -2
  84. package/dist/src/server/csmt/tree.js +57 -6
  85. package/dist/src/server/csmt/types.d.ts +5 -0
  86. package/dist/src/server/index.d.ts +4 -3
  87. package/dist/src/server/index.js +44 -44
  88. package/dist/src/server/migrate/index.js +47 -29
  89. package/dist/src/server/migrate/worker.js +2 -2
  90. package/dist/src/server/save.js +40 -28
  91. package/dist/src/server/start.js +7 -19
  92. package/dist/src/server/tree.d.ts +2 -0
  93. package/dist/src/server/tree.js +34 -2
  94. package/dist/src/server/worker.js +3 -3
  95. package/dist/src/utils.d.ts +3 -1
  96. package/dist/src/utils.js +43 -19
  97. package/package.json +9 -3
  98. package/dist/lib/darwin_aarch64/include/selva/base64.h +0 -59
  99. package/dist/lib/darwin_aarch64/include/selva/base64url.h +0 -59
  100. package/dist/lib/linux_aarch64/include/selva/base64.h +0 -59
  101. package/dist/lib/linux_aarch64/include/selva/base64url.h +0 -59
  102. package/dist/lib/linux_x86_64/include/selva/base64.h +0 -59
  103. package/dist/lib/linux_x86_64/include/selva/base64url.h +0 -59
  104. package/dist/src/client/timestamp.d.ts +0 -1
  105. package/dist/src/client/timestamp.js +0 -68
@@ -4,14 +4,14 @@ import { rm, writeFile } from 'node:fs/promises';
4
4
  import { dirname, join } from 'node:path';
5
5
  import { getPropType, langCodesMap, } from '@based/schema';
6
6
  import { updateTypeDefs, schemaToSelvaBuffer, } from '@based/schema/def';
7
- import { hashEq } from './csmt/index.js';
8
7
  import { start } from './start.js';
9
- import { foreachDirtyBlock, makeCsmtKey, makeCsmtKeyFromNodeId, } from './tree.js';
8
+ import { initCsmt, makeCsmtKey, makeCsmtKeyFromNodeId } from './tree.js';
10
9
  import { save } from './save.js';
11
10
  import { Worker, MessageChannel } from 'node:worker_threads';
12
11
  import { fileURLToPath } from 'node:url';
13
12
  import { setTimeout } from 'node:timers/promises';
14
13
  import { migrate } from './migrate/index.js';
14
+ import { debugServer } from '../utils.js';
15
15
  export const SCHEMA_FILE = 'schema.json';
16
16
  export const WRITELOG_FILE = 'writelog.json';
17
17
  const __filename = fileURLToPath(import.meta.url);
@@ -45,10 +45,6 @@ export class DbWorker {
45
45
  transferList: [port2],
46
46
  });
47
47
  port1.on('message', (buf) => {
48
- // TODO FIX TYPES CHECK IF THIS MAKES A COPY
49
- // It's a copy, if you don't want a copy you'd need to make it an explicit view
50
- // to the underlying buffer:
51
- // new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength)
52
48
  this.resolvers.shift()(new Uint8Array(buf));
53
49
  this.db.onQueryEnd();
54
50
  });
@@ -93,11 +89,14 @@ export class DbServer {
93
89
  stopped;
94
90
  onSchemaChange;
95
91
  unlistenExit;
96
- constructor({ path, maxModifySize = 100 * 1e3 * 1e3, onSchemaChange, }) {
92
+ constructor({ path, maxModifySize = 100 * 1e3 * 1e3, onSchemaChange, debug, }) {
97
93
  this.maxModifySize = maxModifySize;
98
94
  this.fileSystemPath = path;
99
95
  this.sortIndexes = {};
100
96
  this.onSchemaChange = onSchemaChange;
97
+ if (debug) {
98
+ debugServer(this);
99
+ }
101
100
  }
102
101
  #resizeModifyDirtyRanges() {
103
102
  let maxNrChanges = 0;
@@ -141,7 +140,7 @@ export class DbServer {
141
140
  for (const lang in this.sortIndexes[type][field][start]) {
142
141
  const sortIndex = this.sortIndexes[type][field][start][lang];
143
142
  sortIndex.cnt /= 2;
144
- if (sortIndex.cnt < 1) {
143
+ if (sortIndex.cnt < 1 && !this.processingQueries) {
145
144
  native.destroySortIndex(sortIndex.buf, this.dbCtxExternal);
146
145
  delete this.sortIndexes[type][field][start][lang];
147
146
  }
@@ -179,7 +178,7 @@ export class DbServer {
179
178
  if (sortIndex) {
180
179
  return sortIndex;
181
180
  }
182
- const buf = new Uint8Array(8);
181
+ const buf = new Uint8Array(9);
183
182
  // size [2 type] [1 field] [2 start] [2 len] [propIndex] [lang]
184
183
  // call createSortBuf here
185
184
  buf[0] = t.id;
@@ -208,7 +207,6 @@ export class DbServer {
208
207
  }
209
208
  let sortIndex = fields[prop.start];
210
209
  if (sortIndex) {
211
- // [2 type] [1 field] [2 start] [1 lang]
212
210
  const buf = new Uint8Array(6);
213
211
  buf[0] = t.id;
214
212
  buf[1] = t.id >>> 8;
@@ -279,29 +277,6 @@ export class DbServer {
279
277
  fields[start][lang] = sortIndex;
280
278
  return sortIndex;
281
279
  }
282
- updateMerkleTree() {
283
- foreachDirtyBlock(this, (mtKey, typeId, start, end) => {
284
- const oldLeaf = this.merkleTree.search(mtKey);
285
- const hash = new Uint8Array(16);
286
- native.getNodeRangeHash(typeId, start, end, hash, this.dbCtxExternal);
287
- if (oldLeaf) {
288
- if (hashEq(oldLeaf.hash, hash)) {
289
- return;
290
- }
291
- try {
292
- this.merkleTree.delete(mtKey);
293
- }
294
- catch (err) { }
295
- }
296
- const data = {
297
- file: '', // not saved yet
298
- typeId,
299
- start,
300
- end,
301
- };
302
- this.merkleTree.insert(mtKey, hash, data);
303
- });
304
- }
305
280
  setSchema(strictSchema, fromStart = false, transformFns) {
306
281
  if (!fromStart && Object.keys(this.schema.types).length > 0) {
307
282
  return this.migrateSchema(strictSchema, transformFns);
@@ -373,6 +348,7 @@ export class DbServer {
373
348
  }
374
349
  if (strictSchema.props) {
375
350
  // insert a root node
351
+ // TODO fix this add it in schema at least
376
352
  const data = [2, 1, 0, 0, 0, 1, 9, 1, 0, 0, 0, 7, 1, 0, 1];
377
353
  const blockKey = makeCsmtKey(1, 1);
378
354
  const buf = new Uint8Array(data.length + 2 + 8 + 4);
@@ -387,6 +363,7 @@ export class DbServer {
387
363
  view.setUint32(buf.length - 4, data.length, true);
388
364
  this.modify(buf);
389
365
  }
366
+ initCsmt(this);
390
367
  }
391
368
  this.onSchemaChange?.(this.schema);
392
369
  return this.schema;
@@ -401,7 +378,16 @@ export class DbServer {
401
378
  i += 2;
402
379
  const startId = readUint32LE(buf, i);
403
380
  const def = this.schemaTypesParsedById[typeId];
404
- const offset = def.lastId - startId;
381
+ let offset = def.lastId - startId;
382
+ if (offset < 0) {
383
+ console.log('-----------------');
384
+ console.log(def.type, {
385
+ offset,
386
+ serverId: def.lastId,
387
+ clientId: startId,
388
+ });
389
+ offset = 0;
390
+ }
405
391
  buf[i++] = offset;
406
392
  buf[i++] = offset >>> 8;
407
393
  buf[i++] = offset >>> 16;
@@ -411,6 +397,7 @@ export class DbServer {
411
397
  def.lastId = lastId + offset;
412
398
  offsets[typeId] = offset;
413
399
  }
400
+ // console.log('modify', this.processingQueries)
414
401
  if (this.processingQueries) {
415
402
  this.modifyQueue.push(new Uint8Array(buf));
416
403
  }
@@ -449,10 +436,17 @@ export class DbServer {
449
436
  this.dirtyRanges.add(key);
450
437
  }
451
438
  }
452
- getQueryBuf(buf) {
439
+ addToQueryQueue(resolve, buf) {
440
+ if (this.queryQueue.size === 16777216) {
441
+ resolve(new Error('Query queue exceeded'));
442
+ return;
443
+ }
444
+ this.queryQueue.set(resolve, buf);
445
+ }
446
+ getQueryBuf(buf, fromQueue = false) {
453
447
  if (this.modifyQueue.length) {
454
448
  return new Promise((resolve) => {
455
- this.queryQueue.set(resolve, buf);
449
+ this.addToQueryQueue(resolve, buf);
456
450
  });
457
451
  }
458
452
  else {
@@ -469,7 +463,7 @@ export class DbServer {
469
463
  if (!sortIndex) {
470
464
  if (this.processingQueries) {
471
465
  return new Promise((resolve) => {
472
- this.queryQueue.set(resolve, buf);
466
+ this.addToQueryQueue(resolve, buf);
473
467
  });
474
468
  }
475
469
  sortIndex = this.createSortIndexBuffer(typeId, field, start, sort[sort.byteLength - 1]);
@@ -482,6 +476,9 @@ export class DbServer {
482
476
  else if (queryType == 1) {
483
477
  // This will be more advanced - sometimes has indexes / sometimes not
484
478
  }
479
+ if (!fromQueue) {
480
+ native.expire(this.dbCtxExternal);
481
+ }
485
482
  this.availableWorkerIndex =
486
483
  (this.availableWorkerIndex + 1) % this.workers.length;
487
484
  return this.workers[this.availableWorkerIndex].getQueryBuf(buf);
@@ -491,16 +488,19 @@ export class DbServer {
491
488
  this.processingQueries--;
492
489
  if (this.processingQueries === 0) {
493
490
  if (this.modifyQueue.length) {
494
- for (const buf of this.modifyQueue) {
491
+ const modifyQueue = this.modifyQueue;
492
+ this.modifyQueue = [];
493
+ for (const buf of modifyQueue) {
495
494
  this.#modify(buf);
496
495
  }
497
- this.modifyQueue = [];
498
496
  }
499
497
  if (this.queryQueue.size) {
500
- for (const [resolve, buf] of this.queryQueue) {
501
- resolve(this.getQueryBuf(buf));
498
+ const queryQueue = this.queryQueue;
499
+ this.queryQueue = new Map();
500
+ native.expire(this.dbCtxExternal);
501
+ for (const [resolve, buf] of queryQueue) {
502
+ resolve(this.getQueryBuf(buf, true));
502
503
  }
503
- this.queryQueue.clear();
504
504
  }
505
505
  }
506
506
  }
@@ -510,7 +510,7 @@ export class DbServer {
510
510
  }
511
511
  this.stopped = true;
512
512
  clearTimeout(this.cleanupTimer);
513
- this?.unlistenExit();
513
+ this.unlistenExit();
514
514
  try {
515
515
  if (!noSave) {
516
516
  await this.save();
@@ -518,7 +518,7 @@ export class DbServer {
518
518
  await Promise.all(this.workers.map(({ worker }) => worker.terminate()));
519
519
  this.workers = [];
520
520
  native.stop(this.dbCtxExternal);
521
- await setTimeout();
521
+ await setTimeout(20);
522
522
  }
523
523
  catch (e) {
524
524
  this.stopped = false;
@@ -4,7 +4,7 @@ import { tmpdir } from 'os';
4
4
  import { Worker, MessageChannel, receiveMessageOnPort, } from 'node:worker_threads';
5
5
  import native from '../../native.js';
6
6
  import './worker.js';
7
- import { foreachDirtyBlock } from '../tree.js';
7
+ import { destructureCsmtKey, foreachDirtyBlock, specialBlock } from '../tree.js';
8
8
  import { SCHEMA_FILE } from '../index.js';
9
9
  import { fileURLToPath } from 'url';
10
10
  import { deepMerge } from '@saulx/utils';
@@ -35,7 +35,7 @@ export const migrate = async (fromDbServer, toSchema, transform) => {
35
35
  fromDbServer.migrating = migrationId;
36
36
  const abort = () => fromDbServer.migrating !== migrationId;
37
37
  const toDb = new BasedDb({
38
- path: join(tmpdir(), (~~Math.random()).toString(36)),
38
+ path: join(tmpdir(), (~~(Math.random() * 1e9)).toString(36)),
39
39
  });
40
40
  await toDb.start({ clean: true });
41
41
  if (abort()) {
@@ -43,6 +43,10 @@ export const migrate = async (fromDbServer, toSchema, transform) => {
43
43
  return fromDbServer.schema;
44
44
  }
45
45
  toSchema = await toDb.setSchema(toSchema);
46
+ if (abort()) {
47
+ await toDb.destroy();
48
+ return fromDbServer.schema;
49
+ }
46
50
  const fromSchema = fromDbServer.schema;
47
51
  const fromCtx = fromDbServer.dbCtxExternal;
48
52
  const toCtx = toDb.server.dbCtxExternal;
@@ -67,14 +71,18 @@ export const migrate = async (fromDbServer, toSchema, transform) => {
67
71
  worker.on('error', console.error);
68
72
  let i = 0;
69
73
  let ranges = [];
70
- // fromDbServer.updateMerkleTree()
71
- // fromDbServer.dirtyRanges.clear()
72
74
  await fromDbServer.save();
73
75
  fromDbServer.merkleTree.visitLeafNodes((leaf) => {
76
+ const [_typeId, start] = destructureCsmtKey(leaf.key);
77
+ if (start == specialBlock)
78
+ return; // skip the type specialBlock
74
79
  ranges.push(leaf.data);
75
80
  });
76
81
  await Atomics.waitAsync(atomics, 0, 1).value;
77
82
  while (i < ranges.length) {
83
+ if (abort()) {
84
+ break;
85
+ }
78
86
  // block modifies
79
87
  fromDbServer.processingQueries++;
80
88
  const leafData = ranges[i++];
@@ -86,38 +94,48 @@ export const migrate = async (fromDbServer, toSchema, transform) => {
86
94
  await Atomics.waitAsync(atomics, 0, 1).value;
87
95
  // exec queued modifies
88
96
  fromDbServer.onQueryEnd();
89
- if (abort()) {
90
- break;
91
- }
92
- if (i === ranges.length && fromDbServer.dirtyRanges.size) {
93
- ranges = [];
94
- i = 0;
95
- foreachDirtyBlock(fromDbServer, (_mtKey, typeId, start, end) => {
96
- ranges.push({
97
- typeId,
98
- start,
99
- end,
97
+ if (i === ranges.length) {
98
+ if (fromDbServer.dirtyRanges.size) {
99
+ ranges = [];
100
+ i = 0;
101
+ foreachDirtyBlock(fromDbServer, (_mtKey, typeId, start, end) => {
102
+ ranges.push({
103
+ typeId,
104
+ start,
105
+ end,
106
+ });
100
107
  });
101
- });
102
- fromDbServer.dirtyRanges.clear();
108
+ fromDbServer.dirtyRanges.clear();
109
+ }
103
110
  }
104
111
  }
105
- if (!abort()) {
106
- let msg;
107
- let schema;
108
- let schemaTypesParsed;
109
- while ((msg = receiveMessageOnPort(port1))) {
110
- ;
111
- [schema, schemaTypesParsed] = msg.message;
112
- }
113
- fromDbServer.schema = deepMerge(toDb.server.schema, schema);
114
- fromDbServer.schemaTypesParsed = deepMerge(toDb.server.schemaTypesParsed, schemaTypesParsed);
115
- fromDbServer.dbCtxExternal = toCtx;
116
- toDb.server.dbCtxExternal = fromCtx;
112
+ if (abort()) {
113
+ await Promise.all([toDb.destroy(), worker.terminate()]);
114
+ return fromDbServer.schema;
115
+ }
116
+ let msg;
117
+ let schema;
118
+ let schemaTypesParsed;
119
+ while ((msg = receiveMessageOnPort(port1))) {
120
+ ;
121
+ [schema, schemaTypesParsed] = msg.message;
117
122
  }
123
+ fromDbServer.dbCtxExternal = toCtx;
124
+ fromDbServer.sortIndexes = {};
125
+ fromDbServer.schema = deepMerge(toDb.server.schema, schema);
126
+ fromDbServer.schemaTypesParsed = deepMerge(toDb.server.schemaTypesParsed, schemaTypesParsed);
127
+ fromDbServer.schemaTypesParsedById = {};
128
+ for (const key in fromDbServer.schemaTypesParsed) {
129
+ const def = fromDbServer.schemaTypesParsed[key];
130
+ fromDbServer.schemaTypesParsedById[def.id] = def;
131
+ }
132
+ toDb.server.dbCtxExternal = fromCtx;
118
133
  const promises = fromDbServer.workers.map((worker) => worker.updateCtx(toAddress));
119
134
  promises.push(toDb.destroy(), worker.terminate(), fromDbServer.save({ forceFullDump: true }), writeFile(join(fromDbServer.fileSystemPath, SCHEMA_FILE), JSON.stringify(fromDbServer.schema)));
120
135
  await Promise.all(promises);
136
+ if (abort()) {
137
+ return fromDbServer.schema;
138
+ }
121
139
  fromDbServer.onSchemaChange?.(fromDbServer.schema);
122
140
  return fromDbServer.schema;
123
141
  };
@@ -66,7 +66,7 @@ else {
66
66
  const nodes = fromDb
67
67
  .query(type)
68
68
  .include(include)
69
- .range(leafData.start - 1, leafData.end - leafData.start + 1)
69
+ .range(leafData.start - 1, leafData.end - 1)
70
70
  ._getSync(fromCtx);
71
71
  for (const node of nodes) {
72
72
  const res = typeTransformFn(node);
@@ -85,7 +85,7 @@ else {
85
85
  const nodes = fromDb
86
86
  .query(type)
87
87
  .include(include)
88
- .range(leafData.start - 1, leafData.end - leafData.start + 1)
88
+ .range(leafData.start - 1, leafData.end - 1)
89
89
  ._getSync(fromCtx);
90
90
  for (const node of nodes) {
91
91
  toDb.create(type, node, { unsafe: true });
@@ -2,18 +2,21 @@ import native from '../native.js';
2
2
  import { isMainThread } from 'node:worker_threads';
3
3
  import { writeFile } from 'node:fs/promises';
4
4
  import { join } from 'node:path';
5
- import { destructureCsmtKey, foreachBlock, foreachDirtyBlock, makeCsmtKey, } from './tree.js';
5
+ import { destructureCsmtKey, foreachBlock, foreachDirtyBlock, initCsmt, makeCsmtKey, specialBlock, } from './tree.js';
6
6
  import { WRITELOG_FILE } from './index.js';
7
7
  import { writeFileSync } from 'node:fs';
8
8
  import { bufToHex } from '../utils.js';
9
- import { createTree } from './csmt/tree.js';
10
9
  const COMMON_SDB_FILE = 'common.sdb';
11
10
  const block_sdb_file = (typeId, start, end) => `${typeId}_${start}_${end}.sdb`;
12
11
  function saveRange(db, typeId, start, end, hashOut) {
13
12
  const file = block_sdb_file(typeId, start, end);
14
13
  const path = join(db.fileSystemPath, file);
15
14
  const err = native.saveRange(path, typeId, start, end, db.dbCtxExternal, hashOut);
16
- if (err) {
15
+ if (err == -8) {
16
+ // TODO ENOENT
17
+ return '';
18
+ }
19
+ else if (err) {
17
20
  // TODO print the error string
18
21
  console.error(`Save ${typeId}:${start}-${end} failed: ${err}`);
19
22
  return null;
@@ -32,51 +35,58 @@ export function save(db, sync = false, forceFullDump = false) {
32
35
  }
33
36
  if (forceFullDump) {
34
37
  // We just rebuild the whole tree
35
- db.merkleTree = createTree(db.createCsmtHashFun); // TODO This could be somewhere else.
38
+ initCsmt(db);
36
39
  for (const key in db.schemaTypesParsed) {
37
40
  const def = db.schemaTypesParsed[key];
38
41
  foreachBlock(db, def, (start, end, _hash) => {
39
42
  const typeId = def.id;
43
+ const mtKey = makeCsmtKey(typeId, start);
40
44
  const hash = new Uint8Array(16);
41
45
  const file = saveRange(db, typeId, start, end, hash);
42
- if (!file) {
43
- // The previous state should remain in the merkle tree for
44
- // load and sync purposes.
45
- return;
46
+ if (file === null) {
47
+ throw new Error('full dump failed');
48
+ }
49
+ else {
50
+ const data = {
51
+ file,
52
+ typeId,
53
+ start,
54
+ end,
55
+ };
56
+ db.merkleTree.insert(mtKey, hash, data);
46
57
  }
47
- const mtKey = makeCsmtKey(typeId, start);
48
- const data = {
49
- file,
50
- typeId,
51
- start,
52
- end,
53
- };
54
- db.merkleTree.insert(mtKey, hash, data);
55
58
  });
56
59
  }
57
60
  }
58
61
  else {
59
62
  foreachDirtyBlock(db, (mtKey, typeId, start, end) => {
60
63
  const hash = new Uint8Array(16);
64
+ // TODO Don't save if empty
61
65
  const file = saveRange(db, typeId, start, end, hash);
62
- if (!file) {
66
+ if (file === null) {
63
67
  // The previous state should remain in the merkle tree for
64
68
  // load and sync purposes.
65
69
  return;
66
70
  }
67
- const data = {
68
- file,
69
- typeId,
70
- start,
71
- end,
72
- };
73
- try {
74
- db.merkleTree.delete(mtKey);
71
+ else if (file.length === 0) {
72
+ // The range is empty
75
73
  }
76
- catch (err) {
77
- // console.error({ err })
74
+ else {
75
+ const oldLeaf = db.merkleTree.search(mtKey);
76
+ if (oldLeaf) {
77
+ oldLeaf.data.file = file;
78
+ db.merkleTree.update(mtKey, hash);
79
+ }
80
+ else {
81
+ const data = {
82
+ file,
83
+ typeId,
84
+ start,
85
+ end,
86
+ };
87
+ db.merkleTree.insert(mtKey, hash, data);
88
+ }
78
89
  }
79
- db.merkleTree.insert(mtKey, hash, data);
80
90
  });
81
91
  }
82
92
  db.dirtyRanges.clear();
@@ -89,6 +99,8 @@ export function save(db, sync = false, forceFullDump = false) {
89
99
  }
90
100
  db.merkleTree.visitLeafNodes((leaf) => {
91
101
  const [typeId, start] = destructureCsmtKey(leaf.key);
102
+ if (start == specialBlock)
103
+ return; // skip the type specialBlock
92
104
  const data = leaf.data;
93
105
  if (start != data.start) {
94
106
  console.error(`csmtKey start and range start mismatch: ${start} != ${data.start}`);
@@ -3,8 +3,8 @@ import { DbWorker, SCHEMA_FILE, WRITELOG_FILE } from './index.js';
3
3
  import native from '../native.js';
4
4
  import { rm, mkdir, readFile } from 'node:fs/promises';
5
5
  import { join } from 'node:path';
6
- import { createTree, hashEq } from './csmt/index.js';
7
- import { foreachBlock, makeCsmtKey } from './tree.js';
6
+ import { hashEq } from './csmt/index.js';
7
+ import { foreachBlock, initCsmt, makeCsmtKey } from './tree.js';
8
8
  import { availableParallelism } from 'node:os';
9
9
  import exitHook from 'exit-hook';
10
10
  import './worker.js';
@@ -28,7 +28,7 @@ export async function start(db, opts) {
28
28
  native.loadCommon(join(path, writelog.commonDump), db.dbCtxExternal);
29
29
  }
30
30
  catch (e) {
31
- console.log(e.message);
31
+ console.error(e.message);
32
32
  throw e;
33
33
  }
34
34
  // Load all range dumps
@@ -40,7 +40,7 @@ export async function start(db, opts) {
40
40
  native.loadRange(join(path, fname), db.dbCtxExternal);
41
41
  }
42
42
  catch (e) {
43
- console.log(e.message);
43
+ console.error(e.message);
44
44
  }
45
45
  }
46
46
  }
@@ -53,21 +53,9 @@ export async function start(db, opts) {
53
53
  catch (err) {
54
54
  // TODO In some cases we really should give up!
55
55
  }
56
- // The merkle tree should be empty at start.
57
- if (!db.merkleTree || db.merkleTree.getRoot()) {
58
- db.merkleTree = createTree(db.createCsmtHashFun);
59
- }
60
- // FDN-791 CSMT is unstable (not history independent)
61
- // For now we just sort the types to ensure that we always
62
- // load in the same order.
63
- const types = Object.keys(db.schemaTypesParsed)
64
- .sort((a, b) => db.schemaTypesParsed[a].id - db.schemaTypesParsed[b].id)
65
- .reduce((obj, key) => {
66
- obj[key] = db.schemaTypesParsed[key];
67
- return obj;
68
- }, {});
69
- for (const key in types) {
70
- const def = types[key];
56
+ const csmtTypes = initCsmt(db);
57
+ for (const key in csmtTypes) {
58
+ const def = csmtTypes[key];
71
59
  const [total, lastId] = native.getTypeInfo(def.id, db.dbCtxExternal);
72
60
  def.total = total;
73
61
  def.lastId = writelog?.types[def.id]?.lastId || lastId;
@@ -6,8 +6,10 @@ export type CsmtNodeRange = {
6
6
  start: number;
7
7
  end: number;
8
8
  };
9
+ export declare const specialBlock = 2147483647;
9
10
  export declare const destructureCsmtKey: (key: number) => number[];
10
11
  export declare const makeCsmtKey: (typeId: number, start: number) => number;
11
12
  export declare const makeCsmtKeyFromNodeId: (typeId: number, blockCapacity: number, nodeId: number) => number;
13
+ export declare function initCsmt(db: DbServer): {};
12
14
  export declare function foreachBlock(db: DbServer, def: SchemaTypeDef, cb: (start: number, end: number, hash: Uint8Array) => void): void;
13
15
  export declare function foreachDirtyBlock(db: DbServer, cb: (mtKey: number, typeId: number, start: number, end: number) => void): Promise<void>;
@@ -1,4 +1,8 @@
1
1
  import native from '../native.js';
2
+ import { createTree } from './csmt/tree.js';
3
+ // This is a special start id set for every type to somewhat lock the order of the csmt.
4
+ // While the id is valid, it's never a true start id of a block.
5
+ export const specialBlock = 2147483647;
2
6
  export const destructureCsmtKey = (key) => [
3
7
  (key / 4294967296) | 0, // typeId
4
8
  (key >>> 31) * 2147483648 + (key & 0x7fffffff), // start_node_id
@@ -8,13 +12,41 @@ export const makeCsmtKeyFromNodeId = (typeId, blockCapacity, nodeId) => {
8
12
  const tmp = nodeId - +!(nodeId % blockCapacity);
9
13
  return typeId * 4294967296 + ((tmp / blockCapacity) | 0) * blockCapacity + 1;
10
14
  };
15
+ export function initCsmt(db) {
16
+ const types = Object.keys(db.schemaTypesParsed)
17
+ .sort((a, b) => db.schemaTypesParsed[a].id - db.schemaTypesParsed[b].id)
18
+ .reduce((obj, key) => {
19
+ obj[key] = db.schemaTypesParsed[key];
20
+ return obj;
21
+ }, {});
22
+ db.merkleTree = createTree(db.createCsmtHashFun);
23
+ // Insert specialBlocks for types.
24
+ // This should ensure that the insertion order of the actual node ranges is
25
+ // always deterministic.
26
+ for (const key in types) {
27
+ const { id: typeId } = types[key];
28
+ const data = {
29
+ file: '',
30
+ typeId: typeId,
31
+ start: 0,
32
+ end: 0,
33
+ };
34
+ try {
35
+ db.merkleTree.insert(makeCsmtKey(typeId, specialBlock), db.merkleTree.emptyHash, data);
36
+ }
37
+ catch (_) { }
38
+ }
39
+ return types;
40
+ }
11
41
  export function foreachBlock(db, def, cb) {
12
42
  const step = def.blockCapacity;
13
43
  for (let start = 1; start <= def.lastId; start += step) {
14
44
  const end = start + step - 1;
15
45
  const hash = new Uint8Array(16);
16
- native.getNodeRangeHash(def.id, start, end, hash, db.dbCtxExternal);
17
- cb(start, end, hash);
46
+ const res = native.getNodeRangeHash(def.id, start, end, hash, db.dbCtxExternal);
47
+ if (res) {
48
+ cb(start, end, hash);
49
+ }
18
50
  }
19
51
  }
20
52
  export async function foreachDirtyBlock(db, cb) {
@@ -8,7 +8,7 @@ else {
8
8
  let { address, channel } = workerData;
9
9
  let dbCtx = native.externalFromInt(address);
10
10
  workerCtx = native.workerCtxInit();
11
- const transferList = new Array(1);
11
+ // const transferList = new Array(1)
12
12
  const handleMsg = (msg) => {
13
13
  try {
14
14
  if (typeof msg === 'bigint') {
@@ -19,8 +19,8 @@ else {
19
19
  }
20
20
  else {
21
21
  const arrayBuf = native.getQueryBuf(msg, dbCtx);
22
- transferList[0] = arrayBuf;
23
- channel.postMessage(arrayBuf, transferList);
22
+ // transferList[0] = arrayBuf
23
+ channel.postMessage(arrayBuf, [arrayBuf]);
24
24
  }
25
25
  }
26
26
  catch (e) {
@@ -1,10 +1,12 @@
1
+ import { DbServer } from './server/index.js';
1
2
  export declare const DECODER: TextDecoder;
2
3
  export declare const ENCODER: TextEncoder;
4
+ export declare const debugMode: (target: any, getInfo?: any) => void;
5
+ export declare const debugServer: (server: DbServer) => void;
3
6
  export declare const equals: (aB: Uint8Array, bB: Uint8Array) => boolean;
4
7
  export declare function concatUint8Arr(bufs: Uint8Array[], totalByteLength?: number): Uint8Array;
5
8
  export declare const bufToHex: (a: Uint8Array) => string;
6
9
  export declare const hexToBuf: (s: string) => Uint8Array;
7
- export declare const base64encode: (a: Uint8Array, lineMax?: number) => string;
8
10
  export declare const readDoubleLE: (val: Uint8Array, offset: number) => number;
9
11
  export declare const readFloatLE: (val: Uint8Array, offset: number) => number;
10
12
  export declare const readUint32: (val: Uint8Array, offset: number) => number;