@based/db 0.0.28 → 0.0.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -399
- package/dist/lib/darwin_aarch64/include/selva/db.h +8 -4
- package/dist/lib/darwin_aarch64/include/selva/selva_hash128.h +17 -7
- package/dist/lib/darwin_aarch64/include/selva/sort.h +21 -14
- package/dist/lib/darwin_aarch64/libdeflate.dylib +0 -0
- package/dist/lib/darwin_aarch64/libjemalloc_selva.2.dylib +0 -0
- package/dist/lib/darwin_aarch64/libnode-v20.node +0 -0
- package/dist/lib/darwin_aarch64/libnode-v21.node +0 -0
- package/dist/lib/darwin_aarch64/libnode-v22.node +0 -0
- package/dist/lib/darwin_aarch64/libnode-v23.node +0 -0
- package/dist/lib/darwin_aarch64/libselva.dylib +0 -0
- package/dist/lib/darwin_aarch64/libxxhash.dylib +0 -0
- package/dist/lib/linux_aarch64/include/selva/db.h +8 -4
- package/dist/lib/linux_aarch64/include/selva/selva_hash128.h +17 -7
- package/dist/lib/linux_aarch64/include/selva/sort.h +21 -14
- package/dist/lib/linux_aarch64/libdeflate.so +0 -0
- package/dist/lib/linux_aarch64/libjemalloc_selva.so.2 +0 -0
- package/dist/lib/linux_aarch64/libnode-v20.node +0 -0
- package/dist/lib/linux_aarch64/libnode-v21.node +0 -0
- package/dist/lib/linux_aarch64/libnode-v22.node +0 -0
- package/dist/lib/linux_aarch64/libnode-v23.node +0 -0
- package/dist/lib/linux_aarch64/libselva.so +0 -0
- package/dist/lib/linux_x86_64/include/selva/db.h +8 -4
- package/dist/lib/linux_x86_64/include/selva/selva_hash128.h +17 -7
- package/dist/lib/linux_x86_64/include/selva/sort.h +21 -14
- package/dist/lib/linux_x86_64/libjemalloc_selva.so.2 +0 -0
- package/dist/lib/linux_x86_64/libnode-v20.node +0 -0
- package/dist/lib/linux_x86_64/libnode-v21.node +0 -0
- package/dist/lib/linux_x86_64/libnode-v22.node +0 -0
- package/dist/lib/linux_x86_64/libnode-v23.node +0 -0
- package/dist/lib/linux_x86_64/libselva.so +0 -0
- package/dist/src/client/flushModify.d.ts +1 -1
- package/dist/src/client/flushModify.js +12 -17
- package/dist/src/client/index.d.ts +6 -4
- package/dist/src/client/index.js +19 -2
- package/dist/src/client/modify/ModifyRes.d.ts +1 -1
- package/dist/src/client/modify/ModifyRes.js +14 -18
- package/dist/src/client/modify/fixed.js +43 -8
- package/dist/src/client/modify/modify.js +0 -1
- package/dist/src/client/modify/references/{appendRefs.d.ts → appendEdgeRefs.d.ts} +1 -1
- package/dist/src/client/modify/references/{appendRefs.js → appendEdgeRefs.js} +5 -2
- package/dist/src/client/modify/references/edge.js +182 -175
- package/dist/src/client/modify/references/reference.js +4 -8
- package/dist/src/client/modify/references/references.js +18 -14
- package/dist/src/client/modify/string.js +0 -3
- package/dist/src/client/modify/text.js +11 -3
- package/dist/src/client/modify/types.d.ts +11 -0
- package/dist/src/client/modify/types.js +10 -0
- package/dist/src/client/modify/update.js +4 -1
- package/dist/src/client/modify/vector.js +13 -4
- package/dist/src/client/query/BasedDbQuery.d.ts +1 -1
- package/dist/src/client/query/BasedDbQuery.js +2 -2
- package/dist/src/client/query/BasedIterable.d.ts +1 -1
- package/dist/src/client/query/BasedIterable.js +7 -2
- package/dist/src/client/query/filter/createFixedFilterBuffer.d.ts +2 -1
- package/dist/src/client/query/filter/createFixedFilterBuffer.js +11 -28
- package/dist/src/client/query/filter/createReferenceFilter.d.ts +2 -1
- package/dist/src/client/query/filter/createReferenceFilter.js +10 -9
- package/dist/src/client/query/filter/createVariableFilterBuffer.d.ts +2 -1
- package/dist/src/client/query/filter/createVariableFilterBuffer.js +8 -10
- package/dist/src/client/query/filter/parseFilterValue.js +1 -1
- package/dist/src/client/query/filter/primitiveFilter.js +9 -9
- package/dist/src/client/query/filter/toBuffer.js +0 -15
- package/dist/src/client/query/filter/types.d.ts +1 -0
- package/dist/src/client/query/filter/types.js +1 -0
- package/dist/src/client/query/include/walk.js +0 -1
- package/dist/src/client/query/read/read.js +4 -4
- package/dist/src/client/query/search/index.js +11 -15
- package/dist/src/client/query/subscription/markers.js +1 -2
- package/dist/src/client/query/subscription/run.js +0 -2
- package/dist/src/client/query/thresholds.d.ts +0 -2
- package/dist/src/client/query/thresholds.js +0 -2
- package/dist/src/client/query/toBuffer.js +16 -42
- package/dist/src/client/query/types.d.ts +3 -2
- package/dist/src/client/query/validation.d.ts +1 -3
- package/dist/src/client/query/validation.js +6 -18
- package/dist/src/client/string.d.ts +2 -0
- package/dist/src/client/string.js +9 -13
- package/dist/src/index.d.ts +1 -1
- package/dist/src/index.js +7 -15
- package/dist/src/native.d.ts +1 -1
- package/dist/src/native.js +3 -3
- package/dist/src/server/csmt/draw-dot.js +2 -2
- package/dist/src/server/csmt/tree.js +57 -6
- package/dist/src/server/csmt/types.d.ts +5 -0
- package/dist/src/server/index.d.ts +4 -3
- package/dist/src/server/index.js +44 -44
- package/dist/src/server/migrate/index.js +47 -29
- package/dist/src/server/migrate/worker.js +2 -2
- package/dist/src/server/save.js +40 -28
- package/dist/src/server/start.js +7 -19
- package/dist/src/server/tree.d.ts +2 -0
- package/dist/src/server/tree.js +34 -2
- package/dist/src/server/worker.js +3 -3
- package/dist/src/utils.d.ts +3 -1
- package/dist/src/utils.js +43 -19
- package/package.json +9 -3
- package/dist/lib/darwin_aarch64/include/selva/base64.h +0 -59
- package/dist/lib/darwin_aarch64/include/selva/base64url.h +0 -59
- package/dist/lib/linux_aarch64/include/selva/base64.h +0 -59
- package/dist/lib/linux_aarch64/include/selva/base64url.h +0 -59
- package/dist/lib/linux_x86_64/include/selva/base64.h +0 -59
- package/dist/lib/linux_x86_64/include/selva/base64url.h +0 -59
- package/dist/src/client/timestamp.d.ts +0 -1
- package/dist/src/client/timestamp.js +0 -68
package/dist/src/server/index.js
CHANGED
|
@@ -4,14 +4,14 @@ import { rm, writeFile } from 'node:fs/promises';
|
|
|
4
4
|
import { dirname, join } from 'node:path';
|
|
5
5
|
import { getPropType, langCodesMap, } from '@based/schema';
|
|
6
6
|
import { updateTypeDefs, schemaToSelvaBuffer, } from '@based/schema/def';
|
|
7
|
-
import { hashEq } from './csmt/index.js';
|
|
8
7
|
import { start } from './start.js';
|
|
9
|
-
import {
|
|
8
|
+
import { initCsmt, makeCsmtKey, makeCsmtKeyFromNodeId } from './tree.js';
|
|
10
9
|
import { save } from './save.js';
|
|
11
10
|
import { Worker, MessageChannel } from 'node:worker_threads';
|
|
12
11
|
import { fileURLToPath } from 'node:url';
|
|
13
12
|
import { setTimeout } from 'node:timers/promises';
|
|
14
13
|
import { migrate } from './migrate/index.js';
|
|
14
|
+
import { debugServer } from '../utils.js';
|
|
15
15
|
export const SCHEMA_FILE = 'schema.json';
|
|
16
16
|
export const WRITELOG_FILE = 'writelog.json';
|
|
17
17
|
const __filename = fileURLToPath(import.meta.url);
|
|
@@ -45,10 +45,6 @@ export class DbWorker {
|
|
|
45
45
|
transferList: [port2],
|
|
46
46
|
});
|
|
47
47
|
port1.on('message', (buf) => {
|
|
48
|
-
// TODO FIX TYPES CHECK IF THIS MAKES A COPY
|
|
49
|
-
// It's a copy, if you don't want a copy you'd need to make it an explicit view
|
|
50
|
-
// to the underlying buffer:
|
|
51
|
-
// new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength)
|
|
52
48
|
this.resolvers.shift()(new Uint8Array(buf));
|
|
53
49
|
this.db.onQueryEnd();
|
|
54
50
|
});
|
|
@@ -93,11 +89,14 @@ export class DbServer {
|
|
|
93
89
|
stopped;
|
|
94
90
|
onSchemaChange;
|
|
95
91
|
unlistenExit;
|
|
96
|
-
constructor({ path, maxModifySize = 100 * 1e3 * 1e3, onSchemaChange, }) {
|
|
92
|
+
constructor({ path, maxModifySize = 100 * 1e3 * 1e3, onSchemaChange, debug, }) {
|
|
97
93
|
this.maxModifySize = maxModifySize;
|
|
98
94
|
this.fileSystemPath = path;
|
|
99
95
|
this.sortIndexes = {};
|
|
100
96
|
this.onSchemaChange = onSchemaChange;
|
|
97
|
+
if (debug) {
|
|
98
|
+
debugServer(this);
|
|
99
|
+
}
|
|
101
100
|
}
|
|
102
101
|
#resizeModifyDirtyRanges() {
|
|
103
102
|
let maxNrChanges = 0;
|
|
@@ -141,7 +140,7 @@ export class DbServer {
|
|
|
141
140
|
for (const lang in this.sortIndexes[type][field][start]) {
|
|
142
141
|
const sortIndex = this.sortIndexes[type][field][start][lang];
|
|
143
142
|
sortIndex.cnt /= 2;
|
|
144
|
-
if (sortIndex.cnt < 1) {
|
|
143
|
+
if (sortIndex.cnt < 1 && !this.processingQueries) {
|
|
145
144
|
native.destroySortIndex(sortIndex.buf, this.dbCtxExternal);
|
|
146
145
|
delete this.sortIndexes[type][field][start][lang];
|
|
147
146
|
}
|
|
@@ -179,7 +178,7 @@ export class DbServer {
|
|
|
179
178
|
if (sortIndex) {
|
|
180
179
|
return sortIndex;
|
|
181
180
|
}
|
|
182
|
-
const buf = new Uint8Array(
|
|
181
|
+
const buf = new Uint8Array(9);
|
|
183
182
|
// size [2 type] [1 field] [2 start] [2 len] [propIndex] [lang]
|
|
184
183
|
// call createSortBuf here
|
|
185
184
|
buf[0] = t.id;
|
|
@@ -208,7 +207,6 @@ export class DbServer {
|
|
|
208
207
|
}
|
|
209
208
|
let sortIndex = fields[prop.start];
|
|
210
209
|
if (sortIndex) {
|
|
211
|
-
// [2 type] [1 field] [2 start] [1 lang]
|
|
212
210
|
const buf = new Uint8Array(6);
|
|
213
211
|
buf[0] = t.id;
|
|
214
212
|
buf[1] = t.id >>> 8;
|
|
@@ -279,29 +277,6 @@ export class DbServer {
|
|
|
279
277
|
fields[start][lang] = sortIndex;
|
|
280
278
|
return sortIndex;
|
|
281
279
|
}
|
|
282
|
-
updateMerkleTree() {
|
|
283
|
-
foreachDirtyBlock(this, (mtKey, typeId, start, end) => {
|
|
284
|
-
const oldLeaf = this.merkleTree.search(mtKey);
|
|
285
|
-
const hash = new Uint8Array(16);
|
|
286
|
-
native.getNodeRangeHash(typeId, start, end, hash, this.dbCtxExternal);
|
|
287
|
-
if (oldLeaf) {
|
|
288
|
-
if (hashEq(oldLeaf.hash, hash)) {
|
|
289
|
-
return;
|
|
290
|
-
}
|
|
291
|
-
try {
|
|
292
|
-
this.merkleTree.delete(mtKey);
|
|
293
|
-
}
|
|
294
|
-
catch (err) { }
|
|
295
|
-
}
|
|
296
|
-
const data = {
|
|
297
|
-
file: '', // not saved yet
|
|
298
|
-
typeId,
|
|
299
|
-
start,
|
|
300
|
-
end,
|
|
301
|
-
};
|
|
302
|
-
this.merkleTree.insert(mtKey, hash, data);
|
|
303
|
-
});
|
|
304
|
-
}
|
|
305
280
|
setSchema(strictSchema, fromStart = false, transformFns) {
|
|
306
281
|
if (!fromStart && Object.keys(this.schema.types).length > 0) {
|
|
307
282
|
return this.migrateSchema(strictSchema, transformFns);
|
|
@@ -373,6 +348,7 @@ export class DbServer {
|
|
|
373
348
|
}
|
|
374
349
|
if (strictSchema.props) {
|
|
375
350
|
// insert a root node
|
|
351
|
+
// TODO fix this add it in schema at least
|
|
376
352
|
const data = [2, 1, 0, 0, 0, 1, 9, 1, 0, 0, 0, 7, 1, 0, 1];
|
|
377
353
|
const blockKey = makeCsmtKey(1, 1);
|
|
378
354
|
const buf = new Uint8Array(data.length + 2 + 8 + 4);
|
|
@@ -387,6 +363,7 @@ export class DbServer {
|
|
|
387
363
|
view.setUint32(buf.length - 4, data.length, true);
|
|
388
364
|
this.modify(buf);
|
|
389
365
|
}
|
|
366
|
+
initCsmt(this);
|
|
390
367
|
}
|
|
391
368
|
this.onSchemaChange?.(this.schema);
|
|
392
369
|
return this.schema;
|
|
@@ -401,7 +378,16 @@ export class DbServer {
|
|
|
401
378
|
i += 2;
|
|
402
379
|
const startId = readUint32LE(buf, i);
|
|
403
380
|
const def = this.schemaTypesParsedById[typeId];
|
|
404
|
-
|
|
381
|
+
let offset = def.lastId - startId;
|
|
382
|
+
if (offset < 0) {
|
|
383
|
+
console.log('-----------------');
|
|
384
|
+
console.log(def.type, {
|
|
385
|
+
offset,
|
|
386
|
+
serverId: def.lastId,
|
|
387
|
+
clientId: startId,
|
|
388
|
+
});
|
|
389
|
+
offset = 0;
|
|
390
|
+
}
|
|
405
391
|
buf[i++] = offset;
|
|
406
392
|
buf[i++] = offset >>> 8;
|
|
407
393
|
buf[i++] = offset >>> 16;
|
|
@@ -411,6 +397,7 @@ export class DbServer {
|
|
|
411
397
|
def.lastId = lastId + offset;
|
|
412
398
|
offsets[typeId] = offset;
|
|
413
399
|
}
|
|
400
|
+
// console.log('modify', this.processingQueries)
|
|
414
401
|
if (this.processingQueries) {
|
|
415
402
|
this.modifyQueue.push(new Uint8Array(buf));
|
|
416
403
|
}
|
|
@@ -449,10 +436,17 @@ export class DbServer {
|
|
|
449
436
|
this.dirtyRanges.add(key);
|
|
450
437
|
}
|
|
451
438
|
}
|
|
452
|
-
|
|
439
|
+
addToQueryQueue(resolve, buf) {
|
|
440
|
+
if (this.queryQueue.size === 16777216) {
|
|
441
|
+
resolve(new Error('Query queue exceeded'));
|
|
442
|
+
return;
|
|
443
|
+
}
|
|
444
|
+
this.queryQueue.set(resolve, buf);
|
|
445
|
+
}
|
|
446
|
+
getQueryBuf(buf, fromQueue = false) {
|
|
453
447
|
if (this.modifyQueue.length) {
|
|
454
448
|
return new Promise((resolve) => {
|
|
455
|
-
this.
|
|
449
|
+
this.addToQueryQueue(resolve, buf);
|
|
456
450
|
});
|
|
457
451
|
}
|
|
458
452
|
else {
|
|
@@ -469,7 +463,7 @@ export class DbServer {
|
|
|
469
463
|
if (!sortIndex) {
|
|
470
464
|
if (this.processingQueries) {
|
|
471
465
|
return new Promise((resolve) => {
|
|
472
|
-
this.
|
|
466
|
+
this.addToQueryQueue(resolve, buf);
|
|
473
467
|
});
|
|
474
468
|
}
|
|
475
469
|
sortIndex = this.createSortIndexBuffer(typeId, field, start, sort[sort.byteLength - 1]);
|
|
@@ -482,6 +476,9 @@ export class DbServer {
|
|
|
482
476
|
else if (queryType == 1) {
|
|
483
477
|
// This will be more advanced - sometimes has indexes / sometimes not
|
|
484
478
|
}
|
|
479
|
+
if (!fromQueue) {
|
|
480
|
+
native.expire(this.dbCtxExternal);
|
|
481
|
+
}
|
|
485
482
|
this.availableWorkerIndex =
|
|
486
483
|
(this.availableWorkerIndex + 1) % this.workers.length;
|
|
487
484
|
return this.workers[this.availableWorkerIndex].getQueryBuf(buf);
|
|
@@ -491,16 +488,19 @@ export class DbServer {
|
|
|
491
488
|
this.processingQueries--;
|
|
492
489
|
if (this.processingQueries === 0) {
|
|
493
490
|
if (this.modifyQueue.length) {
|
|
494
|
-
|
|
491
|
+
const modifyQueue = this.modifyQueue;
|
|
492
|
+
this.modifyQueue = [];
|
|
493
|
+
for (const buf of modifyQueue) {
|
|
495
494
|
this.#modify(buf);
|
|
496
495
|
}
|
|
497
|
-
this.modifyQueue = [];
|
|
498
496
|
}
|
|
499
497
|
if (this.queryQueue.size) {
|
|
500
|
-
|
|
501
|
-
|
|
498
|
+
const queryQueue = this.queryQueue;
|
|
499
|
+
this.queryQueue = new Map();
|
|
500
|
+
native.expire(this.dbCtxExternal);
|
|
501
|
+
for (const [resolve, buf] of queryQueue) {
|
|
502
|
+
resolve(this.getQueryBuf(buf, true));
|
|
502
503
|
}
|
|
503
|
-
this.queryQueue.clear();
|
|
504
504
|
}
|
|
505
505
|
}
|
|
506
506
|
}
|
|
@@ -510,7 +510,7 @@ export class DbServer {
|
|
|
510
510
|
}
|
|
511
511
|
this.stopped = true;
|
|
512
512
|
clearTimeout(this.cleanupTimer);
|
|
513
|
-
this
|
|
513
|
+
this.unlistenExit();
|
|
514
514
|
try {
|
|
515
515
|
if (!noSave) {
|
|
516
516
|
await this.save();
|
|
@@ -518,7 +518,7 @@ export class DbServer {
|
|
|
518
518
|
await Promise.all(this.workers.map(({ worker }) => worker.terminate()));
|
|
519
519
|
this.workers = [];
|
|
520
520
|
native.stop(this.dbCtxExternal);
|
|
521
|
-
await setTimeout();
|
|
521
|
+
await setTimeout(20);
|
|
522
522
|
}
|
|
523
523
|
catch (e) {
|
|
524
524
|
this.stopped = false;
|
|
@@ -4,7 +4,7 @@ import { tmpdir } from 'os';
|
|
|
4
4
|
import { Worker, MessageChannel, receiveMessageOnPort, } from 'node:worker_threads';
|
|
5
5
|
import native from '../../native.js';
|
|
6
6
|
import './worker.js';
|
|
7
|
-
import { foreachDirtyBlock } from '../tree.js';
|
|
7
|
+
import { destructureCsmtKey, foreachDirtyBlock, specialBlock } from '../tree.js';
|
|
8
8
|
import { SCHEMA_FILE } from '../index.js';
|
|
9
9
|
import { fileURLToPath } from 'url';
|
|
10
10
|
import { deepMerge } from '@saulx/utils';
|
|
@@ -35,7 +35,7 @@ export const migrate = async (fromDbServer, toSchema, transform) => {
|
|
|
35
35
|
fromDbServer.migrating = migrationId;
|
|
36
36
|
const abort = () => fromDbServer.migrating !== migrationId;
|
|
37
37
|
const toDb = new BasedDb({
|
|
38
|
-
path: join(tmpdir(), (~~Math.random()).toString(36)),
|
|
38
|
+
path: join(tmpdir(), (~~(Math.random() * 1e9)).toString(36)),
|
|
39
39
|
});
|
|
40
40
|
await toDb.start({ clean: true });
|
|
41
41
|
if (abort()) {
|
|
@@ -43,6 +43,10 @@ export const migrate = async (fromDbServer, toSchema, transform) => {
|
|
|
43
43
|
return fromDbServer.schema;
|
|
44
44
|
}
|
|
45
45
|
toSchema = await toDb.setSchema(toSchema);
|
|
46
|
+
if (abort()) {
|
|
47
|
+
await toDb.destroy();
|
|
48
|
+
return fromDbServer.schema;
|
|
49
|
+
}
|
|
46
50
|
const fromSchema = fromDbServer.schema;
|
|
47
51
|
const fromCtx = fromDbServer.dbCtxExternal;
|
|
48
52
|
const toCtx = toDb.server.dbCtxExternal;
|
|
@@ -67,14 +71,18 @@ export const migrate = async (fromDbServer, toSchema, transform) => {
|
|
|
67
71
|
worker.on('error', console.error);
|
|
68
72
|
let i = 0;
|
|
69
73
|
let ranges = [];
|
|
70
|
-
// fromDbServer.updateMerkleTree()
|
|
71
|
-
// fromDbServer.dirtyRanges.clear()
|
|
72
74
|
await fromDbServer.save();
|
|
73
75
|
fromDbServer.merkleTree.visitLeafNodes((leaf) => {
|
|
76
|
+
const [_typeId, start] = destructureCsmtKey(leaf.key);
|
|
77
|
+
if (start == specialBlock)
|
|
78
|
+
return; // skip the type specialBlock
|
|
74
79
|
ranges.push(leaf.data);
|
|
75
80
|
});
|
|
76
81
|
await Atomics.waitAsync(atomics, 0, 1).value;
|
|
77
82
|
while (i < ranges.length) {
|
|
83
|
+
if (abort()) {
|
|
84
|
+
break;
|
|
85
|
+
}
|
|
78
86
|
// block modifies
|
|
79
87
|
fromDbServer.processingQueries++;
|
|
80
88
|
const leafData = ranges[i++];
|
|
@@ -86,38 +94,48 @@ export const migrate = async (fromDbServer, toSchema, transform) => {
|
|
|
86
94
|
await Atomics.waitAsync(atomics, 0, 1).value;
|
|
87
95
|
// exec queued modifies
|
|
88
96
|
fromDbServer.onQueryEnd();
|
|
89
|
-
if (
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
end,
|
|
97
|
+
if (i === ranges.length) {
|
|
98
|
+
if (fromDbServer.dirtyRanges.size) {
|
|
99
|
+
ranges = [];
|
|
100
|
+
i = 0;
|
|
101
|
+
foreachDirtyBlock(fromDbServer, (_mtKey, typeId, start, end) => {
|
|
102
|
+
ranges.push({
|
|
103
|
+
typeId,
|
|
104
|
+
start,
|
|
105
|
+
end,
|
|
106
|
+
});
|
|
100
107
|
});
|
|
101
|
-
|
|
102
|
-
|
|
108
|
+
fromDbServer.dirtyRanges.clear();
|
|
109
|
+
}
|
|
103
110
|
}
|
|
104
111
|
}
|
|
105
|
-
if (
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
fromDbServer.dbCtxExternal = toCtx;
|
|
116
|
-
toDb.server.dbCtxExternal = fromCtx;
|
|
112
|
+
if (abort()) {
|
|
113
|
+
await Promise.all([toDb.destroy(), worker.terminate()]);
|
|
114
|
+
return fromDbServer.schema;
|
|
115
|
+
}
|
|
116
|
+
let msg;
|
|
117
|
+
let schema;
|
|
118
|
+
let schemaTypesParsed;
|
|
119
|
+
while ((msg = receiveMessageOnPort(port1))) {
|
|
120
|
+
;
|
|
121
|
+
[schema, schemaTypesParsed] = msg.message;
|
|
117
122
|
}
|
|
123
|
+
fromDbServer.dbCtxExternal = toCtx;
|
|
124
|
+
fromDbServer.sortIndexes = {};
|
|
125
|
+
fromDbServer.schema = deepMerge(toDb.server.schema, schema);
|
|
126
|
+
fromDbServer.schemaTypesParsed = deepMerge(toDb.server.schemaTypesParsed, schemaTypesParsed);
|
|
127
|
+
fromDbServer.schemaTypesParsedById = {};
|
|
128
|
+
for (const key in fromDbServer.schemaTypesParsed) {
|
|
129
|
+
const def = fromDbServer.schemaTypesParsed[key];
|
|
130
|
+
fromDbServer.schemaTypesParsedById[def.id] = def;
|
|
131
|
+
}
|
|
132
|
+
toDb.server.dbCtxExternal = fromCtx;
|
|
118
133
|
const promises = fromDbServer.workers.map((worker) => worker.updateCtx(toAddress));
|
|
119
134
|
promises.push(toDb.destroy(), worker.terminate(), fromDbServer.save({ forceFullDump: true }), writeFile(join(fromDbServer.fileSystemPath, SCHEMA_FILE), JSON.stringify(fromDbServer.schema)));
|
|
120
135
|
await Promise.all(promises);
|
|
136
|
+
if (abort()) {
|
|
137
|
+
return fromDbServer.schema;
|
|
138
|
+
}
|
|
121
139
|
fromDbServer.onSchemaChange?.(fromDbServer.schema);
|
|
122
140
|
return fromDbServer.schema;
|
|
123
141
|
};
|
|
@@ -66,7 +66,7 @@ else {
|
|
|
66
66
|
const nodes = fromDb
|
|
67
67
|
.query(type)
|
|
68
68
|
.include(include)
|
|
69
|
-
.range(leafData.start - 1, leafData.end -
|
|
69
|
+
.range(leafData.start - 1, leafData.end - 1)
|
|
70
70
|
._getSync(fromCtx);
|
|
71
71
|
for (const node of nodes) {
|
|
72
72
|
const res = typeTransformFn(node);
|
|
@@ -85,7 +85,7 @@ else {
|
|
|
85
85
|
const nodes = fromDb
|
|
86
86
|
.query(type)
|
|
87
87
|
.include(include)
|
|
88
|
-
.range(leafData.start - 1, leafData.end -
|
|
88
|
+
.range(leafData.start - 1, leafData.end - 1)
|
|
89
89
|
._getSync(fromCtx);
|
|
90
90
|
for (const node of nodes) {
|
|
91
91
|
toDb.create(type, node, { unsafe: true });
|
package/dist/src/server/save.js
CHANGED
|
@@ -2,18 +2,21 @@ import native from '../native.js';
|
|
|
2
2
|
import { isMainThread } from 'node:worker_threads';
|
|
3
3
|
import { writeFile } from 'node:fs/promises';
|
|
4
4
|
import { join } from 'node:path';
|
|
5
|
-
import { destructureCsmtKey, foreachBlock, foreachDirtyBlock, makeCsmtKey, } from './tree.js';
|
|
5
|
+
import { destructureCsmtKey, foreachBlock, foreachDirtyBlock, initCsmt, makeCsmtKey, specialBlock, } from './tree.js';
|
|
6
6
|
import { WRITELOG_FILE } from './index.js';
|
|
7
7
|
import { writeFileSync } from 'node:fs';
|
|
8
8
|
import { bufToHex } from '../utils.js';
|
|
9
|
-
import { createTree } from './csmt/tree.js';
|
|
10
9
|
const COMMON_SDB_FILE = 'common.sdb';
|
|
11
10
|
const block_sdb_file = (typeId, start, end) => `${typeId}_${start}_${end}.sdb`;
|
|
12
11
|
function saveRange(db, typeId, start, end, hashOut) {
|
|
13
12
|
const file = block_sdb_file(typeId, start, end);
|
|
14
13
|
const path = join(db.fileSystemPath, file);
|
|
15
14
|
const err = native.saveRange(path, typeId, start, end, db.dbCtxExternal, hashOut);
|
|
16
|
-
if (err) {
|
|
15
|
+
if (err == -8) {
|
|
16
|
+
// TODO ENOENT
|
|
17
|
+
return '';
|
|
18
|
+
}
|
|
19
|
+
else if (err) {
|
|
17
20
|
// TODO print the error string
|
|
18
21
|
console.error(`Save ${typeId}:${start}-${end} failed: ${err}`);
|
|
19
22
|
return null;
|
|
@@ -32,51 +35,58 @@ export function save(db, sync = false, forceFullDump = false) {
|
|
|
32
35
|
}
|
|
33
36
|
if (forceFullDump) {
|
|
34
37
|
// We just rebuild the whole tree
|
|
35
|
-
|
|
38
|
+
initCsmt(db);
|
|
36
39
|
for (const key in db.schemaTypesParsed) {
|
|
37
40
|
const def = db.schemaTypesParsed[key];
|
|
38
41
|
foreachBlock(db, def, (start, end, _hash) => {
|
|
39
42
|
const typeId = def.id;
|
|
43
|
+
const mtKey = makeCsmtKey(typeId, start);
|
|
40
44
|
const hash = new Uint8Array(16);
|
|
41
45
|
const file = saveRange(db, typeId, start, end, hash);
|
|
42
|
-
if (
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
+
if (file === null) {
|
|
47
|
+
throw new Error('full dump failed');
|
|
48
|
+
}
|
|
49
|
+
else {
|
|
50
|
+
const data = {
|
|
51
|
+
file,
|
|
52
|
+
typeId,
|
|
53
|
+
start,
|
|
54
|
+
end,
|
|
55
|
+
};
|
|
56
|
+
db.merkleTree.insert(mtKey, hash, data);
|
|
46
57
|
}
|
|
47
|
-
const mtKey = makeCsmtKey(typeId, start);
|
|
48
|
-
const data = {
|
|
49
|
-
file,
|
|
50
|
-
typeId,
|
|
51
|
-
start,
|
|
52
|
-
end,
|
|
53
|
-
};
|
|
54
|
-
db.merkleTree.insert(mtKey, hash, data);
|
|
55
58
|
});
|
|
56
59
|
}
|
|
57
60
|
}
|
|
58
61
|
else {
|
|
59
62
|
foreachDirtyBlock(db, (mtKey, typeId, start, end) => {
|
|
60
63
|
const hash = new Uint8Array(16);
|
|
64
|
+
// TODO Don't save if empty
|
|
61
65
|
const file = saveRange(db, typeId, start, end, hash);
|
|
62
|
-
if (
|
|
66
|
+
if (file === null) {
|
|
63
67
|
// The previous state should remain in the merkle tree for
|
|
64
68
|
// load and sync purposes.
|
|
65
69
|
return;
|
|
66
70
|
}
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
typeId,
|
|
70
|
-
start,
|
|
71
|
-
end,
|
|
72
|
-
};
|
|
73
|
-
try {
|
|
74
|
-
db.merkleTree.delete(mtKey);
|
|
71
|
+
else if (file.length === 0) {
|
|
72
|
+
// The range is empty
|
|
75
73
|
}
|
|
76
|
-
|
|
77
|
-
|
|
74
|
+
else {
|
|
75
|
+
const oldLeaf = db.merkleTree.search(mtKey);
|
|
76
|
+
if (oldLeaf) {
|
|
77
|
+
oldLeaf.data.file = file;
|
|
78
|
+
db.merkleTree.update(mtKey, hash);
|
|
79
|
+
}
|
|
80
|
+
else {
|
|
81
|
+
const data = {
|
|
82
|
+
file,
|
|
83
|
+
typeId,
|
|
84
|
+
start,
|
|
85
|
+
end,
|
|
86
|
+
};
|
|
87
|
+
db.merkleTree.insert(mtKey, hash, data);
|
|
88
|
+
}
|
|
78
89
|
}
|
|
79
|
-
db.merkleTree.insert(mtKey, hash, data);
|
|
80
90
|
});
|
|
81
91
|
}
|
|
82
92
|
db.dirtyRanges.clear();
|
|
@@ -89,6 +99,8 @@ export function save(db, sync = false, forceFullDump = false) {
|
|
|
89
99
|
}
|
|
90
100
|
db.merkleTree.visitLeafNodes((leaf) => {
|
|
91
101
|
const [typeId, start] = destructureCsmtKey(leaf.key);
|
|
102
|
+
if (start == specialBlock)
|
|
103
|
+
return; // skip the type specialBlock
|
|
92
104
|
const data = leaf.data;
|
|
93
105
|
if (start != data.start) {
|
|
94
106
|
console.error(`csmtKey start and range start mismatch: ${start} != ${data.start}`);
|
package/dist/src/server/start.js
CHANGED
|
@@ -3,8 +3,8 @@ import { DbWorker, SCHEMA_FILE, WRITELOG_FILE } from './index.js';
|
|
|
3
3
|
import native from '../native.js';
|
|
4
4
|
import { rm, mkdir, readFile } from 'node:fs/promises';
|
|
5
5
|
import { join } from 'node:path';
|
|
6
|
-
import {
|
|
7
|
-
import { foreachBlock, makeCsmtKey } from './tree.js';
|
|
6
|
+
import { hashEq } from './csmt/index.js';
|
|
7
|
+
import { foreachBlock, initCsmt, makeCsmtKey } from './tree.js';
|
|
8
8
|
import { availableParallelism } from 'node:os';
|
|
9
9
|
import exitHook from 'exit-hook';
|
|
10
10
|
import './worker.js';
|
|
@@ -28,7 +28,7 @@ export async function start(db, opts) {
|
|
|
28
28
|
native.loadCommon(join(path, writelog.commonDump), db.dbCtxExternal);
|
|
29
29
|
}
|
|
30
30
|
catch (e) {
|
|
31
|
-
console.
|
|
31
|
+
console.error(e.message);
|
|
32
32
|
throw e;
|
|
33
33
|
}
|
|
34
34
|
// Load all range dumps
|
|
@@ -40,7 +40,7 @@ export async function start(db, opts) {
|
|
|
40
40
|
native.loadRange(join(path, fname), db.dbCtxExternal);
|
|
41
41
|
}
|
|
42
42
|
catch (e) {
|
|
43
|
-
console.
|
|
43
|
+
console.error(e.message);
|
|
44
44
|
}
|
|
45
45
|
}
|
|
46
46
|
}
|
|
@@ -53,21 +53,9 @@ export async function start(db, opts) {
|
|
|
53
53
|
catch (err) {
|
|
54
54
|
// TODO In some cases we really should give up!
|
|
55
55
|
}
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
}
|
|
60
|
-
// FDN-791 CSMT is unstable (not history independent)
|
|
61
|
-
// For now we just sort the types to ensure that we always
|
|
62
|
-
// load in the same order.
|
|
63
|
-
const types = Object.keys(db.schemaTypesParsed)
|
|
64
|
-
.sort((a, b) => db.schemaTypesParsed[a].id - db.schemaTypesParsed[b].id)
|
|
65
|
-
.reduce((obj, key) => {
|
|
66
|
-
obj[key] = db.schemaTypesParsed[key];
|
|
67
|
-
return obj;
|
|
68
|
-
}, {});
|
|
69
|
-
for (const key in types) {
|
|
70
|
-
const def = types[key];
|
|
56
|
+
const csmtTypes = initCsmt(db);
|
|
57
|
+
for (const key in csmtTypes) {
|
|
58
|
+
const def = csmtTypes[key];
|
|
71
59
|
const [total, lastId] = native.getTypeInfo(def.id, db.dbCtxExternal);
|
|
72
60
|
def.total = total;
|
|
73
61
|
def.lastId = writelog?.types[def.id]?.lastId || lastId;
|
|
@@ -6,8 +6,10 @@ export type CsmtNodeRange = {
|
|
|
6
6
|
start: number;
|
|
7
7
|
end: number;
|
|
8
8
|
};
|
|
9
|
+
export declare const specialBlock = 2147483647;
|
|
9
10
|
export declare const destructureCsmtKey: (key: number) => number[];
|
|
10
11
|
export declare const makeCsmtKey: (typeId: number, start: number) => number;
|
|
11
12
|
export declare const makeCsmtKeyFromNodeId: (typeId: number, blockCapacity: number, nodeId: number) => number;
|
|
13
|
+
export declare function initCsmt(db: DbServer): {};
|
|
12
14
|
export declare function foreachBlock(db: DbServer, def: SchemaTypeDef, cb: (start: number, end: number, hash: Uint8Array) => void): void;
|
|
13
15
|
export declare function foreachDirtyBlock(db: DbServer, cb: (mtKey: number, typeId: number, start: number, end: number) => void): Promise<void>;
|
package/dist/src/server/tree.js
CHANGED
|
@@ -1,4 +1,8 @@
|
|
|
1
1
|
import native from '../native.js';
|
|
2
|
+
import { createTree } from './csmt/tree.js';
|
|
3
|
+
// This is a special start id set for every type to somewhat lock the order of the csmt.
|
|
4
|
+
// While the id is valid, it's never a true start id of a block.
|
|
5
|
+
export const specialBlock = 2147483647;
|
|
2
6
|
export const destructureCsmtKey = (key) => [
|
|
3
7
|
(key / 4294967296) | 0, // typeId
|
|
4
8
|
(key >>> 31) * 2147483648 + (key & 0x7fffffff), // start_node_id
|
|
@@ -8,13 +12,41 @@ export const makeCsmtKeyFromNodeId = (typeId, blockCapacity, nodeId) => {
|
|
|
8
12
|
const tmp = nodeId - +!(nodeId % blockCapacity);
|
|
9
13
|
return typeId * 4294967296 + ((tmp / blockCapacity) | 0) * blockCapacity + 1;
|
|
10
14
|
};
|
|
15
|
+
export function initCsmt(db) {
|
|
16
|
+
const types = Object.keys(db.schemaTypesParsed)
|
|
17
|
+
.sort((a, b) => db.schemaTypesParsed[a].id - db.schemaTypesParsed[b].id)
|
|
18
|
+
.reduce((obj, key) => {
|
|
19
|
+
obj[key] = db.schemaTypesParsed[key];
|
|
20
|
+
return obj;
|
|
21
|
+
}, {});
|
|
22
|
+
db.merkleTree = createTree(db.createCsmtHashFun);
|
|
23
|
+
// Insert specialBlocks for types.
|
|
24
|
+
// This should ensure that the insertion order of the actual node ranges is
|
|
25
|
+
// always deterministic.
|
|
26
|
+
for (const key in types) {
|
|
27
|
+
const { id: typeId } = types[key];
|
|
28
|
+
const data = {
|
|
29
|
+
file: '',
|
|
30
|
+
typeId: typeId,
|
|
31
|
+
start: 0,
|
|
32
|
+
end: 0,
|
|
33
|
+
};
|
|
34
|
+
try {
|
|
35
|
+
db.merkleTree.insert(makeCsmtKey(typeId, specialBlock), db.merkleTree.emptyHash, data);
|
|
36
|
+
}
|
|
37
|
+
catch (_) { }
|
|
38
|
+
}
|
|
39
|
+
return types;
|
|
40
|
+
}
|
|
11
41
|
export function foreachBlock(db, def, cb) {
|
|
12
42
|
const step = def.blockCapacity;
|
|
13
43
|
for (let start = 1; start <= def.lastId; start += step) {
|
|
14
44
|
const end = start + step - 1;
|
|
15
45
|
const hash = new Uint8Array(16);
|
|
16
|
-
native.getNodeRangeHash(def.id, start, end, hash, db.dbCtxExternal);
|
|
17
|
-
|
|
46
|
+
const res = native.getNodeRangeHash(def.id, start, end, hash, db.dbCtxExternal);
|
|
47
|
+
if (res) {
|
|
48
|
+
cb(start, end, hash);
|
|
49
|
+
}
|
|
18
50
|
}
|
|
19
51
|
}
|
|
20
52
|
export async function foreachDirtyBlock(db, cb) {
|
|
@@ -8,7 +8,7 @@ else {
|
|
|
8
8
|
let { address, channel } = workerData;
|
|
9
9
|
let dbCtx = native.externalFromInt(address);
|
|
10
10
|
workerCtx = native.workerCtxInit();
|
|
11
|
-
const transferList = new Array(1)
|
|
11
|
+
// const transferList = new Array(1)
|
|
12
12
|
const handleMsg = (msg) => {
|
|
13
13
|
try {
|
|
14
14
|
if (typeof msg === 'bigint') {
|
|
@@ -19,8 +19,8 @@ else {
|
|
|
19
19
|
}
|
|
20
20
|
else {
|
|
21
21
|
const arrayBuf = native.getQueryBuf(msg, dbCtx);
|
|
22
|
-
transferList[0] = arrayBuf
|
|
23
|
-
channel.postMessage(arrayBuf,
|
|
22
|
+
// transferList[0] = arrayBuf
|
|
23
|
+
channel.postMessage(arrayBuf, [arrayBuf]);
|
|
24
24
|
}
|
|
25
25
|
}
|
|
26
26
|
catch (e) {
|
package/dist/src/utils.d.ts
CHANGED
|
@@ -1,10 +1,12 @@
|
|
|
1
|
+
import { DbServer } from './server/index.js';
|
|
1
2
|
export declare const DECODER: TextDecoder;
|
|
2
3
|
export declare const ENCODER: TextEncoder;
|
|
4
|
+
export declare const debugMode: (target: any, getInfo?: any) => void;
|
|
5
|
+
export declare const debugServer: (server: DbServer) => void;
|
|
3
6
|
export declare const equals: (aB: Uint8Array, bB: Uint8Array) => boolean;
|
|
4
7
|
export declare function concatUint8Arr(bufs: Uint8Array[], totalByteLength?: number): Uint8Array;
|
|
5
8
|
export declare const bufToHex: (a: Uint8Array) => string;
|
|
6
9
|
export declare const hexToBuf: (s: string) => Uint8Array;
|
|
7
|
-
export declare const base64encode: (a: Uint8Array, lineMax?: number) => string;
|
|
8
10
|
export declare const readDoubleLE: (val: Uint8Array, offset: number) => number;
|
|
9
11
|
export declare const readFloatLE: (val: Uint8Array, offset: number) => number;
|
|
10
12
|
export declare const readUint32: (val: Uint8Array, offset: number) => number;
|