@based/db 0.0.66 → 0.0.68

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. package/README.md +2 -2
  2. package/dist/lib/darwin_aarch64/include/selva/colvec.h +71 -0
  3. package/dist/lib/darwin_aarch64/include/selva/db.h +3 -0
  4. package/dist/lib/darwin_aarch64/include/selva/fields.h +1 -1
  5. package/dist/lib/darwin_aarch64/include/selva/types.h +7 -0
  6. package/dist/lib/darwin_aarch64/libdeflate.dylib +0 -0
  7. package/dist/lib/darwin_aarch64/libjemalloc_selva.2.dylib +0 -0
  8. package/dist/lib/darwin_aarch64/libjemalloc_selva.so.2 +0 -0
  9. package/dist/lib/darwin_aarch64/libnode-v20.node +0 -0
  10. package/dist/lib/darwin_aarch64/libnode-v21.node +0 -0
  11. package/dist/lib/darwin_aarch64/libnode-v22.node +0 -0
  12. package/dist/lib/darwin_aarch64/libnode-v23.node +0 -0
  13. package/dist/lib/darwin_aarch64/libnode-v24.node +0 -0
  14. package/dist/lib/darwin_aarch64/libselva.dylib +0 -0
  15. package/dist/lib/darwin_aarch64/libxxhash.dylib +0 -0
  16. package/dist/lib/linux_aarch64/include/selva/colvec.h +71 -0
  17. package/dist/lib/linux_aarch64/include/selva/db.h +3 -0
  18. package/dist/lib/linux_aarch64/include/selva/fields.h +1 -1
  19. package/dist/lib/linux_aarch64/include/selva/types.h +7 -0
  20. package/dist/lib/linux_aarch64/libnode-v20.node +0 -0
  21. package/dist/lib/linux_aarch64/libnode-v21.node +0 -0
  22. package/dist/lib/linux_aarch64/libnode-v22.node +0 -0
  23. package/dist/lib/linux_aarch64/libnode-v23.node +0 -0
  24. package/dist/lib/linux_aarch64/libnode-v24.node +0 -0
  25. package/dist/lib/linux_aarch64/libselva.so +0 -0
  26. package/dist/lib/linux_x86_64/include/selva/colvec.h +71 -0
  27. package/dist/lib/linux_x86_64/include/selva/db.h +3 -0
  28. package/dist/lib/linux_x86_64/include/selva/fields.h +1 -1
  29. package/dist/lib/linux_x86_64/include/selva/types.h +7 -0
  30. package/dist/lib/linux_x86_64/libnode-v20.node +0 -0
  31. package/dist/lib/linux_x86_64/libnode-v21.node +0 -0
  32. package/dist/lib/linux_x86_64/libnode-v22.node +0 -0
  33. package/dist/lib/linux_x86_64/libnode-v23.node +0 -0
  34. package/dist/lib/linux_x86_64/libnode-v24.node +0 -0
  35. package/dist/lib/linux_x86_64/libselva.so +0 -0
  36. package/dist/src/client/flushModify.d.ts +2 -1
  37. package/dist/src/client/flushModify.js +12 -4
  38. package/dist/src/client/modify/create.js +11 -0
  39. package/dist/src/client/modify/delete.js +3 -0
  40. package/dist/src/client/modify/modify.js +2 -2
  41. package/dist/src/client/modify/setCursor.d.ts +2 -1
  42. package/dist/src/client/query/BasedDbQuery.d.ts +7 -2
  43. package/dist/src/client/query/BasedDbQuery.js +111 -14
  44. package/dist/src/client/query/aggregates/aggregation.js +24 -11
  45. package/dist/src/client/query/aggregates/types.d.ts +21 -2
  46. package/dist/src/client/query/aggregates/types.js +34 -1
  47. package/dist/src/client/query/display.js +8 -2
  48. package/dist/src/client/query/filter/createVariableFilterBuffer.d.ts +2 -3
  49. package/dist/src/client/query/filter/createVariableFilterBuffer.js +20 -7
  50. package/dist/src/client/query/filter/filter.js +13 -3
  51. package/dist/src/client/query/filter/primitiveFilter.d.ts +1 -2
  52. package/dist/src/client/query/include/props.js +18 -2
  53. package/dist/src/client/query/include/toBuffer.js +11 -3
  54. package/dist/src/client/query/include/walk.js +5 -1
  55. package/dist/src/client/query/queryDef.js +4 -1
  56. package/dist/src/client/query/read/read.js +50 -23
  57. package/dist/src/client/query/registerQuery.js +1 -0
  58. package/dist/src/client/query/search/index.d.ts +1 -1
  59. package/dist/src/client/query/search/index.js +21 -7
  60. package/dist/src/client/query/sort.d.ts +1 -1
  61. package/dist/src/client/query/toByteCode/default.d.ts +1 -1
  62. package/dist/src/client/query/toByteCode/default.js +0 -2
  63. package/dist/src/client/query/toByteCode/toBuffer.js +0 -7
  64. package/dist/src/client/query/types.d.ts +16 -5
  65. package/dist/src/client/query/validation.js +25 -2
  66. package/dist/src/client/xxHash64.d.ts +1 -1
  67. package/dist/src/index.d.ts +0 -1
  68. package/dist/src/index.js +0 -1
  69. package/dist/src/native.d.ts +2 -2
  70. package/dist/src/native.js +7 -8
  71. package/dist/src/server/IoWorker.d.ts +8 -0
  72. package/dist/src/server/IoWorker.js +39 -0
  73. package/dist/src/server/QueryWorker.d.ts +8 -0
  74. package/dist/src/server/QueryWorker.js +26 -0
  75. package/dist/src/server/blocks.d.ts +24 -0
  76. package/dist/src/server/blocks.js +112 -0
  77. package/dist/src/server/dbHash.d.ts +1 -1
  78. package/dist/src/server/index.d.ts +9 -16
  79. package/dist/src/server/index.js +37 -15
  80. package/dist/src/server/migrate/index.d.ts +5 -0
  81. package/dist/src/server/migrate/index.js +10 -7
  82. package/dist/src/server/save.d.ts +8 -6
  83. package/dist/src/server/save.js +34 -78
  84. package/dist/src/server/schema.js +6 -5
  85. package/dist/src/server/start.js +57 -60
  86. package/dist/src/server/tree.d.ts +24 -13
  87. package/dist/src/server/tree.js +95 -66
  88. package/dist/src/server/workers/DbWorker.d.ts +17 -0
  89. package/dist/src/server/{DbWorker.js → workers/DbWorker.js} +15 -17
  90. package/dist/src/server/workers/io_worker.js +39 -0
  91. package/dist/src/server/workers/io_worker_types.d.ts +12 -0
  92. package/dist/src/server/workers/io_worker_types.js +2 -0
  93. package/dist/src/server/workers/query_worker.d.ts +1 -0
  94. package/dist/src/server/workers/query_worker.js +4 -0
  95. package/dist/src/server/workers/worker.d.ts +1 -0
  96. package/dist/src/server/workers/worker.js +41 -0
  97. package/dist/src/shared/Emitter.d.ts +1 -0
  98. package/dist/src/types.d.ts +1 -1
  99. package/dist/src/types.js +1 -1
  100. package/package.json +3 -3
  101. package/dist/lib/darwin_aarch64/include/selva/history.h +0 -64
  102. package/dist/lib/linux_aarch64/include/selva/history.h +0 -64
  103. package/dist/lib/linux_x86_64/include/selva/history.h +0 -64
  104. package/dist/src/client/query/serialize.d.ts +0 -4
  105. package/dist/src/client/query/serialize.js +0 -26
  106. package/dist/src/server/DbWorker.d.ts +0 -13
  107. package/dist/src/server/csmt/draw-dot.d.ts +0 -4
  108. package/dist/src/server/csmt/draw-dot.js +0 -38
  109. package/dist/src/server/csmt/index.d.ts +0 -4
  110. package/dist/src/server/csmt/index.js +0 -5
  111. package/dist/src/server/csmt/match.d.ts +0 -7
  112. package/dist/src/server/csmt/match.js +0 -10
  113. package/dist/src/server/csmt/memebership-proof.d.ts +0 -7
  114. package/dist/src/server/csmt/memebership-proof.js +0 -122
  115. package/dist/src/server/csmt/tree-utils.d.ts +0 -6
  116. package/dist/src/server/csmt/tree-utils.js +0 -33
  117. package/dist/src/server/csmt/tree.d.ts +0 -3
  118. package/dist/src/server/csmt/tree.js +0 -270
  119. package/dist/src/server/csmt/types.d.ts +0 -46
  120. package/dist/src/server/csmt/types.js +0 -2
  121. package/dist/src/server/worker.js +0 -33
  122. /package/dist/src/server/{worker.d.ts → workers/io_worker.d.ts} +0 -0
@@ -1,15 +1,27 @@
1
- import { DbWorker } from './DbWorker.js';
1
+ import { availableParallelism } from 'node:os';
2
+ import { QueryWorker } from './QueryWorker.js';
3
+ import { IoWorker } from './IoWorker.js';
2
4
  import native from '../native.js';
3
5
  import { rm, mkdir, readFile } from 'node:fs/promises';
4
6
  import { join } from 'node:path';
5
- import { destructureCsmtKey, foreachBlock, initCsmt, makeCsmtKey, specialBlock, } from './tree.js';
6
- import { availableParallelism } from 'node:os';
7
+ import { VerifTree, makeTreeKey } from './tree.js';
8
+ import { foreachBlock } from './blocks.js';
7
9
  import exitHook from 'exit-hook';
8
10
  import { save } from './save.js';
9
- import { DEFAULT_BLOCK_CAPACITY } from '@based/schema/def';
10
- import { bufToHex, wait } from '@saulx/utils';
11
+ import { deSerialize } from '@based/schema';
12
+ import { BLOCK_CAPACITY_DEFAULT } from '@based/schema/def';
13
+ import { bufToHex, equals, hexToBuf, wait } from '@saulx/utils';
11
14
  import { SCHEMA_FILE, WRITELOG_FILE } from '../types.js';
12
15
  import { setSchemaOnServer } from './schema.js';
16
+ function startWorkers(db, opts) {
17
+ const queryThreads = opts?.queryThreads ?? availableParallelism();
18
+ const address = native.intFromExternal(db.dbCtxExternal);
19
+ db.workers = [];
20
+ for (let i = 0; i < queryThreads; i++) {
21
+ db.workers.push(new QueryWorker(address, db, i));
22
+ }
23
+ db.ioWorker = new IoWorker(address, db);
24
+ }
13
25
  export async function start(db, opts) {
14
26
  const path = db.fileSystemPath;
15
27
  const noop = () => { };
@@ -19,6 +31,7 @@ export async function start(db, opts) {
19
31
  await mkdir(path, { recursive: true }).catch(noop);
20
32
  db.dbCtxExternal = native.start();
21
33
  let writelog = null;
34
+ let partials = []; // Blocks that exists but were not loaded [key, hash]
22
35
  try {
23
36
  writelog = JSON.parse((await readFile(join(path, WRITELOG_FILE))).toString());
24
37
  // Load the common dump
@@ -29,81 +42,66 @@ export async function start(db, opts) {
29
42
  console.error(e.message);
30
43
  throw e;
31
44
  }
45
+ const schema = await readFile(join(path, SCHEMA_FILE));
46
+ if (schema) {
47
+ const s = deSerialize(schema);
48
+ setSchemaOnServer(db, s);
49
+ }
32
50
  // Load all range dumps
33
51
  for (const typeId in writelog.rangeDumps) {
34
52
  const dumps = writelog.rangeDumps[typeId];
53
+ const def = db.schemaTypesParsedById[typeId];
35
54
  for (const dump of dumps) {
36
55
  const fname = dump.file;
37
56
  if (fname?.length > 0) {
38
- try {
39
- native.loadBlock(join(path, fname), db.dbCtxExternal);
57
+ if (!def.partial) {
58
+ try {
59
+ // Can't use loadBlock() yet because verifTree is not avail
60
+ native.loadBlock(join(path, fname), db.dbCtxExternal);
61
+ }
62
+ catch (e) {
63
+ console.error(e.message);
64
+ }
40
65
  }
41
- catch (e) {
42
- console.error(e.message);
66
+ else {
67
+ partials.push([
68
+ makeTreeKey(def.id, dump.start),
69
+ hexToBuf(dump.hash),
70
+ ]);
43
71
  }
44
72
  }
45
73
  }
46
74
  }
47
- const schema = await readFile(join(path, SCHEMA_FILE));
48
- if (schema) {
49
- const s = JSON.parse(schema.toString());
50
- setSchemaOnServer(db, s);
51
- }
52
75
  }
53
76
  catch (err) {
54
77
  // TODO In some cases we really should give up!
55
78
  }
56
- const csmtTypes = initCsmt(db);
57
- for (const key in csmtTypes) {
58
- const def = csmtTypes[key];
79
+ db.verifTree = new VerifTree(db.schemaTypesParsed);
80
+ for (const { typeId } of db.verifTree.types()) {
81
+ const def = db.schemaTypesParsedById[typeId];
59
82
  const [total, lastId] = native.getTypeInfo(def.id, db.dbCtxExternal);
60
83
  def.lastId = writelog?.types[def.id]?.lastId || lastId;
61
84
  def.blockCapacity =
62
- writelog?.types[def.id]?.blockCapacity || DEFAULT_BLOCK_CAPACITY;
63
- foreachBlock(db, def, (start, end, hash) => {
64
- const mtKey = makeCsmtKey(def.id, start);
65
- const file = writelog.rangeDumps[def.id]?.find((v) => v.start === start)?.file ||
66
- '';
67
- const data = {
68
- file,
69
- typeId: def.id,
70
- start,
71
- end,
72
- };
73
- db.merkleTree.insert(mtKey, hash, data);
74
- }, true);
85
+ writelog?.types[def.id]?.blockCapacity ||
86
+ def.blockCapacity ||
87
+ BLOCK_CAPACITY_DEFAULT;
88
+ foreachBlock(db, def, (start, _end, hash) => {
89
+ const mtKey = makeTreeKey(def.id, start);
90
+ db.verifTree.update(mtKey, hash);
91
+ });
92
+ }
93
+ // Insert partials to make the hash match
94
+ for (const [key, hash] of partials) {
95
+ db.verifTree.update(key, hash, false);
75
96
  }
76
97
  if (writelog?.hash) {
77
- // FIXME FDN-1301
78
- //const oldHash = hexToBuf(writelog.hash)
79
- //const newHash = db.merkleTree.getRoot()?.hash
80
- //if (!hashEq(oldHash, newHash)) {
81
- // console.error(
82
- // `WARN: CSMT hash mismatch. expected: ${writelog.hash} actual: ${bufToHex(newHash)}`,
83
- // )
84
- //}
85
- const oldHashSet = new Set();
86
- const newHashSet = new Set();
87
- for (let k in writelog.rangeDumps)
88
- writelog.rangeDumps[k].forEach(({ hash }) => oldHashSet.add(hash));
89
- db.merkleTree.visitLeafNodes(({ key, hash }) => {
90
- const [_typeId, start] = destructureCsmtKey(key);
91
- if (start == specialBlock)
92
- return; // skip the type specialBlock
93
- newHashSet.add(bufToHex(hash));
94
- });
95
- const setEq = (a, b) => a.size === b.size && [...a].every((value) => b.has(value));
96
- if (!setEq(oldHashSet, newHashSet)) {
97
- console.error(`WARN: CSMT hash mismatch.`);
98
+ const oldHash = hexToBuf(writelog.hash);
99
+ const newHash = db.verifTree.hash;
100
+ if (!equals(oldHash, newHash)) {
101
+ console.error(`WARN: DB hash mismatch. expected: ${writelog.hash} actual: ${bufToHex(newHash)}`);
98
102
  }
99
103
  }
100
- // start workers
101
- const queryThreads = opts?.queryThreads ?? availableParallelism();
102
- const address = native.intFromExternal(db.dbCtxExternal);
103
- db.workers = [];
104
- for (let i = 0; i < queryThreads; i++) {
105
- db.workers.push(new DbWorker(address, db, i));
106
- }
104
+ startWorkers(db, opts);
107
105
  if (!opts?.hosted) {
108
106
  db.unlistenExit = exitHook((signal) => {
109
107
  const blockSig = () => { };
@@ -118,9 +116,8 @@ export async function start(db, opts) {
118
116
  signals.forEach((sig) => process.off(sig, blockSig));
119
117
  });
120
118
  }
121
- const d = performance.now();
122
119
  await Promise.all(db.workers.map(({ readyPromise }) => readyPromise));
123
- db.emit('info', `Starting workers took ${d}ms`);
120
+ db.emit('info', 'All workers ready');
124
121
  // use timeout
125
122
  if (db.saveIntervalInSeconds > 0) {
126
123
  db.saveInterval ??= setInterval(() => {
@@ -1,15 +1,26 @@
1
- import { DbServer } from './index.js';
2
1
  import { SchemaTypeDef } from '@based/schema/def';
3
- export type CsmtNodeRange = {
4
- file: string;
5
- typeId: number;
6
- start: number;
7
- end: number;
2
+ export declare const destructureTreeKey: (key: number) => number[];
3
+ export declare const makeTreeKey: (typeId: number, start: number) => number;
4
+ export declare const nodeId2Start: (blockCapacity: number, nodeId: number) => number;
5
+ export declare const makeTreeKeyFromNodeId: (typeId: number, blockCapacity: number, nodeId: number) => number;
6
+ type Hash = Uint8Array;
7
+ export type VerifBlock = {
8
+ key: number;
9
+ hash: Hash;
10
+ inmem: boolean;
11
+ loadPromise: null | Promise<void>;
8
12
  };
9
- export declare const specialBlock = 2147483647;
10
- export declare const destructureCsmtKey: (key: number) => number[];
11
- export declare const makeCsmtKey: (typeId: number, start: number) => number;
12
- export declare const makeCsmtKeyFromNodeId: (typeId: number, blockCapacity: number, nodeId: number) => number;
13
- export declare function initCsmt(db: DbServer): {};
14
- export declare function foreachBlock(db: DbServer, def: SchemaTypeDef, cb: (start: number, end: number, hash: Uint8Array) => void, includeEmptyBlocks?: boolean): void;
15
- export declare function foreachDirtyBlock(db: DbServer, cb: (mtKey: number, typeId: number, start: number, end: number) => void): Promise<void>;
13
+ export declare class VerifTree {
14
+ #private;
15
+ constructor(schemaTypesParsed: Record<string, SchemaTypeDef>);
16
+ types(): Generator<any, void, unknown>;
17
+ foreachBlock(cb: (block: VerifBlock) => void): void;
18
+ get hash(): Uint8Array;
19
+ update(key: number, hash: Hash, inmem?: boolean): void;
20
+ remove(key: number): void;
21
+ static blockSdbFile(typeId: number, start: number, end: number): string;
22
+ getBlock(key: number): VerifBlock;
23
+ getBlockFile(block: VerifBlock): string;
24
+ updateTypes(schemaTypesParsed: Record<string, SchemaTypeDef>): void;
25
+ }
26
+ export {};
@@ -1,80 +1,109 @@
1
- import native from '../native.js';
2
- import { createTree } from './csmt/tree.js';
3
- // This is a special start id set for every type to somewhat lock the order of the csmt.
4
- // While the id is valid, it's never a true start id of a block.
5
- export const specialBlock = 2147483647;
6
- export const destructureCsmtKey = (key) => [
1
+ import createDbHash from './dbHash.js';
2
+ export const destructureTreeKey = (key) => [
7
3
  (key / 4294967296) | 0, // typeId
8
4
  (key >>> 31) * 2147483648 + (key & 0x7fffffff), // start_node_id
9
5
  ];
10
- export const makeCsmtKey = (typeId, start) => typeId * 4294967296 + start;
11
- export const makeCsmtKeyFromNodeId = (typeId, blockCapacity, nodeId) => {
6
+ export const makeTreeKey = (typeId, start) => typeId * 4294967296 + start;
7
+ export const nodeId2Start = (blockCapacity, nodeId) => ((nodeId - +!(nodeId % blockCapacity)) / blockCapacity) | 0;
8
+ const nodeId2BlockI = (nodeId, blockCapacity) => ((nodeId - 1) - ((nodeId - 1) % blockCapacity)) / blockCapacity;
9
+ export const makeTreeKeyFromNodeId = (typeId, blockCapacity, nodeId) => {
12
10
  const tmp = nodeId - +!(nodeId % blockCapacity);
13
11
  return typeId * 4294967296 + ((tmp / blockCapacity) | 0) * blockCapacity + 1;
14
12
  };
15
- export function initCsmt(db) {
16
- const types = Object.keys(db.schemaTypesParsed)
17
- .sort((a, b) => db.schemaTypesParsed[a].id - db.schemaTypesParsed[b].id)
18
- .reduce((obj, key) => {
19
- obj[key] = db.schemaTypesParsed[key];
20
- return obj;
21
- }, {});
22
- db.merkleTree = createTree(db.createCsmtHashFun);
23
- // Insert specialBlocks for types.
24
- // This should ensure that the insertion order of the actual node ranges is
25
- // always deterministic.
26
- for (const key in types) {
27
- const { id: typeId } = types[key];
28
- const data = {
29
- file: '',
30
- typeId: typeId,
31
- start: 0,
32
- end: 0,
33
- };
34
- try {
35
- db.merkleTree.insert(makeCsmtKey(typeId, specialBlock), db.merkleTree.emptyHash, data);
13
+ const HASH_SIZE = 16;
14
+ export class VerifTree {
15
+ #types;
16
+ #h = createDbHash();
17
+ constructor(schemaTypesParsed) {
18
+ this.#types = VerifTree.#makeTypes(schemaTypesParsed);
19
+ }
20
+ static #makeTypes(schemaTypesParsed) {
21
+ return Object.preventExtensions(Object.keys(schemaTypesParsed)
22
+ .sort((a, b) => schemaTypesParsed[a].id - schemaTypesParsed[b].id)
23
+ .reduce((obj, key) => {
24
+ const def = schemaTypesParsed[key];
25
+ const typeId = def.id;
26
+ obj[typeId] = {
27
+ typeId,
28
+ blockCapacity: def.blockCapacity,
29
+ hash: new Uint8Array(HASH_SIZE),
30
+ blocks: [],
31
+ };
32
+ return obj;
33
+ }, {}));
34
+ }
35
+ *types() {
36
+ for (const k of Object.keys(this.#types)) {
37
+ yield this.#types[k];
36
38
  }
37
- catch (_) { }
38
39
  }
39
- return types;
40
- }
41
- export function foreachBlock(db, def, cb, includeEmptyBlocks = false) {
42
- const step = def.blockCapacity;
43
- for (let start = 1; start <= def.lastId; start += step) {
44
- const end = start + step - 1;
45
- const hash = new Uint8Array(16);
46
- const res = native.getNodeRangeHash(def.id, start, end, hash, db.dbCtxExternal);
47
- if (res || includeEmptyBlocks) {
48
- cb(start, end, hash);
40
+ foreachBlock(cb) {
41
+ for (const k of Object.keys(this.#types)) {
42
+ const { blocks } = this.#types[k];
43
+ for (let block of blocks) {
44
+ if (block)
45
+ cb(block);
46
+ }
49
47
  }
50
48
  }
51
- }
52
- export async function foreachDirtyBlock(db, cb) {
53
- const typeIdMap = {};
54
- for (const typeName in db.schemaTypesParsed) {
55
- const type = db.schemaTypesParsed[typeName];
56
- const typeId = type.id;
57
- typeIdMap[typeId] = type;
49
+ get hash() {
50
+ this.#h.reset();
51
+ this.foreachBlock((block) => this.#h.update(block.hash));
52
+ return this.#h.digest();
53
+ }
54
+ update(key, hash, inmem = true) {
55
+ const [typeId, start] = destructureTreeKey(key);
56
+ const type = this.#types[typeId];
57
+ if (!type) {
58
+ throw new Error(`type ${typeId} not found`);
59
+ }
60
+ const blockI = nodeId2BlockI(start, type.blockCapacity);
61
+ const block = type.blocks[blockI] ?? (type.blocks[blockI] = Object.preventExtensions({ key, hash, inmem, loadPromise: null }));
62
+ block.hash = hash;
63
+ block.inmem = inmem;
64
+ }
65
+ remove(key) {
66
+ const [typeId, start] = destructureTreeKey(key);
67
+ const type = this.#types[typeId];
68
+ if (!type) {
69
+ throw new Error(`type ${typeId} not found`);
70
+ }
71
+ const blockI = nodeId2BlockI(start, type.blockCapacity);
72
+ delete type.blocks[blockI];
73
+ }
74
+ static blockSdbFile(typeId, start, end) {
75
+ return `${typeId}_${start}_${end}.sdb`;
76
+ }
77
+ getBlock(key) {
78
+ const [typeId, start] = destructureTreeKey(key);
79
+ const type = this.#types[typeId];
80
+ if (!type) {
81
+ throw new Error(`type ${typeId} not found`);
82
+ }
83
+ const blockI = nodeId2BlockI(start, type.blockCapacity);
84
+ return type.blocks[blockI];
58
85
  }
59
- // FDN-791 CSMT is unstable (not history independent)
60
- // For now we just sort the dirty data by type and start
61
- // to make the insertion order more deterministic.
62
- // This doesn't solve the issue completely because
63
- // we might mess the order later with updates.
64
- const dirty = [...db.dirtyRanges].sort((a, b) => {
65
- const [aTypeId, aStart] = destructureCsmtKey(a);
66
- const [bTypeId, bStart] = destructureCsmtKey(b);
67
- const aId = typeIdMap[aTypeId].id;
68
- const bId = typeIdMap[bTypeId].id;
69
- const td = aId - bId;
70
- if (td != 0)
71
- return td;
72
- return aStart - bStart;
73
- });
74
- for (const mtKey of dirty) {
75
- const [typeId, start] = destructureCsmtKey(mtKey);
76
- const end = start + typeIdMap[typeId].blockCapacity - 1;
77
- cb(mtKey, typeId, start, end);
86
+ getBlockFile(block) {
87
+ const [typeId, start] = destructureTreeKey(block.key);
88
+ const type = this.#types[typeId];
89
+ if (!type) {
90
+ throw new Error(`type ${typeId} not found`);
91
+ }
92
+ const end = start + type.blockCapacity - 1;
93
+ return VerifTree.blockSdbFile(typeId, start, end);
94
+ }
95
+ updateTypes(schemaTypesParsed) {
96
+ const oldTypes = this.#types;
97
+ const newTypes = VerifTree.#makeTypes(schemaTypesParsed);
98
+ for (const k of Object.keys(oldTypes)) {
99
+ const oldType = oldTypes[k];
100
+ const newType = newTypes[k];
101
+ if (newType) {
102
+ newType.hash = oldType.hash;
103
+ newType.blocks = oldType.blocks;
104
+ }
105
+ }
106
+ this.#types = newTypes;
78
107
  }
79
108
  }
80
109
  //# sourceMappingURL=tree.js.map
@@ -0,0 +1,17 @@
1
+ import { DbServer } from '../index.js';
2
+ export declare abstract class DbWorker {
3
+ constructor(address: BigInt, db: DbServer, onExit: (code: number) => void, workerName: string);
4
+ protected db: DbServer;
5
+ private channel;
6
+ private worker;
7
+ protected resolvers: ((x: any) => any)[];
8
+ readyPromise: Promise<true>;
9
+ terminate(): Promise<number>;
10
+ abstract handleMsg(buf: any): void;
11
+ protected callback: (resolve: (x: any) => any) => void;
12
+ /**
13
+ * Send msg to the worker thread and return a promise to the response.
14
+ */
15
+ protected call(msg: any): Promise<Uint8Array>;
16
+ updateCtx(address: BigInt): Promise<void>;
17
+ }
@@ -1,16 +1,14 @@
1
1
  import { MessageChannel, Worker } from 'node:worker_threads';
2
2
  import { fileURLToPath } from 'node:url';
3
3
  import { dirname, join } from 'node:path';
4
- import { readUint64 } from '@saulx/utils';
5
4
  const __filename = fileURLToPath(import.meta.url);
6
5
  const __dirname = dirname(__filename);
7
- const workerPath = join(__dirname, 'worker.js');
8
6
  export class DbWorker {
9
- constructor(address, db, workerIndex) {
7
+ constructor(address, db, onExit, workerName) {
10
8
  const { port1, port2 } = new MessageChannel();
11
9
  this.db = db;
12
10
  this.channel = port1;
13
- this.worker = new Worker(workerPath, {
11
+ this.worker = new Worker(join(__dirname, workerName), {
14
12
  workerData: {
15
13
  isDbWorker: true,
16
14
  channel: port2,
@@ -41,12 +39,12 @@ export class DbWorker {
41
39
  resolve(err);
42
40
  }
43
41
  this.resolvers = [];
44
- this.db.workers[workerIndex] = new DbWorker(address, db, workerIndex);
42
+ onExit(code);
45
43
  }
46
44
  });
47
- port1.on('message', (buf) => {
45
+ this.channel.on('message', (buf) => {
48
46
  this.resolvers.shift()(new Uint8Array(buf));
49
- this.db.onQueryEnd();
47
+ this.handleMsg(buf);
50
48
  });
51
49
  }
52
50
  db;
@@ -54,21 +52,21 @@ export class DbWorker {
54
52
  worker;
55
53
  resolvers = [];
56
54
  readyPromise;
55
+ async terminate() {
56
+ return this.worker.terminate();
57
+ }
57
58
  callback = (resolve) => {
58
- this.db.processingQueries++;
59
59
  this.resolvers.push(resolve);
60
60
  };
61
- updateCtx(address) {
62
- this.channel.postMessage(address);
61
+ /**
62
+ * Send msg to the worker thread and return a promise to the response.
63
+ */
64
+ call(msg) {
65
+ this.channel.postMessage(msg);
63
66
  return new Promise(this.callback);
64
67
  }
65
- getQueryBuf(buf) {
66
- const schemaChecksum = readUint64(buf, buf.byteLength - 8);
67
- if (schemaChecksum !== this.db.schema?.hash) {
68
- return Promise.resolve(new Uint8Array(1));
69
- }
70
- this.channel.postMessage(buf);
71
- return new Promise(this.callback);
68
+ updateCtx(address) {
69
+ return this.call(address);
72
70
  }
73
71
  }
74
72
  //# sourceMappingURL=DbWorker.js.map
@@ -0,0 +1,39 @@
1
+ import { registerMsgHandler } from './worker.js';
2
+ import { ENCODER, writeInt32 } from '@saulx/utils';
3
+ import native from '../../native.js';
4
+ function loadBlock(dbCtx, filepath) {
5
+ try {
6
+ native.loadBlock(filepath, dbCtx);
7
+ }
8
+ catch (e) {
9
+ // need to get rid of the shared buffer
10
+ return new Uint8Array(ENCODER.encode(e.toString())).buffer;
11
+ }
12
+ return null;
13
+ }
14
+ function unloadBlock(dbCtx, filepath, typeId, start) {
15
+ const buf = new ArrayBuffer(20); // [[4 bytes err], [16 bytes hash]]
16
+ const hash = new Uint8Array(buf, 4);
17
+ const err = native.saveBlock(filepath, typeId, start, dbCtx, hash);
18
+ if (err) {
19
+ const errCodeBuf = new Uint8Array(buf, 0, 4);
20
+ writeInt32(errCodeBuf, err, 0);
21
+ }
22
+ else {
23
+ native.delBlock(dbCtx, typeId, start);
24
+ }
25
+ return buf;
26
+ }
27
+ registerMsgHandler((dbCtx, msg) => {
28
+ if (typeof msg?.type === 'string') {
29
+ const job = msg;
30
+ if (job.type === 'load') {
31
+ return loadBlock(dbCtx, job.filepath);
32
+ }
33
+ else if (job.type === 'unload') {
34
+ return unloadBlock(dbCtx, job.filepath, job.typeId, job.start);
35
+ }
36
+ }
37
+ return null;
38
+ });
39
+ //# sourceMappingURL=io_worker.js.map
@@ -0,0 +1,12 @@
1
+ type IoJobLoad = {
2
+ type: 'load';
3
+ filepath: string;
4
+ };
5
+ type IoJobUnload = {
6
+ type: 'unload';
7
+ filepath: string;
8
+ typeId: number;
9
+ start: number;
10
+ };
11
+ export type IoJob = IoJobLoad | IoJobUnload;
12
+ export {};
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=io_worker_types.js.map
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,4 @@
1
+ import { registerMsgHandler } from './worker.js';
2
+ import native from '../../native.js';
3
+ registerMsgHandler((dbCtx, msg) => native.getQueryBuf(msg, dbCtx));
4
+ //# sourceMappingURL=query_worker.js.map
@@ -0,0 +1 @@
1
+ export declare function registerMsgHandler(onMsg: (dbCtx: any, msg: any) => Uint8Array<ArrayBufferLike> | ArrayBuffer | null): void;
@@ -0,0 +1,41 @@
1
+ import { isMainThread, parentPort, workerData } from 'node:worker_threads';
2
+ import native from '../../native.js';
3
+ let dbCtx;
4
+ let wCtx; // This must be held until the worker exits otherwise the ctx will be autofreed instantly
5
+ if (isMainThread) {
6
+ console.warn(`running a worker in the mainthread - incorrect`);
7
+ }
8
+ else if (workerData?.isDbWorker) {
9
+ let { address } = workerData;
10
+ dbCtx = native.externalFromInt(address);
11
+ wCtx = native.workerCtxInit();
12
+ }
13
+ else {
14
+ console.info('incorrect worker db query');
15
+ }
16
+ export function registerMsgHandler(onMsg) {
17
+ if (!workerData?.isDbWorker) {
18
+ throw new Error('Not a DbWorker');
19
+ }
20
+ let { channel } = workerData;
21
+ const handleMsg = (msg) => {
22
+ try {
23
+ if (typeof msg === 'bigint') {
24
+ // it's a ctx address
25
+ dbCtx = native.externalFromInt(msg);
26
+ channel.postMessage(null);
27
+ }
28
+ else {
29
+ // a message to the worker handler
30
+ const arrayBuf = onMsg(dbCtx, msg);
31
+ channel.postMessage(arrayBuf, [arrayBuf]);
32
+ }
33
+ }
34
+ catch (e) {
35
+ channel.postMessage(e);
36
+ }
37
+ };
38
+ channel.on('message', handleMsg);
39
+ parentPort.postMessage('READY');
40
+ }
41
+ //# sourceMappingURL=worker.js.map
@@ -2,6 +2,7 @@ import { DbSchema } from '../schema.js';
2
2
  export type EventMap = {
3
3
  schema: DbSchema;
4
4
  info: string;
5
+ error: string;
5
6
  };
6
7
  export type Event = keyof EventMap;
7
8
  export type Listener<T> = (data: T) => void;
@@ -1,4 +1,4 @@
1
- export declare const SCHEMA_FILE = "schema.json";
1
+ export declare const SCHEMA_FILE = "schema.bin";
2
2
  export declare const WRITELOG_FILE = "writelog.json";
3
3
  export declare const COMMON_SDB_FILE = "common.sdb";
4
4
  export type BasedDbOpts = {
package/dist/src/types.js CHANGED
@@ -1,4 +1,4 @@
1
- export const SCHEMA_FILE = 'schema.json';
1
+ export const SCHEMA_FILE = 'schema.bin';
2
2
  export const WRITELOG_FILE = 'writelog.json';
3
3
  export const COMMON_SDB_FILE = 'common.sdb';
4
4
  //# sourceMappingURL=types.js.map
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@based/db",
3
- "version": "0.0.66",
3
+ "version": "0.0.68",
4
4
  "license": "MIT",
5
5
  "type": "module",
6
6
  "main": "./dist/src/index.js",
@@ -38,9 +38,9 @@
38
38
  "basedDbNative.cjs"
39
39
  ],
40
40
  "dependencies": {
41
- "@based/schema": "5.0.0-alpha.20",
41
+ "@based/schema": "5.0.0-alpha.23",
42
42
  "@saulx/hash": "^3.0.0",
43
- "@saulx/utils": "^6.7.0",
43
+ "@saulx/utils": "^6.7.2",
44
44
  "exit-hook": "^4.0.0",
45
45
  "picocolors": "^1.1.0",
46
46
  "@based/crc32c": "^1.0.0"