@based/db 0.0.64 → 0.0.67

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (144) hide show
  1. package/README.md +2 -2
  2. package/dist/lib/darwin_aarch64/include/selva/colvec.h +71 -0
  3. package/dist/lib/darwin_aarch64/include/selva/db.h +33 -4
  4. package/dist/lib/darwin_aarch64/include/selva/fields.h +37 -25
  5. package/dist/lib/darwin_aarch64/include/selva/hll.h +5 -3
  6. package/dist/lib/darwin_aarch64/include/selva/membar.h +23 -0
  7. package/dist/lib/darwin_aarch64/include/selva/types.h +8 -1
  8. package/dist/lib/darwin_aarch64/include/selva/worker_ctx.h +19 -3
  9. package/dist/lib/darwin_aarch64/libdeflate.dylib +0 -0
  10. package/dist/lib/darwin_aarch64/libjemalloc_selva.2.dylib +0 -0
  11. package/dist/lib/darwin_aarch64/libnode-v20.node +0 -0
  12. package/dist/lib/darwin_aarch64/libnode-v21.node +0 -0
  13. package/dist/lib/darwin_aarch64/libnode-v22.node +0 -0
  14. package/dist/lib/darwin_aarch64/libnode-v23.node +0 -0
  15. package/dist/lib/darwin_aarch64/libnode-v24.node +0 -0
  16. package/dist/lib/darwin_aarch64/libselva.dylib +0 -0
  17. package/dist/lib/darwin_aarch64/libxxhash.dylib +0 -0
  18. package/dist/lib/linux_aarch64/include/selva/colvec.h +71 -0
  19. package/dist/lib/linux_aarch64/include/selva/db.h +33 -4
  20. package/dist/lib/linux_aarch64/include/selva/fields.h +37 -25
  21. package/dist/lib/linux_aarch64/include/selva/hll.h +5 -3
  22. package/dist/lib/linux_aarch64/include/selva/membar.h +23 -0
  23. package/dist/lib/linux_aarch64/include/selva/types.h +8 -1
  24. package/dist/lib/linux_aarch64/include/selva/worker_ctx.h +19 -3
  25. package/dist/lib/linux_aarch64/libjemalloc_selva.so.2 +0 -0
  26. package/dist/lib/linux_aarch64/libnode-v20.node +0 -0
  27. package/dist/lib/linux_aarch64/libnode-v21.node +0 -0
  28. package/dist/lib/linux_aarch64/libnode-v22.node +0 -0
  29. package/dist/lib/linux_aarch64/libnode-v23.node +0 -0
  30. package/dist/lib/linux_aarch64/libnode-v24.node +0 -0
  31. package/dist/lib/linux_aarch64/libselva.so +0 -0
  32. package/dist/lib/linux_x86_64/include/selva/colvec.h +71 -0
  33. package/dist/lib/linux_x86_64/include/selva/db.h +33 -4
  34. package/dist/lib/linux_x86_64/include/selva/fields.h +37 -25
  35. package/dist/lib/linux_x86_64/include/selva/hll.h +5 -3
  36. package/dist/lib/linux_x86_64/include/selva/membar.h +23 -0
  37. package/dist/lib/linux_x86_64/include/selva/types.h +8 -1
  38. package/dist/lib/linux_x86_64/include/selva/worker_ctx.h +19 -3
  39. package/dist/lib/linux_x86_64/libjemalloc_selva.so.2 +0 -0
  40. package/dist/lib/linux_x86_64/libnode-v20.node +0 -0
  41. package/dist/lib/linux_x86_64/libnode-v21.node +0 -0
  42. package/dist/lib/linux_x86_64/libnode-v22.node +0 -0
  43. package/dist/lib/linux_x86_64/libnode-v23.node +0 -0
  44. package/dist/lib/linux_x86_64/libnode-v24.node +0 -0
  45. package/dist/lib/linux_x86_64/libselva.so +0 -0
  46. package/dist/src/client/flushModify.d.ts +2 -1
  47. package/dist/src/client/flushModify.js +12 -4
  48. package/dist/src/client/modify/create.js +11 -0
  49. package/dist/src/client/modify/delete.js +3 -0
  50. package/dist/src/client/modify/fixed.js +1 -1
  51. package/dist/src/client/modify/modify.js +2 -2
  52. package/dist/src/client/modify/setCursor.d.ts +2 -1
  53. package/dist/src/client/query/BasedDbQuery.d.ts +10 -4
  54. package/dist/src/client/query/BasedDbQuery.js +114 -6
  55. package/dist/src/client/query/aggregates/aggregation.js +24 -11
  56. package/dist/src/client/query/aggregates/types.d.ts +22 -2
  57. package/dist/src/client/query/aggregates/types.js +34 -1
  58. package/dist/src/client/query/display.js +8 -2
  59. package/dist/src/client/query/filter/createVariableFilterBuffer.d.ts +2 -3
  60. package/dist/src/client/query/filter/createVariableFilterBuffer.js +20 -7
  61. package/dist/src/client/query/filter/filter.js +13 -3
  62. package/dist/src/client/query/filter/primitiveFilter.d.ts +1 -2
  63. package/dist/src/client/query/include/props.js +18 -2
  64. package/dist/src/client/query/include/toBuffer.js +11 -3
  65. package/dist/src/client/query/include/walk.js +5 -1
  66. package/dist/src/client/query/queryDef.js +4 -1
  67. package/dist/src/client/query/read/read.js +52 -22
  68. package/dist/src/client/query/registerQuery.js +1 -0
  69. package/dist/src/client/query/search/index.d.ts +1 -1
  70. package/dist/src/client/query/search/index.js +21 -7
  71. package/dist/src/client/query/sort.d.ts +1 -1
  72. package/dist/src/client/query/toByteCode/default.d.ts +1 -1
  73. package/dist/src/client/query/toByteCode/default.js +0 -2
  74. package/dist/src/client/query/toByteCode/toBuffer.js +0 -7
  75. package/dist/src/client/query/types.d.ts +16 -5
  76. package/dist/src/client/query/validation.d.ts +3 -0
  77. package/dist/src/client/query/validation.js +34 -2
  78. package/dist/src/client/xxHash64.d.ts +1 -1
  79. package/dist/src/index.d.ts +1 -2
  80. package/dist/src/index.js +0 -1
  81. package/dist/src/native.d.ts +7 -4
  82. package/dist/src/native.js +23 -13
  83. package/dist/src/server/IoWorker.d.ts +8 -0
  84. package/dist/src/server/IoWorker.js +39 -0
  85. package/dist/src/server/QueryWorker.d.ts +8 -0
  86. package/dist/src/server/QueryWorker.js +26 -0
  87. package/dist/src/server/blocks.d.ts +24 -0
  88. package/dist/src/server/blocks.js +112 -0
  89. package/dist/src/server/dbHash.d.ts +1 -1
  90. package/dist/src/server/index.d.ts +10 -16
  91. package/dist/src/server/index.js +39 -15
  92. package/dist/src/server/migrate/index.d.ts +5 -0
  93. package/dist/src/server/migrate/index.js +11 -7
  94. package/dist/src/server/migrate/worker.js +3 -0
  95. package/dist/src/server/save.d.ts +8 -6
  96. package/dist/src/server/save.js +34 -78
  97. package/dist/src/server/schema.js +6 -5
  98. package/dist/src/server/start.js +57 -60
  99. package/dist/src/server/tree.d.ts +24 -13
  100. package/dist/src/server/tree.js +95 -66
  101. package/dist/src/server/workers/DbWorker.d.ts +17 -0
  102. package/dist/src/server/{DbWorker.js → workers/DbWorker.js} +15 -17
  103. package/dist/src/server/workers/io_worker.js +39 -0
  104. package/dist/src/server/workers/io_worker_types.d.ts +12 -0
  105. package/dist/src/server/workers/io_worker_types.js +2 -0
  106. package/dist/src/server/workers/query_worker.d.ts +1 -0
  107. package/dist/src/server/workers/query_worker.js +4 -0
  108. package/dist/src/server/workers/worker.d.ts +1 -0
  109. package/dist/src/server/workers/worker.js +41 -0
  110. package/dist/src/shared/Emitter.d.ts +1 -0
  111. package/dist/src/types.d.ts +1 -1
  112. package/dist/src/types.js +1 -1
  113. package/package.json +3 -3
  114. package/dist/lib/darwin_aarch64/include/selva/find.h +0 -47
  115. package/dist/lib/darwin_aarch64/include/selva/history.h +0 -64
  116. package/dist/lib/darwin_aarch64/include/selva/queue_r.h +0 -190
  117. package/dist/lib/darwin_aarch64/include/selva/traverse.h +0 -65
  118. package/dist/lib/linux_aarch64/include/selva/find.h +0 -47
  119. package/dist/lib/linux_aarch64/include/selva/history.h +0 -64
  120. package/dist/lib/linux_aarch64/include/selva/queue_r.h +0 -190
  121. package/dist/lib/linux_aarch64/include/selva/traverse.h +0 -65
  122. package/dist/lib/linux_x86_64/include/selva/find.h +0 -47
  123. package/dist/lib/linux_x86_64/include/selva/history.h +0 -64
  124. package/dist/lib/linux_x86_64/include/selva/queue_r.h +0 -190
  125. package/dist/lib/linux_x86_64/include/selva/traverse.h +0 -65
  126. package/dist/src/client/query/serialize.d.ts +0 -4
  127. package/dist/src/client/query/serialize.js +0 -26
  128. package/dist/src/server/DbWorker.d.ts +0 -13
  129. package/dist/src/server/csmt/draw-dot.d.ts +0 -4
  130. package/dist/src/server/csmt/draw-dot.js +0 -38
  131. package/dist/src/server/csmt/index.d.ts +0 -4
  132. package/dist/src/server/csmt/index.js +0 -5
  133. package/dist/src/server/csmt/match.d.ts +0 -7
  134. package/dist/src/server/csmt/match.js +0 -10
  135. package/dist/src/server/csmt/memebership-proof.d.ts +0 -7
  136. package/dist/src/server/csmt/memebership-proof.js +0 -122
  137. package/dist/src/server/csmt/tree-utils.d.ts +0 -6
  138. package/dist/src/server/csmt/tree-utils.js +0 -33
  139. package/dist/src/server/csmt/tree.d.ts +0 -3
  140. package/dist/src/server/csmt/tree.js +0 -270
  141. package/dist/src/server/csmt/types.d.ts +0 -46
  142. package/dist/src/server/csmt/types.js +0 -2
  143. package/dist/src/server/worker.js +0 -33
  144. /package/dist/src/server/{worker.d.ts → workers/io_worker.d.ts} +0 -0
@@ -4,21 +4,19 @@ import db from '../../basedDbNative.cjs';
4
4
  const DECODER = new TextDecoder('utf-8');
5
5
  const ENCODER = new TextEncoder();
6
6
  const selvaIoErrlog = new Uint8Array(256);
7
- var compressor = db.createCompressor();
7
+ var compressor = db.createCompressor(); // put on threadCtx
8
8
  var decompressor = db.createDecompressor();
9
9
  function SelvaIoErrlogToString(buf) {
10
10
  let i;
11
11
  let len = (i = buf.indexOf(0)) >= 0 ? i : buf.byteLength;
12
12
  return DECODER.decode(selvaIoErrlog.slice(0, len));
13
13
  }
14
+ // add worker CTX HERE
15
+ // then add it to every function
16
+ // worker should allways be here
17
+ // then add ThreadCtx to modify ctx and query ctx
14
18
  const native = {
15
- historyAppend(history, typeId, nodeId, dbCtx) {
16
- return db.historyAppend(history, typeId, nodeId, dbCtx);
17
- },
18
- historyCreate(pathname, mainLen) {
19
- const pathBuf = ENCODER.encode(pathname + '\0');
20
- return db.historyCreate(pathBuf, mainLen + 16 - (mainLen % 16));
21
- },
19
+ threadCtx: null, // add compressors here as well!
22
20
  workerCtxInit: () => {
23
21
  return db.workerCtxInit();
24
22
  },
@@ -45,9 +43,9 @@ const native = {
45
43
  const pathBuf = ENCODER.encode(path + '\0');
46
44
  return db.saveCommon(pathBuf, dbCtx);
47
45
  },
48
- saveRange: (path, typeCode, start, end, dbCtx, hashOut) => {
46
+ saveBlock: (path, typeCode, start, dbCtx, hashOut) => {
49
47
  const pathBuf = ENCODER.encode(path + '\0');
50
- return db.saveRange(pathBuf, typeCode, start, end, dbCtx, hashOut);
48
+ return db.saveBlock(pathBuf, typeCode, start, dbCtx, hashOut);
51
49
  },
52
50
  loadCommon: (path, dbCtx) => {
53
51
  const pathBuf = ENCODER.encode(path + '\0');
@@ -56,13 +54,16 @@ const native = {
56
54
  throw new Error(`Failed to load common. selvaError: ${err} cause:\n${SelvaIoErrlogToString(selvaIoErrlog)}`);
57
55
  }
58
56
  },
59
- loadRange: (path, dbCtx) => {
57
+ loadBlock: (path, dbCtx) => {
60
58
  const pathBuf = ENCODER.encode(path + '\0');
61
- const err = db.loadRange(pathBuf, dbCtx, selvaIoErrlog);
59
+ const err = db.loadBlock(pathBuf, dbCtx, selvaIoErrlog);
62
60
  if (err) {
63
- throw new Error(`Failed to load a range. selvaError: ${err} cause:\n${SelvaIoErrlogToString(selvaIoErrlog)}`);
61
+ throw new Error(`Failed to load a range "${path}". selvaError: ${err} cause:\n${SelvaIoErrlogToString(selvaIoErrlog)}`);
64
62
  }
65
63
  },
64
+ delBlock: (dbCtx, typeId, block) => {
65
+ db.delBlock(dbCtx, typeId, block);
66
+ },
66
67
  updateSchemaType: (prefix, buf, dbCtx) => {
67
68
  return db.updateSchema(prefix, buf, dbCtx);
68
69
  },
@@ -93,6 +94,15 @@ const native = {
93
94
  equals: (a, b) => {
94
95
  return !!db.equals(a, b);
95
96
  },
97
+ membarSyncRead: () => {
98
+ db.membarSyncRead();
99
+ },
100
+ membarSyncWrite: () => {
101
+ db.membarSyncWrite();
102
+ },
103
+ colvecTest: (dbCtx, typeId, field, nodeId, len) => {
104
+ return db.colvecTest(dbCtx, typeId, field, nodeId, len);
105
+ }
96
106
  };
97
107
  global.__basedDb__native__ = native;
98
108
  export default native;
@@ -0,0 +1,8 @@
1
+ import { DbWorker } from './workers/DbWorker.js';
2
+ import { DbServer } from './index.js';
3
+ export declare class IoWorker extends DbWorker {
4
+ constructor(address: BigInt, db: DbServer);
5
+ handleMsg(_buf: any): void;
6
+ loadBlock(filepath: string): Promise<void>;
7
+ unloadBlock(filepath: string, typeId: number, start: number): Promise<Uint8Array>;
8
+ }
@@ -0,0 +1,39 @@
1
+ import { DbWorker } from './workers/DbWorker.js';
2
+ import { DECODER, readInt32 } from '@saulx/utils';
3
+ export class IoWorker extends DbWorker {
4
+ constructor(address, db) {
5
+ const onExit = (_code) => {
6
+ this.db.ioWorker = new IoWorker(address, db);
7
+ };
8
+ super(address, db, onExit, 'io_worker.js');
9
+ }
10
+ handleMsg(_buf) {
11
+ }
12
+ async loadBlock(filepath) {
13
+ const job = {
14
+ type: 'load',
15
+ filepath
16
+ };
17
+ const resBuf = await this.call(job);
18
+ if (resBuf.length) {
19
+ throw new Error(DECODER.decode(resBuf));
20
+ }
21
+ }
22
+ async unloadBlock(filepath, typeId, start) {
23
+ const job = {
24
+ type: 'unload',
25
+ filepath,
26
+ typeId,
27
+ start
28
+ };
29
+ const resBuf = await this.call(job);
30
+ const err = readInt32(resBuf, 0);
31
+ if (err) {
32
+ throw new Error(`selva error: ${err}`);
33
+ }
34
+ // Note that this shares the original buffer which may not be 100% optimal,
35
+ // as the first 4 bytes are no longer needed.
36
+ return new Uint8Array(resBuf.buffer, 4);
37
+ }
38
+ }
39
+ //# sourceMappingURL=IoWorker.js.map
@@ -0,0 +1,8 @@
1
+ import { DbWorker } from './workers/DbWorker.js';
2
+ import { DbServer } from './index.js';
3
+ export declare class QueryWorker extends DbWorker {
4
+ constructor(address: BigInt, db: DbServer, workerIndex: number);
5
+ handleMsg(_buf: any): void;
6
+ protected callback: (resolve: (x: any) => any) => void;
7
+ getQueryBuf(buf: Uint8Array): Promise<Uint8Array>;
8
+ }
@@ -0,0 +1,26 @@
1
+ import { DbWorker } from './workers/DbWorker.js';
2
+ import { readUint64 } from '@saulx/utils';
3
+ export class QueryWorker extends DbWorker {
4
+ constructor(address, db, workerIndex) {
5
+ const onExit = (_code) => {
6
+ this.db.workers[workerIndex] = new QueryWorker(address, db, workerIndex);
7
+ };
8
+ super(address, db, onExit, 'query_worker.js');
9
+ }
10
+ handleMsg(_buf) {
11
+ this.db.processingQueries--;
12
+ this.db.onQueryEnd();
13
+ }
14
+ callback = (resolve) => {
15
+ this.db.processingQueries++;
16
+ this.resolvers.push(resolve);
17
+ };
18
+ getQueryBuf(buf) {
19
+ const schemaChecksum = readUint64(buf, buf.byteLength - 8);
20
+ if (schemaChecksum !== this.db.schema?.hash) {
21
+ return Promise.resolve(new Uint8Array(1));
22
+ }
23
+ return this.call(buf);
24
+ }
25
+ }
26
+ //# sourceMappingURL=QueryWorker.js.map
@@ -0,0 +1,24 @@
1
+ import { SchemaTypeDef } from '@based/schema/def';
2
+ import { DbServer } from './index.js';
3
+ /**
4
+ * Save a block.
5
+ */
6
+ export declare function saveBlock(db: DbServer, typeId: number, start: number, end: number): void;
7
+ /**
8
+ * Load a block (typically of a partial type) back to memory.
9
+ */
10
+ export declare function loadBlock(db: DbServer, def: SchemaTypeDef, start: number): Promise<void>;
11
+ /**
12
+ * Save a block and remove it from memory.
13
+ */
14
+ export declare function unloadBlock(db: DbServer, def: SchemaTypeDef, start: number): Promise<void>;
15
+ /**
16
+ * Execute cb() for each block in memory.
17
+ */
18
+ export declare function foreachBlock(db: DbServer, def: SchemaTypeDef, cb: (start: number, end: number, hash: Uint8Array) => void, includeEmptyBlocks?: boolean): void;
19
+ /**
20
+ * Execute cb() for each dirty block.
21
+ * A dirty block is one that is changed in memory but not yet persisted in the
22
+ * file system.
23
+ */
24
+ export declare function foreachDirtyBlock(db: DbServer, cb: (mtKey: number, typeId: number, start: number, end: number) => void): void;
@@ -0,0 +1,112 @@
1
+ import native from '../native.js';
2
+ import { join } from 'node:path';
3
+ import { equals } from '@saulx/utils';
4
+ import { VerifTree, destructureTreeKey, makeTreeKey, } from './tree.js';
5
+ /**
6
+ * Save a block.
7
+ */
8
+ export function saveBlock(db, typeId, start, end) {
9
+ const hash = new Uint8Array(16);
10
+ const mtKey = makeTreeKey(typeId, start);
11
+ const file = VerifTree.blockSdbFile(typeId, start, end);
12
+ const path = join(db.fileSystemPath, file);
13
+ const err = native.saveBlock(path, typeId, start, db.dbCtxExternal, hash);
14
+ if (err == -8) {
15
+ // TODO ENOENT
16
+ db.verifTree.remove(mtKey);
17
+ }
18
+ else if (err) {
19
+ // TODO print the error string
20
+ console.error(`Save ${typeId}:${start}-${end} failed: ${err}`);
21
+ }
22
+ else {
23
+ db.verifTree.update(mtKey, hash);
24
+ }
25
+ }
26
+ /**
27
+ * Load a block (typically of a partial type) back to memory.
28
+ */
29
+ export async function loadBlock(db, def, start) {
30
+ const key = makeTreeKey(def.id, start);
31
+ const block = db.verifTree.getBlock(key);
32
+ if (!block) {
33
+ throw new Error(`No such block: ${key}`);
34
+ }
35
+ if (block.loadPromise) {
36
+ return block.loadPromise;
37
+ }
38
+ const prevHash = block.hash;
39
+ const filename = db.verifTree.getBlockFile(block);
40
+ const p = db.ioWorker.loadBlock(join(db.fileSystemPath, filename));
41
+ block.loadPromise = p;
42
+ await p;
43
+ delete block.loadPromise;
44
+ // Update and verify the hash
45
+ const hash = new Uint8Array(16);
46
+ const end = start + def.blockCapacity - 1;
47
+ const res = native.getNodeRangeHash(def.id, start, end, hash, db.dbCtxExternal);
48
+ if (res) {
49
+ const key = makeTreeKey(def.id, start);
50
+ db.verifTree.update(key, hash);
51
+ if (!equals(prevHash, hash)) {
52
+ throw new Error('Block hash mismatch');
53
+ }
54
+ }
55
+ }
56
+ /**
57
+ * Save a block and remove it from memory.
58
+ */
59
+ export async function unloadBlock(db, def, start) {
60
+ const typeId = def.id;
61
+ const end = start + def.blockCapacity - 1;
62
+ const key = makeTreeKey(typeId, start);
63
+ const block = db.verifTree.getBlock(key);
64
+ if (!block) {
65
+ throw new Error(`No such block: ${key}`);
66
+ }
67
+ const filepath = join(db.fileSystemPath, VerifTree.blockSdbFile(typeId, start, end));
68
+ try {
69
+ const hash = await db.ioWorker.unloadBlock(filepath, typeId, start);
70
+ native.delBlock(db.dbCtxExternal, typeId, start);
71
+ db.verifTree.update(key, hash, false);
72
+ }
73
+ catch (e) {
74
+ // TODO Proper logging
75
+ // TODO err == -8 == SELVA_ENOENT => db.verifTree.remove(key) ??
76
+ console.error(`Save ${typeId}:${start}-${end} failed`);
77
+ console.error(e);
78
+ }
79
+ }
80
+ /**
81
+ * Execute cb() for each block in memory.
82
+ */
83
+ export function foreachBlock(db, def, cb, includeEmptyBlocks = false) {
84
+ const step = def.blockCapacity;
85
+ for (let start = 1; start <= def.lastId; start += step) {
86
+ const end = start + step - 1;
87
+ const hash = new Uint8Array(16);
88
+ const res = native.getNodeRangeHash(def.id, start, end, hash, db.dbCtxExternal);
89
+ if (res || includeEmptyBlocks) {
90
+ cb(start, end, hash);
91
+ }
92
+ }
93
+ }
94
+ /**
95
+ * Execute cb() for each dirty block.
96
+ * A dirty block is one that is changed in memory but not yet persisted in the
97
+ * file system.
98
+ */
99
+ export function foreachDirtyBlock(db, cb) {
100
+ const typeIdMap = {};
101
+ for (const typeName in db.schemaTypesParsed) {
102
+ const type = db.schemaTypesParsed[typeName];
103
+ const typeId = type.id;
104
+ typeIdMap[typeId] = type;
105
+ }
106
+ for (const mtKey of db.dirtyRanges) {
107
+ const [typeId, start] = destructureTreeKey(mtKey);
108
+ const end = start + typeIdMap[typeId].blockCapacity - 1;
109
+ cb(mtKey, typeId, start, end);
110
+ }
111
+ }
112
+ //# sourceMappingURL=blocks.js.map
@@ -1,5 +1,5 @@
1
1
  export default function createHash(): {
2
- update: (buf: Uint8Array) => any;
2
+ update: (buf: Uint8Array) => /*elided*/ any;
3
3
  digest: (encoding?: "hex") => Uint8Array | string;
4
4
  reset: () => void;
5
5
  };
@@ -1,11 +1,11 @@
1
1
  import { LangName, StrictSchema } from '@based/schema';
2
- import { createTree } from './csmt/index.js';
3
2
  import { StartOpts } from './start.js';
4
- import { CsmtNodeRange } from './tree.js';
3
+ import { VerifTree } from './tree.js';
5
4
  import { TransformFns } from './migrate/index.js';
6
5
  import exitHook from 'exit-hook';
7
6
  import { SchemaChecksum } from '../schema.js';
8
- import { DbWorker } from './DbWorker.js';
7
+ import { IoWorker } from './IoWorker.js';
8
+ import { QueryWorker } from './QueryWorker.js';
9
9
  import { DbShared } from '../shared/DbBase.js';
10
10
  declare class SortIndex {
11
11
  constructor(buf: Uint8Array, dbCtxExternal: any);
@@ -17,17 +17,14 @@ export declare class DbServer extends DbShared {
17
17
  #private;
18
18
  modifyDirtyRanges: Float64Array;
19
19
  dbCtxExternal: any;
20
+ threadCtxExternal: any;
20
21
  migrating: number;
21
22
  saveInProgress: boolean;
22
23
  fileSystemPath: string;
23
- merkleTree: ReturnType<typeof createTree<CsmtNodeRange>>;
24
+ verifTree: VerifTree;
24
25
  dirtyRanges: Set<number>;
25
- csmtHashFun: {
26
- update: (buf: Uint8Array) => any;
27
- digest: (encoding?: "hex") => Uint8Array | string;
28
- reset: () => void;
29
- };
30
- workers: DbWorker[];
26
+ ioWorker: IoWorker;
27
+ workers: QueryWorker[];
31
28
  availableWorkerIndex: number;
32
29
  processingQueries: number;
33
30
  modifyQueue: Uint8Array[];
@@ -46,11 +43,8 @@ export declare class DbServer extends DbShared {
46
43
  save(opts?: {
47
44
  forceFullDump?: boolean;
48
45
  }): Promise<void>;
49
- createCsmtHashFun: () => {
50
- update: (buf: Uint8Array) => any;
51
- digest: (encoding?: "hex") => Uint8Array | string;
52
- reset: () => void;
53
- };
46
+ loadBlock(typeName: string, nodeId: number): Promise<void>;
47
+ unloadBlock(typeName: string, nodeId: number): Promise<void>;
54
48
  sortIndexes: {
55
49
  [type: number]: {
56
50
  [field: number]: {
@@ -68,7 +62,7 @@ export declare class DbServer extends DbShared {
68
62
  createSortIndexBuffer(typeId: number, field: number, start: number, lang: number): SortIndex;
69
63
  setSchema(strictSchema: StrictSchema, transformFns?: TransformFns): Promise<SchemaChecksum>;
70
64
  modify(bufWithHash: Uint8Array): Record<number, number> | null;
71
- addToQueryQueue(resolve: any, buf: any): Promise<Uint8Array>;
65
+ addToQueryQueue(resolve: any, buf: any): Promise<Uint8Array<ArrayBuffer>>;
72
66
  getQueryBuf(buf: Uint8Array, fromQueue?: boolean): Promise<Uint8Array>;
73
67
  onQueryEnd(): any;
74
68
  stop(noSave?: boolean): Promise<void>;
@@ -1,9 +1,8 @@
1
1
  import native from '../native.js';
2
- import createDbHash from './dbHash.js';
3
2
  import { rm } from 'node:fs/promises';
4
3
  import { langCodesMap } from '@based/schema';
5
4
  import { start } from './start.js';
6
- import { makeCsmtKeyFromNodeId } from './tree.js';
5
+ import { destructureTreeKey, makeTreeKeyFromNodeId } from './tree.js';
7
6
  import { save } from './save.js';
8
7
  import { setTimeout } from 'node:timers/promises';
9
8
  import { migrate } from './migrate/index.js';
@@ -14,6 +13,7 @@ import { strictSchemaToDbSchema } from './schema.js';
14
13
  import { DbShared } from '../shared/DbBase.js';
15
14
  import { setNativeSchema, setSchemaOnServer, writeSchemaFile, } from './schema.js';
16
15
  import { resizeModifyDirtyRanges } from './resizeModifyDirtyRanges.js';
16
+ import { loadBlock, unloadBlock } from './blocks.js';
17
17
  const emptyUint8Array = new Uint8Array(0);
18
18
  class SortIndex {
19
19
  constructor(buf, dbCtxExternal) {
@@ -27,15 +27,16 @@ class SortIndex {
27
27
  export class DbServer extends DbShared {
28
28
  modifyDirtyRanges;
29
29
  dbCtxExternal; // pointer to zig dbCtx
30
+ threadCtxExternal; // pointer to zig dbCtx
30
31
  migrating = null;
31
32
  saveInProgress = false;
32
33
  fileSystemPath;
33
- merkleTree;
34
+ verifTree;
34
35
  dirtyRanges = new Set();
35
- csmtHashFun = createDbHash();
36
+ ioWorker;
36
37
  workers = [];
37
38
  availableWorkerIndex = -1;
38
- processingQueries = 0;
39
+ processingQueries = 0; // semaphore for locking writes
39
40
  modifyQueue = [];
40
41
  queryQueue = new Map();
41
42
  stopped; // = true does not work
@@ -59,11 +60,34 @@ export class DbServer extends DbShared {
59
60
  save(opts) {
60
61
  return save(this, false, opts?.forceFullDump ?? false);
61
62
  }
62
- createCsmtHashFun = () => {
63
- // We can just reuse it as long as we only have one tree.
64
- this.csmtHashFun.reset();
65
- return this.csmtHashFun;
66
- };
63
+ async loadBlock(typeName, nodeId) {
64
+ const def = this.schemaTypesParsed[typeName];
65
+ if (!def) {
66
+ throw new Error('Type not found');
67
+ }
68
+ const typeId = def.id;
69
+ const key = makeTreeKeyFromNodeId(typeId, def.blockCapacity, nodeId);
70
+ const [, start] = destructureTreeKey(key);
71
+ const block = this.verifTree.getBlock(key);
72
+ if (!block) {
73
+ throw new Error('Block not found');
74
+ }
75
+ await loadBlock(this, def, start);
76
+ }
77
+ async unloadBlock(typeName, nodeId) {
78
+ const def = this.schemaTypesParsed[typeName];
79
+ if (!def) {
80
+ throw new Error('Type not found');
81
+ }
82
+ const typeId = def.id;
83
+ const key = makeTreeKeyFromNodeId(typeId, def.blockCapacity, nodeId);
84
+ const [, start] = destructureTreeKey(key);
85
+ const block = this.verifTree.getBlock(key);
86
+ if (!block) {
87
+ throw new Error('Block not found');
88
+ }
89
+ await unloadBlock(this, def, start);
90
+ }
67
91
  sortIndexes;
68
92
  cleanupTimer;
69
93
  cleanup() {
@@ -298,15 +322,15 @@ export class DbServer extends DbShared {
298
322
  while (typesSize--) {
299
323
  const typeId = readUint16(buf, i);
300
324
  const def = this.schemaTypesParsedById[typeId];
301
- const key = makeCsmtKeyFromNodeId(def.id, def.blockCapacity, def.lastId);
325
+ const key = makeTreeKeyFromNodeId(def.id, def.blockCapacity, def.lastId);
302
326
  this.dirtyRanges.add(key);
303
327
  i += 10;
304
328
  }
305
329
  const view = new DataView(buf.buffer, buf.byteOffset);
306
330
  while (i < end) {
307
- const key = view.getFloat64(i, true);
331
+ // const key = view.getFloat64(i, true)
308
332
  // These node ranges may not actually exist
309
- //this.dirtyRanges.add(key)
333
+ // this.dirtyRanges.add(key)
310
334
  i += 8;
311
335
  }
312
336
  resizeModifyDirtyRanges(this);
@@ -387,7 +411,6 @@ export class DbServer extends DbShared {
387
411
  }
388
412
  }
389
413
  onQueryEnd() {
390
- this.processingQueries--;
391
414
  if (this.processingQueries === 0) {
392
415
  if (this.modifyQueue.length) {
393
416
  const modifyQueue = this.modifyQueue;
@@ -430,7 +453,8 @@ export class DbServer extends DbShared {
430
453
  if (!noSave) {
431
454
  await this.save();
432
455
  }
433
- await Promise.all(this.workers.map(({ worker }) => worker.terminate()));
456
+ await this.ioWorker.terminate();
457
+ await Promise.all(this.workers.map((worker) => worker.terminate()));
434
458
  this.workers = [];
435
459
  native.stop(this.dbCtxExternal);
436
460
  await setTimeout(100);
@@ -1,5 +1,10 @@
1
1
  import { DbServer } from '../index.js';
2
2
  import { DbSchema } from '../../schema.js';
3
+ export type MigrateRange = {
4
+ typeId: number;
5
+ start: number;
6
+ end: number;
7
+ };
3
8
  type TransformFn = (node: Record<string, any>) => Record<string, any> | [string, Record<string, any>];
4
9
  export type TransformFns = Record<string, TransformFn>;
5
10
  export declare const migrate: (server: DbServer, fromSchema: DbSchema, toSchema: DbSchema, transform?: TransformFns) => Promise<void>;
@@ -2,7 +2,8 @@ import { BasedDb, save } from '../../index.js';
2
2
  import { dirname, join } from 'path';
3
3
  import { Worker, MessageChannel, receiveMessageOnPort, } from 'node:worker_threads';
4
4
  import native from '../../native.js';
5
- import { destructureCsmtKey, foreachDirtyBlock, specialBlock } from '../tree.js';
5
+ import { destructureTreeKey } from '../tree.js';
6
+ import { foreachDirtyBlock } from '../blocks.js';
6
7
  import { fileURLToPath } from 'url';
7
8
  import { setNativeSchema, setSchemaOnServer, writeSchemaFile, } from '../schema.js';
8
9
  import { setToAwake, waitUntilSleeping } from './utils.js';
@@ -90,11 +91,11 @@ export const migrate = async (server, fromSchema, toSchema, transform) => {
90
91
  let i = 0;
91
92
  let rangesToMigrate = [];
92
93
  await save(server, false, false, true);
93
- server.merkleTree.visitLeafNodes((leaf) => {
94
- const [_typeId, start] = destructureCsmtKey(leaf.key);
95
- if (start == specialBlock)
96
- return; // skip the type specialBlock
97
- rangesToMigrate.push(leaf.data);
94
+ server.verifTree.foreachBlock((block) => {
95
+ const [typeId, start] = destructureTreeKey(block.key);
96
+ const def = server.schemaTypesParsedById[typeId];
97
+ const end = start + def.blockCapacity - 1;
98
+ rangesToMigrate.push({ typeId, start, end });
98
99
  });
99
100
  await waitUntilSleeping(workerState);
100
101
  while (i < rangesToMigrate.length) {
@@ -108,12 +109,13 @@ export const migrate = async (server, fromSchema, toSchema, transform) => {
108
109
  setToAwake(workerState, true);
109
110
  await waitUntilSleeping(workerState);
110
111
  // exec queued modifies
112
+ server.processingQueries--;
111
113
  server.onQueryEnd();
112
114
  if (i === rangesToMigrate.length) {
113
115
  if (server.dirtyRanges.size) {
114
116
  rangesToMigrate = [];
115
117
  i = 0;
116
- void foreachDirtyBlock(server, (_mtKey, typeId, start, end) => {
118
+ foreachDirtyBlock(server, (_mtKey, typeId, start, end) => {
117
119
  rangesToMigrate.push({
118
120
  typeId,
119
121
  start,
@@ -140,6 +142,7 @@ export const migrate = async (server, fromSchema, toSchema, transform) => {
140
142
  // pass last node IDS { type: lastId }
141
143
  setSchemaOnServer(server, toSchema);
142
144
  for (const key in schemaTypesParsed) {
145
+ // maybe only send the lastId
143
146
  const def = server.schemaTypesParsed[key];
144
147
  def.lastId = schemaTypesParsed[key].lastId;
145
148
  }
@@ -152,6 +155,7 @@ export const migrate = async (server, fromSchema, toSchema, transform) => {
152
155
  if (abort()) {
153
156
  return;
154
157
  }
158
+ native.membarSyncRead();
155
159
  await save(server, false, true, true);
156
160
  await writeSchemaFile(server, toSchema);
157
161
  server.migrating = 0;
@@ -13,6 +13,8 @@ else if (workerData?.isDbMigrateWorker) {
13
13
  const { from, to, fromSchema, toSchema, channel, workerState, transformFns } = workerData;
14
14
  const fromCtx = native.externalFromInt(from);
15
15
  const toCtx = native.externalFromInt(to);
16
+ // worker ctx init - maybe instead of this just add it on native
17
+ // instead of here just do this in native.js
16
18
  native.workerCtxInit();
17
19
  const fromDb = new BasedDb({ path: null });
18
20
  const toDb = new BasedDb({ path: null });
@@ -109,6 +111,7 @@ else if (workerData?.isDbMigrateWorker) {
109
111
  }
110
112
  }
111
113
  await toDb.drain();
114
+ native.membarSyncWrite();
112
115
  // WE ARE ONLY GOING TO SEND { type: lastNodeId }
113
116
  channel.postMessage(cp(toDb.server.schemaTypesParsed));
114
117
  setToSleep(workerState);
@@ -1,4 +1,10 @@
1
1
  import { DbServer } from './index.js';
2
+ type RangeDump = {
3
+ file: string;
4
+ hash: string;
5
+ start: number;
6
+ end: number;
7
+ };
2
8
  export type Writelog = {
3
9
  ts: number;
4
10
  types: {
@@ -10,12 +16,8 @@ export type Writelog = {
10
16
  hash: string;
11
17
  commonDump: string;
12
18
  rangeDumps: {
13
- [t: number]: {
14
- file: string;
15
- hash: string;
16
- start: number;
17
- end: number;
18
- }[];
19
+ [t: number]: RangeDump[];
19
20
  };
20
21
  };
21
22
  export declare function save<T extends boolean>(db: DbServer, sync?: T, forceFullDump?: boolean, skipMigrationCheck?: boolean): T extends true ? void : Promise<void>;
23
+ export {};