@based/db 0.0.67 → 0.0.69

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/README.md +6 -1
  2. package/dist/lib/darwin_aarch64/include/selva/fields.h +0 -9
  3. package/dist/lib/darwin_aarch64/include/selva_error.h +4 -1
  4. package/dist/lib/darwin_aarch64/libnode-v22.node +0 -0
  5. package/dist/lib/darwin_aarch64/libnode-v23.node +0 -0
  6. package/dist/lib/darwin_aarch64/libnode-v24.node +0 -0
  7. package/dist/lib/darwin_aarch64/libselva.dylib +0 -0
  8. package/dist/lib/linux_aarch64/include/selva/fields.h +0 -9
  9. package/dist/lib/linux_aarch64/include/selva_error.h +4 -1
  10. package/dist/lib/linux_aarch64/libnode-v22.node +0 -0
  11. package/dist/lib/linux_aarch64/libnode-v23.node +0 -0
  12. package/dist/lib/linux_aarch64/libnode-v24.node +0 -0
  13. package/dist/lib/linux_aarch64/libselva.so +0 -0
  14. package/dist/lib/linux_x86_64/include/selva/fields.h +0 -9
  15. package/dist/lib/linux_x86_64/include/selva_error.h +4 -1
  16. package/dist/lib/linux_x86_64/libnode-v22.node +0 -0
  17. package/dist/lib/linux_x86_64/libnode-v23.node +0 -0
  18. package/dist/lib/linux_x86_64/libnode-v24.node +0 -0
  19. package/dist/lib/linux_x86_64/libselva.so +0 -0
  20. package/dist/src/client/modify/alias.js +4 -0
  21. package/dist/src/client/modify/binary.d.ts +1 -1
  22. package/dist/src/client/modify/binary.js +22 -7
  23. package/dist/src/client/modify/create.js +87 -39
  24. package/dist/src/client/modify/fixed.d.ts +3 -3
  25. package/dist/src/client/modify/fixed.js +80 -20
  26. package/dist/src/client/modify/json.js +7 -0
  27. package/dist/src/client/modify/modify.js +3 -3
  28. package/dist/src/client/modify/references/edge.js +3 -3
  29. package/dist/src/client/modify/string.d.ts +1 -1
  30. package/dist/src/client/modify/string.js +17 -7
  31. package/dist/src/client/modify/text.d.ts +1 -1
  32. package/dist/src/client/modify/text.js +27 -15
  33. package/dist/src/client/modify/types.d.ts +2 -1
  34. package/dist/src/client/modify/types.js +6 -0
  35. package/dist/src/client/modify/update.js +1 -1
  36. package/dist/src/client/query/display.js +1 -1
  37. package/dist/src/client/query/filter/createFixedFilterBuffer.js +3 -0
  38. package/dist/src/client/query/filter/createReferenceFilter.js +3 -0
  39. package/dist/src/client/query/filter/createVariableFilterBuffer.js +21 -19
  40. package/dist/src/client/query/read/read.js +9 -10
  41. package/dist/src/native.d.ts +1 -0
  42. package/dist/src/native.js +3 -0
  43. package/dist/src/server/IoWorker.d.ts +16 -0
  44. package/dist/src/server/IoWorker.js +32 -3
  45. package/dist/src/server/QueryWorker.js +2 -2
  46. package/dist/src/server/blocks.d.ts +3 -1
  47. package/dist/src/server/blocks.js +31 -7
  48. package/dist/src/server/index.d.ts +1 -1
  49. package/dist/src/server/index.js +8 -11
  50. package/dist/src/server/migrate/index.js +5 -5
  51. package/dist/src/server/save.d.ts +7 -1
  52. package/dist/src/server/save.js +107 -50
  53. package/dist/src/server/schema.d.ts +1 -1
  54. package/dist/src/server/schema.js +6 -2
  55. package/dist/src/server/start.js +11 -5
  56. package/dist/src/server/tree.d.ts +11 -0
  57. package/dist/src/server/tree.js +0 -1
  58. package/dist/src/server/workers/DbWorker.d.ts +3 -2
  59. package/dist/src/server/workers/DbWorker.js +2 -1
  60. package/dist/src/server/workers/io_worker.js +22 -17
  61. package/dist/src/server/workers/io_worker_types.d.ts +9 -1
  62. package/dist/src/server/workers/worker.js +11 -3
  63. package/dist/src/types.d.ts +1 -0
  64. package/dist/src/types.js +1 -0
  65. package/dist/src/utils.js +1 -1
  66. package/package.json +3 -2
  67. package/dist/lib/darwin_aarch64/libnode-v20.node +0 -0
  68. package/dist/lib/darwin_aarch64/libnode-v21.node +0 -0
  69. package/dist/lib/linux_aarch64/libnode-v20.node +0 -0
  70. package/dist/lib/linux_aarch64/libnode-v21.node +0 -0
  71. package/dist/lib/linux_x86_64/libnode-v20.node +0 -0
  72. package/dist/lib/linux_x86_64/libnode-v21.node +0 -0
@@ -100,6 +100,9 @@ const native = {
100
100
  membarSyncWrite: () => {
101
101
  db.membarSyncWrite();
102
102
  },
103
+ selvaStrerror: (err) => {
104
+ return db.selvaStrerror(err);
105
+ },
103
106
  colvecTest: (dbCtx, typeId, field, nodeId, len) => {
104
107
  return db.colvecTest(dbCtx, typeId, field, nodeId, len);
105
108
  }
@@ -3,6 +3,22 @@ import { DbServer } from './index.js';
3
3
  export declare class IoWorker extends DbWorker {
4
4
  constructor(address: BigInt, db: DbServer);
5
5
  handleMsg(_buf: any): void;
6
+ private cb;
7
+ /**
8
+ * Save given blocks and return errors and hashes in an array.
9
+ * @returns [[4 bytes err], [16 bytes hash]][] with the same length as blocks.
10
+ */
11
+ saveBlocks(blocks: {
12
+ filepath: string;
13
+ typeId: number;
14
+ start: number;
15
+ }[]): Promise<Uint8Array>;
6
16
  loadBlock(filepath: string): Promise<void>;
17
+ /**
18
+ * Save a block and discard it from memory "atomically".
19
+ * Note that this worker doesn't give any protection from other threads
20
+ * accessing the block concurrently, and it must be coordinated in the
21
+ * main thread.
22
+ */
7
23
  unloadBlock(filepath: string, typeId: number, start: number): Promise<Uint8Array>;
8
24
  }
@@ -1,5 +1,6 @@
1
1
  import { DbWorker } from './workers/DbWorker.js';
2
2
  import { DECODER, readInt32 } from '@saulx/utils';
3
+ import native from '../native.js';
3
4
  export class IoWorker extends DbWorker {
4
5
  constructor(address, db) {
5
6
  const onExit = (_code) => {
@@ -9,27 +10,55 @@ export class IoWorker extends DbWorker {
9
10
  }
10
11
  handleMsg(_buf) {
11
12
  }
13
+ cb = (resolve) => {
14
+ this.db.activeReaders++;
15
+ this.resolvers.push((r) => {
16
+ this.db.activeReaders--;
17
+ resolve(r);
18
+ //this.db.onQueryEnd()
19
+ });
20
+ };
21
+ /**
22
+ * Save given blocks and return errors and hashes in an array.
23
+ * @returns [[4 bytes err], [16 bytes hash]][] with the same length as blocks.
24
+ */
25
+ async saveBlocks(blocks) {
26
+ const job = {
27
+ type: 'save',
28
+ blocks,
29
+ };
30
+ this.channel.postMessage(job);
31
+ const resBufP = new Promise(this.cb);
32
+ resBufP.then(() => this.db.onQueryEnd());
33
+ return resBufP;
34
+ }
12
35
  async loadBlock(filepath) {
13
36
  const job = {
14
37
  type: 'load',
15
- filepath
38
+ filepath,
16
39
  };
17
40
  const resBuf = await this.call(job);
18
41
  if (resBuf.length) {
19
42
  throw new Error(DECODER.decode(resBuf));
20
43
  }
21
44
  }
45
+ /**
46
+ * Save a block and discard it from memory "atomically".
47
+ * Note that this worker doesn't give any protection from other threads
48
+ * accessing the block concurrently, and it must be coordinated in the
49
+ * main thread.
50
+ */
22
51
  async unloadBlock(filepath, typeId, start) {
23
52
  const job = {
24
53
  type: 'unload',
25
54
  filepath,
26
55
  typeId,
27
- start
56
+ start,
28
57
  };
29
58
  const resBuf = await this.call(job);
30
59
  const err = readInt32(resBuf, 0);
31
60
  if (err) {
32
- throw new Error(`selva error: ${err}`);
61
+ throw new Error(native.selvaStrerror(err));
33
62
  }
34
63
  // Note that this shares the original buffer which may not be 100% optimal,
35
64
  // as the first 4 bytes are no longer needed.
@@ -8,11 +8,11 @@ export class QueryWorker extends DbWorker {
8
8
  super(address, db, onExit, 'query_worker.js');
9
9
  }
10
10
  handleMsg(_buf) {
11
- this.db.processingQueries--;
11
+ this.db.activeReaders--;
12
12
  this.db.onQueryEnd();
13
13
  }
14
14
  callback = (resolve) => {
15
- this.db.processingQueries++;
15
+ this.db.activeReaders++;
16
16
  this.resolvers.push(resolve);
17
17
  };
18
18
  getQueryBuf(buf) {
@@ -1,11 +1,13 @@
1
1
  import { SchemaTypeDef } from '@based/schema/def';
2
2
  import { DbServer } from './index.js';
3
+ import { IoJobSave } from './workers/io_worker_types.js';
3
4
  /**
4
5
  * Save a block.
5
6
  */
6
7
  export declare function saveBlock(db: DbServer, typeId: number, start: number, end: number): void;
8
+ export declare function saveBlocks(db: DbServer, blocks: IoJobSave['blocks']): Promise<void>;
7
9
  /**
8
- * Load a block (typically of a partial type) back to memory.
10
+ * Load an existing block (typically of a partial type) back to memory.
9
11
  */
10
12
  export declare function loadBlock(db: DbServer, def: SchemaTypeDef, start: number): Promise<void>;
11
13
  /**
@@ -1,7 +1,8 @@
1
1
  import native from '../native.js';
2
2
  import { join } from 'node:path';
3
- import { equals } from '@saulx/utils';
3
+ import { equals, readInt32 } from '@saulx/utils';
4
4
  import { VerifTree, destructureTreeKey, makeTreeKey, } from './tree.js';
5
+ const SELVA_ENOENT = -8;
5
6
  /**
6
7
  * Save a block.
7
8
  */
@@ -11,20 +12,43 @@ export function saveBlock(db, typeId, start, end) {
11
12
  const file = VerifTree.blockSdbFile(typeId, start, end);
12
13
  const path = join(db.fileSystemPath, file);
13
14
  const err = native.saveBlock(path, typeId, start, db.dbCtxExternal, hash);
14
- if (err == -8) {
15
- // TODO ENOENT
15
+ if (err == SELVA_ENOENT) {
16
+ // Generally we don't nor can't remove blocks from verifTree before we
17
+ // attempt to access them.
16
18
  db.verifTree.remove(mtKey);
17
19
  }
18
20
  else if (err) {
19
- // TODO print the error string
20
- console.error(`Save ${typeId}:${start}-${end} failed: ${err}`);
21
+ db.emit('error', `Save ${typeId}:${start}-${end} failed: ${native.selvaStrerror(err)}`);
21
22
  }
22
23
  else {
23
24
  db.verifTree.update(mtKey, hash);
24
25
  }
25
26
  }
27
+ export async function saveBlocks(db, blocks) {
28
+ const res = await db.ioWorker.saveBlocks(blocks);
29
+ if (res.byteOffset !== 0)
30
+ throw new Error('Unexpected offset');
31
+ // if (res.byteLength / 20 !== blocks.length) throw new Error('Invalid res size')
32
+ for (let i = 0; i < blocks.length; i++) {
33
+ const block = blocks[i];
34
+ const key = makeTreeKey(block.typeId, block.start);
35
+ const err = readInt32(res, i * 20);
36
+ const hash = new Uint8Array(res.buffer, i * 20 + 4, 16);
37
+ if (err === SELVA_ENOENT) {
38
+ // Generally we don't nor can't remove blocks from verifTree before we
39
+ // attempt to access them.
40
+ db.verifTree.remove(key);
41
+ }
42
+ else if (err) {
43
+ db.emit('error', `Save ${block.typeId}:${block.start} failed: ${native.selvaStrerror(err)}`);
44
+ }
45
+ else {
46
+ db.verifTree.update(key, hash);
47
+ }
48
+ }
49
+ }
26
50
  /**
27
- * Load a block (typically of a partial type) back to memory.
51
+ * Load an existing block (typically of a partial type) back to memory.
28
52
  */
29
53
  export async function loadBlock(db, def, start) {
30
54
  const key = makeTreeKey(def.id, start);
@@ -72,7 +96,7 @@ export async function unloadBlock(db, def, start) {
72
96
  }
73
97
  catch (e) {
74
98
  // TODO Proper logging
75
- // TODO err == -8 == SELVA_ENOENT => db.verifTree.remove(key) ??
99
+ // TODO SELVA_ENOENT => db.verifTree.remove(key) ??
76
100
  console.error(`Save ${typeId}:${start}-${end} failed`);
77
101
  console.error(e);
78
102
  }
@@ -26,7 +26,7 @@ export declare class DbServer extends DbShared {
26
26
  ioWorker: IoWorker;
27
27
  workers: QueryWorker[];
28
28
  availableWorkerIndex: number;
29
- processingQueries: number;
29
+ activeReaders: number;
30
30
  modifyQueue: Uint8Array[];
31
31
  queryQueue: Map<Function, Uint8Array>;
32
32
  stopped: boolean;
@@ -31,12 +31,12 @@ export class DbServer extends DbShared {
31
31
  migrating = null;
32
32
  saveInProgress = false;
33
33
  fileSystemPath;
34
- verifTree;
34
+ verifTree; // should be updated only when saving/loading
35
35
  dirtyRanges = new Set();
36
36
  ioWorker;
37
37
  workers = [];
38
38
  availableWorkerIndex = -1;
39
- processingQueries = 0; // semaphore for locking writes
39
+ activeReaders = 0; // processing queries or other DB reads
40
40
  modifyQueue = [];
41
41
  queryQueue = new Map();
42
42
  stopped; // = true does not work
@@ -58,7 +58,7 @@ export class DbServer extends DbShared {
58
58
  return start(this, opts);
59
59
  }
60
60
  save(opts) {
61
- return save(this, false, opts?.forceFullDump ?? false);
61
+ return save(this, opts);
62
62
  }
63
63
  async loadBlock(typeName, nodeId) {
64
64
  const def = this.schemaTypesParsed[typeName];
@@ -68,10 +68,6 @@ export class DbServer extends DbShared {
68
68
  const typeId = def.id;
69
69
  const key = makeTreeKeyFromNodeId(typeId, def.blockCapacity, nodeId);
70
70
  const [, start] = destructureTreeKey(key);
71
- const block = this.verifTree.getBlock(key);
72
- if (!block) {
73
- throw new Error('Block not found');
74
- }
75
71
  await loadBlock(this, def, start);
76
72
  }
77
73
  async unloadBlock(typeName, nodeId) {
@@ -103,7 +99,7 @@ export class DbServer extends DbShared {
103
99
  for (const lang in this.sortIndexes[type][field][start]) {
104
100
  const sortIndex = this.sortIndexes[type][field][start][lang];
105
101
  sortIndex.cnt /= 2;
106
- if (sortIndex.cnt < 1 && !this.processingQueries) {
102
+ if (sortIndex.cnt < 1 && !this.activeReaders) {
107
103
  native.destroySortIndex(sortIndex.buf, this.dbCtxExternal);
108
104
  delete this.sortIndexes[type][field][start][lang];
109
105
  }
@@ -299,7 +295,7 @@ export class DbServer extends DbShared {
299
295
  def.lastId = lastId + offset;
300
296
  offsets[typeId] = offset;
301
297
  }
302
- if (this.processingQueries) {
298
+ if (this.activeReaders) {
303
299
  this.modifyQueue.push(new Uint8Array(bufWithHash));
304
300
  }
305
301
  else {
@@ -387,7 +383,7 @@ export class DbServer extends DbShared {
387
383
  const start = readUint16(sort, 3);
388
384
  let sortIndex = this.getSortIndex(typeId, field, start, 0);
389
385
  if (!sortIndex) {
390
- if (this.processingQueries) {
386
+ if (this.activeReaders) {
391
387
  return new Promise((resolve) => {
392
388
  this.addToQueryQueue(resolve, buf);
393
389
  });
@@ -411,7 +407,7 @@ export class DbServer extends DbShared {
411
407
  }
412
408
  }
413
409
  onQueryEnd() {
414
- if (this.processingQueries === 0) {
410
+ if (this.activeReaders === 0) {
415
411
  if (this.modifyQueue.length) {
416
412
  const modifyQueue = this.modifyQueue;
417
413
  this.modifyQueue = [];
@@ -454,6 +450,7 @@ export class DbServer extends DbShared {
454
450
  await this.save();
455
451
  }
456
452
  await this.ioWorker.terminate();
453
+ this.ioWorker = null;
457
454
  await Promise.all(this.workers.map((worker) => worker.terminate()));
458
455
  this.workers = [];
459
456
  native.stop(this.dbCtxExternal);
@@ -90,7 +90,7 @@ export const migrate = async (server, fromSchema, toSchema, transform) => {
90
90
  // Block handling
91
91
  let i = 0;
92
92
  let rangesToMigrate = [];
93
- await save(server, false, false, true);
93
+ await save(server, { skipMigrationCheck: true });
94
94
  server.verifTree.foreachBlock((block) => {
95
95
  const [typeId, start] = destructureTreeKey(block.key);
96
96
  const def = server.schemaTypesParsedById[typeId];
@@ -103,13 +103,13 @@ export const migrate = async (server, fromSchema, toSchema, transform) => {
103
103
  break;
104
104
  }
105
105
  // block modifies
106
- server.processingQueries++;
106
+ server.activeReaders++;
107
107
  const leafData = rangesToMigrate[i++];
108
108
  port1.postMessage(leafData);
109
109
  setToAwake(workerState, true);
110
110
  await waitUntilSleeping(workerState);
111
111
  // exec queued modifies
112
- server.processingQueries--;
112
+ server.activeReaders--;
113
113
  server.onQueryEnd();
114
114
  if (i === rangesToMigrate.length) {
115
115
  if (server.dirtyRanges.size) {
@@ -149,14 +149,14 @@ export const migrate = async (server, fromSchema, toSchema, transform) => {
149
149
  // -----------------------------------------
150
150
  tmpDb.server.dbCtxExternal = fromCtx;
151
151
  // TODO makes this SYNC
152
- const promises = server.workers.map((worker) => worker.updateCtx(toAddress));
152
+ const promises = [server.ioWorker, ...server.workers].map((worker) => worker.updateCtx(toAddress));
153
153
  promises.push(tmpDb.destroy(), worker.terminate());
154
154
  await Promise.all(promises);
155
155
  if (abort()) {
156
156
  return;
157
157
  }
158
158
  native.membarSyncRead();
159
- await save(server, false, true, true);
159
+ await save(server, { forceFullDump: true, skipDirtyCheck: true, skipMigrationCheck: true });
160
160
  await writeSchemaFile(server, toSchema);
161
161
  server.migrating = 0;
162
162
  process.nextTick(() => server.emit('schema', server.schema));
@@ -19,5 +19,11 @@ export type Writelog = {
19
19
  [t: number]: RangeDump[];
20
20
  };
21
21
  };
22
- export declare function save<T extends boolean>(db: DbServer, sync?: T, forceFullDump?: boolean, skipMigrationCheck?: boolean): T extends true ? void : Promise<void>;
22
+ type SaveOpts = {
23
+ forceFullDump?: boolean;
24
+ skipDirtyCheck?: boolean;
25
+ skipMigrationCheck?: boolean;
26
+ };
27
+ export declare function saveSync(db: DbServer, opts?: SaveOpts): void;
28
+ export declare function save(db: DbServer, opts?: SaveOpts): Promise<void>;
23
29
  export {};
@@ -3,7 +3,7 @@ import { isMainThread } from 'node:worker_threads';
3
3
  import { writeFile } from 'node:fs/promises';
4
4
  import { join } from 'node:path';
5
5
  import { VerifTree, destructureTreeKey, } from './tree.js';
6
- import { saveBlock, foreachBlock, foreachDirtyBlock, } from './blocks.js';
6
+ import { saveBlock, foreachBlock, foreachDirtyBlock, saveBlocks, } from './blocks.js';
7
7
  import { writeFileSync } from 'node:fs';
8
8
  import { bufToHex } from '@saulx/utils';
9
9
  import { COMMON_SDB_FILE, WRITELOG_FILE } from '../types.js';
@@ -14,22 +14,56 @@ function hasPartialTypes(db) {
14
14
  }
15
15
  return res;
16
16
  }
17
- export function save(db, sync = false, forceFullDump = false, skipMigrationCheck = false) {
18
- if (!(isMainThread && (db.dirtyRanges.size || forceFullDump))) {
19
- return;
17
+ function inhibitSave(db, { skipDirtyCheck, forceFullDump, skipMigrationCheck }) {
18
+ // RFE isMainThread needed??
19
+ if (!(isMainThread && (skipDirtyCheck || db.dirtyRanges.size || forceFullDump))) {
20
+ return true;
20
21
  }
21
22
  if (forceFullDump && hasPartialTypes(db)) {
22
23
  db.emit('error', 'forceFullDump is not allowed with partial types');
23
- return;
24
+ return true;
24
25
  }
25
26
  if (db.migrating && !skipMigrationCheck) {
26
27
  db.emit('info', 'Block save db is migrating');
27
- return;
28
+ return true;
28
29
  }
29
30
  if (db.saveInProgress) {
30
31
  db.emit('info', 'Already have a save in progress cancel save');
31
- return;
32
+ return true;
33
+ }
34
+ return false;
35
+ }
36
+ function makeWritelog(db, ts) {
37
+ const types = {};
38
+ const rangeDumps = {};
39
+ for (const key in db.schemaTypesParsed) {
40
+ const { id, lastId, blockCapacity } = db.schemaTypesParsed[key];
41
+ types[id] = { lastId, blockCapacity };
42
+ rangeDumps[id] = [];
32
43
  }
44
+ db.verifTree.foreachBlock((block) => {
45
+ const [typeId, start] = destructureTreeKey(block.key);
46
+ const def = db.schemaTypesParsedById[typeId];
47
+ const end = start + def.blockCapacity - 1;
48
+ const data = {
49
+ file: db.verifTree.getBlockFile(block),
50
+ hash: bufToHex(block.hash),
51
+ start,
52
+ end,
53
+ };
54
+ rangeDumps[typeId].push(data);
55
+ });
56
+ return {
57
+ ts,
58
+ types,
59
+ commonDump: COMMON_SDB_FILE,
60
+ rangeDumps,
61
+ hash: bufToHex(db.verifTree.hash), // TODO `hash('hex')`
62
+ };
63
+ }
64
+ export function saveSync(db, opts = {}) {
65
+ if (inhibitSave(db, opts))
66
+ return;
33
67
  let ts = Date.now();
34
68
  db.saveInProgress = true;
35
69
  try {
@@ -39,7 +73,7 @@ export function save(db, sync = false, forceFullDump = false, skipMigrationCheck
39
73
  db.emit('error', `Save common failed: ${err}`);
40
74
  // Return ?
41
75
  }
42
- if (forceFullDump) {
76
+ if (opts.forceFullDump) {
43
77
  // reset the state just in case
44
78
  db.verifTree = new VerifTree(db.schemaTypesParsed);
45
79
  // We use db.verifTree.types instead of db.schemaTypesParsed because it's
@@ -53,58 +87,81 @@ export function save(db, sync = false, forceFullDump = false, skipMigrationCheck
53
87
  foreachDirtyBlock(db, (_mtKey, typeId, start, end) => saveBlock(db, typeId, start, end));
54
88
  }
55
89
  db.dirtyRanges.clear();
56
- const types = {};
57
- const rangeDumps = {};
58
- for (const key in db.schemaTypesParsed) {
59
- const { id, lastId, blockCapacity } = db.schemaTypesParsed[key];
60
- types[id] = { lastId, blockCapacity };
61
- rangeDumps[id] = [];
62
- }
63
- db.verifTree.foreachBlock((block) => {
64
- const [typeId, start] = destructureTreeKey(block.key);
65
- const def = db.schemaTypesParsedById[typeId];
66
- const end = start + def.blockCapacity - 1;
67
- const data = {
68
- file: db.verifTree.getBlockFile(block),
69
- hash: bufToHex(block.hash),
70
- start,
71
- end,
72
- };
73
- rangeDumps[typeId].push(data);
74
- });
75
- const data = {
76
- ts,
77
- types,
78
- commonDump: COMMON_SDB_FILE,
79
- rangeDumps,
80
- hash: bufToHex(db.verifTree.hash), // TODO `hash('hex')`
81
- };
82
- const filePath = join(db.fileSystemPath, WRITELOG_FILE);
90
+ const data = makeWritelog(db, ts);
83
91
  const content = JSON.stringify(data);
84
- db.emit('info', `Save done took ${Date.now() - ts}ms`);
85
- if (sync) {
86
- db.saveInProgress = false;
87
- return writeFileSync(filePath, content);
92
+ db.emit('info', `Save took ${Date.now() - ts}ms`);
93
+ db.saveInProgress = false;
94
+ return writeFileSync(join(db.fileSystemPath, WRITELOG_FILE), content);
95
+ }
96
+ catch (err) {
97
+ db.emit('error', `Save failed ${err.message}`);
98
+ db.saveInProgress = false;
99
+ throw err;
100
+ }
101
+ }
102
+ export async function save(db, opts = {}) {
103
+ if (inhibitSave(db, opts))
104
+ return;
105
+ let ts = Date.now();
106
+ db.saveInProgress = true;
107
+ try {
108
+ let err;
109
+ err = native.saveCommon(join(db.fileSystemPath, COMMON_SDB_FILE), db.dbCtxExternal);
110
+ if (err) {
111
+ db.emit('error', `Save common failed: ${err}`);
112
+ // Return ?
113
+ }
114
+ const blocks = [];
115
+ if (opts.forceFullDump) {
116
+ // reset the state just in case
117
+ db.verifTree = new VerifTree(db.schemaTypesParsed);
118
+ // We use db.verifTree.types instead of db.schemaTypesParsed because it's
119
+ // ordered.
120
+ for (const { typeId } of db.verifTree.types()) {
121
+ const def = db.schemaTypesParsedById[typeId];
122
+ foreachBlock(db, def, (start, end, _hash) => {
123
+ const typeId = def.id;
124
+ const file = VerifTree.blockSdbFile(typeId, start, end);
125
+ const filepath = join(db.fileSystemPath, file);
126
+ blocks.push({
127
+ filepath,
128
+ typeId,
129
+ start,
130
+ });
131
+ });
132
+ }
88
133
  }
89
134
  else {
90
- return new Promise((resolve, reject) => {
91
- return writeFile(filePath, content)
92
- .then((v) => {
93
- db.saveInProgress = false;
94
- resolve(v);
95
- })
96
- .catch((err) => {
97
- db.emit('error', `Save: writing writeLog failed ${err.message}`);
98
- db.saveInProgress = false;
99
- reject(err);
135
+ foreachDirtyBlock(db, (_mtKey, typeId, start, end) => {
136
+ const file = VerifTree.blockSdbFile(typeId, start, end);
137
+ const filepath = join(db.fileSystemPath, file);
138
+ blocks.push({
139
+ filepath,
140
+ typeId,
141
+ start
100
142
  });
101
143
  });
102
144
  }
145
+ db.dirtyRanges.clear();
146
+ await saveBlocks(db, blocks);
147
+ try {
148
+ // Note that we assume here that verifTree didn't change before we call
149
+ // makeWritelog(). This is true as long as db.saveInProgress protects
150
+ // the verifTree from changes.
151
+ const data = makeWritelog(db, ts);
152
+ await writeFile(join(db.fileSystemPath, WRITELOG_FILE), JSON.stringify(data));
153
+ }
154
+ catch (err) {
155
+ db.emit('error', `Save: writing writeLog failed ${err.message}`);
156
+ }
157
+ db.emit('info', `Save took ${Date.now() - ts}ms`);
103
158
  }
104
159
  catch (err) {
105
160
  db.emit('error', `Save failed ${err.message}`);
106
- db.saveInProgress = false;
107
161
  throw err;
108
162
  }
163
+ finally {
164
+ db.saveInProgress = false;
165
+ }
109
166
  }
110
167
  //# sourceMappingURL=save.js.map
@@ -1,6 +1,6 @@
1
+ import { StrictSchema } from '@based/schema';
1
2
  import { DbServer } from './index.js';
2
3
  import { DbSchema } from '../schema.js';
3
- import { StrictSchema } from '@based/schema';
4
4
  export declare const setSchemaOnServer: (server: DbServer, schema: DbSchema) => void;
5
5
  export declare const writeSchemaFile: (server: DbServer, schema: DbSchema) => Promise<void>;
6
6
  export declare const setNativeSchema: (server: DbServer, schema: DbSchema) => void;
@@ -1,12 +1,13 @@
1
1
  import { schemaToSelvaBuffer, updateTypeDefs } from '@based/schema/def';
2
+ import { deepCopy, writeUint64 } from '@saulx/utils';
3
+ import { getPropType, serialize } from '@based/schema';
2
4
  import { join } from 'node:path';
3
5
  import { writeFile } from 'node:fs/promises';
4
6
  import native from '../native.js';
5
7
  import { makeTreeKey } from './tree.js';
6
- import { deepCopy, writeUint64 } from '@saulx/utils';
7
8
  import { SCHEMA_FILE } from '../types.js';
9
+ import { saveSync } from './save.js';
8
10
  import { hash } from '@saulx/hash';
9
- import { getPropType, serialize } from '@based/schema';
10
11
  export const setSchemaOnServer = (server, schema) => {
11
12
  const { schemaTypesParsed, schemaTypesParsedById } = updateTypeDefs(schema);
12
13
  server.schema = schema;
@@ -55,6 +56,9 @@ export const setNativeSchema = (server, schema) => {
55
56
  //server.verifTree = new VerifTree(.schemaTypesParsed)
56
57
  }
57
58
  server.verifTree.updateTypes(server.schemaTypesParsed);
59
+ if (server.fileSystemPath) {
60
+ saveSync(server, { skipDirtyCheck: true });
61
+ }
58
62
  };
59
63
  export const strictSchemaToDbSchema = (schema) => {
60
64
  // @ts-ignore
@@ -7,11 +7,11 @@ import { join } from 'node:path';
7
7
  import { VerifTree, makeTreeKey } from './tree.js';
8
8
  import { foreachBlock } from './blocks.js';
9
9
  import exitHook from 'exit-hook';
10
- import { save } from './save.js';
10
+ import { save, saveSync } from './save.js';
11
11
  import { deSerialize } from '@based/schema';
12
12
  import { BLOCK_CAPACITY_DEFAULT } from '@based/schema/def';
13
13
  import { bufToHex, equals, hexToBuf, wait } from '@saulx/utils';
14
- import { SCHEMA_FILE, WRITELOG_FILE } from '../types.js';
14
+ import { SCHEMA_FILE, WRITELOG_FILE, SCHEMA_FILE_DEPRECATED } from '../types.js';
15
15
  import { setSchemaOnServer } from './schema.js';
16
16
  function startWorkers(db, opts) {
17
17
  const queryThreads = opts?.queryThreads ?? availableParallelism();
@@ -42,11 +42,17 @@ export async function start(db, opts) {
42
42
  console.error(e.message);
43
43
  throw e;
44
44
  }
45
- const schema = await readFile(join(path, SCHEMA_FILE));
45
+ const schema = await readFile(join(path, SCHEMA_FILE)).catch(noop);
46
46
  if (schema) {
47
47
  const s = deSerialize(schema);
48
48
  setSchemaOnServer(db, s);
49
49
  }
50
+ else {
51
+ const schemaJson = await readFile(join(path, SCHEMA_FILE_DEPRECATED));
52
+ if (schemaJson) {
53
+ setSchemaOnServer(db, JSON.parse(schemaJson.toString()));
54
+ }
55
+ }
50
56
  // Load all range dumps
51
57
  for (const typeId in writelog.rangeDumps) {
52
58
  const dumps = writelog.rangeDumps[typeId];
@@ -111,7 +117,7 @@ export async function start(db, opts) {
111
117
  // in Node.js.
112
118
  signals.forEach((sig) => process.on(sig, blockSig));
113
119
  db.emit('info', `Exiting with signal: ${signal}`);
114
- save(db, true);
120
+ saveSync(db);
115
121
  db.emit('info', 'Successfully saved.');
116
122
  signals.forEach((sig) => process.off(sig, blockSig));
117
123
  });
@@ -121,7 +127,7 @@ export async function start(db, opts) {
121
127
  // use timeout
122
128
  if (db.saveIntervalInSeconds > 0) {
123
129
  db.saveInterval ??= setInterval(() => {
124
- void save(db);
130
+ save(db);
125
131
  }, db.saveIntervalInSeconds * 1e3);
126
132
  }
127
133
  if (db.schema) {
@@ -6,8 +6,19 @@ export declare const makeTreeKeyFromNodeId: (typeId: number, blockCapacity: numb
6
6
  type Hash = Uint8Array;
7
7
  export type VerifBlock = {
8
8
  key: number;
9
+ /**
10
+ * Last acquired hash of the block.
11
+ * This is normally updated at load and save time but never during read/modify ops.
12
+ */
9
13
  hash: Hash;
14
+ /**
15
+ * If false the block is offloaded to fs;
16
+ * true doesn't necessarily mean that the block still exists because it could have been deleted.
17
+ */
10
18
  inmem: boolean;
19
+ /**
20
+ * If set, the block is being loaded and it can be awaited with this promise.
21
+ */
11
22
  loadPromise: null | Promise<void>;
12
23
  };
13
24
  export declare class VerifTree {
@@ -26,7 +26,6 @@ export class VerifTree {
26
26
  obj[typeId] = {
27
27
  typeId,
28
28
  blockCapacity: def.blockCapacity,
29
- hash: new Uint8Array(HASH_SIZE),
30
29
  blocks: [],
31
30
  };
32
31
  return obj;