@based/db 0.0.67 → 0.0.69
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -1
- package/dist/lib/darwin_aarch64/include/selva/fields.h +0 -9
- package/dist/lib/darwin_aarch64/include/selva_error.h +4 -1
- package/dist/lib/darwin_aarch64/libnode-v22.node +0 -0
- package/dist/lib/darwin_aarch64/libnode-v23.node +0 -0
- package/dist/lib/darwin_aarch64/libnode-v24.node +0 -0
- package/dist/lib/darwin_aarch64/libselva.dylib +0 -0
- package/dist/lib/linux_aarch64/include/selva/fields.h +0 -9
- package/dist/lib/linux_aarch64/include/selva_error.h +4 -1
- package/dist/lib/linux_aarch64/libnode-v22.node +0 -0
- package/dist/lib/linux_aarch64/libnode-v23.node +0 -0
- package/dist/lib/linux_aarch64/libnode-v24.node +0 -0
- package/dist/lib/linux_aarch64/libselva.so +0 -0
- package/dist/lib/linux_x86_64/include/selva/fields.h +0 -9
- package/dist/lib/linux_x86_64/include/selva_error.h +4 -1
- package/dist/lib/linux_x86_64/libnode-v22.node +0 -0
- package/dist/lib/linux_x86_64/libnode-v23.node +0 -0
- package/dist/lib/linux_x86_64/libnode-v24.node +0 -0
- package/dist/lib/linux_x86_64/libselva.so +0 -0
- package/dist/src/client/modify/alias.js +4 -0
- package/dist/src/client/modify/binary.d.ts +1 -1
- package/dist/src/client/modify/binary.js +22 -7
- package/dist/src/client/modify/create.js +87 -39
- package/dist/src/client/modify/fixed.d.ts +3 -3
- package/dist/src/client/modify/fixed.js +80 -20
- package/dist/src/client/modify/json.js +7 -0
- package/dist/src/client/modify/modify.js +3 -3
- package/dist/src/client/modify/references/edge.js +3 -3
- package/dist/src/client/modify/string.d.ts +1 -1
- package/dist/src/client/modify/string.js +17 -7
- package/dist/src/client/modify/text.d.ts +1 -1
- package/dist/src/client/modify/text.js +27 -15
- package/dist/src/client/modify/types.d.ts +2 -1
- package/dist/src/client/modify/types.js +6 -0
- package/dist/src/client/modify/update.js +1 -1
- package/dist/src/client/query/display.js +1 -1
- package/dist/src/client/query/filter/createFixedFilterBuffer.js +3 -0
- package/dist/src/client/query/filter/createReferenceFilter.js +3 -0
- package/dist/src/client/query/filter/createVariableFilterBuffer.js +21 -19
- package/dist/src/client/query/read/read.js +9 -10
- package/dist/src/native.d.ts +1 -0
- package/dist/src/native.js +3 -0
- package/dist/src/server/IoWorker.d.ts +16 -0
- package/dist/src/server/IoWorker.js +32 -3
- package/dist/src/server/QueryWorker.js +2 -2
- package/dist/src/server/blocks.d.ts +3 -1
- package/dist/src/server/blocks.js +31 -7
- package/dist/src/server/index.d.ts +1 -1
- package/dist/src/server/index.js +8 -11
- package/dist/src/server/migrate/index.js +5 -5
- package/dist/src/server/save.d.ts +7 -1
- package/dist/src/server/save.js +107 -50
- package/dist/src/server/schema.d.ts +1 -1
- package/dist/src/server/schema.js +6 -2
- package/dist/src/server/start.js +11 -5
- package/dist/src/server/tree.d.ts +11 -0
- package/dist/src/server/tree.js +0 -1
- package/dist/src/server/workers/DbWorker.d.ts +3 -2
- package/dist/src/server/workers/DbWorker.js +2 -1
- package/dist/src/server/workers/io_worker.js +22 -17
- package/dist/src/server/workers/io_worker_types.d.ts +9 -1
- package/dist/src/server/workers/worker.js +11 -3
- package/dist/src/types.d.ts +1 -0
- package/dist/src/types.js +1 -0
- package/dist/src/utils.js +1 -1
- package/package.json +3 -2
- package/dist/lib/darwin_aarch64/libnode-v20.node +0 -0
- package/dist/lib/darwin_aarch64/libnode-v21.node +0 -0
- package/dist/lib/linux_aarch64/libnode-v20.node +0 -0
- package/dist/lib/linux_aarch64/libnode-v21.node +0 -0
- package/dist/lib/linux_x86_64/libnode-v20.node +0 -0
- package/dist/lib/linux_x86_64/libnode-v21.node +0 -0
package/dist/src/native.js
CHANGED
|
@@ -100,6 +100,9 @@ const native = {
|
|
|
100
100
|
membarSyncWrite: () => {
|
|
101
101
|
db.membarSyncWrite();
|
|
102
102
|
},
|
|
103
|
+
selvaStrerror: (err) => {
|
|
104
|
+
return db.selvaStrerror(err);
|
|
105
|
+
},
|
|
103
106
|
colvecTest: (dbCtx, typeId, field, nodeId, len) => {
|
|
104
107
|
return db.colvecTest(dbCtx, typeId, field, nodeId, len);
|
|
105
108
|
}
|
|
@@ -3,6 +3,22 @@ import { DbServer } from './index.js';
|
|
|
3
3
|
export declare class IoWorker extends DbWorker {
|
|
4
4
|
constructor(address: BigInt, db: DbServer);
|
|
5
5
|
handleMsg(_buf: any): void;
|
|
6
|
+
private cb;
|
|
7
|
+
/**
|
|
8
|
+
* Save given blocks and return errors and hashes in an array.
|
|
9
|
+
* @returns [[4 bytes err], [16 bytes hash]][] with the same length as blocks.
|
|
10
|
+
*/
|
|
11
|
+
saveBlocks(blocks: {
|
|
12
|
+
filepath: string;
|
|
13
|
+
typeId: number;
|
|
14
|
+
start: number;
|
|
15
|
+
}[]): Promise<Uint8Array>;
|
|
6
16
|
loadBlock(filepath: string): Promise<void>;
|
|
17
|
+
/**
|
|
18
|
+
* Save a block and discard it from memory "atomically".
|
|
19
|
+
* Note that this worker doesn't give any protection from other threads
|
|
20
|
+
* accessing the block concurrently, and it must be coordinated in the
|
|
21
|
+
* main thread.
|
|
22
|
+
*/
|
|
7
23
|
unloadBlock(filepath: string, typeId: number, start: number): Promise<Uint8Array>;
|
|
8
24
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { DbWorker } from './workers/DbWorker.js';
|
|
2
2
|
import { DECODER, readInt32 } from '@saulx/utils';
|
|
3
|
+
import native from '../native.js';
|
|
3
4
|
export class IoWorker extends DbWorker {
|
|
4
5
|
constructor(address, db) {
|
|
5
6
|
const onExit = (_code) => {
|
|
@@ -9,27 +10,55 @@ export class IoWorker extends DbWorker {
|
|
|
9
10
|
}
|
|
10
11
|
handleMsg(_buf) {
|
|
11
12
|
}
|
|
13
|
+
cb = (resolve) => {
|
|
14
|
+
this.db.activeReaders++;
|
|
15
|
+
this.resolvers.push((r) => {
|
|
16
|
+
this.db.activeReaders--;
|
|
17
|
+
resolve(r);
|
|
18
|
+
//this.db.onQueryEnd()
|
|
19
|
+
});
|
|
20
|
+
};
|
|
21
|
+
/**
|
|
22
|
+
* Save given blocks and return errors and hashes in an array.
|
|
23
|
+
* @returns [[4 bytes err], [16 bytes hash]][] with the same length as blocks.
|
|
24
|
+
*/
|
|
25
|
+
async saveBlocks(blocks) {
|
|
26
|
+
const job = {
|
|
27
|
+
type: 'save',
|
|
28
|
+
blocks,
|
|
29
|
+
};
|
|
30
|
+
this.channel.postMessage(job);
|
|
31
|
+
const resBufP = new Promise(this.cb);
|
|
32
|
+
resBufP.then(() => this.db.onQueryEnd());
|
|
33
|
+
return resBufP;
|
|
34
|
+
}
|
|
12
35
|
async loadBlock(filepath) {
|
|
13
36
|
const job = {
|
|
14
37
|
type: 'load',
|
|
15
|
-
filepath
|
|
38
|
+
filepath,
|
|
16
39
|
};
|
|
17
40
|
const resBuf = await this.call(job);
|
|
18
41
|
if (resBuf.length) {
|
|
19
42
|
throw new Error(DECODER.decode(resBuf));
|
|
20
43
|
}
|
|
21
44
|
}
|
|
45
|
+
/**
|
|
46
|
+
* Save a block and discard it from memory "atomically".
|
|
47
|
+
* Note that this worker doesn't give any protection from other threads
|
|
48
|
+
* accessing the block concurrently, and it must be coordinated in the
|
|
49
|
+
* main thread.
|
|
50
|
+
*/
|
|
22
51
|
async unloadBlock(filepath, typeId, start) {
|
|
23
52
|
const job = {
|
|
24
53
|
type: 'unload',
|
|
25
54
|
filepath,
|
|
26
55
|
typeId,
|
|
27
|
-
start
|
|
56
|
+
start,
|
|
28
57
|
};
|
|
29
58
|
const resBuf = await this.call(job);
|
|
30
59
|
const err = readInt32(resBuf, 0);
|
|
31
60
|
if (err) {
|
|
32
|
-
throw new Error(
|
|
61
|
+
throw new Error(native.selvaStrerror(err));
|
|
33
62
|
}
|
|
34
63
|
// Note that this shares the original buffer which may not be 100% optimal,
|
|
35
64
|
// as the first 4 bytes are no longer needed.
|
|
@@ -8,11 +8,11 @@ export class QueryWorker extends DbWorker {
|
|
|
8
8
|
super(address, db, onExit, 'query_worker.js');
|
|
9
9
|
}
|
|
10
10
|
handleMsg(_buf) {
|
|
11
|
-
this.db.
|
|
11
|
+
this.db.activeReaders--;
|
|
12
12
|
this.db.onQueryEnd();
|
|
13
13
|
}
|
|
14
14
|
callback = (resolve) => {
|
|
15
|
-
this.db.
|
|
15
|
+
this.db.activeReaders++;
|
|
16
16
|
this.resolvers.push(resolve);
|
|
17
17
|
};
|
|
18
18
|
getQueryBuf(buf) {
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import { SchemaTypeDef } from '@based/schema/def';
|
|
2
2
|
import { DbServer } from './index.js';
|
|
3
|
+
import { IoJobSave } from './workers/io_worker_types.js';
|
|
3
4
|
/**
|
|
4
5
|
* Save a block.
|
|
5
6
|
*/
|
|
6
7
|
export declare function saveBlock(db: DbServer, typeId: number, start: number, end: number): void;
|
|
8
|
+
export declare function saveBlocks(db: DbServer, blocks: IoJobSave['blocks']): Promise<void>;
|
|
7
9
|
/**
|
|
8
|
-
* Load
|
|
10
|
+
* Load an existing block (typically of a partial type) back to memory.
|
|
9
11
|
*/
|
|
10
12
|
export declare function loadBlock(db: DbServer, def: SchemaTypeDef, start: number): Promise<void>;
|
|
11
13
|
/**
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import native from '../native.js';
|
|
2
2
|
import { join } from 'node:path';
|
|
3
|
-
import { equals } from '@saulx/utils';
|
|
3
|
+
import { equals, readInt32 } from '@saulx/utils';
|
|
4
4
|
import { VerifTree, destructureTreeKey, makeTreeKey, } from './tree.js';
|
|
5
|
+
const SELVA_ENOENT = -8;
|
|
5
6
|
/**
|
|
6
7
|
* Save a block.
|
|
7
8
|
*/
|
|
@@ -11,20 +12,43 @@ export function saveBlock(db, typeId, start, end) {
|
|
|
11
12
|
const file = VerifTree.blockSdbFile(typeId, start, end);
|
|
12
13
|
const path = join(db.fileSystemPath, file);
|
|
13
14
|
const err = native.saveBlock(path, typeId, start, db.dbCtxExternal, hash);
|
|
14
|
-
if (err ==
|
|
15
|
-
//
|
|
15
|
+
if (err == SELVA_ENOENT) {
|
|
16
|
+
// Generally we don't nor can't remove blocks from verifTree before we
|
|
17
|
+
// attempt to access them.
|
|
16
18
|
db.verifTree.remove(mtKey);
|
|
17
19
|
}
|
|
18
20
|
else if (err) {
|
|
19
|
-
|
|
20
|
-
console.error(`Save ${typeId}:${start}-${end} failed: ${err}`);
|
|
21
|
+
db.emit('error', `Save ${typeId}:${start}-${end} failed: ${native.selvaStrerror(err)}`);
|
|
21
22
|
}
|
|
22
23
|
else {
|
|
23
24
|
db.verifTree.update(mtKey, hash);
|
|
24
25
|
}
|
|
25
26
|
}
|
|
27
|
+
export async function saveBlocks(db, blocks) {
|
|
28
|
+
const res = await db.ioWorker.saveBlocks(blocks);
|
|
29
|
+
if (res.byteOffset !== 0)
|
|
30
|
+
throw new Error('Unexpected offset');
|
|
31
|
+
// if (res.byteLength / 20 !== blocks.length) throw new Error('Invalid res size')
|
|
32
|
+
for (let i = 0; i < blocks.length; i++) {
|
|
33
|
+
const block = blocks[i];
|
|
34
|
+
const key = makeTreeKey(block.typeId, block.start);
|
|
35
|
+
const err = readInt32(res, i * 20);
|
|
36
|
+
const hash = new Uint8Array(res.buffer, i * 20 + 4, 16);
|
|
37
|
+
if (err === SELVA_ENOENT) {
|
|
38
|
+
// Generally we don't nor can't remove blocks from verifTree before we
|
|
39
|
+
// attempt to access them.
|
|
40
|
+
db.verifTree.remove(key);
|
|
41
|
+
}
|
|
42
|
+
else if (err) {
|
|
43
|
+
db.emit('error', `Save ${block.typeId}:${block.start} failed: ${native.selvaStrerror(err)}`);
|
|
44
|
+
}
|
|
45
|
+
else {
|
|
46
|
+
db.verifTree.update(key, hash);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
}
|
|
26
50
|
/**
|
|
27
|
-
* Load
|
|
51
|
+
* Load an existing block (typically of a partial type) back to memory.
|
|
28
52
|
*/
|
|
29
53
|
export async function loadBlock(db, def, start) {
|
|
30
54
|
const key = makeTreeKey(def.id, start);
|
|
@@ -72,7 +96,7 @@ export async function unloadBlock(db, def, start) {
|
|
|
72
96
|
}
|
|
73
97
|
catch (e) {
|
|
74
98
|
// TODO Proper logging
|
|
75
|
-
// TODO
|
|
99
|
+
// TODO SELVA_ENOENT => db.verifTree.remove(key) ??
|
|
76
100
|
console.error(`Save ${typeId}:${start}-${end} failed`);
|
|
77
101
|
console.error(e);
|
|
78
102
|
}
|
|
@@ -26,7 +26,7 @@ export declare class DbServer extends DbShared {
|
|
|
26
26
|
ioWorker: IoWorker;
|
|
27
27
|
workers: QueryWorker[];
|
|
28
28
|
availableWorkerIndex: number;
|
|
29
|
-
|
|
29
|
+
activeReaders: number;
|
|
30
30
|
modifyQueue: Uint8Array[];
|
|
31
31
|
queryQueue: Map<Function, Uint8Array>;
|
|
32
32
|
stopped: boolean;
|
package/dist/src/server/index.js
CHANGED
|
@@ -31,12 +31,12 @@ export class DbServer extends DbShared {
|
|
|
31
31
|
migrating = null;
|
|
32
32
|
saveInProgress = false;
|
|
33
33
|
fileSystemPath;
|
|
34
|
-
verifTree;
|
|
34
|
+
verifTree; // should be updated only when saving/loading
|
|
35
35
|
dirtyRanges = new Set();
|
|
36
36
|
ioWorker;
|
|
37
37
|
workers = [];
|
|
38
38
|
availableWorkerIndex = -1;
|
|
39
|
-
|
|
39
|
+
activeReaders = 0; // processing queries or other DB reads
|
|
40
40
|
modifyQueue = [];
|
|
41
41
|
queryQueue = new Map();
|
|
42
42
|
stopped; // = true does not work
|
|
@@ -58,7 +58,7 @@ export class DbServer extends DbShared {
|
|
|
58
58
|
return start(this, opts);
|
|
59
59
|
}
|
|
60
60
|
save(opts) {
|
|
61
|
-
return save(this,
|
|
61
|
+
return save(this, opts);
|
|
62
62
|
}
|
|
63
63
|
async loadBlock(typeName, nodeId) {
|
|
64
64
|
const def = this.schemaTypesParsed[typeName];
|
|
@@ -68,10 +68,6 @@ export class DbServer extends DbShared {
|
|
|
68
68
|
const typeId = def.id;
|
|
69
69
|
const key = makeTreeKeyFromNodeId(typeId, def.blockCapacity, nodeId);
|
|
70
70
|
const [, start] = destructureTreeKey(key);
|
|
71
|
-
const block = this.verifTree.getBlock(key);
|
|
72
|
-
if (!block) {
|
|
73
|
-
throw new Error('Block not found');
|
|
74
|
-
}
|
|
75
71
|
await loadBlock(this, def, start);
|
|
76
72
|
}
|
|
77
73
|
async unloadBlock(typeName, nodeId) {
|
|
@@ -103,7 +99,7 @@ export class DbServer extends DbShared {
|
|
|
103
99
|
for (const lang in this.sortIndexes[type][field][start]) {
|
|
104
100
|
const sortIndex = this.sortIndexes[type][field][start][lang];
|
|
105
101
|
sortIndex.cnt /= 2;
|
|
106
|
-
if (sortIndex.cnt < 1 && !this.
|
|
102
|
+
if (sortIndex.cnt < 1 && !this.activeReaders) {
|
|
107
103
|
native.destroySortIndex(sortIndex.buf, this.dbCtxExternal);
|
|
108
104
|
delete this.sortIndexes[type][field][start][lang];
|
|
109
105
|
}
|
|
@@ -299,7 +295,7 @@ export class DbServer extends DbShared {
|
|
|
299
295
|
def.lastId = lastId + offset;
|
|
300
296
|
offsets[typeId] = offset;
|
|
301
297
|
}
|
|
302
|
-
if (this.
|
|
298
|
+
if (this.activeReaders) {
|
|
303
299
|
this.modifyQueue.push(new Uint8Array(bufWithHash));
|
|
304
300
|
}
|
|
305
301
|
else {
|
|
@@ -387,7 +383,7 @@ export class DbServer extends DbShared {
|
|
|
387
383
|
const start = readUint16(sort, 3);
|
|
388
384
|
let sortIndex = this.getSortIndex(typeId, field, start, 0);
|
|
389
385
|
if (!sortIndex) {
|
|
390
|
-
if (this.
|
|
386
|
+
if (this.activeReaders) {
|
|
391
387
|
return new Promise((resolve) => {
|
|
392
388
|
this.addToQueryQueue(resolve, buf);
|
|
393
389
|
});
|
|
@@ -411,7 +407,7 @@ export class DbServer extends DbShared {
|
|
|
411
407
|
}
|
|
412
408
|
}
|
|
413
409
|
onQueryEnd() {
|
|
414
|
-
if (this.
|
|
410
|
+
if (this.activeReaders === 0) {
|
|
415
411
|
if (this.modifyQueue.length) {
|
|
416
412
|
const modifyQueue = this.modifyQueue;
|
|
417
413
|
this.modifyQueue = [];
|
|
@@ -454,6 +450,7 @@ export class DbServer extends DbShared {
|
|
|
454
450
|
await this.save();
|
|
455
451
|
}
|
|
456
452
|
await this.ioWorker.terminate();
|
|
453
|
+
this.ioWorker = null;
|
|
457
454
|
await Promise.all(this.workers.map((worker) => worker.terminate()));
|
|
458
455
|
this.workers = [];
|
|
459
456
|
native.stop(this.dbCtxExternal);
|
|
@@ -90,7 +90,7 @@ export const migrate = async (server, fromSchema, toSchema, transform) => {
|
|
|
90
90
|
// Block handling
|
|
91
91
|
let i = 0;
|
|
92
92
|
let rangesToMigrate = [];
|
|
93
|
-
await save(server,
|
|
93
|
+
await save(server, { skipMigrationCheck: true });
|
|
94
94
|
server.verifTree.foreachBlock((block) => {
|
|
95
95
|
const [typeId, start] = destructureTreeKey(block.key);
|
|
96
96
|
const def = server.schemaTypesParsedById[typeId];
|
|
@@ -103,13 +103,13 @@ export const migrate = async (server, fromSchema, toSchema, transform) => {
|
|
|
103
103
|
break;
|
|
104
104
|
}
|
|
105
105
|
// block modifies
|
|
106
|
-
server.
|
|
106
|
+
server.activeReaders++;
|
|
107
107
|
const leafData = rangesToMigrate[i++];
|
|
108
108
|
port1.postMessage(leafData);
|
|
109
109
|
setToAwake(workerState, true);
|
|
110
110
|
await waitUntilSleeping(workerState);
|
|
111
111
|
// exec queued modifies
|
|
112
|
-
server.
|
|
112
|
+
server.activeReaders--;
|
|
113
113
|
server.onQueryEnd();
|
|
114
114
|
if (i === rangesToMigrate.length) {
|
|
115
115
|
if (server.dirtyRanges.size) {
|
|
@@ -149,14 +149,14 @@ export const migrate = async (server, fromSchema, toSchema, transform) => {
|
|
|
149
149
|
// -----------------------------------------
|
|
150
150
|
tmpDb.server.dbCtxExternal = fromCtx;
|
|
151
151
|
// TODO makes this SYNC
|
|
152
|
-
const promises = server.workers.map((worker) => worker.updateCtx(toAddress));
|
|
152
|
+
const promises = [server.ioWorker, ...server.workers].map((worker) => worker.updateCtx(toAddress));
|
|
153
153
|
promises.push(tmpDb.destroy(), worker.terminate());
|
|
154
154
|
await Promise.all(promises);
|
|
155
155
|
if (abort()) {
|
|
156
156
|
return;
|
|
157
157
|
}
|
|
158
158
|
native.membarSyncRead();
|
|
159
|
-
await save(server,
|
|
159
|
+
await save(server, { forceFullDump: true, skipDirtyCheck: true, skipMigrationCheck: true });
|
|
160
160
|
await writeSchemaFile(server, toSchema);
|
|
161
161
|
server.migrating = 0;
|
|
162
162
|
process.nextTick(() => server.emit('schema', server.schema));
|
|
@@ -19,5 +19,11 @@ export type Writelog = {
|
|
|
19
19
|
[t: number]: RangeDump[];
|
|
20
20
|
};
|
|
21
21
|
};
|
|
22
|
-
|
|
22
|
+
type SaveOpts = {
|
|
23
|
+
forceFullDump?: boolean;
|
|
24
|
+
skipDirtyCheck?: boolean;
|
|
25
|
+
skipMigrationCheck?: boolean;
|
|
26
|
+
};
|
|
27
|
+
export declare function saveSync(db: DbServer, opts?: SaveOpts): void;
|
|
28
|
+
export declare function save(db: DbServer, opts?: SaveOpts): Promise<void>;
|
|
23
29
|
export {};
|
package/dist/src/server/save.js
CHANGED
|
@@ -3,7 +3,7 @@ import { isMainThread } from 'node:worker_threads';
|
|
|
3
3
|
import { writeFile } from 'node:fs/promises';
|
|
4
4
|
import { join } from 'node:path';
|
|
5
5
|
import { VerifTree, destructureTreeKey, } from './tree.js';
|
|
6
|
-
import { saveBlock, foreachBlock, foreachDirtyBlock, } from './blocks.js';
|
|
6
|
+
import { saveBlock, foreachBlock, foreachDirtyBlock, saveBlocks, } from './blocks.js';
|
|
7
7
|
import { writeFileSync } from 'node:fs';
|
|
8
8
|
import { bufToHex } from '@saulx/utils';
|
|
9
9
|
import { COMMON_SDB_FILE, WRITELOG_FILE } from '../types.js';
|
|
@@ -14,22 +14,56 @@ function hasPartialTypes(db) {
|
|
|
14
14
|
}
|
|
15
15
|
return res;
|
|
16
16
|
}
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
17
|
+
function inhibitSave(db, { skipDirtyCheck, forceFullDump, skipMigrationCheck }) {
|
|
18
|
+
// RFE isMainThread needed??
|
|
19
|
+
if (!(isMainThread && (skipDirtyCheck || db.dirtyRanges.size || forceFullDump))) {
|
|
20
|
+
return true;
|
|
20
21
|
}
|
|
21
22
|
if (forceFullDump && hasPartialTypes(db)) {
|
|
22
23
|
db.emit('error', 'forceFullDump is not allowed with partial types');
|
|
23
|
-
return;
|
|
24
|
+
return true;
|
|
24
25
|
}
|
|
25
26
|
if (db.migrating && !skipMigrationCheck) {
|
|
26
27
|
db.emit('info', 'Block save db is migrating');
|
|
27
|
-
return;
|
|
28
|
+
return true;
|
|
28
29
|
}
|
|
29
30
|
if (db.saveInProgress) {
|
|
30
31
|
db.emit('info', 'Already have a save in progress cancel save');
|
|
31
|
-
return;
|
|
32
|
+
return true;
|
|
33
|
+
}
|
|
34
|
+
return false;
|
|
35
|
+
}
|
|
36
|
+
function makeWritelog(db, ts) {
|
|
37
|
+
const types = {};
|
|
38
|
+
const rangeDumps = {};
|
|
39
|
+
for (const key in db.schemaTypesParsed) {
|
|
40
|
+
const { id, lastId, blockCapacity } = db.schemaTypesParsed[key];
|
|
41
|
+
types[id] = { lastId, blockCapacity };
|
|
42
|
+
rangeDumps[id] = [];
|
|
32
43
|
}
|
|
44
|
+
db.verifTree.foreachBlock((block) => {
|
|
45
|
+
const [typeId, start] = destructureTreeKey(block.key);
|
|
46
|
+
const def = db.schemaTypesParsedById[typeId];
|
|
47
|
+
const end = start + def.blockCapacity - 1;
|
|
48
|
+
const data = {
|
|
49
|
+
file: db.verifTree.getBlockFile(block),
|
|
50
|
+
hash: bufToHex(block.hash),
|
|
51
|
+
start,
|
|
52
|
+
end,
|
|
53
|
+
};
|
|
54
|
+
rangeDumps[typeId].push(data);
|
|
55
|
+
});
|
|
56
|
+
return {
|
|
57
|
+
ts,
|
|
58
|
+
types,
|
|
59
|
+
commonDump: COMMON_SDB_FILE,
|
|
60
|
+
rangeDumps,
|
|
61
|
+
hash: bufToHex(db.verifTree.hash), // TODO `hash('hex')`
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
export function saveSync(db, opts = {}) {
|
|
65
|
+
if (inhibitSave(db, opts))
|
|
66
|
+
return;
|
|
33
67
|
let ts = Date.now();
|
|
34
68
|
db.saveInProgress = true;
|
|
35
69
|
try {
|
|
@@ -39,7 +73,7 @@ export function save(db, sync = false, forceFullDump = false, skipMigrationCheck
|
|
|
39
73
|
db.emit('error', `Save common failed: ${err}`);
|
|
40
74
|
// Return ?
|
|
41
75
|
}
|
|
42
|
-
if (forceFullDump) {
|
|
76
|
+
if (opts.forceFullDump) {
|
|
43
77
|
// reset the state just in case
|
|
44
78
|
db.verifTree = new VerifTree(db.schemaTypesParsed);
|
|
45
79
|
// We use db.verifTree.types instead of db.schemaTypesParsed because it's
|
|
@@ -53,58 +87,81 @@ export function save(db, sync = false, forceFullDump = false, skipMigrationCheck
|
|
|
53
87
|
foreachDirtyBlock(db, (_mtKey, typeId, start, end) => saveBlock(db, typeId, start, end));
|
|
54
88
|
}
|
|
55
89
|
db.dirtyRanges.clear();
|
|
56
|
-
const
|
|
57
|
-
const rangeDumps = {};
|
|
58
|
-
for (const key in db.schemaTypesParsed) {
|
|
59
|
-
const { id, lastId, blockCapacity } = db.schemaTypesParsed[key];
|
|
60
|
-
types[id] = { lastId, blockCapacity };
|
|
61
|
-
rangeDumps[id] = [];
|
|
62
|
-
}
|
|
63
|
-
db.verifTree.foreachBlock((block) => {
|
|
64
|
-
const [typeId, start] = destructureTreeKey(block.key);
|
|
65
|
-
const def = db.schemaTypesParsedById[typeId];
|
|
66
|
-
const end = start + def.blockCapacity - 1;
|
|
67
|
-
const data = {
|
|
68
|
-
file: db.verifTree.getBlockFile(block),
|
|
69
|
-
hash: bufToHex(block.hash),
|
|
70
|
-
start,
|
|
71
|
-
end,
|
|
72
|
-
};
|
|
73
|
-
rangeDumps[typeId].push(data);
|
|
74
|
-
});
|
|
75
|
-
const data = {
|
|
76
|
-
ts,
|
|
77
|
-
types,
|
|
78
|
-
commonDump: COMMON_SDB_FILE,
|
|
79
|
-
rangeDumps,
|
|
80
|
-
hash: bufToHex(db.verifTree.hash), // TODO `hash('hex')`
|
|
81
|
-
};
|
|
82
|
-
const filePath = join(db.fileSystemPath, WRITELOG_FILE);
|
|
90
|
+
const data = makeWritelog(db, ts);
|
|
83
91
|
const content = JSON.stringify(data);
|
|
84
|
-
db.emit('info', `Save
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
92
|
+
db.emit('info', `Save took ${Date.now() - ts}ms`);
|
|
93
|
+
db.saveInProgress = false;
|
|
94
|
+
return writeFileSync(join(db.fileSystemPath, WRITELOG_FILE), content);
|
|
95
|
+
}
|
|
96
|
+
catch (err) {
|
|
97
|
+
db.emit('error', `Save failed ${err.message}`);
|
|
98
|
+
db.saveInProgress = false;
|
|
99
|
+
throw err;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
export async function save(db, opts = {}) {
|
|
103
|
+
if (inhibitSave(db, opts))
|
|
104
|
+
return;
|
|
105
|
+
let ts = Date.now();
|
|
106
|
+
db.saveInProgress = true;
|
|
107
|
+
try {
|
|
108
|
+
let err;
|
|
109
|
+
err = native.saveCommon(join(db.fileSystemPath, COMMON_SDB_FILE), db.dbCtxExternal);
|
|
110
|
+
if (err) {
|
|
111
|
+
db.emit('error', `Save common failed: ${err}`);
|
|
112
|
+
// Return ?
|
|
113
|
+
}
|
|
114
|
+
const blocks = [];
|
|
115
|
+
if (opts.forceFullDump) {
|
|
116
|
+
// reset the state just in case
|
|
117
|
+
db.verifTree = new VerifTree(db.schemaTypesParsed);
|
|
118
|
+
// We use db.verifTree.types instead of db.schemaTypesParsed because it's
|
|
119
|
+
// ordered.
|
|
120
|
+
for (const { typeId } of db.verifTree.types()) {
|
|
121
|
+
const def = db.schemaTypesParsedById[typeId];
|
|
122
|
+
foreachBlock(db, def, (start, end, _hash) => {
|
|
123
|
+
const typeId = def.id;
|
|
124
|
+
const file = VerifTree.blockSdbFile(typeId, start, end);
|
|
125
|
+
const filepath = join(db.fileSystemPath, file);
|
|
126
|
+
blocks.push({
|
|
127
|
+
filepath,
|
|
128
|
+
typeId,
|
|
129
|
+
start,
|
|
130
|
+
});
|
|
131
|
+
});
|
|
132
|
+
}
|
|
88
133
|
}
|
|
89
134
|
else {
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
db.emit('error', `Save: writing writeLog failed ${err.message}`);
|
|
98
|
-
db.saveInProgress = false;
|
|
99
|
-
reject(err);
|
|
135
|
+
foreachDirtyBlock(db, (_mtKey, typeId, start, end) => {
|
|
136
|
+
const file = VerifTree.blockSdbFile(typeId, start, end);
|
|
137
|
+
const filepath = join(db.fileSystemPath, file);
|
|
138
|
+
blocks.push({
|
|
139
|
+
filepath,
|
|
140
|
+
typeId,
|
|
141
|
+
start
|
|
100
142
|
});
|
|
101
143
|
});
|
|
102
144
|
}
|
|
145
|
+
db.dirtyRanges.clear();
|
|
146
|
+
await saveBlocks(db, blocks);
|
|
147
|
+
try {
|
|
148
|
+
// Note that we assume here that verifTree didn't change before we call
|
|
149
|
+
// makeWritelog(). This is true as long as db.saveInProgress protects
|
|
150
|
+
// the verifTree from changes.
|
|
151
|
+
const data = makeWritelog(db, ts);
|
|
152
|
+
await writeFile(join(db.fileSystemPath, WRITELOG_FILE), JSON.stringify(data));
|
|
153
|
+
}
|
|
154
|
+
catch (err) {
|
|
155
|
+
db.emit('error', `Save: writing writeLog failed ${err.message}`);
|
|
156
|
+
}
|
|
157
|
+
db.emit('info', `Save took ${Date.now() - ts}ms`);
|
|
103
158
|
}
|
|
104
159
|
catch (err) {
|
|
105
160
|
db.emit('error', `Save failed ${err.message}`);
|
|
106
|
-
db.saveInProgress = false;
|
|
107
161
|
throw err;
|
|
108
162
|
}
|
|
163
|
+
finally {
|
|
164
|
+
db.saveInProgress = false;
|
|
165
|
+
}
|
|
109
166
|
}
|
|
110
167
|
//# sourceMappingURL=save.js.map
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
+
import { StrictSchema } from '@based/schema';
|
|
1
2
|
import { DbServer } from './index.js';
|
|
2
3
|
import { DbSchema } from '../schema.js';
|
|
3
|
-
import { StrictSchema } from '@based/schema';
|
|
4
4
|
export declare const setSchemaOnServer: (server: DbServer, schema: DbSchema) => void;
|
|
5
5
|
export declare const writeSchemaFile: (server: DbServer, schema: DbSchema) => Promise<void>;
|
|
6
6
|
export declare const setNativeSchema: (server: DbServer, schema: DbSchema) => void;
|
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
import { schemaToSelvaBuffer, updateTypeDefs } from '@based/schema/def';
|
|
2
|
+
import { deepCopy, writeUint64 } from '@saulx/utils';
|
|
3
|
+
import { getPropType, serialize } from '@based/schema';
|
|
2
4
|
import { join } from 'node:path';
|
|
3
5
|
import { writeFile } from 'node:fs/promises';
|
|
4
6
|
import native from '../native.js';
|
|
5
7
|
import { makeTreeKey } from './tree.js';
|
|
6
|
-
import { deepCopy, writeUint64 } from '@saulx/utils';
|
|
7
8
|
import { SCHEMA_FILE } from '../types.js';
|
|
9
|
+
import { saveSync } from './save.js';
|
|
8
10
|
import { hash } from '@saulx/hash';
|
|
9
|
-
import { getPropType, serialize } from '@based/schema';
|
|
10
11
|
export const setSchemaOnServer = (server, schema) => {
|
|
11
12
|
const { schemaTypesParsed, schemaTypesParsedById } = updateTypeDefs(schema);
|
|
12
13
|
server.schema = schema;
|
|
@@ -55,6 +56,9 @@ export const setNativeSchema = (server, schema) => {
|
|
|
55
56
|
//server.verifTree = new VerifTree(.schemaTypesParsed)
|
|
56
57
|
}
|
|
57
58
|
server.verifTree.updateTypes(server.schemaTypesParsed);
|
|
59
|
+
if (server.fileSystemPath) {
|
|
60
|
+
saveSync(server, { skipDirtyCheck: true });
|
|
61
|
+
}
|
|
58
62
|
};
|
|
59
63
|
export const strictSchemaToDbSchema = (schema) => {
|
|
60
64
|
// @ts-ignore
|
package/dist/src/server/start.js
CHANGED
|
@@ -7,11 +7,11 @@ import { join } from 'node:path';
|
|
|
7
7
|
import { VerifTree, makeTreeKey } from './tree.js';
|
|
8
8
|
import { foreachBlock } from './blocks.js';
|
|
9
9
|
import exitHook from 'exit-hook';
|
|
10
|
-
import { save } from './save.js';
|
|
10
|
+
import { save, saveSync } from './save.js';
|
|
11
11
|
import { deSerialize } from '@based/schema';
|
|
12
12
|
import { BLOCK_CAPACITY_DEFAULT } from '@based/schema/def';
|
|
13
13
|
import { bufToHex, equals, hexToBuf, wait } from '@saulx/utils';
|
|
14
|
-
import { SCHEMA_FILE, WRITELOG_FILE } from '../types.js';
|
|
14
|
+
import { SCHEMA_FILE, WRITELOG_FILE, SCHEMA_FILE_DEPRECATED } from '../types.js';
|
|
15
15
|
import { setSchemaOnServer } from './schema.js';
|
|
16
16
|
function startWorkers(db, opts) {
|
|
17
17
|
const queryThreads = opts?.queryThreads ?? availableParallelism();
|
|
@@ -42,11 +42,17 @@ export async function start(db, opts) {
|
|
|
42
42
|
console.error(e.message);
|
|
43
43
|
throw e;
|
|
44
44
|
}
|
|
45
|
-
const schema = await readFile(join(path, SCHEMA_FILE));
|
|
45
|
+
const schema = await readFile(join(path, SCHEMA_FILE)).catch(noop);
|
|
46
46
|
if (schema) {
|
|
47
47
|
const s = deSerialize(schema);
|
|
48
48
|
setSchemaOnServer(db, s);
|
|
49
49
|
}
|
|
50
|
+
else {
|
|
51
|
+
const schemaJson = await readFile(join(path, SCHEMA_FILE_DEPRECATED));
|
|
52
|
+
if (schemaJson) {
|
|
53
|
+
setSchemaOnServer(db, JSON.parse(schemaJson.toString()));
|
|
54
|
+
}
|
|
55
|
+
}
|
|
50
56
|
// Load all range dumps
|
|
51
57
|
for (const typeId in writelog.rangeDumps) {
|
|
52
58
|
const dumps = writelog.rangeDumps[typeId];
|
|
@@ -111,7 +117,7 @@ export async function start(db, opts) {
|
|
|
111
117
|
// in Node.js.
|
|
112
118
|
signals.forEach((sig) => process.on(sig, blockSig));
|
|
113
119
|
db.emit('info', `Exiting with signal: ${signal}`);
|
|
114
|
-
|
|
120
|
+
saveSync(db);
|
|
115
121
|
db.emit('info', 'Successfully saved.');
|
|
116
122
|
signals.forEach((sig) => process.off(sig, blockSig));
|
|
117
123
|
});
|
|
@@ -121,7 +127,7 @@ export async function start(db, opts) {
|
|
|
121
127
|
// use timeout
|
|
122
128
|
if (db.saveIntervalInSeconds > 0) {
|
|
123
129
|
db.saveInterval ??= setInterval(() => {
|
|
124
|
-
|
|
130
|
+
save(db);
|
|
125
131
|
}, db.saveIntervalInSeconds * 1e3);
|
|
126
132
|
}
|
|
127
133
|
if (db.schema) {
|
|
@@ -6,8 +6,19 @@ export declare const makeTreeKeyFromNodeId: (typeId: number, blockCapacity: numb
|
|
|
6
6
|
type Hash = Uint8Array;
|
|
7
7
|
export type VerifBlock = {
|
|
8
8
|
key: number;
|
|
9
|
+
/**
|
|
10
|
+
* Last acquired hash of the block.
|
|
11
|
+
* This is normally updated at load and save time but never during read/modify ops.
|
|
12
|
+
*/
|
|
9
13
|
hash: Hash;
|
|
14
|
+
/**
|
|
15
|
+
* If false the block is offloaded to fs;
|
|
16
|
+
* true doesn't necessarily mean that the block still exists because it could have been deleted.
|
|
17
|
+
*/
|
|
10
18
|
inmem: boolean;
|
|
19
|
+
/**
|
|
20
|
+
* If set, the block is being loaded and it can be awaited with this promise.
|
|
21
|
+
*/
|
|
11
22
|
loadPromise: null | Promise<void>;
|
|
12
23
|
};
|
|
13
24
|
export declare class VerifTree {
|