@based/db 0.0.64 → 0.0.67
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/lib/darwin_aarch64/include/selva/colvec.h +71 -0
- package/dist/lib/darwin_aarch64/include/selva/db.h +33 -4
- package/dist/lib/darwin_aarch64/include/selva/fields.h +37 -25
- package/dist/lib/darwin_aarch64/include/selva/hll.h +5 -3
- package/dist/lib/darwin_aarch64/include/selva/membar.h +23 -0
- package/dist/lib/darwin_aarch64/include/selva/types.h +8 -1
- package/dist/lib/darwin_aarch64/include/selva/worker_ctx.h +19 -3
- package/dist/lib/darwin_aarch64/libdeflate.dylib +0 -0
- package/dist/lib/darwin_aarch64/libjemalloc_selva.2.dylib +0 -0
- package/dist/lib/darwin_aarch64/libnode-v20.node +0 -0
- package/dist/lib/darwin_aarch64/libnode-v21.node +0 -0
- package/dist/lib/darwin_aarch64/libnode-v22.node +0 -0
- package/dist/lib/darwin_aarch64/libnode-v23.node +0 -0
- package/dist/lib/darwin_aarch64/libnode-v24.node +0 -0
- package/dist/lib/darwin_aarch64/libselva.dylib +0 -0
- package/dist/lib/darwin_aarch64/libxxhash.dylib +0 -0
- package/dist/lib/linux_aarch64/include/selva/colvec.h +71 -0
- package/dist/lib/linux_aarch64/include/selva/db.h +33 -4
- package/dist/lib/linux_aarch64/include/selva/fields.h +37 -25
- package/dist/lib/linux_aarch64/include/selva/hll.h +5 -3
- package/dist/lib/linux_aarch64/include/selva/membar.h +23 -0
- package/dist/lib/linux_aarch64/include/selva/types.h +8 -1
- package/dist/lib/linux_aarch64/include/selva/worker_ctx.h +19 -3
- package/dist/lib/linux_aarch64/libjemalloc_selva.so.2 +0 -0
- package/dist/lib/linux_aarch64/libnode-v20.node +0 -0
- package/dist/lib/linux_aarch64/libnode-v21.node +0 -0
- package/dist/lib/linux_aarch64/libnode-v22.node +0 -0
- package/dist/lib/linux_aarch64/libnode-v23.node +0 -0
- package/dist/lib/linux_aarch64/libnode-v24.node +0 -0
- package/dist/lib/linux_aarch64/libselva.so +0 -0
- package/dist/lib/linux_x86_64/include/selva/colvec.h +71 -0
- package/dist/lib/linux_x86_64/include/selva/db.h +33 -4
- package/dist/lib/linux_x86_64/include/selva/fields.h +37 -25
- package/dist/lib/linux_x86_64/include/selva/hll.h +5 -3
- package/dist/lib/linux_x86_64/include/selva/membar.h +23 -0
- package/dist/lib/linux_x86_64/include/selva/types.h +8 -1
- package/dist/lib/linux_x86_64/include/selva/worker_ctx.h +19 -3
- package/dist/lib/linux_x86_64/libjemalloc_selva.so.2 +0 -0
- package/dist/lib/linux_x86_64/libnode-v20.node +0 -0
- package/dist/lib/linux_x86_64/libnode-v21.node +0 -0
- package/dist/lib/linux_x86_64/libnode-v22.node +0 -0
- package/dist/lib/linux_x86_64/libnode-v23.node +0 -0
- package/dist/lib/linux_x86_64/libnode-v24.node +0 -0
- package/dist/lib/linux_x86_64/libselva.so +0 -0
- package/dist/src/client/flushModify.d.ts +2 -1
- package/dist/src/client/flushModify.js +12 -4
- package/dist/src/client/modify/create.js +11 -0
- package/dist/src/client/modify/delete.js +3 -0
- package/dist/src/client/modify/fixed.js +1 -1
- package/dist/src/client/modify/modify.js +2 -2
- package/dist/src/client/modify/setCursor.d.ts +2 -1
- package/dist/src/client/query/BasedDbQuery.d.ts +10 -4
- package/dist/src/client/query/BasedDbQuery.js +114 -6
- package/dist/src/client/query/aggregates/aggregation.js +24 -11
- package/dist/src/client/query/aggregates/types.d.ts +22 -2
- package/dist/src/client/query/aggregates/types.js +34 -1
- package/dist/src/client/query/display.js +8 -2
- package/dist/src/client/query/filter/createVariableFilterBuffer.d.ts +2 -3
- package/dist/src/client/query/filter/createVariableFilterBuffer.js +20 -7
- package/dist/src/client/query/filter/filter.js +13 -3
- package/dist/src/client/query/filter/primitiveFilter.d.ts +1 -2
- package/dist/src/client/query/include/props.js +18 -2
- package/dist/src/client/query/include/toBuffer.js +11 -3
- package/dist/src/client/query/include/walk.js +5 -1
- package/dist/src/client/query/queryDef.js +4 -1
- package/dist/src/client/query/read/read.js +52 -22
- package/dist/src/client/query/registerQuery.js +1 -0
- package/dist/src/client/query/search/index.d.ts +1 -1
- package/dist/src/client/query/search/index.js +21 -7
- package/dist/src/client/query/sort.d.ts +1 -1
- package/dist/src/client/query/toByteCode/default.d.ts +1 -1
- package/dist/src/client/query/toByteCode/default.js +0 -2
- package/dist/src/client/query/toByteCode/toBuffer.js +0 -7
- package/dist/src/client/query/types.d.ts +16 -5
- package/dist/src/client/query/validation.d.ts +3 -0
- package/dist/src/client/query/validation.js +34 -2
- package/dist/src/client/xxHash64.d.ts +1 -1
- package/dist/src/index.d.ts +1 -2
- package/dist/src/index.js +0 -1
- package/dist/src/native.d.ts +7 -4
- package/dist/src/native.js +23 -13
- package/dist/src/server/IoWorker.d.ts +8 -0
- package/dist/src/server/IoWorker.js +39 -0
- package/dist/src/server/QueryWorker.d.ts +8 -0
- package/dist/src/server/QueryWorker.js +26 -0
- package/dist/src/server/blocks.d.ts +24 -0
- package/dist/src/server/blocks.js +112 -0
- package/dist/src/server/dbHash.d.ts +1 -1
- package/dist/src/server/index.d.ts +10 -16
- package/dist/src/server/index.js +39 -15
- package/dist/src/server/migrate/index.d.ts +5 -0
- package/dist/src/server/migrate/index.js +11 -7
- package/dist/src/server/migrate/worker.js +3 -0
- package/dist/src/server/save.d.ts +8 -6
- package/dist/src/server/save.js +34 -78
- package/dist/src/server/schema.js +6 -5
- package/dist/src/server/start.js +57 -60
- package/dist/src/server/tree.d.ts +24 -13
- package/dist/src/server/tree.js +95 -66
- package/dist/src/server/workers/DbWorker.d.ts +17 -0
- package/dist/src/server/{DbWorker.js → workers/DbWorker.js} +15 -17
- package/dist/src/server/workers/io_worker.js +39 -0
- package/dist/src/server/workers/io_worker_types.d.ts +12 -0
- package/dist/src/server/workers/io_worker_types.js +2 -0
- package/dist/src/server/workers/query_worker.d.ts +1 -0
- package/dist/src/server/workers/query_worker.js +4 -0
- package/dist/src/server/workers/worker.d.ts +1 -0
- package/dist/src/server/workers/worker.js +41 -0
- package/dist/src/shared/Emitter.d.ts +1 -0
- package/dist/src/types.d.ts +1 -1
- package/dist/src/types.js +1 -1
- package/package.json +3 -3
- package/dist/lib/darwin_aarch64/include/selva/find.h +0 -47
- package/dist/lib/darwin_aarch64/include/selva/history.h +0 -64
- package/dist/lib/darwin_aarch64/include/selva/queue_r.h +0 -190
- package/dist/lib/darwin_aarch64/include/selva/traverse.h +0 -65
- package/dist/lib/linux_aarch64/include/selva/find.h +0 -47
- package/dist/lib/linux_aarch64/include/selva/history.h +0 -64
- package/dist/lib/linux_aarch64/include/selva/queue_r.h +0 -190
- package/dist/lib/linux_aarch64/include/selva/traverse.h +0 -65
- package/dist/lib/linux_x86_64/include/selva/find.h +0 -47
- package/dist/lib/linux_x86_64/include/selva/history.h +0 -64
- package/dist/lib/linux_x86_64/include/selva/queue_r.h +0 -190
- package/dist/lib/linux_x86_64/include/selva/traverse.h +0 -65
- package/dist/src/client/query/serialize.d.ts +0 -4
- package/dist/src/client/query/serialize.js +0 -26
- package/dist/src/server/DbWorker.d.ts +0 -13
- package/dist/src/server/csmt/draw-dot.d.ts +0 -4
- package/dist/src/server/csmt/draw-dot.js +0 -38
- package/dist/src/server/csmt/index.d.ts +0 -4
- package/dist/src/server/csmt/index.js +0 -5
- package/dist/src/server/csmt/match.d.ts +0 -7
- package/dist/src/server/csmt/match.js +0 -10
- package/dist/src/server/csmt/memebership-proof.d.ts +0 -7
- package/dist/src/server/csmt/memebership-proof.js +0 -122
- package/dist/src/server/csmt/tree-utils.d.ts +0 -6
- package/dist/src/server/csmt/tree-utils.js +0 -33
- package/dist/src/server/csmt/tree.d.ts +0 -3
- package/dist/src/server/csmt/tree.js +0 -270
- package/dist/src/server/csmt/types.d.ts +0 -46
- package/dist/src/server/csmt/types.js +0 -2
- package/dist/src/server/worker.js +0 -33
- /package/dist/src/server/{worker.d.ts → workers/io_worker.d.ts} +0 -0
package/dist/src/native.js
CHANGED
|
@@ -4,21 +4,19 @@ import db from '../../basedDbNative.cjs';
|
|
|
4
4
|
const DECODER = new TextDecoder('utf-8');
|
|
5
5
|
const ENCODER = new TextEncoder();
|
|
6
6
|
const selvaIoErrlog = new Uint8Array(256);
|
|
7
|
-
var compressor = db.createCompressor();
|
|
7
|
+
var compressor = db.createCompressor(); // put on threadCtx
|
|
8
8
|
var decompressor = db.createDecompressor();
|
|
9
9
|
function SelvaIoErrlogToString(buf) {
|
|
10
10
|
let i;
|
|
11
11
|
let len = (i = buf.indexOf(0)) >= 0 ? i : buf.byteLength;
|
|
12
12
|
return DECODER.decode(selvaIoErrlog.slice(0, len));
|
|
13
13
|
}
|
|
14
|
+
// add worker CTX HERE
|
|
15
|
+
// then add it to every function
|
|
16
|
+
// worker should allways be here
|
|
17
|
+
// then add ThreadCtx to modify ctx and query ctx
|
|
14
18
|
const native = {
|
|
15
|
-
|
|
16
|
-
return db.historyAppend(history, typeId, nodeId, dbCtx);
|
|
17
|
-
},
|
|
18
|
-
historyCreate(pathname, mainLen) {
|
|
19
|
-
const pathBuf = ENCODER.encode(pathname + '\0');
|
|
20
|
-
return db.historyCreate(pathBuf, mainLen + 16 - (mainLen % 16));
|
|
21
|
-
},
|
|
19
|
+
threadCtx: null, // add compressors here as well!
|
|
22
20
|
workerCtxInit: () => {
|
|
23
21
|
return db.workerCtxInit();
|
|
24
22
|
},
|
|
@@ -45,9 +43,9 @@ const native = {
|
|
|
45
43
|
const pathBuf = ENCODER.encode(path + '\0');
|
|
46
44
|
return db.saveCommon(pathBuf, dbCtx);
|
|
47
45
|
},
|
|
48
|
-
|
|
46
|
+
saveBlock: (path, typeCode, start, dbCtx, hashOut) => {
|
|
49
47
|
const pathBuf = ENCODER.encode(path + '\0');
|
|
50
|
-
return db.
|
|
48
|
+
return db.saveBlock(pathBuf, typeCode, start, dbCtx, hashOut);
|
|
51
49
|
},
|
|
52
50
|
loadCommon: (path, dbCtx) => {
|
|
53
51
|
const pathBuf = ENCODER.encode(path + '\0');
|
|
@@ -56,13 +54,16 @@ const native = {
|
|
|
56
54
|
throw new Error(`Failed to load common. selvaError: ${err} cause:\n${SelvaIoErrlogToString(selvaIoErrlog)}`);
|
|
57
55
|
}
|
|
58
56
|
},
|
|
59
|
-
|
|
57
|
+
loadBlock: (path, dbCtx) => {
|
|
60
58
|
const pathBuf = ENCODER.encode(path + '\0');
|
|
61
|
-
const err = db.
|
|
59
|
+
const err = db.loadBlock(pathBuf, dbCtx, selvaIoErrlog);
|
|
62
60
|
if (err) {
|
|
63
|
-
throw new Error(`Failed to load a range. selvaError: ${err} cause:\n${SelvaIoErrlogToString(selvaIoErrlog)}`);
|
|
61
|
+
throw new Error(`Failed to load a range "${path}". selvaError: ${err} cause:\n${SelvaIoErrlogToString(selvaIoErrlog)}`);
|
|
64
62
|
}
|
|
65
63
|
},
|
|
64
|
+
delBlock: (dbCtx, typeId, block) => {
|
|
65
|
+
db.delBlock(dbCtx, typeId, block);
|
|
66
|
+
},
|
|
66
67
|
updateSchemaType: (prefix, buf, dbCtx) => {
|
|
67
68
|
return db.updateSchema(prefix, buf, dbCtx);
|
|
68
69
|
},
|
|
@@ -93,6 +94,15 @@ const native = {
|
|
|
93
94
|
equals: (a, b) => {
|
|
94
95
|
return !!db.equals(a, b);
|
|
95
96
|
},
|
|
97
|
+
membarSyncRead: () => {
|
|
98
|
+
db.membarSyncRead();
|
|
99
|
+
},
|
|
100
|
+
membarSyncWrite: () => {
|
|
101
|
+
db.membarSyncWrite();
|
|
102
|
+
},
|
|
103
|
+
colvecTest: (dbCtx, typeId, field, nodeId, len) => {
|
|
104
|
+
return db.colvecTest(dbCtx, typeId, field, nodeId, len);
|
|
105
|
+
}
|
|
96
106
|
};
|
|
97
107
|
global.__basedDb__native__ = native;
|
|
98
108
|
export default native;
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { DbWorker } from './workers/DbWorker.js';
|
|
2
|
+
import { DbServer } from './index.js';
|
|
3
|
+
export declare class IoWorker extends DbWorker {
|
|
4
|
+
constructor(address: BigInt, db: DbServer);
|
|
5
|
+
handleMsg(_buf: any): void;
|
|
6
|
+
loadBlock(filepath: string): Promise<void>;
|
|
7
|
+
unloadBlock(filepath: string, typeId: number, start: number): Promise<Uint8Array>;
|
|
8
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { DbWorker } from './workers/DbWorker.js';
|
|
2
|
+
import { DECODER, readInt32 } from '@saulx/utils';
|
|
3
|
+
export class IoWorker extends DbWorker {
|
|
4
|
+
constructor(address, db) {
|
|
5
|
+
const onExit = (_code) => {
|
|
6
|
+
this.db.ioWorker = new IoWorker(address, db);
|
|
7
|
+
};
|
|
8
|
+
super(address, db, onExit, 'io_worker.js');
|
|
9
|
+
}
|
|
10
|
+
handleMsg(_buf) {
|
|
11
|
+
}
|
|
12
|
+
async loadBlock(filepath) {
|
|
13
|
+
const job = {
|
|
14
|
+
type: 'load',
|
|
15
|
+
filepath
|
|
16
|
+
};
|
|
17
|
+
const resBuf = await this.call(job);
|
|
18
|
+
if (resBuf.length) {
|
|
19
|
+
throw new Error(DECODER.decode(resBuf));
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
async unloadBlock(filepath, typeId, start) {
|
|
23
|
+
const job = {
|
|
24
|
+
type: 'unload',
|
|
25
|
+
filepath,
|
|
26
|
+
typeId,
|
|
27
|
+
start
|
|
28
|
+
};
|
|
29
|
+
const resBuf = await this.call(job);
|
|
30
|
+
const err = readInt32(resBuf, 0);
|
|
31
|
+
if (err) {
|
|
32
|
+
throw new Error(`selva error: ${err}`);
|
|
33
|
+
}
|
|
34
|
+
// Note that this shares the original buffer which may not be 100% optimal,
|
|
35
|
+
// as the first 4 bytes are no longer needed.
|
|
36
|
+
return new Uint8Array(resBuf.buffer, 4);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
//# sourceMappingURL=IoWorker.js.map
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { DbWorker } from './workers/DbWorker.js';
|
|
2
|
+
import { DbServer } from './index.js';
|
|
3
|
+
export declare class QueryWorker extends DbWorker {
|
|
4
|
+
constructor(address: BigInt, db: DbServer, workerIndex: number);
|
|
5
|
+
handleMsg(_buf: any): void;
|
|
6
|
+
protected callback: (resolve: (x: any) => any) => void;
|
|
7
|
+
getQueryBuf(buf: Uint8Array): Promise<Uint8Array>;
|
|
8
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { DbWorker } from './workers/DbWorker.js';
|
|
2
|
+
import { readUint64 } from '@saulx/utils';
|
|
3
|
+
export class QueryWorker extends DbWorker {
|
|
4
|
+
constructor(address, db, workerIndex) {
|
|
5
|
+
const onExit = (_code) => {
|
|
6
|
+
this.db.workers[workerIndex] = new QueryWorker(address, db, workerIndex);
|
|
7
|
+
};
|
|
8
|
+
super(address, db, onExit, 'query_worker.js');
|
|
9
|
+
}
|
|
10
|
+
handleMsg(_buf) {
|
|
11
|
+
this.db.processingQueries--;
|
|
12
|
+
this.db.onQueryEnd();
|
|
13
|
+
}
|
|
14
|
+
callback = (resolve) => {
|
|
15
|
+
this.db.processingQueries++;
|
|
16
|
+
this.resolvers.push(resolve);
|
|
17
|
+
};
|
|
18
|
+
getQueryBuf(buf) {
|
|
19
|
+
const schemaChecksum = readUint64(buf, buf.byteLength - 8);
|
|
20
|
+
if (schemaChecksum !== this.db.schema?.hash) {
|
|
21
|
+
return Promise.resolve(new Uint8Array(1));
|
|
22
|
+
}
|
|
23
|
+
return this.call(buf);
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
//# sourceMappingURL=QueryWorker.js.map
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { SchemaTypeDef } from '@based/schema/def';
|
|
2
|
+
import { DbServer } from './index.js';
|
|
3
|
+
/**
|
|
4
|
+
* Save a block.
|
|
5
|
+
*/
|
|
6
|
+
export declare function saveBlock(db: DbServer, typeId: number, start: number, end: number): void;
|
|
7
|
+
/**
|
|
8
|
+
* Load a block (typically of a partial type) back to memory.
|
|
9
|
+
*/
|
|
10
|
+
export declare function loadBlock(db: DbServer, def: SchemaTypeDef, start: number): Promise<void>;
|
|
11
|
+
/**
|
|
12
|
+
* Save a block and remove it from memory.
|
|
13
|
+
*/
|
|
14
|
+
export declare function unloadBlock(db: DbServer, def: SchemaTypeDef, start: number): Promise<void>;
|
|
15
|
+
/**
|
|
16
|
+
* Execute cb() for each block in memory.
|
|
17
|
+
*/
|
|
18
|
+
export declare function foreachBlock(db: DbServer, def: SchemaTypeDef, cb: (start: number, end: number, hash: Uint8Array) => void, includeEmptyBlocks?: boolean): void;
|
|
19
|
+
/**
|
|
20
|
+
* Execute cb() for each dirty block.
|
|
21
|
+
* A dirty block is one that is changed in memory but not yet persisted in the
|
|
22
|
+
* file system.
|
|
23
|
+
*/
|
|
24
|
+
export declare function foreachDirtyBlock(db: DbServer, cb: (mtKey: number, typeId: number, start: number, end: number) => void): void;
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import native from '../native.js';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { equals } from '@saulx/utils';
|
|
4
|
+
import { VerifTree, destructureTreeKey, makeTreeKey, } from './tree.js';
|
|
5
|
+
/**
|
|
6
|
+
* Save a block.
|
|
7
|
+
*/
|
|
8
|
+
export function saveBlock(db, typeId, start, end) {
|
|
9
|
+
const hash = new Uint8Array(16);
|
|
10
|
+
const mtKey = makeTreeKey(typeId, start);
|
|
11
|
+
const file = VerifTree.blockSdbFile(typeId, start, end);
|
|
12
|
+
const path = join(db.fileSystemPath, file);
|
|
13
|
+
const err = native.saveBlock(path, typeId, start, db.dbCtxExternal, hash);
|
|
14
|
+
if (err == -8) {
|
|
15
|
+
// TODO ENOENT
|
|
16
|
+
db.verifTree.remove(mtKey);
|
|
17
|
+
}
|
|
18
|
+
else if (err) {
|
|
19
|
+
// TODO print the error string
|
|
20
|
+
console.error(`Save ${typeId}:${start}-${end} failed: ${err}`);
|
|
21
|
+
}
|
|
22
|
+
else {
|
|
23
|
+
db.verifTree.update(mtKey, hash);
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Load a block (typically of a partial type) back to memory.
|
|
28
|
+
*/
|
|
29
|
+
export async function loadBlock(db, def, start) {
|
|
30
|
+
const key = makeTreeKey(def.id, start);
|
|
31
|
+
const block = db.verifTree.getBlock(key);
|
|
32
|
+
if (!block) {
|
|
33
|
+
throw new Error(`No such block: ${key}`);
|
|
34
|
+
}
|
|
35
|
+
if (block.loadPromise) {
|
|
36
|
+
return block.loadPromise;
|
|
37
|
+
}
|
|
38
|
+
const prevHash = block.hash;
|
|
39
|
+
const filename = db.verifTree.getBlockFile(block);
|
|
40
|
+
const p = db.ioWorker.loadBlock(join(db.fileSystemPath, filename));
|
|
41
|
+
block.loadPromise = p;
|
|
42
|
+
await p;
|
|
43
|
+
delete block.loadPromise;
|
|
44
|
+
// Update and verify the hash
|
|
45
|
+
const hash = new Uint8Array(16);
|
|
46
|
+
const end = start + def.blockCapacity - 1;
|
|
47
|
+
const res = native.getNodeRangeHash(def.id, start, end, hash, db.dbCtxExternal);
|
|
48
|
+
if (res) {
|
|
49
|
+
const key = makeTreeKey(def.id, start);
|
|
50
|
+
db.verifTree.update(key, hash);
|
|
51
|
+
if (!equals(prevHash, hash)) {
|
|
52
|
+
throw new Error('Block hash mismatch');
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Save a block and remove it from memory.
|
|
58
|
+
*/
|
|
59
|
+
export async function unloadBlock(db, def, start) {
|
|
60
|
+
const typeId = def.id;
|
|
61
|
+
const end = start + def.blockCapacity - 1;
|
|
62
|
+
const key = makeTreeKey(typeId, start);
|
|
63
|
+
const block = db.verifTree.getBlock(key);
|
|
64
|
+
if (!block) {
|
|
65
|
+
throw new Error(`No such block: ${key}`);
|
|
66
|
+
}
|
|
67
|
+
const filepath = join(db.fileSystemPath, VerifTree.blockSdbFile(typeId, start, end));
|
|
68
|
+
try {
|
|
69
|
+
const hash = await db.ioWorker.unloadBlock(filepath, typeId, start);
|
|
70
|
+
native.delBlock(db.dbCtxExternal, typeId, start);
|
|
71
|
+
db.verifTree.update(key, hash, false);
|
|
72
|
+
}
|
|
73
|
+
catch (e) {
|
|
74
|
+
// TODO Proper logging
|
|
75
|
+
// TODO err == -8 == SELVA_ENOENT => db.verifTree.remove(key) ??
|
|
76
|
+
console.error(`Save ${typeId}:${start}-${end} failed`);
|
|
77
|
+
console.error(e);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Execute cb() for each block in memory.
|
|
82
|
+
*/
|
|
83
|
+
export function foreachBlock(db, def, cb, includeEmptyBlocks = false) {
|
|
84
|
+
const step = def.blockCapacity;
|
|
85
|
+
for (let start = 1; start <= def.lastId; start += step) {
|
|
86
|
+
const end = start + step - 1;
|
|
87
|
+
const hash = new Uint8Array(16);
|
|
88
|
+
const res = native.getNodeRangeHash(def.id, start, end, hash, db.dbCtxExternal);
|
|
89
|
+
if (res || includeEmptyBlocks) {
|
|
90
|
+
cb(start, end, hash);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Execute cb() for each dirty block.
|
|
96
|
+
* A dirty block is one that is changed in memory but not yet persisted in the
|
|
97
|
+
* file system.
|
|
98
|
+
*/
|
|
99
|
+
export function foreachDirtyBlock(db, cb) {
|
|
100
|
+
const typeIdMap = {};
|
|
101
|
+
for (const typeName in db.schemaTypesParsed) {
|
|
102
|
+
const type = db.schemaTypesParsed[typeName];
|
|
103
|
+
const typeId = type.id;
|
|
104
|
+
typeIdMap[typeId] = type;
|
|
105
|
+
}
|
|
106
|
+
for (const mtKey of db.dirtyRanges) {
|
|
107
|
+
const [typeId, start] = destructureTreeKey(mtKey);
|
|
108
|
+
const end = start + typeIdMap[typeId].blockCapacity - 1;
|
|
109
|
+
cb(mtKey, typeId, start, end);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
//# sourceMappingURL=blocks.js.map
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { LangName, StrictSchema } from '@based/schema';
|
|
2
|
-
import { createTree } from './csmt/index.js';
|
|
3
2
|
import { StartOpts } from './start.js';
|
|
4
|
-
import {
|
|
3
|
+
import { VerifTree } from './tree.js';
|
|
5
4
|
import { TransformFns } from './migrate/index.js';
|
|
6
5
|
import exitHook from 'exit-hook';
|
|
7
6
|
import { SchemaChecksum } from '../schema.js';
|
|
8
|
-
import {
|
|
7
|
+
import { IoWorker } from './IoWorker.js';
|
|
8
|
+
import { QueryWorker } from './QueryWorker.js';
|
|
9
9
|
import { DbShared } from '../shared/DbBase.js';
|
|
10
10
|
declare class SortIndex {
|
|
11
11
|
constructor(buf: Uint8Array, dbCtxExternal: any);
|
|
@@ -17,17 +17,14 @@ export declare class DbServer extends DbShared {
|
|
|
17
17
|
#private;
|
|
18
18
|
modifyDirtyRanges: Float64Array;
|
|
19
19
|
dbCtxExternal: any;
|
|
20
|
+
threadCtxExternal: any;
|
|
20
21
|
migrating: number;
|
|
21
22
|
saveInProgress: boolean;
|
|
22
23
|
fileSystemPath: string;
|
|
23
|
-
|
|
24
|
+
verifTree: VerifTree;
|
|
24
25
|
dirtyRanges: Set<number>;
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
digest: (encoding?: "hex") => Uint8Array | string;
|
|
28
|
-
reset: () => void;
|
|
29
|
-
};
|
|
30
|
-
workers: DbWorker[];
|
|
26
|
+
ioWorker: IoWorker;
|
|
27
|
+
workers: QueryWorker[];
|
|
31
28
|
availableWorkerIndex: number;
|
|
32
29
|
processingQueries: number;
|
|
33
30
|
modifyQueue: Uint8Array[];
|
|
@@ -46,11 +43,8 @@ export declare class DbServer extends DbShared {
|
|
|
46
43
|
save(opts?: {
|
|
47
44
|
forceFullDump?: boolean;
|
|
48
45
|
}): Promise<void>;
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
digest: (encoding?: "hex") => Uint8Array | string;
|
|
52
|
-
reset: () => void;
|
|
53
|
-
};
|
|
46
|
+
loadBlock(typeName: string, nodeId: number): Promise<void>;
|
|
47
|
+
unloadBlock(typeName: string, nodeId: number): Promise<void>;
|
|
54
48
|
sortIndexes: {
|
|
55
49
|
[type: number]: {
|
|
56
50
|
[field: number]: {
|
|
@@ -68,7 +62,7 @@ export declare class DbServer extends DbShared {
|
|
|
68
62
|
createSortIndexBuffer(typeId: number, field: number, start: number, lang: number): SortIndex;
|
|
69
63
|
setSchema(strictSchema: StrictSchema, transformFns?: TransformFns): Promise<SchemaChecksum>;
|
|
70
64
|
modify(bufWithHash: Uint8Array): Record<number, number> | null;
|
|
71
|
-
addToQueryQueue(resolve: any, buf: any): Promise<Uint8Array
|
|
65
|
+
addToQueryQueue(resolve: any, buf: any): Promise<Uint8Array<ArrayBuffer>>;
|
|
72
66
|
getQueryBuf(buf: Uint8Array, fromQueue?: boolean): Promise<Uint8Array>;
|
|
73
67
|
onQueryEnd(): any;
|
|
74
68
|
stop(noSave?: boolean): Promise<void>;
|
package/dist/src/server/index.js
CHANGED
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
import native from '../native.js';
|
|
2
|
-
import createDbHash from './dbHash.js';
|
|
3
2
|
import { rm } from 'node:fs/promises';
|
|
4
3
|
import { langCodesMap } from '@based/schema';
|
|
5
4
|
import { start } from './start.js';
|
|
6
|
-
import {
|
|
5
|
+
import { destructureTreeKey, makeTreeKeyFromNodeId } from './tree.js';
|
|
7
6
|
import { save } from './save.js';
|
|
8
7
|
import { setTimeout } from 'node:timers/promises';
|
|
9
8
|
import { migrate } from './migrate/index.js';
|
|
@@ -14,6 +13,7 @@ import { strictSchemaToDbSchema } from './schema.js';
|
|
|
14
13
|
import { DbShared } from '../shared/DbBase.js';
|
|
15
14
|
import { setNativeSchema, setSchemaOnServer, writeSchemaFile, } from './schema.js';
|
|
16
15
|
import { resizeModifyDirtyRanges } from './resizeModifyDirtyRanges.js';
|
|
16
|
+
import { loadBlock, unloadBlock } from './blocks.js';
|
|
17
17
|
const emptyUint8Array = new Uint8Array(0);
|
|
18
18
|
class SortIndex {
|
|
19
19
|
constructor(buf, dbCtxExternal) {
|
|
@@ -27,15 +27,16 @@ class SortIndex {
|
|
|
27
27
|
export class DbServer extends DbShared {
|
|
28
28
|
modifyDirtyRanges;
|
|
29
29
|
dbCtxExternal; // pointer to zig dbCtx
|
|
30
|
+
threadCtxExternal; // pointer to zig dbCtx
|
|
30
31
|
migrating = null;
|
|
31
32
|
saveInProgress = false;
|
|
32
33
|
fileSystemPath;
|
|
33
|
-
|
|
34
|
+
verifTree;
|
|
34
35
|
dirtyRanges = new Set();
|
|
35
|
-
|
|
36
|
+
ioWorker;
|
|
36
37
|
workers = [];
|
|
37
38
|
availableWorkerIndex = -1;
|
|
38
|
-
processingQueries = 0;
|
|
39
|
+
processingQueries = 0; // semaphore for locking writes
|
|
39
40
|
modifyQueue = [];
|
|
40
41
|
queryQueue = new Map();
|
|
41
42
|
stopped; // = true does not work
|
|
@@ -59,11 +60,34 @@ export class DbServer extends DbShared {
|
|
|
59
60
|
save(opts) {
|
|
60
61
|
return save(this, false, opts?.forceFullDump ?? false);
|
|
61
62
|
}
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
63
|
+
async loadBlock(typeName, nodeId) {
|
|
64
|
+
const def = this.schemaTypesParsed[typeName];
|
|
65
|
+
if (!def) {
|
|
66
|
+
throw new Error('Type not found');
|
|
67
|
+
}
|
|
68
|
+
const typeId = def.id;
|
|
69
|
+
const key = makeTreeKeyFromNodeId(typeId, def.blockCapacity, nodeId);
|
|
70
|
+
const [, start] = destructureTreeKey(key);
|
|
71
|
+
const block = this.verifTree.getBlock(key);
|
|
72
|
+
if (!block) {
|
|
73
|
+
throw new Error('Block not found');
|
|
74
|
+
}
|
|
75
|
+
await loadBlock(this, def, start);
|
|
76
|
+
}
|
|
77
|
+
async unloadBlock(typeName, nodeId) {
|
|
78
|
+
const def = this.schemaTypesParsed[typeName];
|
|
79
|
+
if (!def) {
|
|
80
|
+
throw new Error('Type not found');
|
|
81
|
+
}
|
|
82
|
+
const typeId = def.id;
|
|
83
|
+
const key = makeTreeKeyFromNodeId(typeId, def.blockCapacity, nodeId);
|
|
84
|
+
const [, start] = destructureTreeKey(key);
|
|
85
|
+
const block = this.verifTree.getBlock(key);
|
|
86
|
+
if (!block) {
|
|
87
|
+
throw new Error('Block not found');
|
|
88
|
+
}
|
|
89
|
+
await unloadBlock(this, def, start);
|
|
90
|
+
}
|
|
67
91
|
sortIndexes;
|
|
68
92
|
cleanupTimer;
|
|
69
93
|
cleanup() {
|
|
@@ -298,15 +322,15 @@ export class DbServer extends DbShared {
|
|
|
298
322
|
while (typesSize--) {
|
|
299
323
|
const typeId = readUint16(buf, i);
|
|
300
324
|
const def = this.schemaTypesParsedById[typeId];
|
|
301
|
-
const key =
|
|
325
|
+
const key = makeTreeKeyFromNodeId(def.id, def.blockCapacity, def.lastId);
|
|
302
326
|
this.dirtyRanges.add(key);
|
|
303
327
|
i += 10;
|
|
304
328
|
}
|
|
305
329
|
const view = new DataView(buf.buffer, buf.byteOffset);
|
|
306
330
|
while (i < end) {
|
|
307
|
-
const key = view.getFloat64(i, true)
|
|
331
|
+
// const key = view.getFloat64(i, true)
|
|
308
332
|
// These node ranges may not actually exist
|
|
309
|
-
//this.dirtyRanges.add(key)
|
|
333
|
+
// this.dirtyRanges.add(key)
|
|
310
334
|
i += 8;
|
|
311
335
|
}
|
|
312
336
|
resizeModifyDirtyRanges(this);
|
|
@@ -387,7 +411,6 @@ export class DbServer extends DbShared {
|
|
|
387
411
|
}
|
|
388
412
|
}
|
|
389
413
|
onQueryEnd() {
|
|
390
|
-
this.processingQueries--;
|
|
391
414
|
if (this.processingQueries === 0) {
|
|
392
415
|
if (this.modifyQueue.length) {
|
|
393
416
|
const modifyQueue = this.modifyQueue;
|
|
@@ -430,7 +453,8 @@ export class DbServer extends DbShared {
|
|
|
430
453
|
if (!noSave) {
|
|
431
454
|
await this.save();
|
|
432
455
|
}
|
|
433
|
-
await
|
|
456
|
+
await this.ioWorker.terminate();
|
|
457
|
+
await Promise.all(this.workers.map((worker) => worker.terminate()));
|
|
434
458
|
this.workers = [];
|
|
435
459
|
native.stop(this.dbCtxExternal);
|
|
436
460
|
await setTimeout(100);
|
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
import { DbServer } from '../index.js';
|
|
2
2
|
import { DbSchema } from '../../schema.js';
|
|
3
|
+
export type MigrateRange = {
|
|
4
|
+
typeId: number;
|
|
5
|
+
start: number;
|
|
6
|
+
end: number;
|
|
7
|
+
};
|
|
3
8
|
type TransformFn = (node: Record<string, any>) => Record<string, any> | [string, Record<string, any>];
|
|
4
9
|
export type TransformFns = Record<string, TransformFn>;
|
|
5
10
|
export declare const migrate: (server: DbServer, fromSchema: DbSchema, toSchema: DbSchema, transform?: TransformFns) => Promise<void>;
|
|
@@ -2,7 +2,8 @@ import { BasedDb, save } from '../../index.js';
|
|
|
2
2
|
import { dirname, join } from 'path';
|
|
3
3
|
import { Worker, MessageChannel, receiveMessageOnPort, } from 'node:worker_threads';
|
|
4
4
|
import native from '../../native.js';
|
|
5
|
-
import {
|
|
5
|
+
import { destructureTreeKey } from '../tree.js';
|
|
6
|
+
import { foreachDirtyBlock } from '../blocks.js';
|
|
6
7
|
import { fileURLToPath } from 'url';
|
|
7
8
|
import { setNativeSchema, setSchemaOnServer, writeSchemaFile, } from '../schema.js';
|
|
8
9
|
import { setToAwake, waitUntilSleeping } from './utils.js';
|
|
@@ -90,11 +91,11 @@ export const migrate = async (server, fromSchema, toSchema, transform) => {
|
|
|
90
91
|
let i = 0;
|
|
91
92
|
let rangesToMigrate = [];
|
|
92
93
|
await save(server, false, false, true);
|
|
93
|
-
server.
|
|
94
|
-
const [
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
rangesToMigrate.push(
|
|
94
|
+
server.verifTree.foreachBlock((block) => {
|
|
95
|
+
const [typeId, start] = destructureTreeKey(block.key);
|
|
96
|
+
const def = server.schemaTypesParsedById[typeId];
|
|
97
|
+
const end = start + def.blockCapacity - 1;
|
|
98
|
+
rangesToMigrate.push({ typeId, start, end });
|
|
98
99
|
});
|
|
99
100
|
await waitUntilSleeping(workerState);
|
|
100
101
|
while (i < rangesToMigrate.length) {
|
|
@@ -108,12 +109,13 @@ export const migrate = async (server, fromSchema, toSchema, transform) => {
|
|
|
108
109
|
setToAwake(workerState, true);
|
|
109
110
|
await waitUntilSleeping(workerState);
|
|
110
111
|
// exec queued modifies
|
|
112
|
+
server.processingQueries--;
|
|
111
113
|
server.onQueryEnd();
|
|
112
114
|
if (i === rangesToMigrate.length) {
|
|
113
115
|
if (server.dirtyRanges.size) {
|
|
114
116
|
rangesToMigrate = [];
|
|
115
117
|
i = 0;
|
|
116
|
-
|
|
118
|
+
foreachDirtyBlock(server, (_mtKey, typeId, start, end) => {
|
|
117
119
|
rangesToMigrate.push({
|
|
118
120
|
typeId,
|
|
119
121
|
start,
|
|
@@ -140,6 +142,7 @@ export const migrate = async (server, fromSchema, toSchema, transform) => {
|
|
|
140
142
|
// pass last node IDS { type: lastId }
|
|
141
143
|
setSchemaOnServer(server, toSchema);
|
|
142
144
|
for (const key in schemaTypesParsed) {
|
|
145
|
+
// maybe only send the lastId
|
|
143
146
|
const def = server.schemaTypesParsed[key];
|
|
144
147
|
def.lastId = schemaTypesParsed[key].lastId;
|
|
145
148
|
}
|
|
@@ -152,6 +155,7 @@ export const migrate = async (server, fromSchema, toSchema, transform) => {
|
|
|
152
155
|
if (abort()) {
|
|
153
156
|
return;
|
|
154
157
|
}
|
|
158
|
+
native.membarSyncRead();
|
|
155
159
|
await save(server, false, true, true);
|
|
156
160
|
await writeSchemaFile(server, toSchema);
|
|
157
161
|
server.migrating = 0;
|
|
@@ -13,6 +13,8 @@ else if (workerData?.isDbMigrateWorker) {
|
|
|
13
13
|
const { from, to, fromSchema, toSchema, channel, workerState, transformFns } = workerData;
|
|
14
14
|
const fromCtx = native.externalFromInt(from);
|
|
15
15
|
const toCtx = native.externalFromInt(to);
|
|
16
|
+
// worker ctx init - maybe instead of this just add it on native
|
|
17
|
+
// instead of here just do this in native.js
|
|
16
18
|
native.workerCtxInit();
|
|
17
19
|
const fromDb = new BasedDb({ path: null });
|
|
18
20
|
const toDb = new BasedDb({ path: null });
|
|
@@ -109,6 +111,7 @@ else if (workerData?.isDbMigrateWorker) {
|
|
|
109
111
|
}
|
|
110
112
|
}
|
|
111
113
|
await toDb.drain();
|
|
114
|
+
native.membarSyncWrite();
|
|
112
115
|
// WE ARE ONLY GOING TO SEND { type: lastNodeId }
|
|
113
116
|
channel.postMessage(cp(toDb.server.schemaTypesParsed));
|
|
114
117
|
setToSleep(workerState);
|
|
@@ -1,4 +1,10 @@
|
|
|
1
1
|
import { DbServer } from './index.js';
|
|
2
|
+
type RangeDump = {
|
|
3
|
+
file: string;
|
|
4
|
+
hash: string;
|
|
5
|
+
start: number;
|
|
6
|
+
end: number;
|
|
7
|
+
};
|
|
2
8
|
export type Writelog = {
|
|
3
9
|
ts: number;
|
|
4
10
|
types: {
|
|
@@ -10,12 +16,8 @@ export type Writelog = {
|
|
|
10
16
|
hash: string;
|
|
11
17
|
commonDump: string;
|
|
12
18
|
rangeDumps: {
|
|
13
|
-
[t: number]:
|
|
14
|
-
file: string;
|
|
15
|
-
hash: string;
|
|
16
|
-
start: number;
|
|
17
|
-
end: number;
|
|
18
|
-
}[];
|
|
19
|
+
[t: number]: RangeDump[];
|
|
19
20
|
};
|
|
20
21
|
};
|
|
21
22
|
export declare function save<T extends boolean>(db: DbServer, sync?: T, forceFullDump?: boolean, skipMigrationCheck?: boolean): T extends true ? void : Promise<void>;
|
|
23
|
+
export {};
|