@based/db 0.0.66 → 0.0.68
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/lib/darwin_aarch64/include/selva/colvec.h +71 -0
- package/dist/lib/darwin_aarch64/include/selva/db.h +3 -0
- package/dist/lib/darwin_aarch64/include/selva/fields.h +1 -1
- package/dist/lib/darwin_aarch64/include/selva/types.h +7 -0
- package/dist/lib/darwin_aarch64/libdeflate.dylib +0 -0
- package/dist/lib/darwin_aarch64/libjemalloc_selva.2.dylib +0 -0
- package/dist/lib/darwin_aarch64/libjemalloc_selva.so.2 +0 -0
- package/dist/lib/darwin_aarch64/libnode-v20.node +0 -0
- package/dist/lib/darwin_aarch64/libnode-v21.node +0 -0
- package/dist/lib/darwin_aarch64/libnode-v22.node +0 -0
- package/dist/lib/darwin_aarch64/libnode-v23.node +0 -0
- package/dist/lib/darwin_aarch64/libnode-v24.node +0 -0
- package/dist/lib/darwin_aarch64/libselva.dylib +0 -0
- package/dist/lib/darwin_aarch64/libxxhash.dylib +0 -0
- package/dist/lib/linux_aarch64/include/selva/colvec.h +71 -0
- package/dist/lib/linux_aarch64/include/selva/db.h +3 -0
- package/dist/lib/linux_aarch64/include/selva/fields.h +1 -1
- package/dist/lib/linux_aarch64/include/selva/types.h +7 -0
- package/dist/lib/linux_aarch64/libnode-v20.node +0 -0
- package/dist/lib/linux_aarch64/libnode-v21.node +0 -0
- package/dist/lib/linux_aarch64/libnode-v22.node +0 -0
- package/dist/lib/linux_aarch64/libnode-v23.node +0 -0
- package/dist/lib/linux_aarch64/libnode-v24.node +0 -0
- package/dist/lib/linux_aarch64/libselva.so +0 -0
- package/dist/lib/linux_x86_64/include/selva/colvec.h +71 -0
- package/dist/lib/linux_x86_64/include/selva/db.h +3 -0
- package/dist/lib/linux_x86_64/include/selva/fields.h +1 -1
- package/dist/lib/linux_x86_64/include/selva/types.h +7 -0
- package/dist/lib/linux_x86_64/libnode-v20.node +0 -0
- package/dist/lib/linux_x86_64/libnode-v21.node +0 -0
- package/dist/lib/linux_x86_64/libnode-v22.node +0 -0
- package/dist/lib/linux_x86_64/libnode-v23.node +0 -0
- package/dist/lib/linux_x86_64/libnode-v24.node +0 -0
- package/dist/lib/linux_x86_64/libselva.so +0 -0
- package/dist/src/client/flushModify.d.ts +2 -1
- package/dist/src/client/flushModify.js +12 -4
- package/dist/src/client/modify/create.js +11 -0
- package/dist/src/client/modify/delete.js +3 -0
- package/dist/src/client/modify/modify.js +2 -2
- package/dist/src/client/modify/setCursor.d.ts +2 -1
- package/dist/src/client/query/BasedDbQuery.d.ts +7 -2
- package/dist/src/client/query/BasedDbQuery.js +111 -14
- package/dist/src/client/query/aggregates/aggregation.js +24 -11
- package/dist/src/client/query/aggregates/types.d.ts +21 -2
- package/dist/src/client/query/aggregates/types.js +34 -1
- package/dist/src/client/query/display.js +8 -2
- package/dist/src/client/query/filter/createVariableFilterBuffer.d.ts +2 -3
- package/dist/src/client/query/filter/createVariableFilterBuffer.js +20 -7
- package/dist/src/client/query/filter/filter.js +13 -3
- package/dist/src/client/query/filter/primitiveFilter.d.ts +1 -2
- package/dist/src/client/query/include/props.js +18 -2
- package/dist/src/client/query/include/toBuffer.js +11 -3
- package/dist/src/client/query/include/walk.js +5 -1
- package/dist/src/client/query/queryDef.js +4 -1
- package/dist/src/client/query/read/read.js +50 -23
- package/dist/src/client/query/registerQuery.js +1 -0
- package/dist/src/client/query/search/index.d.ts +1 -1
- package/dist/src/client/query/search/index.js +21 -7
- package/dist/src/client/query/sort.d.ts +1 -1
- package/dist/src/client/query/toByteCode/default.d.ts +1 -1
- package/dist/src/client/query/toByteCode/default.js +0 -2
- package/dist/src/client/query/toByteCode/toBuffer.js +0 -7
- package/dist/src/client/query/types.d.ts +16 -5
- package/dist/src/client/query/validation.js +25 -2
- package/dist/src/client/xxHash64.d.ts +1 -1
- package/dist/src/index.d.ts +0 -1
- package/dist/src/index.js +0 -1
- package/dist/src/native.d.ts +2 -2
- package/dist/src/native.js +7 -8
- package/dist/src/server/IoWorker.d.ts +8 -0
- package/dist/src/server/IoWorker.js +39 -0
- package/dist/src/server/QueryWorker.d.ts +8 -0
- package/dist/src/server/QueryWorker.js +26 -0
- package/dist/src/server/blocks.d.ts +24 -0
- package/dist/src/server/blocks.js +112 -0
- package/dist/src/server/dbHash.d.ts +1 -1
- package/dist/src/server/index.d.ts +9 -16
- package/dist/src/server/index.js +37 -15
- package/dist/src/server/migrate/index.d.ts +5 -0
- package/dist/src/server/migrate/index.js +10 -7
- package/dist/src/server/save.d.ts +8 -6
- package/dist/src/server/save.js +34 -78
- package/dist/src/server/schema.js +6 -5
- package/dist/src/server/start.js +57 -60
- package/dist/src/server/tree.d.ts +24 -13
- package/dist/src/server/tree.js +95 -66
- package/dist/src/server/workers/DbWorker.d.ts +17 -0
- package/dist/src/server/{DbWorker.js → workers/DbWorker.js} +15 -17
- package/dist/src/server/workers/io_worker.js +39 -0
- package/dist/src/server/workers/io_worker_types.d.ts +12 -0
- package/dist/src/server/workers/io_worker_types.js +2 -0
- package/dist/src/server/workers/query_worker.d.ts +1 -0
- package/dist/src/server/workers/query_worker.js +4 -0
- package/dist/src/server/workers/worker.d.ts +1 -0
- package/dist/src/server/workers/worker.js +41 -0
- package/dist/src/shared/Emitter.d.ts +1 -0
- package/dist/src/types.d.ts +1 -1
- package/dist/src/types.js +1 -1
- package/package.json +3 -3
- package/dist/lib/darwin_aarch64/include/selva/history.h +0 -64
- package/dist/lib/linux_aarch64/include/selva/history.h +0 -64
- package/dist/lib/linux_x86_64/include/selva/history.h +0 -64
- package/dist/src/client/query/serialize.d.ts +0 -4
- package/dist/src/client/query/serialize.js +0 -26
- package/dist/src/server/DbWorker.d.ts +0 -13
- package/dist/src/server/csmt/draw-dot.d.ts +0 -4
- package/dist/src/server/csmt/draw-dot.js +0 -38
- package/dist/src/server/csmt/index.d.ts +0 -4
- package/dist/src/server/csmt/index.js +0 -5
- package/dist/src/server/csmt/match.d.ts +0 -7
- package/dist/src/server/csmt/match.js +0 -10
- package/dist/src/server/csmt/memebership-proof.d.ts +0 -7
- package/dist/src/server/csmt/memebership-proof.js +0 -122
- package/dist/src/server/csmt/tree-utils.d.ts +0 -6
- package/dist/src/server/csmt/tree-utils.js +0 -33
- package/dist/src/server/csmt/tree.d.ts +0 -3
- package/dist/src/server/csmt/tree.js +0 -270
- package/dist/src/server/csmt/types.d.ts +0 -46
- package/dist/src/server/csmt/types.js +0 -2
- package/dist/src/server/worker.js +0 -33
- /package/dist/src/server/{worker.d.ts → workers/io_worker.d.ts} +0 -0
package/dist/src/server/start.js
CHANGED
|
@@ -1,15 +1,27 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { availableParallelism } from 'node:os';
|
|
2
|
+
import { QueryWorker } from './QueryWorker.js';
|
|
3
|
+
import { IoWorker } from './IoWorker.js';
|
|
2
4
|
import native from '../native.js';
|
|
3
5
|
import { rm, mkdir, readFile } from 'node:fs/promises';
|
|
4
6
|
import { join } from 'node:path';
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
+
import { VerifTree, makeTreeKey } from './tree.js';
|
|
8
|
+
import { foreachBlock } from './blocks.js';
|
|
7
9
|
import exitHook from 'exit-hook';
|
|
8
10
|
import { save } from './save.js';
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
+
import { deSerialize } from '@based/schema';
|
|
12
|
+
import { BLOCK_CAPACITY_DEFAULT } from '@based/schema/def';
|
|
13
|
+
import { bufToHex, equals, hexToBuf, wait } from '@saulx/utils';
|
|
11
14
|
import { SCHEMA_FILE, WRITELOG_FILE } from '../types.js';
|
|
12
15
|
import { setSchemaOnServer } from './schema.js';
|
|
16
|
+
function startWorkers(db, opts) {
|
|
17
|
+
const queryThreads = opts?.queryThreads ?? availableParallelism();
|
|
18
|
+
const address = native.intFromExternal(db.dbCtxExternal);
|
|
19
|
+
db.workers = [];
|
|
20
|
+
for (let i = 0; i < queryThreads; i++) {
|
|
21
|
+
db.workers.push(new QueryWorker(address, db, i));
|
|
22
|
+
}
|
|
23
|
+
db.ioWorker = new IoWorker(address, db);
|
|
24
|
+
}
|
|
13
25
|
export async function start(db, opts) {
|
|
14
26
|
const path = db.fileSystemPath;
|
|
15
27
|
const noop = () => { };
|
|
@@ -19,6 +31,7 @@ export async function start(db, opts) {
|
|
|
19
31
|
await mkdir(path, { recursive: true }).catch(noop);
|
|
20
32
|
db.dbCtxExternal = native.start();
|
|
21
33
|
let writelog = null;
|
|
34
|
+
let partials = []; // Blocks that exists but were not loaded [key, hash]
|
|
22
35
|
try {
|
|
23
36
|
writelog = JSON.parse((await readFile(join(path, WRITELOG_FILE))).toString());
|
|
24
37
|
// Load the common dump
|
|
@@ -29,81 +42,66 @@ export async function start(db, opts) {
|
|
|
29
42
|
console.error(e.message);
|
|
30
43
|
throw e;
|
|
31
44
|
}
|
|
45
|
+
const schema = await readFile(join(path, SCHEMA_FILE));
|
|
46
|
+
if (schema) {
|
|
47
|
+
const s = deSerialize(schema);
|
|
48
|
+
setSchemaOnServer(db, s);
|
|
49
|
+
}
|
|
32
50
|
// Load all range dumps
|
|
33
51
|
for (const typeId in writelog.rangeDumps) {
|
|
34
52
|
const dumps = writelog.rangeDumps[typeId];
|
|
53
|
+
const def = db.schemaTypesParsedById[typeId];
|
|
35
54
|
for (const dump of dumps) {
|
|
36
55
|
const fname = dump.file;
|
|
37
56
|
if (fname?.length > 0) {
|
|
38
|
-
|
|
39
|
-
|
|
57
|
+
if (!def.partial) {
|
|
58
|
+
try {
|
|
59
|
+
// Can't use loadBlock() yet because verifTree is not avail
|
|
60
|
+
native.loadBlock(join(path, fname), db.dbCtxExternal);
|
|
61
|
+
}
|
|
62
|
+
catch (e) {
|
|
63
|
+
console.error(e.message);
|
|
64
|
+
}
|
|
40
65
|
}
|
|
41
|
-
|
|
42
|
-
|
|
66
|
+
else {
|
|
67
|
+
partials.push([
|
|
68
|
+
makeTreeKey(def.id, dump.start),
|
|
69
|
+
hexToBuf(dump.hash),
|
|
70
|
+
]);
|
|
43
71
|
}
|
|
44
72
|
}
|
|
45
73
|
}
|
|
46
74
|
}
|
|
47
|
-
const schema = await readFile(join(path, SCHEMA_FILE));
|
|
48
|
-
if (schema) {
|
|
49
|
-
const s = JSON.parse(schema.toString());
|
|
50
|
-
setSchemaOnServer(db, s);
|
|
51
|
-
}
|
|
52
75
|
}
|
|
53
76
|
catch (err) {
|
|
54
77
|
// TODO In some cases we really should give up!
|
|
55
78
|
}
|
|
56
|
-
|
|
57
|
-
for (const
|
|
58
|
-
const def =
|
|
79
|
+
db.verifTree = new VerifTree(db.schemaTypesParsed);
|
|
80
|
+
for (const { typeId } of db.verifTree.types()) {
|
|
81
|
+
const def = db.schemaTypesParsedById[typeId];
|
|
59
82
|
const [total, lastId] = native.getTypeInfo(def.id, db.dbCtxExternal);
|
|
60
83
|
def.lastId = writelog?.types[def.id]?.lastId || lastId;
|
|
61
84
|
def.blockCapacity =
|
|
62
|
-
writelog?.types[def.id]?.blockCapacity ||
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
db.merkleTree.insert(mtKey, hash, data);
|
|
74
|
-
}, true);
|
|
85
|
+
writelog?.types[def.id]?.blockCapacity ||
|
|
86
|
+
def.blockCapacity ||
|
|
87
|
+
BLOCK_CAPACITY_DEFAULT;
|
|
88
|
+
foreachBlock(db, def, (start, _end, hash) => {
|
|
89
|
+
const mtKey = makeTreeKey(def.id, start);
|
|
90
|
+
db.verifTree.update(mtKey, hash);
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
// Insert partials to make the hash match
|
|
94
|
+
for (const [key, hash] of partials) {
|
|
95
|
+
db.verifTree.update(key, hash, false);
|
|
75
96
|
}
|
|
76
97
|
if (writelog?.hash) {
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
// console.error(
|
|
82
|
-
// `WARN: CSMT hash mismatch. expected: ${writelog.hash} actual: ${bufToHex(newHash)}`,
|
|
83
|
-
// )
|
|
84
|
-
//}
|
|
85
|
-
const oldHashSet = new Set();
|
|
86
|
-
const newHashSet = new Set();
|
|
87
|
-
for (let k in writelog.rangeDumps)
|
|
88
|
-
writelog.rangeDumps[k].forEach(({ hash }) => oldHashSet.add(hash));
|
|
89
|
-
db.merkleTree.visitLeafNodes(({ key, hash }) => {
|
|
90
|
-
const [_typeId, start] = destructureCsmtKey(key);
|
|
91
|
-
if (start == specialBlock)
|
|
92
|
-
return; // skip the type specialBlock
|
|
93
|
-
newHashSet.add(bufToHex(hash));
|
|
94
|
-
});
|
|
95
|
-
const setEq = (a, b) => a.size === b.size && [...a].every((value) => b.has(value));
|
|
96
|
-
if (!setEq(oldHashSet, newHashSet)) {
|
|
97
|
-
console.error(`WARN: CSMT hash mismatch.`);
|
|
98
|
+
const oldHash = hexToBuf(writelog.hash);
|
|
99
|
+
const newHash = db.verifTree.hash;
|
|
100
|
+
if (!equals(oldHash, newHash)) {
|
|
101
|
+
console.error(`WARN: DB hash mismatch. expected: ${writelog.hash} actual: ${bufToHex(newHash)}`);
|
|
98
102
|
}
|
|
99
103
|
}
|
|
100
|
-
|
|
101
|
-
const queryThreads = opts?.queryThreads ?? availableParallelism();
|
|
102
|
-
const address = native.intFromExternal(db.dbCtxExternal);
|
|
103
|
-
db.workers = [];
|
|
104
|
-
for (let i = 0; i < queryThreads; i++) {
|
|
105
|
-
db.workers.push(new DbWorker(address, db, i));
|
|
106
|
-
}
|
|
104
|
+
startWorkers(db, opts);
|
|
107
105
|
if (!opts?.hosted) {
|
|
108
106
|
db.unlistenExit = exitHook((signal) => {
|
|
109
107
|
const blockSig = () => { };
|
|
@@ -118,9 +116,8 @@ export async function start(db, opts) {
|
|
|
118
116
|
signals.forEach((sig) => process.off(sig, blockSig));
|
|
119
117
|
});
|
|
120
118
|
}
|
|
121
|
-
const d = performance.now();
|
|
122
119
|
await Promise.all(db.workers.map(({ readyPromise }) => readyPromise));
|
|
123
|
-
db.emit('info',
|
|
120
|
+
db.emit('info', 'All workers ready');
|
|
124
121
|
// use timeout
|
|
125
122
|
if (db.saveIntervalInSeconds > 0) {
|
|
126
123
|
db.saveInterval ??= setInterval(() => {
|
|
@@ -1,15 +1,26 @@
|
|
|
1
|
-
import { DbServer } from './index.js';
|
|
2
1
|
import { SchemaTypeDef } from '@based/schema/def';
|
|
3
|
-
export
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
2
|
+
export declare const destructureTreeKey: (key: number) => number[];
|
|
3
|
+
export declare const makeTreeKey: (typeId: number, start: number) => number;
|
|
4
|
+
export declare const nodeId2Start: (blockCapacity: number, nodeId: number) => number;
|
|
5
|
+
export declare const makeTreeKeyFromNodeId: (typeId: number, blockCapacity: number, nodeId: number) => number;
|
|
6
|
+
type Hash = Uint8Array;
|
|
7
|
+
export type VerifBlock = {
|
|
8
|
+
key: number;
|
|
9
|
+
hash: Hash;
|
|
10
|
+
inmem: boolean;
|
|
11
|
+
loadPromise: null | Promise<void>;
|
|
8
12
|
};
|
|
9
|
-
export declare
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
13
|
+
export declare class VerifTree {
|
|
14
|
+
#private;
|
|
15
|
+
constructor(schemaTypesParsed: Record<string, SchemaTypeDef>);
|
|
16
|
+
types(): Generator<any, void, unknown>;
|
|
17
|
+
foreachBlock(cb: (block: VerifBlock) => void): void;
|
|
18
|
+
get hash(): Uint8Array;
|
|
19
|
+
update(key: number, hash: Hash, inmem?: boolean): void;
|
|
20
|
+
remove(key: number): void;
|
|
21
|
+
static blockSdbFile(typeId: number, start: number, end: number): string;
|
|
22
|
+
getBlock(key: number): VerifBlock;
|
|
23
|
+
getBlockFile(block: VerifBlock): string;
|
|
24
|
+
updateTypes(schemaTypesParsed: Record<string, SchemaTypeDef>): void;
|
|
25
|
+
}
|
|
26
|
+
export {};
|
package/dist/src/server/tree.js
CHANGED
|
@@ -1,80 +1,109 @@
|
|
|
1
|
-
import
|
|
2
|
-
|
|
3
|
-
// This is a special start id set for every type to somewhat lock the order of the csmt.
|
|
4
|
-
// While the id is valid, it's never a true start id of a block.
|
|
5
|
-
export const specialBlock = 2147483647;
|
|
6
|
-
export const destructureCsmtKey = (key) => [
|
|
1
|
+
import createDbHash from './dbHash.js';
|
|
2
|
+
export const destructureTreeKey = (key) => [
|
|
7
3
|
(key / 4294967296) | 0, // typeId
|
|
8
4
|
(key >>> 31) * 2147483648 + (key & 0x7fffffff), // start_node_id
|
|
9
5
|
];
|
|
10
|
-
export const
|
|
11
|
-
export const
|
|
6
|
+
export const makeTreeKey = (typeId, start) => typeId * 4294967296 + start;
|
|
7
|
+
export const nodeId2Start = (blockCapacity, nodeId) => ((nodeId - +!(nodeId % blockCapacity)) / blockCapacity) | 0;
|
|
8
|
+
const nodeId2BlockI = (nodeId, blockCapacity) => ((nodeId - 1) - ((nodeId - 1) % blockCapacity)) / blockCapacity;
|
|
9
|
+
export const makeTreeKeyFromNodeId = (typeId, blockCapacity, nodeId) => {
|
|
12
10
|
const tmp = nodeId - +!(nodeId % blockCapacity);
|
|
13
11
|
return typeId * 4294967296 + ((tmp / blockCapacity) | 0) * blockCapacity + 1;
|
|
14
12
|
};
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
13
|
+
const HASH_SIZE = 16;
|
|
14
|
+
export class VerifTree {
|
|
15
|
+
#types;
|
|
16
|
+
#h = createDbHash();
|
|
17
|
+
constructor(schemaTypesParsed) {
|
|
18
|
+
this.#types = VerifTree.#makeTypes(schemaTypesParsed);
|
|
19
|
+
}
|
|
20
|
+
static #makeTypes(schemaTypesParsed) {
|
|
21
|
+
return Object.preventExtensions(Object.keys(schemaTypesParsed)
|
|
22
|
+
.sort((a, b) => schemaTypesParsed[a].id - schemaTypesParsed[b].id)
|
|
23
|
+
.reduce((obj, key) => {
|
|
24
|
+
const def = schemaTypesParsed[key];
|
|
25
|
+
const typeId = def.id;
|
|
26
|
+
obj[typeId] = {
|
|
27
|
+
typeId,
|
|
28
|
+
blockCapacity: def.blockCapacity,
|
|
29
|
+
hash: new Uint8Array(HASH_SIZE),
|
|
30
|
+
blocks: [],
|
|
31
|
+
};
|
|
32
|
+
return obj;
|
|
33
|
+
}, {}));
|
|
34
|
+
}
|
|
35
|
+
*types() {
|
|
36
|
+
for (const k of Object.keys(this.#types)) {
|
|
37
|
+
yield this.#types[k];
|
|
36
38
|
}
|
|
37
|
-
catch (_) { }
|
|
38
39
|
}
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
const res = native.getNodeRangeHash(def.id, start, end, hash, db.dbCtxExternal);
|
|
47
|
-
if (res || includeEmptyBlocks) {
|
|
48
|
-
cb(start, end, hash);
|
|
40
|
+
foreachBlock(cb) {
|
|
41
|
+
for (const k of Object.keys(this.#types)) {
|
|
42
|
+
const { blocks } = this.#types[k];
|
|
43
|
+
for (let block of blocks) {
|
|
44
|
+
if (block)
|
|
45
|
+
cb(block);
|
|
46
|
+
}
|
|
49
47
|
}
|
|
50
48
|
}
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
49
|
+
get hash() {
|
|
50
|
+
this.#h.reset();
|
|
51
|
+
this.foreachBlock((block) => this.#h.update(block.hash));
|
|
52
|
+
return this.#h.digest();
|
|
53
|
+
}
|
|
54
|
+
update(key, hash, inmem = true) {
|
|
55
|
+
const [typeId, start] = destructureTreeKey(key);
|
|
56
|
+
const type = this.#types[typeId];
|
|
57
|
+
if (!type) {
|
|
58
|
+
throw new Error(`type ${typeId} not found`);
|
|
59
|
+
}
|
|
60
|
+
const blockI = nodeId2BlockI(start, type.blockCapacity);
|
|
61
|
+
const block = type.blocks[blockI] ?? (type.blocks[blockI] = Object.preventExtensions({ key, hash, inmem, loadPromise: null }));
|
|
62
|
+
block.hash = hash;
|
|
63
|
+
block.inmem = inmem;
|
|
64
|
+
}
|
|
65
|
+
remove(key) {
|
|
66
|
+
const [typeId, start] = destructureTreeKey(key);
|
|
67
|
+
const type = this.#types[typeId];
|
|
68
|
+
if (!type) {
|
|
69
|
+
throw new Error(`type ${typeId} not found`);
|
|
70
|
+
}
|
|
71
|
+
const blockI = nodeId2BlockI(start, type.blockCapacity);
|
|
72
|
+
delete type.blocks[blockI];
|
|
73
|
+
}
|
|
74
|
+
static blockSdbFile(typeId, start, end) {
|
|
75
|
+
return `${typeId}_${start}_${end}.sdb`;
|
|
76
|
+
}
|
|
77
|
+
getBlock(key) {
|
|
78
|
+
const [typeId, start] = destructureTreeKey(key);
|
|
79
|
+
const type = this.#types[typeId];
|
|
80
|
+
if (!type) {
|
|
81
|
+
throw new Error(`type ${typeId} not found`);
|
|
82
|
+
}
|
|
83
|
+
const blockI = nodeId2BlockI(start, type.blockCapacity);
|
|
84
|
+
return type.blocks[blockI];
|
|
58
85
|
}
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
const
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
const
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
86
|
+
getBlockFile(block) {
|
|
87
|
+
const [typeId, start] = destructureTreeKey(block.key);
|
|
88
|
+
const type = this.#types[typeId];
|
|
89
|
+
if (!type) {
|
|
90
|
+
throw new Error(`type ${typeId} not found`);
|
|
91
|
+
}
|
|
92
|
+
const end = start + type.blockCapacity - 1;
|
|
93
|
+
return VerifTree.blockSdbFile(typeId, start, end);
|
|
94
|
+
}
|
|
95
|
+
updateTypes(schemaTypesParsed) {
|
|
96
|
+
const oldTypes = this.#types;
|
|
97
|
+
const newTypes = VerifTree.#makeTypes(schemaTypesParsed);
|
|
98
|
+
for (const k of Object.keys(oldTypes)) {
|
|
99
|
+
const oldType = oldTypes[k];
|
|
100
|
+
const newType = newTypes[k];
|
|
101
|
+
if (newType) {
|
|
102
|
+
newType.hash = oldType.hash;
|
|
103
|
+
newType.blocks = oldType.blocks;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
this.#types = newTypes;
|
|
78
107
|
}
|
|
79
108
|
}
|
|
80
109
|
//# sourceMappingURL=tree.js.map
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { DbServer } from '../index.js';
|
|
2
|
+
export declare abstract class DbWorker {
|
|
3
|
+
constructor(address: BigInt, db: DbServer, onExit: (code: number) => void, workerName: string);
|
|
4
|
+
protected db: DbServer;
|
|
5
|
+
private channel;
|
|
6
|
+
private worker;
|
|
7
|
+
protected resolvers: ((x: any) => any)[];
|
|
8
|
+
readyPromise: Promise<true>;
|
|
9
|
+
terminate(): Promise<number>;
|
|
10
|
+
abstract handleMsg(buf: any): void;
|
|
11
|
+
protected callback: (resolve: (x: any) => any) => void;
|
|
12
|
+
/**
|
|
13
|
+
* Send msg to the worker thread and return a promise to the response.
|
|
14
|
+
*/
|
|
15
|
+
protected call(msg: any): Promise<Uint8Array>;
|
|
16
|
+
updateCtx(address: BigInt): Promise<void>;
|
|
17
|
+
}
|
|
@@ -1,16 +1,14 @@
|
|
|
1
1
|
import { MessageChannel, Worker } from 'node:worker_threads';
|
|
2
2
|
import { fileURLToPath } from 'node:url';
|
|
3
3
|
import { dirname, join } from 'node:path';
|
|
4
|
-
import { readUint64 } from '@saulx/utils';
|
|
5
4
|
const __filename = fileURLToPath(import.meta.url);
|
|
6
5
|
const __dirname = dirname(__filename);
|
|
7
|
-
const workerPath = join(__dirname, 'worker.js');
|
|
8
6
|
export class DbWorker {
|
|
9
|
-
constructor(address, db,
|
|
7
|
+
constructor(address, db, onExit, workerName) {
|
|
10
8
|
const { port1, port2 } = new MessageChannel();
|
|
11
9
|
this.db = db;
|
|
12
10
|
this.channel = port1;
|
|
13
|
-
this.worker = new Worker(
|
|
11
|
+
this.worker = new Worker(join(__dirname, workerName), {
|
|
14
12
|
workerData: {
|
|
15
13
|
isDbWorker: true,
|
|
16
14
|
channel: port2,
|
|
@@ -41,12 +39,12 @@ export class DbWorker {
|
|
|
41
39
|
resolve(err);
|
|
42
40
|
}
|
|
43
41
|
this.resolvers = [];
|
|
44
|
-
|
|
42
|
+
onExit(code);
|
|
45
43
|
}
|
|
46
44
|
});
|
|
47
|
-
|
|
45
|
+
this.channel.on('message', (buf) => {
|
|
48
46
|
this.resolvers.shift()(new Uint8Array(buf));
|
|
49
|
-
this.
|
|
47
|
+
this.handleMsg(buf);
|
|
50
48
|
});
|
|
51
49
|
}
|
|
52
50
|
db;
|
|
@@ -54,21 +52,21 @@ export class DbWorker {
|
|
|
54
52
|
worker;
|
|
55
53
|
resolvers = [];
|
|
56
54
|
readyPromise;
|
|
55
|
+
async terminate() {
|
|
56
|
+
return this.worker.terminate();
|
|
57
|
+
}
|
|
57
58
|
callback = (resolve) => {
|
|
58
|
-
this.db.processingQueries++;
|
|
59
59
|
this.resolvers.push(resolve);
|
|
60
60
|
};
|
|
61
|
-
|
|
62
|
-
|
|
61
|
+
/**
|
|
62
|
+
* Send msg to the worker thread and return a promise to the response.
|
|
63
|
+
*/
|
|
64
|
+
call(msg) {
|
|
65
|
+
this.channel.postMessage(msg);
|
|
63
66
|
return new Promise(this.callback);
|
|
64
67
|
}
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
if (schemaChecksum !== this.db.schema?.hash) {
|
|
68
|
-
return Promise.resolve(new Uint8Array(1));
|
|
69
|
-
}
|
|
70
|
-
this.channel.postMessage(buf);
|
|
71
|
-
return new Promise(this.callback);
|
|
68
|
+
updateCtx(address) {
|
|
69
|
+
return this.call(address);
|
|
72
70
|
}
|
|
73
71
|
}
|
|
74
72
|
//# sourceMappingURL=DbWorker.js.map
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { registerMsgHandler } from './worker.js';
|
|
2
|
+
import { ENCODER, writeInt32 } from '@saulx/utils';
|
|
3
|
+
import native from '../../native.js';
|
|
4
|
+
function loadBlock(dbCtx, filepath) {
|
|
5
|
+
try {
|
|
6
|
+
native.loadBlock(filepath, dbCtx);
|
|
7
|
+
}
|
|
8
|
+
catch (e) {
|
|
9
|
+
// need to get rid of the shared buffer
|
|
10
|
+
return new Uint8Array(ENCODER.encode(e.toString())).buffer;
|
|
11
|
+
}
|
|
12
|
+
return null;
|
|
13
|
+
}
|
|
14
|
+
function unloadBlock(dbCtx, filepath, typeId, start) {
|
|
15
|
+
const buf = new ArrayBuffer(20); // [[4 bytes err], [16 bytes hash]]
|
|
16
|
+
const hash = new Uint8Array(buf, 4);
|
|
17
|
+
const err = native.saveBlock(filepath, typeId, start, dbCtx, hash);
|
|
18
|
+
if (err) {
|
|
19
|
+
const errCodeBuf = new Uint8Array(buf, 0, 4);
|
|
20
|
+
writeInt32(errCodeBuf, err, 0);
|
|
21
|
+
}
|
|
22
|
+
else {
|
|
23
|
+
native.delBlock(dbCtx, typeId, start);
|
|
24
|
+
}
|
|
25
|
+
return buf;
|
|
26
|
+
}
|
|
27
|
+
registerMsgHandler((dbCtx, msg) => {
|
|
28
|
+
if (typeof msg?.type === 'string') {
|
|
29
|
+
const job = msg;
|
|
30
|
+
if (job.type === 'load') {
|
|
31
|
+
return loadBlock(dbCtx, job.filepath);
|
|
32
|
+
}
|
|
33
|
+
else if (job.type === 'unload') {
|
|
34
|
+
return unloadBlock(dbCtx, job.filepath, job.typeId, job.start);
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
return null;
|
|
38
|
+
});
|
|
39
|
+
//# sourceMappingURL=io_worker.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function registerMsgHandler(onMsg: (dbCtx: any, msg: any) => Uint8Array<ArrayBufferLike> | ArrayBuffer | null): void;
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { isMainThread, parentPort, workerData } from 'node:worker_threads';
|
|
2
|
+
import native from '../../native.js';
|
|
3
|
+
let dbCtx;
|
|
4
|
+
let wCtx; // This must be held until the worker exits otherwise the ctx will be autofreed instantly
|
|
5
|
+
if (isMainThread) {
|
|
6
|
+
console.warn(`running a worker in the mainthread - incorrect`);
|
|
7
|
+
}
|
|
8
|
+
else if (workerData?.isDbWorker) {
|
|
9
|
+
let { address } = workerData;
|
|
10
|
+
dbCtx = native.externalFromInt(address);
|
|
11
|
+
wCtx = native.workerCtxInit();
|
|
12
|
+
}
|
|
13
|
+
else {
|
|
14
|
+
console.info('incorrect worker db query');
|
|
15
|
+
}
|
|
16
|
+
export function registerMsgHandler(onMsg) {
|
|
17
|
+
if (!workerData?.isDbWorker) {
|
|
18
|
+
throw new Error('Not a DbWorker');
|
|
19
|
+
}
|
|
20
|
+
let { channel } = workerData;
|
|
21
|
+
const handleMsg = (msg) => {
|
|
22
|
+
try {
|
|
23
|
+
if (typeof msg === 'bigint') {
|
|
24
|
+
// it's a ctx address
|
|
25
|
+
dbCtx = native.externalFromInt(msg);
|
|
26
|
+
channel.postMessage(null);
|
|
27
|
+
}
|
|
28
|
+
else {
|
|
29
|
+
// a message to the worker handler
|
|
30
|
+
const arrayBuf = onMsg(dbCtx, msg);
|
|
31
|
+
channel.postMessage(arrayBuf, [arrayBuf]);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
catch (e) {
|
|
35
|
+
channel.postMessage(e);
|
|
36
|
+
}
|
|
37
|
+
};
|
|
38
|
+
channel.on('message', handleMsg);
|
|
39
|
+
parentPort.postMessage('READY');
|
|
40
|
+
}
|
|
41
|
+
//# sourceMappingURL=worker.js.map
|
package/dist/src/types.d.ts
CHANGED
package/dist/src/types.js
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@based/db",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.68",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/src/index.js",
|
|
@@ -38,9 +38,9 @@
|
|
|
38
38
|
"basedDbNative.cjs"
|
|
39
39
|
],
|
|
40
40
|
"dependencies": {
|
|
41
|
-
"@based/schema": "5.0.0-alpha.
|
|
41
|
+
"@based/schema": "5.0.0-alpha.23",
|
|
42
42
|
"@saulx/hash": "^3.0.0",
|
|
43
|
-
"@saulx/utils": "^6.7.
|
|
43
|
+
"@saulx/utils": "^6.7.2",
|
|
44
44
|
"exit-hook": "^4.0.0",
|
|
45
45
|
"picocolors": "^1.1.0",
|
|
46
46
|
"@based/crc32c": "^1.0.0"
|