@based/db 0.0.21 → 0.0.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -10
- package/dist/lib/darwin_aarch64/include/selva/types.h +9 -18
- package/dist/lib/darwin_aarch64/libnode-v20.node +0 -0
- package/dist/lib/darwin_aarch64/libnode-v21.node +0 -0
- package/dist/lib/darwin_aarch64/libnode-v22.node +0 -0
- package/dist/lib/darwin_aarch64/libnode-v23.node +0 -0
- package/dist/lib/darwin_aarch64/libselva.dylib +0 -0
- package/dist/lib/linux_aarch64/include/selva/types.h +9 -18
- package/dist/lib/linux_aarch64/libjemalloc_selva.so.2 +0 -0
- package/dist/lib/linux_aarch64/libnode-v20.node +0 -0
- package/dist/lib/linux_aarch64/libnode-v21.node +0 -0
- package/dist/lib/linux_aarch64/libnode-v22.node +0 -0
- package/dist/lib/linux_aarch64/libnode-v23.node +0 -0
- package/dist/lib/linux_aarch64/libselva.so +0 -0
- package/dist/lib/linux_x86_64/include/selva/types.h +9 -18
- package/dist/lib/linux_x86_64/libjemalloc_selva.so.2 +0 -0
- package/dist/lib/linux_x86_64/libnode-v20.node +0 -0
- package/dist/lib/linux_x86_64/libnode-v21.node +0 -0
- package/dist/lib/linux_x86_64/libnode-v22.node +0 -0
- package/dist/lib/linux_x86_64/libnode-v23.node +0 -0
- package/dist/lib/linux_x86_64/libselva.so +0 -0
- package/dist/src/client/bitWise.d.ts +0 -1
- package/dist/src/client/bitWise.js +0 -10
- package/dist/src/client/{operations.d.ts → flushModify.d.ts} +4 -4
- package/dist/src/client/{operations.js → flushModify.js} +39 -15
- package/dist/src/client/index.d.ts +10 -6
- package/dist/src/client/index.js +12 -4
- package/dist/src/client/modify/ModifyRes.d.ts +2 -8
- package/dist/src/client/modify/ModifyRes.js +23 -29
- package/dist/src/client/modify/alias.d.ts +1 -1
- package/dist/src/client/modify/alias.js +5 -2
- package/dist/src/client/modify/binary.d.ts +1 -1
- package/dist/src/client/modify/binary.js +1 -1
- package/dist/src/client/modify/cardinality.d.ts +2 -2
- package/dist/src/client/modify/cardinality.js +7 -5
- package/dist/src/client/modify/create.js +10 -5
- package/dist/src/client/modify/delete.d.ts +3 -1
- package/dist/src/client/modify/delete.js +12 -5
- package/dist/src/client/modify/expire.js +1 -1
- package/dist/src/client/modify/fixed.js +11 -4
- package/dist/src/client/modify/references/appendRefs.d.ts +4 -0
- package/dist/src/client/modify/references/appendRefs.js +27 -0
- package/dist/src/client/modify/references/edge.d.ts +0 -1
- package/dist/src/client/modify/references/edge.js +191 -71
- package/dist/src/client/modify/references/getEdgeSize.d.ts +3 -0
- package/dist/src/client/modify/references/getEdgeSize.js +27 -0
- package/dist/src/client/modify/references/reference.js +40 -26
- package/dist/src/client/modify/references/references.js +18 -6
- package/dist/src/client/modify/string.js +2 -1
- package/dist/src/client/modify/types.d.ts +1 -0
- package/dist/src/client/modify/types.js +1 -0
- package/dist/src/client/modify/update.js +3 -4
- package/dist/src/client/modify/upsert.js +1 -0
- package/dist/src/client/modify/vector.js +0 -2
- package/dist/src/client/query/BasedDbQuery.d.ts +4 -3
- package/dist/src/client/query/BasedDbQuery.js +12 -3
- package/dist/src/client/query/aggregationFn.d.ts +3 -0
- package/dist/src/client/query/aggregationFn.js +9 -0
- package/dist/src/client/query/debug.js +2 -6
- package/dist/src/client/query/display.js +5 -2
- package/dist/src/client/query/filter/createFixedFilterBuffer.d.ts +2 -2
- package/dist/src/client/query/filter/createFixedFilterBuffer.js +34 -26
- package/dist/src/client/query/filter/createReferenceFilter.d.ts +1 -1
- package/dist/src/client/query/filter/createReferenceFilter.js +17 -7
- package/dist/src/client/query/filter/createVariableFilterBuffer.d.ts +1 -1
- package/dist/src/client/query/filter/createVariableFilterBuffer.js +45 -22
- package/dist/src/client/query/filter/filter.d.ts +1 -1
- package/dist/src/client/query/filter/filter.js +5 -0
- package/dist/src/client/query/filter/parseFilterValue.js +1 -1
- package/dist/src/client/query/filter/toBuffer.d.ts +2 -2
- package/dist/src/client/query/filter/toBuffer.js +16 -9
- package/dist/src/client/query/include/props.js +18 -1
- package/dist/src/client/query/include/toBuffer.d.ts +1 -1
- package/dist/src/client/query/include/toBuffer.js +22 -12
- package/dist/src/client/query/include/walk.js +2 -1
- package/dist/src/client/query/query.d.ts +1 -0
- package/dist/src/client/query/query.js +1 -0
- package/dist/src/client/query/queryDef.js +2 -1
- package/dist/src/client/query/read/read.d.ts +2 -1
- package/dist/src/client/query/read/read.js +95 -62
- package/dist/src/client/query/registerQuery.d.ts +1 -1
- package/dist/src/client/query/registerQuery.js +2 -1
- package/dist/src/client/query/search/index.d.ts +1 -1
- package/dist/src/client/query/search/index.js +56 -24
- package/dist/src/client/query/subscription/markers.js +2 -1
- package/dist/src/client/query/toBuffer.d.ts +1 -1
- package/dist/src/client/query/toBuffer.js +80 -32
- package/dist/src/client/query/types.d.ts +25 -3
- package/dist/src/client/query/types.js +6 -0
- package/dist/src/client/query/validation.js +1 -1
- package/dist/src/client/string.d.ts +1 -1
- package/dist/src/client/string.js +1 -2
- package/dist/src/client/xxHash64.d.ts +1 -1
- package/dist/src/index.d.ts +4 -3
- package/dist/src/index.js +20 -11
- package/dist/src/native.d.ts +8 -13
- package/dist/src/native.js +9 -31
- package/dist/src/server/dbHash.d.ts +5 -0
- package/dist/src/server/dbHash.js +27 -0
- package/dist/src/server/index.d.ts +10 -10
- package/dist/src/server/index.js +54 -35
- package/dist/src/server/migrate/index.js +1 -1
- package/dist/src/server/migrate/worker.js +2 -2
- package/dist/src/server/save.d.ts +1 -1
- package/dist/src/server/save.js +10 -10
- package/dist/src/server/start.js +4 -2
- package/dist/src/server/tree.d.ts +1 -1
- package/dist/src/server/tree.js +1 -1
- package/dist/src/utils.d.ts +3 -0
- package/dist/src/utils.js +23 -5
- package/package.json +3 -2
- package/dist/src/client/query/read/types.d.ts +0 -4
- package/dist/src/client/query/read/types.js +0 -5
package/dist/src/native.js
CHANGED
|
@@ -1,21 +1,22 @@
|
|
|
1
1
|
// @ts-ignore
|
|
2
2
|
import db from '../../basedDbNative.cjs';
|
|
3
|
-
|
|
3
|
+
// Can't import these from utils or it would be a cyclic import.
|
|
4
|
+
const DECODER = new TextDecoder('utf-8');
|
|
5
|
+
const ENCODER = new TextEncoder();
|
|
4
6
|
const selvaIoErrlog = new Uint8Array(256);
|
|
5
|
-
const textEncoder = new TextEncoder();
|
|
6
7
|
var compressor = db.createCompressor();
|
|
7
8
|
var decompressor = db.createDecompressor();
|
|
8
9
|
function SelvaIoErrlogToString(buf) {
|
|
9
10
|
let i;
|
|
10
11
|
let len = (i = buf.indexOf(0)) >= 0 ? i : buf.byteLength;
|
|
11
|
-
return
|
|
12
|
+
return DECODER.decode(selvaIoErrlog.slice(0, len));
|
|
12
13
|
}
|
|
13
14
|
export default {
|
|
14
15
|
historyAppend(history, typeId, nodeId, dbCtx) {
|
|
15
16
|
return db.historyAppend(history, typeId, nodeId, dbCtx);
|
|
16
17
|
},
|
|
17
18
|
historyCreate(pathname, mainLen) {
|
|
18
|
-
const pathBuf =
|
|
19
|
+
const pathBuf = ENCODER.encode(pathname + '\0');
|
|
19
20
|
return db.historyCreate(pathBuf, mainLen + 16 - (mainLen % 16));
|
|
20
21
|
},
|
|
21
22
|
workerCtxInit: () => {
|
|
@@ -41,22 +42,22 @@ export default {
|
|
|
41
42
|
return db.stop(dbCtx);
|
|
42
43
|
},
|
|
43
44
|
saveCommon: (path, dbCtx) => {
|
|
44
|
-
const pathBuf =
|
|
45
|
+
const pathBuf = ENCODER.encode(path + '\0');
|
|
45
46
|
return db.saveCommon(pathBuf, dbCtx);
|
|
46
47
|
},
|
|
47
48
|
saveRange: (path, typeCode, start, end, dbCtx, hashOut) => {
|
|
48
|
-
const pathBuf =
|
|
49
|
+
const pathBuf = ENCODER.encode(path + '\0');
|
|
49
50
|
return db.saveRange(pathBuf, typeCode, start, end, dbCtx, hashOut);
|
|
50
51
|
},
|
|
51
52
|
loadCommon: (path, dbCtx) => {
|
|
52
|
-
const pathBuf =
|
|
53
|
+
const pathBuf = ENCODER.encode(path + '\0');
|
|
53
54
|
const err = db.loadCommon(pathBuf, dbCtx, selvaIoErrlog);
|
|
54
55
|
if (err) {
|
|
55
56
|
throw new Error(`Failed to load common. selvaError: ${err} cause:\n${SelvaIoErrlogToString(selvaIoErrlog)}`);
|
|
56
57
|
}
|
|
57
58
|
},
|
|
58
59
|
loadRange: (path, dbCtx) => {
|
|
59
|
-
const pathBuf =
|
|
60
|
+
const pathBuf = ENCODER.encode(path + '\0');
|
|
60
61
|
const err = db.loadRange(pathBuf, dbCtx, selvaIoErrlog);
|
|
61
62
|
if (err) {
|
|
62
63
|
throw new Error(`Failed to load a range. selvaError: ${err} cause:\n${SelvaIoErrlogToString(selvaIoErrlog)}`);
|
|
@@ -71,29 +72,6 @@ export default {
|
|
|
71
72
|
getNodeRangeHash: (typeId, start, end, bufOut, dbCtx) => {
|
|
72
73
|
return db.getNodeRangeHash(typeId, start, end, bufOut, dbCtx);
|
|
73
74
|
},
|
|
74
|
-
createHash: () => {
|
|
75
|
-
const state = db.hashCreate();
|
|
76
|
-
const hash = {
|
|
77
|
-
update: (buf) => {
|
|
78
|
-
db.hashUpdate(state, buf);
|
|
79
|
-
return hash;
|
|
80
|
-
},
|
|
81
|
-
digest: (encoding) => {
|
|
82
|
-
const buf = new Uint8Array(16);
|
|
83
|
-
db.hashDigest(state, buf);
|
|
84
|
-
if (encoding === 'hex') {
|
|
85
|
-
return bufToHex(buf);
|
|
86
|
-
}
|
|
87
|
-
else {
|
|
88
|
-
return buf;
|
|
89
|
-
}
|
|
90
|
-
},
|
|
91
|
-
reset: () => {
|
|
92
|
-
db.hashReset(state);
|
|
93
|
-
},
|
|
94
|
-
};
|
|
95
|
-
return hash;
|
|
96
|
-
},
|
|
97
75
|
compress: (buf, offset, stringSize) => {
|
|
98
76
|
return db.compress(compressor, buf, offset, stringSize);
|
|
99
77
|
},
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
// @ts-ignore
|
|
2
|
+
import db from '../../../basedDbNative.cjs';
|
|
3
|
+
import { bufToHex } from '../utils.js';
|
|
4
|
+
export default function createHash() {
|
|
5
|
+
const state = db.hashCreate();
|
|
6
|
+
const hash = {
|
|
7
|
+
update: (buf) => {
|
|
8
|
+
db.hashUpdate(state, buf);
|
|
9
|
+
return hash;
|
|
10
|
+
},
|
|
11
|
+
digest: (encoding) => {
|
|
12
|
+
const buf = new Uint8Array(16);
|
|
13
|
+
db.hashDigest(state, buf);
|
|
14
|
+
if (encoding === 'hex') {
|
|
15
|
+
return bufToHex(buf);
|
|
16
|
+
}
|
|
17
|
+
else {
|
|
18
|
+
return buf;
|
|
19
|
+
}
|
|
20
|
+
},
|
|
21
|
+
reset: () => {
|
|
22
|
+
db.hashReset(state);
|
|
23
|
+
},
|
|
24
|
+
};
|
|
25
|
+
return hash;
|
|
26
|
+
}
|
|
27
|
+
//# sourceMappingURL=dbHash.js.map
|
|
@@ -5,8 +5,8 @@ import { Worker, MessagePort } from 'node:worker_threads';
|
|
|
5
5
|
import { TransformFns } from './migrate/index.js';
|
|
6
6
|
import exitHook from 'exit-hook';
|
|
7
7
|
declare class SortIndex {
|
|
8
|
-
constructor(buf:
|
|
9
|
-
buf:
|
|
8
|
+
constructor(buf: Uint8Array, dbCtxExternal: any);
|
|
9
|
+
buf: Uint8Array;
|
|
10
10
|
idx: any;
|
|
11
11
|
cnt: number;
|
|
12
12
|
}
|
|
@@ -18,7 +18,7 @@ export declare class DbWorker {
|
|
|
18
18
|
resolvers: any[];
|
|
19
19
|
callback: (resolve: any) => void;
|
|
20
20
|
updateCtx(address: BigInt): Promise<void>;
|
|
21
|
-
getQueryBuf(buf:
|
|
21
|
+
getQueryBuf(buf: Uint8Array): Promise<Uint8Array>;
|
|
22
22
|
}
|
|
23
23
|
type OnSchemaChange = (schema: StrictSchema) => void;
|
|
24
24
|
export declare class DbServer {
|
|
@@ -36,15 +36,15 @@ export declare class DbServer {
|
|
|
36
36
|
merkleTree: ReturnType<typeof createTree>;
|
|
37
37
|
dirtyRanges: Set<number>;
|
|
38
38
|
csmtHashFun: {
|
|
39
|
-
update: (buf:
|
|
39
|
+
update: (buf: Uint8Array) => any;
|
|
40
40
|
digest: (encoding?: "hex") => Uint8Array | string;
|
|
41
41
|
reset: () => void;
|
|
42
42
|
};
|
|
43
43
|
workers: DbWorker[];
|
|
44
44
|
availableWorkerIndex: number;
|
|
45
45
|
processingQueries: number;
|
|
46
|
-
modifyQueue:
|
|
47
|
-
queryQueue: Map<Function,
|
|
46
|
+
modifyQueue: Uint8Array[];
|
|
47
|
+
queryQueue: Map<Function, Uint8Array>;
|
|
48
48
|
stopped: boolean;
|
|
49
49
|
onSchemaChange: OnSchemaChange;
|
|
50
50
|
unlistenExit: ReturnType<typeof exitHook>;
|
|
@@ -59,7 +59,7 @@ export declare class DbServer {
|
|
|
59
59
|
}): Promise<void>;
|
|
60
60
|
save(): void | Promise<void>;
|
|
61
61
|
createCsmtHashFun: () => {
|
|
62
|
-
update: (buf:
|
|
62
|
+
update: (buf: Uint8Array) => any;
|
|
63
63
|
digest: (encoding?: "hex") => Uint8Array | string;
|
|
64
64
|
reset: () => void;
|
|
65
65
|
};
|
|
@@ -82,13 +82,13 @@ export declare class DbServer {
|
|
|
82
82
|
}>;
|
|
83
83
|
createSortIndexBuffer(typeId: number, field: number, start: number, lang: number): SortIndex;
|
|
84
84
|
updateMerkleTree(): void;
|
|
85
|
-
|
|
85
|
+
setSchema(strictSchema: StrictSchema, fromStart?: boolean, transformFns?: TransformFns): (StrictSchema & {
|
|
86
86
|
lastId: number;
|
|
87
87
|
}) | Promise<StrictSchema & {
|
|
88
88
|
lastId: number;
|
|
89
89
|
}>;
|
|
90
|
-
modify(buf:
|
|
91
|
-
getQueryBuf(buf:
|
|
90
|
+
modify(buf: Uint8Array): Record<number, number>;
|
|
91
|
+
getQueryBuf(buf: Uint8Array): Promise<Uint8Array>;
|
|
92
92
|
onQueryEnd(): void;
|
|
93
93
|
stop(noSave?: boolean): Promise<void>;
|
|
94
94
|
destroy(): Promise<void>;
|
package/dist/src/server/index.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import native from '../native.js';
|
|
2
|
+
import createDbHash from './dbHash.js';
|
|
2
3
|
import { rm, writeFile } from 'node:fs/promises';
|
|
3
4
|
import { dirname, join } from 'node:path';
|
|
4
5
|
import { getPropType, langCodesMap, } from '@based/schema';
|
|
@@ -24,6 +25,12 @@ class SortIndex {
|
|
|
24
25
|
idx;
|
|
25
26
|
cnt = 0;
|
|
26
27
|
}
|
|
28
|
+
function readUint16LE(buf, off) {
|
|
29
|
+
return buf[off] | (buf[off + 1] << 8);
|
|
30
|
+
}
|
|
31
|
+
function readUint32LE(buf, off) {
|
|
32
|
+
return (buf[off] | (buf[off + 1] << 8) | (buf[off + 2] << 16) | (buf[off + 3] << 24));
|
|
33
|
+
}
|
|
27
34
|
export class DbWorker {
|
|
28
35
|
constructor(address, db) {
|
|
29
36
|
const { port1, port2 } = new MessageChannel();
|
|
@@ -73,7 +80,7 @@ export class DbServer {
|
|
|
73
80
|
maxModifySize;
|
|
74
81
|
merkleTree;
|
|
75
82
|
dirtyRanges = new Set();
|
|
76
|
-
csmtHashFun =
|
|
83
|
+
csmtHashFun = createDbHash();
|
|
77
84
|
workers = [];
|
|
78
85
|
availableWorkerIndex = -1;
|
|
79
86
|
processingQueries = 0;
|
|
@@ -152,13 +159,16 @@ export class DbServer {
|
|
|
152
159
|
if (sortIndex) {
|
|
153
160
|
return sortIndex;
|
|
154
161
|
}
|
|
155
|
-
const buf =
|
|
162
|
+
const buf = new Uint8Array(8);
|
|
156
163
|
// size [2 type] [1 field] [2 start] [2 len] [propIndex] [lang]
|
|
157
164
|
// call createSortBuf here
|
|
158
|
-
buf
|
|
165
|
+
buf[0] = t.id;
|
|
166
|
+
buf[1] = t.id >>> 8;
|
|
159
167
|
buf[2] = prop.prop;
|
|
160
|
-
buf
|
|
161
|
-
buf
|
|
168
|
+
buf[3] = prop.start;
|
|
169
|
+
buf[4] = prop.start >>> 8;
|
|
170
|
+
buf[5] = prop.len;
|
|
171
|
+
buf[6] = prop.len >>> 8;
|
|
162
172
|
buf[7] = prop.typeIndex;
|
|
163
173
|
buf[8] = langCode;
|
|
164
174
|
sortIndex = new SortIndex(buf, this.dbCtxExternal);
|
|
@@ -179,10 +189,12 @@ export class DbServer {
|
|
|
179
189
|
let sortIndex = fields[prop.start];
|
|
180
190
|
if (sortIndex) {
|
|
181
191
|
// [2 type] [1 field] [2 start] [1 lang]
|
|
182
|
-
const buf =
|
|
183
|
-
buf
|
|
192
|
+
const buf = new Uint8Array(6);
|
|
193
|
+
buf[0] = t.id;
|
|
194
|
+
buf[1] = t.id >>> 8;
|
|
184
195
|
buf[2] = prop.prop;
|
|
185
|
-
buf
|
|
196
|
+
buf[3] = prop.start;
|
|
197
|
+
buf[4] = prop.start >>> 8;
|
|
186
198
|
buf[5] =
|
|
187
199
|
langCodesMap.get(lang ?? Object.keys(this.schema?.locales ?? 'en')[0]) ?? 0;
|
|
188
200
|
native.destroySortIndex(buf, this.dbCtxExternal);
|
|
@@ -208,10 +220,12 @@ export class DbServer {
|
|
|
208
220
|
return migrate(this, schema, transform);
|
|
209
221
|
}
|
|
210
222
|
createSortIndexBuffer(typeId, field, start, lang) {
|
|
211
|
-
const buf =
|
|
212
|
-
buf
|
|
223
|
+
const buf = new Uint8Array(9);
|
|
224
|
+
buf[0] = typeId;
|
|
225
|
+
buf[1] = typeId >>> 8;
|
|
213
226
|
buf[2] = field;
|
|
214
|
-
buf
|
|
227
|
+
buf[3] = start;
|
|
228
|
+
buf[4] = start >>> 8;
|
|
215
229
|
let typeDef;
|
|
216
230
|
let prop;
|
|
217
231
|
for (const t in this.schemaTypesParsed) {
|
|
@@ -233,7 +247,8 @@ export class DbServer {
|
|
|
233
247
|
if (!prop) {
|
|
234
248
|
throw new Error(`Cannot find prop on db from query for sort ${field}`);
|
|
235
249
|
}
|
|
236
|
-
buf
|
|
250
|
+
buf[5] = prop.len;
|
|
251
|
+
buf[6] = prop.len >>> 8;
|
|
237
252
|
buf[7] = prop.typeIndex;
|
|
238
253
|
buf[8] = lang;
|
|
239
254
|
// put in modify stuff
|
|
@@ -247,7 +262,7 @@ export class DbServer {
|
|
|
247
262
|
updateMerkleTree() {
|
|
248
263
|
foreachDirtyBlock(this, (mtKey, typeId, start, end) => {
|
|
249
264
|
const oldLeaf = this.merkleTree.search(mtKey);
|
|
250
|
-
const hash =
|
|
265
|
+
const hash = new Uint8Array(16);
|
|
251
266
|
native.getNodeRangeHash(typeId, start, end, hash, this.dbCtxExternal);
|
|
252
267
|
if (oldLeaf) {
|
|
253
268
|
if (hashEq(oldLeaf.hash, hash)) {
|
|
@@ -267,7 +282,7 @@ export class DbServer {
|
|
|
267
282
|
this.merkleTree.insert(mtKey, hash, data);
|
|
268
283
|
});
|
|
269
284
|
}
|
|
270
|
-
|
|
285
|
+
setSchema(strictSchema, fromStart = false, transformFns) {
|
|
271
286
|
if (!fromStart && Object.keys(this.schema.types).length > 0) {
|
|
272
287
|
return this.migrateSchema(strictSchema, transformFns);
|
|
273
288
|
}
|
|
@@ -338,17 +353,18 @@ export class DbServer {
|
|
|
338
353
|
}
|
|
339
354
|
if (strictSchema.props) {
|
|
340
355
|
// insert a root node
|
|
341
|
-
const data = [2, 1, 0, 0, 0,
|
|
356
|
+
const data = [2, 1, 0, 0, 0, 1, 9, 1, 0, 0, 0, 7, 1, 0, 1];
|
|
342
357
|
const blockKey = makeCsmtKey(1, 1);
|
|
343
|
-
const buf =
|
|
358
|
+
const buf = new Uint8Array(data.length + 2 + 8 + 4);
|
|
359
|
+
const view = new DataView(buf.buffer, 0, buf.byteLength);
|
|
344
360
|
// add content
|
|
345
361
|
buf.set(data);
|
|
346
362
|
// add typesLen
|
|
347
|
-
|
|
363
|
+
view.setFloat64(data.length, 0, true);
|
|
348
364
|
// add dirty key
|
|
349
|
-
|
|
365
|
+
view.setFloat64(data.length + 2, blockKey, true);
|
|
350
366
|
// add dataLen
|
|
351
|
-
|
|
367
|
+
view.setUint32(buf.length - 4, data.length, true);
|
|
352
368
|
this.modify(buf);
|
|
353
369
|
}
|
|
354
370
|
}
|
|
@@ -357,24 +373,26 @@ export class DbServer {
|
|
|
357
373
|
}
|
|
358
374
|
modify(buf) {
|
|
359
375
|
const offsets = {};
|
|
360
|
-
const dataLen =
|
|
361
|
-
let typesSize =
|
|
376
|
+
const dataLen = readUint32LE(buf, buf.length - 4);
|
|
377
|
+
let typesSize = readUint16LE(buf, dataLen);
|
|
362
378
|
let i = dataLen + 2;
|
|
363
379
|
while (typesSize--) {
|
|
364
|
-
const typeId =
|
|
380
|
+
const typeId = readUint16LE(buf, i);
|
|
365
381
|
i += 2;
|
|
366
|
-
const startId =
|
|
382
|
+
const startId = readUint32LE(buf, i);
|
|
367
383
|
const def = this.schemaTypesParsedById[typeId];
|
|
368
384
|
const offset = def.lastId - startId;
|
|
369
|
-
buf
|
|
370
|
-
i
|
|
371
|
-
|
|
385
|
+
buf[i++] = offset;
|
|
386
|
+
buf[i++] = offset >>> 8;
|
|
387
|
+
buf[i++] = offset >>> 16;
|
|
388
|
+
buf[i++] = offset >>> 24;
|
|
389
|
+
const lastId = readUint32LE(buf, i);
|
|
372
390
|
i += 4;
|
|
373
391
|
def.lastId = lastId + offset;
|
|
374
392
|
offsets[typeId] = offset;
|
|
375
393
|
}
|
|
376
394
|
if (this.processingQueries) {
|
|
377
|
-
this.modifyQueue.push(
|
|
395
|
+
this.modifyQueue.push(new Uint8Array(buf));
|
|
378
396
|
}
|
|
379
397
|
else {
|
|
380
398
|
this.#modify(buf);
|
|
@@ -383,21 +401,22 @@ export class DbServer {
|
|
|
383
401
|
}
|
|
384
402
|
#modify(buf) {
|
|
385
403
|
const end = buf.length - 4;
|
|
386
|
-
const dataLen =
|
|
387
|
-
let typesSize =
|
|
404
|
+
const dataLen = readUint32LE(buf, end);
|
|
405
|
+
let typesSize = readUint16LE(buf, dataLen);
|
|
388
406
|
const typesLen = typesSize * 10;
|
|
389
407
|
const types = buf.subarray(dataLen + 2, dataLen + typesLen + 2);
|
|
390
408
|
const data = buf.subarray(0, dataLen);
|
|
391
409
|
let i = dataLen + 2;
|
|
392
410
|
while (typesSize--) {
|
|
393
|
-
const typeId =
|
|
411
|
+
const typeId = readUint16LE(buf, i);
|
|
394
412
|
const def = this.schemaTypesParsedById[typeId];
|
|
395
413
|
const key = makeCsmtKeyFromNodeId(def.id, def.blockCapacity, def.lastId);
|
|
396
414
|
this.dirtyRanges.add(key);
|
|
397
415
|
i += 10;
|
|
398
416
|
}
|
|
417
|
+
const view = new DataView(buf.buffer, buf.byteOffset);
|
|
399
418
|
while (i < end) {
|
|
400
|
-
const key =
|
|
419
|
+
const key = view.getFloat64(i, true);
|
|
401
420
|
this.dirtyRanges.add(key);
|
|
402
421
|
i += 8;
|
|
403
422
|
}
|
|
@@ -412,13 +431,13 @@ export class DbServer {
|
|
|
412
431
|
else {
|
|
413
432
|
const queryType = buf[0];
|
|
414
433
|
if (queryType == 2) {
|
|
415
|
-
const s = 13 +
|
|
416
|
-
const sortLen =
|
|
434
|
+
const s = 13 + readUint16LE(buf, 11);
|
|
435
|
+
const sortLen = readUint16LE(buf, s);
|
|
417
436
|
if (sortLen) {
|
|
418
|
-
const typeId =
|
|
437
|
+
const typeId = readUint16LE(buf, 1);
|
|
419
438
|
const sort = buf.slice(s + 2, s + 2 + sortLen);
|
|
420
439
|
const field = sort[1];
|
|
421
|
-
const start =
|
|
440
|
+
const start = readUint16LE(sort, 3);
|
|
422
441
|
let sortIndex = this.getSortIndex(typeId, field, start, 0);
|
|
423
442
|
if (!sortIndex) {
|
|
424
443
|
if (this.processingQueries) {
|
|
@@ -39,7 +39,7 @@ export const migrate = async (fromDbServer, toSchema, transform) => {
|
|
|
39
39
|
await toDb.destroy();
|
|
40
40
|
return fromDbServer.schema;
|
|
41
41
|
}
|
|
42
|
-
toSchema = await toDb.
|
|
42
|
+
toSchema = await toDb.setSchema(toSchema);
|
|
43
43
|
const fromSchema = fromDbServer.schema;
|
|
44
44
|
const fromCtx = fromDbServer.dbCtxExternal;
|
|
45
45
|
const toCtx = toDb.server.dbCtxExternal;
|
|
@@ -14,8 +14,8 @@ else {
|
|
|
14
14
|
const toDb = new BasedDb({ path });
|
|
15
15
|
fromDb.server.dbCtxExternal = fromCtx;
|
|
16
16
|
toDb.server.dbCtxExternal = toCtx;
|
|
17
|
-
await fromDb.
|
|
18
|
-
await toDb.
|
|
17
|
+
await fromDb.setSchema(fromSchema, true);
|
|
18
|
+
await toDb.setSchema(toSchema, true);
|
|
19
19
|
const map = {};
|
|
20
20
|
for (const type in fromDb.client.schemaTypesParsed) {
|
|
21
21
|
const { id, props } = fromDb.client.schemaTypesParsed[type];
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
import { DbServer } from './index.js';
|
|
2
|
-
export declare function save(db: DbServer, sync?:
|
|
2
|
+
export declare function save<T extends boolean>(db: DbServer, sync?: T): T extends true ? void : Promise<void>;
|
package/dist/src/server/save.js
CHANGED
|
@@ -42,25 +42,25 @@ export function save(db, sync = false) {
|
|
|
42
42
|
});
|
|
43
43
|
db.dirtyRanges.clear();
|
|
44
44
|
const types = {};
|
|
45
|
+
const rangeDumps = {};
|
|
45
46
|
for (const key in db.schemaTypesParsed) {
|
|
46
|
-
const
|
|
47
|
-
types[
|
|
48
|
-
|
|
49
|
-
const dumps = {};
|
|
50
|
-
for (const key in db.schemaTypesParsed) {
|
|
51
|
-
const def = db.schemaTypesParsed[key];
|
|
52
|
-
dumps[def.id] = [];
|
|
47
|
+
const { id, lastId, blockCapacity } = db.schemaTypesParsed[key];
|
|
48
|
+
types[id] = { lastId, blockCapacity };
|
|
49
|
+
rangeDumps[id] = [];
|
|
53
50
|
}
|
|
54
51
|
db.merkleTree.visitLeafNodes((leaf) => {
|
|
55
|
-
const [typeId] = destructureCsmtKey(leaf.key);
|
|
52
|
+
const [typeId, start] = destructureCsmtKey(leaf.key);
|
|
56
53
|
const data = leaf.data;
|
|
57
|
-
|
|
54
|
+
if (start != data.start) {
|
|
55
|
+
console.error(`csmtKey start and range start mismatch: ${start} != ${data.start}`);
|
|
56
|
+
}
|
|
57
|
+
rangeDumps[typeId].push({ ...data, hash: bufToHex(leaf.hash) });
|
|
58
58
|
});
|
|
59
59
|
const data = {
|
|
60
60
|
ts,
|
|
61
61
|
types,
|
|
62
62
|
commonDump: COMMON_SDB_FILE,
|
|
63
|
-
rangeDumps
|
|
63
|
+
rangeDumps,
|
|
64
64
|
};
|
|
65
65
|
const mtRoot = db.merkleTree.getRoot();
|
|
66
66
|
if (mtRoot) {
|
package/dist/src/server/start.js
CHANGED
|
@@ -52,7 +52,7 @@ export async function start(db, opts) {
|
|
|
52
52
|
const schema = await readFile(join(path, SCHEMA_FILE));
|
|
53
53
|
if (schema) {
|
|
54
54
|
// Prop need to not call setting in selva
|
|
55
|
-
db.
|
|
55
|
+
db.setSchema(JSON.parse(schema.toString()), true);
|
|
56
56
|
}
|
|
57
57
|
}
|
|
58
58
|
catch (err) {
|
|
@@ -65,7 +65,9 @@ export async function start(db, opts) {
|
|
|
65
65
|
// FDN-791 CSMT is unstable (not history independent)
|
|
66
66
|
// For now we just sort the types to ensure that we always
|
|
67
67
|
// load in the same order.
|
|
68
|
-
const types = Object.keys(db.schemaTypesParsed)
|
|
68
|
+
const types = Object.keys(db.schemaTypesParsed)
|
|
69
|
+
.sort((a, b) => db.schemaTypesParsed[a].id - db.schemaTypesParsed[b].id)
|
|
70
|
+
.reduce((obj, key) => {
|
|
69
71
|
obj[key] = db.schemaTypesParsed[key];
|
|
70
72
|
return obj;
|
|
71
73
|
}, {});
|
|
@@ -6,8 +6,8 @@ export type CsmtNodeRange = {
|
|
|
6
6
|
start: number;
|
|
7
7
|
end: number;
|
|
8
8
|
};
|
|
9
|
-
export declare const makeCsmtKey: (typeId: number, start: number) => number;
|
|
10
9
|
export declare const destructureCsmtKey: (key: number) => number[];
|
|
10
|
+
export declare const makeCsmtKey: (typeId: number, start: number) => number;
|
|
11
11
|
export declare const makeCsmtKeyFromNodeId: (typeId: number, blockCapacity: number, nodeId: number) => number;
|
|
12
12
|
export declare function foreachBlock(db: DbServer, def: SchemaTypeDef, cb: (start: number, end: number, hash: Uint8Array) => void): Promise<void>;
|
|
13
13
|
export declare function foreachDirtyBlock(db: DbServer, cb: (mtKey: number, typeId: number, start: number, end: number) => void): Promise<void>;
|
package/dist/src/server/tree.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import native from '../native.js';
|
|
2
|
-
export const makeCsmtKey = (typeId, start) => typeId * 4294967296 + start;
|
|
3
2
|
export const destructureCsmtKey = (key) => [
|
|
4
3
|
(key / 4294967296) | 0, // typeId
|
|
5
4
|
(key >>> 31) * 2147483648 + (key & 0x7fffffff), // start_node_id
|
|
6
5
|
];
|
|
6
|
+
export const makeCsmtKey = (typeId, start) => typeId * 4294967296 + start;
|
|
7
7
|
export const makeCsmtKeyFromNodeId = (typeId, blockCapacity, nodeId) => {
|
|
8
8
|
const tmp = nodeId - +!(nodeId % blockCapacity);
|
|
9
9
|
return typeId * 4294967296 + ((tmp / blockCapacity) | 0) * blockCapacity + 1;
|
package/dist/src/utils.d.ts
CHANGED
|
@@ -1,4 +1,7 @@
|
|
|
1
|
+
export declare const DECODER: TextDecoder;
|
|
2
|
+
export declare const ENCODER: TextEncoder;
|
|
1
3
|
export declare const equals: (aB: Uint8Array, bB: Uint8Array) => boolean;
|
|
4
|
+
export declare function concatUint8Arr(bufs: Uint8Array[], totalByteLength?: number): Uint8Array;
|
|
2
5
|
export declare const bufToHex: (a: Uint8Array) => string;
|
|
3
6
|
export declare const hexToBuf: (s: string) => Uint8Array;
|
|
4
7
|
export declare const base64encode: (a: Uint8Array, lineMax?: number) => string;
|
package/dist/src/utils.js
CHANGED
|
@@ -1,11 +1,14 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { encodeBase64 } from '@saulx/utils';
|
|
2
|
+
const native = (typeof window === 'undefined') ? (await import('./native.js')).default : null;
|
|
3
|
+
export const DECODER = new TextDecoder('utf-8');
|
|
4
|
+
export const ENCODER = new TextEncoder();
|
|
2
5
|
export const equals = (aB, bB) => {
|
|
3
6
|
const len = aB.byteLength;
|
|
4
7
|
if (len != bB.byteLength) {
|
|
5
8
|
return false;
|
|
6
9
|
}
|
|
7
10
|
let i = 0;
|
|
8
|
-
if (len < 50) {
|
|
11
|
+
if (len < 50 || !native) {
|
|
9
12
|
while (i < len) {
|
|
10
13
|
if (aB[i] != bB[i]) {
|
|
11
14
|
return false;
|
|
@@ -24,8 +27,18 @@ export const equals = (aB, bB) => {
|
|
|
24
27
|
return native.equals(aB, bB);
|
|
25
28
|
}
|
|
26
29
|
};
|
|
27
|
-
|
|
28
|
-
|
|
30
|
+
export function concatUint8Arr(bufs, totalByteLength) {
|
|
31
|
+
totalByteLength = totalByteLength ?? bufs.reduce((acc, cur) => acc + cur.byteLength, 0);
|
|
32
|
+
const res = new Uint8Array(totalByteLength);
|
|
33
|
+
let off = 0;
|
|
34
|
+
for (let i = 0; i < bufs.length; i++) {
|
|
35
|
+
const buf = bufs[i];
|
|
36
|
+
res.set(buf, off);
|
|
37
|
+
off += buf.byteLength;
|
|
38
|
+
}
|
|
39
|
+
return res;
|
|
40
|
+
}
|
|
41
|
+
const charMap = ENCODER.encode('0123456789abcdef');
|
|
29
42
|
// Uint8Array.fromHex() and Uint8Array.toHex() are not available in V8
|
|
30
43
|
// https://issues.chromium.org/issues/42204568
|
|
31
44
|
export const bufToHex = (a) => {
|
|
@@ -79,6 +92,11 @@ function base64OutLen(n, lineMax) {
|
|
|
79
92
|
export const base64encode = (a, lineMax = 72) => {
|
|
80
93
|
// TODO Could fallback to @saulx/utils if native is not available
|
|
81
94
|
const tmp = new Uint8Array(base64OutLen(a.byteLength, lineMax));
|
|
82
|
-
|
|
95
|
+
if ((a.length < 10 && lineMax === 72) || !native) {
|
|
96
|
+
return encodeBase64(a);
|
|
97
|
+
}
|
|
98
|
+
else {
|
|
99
|
+
return DECODER.decode(native.base64encode(tmp, a, lineMax));
|
|
100
|
+
}
|
|
83
101
|
};
|
|
84
102
|
//# sourceMappingURL=utils.js.map
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@based/db",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.23",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/src/index.js",
|
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
"test": "npm run build && LOCPATH=../locale/locale-x86_64-gnu/locale ./scripts/lldb-node ./scripts/test.js",
|
|
18
18
|
"test-gdb": "npm run build && LOCPATH=../locale/locale-x86_64-gnu/locale gdb -ex run --args node ./scripts/test.js",
|
|
19
19
|
"test-fast": "LOCPATH=../locale/locale-x86_64-gnu/locale node ./scripts/test.js",
|
|
20
|
+
"test-fast-linux_aarch64": "podman run --rm -v \"$PWD/../..:/usr/src/based-db\" based-db-clibs-build-linux_aarch64 sh -c '\\. \"/usr/local/nvm/nvm.sh\"; cd /usr/src/based-db/packages/db; npm run test-fast'",
|
|
20
21
|
"test-zig": "npm run build-zig -- debug && tsc && npm run test-fast",
|
|
21
22
|
"test-ts": "tsc && node ./scripts/test.js",
|
|
22
23
|
"perf": "npm run build && node benchmarks/references.js && node benchmarks/transfermarkt/transfermarkt-based.js"
|
|
@@ -31,7 +32,7 @@
|
|
|
31
32
|
"basedDbNative.cjs"
|
|
32
33
|
],
|
|
33
34
|
"dependencies": {
|
|
34
|
-
"@based/schema": "5.0.0-alpha.
|
|
35
|
+
"@based/schema": "5.0.0-alpha.6",
|
|
35
36
|
"@saulx/hash": "^3.0.0",
|
|
36
37
|
"@saulx/utils": "^4.3.2",
|
|
37
38
|
"exit-hook": "^4.0.0",
|