@based/db 0.0.25 → 0.0.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/darwin_aarch64/include/cdefs.h +1 -1
- package/dist/lib/darwin_aarch64/include/selva/db.h +45 -11
- package/dist/lib/darwin_aarch64/include/selva/fields.h +48 -13
- package/dist/lib/darwin_aarch64/include/selva/hll.h +59 -0
- package/dist/lib/darwin_aarch64/include/selva/types.h +2 -0
- package/dist/lib/darwin_aarch64/include/tree.h +69 -69
- package/dist/lib/darwin_aarch64/libnode-v20.node +0 -0
- package/dist/lib/darwin_aarch64/libnode-v21.node +0 -0
- package/dist/lib/darwin_aarch64/libnode-v22.node +0 -0
- package/dist/lib/darwin_aarch64/libnode-v23.node +0 -0
- package/dist/lib/darwin_aarch64/libselva.dylib +0 -0
- package/dist/lib/linux_aarch64/include/cdefs.h +1 -1
- package/dist/lib/linux_aarch64/include/selva/db.h +45 -11
- package/dist/lib/linux_aarch64/include/selva/fields.h +48 -13
- package/dist/lib/linux_aarch64/include/selva/hll.h +59 -0
- package/dist/lib/linux_aarch64/include/selva/types.h +2 -0
- package/dist/lib/linux_aarch64/include/tree.h +69 -69
- package/dist/lib/linux_aarch64/libdeflate.so +0 -0
- package/dist/lib/linux_aarch64/libnode-v20.node +0 -0
- package/dist/lib/linux_aarch64/libnode-v21.node +0 -0
- package/dist/lib/linux_aarch64/libnode-v22.node +0 -0
- package/dist/lib/linux_aarch64/libnode-v23.node +0 -0
- package/dist/lib/linux_aarch64/libselva.so +0 -0
- package/dist/lib/linux_x86_64/include/cdefs.h +1 -1
- package/dist/lib/linux_x86_64/include/selva/db.h +45 -11
- package/dist/lib/linux_x86_64/include/selva/fields.h +48 -13
- package/dist/lib/linux_x86_64/include/selva/hll.h +59 -0
- package/dist/lib/linux_x86_64/include/selva/types.h +2 -0
- package/dist/lib/linux_x86_64/include/tree.h +69 -69
- package/dist/lib/linux_x86_64/libdeflate.so +0 -0
- package/dist/lib/linux_x86_64/libnode-v20.node +0 -0
- package/dist/lib/linux_x86_64/libnode-v21.node +0 -0
- package/dist/lib/linux_x86_64/libnode-v22.node +0 -0
- package/dist/lib/linux_x86_64/libnode-v23.node +0 -0
- package/dist/lib/linux_x86_64/libselva.so +0 -0
- package/dist/src/client/flushModify.js +5 -1
- package/dist/src/client/index.d.ts +12 -6
- package/dist/src/client/index.js +33 -1
- package/dist/src/client/modify/alias.js +3 -0
- package/dist/src/client/modify/binary.js +1 -1
- package/dist/src/client/modify/cardinality.d.ts +2 -2
- package/dist/src/client/modify/cardinality.js +17 -6
- package/dist/src/client/modify/create.js +17 -1
- package/dist/src/client/modify/fixed.js +27 -23
- package/dist/src/client/modify/json.js +15 -1
- package/dist/src/client/modify/modify.js +3 -1
- package/dist/src/client/modify/references/edge.js +4 -2
- package/dist/src/client/modify/references/references.js +21 -6
- package/dist/src/client/modify/string.js +5 -6
- package/dist/src/client/modify/text.js +0 -11
- package/dist/src/client/modify/upsert.js +1 -1
- package/dist/src/client/modify/vector.js +3 -3
- package/dist/src/client/query/BasedDbQuery.js +9 -1
- package/dist/src/client/query/BasedIterable.js +10 -3
- package/dist/src/client/query/aggregation.d.ts +3 -0
- package/dist/src/client/query/aggregation.js +9 -0
- package/dist/src/client/query/display.js +12 -2
- package/dist/src/client/query/filter/parseFilterValue.js +2 -4
- package/dist/src/client/query/filter/toBuffer.js +2 -2
- package/dist/src/client/query/include/walk.js +1 -0
- package/dist/src/client/query/query.d.ts +1 -1
- package/dist/src/client/query/query.js +1 -1
- package/dist/src/client/query/queryDef.js +0 -1
- package/dist/src/client/query/read/read.js +17 -5
- package/dist/src/client/query/toBuffer.js +2 -2
- package/dist/src/client/query/types.d.ts +4 -3
- package/dist/src/client/query/validation.js +8 -1
- package/dist/src/client/string.js +1 -1
- package/dist/src/index.d.ts +7 -1
- package/dist/src/index.js +19 -4
- package/dist/src/native.d.ts +1 -1
- package/dist/src/native.js +2 -2
- package/dist/src/server/csmt/tree.js +2 -2
- package/dist/src/server/index.d.ts +1 -1
- package/dist/src/server/index.js +36 -4
- package/dist/src/server/migrate/index.js +9 -5
- package/dist/src/server/migrate/worker.js +26 -1
- package/dist/src/server/save.js +20 -12
- package/dist/src/server/start.js +0 -2
- package/dist/src/utils.d.ts +6 -0
- package/dist/src/utils.js +81 -9
- package/package.json +4 -3
- package/dist/src/client/bitWise.d.ts +0 -6
- package/dist/src/client/bitWise.js +0 -72
- package/dist/src/client/query/aggregationFn.d.ts +0 -3
- package/dist/src/client/query/aggregationFn.js +0 -9
- package/dist/src/client/tree.d.ts +0 -1
- package/dist/src/client/tree.js +0 -5
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import picocolors from 'picocolors';
|
|
2
|
-
import { ALIAS, BINARY, BOOLEAN, REFERENCE, REFERENCES, REVERSE_TYPE_INDEX_MAP, STRING, TEXT, TIMESTAMP, VECTOR, propIsNumerical, createEmptyDef, } from '@based/schema/def';
|
|
2
|
+
import { ALIAS, BINARY, BOOLEAN, REFERENCE, REFERENCES, REVERSE_TYPE_INDEX_MAP, STRING, TEXT, TIMESTAMP, VECTOR, propIsNumerical, createEmptyDef, DEFAULT_MAP, } from '@based/schema/def';
|
|
3
3
|
import { EQUAL, EXISTS, HAS, isNumerical, LIKE, operatorReverseMap, VECTOR_FNS, } from './filter/types.js';
|
|
4
4
|
import { MAX_ID, MAX_ID_VALUE, MAX_IDS_PER_QUERY, MIN_ID_VALUE, } from './thresholds.js';
|
|
5
5
|
import { displayTarget, safeStringify } from './display.js';
|
|
@@ -418,8 +418,10 @@ export const EMPTY_ALIAS_PROP_DEF = {
|
|
|
418
418
|
typeIndex: ALIAS,
|
|
419
419
|
__isPropDef: true,
|
|
420
420
|
separate: true,
|
|
421
|
+
validation: () => true,
|
|
421
422
|
len: 0,
|
|
422
423
|
start: 0,
|
|
424
|
+
default: DEFAULT_MAP[ALIAS],
|
|
423
425
|
path: ['ERROR_ALIAS'],
|
|
424
426
|
};
|
|
425
427
|
export const ERROR_STRING = {
|
|
@@ -427,8 +429,10 @@ export const ERROR_STRING = {
|
|
|
427
429
|
typeIndex: STRING,
|
|
428
430
|
__isPropDef: true,
|
|
429
431
|
separate: true,
|
|
432
|
+
validation: () => true,
|
|
430
433
|
len: 0,
|
|
431
434
|
start: 0,
|
|
435
|
+
default: DEFAULT_MAP[STRING],
|
|
432
436
|
path: ['ERROR_STRING'],
|
|
433
437
|
};
|
|
434
438
|
export const ERROR_VECTOR = {
|
|
@@ -436,8 +440,10 @@ export const ERROR_VECTOR = {
|
|
|
436
440
|
typeIndex: VECTOR,
|
|
437
441
|
__isPropDef: true,
|
|
438
442
|
separate: true,
|
|
443
|
+
validation: () => true,
|
|
439
444
|
len: 0,
|
|
440
445
|
start: 0,
|
|
446
|
+
default: DEFAULT_MAP[VECTOR],
|
|
441
447
|
path: ['ERROR_VECTOR'],
|
|
442
448
|
};
|
|
443
449
|
export const EMPTY_SCHEMA_DEF = {
|
|
@@ -446,5 +452,6 @@ export const EMPTY_SCHEMA_DEF = {
|
|
|
446
452
|
propNames: new Uint8Array([]),
|
|
447
453
|
packed: new Uint8Array([]),
|
|
448
454
|
idUint8: new Uint8Array([0, 0]),
|
|
455
|
+
mainEmptyAllZeroes: true,
|
|
449
456
|
};
|
|
450
457
|
//# sourceMappingURL=validation.js.map
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import native from '../native.js';
|
|
2
|
-
import { readUint32 } from '
|
|
2
|
+
import { readUint32 } from './../utils.js';
|
|
3
3
|
import makeTmpBuffer from './tmpBuffer.js';
|
|
4
4
|
import { DECODER, ENCODER } from '../utils.js';
|
|
5
5
|
const { getUint8Array: getTmpBuffer } = makeTmpBuffer(4096); // the usual page size?
|
package/dist/src/index.d.ts
CHANGED
|
@@ -10,6 +10,9 @@ export { xxHash64 } from './client/xxHash64.js';
|
|
|
10
10
|
export { crc32 } from './client/crc32.js';
|
|
11
11
|
export * from './client/query/serialize.js';
|
|
12
12
|
export * from './utils.js';
|
|
13
|
+
export * from './client/query/query.js';
|
|
14
|
+
export * from './client/query/BasedDbQuery.js';
|
|
15
|
+
export * from './client/query/BasedIterable.js';
|
|
13
16
|
export declare class BasedDb {
|
|
14
17
|
#private;
|
|
15
18
|
client: DbClient;
|
|
@@ -35,7 +38,10 @@ export declare class BasedDb {
|
|
|
35
38
|
stop: DbServer['stop'];
|
|
36
39
|
save: DbServer['save'];
|
|
37
40
|
migrateSchema: DbServer['migrateSchema'];
|
|
38
|
-
|
|
41
|
+
isModified: DbClient['isModified'];
|
|
42
|
+
schemaIsSet: DbClient['schemaIsSet'];
|
|
39
43
|
destroy(): Promise<void>;
|
|
40
44
|
wipe(): Promise<void>;
|
|
45
|
+
on: DbClient['on'];
|
|
46
|
+
off: DbClient['off'];
|
|
41
47
|
}
|
package/dist/src/index.js
CHANGED
|
@@ -12,6 +12,9 @@ export { xxHash64 } from './client/xxHash64.js';
|
|
|
12
12
|
export { crc32 } from './client/crc32.js';
|
|
13
13
|
export * from './client/query/serialize.js';
|
|
14
14
|
export * from './utils.js';
|
|
15
|
+
export * from './client/query/query.js';
|
|
16
|
+
export * from './client/query/BasedDbQuery.js';
|
|
17
|
+
export * from './client/query/BasedIterable.js';
|
|
15
18
|
export class BasedDb {
|
|
16
19
|
client;
|
|
17
20
|
server;
|
|
@@ -102,20 +105,26 @@ export class BasedDb {
|
|
|
102
105
|
start = function () {
|
|
103
106
|
return this.server.start.apply(this.server, arguments);
|
|
104
107
|
};
|
|
105
|
-
stop = function () {
|
|
108
|
+
stop = async function () {
|
|
109
|
+
await this.isModified();
|
|
106
110
|
this.client.stop();
|
|
107
111
|
return this.server.stop.apply(this.server, arguments);
|
|
108
112
|
};
|
|
109
|
-
save = function () {
|
|
113
|
+
save = async function () {
|
|
114
|
+
await this.isModified();
|
|
110
115
|
return this.server.save.apply(this.server, arguments);
|
|
111
116
|
};
|
|
112
117
|
migrateSchema = function () {
|
|
113
118
|
return this.server.migrateSchema.apply(this.server, arguments);
|
|
114
119
|
};
|
|
115
|
-
|
|
116
|
-
return this.client.
|
|
120
|
+
isModified = function () {
|
|
121
|
+
return this.client.isModified.apply(this.client, arguments);
|
|
122
|
+
};
|
|
123
|
+
schemaIsSet = function () {
|
|
124
|
+
return this.client.schemaIsSet.apply(this.client, arguments);
|
|
117
125
|
};
|
|
118
126
|
async destroy() {
|
|
127
|
+
await this.isModified();
|
|
119
128
|
// Tmp fix: Gives node time to GC existing buffers else it can incorrectly re-asign to mem
|
|
120
129
|
// Todo: clear all active queries, queues ETC
|
|
121
130
|
await wait(Math.max(this.client.hooks.flushTime + 10, 10));
|
|
@@ -131,5 +140,11 @@ export class BasedDb {
|
|
|
131
140
|
this.#init(opts);
|
|
132
141
|
await this.start({ clean: true });
|
|
133
142
|
}
|
|
143
|
+
on = function () {
|
|
144
|
+
return this.client.on.apply(this.client, arguments);
|
|
145
|
+
};
|
|
146
|
+
off = function () {
|
|
147
|
+
return this.client.on.apply(this.client, arguments);
|
|
148
|
+
};
|
|
134
149
|
}
|
|
135
150
|
//# sourceMappingURL=index.js.map
|
package/dist/src/native.d.ts
CHANGED
|
@@ -4,7 +4,7 @@ declare const _default: {
|
|
|
4
4
|
workerCtxInit: () => void;
|
|
5
5
|
externalFromInt(address: BigInt): any;
|
|
6
6
|
intFromExternal(external: any): BigInt;
|
|
7
|
-
modify: (data: Uint8Array, types: Uint8Array, dbCtx: any) => any;
|
|
7
|
+
modify: (data: Uint8Array, types: Uint8Array, dbCtx: any, dirtyBlocksOut: Float64Array) => any;
|
|
8
8
|
getQueryBuf: (q: Uint8Array, dbCtx: any) => ArrayBuffer | null;
|
|
9
9
|
start: (id: number) => any;
|
|
10
10
|
stop: (dbCtx: any) => any;
|
package/dist/src/native.js
CHANGED
|
@@ -28,8 +28,8 @@ export default {
|
|
|
28
28
|
intFromExternal(external) {
|
|
29
29
|
return db.intFromExternal(external);
|
|
30
30
|
},
|
|
31
|
-
modify: (data, types, dbCtx) => {
|
|
32
|
-
db.modify(data, types, dbCtx);
|
|
31
|
+
modify: (data, types, dbCtx, dirtyBlocksOut) => {
|
|
32
|
+
db.modify(data, types, dbCtx, dirtyBlocksOut);
|
|
33
33
|
},
|
|
34
34
|
getQueryBuf: (q, dbCtx) => {
|
|
35
35
|
const x = db.getQueryBuf(dbCtx, q);
|
|
@@ -199,8 +199,8 @@ export function createTree(createHash) {
|
|
|
199
199
|
return {
|
|
200
200
|
getRoot: () => root,
|
|
201
201
|
insert: (k, h, data = null) => {
|
|
202
|
-
if (!(h instanceof Uint8Array)) {
|
|
203
|
-
throw new TypeError('`h` must be a Uint8Array');
|
|
202
|
+
if (!(h instanceof Uint8Array)) {
|
|
203
|
+
throw new TypeError('`h` must be a Uint8Array');
|
|
204
204
|
}
|
|
205
205
|
const newLeaf = createLeaf(k, h, data);
|
|
206
206
|
root = root ? insert(root, newLeaf) : newLeaf;
|
|
@@ -25,7 +25,7 @@ export declare class DbWorker {
|
|
|
25
25
|
type OnSchemaChange = (schema: StrictSchema) => void;
|
|
26
26
|
export declare class DbServer {
|
|
27
27
|
#private;
|
|
28
|
-
|
|
28
|
+
modifyDirtyRanges: Float64Array;
|
|
29
29
|
dbCtxExternal: any;
|
|
30
30
|
schema: StrictSchema & {
|
|
31
31
|
lastId: number;
|
package/dist/src/server/index.js
CHANGED
|
@@ -46,6 +46,9 @@ export class DbWorker {
|
|
|
46
46
|
});
|
|
47
47
|
port1.on('message', (buf) => {
|
|
48
48
|
// TODO FIX TYPES CHECK IF THIS MAKES A COPY
|
|
49
|
+
// It's a copy, if you don't want a copy you'd need to make it an explicit view
|
|
50
|
+
// to the underlying buffer:
|
|
51
|
+
// new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength)
|
|
49
52
|
this.resolvers.shift()(new Uint8Array(buf));
|
|
50
53
|
this.db.onQueryEnd();
|
|
51
54
|
});
|
|
@@ -68,7 +71,7 @@ export class DbWorker {
|
|
|
68
71
|
}
|
|
69
72
|
}
|
|
70
73
|
export class DbServer {
|
|
71
|
-
|
|
74
|
+
modifyDirtyRanges;
|
|
72
75
|
dbCtxExternal; // pointer to zig dbCtx
|
|
73
76
|
schema = {
|
|
74
77
|
lastId: 1, // we reserve one for root props
|
|
@@ -96,6 +99,22 @@ export class DbServer {
|
|
|
96
99
|
this.sortIndexes = {};
|
|
97
100
|
this.onSchemaChange = onSchemaChange;
|
|
98
101
|
}
|
|
102
|
+
#resizeModifyDirtyRanges() {
|
|
103
|
+
let maxNrChanges = 0;
|
|
104
|
+
for (const typeId in this.schemaTypesParsedById) {
|
|
105
|
+
const def = this.schemaTypesParsedById[typeId];
|
|
106
|
+
const lastId = def.lastId;
|
|
107
|
+
const blockCapacity = def.blockCapacity;
|
|
108
|
+
const tmp = lastId - +!(lastId % def.blockCapacity);
|
|
109
|
+
const lastBlock = Math.ceil((((tmp / blockCapacity) | 0) * blockCapacity + 1) / blockCapacity);
|
|
110
|
+
maxNrChanges += lastBlock;
|
|
111
|
+
}
|
|
112
|
+
if (!this.modifyDirtyRanges ||
|
|
113
|
+
this.modifyDirtyRanges.length < maxNrChanges) {
|
|
114
|
+
const min = Math.max(maxNrChanges * 1.2, 1024) | 0;
|
|
115
|
+
this.modifyDirtyRanges = new Float64Array(min);
|
|
116
|
+
}
|
|
117
|
+
}
|
|
99
118
|
start(opts) {
|
|
100
119
|
return start(this, opts);
|
|
101
120
|
}
|
|
@@ -338,7 +357,7 @@ export class DbServer {
|
|
|
338
357
|
}
|
|
339
358
|
updateTypeDefs(this.schema, this.schemaTypesParsed, this.schemaTypesParsedById);
|
|
340
359
|
if (!fromStart) {
|
|
341
|
-
writeFile(join(this.fileSystemPath, SCHEMA_FILE), JSON.stringify(this.schema)).catch((err) => console.error(SCHEMA_FILE, err));
|
|
360
|
+
writeFile(join(this.fileSystemPath, SCHEMA_FILE), JSON.stringify(this.schema)).catch((err) => console.error('!!!', SCHEMA_FILE, err));
|
|
342
361
|
let types = Object.keys(this.schemaTypesParsed);
|
|
343
362
|
const s = schemaToSelvaBuffer(this.schemaTypesParsed);
|
|
344
363
|
for (let i = 0; i < s.length; i++) {
|
|
@@ -421,7 +440,14 @@ export class DbServer {
|
|
|
421
440
|
this.dirtyRanges.add(key);
|
|
422
441
|
i += 8;
|
|
423
442
|
}
|
|
424
|
-
|
|
443
|
+
this.#resizeModifyDirtyRanges();
|
|
444
|
+
native.modify(data, types, this.dbCtxExternal, this.modifyDirtyRanges);
|
|
445
|
+
for (let key of this.modifyDirtyRanges) {
|
|
446
|
+
if (key === 0) {
|
|
447
|
+
break;
|
|
448
|
+
}
|
|
449
|
+
this.dirtyRanges.add(key);
|
|
450
|
+
}
|
|
425
451
|
}
|
|
426
452
|
getQueryBuf(buf) {
|
|
427
453
|
if (this.modifyQueue.length) {
|
|
@@ -501,7 +527,13 @@ export class DbServer {
|
|
|
501
527
|
}
|
|
502
528
|
async destroy() {
|
|
503
529
|
await this.stop(true);
|
|
504
|
-
await rm(this.fileSystemPath, { recursive: true }).catch((err) =>
|
|
530
|
+
await rm(this.fileSystemPath, { recursive: true }).catch((err) => {
|
|
531
|
+
// console.warn(
|
|
532
|
+
// 'Error removing dump folder',
|
|
533
|
+
// this.fileSystemPath,
|
|
534
|
+
// err.message,
|
|
535
|
+
// ),
|
|
536
|
+
});
|
|
505
537
|
}
|
|
506
538
|
}
|
|
507
539
|
//# sourceMappingURL=index.js.map
|
|
@@ -5,7 +5,10 @@ import { Worker, MessageChannel, receiveMessageOnPort, } from 'node:worker_threa
|
|
|
5
5
|
import native from '../../native.js';
|
|
6
6
|
import './worker.js';
|
|
7
7
|
import { foreachDirtyBlock } from '../tree.js';
|
|
8
|
+
import { SCHEMA_FILE } from '../index.js';
|
|
8
9
|
import { fileURLToPath } from 'url';
|
|
10
|
+
import { deepMerge } from '@saulx/utils';
|
|
11
|
+
import { writeFile } from 'fs/promises';
|
|
9
12
|
const __filename = fileURLToPath(import.meta.url);
|
|
10
13
|
const __dirname = dirname(__filename);
|
|
11
14
|
const workerPath = join(__dirname, 'worker.js');
|
|
@@ -64,8 +67,9 @@ export const migrate = async (fromDbServer, toSchema, transform) => {
|
|
|
64
67
|
worker.on('error', console.error);
|
|
65
68
|
let i = 0;
|
|
66
69
|
let ranges = [];
|
|
67
|
-
fromDbServer.updateMerkleTree()
|
|
68
|
-
fromDbServer.dirtyRanges.clear()
|
|
70
|
+
// fromDbServer.updateMerkleTree()
|
|
71
|
+
// fromDbServer.dirtyRanges.clear()
|
|
72
|
+
await fromDbServer.save();
|
|
69
73
|
fromDbServer.merkleTree.visitLeafNodes((leaf) => {
|
|
70
74
|
ranges.push(leaf.data);
|
|
71
75
|
});
|
|
@@ -106,13 +110,13 @@ export const migrate = async (fromDbServer, toSchema, transform) => {
|
|
|
106
110
|
;
|
|
107
111
|
[schema, schemaTypesParsed] = msg.message;
|
|
108
112
|
}
|
|
109
|
-
fromDbServer.schema = schema;
|
|
110
|
-
fromDbServer.schemaTypesParsed = schemaTypesParsed;
|
|
113
|
+
fromDbServer.schema = deepMerge(toDb.server.schema, schema);
|
|
114
|
+
fromDbServer.schemaTypesParsed = deepMerge(toDb.server.schemaTypesParsed, schemaTypesParsed);
|
|
111
115
|
fromDbServer.dbCtxExternal = toCtx;
|
|
112
116
|
toDb.server.dbCtxExternal = fromCtx;
|
|
113
117
|
}
|
|
114
118
|
const promises = fromDbServer.workers.map((worker) => worker.updateCtx(toAddress));
|
|
115
|
-
promises.push(toDb.destroy(), worker.terminate());
|
|
119
|
+
promises.push(toDb.destroy(), worker.terminate(), fromDbServer.save({ forceFullDump: true }), writeFile(join(fromDbServer.fileSystemPath, SCHEMA_FILE), JSON.stringify(fromDbServer.schema)));
|
|
116
120
|
await Promise.all(promises);
|
|
117
121
|
fromDbServer.onSchemaChange?.(fromDbServer.schema);
|
|
118
122
|
return fromDbServer.schema;
|
|
@@ -2,6 +2,7 @@ import { isMainThread, receiveMessageOnPort, workerData, } from 'node:worker_thr
|
|
|
2
2
|
import native from '../../native.js';
|
|
3
3
|
import { BasedDb } from '../../index.js';
|
|
4
4
|
import { REFERENCE, REFERENCES } from '@based/schema/def';
|
|
5
|
+
import { isTypedArray } from 'node:util/types';
|
|
5
6
|
if (isMainThread) {
|
|
6
7
|
console.warn('running worker.ts in mainthread');
|
|
7
8
|
}
|
|
@@ -12,6 +13,27 @@ else {
|
|
|
12
13
|
const path = null;
|
|
13
14
|
const fromDb = new BasedDb({ path });
|
|
14
15
|
const toDb = new BasedDb({ path });
|
|
16
|
+
const cp = (obj) => {
|
|
17
|
+
let copy;
|
|
18
|
+
for (const key in obj) {
|
|
19
|
+
const val = obj[key];
|
|
20
|
+
if (typeof val === 'number') {
|
|
21
|
+
// only copy numbers
|
|
22
|
+
copy ??= Array.isArray(obj) ? [] : {};
|
|
23
|
+
copy[key] = val;
|
|
24
|
+
}
|
|
25
|
+
else if (typeof val === 'object' &&
|
|
26
|
+
val !== null &&
|
|
27
|
+
!isTypedArray(val)) {
|
|
28
|
+
const res = cp(val);
|
|
29
|
+
if (res) {
|
|
30
|
+
copy ??= Array.isArray(obj) ? [] : {};
|
|
31
|
+
copy[key] = cp(val);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
return copy;
|
|
36
|
+
};
|
|
15
37
|
fromDb.server.dbCtxExternal = fromCtx;
|
|
16
38
|
toDb.server.dbCtxExternal = toCtx;
|
|
17
39
|
await fromDb.setSchema(fromSchema, true);
|
|
@@ -71,7 +93,10 @@ else {
|
|
|
71
93
|
}
|
|
72
94
|
}
|
|
73
95
|
await toDb.drain();
|
|
74
|
-
channel.postMessage([
|
|
96
|
+
channel.postMessage([
|
|
97
|
+
cp(toDb.server.schema),
|
|
98
|
+
cp(toDb.server.schemaTypesParsed),
|
|
99
|
+
]);
|
|
75
100
|
// put it to sleep
|
|
76
101
|
atomics[0] = 0;
|
|
77
102
|
Atomics.notify(atomics, 0);
|
package/dist/src/server/save.js
CHANGED
|
@@ -2,13 +2,24 @@ import native from '../native.js';
|
|
|
2
2
|
import { isMainThread } from 'node:worker_threads';
|
|
3
3
|
import { writeFile } from 'node:fs/promises';
|
|
4
4
|
import { join } from 'node:path';
|
|
5
|
-
import { destructureCsmtKey, foreachBlock, foreachDirtyBlock, makeCsmtKey } from './tree.js';
|
|
5
|
+
import { destructureCsmtKey, foreachBlock, foreachDirtyBlock, makeCsmtKey, } from './tree.js';
|
|
6
6
|
import { WRITELOG_FILE } from './index.js';
|
|
7
7
|
import { writeFileSync } from 'node:fs';
|
|
8
8
|
import { bufToHex } from '../utils.js';
|
|
9
9
|
import { createTree } from './csmt/tree.js';
|
|
10
10
|
const COMMON_SDB_FILE = 'common.sdb';
|
|
11
11
|
const block_sdb_file = (typeId, start, end) => `${typeId}_${start}_${end}.sdb`;
|
|
12
|
+
function saveRange(db, typeId, start, end, hashOut) {
|
|
13
|
+
const file = block_sdb_file(typeId, start, end);
|
|
14
|
+
const path = join(db.fileSystemPath, file);
|
|
15
|
+
const err = native.saveRange(path, typeId, start, end, db.dbCtxExternal, hashOut);
|
|
16
|
+
if (err) {
|
|
17
|
+
// TODO print the error string
|
|
18
|
+
console.error(`Save ${typeId}:${start}-${end} failed: ${err}`);
|
|
19
|
+
return null;
|
|
20
|
+
}
|
|
21
|
+
return file;
|
|
22
|
+
}
|
|
12
23
|
export function save(db, sync = false, forceFullDump = false) {
|
|
13
24
|
if (!(isMainThread && (db.dirtyRanges.size || forceFullDump))) {
|
|
14
25
|
return;
|
|
@@ -26,12 +37,12 @@ export function save(db, sync = false, forceFullDump = false) {
|
|
|
26
37
|
const def = db.schemaTypesParsed[key];
|
|
27
38
|
foreachBlock(db, def, (start, end, _hash) => {
|
|
28
39
|
const typeId = def.id;
|
|
29
|
-
const
|
|
30
|
-
const
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
return;
|
|
40
|
+
const hash = new Uint8Array(16);
|
|
41
|
+
const file = saveRange(db, typeId, start, end, hash);
|
|
42
|
+
if (!file) {
|
|
43
|
+
// The previous state should remain in the merkle tree for
|
|
44
|
+
// load and sync purposes.
|
|
45
|
+
return;
|
|
35
46
|
}
|
|
36
47
|
const mtKey = makeCsmtKey(typeId, start);
|
|
37
48
|
const data = {
|
|
@@ -46,12 +57,9 @@ export function save(db, sync = false, forceFullDump = false) {
|
|
|
46
57
|
}
|
|
47
58
|
else {
|
|
48
59
|
foreachDirtyBlock(db, (mtKey, typeId, start, end) => {
|
|
49
|
-
const file = block_sdb_file(typeId, start, end);
|
|
50
|
-
const path = join(db.fileSystemPath, file);
|
|
51
60
|
const hash = new Uint8Array(16);
|
|
52
|
-
|
|
53
|
-
if (
|
|
54
|
-
console.error(`Save ${typeId}:${start}-${end} failed: ${err}`);
|
|
61
|
+
const file = saveRange(db, typeId, start, end, hash);
|
|
62
|
+
if (!file) {
|
|
55
63
|
// The previous state should remain in the merkle tree for
|
|
56
64
|
// load and sync purposes.
|
|
57
65
|
return;
|
package/dist/src/server/start.js
CHANGED
|
@@ -19,8 +19,6 @@ export async function start(db, opts) {
|
|
|
19
19
|
await rm(path, { recursive: true, force: true }).catch(noop);
|
|
20
20
|
}
|
|
21
21
|
await mkdir(path, { recursive: true }).catch(noop);
|
|
22
|
-
// not doing db yet
|
|
23
|
-
// db.modifyBuf = new SharedArrayBuffer(db.maxModifySize)
|
|
24
22
|
db.dbCtxExternal = native.start(id);
|
|
25
23
|
let writelog = null;
|
|
26
24
|
try {
|
package/dist/src/utils.d.ts
CHANGED
|
@@ -5,3 +5,9 @@ export declare function concatUint8Arr(bufs: Uint8Array[], totalByteLength?: num
|
|
|
5
5
|
export declare const bufToHex: (a: Uint8Array) => string;
|
|
6
6
|
export declare const hexToBuf: (s: string) => Uint8Array;
|
|
7
7
|
export declare const base64encode: (a: Uint8Array, lineMax?: number) => string;
|
|
8
|
+
export declare const readDoubleLE: (val: Uint8Array, offset: number) => number;
|
|
9
|
+
export declare const readFloatLE: (val: Uint8Array, offset: number) => number;
|
|
10
|
+
export declare const readUint32: (val: Uint8Array, offset: number) => number;
|
|
11
|
+
export declare const readInt32: (val: Uint8Array, offset: number) => number;
|
|
12
|
+
export declare const readInt16: (val: Uint8Array, offset: number) => number;
|
|
13
|
+
export declare const readUint16: (val: Uint8Array, offset: number) => number;
|
package/dist/src/utils.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { encodeBase64 } from '@saulx/utils';
|
|
2
|
-
const native =
|
|
2
|
+
const native = typeof window === 'undefined' ? (await import('./native.js')).default : null;
|
|
3
3
|
export const DECODER = new TextDecoder('utf-8');
|
|
4
4
|
export const ENCODER = new TextEncoder();
|
|
5
5
|
export const equals = (aB, bB) => {
|
|
@@ -28,7 +28,8 @@ export const equals = (aB, bB) => {
|
|
|
28
28
|
}
|
|
29
29
|
};
|
|
30
30
|
export function concatUint8Arr(bufs, totalByteLength) {
|
|
31
|
-
totalByteLength =
|
|
31
|
+
totalByteLength =
|
|
32
|
+
totalByteLength ?? bufs.reduce((acc, cur) => acc + cur.byteLength, 0);
|
|
32
33
|
const res = new Uint8Array(totalByteLength);
|
|
33
34
|
let off = 0;
|
|
34
35
|
for (let i = 0; i < bufs.length; i++) {
|
|
@@ -62,12 +63,12 @@ const intMap = {
|
|
|
62
63
|
'7': 0x7,
|
|
63
64
|
'8': 0x8,
|
|
64
65
|
'9': 0x9,
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
66
|
+
a: 0xa,
|
|
67
|
+
b: 0xb,
|
|
68
|
+
c: 0xc,
|
|
69
|
+
d: 0xd,
|
|
70
|
+
e: 0xe,
|
|
71
|
+
f: 0xf,
|
|
71
72
|
};
|
|
72
73
|
// Uint8Array.fromHex() and Uint8Array.toHex() are not available in V8
|
|
73
74
|
// https://issues.chromium.org/issues/42204568
|
|
@@ -85,7 +86,7 @@ function base64OutLen(n, lineMax) {
|
|
|
85
86
|
let olen;
|
|
86
87
|
/* This version would be with padding but we don't pad */
|
|
87
88
|
//olen = n * 4 / 3 + 4; /* 3-byte blocks to 4-byte */
|
|
88
|
-
olen = ((4 * n / 3
|
|
89
|
+
olen = ((4 * n) / 3 + 3) & ~3;
|
|
89
90
|
olen += lineMax > 0 ? olen / lineMax : 0; // line feeds
|
|
90
91
|
return olen;
|
|
91
92
|
}
|
|
@@ -99,4 +100,75 @@ export const base64encode = (a, lineMax = 72) => {
|
|
|
99
100
|
return DECODER.decode(native.base64encode(tmp, a, lineMax));
|
|
100
101
|
}
|
|
101
102
|
};
|
|
103
|
+
export const readDoubleLE = (val, offset) => {
|
|
104
|
+
const low = (val[offset] |
|
|
105
|
+
(val[offset + 1] << 8) |
|
|
106
|
+
(val[offset + 2] << 16) |
|
|
107
|
+
(val[offset + 3] << 24)) >>>
|
|
108
|
+
0;
|
|
109
|
+
const high = (val[offset + 4] |
|
|
110
|
+
(val[offset + 5] << 8) |
|
|
111
|
+
(val[offset + 6] << 16) |
|
|
112
|
+
(val[offset + 7] << 24)) >>>
|
|
113
|
+
0;
|
|
114
|
+
const sign = high >>> 31 ? -1 : 1;
|
|
115
|
+
let exponent = (high >>> 20) & 0x7ff;
|
|
116
|
+
let fraction = (high & 0xfffff) * 2 ** 32 + low;
|
|
117
|
+
if (exponent === 0x7ff) {
|
|
118
|
+
if (fraction === 0)
|
|
119
|
+
return sign * Infinity;
|
|
120
|
+
return NaN;
|
|
121
|
+
}
|
|
122
|
+
if (exponent === 0) {
|
|
123
|
+
if (fraction === 0)
|
|
124
|
+
return sign * 0;
|
|
125
|
+
exponent = 1;
|
|
126
|
+
}
|
|
127
|
+
else {
|
|
128
|
+
fraction += 2 ** 52;
|
|
129
|
+
}
|
|
130
|
+
return sign * fraction * 2 ** (exponent - 1075);
|
|
131
|
+
};
|
|
132
|
+
export const readFloatLE = (val, offset) => {
|
|
133
|
+
const bits = val[offset] |
|
|
134
|
+
(val[offset + 1] << 8) |
|
|
135
|
+
(val[offset + 2] << 16) |
|
|
136
|
+
(val[offset + 3] << 24);
|
|
137
|
+
const sign = bits >>> 31 ? -1 : 1;
|
|
138
|
+
let exponent = (bits >>> 23) & 0xff;
|
|
139
|
+
let fraction = bits & 0x7fffff;
|
|
140
|
+
if (exponent === 0xff) {
|
|
141
|
+
if (fraction === 0)
|
|
142
|
+
return sign * Infinity;
|
|
143
|
+
return NaN;
|
|
144
|
+
}
|
|
145
|
+
if (exponent === 0) {
|
|
146
|
+
if (fraction === 0)
|
|
147
|
+
return sign * 0;
|
|
148
|
+
exponent = 1;
|
|
149
|
+
}
|
|
150
|
+
else {
|
|
151
|
+
fraction |= 0x800000;
|
|
152
|
+
}
|
|
153
|
+
return sign * fraction * 2 ** (exponent - 150);
|
|
154
|
+
};
|
|
155
|
+
export const readUint32 = (val, offset) => {
|
|
156
|
+
return ((val[offset] |
|
|
157
|
+
(val[offset + 1] << 8) |
|
|
158
|
+
(val[offset + 2] << 16) |
|
|
159
|
+
(val[offset + 3] << 24)) >>>
|
|
160
|
+
0);
|
|
161
|
+
};
|
|
162
|
+
export const readInt32 = (val, offset) => {
|
|
163
|
+
return (val[offset] |
|
|
164
|
+
(val[offset + 1] << 8) |
|
|
165
|
+
(val[offset + 2] << 16) |
|
|
166
|
+
(val[offset + 3] << 24));
|
|
167
|
+
};
|
|
168
|
+
export const readInt16 = (val, offset) => {
|
|
169
|
+
return ((val[offset] | (val[offset + 1] << 8)) << 16) >> 16;
|
|
170
|
+
};
|
|
171
|
+
export const readUint16 = (val, offset) => {
|
|
172
|
+
return (val[offset] | (val[offset + 1] << 8)) >>> 0;
|
|
173
|
+
};
|
|
102
174
|
//# sourceMappingURL=utils.js.map
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@based/db",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.27",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/src/index.js",
|
|
@@ -32,9 +32,9 @@
|
|
|
32
32
|
"basedDbNative.cjs"
|
|
33
33
|
],
|
|
34
34
|
"dependencies": {
|
|
35
|
-
"@based/schema": "5.0.0-alpha.
|
|
35
|
+
"@based/schema": "5.0.0-alpha.8",
|
|
36
36
|
"@saulx/hash": "^3.0.0",
|
|
37
|
-
"@saulx/utils": "^
|
|
37
|
+
"@saulx/utils": "^6.1.1",
|
|
38
38
|
"exit-hook": "^4.0.0",
|
|
39
39
|
"picocolors": "^1.1.0",
|
|
40
40
|
"@based/crc32c": "^1.0.0"
|
|
@@ -43,6 +43,7 @@
|
|
|
43
43
|
"@based/locale-x86-64-gnu": "*"
|
|
44
44
|
},
|
|
45
45
|
"devDependencies": {
|
|
46
|
+
"jsondiffpatch": "^0.7.3",
|
|
46
47
|
"@based/crc32c": "^1.0.0",
|
|
47
48
|
"@types/node": "^22.5.3",
|
|
48
49
|
"axios": "^1.7.9",
|
|
@@ -1,6 +0,0 @@
|
|
|
1
|
-
export declare const readDoubleLE: (val: Uint8Array, offset: number) => number;
|
|
2
|
-
export declare const readFloatLE: (val: Uint8Array, offset: number) => number;
|
|
3
|
-
export declare const readUint32: (val: Uint8Array, offset: number) => number;
|
|
4
|
-
export declare const readInt32: (val: Uint8Array, offset: number) => number;
|
|
5
|
-
export declare const readInt16: (val: Uint8Array, offset: number) => number;
|
|
6
|
-
export declare const readUint16: (val: Uint8Array, offset: number) => number;
|
|
@@ -1,72 +0,0 @@
|
|
|
1
|
-
export const readDoubleLE = (val, offset) => {
|
|
2
|
-
const low = (val[offset] |
|
|
3
|
-
(val[offset + 1] << 8) |
|
|
4
|
-
(val[offset + 2] << 16) |
|
|
5
|
-
(val[offset + 3] << 24)) >>>
|
|
6
|
-
0;
|
|
7
|
-
const high = (val[offset + 4] |
|
|
8
|
-
(val[offset + 5] << 8) |
|
|
9
|
-
(val[offset + 6] << 16) |
|
|
10
|
-
(val[offset + 7] << 24)) >>>
|
|
11
|
-
0;
|
|
12
|
-
const sign = high >>> 31 ? -1 : 1;
|
|
13
|
-
let exponent = (high >>> 20) & 0x7ff;
|
|
14
|
-
let fraction = (high & 0xfffff) * 2 ** 32 + low;
|
|
15
|
-
if (exponent === 0x7ff) {
|
|
16
|
-
if (fraction === 0)
|
|
17
|
-
return sign * Infinity;
|
|
18
|
-
return NaN;
|
|
19
|
-
}
|
|
20
|
-
if (exponent === 0) {
|
|
21
|
-
if (fraction === 0)
|
|
22
|
-
return sign * 0;
|
|
23
|
-
exponent = 1;
|
|
24
|
-
}
|
|
25
|
-
else {
|
|
26
|
-
fraction += 2 ** 52;
|
|
27
|
-
}
|
|
28
|
-
return sign * fraction * 2 ** (exponent - 1075);
|
|
29
|
-
};
|
|
30
|
-
export const readFloatLE = (val, offset) => {
|
|
31
|
-
const bits = val[offset] |
|
|
32
|
-
(val[offset + 1] << 8) |
|
|
33
|
-
(val[offset + 2] << 16) |
|
|
34
|
-
(val[offset + 3] << 24);
|
|
35
|
-
const sign = bits >>> 31 ? -1 : 1;
|
|
36
|
-
let exponent = (bits >>> 23) & 0xff;
|
|
37
|
-
let fraction = bits & 0x7fffff;
|
|
38
|
-
if (exponent === 0xff) {
|
|
39
|
-
if (fraction === 0)
|
|
40
|
-
return sign * Infinity;
|
|
41
|
-
return NaN;
|
|
42
|
-
}
|
|
43
|
-
if (exponent === 0) {
|
|
44
|
-
if (fraction === 0)
|
|
45
|
-
return sign * 0;
|
|
46
|
-
exponent = 1;
|
|
47
|
-
}
|
|
48
|
-
else {
|
|
49
|
-
fraction |= 0x800000;
|
|
50
|
-
}
|
|
51
|
-
return sign * fraction * 2 ** (exponent - 150);
|
|
52
|
-
};
|
|
53
|
-
export const readUint32 = (val, offset) => {
|
|
54
|
-
return ((val[offset] |
|
|
55
|
-
(val[offset + 1] << 8) |
|
|
56
|
-
(val[offset + 2] << 16) |
|
|
57
|
-
(val[offset + 3] << 24)) >>>
|
|
58
|
-
0);
|
|
59
|
-
};
|
|
60
|
-
export const readInt32 = (val, offset) => {
|
|
61
|
-
return (val[offset] |
|
|
62
|
-
(val[offset + 1] << 8) |
|
|
63
|
-
(val[offset + 2] << 16) |
|
|
64
|
-
(val[offset + 3] << 24));
|
|
65
|
-
};
|
|
66
|
-
export const readInt16 = (val, offset) => {
|
|
67
|
-
return ((val[offset] | (val[offset + 1] << 8)) << 16) >> 16;
|
|
68
|
-
};
|
|
69
|
-
export const readUint16 = (val, offset) => {
|
|
70
|
-
return (val[offset] | (val[offset + 1] << 8)) >>> 0;
|
|
71
|
-
};
|
|
72
|
-
//# sourceMappingURL=bitWise.js.map
|