@based/db 0.0.52 → 0.0.56
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/darwin_aarch64/libdeflate.dylib +0 -0
- package/dist/lib/darwin_aarch64/libjemalloc_selva.2.dylib +0 -0
- package/dist/lib/darwin_aarch64/libnode-v20.node +0 -0
- package/dist/lib/darwin_aarch64/libnode-v21.node +0 -0
- package/dist/lib/darwin_aarch64/libnode-v22.node +0 -0
- package/dist/lib/darwin_aarch64/libnode-v23.node +0 -0
- package/dist/lib/darwin_aarch64/libnode-v24.node +0 -0
- package/dist/lib/darwin_aarch64/libselva.dylib +0 -0
- package/dist/lib/linux_aarch64/libnode-v20.node +0 -0
- package/dist/lib/linux_aarch64/libnode-v21.node +0 -0
- package/dist/lib/linux_aarch64/libnode-v22.node +0 -0
- package/dist/lib/linux_aarch64/libnode-v23.node +0 -0
- package/dist/lib/linux_aarch64/libnode-v24.node +0 -0
- package/dist/lib/linux_aarch64/libselva.so +0 -0
- package/dist/lib/linux_x86_64/libnode-v20.node +0 -0
- package/dist/lib/linux_x86_64/libnode-v21.node +0 -0
- package/dist/lib/linux_x86_64/libnode-v22.node +0 -0
- package/dist/lib/linux_x86_64/libnode-v23.node +0 -0
- package/dist/lib/linux_x86_64/libnode-v24.node +0 -0
- package/dist/lib/linux_x86_64/libselva.so +0 -0
- package/dist/src/client/flushModify.d.ts +1 -0
- package/dist/src/client/flushModify.js +8 -0
- package/dist/src/client/index.d.ts +8 -32
- package/dist/src/client/index.js +22 -84
- package/dist/src/client/modify/ModifyRes.js +4 -1
- package/dist/src/client/modify/references/reference.js +3 -0
- package/dist/src/client/query/BasedDbQuery.d.ts +2 -3
- package/dist/src/client/query/BasedDbQuery.js +150 -139
- package/dist/src/client/query/aggregates/aggregation.js +3 -0
- package/dist/src/client/query/registerQuery.js +16 -0
- package/dist/src/client/query/subscription/index.d.ts +1 -1
- package/dist/src/client/query/subscription/index.js +32 -9
- package/dist/src/client/setLocalClientSchema.d.ts +3 -0
- package/dist/src/client/setLocalClientSchema.js +30 -0
- package/dist/src/hooks.d.ts +9 -8
- package/dist/src/hooks.js +10 -2
- package/dist/src/index.d.ts +6 -7
- package/dist/src/index.js +24 -41
- package/dist/src/schema.d.ts +5 -2
- package/dist/src/schema.js +1 -93
- package/dist/src/server/DbWorker.d.ts +12 -0
- package/dist/src/server/DbWorker.js +42 -0
- package/dist/src/server/index.d.ts +6 -38
- package/dist/src/server/index.js +39 -146
- package/dist/src/server/migrate/index.d.ts +2 -2
- package/dist/src/server/migrate/index.js +66 -61
- package/dist/src/server/migrate/types.d.ts +4 -0
- package/dist/src/server/migrate/types.js +6 -0
- package/dist/src/server/migrate/utils.d.ts +3 -0
- package/dist/src/server/migrate/utils.js +16 -0
- package/dist/src/server/migrate/worker.js +18 -19
- package/dist/src/server/resizeModifyDirtyRanges.d.ts +2 -0
- package/dist/src/server/resizeModifyDirtyRanges.js +17 -0
- package/dist/src/server/save.d.ts +1 -1
- package/dist/src/server/save.js +5 -3
- package/dist/src/server/schema.d.ts +7 -0
- package/dist/src/server/schema.js +111 -0
- package/dist/src/server/start.js +10 -4
- package/dist/src/shared/DbBase.d.ts +13 -0
- package/dist/src/shared/DbBase.js +7 -0
- package/dist/src/shared/Emitter.d.ts +17 -0
- package/dist/src/shared/Emitter.js +66 -0
- package/dist/src/types.d.ts +3 -0
- package/dist/src/types.js +4 -0
- package/package.json +3 -3
|
@@ -1,13 +1,12 @@
|
|
|
1
|
-
import { BasedDb } from '../../index.js';
|
|
1
|
+
import { BasedDb, save } from '../../index.js';
|
|
2
2
|
import { dirname, join } from 'path';
|
|
3
|
-
import { tmpdir } from 'os';
|
|
4
3
|
import { Worker, MessageChannel, receiveMessageOnPort, } from 'node:worker_threads';
|
|
5
4
|
import native from '../../native.js';
|
|
6
5
|
import { destructureCsmtKey, foreachDirtyBlock, specialBlock } from '../tree.js';
|
|
7
|
-
import { SCHEMA_FILE } from '../index.js';
|
|
8
6
|
import { fileURLToPath } from 'url';
|
|
7
|
+
import { setNativeSchema, setSchemaOnServer, writeSchemaFile, } from '../schema.js';
|
|
8
|
+
import { setToAwake, waitUntilSleeping } from './utils.js';
|
|
9
9
|
import { deepMerge } from '@saulx/utils';
|
|
10
|
-
import { writeFile } from 'fs/promises';
|
|
11
10
|
const __filename = fileURLToPath(import.meta.url);
|
|
12
11
|
const __dirname = dirname(__filename);
|
|
13
12
|
const workerPath = join(__dirname, 'worker.js');
|
|
@@ -29,32 +28,33 @@ const parseTransform = (transform) => {
|
|
|
29
28
|
}
|
|
30
29
|
return res;
|
|
31
30
|
};
|
|
32
|
-
export const migrate = async (
|
|
33
|
-
const migrationId =
|
|
34
|
-
|
|
35
|
-
const abort = () =>
|
|
36
|
-
const
|
|
37
|
-
path:
|
|
31
|
+
export const migrate = async (server, fromSchema, toSchema, transform) => {
|
|
32
|
+
const migrationId = toSchema.hash;
|
|
33
|
+
server.migrating = migrationId;
|
|
34
|
+
const abort = () => server.migrating !== migrationId;
|
|
35
|
+
const tmpDb = new BasedDb({
|
|
36
|
+
path: null,
|
|
38
37
|
});
|
|
39
|
-
await
|
|
38
|
+
await tmpDb.start({ clean: true });
|
|
40
39
|
if (abort()) {
|
|
41
|
-
await
|
|
42
|
-
return
|
|
40
|
+
await tmpDb.destroy();
|
|
41
|
+
return;
|
|
43
42
|
}
|
|
44
|
-
|
|
43
|
+
setSchemaOnServer(tmpDb.server, toSchema);
|
|
44
|
+
// await writeSchemaFile(this, toSchema)
|
|
45
|
+
await setNativeSchema(tmpDb.server, toSchema);
|
|
45
46
|
if (abort()) {
|
|
46
|
-
await
|
|
47
|
-
return
|
|
47
|
+
await tmpDb.destroy();
|
|
48
|
+
return;
|
|
48
49
|
}
|
|
49
|
-
const
|
|
50
|
-
const
|
|
51
|
-
const toCtx = toDb.server.dbCtxExternal;
|
|
50
|
+
const fromCtx = server.dbCtxExternal;
|
|
51
|
+
const toCtx = tmpDb.server.dbCtxExternal;
|
|
52
52
|
const { port1, port2 } = new MessageChannel();
|
|
53
|
-
const
|
|
53
|
+
const workerState = new Int32Array(new SharedArrayBuffer(4));
|
|
54
54
|
const fromAddress = native.intFromExternal(fromCtx);
|
|
55
55
|
const toAddress = native.intFromExternal(toCtx);
|
|
56
56
|
const transformFns = parseTransform(transform);
|
|
57
|
-
|
|
57
|
+
setToAwake(workerState, false);
|
|
58
58
|
const worker = new Worker(workerPath, {
|
|
59
59
|
workerData: {
|
|
60
60
|
isDbMigrateWorker: true,
|
|
@@ -63,80 +63,85 @@ export const migrate = async (fromDbServer, toSchema, transform) => {
|
|
|
63
63
|
fromSchema,
|
|
64
64
|
toSchema,
|
|
65
65
|
channel: port2,
|
|
66
|
-
|
|
66
|
+
workerState,
|
|
67
67
|
transformFns,
|
|
68
68
|
},
|
|
69
69
|
transferList: [port2],
|
|
70
70
|
});
|
|
71
|
+
// handle?
|
|
71
72
|
worker.on('error', console.error);
|
|
73
|
+
// Block handling
|
|
72
74
|
let i = 0;
|
|
73
|
-
let
|
|
74
|
-
await
|
|
75
|
-
|
|
75
|
+
let rangesToMigrate = [];
|
|
76
|
+
await save(server, false, false, true);
|
|
77
|
+
server.merkleTree.visitLeafNodes((leaf) => {
|
|
76
78
|
const [_typeId, start] = destructureCsmtKey(leaf.key);
|
|
77
79
|
if (start == specialBlock)
|
|
78
80
|
return; // skip the type specialBlock
|
|
79
|
-
|
|
81
|
+
rangesToMigrate.push(leaf.data);
|
|
80
82
|
});
|
|
81
|
-
await
|
|
82
|
-
while (i <
|
|
83
|
+
await waitUntilSleeping(workerState);
|
|
84
|
+
while (i < rangesToMigrate.length) {
|
|
83
85
|
if (abort()) {
|
|
84
86
|
break;
|
|
85
87
|
}
|
|
86
88
|
// block modifies
|
|
87
|
-
|
|
88
|
-
const leafData =
|
|
89
|
+
server.processingQueries++;
|
|
90
|
+
const leafData = rangesToMigrate[i++];
|
|
89
91
|
port1.postMessage(leafData);
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
Atomics.notify(atomics, 0);
|
|
93
|
-
// wait until it's done
|
|
94
|
-
await Atomics.waitAsync(atomics, 0, 1).value;
|
|
92
|
+
setToAwake(workerState, true);
|
|
93
|
+
await waitUntilSleeping(workerState);
|
|
95
94
|
// exec queued modifies
|
|
96
|
-
|
|
97
|
-
if (i ===
|
|
98
|
-
if (
|
|
99
|
-
|
|
95
|
+
server.onQueryEnd();
|
|
96
|
+
if (i === rangesToMigrate.length) {
|
|
97
|
+
if (server.dirtyRanges.size) {
|
|
98
|
+
rangesToMigrate = [];
|
|
100
99
|
i = 0;
|
|
101
|
-
foreachDirtyBlock(
|
|
102
|
-
|
|
100
|
+
foreachDirtyBlock(server, (_mtKey, typeId, start, end) => {
|
|
101
|
+
rangesToMigrate.push({
|
|
103
102
|
typeId,
|
|
104
103
|
start,
|
|
105
104
|
end,
|
|
106
105
|
});
|
|
107
106
|
});
|
|
108
|
-
|
|
107
|
+
server.dirtyRanges.clear();
|
|
109
108
|
}
|
|
110
109
|
}
|
|
111
110
|
}
|
|
111
|
+
// ---------------------------------
|
|
112
112
|
if (abort()) {
|
|
113
|
-
await Promise.all([
|
|
114
|
-
return
|
|
113
|
+
await Promise.all([tmpDb.destroy(), worker.terminate()]);
|
|
114
|
+
return;
|
|
115
115
|
}
|
|
116
116
|
let msg;
|
|
117
|
-
let schema;
|
|
118
117
|
let schemaTypesParsed;
|
|
119
118
|
while ((msg = receiveMessageOnPort(port1))) {
|
|
120
|
-
;
|
|
121
|
-
[schema, schemaTypesParsed] = msg.message;
|
|
119
|
+
schemaTypesParsed = msg.message;
|
|
122
120
|
}
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
121
|
+
server.dbCtxExternal = toCtx;
|
|
122
|
+
server.sortIndexes = {};
|
|
123
|
+
// ----------------MAKE NICE THIS------------------
|
|
124
|
+
// pass last node IDS { type: lastId }
|
|
125
|
+
setSchemaOnServer(server, toSchema);
|
|
126
|
+
// make schema util for this later
|
|
127
|
+
server.schemaTypesParsed = deepMerge(tmpDb.server.schemaTypesParsed, schemaTypesParsed);
|
|
128
|
+
server.schemaTypesParsedById = {};
|
|
129
|
+
for (const key in server.schemaTypesParsed) {
|
|
130
|
+
const def = server.schemaTypesParsed[key];
|
|
131
|
+
server.schemaTypesParsedById[def.id] = def;
|
|
131
132
|
}
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
133
|
+
// -----------------------------------------
|
|
134
|
+
tmpDb.server.dbCtxExternal = fromCtx;
|
|
135
|
+
// TODO makes this SYNC
|
|
136
|
+
const promises = server.workers.map((worker) => worker.updateCtx(toAddress));
|
|
137
|
+
promises.push(tmpDb.destroy(), worker.terminate());
|
|
135
138
|
await Promise.all(promises);
|
|
136
139
|
if (abort()) {
|
|
137
|
-
return
|
|
140
|
+
return;
|
|
138
141
|
}
|
|
139
|
-
|
|
140
|
-
|
|
142
|
+
await save(server, false, true, true);
|
|
143
|
+
await writeSchemaFile(server, toSchema);
|
|
144
|
+
server.migrating = 0;
|
|
145
|
+
process.nextTick(() => server.emit('schema', server.schema));
|
|
141
146
|
};
|
|
142
147
|
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { MigrationState } from './types.js';
|
|
2
|
+
export const waitUntilSleeping = async (workerState) => {
|
|
3
|
+
await Atomics.waitAsync(workerState, 0, MigrationState.AWAKE).value;
|
|
4
|
+
};
|
|
5
|
+
export const setToAwake = (workerState, notify) => {
|
|
6
|
+
workerState[0] = MigrationState.AWAKE;
|
|
7
|
+
if (notify) {
|
|
8
|
+
Atomics.notify(workerState, 0);
|
|
9
|
+
}
|
|
10
|
+
};
|
|
11
|
+
export const setToSleep = (workerState) => {
|
|
12
|
+
workerState[0] = MigrationState.SLEEP;
|
|
13
|
+
Atomics.notify(workerState, 0);
|
|
14
|
+
Atomics.wait(workerState, 0, MigrationState.SLEEP);
|
|
15
|
+
};
|
|
16
|
+
//# sourceMappingURL=utils.js.map
|
|
@@ -3,16 +3,18 @@ import native from '../../native.js';
|
|
|
3
3
|
import { BasedDb } from '../../index.js';
|
|
4
4
|
import { REFERENCE, REFERENCES } from '@based/schema/def';
|
|
5
5
|
import { isTypedArray } from 'node:util/types';
|
|
6
|
+
import { setSchemaOnServer } from '../schema.js';
|
|
7
|
+
import { setToSleep } from './utils.js';
|
|
8
|
+
import { setLocalClientSchema } from '../../client/setLocalClientSchema.js';
|
|
6
9
|
if (isMainThread) {
|
|
7
10
|
console.warn('running worker.ts in mainthread');
|
|
8
11
|
}
|
|
9
12
|
else if (workerData?.isDbMigrateWorker) {
|
|
10
|
-
const { from, to, fromSchema, toSchema, channel,
|
|
13
|
+
const { from, to, fromSchema, toSchema, channel, workerState, transformFns } = workerData;
|
|
11
14
|
const fromCtx = native.externalFromInt(from);
|
|
12
15
|
const toCtx = native.externalFromInt(to);
|
|
13
|
-
const
|
|
14
|
-
const
|
|
15
|
-
const toDb = new BasedDb({ path });
|
|
16
|
+
const fromDb = new BasedDb({ path: null });
|
|
17
|
+
const toDb = new BasedDb({ path: null });
|
|
16
18
|
const cp = (obj) => {
|
|
17
19
|
let copy;
|
|
18
20
|
for (const key in obj) {
|
|
@@ -36,11 +38,13 @@ else if (workerData?.isDbMigrateWorker) {
|
|
|
36
38
|
};
|
|
37
39
|
fromDb.server.dbCtxExternal = fromCtx;
|
|
38
40
|
toDb.server.dbCtxExternal = toCtx;
|
|
39
|
-
|
|
40
|
-
|
|
41
|
+
setSchemaOnServer(fromDb.server, fromSchema);
|
|
42
|
+
setSchemaOnServer(toDb.server, toSchema);
|
|
43
|
+
setLocalClientSchema(fromDb.client, fromDb.server.schema);
|
|
44
|
+
setLocalClientSchema(toDb.client, toDb.server.schema);
|
|
41
45
|
const map = {};
|
|
42
|
-
for (const type in fromDb.
|
|
43
|
-
const { id, props } = fromDb.
|
|
46
|
+
for (const type in fromDb.server.schemaTypesParsed) {
|
|
47
|
+
const { id, props } = fromDb.server.schemaTypesParsed[type];
|
|
44
48
|
const include = Object.keys(props);
|
|
45
49
|
let i = include.length;
|
|
46
50
|
while (i--) {
|
|
@@ -77,7 +81,7 @@ else if (workerData?.isDbMigrateWorker) {
|
|
|
77
81
|
const nodes = fromDb
|
|
78
82
|
.query(type)
|
|
79
83
|
.include(include)
|
|
80
|
-
.range(leafData.start - 1, leafData.end
|
|
84
|
+
.range(leafData.start - 1, leafData.end)
|
|
81
85
|
._getSync(fromCtx);
|
|
82
86
|
for (const node of nodes) {
|
|
83
87
|
const res = typeTransformFn(node);
|
|
@@ -92,11 +96,11 @@ else if (workerData?.isDbMigrateWorker) {
|
|
|
92
96
|
}
|
|
93
97
|
}
|
|
94
98
|
}
|
|
95
|
-
else if (type in toDb.
|
|
99
|
+
else if (type in toDb.server.schemaTypesParsed) {
|
|
96
100
|
const nodes = fromDb
|
|
97
101
|
.query(type)
|
|
98
102
|
.include(include)
|
|
99
|
-
.range(leafData.start - 1, leafData.end
|
|
103
|
+
.range(leafData.start - 1, leafData.end)
|
|
100
104
|
._getSync(fromCtx);
|
|
101
105
|
for (const node of nodes) {
|
|
102
106
|
toDb.create(type, node, { unsafe: true });
|
|
@@ -104,14 +108,9 @@ else if (workerData?.isDbMigrateWorker) {
|
|
|
104
108
|
}
|
|
105
109
|
}
|
|
106
110
|
await toDb.drain();
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
]);
|
|
111
|
-
// put it to sleep
|
|
112
|
-
atomics[0] = 0;
|
|
113
|
-
Atomics.notify(atomics, 0);
|
|
114
|
-
Atomics.wait(atomics, 0, 0);
|
|
111
|
+
// WE ARE ONLY GOING TO SEND { type: lastNodeId }
|
|
112
|
+
channel.postMessage(cp(toDb.server.schemaTypesParsed));
|
|
113
|
+
setToSleep(workerState);
|
|
115
114
|
}
|
|
116
115
|
}
|
|
117
116
|
else {
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
export const resizeModifyDirtyRanges = (server) => {
|
|
2
|
+
let maxNrChanges = 0;
|
|
3
|
+
for (const typeId in server.schemaTypesParsedById) {
|
|
4
|
+
const def = server.schemaTypesParsedById[typeId];
|
|
5
|
+
const lastId = def.lastId;
|
|
6
|
+
const blockCapacity = def.blockCapacity;
|
|
7
|
+
const tmp = lastId - +!(lastId % def.blockCapacity);
|
|
8
|
+
const lastBlock = Math.ceil((((tmp / blockCapacity) | 0) * blockCapacity + 1) / blockCapacity);
|
|
9
|
+
maxNrChanges += lastBlock;
|
|
10
|
+
}
|
|
11
|
+
if (!server.modifyDirtyRanges ||
|
|
12
|
+
server.modifyDirtyRanges.length < maxNrChanges) {
|
|
13
|
+
const min = Math.max(maxNrChanges * 1.2, 1024) | 0;
|
|
14
|
+
server.modifyDirtyRanges = new Float64Array(min);
|
|
15
|
+
}
|
|
16
|
+
};
|
|
17
|
+
//# sourceMappingURL=resizeModifyDirtyRanges.js.map
|
|
@@ -18,4 +18,4 @@ export type Writelog = {
|
|
|
18
18
|
}[];
|
|
19
19
|
};
|
|
20
20
|
};
|
|
21
|
-
export declare function save<T extends boolean>(db: DbServer, sync?: T, forceFullDump?: boolean): T extends true ? void : Promise<void>;
|
|
21
|
+
export declare function save<T extends boolean>(db: DbServer, sync?: T, forceFullDump?: boolean, skipMigrationCheck?: boolean): T extends true ? void : Promise<void>;
|
package/dist/src/server/save.js
CHANGED
|
@@ -3,10 +3,9 @@ import { isMainThread } from 'node:worker_threads';
|
|
|
3
3
|
import { writeFile } from 'node:fs/promises';
|
|
4
4
|
import { join } from 'node:path';
|
|
5
5
|
import { destructureCsmtKey, foreachBlock, foreachDirtyBlock, initCsmt, makeCsmtKey, specialBlock, } from './tree.js';
|
|
6
|
-
import { WRITELOG_FILE } from './index.js';
|
|
7
6
|
import { writeFileSync } from 'node:fs';
|
|
8
7
|
import { bufToHex } from '@saulx/utils';
|
|
9
|
-
|
|
8
|
+
import { COMMON_SDB_FILE, WRITELOG_FILE } from '../types.js';
|
|
10
9
|
const block_sdb_file = (typeId, start, end) => `${typeId}_${start}_${end}.sdb`;
|
|
11
10
|
function saveRange(db, typeId, start, end, hashOut) {
|
|
12
11
|
const file = block_sdb_file(typeId, start, end);
|
|
@@ -23,10 +22,13 @@ function saveRange(db, typeId, start, end, hashOut) {
|
|
|
23
22
|
}
|
|
24
23
|
return file;
|
|
25
24
|
}
|
|
26
|
-
export function save(db, sync = false, forceFullDump = false) {
|
|
25
|
+
export function save(db, sync = false, forceFullDump = false, skipMigrationCheck = false) {
|
|
27
26
|
if (!(isMainThread && (db.dirtyRanges.size || forceFullDump))) {
|
|
28
27
|
return;
|
|
29
28
|
}
|
|
29
|
+
if (db.migrating && !skipMigrationCheck) {
|
|
30
|
+
return;
|
|
31
|
+
}
|
|
30
32
|
let err;
|
|
31
33
|
const ts = Date.now();
|
|
32
34
|
err = native.saveCommon(join(db.fileSystemPath, COMMON_SDB_FILE), db.dbCtxExternal);
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { DbServer } from './index.js';
|
|
2
|
+
import { DbSchema } from '../schema.js';
|
|
3
|
+
import { StrictSchema } from '@based/schema';
|
|
4
|
+
export declare const setSchemaOnServer: (server: DbServer, schema: DbSchema) => void;
|
|
5
|
+
export declare const writeSchemaFile: (server: DbServer, schema: DbSchema) => Promise<void>;
|
|
6
|
+
export declare const setNativeSchema: (server: DbServer, schema: DbSchema) => Promise<void>;
|
|
7
|
+
export declare const strictSchemaToDbSchema: (schema: StrictSchema) => DbSchema;
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import { schemaToSelvaBuffer, updateTypeDefs } from '@based/schema/def';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { writeFile } from 'node:fs/promises';
|
|
4
|
+
import native from '../native.js';
|
|
5
|
+
import { initCsmt, makeCsmtKey } from './tree.js';
|
|
6
|
+
import { deepCopy, writeUint64 } from '@saulx/utils';
|
|
7
|
+
import { SCHEMA_FILE } from '../types.js';
|
|
8
|
+
import { hash } from '@saulx/hash';
|
|
9
|
+
import { getPropType } from '@based/schema';
|
|
10
|
+
export const setSchemaOnServer = (server, schema) => {
|
|
11
|
+
const { schemaTypesParsed, schemaTypesParsedById } = updateTypeDefs(schema);
|
|
12
|
+
server.schema = schema;
|
|
13
|
+
server.schemaTypesParsed = schemaTypesParsed;
|
|
14
|
+
server.schemaTypesParsedById = schemaTypesParsedById;
|
|
15
|
+
};
|
|
16
|
+
export const writeSchemaFile = async (server, schema) => {
|
|
17
|
+
const schemaFilePath = join(server.fileSystemPath, SCHEMA_FILE);
|
|
18
|
+
try {
|
|
19
|
+
await writeFile(schemaFilePath, JSON.stringify(schema));
|
|
20
|
+
}
|
|
21
|
+
catch (err) {
|
|
22
|
+
throw new Error(`Error writing schema to a file path ${schemaFilePath}}`);
|
|
23
|
+
}
|
|
24
|
+
};
|
|
25
|
+
export const setNativeSchema = async (server, schema) => {
|
|
26
|
+
const types = Object.keys(server.schemaTypesParsed);
|
|
27
|
+
const s = schemaToSelvaBuffer(server.schemaTypesParsed);
|
|
28
|
+
for (let i = 0; i < s.length; i++) {
|
|
29
|
+
const type = server.schemaTypesParsed[types[i]];
|
|
30
|
+
try {
|
|
31
|
+
native.updateSchemaType(type.id, new Uint8Array(s[i]), server.dbCtxExternal);
|
|
32
|
+
}
|
|
33
|
+
catch (err) {
|
|
34
|
+
throw new Error(`Cannot update schema on selva (native) ${type.type} ${err.message}`);
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
// Insert a root node
|
|
38
|
+
if (schema.types._root) {
|
|
39
|
+
// TODO fix server add it in schema at least
|
|
40
|
+
const data = [2, 1, 0, 0, 0, 1, 9, 1, 0, 0, 0, 7, 1, 0, 1];
|
|
41
|
+
const blockKey = makeCsmtKey(1, 1);
|
|
42
|
+
const buf = new Uint8Array(8 + data.length + 2 + 8 + 4);
|
|
43
|
+
const view = new DataView(buf.buffer, 0, buf.byteLength);
|
|
44
|
+
// set schema hash
|
|
45
|
+
writeUint64(buf, server.schema.hash, 0);
|
|
46
|
+
// add content
|
|
47
|
+
buf.set(data, 8);
|
|
48
|
+
// add typesLen
|
|
49
|
+
view.setFloat64(8 + data.length, 0, true);
|
|
50
|
+
// add dirty key
|
|
51
|
+
view.setFloat64(8 + data.length + 2, blockKey, true);
|
|
52
|
+
// add dataLen
|
|
53
|
+
view.setUint32(buf.length - 4, data.length, true);
|
|
54
|
+
server.modify(buf);
|
|
55
|
+
initCsmt(server);
|
|
56
|
+
}
|
|
57
|
+
};
|
|
58
|
+
export const strictSchemaToDbSchema = (schema) => {
|
|
59
|
+
// @ts-ignore
|
|
60
|
+
let dbSchema = deepCopy(schema);
|
|
61
|
+
// reserve 1 for root (even if you dont have it)
|
|
62
|
+
dbSchema.lastId = 1;
|
|
63
|
+
if (dbSchema.props) {
|
|
64
|
+
for (const key in dbSchema.props) {
|
|
65
|
+
const prop = dbSchema.props[key];
|
|
66
|
+
const propType = getPropType(prop);
|
|
67
|
+
let refProp;
|
|
68
|
+
if (propType === 'reference') {
|
|
69
|
+
refProp = prop;
|
|
70
|
+
}
|
|
71
|
+
else if (propType === 'references') {
|
|
72
|
+
refProp = prop.items;
|
|
73
|
+
prop.items = refProp;
|
|
74
|
+
}
|
|
75
|
+
if (refProp) {
|
|
76
|
+
const type = dbSchema.types[refProp.ref];
|
|
77
|
+
const inverseKey = '_' + key;
|
|
78
|
+
dbSchema.types[refProp.ref] = {
|
|
79
|
+
...type,
|
|
80
|
+
props: {
|
|
81
|
+
...type.props,
|
|
82
|
+
[inverseKey]: {
|
|
83
|
+
items: {
|
|
84
|
+
ref: '_root',
|
|
85
|
+
prop: key,
|
|
86
|
+
},
|
|
87
|
+
},
|
|
88
|
+
},
|
|
89
|
+
};
|
|
90
|
+
refProp.prop = inverseKey;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
dbSchema.types ??= {};
|
|
94
|
+
// @ts-ignore This creates an internal type to use for root props
|
|
95
|
+
dbSchema.types._root = {
|
|
96
|
+
id: 1,
|
|
97
|
+
props: dbSchema.props,
|
|
98
|
+
};
|
|
99
|
+
delete dbSchema.props;
|
|
100
|
+
}
|
|
101
|
+
for (const field in dbSchema.types) {
|
|
102
|
+
if (!('id' in dbSchema.types[field])) {
|
|
103
|
+
dbSchema.lastId++;
|
|
104
|
+
dbSchema.types[field].id = dbSchema.lastId;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
const { hash: _, ...rest } = dbSchema;
|
|
108
|
+
dbSchema.hash = hash(rest);
|
|
109
|
+
return dbSchema;
|
|
110
|
+
};
|
|
111
|
+
//# sourceMappingURL=schema.js.map
|
package/dist/src/server/start.js
CHANGED
|
@@ -1,13 +1,15 @@
|
|
|
1
|
-
import { DbWorker
|
|
1
|
+
import { DbWorker } from './DbWorker.js';
|
|
2
2
|
import native from '../native.js';
|
|
3
3
|
import { rm, mkdir, readFile } from 'node:fs/promises';
|
|
4
4
|
import { join } from 'node:path';
|
|
5
|
-
import { destructureCsmtKey, foreachBlock, initCsmt, makeCsmtKey, specialBlock } from './tree.js';
|
|
5
|
+
import { destructureCsmtKey, foreachBlock, initCsmt, makeCsmtKey, specialBlock, } from './tree.js';
|
|
6
6
|
import { availableParallelism } from 'node:os';
|
|
7
7
|
import exitHook from 'exit-hook';
|
|
8
8
|
import { save } from './save.js';
|
|
9
9
|
import { DEFAULT_BLOCK_CAPACITY } from '@based/schema/def';
|
|
10
10
|
import { bufToHex } from '@saulx/utils';
|
|
11
|
+
import { SCHEMA_FILE, WRITELOG_FILE } from '../types.js';
|
|
12
|
+
import { setSchemaOnServer } from './schema.js';
|
|
11
13
|
export async function start(db, opts) {
|
|
12
14
|
const path = db.fileSystemPath;
|
|
13
15
|
const noop = () => { };
|
|
@@ -45,7 +47,8 @@ export async function start(db, opts) {
|
|
|
45
47
|
const schema = await readFile(join(path, SCHEMA_FILE));
|
|
46
48
|
if (schema) {
|
|
47
49
|
// Prop need to not call setting in selva
|
|
48
|
-
|
|
50
|
+
setSchemaOnServer(db, JSON.parse(schema.toString()));
|
|
51
|
+
// setNativeSchema might still be nessecary...
|
|
49
52
|
}
|
|
50
53
|
}
|
|
51
54
|
catch (err) {
|
|
@@ -91,7 +94,7 @@ export async function start(db, opts) {
|
|
|
91
94
|
return; // skip the type specialBlock
|
|
92
95
|
newHashSet.add(bufToHex(hash));
|
|
93
96
|
});
|
|
94
|
-
const setEq = (a, b) => a.size === b.size && [...a].every(value => b.has(value));
|
|
97
|
+
const setEq = (a, b) => a.size === b.size && [...a].every((value) => b.has(value));
|
|
95
98
|
if (!setEq(oldHashSet, newHashSet)) {
|
|
96
99
|
console.error(`WARN: CSMT hash mismatch.`);
|
|
97
100
|
}
|
|
@@ -122,5 +125,8 @@ export async function start(db, opts) {
|
|
|
122
125
|
save(db);
|
|
123
126
|
}, db.saveIntervalInSeconds * 1e3);
|
|
124
127
|
}
|
|
128
|
+
if (db.schema) {
|
|
129
|
+
db.emit('schema', db.schema);
|
|
130
|
+
}
|
|
125
131
|
}
|
|
126
132
|
//# sourceMappingURL=start.js.map
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { SchemaTypeDef } from '@based/schema/def';
|
|
2
|
+
import { DbSchema } from '../schema.js';
|
|
3
|
+
import { Emitter } from './Emitter.js';
|
|
4
|
+
export type EventMap = {
|
|
5
|
+
schema: DbSchema;
|
|
6
|
+
};
|
|
7
|
+
export type Event = keyof EventMap;
|
|
8
|
+
export type Listener<T> = (data: T) => void;
|
|
9
|
+
export declare class DbShared extends Emitter {
|
|
10
|
+
schema?: DbSchema;
|
|
11
|
+
schemaTypesParsed?: Record<string, SchemaTypeDef>;
|
|
12
|
+
schemaTypesParsedById?: Record<number, SchemaTypeDef>;
|
|
13
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { DbSchema } from '../schema.js';
|
|
2
|
+
export type EventMap = {
|
|
3
|
+
schema: DbSchema;
|
|
4
|
+
};
|
|
5
|
+
export type Event = keyof EventMap;
|
|
6
|
+
export type Listener<T> = (data: T) => void;
|
|
7
|
+
export declare class Emitter {
|
|
8
|
+
listeners: {
|
|
9
|
+
[E in Event]?: Listener<EventMap[E]>[];
|
|
10
|
+
};
|
|
11
|
+
emit<E extends Event>(type: E, val: EventMap[E]): void;
|
|
12
|
+
on<E extends Event>(type: E, fn: Listener<EventMap[E]>): void;
|
|
13
|
+
removeAllListeners(): void;
|
|
14
|
+
once<E extends Event>(type: E): Promise<EventMap[E]>;
|
|
15
|
+
once<E extends Event>(type: E, fn: Listener<EventMap[E]>): void;
|
|
16
|
+
off<E extends Event>(type: E, fn?: Listener<EventMap[E]>): void;
|
|
17
|
+
}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
export class Emitter {
|
|
2
|
+
listeners = {};
|
|
3
|
+
emit(type, val) {
|
|
4
|
+
if (this.listeners[type]) {
|
|
5
|
+
const lis = this.listeners[type];
|
|
6
|
+
for (let i = 0, len = lis.length; i < lis.length; i++) {
|
|
7
|
+
const fn = lis[i];
|
|
8
|
+
// @ts-ignore
|
|
9
|
+
fn(val);
|
|
10
|
+
if (len > lis.length) {
|
|
11
|
+
if (lis[i] !== fn) {
|
|
12
|
+
i--;
|
|
13
|
+
len = lis.length;
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
on(type, fn) {
|
|
20
|
+
if (!this.listeners[type]) {
|
|
21
|
+
this.listeners[type] = [];
|
|
22
|
+
}
|
|
23
|
+
this.listeners[type].push(fn);
|
|
24
|
+
}
|
|
25
|
+
removeAllListeners() {
|
|
26
|
+
this.listeners = {};
|
|
27
|
+
}
|
|
28
|
+
once(type, fn) {
|
|
29
|
+
if (!fn) {
|
|
30
|
+
return new Promise((resolve) => {
|
|
31
|
+
const listener = (v) => {
|
|
32
|
+
resolve(v);
|
|
33
|
+
this.off(type, listener);
|
|
34
|
+
};
|
|
35
|
+
this.on(type, listener);
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
// TODO: optmize this
|
|
39
|
+
const listener = () => {
|
|
40
|
+
this.off(type, listener);
|
|
41
|
+
this.off(type, fn);
|
|
42
|
+
};
|
|
43
|
+
this.on(type, fn);
|
|
44
|
+
this.on(type, listener);
|
|
45
|
+
}
|
|
46
|
+
off(type, fn) {
|
|
47
|
+
const listeners = this.listeners[type];
|
|
48
|
+
if (listeners) {
|
|
49
|
+
if (!fn) {
|
|
50
|
+
delete this.listeners[type];
|
|
51
|
+
}
|
|
52
|
+
else {
|
|
53
|
+
for (let i = 0; i < listeners.length; i++) {
|
|
54
|
+
if (listeners[i] === fn) {
|
|
55
|
+
listeners.splice(i, 1);
|
|
56
|
+
break;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
if (listeners.length === 0) {
|
|
60
|
+
delete this.listeners[type];
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
//# sourceMappingURL=Emitter.js.map
|