@based/db 0.0.1

package/dist/src/server/start.js

@@ -0,0 +1,86 @@
+import { stringHash } from '@saulx/hash';
+import { DbWorker } from './index.js';
+import native from '../native.js';
+import { rm, mkdir, readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { createTree } from './csmt/index.js';
+import { foreachBlock } from './tree.js';
+import { availableParallelism } from 'node:os';
+import './worker.js';
+const SCHEMA_FILE = 'schema.json';
+const WRITELOG_FILE = 'writelog.json';
+export const DEFAULT_BLOCK_CAPACITY = 100_000;
+const makeCsmtKey = (typeId, start) => typeId * 4294967296 + start;
+export async function start(db, opts) {
+    const path = db.fileSystemPath;
+    const id = stringHash(path) >>> 0;
+    const noop = () => { };
+    if (opts?.clean) {
+        await rm(path, { recursive: true, force: true }).catch(noop);
+    }
+    await mkdir(path, { recursive: true }).catch(noop);
+    // not doing db yet
+    // db.modifyBuf = new SharedArrayBuffer(db.maxModifySize)
+    db.dbCtxExternal = native.start(id);
+    let writelog = null;
+    try {
+        writelog = JSON.parse((await readFile(join(path, WRITELOG_FILE))).toString());
+        // Load the common dump
+        native.loadCommon(join(path, writelog.commonDump), db.dbCtxExternal);
+        // Load all range dumps
+        for (const typeId in writelog.rangeDumps) {
+            const dumps = writelog.rangeDumps[typeId];
+            for (const dump of dumps) {
+                const fname = dump.file;
+                const err = native.loadRange(join(path, fname), db.dbCtxExternal);
+                if (err) {
+                    console.log(`Failed to load a range. file: "${fname}": ${err}`);
+                }
+            }
+        }
+        const schema = await readFile(join(path, SCHEMA_FILE));
+        if (schema) {
+            // Prop need to not call setting in selva
+            db.putSchema(JSON.parse(schema.toString()), true);
+        }
+    }
+    catch (err) { }
+    // The merkle tree should be empty at start.
+    if (!db.merkleTree || db.merkleTree.getRoot()) {
+        db.merkleTree = createTree(db.createCsmtHashFun);
+    }
+    for (const key in db.schemaTypesParsed) {
+        const def = db.schemaTypesParsed[key];
+        const [total, lastId] = native.getTypeInfo(def.id, db.dbCtxExternal);
+        def.total = total;
+        def.lastId = writelog?.types[def.id].lastId || lastId;
+        def.blockCapacity =
+            writelog?.types[def.id].blockCapacity || DEFAULT_BLOCK_CAPACITY;
+        foreachBlock(db, def, (start, end, hash) => {
+            const mtKey = makeCsmtKey(def.id, start);
+            const file = writelog.rangeDumps[def.id].find((v) => v.start === start)?.file || '';
+            const data = {
+                file,
+                typeId: def.id,
+                start,
+                end,
+            };
+            db.merkleTree.insert(mtKey, hash, data);
+        });
+    }
+    if (writelog?.hash) {
+        const oldHash = Buffer.from(writelog.hash, 'hex');
+        const newHash = db.merkleTree.getRoot()?.hash;
+        if (!oldHash.equals(newHash)) {
+            console.error(`WARN: CSMT hash mismatch: ${writelog.hash} != ${newHash.toString('hex')}`);
+        }
+    }
+    // start workers
+    let i = availableParallelism();
+    const address = native.intFromExternal(db.dbCtxExternal);
+    db.workers = new Array(i);
+    while (i--) {
+        db.workers[i] = new DbWorker(address, db);
+    }
+}
+//# sourceMappingURL=start.js.map

package/dist/src/server/tree.d.ts

@@ -0,0 +1,13 @@
+import { DbServer } from './index.js';
+import { SchemaTypeDef } from './schema/types.js';
+export type CsmtNodeRange = {
+    file: string;
+    typeId: number;
+    start: number;
+    end: number;
+};
+export declare const makeCsmtKey: (typeId: number, start: number) => number;
+export declare const destructureCsmtKey: (key: number) => number[];
+export declare const makeCsmtKeyFromNodeId: (typeId: number, blockCapacity: number, nodeId: number) => number;
+export declare function foreachBlock(db: DbServer, def: SchemaTypeDef, cb: (start: number, end: number, hash: Buffer) => void): Promise<void>;
+export declare function foreachDirtyBlock(db: DbServer, cb: (mtKey: number, typeId: number, start: number, end: number) => void): Promise<void>;

package/dist/src/server/tree.js

@@ -0,0 +1,33 @@
+import native from '../native.js';
+export const makeCsmtKey = (typeId, start) => typeId * 4294967296 + start;
+export const destructureCsmtKey = (key) => [
+    (key / 4294967296) | 0, // typeId
+    (key >>> 31) * 2147483648 + (key & 0x7fffffff), // start_node_id
+];
+export const makeCsmtKeyFromNodeId = (typeId, blockCapacity, nodeId) => {
+    const tmp = nodeId - +!(nodeId % blockCapacity);
+    return typeId * 4294967296 + ((tmp / blockCapacity) | 0) * blockCapacity + 1;
+};
+export async function foreachBlock(db, def, cb) {
+    const step = def.blockCapacity;
+    for (let start = 1; start <= def.lastId; start += step) {
+        const end = start + step - 1;
+        const hash = Buffer.allocUnsafe(16);
+        native.getNodeRangeHash(def.id, start, end, hash, db.dbCtxExternal);
+        cb(start, end, hash);
+    }
+}
+export async function foreachDirtyBlock(db, cb) {
+    const typeIdMap = {};
+    for (const typeName in db.schemaTypesParsed) {
+        const type = db.schemaTypesParsed[typeName];
+        const typeId = type.id;
+        typeIdMap[typeId] = type;
+    }
+    for (const mtKey of db.dirtyRanges) {
+        const [typeId, start] = destructureCsmtKey(mtKey);
+        const end = start + typeIdMap[typeId].blockCapacity - 1;
+        cb(mtKey, typeId, start, end);
+    }
+}
+//# sourceMappingURL=tree.js.map

package/dist/src/server/worker.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/src/server/worker.js

@@ -0,0 +1,35 @@
+import { isMainThread, receiveMessageOnPort, workerData, } from 'node:worker_threads';
+import native from '../native.js';
+if (isMainThread) {
+    console.warn('running query worker.ts in mainthread');
+}
+else {
+    let { address, channel } = workerData;
+    let dbCtx = native.externalFromInt(address);
+    native.workerCtxInit();
+    const transferList = new Array(1);
+    const handleMsg = (msg) => {
+        try {
+            if (typeof msg === 'bigint') {
+                // it's a ctx address
+                address = msg;
+                dbCtx = native.externalFromInt(address);
+                channel.postMessage(null);
+            }
+            else {
+                const arrayBuf = native.getQueryBuf(msg, dbCtx).buffer;
+                transferList[0] = arrayBuf;
+                channel.postMessage(arrayBuf, transferList);
+            }
+        }
+        catch (e) {
+            channel.postMessage(e);
+        }
+    };
+    channel.on('message', handleMsg);
+    const msg = receiveMessageOnPort(channel);
+    if (msg) {
+        handleMsg(msg.message);
+    }
+}
+//# sourceMappingURL=worker.js.map

package/package.json

@@ -0,0 +1,48 @@
+{
+  "name": "@based/db",
+  "version": "0.0.1",
+  "license": "MIT",
+  "type": "module",
+  "main": "./dist/src/index.js",
+  "scripts": {
+    "build": "make -C clibs && npm run build-zig && tsc",
+    "build-zig": "npx tsx scripts/zig_cross_compile.ts",
+    "build-release": "(cd clibs && ./build-release.sh) && npm run build-zig -- release && tsc",
+    "watch": "npx tsc --watch",
+    "clean-selva": "make -C clibs clean",
+    "clean": "rimraf {.turbo,dist,node_modules,.zig-cache,.zig-out,tmp} && npm run clean-selva",
+    "playground-nodebug": "npm run build && node ./dist/test/shared/playground.js",
+    "playground": "npm run build && LOCPATH=../locale/locale-x86_64-gnu/locale lldb -b -o run --one-line-on-crash bt -o \"script import os; os._exit(0)\" node ./dist/test/shared/playground.js",
+    "playground-fast": "node ./dist/test/shared/playground.js",
+    "test": "npm run build && LOCPATH=../locale/locale-x86_64-gnu/locale lldb -b -o run --one-line-on-crash bt -o \"script import os; os._exit(0)\" node ./scripts/test.js",
+    "test-gdb": "npm run build && LOCPATH=../locale/locale-x86_64-gnu/locale gdb -ex run --args node ./scripts/test.js",
+    "test-fast": "LOCPATH=../locale/locale-x86_64-gnu/locale node ./scripts/test.js",
+    "test-zig": "npm run build-zig && tsc && npm run test-fast",
+    "test-ts": "tsc && node ./scripts/test.js",
+    "perf": "npm run build && node benchmarks/references.js && node benchmarks/transfermarkt/transfermarkt-based.js"
+  },
+  "sideEffects": false,
+  "files": [
+    "dist/src",
+    "dist/lib",
+    "!*.map",
+    "README.md",
+    "package.json"
+  ],
+  "dependencies": {
+    "@based/schema": "5.0.0-alpha.0",
+    "@saulx/hash": "^3.0.0",
+    "@saulx/utils": "^4.3.2",
+    "picocolors": "^1.1.0"
+  },
+  "optionalDependencies": {
+    "@based/locale-x86-64-gnu": "*"
+  },
+  "devDependencies": {
+    "@types/node": "^22.5.3",
+    "axios": "^1.7.9",
+    "rimraf": "^3.0.2",
+    "tar": "^7.4.3",
+    "typescript": "^5.6.3"
+  }
+}