@based/db 0.0.64 → 0.0.67

package/dist/src/server/{DbWorker.js → workers/DbWorker.js}

@@ -1,16 +1,14 @@
 import { MessageChannel, Worker } from 'node:worker_threads';
 import { fileURLToPath } from 'node:url';
 import { dirname, join } from 'node:path';
-import { readUint64 } from '@saulx/utils';
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
-const workerPath = join(__dirname, 'worker.js');
 export class DbWorker {
-    constructor(address, db, workerIndex) {
+    constructor(address, db, onExit, workerName) {
         const { port1, port2 } = new MessageChannel();
         this.db = db;
         this.channel = port1;
-        this.worker = new Worker(workerPath, {
+        this.worker = new Worker(join(__dirname, workerName), {
             workerData: {
                 isDbWorker: true,
                 channel: port2,
@@ -41,12 +39,12 @@ export class DbWorker {
                     resolve(err);
                 }
                 this.resolvers = [];
-                this.db.workers[workerIndex] = new DbWorker(address, db, workerIndex);
+                onExit(code);
             }
         });
-        port1.on('message', (buf) => {
+        this.channel.on('message', (buf) => {
             this.resolvers.shift()(new Uint8Array(buf));
-            this.db.onQueryEnd();
+            this.handleMsg(buf);
         });
     }
     db;
@@ -54,21 +52,21 @@ export class DbWorker {
     worker;
     resolvers = [];
     readyPromise;
+    async terminate() {
+        return this.worker.terminate();
+    }
     callback = (resolve) => {
-        this.db.processingQueries++;
         this.resolvers.push(resolve);
     };
-    updateCtx(address) {
-        this.channel.postMessage(address);
+    /**
+     * Send msg to the worker thread and return a promise to the response.
+     */
+    call(msg) {
+        this.channel.postMessage(msg);
         return new Promise(this.callback);
     }
-    getQueryBuf(buf) {
-        const schemaChecksum = readUint64(buf, buf.byteLength - 8);
-        if (schemaChecksum !== this.db.schema?.hash) {
-            return Promise.resolve(new Uint8Array(1));
-        }
-        this.channel.postMessage(buf);
-        return new Promise(this.callback);
+    updateCtx(address) {
+        return this.call(address);
     }
 }
 //# sourceMappingURL=DbWorker.js.map

package/dist/src/server/workers/io_worker.js

@@ -0,0 +1,39 @@
+import { registerMsgHandler } from './worker.js';
+import { ENCODER, writeInt32 } from '@saulx/utils';
+import native from '../../native.js';
+function loadBlock(dbCtx, filepath) {
+    try {
+        native.loadBlock(filepath, dbCtx);
+    }
+    catch (e) {
+        // need to get rid of the shared buffer
+        return new Uint8Array(ENCODER.encode(e.toString())).buffer;
+    }
+    return null;
+}
+function unloadBlock(dbCtx, filepath, typeId, start) {
+    const buf = new ArrayBuffer(20); // [[4 bytes err], [16 bytes hash]]
+    const hash = new Uint8Array(buf, 4);
+    const err = native.saveBlock(filepath, typeId, start, dbCtx, hash);
+    if (err) {
+        const errCodeBuf = new Uint8Array(buf, 0, 4);
+        writeInt32(errCodeBuf, err, 0);
+    }
+    else {
+        native.delBlock(dbCtx, typeId, start);
+    }
+    return buf;
+}
+registerMsgHandler((dbCtx, msg) => {
+    if (typeof msg?.type === 'string') {
+        const job = msg;
+        if (job.type === 'load') {
+            return loadBlock(dbCtx, job.filepath);
+        }
+        else if (job.type === 'unload') {
+            return unloadBlock(dbCtx, job.filepath, job.typeId, job.start);
+        }
+    }
+    return null;
+});
+//# sourceMappingURL=io_worker.js.map

package/dist/src/server/workers/io_worker_types.d.ts

@@ -0,0 +1,12 @@
+type IoJobLoad = {
+    type: 'load';
+    filepath: string;
+};
+type IoJobUnload = {
+    type: 'unload';
+    filepath: string;
+    typeId: number;
+    start: number;
+};
+export type IoJob = IoJobLoad | IoJobUnload;
+export {};

package/dist/src/server/workers/io_worker_types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=io_worker_types.js.map

package/dist/src/server/workers/query_worker.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/src/server/workers/query_worker.js

@@ -0,0 +1,4 @@
+import { registerMsgHandler } from './worker.js';
+import native from '../../native.js';
+registerMsgHandler((dbCtx, msg) => native.getQueryBuf(msg, dbCtx));
+//# sourceMappingURL=query_worker.js.map

package/dist/src/server/workers/worker.d.ts

@@ -0,0 +1 @@
+export declare function registerMsgHandler(onMsg: (dbCtx: any, msg: any) => Uint8Array<ArrayBufferLike> | ArrayBuffer | null): void;

package/dist/src/server/workers/worker.js

@@ -0,0 +1,41 @@
+import { isMainThread, parentPort, workerData } from 'node:worker_threads';
+import native from '../../native.js';
+let dbCtx;
+let wCtx; // This must be held until the worker exits otherwise the ctx will be autofreed instantly
+if (isMainThread) {
+    console.warn(`running a worker in the mainthread - incorrect`);
+}
+else if (workerData?.isDbWorker) {
+    let { address } = workerData;
+    dbCtx = native.externalFromInt(address);
+    wCtx = native.workerCtxInit();
+}
+else {
+    console.info('incorrect worker db query');
+}
+export function registerMsgHandler(onMsg) {
+    if (!workerData?.isDbWorker) {
+        throw new Error('Not a DbWorker');
+    }
+    let { channel } = workerData;
+    const handleMsg = (msg) => {
+        try {
+            if (typeof msg === 'bigint') {
+                // it's a ctx address
+                dbCtx = native.externalFromInt(msg);
+                channel.postMessage(null);
+            }
+            else {
+                // a message to the worker handler
+                const arrayBuf = onMsg(dbCtx, msg);
+                channel.postMessage(arrayBuf, [arrayBuf]);
+            }
+        }
+        catch (e) {
+            channel.postMessage(e);
+        }
+    };
+    channel.on('message', handleMsg);
+    parentPort.postMessage('READY');
+}
+//# sourceMappingURL=worker.js.map

package/dist/src/shared/Emitter.d.ts

@@ -2,6 +2,7 @@ import { DbSchema } from '../schema.js';
 export type EventMap = {
     schema: DbSchema;
     info: string;
+    error: string;
 };
 export type Event = keyof EventMap;
 export type Listener<T> = (data: T) => void;

package/dist/src/types.d.ts

@@ -1,4 +1,4 @@
-export declare const SCHEMA_FILE = "schema.json";
+export declare const SCHEMA_FILE = "schema.bin";
 export declare const WRITELOG_FILE = "writelog.json";
 export declare const COMMON_SDB_FILE = "common.sdb";
 export type BasedDbOpts = {

package/dist/src/types.js

@@ -1,4 +1,4 @@
-export const SCHEMA_FILE = 'schema.json';
+export const SCHEMA_FILE = 'schema.bin';
 export const WRITELOG_FILE = 'writelog.json';
 export const COMMON_SDB_FILE = 'common.sdb';
 //# sourceMappingURL=types.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@based/db",
-  "version": "0.0.64",
+  "version": "0.0.67",
   "license": "MIT",
   "type": "module",
   "main": "./dist/src/index.js",
@@ -38,9 +38,9 @@
     "basedDbNative.cjs"
   ],
   "dependencies": {
-    "@based/schema": "5.0.0-alpha.18",
+    "@based/schema": "5.0.0-alpha.23",
     "@saulx/hash": "^3.0.0",
-    "@saulx/utils": "^6.7.0",
+    "@saulx/utils": "^6.7.2",
     "exit-hook": "^4.0.0",
     "picocolors": "^1.1.0",
     "@based/crc32c": "^1.0.0"

package/dist/lib/darwin_aarch64/include/selva/find.h

@@ -1,47 +0,0 @@
-/*
- * Copyright (c) 2024 SAULX
- * SPDX-License-Identifier: MIT
- */
-#pragma once
-
-#include "selva/_export.h"
-#include "selva/types.h"
-#include "selva/traverse.h"
-
-struct SelvaFindParam {
-    SelvaTraversalNodeCallback node_cb;
-    void *node_arg;
-
-    /**
-     * Traverse field selector.
-     */
-    const struct FindFields {
-        uint8_t len;
-        struct {
-            node_type_t type;
-            field_t field;
-        } __packed data[];
-    } __packed *fields;
-
-    /**
-     * Skip the first n nodes in the traversal.
-     * This happens before filtering.
-     */
-    ssize_t skip;
-
-    /**
-     * Skip the first n - 1 results.
-     * This skipping is executed on the final result set.
-     * 0 for no offset.
-     */
-    ssize_t offset;
-
-    /**
-     * Limit the number of results.
-     * -1 for no limit (inf).
-     */
-    ssize_t limit;
-};
-
-SELVA_EXPORT
-int selva_find(struct SelvaDb *db, struct SelvaNode *node, const struct SelvaFindParam *param);

package/dist/lib/darwin_aarch64/include/selva/history.h

@@ -1,64 +0,0 @@
-/*
- * Copyright (c) 2025 SAULX
- * SPDX-License-Identifier: MIT
- */
-#pragma once
-
-#include <stddef.h>
-#include <stdint.h>
-#include "selva/types.h"
-#include "selva/_export.h"
-
-struct selva_history;
-
-struct selva_history_event {
-    int64_t ts;
-    node_id_t node_id;
-    uint32_t crc;
-} __packed __attribute__((aligned(4)));
-
-static_assert(alignof(struct selva_history_event) == alignof(uint32_t));
-
-/**
- * Create a new history object.
- */
-SELVA_EXPORT
-int selva_history_create(const char *pathname, size_t bsize, struct selva_history **hist_out);
-
-/**
- * Destroy a history object.
- */
-SELVA_EXPORT
-void selva_history_destroy(struct selva_history *hist);
-
-/**
- * Append a block of size `bsize` to a history object.
- */
-SELVA_EXPORT
-void selva_history_append(struct selva_history *hist, int64_t ts, node_id_t node_id, void *buf);
-
-/**
- * Ensure that the history file is fully written to the disk.
- */
-SELVA_EXPORT
-void selva_history_fsync(struct selva_history *hist);
-
-/**
- * Find a range.
- * The returned buffer must be freed with selva_history_free_range().
- */
-SELVA_EXPORT
-uint32_t *selva_history_find_range(struct selva_history *hist, int64_t from, int64_t to, size_t *size_out);
-
-/**
- * Find a range.
- * The returned buffer must be freed with selva_history_free_range().
- */
-SELVA_EXPORT
-uint32_t *selva_history_find_range_node(struct selva_history *hist, int64_t from, int64_t to, node_id_t node_id, size_t *size_out);
-
-/**
- * Free a range returned by selva_history_find_range() or selva_history_find_range_node().
- */
-SELVA_EXPORT
-void selva_history_free_range(uint32_t *range);

package/dist/lib/darwin_aarch64/include/selva/queue_r.h

@@ -1,190 +0,0 @@
-/**
- *******************************************************************************
- * @file    queue_r.h
- * @author  Olli Vanhoja
- * @brief   Thread-safe queue
- * @section LICENSE
- * Copyright (c) 2020-2021, 2023, 2025 SAULX
- * Copyright (c) 2013, 2015, 2016 Olli Vanhoja <olli.vanhoja@cs.helsinki.fi>
- * Copyright (c) 2012, 2013 Ninjaware Oy,
- *                          Olli Vanhoja <olli.vanhoja@ninjaware.fi>
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * 1. Redistributions of source code must retain the above copyright notice,
- *    this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright notice,
- *    this list of conditions and the following disclaimer in the documentation
- *    and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- *
- * SPDX-License-Identifier: BSD-2-Clause
- *******************************************************************************
- */
-
-/**
- * @addtogroup queue_r
- * @{
- */
-
-#pragma once
-
-#include <stddef.h>
-#include "selva/_export.h"
-
-/**
- * Queue control block.
- */
-typedef struct queue_cb {
-    void * data;    /*!< Pointer to the data array used for queue. */
-    size_t b_size;  /*!< Block size in bytes. */
-    size_t a_len;   /*!< Array length. */
-    size_t m_write; /*!< Write index. */
-    size_t m_read;  /*!< Read index. */
-} queue_cb_t;
-
-#define QUEUE_INITIALIZER(data_array, block_size, array_size)   \
-    (struct queue_cb){                                          \
-        .data = (data_array),                                   \
-        .b_size = (block_size),                                 \
-        .a_len = (array_size / block_size),                     \
-        .m_write = 0,                                           \
-        .m_read = 0,                                            \
-    }
-
-/**
- * Create a new queue control block.
- * Initializes a new queue control block and returns it as a value.
- * @param data_array an array where the actual queue data is stored.
- * @param block_size the size of single data block/struct/data type in
- *                   data_array in bytes.
- * @param array_size the size of the data_array in bytes.
- * @return a new queue_cb_t queue control block structure.
- */
-SELVA_EXPORT
-queue_cb_t queue_create(void * data_array, size_t block_size,
-                        size_t array_size)
-    __attribute__((pure, access(read_only, 1, 3)));
-
-/**
- * Push an element to the queue.
- * @param cb is a pointer to the queue control block.
- * @param element the new element to be copied.
- * @return 0 if queue is already full; otherwise operation was succeed.
- * @note element is always copied to the queue, so it is safe to remove the
- * original data after a push.
- */
-SELVA_EXPORT
-int queue_push(queue_cb_t * cb, const void * element)
-    __attribute__((access(read_write, 1)));
-
-/**
- * Allocate an element from the queue.
- * @param cb is a pointer to the queue control block.
- * @return A pointer to the element in the queue.
- */
-SELVA_EXPORT
-void * queue_alloc_get(queue_cb_t * cb)
-    __attribute__((access(read_write, 1)));
-
-/**
- * Commit previous allocation from the queue.
- * @note queue_push() should not be called between calls to queue_alloc_get()
- *       and queue_alloc_commit().
- */
-SELVA_EXPORT
-void queue_alloc_commit(queue_cb_t * cb)
-    __attribute__((access(read_write, 1)));
-
-/**
- * Pop an element from the queue.
- * @param cb is a pointer to the queue control block.
- * @param element is the location where element is copied to from the queue.
- * @return 0 if queue is empty; otherwise operation was succeed.
- */
-SELVA_EXPORT
-int queue_pop(queue_cb_t * cb, void * element)
-    __attribute__((access(read_write, 1)));
-
-/**
- * Peek an element from the queue.
- * @param cb is a pointer to the queue control block.
- * @param element is the location where element is copied to from the queue.
- * @return 0 if queue is empty; otherwise operation was succeed.
- */
-SELVA_EXPORT
-int queue_peek(queue_cb_t * cb, void ** element)
-    __attribute__((access(read_only, 1), access(write_only, 2)));
-
-/**
- * Skip n number of elements in the queue.
- * @param cb is a pointer to the queue control block.
- * @return Returns the number of elements skipped.
- */
-SELVA_EXPORT
-int queue_skip(queue_cb_t * cb, size_t n)
-    __attribute__((access(read_write, 1)));
-
-/**
- * Clear the queue.
- * This operation is considered safe when committed from the push end thread.
- * @param cb is a pointer to the queue control block.
- */
-SELVA_EXPORT
-void queue_clear_from_push_end(queue_cb_t * cb)
-    __attribute__((access(read_write, 1)));
-
-/**
- * Clear the queue.
- * This operation is considered safe when committed from the pop end thread.
- * @param cb is a pointer to the queue control block.
- */
-SELVA_EXPORT
-void queue_clear_from_pop_end(queue_cb_t * cb)
-    __attribute__((access(read_write, 1)));
-
-/**
- * Check if the queue is empty.
- * @param cb is a pointer to the queue control block.
- * @return 0 if the queue is not empty.
- */
-SELVA_EXPORT
-int queue_isempty(const queue_cb_t * cb)
-    __attribute__((access(read_only, 1)));
-
-/**
- * Check if the queue is full.
- * @param cb is a pointer to the queue control block.
- * @return 0 if the queue is not full.
- */
-SELVA_EXPORT
-int queue_isfull(const queue_cb_t * cb)
-    __attribute__((access(read_only, 1)));
-
-/**
- * Seek queue.
- * @param cb is a pointer to the queue control block.
- * @param i index.
- * @param[in] element returned element.
- * @return 0 if failed; otherwise succeed.
- */
-SELVA_EXPORT
-int seek(const queue_cb_t * cb, size_t i, void * element)
-    __attribute__((access(read_only, 1), access(write_only, 3)));
-
-/**
- * @}
- */

package/dist/lib/darwin_aarch64/include/selva/traverse.h

@@ -1,65 +0,0 @@
-/*
- * Copyright (c) 2024 SAULX
- * SPDX-License-Identifier: MIT
- */
-#pragma once
-
-#include "selva/_export.h"
-
-struct SelvaNode;
-struct SelvaDb;
-
-/**
- * Traversal metadata for child/adjacent nodes.
- * Note that SelvaTraversalOrder expects this to be copyable.
- */
-struct SelvaTraversalMetadata {
-    const struct SelvaFields *edge_data; /*!< Edge metadata. */
-    long long depth;
-};
-
-#define SELVA_TRAVERSAL_ABORT (-2)
-#define SELVA_TRAVERSAL_STOP (-1)
-
-/**
- * Called for each node found during a traversal.
- * @param node a pointer to the node.
- * @param arg a pointer to node_arg give in SelvaTraversalCallback structure.
- */
-typedef int (*SelvaTraversalNodeCallback)(
-        struct SelvaDb *db,
-        const struct SelvaTraversalMetadata *metadata,
-        struct SelvaNode *node,
-        void *arg);
-
-/**
- * Callback descriptor used for traversals.
- */
-struct SelvaTraversalParam {
-    /**
-     * Called for each orphan head in the hierarchy.
-     */
-    SelvaTraversalNodeCallback head_cb;
-    void *head_arg;
-
-    /**
-     * Called for each node in the hierarchy.
-     */
-    SelvaTraversalNodeCallback node_cb;
-    void *node_arg;
-
-    /**
-     * Called for each child of current node.
-     */
-    SelvaTraversalNodeCallback child_cb;
-    void *child_arg;
-};
-
-SELVA_EXPORT
-int selva_traverse_field_bfs(
-        struct SelvaDb *db,
-        struct SelvaNode *head,
-        const struct SelvaTraversalParam *cb);
-
-SELVA_EXPORT
-int selva_traverse_type(struct SelvaDb *db, struct SelvaTypeEntry *te, SelvaTraversalNodeCallback node_cb, void *node_arg);

package/dist/lib/linux_aarch64/include/selva/find.h

@@ -1,47 +0,0 @@
-/*
- * Copyright (c) 2024 SAULX
- * SPDX-License-Identifier: MIT
- */
-#pragma once
-
-#include "selva/_export.h"
-#include "selva/types.h"
-#include "selva/traverse.h"
-
-struct SelvaFindParam {
-    SelvaTraversalNodeCallback node_cb;
-    void *node_arg;
-
-    /**
-     * Traverse field selector.
-     */
-    const struct FindFields {
-        uint8_t len;
-        struct {
-            node_type_t type;
-            field_t field;
-        } __packed data[];
-    } __packed *fields;
-
-    /**
-     * Skip the first n nodes in the traversal.
-     * This happens before filtering.
-     */
-    ssize_t skip;
-
-    /**
-     * Skip the first n - 1 results.
-     * This skipping is executed on the final result set.
-     * 0 for no offset.
-     */
-    ssize_t offset;
-
-    /**
-     * Limit the number of results.
-     * -1 for no limit (inf).
-     */
-    ssize_t limit;
-};
-
-SELVA_EXPORT
-int selva_find(struct SelvaDb *db, struct SelvaNode *node, const struct SelvaFindParam *param);