@dikolab/kbdb 0.1.0

package/src/shared/worker-daemon/constants/daemon-defaults.constant.ts

@@ -0,0 +1,29 @@
+/** Default maximum number of items held in daemon memory. */
+export const DEFAULT_ITEM_LIMIT = 1000;
+
+/** Default maximum number of index entries held in daemon memory. */
+export const DEFAULT_INDEX_LIMIT = 100;
+
+/**
+ * Default idle timeout in milliseconds before the daemon shuts
+ * down automatically (5 minutes).
+ */
+export const DEFAULT_TIMEOUT_MS = 300_000;
+
+/**
+ * Default time-to-live in milliseconds for each cached item
+ * (1 minute).
+ */
+export const DEFAULT_ITEM_TTL_MS = 60_000;
+
+/** Filename of the daemon configuration file inside `.kbdb/`. */
+export const WORKER_TOML_FILENAME = 'worker.toml';
+
+/**
+ * Database format version the daemon supports.
+ *
+ * Read from `catalog.toml` at startup. If the catalog version
+ * differs, the daemon prints a structured error to stderr and
+ * exits with code 1 before loading any WASM modules.
+ */
+export const DB_FORMAT_VERSION = 1;

package/src/shared/worker-daemon/functions/create-host-imports.function.ts

@@ -0,0 +1,312 @@
+import {
+   readFileSync,
+   writeFileSync,
+   renameSync,
+   unlinkSync,
+   readdirSync,
+   mkdirSync,
+   existsSync,
+} from 'node:fs';
+import { join } from 'node:path';
+
+// ---------------------------------------------------------------------------
+// FsAdapter interface (enables dependency injection for testing)
+// ---------------------------------------------------------------------------
+
+/**
+ * Minimal filesystem operations required by the host imports.
+ *
+ * The default implementation delegates to `node:fs` synchronous
+ * APIs. Tests inject a stub to avoid touching the real filesystem.
+ */
+export interface FsAdapter {
+   /** Read entire file as a Buffer. */
+   readFileSync(path: string): Buffer;
+   /** Write data buffer to file. */
+   writeFileSync(path: string, data: Uint8Array): void;
+   /** Rename `oldPath` to `newPath`. */
+   renameSync(oldPath: string, newPath: string): void;
+   /** Delete a file. */
+   unlinkSync(path: string): void;
+   /** List directory entries as strings. */
+   readdirSync(path: string): string[];
+   /** Create a directory (and parents) recursively. */
+   mkdirSync(path: string): void;
+   /** Return true if the path exists. */
+   existsSync(path: string): boolean;
+}
+
+// ---------------------------------------------------------------------------
+// MemoryRef interface
+// ---------------------------------------------------------------------------
+
+/**
+ * A mutable reference to the WASM linear memory.
+ *
+ * The `memory` field starts as `null` and is wired up after the
+ * `kb-worker` WASM module is instantiated (the module exports its
+ * own memory rather than importing one). Host import functions
+ * read `memory` at call time, not at creation time, so the ref
+ * can be populated after the import record is built.
+ */
+export interface MemoryRef {
+   memory: WebAssembly.Memory | null;
+}
+
+// ---------------------------------------------------------------------------
+// AllocatorRef interface
+// ---------------------------------------------------------------------------
+
+/**
+ * A mutable reference to the WASM allocator function.
+ *
+ * The `alloc` field starts as `null` and is wired up after the
+ * `kb-worker` WASM module is instantiated. Host import functions
+ * that must allocate memory in WASM linear memory check this
+ * reference before proceeding and return -1 when it is null.
+ */
+export interface AllocatorRef {
+   /**
+    * Allocates `size` bytes in WASM linear memory and returns the
+    * byte offset of the allocated region, or `null` when the
+    * allocator is not yet available.
+    */
+   alloc: ((size: number) => number) | null;
+}
+
+/** Default `FsAdapter` backed by `node:fs` synchronous APIs. */
+const nodeFsAdapter: FsAdapter = {
+   readFileSync: (p) => readFileSync(p),
+   writeFileSync: (p, d) => {
+      writeFileSync(p, d);
+   },
+   renameSync: (o, n) => {
+      renameSync(o, n);
+   },
+   unlinkSync: (p) => {
+      unlinkSync(p);
+   },
+   readdirSync: (p) => readdirSync(p),
+   mkdirSync: (p) => {
+      mkdirSync(p, { recursive: true });
+   },
+   existsSync: (p) => existsSync(p),
+};
+
+/** Default `MemoryRef` with no memory wired. */
+const nullMemory: MemoryRef = { memory: null };
+
+/** Default `AllocatorRef` with no allocator wired. */
+const nullAllocator: AllocatorRef = { alloc: null };
+
+// ---------------------------------------------------------------------------
+// createHostImports
+// ---------------------------------------------------------------------------
+
+/**
+ * Creates the host I/O import functions required by `kb-worker.wasm`.
+ *
+ * All functions read the WASM linear memory through `memoryRef`,
+ * a deferred reference wired up after module instantiation (the
+ * module exports its own memory). Paths from WASM are resolved
+ * relative to `contextPath`.
+ *
+ * @param contextPath - absolute path to the `.kbdb` database directory
+ * @param memoryRef - deferred reference to the WASM module's
+ *   exported memory; wired after instantiation
+ * @param fs - filesystem adapter; defaults to real `node:fs` operations
+ * @param allocator - reference to the WASM allocator; wired after
+ *   `kb-worker` instantiation. Defaults to a null (uninitialized) ref.
+ * @returns a record of host import functions keyed by their WASM import
+ *   names
+ */
+export function createHostImports(
+   contextPath: string,
+   memoryRef: MemoryRef = nullMemory,
+   fs: FsAdapter = nodeFsAdapter,
+   allocator: AllocatorRef = nullAllocator,
+): Record<string, WebAssembly.ImportValue> {
+   function getBuffer(): ArrayBuffer {
+      if (!memoryRef.memory) {
+         throw new Error('WASM memory not initialized');
+      }
+      return memoryRef.memory.buffer;
+   }
+
+   function readPathArg(ptr: number, len: number): string {
+      const bytes = new Uint8Array(getBuffer(), ptr, len);
+      const relative = new TextDecoder().decode(bytes);
+      return join(contextPath, relative);
+   }
+
+   return {
+      /**
+       * Reads a file at the path encoded in shared memory and writes
+       * the result back via the out-pointer protocol.
+       *
+       * `outPtr` and `outLen` are WASM linear-memory addresses. On
+       * success, the host allocates a buffer via `allocator.alloc`,
+       * copies the file data there, and writes the allocated pointer
+       * as a LE u32 to `outPtr` and the data length as a LE u32 to
+       * `outLen`. Returns 0 on success, -1 on error or when the
+       * allocator is not yet available.
+       */
+      host_read_file(
+         pathPtr: number,
+         pathLen: number,
+         outPtr: number,
+         outLen: number,
+      ): number {
+         try {
+            if (!allocator.alloc) return -1;
+            const filePath = readPathArg(pathPtr, pathLen);
+            const data = fs.readFileSync(filePath);
+            const dataLen = data.byteLength;
+            const dv = new DataView(getBuffer());
+            if (dataLen === 0) {
+               dv.setUint32(outPtr, 0, true);
+               dv.setUint32(outLen, 0, true);
+               return 0;
+            }
+            const dataPtr = allocator.alloc(dataLen);
+            if (dataPtr === 0) return -1;
+            const memView = new Uint8Array(getBuffer());
+            memView.set(
+               new Uint8Array(
+                  data.buffer,
+                  data.byteOffset,
+                  data.byteLength,
+               ),
+               dataPtr,
+            );
+            dv.setUint32(outPtr, dataPtr, true);
+            dv.setUint32(outLen, dataLen, true);
+            return 0;
+         } catch {
+            return -1;
+         }
+      },
+
+      /**
+       * Writes bytes from shared memory (`dataPtr`, `dataLen`) to the
+       * file at the path encoded in shared memory. Returns 0 on
+       * success, -1 on error.
+       */
+      host_write_file(
+         pathPtr: number,
+         pathLen: number,
+         dataPtr: number,
+         dataLen: number,
+      ): number {
+         try {
+            const filePath = readPathArg(pathPtr, pathLen);
+            const data = new Uint8Array(getBuffer(), dataPtr, dataLen);
+            fs.writeFileSync(filePath, data);
+            return 0;
+         } catch {
+            return -1;
+         }
+      },
+
+      /**
+       * Renames the file at `oldPtr/oldLen` to `newPtr/newLen`.
+       * Both paths are resolved relative to `contextPath`. Returns 0
+       * on success, -1 on error.
+       */
+      host_rename(
+         oldPtr: number,
+         oldLen: number,
+         newPtr: number,
+         newLen: number,
+      ): number {
+         try {
+            const oldPath = readPathArg(oldPtr, oldLen);
+            const newPath = readPathArg(newPtr, newLen);
+            fs.renameSync(oldPath, newPath);
+            return 0;
+         } catch {
+            return -1;
+         }
+      },
+
+      /**
+       * Deletes the file at the path encoded in shared memory.
+       * Returns 0 on success, -1 on error.
+       */
+      host_remove_file(pathPtr: number, pathLen: number): number {
+         try {
+            const filePath = readPathArg(pathPtr, pathLen);
+            fs.unlinkSync(filePath);
+            return 0;
+         } catch {
+            return -1;
+         }
+      },
+
+      /**
+       * Lists the directory at the path encoded in shared memory and
+       * writes the result back via the out-pointer protocol.
+       *
+       * `outPtr` and `outLen` are WASM linear-memory addresses. On
+       * success, the host allocates a buffer via `allocator.alloc`,
+       * copies a newline-delimited UTF-8 listing there, and writes
+       * the allocated pointer as a LE u32 to `outPtr` and the byte
+       * length as a LE u32 to `outLen`. Returns 0 on success, -1 on
+       * error or when the allocator is not yet available.
+       */
+      host_list_dir(
+         pathPtr: number,
+         pathLen: number,
+         outPtr: number,
+         outLen: number,
+      ): number {
+         try {
+            if (!allocator.alloc) return -1;
+            const dirPath = readPathArg(pathPtr, pathLen);
+            const entries = fs.readdirSync(dirPath);
+            const text = entries.join('\n');
+            const encoded = new TextEncoder().encode(text);
+            const dv = new DataView(getBuffer());
+            if (encoded.length === 0) {
+               dv.setUint32(outPtr, 0, true);
+               dv.setUint32(outLen, 0, true);
+               return 0;
+            }
+            const dataPtr = allocator.alloc(encoded.length);
+            if (dataPtr === 0) return -1;
+            new Uint8Array(getBuffer(), dataPtr, encoded.length).set(
+               encoded,
+            );
+            dv.setUint32(outPtr, dataPtr, true);
+            dv.setUint32(outLen, encoded.length, true);
+            return 0;
+         } catch {
+            return -1;
+         }
+      },
+
+      /**
+       * Creates the directory at the path encoded in shared memory,
+       * including any missing parent directories (recursive). Returns
+       * 0 on success, -1 on error.
+       */
+      host_mkdir(pathPtr: number, pathLen: number): number {
+         try {
+            const dirPath = readPathArg(pathPtr, pathLen);
+            fs.mkdirSync(dirPath);
+            return 0;
+         } catch {
+            return -1;
+         }
+      },
+
+      /**
+       * Checks whether the file at the path encoded in shared memory
+       * exists. Returns 1 if it exists, 0 if it does not.
+       */
+      host_file_exists(pathPtr: number, pathLen: number): number {
+         const filePath = readPathArg(pathPtr, pathLen);
+         return fs.existsSync(filePath) ? 1 : 0;
+      },
+   };
+}

package/src/shared/worker-daemon/functions/create-ipc-server.function.ts

@@ -0,0 +1,83 @@
+import { createServer } from 'node:net';
+import type { Server, Socket } from 'node:net';
+import type {
+   IpcHandler,
+   IpcRequest,
+   IpcResponse,
+} from '../typings/ipc-handler.interface.ts';
+
+/**
+ * Creates a Unix socket (or named pipe on Windows) IPC server that
+ * reads newline-delimited JSON-RPC requests and writes responses.
+ *
+ * Each line received on a socket connection is parsed as an
+ * `IpcRequest`, dispatched to `handler`, and the resulting
+ * `IpcResponse` is written back as a newline-terminated JSON string.
+ * A malformed line that cannot be parsed results in a JSON-RPC
+ * parse-error response (`-32700`).
+ *
+ * @param socketPath - path to the Unix domain socket or Windows
+ *   named pipe
+ * @param handler - async function invoked for each valid request
+ * @returns the listening `net.Server` instance
+ */
+export function createIpcServer(
+   socketPath: string,
+   handler: IpcHandler,
+): Server {
+   const server = createServer((socket) => {
+      let buffer = '';
+
+      socket.on('data', (chunk: Buffer) => {
+         buffer += chunk.toString();
+         const lines = buffer.split('\n');
+         buffer = lines.pop() ?? '';
+
+         for (const line of lines) {
+            if (!line.trim()) continue;
+            void handleLine(line, handler, socket);
+         }
+      });
+
+      socket.on('error', (err: Error) => {
+         // eslint-disable-next-line no-console
+         console.error(
+            `kbdb-worker: ipc connection error: ${err.message}`,
+         );
+      });
+   });
+
+   server.on('error', (err: Error) => {
+      // eslint-disable-next-line no-console
+      console.error(`kbdb-worker: ipc server error: ${err.message}`);
+   });
+
+   server.listen(socketPath);
+   return server;
+}
+
+function bigintReplacer(_key: string, value: unknown): unknown {
+   return typeof value === 'bigint' ? Number(value) : value;
+}
+
+async function handleLine(
+   line: string,
+   handler: IpcHandler,
+   socket: Socket,
+): Promise<void> {
+   let response: IpcResponse;
+   try {
+      const request = JSON.parse(line) as IpcRequest;
+      response = await handler(request);
+   } catch {
+      response = {
+         jsonrpc: '2.0',
+         id: 0,
+         error: {
+            code: -32700,
+            message: 'Parse error',
+         },
+      };
+   }
+   socket.write(JSON.stringify(response, bigintReplacer) + '\n');
+}

package/src/shared/worker-daemon/functions/load-wasm-modules.function.ts

@@ -0,0 +1,98 @@
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import type { DaemonConfig } from '../typings/daemon-config.interface.ts';
+import type { WasmModules } from '../typings/wasm-modules.interface.ts';
+import { createHostImports } from './create-host-imports.function.ts';
+import type {
+   AllocatorRef,
+   MemoryRef,
+} from './create-host-imports.function.ts';
+
+/**
+ * Loads and instantiates the `kb-worker` WASM module.
+ *
+ * `kb-worker.wasm` statically links the `fs-database` and
+ * `query-parser` crates, so only one module needs to be
+ * instantiated.  The module exports its own linear memory;
+ * host I/O imports read it through a deferred `MemoryRef`
+ * that is wired up immediately after instantiation.
+ *
+ * @param config - daemon configuration, used to supply the context
+ *   path to the host I/O imports
+ * @param wasmDir - absolute path to the directory containing the
+ *   compiled WASM subdirectories (`query-parser/`, `fs-database/`,
+ *   `kb-worker/`)
+ * @throws when the WASM file cannot be read or instantiated
+ */
+export async function loadWasmModules(
+   config: DaemonConfig,
+   wasmDir: string,
+): Promise<WasmModules> {
+   const memoryRef: MemoryRef = { memory: null };
+   const allocator: AllocatorRef = { alloc: null };
+
+   const hostImports = createHostImports(
+      config.contextPath,
+      memoryRef,
+      undefined,
+      allocator,
+   );
+
+   const wkPath = join(wasmDir, 'kb-worker', 'kbdb_worker_bg.wasm');
+   const wkBytes = readWasmFileOrThrow(wkPath, 'kb-worker');
+
+   let wkModule: WebAssembly.Module;
+   try {
+      wkModule = await WebAssembly.compile(wkBytes);
+   } catch (err) {
+      rethrow('compile', 'kb-worker', err);
+   }
+
+   let kbWorker: WebAssembly.Instance;
+   try {
+      kbWorker = await WebAssembly.instantiate(wkModule, {
+         env: { ...hostImports },
+      });
+   } catch (err) {
+      rethrow('instantiate', 'kb-worker', err);
+   }
+
+   const memory = kbWorker.exports['memory'] as WebAssembly.Memory;
+   memoryRef.memory = memory;
+
+   allocator.alloc = (size: number): number => {
+      const fn_ = kbWorker.exports['kbdb_alloc'] as
+         | ((s: number) => number)
+         | undefined;
+      if (typeof fn_ !== 'function') return 0;
+      return fn_(size);
+   };
+
+   return { memory, kbWorker };
+}
+
+function readWasmFileOrThrow(path: string, label: string): ArrayBuffer {
+   try {
+      const buf = readFileSync(path);
+      return buf.buffer.slice(
+         buf.byteOffset,
+         buf.byteOffset + buf.byteLength,
+      );
+   } catch (err) {
+      rethrow('load', label, err, path);
+   }
+}
+
+function rethrow(
+   stage: string,
+   label: string,
+   err: unknown,
+   path?: string,
+): never {
+   const detail = err instanceof Error ? err.message : String(err);
+   const suffix = path ? ` (path: ${path})` : '';
+   const message = `failed to ${stage} ${label} WASM: ${detail}${suffix}`;
+   // eslint-disable-next-line no-console
+   console.error(`kbdb-worker: ${message}`);
+   throw new Error(message, { cause: err });
+}

package/src/shared/worker-daemon/functions/read-config.function.ts

@@ -0,0 +1,83 @@
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import type { DaemonConfig } from '../typings/daemon-config.interface.ts';
+import {
+   DEFAULT_ITEM_LIMIT,
+   DEFAULT_INDEX_LIMIT,
+   DEFAULT_TIMEOUT_MS,
+   DEFAULT_ITEM_TTL_MS,
+   WORKER_TOML_FILENAME,
+} from '../constants/daemon-defaults.constant.ts';
+
+/**
+ * Reads the `worker.toml` configuration file from the given
+ * context directory and returns a `DaemonConfig`.
+ *
+ * Falls back to all-default values when the file is absent or
+ * unreadable.
+ *
+ * @param contextPath - absolute path to the `.kbdb` context
+ *   directory
+ */
+export function readConfig(contextPath: string): DaemonConfig {
+   const configPath = join(contextPath, WORKER_TOML_FILENAME);
+   try {
+      const text = readFileSync(configPath, 'utf-8');
+      return parseWorkerToml(text, contextPath);
+   } catch {
+      return {
+         contextPath,
+         itemLimit: DEFAULT_ITEM_LIMIT,
+         indexLimit: DEFAULT_INDEX_LIMIT,
+         timeoutMs: DEFAULT_TIMEOUT_MS,
+         itemTtlMs: DEFAULT_ITEM_TTL_MS,
+      };
+   }
+}
+
+function parseWorkerToml(
+   text: string,
+   contextPath: string,
+): DaemonConfig {
+   const config: DaemonConfig = {
+      contextPath,
+      itemLimit: DEFAULT_ITEM_LIMIT,
+      indexLimit: DEFAULT_INDEX_LIMIT,
+      timeoutMs: DEFAULT_TIMEOUT_MS,
+      itemTtlMs: DEFAULT_ITEM_TTL_MS,
+   };
+
+   const result = { ...config };
+
+   for (const line of text.split('\n')) {
+      const trimmed = line.trim();
+      if (trimmed.startsWith('#') || !trimmed.includes('=')) {
+         continue;
+      }
+      const eqIdx = trimmed.indexOf('=');
+      const key = trimmed.slice(0, eqIdx).trim();
+      const value = trimmed.slice(eqIdx + 1).trim();
+      switch (key) {
+         case 'item_limit':
+            result.itemLimit =
+               parseInt(value, 10) || DEFAULT_ITEM_LIMIT;
+            break;
+         case 'index_limit':
+            result.indexLimit =
+               parseInt(value, 10) || DEFAULT_INDEX_LIMIT;
+            break;
+         case 'timeout_ms':
+            result.timeoutMs =
+               parseInt(value, 10) || DEFAULT_TIMEOUT_MS;
+            break;
+         case 'item_ttl_ms':
+            result.itemTtlMs =
+               parseInt(value, 10) || DEFAULT_ITEM_TTL_MS;
+            break;
+         default:
+            break;
+      }
+   }
+
+   return result;
+}

package/src/shared/worker-daemon/functions/run-worker.function.ts

@@ -0,0 +1,57 @@
+import { resolve } from 'node:path';
+
+/** Result of the worker entry point. */
+export interface WorkerResult {
+   /** Error message (if any). */
+   error?: string;
+   /** Process exit code. `0` means success. */
+   exitCode: number;
+}
+
+/**
+ * Injectable dependencies for the worker entry point.
+ */
+export interface WorkerDeps {
+   existsSync: (path: string) => boolean;
+   startDaemon: (
+      contextPath: string,
+      wasmDir?: string,
+   ) => Promise<void>;
+   wasmDir?: string;
+}
+
+/**
+ * Worker daemon entry point extracted from `src/worker.ts`.
+ *
+ * Validates the context path argument, resolves it to an
+ * absolute path, checks existence, and delegates to
+ * `startDaemon`.
+ *
+ * @param argv - raw arguments (expects `[contextPath]`)
+ * @param deps - injectable dependencies
+ */
+export async function runWorker(
+   argv: string[],
+   deps: WorkerDeps,
+): Promise<WorkerResult> {
+   const contextPath = argv[0];
+
+   if (!contextPath) {
+      return {
+         error: 'usage: kbdb-worker <context-path>',
+         exitCode: 1,
+      };
+   }
+
+   const resolved = resolve(contextPath);
+
+   if (!deps.existsSync(resolved)) {
+      return {
+         error: `error: path not found: ${resolved}`,
+         exitCode: 1,
+      };
+   }
+
+   await deps.startDaemon(resolved, deps.wasmDir);
+   return { exitCode: 0 };
+}