@dikolab/kbdb 0.3.1 → 0.4.0

package/src/shared/worker-daemon/functions/run-startup-integrity-check.function.ts

@@ -1,55 +0,0 @@
-import type { WasmModules } from '../typings/wasm-modules.interface.ts';
-import {
-   callWasmZeroArg,
-   readReturnSlot,
-} from './wasm-bridge.function.ts';
-import { decodeKeyValueText } from '../../wasm-codec/index.ts';
-
-/** Result of a startup integrity check. */
-export interface StartupIntegrityResult {
-   /** Whether the check passed without errors. */
-   readonly integrityOk: boolean;
-   /** Integrity error descriptions found at startup. */
-   readonly integrityErrors: string[];
-}
-
-/**
- * Runs `worker_check_integrity` immediately after WASM context
- * initialisation and returns the result.
- *
- * Logs a warning to stderr when issues are found. Never throws --
- * failures are captured and returned as part of the result.
- *
- * @param modules - loaded WASM module instances
- */
-export function runStartupIntegrityCheck(
-   modules: WasmModules,
-): StartupIntegrityResult {
-   try {
-      callWasmZeroArg(modules, 'worker_check_integrity');
-      const bytes = readReturnSlot(modules);
-      const kv = decodeKeyValueText(new TextDecoder().decode(bytes));
-      const integrityOk = kv['ok'] === 'true';
-      if (!integrityOk) {
-         const integrityErrors = Object.entries(kv)
-            .filter(([k]) => k !== 'ok')
-            .map(([k, v]) => `${k}=${v}`);
-         // eslint-disable-next-line no-console
-         console.error(
-            '[kbdb-worker] startup integrity check failed:',
-            integrityErrors.join(', '),
-         );
-         return { integrityOk: false, integrityErrors };
-      }
-      return { integrityOk: true, integrityErrors: [] };
-   } catch {
-      // eslint-disable-next-line no-console
-      console.error(
-         '[kbdb-worker] startup integrity check threw an error',
-      );
-      return {
-         integrityOk: false,
-         integrityErrors: ['integrity_check_failed'],
-      };
-   }
-}

package/src/shared/worker-daemon/functions/run-worker.function.ts

@@ -1,57 +0,0 @@
-import { resolve } from 'node:path';
-
-/** Result of the worker entry point. */
-export interface WorkerResult {
-   /** Error message (if any). */
-   error?: string;
-   /** Process exit code. `0` means success. */
-   exitCode: number;
-}
-
-/**
- * Injectable dependencies for the worker entry point.
- */
-export interface WorkerDeps {
-   existsSync: (path: string) => boolean;
-   startDaemon: (
-      contextPath: string,
-      wasmDir?: string,
-   ) => Promise<void>;
-   wasmDir?: string;
-}
-
-/**
- * Worker daemon entry point extracted from `src/worker.ts`.
- *
- * Validates the context path argument, resolves it to an
- * absolute path, checks existence, and delegates to
- * `startDaemon`.
- *
- * @param argv - raw arguments (expects `[contextPath]`)
- * @param deps - injectable dependencies
- */
-export async function runWorker(
-   argv: string[],
-   deps: WorkerDeps,
-): Promise<WorkerResult> {
-   const contextPath = argv[0];
-
-   if (!contextPath) {
-      return {
-         error: 'usage: kbdb-worker <context-path>',
-         exitCode: 1,
-      };
-   }
-
-   const resolved = resolve(contextPath);
-
-   if (!deps.existsSync(resolved)) {
-      return {
-         error: `error: path not found: ${resolved}`,
-         exitCode: 1,
-      };
-   }
-
-   await deps.startDaemon(resolved, deps.wasmDir);
-   return { exitCode: 0 };
-}

package/src/shared/worker-daemon/functions/scan-near-duplicates.function.ts

@@ -1,92 +0,0 @@
-import { readdirSync, readFileSync } from 'node:fs';
-import { join } from 'node:path';
-
-/** A near-duplicate match found during add-section scanning. */
-export interface NearDuplicateMatch {
-   /** Knowledge-base identifier of the near-duplicate section. */
-   readonly kbid: string;
-   /** Jaccard similarity score, rounded to 2 decimal places (0–1). */
-   readonly similarity: number;
-}
-
-/**
- * Scans existing section files for near-duplicates of `content`.
- *
- * Uses Jaccard similarity on a set of lowercased tokens (3+ chars)
- * extracted from the section body. Sections with a similarity of
- * 0.85 or above are included in the result. The `newKbid` section
- * is excluded from comparison.
- *
- * @param targetDir - absolute path to the `.kbdb` directory
- * @param content - raw text content of the newly added section
- * @param newKbid - identifier of the just-added section to exclude
- * @returns near-duplicate matches sorted by insertion order
- */
-export function scanNearDuplicates(
-   targetDir: string,
-   content: string,
-   newKbid: string,
-): NearDuplicateMatch[] {
-   const newTerms = extractTermSet(content);
-   if (newTerms.size === 0) return [];
-
-   const sectionsDir = join(targetDir, 'sections');
-   let shards: string[];
-   try {
-      shards = readdirSync(sectionsDir);
-   } catch {
-      return [];
-   }
-
-   const results: NearDuplicateMatch[] = [];
-   for (const shard of shards) {
-      let files: string[];
-      try {
-         files = readdirSync(join(sectionsDir, shard));
-      } catch {
-         continue;
-      }
-      for (const file of files) {
-         if (!file.endsWith('.section')) continue;
-         const kbid = file.slice(0, -8);
-         if (kbid === newKbid) continue;
-         try {
-            const data = readFileSync(
-               join(sectionsDir, shard, file),
-               'utf-8',
-            );
-            const sep = data.indexOf('\n---\n');
-            const body = sep >= 0 ? data.slice(sep + 5) : data;
-            const existing = extractTermSet(body);
-            let intersection = 0;
-            for (const t of newTerms) {
-               if (existing.has(t)) intersection++;
-            }
-            const union = newTerms.size + existing.size - intersection;
-            const sim = union > 0 ? intersection / union : 0;
-            if (sim >= 0.85) {
-               results.push({
-                  kbid,
-                  similarity: Math.round(sim * 100) / 100,
-               });
-            }
-         } catch {
-            continue;
-         }
-      }
-   }
-   return results;
-}
-
-/**
- * Extracts a set of lowercased word tokens with 3 or more
- * characters from `text`.
- */
-function extractTermSet(text: string): Set<string> {
-   return new Set(
-      text
-         .toLowerCase()
-         .split(/\W+/)
-         .filter((w) => w.length > 2),
-   );
-}

package/src/shared/worker-daemon/functions/start-daemon.function.ts

@@ -1,74 +0,0 @@
-import { writeFileSync } from 'node:fs';
-import { join } from 'node:path';
-import { tmpdir } from 'node:os';
-import { readConfig } from './read-config.function.ts';
-import { computeSha256 } from '../../hash/functions/compute-sha256.ts';
-import { WorkerDaemon } from '../classes/worker-daemon.class.ts';
-
-/**
- * Starts the worker daemon for the given knowledge-base context.
- *
- * Reads configuration from `worker.toml` in the context directory,
- * validates startup preconditions (directory exists, accessible, and
- * database version compatible), and then starts the `WorkerDaemon`.
- *
- * Signal handlers for `SIGTERM` and `SIGINT` trigger a graceful stop.
- * Precondition failures are printed to stderr and cause exit code 1.
- *
- * @param contextPath - absolute path to the `.kbdb` context directory
- */
-export async function startDaemon(
-   contextPath: string,
-   wasmDir?: string,
-): Promise<void> {
-   const config = readConfig(contextPath);
-   const hash = await computeSha256(contextPath);
-   const contextId = hash.slice(0, 16);
-
-   const daemon = new WorkerDaemon(
-      contextId,
-      contextPath,
-      config,
-      wasmDir,
-   );
-
-   try {
-      daemon.checkPreconditions();
-   } catch (err) {
-      const message = err instanceof Error ? err.message : String(err);
-      // eslint-disable-next-line no-console
-      console.error(`kbdb-worker: ${message}`);
-      writeCrashLog(contextId, message);
-      process.exit(1);
-   }
-
-   try {
-      await daemon.start();
-   } catch (err) {
-      const message = err instanceof Error ? err.message : String(err);
-      // eslint-disable-next-line no-console
-      console.error(`kbdb-worker: ${message}`);
-      writeCrashLog(contextId, message);
-      process.exit(1);
-   }
-
-   process.on('SIGTERM', () => {
-      daemon.stop();
-   });
-   process.on('SIGINT', () => {
-      daemon.stop();
-   });
-}
-
-function writeCrashLog(contextId: string, message: string): void {
-   try {
-      const crashPath = join(tmpdir(), `kbdb-${contextId}.crash`);
-      const payload = JSON.stringify({
-         error: message,
-         timestamp: Date.now(),
-      });
-      writeFileSync(crashPath, payload);
-   } catch {
-      // never mask the original error
-   }
-}

package/src/shared/worker-daemon/functions/wasm-bridge.function.ts

@@ -1,144 +0,0 @@
-import type { WasmModules } from '../typings/wasm-modules.interface.ts';
-import type { WasmFn } from '../typings/wasm-fn.type.ts';
-import { IpcError } from '../classes/ipc-error.class.ts';
-
-/**
- * Retrieves a named export from the kb-worker WASM instance.
- *
- * @throws {Error} if WASM modules are not yet loaded
- * @throws {Error} if the named export does not exist or is not a
- *   function
- */
-export function getWasmExport(
-   modules: WasmModules,
-   name: string,
-): WasmFn {
-   const fn = modules.kbWorker.exports[name] as WasmFn | undefined;
-   if (typeof fn !== 'function') {
-      throw new Error(`WASM export not found: ${name}`);
-   }
-   return fn;
-}
-
-/**
- * Allocates `size` bytes in WASM linear memory.
- *
- * Returns `0` when `size` is zero or the allocator export is
- * absent.
- */
-export function wasmAlloc(modules: WasmModules, size: number): number {
-   if (size === 0) return 0;
-   const allocFn = modules.kbWorker.exports['kbdb_alloc'] as
-      | WasmFn
-      | undefined;
-   if (typeof allocFn !== 'function') return 0;
-   return allocFn(size) as number;
-}
-
-/**
- * Frees a previously allocated WASM memory region.
- *
- * No-ops when `ptr` or `size` is zero, or when the free export
- * is absent.
- */
-export function wasmFree(
-   modules: WasmModules,
-   ptr: number,
-   size: number,
-): void {
-   if (ptr === 0 || size === 0) return;
-   const freeFn = modules.kbWorker.exports['kbdb_free'] as
-      | WasmFn
-      | undefined;
-   if (typeof freeFn === 'function') {
-      freeFn(ptr, size);
-   }
-}
-
-/**
- * Writes `data` into WASM linear memory at `ptr`.
- *
- * @throws {Error} if the write would exceed memory bounds
- */
-export function wasmWrite(
-   modules: WasmModules,
-   ptr: number,
-   data: Uint8Array,
-): void {
-   if (data.length === 0) return;
-   const memView = new Uint8Array(modules.memory.buffer);
-   if (ptr + data.length > memView.length) {
-      throw new Error('WASM memory overflow');
-   }
-   memView.set(data, ptr);
-}
-
-/**
- * Reads the WASM return slot, frees the underlying buffer, and
- * returns its contents as a `Uint8Array`.
- *
- * Returns an empty array when the slot is unset or memory is
- * unavailable.
- */
-export function readReturnSlot(modules: WasmModules): Uint8Array {
-   const getPtrFn = modules.kbWorker.exports['get_return_ptr'] as
-      | WasmFn
-      | undefined;
-   const getLenFn = modules.kbWorker.exports['get_return_len'] as
-      | WasmFn
-      | undefined;
-   if (
-      typeof getPtrFn !== 'function' ||
-      typeof getLenFn !== 'function'
-   ) {
-      return new Uint8Array(0);
-   }
-   const ptr = getPtrFn() as number;
-   const len = getLenFn() as number;
-   if (ptr === 0 || len === 0) return new Uint8Array(0);
-   const data = new Uint8Array(modules.memory.buffer, ptr, len).slice();
-   wasmFree(modules, ptr, len);
-   return data;
-}
-
-/**
- * Calls a WASM export with binary-encoded input via the
- * alloc/write/call/read/free protocol.
- *
- * @throws {IpcError} when the export returns `-1` (error sentinel)
- */
-export function callWasmSync(
-   modules: WasmModules,
-   exportName: string,
-   input: Uint8Array,
-): Uint8Array {
-   const fn = getWasmExport(modules, exportName);
-   const ptr = wasmAlloc(modules, input.length);
-   wasmWrite(modules, ptr, input);
-   const rc = fn(ptr, input.length) as number;
-   wasmFree(modules, ptr, input.length);
-   if (rc === -1) {
-      const errBytes = readReturnSlot(modules);
-      const errMsg = new TextDecoder().decode(errBytes);
-      throw new IpcError(-32000, errMsg || exportName + ' failed');
-   }
-   return readReturnSlot(modules);
-}
-
-/**
- * Calls a zero-argument WASM export and checks its return code.
- *
- * @throws {IpcError} when the export returns `-1` (error sentinel)
- */
-export function callWasmZeroArg(
-   modules: WasmModules,
-   exportName: string,
-): void {
-   const fn = getWasmExport(modules, exportName);
-   const rc = fn() as number;
-   if (rc === -1) {
-      const errBytes = readReturnSlot(modules);
-      const errMsg = new TextDecoder().decode(errBytes);
-      throw new IpcError(-32000, errMsg || exportName + ' failed');
-   }
-}

package/src/shared/worker-daemon/functions/wasm-memory-helpers.function.ts

@@ -1,41 +0,0 @@
-import { join } from 'node:path';
-import type { MemoryRef } from '../typings/memory-ref.interface.ts';
-
-/**
- * Returns the current `ArrayBuffer` backing WASM linear memory.
- *
- * Must be called after WASM module instantiation. The buffer
- * reference may change after WASM memory growth, so callers
- * should call this function each time they need a fresh view.
- *
- * @param memoryRef - deferred reference to WASM linear memory
- * @throws {Error} when memory is not yet initialised
- */
-export function getWasmBuffer(memoryRef: MemoryRef): ArrayBuffer {
-   if (!memoryRef.memory) {
-      throw new Error('WASM memory not initialized');
-   }
-   // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
-   return memoryRef.memory.buffer as ArrayBuffer;
-}
-
-/**
- * Decodes a UTF-8 path from WASM linear memory and resolves it
- * relative to `contextPath` when the path is not absolute.
- *
- * @param memoryRef - deferred reference to WASM linear memory
- * @param contextPath - base directory for relative paths
- * @param ptr - byte offset of the path string in WASM memory
- * @param len - byte length of the path string
- */
-export function readWasmPathArg(
-   memoryRef: MemoryRef,
-   contextPath: string,
-   ptr: number,
-   len: number,
-): string {
-   const bytes = new Uint8Array(getWasmBuffer(memoryRef), ptr, len);
-   const decoded = new TextDecoder().decode(bytes);
-   if (decoded.startsWith('/')) return decoded;
-   return join(contextPath, decoded);
-}

package/src/shared/worker-daemon/functions/write-lock.function.ts

@@ -1,49 +0,0 @@
-import type { WriteLockState } from '../typings/write-lock.interface.ts';
-
-/**
- * Acquires the single-writer lock.
- *
- * Enforces the single-writer guarantee: at most one write
- * operation runs at a time inside the daemon. Concurrent writes
- * are serialized through an in-memory FIFO queue -- each caller
- * awaits this function and proceeds only when it is at the head
- * of the queue. Read operations bypass this lock entirely and
- * may proceed concurrently with writes and with each other.
- *
- * Crash recovery: the lock is purely in-memory. If the daemon
- * process crashes mid-write, the OS reclaims process memory,
- * releasing the lock automatically. The WASM engine is
- * responsible for leaving on-disk state consistent (it uses
- * atomic rename: write to a `.tmp` file, then `host_rename` to
- * the final path).
- *
- * @param state - shared write-lock state
- */
-export function acquireWriteLock(state: WriteLockState): Promise<void> {
-   if (!state.locked) {
-      state.locked = true;
-      return Promise.resolve();
-   }
-   return new Promise<void>((resolve) => {
-      state.queue.push({ resolve });
-   });
-}
-
-/**
- * Releases the single-writer lock.
- *
- * Grants the lock to the next queued waiter if one exists;
- * otherwise marks the lock as free. Must always be called in a
- * `finally` block after `acquireWriteLock` to prevent permanent
- * lock starvation.
- *
- * @param state - shared write-lock state
- */
-export function releaseWriteLock(state: WriteLockState): void {
-   const next = state.queue.shift();
-   if (next !== undefined) {
-      next.resolve();
-   } else {
-      state.locked = false;
-   }
-}

package/src/shared/worker-daemon/index.ts

@@ -1,16 +0,0 @@
-export { readConfig } from './functions/read-config.function.ts';
-export { startDaemon } from './functions/start-daemon.function.ts';
-export { createIpcServer } from './functions/create-ipc-server.function.ts';
-export { loadWasmModules } from './functions/load-wasm-modules.function.ts';
-export type { DaemonConfig } from './typings/daemon-config.interface.ts';
-export type { WasmModules } from './typings/wasm-modules.interface.ts';
-export type {
-   IpcRequest,
-   IpcResponse,
-   IpcHandler,
-} from './typings/ipc-handler.interface.ts';
-export { runWorker } from './functions/run-worker.function.ts';
-export type {
-   WorkerResult,
-   WorkerDeps,
-} from './functions/run-worker.function.ts';

package/src/shared/worker-daemon/typings/allocator-ref.interface.ts

@@ -1,16 +0,0 @@
-/**
- * A mutable reference to the WASM allocator function.
- *
- * The `alloc` field starts as `null` and is wired up after the
- * `kb-worker` WASM module is instantiated. Host import functions
- * that must allocate memory in WASM linear memory check this
- * reference before proceeding and return `-1` when it is null.
- */
-export interface AllocatorRef {
-   /**
-    * Allocates `size` bytes in WASM linear memory and returns the
-    * byte offset of the allocated region, or `null` when the
-    * allocator is not yet available.
-    */
-   alloc: ((size: number) => number) | null;
-}

package/src/shared/worker-daemon/typings/daemon-config.interface.ts

@@ -1,20 +0,0 @@
-import type { EmbeddingConfig } from '../../embedding/typings/embedding-config.interface.ts';
-import type { AutoCaptureConfig } from '../../auto-capture/typings/auto-capture-config.interface.ts';
-
-/** Configuration for the worker daemon process. */
-export interface DaemonConfig {
-   /** Absolute path to the knowledge-base context directory. */
-   readonly contextPath: string;
-   /** Maximum number of items held in memory at once. */
-   readonly itemLimit: number;
-   /** Maximum number of index entries held in memory at once. */
-   readonly indexLimit: number;
-   /** Idle timeout in milliseconds before the daemon shuts down. */
-   readonly timeoutMs: number;
-   /** Time-to-live in milliseconds for each cached item. */
-   readonly itemTtlMs: number;
-   /** Embedding provider configuration. */
-   readonly embedding?: EmbeddingConfig;
-   /** Auto-capture behaviour configuration. */
-   readonly autoCapture?: AutoCaptureConfig;
-}

package/src/shared/worker-daemon/typings/fs-adapter.interface.ts

@@ -1,22 +0,0 @@
-/**
- * Minimal filesystem operations required by the WASM host imports.
- *
- * The default implementation delegates to `node:fs` synchronous
- * APIs. Tests inject a stub to avoid touching the real filesystem.
- */
-export interface FsAdapter {
-   /** Read entire file as a Buffer. */
-   readFileSync(path: string): Buffer;
-   /** Write data buffer to file. */
-   writeFileSync(path: string, data: Uint8Array): void;
-   /** Rename `oldPath` to `newPath`. */
-   renameSync(oldPath: string, newPath: string): void;
-   /** Delete a file. */
-   unlinkSync(path: string): void;
-   /** List directory entries as strings. */
-   readdirSync(path: string): string[];
-   /** Create a directory (and parents) recursively. */
-   mkdirSync(path: string): void;
-   /** Return true if the path exists. */
-   existsSync(path: string): boolean;
-}

package/src/shared/worker-daemon/typings/handler-context.interface.ts

@@ -1,19 +0,0 @@
-import type { DaemonConfig } from './daemon-config.interface.ts';
-import type { WasmModules } from './wasm-modules.interface.ts';
-
-/**
- * Runtime context passed to all IPC request handler functions.
- *
- * Groups the minimal shared state needed by every request handler
- * so that handler functions can be standalone (not methods on a
- * class) while still accessing WASM modules, configuration, and
- * the database directory path.
- */
-export interface HandlerContext {
-   /** Absolute path to the `.kbdb` directory. */
-   readonly targetDir: string;
-   /** Parsed daemon configuration from `worker.toml`. */
-   readonly config: DaemonConfig;
-   /** Loaded WASM module instances. */
-   readonly modules: WasmModules;
-}

package/src/shared/worker-daemon/typings/ipc-handler.interface.ts

@@ -1,39 +0,0 @@
-/** A JSON-RPC 2.0 request sent to the daemon over IPC. */
-export interface IpcRequest {
-   /** JSON-RPC version string -- always `'2.0'`. */
-   readonly jsonrpc: '2.0';
-   /** Request identifier, echoed back in the response. */
-   readonly id: number | string;
-   /** Method name to invoke on the daemon. */
-   readonly method: string;
-   /** Optional method parameters. */
-   readonly params?: unknown;
-}
-
-/** Error detail included in a failed JSON-RPC response. */
-export interface IpcErrorDetail {
-   /** JSON-RPC error code. */
-   readonly code: number;
-   /** Human-readable error message. */
-   readonly message: string;
-   /** Optional additional error data. */
-   readonly data?: unknown;
-}
-
-/** A JSON-RPC 2.0 response returned by the daemon over IPC. */
-export interface IpcResponse {
-   /** JSON-RPC version string -- always `'2.0'`. */
-   readonly jsonrpc: '2.0';
-   /** Identifier matching the originating request. */
-   readonly id: number | string;
-   /** Successful result payload. Present on success. */
-   readonly result?: unknown;
-   /** Error detail. Present on failure. */
-   readonly error?: IpcErrorDetail;
-}
-
-/**
- * Function that handles a single IPC request and returns a
- * JSON-RPC response.
- */
-export type IpcHandler = (request: IpcRequest) => Promise<IpcResponse>;

package/src/shared/worker-daemon/typings/memory-ref.interface.ts

@@ -1,13 +0,0 @@
-/**
- * A mutable reference to the WASM linear memory.
- *
- * The `memory` field starts as `null` and is wired up after the
- * `kb-worker` WASM module is instantiated (the module exports its
- * own memory rather than importing one). Host import functions
- * read `memory` at call time, not at creation time, so the ref
- * can be populated after the import record is built.
- */
-export interface MemoryRef {
-   /** WASM linear memory; `null` until the module is instantiated. */
-   memory: WebAssembly.Memory | null;
-}

package/src/shared/worker-daemon/typings/wasm-fn.type.ts

@@ -1,2 +0,0 @@
-/** Callable WASM export function signature. */
-export type WasmFn = (...args: unknown[]) => unknown;

package/src/shared/worker-daemon/typings/wasm-modules.interface.ts

@@ -1,7 +0,0 @@
-/** Loaded and instantiated WASM module instances. */
-export interface WasmModules {
-   /** Linear memory exported by the kb-worker WASM module. */
-   readonly memory: WebAssembly.Memory;
-   /** Instantiated kb-worker WASM module. */
-   readonly kbWorker: WebAssembly.Instance;
-}