@dikolab/kbdb 0.3.0 → 0.3.1

package/src/shared/worker-daemon/functions/run-startup-integrity-check.function.ts

@@ -0,0 +1,55 @@
+import type { WasmModules } from '../typings/wasm-modules.interface.ts';
+import {
+   callWasmZeroArg,
+   readReturnSlot,
+} from './wasm-bridge.function.ts';
+import { decodeKeyValueText } from '../../wasm-codec/index.ts';
+
+/** Result of a startup integrity check. */
+export interface StartupIntegrityResult {
+   /** Whether the check passed without errors. */
+   readonly integrityOk: boolean;
+   /** Integrity error descriptions found at startup. */
+   readonly integrityErrors: string[];
+}
+
+/**
+ * Runs `worker_check_integrity` immediately after WASM context
+ * initialisation and returns the result.
+ *
+ * Logs a warning to stderr when issues are found. Never throws --
+ * failures are captured and returned as part of the result.
+ *
+ * @param modules - loaded WASM module instances
+ */
+export function runStartupIntegrityCheck(
+   modules: WasmModules,
+): StartupIntegrityResult {
+   try {
+      callWasmZeroArg(modules, 'worker_check_integrity');
+      const bytes = readReturnSlot(modules);
+      const kv = decodeKeyValueText(new TextDecoder().decode(bytes));
+      const integrityOk = kv['ok'] === 'true';
+      if (!integrityOk) {
+         const integrityErrors = Object.entries(kv)
+            .filter(([k]) => k !== 'ok')
+            .map(([k, v]) => `${k}=${v}`);
+         // eslint-disable-next-line no-console
+         console.error(
+            '[kbdb-worker] startup integrity check failed:',
+            integrityErrors.join(', '),
+         );
+         return { integrityOk: false, integrityErrors };
+      }
+      return { integrityOk: true, integrityErrors: [] };
+   } catch {
+      // eslint-disable-next-line no-console
+      console.error(
+         '[kbdb-worker] startup integrity check threw an error',
+      );
+      return {
+         integrityOk: false,
+         integrityErrors: ['integrity_check_failed'],
+      };
+   }
+}

package/src/shared/worker-daemon/functions/scan-near-duplicates.function.ts

@@ -0,0 +1,92 @@
+import { readdirSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+/** A near-duplicate match found during add-section scanning. */
+export interface NearDuplicateMatch {
+   /** Knowledge-base identifier of the near-duplicate section. */
+   readonly kbid: string;
+   /** Jaccard similarity score, rounded to 2 decimal places (0–1). */
+   readonly similarity: number;
+}
+
+/**
+ * Scans existing section files for near-duplicates of `content`.
+ *
+ * Uses Jaccard similarity on a set of lowercased tokens (3+ chars)
+ * extracted from the section body. Sections with a similarity of
+ * 0.85 or above are included in the result. The `newKbid` section
+ * is excluded from comparison.
+ *
+ * @param targetDir - absolute path to the `.kbdb` directory
+ * @param content - raw text content of the newly added section
+ * @param newKbid - identifier of the just-added section to exclude
+ * @returns near-duplicate matches sorted by insertion order
+ */
+export function scanNearDuplicates(
+   targetDir: string,
+   content: string,
+   newKbid: string,
+): NearDuplicateMatch[] {
+   const newTerms = extractTermSet(content);
+   if (newTerms.size === 0) return [];
+
+   const sectionsDir = join(targetDir, 'sections');
+   let shards: string[];
+   try {
+      shards = readdirSync(sectionsDir);
+   } catch {
+      return [];
+   }
+
+   const results: NearDuplicateMatch[] = [];
+   for (const shard of shards) {
+      let files: string[];
+      try {
+         files = readdirSync(join(sectionsDir, shard));
+      } catch {
+         continue;
+      }
+      for (const file of files) {
+         if (!file.endsWith('.section')) continue;
+         const kbid = file.slice(0, -8);
+         if (kbid === newKbid) continue;
+         try {
+            const data = readFileSync(
+               join(sectionsDir, shard, file),
+               'utf-8',
+            );
+            const sep = data.indexOf('\n---\n');
+            const body = sep >= 0 ? data.slice(sep + 5) : data;
+            const existing = extractTermSet(body);
+            let intersection = 0;
+            for (const t of newTerms) {
+               if (existing.has(t)) intersection++;
+            }
+            const union = newTerms.size + existing.size - intersection;
+            const sim = union > 0 ? intersection / union : 0;
+            if (sim >= 0.85) {
+               results.push({
+                  kbid,
+                  similarity: Math.round(sim * 100) / 100,
+               });
+            }
+         } catch {
+            continue;
+         }
+      }
+   }
+   return results;
+}
+
+/**
+ * Extracts a set of lowercased word tokens with 3 or more
+ * characters from `text`.
+ */
+function extractTermSet(text: string): Set<string> {
+   return new Set(
+      text
+         .toLowerCase()
+         .split(/\W+/)
+         .filter((w) => w.length > 2),
+   );
+}

package/src/shared/worker-daemon/functions/wasm-bridge.function.ts

@@ -0,0 +1,144 @@
+import type { WasmModules } from '../typings/wasm-modules.interface.ts';
+import type { WasmFn } from '../typings/wasm-fn.type.ts';
+import { IpcError } from '../classes/ipc-error.class.ts';
+
+/**
+ * Retrieves a named export from the kb-worker WASM instance.
+ *
+ * @throws {Error} if WASM modules are not yet loaded
+ * @throws {Error} if the named export does not exist or is not a
+ *   function
+ */
+export function getWasmExport(
+   modules: WasmModules,
+   name: string,
+): WasmFn {
+   const fn = modules.kbWorker.exports[name] as WasmFn | undefined;
+   if (typeof fn !== 'function') {
+      throw new Error(`WASM export not found: ${name}`);
+   }
+   return fn;
+}
+
+/**
+ * Allocates `size` bytes in WASM linear memory.
+ *
+ * Returns `0` when `size` is zero or the allocator export is
+ * absent.
+ */
+export function wasmAlloc(modules: WasmModules, size: number): number {
+   if (size === 0) return 0;
+   const allocFn = modules.kbWorker.exports['kbdb_alloc'] as
+      | WasmFn
+      | undefined;
+   if (typeof allocFn !== 'function') return 0;
+   return allocFn(size) as number;
+}
+
+/**
+ * Frees a previously allocated WASM memory region.
+ *
+ * No-ops when `ptr` or `size` is zero, or when the free export
+ * is absent.
+ */
+export function wasmFree(
+   modules: WasmModules,
+   ptr: number,
+   size: number,
+): void {
+   if (ptr === 0 || size === 0) return;
+   const freeFn = modules.kbWorker.exports['kbdb_free'] as
+      | WasmFn
+      | undefined;
+   if (typeof freeFn === 'function') {
+      freeFn(ptr, size);
+   }
+}
+
+/**
+ * Writes `data` into WASM linear memory at `ptr`.
+ *
+ * @throws {Error} if the write would exceed memory bounds
+ */
+export function wasmWrite(
+   modules: WasmModules,
+   ptr: number,
+   data: Uint8Array,
+): void {
+   if (data.length === 0) return;
+   const memView = new Uint8Array(modules.memory.buffer);
+   if (ptr + data.length > memView.length) {
+      throw new Error('WASM memory overflow');
+   }
+   memView.set(data, ptr);
+}
+
+/**
+ * Reads the WASM return slot, frees the underlying buffer, and
+ * returns its contents as a `Uint8Array`.
+ *
+ * Returns an empty array when the slot is unset or memory is
+ * unavailable.
+ */
+export function readReturnSlot(modules: WasmModules): Uint8Array {
+   const getPtrFn = modules.kbWorker.exports['get_return_ptr'] as
+      | WasmFn
+      | undefined;
+   const getLenFn = modules.kbWorker.exports['get_return_len'] as
+      | WasmFn
+      | undefined;
+   if (
+      typeof getPtrFn !== 'function' ||
+      typeof getLenFn !== 'function'
+   ) {
+      return new Uint8Array(0);
+   }
+   const ptr = getPtrFn() as number;
+   const len = getLenFn() as number;
+   if (ptr === 0 || len === 0) return new Uint8Array(0);
+   const data = new Uint8Array(modules.memory.buffer, ptr, len).slice();
+   wasmFree(modules, ptr, len);
+   return data;
+}
+
+/**
+ * Calls a WASM export with binary-encoded input via the
+ * alloc/write/call/read/free protocol.
+ *
+ * @throws {IpcError} when the export returns `-1` (error sentinel)
+ */
+export function callWasmSync(
+   modules: WasmModules,
+   exportName: string,
+   input: Uint8Array,
+): Uint8Array {
+   const fn = getWasmExport(modules, exportName);
+   const ptr = wasmAlloc(modules, input.length);
+   wasmWrite(modules, ptr, input);
+   const rc = fn(ptr, input.length) as number;
+   wasmFree(modules, ptr, input.length);
+   if (rc === -1) {
+      const errBytes = readReturnSlot(modules);
+      const errMsg = new TextDecoder().decode(errBytes);
+      throw new IpcError(-32000, errMsg || exportName + ' failed');
+   }
+   return readReturnSlot(modules);
+}
+
+/**
+ * Calls a zero-argument WASM export and checks its return code.
+ *
+ * @throws {IpcError} when the export returns `-1` (error sentinel)
+ */
+export function callWasmZeroArg(
+   modules: WasmModules,
+   exportName: string,
+): void {
+   const fn = getWasmExport(modules, exportName);
+   const rc = fn() as number;
+   if (rc === -1) {
+      const errBytes = readReturnSlot(modules);
+      const errMsg = new TextDecoder().decode(errBytes);
+      throw new IpcError(-32000, errMsg || exportName + ' failed');
+   }
+}

package/src/shared/worker-daemon/functions/wasm-memory-helpers.function.ts

@@ -0,0 +1,41 @@
+import { join } from 'node:path';
+import type { MemoryRef } from '../typings/memory-ref.interface.ts';
+
+/**
+ * Returns the current `ArrayBuffer` backing WASM linear memory.
+ *
+ * Must be called after WASM module instantiation. The buffer
+ * reference may change after WASM memory growth, so callers
+ * should call this function each time they need a fresh view.
+ *
+ * @param memoryRef - deferred reference to WASM linear memory
+ * @throws {Error} when memory is not yet initialised
+ */
+export function getWasmBuffer(memoryRef: MemoryRef): ArrayBuffer {
+   if (!memoryRef.memory) {
+      throw new Error('WASM memory not initialized');
+   }
+   // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
+   return memoryRef.memory.buffer as ArrayBuffer;
+}
+
+/**
+ * Decodes a UTF-8 path from WASM linear memory and resolves it
+ * relative to `contextPath` when the path is not absolute.
+ *
+ * @param memoryRef - deferred reference to WASM linear memory
+ * @param contextPath - base directory for relative paths
+ * @param ptr - byte offset of the path string in WASM memory
+ * @param len - byte length of the path string
+ */
+export function readWasmPathArg(
+   memoryRef: MemoryRef,
+   contextPath: string,
+   ptr: number,
+   len: number,
+): string {
+   const bytes = new Uint8Array(getWasmBuffer(memoryRef), ptr, len);
+   const decoded = new TextDecoder().decode(bytes);
+   if (decoded.startsWith('/')) return decoded;
+   return join(contextPath, decoded);
+}

package/src/shared/worker-daemon/functions/write-lock.function.ts

@@ -0,0 +1,49 @@
+import type { WriteLockState } from '../typings/write-lock.interface.ts';
+
+/**
+ * Acquires the single-writer lock.
+ *
+ * Enforces the single-writer guarantee: at most one write
+ * operation runs at a time inside the daemon. Concurrent writes
+ * are serialized through an in-memory FIFO queue -- each caller
+ * awaits this function and proceeds only when it is at the head
+ * of the queue. Read operations bypass this lock entirely and
+ * may proceed concurrently with writes and with each other.
+ *
+ * Crash recovery: the lock is purely in-memory. If the daemon
+ * process crashes mid-write, the OS reclaims process memory,
+ * releasing the lock automatically. The WASM engine is
+ * responsible for leaving on-disk state consistent (it uses
+ * atomic rename: write to a `.tmp` file, then `host_rename` to
+ * the final path).
+ *
+ * @param state - shared write-lock state
+ */
+export function acquireWriteLock(state: WriteLockState): Promise<void> {
+   if (!state.locked) {
+      state.locked = true;
+      return Promise.resolve();
+   }
+   return new Promise<void>((resolve) => {
+      state.queue.push({ resolve });
+   });
+}
+
+/**
+ * Releases the single-writer lock.
+ *
+ * Grants the lock to the next queued waiter if one exists;
+ * otherwise marks the lock as free. Must always be called in a
+ * `finally` block after `acquireWriteLock` to prevent permanent
+ * lock starvation.
+ *
+ * @param state - shared write-lock state
+ */
+export function releaseWriteLock(state: WriteLockState): void {
+   const next = state.queue.shift();
+   if (next !== undefined) {
+      next.resolve();
+   } else {
+      state.locked = false;
+   }
+}

package/src/shared/worker-daemon/typings/allocator-ref.interface.ts

@@ -0,0 +1,16 @@
+/**
+ * A mutable reference to the WASM allocator function.
+ *
+ * The `alloc` field starts as `null` and is wired up after the
+ * `kb-worker` WASM module is instantiated. Host import functions
+ * that must allocate memory in WASM linear memory check this
+ * reference before proceeding and return `-1` when it is null.
+ */
+export interface AllocatorRef {
+   /**
+    * Allocates `size` bytes in WASM linear memory and returns the
+    * byte offset of the allocated region, or `null` when the
+    * allocator is not yet available.
+    */
+   alloc: ((size: number) => number) | null;
+}

package/src/shared/worker-daemon/typings/daemon-config.interface.ts

@@ -1,3 +1,6 @@
+import type { EmbeddingConfig } from '../../embedding/typings/embedding-config.interface.ts';
+import type { AutoCaptureConfig } from '../../auto-capture/typings/auto-capture-config.interface.ts';
+
 /** Configuration for the worker daemon process. */
 export interface DaemonConfig {
    /** Absolute path to the knowledge-base context directory. */
@@ -10,4 +13,8 @@ export interface DaemonConfig {
    readonly timeoutMs: number;
    /** Time-to-live in milliseconds for each cached item. */
    readonly itemTtlMs: number;
+   /** Embedding provider configuration. */
+   readonly embedding?: EmbeddingConfig;
+   /** Auto-capture behaviour configuration. */
+   readonly autoCapture?: AutoCaptureConfig;
 }

package/src/shared/worker-daemon/typings/fs-adapter.interface.ts

@@ -0,0 +1,22 @@
+/**
+ * Minimal filesystem operations required by the WASM host imports.
+ *
+ * The default implementation delegates to `node:fs` synchronous
+ * APIs. Tests inject a stub to avoid touching the real filesystem.
+ */
+export interface FsAdapter {
+   /** Read entire file as a Buffer. */
+   readFileSync(path: string): Buffer;
+   /** Write data buffer to file. */
+   writeFileSync(path: string, data: Uint8Array): void;
+   /** Rename `oldPath` to `newPath`. */
+   renameSync(oldPath: string, newPath: string): void;
+   /** Delete a file. */
+   unlinkSync(path: string): void;
+   /** List directory entries as strings. */
+   readdirSync(path: string): string[];
+   /** Create a directory (and parents) recursively. */
+   mkdirSync(path: string): void;
+   /** Return true if the path exists. */
+   existsSync(path: string): boolean;
+}

package/src/shared/worker-daemon/typings/handler-context.interface.ts

@@ -0,0 +1,19 @@
+import type { DaemonConfig } from './daemon-config.interface.ts';
+import type { WasmModules } from './wasm-modules.interface.ts';
+
+/**
+ * Runtime context passed to all IPC request handler functions.
+ *
+ * Groups the minimal shared state needed by every request handler
+ * so that handler functions can be standalone (not methods on a
+ * class) while still accessing WASM modules, configuration, and
+ * the database directory path.
+ */
+export interface HandlerContext {
+   /** Absolute path to the `.kbdb` directory. */
+   readonly targetDir: string;
+   /** Parsed daemon configuration from `worker.toml`. */
+   readonly config: DaemonConfig;
+   /** Loaded WASM module instances. */
+   readonly modules: WasmModules;
+}

package/src/shared/worker-daemon/typings/memory-ref.interface.ts

@@ -0,0 +1,13 @@
+/**
+ * A mutable reference to the WASM linear memory.
+ *
+ * The `memory` field starts as `null` and is wired up after the
+ * `kb-worker` WASM module is instantiated (the module exports its
+ * own memory rather than importing one). Host import functions
+ * read `memory` at call time, not at creation time, so the ref
+ * can be populated after the import record is built.
+ */
+export interface MemoryRef {
+   /** WASM linear memory; `null` until the module is instantiated. */
+   memory: WebAssembly.Memory | null;
+}

package/src/shared/worker-daemon/typings/wasm-fn.type.ts

@@ -0,0 +1,2 @@
+/** Callable WASM export function signature. */
+export type WasmFn = (...args: unknown[]) => unknown;

package/src/shared/worker-daemon/typings/write-lock.interface.ts

@@ -0,0 +1,13 @@
+/**
+ * Mutable state for the single-writer lock used by WorkerDaemon.
+ *
+ * Holds the current lock flag and the queue of pending waiters.
+ * Both fields are mutated by `acquireWriteLock` and
+ * `releaseWriteLock`.
+ */
+export interface WriteLockState {
+   /** Whether the write lock is currently held. */
+   locked: boolean;
+   /** Queue of resolve callbacks for pending lock waiters. */
+   queue: { resolve: () => void }[];
+}