@dikolab/kbdb 0.2.2 → 0.3.1

package/src/shared/worker-daemon/classes/worker-daemon.class.ts

@@ -1,6 +1,7 @@
 import {
    existsSync,
    readFileSync,
+   readdirSync,
    accessSync,
    writeFileSync,
    unlinkSync,
@@ -38,6 +39,7 @@ import {
    decodeKeyValueText,
    decodeRemoveFlag,
    decodeRecallResult,
+   decodeAddSectionResult,
 } from '../../wasm-codec/index.ts';
 
 import type { AddSectionInput } from '../../wasm-codec/index.ts';
@@ -89,6 +91,8 @@ const WASM_METHOD_MAP: Readonly<Partial<Record<string, string>>> = {
    listDocumentsByType: 'worker_list_docs_by_type',
    list_docs_by_type: 'worker_list_docs_by_type',
    recall: 'worker_recall',
+   export: 'worker_export',
+   exportDb: 'worker_export',
 };
 
 /** Config method name prefix. */
@@ -155,6 +159,18 @@ export class WorkerDaemon {
    private _server: Server | null = null;
    private readonly _wasmDir: string;
 
+   /** Queue of waiters for the single-writer lock. */
+   private _writeQueue: { resolve: () => void }[] = [];
+
+   /** Whether the write lock is currently held. */
+   private _writeLocked = false;
+
+   /** Whether the startup integrity check passed. */
+   private _integrityOk = true;
+
+   /** Integrity errors found at startup, if any. */
+   private _integrityErrors: string[] = [];
+
    /**
     * @param ctx - context ID (16-hex characters), the first 16 of
     *   the SHA-256 hash of `targetDir`
@@ -229,6 +245,7 @@ export class WorkerDaemon {
       this._wasmModules = wasmModules;
 
       this.initWorkerContext();
+      this.runStartupIntegrityCheck();
 
       const pidPath = join(tmpdir(), `kbdb-${this.ctx}.pid`);
       writeFileSync(pidPath, process.pid.toString());
@@ -470,7 +487,11 @@ export class WorkerDaemon {
       params: Record<string, unknown>,
    ): unknown {
       if (method === 'status') {
-         return this._connectionCount > 0 ? 'busy' : 'idle';
+         return {
+            status: this._connectionCount > 0 ? 'busy' : 'idle',
+            integrityOk: this._integrityOk,
+            integrityErrors: this._integrityErrors,
+         };
       }
 
       if (method === 'idle_at') {
@@ -493,71 +514,36 @@ export class WorkerDaemon {
       throw new IpcError(-32601, `Method not found: ${method}`);
    }
 
-   private dispatchWasm(
+   private async dispatchWasm(
       method: string,
       wasmExport: string,
       params: Record<string, unknown>,
-   ): unknown {
+   ): Promise<unknown> {
+      // Read-only operations — no write lock required.
       if (method === 'search' || method === 'query') {
          return this.handleSearch(params);
       }
-      if (method === 'addSection' || method === 'add_section') {
-         return this.handleAddSection(params);
-      }
-      if (method === 'updateSection' || method === 'update_section') {
-         return this.handleUpdateSection(params);
-      }
-      if (method === 'removeSection' || method === 'remove_section') {
-         return this.handleRemoveSection(params);
-      }
       if (method === 'readSections' || method === 'read_sections') {
          return this.handleReadSections(params);
       }
       if (method === 'retrieveDocument' || method === 'retrieve_doc') {
          return this.handleRetrieveDoc(params);
       }
-      if (method === 'groupSections' || method === 'group_sections') {
-         return this.handleGroupSections(params);
-      }
-      if (method === 'removeGrouping' || method === 'remove_grouping') {
-         return this.handleRemoveGrouping(params);
-      }
       if (method === 'content' || method === 'compose') {
          return this.handleCompose(params);
       }
       if (method === 'integrityCheck' || method === 'integrity_check') {
          return this.handleIntegrityCheck();
       }
-      if (method === 'gc') {
-         return this.handleGc();
-      }
-      if (method === 'rebuildIndexes' || method === 'rebuild_indexes') {
-         return this.handleRebuild();
-      }
       if (method === 'dbStatus' || method === 'db_status') {
          return this.handleDbStatus();
       }
-      if (method === 'initDirectory' || method === 'init_directory') {
-         return this.handleInitDirectory(params);
-      }
-      if (method === 'invalidate') {
-         return this.handleInvalidate(params);
-      }
-      if (method === 'invalidateAll' || method === 'invalidate_all') {
-         return this.handleInvalidateAll();
-      }
       if (method === 'get_result_status') {
          return this.handleResultStatus(params);
       }
       if (method === 'get_result') {
          return this.handleGetResult(params);
       }
-      if (method === 'migrateDatabase') {
-         throw new IpcError(
-            -32601,
-            'migrateDatabase is not yet supported',
-         );
-      }
       if (method === 'listByType' || method === 'list_by_type') {
          return this.handleListByType(params);
       }
@@ -570,7 +556,74 @@ export class WorkerDaemon {
       if (method === 'recall') {
          return this.handleRecall(params);
       }
-      return this.invokeWasmRaw(wasmExport, new Uint8Array(0));
+      if (method === 'export' || method === 'exportDb') {
+         return this.handleExport(params);
+      }
+
+      // Write operations — serialised through the single-writer lock.
+      await this.acquireWriteLock();
+      try {
+         if (method === 'invalidate') {
+            return this.handleInvalidate(params);
+         }
+         if (
+            method === 'invalidateAll' ||
+            method === 'invalidate_all'
+         ) {
+            return this.handleInvalidateAll();
+         }
+         if (method === 'addSection' || method === 'add_section') {
+            return this.handleAddSection(params);
+         }
+         if (
+            method === 'updateSection' ||
+            method === 'update_section'
+         ) {
+            return this.handleUpdateSection(params);
+         }
+         if (
+            method === 'removeSection' ||
+            method === 'remove_section'
+         ) {
+            return this.handleRemoveSection(params);
+         }
+         if (
+            method === 'groupSections' ||
+            method === 'group_sections'
+         ) {
+            return this.handleGroupSections(params);
+         }
+         if (
+            method === 'removeGrouping' ||
+            method === 'remove_grouping'
+         ) {
+            return this.handleRemoveGrouping(params);
+         }
+         if (method === 'gc') {
+            return this.handleGc();
+         }
+         if (
+            method === 'rebuildIndexes' ||
+            method === 'rebuild_indexes'
+         ) {
+            return this.handleRebuild();
+         }
+         if (
+            method === 'initDirectory' ||
+            method === 'init_directory'
+         ) {
+            return this.handleInitDirectory(params);
+         }
+         if (method === 'migrateDatabase') {
+            throw new IpcError(
+               -32601,
+               'migrateDatabase is not yet supported',
+            );
+         }
+         return this.invokeWasmRaw(wasmExport, new Uint8Array(0));
+      } finally {
+         this.releaseWriteLock();
+      }
    }
 
    // -- Search/query (token-based async) --
@@ -643,6 +696,9 @@ export class WorkerDaemon {
             : 'text';
       const typeName =
          sectionType !== semanticType ? sectionType : null;
+      const tags = Array.isArray(params['tags'])
+         ? (params['tags'] as string[])
+         : [];
       const input: AddSectionInput = {
          sectionType: semanticType,
          mimeType: params['mimeType'] as string | undefined,
@@ -651,11 +707,32 @@ export class WorkerDaemon {
          description: (params['description'] as string | null) ?? null,
          docid: (params['docid'] as string | null) ?? null,
          typeName,
+         sourcePath:
+            typeof params['sourcePath'] === 'string'
+               ? params['sourcePath']
+               : undefined,
+         sourceMtime:
+            typeof params['sourceMtime'] === 'string'
+               ? params['sourceMtime']
+               : undefined,
+         tags,
+         replace: params['replace'] === true,
       };
       const encoded = encodeAddSectionParams(input);
       const bytes = this.callWasmSync('worker_add_section', encoded);
-      const kbid = new TextDecoder().decode(bytes);
-      return { kbid };
+      const decoded = decodeAddSectionResult(bytes);
+      const nearDups =
+         decoded.nearDuplicates.length > 0
+            ? decoded.nearDuplicates
+            : this.scanNearDuplicates(input.content, decoded.kbid);
+      return {
+         kbid: decoded.kbid,
+         indexed_terms_count: decoded.indexedTermsCount,
+         elapsed_ms: decoded.elapsedMs,
+         superseded: decoded.superseded,
+         warnings: decoded.warnings,
+         near_duplicates: nearDups,
+      };
    }
 
    private handleUpdateSection(
@@ -759,7 +836,18 @@ export class WorkerDaemon {
    private handleInvalidate(params: Record<string, unknown>): unknown {
       const kbid = (params['kbid'] as string | undefined) ?? '';
       const encoded = new TextEncoder().encode(kbid);
-      this.callWasmSync('worker_invalidate', encoded);
+      const fn = this.getExport('worker_invalidate');
+      const ptr = this.wasmAlloc(encoded.length);
+      this.wasmWrite(ptr, encoded);
+      const rc = fn(ptr, encoded.length) as number;
+      this.wasmFree(ptr, encoded.length);
+      if (rc === -1) {
+         const errBytes = this.readReturnSlot();
+         throw new IpcError(
+            -32000,
+            new TextDecoder().decode(errBytes) || 'invalidate failed',
+         );
+      }
       return { ok: true };
    }
 
@@ -819,15 +907,161 @@ export class WorkerDaemon {
    }
 
    private handleGc(): unknown {
+      const start = Date.now();
       this.callWasmZeroArg('worker_garbage_collect');
       const bytes = this.readReturnSlot();
       const text = new TextDecoder().decode(bytes);
-      return decodeKeyValueText(text);
+      const kv = decodeKeyValueText(text);
+      const removedCount = Number(kv['removed'] ?? 0);
+      const staleSources = Number(kv['stale_sources'] ?? 0);
+      return {
+         removed_sections: removedCount,
+         removed_bytes: 0,
+         elapsed_ms: Date.now() - start,
+         stale_sources: staleSources,
+      };
    }
 
    private handleRebuild(): unknown {
-      this.callWasmZeroArg('worker_rebuild_indexes');
-      return { ok: true };
+      const fn = this.getExport('worker_rebuild_indexes');
+      const rc = fn() as number;
+      if (rc === -1) {
+         const errBytes = this.readReturnSlot();
+         const errMsg = new TextDecoder().decode(errBytes);
+         throw new IpcError(-32000, errMsg || 'rebuild failed');
+      }
+      const resultBytes = this.readReturnSlot();
+      const reindexed =
+         resultBytes.length >= 4
+            ? new DataView(
+                 resultBytes.buffer,
+                 resultBytes.byteOffset,
+                 resultBytes.byteLength,
+              ).getUint32(0, true)
+            : 0;
+      return { ok: true, sections_reindexed: reindexed };
+   }
+
+   // -- Export (read operation, no lock required) --
+
+   private handleExport(params: Record<string, unknown>): unknown {
+      const targetPath =
+         (params['targetPath'] as string | undefined) ??
+         (params['path'] as string | undefined) ??
+         './kbdb-export/';
+      const encoded = new TextEncoder().encode(targetPath);
+      const start = Date.now();
+      const bytes = this.callWasmSync('worker_export', encoded);
+      const elapsed = Date.now() - start;
+      const text = new TextDecoder().decode(bytes);
+      const kv = decodeKeyValueText(text);
+      return {
+         path: kv['path'] ?? targetPath,
+         sections_exported: Number(kv['sections_exported'] ?? 0),
+         documents_exported: Number(kv['documents_exported'] ?? 0),
+         elapsed_ms: elapsed,
+      };
+   }
+
+   // -- Near-duplicate detection --
+
+   private scanNearDuplicates(
+      content: string,
+      newKbid: string,
+   ): { kbid: string; similarity: number }[] {
+      const newTerms = this.extractTermSet(content);
+      if (newTerms.size === 0) return [];
+
+      const sectionsDir = join(this.targetDir, 'sections');
+      let shards: string[];
+      try {
+         shards = readdirSync(sectionsDir);
+      } catch {
+         return [];
+      }
+
+      const results: { kbid: string; similarity: number }[] = [];
+      for (const shard of shards) {
+         let files: string[];
+         try {
+            files = readdirSync(join(sectionsDir, shard));
+         } catch {
+            continue;
+         }
+         for (const file of files) {
+            if (!file.endsWith('.section')) continue;
+            const kbid = file.slice(0, -8);
+            if (kbid === newKbid) continue;
+            try {
+               const data = readFileSync(
+                  join(sectionsDir, shard, file),
+                  'utf-8',
+               );
+               const sep = data.indexOf('\n---\n');
+               const body = sep >= 0 ? data.slice(sep + 5) : data;
+               const existing = this.extractTermSet(body);
+               let intersection = 0;
+               for (const t of newTerms) {
+                  if (existing.has(t)) intersection++;
+               }
+               const union =
+                  newTerms.size + existing.size - intersection;
+               const sim = union > 0 ? intersection / union : 0;
+               if (sim >= 0.85) {
+                  results.push({
+                     kbid,
+                     similarity: Math.round(sim * 100) / 100,
+                  });
+               }
+            } catch {
+               continue;
+            }
+         }
+      }
+      return results;
+   }
+
+   private extractTermSet(text: string): Set<string> {
+      return new Set(
+         text
+            .toLowerCase()
+            .split(/\W+/)
+            .filter((w) => w.length > 2),
+      );
+   }
+
+   // -- Single-writer lock --
+
+   /**
+    * Acquires the write lock.
+    *
+    * If the lock is free it is granted immediately. If it is held the
+    * caller is queued and its promise resolves once the lock is
+    * released by the current holder.
+    */
+   private acquireWriteLock(): Promise<void> {
+      if (!this._writeLocked) {
+         this._writeLocked = true;
+         return Promise.resolve();
+      }
+      return new Promise<void>((resolve) => {
+         this._writeQueue.push({ resolve });
+      });
+   }
+
+   /**
+    * Releases the write lock.
+    *
+    * If waiters are queued the next one is granted the lock
+    * immediately. Otherwise the lock is marked free.
+    */
+   private releaseWriteLock(): void {
+      const next = this._writeQueue.shift();
+      if (next !== undefined) {
+         next.resolve();
+      } else {
+         this._writeLocked = false;
+      }
    }
 
    // -- WASM lifecycle --
@@ -837,6 +1071,41 @@ export class WorkerDaemon {
       this.callWasmSync('worker_init', pathBytes);
    }
 
+   /**
+    * Runs a database integrity check immediately after context
+    * initialisation.
+    *
+    * Stores the result in `_integrityOk` and `_integrityErrors`.
+    * Logs a warning to stderr when issues are found so operators
+    * are alerted without aborting the daemon.
+    */
+   private runStartupIntegrityCheck(): void {
+      try {
+         this.callWasmZeroArg('worker_check_integrity');
+         const bytes = this.readReturnSlot();
+         const text = new TextDecoder().decode(bytes);
+         const kv = decodeKeyValueText(text);
+         this._integrityOk = kv['ok'] === 'true';
+         if (!this._integrityOk) {
+            this._integrityErrors = Object.entries(kv)
+               .filter(([k]) => k !== 'ok')
+               .map(([k, v]) => `${k}=${v}`);
+            // eslint-disable-next-line no-console
+            console.error(
+               '[kbdb-worker] startup integrity check failed:',
+               this._integrityErrors.join(', '),
+            );
+         }
+      } catch {
+         this._integrityOk = false;
+         this._integrityErrors = ['integrity_check_failed'];
+         // eslint-disable-next-line no-console
+         console.error(
+            '[kbdb-worker] startup integrity check threw an error',
+         );
+      }
+   }
+
    // -- WASM bridge primitives --
 
    private getExport(name: string): WasmFn {

package/src/shared/worker-daemon/constants/wasm-method-map.constant.ts

@@ -0,0 +1,69 @@
+import type { DaemonConfig } from '../typings/daemon-config.interface.ts';
+
+/**
+ * Maps incoming IPC method names to `kb-worker.wasm` export names.
+ *
+ * Both camelCase and snake_case variants of each method are
+ * included so that clients may use either convention.
+ */
+export const WASM_METHOD_MAP: Readonly<
+   Partial<Record<string, string>>
+> = {
+   search: 'worker_search',
+   addSection: 'worker_add_section',
+   add_section: 'worker_add_section',
+   updateSection: 'worker_update_section',
+   update_section: 'worker_update_section',
+   removeSection: 'worker_remove_section',
+   remove_section: 'worker_remove_section',
+   readSections: 'worker_read_sections',
+   read_sections: 'worker_read_sections',
+   groupSections: 'worker_group_sections',
+   group_sections: 'worker_group_sections',
+   removeGrouping: 'worker_remove_grouping',
+   remove_grouping: 'worker_remove_grouping',
+   retrieveDocument: 'worker_retrieve_doc',
+   retrieve_doc: 'worker_retrieve_doc',
+   content: 'worker_compose',
+   compose: 'worker_compose',
+   integrity_check: 'worker_check_integrity',
+   integrityCheck: 'worker_check_integrity',
+   gc: 'worker_garbage_collect',
+   rebuild_indexes: 'worker_rebuild_indexes',
+   rebuildIndexes: 'worker_rebuild_indexes',
+   init_directory: 'worker_init_directory',
+   initDirectory: 'worker_init_directory',
+   migrateDatabase: 'worker_migrate',
+   query: 'worker_query',
+   get_result_status: 'worker_result_status',
+   get_result: 'worker_get_result',
+   invalidate: 'worker_invalidate',
+   invalidate_all: 'worker_invalidate_all',
+   invalidateAll: 'worker_invalidate_all',
+   db_status: 'worker_status',
+   dbStatus: 'worker_status',
+   listByType: 'worker_list_by_type',
+   list_by_type: 'worker_list_by_type',
+   listDocumentsByType: 'worker_list_docs_by_type',
+   list_docs_by_type: 'worker_list_docs_by_type',
+   recall: 'worker_recall',
+   export: 'worker_export',
+   exportDb: 'worker_export',
+};
+
+/** Prefix shared by all IPC config-accessor method names. */
+export const CONFIG_PREFIX = 'config_';
+
+/**
+ * Maps config IPC key suffixes to `DaemonConfig` property names.
+ *
+ * A client sends `config_timeout_ms` to read `config.timeoutMs`.
+ */
+export const CONFIG_KEY_MAP: Readonly<
+   Partial<Record<string, keyof DaemonConfig>>
+> = {
+   timeout_ms: 'timeoutMs',
+   item_ttl_ms: 'itemTtlMs',
+   item_limit: 'itemLimit',
+   index_limit: 'indexLimit',
+};

package/src/shared/worker-daemon/functions/build-ipc-error-response.function.ts

@@ -0,0 +1,16 @@
+import type { IpcResponse } from '../typings/ipc-handler.interface.ts';
+
+/**
+ * Builds a JSON-RPC 2.0 error response object.
+ *
+ * @param id - request identifier to echo back in the response
+ * @param code - JSON-RPC error code
+ * @param message - human-readable error description
+ */
+export function buildIpcErrorResponse(
+   id: number | string,
+   code: number,
+   message: string,
+): IpcResponse {
+   return { jsonrpc: '2.0', id, error: { code, message } };
+}

package/src/shared/worker-daemon/functions/check-catalog-version.function.ts

@@ -0,0 +1,54 @@
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { DB_FORMAT_VERSION } from '../constants/daemon-defaults.constant.ts';
+import { parseCatalogVersion } from './parse-catalog-version.function.ts';
+
+/**
+ * Reads `catalog.toml` in `targetDir` and validates its version
+ * against `DB_FORMAT_VERSION`.
+ *
+ * @param targetDir - absolute path to the `.kbdb` directory
+ * @throws {Error} when the catalog cannot be read, is missing a
+ *   version, or the version is incompatible with the current engine
+ */
+export function checkCatalogVersion(targetDir: string): void {
+   const catalogPath = join(targetDir, 'catalog.toml');
+   let text: string;
+   try {
+      text = readFileSync(catalogPath, 'utf-8');
+   } catch {
+      throw new Error(
+         `cannot read database directory: ${targetDir}` +
+            ' -- permission denied',
+      );
+   }
+
+   const version = parseCatalogVersion(text);
+   if (version === null) {
+      throw new Error(
+         'database version 0 requires migration to version ' +
+            String(DB_FORMAT_VERSION) +
+            ' -- run `kbdb --migrate` before starting the daemon',
+      );
+   }
+
+   if (version > DB_FORMAT_VERSION) {
+      throw new Error(
+         'database version ' +
+            String(version) +
+            ' is newer than kbdb version ' +
+            String(DB_FORMAT_VERSION) +
+            ' -- upgrade kbdb to open this database',
+      );
+   }
+
+   if (version < DB_FORMAT_VERSION) {
+      throw new Error(
+         'database version ' +
+            String(version) +
+            ' requires migration to version ' +
+            String(DB_FORMAT_VERSION) +
+            ' -- run `kbdb --migrate` before starting the daemon',
+      );
+   }
+}

package/src/shared/worker-daemon/functions/check-preconditions.function.ts

@@ -0,0 +1,28 @@
+import { existsSync, accessSync, constants } from 'node:fs';
+import { checkCatalogVersion } from './check-catalog-version.function.ts';
+
+/**
+ * Validates startup preconditions for the worker daemon.
+ *
+ * Checks that `targetDir` exists, is readable and writable, and
+ * that `catalog.toml` reports a compatible format version.
+ *
+ * @param targetDir - absolute path to the `.kbdb` directory
+ * @throws {Error} when any precondition fails
+ */
+export function checkPreconditions(targetDir: string): void {
+   if (!existsSync(targetDir)) {
+      throw new Error(`database directory not found: ${targetDir}`);
+   }
+
+   try {
+      accessSync(targetDir, constants.R_OK | constants.W_OK);
+   } catch {
+      throw new Error(
+         `cannot access database directory: ${targetDir}` +
+            ' -- permission denied',
+      );
+   }
+
+   checkCatalogVersion(targetDir);
+}