@dikolab/kbdb 0.2.1 → 0.3.0

package/src/shared/worker-daemon/classes/worker-daemon.class.ts

@@ -1,6 +1,7 @@
 import {
    existsSync,
    readFileSync,
+   readdirSync,
    accessSync,
    writeFileSync,
    unlinkSync,
@@ -38,6 +39,7 @@ import {
    decodeKeyValueText,
    decodeRemoveFlag,
    decodeRecallResult,
+   decodeAddSectionResult,
 } from '../../wasm-codec/index.ts';
 
 import type { AddSectionInput } from '../../wasm-codec/index.ts';
@@ -89,6 +91,8 @@ const WASM_METHOD_MAP: Readonly<Partial<Record<string, string>>> = {
    listDocumentsByType: 'worker_list_docs_by_type',
    list_docs_by_type: 'worker_list_docs_by_type',
    recall: 'worker_recall',
+   export: 'worker_export',
+   exportDb: 'worker_export',
 };
 
 /** Config method name prefix. */
@@ -155,6 +159,18 @@ export class WorkerDaemon {
    private _server: Server | null = null;
    private readonly _wasmDir: string;
 
+   /** Queue of waiters for the single-writer lock. */
+   private _writeQueue: { resolve: () => void }[] = [];
+
+   /** Whether the write lock is currently held. */
+   private _writeLocked = false;
+
+   /** Whether the startup integrity check passed. */
+   private _integrityOk = true;
+
+   /** Integrity errors found at startup, if any. */
+   private _integrityErrors: string[] = [];
+
    /**
     * @param ctx - context ID (16-hex characters), the first 16 of
     *   the SHA-256 hash of `targetDir`
@@ -229,6 +245,7 @@ export class WorkerDaemon {
       this._wasmModules = wasmModules;
 
       this.initWorkerContext();
+      this.runStartupIntegrityCheck();
 
       const pidPath = join(tmpdir(), `kbdb-${this.ctx}.pid`);
       writeFileSync(pidPath, process.pid.toString());
@@ -470,7 +487,11 @@ export class WorkerDaemon {
       params: Record<string, unknown>,
    ): unknown {
       if (method === 'status') {
-         return this._connectionCount > 0 ? 'busy' : 'idle';
+         return {
+            status: this._connectionCount > 0 ? 'busy' : 'idle',
+            integrityOk: this._integrityOk,
+            integrityErrors: this._integrityErrors,
+         };
       }
 
       if (method === 'idle_at') {
@@ -493,71 +514,36 @@ export class WorkerDaemon {
       throw new IpcError(-32601, `Method not found: ${method}`);
    }
 
-   private dispatchWasm(
+   private async dispatchWasm(
       method: string,
       wasmExport: string,
       params: Record<string, unknown>,
-   ): unknown {
+   ): Promise<unknown> {
+      // Read-only operations — no write lock required.
       if (method === 'search' || method === 'query') {
          return this.handleSearch(params);
       }
-      if (method === 'addSection' || method === 'add_section') {
-         return this.handleAddSection(params);
-      }
-      if (method === 'updateSection' || method === 'update_section') {
-         return this.handleUpdateSection(params);
-      }
-      if (method === 'removeSection' || method === 'remove_section') {
-         return this.handleRemoveSection(params);
-      }
       if (method === 'readSections' || method === 'read_sections') {
          return this.handleReadSections(params);
       }
       if (method === 'retrieveDocument' || method === 'retrieve_doc') {
          return this.handleRetrieveDoc(params);
       }
-      if (method === 'groupSections' || method === 'group_sections') {
-         return this.handleGroupSections(params);
-      }
-      if (method === 'removeGrouping' || method === 'remove_grouping') {
-         return this.handleRemoveGrouping(params);
-      }
       if (method === 'content' || method === 'compose') {
          return this.handleCompose(params);
       }
       if (method === 'integrityCheck' || method === 'integrity_check') {
          return this.handleIntegrityCheck();
       }
-      if (method === 'gc') {
-         return this.handleGc();
-      }
-      if (method === 'rebuildIndexes' || method === 'rebuild_indexes') {
-         return this.handleRebuild();
-      }
       if (method === 'dbStatus' || method === 'db_status') {
          return this.handleDbStatus();
       }
-      if (method === 'initDirectory' || method === 'init_directory') {
-         return this.handleInitDirectory(params);
-      }
-      if (method === 'invalidate') {
-         return this.handleInvalidate(params);
-      }
-      if (method === 'invalidateAll' || method === 'invalidate_all') {
-         return this.handleInvalidateAll();
-      }
       if (method === 'get_result_status') {
          return this.handleResultStatus(params);
       }
       if (method === 'get_result') {
          return this.handleGetResult(params);
       }
-      if (method === 'migrateDatabase') {
-         throw new IpcError(
-            -32601,
-            'migrateDatabase is not yet supported',
-         );
-      }
       if (method === 'listByType' || method === 'list_by_type') {
          return this.handleListByType(params);
       }
@@ -570,7 +556,74 @@ export class WorkerDaemon {
       if (method === 'recall') {
          return this.handleRecall(params);
       }
-      return this.invokeWasmRaw(wasmExport, new Uint8Array(0));
+      if (method === 'export' || method === 'exportDb') {
+         return this.handleExport(params);
+      }
+
+      // Write operations — serialised through the single-writer lock.
+      await this.acquireWriteLock();
+      try {
+         if (method === 'invalidate') {
+            return this.handleInvalidate(params);
+         }
+         if (
+            method === 'invalidateAll' ||
+            method === 'invalidate_all'
+         ) {
+            return this.handleInvalidateAll();
+         }
+         if (method === 'addSection' || method === 'add_section') {
+            return this.handleAddSection(params);
+         }
+         if (
+            method === 'updateSection' ||
+            method === 'update_section'
+         ) {
+            return this.handleUpdateSection(params);
+         }
+         if (
+            method === 'removeSection' ||
+            method === 'remove_section'
+         ) {
+            return this.handleRemoveSection(params);
+         }
+         if (
+            method === 'groupSections' ||
+            method === 'group_sections'
+         ) {
+            return this.handleGroupSections(params);
+         }
+         if (
+            method === 'removeGrouping' ||
+            method === 'remove_grouping'
+         ) {
+            return this.handleRemoveGrouping(params);
+         }
+         if (method === 'gc') {
+            return this.handleGc();
+         }
+         if (
+            method === 'rebuildIndexes' ||
+            method === 'rebuild_indexes'
+         ) {
+            return this.handleRebuild();
+         }
+         if (
+            method === 'initDirectory' ||
+            method === 'init_directory'
+         ) {
+            return this.handleInitDirectory(params);
+         }
+         if (method === 'migrateDatabase') {
+            throw new IpcError(
+               -32601,
+               'migrateDatabase is not yet supported',
+            );
+         }
+         return this.invokeWasmRaw(wasmExport, new Uint8Array(0));
+      } finally {
+         this.releaseWriteLock();
+      }
    }
 
    // -- Search/query (token-based async) --
@@ -643,6 +696,9 @@ export class WorkerDaemon {
             : 'text';
       const typeName =
          sectionType !== semanticType ? sectionType : null;
+      const tags = Array.isArray(params['tags'])
+         ? (params['tags'] as string[])
+         : [];
       const input: AddSectionInput = {
          sectionType: semanticType,
          mimeType: params['mimeType'] as string | undefined,
@@ -651,11 +707,32 @@ export class WorkerDaemon {
          description: (params['description'] as string | null) ?? null,
          docid: (params['docid'] as string | null) ?? null,
          typeName,
+         sourcePath:
+            typeof params['sourcePath'] === 'string'
+               ? params['sourcePath']
+               : undefined,
+         sourceMtime:
+            typeof params['sourceMtime'] === 'string'
+               ? params['sourceMtime']
+               : undefined,
+         tags,
+         replace: params['replace'] === true,
       };
       const encoded = encodeAddSectionParams(input);
       const bytes = this.callWasmSync('worker_add_section', encoded);
-      const kbid = new TextDecoder().decode(bytes);
-      return { kbid };
+      const decoded = decodeAddSectionResult(bytes);
+      const nearDups =
+         decoded.nearDuplicates.length > 0
+            ? decoded.nearDuplicates
+            : this.scanNearDuplicates(input.content, decoded.kbid);
+      return {
+         kbid: decoded.kbid,
+         indexed_terms_count: decoded.indexedTermsCount,
+         elapsed_ms: decoded.elapsedMs,
+         superseded: decoded.superseded,
+         warnings: decoded.warnings,
+         near_duplicates: nearDups,
+      };
    }
 
    private handleUpdateSection(
@@ -759,7 +836,18 @@ export class WorkerDaemon {
    private handleInvalidate(params: Record<string, unknown>): unknown {
       const kbid = (params['kbid'] as string | undefined) ?? '';
       const encoded = new TextEncoder().encode(kbid);
-      this.callWasmSync('worker_invalidate', encoded);
+      const fn = this.getExport('worker_invalidate');
+      const ptr = this.wasmAlloc(encoded.length);
+      this.wasmWrite(ptr, encoded);
+      const rc = fn(ptr, encoded.length) as number;
+      this.wasmFree(ptr, encoded.length);
+      if (rc === -1) {
+         const errBytes = this.readReturnSlot();
+         throw new IpcError(
+            -32000,
+            new TextDecoder().decode(errBytes) || 'invalidate failed',
+         );
+      }
       return { ok: true };
    }
 
@@ -819,15 +907,161 @@ export class WorkerDaemon {
    }
 
    private handleGc(): unknown {
+      const start = Date.now();
       this.callWasmZeroArg('worker_garbage_collect');
       const bytes = this.readReturnSlot();
       const text = new TextDecoder().decode(bytes);
-      return decodeKeyValueText(text);
+      const kv = decodeKeyValueText(text);
+      const removedCount = Number(kv['removed'] ?? 0);
+      const staleSources = Number(kv['stale_sources'] ?? 0);
+      return {
+         removed_sections: removedCount,
+         removed_bytes: 0,
+         elapsed_ms: Date.now() - start,
+         stale_sources: staleSources,
+      };
    }
 
    private handleRebuild(): unknown {
-      this.callWasmZeroArg('worker_rebuild_indexes');
-      return { ok: true };
+      const fn = this.getExport('worker_rebuild_indexes');
+      const rc = fn() as number;
+      if (rc === -1) {
+         const errBytes = this.readReturnSlot();
+         const errMsg = new TextDecoder().decode(errBytes);
+         throw new IpcError(-32000, errMsg || 'rebuild failed');
+      }
+      const resultBytes = this.readReturnSlot();
+      const reindexed =
+         resultBytes.length >= 4
+            ? new DataView(
+                 resultBytes.buffer,
+                 resultBytes.byteOffset,
+                 resultBytes.byteLength,
+              ).getUint32(0, true)
+            : 0;
+      return { ok: true, sections_reindexed: reindexed };
+   }
+
+   // -- Export (read operation, no lock required) --
+
+   private handleExport(params: Record<string, unknown>): unknown {
+      const targetPath =
+         (params['targetPath'] as string | undefined) ??
+         (params['path'] as string | undefined) ??
+         './kbdb-export/';
+      const encoded = new TextEncoder().encode(targetPath);
+      const start = Date.now();
+      const bytes = this.callWasmSync('worker_export', encoded);
+      const elapsed = Date.now() - start;
+      const text = new TextDecoder().decode(bytes);
+      const kv = decodeKeyValueText(text);
+      return {
+         path: kv['path'] ?? targetPath,
+         sections_exported: Number(kv['sections_exported'] ?? 0),
+         documents_exported: Number(kv['documents_exported'] ?? 0),
+         elapsed_ms: elapsed,
+      };
+   }
+
+   // -- Near-duplicate detection --
+
+   private scanNearDuplicates(
+      content: string,
+      newKbid: string,
+   ): { kbid: string; similarity: number }[] {
+      const newTerms = this.extractTermSet(content);
+      if (newTerms.size === 0) return [];
+
+      const sectionsDir = join(this.targetDir, 'sections');
+      let shards: string[];
+      try {
+         shards = readdirSync(sectionsDir);
+      } catch {
+         return [];
+      }
+
+      const results: { kbid: string; similarity: number }[] = [];
+      for (const shard of shards) {
+         let files: string[];
+         try {
+            files = readdirSync(join(sectionsDir, shard));
+         } catch {
+            continue;
+         }
+         for (const file of files) {
+            if (!file.endsWith('.section')) continue;
+            const kbid = file.slice(0, -8);
+            if (kbid === newKbid) continue;
+            try {
+               const data = readFileSync(
+                  join(sectionsDir, shard, file),
+                  'utf-8',
+               );
+               const sep = data.indexOf('\n---\n');
+               const body = sep >= 0 ? data.slice(sep + 5) : data;
+               const existing = this.extractTermSet(body);
+               let intersection = 0;
+               for (const t of newTerms) {
+                  if (existing.has(t)) intersection++;
+               }
+               const union =
+                  newTerms.size + existing.size - intersection;
+               const sim = union > 0 ? intersection / union : 0;
+               if (sim >= 0.85) {
+                  results.push({
+                     kbid,
+                     similarity: Math.round(sim * 100) / 100,
+                  });
+               }
+            } catch {
+               continue;
+            }
+         }
+      }
+      return results;
+   }
+
+   private extractTermSet(text: string): Set<string> {
+      return new Set(
+         text
+            .toLowerCase()
+            .split(/\W+/)
+            .filter((w) => w.length > 2),
+      );
+   }
+
+   // -- Single-writer lock --
+
+   /**
+    * Acquires the write lock.
+    *
+    * If the lock is free it is granted immediately. If it is held the
+    * caller is queued and its promise resolves once the lock is
+    * released by the current holder.
+    */
+   private acquireWriteLock(): Promise<void> {
+      if (!this._writeLocked) {
+         this._writeLocked = true;
+         return Promise.resolve();
+      }
+      return new Promise<void>((resolve) => {
+         this._writeQueue.push({ resolve });
+      });
+   }
+
+   /**
+    * Releases the write lock.
+    *
+    * If waiters are queued the next one is granted the lock
+    * immediately. Otherwise the lock is marked free.
+    */
+   private releaseWriteLock(): void {
+      const next = this._writeQueue.shift();
+      if (next !== undefined) {
+         next.resolve();
+      } else {
+         this._writeLocked = false;
+      }
    }
 
    // -- WASM lifecycle --
@@ -837,6 +1071,41 @@ export class WorkerDaemon {
       this.callWasmSync('worker_init', pathBytes);
    }
 
+   /**
+    * Runs a database integrity check immediately after context
+    * initialisation.
+    *
+    * Stores the result in `_integrityOk` and `_integrityErrors`.
+    * Logs a warning to stderr when issues are found so operators
+    * are alerted without aborting the daemon.
+    */
+   private runStartupIntegrityCheck(): void {
+      try {
+         this.callWasmZeroArg('worker_check_integrity');
+         const bytes = this.readReturnSlot();
+         const text = new TextDecoder().decode(bytes);
+         const kv = decodeKeyValueText(text);
+         this._integrityOk = kv['ok'] === 'true';
+         if (!this._integrityOk) {
+            this._integrityErrors = Object.entries(kv)
+               .filter(([k]) => k !== 'ok')
+               .map(([k, v]) => `${k}=${v}`);
+            // eslint-disable-next-line no-console
+            console.error(
+               '[kbdb-worker] startup integrity check failed:',
+               this._integrityErrors.join(', '),
+            );
+         }
+      } catch {
+         this._integrityOk = false;
+         this._integrityErrors = ['integrity_check_failed'];
+         // eslint-disable-next-line no-console
+         console.error(
+            '[kbdb-worker] startup integrity check threw an error',
+         );
+      }
+   }
+
    // -- WASM bridge primitives --
 
    private getExport(name: string): WasmFn {

package/src/shared/worker-daemon/functions/create-host-imports.function.ts

@@ -136,8 +136,9 @@ export function createHostImports(
 
    function readPathArg(ptr: number, len: number): string {
       const bytes = new Uint8Array(getBuffer(), ptr, len);
-      const relative = new TextDecoder().decode(bytes);
-      return join(contextPath, relative);
+      const decoded = new TextDecoder().decode(bytes);
+      if (decoded.startsWith('/')) return decoded;
+      return join(contextPath, decoded);
    }
 
    return {
@@ -163,14 +164,21 @@ export function createHostImports(
             const filePath = readPathArg(pathPtr, pathLen);
             const data = fs.readFileSync(filePath);
             const dataLen = data.byteLength;
-            const dv = new DataView(getBuffer());
             if (dataLen === 0) {
+               // No allocation -- buffer cannot have grown; safe to
+               // create DataView now.
+               const dv = new DataView(getBuffer());
                dv.setUint32(outPtr, 0, true);
                dv.setUint32(outLen, 0, true);
                return 0;
             }
             const dataPtr = allocator.alloc(dataLen);
             if (dataPtr === 0) return -1;
+            // Create DataView AFTER alloc() -- alloc may trigger WASM
+            // memory growth, which detaches the old ArrayBuffer and
+            // replaces it with a new one. A DataView captured before
+            // growth would reference the detached buffer and throw.
+            const dv = new DataView(getBuffer());
             const memView = new Uint8Array(getBuffer());
             memView.set(
                new Uint8Array(
@@ -267,14 +275,21 @@ export function createHostImports(
             const entries = fs.readdirSync(dirPath);
             const text = entries.join('\n');
             const encoded = new TextEncoder().encode(text);
-            const dv = new DataView(getBuffer());
             if (encoded.length === 0) {
+               // No allocation -- buffer cannot have grown; safe to
+               // create DataView now.
+               const dv = new DataView(getBuffer());
                dv.setUint32(outPtr, 0, true);
                dv.setUint32(outLen, 0, true);
                return 0;
             }
             const dataPtr = allocator.alloc(encoded.length);
             if (dataPtr === 0) return -1;
+            // Create DataView AFTER alloc() -- alloc may trigger WASM
+            // memory growth, which detaches the old ArrayBuffer and
+            // replaces it with a new one. A DataView captured before
+            // growth would reference the detached buffer and throw.
+            const dv = new DataView(getBuffer());
             new Uint8Array(getBuffer(), dataPtr, encoded.length).set(
                encoded,
             );

package/src/shared/worker-daemon/functions/create-ipc-server.function.ts

@@ -28,6 +28,8 @@ export function createIpcServer(
    const server = createServer((socket) => {
       let buffer = '';
 
+      let processingQueue = Promise.resolve();
+
       socket.on('data', (chunk: Buffer) => {
          buffer += chunk.toString();
          const lines = buffer.split('\n');
@@ -35,7 +37,9 @@ export function createIpcServer(
 
          for (const line of lines) {
             if (!line.trim()) continue;
-            void handleLine(line, handler, socket);
+            processingQueue = processingQueue.then(() =>
+               handleLine(line, handler, socket),
+            );
          }
       });
 

package/src/wasm/fs-database/kbdb_fs_database.d.ts

@@ -5,7 +5,10 @@ declare namespace wasm_bindgen {
     /**
      * Adds a section. `ptr`/`len` point to an encoded `AddSectionParams`.
      *
-     * On success: writes encoded kbid string to return slot, returns 0.
+     * On success: writes an encoded `AddSectionResult` to the return slot and
+     * returns 0.  When `params.replace` is true and a section with the same
+     * `source_key` exists, the superseded section's index entries are removed
+     * and the returned `AddSectionResult.superseded` is set to its kbid.
      * On error: writes error message to return slot, returns -1.
      */
     export function db_add_section(ptr: number, len: number): number;
@@ -42,6 +45,23 @@ declare namespace wasm_bindgen {
      */
     export function db_check_version(ptr: number, len: number): number;
 
+    /**
+     * Exports the database to a target directory.
+     *
+     * `ptr`/`len` point to a UTF-8 string containing the target path.
+     *
+     * Copies `sections/`, `documents/`, and `catalog.toml` to the target.
+     * Indexes are not copied because they can be rebuilt from the data.
+     *
+     * The target path may be absolute or relative. It is an external output
+     * destination, not a database-internal path, so no path validation is
+     * applied to it.
+     *
+     * On success a summary string is placed in the return slot:
+     * `path={target}\nsections_exported={N}\ndocuments_exported={M}`
+     */
+    export function db_export(ptr: number, len: number): number;
+
     /**
      * Runs garbage collection. Returns 0 on success.
      * On success a summary string is placed in the return slot.
@@ -200,6 +220,7 @@ declare interface InitOutput {
     readonly db_add_to_index: (a: number, b: number) => number;
     readonly db_check_integrity: (a: number, b: number) => number;
     readonly db_check_version: (a: number, b: number) => number;
+    readonly db_export: (a: number, b: number) => number;
     readonly db_gc: (a: number, b: number) => number;
     readonly db_get_back_refs: (a: number, b: number) => number;
     readonly db_get_forward_refs: (a: number, b: number) => number;

package/src/wasm/fs-database/kbdb_fs_database.js

@@ -7,7 +7,10 @@ let wasm_bindgen = (function(exports) {
     /**
      * Adds a section. `ptr`/`len` point to an encoded `AddSectionParams`.
      *
-     * On success: writes encoded kbid string to return slot, returns 0.
+     * On success: writes an encoded `AddSectionResult` to the return slot and
+     * returns 0.  When `params.replace` is true and a section with the same
+     * `source_key` exists, the superseded section's index entries are removed
+     * and the returned `AddSectionResult.superseded` is set to its kbid.
      * On error: writes error message to return slot, returns -1.
      * @param {number} ptr
      * @param {number} len
@@ -72,6 +75,30 @@ let wasm_bindgen = (function(exports) {
     }
     exports.db_check_version = db_check_version;
 
+    /**
+     * Exports the database to a target directory.
+     *
+     * `ptr`/`len` point to a UTF-8 string containing the target path.
+     *
+     * Copies `sections/`, `documents/`, and `catalog.toml` to the target.
+     * Indexes are not copied because they can be rebuilt from the data.
+     *
+     * The target path may be absolute or relative. It is an external output
+     * destination, not a database-internal path, so no path validation is
+     * applied to it.
+     *
+     * On success a summary string is placed in the return slot:
+     * `path={target}\nsections_exported={N}\ndocuments_exported={M}`
+     * @param {number} ptr
+     * @param {number} len
+     * @returns {number}
+     */
+    function db_export(ptr, len) {
+        const ret = wasm.db_export(ptr, len);
+        return ret;
+    }
+    exports.db_export = db_export;
+
     /**
      * Runs garbage collection. Returns 0 on success.
      * On success a summary string is placed in the return slot.

package/src/wasm/fs-database/kbdb_fs_database_bg.wasm.d.ts

@@ -7,6 +7,7 @@ export const db_add_section: (a: number, b: number) => number;
 export const db_add_to_index: (a: number, b: number) => number;
 export const db_check_integrity: (a: number, b: number) => number;
 export const db_check_version: (a: number, b: number) => number;
+export const db_export: (a: number, b: number) => number;
 export const db_gc: (a: number, b: number) => number;
 export const db_get_back_refs: (a: number, b: number) => number;
 export const db_get_forward_refs: (a: number, b: number) => number;