@dikolab/kbdb 0.2.1 → 0.3.0

package/dist/worker.mjs

@@ -85,6 +85,7 @@ import { tmpdir as tmpdir2 } from "node:os";
 import {
   existsSync as existsSync2,
   readFileSync as readFileSync4,
+  readdirSync as readdirSync2,
   accessSync,
   writeFileSync as writeFileSync2,
   unlinkSync as unlinkSync2,
@@ -135,8 +136,9 @@ function createHostImports(contextPath, memoryRef = nullMemory, fs = nodeFsAdapt
   }
   function readPathArg(ptr, len) {
     const bytes = new Uint8Array(getBuffer(), ptr, len);
-    const relative = new TextDecoder().decode(bytes);
-    return join2(contextPath, relative);
+    const decoded = new TextDecoder().decode(bytes);
+    if (decoded.startsWith("/")) return decoded;
+    return join2(contextPath, decoded);
   }
   return {
     /**
@@ -156,14 +158,15 @@ function createHostImports(contextPath, memoryRef = nullMemory, fs = nodeFsAdapt
         const filePath = readPathArg(pathPtr, pathLen);
         const data = fs.readFileSync(filePath);
         const dataLen = data.byteLength;
-        const dv = new DataView(getBuffer());
         if (dataLen === 0) {
-          dv.setUint32(outPtr, 0, true);
-          dv.setUint32(outLen, 0, true);
+          const dv2 = new DataView(getBuffer());
+          dv2.setUint32(outPtr, 0, true);
+          dv2.setUint32(outLen, 0, true);
           return 0;
         }
         const dataPtr = allocator.alloc(dataLen);
         if (dataPtr === 0) return -1;
+        const dv = new DataView(getBuffer());
         const memView = new Uint8Array(getBuffer());
         memView.set(
           new Uint8Array(
@@ -241,14 +244,15 @@ function createHostImports(contextPath, memoryRef = nullMemory, fs = nodeFsAdapt
         const entries = fs.readdirSync(dirPath);
         const text = entries.join("\n");
         const encoded = new TextEncoder().encode(text);
-        const dv = new DataView(getBuffer());
         if (encoded.length === 0) {
-          dv.setUint32(outPtr, 0, true);
-          dv.setUint32(outLen, 0, true);
+          const dv2 = new DataView(getBuffer());
+          dv2.setUint32(outPtr, 0, true);
+          dv2.setUint32(outLen, 0, true);
           return 0;
         }
         const dataPtr = allocator.alloc(encoded.length);
         if (dataPtr === 0) return -1;
+        const dv = new DataView(getBuffer());
         new Uint8Array(getBuffer(), dataPtr, encoded.length).set(
           encoded
         );
@@ -352,13 +356,16 @@ import { createServer } from "node:net";
 function createIpcServer(socketPath, handler) {
   const server = createServer((socket) => {
     let buffer = "";
+    let processingQueue = Promise.resolve();
     socket.on("data", (chunk) => {
       buffer += chunk.toString();
       const lines = buffer.split("\n");
       buffer = lines.pop() ?? "";
       for (const line of lines) {
         if (!line.trim()) continue;
-        void handleLine(line, handler, socket);
+        processingQueue = processingQueue.then(
+          () => handleLine(line, handler, socket)
+        );
       }
     });
     socket.on("error", (err) => {
@@ -439,13 +446,27 @@ function encodeSectionType(kind, mimeType) {
 
 // src/shared/wasm-codec/functions/encode-add-section-params.function.ts
 function encodeAddSectionParams(params) {
+  const tags = params.tags ?? [];
+  const tagParts = [];
+  for (const tag of tags) {
+    tagParts.push(encodeString(tag));
+  }
+  const tagsCountBytes = encodeU32(tags.length);
+  const replaceByte = new Uint8Array([
+    params.replace === true ? 1 : 0
+  ]);
   const parts = [
     encodeSectionType(params.sectionType, params.mimeType),
     encodeString(params.content),
     encodeOptionString(params.title),
     encodeOptionString(params.description),
     encodeOptionString(params.docid),
-    encodeOptionString(params.typeName)
+    encodeOptionString(params.typeName),
+    encodeOptionString(params.sourcePath),
+    encodeOptionString(params.sourceMtime),
+    tagsCountBytes,
+    ...tagParts,
+    replaceByte
   ];
   const totalLen = parts.reduce((sum, p) => sum + p.length, 0);
   const result = new Uint8Array(totalLen);
@@ -602,6 +623,10 @@ function decodePagedSearchResult(bytes) {
   offset += limit.bytesRead;
   const hasMore = bytes[offset] === 1;
   offset += 1;
+  const relaxed = offset < bytes.length && bytes[offset] === 1;
+  if (offset < bytes.length) {
+    offset += 1;
+  }
   const itemCount = decodeU32(bytes, offset);
   offset += itemCount.bytesRead;
   const items = [];
@@ -622,6 +647,8 @@ function decodePagedSearchResult(bytes) {
     }
     const score = decodeF32(bytes, offset);
     offset += score.bytesRead;
+    const confidence = decodeF32(bytes, offset);
+    offset += confidence.bytesRead;
     const snippet = decodeString(bytes, offset);
     offset += snippet.bytesRead;
     const termCount = decodeU32(bytes, offset);
@@ -632,14 +659,27 @@ function decodePagedSearchResult(bytes) {
       offset += term.bytesRead;
       matched_terms.push(term.value);
     }
+    const fieldCount = decodeU32(bytes, offset);
+    offset += fieldCount.bytesRead;
+    const matched_fields = [];
+    for (let j = 0; j < fieldCount.value; j++) {
+      const field = decodeString(bytes, offset);
+      offset += field.bytesRead;
+      matched_fields.push(field.value);
+    }
+    const createdAt = decodeString(bytes, offset);
+    offset += createdAt.bytesRead;
     items.push({
       kbid: kbid.value,
       heading: heading.value,
       type: sectionType.value,
       docids,
       score: score.value,
+      confidence: confidence.value,
       snippet: snippet.value,
-      matched_terms
+      matched_terms,
+      matched_fields,
+      created_at: createdAt.value
     });
   }
   return {
@@ -647,7 +687,8 @@ function decodePagedSearchResult(bytes) {
     total: total.value,
     offset: resultOffset.value,
     limit: limit.value,
-    has_more: hasMore
+    has_more: hasMore,
+    relaxed
   };
 }
 
@@ -707,6 +748,47 @@ function decodeSectionRecords(bytes) {
     offset += codeTokenCount.bytesRead;
     const typeName = decodeString(bytes, offset);
     offset += typeName.bytesRead;
+    let sourceKey = null;
+    if (offset < bytes.length) {
+      const v = decodeOptionString2(bytes, offset);
+      offset += v.bytesRead;
+      sourceKey = v.value;
+    }
+    let sourcePath = null;
+    if (offset < bytes.length) {
+      const v = decodeOptionString2(bytes, offset);
+      offset += v.bytesRead;
+      sourcePath = v.value;
+    }
+    let sourceMtime = null;
+    if (offset < bytes.length) {
+      const v = decodeOptionString2(bytes, offset);
+      offset += v.bytesRead;
+      sourceMtime = v.value;
+    }
+    let updatedAt = null;
+    if (offset < bytes.length) {
+      const v = decodeOptionString2(bytes, offset);
+      offset += v.bytesRead;
+      updatedAt = v.value;
+    }
+    let tags = [];
+    if (offset < bytes.length) {
+      const tagCount = decodeU32(bytes, offset);
+      offset += tagCount.bytesRead;
+      tags = [];
+      for (let j = 0; j < tagCount.value; j++) {
+        const tag = decodeString(bytes, offset);
+        offset += tag.bytesRead;
+        tags.push(tag.value);
+      }
+    }
+    let supersedes = null;
+    if (offset < bytes.length) {
+      const v = decodeOptionString2(bytes, offset);
+      offset += v.bytesRead;
+      supersedes = v.value;
+    }
     const record = {
       kbid: kbid.value,
       docid: docid.value,
@@ -721,7 +803,13 @@ function decodeSectionRecords(bytes) {
       headingTokenCount: headingTokenCount.value,
       bodyTokenCount: bodyTokenCount.value,
       codeTokenCount: codeTokenCount.value,
-      typeName: typeName.value
+      typeName: typeName.value,
+      sourceKey,
+      sourcePath,
+      sourceMtime,
+      updatedAt,
+      tags,
+      supersedes
     };
     if (st.mimeType !== void 0) {
       record.mimeType = st.mimeType;
@@ -1041,6 +1129,54 @@ function decodeOneResult(bytes, startOffset) {
   };
 }
 
+// src/shared/wasm-codec/functions/decode-add-section-result.function.ts
+function decodeAddSectionResult(bytes) {
+  let offset = 0;
+  const kbid = decodeString(bytes, offset);
+  offset += kbid.bytesRead;
+  const indexedTermsCount = decodeU32(bytes, offset);
+  offset += indexedTermsCount.bytesRead;
+  const elapsedMsBig = decodeU64(bytes, offset);
+  offset += elapsedMsBig.bytesRead;
+  const supersededTag = bytes[offset];
+  offset += 1;
+  let superseded = null;
+  if (supersededTag === 1) {
+    const supersededStr = decodeString(bytes, offset);
+    offset += supersededStr.bytesRead;
+    superseded = supersededStr.value;
+  }
+  const warningCount = decodeU32(bytes, offset);
+  offset += warningCount.bytesRead;
+  const warnings = [];
+  for (let i = 0; i < warningCount.value; i++) {
+    const warning = decodeString(bytes, offset);
+    offset += warning.bytesRead;
+    warnings.push(warning.value);
+  }
+  const dupCount = decodeU32(bytes, offset);
+  offset += dupCount.bytesRead;
+  const nearDuplicates = [];
+  for (let i = 0; i < dupCount.value; i++) {
+    const dupKbid = decodeString(bytes, offset);
+    offset += dupKbid.bytesRead;
+    const similarity = decodeF32(bytes, offset);
+    offset += similarity.bytesRead;
+    nearDuplicates.push({
+      kbid: dupKbid.value,
+      similarity: similarity.value
+    });
+  }
+  return {
+    kbid: kbid.value,
+    indexedTermsCount: indexedTermsCount.value,
+    elapsedMs: Number(elapsedMsBig.value),
+    superseded,
+    warnings,
+    nearDuplicates
+  };
+}
+
 // src/shared/worker-daemon/classes/worker-daemon.class.ts
 var WASM_METHOD_MAP = {
   search: "worker_search",
@@ -1080,7 +1216,9 @@ var WASM_METHOD_MAP = {
   list_by_type: "worker_list_by_type",
   listDocumentsByType: "worker_list_docs_by_type",
   list_docs_by_type: "worker_list_docs_by_type",
-  recall: "worker_recall"
+  recall: "worker_recall",
+  export: "worker_export",
+  exportDb: "worker_export"
 };
 var CONFIG_PREFIX = "config_";
 var CONFIG_KEY_MAP = {
@@ -1115,6 +1253,14 @@ var WorkerDaemon = class {
   _idleTimer = null;
   _server = null;
   _wasmDir;
+  /** Queue of waiters for the single-writer lock. */
+  _writeQueue = [];
+  /** Whether the write lock is currently held. */
+  _writeLocked = false;
+  /** Whether the startup integrity check passed. */
+  _integrityOk = true;
+  /** Integrity errors found at startup, if any. */
+  _integrityErrors = [];
   /**
    * @param ctx - context ID (16-hex characters), the first 16 of
    *   the SHA-256 hash of `targetDir`
@@ -1176,6 +1322,7 @@ var WorkerDaemon = class {
     );
     this._wasmModules = wasmModules;
     this.initWorkerContext();
+    this.runStartupIntegrityCheck();
     const pidPath = join3(tmpdir(), `kbdb-${this.ctx}.pid`);
     writeFileSync2(pidPath, process.pid.toString());
     const ipcPath = getIpcPath(this.ctx);
@@ -1364,7 +1511,11 @@ var WorkerDaemon = class {
    */
   dispatch(method, params) {
     if (method === "status") {
-      return this._connectionCount > 0 ? "busy" : "idle";
+      return {
+        status: this._connectionCount > 0 ? "busy" : "idle",
+        integrityOk: this._integrityOk,
+        integrityErrors: this._integrityErrors
+      };
     }
     if (method === "idle_at") {
       return this._idleAt;
@@ -1381,67 +1532,31 @@ var WorkerDaemon = class {
     }
     throw new IpcError(-32601, `Method not found: ${method}`);
   }
-  dispatchWasm(method, wasmExport, params) {
+  async dispatchWasm(method, wasmExport, params) {
     if (method === "search" || method === "query") {
       return this.handleSearch(params);
     }
-    if (method === "addSection" || method === "add_section") {
-      return this.handleAddSection(params);
-    }
-    if (method === "updateSection" || method === "update_section") {
-      return this.handleUpdateSection(params);
-    }
-    if (method === "removeSection" || method === "remove_section") {
-      return this.handleRemoveSection(params);
-    }
     if (method === "readSections" || method === "read_sections") {
       return this.handleReadSections(params);
     }
     if (method === "retrieveDocument" || method === "retrieve_doc") {
       return this.handleRetrieveDoc(params);
     }
-    if (method === "groupSections" || method === "group_sections") {
-      return this.handleGroupSections(params);
-    }
-    if (method === "removeGrouping" || method === "remove_grouping") {
-      return this.handleRemoveGrouping(params);
-    }
     if (method === "content" || method === "compose") {
       return this.handleCompose(params);
     }
     if (method === "integrityCheck" || method === "integrity_check") {
       return this.handleIntegrityCheck();
     }
-    if (method === "gc") {
-      return this.handleGc();
-    }
-    if (method === "rebuildIndexes" || method === "rebuild_indexes") {
-      return this.handleRebuild();
-    }
     if (method === "dbStatus" || method === "db_status") {
       return this.handleDbStatus();
     }
-    if (method === "initDirectory" || method === "init_directory") {
-      return this.handleInitDirectory(params);
-    }
-    if (method === "invalidate") {
-      return this.handleInvalidate(params);
-    }
-    if (method === "invalidateAll" || method === "invalidate_all") {
-      return this.handleInvalidateAll();
-    }
     if (method === "get_result_status") {
       return this.handleResultStatus(params);
     }
     if (method === "get_result") {
       return this.handleGetResult(params);
     }
-    if (method === "migrateDatabase") {
-      throw new IpcError(
-        -32601,
-        "migrateDatabase is not yet supported"
-      );
-    }
     if (method === "listByType" || method === "list_by_type") {
       return this.handleListByType(params);
     }
@@ -1451,7 +1566,51 @@ var WorkerDaemon = class {
     if (method === "recall") {
       return this.handleRecall(params);
     }
-    return this.invokeWasmRaw(wasmExport, new Uint8Array(0));
+    if (method === "export" || method === "exportDb") {
+      return this.handleExport(params);
+    }
+    await this.acquireWriteLock();
+    try {
+      if (method === "invalidate") {
+        return this.handleInvalidate(params);
+      }
+      if (method === "invalidateAll" || method === "invalidate_all") {
+        return this.handleInvalidateAll();
+      }
+      if (method === "addSection" || method === "add_section") {
+        return this.handleAddSection(params);
+      }
+      if (method === "updateSection" || method === "update_section") {
+        return this.handleUpdateSection(params);
+      }
+      if (method === "removeSection" || method === "remove_section") {
+        return this.handleRemoveSection(params);
+      }
+      if (method === "groupSections" || method === "group_sections") {
+        return this.handleGroupSections(params);
+      }
+      if (method === "removeGrouping" || method === "remove_grouping") {
+        return this.handleRemoveGrouping(params);
+      }
+      if (method === "gc") {
+        return this.handleGc();
+      }
+      if (method === "rebuildIndexes" || method === "rebuild_indexes") {
+        return this.handleRebuild();
+      }
+      if (method === "initDirectory" || method === "init_directory") {
+        return this.handleInitDirectory(params);
+      }
+      if (method === "migrateDatabase") {
+        throw new IpcError(
+          -32601,
+          "migrateDatabase is not yet supported"
+        );
+      }
+      return this.invokeWasmRaw(wasmExport, new Uint8Array(0));
+    } finally {
+      this.releaseWriteLock();
+    }
   }
   // -- Search/query (token-based async) --
   handleSearch(params) {
@@ -1506,6 +1665,7 @@ var WorkerDaemon = class {
     const sectionType = params["sectionType"] ?? "text";
     const semanticType = sectionType === "text" || sectionType === "code" || sectionType === "image" ? sectionType : "text";
     const typeName = sectionType !== semanticType ? sectionType : null;
+    const tags = Array.isArray(params["tags"]) ? params["tags"] : [];
     const input = {
       sectionType: semanticType,
       mimeType: params["mimeType"],
@@ -1513,12 +1673,24 @@ var WorkerDaemon = class {
       title: params["title"] ?? null,
       description: params["description"] ?? null,
       docid: params["docid"] ?? null,
-      typeName
+      typeName,
+      sourcePath: typeof params["sourcePath"] === "string" ? params["sourcePath"] : void 0,
+      sourceMtime: typeof params["sourceMtime"] === "string" ? params["sourceMtime"] : void 0,
+      tags,
+      replace: params["replace"] === true
     };
     const encoded = encodeAddSectionParams(input);
     const bytes = this.callWasmSync("worker_add_section", encoded);
-    const kbid = new TextDecoder().decode(bytes);
-    return { kbid };
+    const decoded = decodeAddSectionResult(bytes);
+    const nearDups = decoded.nearDuplicates.length > 0 ? decoded.nearDuplicates : this.scanNearDuplicates(input.content, decoded.kbid);
+    return {
+      kbid: decoded.kbid,
+      indexed_terms_count: decoded.indexedTermsCount,
+      elapsed_ms: decoded.elapsedMs,
+      superseded: decoded.superseded,
+      warnings: decoded.warnings,
+      near_duplicates: nearDups
+    };
   }
   handleUpdateSection(params) {
     const oldKbid = params["kbid"] ?? "";
@@ -1592,7 +1764,18 @@ var WorkerDaemon = class {
   handleInvalidate(params) {
     const kbid = params["kbid"] ?? "";
     const encoded = new TextEncoder().encode(kbid);
-    this.callWasmSync("worker_invalidate", encoded);
+    const fn = this.getExport("worker_invalidate");
+    const ptr = this.wasmAlloc(encoded.length);
+    this.wasmWrite(ptr, encoded);
+    const rc = fn(ptr, encoded.length);
+    this.wasmFree(ptr, encoded.length);
+    if (rc === -1) {
+      const errBytes = this.readReturnSlot();
+      throw new IpcError(
+        -32e3,
+        new TextDecoder().decode(errBytes) || "invalidate failed"
+      );
+    }
     return { ok: true };
   }
   handleListByType(params) {
@@ -1636,20 +1819,173 @@ var WorkerDaemon = class {
     return decodeKeyValueText(text);
   }
   handleGc() {
+    const start = Date.now();
     this.callWasmZeroArg("worker_garbage_collect");
     const bytes = this.readReturnSlot();
     const text = new TextDecoder().decode(bytes);
-    return decodeKeyValueText(text);
+    const kv = decodeKeyValueText(text);
+    const removedCount = Number(kv["removed"] ?? 0);
+    const staleSources = Number(kv["stale_sources"] ?? 0);
+    return {
+      removed_sections: removedCount,
+      removed_bytes: 0,
+      elapsed_ms: Date.now() - start,
+      stale_sources: staleSources
+    };
   }
   handleRebuild() {
-    this.callWasmZeroArg("worker_rebuild_indexes");
-    return { ok: true };
+    const fn = this.getExport("worker_rebuild_indexes");
+    const rc = fn();
+    if (rc === -1) {
+      const errBytes = this.readReturnSlot();
+      const errMsg = new TextDecoder().decode(errBytes);
+      throw new IpcError(-32e3, errMsg || "rebuild failed");
+    }
+    const resultBytes = this.readReturnSlot();
+    const reindexed = resultBytes.length >= 4 ? new DataView(
+      resultBytes.buffer,
+      resultBytes.byteOffset,
+      resultBytes.byteLength
+    ).getUint32(0, true) : 0;
+    return { ok: true, sections_reindexed: reindexed };
+  }
+  // -- Export (read operation, no lock required) --
+  handleExport(params) {
+    const targetPath = params["targetPath"] ?? params["path"] ?? "./kbdb-export/";
+    const encoded = new TextEncoder().encode(targetPath);
+    const start = Date.now();
+    const bytes = this.callWasmSync("worker_export", encoded);
+    const elapsed = Date.now() - start;
+    const text = new TextDecoder().decode(bytes);
+    const kv = decodeKeyValueText(text);
+    return {
+      path: kv["path"] ?? targetPath,
+      sections_exported: Number(kv["sections_exported"] ?? 0),
+      documents_exported: Number(kv["documents_exported"] ?? 0),
+      elapsed_ms: elapsed
+    };
+  }
+  // -- Near-duplicate detection --
+  scanNearDuplicates(content, newKbid) {
+    const newTerms = this.extractTermSet(content);
+    if (newTerms.size === 0) return [];
+    const sectionsDir = join3(this.targetDir, "sections");
+    let shards;
+    try {
+      shards = readdirSync2(sectionsDir);
+    } catch {
+      return [];
+    }
+    const results = [];
+    for (const shard of shards) {
+      let files;
+      try {
+        files = readdirSync2(join3(sectionsDir, shard));
+      } catch {
+        continue;
+      }
+      for (const file of files) {
+        if (!file.endsWith(".section")) continue;
+        const kbid = file.slice(0, -8);
+        if (kbid === newKbid) continue;
+        try {
+          const data = readFileSync4(
+            join3(sectionsDir, shard, file),
+            "utf-8"
+          );
+          const sep = data.indexOf("\n---\n");
+          const body = sep >= 0 ? data.slice(sep + 5) : data;
+          const existing = this.extractTermSet(body);
+          let intersection = 0;
+          for (const t of newTerms) {
+            if (existing.has(t)) intersection++;
+          }
+          const union = newTerms.size + existing.size - intersection;
+          const sim = union > 0 ? intersection / union : 0;
+          if (sim >= 0.85) {
+            results.push({
+              kbid,
+              similarity: Math.round(sim * 100) / 100
+            });
+          }
+        } catch {
+          continue;
+        }
+      }
+    }
+    return results;
+  }
+  extractTermSet(text) {
+    return new Set(
+      text.toLowerCase().split(/\W+/).filter((w) => w.length > 2)
+    );
+  }
+  // -- Single-writer lock --
+  /**
+   * Acquires the write lock.
+   *
+   * If the lock is free it is granted immediately. If it is held the
+   * caller is queued and its promise resolves once the lock is
+   * released by the current holder.
+   */
+  acquireWriteLock() {
+    if (!this._writeLocked) {
+      this._writeLocked = true;
+      return Promise.resolve();
+    }
+    return new Promise((resolve2) => {
+      this._writeQueue.push({ resolve: resolve2 });
+    });
+  }
+  /**
+   * Releases the write lock.
+   *
+   * If waiters are queued the next one is granted the lock
+   * immediately. Otherwise the lock is marked free.
+   */
+  releaseWriteLock() {
+    const next = this._writeQueue.shift();
+    if (next !== void 0) {
+      next.resolve();
+    } else {
+      this._writeLocked = false;
+    }
   }
   // -- WASM lifecycle --
   initWorkerContext() {
     const pathBytes = new TextEncoder().encode(".");
     this.callWasmSync("worker_init", pathBytes);
   }
+  /**
+   * Runs a database integrity check immediately after context
+   * initialisation.
+   *
+   * Stores the result in `_integrityOk` and `_integrityErrors`.
+   * Logs a warning to stderr when issues are found so operators
+   * are alerted without aborting the daemon.
+   */
+  runStartupIntegrityCheck() {
+    try {
+      this.callWasmZeroArg("worker_check_integrity");
+      const bytes = this.readReturnSlot();
+      const text = new TextDecoder().decode(bytes);
+      const kv = decodeKeyValueText(text);
+      this._integrityOk = kv["ok"] === "true";
+      if (!this._integrityOk) {
+        this._integrityErrors = Object.entries(kv).filter(([k]) => k !== "ok").map(([k, v]) => `${k}=${v}`);
+        console.error(
+          "[kbdb-worker] startup integrity check failed:",
+          this._integrityErrors.join(", ")
+        );
+      }
+    } catch {
+      this._integrityOk = false;
+      this._integrityErrors = ["integrity_check_failed"];
+      console.error(
+        "[kbdb-worker] startup integrity check threw an error"
+      );
+    }
+  }
   // -- WASM bridge primitives --
   getExport(name) {
     if (this._wasmModules === null) {