@dikolab/kbdb 0.2.2 → 0.3.1

package/src/shared/worker-daemon/functions/handle-write-ops.function.ts

@@ -0,0 +1,257 @@
+import {
+   encodeAddSectionParams,
+   encodeUpdateSectionParams,
+   encodeGroupParams,
+   decodeKeyValueText,
+   decodeRemoveFlag,
+   decodeAddSectionResult,
+} from '../../wasm-codec/index.ts';
+import type { AddSectionInput } from '../../wasm-codec/index.ts';
+import type { HandlerContext } from '../typings/handler-context.interface.ts';
+import { IpcError } from '../classes/ipc-error.class.ts';
+import {
+   getWasmExport,
+   wasmAlloc,
+   wasmWrite,
+   wasmFree,
+   readReturnSlot,
+   callWasmSync,
+   callWasmZeroArg,
+} from './wasm-bridge.function.ts';
+import { scanNearDuplicates } from './scan-near-duplicates.function.ts';
+
+/**
+ * Handles the `addSection` IPC method.
+ *
+ * Encodes the section input, delegates to `worker_add_section`, and
+ * runs a near-duplicate scan when the WASM result reports none.
+ */
+export function handleAddSection(
+   ctx: HandlerContext,
+   params: Record<string, unknown>,
+): unknown {
+   const sectionType = (params['sectionType'] ?? 'text') as string;
+   const semanticType =
+      sectionType === 'text' ||
+      sectionType === 'code' ||
+      sectionType === 'image'
+         ? sectionType
+         : 'text';
+   const typeName = sectionType !== semanticType ? sectionType : null;
+   const tags = Array.isArray(params['tags'])
+      ? (params['tags'] as string[])
+      : [];
+   const content = (params['content'] as string | undefined) ?? '';
+   const input: AddSectionInput = {
+      sectionType: semanticType,
+      mimeType: params['mimeType'] as string | undefined,
+      content,
+      title: (params['title'] as string | null) ?? null,
+      description: (params['description'] as string | null) ?? null,
+      docid: (params['docid'] as string | null) ?? null,
+      typeName,
+      sourcePath:
+         typeof params['sourcePath'] === 'string'
+            ? params['sourcePath']
+            : undefined,
+      sourceMtime:
+         typeof params['sourceMtime'] === 'string'
+            ? params['sourceMtime']
+            : undefined,
+      tags,
+      replace: params['replace'] === true,
+   };
+   const encoded = encodeAddSectionParams(input);
+   const bytes = callWasmSync(
+      ctx.modules,
+      'worker_add_section',
+      encoded,
+   );
+   const decoded = decodeAddSectionResult(bytes);
+   const nearDups =
+      decoded.nearDuplicates.length > 0
+         ? decoded.nearDuplicates
+         : scanNearDuplicates(ctx.targetDir, content, decoded.kbid);
+   return {
+      kbid: decoded.kbid,
+      indexed_terms_count: decoded.indexedTermsCount,
+      elapsed_ms: decoded.elapsedMs,
+      superseded: decoded.superseded,
+      warnings: decoded.warnings,
+      near_duplicates: nearDups,
+   };
+}
+
+/**
+ * Handles the `updateSection` IPC method.
+ */
+export function handleUpdateSection(
+   ctx: HandlerContext,
+   params: Record<string, unknown>,
+): unknown {
+   const oldKbid = (params['kbid'] as string | undefined) ?? '';
+   const updSectionType = (params['sectionType'] ?? 'text') as string;
+   const updSemanticType =
+      updSectionType === 'text' ||
+      updSectionType === 'code' ||
+      updSectionType === 'image'
+         ? updSectionType
+         : 'text';
+   const updTypeName =
+      updSectionType !== updSemanticType ? updSectionType : null;
+   const input: AddSectionInput = {
+      sectionType: updSemanticType,
+      mimeType: params['mimeType'] as string | undefined,
+      content: (params['content'] as string | undefined) ?? '',
+      title: (params['title'] as string | null) ?? null,
+      description: (params['description'] as string | null) ?? null,
+      docid: (params['docid'] as string | null) ?? null,
+      typeName: updTypeName,
+   };
+   const encoded = encodeUpdateSectionParams(oldKbid, input);
+   const bytes = callWasmSync(
+      ctx.modules,
+      'worker_update_section',
+      encoded,
+   );
+   return { kbid: new TextDecoder().decode(bytes) };
+}
+
+/**
+ * Handles the `removeSection` IPC method.
+ */
+export function handleRemoveSection(
+   ctx: HandlerContext,
+   params: Record<string, unknown>,
+): unknown {
+   const kbid = (params['kbid'] as string | undefined) ?? '';
+   const encoded = new TextEncoder().encode(kbid);
+   const bytes = callWasmSync(
+      ctx.modules,
+      'worker_remove_section',
+      encoded,
+   );
+   return { removed: decodeRemoveFlag(bytes) };
+}
+
+/**
+ * Handles the `groupSections` IPC method.
+ */
+export function handleGroupSections(
+   ctx: HandlerContext,
+   params: Record<string, unknown>,
+): unknown {
+   const title = (params['title'] as string | undefined) ?? '';
+   const kbids = (params['kbids'] ?? []) as string[];
+   const docType = (params['docType'] as string | undefined) ?? '';
+   const encoded = encodeGroupParams(title, kbids, docType);
+   const bytes = callWasmSync(
+      ctx.modules,
+      'worker_group_sections',
+      encoded,
+   );
+   return { docid: new TextDecoder().decode(bytes) };
+}
+
+/**
+ * Handles the `removeGrouping` IPC method.
+ */
+export function handleRemoveGrouping(
+   ctx: HandlerContext,
+   params: Record<string, unknown>,
+): unknown {
+   const docid = (params['docid'] as string | undefined) ?? '';
+   callWasmSync(
+      ctx.modules,
+      'worker_remove_grouping',
+      new TextEncoder().encode(docid),
+   );
+   return { ok: true };
+}
+
+/**
+ * Handles the `initDirectory` IPC method.
+ */
+export function handleInitDirectory(
+   ctx: HandlerContext,
+   params: Record<string, unknown>,
+): unknown {
+   const path = (params['path'] as string | undefined) ?? ctx.targetDir;
+   callWasmSync(
+      ctx.modules,
+      'worker_init_directory',
+      new TextEncoder().encode(path),
+   );
+   return { ok: true };
+}
+
+/**
+ * Handles the `invalidate` IPC method.
+ */
+export function handleInvalidate(
+   ctx: HandlerContext,
+   params: Record<string, unknown>,
+): unknown {
+   const kbid = (params['kbid'] as string | undefined) ?? '';
+   const encoded = new TextEncoder().encode(kbid);
+   const fn = getWasmExport(ctx.modules, 'worker_invalidate');
+   const ptr = wasmAlloc(ctx.modules, encoded.length);
+   wasmWrite(ctx.modules, ptr, encoded);
+   const rc = fn(ptr, encoded.length) as number;
+   wasmFree(ctx.modules, ptr, encoded.length);
+   if (rc === -1) {
+      const errBytes = readReturnSlot(ctx.modules);
+      throw new IpcError(
+         -32000,
+         new TextDecoder().decode(errBytes) || 'invalidate failed',
+      );
+   }
+   return { ok: true };
+}
+
+/**
+ * Handles the `invalidateAll` IPC method.
+ */
+export function handleInvalidateAll(ctx: HandlerContext): unknown {
+   callWasmZeroArg(ctx.modules, 'worker_invalidate_all');
+   return { ok: true };
+}
+
+/**
+ * Handles the `gc` IPC method.
+ */
+export function handleGc(ctx: HandlerContext): unknown {
+   const start = Date.now();
+   callWasmZeroArg(ctx.modules, 'worker_garbage_collect');
+   const bytes = readReturnSlot(ctx.modules);
+   const kv = decodeKeyValueText(new TextDecoder().decode(bytes));
+   return {
+      removed_sections: Number(kv['removed'] ?? 0),
+      removed_bytes: 0,
+      elapsed_ms: Date.now() - start,
+      stale_sources: Number(kv['stale_sources'] ?? 0),
+   };
+}
+
+/**
+ * Handles the `rebuildIndexes` IPC method.
+ */
+export function handleRebuild(ctx: HandlerContext): unknown {
+   const fn = getWasmExport(ctx.modules, 'worker_rebuild_indexes');
+   const rc = fn() as number;
+   if (rc === -1) {
+      const errBytes = readReturnSlot(ctx.modules);
+      const errMsg = new TextDecoder().decode(errBytes);
+      throw new IpcError(-32000, errMsg || 'rebuild failed');
+   }
+   const resultBytes = readReturnSlot(ctx.modules);
+   const reindexed =
+      resultBytes.length >= 4
+         ? new DataView(
+              resultBytes.buffer,
+              resultBytes.byteOffset,
+              resultBytes.byteLength,
+           ).getUint32(0, true)
+         : 0;
+   return { ok: true, sections_reindexed: reindexed };
+}

package/src/shared/worker-daemon/functions/parse-auto-capture-section.function.ts

@@ -0,0 +1,51 @@
+import type { AutoCaptureConfig } from '../../auto-capture/typings/auto-capture-config.interface.ts';
+import { AUTO_CAPTURE_DEFAULTS } from '../../auto-capture/constants/auto-capture-defaults.constant.ts';
+
+/**
+ * Parses key-value pairs from an `[auto_capture]` TOML section
+ * into an `AutoCaptureConfig`.
+ *
+ * Recognised keys: `enabled`, `trigger_after_tool_calls`,
+ * `min_interval_ms`, `max_captures_per_session`, `dry_run`.
+ * Unknown keys are silently ignored. Falls back to
+ * `AUTO_CAPTURE_DEFAULTS` for any absent key.
+ *
+ * @param entries - key-value pairs from inside the
+ *   `[auto_capture]` section
+ */
+export function parseAutoCaptureSection(
+   entries: ReadonlyMap<string, string>,
+): AutoCaptureConfig {
+   const enabled = entries.has('enabled')
+      ? entries.get('enabled') === 'true'
+      : AUTO_CAPTURE_DEFAULTS.enabled;
+
+   const dryRun = entries.has('dry_run')
+      ? entries.get('dry_run') === 'true'
+      : AUTO_CAPTURE_DEFAULTS.dryRun;
+
+   const triggerRaw = entries.get('trigger_after_tool_calls');
+   const triggerAfterToolCalls = triggerRaw
+      ? parseInt(triggerRaw, 10) ||
+        AUTO_CAPTURE_DEFAULTS.triggerAfterToolCalls
+      : AUTO_CAPTURE_DEFAULTS.triggerAfterToolCalls;
+
+   const intervalRaw = entries.get('min_interval_ms');
+   const minIntervalMs = intervalRaw
+      ? parseInt(intervalRaw, 10) || AUTO_CAPTURE_DEFAULTS.minIntervalMs
+      : AUTO_CAPTURE_DEFAULTS.minIntervalMs;
+
+   const capRaw = entries.get('max_captures_per_session');
+   const maxCapturesPerSession = capRaw
+      ? parseInt(capRaw, 10) ||
+        AUTO_CAPTURE_DEFAULTS.maxCapturesPerSession
+      : AUTO_CAPTURE_DEFAULTS.maxCapturesPerSession;
+
+   return {
+      enabled,
+      triggerAfterToolCalls,
+      minIntervalMs,
+      maxCapturesPerSession,
+      dryRun,
+   };
+}

package/src/shared/worker-daemon/functions/parse-catalog-version.function.ts

@@ -0,0 +1,20 @@
+/**
+ * Parses the `version` field from a minimal TOML string.
+ *
+ * Supports only `version = <integer>` syntax. Comment lines are
+ * skipped. Returns `null` when no `version` field is present.
+ *
+ * @param text - raw TOML text from `catalog.toml`
+ * @returns the integer version, or `null` if absent
+ */
+export function parseCatalogVersion(text: string): number | null {
+   for (const line of text.split('\n')) {
+      const trimmed = line.trim();
+      if (trimmed.startsWith('#')) continue;
+      const match = /^version\s*=\s*(\d+)/.exec(trimmed);
+      if (match?.[1] !== undefined) {
+         return parseInt(match[1], 10);
+      }
+   }
+   return null;
+}

package/src/shared/worker-daemon/functions/parse-embedding-section.function.ts

@@ -0,0 +1,38 @@
+import type { EmbeddingConfig } from '../../embedding/typings/embedding-config.interface.ts';
+
+/**
+ * Parses key-value pairs from a `[embedding]` TOML section into an
+ * `EmbeddingConfig`.
+ *
+ * Recognises: `provider`, `model`, `endpoint`, `dimensions`.
+ * Unknown keys are silently ignored. Returns `undefined` when the
+ * `provider` key is absent from the supplied entries.
+ *
+ * @param entries - key-value pairs from inside the `[embedding]`
+ *   section
+ */
+export function parseEmbeddingSection(
+   entries: ReadonlyMap<string, string>,
+): EmbeddingConfig | undefined {
+   const provider = entries.get('provider') as
+      | EmbeddingConfig['provider']
+      | undefined;
+
+   if (!provider) {
+      return undefined;
+   }
+
+   const model = entries.get('model');
+   const endpoint = entries.get('endpoint');
+   const dimensionsRaw = entries.get('dimensions');
+   const dimensions = dimensionsRaw
+      ? parseInt(dimensionsRaw, 10) || undefined
+      : undefined;
+
+   return {
+      provider,
+      ...(model !== undefined ? { model } : {}),
+      ...(endpoint !== undefined ? { endpoint } : {}),
+      ...(dimensions !== undefined ? { dimensions } : {}),
+   };
+}

package/src/shared/worker-daemon/functions/parse-worker-toml.function.ts

@@ -0,0 +1,104 @@
+import type { DaemonConfig } from '../typings/daemon-config.interface.ts';
+import {
+   DEFAULT_ITEM_LIMIT,
+   DEFAULT_INDEX_LIMIT,
+   DEFAULT_TIMEOUT_MS,
+   DEFAULT_ITEM_TTL_MS,
+} from '../constants/daemon-defaults.constant.ts';
+import { parseEmbeddingSection } from './parse-embedding-section.function.ts';
+import { parseAutoCaptureSection } from './parse-auto-capture-section.function.ts';
+
+interface RootAccum {
+   itemLimit: number;
+   indexLimit: number;
+   timeoutMs: number;
+   itemTtlMs: number;
+}
+
+function applyRootKey(
+   acc: RootAccum,
+   key: string,
+   value: string,
+): void {
+   switch (key) {
+      case 'item_limit':
+         acc.itemLimit = parseInt(value, 10) || DEFAULT_ITEM_LIMIT;
+         break;
+      case 'index_limit':
+         acc.indexLimit = parseInt(value, 10) || DEFAULT_INDEX_LIMIT;
+         break;
+      case 'timeout_ms':
+         acc.timeoutMs = parseInt(value, 10) || DEFAULT_TIMEOUT_MS;
+         break;
+      case 'item_ttl_ms':
+         acc.itemTtlMs = parseInt(value, 10) || DEFAULT_ITEM_TTL_MS;
+         break;
+      default:
+         break;
+   }
+}
+
+/**
+ * Parses a `worker.toml` file body into a partial `DaemonConfig`.
+ *
+ * Handles `[embedding]` sub-sections and root key-value pairs.
+ * Unknown keys are silently ignored.
+ *
+ * @param text - raw content of `worker.toml`
+ * @param contextPath - daemon context directory path
+ */
+export function parseWorkerToml(
+   text: string,
+   contextPath: string,
+): DaemonConfig {
+   let section = '';
+   const embeddingEntries = new Map<string, string>();
+   const autoCaptureEntries = new Map<string, string>();
+   const acc: RootAccum = {
+      itemLimit: DEFAULT_ITEM_LIMIT,
+      indexLimit: DEFAULT_INDEX_LIMIT,
+      timeoutMs: DEFAULT_TIMEOUT_MS,
+      itemTtlMs: DEFAULT_ITEM_TTL_MS,
+   };
+
+   for (const line of text.split('\n')) {
+      const trimmed = line.trim();
+      if (trimmed === '[embedding]') {
+         section = 'embedding';
+         continue;
+      }
+      if (trimmed === '[auto_capture]') {
+         section = 'auto_capture';
+         continue;
+      }
+      if (trimmed.startsWith('[')) {
+         section = '';
+         continue;
+      }
+      if (trimmed.startsWith('#') || !trimmed.includes('=')) continue;
+      const eqIdx = trimmed.indexOf('=');
+      const key = trimmed.slice(0, eqIdx).trim();
+      const value = trimmed.slice(eqIdx + 1).trim();
+      if (section === 'embedding') {
+         embeddingEntries.set(key, value);
+         continue;
+      }
+      if (section === 'auto_capture') {
+         autoCaptureEntries.set(key, value);
+         continue;
+      }
+      applyRootKey(acc, key, value);
+   }
+
+   const embedding = parseEmbeddingSection(embeddingEntries);
+   const autoCapture =
+      autoCaptureEntries.size > 0
+         ? parseAutoCaptureSection(autoCaptureEntries)
+         : undefined;
+   return {
+      contextPath,
+      ...acc,
+      ...(embedding !== undefined ? { embedding } : {}),
+      ...(autoCapture !== undefined ? { autoCapture } : {}),
+   };
+}

package/src/shared/worker-daemon/functions/run-startup-integrity-check.function.ts

@@ -0,0 +1,55 @@
+import type { WasmModules } from '../typings/wasm-modules.interface.ts';
+import {
+   callWasmZeroArg,
+   readReturnSlot,
+} from './wasm-bridge.function.ts';
+import { decodeKeyValueText } from '../../wasm-codec/index.ts';
+
+/** Result of a startup integrity check. */
+export interface StartupIntegrityResult {
+   /** Whether the check passed without errors. */
+   readonly integrityOk: boolean;
+   /** Integrity error descriptions found at startup. */
+   readonly integrityErrors: string[];
+}
+
+/**
+ * Runs `worker_check_integrity` immediately after WASM context
+ * initialisation and returns the result.
+ *
+ * Logs a warning to stderr when issues are found. Never throws --
+ * failures are captured and returned as part of the result.
+ *
+ * @param modules - loaded WASM module instances
+ */
+export function runStartupIntegrityCheck(
+   modules: WasmModules,
+): StartupIntegrityResult {
+   try {
+      callWasmZeroArg(modules, 'worker_check_integrity');
+      const bytes = readReturnSlot(modules);
+      const kv = decodeKeyValueText(new TextDecoder().decode(bytes));
+      const integrityOk = kv['ok'] === 'true';
+      if (!integrityOk) {
+         const integrityErrors = Object.entries(kv)
+            .filter(([k]) => k !== 'ok')
+            .map(([k, v]) => `${k}=${v}`);
+         // eslint-disable-next-line no-console
+         console.error(
+            '[kbdb-worker] startup integrity check failed:',
+            integrityErrors.join(', '),
+         );
+         return { integrityOk: false, integrityErrors };
+      }
+      return { integrityOk: true, integrityErrors: [] };
+   } catch {
+      // eslint-disable-next-line no-console
+      console.error(
+         '[kbdb-worker] startup integrity check threw an error',
+      );
+      return {
+         integrityOk: false,
+         integrityErrors: ['integrity_check_failed'],
+      };
+   }
+}

package/src/shared/worker-daemon/functions/scan-near-duplicates.function.ts

@@ -0,0 +1,92 @@
+import { readdirSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+/** A near-duplicate match found during add-section scanning. */
+export interface NearDuplicateMatch {
+   /** Knowledge-base identifier of the near-duplicate section. */
+   readonly kbid: string;
+   /** Jaccard similarity score, rounded to 2 decimal places (0–1). */
+   readonly similarity: number;
+}
+
+/**
+ * Scans existing section files for near-duplicates of `content`.
+ *
+ * Uses Jaccard similarity on a set of lowercased tokens (3+ chars)
+ * extracted from the section body. Sections with a similarity of
+ * 0.85 or above are included in the result. The `newKbid` section
+ * is excluded from comparison.
+ *
+ * @param targetDir - absolute path to the `.kbdb` directory
+ * @param content - raw text content of the newly added section
+ * @param newKbid - identifier of the just-added section to exclude
+ * @returns near-duplicate matches sorted by insertion order
+ */
+export function scanNearDuplicates(
+   targetDir: string,
+   content: string,
+   newKbid: string,
+): NearDuplicateMatch[] {
+   const newTerms = extractTermSet(content);
+   if (newTerms.size === 0) return [];
+
+   const sectionsDir = join(targetDir, 'sections');
+   let shards: string[];
+   try {
+      shards = readdirSync(sectionsDir);
+   } catch {
+      return [];
+   }
+
+   const results: NearDuplicateMatch[] = [];
+   for (const shard of shards) {
+      let files: string[];
+      try {
+         files = readdirSync(join(sectionsDir, shard));
+      } catch {
+         continue;
+      }
+      for (const file of files) {
+         if (!file.endsWith('.section')) continue;
+         const kbid = file.slice(0, -8);
+         if (kbid === newKbid) continue;
+         try {
+            const data = readFileSync(
+               join(sectionsDir, shard, file),
+               'utf-8',
+            );
+            const sep = data.indexOf('\n---\n');
+            const body = sep >= 0 ? data.slice(sep + 5) : data;
+            const existing = extractTermSet(body);
+            let intersection = 0;
+            for (const t of newTerms) {
+               if (existing.has(t)) intersection++;
+            }
+            const union = newTerms.size + existing.size - intersection;
+            const sim = union > 0 ? intersection / union : 0;
+            if (sim >= 0.85) {
+               results.push({
+                  kbid,
+                  similarity: Math.round(sim * 100) / 100,
+               });
+            }
+         } catch {
+            continue;
+         }
+      }
+   }
+   return results;
+}
+
+/**
+ * Extracts a set of lowercased word tokens with 3 or more
+ * characters from `text`.
+ */
+function extractTermSet(text: string): Set<string> {
+   return new Set(
+      text
+         .toLowerCase()
+         .split(/\W+/)
+         .filter((w) => w.length > 2),
+   );
+}