@dikolab/kbdb 0.1.0

package/src/shared/wasm-codec/functions/encode-update-section-params.function.ts

@@ -0,0 +1,16 @@
+import { encodeString } from './encode-string.function.ts';
+import { encodeAddSectionParams } from './encode-add-section-params.function.ts';
+
+import type { AddSectionInput } from './encode-add-section-params.function.ts';
+
+export function encodeUpdateSectionParams(
+   oldKbid: string,
+   params: AddSectionInput,
+): Uint8Array {
+   const kbidBytes = encodeString(oldKbid);
+   const paramBytes = encodeAddSectionParams(params);
+   const result = new Uint8Array(kbidBytes.length + paramBytes.length);
+   result.set(kbidBytes, 0);
+   result.set(paramBytes, kbidBytes.length);
+   return result;
+}

package/src/shared/wasm-codec/index.ts

@@ -0,0 +1,36 @@
+export { encodeString } from './functions/encode-string.function.ts';
+export { encodeU32 } from './functions/encode-u32.function.ts';
+export { encodeOptionString } from './functions/encode-option-string.function.ts';
+export { encodeSectionType } from './functions/encode-section-type.function.ts';
+export { encodeAddSectionParams } from './functions/encode-add-section-params.function.ts';
+export { encodeUpdateSectionParams } from './functions/encode-update-section-params.function.ts';
+export { encodeKbidList } from './functions/encode-kbid-list.function.ts';
+export { encodeGroupParams } from './functions/encode-group-params.function.ts';
+export { encodeIdentifierList } from './functions/encode-identifier-list.function.ts';
+export { encodeSearchParams } from './functions/encode-search-params.function.ts';
+export { encodeTypeFilterParam } from './functions/encode-type-filter-param.function.ts';
+export { decodeString } from './functions/decode-string.function.ts';
+export { decodeU32 } from './functions/decode-u32.function.ts';
+export { decodeF32 } from './functions/decode-f32.function.ts';
+export { decodeU64 } from './functions/decode-u64.function.ts';
+export { decodeScoredResults } from './functions/decode-scored-results.function.ts';
+export { decodeSectionRecords } from './functions/decode-section-records.function.ts';
+export { decodeSectionRecordList } from './functions/decode-section-record-list.function.ts';
+export { decodeDocumentManifest } from './functions/decode-document-manifest.function.ts';
+export { decodeDocumentManifestList } from './functions/decode-document-manifest-list.function.ts';
+export { decodeComposeOutput } from './functions/decode-compose-output.function.ts';
+export { decodeKeyValueText } from './functions/decode-key-value-text.function.ts';
+export { decodeRemoveFlag } from './functions/decode-remove-flag.function.ts';
+
+export type { DecodeResult } from './functions/decode-string.function.ts';
+export type { AddSectionInput } from './functions/encode-add-section-params.function.ts';
+export type {
+   ScoredResult,
+   TermMatch,
+} from './functions/decode-scored-results.function.ts';
+export type { SectionRecord } from './functions/decode-section-records.function.ts';
+export type {
+   DocumentManifest,
+   DocumentSection,
+} from './functions/decode-document-manifest.function.ts';
+export type { ComposeOutput } from './functions/decode-compose-output.function.ts';

package/src/shared/worker-client/classes/worker-client.class.ts

@@ -0,0 +1,427 @@
+import { connect } from 'node:net';
+import type { Socket } from 'node:net';
+import type { SearchParams } from '../typings/search-params.model.ts';
+import type { SearchResult } from '../typings/search-result.model.ts';
+import type {
+   AddSectionParams,
+   Section,
+} from '../typings/section.model.ts';
+import type { StatusResult } from '../typings/status-result.model.ts';
+import type { AddSectionResult } from '../typings/add-section-result.model.ts';
+import type { RemoveSectionResult } from '../typings/remove-section-result.model.ts';
+import type { ContentResult } from '../typings/content-result.model.ts';
+import type { IntegrityCheckResult } from '../typings/integrity-check-result.model.ts';
+import type { GcResult } from '../typings/gc-result.model.ts';
+import type { RebuildResult } from '../typings/rebuild-result.model.ts';
+import type { MigrateResult } from '../typings/migrate-result.model.ts';
+import type { VersionStatus } from '../typings/version-status.model.ts';
+import type { SectionRecord } from '../../wasm-codec/functions/decode-section-records.function.ts';
+import type { DocumentManifest } from '../../wasm-codec/functions/decode-document-manifest.function.ts';
+
+/** Pending JSON-RPC call awaiting a response from the daemon. */
+interface PendingCall {
+   resolve: (v: unknown) => void;
+   reject: (e: Error) => void;
+}
+
+/** Shape of a JSON-RPC 2.0 error object from the daemon. */
+interface JsonRpcError {
+   readonly code: number;
+   readonly message: string;
+}
+
+/** Shape of a JSON-RPC 2.0 response from the daemon. */
+interface JsonRpcResponse {
+   readonly id: number;
+   readonly result?: unknown;
+   readonly error?: JsonRpcError;
+}
+
+/**
+ * IPC client that communicates with the worker daemon over a Unix
+ * domain socket (or Windows named pipe) using JSON-RPC 2.0 with
+ * newline-delimited message framing.
+ *
+ * Call `connect()` before any other method and `disconnect()` when
+ * done to release the underlying socket.
+ */
+export class WorkerClient {
+   /** Underlying socket; `null` when not connected. */
+   private socket: Socket | null = null;
+
+   /** Monotonically increasing request ID counter. */
+   private nextId = 1;
+
+   /** Map of outstanding request IDs to their resolve/reject pairs. */
+   private readonly pending = new Map<number, PendingCall>();
+
+   /** Partial line buffer for split TCP/IPC frames. */
+   private buffer = '';
+
+   /**
+    * @param socketPath - absolute path to the daemon's IPC socket
+    *   (or Windows named pipe path)
+    * @param contextId - 16-character context identifier sent as
+    *   `ctx` in every JSON-RPC request
+    */
+   constructor(
+      private readonly socketPath: string,
+      private readonly contextId: string,
+   ) {}
+
+   /**
+    * Establishes the IPC connection to the worker daemon.
+    *
+    * @throws {Error} if the connection cannot be established
+    */
+   async connect(): Promise<void> {
+      return new Promise<void>((resolve, reject) => {
+         this.socket = connect(this.socketPath, () => {
+            resolve();
+         });
+         this.socket.on('error', (err: Error) => {
+            reject(err);
+         });
+         this.socket.on('data', (chunk: Buffer) => {
+            this.handleData(chunk.toString());
+         });
+      });
+   }
+
+   /** Destroys the IPC socket and clears the connection state. */
+   disconnect(): void {
+      if (this.socket) {
+         this.socket.destroy();
+         this.socket = null;
+      }
+   }
+
+   /**
+    * Searches the knowledge base.
+    *
+    * @param params - query parameters including the search string
+    * @returns ranked list of matching sections
+    */
+   async search(params: SearchParams): Promise<SearchResult[]> {
+      return this.call('search', params) as Promise<SearchResult[]>;
+   }
+
+   /**
+    * Adds a new section to the knowledge base.
+    *
+    * @param params - section content and metadata
+    * @returns the result including the assigned knowledge-base
+    *   identifier and indexing statistics
+    */
+   async addSection(
+      params: AddSectionParams,
+   ): Promise<AddSectionResult> {
+      return this.call(
+         'addSection',
+         params,
+      ) as Promise<AddSectionResult>;
+   }
+
+   /**
+    * Replaces an existing section's content and metadata.
+    *
+    * @param kbid - identifier of the section to update
+    * @param params - new content and metadata
+    * @returns the knowledge-base identifier of the updated section
+    */
+   async updateSection(
+      kbid: string,
+      params: AddSectionParams,
+   ): Promise<string> {
+      return this.call('updateSection', {
+         kbid,
+         ...params,
+      }) as Promise<string>;
+   }
+
+   /**
+    * Removes a section from the knowledge base.
+    *
+    * @param kbid - identifier of the section to remove
+    * @returns removal result with `removed` flag and elapsed time
+    */
+   async removeSection(kbid: string): Promise<RemoveSectionResult> {
+      return this.call('removeSection', {
+         kbid,
+      }) as Promise<RemoveSectionResult>;
+   }
+
+   /**
+    * Retrieves multiple sections by their identifiers.
+    *
+    * @param kbids - list of knowledge-base identifiers to fetch
+    * @returns sections in the order their IDs were requested
+    */
+   async readSections(kbids: string[]): Promise<Section[]> {
+      return this.call('readSections', {
+         kbids,
+      }) as Promise<Section[]>;
+   }
+
+   /**
+    * Groups a set of sections under a named document.
+    *
+    * @param title - human-readable document title
+    * @param kbids - identifiers of the sections to group
+    * @param docType - application-level document type tag; omit or
+    *   pass an empty string for untagged documents
+    * @returns the assigned document identifier
+    */
+   async groupSections(
+      title: string,
+      kbids: string[],
+      docType?: string,
+   ): Promise<string> {
+      const result = (await this.call('groupSections', {
+         title,
+         kbids,
+         docType: docType ?? '',
+      })) as { docid: string } | string;
+      return typeof result === 'string' ? result : result.docid;
+   }
+
+   /**
+    * Removes the document grouping for the given document.
+    *
+    * The sections themselves are not deleted; only the grouping
+    * record is removed.
+    *
+    * @param docid - identifier of the document grouping to remove
+    */
+   async removeGrouping(docid: string): Promise<void> {
+      await this.call('removeGrouping', { docid });
+   }
+
+   /**
+    * Retrieves a full document record including its sections.
+    *
+    * @param docid - identifier of the document to retrieve
+    * @returns the document record as returned by the daemon
+    */
+   async retrieveDocument(docid: string): Promise<unknown> {
+      return this.call('retrieveDocument', { docid });
+   }
+
+   /**
+    * Composes a rendered Markdown document from one or more kbid
+    * or docid identifiers.
+    *
+    * @param ids - list of kbid or docid identifiers (may be mixed)
+    * @returns composed Markdown result with character count
+    */
+   async content(ids: string[]): Promise<ContentResult> {
+      return this.call('content', { ids }) as Promise<ContentResult>;
+   }
+
+   /**
+    * Runs an integrity check on the knowledge-base database.
+    *
+    * @returns a report of all issues found; `ok` is `true` when
+    *   no errors were detected
+    */
+   async integrityCheck(): Promise<IntegrityCheckResult> {
+      return this.call(
+         'integrityCheck',
+         {},
+      ) as Promise<IntegrityCheckResult>;
+   }
+
+   /**
+    * Removes unreferenced sections from the database.
+    *
+    * @returns a report of sections removed and bytes freed
+    */
+   async gc(): Promise<GcResult> {
+      return this.call('gc', {}) as Promise<GcResult>;
+   }
+
+   /**
+    * Reconstructs all index files from section files and document
+    * manifests on disk. Text and code sections are re-tokenized;
+    * image sections are skipped.
+    *
+    * @returns a report of sections and terms re-indexed
+    */
+   async rebuildIndexes(): Promise<RebuildResult> {
+      return this.call('rebuildIndexes', {}) as Promise<RebuildResult>;
+   }
+
+   /**
+    * Invalidates the in-memory cache entry for a single section.
+    *
+    * @param kbid - identifier of the section to evict from cache
+    */
+   async invalidate(kbid: string): Promise<void> {
+      await this.call('invalidate', { kbid });
+   }
+
+   /** Clears all entries from the daemon's in-memory cache. */
+   async invalidateAll(): Promise<void> {
+      await this.call('invalidateAll', {});
+   }
+
+   /**
+    * Retrieves diagnostic status information from the daemon.
+    *
+    * @returns current initialization state and cache statistics
+    */
+   async status(): Promise<StatusResult> {
+      return this.call('status', {}) as Promise<StatusResult>;
+   }
+
+   /**
+    * Returns database metadata and status from the daemon,
+    * including document count, section count, index size, and
+    * cache statistics.
+    *
+    * @returns database metadata from the daemon's cached state
+    */
+   async dbStatus(): Promise<StatusResult> {
+      return this.call('dbStatus', {}) as Promise<StatusResult>;
+   }
+
+   /**
+    * Initialises a new `.kbdb` database directory at the given path
+    * by sending an init request to the worker daemon.
+    *
+    * The daemon creates all required subdirectories and seed files via
+    * the `worker_init_directory` WASM export.
+    *
+    * @param targetPath - absolute path to the directory that will
+    *   contain the new `.kbdb` database
+    * @throws {Error} if the `.kbdb` directory already exists or the
+    *   daemon cannot create the layout
+    */
+   async initDirectory(targetPath: string): Promise<void> {
+      await this.call('initDirectory', { targetPath });
+   }
+
+   /**
+    * Runs all pending database migrations for the `.kbdb` directory
+    * at the given path.
+    *
+    * Delegates to the `worker_migrate` WASM export on the daemon.
+    * Each migration step updates `catalog.toml` on success, so
+    * retrying after a partial failure is safe.
+    *
+    * @param targetPath - absolute path to the `.kbdb` database root
+    * @returns migration report with old version, new version, and
+    *   number of migration steps applied
+    */
+   async migrateDatabase(targetPath: string): Promise<MigrateResult> {
+      return this.call('migrateDatabase', {
+         targetPath,
+      }) as Promise<MigrateResult>;
+   }
+
+   /**
+    * Checks the format version of the `.kbdb` database at the given
+    * path against the version required by the current WASM engine.
+    *
+    * This is a read-only operation: it reads `catalog.toml` and
+    * returns the compatibility status without making any changes.
+    *
+    * @param targetPath - absolute path to the `.kbdb` database root
+    * @returns version compatibility report
+    */
+   async checkVersion(targetPath: string): Promise<VersionStatus> {
+      return this.call('checkVersion', {
+         targetPath,
+      }) as Promise<VersionStatus>;
+   }
+
+   /**
+    * Retrieves all sections whose type name matches the given string.
+    *
+    * @param sectionType - the application-level type name to filter by
+    * @returns all section records with a matching type name
+    */
+   async listByType(sectionType: string): Promise<SectionRecord[]> {
+      return this.call('listByType', {
+         sectionType,
+      }) as Promise<SectionRecord[]>;
+   }
+
+   /**
+    * Retrieves all documents whose type name matches the given string.
+    *
+    * @param docType - the application-level document type to filter by
+    * @returns all document manifests with a matching type name
+    */
+   async listDocumentsByType(
+      docType: string,
+   ): Promise<DocumentManifest[]> {
+      return this.call('listDocumentsByType', {
+         docType,
+      }) as Promise<DocumentManifest[]>;
+   }
+
+   /**
+    * Sends a JSON-RPC 2.0 request and awaits the response.
+    *
+    * @param method - the RPC method name
+    * @param params - parameters to include in the request
+    * @throws {Error} if not connected
+    * @throws {Error} when the daemon returns an error response
+    */
+   private async call(
+      method: string,
+      params: unknown,
+   ): Promise<unknown> {
+      const socket = this.socket;
+      if (!socket) {
+         throw new Error('not connected');
+      }
+
+      const id = this.nextId++;
+      const request = {
+         jsonrpc: '2.0',
+         id,
+         method,
+         params: { ctx: this.contextId, ...(params as object) },
+      };
+
+      return new Promise<unknown>((resolve, reject) => {
+         this.pending.set(id, { resolve, reject });
+         socket.write(JSON.stringify(request) + '\n');
+      });
+   }
+
+   /**
+    * Processes incoming data from the socket, splitting on newlines
+    * and dispatching complete JSON-RPC response objects.
+    *
+    * @param data - raw string chunk received from the socket
+    */
+   private handleData(data: string): void {
+      this.buffer += data;
+      const lines = this.buffer.split('\n');
+      this.buffer = lines.pop() ?? '';
+
+      for (const line of lines) {
+         if (!line.trim()) continue;
+         try {
+            const response = JSON.parse(line) as JsonRpcResponse;
+            const pending = this.pending.get(response.id);
+            if (pending) {
+               this.pending.delete(response.id);
+               if (response.error) {
+                  const err = new Error(response.error.message);
+                  (err as unknown as Record<string, unknown>).code =
+                     response.error.code;
+                  pending.reject(err);
+               } else {
+                  pending.resolve(response.result);
+               }
+            }
+         } catch {
+            // Ignore unparseable lines
+         }
+      }
+   }
+}

package/src/shared/worker-client/functions/compute-context-id.function.ts

@@ -0,0 +1,18 @@
+import { computeSha256 } from '../../hash/functions/compute-sha256.ts';
+
+/**
+ * Computes a short context identifier for an absolute path.
+ *
+ * Hashes the path with SHA-256 and returns the first 16 hex
+ * characters. The result is stable for the same input and unique
+ * enough to avoid accidental collisions across typical path sets.
+ *
+ * @param absolutePath - the absolute filesystem path to identify
+ * @returns a 16-character lowercase hex string
+ */
+export async function computeContextId(
+   absolutePath: string,
+): Promise<string> {
+   const hash = await computeSha256(absolutePath);
+   return hash.slice(0, 16);
+}

package/src/shared/worker-client/functions/create-worker-client.function.ts

@@ -0,0 +1,70 @@
+import { existsSync, readFileSync, unlinkSync } from 'node:fs';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import { WorkerClient } from '../classes/worker-client.class.ts';
+import { resolveDbPath } from './resolve-db-path.function.ts';
+import { computeContextId } from './compute-context-id.function.ts';
+import { discoverDaemon } from './discover-daemon.function.ts';
+import { spawnDaemon } from './spawn-daemon.function.ts';
+import { waitForDaemon } from './wait-for-daemon.function.ts';
+import { detectRuntime } from '../../runtime/functions/detect-runtime.ts';
+import { getIpcPath } from '../../platform/functions/get-ipc-path.function.ts';
+import type { WorkerClientOptions } from '../typings/worker-client-options.interface.ts';
+
+/**
+ * Creates and connects a `WorkerClient` to the worker daemon.
+ *
+ * Resolves the context path, derives a stable context identifier,
+ * and checks whether a daemon is already running. If no daemon is
+ * found and `autoSpawn` is not `false`, a new daemon process is
+ * spawned and the factory waits for its PID file to appear before
+ * connecting.
+ *
+ * @param options - optional configuration for path resolution,
+ *   auto-spawn behaviour, and connection timeout
+ * @returns a connected `WorkerClient` ready to send requests
+ * @throws {Error} if the `.kbdb` directory cannot be found
+ * @throws {Error} if the socket connection cannot be established
+ */
+export async function createWorkerClient(
+   options?: WorkerClientOptions,
+): Promise<WorkerClient> {
+   const dbPath =
+      options?.contextPath ?? resolveDbPath(options?.dbPath);
+   const contextId = await computeContextId(dbPath);
+   const socketPath = getIpcPath(contextId);
+
+   let daemon = discoverDaemon(contextId);
+
+   if (!daemon && options?.autoSpawn !== false) {
+      const runtime = detectRuntime();
+      spawnDaemon(dbPath, runtime, options?.workerScript);
+      await waitForDaemon(contextId, options?.connectTimeoutMs);
+      daemon = discoverDaemon(contextId);
+   }
+
+   if (!daemon) {
+      throw new Error(
+         `kbdb worker daemon did not start: ${readCrashReason(contextId)}`,
+      );
+   }
+
+   const client = new WorkerClient(socketPath, contextId);
+   await client.connect();
+   return client;
+}
+
+function readCrashReason(contextId: string): string {
+   const crashPath = join(tmpdir(), `kbdb-${contextId}.crash`);
+   if (!existsSync(crashPath)) {
+      return 'daemon failed to start (no details available)';
+   }
+   try {
+      const raw = readFileSync(crashPath, 'utf-8');
+      const crash = JSON.parse(raw) as { error?: string };
+      unlinkSync(crashPath);
+      return crash.error ?? 'unknown error (crash log empty)';
+   } catch {
+      return 'daemon failed to start (crash log unreadable)';
+   }
+}

package/src/shared/worker-client/functions/discover-daemon.function.ts

@@ -0,0 +1,47 @@
+import { existsSync, readFileSync, unlinkSync } from 'node:fs';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import { isProcessAlive } from '../../platform/functions/is-process-alive.function.ts';
+import type { DaemonInfo } from '../typings/daemon-info.model.ts';
+
+export type { DaemonInfo };
+
+/**
+ * Discovers a running worker daemon for the given context.
+ *
+ * Reads the PID file at `<tmpdir>/kbdb-<contextId>.pid`. If the
+ * file exists and the process is alive, returns its PID and the
+ * expected socket path. If the file is stale (process dead or PID
+ * unparseable), removes the PID file and returns `null`.
+ *
+ * @param contextId - the 16-character context identifier
+ * @returns daemon location info, or `null` if no live daemon found
+ */
+export function discoverDaemon(contextId: string): DaemonInfo | null {
+   const pidPath = join(tmpdir(), `kbdb-${contextId}.pid`);
+
+   if (!existsSync(pidPath)) return null;
+
+   const pidStr = readFileSync(pidPath, 'utf-8').trim();
+   const pid = parseInt(pidStr, 10);
+
+   if (isNaN(pid) || !isProcessAlive(pid)) {
+      try {
+         unlinkSync(pidPath);
+      } catch {
+         // Stale PID file removal is best-effort
+      }
+      return null;
+   }
+
+   const socketPath = join(tmpdir(), `kbdb-${contextId}.sock`);
+
+   const crashPath = join(tmpdir(), `kbdb-${contextId}.crash`);
+   try {
+      unlinkSync(crashPath);
+   } catch {
+      // stale crash file removal is best-effort
+   }
+
+   return { pid, socketPath };
+}

package/src/shared/worker-client/functions/resolve-db-path.function.ts

@@ -0,0 +1,21 @@
+import { existsSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+
+/**
+ * Resolves the absolute path to the `.kbdb` database directory.
+ *
+ * Searches for a `.kbdb` sub-directory inside `targetDir`. When
+ * `targetDir` is omitted, `process.cwd()` is used as the base.
+ *
+ * @param targetDir - directory to search in; defaults to cwd
+ * @throws {Error} if no `.kbdb` directory exists at the resolved
+ *   path
+ */
+export function resolveDbPath(targetDir?: string): string {
+   const base = targetDir ? resolve(targetDir) : process.cwd();
+   const dbPath = join(base, '.kbdb');
+   if (!existsSync(dbPath)) {
+      throw new Error(`database not found: ${dbPath}`);
+   }
+   return dbPath;
+}

package/src/shared/worker-client/functions/spawn-daemon.function.ts

@@ -0,0 +1,35 @@
+import { spawn } from 'node:child_process';
+import type { Runtime } from '../../runtime/typings/runtime.ts';
+
+/**
+ * Spawns a detached worker daemon process for the given context.
+ *
+ * @param contextPath - absolute path to the context directory the
+ *   daemon should serve
+ * @param runtime - the current JavaScript runtime environment
+ * @param workerScript - absolute path to the worker daemon script,
+ *   resolved at the entry point by `resolveWorkerScript`
+ */
+export function spawnDaemon(
+   contextPath: string,
+   runtime: Runtime,
+   workerScript?: string,
+): void {
+   if (!workerScript) {
+      throw new Error(
+         'workerScript must be provided' + ' (resolve at entry point)',
+      );
+   }
+   const execPath = runtime === 'deno' ? 'deno' : process.execPath;
+
+   const args =
+      runtime === 'deno'
+         ? ['run', '--allow-all', workerScript, contextPath]
+         : [workerScript, contextPath];
+
+   const child = spawn(execPath, args, {
+      detached: true,
+      stdio: 'ignore',
+   });
+   child.unref();
+}