@dikolab/kbdb 0.2.2 → 0.3.1

package/src/shared/wasm-codec/functions/decode-section-records.function.ts

@@ -40,6 +40,33 @@ export interface SectionRecord {
    codeTokenCount: number;
    /** Application-level type name assigned to this section. */
    typeName: string;
+   /**
+    * Stable logical identity derived from source path and heading.
+    * `null` for stdin input or sections without a source path.
+    */
+   sourceKey: string | null;
+   /**
+    * Absolute path of the source file that was learned.
+    * `null` for stdin input or programmatic API calls.
+    */
+   sourcePath: string | null;
+   /**
+    * ISO 8601 modified time of the source file at learn time.
+    * `null` when no source file exists.
+    */
+   sourceMtime: string | null;
+   /**
+    * ISO 8601 timestamp of the last supersession.
+    * `null` on first creation.
+    */
+   updatedAt: string | null;
+   /** User-supplied tags for scoping and filtering. */
+   tags: string[];
+   /**
+    * The kbid of the section this one replaced via supersession.
+    * `null` on first creation or non-superseding additions.
+    */
+   supersedes: string | null;
 }
 
 function decodeOptionString(
@@ -82,6 +109,12 @@ function decodeSectionType(
  * Each record is decoded in field order as produced by the Rust
  * `SectionRecord::encode` implementation.
  *
+ * Trailing fields (`sourceKey`, `sourcePath`, `sourceMtime`,
+ * `updatedAt`, `tags`, `supersedes`) were added after the initial
+ * format. When decoding legacy bytes that stop after `typeName`,
+ * those fields default to `null` / `[]`. This matches the Rust
+ * decoder's backward-compatibility guards.
+ *
  * @param bytes - raw bytes returned by a WASM section-list export
  * @returns decoded array of section records, empty when count is zero
  */
@@ -136,6 +169,54 @@ export function decodeSectionRecords(
       const typeName = decodeString(bytes, offset);
       offset += typeName.bytesRead;
 
+      // New trailing fields -- use length guards for backward compat.
+      let sourceKey: string | null = null;
+      if (offset < bytes.length) {
+         const v = decodeOptionString(bytes, offset);
+         offset += v.bytesRead;
+         sourceKey = v.value;
+      }
+
+      let sourcePath: string | null = null;
+      if (offset < bytes.length) {
+         const v = decodeOptionString(bytes, offset);
+         offset += v.bytesRead;
+         sourcePath = v.value;
+      }
+
+      let sourceMtime: string | null = null;
+      if (offset < bytes.length) {
+         const v = decodeOptionString(bytes, offset);
+         offset += v.bytesRead;
+         sourceMtime = v.value;
+      }
+
+      let updatedAt: string | null = null;
+      if (offset < bytes.length) {
+         const v = decodeOptionString(bytes, offset);
+         offset += v.bytesRead;
+         updatedAt = v.value;
+      }
+
+      let tags: string[] = [];
+      if (offset < bytes.length) {
+         const tagCount = decodeU32(bytes, offset);
+         offset += tagCount.bytesRead;
+         tags = [];
+         for (let j = 0; j < tagCount.value; j++) {
+            const tag = decodeString(bytes, offset);
+            offset += tag.bytesRead;
+            tags.push(tag.value);
+         }
+      }
+
+      let supersedes: string | null = null;
+      if (offset < bytes.length) {
+         const v = decodeOptionString(bytes, offset);
+         offset += v.bytesRead;
+         supersedes = v.value;
+      }
+
       const record: SectionRecord = {
          kbid: kbid.value,
          docid: docid.value,
@@ -151,6 +232,12 @@ export function decodeSectionRecords(
          bodyTokenCount: bodyTokenCount.value,
          codeTokenCount: codeTokenCount.value,
          typeName: typeName.value,
+         sourceKey,
+         sourcePath,
+         sourceMtime,
+         updatedAt,
+         tags,
+         supersedes,
       };
       if (st.mimeType !== undefined) {
          record.mimeType = st.mimeType;

package/src/shared/wasm-codec/functions/decode-section-type.function.ts

@@ -0,0 +1,30 @@
+import { decodeString } from './decode-string.function.ts';
+import type { SectionTypeResult } from '../typings/section-type-result.model.ts';
+
+export type { SectionTypeResult };
+
+/**
+ * Decodes a section type discriminant from WASM binary output.
+ *
+ * Wire format: `u8` tag (`0x00` = text, `0x01` = code,
+ * `0x02` = image), followed by a MIME string for code/image.
+ *
+ * @param bytes - raw binary buffer
+ * @param offset - byte position of the tag byte
+ * @returns section type, optional MIME type, and bytes consumed
+ */
+export function decodeSectionType(
+   bytes: Uint8Array,
+   offset: number,
+): SectionTypeResult {
+   const tag = bytes[offset];
+   if (tag === 0x00) {
+      return { sectionType: 'text', bytesRead: 1 };
+   }
+   const mime = decodeString(bytes, offset + 1);
+   return {
+      sectionType: tag === 0x01 ? 'code' : 'image',
+      mimeType: mime.value,
+      bytesRead: 1 + mime.bytesRead,
+   };
+}

package/src/shared/wasm-codec/functions/decode-wasm-list.function.ts

@@ -0,0 +1,31 @@
+import { decodeU32 } from './decode-u32.function.ts';
+import type { ItemDecoder } from '../typings/item-decoder.type.ts';
+
+export type { ItemDecoder };
+
+/**
+ * Decodes a length-prefixed list from WASM binary output.
+ *
+ * Wire format: u32 item count, then that many items decoded
+ * sequentially by `decode`.
+ *
+ * @param bytes - raw binary buffer
+ * @param offset - byte position of the u32 count
+ * @param decode - decoder function for each item
+ * @returns decoded item array and total bytes consumed
+ */
+export function decodeWasmList<T>(
+   bytes: Uint8Array,
+   offset: number,
+   decode: ItemDecoder<T>,
+): { items: T[]; bytesRead: number } {
+   const count = decodeU32(bytes, offset);
+   let pos = offset + count.bytesRead;
+   const items: T[] = [];
+   for (let i = 0; i < count.value; i++) {
+      const result = decode(bytes, pos);
+      pos += result.bytesRead;
+      items.push(result.value);
+   }
+   return { items, bytesRead: pos - offset };
+}

package/src/shared/wasm-codec/functions/encode-add-section-params.function.ts

@@ -1,20 +1,83 @@
 import { encodeSectionType } from './encode-section-type.function.ts';
 import { encodeString } from './encode-string.function.ts';
 import { encodeOptionString } from './encode-option-string.function.ts';
+import { encodeU32 } from './encode-u32.function.ts';
 
+/**
+ * Parameters for adding a new section to the knowledge base.
+ *
+ * Maps to the Rust `AddSectionParams` struct encoded for the
+ * `worker_add_section` WASM export.
+ */
 export interface AddSectionInput {
+   /** Semantic content category. */
    sectionType: 'text' | 'code' | 'image';
+   /** MIME type; required for `code` and `image` section types. */
    mimeType?: string;
+   /** Raw section content. */
    content: string;
+   /** Optional section heading. */
    title?: string | null;
+   /** Optional description summary. */
    description?: string | null;
+   /** Optional document identifier to attach the section to. */
    docid?: string | null;
+   /**
+    * Application-level type name for the section (e.g. `'skill'`,
+    * `'fact'`). `null` means no custom type.
+    */
    typeName?: string | null;
+   /**
+    * Absolute path of the source file being learned. `undefined`
+    * means stdin input or programmatic API call.
+    */
+   sourcePath?: string;
+   /**
+    * ISO 8601 modified time of the source file. `undefined` means
+    * no source file exists.
+    */
+   sourceMtime?: string;
+   /**
+    * User-supplied tags for scoping and filtering. Defaults to
+    * an empty list when not provided.
+    */
+   tags?: string[];
+   /**
+    * When `true`, supersede any existing section with the same
+    * source key. Defaults to `false`.
+    */
+   replace?: boolean;
 }
 
+/**
+ * Encodes `AddSectionInput` as the binary format expected by the
+ * `worker_add_section` WASM export.
+ *
+ * Wire format (little-endian):
+ * - `SectionType` tag byte (+ optional MIME string for code/image)
+ * - `content`: `u32` length + UTF-8 bytes
+ * - `title`: `u8` flag + optional `String`
+ * - `description`: `u8` flag + optional `String`
+ * - `docid`: `u8` flag + optional `String`
+ * - `typeName`: `u8` flag + optional `String`
+ * - `sourcePath`: `u8` flag + optional `String`
+ * - `sourceMtime`: `u8` flag + optional `String`
+ * - `tags`: `u32` count + each `String`
+ * - `replace`: `u8` (0 or 1)
+ */
 export function encodeAddSectionParams(
    params: AddSectionInput,
 ): Uint8Array {
+   const tags = params.tags ?? [];
+   const tagParts: Uint8Array[] = [];
+   for (const tag of tags) {
+      tagParts.push(encodeString(tag));
+   }
+   const tagsCountBytes = encodeU32(tags.length);
+   const replaceByte = new Uint8Array([
+      params.replace === true ? 1 : 0,
+   ]);
+
    const parts = [
       encodeSectionType(params.sectionType, params.mimeType),
       encodeString(params.content),
@@ -22,6 +85,11 @@ export function encodeAddSectionParams(
       encodeOptionString(params.description),
       encodeOptionString(params.docid),
       encodeOptionString(params.typeName),
+      encodeOptionString(params.sourcePath),
+      encodeOptionString(params.sourceMtime),
+      tagsCountBytes,
+      ...tagParts,
+      replaceByte,
    ];
    const totalLen = parts.reduce((sum, p) => sum + p.length, 0);
    const result = new Uint8Array(totalLen);

package/src/shared/wasm-codec/index.ts

@@ -23,9 +23,14 @@ export { decodeKeyValueText } from './functions/decode-key-value-text.function.t
 export { decodeRemoveFlag } from './functions/decode-remove-flag.function.ts';
 export { encodeRecallParams } from './functions/encode-recall-params.function.ts';
 export { decodeRecallResult } from './functions/decode-recall-result.function.ts';
+export { decodeAddSectionResult } from './functions/decode-add-section-result.function.ts';
 
 export type { DecodeResult } from './functions/decode-string.function.ts';
 export type { AddSectionInput } from './functions/encode-add-section-params.function.ts';
+export type {
+   AddSectionResult,
+   NearDuplicate,
+} from './functions/decode-add-section-result.function.ts';
 export type {
    SectionMatch,
    PagedSearchResult,

package/src/shared/wasm-codec/typings/add-section-input.model.ts

@@ -0,0 +1,45 @@
+/**
+ * Parameters for adding a new section to the knowledge base.
+ *
+ * Maps to the Rust `AddSectionParams` struct encoded for the
+ * `worker_add_section` WASM export.
+ */
+export interface AddSectionInput {
+   /** Semantic content category. */
+   sectionType: 'text' | 'code' | 'image';
+   /** MIME type; required for `code` and `image` section types. */
+   mimeType?: string;
+   /** Raw section content. */
+   content: string;
+   /** Optional section heading. */
+   title?: string | null;
+   /** Optional description summary. */
+   description?: string | null;
+   /** Optional document identifier to attach the section to. */
+   docid?: string | null;
+   /**
+    * Application-level type name for the section (e.g. `'skill'`,
+    * `'fact'`). `null` means no custom type.
+    */
+   typeName?: string | null;
+   /**
+    * Absolute path of the source file being learned. `undefined`
+    * means stdin input or programmatic API call.
+    */
+   sourcePath?: string;
+   /**
+    * ISO 8601 modified time of the source file. `undefined` means
+    * no source file exists.
+    */
+   sourceMtime?: string;
+   /**
+    * User-supplied tags for scoping and filtering. Defaults to
+    * an empty list when not provided.
+    */
+   tags?: string[];
+   /**
+    * When `true`, supersede any existing section with the same
+    * source key. Defaults to `false`.
+    */
+   replace?: boolean;
+}

package/src/shared/wasm-codec/typings/add-section-result.model.ts

@@ -0,0 +1,28 @@
+import type { NearDuplicate } from './near-duplicate.model.ts';
+
+/**
+ * Decoded result of a `worker_add_section` WASM call.
+ *
+ * Mirrors the Rust `AddSectionResult` struct encoded via the
+ * `Encode` trait wire format.
+ */
+export interface AddSectionResult {
+   /** Knowledge-base identifier assigned to the new section. */
+   readonly kbid: string;
+   /** Number of unique terms indexed for the section. */
+   readonly indexedTermsCount: number;
+   /** Wall-clock time the add operation took, in milliseconds. */
+   readonly elapsedMs: number;
+   /**
+    * Knowledge-base identifier of the section that was superseded,
+    * or `null` if no supersession occurred.
+    */
+   readonly superseded: string | null;
+   /** Advisory warnings emitted during indexing. */
+   readonly warnings: string[];
+   /**
+    * Near-duplicate sections detected above the similarity threshold
+    * (Jaccard ≥ 0.85).
+    */
+   readonly nearDuplicates: NearDuplicate[];
+}

package/src/shared/wasm-codec/typings/document-manifest.model.ts

@@ -0,0 +1,20 @@
+import type { DocumentSection } from './document-section.model.ts';
+
+/**
+ * A document manifest decoded from WASM binary output.
+ *
+ * Represents a grouped document record with its ordered list of
+ * section references as returned by `worker_retrieve_doc`.
+ */
+export interface DocumentManifest {
+   /** Unique document identifier. */
+   docid: string;
+   /** Human-readable document title. */
+   title: string;
+   /** Document creation timestamp as a Unix epoch value. */
+   createdAt: bigint;
+   /** Application-level type name assigned to this document. */
+   docType: string;
+   /** Ordered list of section references belonging to the document. */
+   sections: DocumentSection[];
+}

package/src/shared/wasm-codec/typings/document-section.model.ts

@@ -0,0 +1,12 @@
+/**
+ * A single section reference within a document manifest.
+ *
+ * Associates a knowledge-base identifier with the section's
+ * ordinal position in the document.
+ */
+export interface DocumentSection {
+   /** Knowledge-base identifier of the referenced section. */
+   kbid: string;
+   /** Zero-based position of this section within its document. */
+   position: number;
+}

package/src/shared/wasm-codec/typings/item-decoder.type.ts

@@ -0,0 +1,10 @@
+import type { DecodeResult } from '../functions/decode-string.function.ts';
+
+/**
+ * A function that decodes one item from a binary buffer at the
+ * given byte offset, returning the decoded value and bytes consumed.
+ */
+export type ItemDecoder<T> = (
+   bytes: Uint8Array,
+   offset: number,
+) => DecodeResult<T>;

package/src/shared/wasm-codec/typings/near-duplicate.model.ts

@@ -0,0 +1,16 @@
+/**
+ * A near-duplicate section identified during add-section indexing.
+ */
+export interface NearDuplicate {
+   /** Knowledge-base identifier of the near-duplicate section. */
+   readonly kbid: string;
+   /** Similarity score (0.0–1.0). */
+   readonly similarity: number;
+   /**
+    * Detection method that produced this match.
+    *
+    * Optional for backward compatibility with WASM-decoded results
+    * that do not carry method information.
+    */
+   readonly method?: 'jaccard' | 'cosine';
+}

package/src/shared/wasm-codec/typings/paged-search-result.model.ts

@@ -0,0 +1,22 @@
+import type { SectionMatch } from './section-match.model.ts';
+
+/**
+ * Paginated search response envelope returned by the WASM engine.
+ */
+export interface PagedSearchResult {
+   /** Matching sections for the current page. */
+   items: SectionMatch[];
+   /** Total number of matches across all pages. */
+   total: number;
+   /** Number of results skipped (pagination offset). */
+   offset: number;
+   /** Maximum results per page. */
+   limit: number;
+   /** Whether more results exist beyond this page. */
+   has_more: boolean;
+   /**
+    * Whether the engine fell back to relaxed (OR-mode) search
+    * after strict (AND-mode) returned no results.
+    */
+   relaxed: boolean;
+}

package/src/shared/wasm-codec/typings/section-match.model.ts

@@ -0,0 +1,31 @@
+/**
+ * A single ranked result item from a paged search response.
+ */
+export interface SectionMatch {
+   /** Knowledge-base identifier for the matched section. */
+   kbid: string;
+   /** Section heading, or `null` when untitled. */
+   heading: string | null;
+   /** MIME type of the section content. */
+   type: string;
+   /** Document IDs this section belongs to. */
+   docids: string[];
+   /** Relevance score from the ranking engine. */
+   score: number;
+   /**
+    * Confidence modifier (0.0–1.0) applied after relaxed search.
+    * Always 1.0 for strict-mode results.
+    */
+   confidence: number;
+   /** KWIC snippet (~150 chars around the best match). */
+   snippet: string;
+   /** Query terms that matched within this section. */
+   matched_terms: string[];
+   /**
+    * Document fields that contained at least one matched term
+    * (e.g. `'title'`, `'body'`).
+    */
+   matched_fields: string[];
+   /** ISO 8601 creation timestamp of the section. */
+   created_at: string;
+}

package/src/shared/wasm-codec/typings/section-record-trailing.model.ts

@@ -0,0 +1,38 @@
+/**
+ * Trailing backward-compatible fields of a section record.
+ *
+ * These fields were added after the initial wire format. When
+ * decoding legacy binary output that stops before these fields,
+ * each defaults to `null` or `[]`.
+ */
+export interface SectionRecordTrailing {
+   /**
+    * Stable logical identity derived from source path and heading.
+    * `null` for stdin input or sections without a source path.
+    */
+   sourceKey: string | null;
+   /**
+    * Absolute path of the source file that was learned.
+    * `null` for stdin input or programmatic API calls.
+    */
+   sourcePath: string | null;
+   /**
+    * ISO 8601 modified time of the source file at learn time.
+    * `null` when no source file exists.
+    */
+   sourceMtime: string | null;
+   /**
+    * ISO 8601 timestamp of the last supersession.
+    * `null` on first creation.
+    */
+   updatedAt: string | null;
+   /** User-supplied tags for scoping and filtering. */
+   tags: string[];
+   /**
+    * The kbid of the section this one replaced via supersession.
+    * `null` on first creation or non-superseding additions.
+    */
+   supersedes: string | null;
+   /** Number of bytes consumed from the buffer. */
+   bytesRead: number;
+}

package/src/shared/wasm-codec/typings/section-record.model.ts

@@ -0,0 +1,66 @@
+/**
+ * A single section record decoded from WASM binary output.
+ *
+ * Represents a stored knowledge-base section with all its metadata
+ * and token counts as returned by `worker_read_sections` or
+ * `worker_list_by_type`.
+ */
+export interface SectionRecord {
+   /** Unique knowledge-base identifier for this section. */
+   kbid: string;
+   /** Document identifier this section belongs to, or `null`. */
+   docid: string | null;
+   /** Semantic type discriminant: `'text'`, `'code'`, or `'image'`. */
+   sectionType: string;
+   /** MIME type string, present only for `'code'` and `'image'` types. */
+   mimeType?: string;
+   /** Human-readable title for the section, or `null`. */
+   title: string | null;
+   /** Optional description of the section content, or `null`. */
+   description: string | null;
+   /** Raw text or base64-encoded content of the section. */
+   content: string;
+   /** Byte size of the stored content. */
+   size: number;
+   /** Section creation timestamp as a Unix epoch value. */
+   createdAt: bigint;
+   /** Integrity checksum of the stored content. */
+   checksum: string;
+   /** Total number of tokens in the section. */
+   tokenCount: number;
+   /** Number of heading tokens in the section. */
+   headingTokenCount: number;
+   /** Number of body tokens in the section. */
+   bodyTokenCount: number;
+   /** Number of code tokens in the section. */
+   codeTokenCount: number;
+   /** Application-level type name assigned to this section. */
+   typeName: string;
+   /**
+    * Stable logical identity derived from source path and heading.
+    * `null` for stdin input or sections without a source path.
+    */
+   sourceKey: string | null;
+   /**
+    * Absolute path of the source file that was learned.
+    * `null` for stdin input or programmatic API calls.
+    */
+   sourcePath: string | null;
+   /**
+    * ISO 8601 modified time of the source file at learn time.
+    * `null` when no source file exists.
+    */
+   sourceMtime: string | null;
+   /**
+    * ISO 8601 timestamp of the last supersession.
+    * `null` on first creation.
+    */
+   updatedAt: string | null;
+   /** User-supplied tags for scoping and filtering. */
+   tags: string[];
+   /**
+    * The kbid of the section this one replaced via supersession.
+    * `null` on first creation or non-superseding additions.
+    */
+   supersedes: string | null;
+}

package/src/shared/wasm-codec/typings/section-type-result.model.ts

@@ -0,0 +1,12 @@
+/** Result of decoding a section type discriminant from binary output. */
+export interface SectionTypeResult {
+   /** Semantic section category: `'text'`, `'code'`, or `'image'`. */
+   sectionType: string;
+   /**
+    * MIME type string, present only for `'code'` and `'image'`
+    * types.
+    */
+   mimeType?: string;
+   /** Number of bytes consumed from the buffer. */
+   bytesRead: number;
+}

package/src/shared/worker-client/classes/worker-client.class.ts

@@ -21,6 +21,7 @@ import type {
    RecallParams,
    RecallResult,
 } from '../typings/recall-result.model.ts';
+import type { ExportResult } from '../typings/export-result.model.ts';
 
 /** Pending JSON-RPC call awaiting a response from the daemon. */
 interface PendingCall {
@@ -381,6 +382,24 @@ export class WorkerClient {
       }) as Promise<DocumentManifest[]>;
    }
 
+   /**
+    * Exports the database to a target directory on disk.
+    *
+    * Copies section files, document manifests, and `catalog.toml`
+    * to `params.path`. Index files are not exported because they
+    * are derivable from the section and document data.
+    *
+    * @param params - optional export parameters
+    * @param params.path - target directory path; defaults to
+    *   `./kbdb-export/` when not provided
+    * @returns export report with the resolved path and file counts
+    */
+   async export(params?: { path?: string }): Promise<ExportResult> {
+      return this.call('export', {
+         targetPath: params?.path || undefined,
+      }) as Promise<ExportResult>;
+   }
+
    /**
     * Sends a JSON-RPC 2.0 request and awaits the response.
     *

package/src/shared/worker-client/functions/handle-ipc-data.function.ts

@@ -0,0 +1,49 @@
+import type { PendingCall } from '../typings/pending-call.interface.ts';
+import type { JsonRpcResponse } from '../typings/json-rpc-response.interface.ts';
+
+/**
+ * Processes an incoming data chunk from the IPC socket.
+ *
+ * Appends the chunk to `buffer`, splits on newline boundaries,
+ * parses complete JSON-RPC response lines, and resolves or rejects
+ * the matching pending call. Returns the updated buffer with any
+ * incomplete trailing line retained.
+ *
+ * @param data - raw string chunk received from the socket
+ * @param buffer - partial line buffer from the previous call
+ * @param pending - map of outstanding request IDs to their
+ *   resolve/reject pairs; mutated by this function
+ * @returns the updated buffer (incomplete trailing line, if any)
+ */
+export function handleIpcData(
+   data: string,
+   buffer: string,
+   pending: Map<number, PendingCall>,
+): string {
+   const combined = buffer + data;
+   const lines = combined.split('\n');
+   const nextBuffer = lines.pop() ?? '';
+
+   for (const line of lines) {
+      if (!line.trim()) continue;
+      try {
+         const response = JSON.parse(line) as JsonRpcResponse;
+         const call = pending.get(response.id);
+         if (call !== undefined) {
+            pending.delete(response.id);
+            if (response.error !== undefined) {
+               const err = new Error(response.error.message);
+               (err as unknown as Record<string, unknown>)['code'] =
+                  response.error.code;
+               call.reject(err);
+            } else {
+               call.resolve(response.result);
+            }
+         }
+      } catch {
+         // Ignore unparseable lines
+      }
+   }
+
+   return nextBuffer;
+}