@dikolab/kbdb 0.1.5 → 0.2.0

package/src/shared/wasm-codec/functions/decode-scored-results.function.ts

@@ -2,53 +2,107 @@ import { decodeString } from './decode-string.function.ts';
 import { decodeU32 } from './decode-u32.function.ts';
 import { decodeF32 } from './decode-f32.function.ts';
 
-export interface TermMatch {
-   term: string;
-   frequency: number;
-}
-
-export interface ScoredResult {
+export interface SectionMatch {
    kbid: string;
+   heading: string | null;
+   type: string;
+   docids: string[];
    score: number;
-   matches: TermMatch[];
+   snippet: string;
+   matched_terms: string[];
 }
 
-export function decodeScoredResults(bytes: Uint8Array): ScoredResult[] {
+export interface PagedSearchResult {
+   items: SectionMatch[];
+   total: number;
+   offset: number;
+   limit: number;
+   has_more: boolean;
+}
+
+function decodeOptionString(
+   bytes: Uint8Array,
+   offset: number,
+): { value: string | null; bytesRead: number } {
+   const tag = bytes[offset];
+   if (tag === 0x00) {
+      return { value: null, bytesRead: 1 };
+   }
+   const str = decodeString(bytes, offset + 1);
+   return { value: str.value, bytesRead: 1 + str.bytesRead };
+}
+
+export function decodePagedSearchResult(
+   bytes: Uint8Array,
+): PagedSearchResult {
    let offset = 0;
-   const count = decodeU32(bytes, offset);
-   offset += count.bytesRead;
 
-   const results: ScoredResult[] = [];
-   for (let i = 0; i < count.value; i++) {
+   const total = decodeU32(bytes, offset);
+   offset += total.bytesRead;
+
+   const resultOffset = decodeU32(bytes, offset);
+   offset += resultOffset.bytesRead;
+
+   const limit = decodeU32(bytes, offset);
+   offset += limit.bytesRead;
+
+   const hasMore = bytes[offset] === 1;
+   offset += 1;
+
+   const itemCount = decodeU32(bytes, offset);
+   offset += itemCount.bytesRead;
+
+   const items: SectionMatch[] = [];
+   for (let i = 0; i < itemCount.value; i++) {
       const kbid = decodeString(bytes, offset);
       offset += kbid.bytesRead;
 
+      const heading = decodeOptionString(bytes, offset);
+      offset += heading.bytesRead;
+
+      const sectionType = decodeString(bytes, offset);
+      offset += sectionType.bytesRead;
+
+      const docidCount = decodeU32(bytes, offset);
+      offset += docidCount.bytesRead;
+      const docids: string[] = [];
+      for (let j = 0; j < docidCount.value; j++) {
+         const docid = decodeString(bytes, offset);
+         offset += docid.bytesRead;
+         docids.push(docid.value);
+      }
+
       const score = decodeF32(bytes, offset);
       offset += score.bytesRead;
 
-      const matchCount = decodeU32(bytes, offset);
-      offset += matchCount.bytesRead;
+      const snippet = decodeString(bytes, offset);
+      offset += snippet.bytesRead;
 
-      const matches: TermMatch[] = [];
-      for (let j = 0; j < matchCount.value; j++) {
+      const termCount = decodeU32(bytes, offset);
+      offset += termCount.bytesRead;
+      const matched_terms: string[] = [];
+      for (let j = 0; j < termCount.value; j++) {
          const term = decodeString(bytes, offset);
          offset += term.bytesRead;
-
-         const freq = decodeF32(bytes, offset);
-         offset += freq.bytesRead;
-
-         matches.push({
-            term: term.value,
-            frequency: freq.value,
-         });
+         matched_terms.push(term.value);
       }
 
-      results.push({
+      items.push({
          kbid: kbid.value,
+         heading: heading.value,
+         type: sectionType.value,
+         docids,
          score: score.value,
-         matches,
+         snippet: snippet.value,
+         matched_terms,
       });
    }
 
-   return results;
+   return {
+      items,
+      total: total.value,
+      offset: resultOffset.value,
+      limit: limit.value,
+      has_more: hasMore,
+   };
 }

package/src/shared/wasm-codec/functions/encode-recall-params.function.ts

@@ -0,0 +1,68 @@
+import { encodeU32 } from './encode-u32.function.ts';
+import { encodeString } from './encode-string.function.ts';
+
+/**
+ * Encodes recall parameters to the binary format expected by the
+ * `worker_recall` WASM export.
+ *
+ * Binary layout:
+ * - `u8 tag`: `0` = single kbid, `1` = batch
+ * - If tag=0: `u32 len + utf8` kbid
+ * - If tag=1: `u32 count` + (`u32 len + utf8`) per kbid
+ * - `u32 depth` (little-endian)
+ *
+ * When both `kbid` and `kbids` are provided, `kbids` takes precedence
+ * and `kbid` is appended to the batch.
+ *
+ * @param params - recall parameters; at least one of `kbid` or
+ *   `kbids` must be present
+ */
+export function encodeRecallParams(params: {
+   kbid?: string;
+   kbids?: readonly string[];
+   depth?: number;
+}): Uint8Array {
+   const depth = params.depth ?? 0;
+   const depthBytes = encodeU32(depth);
+
+   // Batch: kbids provided (kbid optionally appended)
+   if (params.kbids !== undefined) {
+      const allKbids: string[] =
+         params.kbid !== undefined
+            ? [...params.kbids, params.kbid]
+            : [...params.kbids];
+
+      const countBytes = encodeU32(allKbids.length);
+      const encodedKbids = allKbids.map(encodeString);
+
+      const totalLen =
+         1 + // tag
+         countBytes.length +
+         encodedKbids.reduce((sum, b) => sum + b.length, 0) +
+         depthBytes.length;
+
+      const result = new Uint8Array(totalLen);
+      let offset = 0;
+      result[offset++] = 0x01; // tag = batch
+      result.set(countBytes, offset);
+      offset += countBytes.length;
+      for (const kbidBytes of encodedKbids) {
+         result.set(kbidBytes, offset);
+         offset += kbidBytes.length;
+      }
+      result.set(depthBytes, offset);
+      return result;
+   }
+
+   // Single kbid
+   const kbid = params.kbid ?? '';
+   const kbidBytes = encodeString(kbid);
+   const totalLen = 1 + kbidBytes.length + depthBytes.length;
+   const result = new Uint8Array(totalLen);
+   let offset = 0;
+   result[offset++] = 0x00; // tag = single
+   result.set(kbidBytes, offset);
+   offset += kbidBytes.length;
+   result.set(depthBytes, offset);
+   return result;
+}

package/src/shared/wasm-codec/functions/encode-search-params.function.ts

@@ -4,6 +4,9 @@ export function encodeSearchParams(
    query: string,
    mode = 'and',
    limit = 20,
+   offset = 0,
 ): Uint8Array {
-   return encoder.encode(`${query}\n${mode}\n${String(limit)}`);
+   return encoder.encode(
+      `${query}\n${mode}\n${String(limit)}\n${String(offset)}`,
+   );
 }

package/src/shared/wasm-codec/index.ts

@@ -13,7 +13,7 @@ export { decodeString } from './functions/decode-string.function.ts';
 export { decodeU32 } from './functions/decode-u32.function.ts';
 export { decodeF32 } from './functions/decode-f32.function.ts';
 export { decodeU64 } from './functions/decode-u64.function.ts';
-export { decodeScoredResults } from './functions/decode-scored-results.function.ts';
+export { decodePagedSearchResult } from './functions/decode-scored-results.function.ts';
 export { decodeSectionRecords } from './functions/decode-section-records.function.ts';
 export { decodeSectionRecordList } from './functions/decode-section-record-list.function.ts';
 export { decodeDocumentManifest } from './functions/decode-document-manifest.function.ts';
@@ -21,12 +21,14 @@ export { decodeDocumentManifestList } from './functions/decode-document-manifest
 export { decodeComposeOutput } from './functions/decode-compose-output.function.ts';
 export { decodeKeyValueText } from './functions/decode-key-value-text.function.ts';
 export { decodeRemoveFlag } from './functions/decode-remove-flag.function.ts';
+export { encodeRecallParams } from './functions/encode-recall-params.function.ts';
+export { decodeRecallResult } from './functions/decode-recall-result.function.ts';
 
 export type { DecodeResult } from './functions/decode-string.function.ts';
 export type { AddSectionInput } from './functions/encode-add-section-params.function.ts';
 export type {
-   ScoredResult,
-   TermMatch,
+   SectionMatch,
+   PagedSearchResult,
 } from './functions/decode-scored-results.function.ts';
 export type { SectionRecord } from './functions/decode-section-records.function.ts';
 export type {
@@ -34,3 +36,9 @@ export type {
    DocumentSection,
 } from './functions/decode-document-manifest.function.ts';
 export type { ComposeOutput } from './functions/decode-compose-output.function.ts';
+export type {
+   RecallResult,
+   RecallDocument,
+   RecallSibling,
+   RecallReference,
+} from '../worker-client/typings/recall-result.model.ts';

package/src/shared/worker-client/classes/worker-client.class.ts

@@ -1,7 +1,7 @@
 import { connect } from 'node:net';
 import type { Socket } from 'node:net';
 import type { SearchParams } from '../typings/search-params.model.ts';
-import type { SearchResult } from '../typings/search-result.model.ts';
+import type { PagedSearchResult } from '../typings/search-result.model.ts';
 import type {
    AddSectionParams,
    Section,
@@ -17,6 +17,10 @@ import type { MigrateResult } from '../typings/migrate-result.model.ts';
 import type { VersionStatus } from '../typings/version-status.model.ts';
 import type { SectionRecord } from '../../wasm-codec/functions/decode-section-records.function.ts';
 import type { DocumentManifest } from '../../wasm-codec/functions/decode-document-manifest.function.ts';
+import type {
+   RecallParams,
+   RecallResult,
+} from '../typings/recall-result.model.ts';
 
 /** Pending JSON-RPC call awaiting a response from the daemon. */
 interface PendingCall {
@@ -100,10 +104,26 @@ export class WorkerClient {
     * Searches the knowledge base.
     *
     * @param params - query parameters including the search string
-    * @returns ranked list of matching sections
+    * @returns paginated search results with metadata
+    */
+   async search(params: SearchParams): Promise<PagedSearchResult> {
+      return this.call('search', params) as Promise<PagedSearchResult>;
+   }
+
+   /**
+    * Retrieves full content and context for one or more sections by
+    * their knowledge-base identifiers.
+    *
+    * @param params - section identifier(s) and optional expansion depth
+    * @returns a single `RecallResult` when `kbid` is provided without
+    *   `kbids`, or a `RecallResult[]` when `kbids` is provided
     */
-   async search(params: SearchParams): Promise<SearchResult[]> {
-      return this.call('search', params) as Promise<SearchResult[]>;
+   async recall(
+      params: RecallParams,
+   ): Promise<RecallResult | RecallResult[]> {
+      return this.call('recall', params) as Promise<
+         RecallResult | RecallResult[]
+      >;
    }
 
    /**

package/src/shared/worker-client/index.ts

@@ -7,7 +7,10 @@ export { spawnDaemon } from './functions/spawn-daemon.function.ts';
 export { waitForDaemon } from './functions/wait-for-daemon.function.ts';
 export type { WorkerClientOptions } from './typings/worker-client-options.interface.ts';
 export type { SearchParams } from './typings/search-params.model.ts';
-export type { SearchResult } from './typings/search-result.model.ts';
+export type {
+   SearchResult,
+   PagedSearchResult,
+} from './typings/search-result.model.ts';
 export type {
    Section,
    AddSectionParams,
@@ -29,3 +32,10 @@ export type {
    VersionCompatibility,
    VersionStatus,
 } from './typings/version-status.model.ts';
+export type {
+   RecallParams,
+   RecallResult,
+   RecallDocument,
+   RecallSibling,
+   RecallReference,
+} from './typings/recall-result.model.ts';

package/src/shared/worker-client/typings/recall-result.model.ts

@@ -0,0 +1,98 @@
+/** Parameters for a recall operation. */
+export interface RecallParams {
+   /** Single section identifier to recall. */
+   readonly kbid?: string;
+   /** Batch of section identifiers to recall. */
+   readonly kbids?: readonly string[];
+   /**
+    * Context expansion depth (0-3).
+    *
+    * At depth 0 only the section itself is returned.
+    * Higher values include progressively more context:
+    * documents, siblings, and references.
+    */
+   readonly depth?: number;
+}
+
+/** A document record included at depth >= 1. */
+export interface RecallDocument {
+   /** Unique document identifier. */
+   readonly docid: string;
+   /** Human-readable document title. */
+   readonly title: string;
+   /** Ordered list of section summaries belonging to this document. */
+   readonly sections: readonly {
+      /** Knowledge-base identifier of the section. */
+      readonly kbid: string;
+      /** Section heading text, or `null` when absent. */
+      readonly heading: string | null;
+      /** Semantic type label for the section. */
+      readonly type: string;
+   }[];
+}
+
+/** A sibling section adjacent to the recalled section. */
+export interface RecallSibling {
+   /** Knowledge-base identifier of the sibling section. */
+   readonly kbid: string;
+   /** Sibling heading text, or `null` when absent. */
+   readonly heading: string | null;
+   /** Semantic type label for the sibling section. */
+   readonly type: string;
+   /** Full text content of the sibling section. */
+   readonly content: string;
+}
+
+/** A section that references or is referenced by the recalled section. */
+export interface RecallReference {
+   /** Knowledge-base identifier of the referencing/referenced section. */
+   readonly kbid: string;
+   /** Heading text of the referencing section, or `null` when absent. */
+   readonly heading: string | null;
+   /** Semantic type label of the referencing section. */
+   readonly type: string;
+   /** Application-level marker type (e.g. `'see_also'`, `'link'`). */
+   readonly marker_type: string;
+   /** Short excerpt from the referencing section surrounding the marker. */
+   readonly context_snippet: string;
+   /** Whether the reference is `'forward'` (outgoing) or `'back'` (incoming). */
+   readonly direction: 'forward' | 'back';
+}
+
+/** Full content and context for a recalled knowledge-base section. */
+export interface RecallResult {
+   /** Knowledge-base identifier of the recalled section. */
+   readonly kbid: string;
+   /** Section heading text, or `null` when absent. */
+   readonly heading: string | null;
+   /** Semantic type label for the section. */
+   readonly type: string;
+   /** Full text content of the section. */
+   readonly content: string;
+   /** Identifiers of documents that contain this section. */
+   readonly docids: readonly string[];
+   /**
+    * Full document records containing this section.
+    *
+    * Populated when `depth >= 1`.
+    */
+   readonly documents?: readonly RecallDocument[];
+   /**
+    * Sections that contain a marker pointing back to this section.
+    *
+    * Populated when `depth >= 1`.
+    */
+   readonly back_references?: readonly RecallReference[];
+   /**
+    * Sections adjacent to this section in the same document.
+    *
+    * Populated when `depth >= 1`.
+    */
+   readonly siblings?: readonly RecallSibling[];
+   /**
+    * Sections that this section points to via markers.
+    *
+    * Populated when `depth >= 1`.
+    */
+   readonly references?: readonly RecallReference[];
+}

package/src/shared/worker-client/typings/search-params.model.ts

@@ -9,4 +9,6 @@ export interface SearchParams {
    readonly mode?: 'sections' | 'documents' | 'stats';
    /** Maximum number of results to return. */
    readonly limit?: number;
+   /** Number of results to skip (pagination offset). */
+   readonly offset?: number;
 }

package/src/shared/worker-client/typings/search-result.model.ts

@@ -2,13 +2,30 @@
 export interface SearchResult {
    /** Knowledge-base identifier for the matched section. */
    readonly kbid: string;
+   /** Section heading, or null when untitled. */
+   readonly heading: string | null;
+   /** MIME type of the section content. */
+   readonly type: string;
+   /** Document IDs this section belongs to. */
+   readonly docids: readonly string[];
    /** Relevance score assigned by the ranking engine. */
    readonly score: number;
-   /** Terms that matched within this section. */
-   readonly matches: readonly {
-      /** The matched term. */
-      readonly term: string;
-      /** Number of times the term appears in the section. */
-      readonly frequency: number;
-   }[];
+   /** KWIC snippet (~150 chars around the best match). */
+   readonly snippet: string;
+   /** Query terms that matched within this section. */
+   readonly matched_terms: readonly string[];
+}
+
+/** Paginated search result envelope. */
+export interface PagedSearchResult {
+   /** Matching sections for the current page. */
+   readonly items: readonly SearchResult[];
+   /** Total number of matches across all pages. */
+   readonly total: number;
+   /** Number of results skipped (pagination offset). */
+   readonly offset: number;
+   /** Maximum results per page. */
+   readonly limit: number;
+   /** Whether more results exist beyond this page. */
+   readonly has_more: boolean;
 }

package/src/shared/worker-daemon/classes/worker-daemon.class.ts

@@ -28,7 +28,8 @@ import {
    encodeGroupParams,
    encodeIdentifierList,
    encodeTypeFilterParam,
-   decodeScoredResults,
+   encodeRecallParams,
+   decodePagedSearchResult,
    decodeSectionRecords,
    decodeSectionRecordList,
    decodeDocumentManifest,
@@ -36,6 +37,7 @@ import {
    decodeComposeOutput,
    decodeKeyValueText,
    decodeRemoveFlag,
+   decodeRecallResult,
 } from '../../wasm-codec/index.ts';
 
 import type { AddSectionInput } from '../../wasm-codec/index.ts';
@@ -86,6 +88,7 @@ const WASM_METHOD_MAP: Readonly<Partial<Record<string, string>>> = {
    list_by_type: 'worker_list_by_type',
    listDocumentsByType: 'worker_list_docs_by_type',
    list_docs_by_type: 'worker_list_docs_by_type',
+   recall: 'worker_recall',
 };
 
 /** Config method name prefix. */
@@ -564,6 +567,9 @@ export class WorkerDaemon {
       ) {
          return this.handleListDocsByType(params);
       }
+      if (method === 'recall') {
+         return this.handleRecall(params);
+      }
       return this.invokeWasmRaw(wasmExport, new Uint8Array(0));
    }
 
@@ -573,7 +579,8 @@ export class WorkerDaemon {
       const query = (params['query'] as string | undefined) ?? '';
       const mode = (params['mode'] as string | undefined) ?? 'and';
       const limit = Number(params['limit'] ?? 20);
-      const encoded = encodeSearchParams(query, mode, limit);
+      const offset = Number(params['offset'] ?? 0);
+      const encoded = encodeSearchParams(query, mode, limit, offset);
 
       const searchFn = this.getExport('worker_search');
       const ptr = this.wasmAlloc(encoded.length);
@@ -598,7 +605,7 @@ export class WorkerDaemon {
          throw new IpcError(-32000, new TextDecoder().decode(errBytes));
       }
       const resultBytes = this.readReturnSlot();
-      return decodeScoredResults(resultBytes);
+      return decodePagedSearchResult(resultBytes);
    }
 
    // -- Token status/result (u32 direct params) --
@@ -621,7 +628,7 @@ export class WorkerDaemon {
          throw new IpcError(-32000, new TextDecoder().decode(errBytes));
       }
       const bytes = this.readReturnSlot();
-      return decodeScoredResults(bytes);
+      return decodePagedSearchResult(bytes);
    }
 
    // -- Binary-encoded input methods --
@@ -776,6 +783,20 @@ export class WorkerDaemon {
       return decodeDocumentManifestList(bytes);
    }
 
+   private handleRecall(params: Record<string, unknown>): unknown {
+      const kbid =
+         typeof params['kbid'] === 'string'
+            ? params['kbid']
+            : undefined;
+      const kbids = Array.isArray(params['kbids'])
+         ? (params['kbids'] as string[])
+         : undefined;
+      const depth = Number(params['depth'] ?? 0);
+      const encoded = encodeRecallParams({ kbid, kbids, depth });
+      const bytes = this.callWasmSync('worker_recall', encoded);
+      return decodeRecallResult(bytes);
+   }
+
    // -- Zero-arg methods --
 
    private handleInvalidateAll(): unknown {

package/src/wasm/fs-database/kbdb_fs_database.d.ts

@@ -48,6 +48,22 @@ declare namespace wasm_bindgen {
      */
     export function db_gc(_ptr: number, _len: number): number;
 
+    /**
+     * Returns back-references pointing to a section.
+     * `ptr`/`len` point to a UTF-8 kbid string.
+     * On success encodes `u32 count` followed by entries (peer_id, marker_type,
+     * context_snippet) into the return slot.
+     */
+    export function db_get_back_refs(ptr: number, len: number): number;
+
+    /**
+     * Returns forward references for a section.
+     * `ptr`/`len` point to a UTF-8 kbid string.
+     * On success encodes `u32 count` followed by entries (peer_id, marker_type,
+     * context_snippet) into the return slot.
+     */
+    export function db_get_forward_refs(ptr: number, len: number): number;
+
     /**
      * Initialises the database context path.
      *
@@ -185,6 +201,8 @@ declare interface InitOutput {
     readonly db_check_integrity: (a: number, b: number) => number;
     readonly db_check_version: (a: number, b: number) => number;
     readonly db_gc: (a: number, b: number) => number;
+    readonly db_get_back_refs: (a: number, b: number) => number;
+    readonly db_get_forward_refs: (a: number, b: number) => number;
     readonly db_init: (a: number, b: number) => number;
     readonly db_init_catalog: (a: number, b: number) => number;
     readonly db_init_directory: (a: number, b: number) => number;

package/src/wasm/fs-database/kbdb_fs_database.js

@@ -85,6 +85,36 @@ let wasm_bindgen = (function(exports) {
     }
     exports.db_gc = db_gc;
 
+    /**
+     * Returns back-references pointing to a section.
+     * `ptr`/`len` point to a UTF-8 kbid string.
+     * On success encodes `u32 count` followed by entries (peer_id, marker_type,
+     * context_snippet) into the return slot.
+     * @param {number} ptr
+     * @param {number} len
+     * @returns {number}
+     */
+    function db_get_back_refs(ptr, len) {
+        const ret = wasm.db_get_back_refs(ptr, len);
+        return ret;
+    }
+    exports.db_get_back_refs = db_get_back_refs;
+
+    /**
+     * Returns forward references for a section.
+     * `ptr`/`len` point to a UTF-8 kbid string.
+     * On success encodes `u32 count` followed by entries (peer_id, marker_type,
+     * context_snippet) into the return slot.
+     * @param {number} ptr
+     * @param {number} len
+     * @returns {number}
+     */
+    function db_get_forward_refs(ptr, len) {
+        const ret = wasm.db_get_forward_refs(ptr, len);
+        return ret;
+    }
+    exports.db_get_forward_refs = db_get_forward_refs;
+
     /**
      * Initialises the database context path.
      *

package/src/wasm/fs-database/kbdb_fs_database_bg.wasm.d.ts

@@ -8,6 +8,8 @@ export const db_add_to_index: (a: number, b: number) => number;
 export const db_check_integrity: (a: number, b: number) => number;
 export const db_check_version: (a: number, b: number) => number;
 export const db_gc: (a: number, b: number) => number;
+export const db_get_back_refs: (a: number, b: number) => number;
+export const db_get_forward_refs: (a: number, b: number) => number;
 export const db_init: (a: number, b: number) => number;
 export const db_init_catalog: (a: number, b: number) => number;
 export const db_init_directory: (a: number, b: number) => number;