@dikolab/kbdb 0.1.5 → 0.2.0

package/docs/release-notes/0.2.0.md

@@ -0,0 +1,147 @@
+# 0.2.0
+
+*Minor release -- 2026-06-21*
+
+Adds progressive context recall, paginated search
+results with rich metadata, and a bidirectional
+reference graph. These features give AI agents
+the ability to explore related content across
+sections and documents without loading everything
+at once.
+
+---
+
+## New Features
+
+- **Recall with layered context (depths 0--3)** --
+  New `recall` command (CLI), MCP tool, and IPC
+  method. Accepts a `kbid` (or `kbids` for batch)
+  and a `depth` parameter that controls how much
+  context is returned:
+  - Depth 0: section content, heading, type, docids
+  - Depth 1: adds parent documents and
+    back-references
+  - Depth 2: adds sibling sections and forward
+    references
+  - Depth 3: referenced sections include full
+    content
+
+- **Paginated search results** -- Search now
+  returns a `PagedSearchResult` envelope:
+  `{ items, total, offset, limit, has_more }`.
+  New `--offset` / `-o` CLI flag. The `offset`
+  parameter is available in MCP and IPC as well.
+
+- **Rich search result metadata** -- Each search
+  result item now includes `heading`, `type`,
+  `snippet` (KWIC ~150 chars around matched terms),
+  `matched_terms`, and `docids`. Replaces the
+  previous `matches: [{term, frequency}]` format.
+
+- **Bidirectional reference graph** -- New
+  `references.idx` tracks content-marker references
+  (`$(link: ...)`, `$(doclink: ...)`, etc.) between
+  sections. New WASM exports `db_get_forward_refs`
+  and `db_get_back_refs`. Built automatically on
+  `learn`/`update`, cleaned on `unlearn`,
+  regenerated on `rebuild`.
+
+---
+
+## Improvements
+
+- **KWIC snippet generation** -- New
+  `extract_kwic_snippet()` in the Rust engine
+  produces ~150-character context windows around
+  matched terms with case-insensitive matching,
+  Unicode-safe boundaries, and ellipsis markers.
+
+- **MCP search strips score** -- The MCP layer
+  omits the `score` field from search results.
+  Rank order is the signal for AI agents; raw
+  scores are kept in CLI and IPC output for
+  debugging.
+
+- **Tool annotations** -- All 18 MCP tool
+  definitions now include `openWorldHint: false`,
+  `readOnlyHint`, `destructiveHint`, and
+  `idempotentHint` annotations per the MCP spec.
+
+- **Text format paging** -- CLI text output now
+  shows a paging footer:
+  `Showing 1--20 of 42 results. Use --offset 20 to see more.`
+
+- **MCP tool descriptions** -- `search` guides
+  agents to use `recall` for full content.
+  `retrieve` updated to prefer `recall`.
+
+- **Documentation included in package** -- Both
+  npm and JSR packages now ship `docs/details/`,
+  `docs/goals/`, `docs/modules/`,
+  `docs/release-notes/`, and `docs/overview.md`.
+
+---
+
+## Testing
+
+- **383 e2e assertions (Node.js), 378 (Deno)** --
+  up from ~300. All pass with 0 failures.
+
+- **Recall depth testing** -- Cross-referenced
+  test data (sections with `$(link: ...)` markers)
+  validates depth 0--3 field presence across CLI,
+  MCP, and worker IPC.
+
+- **Response shape validation** -- Every field in
+  the public API is asserted: paged search envelope,
+  search item metadata, recall depth fields, skill
+  and agent CRUD responses, MCP prompts protocol,
+  resource templates, and completions.
+
+- **Rust unit tests** -- 21+ new tests for KWIC
+  snippet extraction, matched-terms ordering,
+  paged result pagination, and section matching.
+
+---
+
+## Breaking Changes
+
+- **Search result shape changed** -- Results are
+  now wrapped in a `PagedSearchResult` envelope.
+  Code that expected a flat array of results must
+  read `result.items` instead. Each item's
+  `matches` array is replaced by `snippet`,
+  `matched_terms`, `heading`, `type`, and `docids`.
+
+- **Codec exports renamed** -- `decodeScoredResults`
+  is now `decodePagedSearchResult`. `ScoredResult`
+  is now `SectionMatch`.
+
+---
+
+## Internal
+
+- Rust engine: reference tracking wired into
+  `db_add_section`, `db_update_section`, and
+  `db_remove_section`. `worker_search` parses
+  4-part format (query/mode/limit/offset).
+  `worker_recall` WASM export added.
+- `content-creator` renamed to `content-composer`
+  across all code and docs.
+- WASM binaries rebuilt (~8KB and ~35KB larger).
+
+---
+
+## Upgrade
+
+```sh
+# Node.js
+npm install -g @dikolab/kbdb@0.2.0
+
+# Deno
+deno install -Agf jsr:@dikolab/kbdb/cli
+```
+
+Review the **Breaking Changes** section above if
+you use the library API or parse search results
+programmatically.

package/docs/release-notes/README.md

@@ -0,0 +1,9 @@
+# Release Notes
+
+## Releases
+
+- [0.2.0](0.2.0.md) -- Recall, paginated search, reference graph
+- [0.1.3](0.1.3.md) -- Include WASM binaries in JSR package
+- [0.1.2](0.1.2.md) -- Fix WASM loading from JSR
+- [0.1.1](0.1.1.md) -- Deno type fix, build pipeline cleanup
+- [0.1.0](0.1.0.md) -- Initial release

package/package.json

@@ -1,6 +1,6 @@
 {
    "name": "@dikolab/kbdb",
-   "version": "0.1.5",
+   "version": "0.2.0",
    "description": "CLI tool and MCP server for a file-based knowledge base database",
    "publishConfig": {
       "access": "public"
@@ -24,7 +24,13 @@
       "mod.ts",
       "src/**/*",
       "!src/**/*.spec.ts",
-      "LICENSE"
+      "LICENSE",
+      "README.md",
+      "docs/details/**/*.md",
+      "docs/goals/**",
+      "docs/modules/**/*.md",
+      "docs/release-notes/**/*.md",
+      "docs/overview.md"
    ],
    "type": "module",
    "main": "./dist/mod.cjs",

package/src/shared/cli/functions/dispatch-command.function.ts

@@ -1,6 +1,6 @@
 import type { CliOptions } from '../typings/cli-options.interface.ts';
 import type { CommandResult } from '../typings/command-result.model.ts';
-import type { SearchDisplay } from '../typings/search-display.model.ts';
+import type { PagedSearchDisplay } from '../typings/search-display.model.ts';
 import type { LearnIO } from './run-learn.function.ts';
 import type { SearchClient } from './run-search.function.ts';
 import type { LearnClient } from './run-learn.function.ts';
@@ -323,10 +323,10 @@ async function handleDbMigrate(
  */
 async function hydrateSearchPreviews(
    client: ContentClient,
-   results: SearchDisplay[],
+   paged: PagedSearchDisplay,
 ): Promise<void> {
    await Promise.all(
-      results.map(async (result) => {
+      paged.items.map(async (result) => {
          try {
             const contentResult = await client.content([result.kbid]);
             result.preview = contentResult.data;

package/src/shared/cli/functions/format-output.function.ts

@@ -1,63 +1,70 @@
 import type { OutputFormat } from '../typings/cli-options.interface.ts';
-import type { SearchDisplay } from '../typings/search-display.model.ts';
+import type { PagedSearchDisplay } from '../typings/search-display.model.ts';
 
 /**
- * Serialises an array of search results into the requested output
+ * Serialises a paged search result into the requested output
  * format string.
  *
- * - `'json'` -- pretty-printed JSON array
- * - `'text'` -- numbered human-readable list
- * - `'mcp'` -- JSON envelope compatible with MCP protocol consumers
+ * - `'json'` -- pretty-printed JSON envelope
+ * - `'text'` -- numbered human-readable list with paging footer
+ * - `'mcp'` -- JSON envelope compatible with MCP protocol
  *
- * @param results - ranked search results to format
+ * @param paged - paginated search results to format
  * @param format - target output format
  * @returns formatted string ready for stdout
  */
 export function formatOutput(
-   results: SearchDisplay[],
+   paged: PagedSearchDisplay,
    format: OutputFormat,
 ): string {
    switch (format) {
       case 'json':
-         return JSON.stringify(results, null, 2);
+         return JSON.stringify(paged, null, 2);
       case 'text':
-         return formatText(results);
+         return formatText(paged);
       case 'mcp':
-         return formatMcp(results);
+         return formatMcp(paged);
    }
 }
 
-/**
- * Renders results as a numbered human-readable list.
- *
- * @param results - search results to render
- */
-function formatText(results: SearchDisplay[]): string {
-   if (results.length === 0) return 'No results found.';
-   return results
-      .map(
-         (r, i) =>
-            `${String(i + 1)}. [${r.kbid}] ` +
-            `(score: ${r.score.toFixed(4)})\n` +
-            `   ${r.preview}\n` +
-            `   matches: ${r.matches.map((m) => m.term).join(', ')}`,
-      )
+function formatText(paged: PagedSearchDisplay): string {
+   if (paged.items.length === 0) return 'No results found.';
+   const start = paged.offset + 1;
+   const end = paged.offset + paged.items.length;
+
+   const lines = paged.items
+      .map((r, i) => {
+         const num = paged.offset + i + 1;
+         const heading = r.heading ?? '(untitled)';
+         const terms = r.matched_terms.join(', ');
+         return (
+            `${String(num)}. [${r.kbid}] ${heading}` +
+            ` (score: ${r.score.toFixed(4)})\n` +
+            `   ${r.snippet || r.preview || '(no snippet)'}\n` +
+            `   type: ${r.type}` +
+            `  terms: ${terms}`
+         );
+      })
       .join('\n\n');
+
+   const footer =
+      `\nShowing ${String(start)}-${String(end)}` +
+      ` of ${String(paged.total)} results`;
+   const hint = paged.has_more
+      ? `\nUse --offset ${String(end)} to see more`
+      : '';
+
+   return lines + footer + hint;
 }
 
-/**
- * Renders results as a JSON envelope for MCP protocol consumers.
- *
- * @param results - search results to render
- */
-function formatMcp(results: SearchDisplay[]): string {
+function formatMcp(paged: PagedSearchDisplay): string {
    return JSON.stringify({
       type: 'text',
-      text: results
+      text: paged.items
          .map(
             (r) =>
-               `[${r.kbid}] score=${r.score.toFixed(4)}` +
-               ` — ${r.preview}`,
+               `[${r.kbid}] ${r.heading ?? '(untitled)'}` +
+               ` — ${r.snippet || r.preview}`,
          )
          .join('\n'),
    });

package/src/shared/cli/functions/parse-args.function.ts

@@ -87,6 +87,13 @@ export function parseArgs(argv: string[]): CliOptions {
             options.limit =
                parseInt(argv[++i] ?? '', 10) || DEFAULT_LIMIT;
             break;
+         case '--offset':
+         case '-o':
+            options.offset = Math.max(
+               0,
+               parseInt(argv[++i] ?? '', 10) || 0,
+            );
+            break;
          case '--title':
             options.title = argv[++i];
             break;

package/src/shared/cli/functions/run-search.function.ts

@@ -1,5 +1,5 @@
 import type { CliOptions } from '../typings/cli-options.interface.ts';
-import type { SearchDisplay } from '../typings/search-display.model.ts';
+import type { PagedSearchDisplay } from '../typings/search-display.model.ts';
 
 /**
  * Minimal client interface required by `runSearch`.
@@ -9,54 +9,80 @@ import type { SearchDisplay } from '../typings/search-display.model.ts';
  */
 export interface SearchClient {
    /**
-    * Searches the knowledge base and returns ranked results.
+    * Searches the knowledge base and returns paginated results.
     *
-    * @param params - query, mode, and result limit
+    * @param params - query, mode, limit, and offset
     */
    search(params: {
       query: string;
       mode?: string;
       limit?: number;
-   }): Promise<
-      {
-         kbid: string;
-         score: number;
-         readonly matches: readonly {
-            readonly term: string;
-            readonly frequency: number;
-         }[];
-      }[]
-   >;
+      offset?: number;
+   }): Promise<{
+      readonly items: readonly {
+         readonly kbid: string;
+         readonly heading: string | null;
+         readonly type: string;
+         readonly docids: readonly string[];
+         readonly score: number;
+         readonly snippet: string;
+         readonly matched_terms: readonly string[];
+      }[];
+      readonly total: number;
+      readonly offset: number;
+      readonly limit: number;
+      readonly has_more: boolean;
+   }>;
 }
 
 /**
- * Executes a search query via the provided client and maps the raw
- * daemon results to the `SearchDisplay` format used by the CLI.
+ * Executes a search query via the provided client and maps the
+ * daemon results to the `PagedSearchDisplay` format used by the
+ * CLI.
  *
- * Returns an empty array without calling the client when `options.args`
- * contains no query terms.
+ * Returns an empty paged result without calling the client when
+ * `options.args` contains no query terms.
  *
  * @param client - search client to delegate to
- * @param options - parsed CLI options supplying query args and search
- *   parameters
+ * @param options - parsed CLI options supplying query args and
+ *   search parameters
  */
 export async function runSearch(
    client: SearchClient,
    options: CliOptions,
-): Promise<SearchDisplay[]> {
+): Promise<PagedSearchDisplay> {
    const query = options.args.join(' ').trim();
-   if (!query) return [];
+   if (!query) {
+      return {
+         items: [],
+         total: 0,
+         offset: 0,
+         limit: options.limit,
+         has_more: false,
+      };
+   }
 
    const results = await client.search({
       query,
       mode: options.mode,
       limit: options.limit,
+      offset: options.offset,
    });
 
-   return results.map((r) => ({
-      kbid: r.kbid,
-      score: r.score,
-      preview: '',
-      matches: r.matches,
-   }));
+   return {
+      items: results.items.map((r) => ({
+         kbid: r.kbid,
+         heading: r.heading,
+         type: r.type,
+         docids: r.docids,
+         score: r.score,
+         snippet: r.snippet,
+         matched_terms: r.matched_terms,
+         preview: '',
+      })),
+      total: results.total,
+      offset: results.offset,
+      limit: results.limit,
+      has_more: results.has_more,
+   };
 }

package/src/shared/cli/index.ts

@@ -23,7 +23,10 @@ export type {
    SearchMode,
 } from './typings/cli-options.interface.ts';
 export type { LearnResult } from './typings/learn-result.model.ts';
-export type { SearchDisplay } from './typings/search-display.model.ts';
+export type {
+   SearchDisplay,
+   PagedSearchDisplay,
+} from './typings/search-display.model.ts';
 export type { SearchClient } from './functions/run-search.function.ts';
 export type { LearnClient } from './functions/run-learn.function.ts';
 export type {

package/src/shared/cli/typings/cli-options.interface.ts

@@ -31,6 +31,8 @@ export interface CliOptions {
    mode: SearchMode;
    /** Maximum number of results to return. Defaults to 20. */
    limit: number;
+   /** Number of results to skip for pagination. */
+   offset?: number;
    /** Section title for learn command. */
    title?: string;
    /** Section description for learn command. */

package/src/shared/cli/typings/search-display.model.ts

@@ -2,16 +2,35 @@
 export interface SearchDisplay {
    /** Knowledge-base identifier of the matched section. */
    kbid: string;
+   /** Section heading, or null when untitled. */
+   heading: string | null;
+   /** MIME type of the section content. */
+   type: string;
+   /** Document IDs this section belongs to. */
+   docids: readonly string[];
    /** Relevance score assigned by the ranking engine. */
    score: number;
+   /** KWIC snippet from the WASM engine (~150 chars). */
+   snippet: string;
+   /** Query terms that matched within this section. */
+   matched_terms: readonly string[];
    /**
-    * Short excerpt from the section content.
-    * Empty string when no preview text is available.
+    * CLI-side truncated content preview for terminal display.
+    * Populated when `--content` / `-c` is used.
     */
    preview: string;
-   /** Terms that matched within this section and their frequencies. */
-   readonly matches: readonly {
-      readonly term: string;
-      readonly frequency: number;
-   }[];
+}
+
+/** Paginated search display envelope for CLI output. */
+export interface PagedSearchDisplay {
+   /** Search results for the current page. */
+   items: SearchDisplay[];
+   /** Total number of matches across all pages. */
+   total: number;
+   /** Number of results skipped. */
+   offset: number;
+   /** Maximum results per page. */
+   limit: number;
+   /** Whether more results exist beyond this page. */
+   has_more: boolean;
 }