@dikolab/kbdb 0.2.2 → 0.3.1

package/src/shared/cli/functions/collect-files.function.ts

@@ -0,0 +1,55 @@
+import { readdirSync, statSync } from 'node:fs';
+import { basename, extname, join } from 'node:path';
+import { SUPPORTED_EXTENSIONS } from '../constants/defaults.constant.ts';
+
+/**
+ * Resolves a single target path to a flat list of supported files.
+ *
+ * When `target` is a file, returns it wrapped in an array if its
+ * extension is in `SUPPORTED_EXTENSIONS` and it is not a dotfile;
+ * otherwise returns `[]`. When `target` is a directory, recursively
+ * collects all matching files, skipping dot-entries at every level.
+ * Returns `[]` for non-existent or inaccessible paths.
+ *
+ * @param target - file or directory path to resolve
+ * @returns flat array of absolute or relative paths to supported files
+ */
+export function collectFiles(target: string): string[] {
+   try {
+      const stat = statSync(target);
+      if (stat.isFile()) {
+         return isSupportedFile(target) ? [target] : [];
+      }
+      if (stat.isDirectory()) {
+         return scanDirectory(target);
+      }
+   } catch {
+      // Non-existent or inaccessible path -- skip
+   }
+   return [];
+}
+
+function scanDirectory(dir: string): string[] {
+   const files: string[] = [];
+   for (const entry of readdirSync(dir)) {
+      if (entry.startsWith('.')) continue;
+      const full = join(dir, entry);
+      try {
+         const stat = statSync(full);
+         if (stat.isDirectory()) {
+            files.push(...scanDirectory(full));
+         } else if (isSupportedFile(full)) {
+            files.push(full);
+         }
+      } catch {
+         // Skip inaccessible entries
+      }
+   }
+   return files;
+}
+
+function isSupportedFile(path: string): boolean {
+   if (basename(path).startsWith('.')) return false;
+   const ext = extname(path).toLowerCase();
+   return (SUPPORTED_EXTENSIONS as readonly string[]).includes(ext);
+}

package/src/shared/cli/functions/compute-section-kbid.function.ts

@@ -0,0 +1,49 @@
+import { createHash } from 'node:crypto';
+
+/** RFC 4648 base32 alphabet (upper-case, no padding). */
+const BASE32_ALPHABET = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567';
+
+function base32Encode(bytes: Uint8Array): string {
+   let result = '';
+   let bits = 0;
+   let value = 0;
+   for (const byte of bytes) {
+      value = (value << 8) | byte;
+      bits += 8;
+      while (bits >= 5) {
+         result += BASE32_ALPHABET[(value >>> (bits - 5)) & 0x1f];
+         bits -= 5;
+      }
+   }
+   if (bits > 0) {
+      result += BASE32_ALPHABET[(value << (5 - bits)) & 0x1f];
+   }
+   return result.toLowerCase();
+}
+
+/**
+ * Computes a 26-character kbid for a section using the same algorithm
+ * as the Rust engine's `hash_section`.
+ *
+ * Algorithm:
+ * 1. Build input: `"section <mime> <byteLen>\0<content>"`
+ * 2. SHA-256 hash the input
+ * 3. Take the first 16 bytes
+ * 4. Base32 encode (RFC 4648, no padding), lowercased → 26 chars
+ *
+ * @param mimeType - MIME type string (e.g. `"text/markdown"`)
+ * @param content - UTF-8 section content string
+ * @returns 26-character lowercase base32 kbid
+ */
+export function computeSectionKbid(
+   mimeType: string,
+   content: string,
+): string {
+   const contentBytes = Buffer.from(content, 'utf-8');
+   const header = `section ${mimeType} ${contentBytes.length.toString()}\0`;
+   const headerBytes = Buffer.from(header, 'utf-8');
+   const input = Buffer.concat([headerBytes, contentBytes]);
+   const hash = createHash('sha256').update(input).digest();
+   const truncated = new Uint8Array(hash.buffer, 0, 16);
+   return base32Encode(truncated);
+}

package/src/shared/cli/functions/dispatch-command.function.ts

@@ -19,6 +19,8 @@ import type { AgentCreateClient } from './run-agent-create.function.ts';
 import type { AgentListClient } from './run-agent-list.function.ts';
 import type { AgentGetClient } from './run-agent-get.function.ts';
 import type { AgentDeleteClient } from './run-agent-delete.function.ts';
+import type { RecallClient } from './run-recall.function.ts';
+import type { ExportClient } from './run-export.function.ts';
 import { runSearch } from './run-search.function.ts';
 import { runLearn } from './run-learn.function.ts';
 import { runUnlearn } from './run-unlearn.function.ts';
@@ -38,6 +40,8 @@ import { runAgentCreate } from './run-agent-create.function.ts';
 import { runAgentList } from './run-agent-list.function.ts';
 import { runAgentGet } from './run-agent-get.function.ts';
 import { runAgentDelete } from './run-agent-delete.function.ts';
+import { runRecall } from './run-recall.function.ts';
+import { runExport } from './run-export.function.ts';
 import { formatOutput } from './format-output.function.ts';
 import { errorToMessage } from './error-to-message.function.ts';
 
@@ -64,7 +68,9 @@ export type DispatchClient = SearchClient &
    AgentCreateClient &
    AgentListClient &
    AgentGetClient &
-   AgentDeleteClient;
+   AgentDeleteClient &
+   RecallClient &
+   ExportClient;
 
 /**
  * Routes a parsed CLI command to the appropriate handler and
@@ -86,6 +92,8 @@ export async function dispatchCommand(
          return handleUnlearn(client, options);
       case 'search':
          return handleSearch(client, options);
+      case 'recall':
+         return handleRecall(client, options);
       case 'content':
          return handleContent(client, options);
       case 'check':
@@ -96,6 +104,8 @@ export async function dispatchCommand(
          return handleRebuild(client, options);
       case 'status':
          return handleStatus(client, options);
+      case 'export':
+         return handleExport(client, options);
       case 'db-init':
          return handleDbInit(options);
       case 'db-migrate':
@@ -198,6 +208,48 @@ async function handleSearch(
    }
 }
 
+async function handleRecall(
+   client: DispatchClient,
+   options: CliOptions,
+): Promise<CommandResult> {
+   try {
+      const result = await runRecall(client, options);
+      if ('error' in result) {
+         return {
+            error: `error: ${result.error}`,
+            exitCode: 1,
+         };
+      }
+      return {
+         output: JSON.stringify(result, null, 2),
+         exitCode: 0,
+      };
+   } catch (err) {
+      return {
+         error: `recall failed: ${errorToMessage(err)}`,
+         exitCode: 1,
+      };
+   }
+}
+
+async function handleExport(
+   client: DispatchClient,
+   options: CliOptions,
+): Promise<CommandResult> {
+   try {
+      const result = await runExport(client, options);
+      return {
+         output: JSON.stringify(result, null, 2),
+         exitCode: 0,
+      };
+   } catch (err) {
+      return {
+         error: `export failed: ${errorToMessage(err)}`,
+         exitCode: 1,
+      };
+   }
+}
+
 async function handleContent(
    client: DispatchClient,
    options: CliOptions,

package/src/shared/cli/functions/format-mcp-output.function.ts

@@ -0,0 +1,41 @@
+import type { PagedSearchDisplay } from '../typings/search-display.model.ts';
+
+/**
+ * Formats a paged search result as a JSON string compatible
+ * with the MCP (Model Context Protocol) `text` content block.
+ *
+ * @param paged - paginated search results to serialise
+ * @returns MCP-compatible JSON string ready for stdout
+ */
+export function formatMcpOutput(paged: PagedSearchDisplay): string {
+   const mcpItems = paged.items.map((r) => ({
+      kbid: r.kbid,
+      heading: r.heading,
+      type: r.type,
+      docids: r.docids,
+      snippet: r.snippet,
+      matched_terms: r.matched_terms,
+      matched_fields: r.matched_fields,
+      created_at: r.created_at,
+   }));
+   return JSON.stringify(
+      {
+         type: 'text',
+         text: paged.items
+            .map(
+               (r) =>
+                  `[${r.kbid}] ${r.heading ?? '(untitled)'}` +
+                  ` — ${r.snippet || r.preview}`,
+            )
+            .join('\n'),
+         items: mcpItems,
+         total: paged.total,
+         offset: paged.offset,
+         limit: paged.limit,
+         has_more: paged.has_more,
+         relaxed: paged.relaxed,
+      },
+      null,
+      2,
+   );
+}

package/src/shared/cli/functions/format-output.function.ts

@@ -37,12 +37,16 @@ function formatText(paged: PagedSearchDisplay): string {
          const num = paged.offset + i + 1;
          const heading = r.heading ?? '(untitled)';
          const terms = r.matched_terms.join(', ');
+         const fields = r.matched_fields.join(', ');
+         const confidencePct = (r.confidence * 100).toFixed(0);
          return (
             `${String(num)}. [${r.kbid}] ${heading}` +
-            ` (score: ${r.score.toFixed(4)})\n` +
+            ` (score: ${r.score.toFixed(4)},` +
+            ` confidence: ${confidencePct}%)\n` +
             `   ${r.snippet || r.preview || '(no snippet)'}\n` +
             `   type: ${r.type}` +
-            `  terms: ${terms}`
+            `  terms: ${terms}` +
+            (fields ? `  fields: ${fields}` : '')
          );
       })
       .join('\n\n');
@@ -58,14 +62,34 @@ function formatText(paged: PagedSearchDisplay): string {
 }
 
 function formatMcp(paged: PagedSearchDisplay): string {
-   return JSON.stringify({
-      type: 'text',
-      text: paged.items
-         .map(
-            (r) =>
-               `[${r.kbid}] ${r.heading ?? '(untitled)'}` +
-               ` — ${r.snippet || r.preview}`,
-         )
-         .join('\n'),
-   });
+   const mcpItems = paged.items.map((r) => ({
+      kbid: r.kbid,
+      heading: r.heading,
+      type: r.type,
+      docids: r.docids,
+      snippet: r.snippet,
+      matched_terms: r.matched_terms,
+      matched_fields: r.matched_fields,
+      created_at: r.created_at,
+   }));
+   return JSON.stringify(
+      {
+         type: 'text',
+         text: paged.items
+            .map(
+               (r) =>
+                  `[${r.kbid}] ${r.heading ?? '(untitled)'}` +
+                  ` — ${r.snippet || r.preview}`,
+            )
+            .join('\n'),
+         items: mcpItems,
+         total: paged.total,
+         offset: paged.offset,
+         limit: paged.limit,
+         has_more: paged.has_more,
+         relaxed: paged.relaxed,
+      },
+      null,
+      2,
+   );
 }

package/src/shared/cli/functions/format-text-output.function.ts

@@ -0,0 +1,42 @@
+import type { PagedSearchDisplay } from '../typings/search-display.model.ts';
+
+/**
+ * Formats a paged search result as a numbered human-readable
+ * text list with a paging footer and optional "more" hint.
+ *
+ * @param paged - paginated search results to render
+ * @returns formatted string ready for stdout
+ */
+export function formatTextOutput(paged: PagedSearchDisplay): string {
+   if (paged.items.length === 0) return 'No results found.';
+   const start = paged.offset + 1;
+   const end = paged.offset + paged.items.length;
+
+   const lines = paged.items
+      .map((r, i) => {
+         const num = paged.offset + i + 1;
+         const heading = r.heading ?? '(untitled)';
+         const terms = r.matched_terms.join(', ');
+         const fields = r.matched_fields.join(', ');
+         const confidencePct = (r.confidence * 100).toFixed(0);
+         return (
+            `${String(num)}. [${r.kbid}] ${heading}` +
+            ` (score: ${r.score.toFixed(4)},` +
+            ` confidence: ${confidencePct}%)\n` +
+            `   ${r.snippet || r.preview || '(no snippet)'}\n` +
+            `   type: ${r.type}` +
+            `  terms: ${terms}` +
+            (fields ? `  fields: ${fields}` : '')
+         );
+      })
+      .join('\n\n');
+
+   const footer =
+      `\nShowing ${String(start)}-${String(end)}` +
+      ` of ${String(paged.total)} results`;
+   const hint = paged.has_more
+      ? `\nUse --offset ${String(end)} to see more`
+      : '';
+
+   return lines + footer + hint;
+}

package/src/shared/cli/functions/generate-group-help.function.ts

@@ -0,0 +1,41 @@
+/** One-line description for each group command. */
+const GROUP_DESCRIPTIONS: Readonly<Record<string, string>> = {
+   db: 'Database management',
+   skill: 'Skill management',
+   agent: 'Agent management',
+};
+
+/**
+ * Generates the level-2 help text for a group command.
+ *
+ * Lists all subcommands with their descriptions and
+ * includes a footer pointing to per-subcommand help.
+ *
+ * @param group - the group command name (e.g. `'db'`, `'skill'`)
+ * @param subs - ordered list of `[name, description]` pairs
+ * @returns formatted help string for the group command
+ */
+export function generateGroupHelp(
+   group: string,
+   subs: readonly (readonly [string, string])[],
+): string {
+   const subLines = subs
+      .map(([name, desc]) => `   ${name.padEnd(20)}${desc}`)
+      .join('\n');
+   const desc = GROUP_DESCRIPTIONS[group] ?? group;
+   return (
+      `kbdb ${group} - ${desc}\n` +
+      `\n` +
+      `Usage: kbdb ${group} <subcommand> [options]\n` +
+      `\n` +
+      `Subcommands:\n` +
+      `${subLines}\n` +
+      `\n` +
+      `Global Options:\n` +
+      `   --db <path>         Path to .kbdb directory\n` +
+      `   -h, --help          Show this help message\n` +
+      `\n` +
+      `Run "kbdb ${group} <subcommand> --help"` +
+      ` for details.\n`
+   );
+}

package/src/shared/cli/functions/generate-help.function.ts

@@ -8,11 +8,13 @@ const KNOWN_COMMANDS = new Set([
    'learn',
    'unlearn',
    'search',
+   'recall',
    'content',
    'check',
    'gc',
    'rebuild',
    'status',
+   'export',
    'mcp',
    'db',
    'skill',
@@ -125,6 +127,8 @@ function generateGlobalHelp(): string {
       ` files/stdin\n` +
       `   unlearn             Remove sections by kbid\n` +
       `   search              Query the knowledge base\n` +
+      `   recall              Retrieve section context` +
+      ` by kbid\n` +
       `   content             Compose rendered content` +
       ` from ids\n` +
       `\n` +
@@ -142,6 +146,8 @@ function generateGlobalHelp(): string {
       `   rebuild             Reconstruct all index` +
       ` files\n` +
       `   status              Show database status\n` +
+      `   export              Snapshot database for` +
+      ` backup\n` +
       `\n` +
       `Skills:\n` +
       `   skill learn         Create a new skill` +
@@ -238,14 +244,20 @@ const LEAF_HELP: Readonly<Record<string, string>> = {
       `   --description <d>   Section description\n` +
       `   --docid <docid>     Document ID to group` +
       ` under\n` +
+      `   --tags <t,...>      Comma-separated tags\n` +
+      `   --replace           Supersede existing section` +
+      ` by source key\n` +
       `   --db <path>         Path to .kbdb directory\n` +
       `\n` +
       `Output:\n` +
-      `   JSON array of { kbid, path, size, skipped }\n` +
+      `   JSON array of { kbid, path, size, skipped,\n` +
+      `                   superseded, warnings,` +
+      ` near_duplicates }\n` +
       `\n` +
       `Examples:\n` +
       `   kbdb learn README.md\n` +
       `   kbdb learn ./docs/\n` +
+      `   kbdb learn --tags auth,api --replace doc.md\n` +
       `   cat notes.txt | kbdb learn -\n`,
 
    unlearn:
@@ -294,6 +306,28 @@ const LEAF_HELP: Readonly<Record<string, string>> = {
       `   kbdb search --limit 5 --format text query\n` +
       `   kbdb search --mode documents --content term\n`,
 
+   recall:
+      `kbdb recall - Retrieve section context by kbid\n` +
+      `\n` +
+      `Usage: kbdb recall [options] <kbid>...\n` +
+      `\n` +
+      `Arguments:\n` +
+      `   <kbid>              26-character knowledge` +
+      ` base identifier\n` +
+      `\n` +
+      `Options:\n` +
+      `   --depth <n>, -d     Expansion depth (0-3,` +
+      ` default: 0)\n` +
+      `   --tags <t,...>      Filter by tags\n` +
+      `   --db <path>         Path to .kbdb directory\n` +
+      `\n` +
+      `Output:\n` +
+      `   JSON RecallResult or RecallResult[]\n` +
+      `\n` +
+      `Examples:\n` +
+      `   kbdb recall abcdef1234567890abcdef1234\n` +
+      `   kbdb recall kbid1 --depth 2\n`,
+
    content:
       `kbdb content - Compose rendered content from ids\n` +
       `\n` +
@@ -372,6 +406,27 @@ const LEAF_HELP: Readonly<Record<string, string>> = {
       `   kbdb status\n` +
       `   kbdb status --db /path/to/project\n`,
 
+   export:
+      `kbdb export - Snapshot database for backup\n` +
+      `\n` +
+      `Usage: kbdb export [options] [<path>]\n` +
+      `\n` +
+      `Arguments:\n` +
+      `   <path>              Output directory path\n` +
+      `                       (default: ./kbdb-export/)\n` +
+      `\n` +
+      `Options:\n` +
+      `   --db <path>         Path to .kbdb directory\n` +
+      `\n` +
+      `Output:\n` +
+      `   JSON { path, sections_exported,` +
+      ` documents_exported }\n` +
+      `\n` +
+      `Examples:\n` +
+      `   kbdb export\n` +
+      `   kbdb export ./my-backup\n` +
+      `   kbdb export --db /path/to/project\n`,
+
    mcp:
       `kbdb mcp - Start MCP server mode (stdio)\n` +
       `\n` +

package/src/shared/cli/functions/handle-data-command.function.ts

@@ -0,0 +1,67 @@
+import type { CliOptions } from '../typings/cli-options.interface.ts';
+import type { CommandResult } from '../typings/command-result.model.ts';
+import type { CliDeps } from '../typings/cli-deps.interface.ts';
+import type { DisconnectableClient } from '../typings/disconnectable-client.interface.ts';
+import { errorToMessage } from './error-to-message.function.ts';
+import { versionGate } from './version-gate.function.ts';
+import { dispatchCommand } from './dispatch-command.function.ts';
+
+/**
+ * Handles data commands (search, learn, recall, etc.) by finding or
+ * creating the database, running the version gate, and dispatching to
+ * the appropriate command handler.
+ *
+ * When the database is not found in non-TTY mode, returns an error.
+ * In TTY mode, prompts the user to create a new database.
+ *
+ * @param options - parsed CLI options with command and args
+ * @param deps - injectable CLI dependencies
+ * @returns a `CommandResult` from the dispatched command
+ */
+export async function handleDataCommand(
+   options: CliOptions,
+   deps: CliDeps,
+): Promise<CommandResult> {
+   let dbDir = deps.findDb(options.dbPath);
+
+   if (!dbDir) {
+      const basePath = options.dbPath ?? process.cwd();
+      if (!deps.isStdinTty()) {
+         return {
+            error:
+               `error: no .kbdb database found at ${basePath}` +
+               ' -- run `kbdb --init` to create one',
+            exitCode: 1,
+         };
+      }
+      const answer = await deps.promptUser(
+         `No .kbdb database found. Create one in ${basePath}? [y/N] `,
+      );
+      if (answer.toLowerCase() !== 'y') {
+         return { error: 'aborted: no database created', exitCode: 1 };
+      }
+      const initResult = deps.runInit(options);
+      if (!initResult.ok) {
+         return {
+            error: `error: ${initResult.errorMessage ?? 'init failed'}`,
+            exitCode: 1,
+         };
+      }
+      dbDir = initResult.targetPath;
+   }
+
+   let client: DisconnectableClient;
+   try {
+      client = await deps.createWorkerClient({ contextPath: dbDir });
+   } catch (err) {
+      return { error: `error: ${errorToMessage(err)}`, exitCode: 1 };
+   }
+
+   try {
+      const gateResult = await versionGate(client, dbDir);
+      if (gateResult) return gateResult;
+      return await dispatchCommand(client, options);
+   } finally {
+      client.disconnect();
+   }
+}

package/src/shared/cli/functions/handle-mcp.function.ts

@@ -0,0 +1,56 @@
+import type { CliOptions } from '../typings/cli-options.interface.ts';
+import type { CommandResult } from '../typings/command-result.model.ts';
+import type { CliDeps } from '../typings/cli-deps.interface.ts';
+import type { DisconnectableClient } from '../typings/disconnectable-client.interface.ts';
+import { errorToMessage } from './error-to-message.function.ts';
+import { versionGate } from './version-gate.function.ts';
+
+/**
+ * Handles the `mcp` command sub-flow.
+ *
+ * Resolves the database directory, creates a worker client, runs
+ * the version gate, and starts the MCP server. Disconnects the
+ * worker client when finished.
+ *
+ * @param options - parsed CLI options (uses `dbPath` and env fallback)
+ * @param deps - injectable CLI dependencies
+ * @returns a `CommandResult` with `exitCode: 0` on success
+ */
+export async function handleMcp(
+   options: CliOptions,
+   deps: CliDeps,
+): Promise<CommandResult> {
+   const rawDbPath = options.dbPath ?? process.env['KBDB_DIR'];
+   if (!rawDbPath) {
+      return {
+         error: 'error: --db or KBDB_DIR required in MCP mode',
+         exitCode: 1,
+      };
+   }
+   const dbDir = deps.findDb(rawDbPath);
+   if (!dbDir) {
+      return {
+         error: `error: no .kbdb database found at ${rawDbPath}`,
+         exitCode: 1,
+      };
+   }
+   let client: DisconnectableClient;
+   try {
+      client = await deps.createWorkerClient({ contextPath: dbDir });
+   } catch (err) {
+      return { error: `error: ${errorToMessage(err)}`, exitCode: 1 };
+   }
+   try {
+      const gateResult = await versionGate(client, dbDir);
+      if (gateResult) return gateResult;
+      if (deps.startMcpServer) await deps.startMcpServer(dbDir);
+      return { exitCode: 0 };
+   } catch (err) {
+      return {
+         error: `error: mcp failed: ${errorToMessage(err)}`,
+         exitCode: 1,
+      };
+   } finally {
+      client.disconnect();
+   }
+}

package/src/shared/cli/functions/handle-migrate.function.ts

@@ -0,0 +1,64 @@
+import type { CliOptions } from '../typings/cli-options.interface.ts';
+import type { CommandResult } from '../typings/command-result.model.ts';
+import type { CliDeps } from '../typings/cli-deps.interface.ts';
+import type { DisconnectableClient } from '../typings/disconnectable-client.interface.ts';
+import { errorToMessage } from './error-to-message.function.ts';
+
+/**
+ * Handles the `migrate` / `db-migrate` command sub-flow.
+ *
+ * Resolves the database directory, creates a worker client,
+ * checks the version, and runs migrations when needed. Disconnects
+ * the worker client when finished.
+ *
+ * @param options - parsed CLI options (uses `dbPath`)
+ * @param deps - injectable CLI dependencies
+ * @returns a `CommandResult` with migration output or an error
+ */
+export async function handleMigrate(
+   options: CliOptions,
+   deps: CliDeps,
+): Promise<CommandResult> {
+   const dbDir = deps.findDb(options.dbPath);
+   if (!dbDir) {
+      return {
+         error: `error: no .kbdb database found at ${options.dbPath ?? process.cwd()}`,
+         exitCode: 1,
+      };
+   }
+   let client: DisconnectableClient;
+   try {
+      client = await deps.createWorkerClient({ contextPath: dbDir });
+   } catch (err) {
+      return { error: `error: ${errorToMessage(err)}`, exitCode: 1 };
+   }
+   try {
+      const versionStatus = await client.checkVersion(dbDir);
+      if (versionStatus.status === 'too_new') {
+         return {
+            error:
+               `error: database version` +
+               ` ${versionStatus.currentVersion.toString()}` +
+               ` is newer than kbdb version` +
+               ` ${versionStatus.requiredVersion.toString()}` +
+               ` -- upgrade kbdb`,
+            exitCode: 1,
+         };
+      }
+      if (versionStatus.status === 'needs_migration') {
+         const migrateResult = await client.migrateDatabase(dbDir);
+         return {
+            output: JSON.stringify(migrateResult, null, 2),
+            exitCode: 0,
+         };
+      }
+      return { output: 'already up to date', exitCode: 0 };
+   } catch (err) {
+      return {
+         error: `error: migrate failed: ${errorToMessage(err)}`,
+         exitCode: 1,
+      };
+   } finally {
+      client.disconnect();
+   }
+}