@dikolab/kbdb 0.2.2 → 0.3.0

package/src/shared/cli/functions/install-default-skills.function.ts

@@ -0,0 +1,296 @@
+import { createHash } from 'node:crypto';
+import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+/**
+ * RFC 4648 base32 alphabet (upper-case, no padding).
+ * Used to encode the SHA-256-derived kbid bytes.
+ */
+const BASE32_ALPHABET = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567';
+
+/**
+ * Encodes a byte array as a lowercase base32 string without padding.
+ *
+ * Mirrors the `data_encoding::BASE32_NOPAD` encoding used in the
+ * Rust engine's `KbId::from_content`.
+ *
+ * @param bytes - raw bytes to encode
+ * @returns lowercase base32-encoded string
+ */
+function base32Encode(bytes: Uint8Array): string {
+   let result = '';
+   let bits = 0;
+   let value = 0;
+   for (const byte of bytes) {
+      value = (value << 8) | byte;
+      bits += 8;
+      while (bits >= 5) {
+         result += BASE32_ALPHABET[(value >>> (bits - 5)) & 0x1f];
+         bits -= 5;
+      }
+   }
+   if (bits > 0) {
+      result += BASE32_ALPHABET[(value << (5 - bits)) & 0x1f];
+   }
+   return result.toLowerCase();
+}
+
+/**
+ * Computes a 26-character kbid for a section using the same algorithm
+ * as the Rust engine's `hash_section`.
+ *
+ * Algorithm:
+ * 1. Build input: `"section <mime> <byteLen>\0<content>"`
+ * 2. SHA-256 hash the input
+ * 3. Take the first 16 bytes
+ * 4. Base32 encode (RFC 4648, no padding), lowercased → 26 chars
+ *
+ * @param mimeType - MIME type string (e.g. `"text/markdown"`)
+ * @param content - UTF-8 section content string
+ * @returns 26-character lowercase base32 kbid
+ */
+function computeSectionKbid(mimeType: string, content: string): string {
+   const contentBytes = Buffer.from(content, 'utf-8');
+   const header = `section ${mimeType} ${contentBytes.length.toString()}\0`;
+   const headerBytes = Buffer.from(header, 'utf-8');
+   const input = Buffer.concat([headerBytes, contentBytes]);
+   const hash = createHash('sha256').update(input).digest();
+   const truncated = new Uint8Array(hash.buffer, 0, 16);
+   return base32Encode(truncated);
+}
+
+/**
+ * Escapes backslashes and double-quotes in a string for TOML inline
+ * string values.
+ *
+ * Mirrors the `escape_toml_str` function in the Rust engine.
+ *
+ * @param s - string to escape
+ * @returns escaped string safe for use inside TOML double-quoted strings
+ */
+function escapeToml(s: string): string {
+   return s.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
+}
+
+/**
+ * Serializes a skill section record to the `.section` file format.
+ *
+ * Format:
+ * ```
+ * kbid = "..."
+ * section_type = "text/markdown"
+ * type_name = "skill"
+ * title = "..."
+ * description = "..."
+ * size = 0
+ * token_count = 0
+ * heading_token_count = 0
+ * body_token_count = 0
+ * code_token_count = 0
+ * created_at = 0
+ * checksum = "..."
+ *
+ * ---
+ * <json content>
+ * ```
+ *
+ * @param kbid - 26-char content-addressed identifier
+ * @param title - skill name used as section title
+ * @param description - human-readable description
+ * @param content - serialised JSON skill definition
+ * @returns file content string
+ */
+function serializeSkillSection(
+   kbid: string,
+   title: string,
+   description: string,
+   content: string,
+): string {
+   const size = Buffer.byteLength(content, 'utf-8');
+   const header = [
+      `kbid = "${escapeToml(kbid)}"`,
+      `section_type = "text/markdown"`,
+      `type_name = "skill"`,
+      `title = "${escapeToml(title)}"`,
+      `description = "${escapeToml(description)}"`,
+      `size = ${size.toString()}`,
+      `token_count = 0`,
+      `heading_token_count = 0`,
+      `body_token_count = 0`,
+      `code_token_count = 0`,
+      `created_at = 0`,
+      `checksum = "${escapeToml(kbid)}"`,
+      '',
+   ].join('\n');
+   return `${header}\n---\n${content}`;
+}
+
+/** Descriptor for a skill to install. */
+interface SkillDescriptor {
+   /** Unique skill name (kebab-case). */
+   readonly name: string;
+   /** Human-readable description. */
+   readonly description: string;
+   /** Template body text. */
+   readonly body: string;
+}
+
+/**
+ * Installs a single skill section file under `sectionsDir`.
+ *
+ * Creates the shard subdirectory if it does not exist. Skips
+ * writing if the section file already exists, preserving any
+ * user modifications.
+ *
+ * @param sectionsDir - absolute path to `.kbdb/sections`
+ * @param skill - skill definition to serialise and store
+ */
+function installSkill(
+   sectionsDir: string,
+   skill: SkillDescriptor,
+): void {
+   const content = JSON.stringify({
+      name: skill.name,
+      description: skill.description,
+      arguments: [],
+      body: skill.body,
+   });
+   const kbid = computeSectionKbid('text/markdown', content);
+   const prefix = kbid.slice(0, 2);
+   const shardDir = join(sectionsDir, prefix);
+   const filePath = join(shardDir, `${kbid}.section`);
+
+   if (existsSync(filePath)) {
+      return;
+   }
+
+   mkdirSync(shardDir, { recursive: true });
+
+   const fileContent = serializeSkillSection(
+      kbid,
+      skill.name,
+      skill.description,
+      content,
+   );
+   writeFileSync(filePath, fileContent, 'utf-8');
+}
+
+/**
+ * Body text for the `query-guidance` default skill.
+ *
+ * Teaches AI agents how to effectively search the knowledge base
+ * using lexical BM25F queries.
+ */
+const QUERY_GUIDANCE_BODY =
+   'This knowledge base uses lexical search (stemmed BM25F), not' +
+   ' semantic/vector search. Follow these guidelines for effective' +
+   ' retrieval:\n\n' +
+   '1. Issue MULTIPLE short keyword queries rather than one long' +
+   ' natural-language question. Each query should target 2-3' +
+   ' specific terms.\n\n' +
+   '2. EXPAND SYNONYMS yourself. "login" will not match "sign-in"' +
+   ' unless both terms appear in the content. Try alternative' +
+   ' phrasings: search for "login", then "sign-in", then' +
+   ' "authentication".\n\n' +
+   '3. USE BOOLEAN AND PHRASE SYNTAX:\n' +
+   '   - Exact phrases: "error handling"\n' +
+   '   - Boolean AND: auth AND token\n' +
+   '   - Boolean OR: cache OR redis\n' +
+   '   - Exclusion: auth -oauth\n\n' +
+   '4. LEAN ON HEADINGS. Headings carry 2x weight in ranking.' +
+   ' Search for terms that are likely in section titles.\n\n' +
+   '5. STEMMING IS AUTOMATIC. "connections" matches "connect",' +
+   ' "running" matches "run". You do not need to try inflected' +
+   ' forms.\n\n' +
+   '6. USE RECALL AFTER SEARCH. After finding relevant sections' +
+   ' with search, use recall with increasing depth (0-3) to' +
+   ' progressively expand context: depth 0 = full content, depth' +
+   ' 1 = parent documents + back-references, depth 2 = siblings' +
+   ' + forward references, depth 3 = referenced sections.\n\n' +
+   '7. CODE IDENTIFIERS ARE SPLIT. camelCase and snake_case names' +
+   ' are decomposed: getUserById becomes searchable as "get",' +
+   ' "user", "by", "id".\n\n' +
+   '8. PAGINATION. Default limit is 20 results. Use offset for' +
+   ' additional pages if total > limit.';
+
+/**
+ * Body text for the `memory-protocol` default skill.
+ *
+ * Teaches AI agents when and what to store in the knowledge base
+ * to keep it accurate and useful.
+ */
+const MEMORY_PROTOCOL_BODY =
+   'Follow these guidelines to keep the knowledge base accurate' +
+   ' and useful:\n\n' +
+   '1. STORE DURABLE FACTS: decisions, architecture choices,' +
+   ' configuration values, corrections to prior knowledge, API' +
+   ' contracts, and resolved incidents. These are the facts that' +
+   ' matter weeks later.\n\n' +
+   '2. DO NOT STORE: transient conversation context, debugging' +
+   ' output, ephemeral task state, raw log dumps, or information' +
+   ' that changes every session. The brain should contain' +
+   ' knowledge, not activity logs.\n\n' +
+   '3. SEARCH BEFORE STORING. Always search the knowledge base' +
+   ' before adding new content. If a section already covers the' +
+   ' topic, use learn with --replace to update it rather than' +
+   ' creating a duplicate.\n\n' +
+   '4. USE --replace FOR CORRECTIONS. When a fact changes (e.g.,' +
+   ' rate limit updated from 100 to 200 req/min), re-learn the' +
+   ' corrected content with --replace so the old version is' +
+   ' superseded. Without --replace, both the old and new versions' +
+   ' coexist with equal ranking.\n\n' +
+   '5. ATTACH TAGS for scoping. Use tags like project name,' +
+   ' domain, or topic (e.g., --tags auth,api,v2) so queries can' +
+   ' be filtered by scope later.\n\n' +
+   '6. PREFER RECENT RESULTS. When search returns multiple' +
+   ' sections covering the same topic, prefer the one with the' +
+   ' most recent created_at timestamp -- it is more likely to' +
+   ' reflect the current state.\n\n' +
+   '7. USE RECALL AFTER SEARCH. After finding relevant sections,' +
+   ' use recall with increasing depth (0-3) to expand context' +
+   ' progressively before making decisions.\n\n' +
+   '8. INCLUDE HEADINGS. Always provide a descriptive title when' +
+   ' storing content. Headings carry 2x ranking weight -- omitting' +
+   ' them significantly reduces searchability.';
+
+/**
+ * Default skills installed on every fresh `kbdb db init`.
+ *
+ * These provide baseline guidance that improves AI agent
+ * effectiveness out of the box. They are regular skill sections
+ * and can be deleted by users.
+ */
+const DEFAULT_SKILLS: readonly SkillDescriptor[] = [
+   {
+      name: 'query-guidance',
+      description:
+         'How to effectively search this knowledge base.' +
+         ' Read this before issuing search queries.',
+      body: QUERY_GUIDANCE_BODY,
+   },
+   {
+      name: 'memory-protocol',
+      description:
+         'Guidelines for what to store in this knowledge base' +
+         ' and when. Read this before using the learn tool.',
+      body: MEMORY_PROTOCOL_BODY,
+   },
+];
+
+/**
+ * Writes the two default skills (`query-guidance` and
+ * `memory-protocol`) as section files directly into the `.kbdb`
+ * directory scaffold.
+ *
+ * Called by `runInit` after the directory scaffold is created.
+ * Does not require a running worker daemon. Skips any skill whose
+ * section file already exists so user modifications are preserved.
+ *
+ * @param dbDir - absolute path to the `.kbdb` directory
+ */
+export function installDefaultSkills(dbDir: string): void {
+   const sectionsDir = join(dbDir, 'sections');
+   for (const skill of DEFAULT_SKILLS) {
+      installSkill(sectionsDir, skill);
+   }
+}

package/src/shared/cli/functions/parse-args.function.ts

@@ -115,6 +115,19 @@ export function parseArgs(argv: string[]): CliOptions {
          case '--skills':
             options.skills = (argv[++i] ?? '').split(',');
             break;
+         case '--tags':
+            options.tags = (argv[++i] ?? '').split(',');
+            break;
+         case '--replace':
+            options.replace = true;
+            break;
+         case '--depth':
+         case '-d':
+            options.depth = Math.min(
+               3,
+               Math.max(0, parseInt(argv[++i] ?? '', 10) || 0),
+            );
+            break;
          case '--content':
          case '-c':
             options.content = true;

package/src/shared/cli/functions/run-check.function.ts

@@ -25,6 +25,17 @@ interface IntegrityError {
    readonly detail: string;
 }
 
+/**
+ * A single entry in the divergent-content map.
+ * Groups kbids under a common source key.
+ */
+interface DivergentContentEntry {
+   /** Source key that has multiple divergent content versions. */
+   readonly sourceKey: string;
+   /** Kbids of sections sharing this source key but with different content. */
+   readonly kbids: string[];
+}
+
 /**
  * Integrity check report returned by the daemon.
  */
@@ -41,6 +52,38 @@ interface IntegrityCheckResult {
    readonly references_checked: number;
    /** Time taken to complete the check, in milliseconds. */
    readonly elapsed_ms: number;
+   /**
+    * Sections that share a source key but have divergent content.
+    * Each entry groups the kbids under a common source key.
+    * Empty when no divergent content is found.
+    */
+   readonly divergent_content: DivergentContentEntry[];
+}
+
+/** Raw daemon response from `integrityCheck`. */
+interface RawIntegrityCheckResult {
+   /** `true` when all checks passed and no errors were found. */
+   readonly ok: boolean;
+   /** All integrity errors found; empty when `ok` is `true`. */
+   readonly errors: IntegrityError[];
+   /** Total number of sections examined. */
+   readonly sections_checked: number;
+   /** Total number of documents examined. */
+   readonly documents_checked: number;
+   /** Total number of marker references checked. */
+   readonly references_checked: number;
+   /** Time taken to complete the check, in milliseconds. */
+   readonly elapsed_ms: number;
+   /**
+    * Encoded divergent-content map.
+    *
+    * Format: `sk1:kbid1,kbid2;sk2:kbid3,kbid4`
+    *
+    * Each semicolon-separated segment is a source key followed by
+    * a colon and a comma-separated list of kbids. Absent when no
+    * divergent content exists.
+    */
+   readonly divergent_content?: string;
 }
 
 /**
@@ -56,23 +99,61 @@ export interface CheckClient {
     * @returns a report of all issues found; `ok` is `true` when
     *   no errors were detected
     */
-   integrityCheck(): Promise<IntegrityCheckResult>;
+   integrityCheck(): Promise<RawIntegrityCheckResult>;
+}
+
+/**
+ * Parses the encoded `divergent_content` string from the daemon
+ * into a structured array.
+ *
+ * Input format: `sk1:kbid1,kbid2;sk2:kbid3,kbid4`
+ *
+ * @param encoded - raw encoded string, or empty / absent
+ * @returns structured entries grouped by source key
+ */
+function parseDivergentContent(
+   encoded: string | undefined,
+): DivergentContentEntry[] {
+   if (!encoded) return [];
+   return encoded
+      .split(';')
+      .filter((segment) => segment.includes(':'))
+      .map((segment) => {
+         const colonIdx = segment.indexOf(':');
+         const sourceKey = segment.slice(0, colonIdx);
+         const kbids = segment
+            .slice(colonIdx + 1)
+            .split(',')
+            .filter((k) => k.length > 0);
+         return { sourceKey, kbids };
+      })
+      .filter((entry) => entry.kbids.length > 0);
 }
 
 /**
  * Runs an integrity check on the database and returns the report.
  *
- * Delegates to `client.integrityCheck()`. The caller is responsible
- * for printing the result to stdout and setting the process exit code
- * based on `result.ok`.
+ * Delegates to `client.integrityCheck()`, then parses the encoded
+ * `divergent_content` field into structured form. The caller is
+ * responsible for printing the result to stdout and setting the
+ * process exit code based on `result.ok`.
  *
  * @param client - check client to delegate to
  * @param _options - parsed CLI options (unused; reserved for future flags)
- * @returns the integrity check report from the daemon
+ * @returns the integrity check report with parsed `divergent_content`
  */
 export async function runCheck(
    client: CheckClient,
    _options: CliOptions,
 ): Promise<IntegrityCheckResult> {
-   return client.integrityCheck();
+   const raw = await client.integrityCheck();
+   return {
+      ok: raw.ok,
+      errors: raw.errors,
+      sections_checked: raw.sections_checked,
+      documents_checked: raw.documents_checked,
+      references_checked: raw.references_checked,
+      elapsed_ms: raw.elapsed_ms,
+      divergent_content: parseDivergentContent(raw.divergent_content),
+   };
 }

package/src/shared/cli/functions/run-export.function.ts

@@ -0,0 +1,40 @@
+import type { CliOptions } from '../typings/cli-options.interface.ts';
+import type { ExportResult } from '../../worker-client/typings/export-result.model.ts';
+
+/**
+ * Minimal client interface required by `runExport`.
+ *
+ * Decouples the CLI function from the concrete `WorkerClient`
+ * implementation, making it straightforward to test with mocks.
+ */
+export interface ExportClient {
+   /**
+    * Exports the database to the given directory path.
+    *
+    * @param params - optional target path; the daemon defaults to
+    *   `./kbdb-export/` when `path` is `undefined`
+    * @returns export report with the resolved path and file counts
+    */
+   export(params: { path?: string }): Promise<ExportResult>;
+}
+
+/**
+ * Exports the knowledge-base database to a target directory via the
+ * provided client.
+ *
+ * The target path is taken from `options.args[0]`. When no argument
+ * is given the daemon resolves its own default (`./kbdb-export/`).
+ *
+ * @param client - export client to delegate to
+ * @param options - parsed CLI options whose first positional
+ *   argument, if present, is the target directory path
+ * @returns the export result containing the resolved path and file
+ *   counts
+ */
+export async function runExport(
+   client: ExportClient,
+   options: CliOptions,
+): Promise<ExportResult> {
+   const path = options.args[0];
+   return client.export({ path });
+}

package/src/shared/cli/functions/run-gc.function.ts

@@ -10,6 +10,12 @@ interface GcResult {
    readonly removed_bytes: number;
    /** Time taken to complete garbage collection, in milliseconds. */
    readonly elapsed_ms: number;
+   /**
+    * Number of sections whose `source_path` file no longer
+    * contains their heading text (advisory; not auto-removed).
+    * Absent on older daemon versions.
+    */
+   readonly stale_sources?: number;
 }
 
 /**

package/src/shared/cli/functions/run-init.function.ts

@@ -1,6 +1,7 @@
 import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
 import { join, resolve } from 'node:path';
 import type { CliOptions } from '../typings/cli-options.interface.ts';
+import { installDefaultSkills } from './install-default-skills.function.ts';
 
 /**
  * Result returned by `runInit` describing whether the directory was
@@ -82,6 +83,7 @@ export function runInit(options: CliOptions): InitResult {
          };
       }
       createScaffold(dbDir);
+      installDefaultSkills(dbDir);
       return { ok: true, targetPath: dbDir };
    } catch (err: unknown) {
       const errorMessage =

package/src/shared/cli/functions/run-learn.function.ts

@@ -4,6 +4,30 @@ import type { CliOptions } from '../typings/cli-options.interface.ts';
 import type { LearnResult } from '../typings/learn-result.model.ts';
 import { SUPPORTED_EXTENSIONS } from '../constants/defaults.constant.ts';
 
+/** Enriched add-section result returned by the daemon. */
+interface AddSectionResult {
+   /** Knowledge-base identifier assigned to the new section. */
+   readonly kbid: string;
+   /**
+    * Kbid of the section superseded via `--replace`,
+    * or `null` / `undefined` when none.
+    */
+   readonly superseded?: string | null;
+   /**
+    * Advisory warning strings (e.g. `"heading_missing"`).
+    * May be absent from older daemon versions.
+    */
+   readonly warnings?: string[];
+   /**
+    * Near-duplicate sections with similarity >= 0.85.
+    * May be absent from older daemon versions.
+    */
+   readonly near_duplicates?: {
+      readonly kbid: string;
+      readonly similarity: number;
+   }[];
+}
+
 /**
  * Minimal client interface required by `runLearn`.
  *
@@ -14,9 +38,9 @@ export interface LearnClient {
    /**
     * Ingests a content section into the knowledge base.
     *
-    * @param params - section type label and raw text content
-    * @returns result object containing the knowledge-base identifier
-    *   assigned to the section
+    * @param params - section metadata and raw text content
+    * @returns enriched result containing the new kbid plus
+    *   optional supersession, warnings, and near-duplicate info
     */
    addSection(params: {
       sectionType: string;
@@ -24,7 +48,14 @@ export interface LearnClient {
       title?: string;
       description?: string;
       docid?: string;
-   }): Promise<{ kbid: string }>;
+      tags?: string[];
+      replace?: boolean;
+      /**
+       * Absolute path to the source file. Required for supersession
+       * via source key when `replace` is `true`.
+       */
+      sourcePath?: string;
+   }): Promise<AddSectionResult>;
 }
 
 /** Injectable I/O for testing without real stdin. */
@@ -67,13 +98,10 @@ export async function runLearn(
             title: options.title,
             description: options.description,
             docid: options.docid,
+            tags: options.tags,
+            replace: options.replace,
          });
-         results.push({
-            kbid: result.kbid,
-            path: '-',
-            size: content.length,
-            skipped: false,
-         });
+         results.push(mapAddSectionResult(result, '-', content.length));
          continue;
       }
 
@@ -84,20 +112,44 @@ export async function runLearn(
             sectionType: 'text',
             content,
             title: options.title,
+            description: options.description,
             docid: options.docid,
+            tags: options.tags,
+            replace: options.replace,
+            sourcePath: file,
          });
-         results.push({
-            kbid: result.kbid,
-            path: file,
-            size: content.length,
-            skipped: false,
-         });
+         results.push(
+            mapAddSectionResult(result, file, content.length),
+         );
       }
    }
 
    return results;
 }
 
+/**
+ * Maps an `AddSectionResult` to a `LearnResult` record.
+ *
+ * @param result - raw result from `client.addSection()`
+ * @param path - source file path or `'-'` for stdin
+ * @param size - UTF-8 character count of the ingested content
+ */
+function mapAddSectionResult(
+   result: AddSectionResult,
+   path: string,
+   size: number,
+): LearnResult {
+   return {
+      kbid: result.kbid,
+      path,
+      size,
+      skipped: false,
+      superseded: result.superseded ?? null,
+      warnings: result.warnings ?? [],
+      near_duplicates: result.near_duplicates ?? [],
+   };
+}
+
 function readProcessStdin(): Promise<string> {
    return new Promise<string>((resolve) => {
       const chunks: Buffer[] = [];