@dikolab/kbdb 0.3.0 → 0.3.2

package/src/shared/cli/typings/dispatch-client.type.ts

@@ -0,0 +1,46 @@
+import type { SearchClient } from './search-client.interface.ts';
+import type { LearnClient } from './learn-client.interface.ts';
+import type { UnlearnClient } from '../functions/run-unlearn.function.ts';
+import type { ContentClient } from '../functions/run-content.function.ts';
+import type { CheckClient } from './check-client.interface.ts';
+import type { GcClient } from '../functions/run-gc.function.ts';
+import type { RebuildClient } from '../functions/run-rebuild.function.ts';
+import type { StatusClient } from '../functions/run-status.function.ts';
+import type { DbMigrateClient } from '../functions/run-db-migrate.function.ts';
+import type { SkillLearnClient } from '../functions/run-skill-learn.function.ts';
+import type { SkillListClient } from '../functions/run-skill-list.function.ts';
+import type { SkillGetClient } from '../functions/run-skill-get.function.ts';
+import type { SkillDeleteClient } from '../functions/run-skill-delete.function.ts';
+import type { AgentCreateClient } from './agent-create-client.interface.ts';
+import type { AgentListClient } from '../functions/run-agent-list.function.ts';
+import type { AgentGetClient } from './agent-get-client.interface.ts';
+import type { AgentDeleteClient } from '../functions/run-agent-delete.function.ts';
+import type { RecallClient } from '../functions/run-recall.function.ts';
+import type { ExportClient } from '../functions/run-export.function.ts';
+
+/**
+ * Intersection of all data-command client interfaces.
+ *
+ * Each run* function defines its own minimal interface; this
+ * intersection combines them so `dispatchCommand` can route
+ * to any command handler with full type safety.
+ */
+export type DispatchClient = SearchClient &
+   LearnClient &
+   UnlearnClient &
+   ContentClient &
+   CheckClient &
+   GcClient &
+   RebuildClient &
+   StatusClient &
+   DbMigrateClient &
+   SkillLearnClient &
+   SkillListClient &
+   SkillGetClient &
+   SkillDeleteClient &
+   AgentCreateClient &
+   AgentListClient &
+   AgentGetClient &
+   AgentDeleteClient &
+   RecallClient &
+   ExportClient;

package/src/shared/cli/typings/learn-client.interface.ts

@@ -0,0 +1,53 @@
+/**
+ * Minimal client interface required by `runLearn`.
+ *
+ * Decouples the CLI function from the concrete `WorkerClient`
+ * implementation, making it straightforward to test with mocks.
+ */
+export interface LearnClient {
+   /**
+    * Ingests a content section into the knowledge base.
+    *
+    * @param params - section metadata and raw text content
+    * @returns enriched result containing the new kbid plus
+    *   optional supersession, warnings, and near-duplicate info
+    */
+   addSection(params: {
+      sectionType: string;
+      content: string;
+      title?: string;
+      description?: string;
+      docid?: string;
+      tags?: string[];
+      replace?: boolean;
+      /**
+       * Absolute path to the source file. Required for supersession
+       * via source key when `replace` is `true`.
+       */
+      sourcePath?: string;
+   }): Promise<LearnAddSectionResult>;
+}
+
+/** Enriched add-section result returned by the daemon. */
+export interface LearnAddSectionResult {
+   /** Knowledge-base identifier assigned to the new section. */
+   readonly kbid: string;
+   /**
+    * Kbid of the section superseded via `--replace`,
+    * or `null` / `undefined` when none.
+    */
+   readonly superseded?: string | null;
+   /**
+    * Advisory warning strings (e.g. `"heading_missing"`).
+    * May be absent from older daemon versions.
+    */
+   readonly warnings?: string[];
+   /**
+    * Near-duplicate sections with similarity >= 0.85.
+    * May be absent from older daemon versions.
+    */
+   readonly near_duplicates?: {
+      readonly kbid: string;
+      readonly similarity: number;
+   }[];
+}

package/src/shared/cli/typings/learn-io.interface.ts

@@ -0,0 +1,5 @@
+/** Injectable I/O for `runLearn` without real stdin. */
+export interface LearnIO {
+   /** Reads all stdin content and resolves with the full string. */
+   readStdin: () => Promise<string>;
+}

package/src/shared/cli/typings/search-client.interface.ts

@@ -0,0 +1,73 @@
+/**
+ * Minimal client interface required by `runSearch`.
+ *
+ * Decouples the CLI function from the concrete `WorkerClient`
+ * implementation, making it straightforward to test with mocks.
+ */
+export interface SearchClient {
+   /**
+    * Searches the knowledge base and returns paginated results.
+    *
+    * @param params - query, mode, limit, and offset
+    */
+   search(params: {
+      query: string;
+      mode?: string;
+      /** Search algorithm: `'lexical'`, `'vector'`, or `'hybrid'`. */
+      algo?: string;
+      limit?: number;
+      offset?: number;
+   }): Promise<SearchClientResponse>;
+}
+
+/** Raw paginated search response from the daemon. */
+export interface SearchClientResponse {
+   /** Result items for the current page. */
+   readonly items: readonly SearchClientItem[];
+   /** Total number of matches across all pages. */
+   readonly total: number;
+   /** Number of results skipped. */
+   readonly offset: number;
+   /** Maximum results per page. */
+   readonly limit: number;
+   /** Whether more results exist beyond this page. */
+   readonly has_more: boolean;
+   /**
+    * When `true`, the engine relaxed the query to produce results.
+    * May be absent from older daemon versions.
+    */
+   readonly relaxed?: boolean;
+}
+
+/** A single raw search result item returned by the daemon. */
+export interface SearchClientItem {
+   /** Knowledge-base identifier of the matched section. */
+   readonly kbid: string;
+   /** Section heading, or null when untitled. */
+   readonly heading: string | null;
+   /** MIME type of the section content. */
+   readonly type: string;
+   /** Document IDs this section belongs to. */
+   readonly docids: readonly string[];
+   /** Relevance score from the ranking engine. */
+   readonly score: number;
+   /** KWIC excerpt centered on the highest-scoring matched term. */
+   readonly snippet: string;
+   /** Query terms that matched within this section. */
+   readonly matched_terms: readonly string[];
+   /**
+    * Fields that contained at least one match.
+    * May be absent from older daemon versions.
+    */
+   readonly matched_fields?: readonly string[];
+   /**
+    * Normalised confidence score (0.0-1.0) relative to the top result.
+    * May be absent from older daemon versions.
+    */
+   readonly confidence?: number;
+   /**
+    * ISO 8601 creation timestamp of the section.
+    * May be absent from older daemon versions.
+    */
+   readonly created_at?: string;
+}

package/src/shared/embedding/classes/null-embedding-provider.class.ts

@@ -0,0 +1,29 @@
+import type { EmbeddingProvider } from '../typings/embedding-provider.interface.ts';
+
+/**
+ * No-op embedding provider used when embeddings are not configured.
+ *
+ * Satisfies the `EmbeddingProvider` contract without performing any
+ * computation. Use this as the default when no provider is selected
+ * in the config.
+ */
+export class NullEmbeddingProvider implements EmbeddingProvider {
+   /**
+    * Returns an empty `Float32Array` for each input text.
+    *
+    * @param texts - input strings (contents are ignored)
+    */
+   embed(texts: string[]): Promise<Float32Array[]> {
+      return Promise.resolve(texts.map(() => new Float32Array(0)));
+   }
+
+   /** Always returns 0 -- no dimensions when disabled. */
+   dimensions(): number {
+      return 0;
+   }
+
+   /** Always returns `'none'`. */
+   modelName(): string {
+      return 'none';
+   }
+}

package/src/shared/embedding/constants/near-duplicate-thresholds.constant.ts

@@ -0,0 +1,17 @@
+/**
+ * Minimum Jaccard coefficient to flag a section as a near-duplicate
+ * directly via the Jaccard pass.
+ */
+export const JACCARD_THRESHOLD = 0.85;
+
+/**
+ * Lower bound of the Jaccard pre-filter band that triggers the
+ * cosine-similarity pass.
+ */
+export const JACCARD_BAND_MIN = 0.4;
+
+/**
+ * Minimum cosine similarity to flag a section as a near-duplicate
+ * via the cosine pass.
+ */
+export const COSINE_THRESHOLD = 0.92;

package/src/shared/embedding/functions/build-vector-index.function.ts

@@ -0,0 +1,79 @@
+import { writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import type { VectorIndexEntry } from '../typings/vector-index-entry.model.ts';
+
+const MAGIC = 'KBVX';
+const VERSION = 1;
+const KBID_WIDTH = 64;
+const HEADER_SIZE = 32;
+
+function writeHeader(
+   buf: Buffer,
+   dimensions: number,
+   count: number,
+   modelLen: number,
+): void {
+   buf.write(MAGIC, 0, 'ascii');
+   buf.writeUInt32LE(VERSION, 4);
+   buf.writeUInt32LE(dimensions, 8);
+   buf.writeUInt32LE(count, 12);
+   buf.writeUInt32LE(modelLen, 16);
+}
+
+function writeRecord(
+   buf: Buffer,
+   offset: number,
+   entry: VectorIndexEntry,
+   dimensions: number,
+   encoder: TextEncoder,
+): void {
+   const kbidBytes = encoder.encode(entry.kbid);
+   kbidBytes.forEach((b, i) => {
+      if (i < KBID_WIDTH) buf.writeUInt8(b, offset + i);
+   });
+   for (let d = 0; d < dimensions; d++) {
+      buf.writeFloatLE(
+         entry.vector[d] ?? 0,
+         offset + KBID_WIDTH + d * 4,
+      );
+   }
+}
+
+/**
+ * Writes a binary `vectors.idx` file to the given directory.
+ *
+ * Header: 32 bytes (magic KBVX, version u32, dimensions u32, count
+ * u32, model-name length u32, 12 reserved bytes). Followed by the
+ * model name (UTF-8) then records (64-byte kbid + dims*4 floats).
+ *
+ * @param dirPath - directory where `vectors.idx` will be written
+ * @param modelName - name of the embedding model used
+ * @param dimensions - vector dimension count
+ * @param entries - kbid/vector pairs to index
+ */
+export function buildVectorIndex(
+   dirPath: string,
+   modelName: string,
+   dimensions: number,
+   entries: VectorIndexEntry[],
+): Promise<void> {
+   const encoder = new TextEncoder();
+   const modelBytes = encoder.encode(modelName);
+   const modelLen = modelBytes.length;
+   const recordSize = KBID_WIDTH + dimensions * 4;
+   const totalSize =
+      HEADER_SIZE + modelLen + entries.length * recordSize;
+
+   const buf = Buffer.alloc(totalSize, 0);
+   writeHeader(buf, dimensions, entries.length, modelLen);
+   modelBytes.forEach((b, i) => buf.writeUInt8(b, HEADER_SIZE + i));
+
+   let recOffset = HEADER_SIZE + modelLen;
+   for (const entry of entries) {
+      writeRecord(buf, recOffset, entry, dimensions, encoder);
+      recOffset += recordSize;
+   }
+
+   writeFileSync(join(dirPath, 'vectors.idx'), buf);
+   return Promise.resolve();
+}

package/src/shared/embedding/functions/cosine-similarity.function.ts

@@ -0,0 +1,31 @@
+/**
+ * Computes cosine similarity between two vectors.
+ *
+ * Returns `dot(a, b) / (||a|| * ||b||)`. Returns `0` when either
+ * vector has zero magnitude to avoid division by zero.
+ *
+ * @param a - first embedding vector
+ * @param b - second embedding vector
+ */
+export function cosineSimilarity(
+   a: Float32Array,
+   b: Float32Array,
+): number {
+   let dot = 0;
+   let magA = 0;
+   let magB = 0;
+
+   for (let i = 0; i < a.length; i++) {
+      const ai = a[i] ?? 0;
+      const bi = b[i] ?? 0;
+      dot += ai * bi;
+      magA += ai * ai;
+      magB += bi * bi;
+   }
+
+   if (magA === 0 || magB === 0) {
+      return 0;
+   }
+
+   return dot / (Math.sqrt(magA) * Math.sqrt(magB));
+}

package/src/shared/embedding/functions/detect-near-duplicates.function.ts

@@ -0,0 +1,69 @@
+import type { EmbeddingProvider } from '../typings/embedding-provider.interface.ts';
+import type { NearDuplicateCandidate } from '../typings/near-duplicate-candidate.model.ts';
+import type { NearDuplicateResult } from '../typings/near-duplicate-result.model.ts';
+import { jaccardCoefficient } from './jaccard-coefficient.function.ts';
+import { cosineSimilarity } from './cosine-similarity.function.ts';
+import {
+   JACCARD_THRESHOLD,
+   JACCARD_BAND_MIN,
+   COSINE_THRESHOLD,
+} from '../constants/near-duplicate-thresholds.constant.ts';
+
+/**
+ * Two-pass near-duplicate detection.
+ *
+ * Pass 1: Jaccard coefficient (always runs).
+ * Pass 2: Cosine similarity on embeddings (only when `provider` is
+ * available and the Jaccard score falls in the pre-filter band
+ * [0.40, 0.85)).
+ *
+ * @param candidate - the new section to test against existing ones
+ * @param existing - known sections to compare against
+ * @param provider - optional embedding provider; when absent or null,
+ *   the cosine pass is skipped entirely
+ * @returns detected near-duplicates in the same order as `existing`
+ */
+export function detectNearDuplicates(
+   candidate: NearDuplicateCandidate,
+   existing: NearDuplicateCandidate[],
+   provider?: EmbeddingProvider | null,
+): NearDuplicateResult[] {
+   const results: NearDuplicateResult[] = [];
+
+   for (const entry of existing) {
+      const jaccard = jaccardCoefficient(
+         candidate.tokens,
+         entry.tokens,
+      );
+
+      if (jaccard >= JACCARD_THRESHOLD) {
+         results.push({
+            kbid: entry.kbid,
+            score: jaccard,
+            method: 'jaccard',
+         });
+         continue;
+      }
+
+      if (
+         provider != null &&
+         jaccard >= JACCARD_BAND_MIN &&
+         candidate.embedding != null &&
+         entry.embedding != null
+      ) {
+         const cosine = cosineSimilarity(
+            candidate.embedding,
+            entry.embedding,
+         );
+         if (cosine >= COSINE_THRESHOLD) {
+            results.push({
+               kbid: entry.kbid,
+               score: cosine,
+               method: 'cosine',
+            });
+         }
+      }
+   }
+
+   return results;
+}

package/src/shared/embedding/functions/jaccard-coefficient.function.ts

@@ -0,0 +1,26 @@
+/**
+ * Computes Jaccard coefficient between two token sets.
+ *
+ * `|A ∩ B| / |A ∪ B|`. Returns `0` if both sets are empty.
+ *
+ * @param a - first token set
+ * @param b - second token set
+ */
+export function jaccardCoefficient(
+   a: ReadonlySet<string>,
+   b: ReadonlySet<string>,
+): number {
+   if (a.size === 0 || b.size === 0) {
+      return 0;
+   }
+
+   let intersection = 0;
+   for (const token of a) {
+      if (b.has(token)) {
+         intersection++;
+      }
+   }
+
+   const union = a.size + b.size - intersection;
+   return union > 0 ? intersection / union : 0;
+}

package/src/shared/embedding/functions/read-vector-index-meta.function.ts

@@ -0,0 +1,44 @@
+import { readFileSync } from 'node:fs';
+import type { VectorIndexMeta } from '../typings/vector-index-meta.model.ts';
+
+/** Expected magic string in the first 4 bytes of a KBVX index. */
+const MAGIC = 'KBVX';
+
+/** Fixed header size in bytes. */
+const HEADER_SIZE = 32;
+
+/**
+ * Reads the 32-byte header from a `vectors.idx` file and returns
+ * the embedded `VectorIndexMeta`.
+ *
+ * @param filePath - absolute path to the `vectors.idx` file
+ * @throws {Error} if the file does not exist or the magic bytes do
+ *   not match the expected `KBVX` signature
+ */
+export function readVectorIndexMeta(filePath: string): VectorIndexMeta {
+   const buf = readFileSync(filePath);
+
+   if (buf.length < HEADER_SIZE) {
+      throw new Error(
+         `vectors.idx too small: ${String(buf.length)} bytes`,
+      );
+   }
+
+   const magic = buf.toString('ascii', 0, 4);
+   if (magic !== MAGIC) {
+      throw new Error(
+         `invalid vectors.idx: expected magic ${MAGIC}, got ${magic}`,
+      );
+   }
+
+   const version = buf.readUInt32LE(4);
+   const dimensions = buf.readUInt32LE(8);
+   const count = buf.readUInt32LE(12);
+   const modelLen = buf.readUInt32LE(16);
+
+   const modelName = buf
+      .toString('utf-8', HEADER_SIZE, HEADER_SIZE + modelLen)
+      .replace(/\0/g, '');
+
+   return { modelName, dimensions, count, version };
+}

package/src/shared/embedding/functions/rrf-fuse.function.ts

@@ -0,0 +1,33 @@
+/** Default RRF smoothing constant. */
+const DEFAULT_K = 60;
+
+/**
+ * Fuses lexical and vector rankings using Reciprocal Rank Fusion.
+ *
+ * For each kbid in either map the fused score is:
+ * `1/(k + lexicalRank) + 1/(k + vectorRank)`.
+ * When a kbid appears in only one map the missing rank contributes
+ * `0` (treated as infinite rank -- no contribution).
+ *
+ * @param lexicalRanks - map of kbid to 1-based lexical rank position
+ * @param vectorRanks - map of kbid to 1-based vector rank position
+ * @param k - RRF smoothing constant (default 60)
+ * @returns map of kbid to fused RRF score
+ */
+export function rrfFuse(
+   lexicalRanks: Map<string, number>,
+   vectorRanks: Map<string, number>,
+   k: number = DEFAULT_K,
+): Map<string, number> {
+   const scores = new Map<string, number>();
+
+   for (const [kbid, rank] of lexicalRanks) {
+      scores.set(kbid, (scores.get(kbid) ?? 0) + 1 / (k + rank));
+   }
+
+   for (const [kbid, rank] of vectorRanks) {
+      scores.set(kbid, (scores.get(kbid) ?? 0) + 1 / (k + rank));
+   }
+
+   return scores;
+}

package/src/shared/embedding/functions/search-vectors.function.ts

@@ -0,0 +1,59 @@
+import { readFileSync } from 'node:fs';
+import type { VectorSearchResult } from '../typings/vector-search-result.model.ts';
+import { cosineSimilarity } from './cosine-similarity.function.ts';
+
+/** Fixed header size in bytes. */
+const HEADER_SIZE = 32;
+
+/** Fixed width in bytes for the stored kbid string. */
+const KBID_WIDTH = 64;
+
+/**
+ * Brute-force linear scan of a `vectors.idx` file.
+ *
+ * Reads every stored vector, computes cosine similarity against
+ * `queryVector`, then returns the top `topN` results sorted by
+ * score descending.
+ *
+ * @param filePath - absolute path to the `vectors.idx` file
+ * @param queryVector - embedding to search against
+ * @param topN - maximum number of results to return
+ * @returns results sorted by descending cosine similarity score
+ */
+export function searchVectors(
+   filePath: string,
+   queryVector: Float32Array,
+   topN: number,
+): VectorSearchResult[] {
+   const buf = readFileSync(filePath);
+
+   if (buf.length < HEADER_SIZE) {
+      return [];
+   }
+
+   const dimensions = buf.readUInt32LE(8);
+   const count = buf.readUInt32LE(12);
+   const modelLen = buf.readUInt32LE(16);
+   const dataStart = HEADER_SIZE + modelLen;
+   const recordSize = KBID_WIDTH + dimensions * 4;
+
+   const results: VectorSearchResult[] = [];
+
+   for (let i = 0; i < count; i++) {
+      const recOffset = dataStart + i * recordSize;
+      const kbid = buf
+         .toString('utf-8', recOffset, recOffset + KBID_WIDTH)
+         .replace(/\0/g, '');
+
+      const vector = new Float32Array(dimensions);
+      for (let d = 0; d < dimensions; d++) {
+         vector[d] = buf.readFloatLE(recOffset + KBID_WIDTH + d * 4);
+      }
+
+      const score = cosineSimilarity(queryVector, vector);
+      results.push({ kbid, score });
+   }
+
+   results.sort((a, b) => b.score - a.score);
+   return results.slice(0, topN);
+}

package/src/shared/embedding/index.ts

@@ -0,0 +1,20 @@
+export { NullEmbeddingProvider } from './classes/null-embedding-provider.class.ts';
+export { cosineSimilarity } from './functions/cosine-similarity.function.ts';
+export { rrfFuse } from './functions/rrf-fuse.function.ts';
+export { buildVectorIndex } from './functions/build-vector-index.function.ts';
+export { searchVectors } from './functions/search-vectors.function.ts';
+export { readVectorIndexMeta } from './functions/read-vector-index-meta.function.ts';
+export { jaccardCoefficient } from './functions/jaccard-coefficient.function.ts';
+export { detectNearDuplicates } from './functions/detect-near-duplicates.function.ts';
+export {
+   JACCARD_THRESHOLD,
+   JACCARD_BAND_MIN,
+   COSINE_THRESHOLD,
+} from './constants/near-duplicate-thresholds.constant.ts';
+export type { EmbeddingProvider } from './typings/embedding-provider.interface.ts';
+export type { EmbeddingConfig } from './typings/embedding-config.interface.ts';
+export type { VectorSearchResult } from './typings/vector-search-result.model.ts';
+export type { VectorIndexMeta } from './typings/vector-index-meta.model.ts';
+export type { VectorIndexEntry } from './typings/vector-index-entry.model.ts';
+export type { NearDuplicateCandidate } from './typings/near-duplicate-candidate.model.ts';
+export type { NearDuplicateResult } from './typings/near-duplicate-result.model.ts';

package/src/shared/embedding/typings/embedding-config.interface.ts

@@ -0,0 +1,23 @@
+/** Configuration for the embedding provider. */
+export interface EmbeddingConfig {
+   /**
+    * Provider type.
+    *
+    * - `'none'` -- embeddings disabled (default)
+    * - `'onnx'` -- local ONNX Runtime model
+    * - `'api'` -- remote HTTP embedding API
+    */
+   readonly provider: 'none' | 'onnx' | 'api';
+
+   /** Model name or path (for onnx/api providers). */
+   readonly model?: string;
+
+   /** API endpoint URL (for api provider). */
+   readonly endpoint?: string;
+
+   /**
+    * Number of dimensions the model produces
+    * (required for api provider).
+    */
+   readonly dimensions?: number;
+}

package/src/shared/embedding/typings/embedding-provider.interface.ts

@@ -0,0 +1,25 @@
+/** Pluggable embedding provider interface. */
+export interface EmbeddingProvider {
+   /**
+    * Compute embedding vectors for text inputs.
+    *
+    * @param texts - list of text strings to embed
+    * @returns a Float32Array embedding for each input text,
+    *   in the same order
+    */
+   embed(texts: string[]): Promise<Float32Array[]>;
+
+   /**
+    * Number of dimensions in the embedding vector.
+    *
+    * Returns 0 when embeddings are disabled.
+    */
+   dimensions(): number;
+
+   /**
+    * Name of the underlying model.
+    *
+    * Returns `'none'` when embeddings are disabled.
+    */
+   modelName(): string;
+}

package/src/shared/embedding/typings/near-duplicate-candidate.model.ts

@@ -0,0 +1,25 @@
+/**
+ * A document section candidate for near-duplicate detection.
+ *
+ * Carries the token set for Jaccard comparison and an optional
+ * embedding vector for cosine-similarity comparison.
+ */
+export interface NearDuplicateCandidate {
+   /** Knowledge-base identifier for this section. */
+   readonly kbid: string;
+
+   /**
+    * Set of lowercased word tokens extracted from the section body.
+    *
+    * Used by the Jaccard coefficient pass.
+    */
+   readonly tokens: Set<string>;
+
+   /**
+    * Optional embedding vector for this section.
+    *
+    * When present and a provider is available, used as the
+    * cosine-similarity pass input.
+    */
+   readonly embedding?: Float32Array;
+}