@dikolab/kbdb 0.3.2 → 0.4.1

package/src/shared/cli/typings/disconnectable-client.interface.ts

@@ -1,29 +0,0 @@
-import type { VersionStatus } from '../../worker-client/typings/version-status.model.ts';
-import type { MigrateResult } from '../../worker-client/typings/migrate-result.model.ts';
-import type { DispatchClient } from './dispatch-client.type.ts';
-
-/**
- * Extension of `DispatchClient` with lifecycle methods needed by
- * the CLI orchestrator.
- *
- * Adds version checking, migration, and disconnect so the orchestrator
- * can manage a worker-client connection across all CLI commands.
- */
-export interface DisconnectableClient extends DispatchClient {
-   /**
-    * Checks the database version against the worker engine.
-    *
-    * @param targetPath - absolute path to the `.kbdb` database root
-    */
-   checkVersion(targetPath: string): Promise<VersionStatus>;
-
-   /**
-    * Runs all pending database migrations.
-    *
-    * @param targetPath - absolute path to the `.kbdb` database root
-    */
-   migrateDatabase(targetPath: string): Promise<MigrateResult>;
-
-   /** Disconnects from the worker daemon and releases resources. */
-   disconnect(): void;
-}

package/src/shared/cli/typings/dispatch-client.type.ts

@@ -1,46 +0,0 @@
-import type { SearchClient } from './search-client.interface.ts';
-import type { LearnClient } from './learn-client.interface.ts';
-import type { UnlearnClient } from '../functions/run-unlearn.function.ts';
-import type { ContentClient } from '../functions/run-content.function.ts';
-import type { CheckClient } from './check-client.interface.ts';
-import type { GcClient } from '../functions/run-gc.function.ts';
-import type { RebuildClient } from '../functions/run-rebuild.function.ts';
-import type { StatusClient } from '../functions/run-status.function.ts';
-import type { DbMigrateClient } from '../functions/run-db-migrate.function.ts';
-import type { SkillLearnClient } from '../functions/run-skill-learn.function.ts';
-import type { SkillListClient } from '../functions/run-skill-list.function.ts';
-import type { SkillGetClient } from '../functions/run-skill-get.function.ts';
-import type { SkillDeleteClient } from '../functions/run-skill-delete.function.ts';
-import type { AgentCreateClient } from './agent-create-client.interface.ts';
-import type { AgentListClient } from '../functions/run-agent-list.function.ts';
-import type { AgentGetClient } from './agent-get-client.interface.ts';
-import type { AgentDeleteClient } from '../functions/run-agent-delete.function.ts';
-import type { RecallClient } from '../functions/run-recall.function.ts';
-import type { ExportClient } from '../functions/run-export.function.ts';
-
-/**
- * Intersection of all data-command client interfaces.
- *
- * Each run* function defines its own minimal interface; this
- * intersection combines them so `dispatchCommand` can route
- * to any command handler with full type safety.
- */
-export type DispatchClient = SearchClient &
-   LearnClient &
-   UnlearnClient &
-   ContentClient &
-   CheckClient &
-   GcClient &
-   RebuildClient &
-   StatusClient &
-   DbMigrateClient &
-   SkillLearnClient &
-   SkillListClient &
-   SkillGetClient &
-   SkillDeleteClient &
-   AgentCreateClient &
-   AgentListClient &
-   AgentGetClient &
-   AgentDeleteClient &
-   RecallClient &
-   ExportClient;

package/src/shared/cli/typings/learn-client.interface.ts

@@ -1,53 +0,0 @@
-/**
- * Minimal client interface required by `runLearn`.
- *
- * Decouples the CLI function from the concrete `WorkerClient`
- * implementation, making it straightforward to test with mocks.
- */
-export interface LearnClient {
-   /**
-    * Ingests a content section into the knowledge base.
-    *
-    * @param params - section metadata and raw text content
-    * @returns enriched result containing the new kbid plus
-    *   optional supersession, warnings, and near-duplicate info
-    */
-   addSection(params: {
-      sectionType: string;
-      content: string;
-      title?: string;
-      description?: string;
-      docid?: string;
-      tags?: string[];
-      replace?: boolean;
-      /**
-       * Absolute path to the source file. Required for supersession
-       * via source key when `replace` is `true`.
-       */
-      sourcePath?: string;
-   }): Promise<LearnAddSectionResult>;
-}
-
-/** Enriched add-section result returned by the daemon. */
-export interface LearnAddSectionResult {
-   /** Knowledge-base identifier assigned to the new section. */
-   readonly kbid: string;
-   /**
-    * Kbid of the section superseded via `--replace`,
-    * or `null` / `undefined` when none.
-    */
-   readonly superseded?: string | null;
-   /**
-    * Advisory warning strings (e.g. `"heading_missing"`).
-    * May be absent from older daemon versions.
-    */
-   readonly warnings?: string[];
-   /**
-    * Near-duplicate sections with similarity >= 0.85.
-    * May be absent from older daemon versions.
-    */
-   readonly near_duplicates?: {
-      readonly kbid: string;
-      readonly similarity: number;
-   }[];
-}

package/src/shared/cli/typings/learn-io.interface.ts

@@ -1,5 +0,0 @@
-/** Injectable I/O for `runLearn` without real stdin. */
-export interface LearnIO {
-   /** Reads all stdin content and resolves with the full string. */
-   readStdin: () => Promise<string>;
-}

package/src/shared/cli/typings/learn-result.model.ts

@@ -1,27 +0,0 @@
-/** Result record produced for each file processed by the learn command. */
-export interface LearnResult {
-   /** Knowledge-base identifier assigned to the ingested section. */
-   kbid: string;
-   /** Absolute path to the source file that was ingested. */
-   path: string;
-   /** Size of the file content in bytes (UTF-8 character count). */
-   size: number;
-   /** `true` when the file was skipped without being ingested. */
-   skipped: boolean;
-   /**
-    * Kbid of the section that was superseded via `--replace`,
-    * or `null` when no supersession occurred.
-    */
-   superseded: string | null;
-   /**
-    * Advisory warning strings returned by the engine
-    * (e.g. `"heading_missing"` when no title is provided).
-    */
-   warnings: string[];
-   /**
-    * Sections with content similarity >= 0.85 to this section.
-    * Each entry identifies a near-duplicate section and its
-    * similarity score.
-    */
-   near_duplicates: { kbid: string; similarity: number }[];
-}

package/src/shared/cli/typings/search-client.interface.ts

@@ -1,73 +0,0 @@
-/**
- * Minimal client interface required by `runSearch`.
- *
- * Decouples the CLI function from the concrete `WorkerClient`
- * implementation, making it straightforward to test with mocks.
- */
-export interface SearchClient {
-   /**
-    * Searches the knowledge base and returns paginated results.
-    *
-    * @param params - query, mode, limit, and offset
-    */
-   search(params: {
-      query: string;
-      mode?: string;
-      /** Search algorithm: `'lexical'`, `'vector'`, or `'hybrid'`. */
-      algo?: string;
-      limit?: number;
-      offset?: number;
-   }): Promise<SearchClientResponse>;
-}
-
-/** Raw paginated search response from the daemon. */
-export interface SearchClientResponse {
-   /** Result items for the current page. */
-   readonly items: readonly SearchClientItem[];
-   /** Total number of matches across all pages. */
-   readonly total: number;
-   /** Number of results skipped. */
-   readonly offset: number;
-   /** Maximum results per page. */
-   readonly limit: number;
-   /** Whether more results exist beyond this page. */
-   readonly has_more: boolean;
-   /**
-    * When `true`, the engine relaxed the query to produce results.
-    * May be absent from older daemon versions.
-    */
-   readonly relaxed?: boolean;
-}
-
-/** A single raw search result item returned by the daemon. */
-export interface SearchClientItem {
-   /** Knowledge-base identifier of the matched section. */
-   readonly kbid: string;
-   /** Section heading, or null when untitled. */
-   readonly heading: string | null;
-   /** MIME type of the section content. */
-   readonly type: string;
-   /** Document IDs this section belongs to. */
-   readonly docids: readonly string[];
-   /** Relevance score from the ranking engine. */
-   readonly score: number;
-   /** KWIC excerpt centered on the highest-scoring matched term. */
-   readonly snippet: string;
-   /** Query terms that matched within this section. */
-   readonly matched_terms: readonly string[];
-   /**
-    * Fields that contained at least one match.
-    * May be absent from older daemon versions.
-    */
-   readonly matched_fields?: readonly string[];
-   /**
-    * Normalised confidence score (0.0-1.0) relative to the top result.
-    * May be absent from older daemon versions.
-    */
-   readonly confidence?: number;
-   /**
-    * ISO 8601 creation timestamp of the section.
-    * May be absent from older daemon versions.
-    */
-   readonly created_at?: string;
-}

package/src/shared/cli/typings/search-display.model.ts

@@ -1,54 +0,0 @@
-/** A single search result formatted for display to the user. */
-export interface SearchDisplay {
-   /** Knowledge-base identifier of the matched section. */
-   kbid: string;
-   /** Section heading, or null when untitled. */
-   heading: string | null;
-   /** MIME type of the section content. */
-   type: string;
-   /** Document IDs this section belongs to. */
-   docids: readonly string[];
-   /** Relevance score assigned by the ranking engine. */
-   score: number;
-   /**
-    * Normalised confidence score (0.0-1.0) relative to the
-    * top result in this result set.
-    */
-   confidence: number;
-   /** KWIC snippet from the WASM engine (~150 chars). */
-   snippet: string;
-   /** Query terms that matched within this section. */
-   matched_terms: readonly string[];
-   /**
-    * Fields that contained at least one match:
-    * subset of `["heading", "body", "code"]`.
-    */
-   matched_fields: readonly string[];
-   /** ISO 8601 creation timestamp of the section. */
-   created_at: string;
-   /**
-    * CLI-side truncated content preview for terminal display.
-    * Populated when `--content` / `-c` is used.
-    */
-   preview: string;
-}
-
-/** Paginated search display envelope for CLI output. */
-export interface PagedSearchDisplay {
-   /** Search results for the current page. */
-   items: SearchDisplay[];
-   /** Total number of matches across all pages. */
-   total: number;
-   /** Number of results skipped. */
-   offset: number;
-   /** Maximum results per page. */
-   limit: number;
-   /** Whether more results exist beyond this page. */
-   has_more: boolean;
-   /**
-    * When `true`, the engine relaxed the query (e.g. dropped
-    * rare terms) to return results that would otherwise be
-    * empty.
-    */
-   relaxed: boolean;
-}

package/src/shared/dir-resolver/functions/join-path.function.ts

@@ -1,13 +0,0 @@
-import { join } from 'node:path';
-
-export function isRemoteUrl(path: string): boolean {
-   return path.startsWith('http://') || path.startsWith('https://');
-}
-
-export function joinPath(base: string, ...segments: string[]): string {
-   if (isRemoteUrl(base)) {
-      const trailing = base.endsWith('/') ? '' : '/';
-      return base + trailing + segments.join('/');
-   }
-   return join(base, ...segments);
-}

package/src/shared/dir-resolver/functions/resolve-base-dir.function.ts

@@ -1,25 +0,0 @@
-import { dirname } from 'node:path';
-import { fileURLToPath } from 'node:url';
-
-import type { BaseDir } from '../typings/base-dir.interface.ts';
-
-export function resolveScriptDir(
-   entryUrl: string,
-   entryDirname?: string,
-): string {
-   if (entryDirname) return entryDirname;
-   if (entryUrl.startsWith('file:'))
-      return dirname(fileURLToPath(entryUrl));
-   return dirname(entryUrl);
-}
-
-export function resolveBaseDir(
-   entryUrl: string,
-   entryDirname?: string,
-): BaseDir {
-   const scriptDir =
-      typeof __dirname !== 'undefined'
-         ? __dirname
-         : resolveScriptDir(entryUrl, entryDirname);
-   return { scriptDir, cwd: process.cwd() };
-}

package/src/shared/dir-resolver/functions/resolve-wasm-dir.function.ts

@@ -1,5 +0,0 @@
-import { joinPath } from './join-path.function.ts';
-
-export function resolveWasmDir(scriptDir: string): string {
-   return joinPath(scriptDir, 'wasm');
-}

package/src/shared/dir-resolver/functions/resolve-worker-script.function.ts

@@ -1,13 +0,0 @@
-import { joinPath } from './join-path.function.ts';
-
-import type { Runtime } from '../../runtime/typings/runtime.ts';
-
-export function resolveWorkerScript(
-   scriptDir: string,
-   runtime: Runtime,
-): string {
-   if (runtime === 'deno') {
-      return joinPath(scriptDir, 'worker.ts');
-   }
-   return joinPath(scriptDir, 'kbdb-worker.cjs');
-}

package/src/shared/dir-resolver/index.ts

@@ -1,11 +0,0 @@
-export {
-   resolveBaseDir,
-   resolveScriptDir,
-} from './functions/resolve-base-dir.function.ts';
-export { resolveWorkerScript } from './functions/resolve-worker-script.function.ts';
-export { resolveWasmDir } from './functions/resolve-wasm-dir.function.ts';
-export {
-   isRemoteUrl,
-   joinPath,
-} from './functions/join-path.function.ts';
-export type { BaseDir } from './typings/base-dir.interface.ts';

package/src/shared/dir-resolver/typings/base-dir.interface.ts

@@ -1,4 +0,0 @@
-export interface BaseDir {
-   readonly scriptDir: string;
-   readonly cwd: string;
-}

package/src/shared/embedding/classes/null-embedding-provider.class.ts

@@ -1,29 +0,0 @@
-import type { EmbeddingProvider } from '../typings/embedding-provider.interface.ts';
-
-/**
- * No-op embedding provider used when embeddings are not configured.
- *
- * Satisfies the `EmbeddingProvider` contract without performing any
- * computation. Use this as the default when no provider is selected
- * in the config.
- */
-export class NullEmbeddingProvider implements EmbeddingProvider {
-   /**
-    * Returns an empty `Float32Array` for each input text.
-    *
-    * @param texts - input strings (contents are ignored)
-    */
-   embed(texts: string[]): Promise<Float32Array[]> {
-      return Promise.resolve(texts.map(() => new Float32Array(0)));
-   }
-
-   /** Always returns 0 -- no dimensions when disabled. */
-   dimensions(): number {
-      return 0;
-   }
-
-   /** Always returns `'none'`. */
-   modelName(): string {
-      return 'none';
-   }
-}

package/src/shared/embedding/constants/near-duplicate-thresholds.constant.ts

@@ -1,17 +0,0 @@
-/**
- * Minimum Jaccard coefficient to flag a section as a near-duplicate
- * directly via the Jaccard pass.
- */
-export const JACCARD_THRESHOLD = 0.85;
-
-/**
- * Lower bound of the Jaccard pre-filter band that triggers the
- * cosine-similarity pass.
- */
-export const JACCARD_BAND_MIN = 0.4;
-
-/**
- * Minimum cosine similarity to flag a section as a near-duplicate
- * via the cosine pass.
- */
-export const COSINE_THRESHOLD = 0.92;

package/src/shared/embedding/functions/build-vector-index.function.ts

@@ -1,79 +0,0 @@
-import { writeFileSync } from 'node:fs';
-import { join } from 'node:path';
-import type { VectorIndexEntry } from '../typings/vector-index-entry.model.ts';
-
-const MAGIC = 'KBVX';
-const VERSION = 1;
-const KBID_WIDTH = 64;
-const HEADER_SIZE = 32;
-
-function writeHeader(
-   buf: Buffer,
-   dimensions: number,
-   count: number,
-   modelLen: number,
-): void {
-   buf.write(MAGIC, 0, 'ascii');
-   buf.writeUInt32LE(VERSION, 4);
-   buf.writeUInt32LE(dimensions, 8);
-   buf.writeUInt32LE(count, 12);
-   buf.writeUInt32LE(modelLen, 16);
-}
-
-function writeRecord(
-   buf: Buffer,
-   offset: number,
-   entry: VectorIndexEntry,
-   dimensions: number,
-   encoder: TextEncoder,
-): void {
-   const kbidBytes = encoder.encode(entry.kbid);
-   kbidBytes.forEach((b, i) => {
-      if (i < KBID_WIDTH) buf.writeUInt8(b, offset + i);
-   });
-   for (let d = 0; d < dimensions; d++) {
-      buf.writeFloatLE(
-         entry.vector[d] ?? 0,
-         offset + KBID_WIDTH + d * 4,
-      );
-   }
-}
-
-/**
- * Writes a binary `vectors.idx` file to the given directory.
- *
- * Header: 32 bytes (magic KBVX, version u32, dimensions u32, count
- * u32, model-name length u32, 12 reserved bytes). Followed by the
- * model name (UTF-8) then records (64-byte kbid + dims*4 floats).
- *
- * @param dirPath - directory where `vectors.idx` will be written
- * @param modelName - name of the embedding model used
- * @param dimensions - vector dimension count
- * @param entries - kbid/vector pairs to index
- */
-export function buildVectorIndex(
-   dirPath: string,
-   modelName: string,
-   dimensions: number,
-   entries: VectorIndexEntry[],
-): Promise<void> {
-   const encoder = new TextEncoder();
-   const modelBytes = encoder.encode(modelName);
-   const modelLen = modelBytes.length;
-   const recordSize = KBID_WIDTH + dimensions * 4;
-   const totalSize =
-      HEADER_SIZE + modelLen + entries.length * recordSize;
-
-   const buf = Buffer.alloc(totalSize, 0);
-   writeHeader(buf, dimensions, entries.length, modelLen);
-   modelBytes.forEach((b, i) => buf.writeUInt8(b, HEADER_SIZE + i));
-
-   let recOffset = HEADER_SIZE + modelLen;
-   for (const entry of entries) {
-      writeRecord(buf, recOffset, entry, dimensions, encoder);
-      recOffset += recordSize;
-   }
-
-   writeFileSync(join(dirPath, 'vectors.idx'), buf);
-   return Promise.resolve();
-}

package/src/shared/embedding/functions/cosine-similarity.function.ts

@@ -1,31 +0,0 @@
-/**
- * Computes cosine similarity between two vectors.
- *
- * Returns `dot(a, b) / (||a|| * ||b||)`. Returns `0` when either
- * vector has zero magnitude to avoid division by zero.
- *
- * @param a - first embedding vector
- * @param b - second embedding vector
- */
-export function cosineSimilarity(
-   a: Float32Array,
-   b: Float32Array,
-): number {
-   let dot = 0;
-   let magA = 0;
-   let magB = 0;
-
-   for (let i = 0; i < a.length; i++) {
-      const ai = a[i] ?? 0;
-      const bi = b[i] ?? 0;
-      dot += ai * bi;
-      magA += ai * ai;
-      magB += bi * bi;
-   }
-
-   if (magA === 0 || magB === 0) {
-      return 0;
-   }
-
-   return dot / (Math.sqrt(magA) * Math.sqrt(magB));
-}

package/src/shared/embedding/functions/detect-near-duplicates.function.ts

@@ -1,69 +0,0 @@
-import type { EmbeddingProvider } from '../typings/embedding-provider.interface.ts';
-import type { NearDuplicateCandidate } from '../typings/near-duplicate-candidate.model.ts';
-import type { NearDuplicateResult } from '../typings/near-duplicate-result.model.ts';
-import { jaccardCoefficient } from './jaccard-coefficient.function.ts';
-import { cosineSimilarity } from './cosine-similarity.function.ts';
-import {
-   JACCARD_THRESHOLD,
-   JACCARD_BAND_MIN,
-   COSINE_THRESHOLD,
-} from '../constants/near-duplicate-thresholds.constant.ts';
-
-/**
- * Two-pass near-duplicate detection.
- *
- * Pass 1: Jaccard coefficient (always runs).
- * Pass 2: Cosine similarity on embeddings (only when `provider` is
- * available and the Jaccard score falls in the pre-filter band
- * [0.40, 0.85)).
- *
- * @param candidate - the new section to test against existing ones
- * @param existing - known sections to compare against
- * @param provider - optional embedding provider; when absent or null,
- *   the cosine pass is skipped entirely
- * @returns detected near-duplicates in the same order as `existing`
- */
-export function detectNearDuplicates(
-   candidate: NearDuplicateCandidate,
-   existing: NearDuplicateCandidate[],
-   provider?: EmbeddingProvider | null,
-): NearDuplicateResult[] {
-   const results: NearDuplicateResult[] = [];
-
-   for (const entry of existing) {
-      const jaccard = jaccardCoefficient(
-         candidate.tokens,
-         entry.tokens,
-      );
-
-      if (jaccard >= JACCARD_THRESHOLD) {
-         results.push({
-            kbid: entry.kbid,
-            score: jaccard,
-            method: 'jaccard',
-         });
-         continue;
-      }
-
-      if (
-         provider != null &&
-         jaccard >= JACCARD_BAND_MIN &&
-         candidate.embedding != null &&
-         entry.embedding != null
-      ) {
-         const cosine = cosineSimilarity(
-            candidate.embedding,
-            entry.embedding,
-         );
-         if (cosine >= COSINE_THRESHOLD) {
-            results.push({
-               kbid: entry.kbid,
-               score: cosine,
-               method: 'cosine',
-            });
-         }
-      }
-   }
-
-   return results;
-}

package/src/shared/embedding/functions/jaccard-coefficient.function.ts

@@ -1,26 +0,0 @@
-/**
- * Computes Jaccard coefficient between two token sets.
- *
- * `|A ∩ B| / |A ∪ B|`. Returns `0` if both sets are empty.
- *
- * @param a - first token set
- * @param b - second token set
- */
-export function jaccardCoefficient(
-   a: ReadonlySet<string>,
-   b: ReadonlySet<string>,
-): number {
-   if (a.size === 0 || b.size === 0) {
-      return 0;
-   }
-
-   let intersection = 0;
-   for (const token of a) {
-      if (b.has(token)) {
-         intersection++;
-      }
-   }
-
-   const union = a.size + b.size - intersection;
-   return union > 0 ? intersection / union : 0;
-}

package/src/shared/embedding/functions/read-vector-index-meta.function.ts

@@ -1,44 +0,0 @@
-import { readFileSync } from 'node:fs';
-import type { VectorIndexMeta } from '../typings/vector-index-meta.model.ts';
-
-/** Expected magic string in the first 4 bytes of a KBVX index. */
-const MAGIC = 'KBVX';
-
-/** Fixed header size in bytes. */
-const HEADER_SIZE = 32;
-
-/**
- * Reads the 32-byte header from a `vectors.idx` file and returns
- * the embedded `VectorIndexMeta`.
- *
- * @param filePath - absolute path to the `vectors.idx` file
- * @throws {Error} if the file does not exist or the magic bytes do
- *   not match the expected `KBVX` signature
- */
-export function readVectorIndexMeta(filePath: string): VectorIndexMeta {
-   const buf = readFileSync(filePath);
-
-   if (buf.length < HEADER_SIZE) {
-      throw new Error(
-         `vectors.idx too small: ${String(buf.length)} bytes`,
-      );
-   }
-
-   const magic = buf.toString('ascii', 0, 4);
-   if (magic !== MAGIC) {
-      throw new Error(
-         `invalid vectors.idx: expected magic ${MAGIC}, got ${magic}`,
-      );
-   }
-
-   const version = buf.readUInt32LE(4);
-   const dimensions = buf.readUInt32LE(8);
-   const count = buf.readUInt32LE(12);
-   const modelLen = buf.readUInt32LE(16);
-
-   const modelName = buf
-      .toString('utf-8', HEADER_SIZE, HEADER_SIZE + modelLen)
-      .replace(/\0/g, '');
-
-   return { modelName, dimensions, count, version };
-}