@dikolab/kbdb 0.3.2 → 0.4.0

package/src/shared/embedding/functions/rrf-fuse.function.ts

@@ -1,33 +0,0 @@
-/** Default RRF smoothing constant. */
-const DEFAULT_K = 60;
-
-/**
- * Fuses lexical and vector rankings using Reciprocal Rank Fusion.
- *
- * For each kbid in either map the fused score is:
- * `1/(k + lexicalRank) + 1/(k + vectorRank)`.
- * When a kbid appears in only one map the missing rank contributes
- * `0` (treated as infinite rank -- no contribution).
- *
- * @param lexicalRanks - map of kbid to 1-based lexical rank position
- * @param vectorRanks - map of kbid to 1-based vector rank position
- * @param k - RRF smoothing constant (default 60)
- * @returns map of kbid to fused RRF score
- */
-export function rrfFuse(
-   lexicalRanks: Map<string, number>,
-   vectorRanks: Map<string, number>,
-   k: number = DEFAULT_K,
-): Map<string, number> {
-   const scores = new Map<string, number>();
-
-   for (const [kbid, rank] of lexicalRanks) {
-      scores.set(kbid, (scores.get(kbid) ?? 0) + 1 / (k + rank));
-   }
-
-   for (const [kbid, rank] of vectorRanks) {
-      scores.set(kbid, (scores.get(kbid) ?? 0) + 1 / (k + rank));
-   }
-
-   return scores;
-}

package/src/shared/embedding/functions/search-vectors.function.ts

@@ -1,59 +0,0 @@
-import { readFileSync } from 'node:fs';
-import type { VectorSearchResult } from '../typings/vector-search-result.model.ts';
-import { cosineSimilarity } from './cosine-similarity.function.ts';
-
-/** Fixed header size in bytes. */
-const HEADER_SIZE = 32;
-
-/** Fixed width in bytes for the stored kbid string. */
-const KBID_WIDTH = 64;
-
-/**
- * Brute-force linear scan of a `vectors.idx` file.
- *
- * Reads every stored vector, computes cosine similarity against
- * `queryVector`, then returns the top `topN` results sorted by
- * score descending.
- *
- * @param filePath - absolute path to the `vectors.idx` file
- * @param queryVector - embedding to search against
- * @param topN - maximum number of results to return
- * @returns results sorted by descending cosine similarity score
- */
-export function searchVectors(
-   filePath: string,
-   queryVector: Float32Array,
-   topN: number,
-): VectorSearchResult[] {
-   const buf = readFileSync(filePath);
-
-   if (buf.length < HEADER_SIZE) {
-      return [];
-   }
-
-   const dimensions = buf.readUInt32LE(8);
-   const count = buf.readUInt32LE(12);
-   const modelLen = buf.readUInt32LE(16);
-   const dataStart = HEADER_SIZE + modelLen;
-   const recordSize = KBID_WIDTH + dimensions * 4;
-
-   const results: VectorSearchResult[] = [];
-
-   for (let i = 0; i < count; i++) {
-      const recOffset = dataStart + i * recordSize;
-      const kbid = buf
-         .toString('utf-8', recOffset, recOffset + KBID_WIDTH)
-         .replace(/\0/g, '');
-
-      const vector = new Float32Array(dimensions);
-      for (let d = 0; d < dimensions; d++) {
-         vector[d] = buf.readFloatLE(recOffset + KBID_WIDTH + d * 4);
-      }
-
-      const score = cosineSimilarity(queryVector, vector);
-      results.push({ kbid, score });
-   }
-
-   results.sort((a, b) => b.score - a.score);
-   return results.slice(0, topN);
-}

package/src/shared/embedding/index.ts

@@ -1,20 +0,0 @@
-export { NullEmbeddingProvider } from './classes/null-embedding-provider.class.ts';
-export { cosineSimilarity } from './functions/cosine-similarity.function.ts';
-export { rrfFuse } from './functions/rrf-fuse.function.ts';
-export { buildVectorIndex } from './functions/build-vector-index.function.ts';
-export { searchVectors } from './functions/search-vectors.function.ts';
-export { readVectorIndexMeta } from './functions/read-vector-index-meta.function.ts';
-export { jaccardCoefficient } from './functions/jaccard-coefficient.function.ts';
-export { detectNearDuplicates } from './functions/detect-near-duplicates.function.ts';
-export {
-   JACCARD_THRESHOLD,
-   JACCARD_BAND_MIN,
-   COSINE_THRESHOLD,
-} from './constants/near-duplicate-thresholds.constant.ts';
-export type { EmbeddingProvider } from './typings/embedding-provider.interface.ts';
-export type { EmbeddingConfig } from './typings/embedding-config.interface.ts';
-export type { VectorSearchResult } from './typings/vector-search-result.model.ts';
-export type { VectorIndexMeta } from './typings/vector-index-meta.model.ts';
-export type { VectorIndexEntry } from './typings/vector-index-entry.model.ts';
-export type { NearDuplicateCandidate } from './typings/near-duplicate-candidate.model.ts';
-export type { NearDuplicateResult } from './typings/near-duplicate-result.model.ts';

package/src/shared/embedding/typings/embedding-config.interface.ts

@@ -1,23 +0,0 @@
-/** Configuration for the embedding provider. */
-export interface EmbeddingConfig {
-   /**
-    * Provider type.
-    *
-    * - `'none'` -- embeddings disabled (default)
-    * - `'onnx'` -- local ONNX Runtime model
-    * - `'api'` -- remote HTTP embedding API
-    */
-   readonly provider: 'none' | 'onnx' | 'api';
-
-   /** Model name or path (for onnx/api providers). */
-   readonly model?: string;
-
-   /** API endpoint URL (for api provider). */
-   readonly endpoint?: string;
-
-   /**
-    * Number of dimensions the model produces
-    * (required for api provider).
-    */
-   readonly dimensions?: number;
-}

package/src/shared/embedding/typings/embedding-provider.interface.ts

@@ -1,25 +0,0 @@
-/** Pluggable embedding provider interface. */
-export interface EmbeddingProvider {
-   /**
-    * Compute embedding vectors for text inputs.
-    *
-    * @param texts - list of text strings to embed
-    * @returns a Float32Array embedding for each input text,
-    *   in the same order
-    */
-   embed(texts: string[]): Promise<Float32Array[]>;
-
-   /**
-    * Number of dimensions in the embedding vector.
-    *
-    * Returns 0 when embeddings are disabled.
-    */
-   dimensions(): number;
-
-   /**
-    * Name of the underlying model.
-    *
-    * Returns `'none'` when embeddings are disabled.
-    */
-   modelName(): string;
-}

package/src/shared/embedding/typings/near-duplicate-candidate.model.ts

@@ -1,25 +0,0 @@
-/**
- * A document section candidate for near-duplicate detection.
- *
- * Carries the token set for Jaccard comparison and an optional
- * embedding vector for cosine-similarity comparison.
- */
-export interface NearDuplicateCandidate {
-   /** Knowledge-base identifier for this section. */
-   readonly kbid: string;
-
-   /**
-    * Set of lowercased word tokens extracted from the section body.
-    *
-    * Used by the Jaccard coefficient pass.
-    */
-   readonly tokens: Set<string>;
-
-   /**
-    * Optional embedding vector for this section.
-    *
-    * When present and a provider is available, used as the
-    * cosine-similarity pass input.
-    */
-   readonly embedding?: Float32Array;
-}

package/src/shared/embedding/typings/near-duplicate-result.model.ts

@@ -1,26 +0,0 @@
-/**
- * A near-duplicate detected by the two-pass detection algorithm.
- *
- * Reports which existing section matched the candidate and which
- * detection method produced the match.
- */
-export interface NearDuplicateResult {
-   /** Knowledge-base identifier of the matched existing section. */
-   readonly kbid: string;
-
-   /**
-    * Similarity score of the match (0.0–1.0).
-    *
-    * Jaccard matches use the Jaccard coefficient; cosine matches use
-    * cosine similarity.
-    */
-   readonly score: number;
-
-   /**
-    * Detection method that produced this match.
-    *
-    * - `'jaccard'`: Jaccard coefficient >= 0.85
-    * - `'cosine'`: cosine similarity >= 0.92 with Jaccard 0.40–0.84
-    */
-   readonly method: 'jaccard' | 'cosine';
-}

package/src/shared/embedding/typings/vector-index-entry.model.ts

@@ -1,13 +0,0 @@
-/** A single record written into the vector index file. */
-export interface VectorIndexEntry {
-   /**
-    * Knowledge-base identifier for this section.
-    *
-    * Stored as a fixed 64-byte UTF-8 field in the index file;
-    * truncated or zero-padded as needed.
-    */
-   readonly kbid: string;
-
-   /** Embedding vector for this section. */
-   readonly vector: Float32Array;
-}

package/src/shared/embedding/typings/vector-index-meta.model.ts

@@ -1,14 +0,0 @@
-/** Metadata for a vector index file. */
-export interface VectorIndexMeta {
-   /** Name of the embedding model used. */
-   readonly modelName: string;
-
-   /** Embedding vector dimensions. */
-   readonly dimensions: number;
-
-   /** Number of vectors stored. */
-   readonly count: number;
-
-   /** Schema version. */
-   readonly version: number;
-}

package/src/shared/embedding/typings/vector-search-result.model.ts

@@ -1,8 +0,0 @@
-/** Result from a vector similarity search. */
-export interface VectorSearchResult {
-   /** Knowledge base ID of the matched section. */
-   readonly kbid: string;
-
-   /** Cosine similarity score (0.0 to 1.0). */
-   readonly score: number;
-}

package/src/shared/hash/functions/compute-sha256.ts

@@ -1,17 +0,0 @@
-/**
- * Computes the SHA-256 hash of a UTF-8 string.
- *
- * Uses the Web Crypto API (`crypto.subtle`), which is available
- * in both Deno and Node.js >= 20 without additional imports.
- *
- * @param input - the string to hash
- * @returns lowercase hex-encoded SHA-256 digest (64 characters)
- */
-export async function computeSha256(input: string): Promise<string> {
-   const data = new TextEncoder().encode(input);
-   const hash = await crypto.subtle.digest('SHA-256', data);
-   const bytes = new Uint8Array(hash);
-   return Array.from(bytes)
-      .map((b) => b.toString(16).padStart(2, '0'))
-      .join('');
-}

package/src/shared/hash/index.ts

@@ -1 +0,0 @@
-export { computeSha256 } from './functions/compute-sha256.ts';