ocr-provenance-mcp 1.0.0

package/dist/services/storage/vector.d.ts

@@ -0,0 +1,208 @@
+/**
+ * VectorService - sqlite-vec vector storage and similarity search
+ *
+ * Handles vector CRUD operations on vec_embeddings virtual table.
+ * Search results are self-contained per CP-002 (original text always included).
+ *
+ * Uses vec_distance_cosine() for all similarity calculations to ensure
+ * consistent cosine similarity scoring regardless of underlying index configuration.
+ *
+ * @module services/storage/vector
+ */
+import Database from 'better-sqlite3';
+/**
+ * Error codes for vector operations
+ */
+declare enum VectorErrorCode {
+    INVALID_VECTOR_DIMENSIONS = "INVALID_VECTOR_DIMENSIONS",
+    EMBEDDING_NOT_FOUND = "EMBEDDING_NOT_FOUND",
+    VEC_EXTENSION_NOT_LOADED = "VEC_EXTENSION_NOT_LOADED",
+    STORE_FAILED = "STORE_FAILED",
+    SEARCH_FAILED = "SEARCH_FAILED",
+    DELETE_FAILED = "DELETE_FAILED"
+}
+/**
+ * Custom error class for vector operations
+ * Includes error code and optional details for debugging
+ */
+export declare class VectorError extends Error {
+    readonly code: VectorErrorCode;
+    readonly details?: Record<string, unknown> | undefined;
+    constructor(message: string, code: VectorErrorCode, details?: Record<string, unknown> | undefined);
+}
+/**
+ * Search result from vector similarity search
+ * FLAT structure - no nested objects - direct from SQL join
+ *
+ * CRITICAL: original_text is ALWAYS included per CP-002
+ */
+export interface VectorSearchResult {
+    embedding_id: string;
+    chunk_id: string | null;
+    image_id: string | null;
+    extraction_id: string | null;
+    document_id: string;
+    result_type: 'chunk' | 'vlm' | 'extraction';
+    similarity_score: number;
+    distance: number;
+    original_text: string;
+    original_text_length: number;
+    source_file_path: string;
+    source_file_name: string;
+    source_file_hash: string;
+    page_number: number | null;
+    page_range: string | null;
+    character_start: number;
+    character_end: number;
+    chunk_index: number;
+    total_chunks: number;
+    model_name: string;
+    model_version: string;
+    provenance_id: string;
+    content_hash: string;
+    heading_context?: string | null;
+    section_path?: string | null;
+    content_types?: string | null;
+    is_atomic?: boolean;
+    chunk_page_range?: string | null;
+    heading_level?: number | null;
+    ocr_quality_score?: number | null;
+    doc_title?: string | null;
+    doc_author?: string | null;
+    doc_subject?: string | null;
+    overlap_previous?: number;
+    overlap_next?: number;
+    chunking_strategy?: string | null;
+    embedding_status?: string;
+    doc_page_count?: number | null;
+    datalab_mode?: string | null;
+}
+/**
+ * Options for vector similarity search
+ */
+export interface VectorSearchOptions {
+    /** Maximum results to return. Default: 10, Max: 100 */
+    limit?: number;
+    /** Similarity threshold 0-1. Default: 0.0 (no threshold) */
+    threshold?: number;
+    /** Filter by document IDs */
+    documentFilter?: string[];
+    /** Chunk-level filter conditions (from resolveChunkFilter) */
+    chunkFilter?: {
+        conditions: string[];
+        params: unknown[];
+    };
+    /** Page range filter for VLM/extraction results */
+    pageRangeFilter?: {
+        min_page?: number;
+        max_page?: number;
+    };
+}
+/**
+ * VectorService - handles vector storage and similarity search
+ *
+ * Wraps sqlite-vec operations for 768-dimensional vectors.
+ * All search results are self-contained with original text per CP-002.
+ */
+export declare class VectorService {
+    private readonly db;
+    private vecLoaded;
+    /**
+     * Create a new VectorService
+     *
+     * @param db - Better-sqlite3 database connection with schema already initialized
+     * @throws VectorError if sqlite-vec extension cannot be loaded
+     */
+    constructor(db: Database.Database);
+    /**
+     * Ensure sqlite-vec extension is loaded
+     * FAIL FAST if not available - no workarounds
+     */
+    private ensureVecExtension;
+    /**
+     * Store a vector in vec_embeddings
+     *
+     * PREREQUISITE: Embedding record must already exist in embeddings table
+     * (inserted via DatabaseService.insertEmbedding)
+     *
+     * @param embeddingId - Must match an existing embeddings.id
+     * @param vector - 768-dimensional Float32Array
+     * @throws VectorError if dimensions != 768 or embedding doesn't exist
+     */
+    storeVector(embeddingId: string, vector: Float32Array): void;
+    /**
+     * Store multiple vectors in a single transaction
+     *
+     * @param items - Array of {embeddingId, vector} pairs
+     * @returns Count of vectors stored
+     * @throws VectorError on any failure (transaction rolls back)
+     */
+    batchStoreVectors(items: Array<{
+        embeddingId: string;
+        vector: Float32Array;
+    }>): number;
+    /**
+     * Search for similar vectors using cosine distance
+     *
+     * CRITICAL: Results ALWAYS include original_text per CP-002.
+     * No follow-up queries needed - results are self-contained.
+     *
+     * Uses vec_distance_cosine() for consistent cosine similarity.
+     *
+     * @param queryVector - 768-dim query vector
+     * @param options - Search options (limit, threshold, filters)
+     * @returns Array of VectorSearchResult with original text and source info
+     */
+    searchSimilar(queryVector: Float32Array, options?: VectorSearchOptions): VectorSearchResult[];
+    /**
+     * Build chunk filter SQL fragment for vector search.
+     * Translates chunk filter conditions from 'c.' alias to 'ch.' alias used in vector queries.
+     */
+    private buildChunkFilterSQL;
+    /**
+     * Search with document filter
+     */
+    private searchWithFilter;
+    /**
+     * Search all vectors without filter
+     */
+    private searchAll;
+    /**
+     * Determine result type from row identity fields.
+     */
+    private static getResultType;
+    /**
+     * Map database rows to VectorSearchResult and filter by threshold.
+     * Applies quality-weighted scoring to all results.
+     */
+    private mapAndFilterResults;
+    /**
+     * Delete a vector from vec_embeddings
+     *
+     * @param embeddingId - The embedding ID
+     * @returns true if deleted, false if not found
+     */
+    deleteVector(embeddingId: string): boolean;
+    /**
+     * Delete all vectors for a document
+     *
+     * @param documentId - The document ID
+     * @returns Count of vectors deleted
+     */
+    deleteVectorsByDocumentId(documentId: string): number;
+    /**
+     * Get count of vectors stored
+     */
+    getVectorCount(): number;
+    /**
+     * Check if vector exists
+     */
+    vectorExists(embeddingId: string): boolean;
+    /**
+     * Get raw vector by embedding ID
+     * @returns Float32Array or null if not found
+     */
+    getVector(embeddingId: string): Float32Array | null;
+}
+export {};
+//# sourceMappingURL=vector.d.ts.map

package/dist/services/storage/vector.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vector.d.ts","sourceRoot":"","sources":["../../../src/services/storage/vector.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AAWtC;;GAEG;AACH,aAAK,eAAe;IAClB,yBAAyB,8BAA8B;IACvD,mBAAmB,wBAAwB;IAC3C,wBAAwB,6BAA6B;IACrD,YAAY,iBAAiB;IAC7B,aAAa,kBAAkB;IAC/B,aAAa,kBAAkB;CAChC;AAED;;;GAGG;AACH,qBAAa,WAAY,SAAQ,KAAK;aAGlB,IAAI,EAAE,eAAe;aACrB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;gBAFjD,OAAO,EAAE,MAAM,EACC,IAAI,EAAE,eAAe,EACrB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,YAAA;CAKpD;AAMD;;;;;GAKG;AACH,MAAM,WAAW,kBAAkB;IAEjC,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IAGpB,WAAW,EAAE,OAAO,GAAG,KAAK,GAAG,YAAY,CAAC;IAG5C,gBAAgB,EAAE,MAAM,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;IAGjB,aAAa,EAAE,MAAM,CAAC;IACtB,oBAAoB,EAAE,MAAM,CAAC;IAG7B,gBAAgB,EAAE,MAAM,CAAC;IACzB,gBAAgB,EAAE,MAAM,CAAC;IACzB,gBAAgB,EAAE,MAAM,CAAC;IAGzB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IAGrB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IAGtB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IAGrB,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAG9B,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAGlC,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,uDAAuD;IACvD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,4DAA4D;IAC5D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,6BAA6B;IAC7B,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,8DAA8D;IAC9D,WAAW,CAAC,EAAE;QAAE,UAAU,EAAE,MAAM,EAAE,CAAC;QAAC,MAAM,EAAE,OAAO,EAAE,CAAA;KAAE,CAAC;IAC1D,mDAAmD;IACnD,eAAe,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAC5D;AAqDD;;;;;GAKG;AACH,qBAAa,aAAa;IACxB,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAoB;IACvC,OAAO,CAAC,SAAS,CAAS;IAE1B;;;;;OAKG;gBACS,EAAE,EAAE,QAAQ,CAAC,QAAQ;IAKjC;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAoB1B;;;;;;;;;OASG;IACH,WAAW,CAAC,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,GAAG,IAAI;IAoC5D;;;;;;OAMG;IACH,iBAAiB,CAAC,KAAK,EAAE,KAAK,CAAC;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,YAAY,CAAA;KAAE,CAAC,GAAG,MAAM;IA0CtF;;;;;;;;;;;OAWG;IACH,aAAa,CACX,WAAW,EAAE,YAAY,EACzB,OAAO,GAAE,mBAAwB,GAChC,kBAAkB,EAAE;IA2BvB;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IAwB3B;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAoFxB;;OAEG;IACH,OAAO,CAAC,SAAS;IA8EjB;;OAEG;IACH,OAAO,CAAC,MAAM,CAAC,aAAa;IAM5B;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IAkF3B;;;;;OAKG;IACH,YAAY,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO;IAe1C;;;;;OAKG;IACH,yBAAyB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM;IAkCrD;;OAEG;IACH,cAAc,IAAI,MAAM;IAOxB;;OAEG;IACH,YAAY,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO;IAO1C;;;OAGG;IACH,SAAS,CAAC,WAAW,EAAE,MAAM,GAAG,YAAY,GAAG,IAAI;CAapD"}