ocr-provenance-mcp 1.0.0

package/dist/services/storage/database/chunk-operations.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunk-operations.d.ts","sourceRoot":"","sources":["../../../../src/services/storage/database/chunk-operations.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,KAAK,EAAE,MAAM,0BAA0B,CAAC;AAKjD;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC/B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,gBAAgB,CAAC,EAAE,SAAS,GAAG,UAAU,GAAG,QAAQ,CAAC;IACrD,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,UAAU,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACtD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AAED;;;;;;;GAOG;AACH,wBAAgB,WAAW,CACzB,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,YAAY,GAAG,kBAAkB,GAAG,aAAa,CAAC,EACrE,oBAAoB,EAAE,MAAM,IAAI,GAC/B,MAAM,CA8CR;AAED;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAC1B,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,MAAM,EAAE,IAAI,CAAC,KAAK,EAAE,YAAY,GAAG,kBAAkB,GAAG,aAAa,CAAC,EAAE,EACxE,oBAAoB,EAAE,MAAM,IAAI,EAChC,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,KAAK,CAAC,GACjC,MAAM,EAAE,CAwDV;AAED;;;;;;GAMG;AACH,wBAAgB,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EAAE,EAAE,EAAE,MAAM,GAAG,KAAK,GAAG,IAAI,CAIxE;AAED;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAGxF;AAED;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CACnC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,UAAU,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,GAC5C,KAAK,EAAE,CAgBT;AAED;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CACpC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,WAAW,EAAE,MAAM,EACnB,OAAO,CAAC,EAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,GAC5C,KAAK,EAAE,CAgBT;AAED;;;;;;GAMG;AACH,wBAAgB,yBAAyB,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,KAAK,EAAE,CAQxF;AAED;;;;;;;;GAQG;AACH,wBAAgB,0BAA0B,CACxC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,EAAE,EAAE,MAAM,EACV,MAAM,EAAE,SAAS,GAAG,UAAU,GAAG,QAAQ,EACzC,UAAU,EAAE,MAAM,GAAG,SAAS,EAC9B,sBAAsB,EAAE,MAAM,IAAI,GACjC,IAAI,CAcN;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,iBAAiB,CAC/B,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,kBAAkB,GAC1B;IAAE,MAAM,EAAE,KAAK,EAAE,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAmEpC;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,iBAAiB,CAC/B,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,MAAM,GACZ,KAAK,EAAE,CAST"}

package/dist/services/storage/database/chunk-operations.js

@@ -0,0 +1,306 @@
+/**
+ * Chunk operations for DatabaseService
+ *
+ * Handles all CRUD operations for text chunks including
+ * batch inserts, embedding status updates, filtered queries,
+ * and neighbor lookups.
+ */
+import { DatabaseError, DatabaseErrorCode } from './types.js';
+import { runWithForeignKeyCheck } from './helpers.js';
+import { rowToChunk } from './converters.js';
+/**
+ * Insert a chunk
+ *
+ * @param db - Database connection
+ * @param chunk - Chunk data (created_at, embedding_status, embedded_at will be generated)
+ * @param updateMetadataCounts - Callback to update metadata counts
+ * @returns string - The chunk ID
+ */
+export function insertChunk(db, chunk, updateMetadataCounts) {
+    const created_at = new Date().toISOString();
+    const stmt = db.prepare(`
+    INSERT INTO chunks (
+      id, document_id, ocr_result_id, text, text_hash, chunk_index,
+      character_start, character_end, page_number, page_range,
+      overlap_previous, overlap_next, provenance_id, created_at,
+      embedding_status, embedded_at, ocr_quality_score,
+      heading_context, heading_level, section_path,
+      content_types, is_atomic, chunking_strategy
+    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+  `);
+    runWithForeignKeyCheck(stmt, [
+        chunk.id,
+        chunk.document_id,
+        chunk.ocr_result_id,
+        chunk.text,
+        chunk.text_hash,
+        chunk.chunk_index,
+        chunk.character_start,
+        chunk.character_end,
+        chunk.page_number,
+        chunk.page_range,
+        chunk.overlap_previous,
+        chunk.overlap_next,
+        chunk.provenance_id,
+        created_at,
+        'pending',
+        null,
+        chunk.ocr_quality_score ?? null,
+        chunk.heading_context ?? null,
+        chunk.heading_level ?? null,
+        chunk.section_path ?? null,
+        chunk.content_types ?? null,
+        chunk.is_atomic ?? 0,
+        chunk.chunking_strategy ?? 'fixed',
+    ], 'inserting chunk: document_id, ocr_result_id, or provenance_id does not exist');
+    updateMetadataCounts();
+    return chunk.id;
+}
+/**
+ * Insert multiple chunks in a batch transaction
+ *
+ * @param db - Database connection
+ * @param chunks - Array of chunk data
+ * @param updateMetadataCounts - Callback to update metadata counts
+ * @param transaction - Transaction wrapper function
+ * @returns string[] - Array of chunk IDs
+ */
+export function insertChunks(db, chunks, updateMetadataCounts, transaction) {
+    if (chunks.length === 0) {
+        return [];
+    }
+    return transaction(() => {
+        const created_at = new Date().toISOString();
+        const ids = [];
+        const stmt = db.prepare(`
+      INSERT INTO chunks (
+        id, document_id, ocr_result_id, text, text_hash, chunk_index,
+        character_start, character_end, page_number, page_range,
+        overlap_previous, overlap_next, provenance_id, created_at,
+        embedding_status, embedded_at, ocr_quality_score,
+        heading_context, heading_level, section_path,
+        content_types, is_atomic, chunking_strategy
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `);
+        for (const chunk of chunks) {
+            runWithForeignKeyCheck(stmt, [
+                chunk.id,
+                chunk.document_id,
+                chunk.ocr_result_id,
+                chunk.text,
+                chunk.text_hash,
+                chunk.chunk_index,
+                chunk.character_start,
+                chunk.character_end,
+                chunk.page_number,
+                chunk.page_range,
+                chunk.overlap_previous,
+                chunk.overlap_next,
+                chunk.provenance_id,
+                created_at,
+                'pending',
+                null,
+                chunk.ocr_quality_score ?? null,
+                chunk.heading_context ?? null,
+                chunk.heading_level ?? null,
+                chunk.section_path ?? null,
+                chunk.content_types ?? null,
+                chunk.is_atomic ?? 0,
+                chunk.chunking_strategy ?? 'fixed',
+            ], `inserting chunk "${chunk.id}"`);
+            ids.push(chunk.id);
+        }
+        updateMetadataCounts();
+        return ids;
+    });
+}
+/**
+ * Get a chunk by ID
+ *
+ * @param db - Database connection
+ * @param id - Chunk ID
+ * @returns Chunk | null - The chunk or null if not found
+ */
+export function getChunk(db, id) {
+    const stmt = db.prepare('SELECT * FROM chunks WHERE id = ?');
+    const row = stmt.get(id);
+    return row ? rowToChunk(row) : null;
+}
+/**
+ * Check if a document has any chunks (M-9: avoids loading all chunk rows)
+ *
+ * @param db - Database connection
+ * @param documentId - Document ID
+ * @returns boolean - true if document has at least one chunk
+ */
+export function hasChunksByDocumentId(db, documentId) {
+    const stmt = db.prepare('SELECT 1 FROM chunks WHERE document_id = ? LIMIT 1');
+    return stmt.get(documentId) !== undefined;
+}
+/**
+ * Get all chunks for a document
+ *
+ * @param db - Database connection
+ * @param documentId - Document ID
+ * @returns Chunk[] - Array of chunks ordered by chunk_index
+ */
+export function getChunksByDocumentId(db, documentId, options) {
+    let sql = 'SELECT * FROM chunks WHERE document_id = ? ORDER BY chunk_index';
+    const params = [documentId];
+    const limit = options?.limit ?? 10000;
+    sql += ' LIMIT ?';
+    params.push(limit);
+    if (options?.offset !== undefined) {
+        sql += ' OFFSET ?';
+        params.push(options.offset);
+    }
+    const stmt = db.prepare(sql);
+    const rows = stmt.all(...params);
+    return rows.map(rowToChunk);
+}
+/**
+ * Get all chunks for an OCR result
+ *
+ * @param db - Database connection
+ * @param ocrResultId - OCR result ID
+ * @returns Chunk[] - Array of chunks ordered by chunk_index
+ */
+export function getChunksByOCRResultId(db, ocrResultId, options) {
+    let sql = 'SELECT * FROM chunks WHERE ocr_result_id = ? ORDER BY chunk_index';
+    const params = [ocrResultId];
+    const limit = options?.limit ?? 10000;
+    sql += ' LIMIT ?';
+    params.push(limit);
+    if (options?.offset !== undefined) {
+        sql += ' OFFSET ?';
+        params.push(options.offset);
+    }
+    const stmt = db.prepare(sql);
+    const rows = stmt.all(...params);
+    return rows.map(rowToChunk);
+}
+/**
+ * Get chunks pending embedding generation
+ *
+ * @param db - Database connection
+ * @param limit - Optional maximum number of chunks to return
+ * @returns Chunk[] - Array of pending chunks
+ */
+export function getPendingEmbeddingChunks(db, limit) {
+    // M-15: Default limit prevents unbounded loading of all pending chunks
+    const effectiveLimit = limit ?? 1000;
+    const query = "SELECT * FROM chunks WHERE embedding_status = 'pending' ORDER BY created_at LIMIT ?";
+    const stmt = db.prepare(query);
+    const rows = stmt.all(effectiveLimit);
+    return rows.map(rowToChunk);
+}
+/**
+ * Update chunk embedding status
+ *
+ * @param db - Database connection
+ * @param id - Chunk ID
+ * @param status - New embedding status
+ * @param embeddedAt - Optional ISO 8601 timestamp when embedded
+ * @param updateMetadataModified - Callback to update metadata modified timestamp
+ */
+export function updateChunkEmbeddingStatus(db, id, status, embeddedAt, updateMetadataModified) {
+    const stmt = db.prepare(`
+    UPDATE chunks
+    SET embedding_status = ?, embedded_at = ?
+    WHERE id = ?
+  `);
+    const result = stmt.run(status, embeddedAt ?? null, id);
+    if (result.changes === 0) {
+        throw new DatabaseError(`Chunk "${id}" not found`, DatabaseErrorCode.CHUNK_NOT_FOUND);
+    }
+    updateMetadataModified();
+}
+/**
+ * Get chunks for a document with dynamic filtering.
+ *
+ * Builds a parameterized WHERE clause from the provided filters.
+ * For content_type_filter, uses LIKE matching against the JSON-encoded
+ * content_types column (e.g., content_types LIKE '%table%').
+ *
+ * @param db - Database connection
+ * @param documentId - Document ID to filter by
+ * @param filters - Filter options
+ * @returns Object with chunks array and total count
+ */
+export function getChunksFiltered(db, documentId, filters) {
+    const conditions = ['document_id = ?'];
+    const params = [documentId];
+    if (filters.section_path_filter) {
+        conditions.push("section_path LIKE ? || '%'");
+        params.push(filters.section_path_filter);
+    }
+    if (filters.heading_filter) {
+        conditions.push("heading_context LIKE '%' || ? || '%'");
+        params.push(filters.heading_filter);
+    }
+    if (filters.content_type_filter && filters.content_type_filter.length > 0) {
+        // Each content type must be present in the JSON array string.
+        // Wrap value in JSON quotes to prevent substring false positives
+        // (e.g. "text" matching "context_text"). Matches search.ts resolveChunkFilter.
+        for (const ct of filters.content_type_filter) {
+            conditions.push("content_types LIKE '%' || ? || '%'");
+            params.push('"' + ct + '"');
+        }
+    }
+    if (filters.min_quality_score !== undefined) {
+        conditions.push('ocr_quality_score >= ?');
+        params.push(filters.min_quality_score);
+    }
+    if (filters.embedding_status) {
+        conditions.push('embedding_status = ?');
+        params.push(filters.embedding_status);
+    }
+    if (filters.is_atomic !== undefined) {
+        conditions.push('is_atomic = ?');
+        params.push(filters.is_atomic ? 1 : 0);
+    }
+    if (filters.page_range) {
+        if (filters.page_range.min_page !== undefined) {
+            conditions.push('page_number >= ?');
+            params.push(filters.page_range.min_page);
+        }
+        if (filters.page_range.max_page !== undefined) {
+            conditions.push('page_number <= ?');
+            params.push(filters.page_range.max_page);
+        }
+    }
+    const whereClause = ' WHERE ' + conditions.join(' AND ');
+    // Get total count with same filters
+    const countRow = db
+        .prepare(`SELECT COUNT(*) as total FROM chunks${whereClause}`)
+        .get(...params);
+    // Get paginated results
+    const limit = filters.limit ?? 50;
+    const offset = filters.offset ?? 0;
+    const dataQuery = `SELECT * FROM chunks${whereClause} ORDER BY chunk_index LIMIT ? OFFSET ?`;
+    const rows = db.prepare(dataQuery).all(...params, limit, offset);
+    return {
+        chunks: rows.map(rowToChunk),
+        total: countRow.total,
+    };
+}
+/**
+ * Get neighboring chunks around a given chunk index for context building.
+ *
+ * Returns chunks with chunk_index in range [chunkIndex - count, chunkIndex + count],
+ * ordered by chunk_index.
+ *
+ * @param db - Database connection
+ * @param documentId - Document ID
+ * @param chunkIndex - Center chunk index
+ * @param count - Number of neighbors on each side
+ * @returns Chunk[] - Array of neighboring chunks (including center)
+ */
+export function getChunkNeighbors(db, documentId, chunkIndex, count) {
+    const minIndex = Math.max(0, chunkIndex - count);
+    const maxIndex = chunkIndex + count;
+    const stmt = db.prepare('SELECT * FROM chunks WHERE document_id = ? AND chunk_index BETWEEN ? AND ? ORDER BY chunk_index');
+    const rows = stmt.all(documentId, minIndex, maxIndex);
+    return rows.map(rowToChunk);
+}
+//# sourceMappingURL=chunk-operations.js.map

package/dist/services/storage/database/chunk-operations.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunk-operations.js","sourceRoot":"","sources":["../../../../src/services/storage/database/chunk-operations.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAIH,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAY,MAAM,YAAY,CAAC;AACxE,OAAO,EAAE,sBAAsB,EAAE,MAAM,cAAc,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAkB7C;;;;;;;GAOG;AACH,MAAM,UAAU,WAAW,CACzB,EAAqB,EACrB,KAAqE,EACrE,oBAAgC;IAEhC,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAE5C,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC;;;;;;;;;GASvB,CAAC,CAAC;IAEH,sBAAsB,CACpB,IAAI,EACJ;QACE,KAAK,CAAC,EAAE;QACR,KAAK,CAAC,WAAW;QACjB,KAAK,CAAC,aAAa;QACnB,KAAK,CAAC,IAAI;QACV,KAAK,CAAC,SAAS;QACf,KAAK,CAAC,WAAW;QACjB,KAAK,CAAC,eAAe;QACrB,KAAK,CAAC,aAAa;QACnB,KAAK,CAAC,WAAW;QACjB,KAAK,CAAC,UAAU;QAChB,KAAK,CAAC,gBAAgB;QACtB,KAAK,CAAC,YAAY;QAClB,KAAK,CAAC,aAAa;QACnB,UAAU;QACV,SAAS;QACT,IAAI;QACJ,KAAK,CAAC,iBAAiB,IAAI,IAAI;QAC/B,KAAK,CAAC,eAAe,IAAI,IAAI;QAC7B,KAAK,CAAC,aAAa,IAAI,IAAI;QAC3B,KAAK,CAAC,YAAY,IAAI,IAAI;QAC1B,KAAK,CAAC,aAAa,IAAI,IAAI;QAC3B,KAAK,CAAC,SAAS,IAAI,CAAC;QACpB,KAAK,CAAC,iBAAiB,IAAI,OAAO;KACnC,EACD,8EAA8E,CAC/E,CAAC;IAEF,oBAAoB,EAAE,CAAC;IACvB,OAAO,KAAK,CAAC,EAAE,CAAC;AAClB,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,YAAY,CAC1B,EAAqB,EACrB,MAAwE,EACxE,oBAAgC,EAChC,WAAkC;IAElC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,OAAO,WAAW,CAAC,GAAG,EAAE;QACtB,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC5C,MAAM,GAAG,GAAa,EAAE,CAAC;QAEzB,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC;;;;;;;;;KASvB,CAAC,CAAC;QAEH,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,sBAAsB,CACpB,IAAI,EACJ;gBACE,KAAK,CAAC,EAAE;gBACR,KAAK,CAAC,WAAW;gBACjB,KAAK,CAAC,aAAa;gBACnB,KAAK,CAAC,IAAI;gBACV,KAAK,CAAC,SAAS;gBACf,KAAK,CAAC,WAAW;gBACjB,KAAK,CAAC,eAAe;gBACrB,KAAK,CAAC,aAAa;gBACnB,KAAK,CAAC,WAAW;gBACjB,KAAK,CAAC,UAAU;gBAChB,KAAK,CAAC,gBAAgB;gBACtB,KAAK,CAAC,YAAY;gBAClB,KAAK,CAAC,aAAa;gBACnB,UAAU;gBACV,SAAS;gBACT,IAAI;gBACJ,KAAK,CAAC,iBAAiB,IAAI,IAAI;gBAC/B,KAAK,CAAC,eAAe,IAAI,IAAI;gBAC7B,KAAK,CAAC,aAAa,IAAI,IAAI;gBAC3B,KAAK,CAAC,YAAY,IAAI,IAAI;gBAC1B,KAAK,CAAC,aAAa,IAAI,IAAI;gBAC3B,KAAK,CAAC,SAAS,IAAI,CAAC;gBACpB,KAAK,CAAC,iBAAiB,IAAI,OAAO;aACnC,EACD,oBAAoB,KAAK,CAAC,EAAE,GAAG,CAChC,CAAC;YACF,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACrB,CAAC;QAED,oBAAoB,EAAE,CAAC;QACvB,OAAO,GAAG,CAAC;IACb,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,QAAQ,CAAC,EAAqB,EAAE,EAAU;IACxD,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC,mCAAmC,CAAC,CAAC;IAC7D,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAyB,CAAC;IACjD,OAAO,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACtC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,qBAAqB,CAAC,EAAqB,EAAE,UAAkB;IAC7E,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC,oDAAoD,CAAC,CAAC;IAC9E,OAAO,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,KAAK,SAAS,CAAC;AAC5C,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,qBAAqB,CACnC,EAAqB,EACrB,UAAkB,EAClB,OAA6C;IAE7C,IAAI,GAAG,GAAG,iEAAiE,CAAC;IAC5E,MAAM,MAAM,GAAwB,CAAC,UAAU,CAAC,CAAC;IAEjD,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,KAAK,CAAC;IACtC,GAAG,IAAI,UAAU,CAAC;IAClB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAEnB,IAAI,OAAO,EAAE,MAAM,KAAK,SAAS,EAAE,CAAC;QAClC,GAAG,IAAI,WAAW,CAAC;QACnB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC9B,CAAC;IAED,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC7B,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAe,CAAC;IAC/C,OAAO,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;AAC9B,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,sBAAsB,CACpC,EAAqB,EACrB,WAAmB,EACnB,OAA6C;IAE7C,IAAI,GAAG,GAAG,mEAAmE,CAAC;IAC9E,MAAM,MAAM,GAAwB,CAAC,WAAW,CAAC,CAAC;IAElD,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,KAAK,CAAC;IACtC,GAAG,IAAI,UAAU,CAAC;IAClB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAEnB,IAAI,OAAO,EAAE,MAAM,KAAK,SAAS,EAAE,CAAC;QAClC,GAAG,IAAI,WAAW,CAAC;QACnB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC9B,CAAC;IAED,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC7B,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAe,CAAC;IAC/C,OAAO,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;AAC9B,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,yBAAyB,CAAC,EAAqB,EAAE,KAAc;IAC7E,uEAAuE;IACvE,MAAM,cAAc,GAAG,KAAK,IAAI,IAAI,CAAC;IACrC,MAAM,KAAK,GACT,qFAAqF,CAAC;IACxF,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,cAAc,CAAe,CAAC;IACpD,OAAO,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;AAC9B,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,0BAA0B,CACxC,EAAqB,EACrB,EAAU,EACV,MAAyC,EACzC,UAA8B,EAC9B,sBAAkC;IAElC,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC;;;;GAIvB,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,UAAU,IAAI,IAAI,EAAE,EAAE,CAAC,CAAC;IAExD,IAAI,MAAM,CAAC,OAAO,KAAK,CAAC,EAAE,CAAC;QACzB,MAAM,IAAI,aAAa,CAAC,UAAU,EAAE,aAAa,EAAE,iBAAiB,CAAC,eAAe,CAAC,CAAC;IACxF,CAAC;IAED,sBAAsB,EAAE,CAAC;AAC3B,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,iBAAiB,CAC/B,EAAqB,EACrB,UAAkB,EAClB,OAA2B;IAE3B,MAAM,UAAU,GAAa,CAAC,iBAAiB,CAAC,CAAC;IACjD,MAAM,MAAM,GAAwB,CAAC,UAAU,CAAC,CAAC;IAEjD,IAAI,OAAO,CAAC,mBAAmB,EAAE,CAAC;QAChC,UAAU,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;QAC9C,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC;IAC3C,CAAC;IAED,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;QAC3B,UAAU,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC;QACxD,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;IACtC,CAAC;IAED,IAAI,OAAO,CAAC,mBAAmB,IAAI,OAAO,CAAC,mBAAmB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1E,8DAA8D;QAC9D,iEAAiE;QACjE,+EAA+E;QAC/E,KAAK,MAAM,EAAE,IAAI,OAAO,CAAC,mBAAmB,EAAE,CAAC;YAC7C,UAAU,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;YACtD,MAAM,CAAC,IAAI,CAAC,GAAG,GAAG,EAAE,GAAG,GAAG,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;IAED,IAAI,OAAO,CAAC,iBAAiB,KAAK,SAAS,EAAE,CAAC;QAC5C,UAAU,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;QAC1C,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;IACzC,CAAC;IAED,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC;QAC7B,UAAU,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;QACxC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;IACxC,CAAC;IAED,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QACpC,UAAU,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QACjC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzC,CAAC;IAED,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;QACvB,IAAI,OAAO,CAAC,UAAU,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC9C,UAAU,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;YACpC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;QAC3C,CAAC;QACD,IAAI,OAAO,CAAC,UAAU,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC9C,UAAU,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;YACpC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;IAED,MAAM,WAAW,GAAG,SAAS,GAAG,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAEzD,oCAAoC;IACpC,MAAM,QAAQ,GAAG,EAAE;SAChB,OAAO,CAAC,uCAAuC,WAAW,EAAE,CAAC;SAC7D,GAAG,CAAC,GAAG,MAAM,CAAsB,CAAC;IAEvC,wBAAwB;IACxB,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;IAClC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;IACnC,MAAM,SAAS,GAAG,uBAAuB,WAAW,wCAAwC,CAAC;IAC7F,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,EAAE,KAAK,EAAE,MAAM,CAAe,CAAC;IAE/E,OAAO;QACL,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC;QAC5B,KAAK,EAAE,QAAQ,CAAC,KAAK;KACtB,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,iBAAiB,CAC/B,EAAqB,EACrB,UAAkB,EAClB,UAAkB,EAClB,KAAa;IAEb,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,GAAG,KAAK,CAAC,CAAC;IACjD,MAAM,QAAQ,GAAG,UAAU,GAAG,KAAK,CAAC;IAEpC,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CACrB,iGAAiG,CAClG,CAAC;IACF,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,QAAQ,EAAE,QAAQ,CAAe,CAAC;IACpE,OAAO,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;AAC9B,CAAC"}

package/dist/services/storage/database/cluster-operations.d.ts

@@ -0,0 +1,97 @@
+/**
+ * Cluster operations for DatabaseService
+ *
+ * Handles CRUD operations for the clusters and document_clusters tables.
+ */
+import Database from 'better-sqlite3';
+import type { Cluster, DocumentCluster } from '../../../models/cluster.js';
+/**
+ * Insert a cluster record
+ */
+export declare function insertCluster(db: Database.Database, cluster: Cluster): string;
+/**
+ * Get a cluster by ID
+ */
+export declare function getCluster(db: Database.Database, id: string): Cluster | null;
+/**
+ * List clusters with optional filters and pagination
+ */
+export declare function listClusters(db: Database.Database, options?: {
+    run_id?: string;
+    classification_tag?: string;
+    limit?: number;
+    offset?: number;
+}): Cluster[];
+/**
+ * Delete all clusters and their document assignments for a run.
+ * First deletes document_clusters, then clusters.
+ * Returns the number of clusters deleted.
+ */
+export declare function deleteClustersByRunId(db: Database.Database, runId: string): number;
+/**
+ * Insert a document-cluster assignment
+ */
+export declare function insertDocumentCluster(db: Database.Database, dc: DocumentCluster): string;
+/**
+ * Get all documents in a cluster, joined with documents for file_name
+ */
+export declare function getClusterDocuments(db: Database.Database, clusterId: string): Array<{
+    document_id: string;
+    file_name: string;
+    similarity_to_centroid: number;
+    membership_probability: number;
+}>;
+/**
+ * Lightweight cluster summary (excludes large JSON fields)
+ */
+interface ClusterSummary {
+    id: string;
+    run_id: string;
+    cluster_index: number;
+    label: string | null;
+    classification_tag: string | null;
+    document_count: number;
+    coherence_score: number | null;
+    created_at: string;
+}
+/**
+ * Get cluster summaries for a run (lightweight: no JSON blobs)
+ */
+export declare function getClusterSummariesByRunId(db: Database.Database, runId: string): ClusterSummary[];
+/**
+ * Get cluster summaries for a document (via document_clusters join)
+ */
+export declare function getClusterSummariesForDocument(db: Database.Database, documentId: string): ClusterSummary[];
+/**
+ * Reassign a document from its current cluster to a different target cluster.
+ * Deletes existing document_clusters entries for this document within the same run,
+ * inserts a new assignment, and updates member_count on both old and new clusters.
+ *
+ * @returns Object with old_cluster_id (null if not previously assigned) and run_id
+ */
+export declare function reassignDocument(db: Database.Database, documentId: string, targetClusterId: string): {
+    old_cluster_id: string | null;
+    run_id: string;
+};
+/**
+ * Merge two clusters into one. All documents from cluster2 are moved to cluster1.
+ * cluster2 is deleted after the merge.
+ *
+ * Both clusters must belong to the same run_id.
+ *
+ * @returns Object with merged_cluster_id and documents_moved count
+ */
+export declare function mergeClusters(db: Database.Database, clusterId1: string, clusterId2: string): {
+    merged_cluster_id: string;
+    documents_moved: number;
+};
+/**
+ * Get aggregate clustering statistics
+ */
+export declare function getClusteringStats(db: Database.Database): {
+    total_clusters: number;
+    total_runs: number;
+    avg_coherence: number | null;
+};
+export {};
+//# sourceMappingURL=cluster-operations.d.ts.map

package/dist/services/storage/database/cluster-operations.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cluster-operations.d.ts","sourceRoot":"","sources":["../../../../src/services/storage/database/cluster-operations.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AAEtC,OAAO,KAAK,EAAE,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAC;AAK3E;;GAEG;AACH,wBAAgB,aAAa,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,GAAG,MAAM,CAkC7E;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,GAAG,IAAI,CAG5E;AAED;;GAEG;AACH,wBAAgB,YAAY,CAC1B,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,OAAO,CAAC,EAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,GAC1F,OAAO,EAAE,CAoBX;AAED;;;;GAIG;AACH,wBAAgB,qBAAqB,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CA0BlF;AAID;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EAAE,EAAE,EAAE,eAAe,GAAG,MAAM,CAuBxF;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CACjC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,SAAS,EAAE,MAAM,GAChB,KAAK,CAAC;IACP,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,sBAAsB,EAAE,MAAM,CAAC;CAChC,CAAC,CAeD;AAID;;GAEG;AACH,UAAU,cAAc;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,wBAAgB,0BAA0B,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,GAAG,cAAc,EAAE,CAUjG;AAED;;GAEG;AACH,wBAAgB,8BAA8B,CAC5C,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,UAAU,EAAE,MAAM,GACjB,cAAc,EAAE,CAWlB;AAID;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAC9B,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,UAAU,EAAE,MAAM,EAClB,eAAe,EAAE,MAAM,GACtB;IAAE,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAkDnD;AAED;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAC3B,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,GACjB;IAAE,iBAAiB,EAAE,MAAM,CAAC;IAAC,eAAe,EAAE,MAAM,CAAA;CAAE,CAuDxD;AAID;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,GAAG;IACzD,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B,CAWA"}