ocr-provenance-mcp 1.0.0

package/dist/services/storage/database/cluster-operations.js

@@ -0,0 +1,258 @@
+/**
+ * Cluster operations for DatabaseService
+ *
+ * Handles CRUD operations for the clusters and document_clusters tables.
+ */
+import { v4 as uuidv4 } from 'uuid';
+import { runWithForeignKeyCheck } from './helpers.js';
+// --- Cluster CRUD ---
+/**
+ * Insert a cluster record
+ */
+export function insertCluster(db, cluster) {
+    const stmt = db.prepare(`
+    INSERT INTO clusters (id, run_id, cluster_index, label, description,
+      classification_tag, document_count, centroid_json, top_terms_json,
+      coherence_score, algorithm, algorithm_params_json, silhouette_score,
+      content_hash, provenance_id, created_at, processing_duration_ms)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+  `);
+    runWithForeignKeyCheck(stmt, [
+        cluster.id,
+        cluster.run_id,
+        cluster.cluster_index,
+        cluster.label,
+        cluster.description,
+        cluster.classification_tag,
+        cluster.document_count,
+        cluster.centroid_json,
+        cluster.top_terms_json,
+        cluster.coherence_score,
+        cluster.algorithm,
+        cluster.algorithm_params_json,
+        cluster.silhouette_score,
+        cluster.content_hash,
+        cluster.provenance_id,
+        cluster.created_at,
+        cluster.processing_duration_ms,
+    ], `inserting cluster: FK violation for provenance_id="${cluster.provenance_id}"`);
+    return cluster.id;
+}
+/**
+ * Get a cluster by ID
+ */
+export function getCluster(db, id) {
+    const row = db.prepare('SELECT * FROM clusters WHERE id = ?').get(id);
+    return row ?? null;
+}
+/**
+ * List clusters with optional filters and pagination
+ */
+export function listClusters(db, options) {
+    const params = [];
+    const conditions = [];
+    if (options?.run_id) {
+        conditions.push('run_id = ?');
+        params.push(options.run_id);
+    }
+    if (options?.classification_tag) {
+        conditions.push('classification_tag = ?');
+        params.push(options.classification_tag);
+    }
+    const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
+    params.push(options?.limit ?? 50, options?.offset ?? 0);
+    return db
+        .prepare(`SELECT * FROM clusters ${where} ORDER BY created_at DESC LIMIT ? OFFSET ?`)
+        .all(...params);
+}
+/**
+ * Delete all clusters and their document assignments for a run.
+ * First deletes document_clusters, then clusters.
+ * Returns the number of clusters deleted.
+ */
+export function deleteClustersByRunId(db, runId) {
+    // Collect provenance IDs before deleting clusters (clusters.provenance_id NOT NULL REFERENCES provenance(id))
+    const provenanceIds = db
+        .prepare('SELECT provenance_id FROM clusters WHERE run_id = ?')
+        .all(runId);
+    db.prepare('DELETE FROM document_clusters WHERE run_id = ?').run(runId);
+    const result = db.prepare('DELETE FROM clusters WHERE run_id = ?').run(runId);
+    // Clean up orphaned provenance records now that the FK references are gone
+    if (provenanceIds.length > 0) {
+        const deleteProvStmt = db.prepare('DELETE FROM provenance WHERE id = ?');
+        for (const { provenance_id } of provenanceIds) {
+            try {
+                deleteProvStmt.run(provenance_id);
+            }
+            catch (e) {
+                // Log but don't fail if provenance record is still referenced elsewhere
+                console.error(`[cluster-operations] Failed to delete provenance ${provenance_id}:`, e instanceof Error ? e.message : String(e));
+            }
+        }
+    }
+    return result.changes;
+}
+// --- DocumentCluster CRUD ---
+/**
+ * Insert a document-cluster assignment
+ */
+export function insertDocumentCluster(db, dc) {
+    const stmt = db.prepare(`
+    INSERT INTO document_clusters (id, document_id, cluster_id, run_id,
+      similarity_to_centroid, membership_probability, is_noise, assigned_at)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+  `);
+    runWithForeignKeyCheck(stmt, [
+        dc.id,
+        dc.document_id,
+        dc.cluster_id,
+        dc.run_id,
+        dc.similarity_to_centroid,
+        dc.membership_probability,
+        dc.is_noise ? 1 : 0,
+        dc.assigned_at,
+    ], `inserting document_cluster: FK violation for document_id="${dc.document_id}" or cluster_id="${dc.cluster_id}"`);
+    return dc.id;
+}
+/**
+ * Get all documents in a cluster, joined with documents for file_name
+ */
+export function getClusterDocuments(db, clusterId) {
+    return db
+        .prepare(`SELECT dc.document_id, d.file_name, dc.similarity_to_centroid, dc.membership_probability
+     FROM document_clusters dc
+     JOIN documents d ON d.id = dc.document_id
+     WHERE dc.cluster_id = ?
+     ORDER BY dc.similarity_to_centroid DESC`)
+        .all(clusterId);
+}
+/**
+ * Get cluster summaries for a run (lightweight: no JSON blobs)
+ */
+export function getClusterSummariesByRunId(db, runId) {
+    return db
+        .prepare(`SELECT id, run_id, cluster_index, label, classification_tag, document_count,
+            coherence_score, created_at
+     FROM clusters
+     WHERE run_id = ?
+     ORDER BY cluster_index ASC`)
+        .all(runId);
+}
+/**
+ * Get cluster summaries for a document (via document_clusters join)
+ */
+export function getClusterSummariesForDocument(db, documentId) {
+    return db
+        .prepare(`SELECT c.id, c.run_id, c.cluster_index, c.label, c.classification_tag,
+            c.document_count, c.coherence_score, c.created_at
+     FROM clusters c
+     JOIN document_clusters dc ON dc.cluster_id = c.id
+     WHERE dc.document_id = ?
+     ORDER BY c.created_at DESC`)
+        .all(documentId);
+}
+// --- Reassign & Merge ---
+/**
+ * Reassign a document from its current cluster to a different target cluster.
+ * Deletes existing document_clusters entries for this document within the same run,
+ * inserts a new assignment, and updates member_count on both old and new clusters.
+ *
+ * @returns Object with old_cluster_id (null if not previously assigned) and run_id
+ */
+export function reassignDocument(db, documentId, targetClusterId) {
+    // Get the target cluster to know the run_id
+    const targetCluster = getCluster(db, targetClusterId);
+    if (!targetCluster) {
+        throw new Error(`Target cluster "${targetClusterId}" not found`);
+    }
+    const runId = targetCluster.run_id;
+    // Find existing assignment for this document in this run
+    const existing = db
+        .prepare('SELECT id, cluster_id FROM document_clusters WHERE document_id = ? AND run_id = ?')
+        .get(documentId, runId);
+    const oldClusterId = existing?.cluster_id ?? null;
+    if (oldClusterId === targetClusterId) {
+        // Already in the target cluster, no-op
+        return { old_cluster_id: oldClusterId, run_id: runId };
+    }
+    // Delete existing assignment for this document in this run
+    if (existing) {
+        db.prepare('DELETE FROM document_clusters WHERE id = ?').run(existing.id);
+        // Decrement old cluster's document_count (if it was in a cluster, not noise)
+        if (oldClusterId) {
+            db.prepare('UPDATE clusters SET document_count = MAX(0, document_count - 1) WHERE id = ?').run(oldClusterId);
+        }
+    }
+    // Insert new assignment
+    const now = new Date().toISOString();
+    const dcId = uuidv4();
+    db.prepare(`
+    INSERT INTO document_clusters (id, document_id, cluster_id, run_id,
+      similarity_to_centroid, membership_probability, is_noise, assigned_at)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+  `).run(dcId, documentId, targetClusterId, runId, 0, 1.0, 0, now);
+    // Increment target cluster's document_count
+    db.prepare('UPDATE clusters SET document_count = document_count + 1 WHERE id = ?').run(targetClusterId);
+    return { old_cluster_id: oldClusterId, run_id: runId };
+}
+/**
+ * Merge two clusters into one. All documents from cluster2 are moved to cluster1.
+ * cluster2 is deleted after the merge.
+ *
+ * Both clusters must belong to the same run_id.
+ *
+ * @returns Object with merged_cluster_id and documents_moved count
+ */
+export function mergeClusters(db, clusterId1, clusterId2) {
+    // Validation outside transaction - read-only lookups
+    const cluster1 = getCluster(db, clusterId1);
+    if (!cluster1) {
+        throw new Error(`Cluster "${clusterId1}" not found`);
+    }
+    const cluster2 = getCluster(db, clusterId2);
+    if (!cluster2) {
+        throw new Error(`Cluster "${clusterId2}" not found`);
+    }
+    if (cluster1.run_id !== cluster2.run_id) {
+        throw new Error(`Cannot merge clusters from different runs: "${cluster1.run_id}" vs "${cluster2.run_id}"`);
+    }
+    // M-8: Wrap all mutations in a transaction so a crash mid-merge
+    // cannot leave cluster state inconsistent (e.g., documents moved
+    // but old cluster not deleted, or count not updated).
+    const cluster2ProvId = cluster2.provenance_id;
+    const runInTransaction = db.transaction(() => {
+        // Move all document_clusters from cluster2 to cluster1
+        const moveResult = db
+            .prepare('UPDATE document_clusters SET cluster_id = ? WHERE cluster_id = ?')
+            .run(clusterId1, clusterId2);
+        const documentsMoved = moveResult.changes;
+        // Update cluster1's document_count
+        db.prepare('UPDATE clusters SET document_count = document_count + ? WHERE id = ?').run(documentsMoved, clusterId1);
+        // Delete cluster2 record
+        db.prepare('DELETE FROM clusters WHERE id = ?').run(clusterId2);
+        // Clean up cluster2's provenance record
+        try {
+            db.prepare('DELETE FROM provenance WHERE id = ?').run(cluster2ProvId);
+        }
+        catch (e) {
+            console.error(`[cluster-operations] Failed to delete provenance ${cluster2ProvId}:`, e instanceof Error ? e.message : String(e));
+        }
+        return documentsMoved;
+    });
+    const documentsMoved = runInTransaction();
+    return { merged_cluster_id: clusterId1, documents_moved: documentsMoved };
+}
+// --- Stats ---
+/**
+ * Get aggregate clustering statistics
+ */
+export function getClusteringStats(db) {
+    const row = db
+        .prepare(`SELECT COUNT(*) AS total_clusters,
+            COUNT(DISTINCT run_id) AS total_runs,
+            AVG(coherence_score) AS avg_coherence
+     FROM clusters`)
+        .get();
+    return row;
+}
+//# sourceMappingURL=cluster-operations.js.map

package/dist/services/storage/database/cluster-operations.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cluster-operations.js","sourceRoot":"","sources":["../../../../src/services/storage/database/cluster-operations.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,MAAM,CAAC;AAEpC,OAAO,EAAE,sBAAsB,EAAE,MAAM,cAAc,CAAC;AAEtD,uBAAuB;AAEvB;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,EAAqB,EAAE,OAAgB;IACnE,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC;;;;;;GAMvB,CAAC,CAAC;IAEH,sBAAsB,CACpB,IAAI,EACJ;QACE,OAAO,CAAC,EAAE;QACV,OAAO,CAAC,MAAM;QACd,OAAO,CAAC,aAAa;QACrB,OAAO,CAAC,KAAK;QACb,OAAO,CAAC,WAAW;QACnB,OAAO,CAAC,kBAAkB;QAC1B,OAAO,CAAC,cAAc;QACtB,OAAO,CAAC,aAAa;QACrB,OAAO,CAAC,cAAc;QACtB,OAAO,CAAC,eAAe;QACvB,OAAO,CAAC,SAAS;QACjB,OAAO,CAAC,qBAAqB;QAC7B,OAAO,CAAC,gBAAgB;QACxB,OAAO,CAAC,YAAY;QACpB,OAAO,CAAC,aAAa;QACrB,OAAO,CAAC,UAAU;QAClB,OAAO,CAAC,sBAAsB;KAC/B,EACD,sDAAsD,OAAO,CAAC,aAAa,GAAG,CAC/E,CAAC;IAEF,OAAO,OAAO,CAAC,EAAE,CAAC;AACpB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,UAAU,CAAC,EAAqB,EAAE,EAAU;IAC1D,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,qCAAqC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAwB,CAAC;IAC7F,OAAO,GAAG,IAAI,IAAI,CAAC;AACrB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAC1B,EAAqB,EACrB,OAA2F;IAE3F,MAAM,MAAM,GAAwB,EAAE,CAAC;IACvC,MAAM,UAAU,GAAa,EAAE,CAAC;IAEhC,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QACpB,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC9B,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC9B,CAAC;IAED,IAAI,OAAO,EAAE,kBAAkB,EAAE,CAAC;QAChC,UAAU,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;QAC1C,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;IAC1C,CAAC;IAED,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAC/E,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC;IAExD,OAAO,EAAE;SACN,OAAO,CAAC,0BAA0B,KAAK,4CAA4C,CAAC;SACpF,GAAG,CAAC,GAAG,MAAM,CAAc,CAAC;AACjC,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,qBAAqB,CAAC,EAAqB,EAAE,KAAa;IACxE,8GAA8G;IAC9G,MAAM,aAAa,GAAG,EAAE;SACrB,OAAO,CAAC,qDAAqD,CAAC;SAC9D,GAAG,CAAC,KAAK,CAAgC,CAAC;IAE7C,EAAE,CAAC,OAAO,CAAC,gDAAgD,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACxE,MAAM,MAAM,GAAG,EAAE,CAAC,OAAO,CAAC,uCAAuC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAE9E,2EAA2E;IAC3E,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,MAAM,cAAc,GAAG,EAAE,CAAC,OAAO,CAAC,qCAAqC,CAAC,CAAC;QACzE,KAAK,MAAM,EAAE,aAAa,EAAE,IAAI,aAAa,EAAE,CAAC;YAC9C,IAAI,CAAC;gBACH,cAAc,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;YACpC,CAAC;YAAC,OAAO,CAAU,EAAE,CAAC;gBACpB,wEAAwE;gBACxE,OAAO,CAAC,KAAK,CACX,oDAAoD,aAAa,GAAG,EACpE,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAC3C,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC,OAAO,CAAC;AACxB,CAAC;AAED,+BAA+B;AAE/B;;GAEG;AACH,MAAM,UAAU,qBAAqB,CAAC,EAAqB,EAAE,EAAmB;IAC9E,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC;;;;GAIvB,CAAC,CAAC;IAEH,sBAAsB,CACpB,IAAI,EACJ;QACE,EAAE,CAAC,EAAE;QACL,EAAE,CAAC,WAAW;QACd,EAAE,CAAC,UAAU;QACb,EAAE,CAAC,MAAM;QACT,EAAE,CAAC,sBAAsB;QACzB,EAAE,CAAC,sBAAsB;QACzB,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,EAAE,CAAC,WAAW;KACf,EACD,6DAA6D,EAAE,CAAC,WAAW,oBAAoB,EAAE,CAAC,UAAU,GAAG,CAChH,CAAC;IAEF,OAAO,EAAE,CAAC,EAAE,CAAC;AACf,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CACjC,EAAqB,EACrB,SAAiB;IAOjB,OAAO,EAAE;SACN,OAAO,CACN;;;;6CAIuC,CACxC;SACA,GAAG,CAAC,SAAS,CAKd,CAAC;AACL,CAAC;AAkBD;;GAEG;AACH,MAAM,UAAU,0BAA0B,CAAC,EAAqB,EAAE,KAAa;IAC7E,OAAO,EAAE;SACN,OAAO,CACN;;;;gCAI0B,CAC3B;SACA,GAAG,CAAC,KAAK,CAAqB,CAAC;AACpC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,8BAA8B,CAC5C,EAAqB,EACrB,UAAkB;IAElB,OAAO,EAAE;SACN,OAAO,CACN;;;;;gCAK0B,CAC3B;SACA,GAAG,CAAC,UAAU,CAAqB,CAAC;AACzC,CAAC;AAED,2BAA2B;AAE3B;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAC9B,EAAqB,EACrB,UAAkB,EAClB,eAAuB;IAEvB,4CAA4C;IAC5C,MAAM,aAAa,GAAG,UAAU,CAAC,EAAE,EAAE,eAAe,CAAC,CAAC;IACtD,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,mBAAmB,eAAe,aAAa,CAAC,CAAC;IACnE,CAAC;IAED,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC;IAEnC,yDAAyD;IACzD,MAAM,QAAQ,GAAG,EAAE;SAChB,OAAO,CACN,mFAAmF,CACpF;SACA,GAAG,CAAC,UAAU,EAAE,KAAK,CAA0D,CAAC;IAEnF,MAAM,YAAY,GAAG,QAAQ,EAAE,UAAU,IAAI,IAAI,CAAC;IAElD,IAAI,YAAY,KAAK,eAAe,EAAE,CAAC;QACrC,uCAAuC;QACvC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;IACzD,CAAC;IAED,2DAA2D;IAC3D,IAAI,QAAQ,EAAE,CAAC;QACb,EAAE,CAAC,OAAO,CAAC,4CAA4C,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QAE1E,6EAA6E;QAC7E,IAAI,YAAY,EAAE,CAAC;YACjB,EAAE,CAAC,OAAO,CACR,8EAA8E,CAC/E,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAED,wBAAwB;IACxB,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACrC,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC;IACtB,EAAE,CAAC,OAAO,CAAC;;;;GAIV,CAAC,CAAC,GAAG,CAAC,IAAI,EAAE,UAAU,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC;IAEjE,4CAA4C;IAC5C,EAAE,CAAC,OAAO,CACR,sEAAsE,CACvE,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;IAEvB,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AACzD,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,aAAa,CAC3B,EAAqB,EACrB,UAAkB,EAClB,UAAkB;IAElB,qDAAqD;IACrD,MAAM,QAAQ,GAAG,UAAU,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;IAC5C,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,YAAY,UAAU,aAAa,CAAC,CAAC;IACvD,CAAC;IAED,MAAM,QAAQ,GAAG,UAAU,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;IAC5C,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,YAAY,UAAU,aAAa,CAAC,CAAC;IACvD,CAAC;IAED,IAAI,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CACb,+CAA+C,QAAQ,CAAC,MAAM,SAAS,QAAQ,CAAC,MAAM,GAAG,CAC1F,CAAC;IACJ,CAAC;IAED,gEAAgE;IAChE,iEAAiE;IACjE,sDAAsD;IACtD,MAAM,cAAc,GAAG,QAAQ,CAAC,aAAa,CAAC;IAE9C,MAAM,gBAAgB,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;QAC3C,uDAAuD;QACvD,MAAM,UAAU,GAAG,EAAE;aAClB,OAAO,CAAC,kEAAkE,CAAC;aAC3E,GAAG,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QAE/B,MAAM,cAAc,GAAG,UAAU,CAAC,OAAO,CAAC;QAE1C,mCAAmC;QACnC,EAAE,CAAC,OAAO,CACR,sEAAsE,CACvE,CAAC,GAAG,CAAC,cAAc,EAAE,UAAU,CAAC,CAAC;QAElC,yBAAyB;QACzB,EAAE,CAAC,OAAO,CAAC,mCAAmC,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAEhE,wCAAwC;QACxC,IAAI,CAAC;YACH,EAAE,CAAC,OAAO,CAAC,qCAAqC,CAAC,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;QACxE,CAAC;QAAC,OAAO,CAAU,EAAE,CAAC;YACpB,OAAO,CAAC,KAAK,CACX,oDAAoD,cAAc,GAAG,EACrE,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAC3C,CAAC;QACJ,CAAC;QAED,OAAO,cAAc,CAAC;IACxB,CAAC,CAAC,CAAC;IAEH,MAAM,cAAc,GAAG,gBAAgB,EAAE,CAAC;IAE1C,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,eAAe,EAAE,cAAc,EAAE,CAAC;AAC5E,CAAC;AAED,gBAAgB;AAEhB;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,EAAqB;IAKtD,MAAM,GAAG,GAAG,EAAE;SACX,OAAO,CACN;;;mBAGa,CACd;SACA,GAAG,EAAkF,CAAC;IAEzF,OAAO,GAAG,CAAC;AACb,CAAC"}

package/dist/services/storage/database/comparison-operations.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Comparison operations for DatabaseService
+ *
+ * Handles CRUD operations for the comparisons table.
+ */
+import Database from 'better-sqlite3';
+import type { Comparison } from '../../../models/comparison.js';
+/**
+ * Insert a comparison record
+ */
+export declare function insertComparison(db: Database.Database, comparison: Comparison): string;
+/**
+ * Get a comparison by ID
+ */
+export declare function getComparison(db: Database.Database, id: string): Comparison | null;
+/**
+ * List comparisons with optional document filter and pagination
+ */
+export declare function listComparisons(db: Database.Database, options?: {
+    document_id?: string;
+    limit?: number;
+    offset?: number;
+}): Comparison[];
+/**
+ * Summary of a comparison (excludes large JSON diff fields)
+ */
+interface ComparisonSummary {
+    id: string;
+    document_id_1: string;
+    document_id_2: string;
+    similarity_ratio: number;
+    summary: string;
+    created_at: string;
+    processing_duration_ms: number | null;
+}
+/**
+ * Get comparison summaries for a document (lightweight: no JSON blobs)
+ */
+export declare function getComparisonSummariesByDocument(db: Database.Database, documentId: string): ComparisonSummary[];
+export {};
+//# sourceMappingURL=comparison-operations.d.ts.map

package/dist/services/storage/database/comparison-operations.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"comparison-operations.d.ts","sourceRoot":"","sources":["../../../../src/services/storage/database/comparison-operations.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAGhE;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EAAE,UAAU,EAAE,UAAU,GAAG,MAAM,CA2BtF;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EAAE,EAAE,EAAE,MAAM,GAAG,UAAU,GAAG,IAAI,CAKlF;AAED;;GAEG;AACH,wBAAgB,eAAe,CAC7B,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,OAAO,CAAC,EAAE;IAAE,WAAW,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,GAClE,UAAU,EAAE,CAcd;AAED;;GAEG;AACH,UAAU,iBAAiB;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,sBAAsB,EAAE,MAAM,GAAG,IAAI,CAAC;CACvC;AAED;;GAEG;AACH,wBAAgB,gCAAgC,CAC9C,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,UAAU,EAAE,MAAM,GACjB,iBAAiB,EAAE,CASrB"}

package/dist/services/storage/database/comparison-operations.js

@@ -0,0 +1,65 @@
+/**
+ * Comparison operations for DatabaseService
+ *
+ * Handles CRUD operations for the comparisons table.
+ */
+import { runWithForeignKeyCheck } from './helpers.js';
+/**
+ * Insert a comparison record
+ */
+export function insertComparison(db, comparison) {
+    const stmt = db.prepare(`
+    INSERT INTO comparisons (id, document_id_1, document_id_2, similarity_ratio,
+      text_diff_json, structural_diff_json, summary,
+      content_hash, provenance_id, created_at, processing_duration_ms)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+  `);
+    runWithForeignKeyCheck(stmt, [
+        comparison.id,
+        comparison.document_id_1,
+        comparison.document_id_2,
+        comparison.similarity_ratio,
+        comparison.text_diff_json,
+        comparison.structural_diff_json,
+        comparison.summary,
+        comparison.content_hash,
+        comparison.provenance_id,
+        comparison.created_at,
+        comparison.processing_duration_ms,
+    ], `inserting comparison: FK violation for document_id_1="${comparison.document_id_1}" or document_id_2="${comparison.document_id_2}"`);
+    return comparison.id;
+}
+/**
+ * Get a comparison by ID
+ */
+export function getComparison(db, id) {
+    const row = db.prepare('SELECT * FROM comparisons WHERE id = ?').get(id);
+    return row ?? null;
+}
+/**
+ * List comparisons with optional document filter and pagination
+ */
+export function listComparisons(db, options) {
+    const params = [];
+    let where = '';
+    if (options?.document_id) {
+        where = 'WHERE document_id_1 = ? OR document_id_2 = ?';
+        params.push(options.document_id, options.document_id);
+    }
+    params.push(options?.limit ?? 50, options?.offset ?? 0);
+    return db
+        .prepare(`SELECT * FROM comparisons ${where} ORDER BY created_at DESC LIMIT ? OFFSET ?`)
+        .all(...params);
+}
+/**
+ * Get comparison summaries for a document (lightweight: no JSON blobs)
+ */
+export function getComparisonSummariesByDocument(db, documentId) {
+    return db
+        .prepare(`SELECT id, document_id_1, document_id_2, similarity_ratio, summary, created_at, processing_duration_ms
+     FROM comparisons
+     WHERE document_id_1 = ? OR document_id_2 = ?
+     ORDER BY created_at DESC`)
+        .all(documentId, documentId);
+}
+//# sourceMappingURL=comparison-operations.js.map

package/dist/services/storage/database/comparison-operations.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"comparison-operations.js","sourceRoot":"","sources":["../../../../src/services/storage/database/comparison-operations.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH,OAAO,EAAE,sBAAsB,EAAE,MAAM,cAAc,CAAC;AAEtD;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,EAAqB,EAAE,UAAsB;IAC5E,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC;;;;;GAKvB,CAAC,CAAC;IAEH,sBAAsB,CACpB,IAAI,EACJ;QACE,UAAU,CAAC,EAAE;QACb,UAAU,CAAC,aAAa;QACxB,UAAU,CAAC,aAAa;QACxB,UAAU,CAAC,gBAAgB;QAC3B,UAAU,CAAC,cAAc;QACzB,UAAU,CAAC,oBAAoB;QAC/B,UAAU,CAAC,OAAO;QAClB,UAAU,CAAC,YAAY;QACvB,UAAU,CAAC,aAAa;QACxB,UAAU,CAAC,UAAU;QACrB,UAAU,CAAC,sBAAsB;KAClC,EACD,yDAAyD,UAAU,CAAC,aAAa,uBAAuB,UAAU,CAAC,aAAa,GAAG,CACpI,CAAC;IAEF,OAAO,UAAU,CAAC,EAAE,CAAC;AACvB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,EAAqB,EAAE,EAAU;IAC7D,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,wCAAwC,CAAC,CAAC,GAAG,CAAC,EAAE,CAE1D,CAAC;IACd,OAAO,GAAG,IAAI,IAAI,CAAC;AACrB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAC7B,EAAqB,EACrB,OAAmE;IAEnE,MAAM,MAAM,GAAwB,EAAE,CAAC;IACvC,IAAI,KAAK,GAAG,EAAE,CAAC;IAEf,IAAI,OAAO,EAAE,WAAW,EAAE,CAAC;QACzB,KAAK,GAAG,8CAA8C,CAAC;QACvD,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;IACxD,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC;IAExD,OAAO,EAAE;SACN,OAAO,CAAC,6BAA6B,KAAK,4CAA4C,CAAC;SACvF,GAAG,CAAC,GAAG,MAAM,CAAiB,CAAC;AACpC,CAAC;AAeD;;GAEG;AACH,MAAM,UAAU,gCAAgC,CAC9C,EAAqB,EACrB,UAAkB;IAElB,OAAO,EAAE;SACN,OAAO,CACN;;;8BAGwB,CACzB;SACA,GAAG,CAAC,UAAU,EAAE,UAAU,CAAwB,CAAC;AACxD,CAAC"}

package/dist/services/storage/database/converters.d.ts

@@ -0,0 +1,36 @@
+/**
+ * Row conversion functions for DatabaseService
+ *
+ * Converts database row objects to domain model interfaces.
+ */
+import { Document, OCRResult } from '../../../models/document.js';
+import { Chunk } from '../../../models/chunk.js';
+import { Embedding } from '../../../models/embedding.js';
+import { ProvenanceRecord } from '../../../models/provenance.js';
+import { ImageReference } from '../../../models/image.js';
+import { DocumentRow, OCRResultRow, ChunkRow, EmbeddingRow, ProvenanceRow, ImageRow } from './types.js';
+/**
+ * Convert document row to Document interface
+ */
+export declare function rowToDocument(row: DocumentRow): Document;
+/**
+ * Convert OCR result row to OCRResult interface
+ */
+export declare function rowToOCRResult(row: OCRResultRow): OCRResult;
+/**
+ * Convert chunk row to Chunk interface
+ */
+export declare function rowToChunk(row: ChunkRow): Chunk;
+/**
+ * Convert embedding row to Embedding interface (without vector)
+ */
+export declare function rowToEmbedding(row: EmbeddingRow): Omit<Embedding, 'vector'>;
+/**
+ * Convert provenance row to ProvenanceRecord interface
+ */
+export declare function rowToProvenance(row: ProvenanceRow): ProvenanceRecord;
+/**
+ * Convert image row to ImageReference interface
+ */
+export declare function rowToImage(row: ImageRow): ImageReference;
+//# sourceMappingURL=converters.d.ts.map

package/dist/services/storage/database/converters.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"converters.d.ts","sourceRoot":"","sources":["../../../../src/services/storage/database/converters.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,QAAQ,EAAkB,SAAS,EAAE,MAAM,6BAA6B,CAAC;AAClF,OAAO,EAAE,KAAK,EAAE,MAAM,0BAA0B,CAAC;AACjD,OAAO,EAAE,SAAS,EAAE,MAAM,8BAA8B,CAAC;AACzD,OAAO,EACL,gBAAgB,EAIjB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,cAAc,EAAgC,MAAM,0BAA0B,CAAC;AACxF,OAAO,EACL,WAAW,EACX,YAAY,EACZ,QAAQ,EACR,YAAY,EACZ,aAAa,EACb,QAAQ,EACT,MAAM,YAAY,CAAC;AAwEpB;;GAEG;AACH,wBAAgB,aAAa,CAAC,GAAG,EAAE,WAAW,GAAG,QAAQ,CAoBxD;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,YAAY,GAAG,SAAS,CAmB3D;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,GAAG,EAAE,QAAQ,GAAG,KAAK,CA0B/C;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,YAAY,GAAG,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CA4B3E;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,GAAG,EAAE,aAAa,GAAG,gBAAgB,CA0BpE;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,GAAG,EAAE,QAAQ,GAAG,cAAc,CAsCxD"}

package/dist/services/storage/database/converters.js

@@ -0,0 +1,244 @@
+/**
+ * Row conversion functions for DatabaseService
+ *
+ * Converts database row objects to domain model interfaces.
+ */
+/** Valid DocumentStatus values for runtime validation */
+const VALID_DOCUMENT_STATUSES = ['pending', 'processing', 'complete', 'failed'];
+/** Valid ProvenanceType values for runtime validation */
+const VALID_PROVENANCE_TYPES = [
+    'DOCUMENT', 'OCR_RESULT', 'CHUNK', 'IMAGE', 'VLM_DESCRIPTION',
+    'EXTRACTION', 'FORM_FILL', 'COMPARISON', 'CLUSTERING', 'EMBEDDING',
+];
+/** Valid VLMStatus values for runtime validation */
+const VALID_VLM_STATUSES = ['pending', 'processing', 'complete', 'failed'];
+/**
+ * Validate that a string value is a member of an enum/union type at runtime.
+ * Throws a descriptive error if the value is invalid, preventing silent data corruption.
+ */
+function validateEnum(value, validValues, fieldName, id) {
+    if (!validValues.includes(value)) {
+        throw new Error(`Invalid ${fieldName} "${value}" in record ${id}. Valid values: ${validValues.join(', ')}`);
+    }
+    return value;
+}
+/**
+ * Safely parse JSON processing_params, returning a fallback on corrupt data.
+ */
+function parseProcessingParams(id, raw) {
+    try {
+        return JSON.parse(raw);
+    }
+    catch (error) {
+        console.error(`[converters] Corrupt processing_params in provenance ${id}: ${raw}:`, error instanceof Error ? error.message : String(error));
+        return { _parse_error: true, _raw: raw };
+    }
+}
+/**
+ * Safely parse JSON location, returning null on corrupt data.
+ * Callers already handle null (ProvenanceRecord.location is ProvenanceLocation | null).
+ */
+function parseLocation(id, raw) {
+    try {
+        return JSON.parse(raw);
+    }
+    catch (error) {
+        console.error(`[converters] Corrupt location in provenance ${id}: ${raw}:`, error instanceof Error ? error.message : String(error));
+        return null;
+    }
+}
+/**
+ * Safely parse JSON vlm_structured_data, returning null on corrupt data.
+ * Callers already handle null (ImageReference.vlm_structured_data is VLMStructuredData | null).
+ */
+function parseVLMStructuredData(id, raw) {
+    try {
+        return JSON.parse(raw);
+    }
+    catch (error) {
+        console.error(`[converters] Corrupt vlm_structured_data in image ${id}: ${raw}:`, error instanceof Error ? error.message : String(error));
+        return null;
+    }
+}
+/**
+ * Convert document row to Document interface
+ */
+export function rowToDocument(row) {
+    return {
+        id: row.id,
+        file_path: row.file_path,
+        file_name: row.file_name,
+        file_hash: row.file_hash,
+        file_size: row.file_size,
+        file_type: row.file_type,
+        status: validateEnum(row.status, VALID_DOCUMENT_STATUSES, 'DocumentStatus', row.id),
+        page_count: row.page_count,
+        provenance_id: row.provenance_id,
+        created_at: row.created_at,
+        modified_at: row.modified_at,
+        ocr_completed_at: row.ocr_completed_at,
+        error_message: row.error_message,
+        doc_title: row.doc_title ?? null,
+        doc_author: row.doc_author ?? null,
+        doc_subject: row.doc_subject ?? null,
+        datalab_file_id: row.datalab_file_id ?? null,
+    };
+}
+/**
+ * Convert OCR result row to OCRResult interface
+ */
+export function rowToOCRResult(row) {
+    return {
+        id: row.id,
+        provenance_id: row.provenance_id,
+        document_id: row.document_id,
+        extracted_text: row.extracted_text,
+        text_length: row.text_length,
+        datalab_request_id: row.datalab_request_id,
+        datalab_mode: row.datalab_mode,
+        parse_quality_score: row.parse_quality_score,
+        page_count: row.page_count,
+        cost_cents: row.cost_cents,
+        content_hash: row.content_hash,
+        processing_started_at: row.processing_started_at,
+        processing_completed_at: row.processing_completed_at,
+        processing_duration_ms: row.processing_duration_ms,
+        json_blocks: row.json_blocks ?? null,
+        extras_json: row.extras_json ?? null,
+    };
+}
+/**
+ * Convert chunk row to Chunk interface
+ */
+export function rowToChunk(row) {
+    return {
+        id: row.id,
+        document_id: row.document_id,
+        ocr_result_id: row.ocr_result_id,
+        text: row.text,
+        text_hash: row.text_hash,
+        chunk_index: row.chunk_index,
+        character_start: row.character_start,
+        character_end: row.character_end,
+        page_number: row.page_number,
+        page_range: row.page_range,
+        overlap_previous: row.overlap_previous,
+        overlap_next: row.overlap_next,
+        provenance_id: row.provenance_id,
+        created_at: row.created_at,
+        embedding_status: row.embedding_status,
+        embedded_at: row.embedded_at,
+        ocr_quality_score: row.ocr_quality_score ?? null,
+        heading_context: row.heading_context,
+        heading_level: row.heading_level,
+        section_path: row.section_path,
+        content_types: row.content_types,
+        is_atomic: row.is_atomic,
+        chunking_strategy: row.chunking_strategy,
+    };
+}
+/**
+ * Convert embedding row to Embedding interface (without vector)
+ */
+export function rowToEmbedding(row) {
+    return {
+        id: row.id,
+        chunk_id: row.chunk_id,
+        image_id: row.image_id,
+        extraction_id: row.extraction_id,
+        document_id: row.document_id,
+        original_text: row.original_text,
+        original_text_length: row.original_text_length,
+        source_file_path: row.source_file_path,
+        source_file_name: row.source_file_name,
+        source_file_hash: row.source_file_hash,
+        page_number: row.page_number,
+        page_range: row.page_range,
+        character_start: row.character_start,
+        character_end: row.character_end,
+        chunk_index: row.chunk_index,
+        total_chunks: row.total_chunks,
+        model_name: row.model_name,
+        model_version: row.model_version,
+        task_type: row.task_type,
+        inference_mode: row.inference_mode,
+        gpu_device: row.gpu_device ?? '',
+        provenance_id: row.provenance_id,
+        content_hash: row.content_hash,
+        created_at: row.created_at,
+        generation_duration_ms: row.generation_duration_ms,
+    };
+}
+/**
+ * Convert provenance row to ProvenanceRecord interface
+ */
+export function rowToProvenance(row) {
+    return {
+        id: row.id,
+        type: validateEnum(row.type, VALID_PROVENANCE_TYPES, 'ProvenanceType', row.id),
+        created_at: row.created_at,
+        processed_at: row.processed_at,
+        source_file_created_at: row.source_file_created_at,
+        source_file_modified_at: row.source_file_modified_at,
+        source_type: row.source_type,
+        source_path: row.source_path,
+        source_id: row.source_id,
+        root_document_id: row.root_document_id,
+        location: row.location ? parseLocation(row.id, row.location) : null,
+        content_hash: row.content_hash,
+        input_hash: row.input_hash,
+        file_hash: row.file_hash,
+        processor: row.processor,
+        processor_version: row.processor_version,
+        processing_params: parseProcessingParams(row.id, row.processing_params),
+        processing_duration_ms: row.processing_duration_ms,
+        processing_quality_score: row.processing_quality_score,
+        parent_id: row.parent_id,
+        parent_ids: row.parent_ids,
+        chain_depth: row.chain_depth,
+        chain_path: row.chain_path,
+    };
+}
+/**
+ * Convert image row to ImageReference interface
+ */
+export function rowToImage(row) {
+    return {
+        id: row.id,
+        document_id: row.document_id,
+        ocr_result_id: row.ocr_result_id,
+        page_number: row.page_number,
+        bounding_box: {
+            x: row.bbox_x,
+            y: row.bbox_y,
+            width: row.bbox_width,
+            height: row.bbox_height,
+        },
+        image_index: row.image_index,
+        format: row.format,
+        dimensions: {
+            width: row.width,
+            height: row.height,
+        },
+        extracted_path: row.extracted_path,
+        file_size: row.file_size,
+        vlm_status: validateEnum(row.vlm_status, VALID_VLM_STATUSES, 'VLMStatus', row.id),
+        vlm_description: row.vlm_description,
+        vlm_structured_data: row.vlm_structured_data
+            ? parseVLMStructuredData(row.id, row.vlm_structured_data)
+            : null,
+        vlm_embedding_id: row.vlm_embedding_id,
+        vlm_model: row.vlm_model,
+        vlm_confidence: row.vlm_confidence,
+        vlm_processed_at: row.vlm_processed_at,
+        vlm_tokens_used: row.vlm_tokens_used,
+        context_text: row.context_text,
+        provenance_id: row.provenance_id,
+        created_at: row.created_at,
+        error_message: row.error_message,
+        block_type: row.block_type ?? null,
+        is_header_footer: Boolean(row.is_header_footer),
+        content_hash: row.content_hash ?? null,
+    };
+}
+//# sourceMappingURL=converters.js.map