ocr-provenance-mcp 1.0.0

package/dist/services/clustering/clustering-service.js

@@ -0,0 +1,467 @@
+/**
+ * Clustering Service - Orchestrates document clustering pipeline
+ *
+ * Pipeline:
+ *   1. Fetch document-level embeddings (average-pool chunk embeddings, L2-normalize)
+ *   2. Call Python clustering worker (HDBSCAN / agglomerative / kmeans)
+ *   3. Store cluster + document_cluster records with provenance
+ *
+ * CRITICAL: NEVER use console.log() - stdout is JSON-RPC protocol.
+ * Use console.error() for all logging.
+ *
+ * @module services/clustering/clustering-service
+ */
+import { v4 as uuidv4 } from 'uuid';
+import { PythonShell } from 'python-shell';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import { getProvenanceTracker } from '../provenance/index.js';
+import { ProvenanceType } from '../../models/provenance.js';
+import { insertCluster, insertDocumentCluster } from '../storage/database/cluster-operations.js';
+import { computeHash } from '../../utils/hash.js';
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+class ClusteringError extends Error {
+    code;
+    details;
+    constructor(message, code, details) {
+        super(message);
+        this.code = code;
+        this.details = details;
+        this.name = 'ClusteringError';
+    }
+}
+// ═══════════════════════════════════════════════════════════════════════════════
+// SERVICE
+// ═══════════════════════════════════════════════════════════════════════════════
+/** Worker timeout: 5 minutes */
+const WORKER_TIMEOUT_MS = 300_000;
+/** Max stderr accumulation: 10KB */
+const MAX_STDERR_LENGTH = 10_240;
+/**
+ * Compute document-level embeddings by average-pooling chunk embeddings.
+ *
+ * For each document that has embeddings, fetches all chunk-based vectors
+ * from vec_embeddings, computes the element-wise mean, and L2-normalizes.
+ *
+ * sqlite-vec has NO native vector averaging -- we extract to TypeScript.
+ *
+ * @param conn - Raw better-sqlite3 connection (for direct vec_embeddings queries)
+ * @param documentIds - Optional filter; if empty, includes all documents with embeddings
+ * @returns Array of DocumentEmbedding with 768-dim Float32Array per document
+ */
+export function computeDocumentEmbeddings(conn, documentIds) {
+    // chunk_id IS NOT NULL ensures we only get chunk embeddings (not VLM or extraction)
+    const hasFilter = documentIds && documentIds.length > 0;
+    const filterClause = hasFilter
+        ? ` AND e.document_id IN (${documentIds.map(() => '?').join(', ')})`
+        : '';
+    const rows = conn
+        .prepare(`
+    SELECT e.document_id, v.vector
+    FROM vec_embeddings v
+    JOIN embeddings e ON e.id = v.embedding_id
+    WHERE e.chunk_id IS NOT NULL${filterClause}
+    ORDER BY e.document_id, e.chunk_index
+  `)
+        .all(...(hasFilter ? documentIds : []));
+    // Group by document_id
+    const docVectors = new Map();
+    for (const row of rows) {
+        const existing = docVectors.get(row.document_id);
+        if (existing) {
+            existing.push(row.vector);
+        }
+        else {
+            docVectors.set(row.document_id, [row.vector]);
+        }
+    }
+    // Average-pool + L2-normalize per document
+    const results = [];
+    for (const [docId, vectors] of docVectors) {
+        const averaged = averageVectors(vectors);
+        results.push({
+            document_id: docId,
+            embedding: averaged,
+            chunk_count: vectors.length,
+        });
+    }
+    return results;
+}
+/**
+ * Average-pool vectors and L2-normalize the result.
+ *
+ * @param vectors - Array of 768-dim vectors as Buffers (from sqlite-vec)
+ * @returns L2-normalized 768-dim Float32Array
+ */
+function averageVectors(vectors) {
+    if (vectors.length === 0) {
+        return new Float32Array(768);
+    }
+    const dim = 768;
+    const sum = new Float64Array(dim); // Use float64 for accumulation precision
+    for (const buf of vectors) {
+        const f32 = new Float32Array(buf.buffer, buf.byteOffset, dim);
+        for (let i = 0; i < dim; i++) {
+            sum[i] += f32[i];
+        }
+    }
+    // Mean
+    const n = vectors.length;
+    const result = new Float32Array(dim);
+    for (let i = 0; i < dim; i++) {
+        result[i] = sum[i] / n;
+    }
+    // L2 normalize
+    let norm = 0;
+    for (let i = 0; i < dim; i++) {
+        norm += result[i] * result[i];
+    }
+    norm = Math.sqrt(norm);
+    if (norm > 0) {
+        for (let i = 0; i < dim; i++) {
+            result[i] /= norm;
+        }
+    }
+    return result;
+}
+/**
+ * Run the Python clustering worker via python-shell.
+ *
+ * Sends JSON to stdin, parses JSON from stdout.
+ * Uses the same PythonShell pattern as embedding_worker.py.
+ *
+ * @param embeddings - 2D array of embeddings [n_docs][768]
+ * @param documentIds - Document IDs matching embedding order
+ * @param config - Clustering algorithm configuration
+ * @param distanceMatrix - Optional precomputed distance matrix [n_docs][n_docs]
+ * @returns WorkerResult from Python
+ */
+async function runClusteringWorker(embeddings, documentIds, config, distanceMatrix) {
+    const workerPath = path.resolve(__dirname, '../../../python/clustering_worker.py');
+    const workerInput = {
+        embeddings,
+        document_ids: documentIds,
+        algorithm: config.algorithm,
+        n_clusters: config.n_clusters,
+        min_cluster_size: config.min_cluster_size,
+        distance_threshold: config.distance_threshold,
+        linkage: config.linkage,
+    };
+    if (distanceMatrix) {
+        workerInput.distance_matrix = distanceMatrix;
+    }
+    const input = JSON.stringify(workerInput);
+    return new Promise((resolve, reject) => {
+        let settled = false;
+        const options = {
+            mode: 'text',
+            pythonOptions: ['-u'],
+            args: [],
+        };
+        const shell = new PythonShell(workerPath, options);
+        let stderr = '';
+        let sigkillTimer = null;
+        const cleanup = () => {
+            if (sigkillTimer) {
+                clearTimeout(sigkillTimer);
+                sigkillTimer = null;
+            }
+        };
+        const timer = setTimeout(() => {
+            if (settled)
+                return;
+            try {
+                shell.kill();
+            }
+            catch (error) {
+                console.error('[ClusteringService] Failed to kill shell on timeout:', error instanceof Error ? error.message : String(error));
+                /* ignore */
+            }
+            // M-6: SIGKILL escalation if SIGTERM doesn't exit within 5s
+            sigkillTimer = setTimeout(() => {
+                if (!settled) {
+                    console.error(`[ClusteringService] Process did not exit after SIGTERM, sending SIGKILL (pid: ${shell.childProcess?.pid})`);
+                    try {
+                        shell.childProcess?.kill('SIGKILL');
+                    }
+                    catch (error) {
+                        console.error('[ClusteringService] Failed to SIGKILL process (may already be gone):', error instanceof Error ? error.message : String(error));
+                    }
+                }
+                if (!settled) {
+                    settled = true;
+                    reject(new ClusteringError(`Clustering worker timeout after ${WORKER_TIMEOUT_MS}ms (SIGKILL after 5s grace)`, 'WORKER_TIMEOUT', { stderr: stderr.substring(0, 1000) }));
+                }
+            }, 5000);
+        }, WORKER_TIMEOUT_MS);
+        const outputChunks = [];
+        shell.on('message', (msg) => {
+            outputChunks.push(msg);
+        });
+        shell.on('stderr', (err) => {
+            if (stderr.length < MAX_STDERR_LENGTH) {
+                stderr += err + '\n';
+            }
+        });
+        const handleEnd = (err) => {
+            clearTimeout(timer);
+            cleanup();
+            if (settled)
+                return;
+            settled = true;
+            if (err) {
+                console.error('[ClusterWorker] Error:', err.message);
+                if (stderr)
+                    console.error('[ClusterWorker] Stderr:', stderr.substring(0, 1000));
+            }
+            const output = outputChunks.join('\n');
+            if (!output.trim()) {
+                if (err) {
+                    reject(new ClusteringError(`Clustering worker failed: ${err.message}`, 'WORKER_FAILED', {
+                        stderr: stderr.substring(0, 1000),
+                    }));
+                }
+                else {
+                    reject(new ClusteringError('Clustering worker produced no output', 'WORKER_FAILED', {
+                        stderr: stderr.substring(0, 1000),
+                    }));
+                }
+                return;
+            }
+            // Parse the last JSON line
+            const lines = output.trim().split('\n');
+            let parsed;
+            for (let i = lines.length - 1; i >= 0; i--) {
+                try {
+                    parsed = JSON.parse(lines[i].trim());
+                    break;
+                }
+                catch (error) {
+                    console.error('[ClusteringService] JSON parse failed for output line, trying previous:', error instanceof Error ? error.message : String(error));
+                    /* not JSON, try previous line */
+                }
+            }
+            if (parsed !== undefined) {
+                // Validate required fields exist on parsed worker result
+                if (typeof parsed !== 'object' || parsed === null) {
+                    reject(new ClusteringError('Clustering worker returned non-object result', 'WORKER_PARSE_ERROR', { output: output.substring(0, 1000) }));
+                    return;
+                }
+                if (typeof parsed.success !== 'boolean') {
+                    reject(new ClusteringError('Clustering worker result missing required "success" field', 'WORKER_PARSE_ERROR', { output: output.substring(0, 1000) }));
+                    return;
+                }
+                if (parsed.success &&
+                    parsed.silhouette_score === undefined &&
+                    parsed.labels === undefined) {
+                    reject(new ClusteringError('Clustering worker success=true but missing "labels" and "silhouette_score" fields', 'WORKER_PARSE_ERROR', { output: output.substring(0, 1000) }));
+                    return;
+                }
+                resolve(parsed);
+            }
+            else {
+                reject(new ClusteringError('Failed to parse clustering worker output as JSON', 'WORKER_PARSE_ERROR', { output: output.substring(0, 1000) }));
+            }
+        };
+        shell.send(input);
+        shell.end(handleEnd);
+    });
+}
+/**
+ * Compute cosine similarity between a document embedding and a centroid.
+ * Both vectors are assumed to be L2-normalized, so similarity = dot product.
+ */
+export function cosineSimilarity(a, b) {
+    let dot = 0;
+    for (let i = 0; i < a.length; i++) {
+        dot += (a[i] ?? 0) * (b[i] ?? 0);
+    }
+    // Clamp to [0, 1] to handle floating point drift
+    return Math.max(0, Math.min(1, dot));
+}
+// ═══════════════════════════════════════════════════════════════════════════════
+// MAIN CLUSTERING PIPELINE
+// ═══════════════════════════════════════════════════════════════════════════════
+/**
+ * Run the full clustering pipeline:
+ *   1. Compute document-level embeddings
+ *   2. Validate minimum document count
+ *   3. Call Python clustering worker
+ *   4. Create provenance records
+ *   5. Store clusters + document_cluster assignments
+ *
+ * @param db - DatabaseService instance
+ * @param vector - VectorService instance (unused directly but validates vec loaded)
+ * @param config - Clustering configuration
+ * @param documentIds - Optional filter (empty = all documents with embeddings)
+ * @returns ClusterRunResult with full results
+ */
+export async function runClustering(db, _vector, config, documentIds) {
+    const startTime = performance.now();
+    const runId = uuidv4();
+    const conn = db.getConnection();
+    // Step 1: Compute document-level embeddings
+    console.error(`[CLUSTER] Computing document embeddings...`);
+    const docEmbeddings = computeDocumentEmbeddings(conn, documentIds);
+    if (docEmbeddings.length < 2) {
+        throw new ClusteringError(`At least 2 documents with embeddings required for clustering, got ${docEmbeddings.length}`, 'INSUFFICIENT_DOCUMENTS', { found: docEmbeddings.length, requested: documentIds?.length ?? 'all' });
+    }
+    console.error(`[CLUSTER] ${docEmbeddings.length} documents with embeddings`);
+    // Step 2: Prepare data for Python worker
+    const orderedDocIds = docEmbeddings.map((d) => d.document_id);
+    const embeddingMatrix = docEmbeddings.map((d) => Array.from(d.embedding));
+    // Step 3: Call Python clustering worker
+    console.error(`[CLUSTER] Running ${config.algorithm} clustering...`);
+    const workerResult = await runClusteringWorker(embeddingMatrix, orderedDocIds, config);
+    if (!workerResult.success) {
+        throw new ClusteringError(`Clustering worker failed: ${workerResult.error}`, 'WORKER_FAILED', {
+            error_type: workerResult.error_type,
+            error: workerResult.error,
+        });
+    }
+    // Step 4: Store results in database
+    console.error(`[CLUSTER] Found ${workerResult.n_clusters} clusters, storing results...`);
+    const tracker = getProvenanceTracker(db);
+    const now = new Date().toISOString();
+    const processingDurationMs = Math.round(performance.now() - startTime);
+    const algorithmParamsJson = JSON.stringify({
+        algorithm: config.algorithm,
+        n_clusters: config.n_clusters,
+        min_cluster_size: config.min_cluster_size,
+        distance_threshold: config.distance_threshold,
+        linkage: config.linkage,
+    });
+    // Build cluster result items and store cluster records
+    const clusterItems = [];
+    const labels = workerResult.labels;
+    const probabilities = workerResult.probabilities;
+    const centroids = workerResult.centroids;
+    const coherenceScores = workerResult.coherence_scores;
+    // Group documents by cluster label
+    const clusterGroups = new Map(); // label -> doc indices
+    for (let i = 0; i < labels.length; i++) {
+        const label = labels[i];
+        if (label === -1)
+            continue; // Skip noise
+        const existing = clusterGroups.get(label);
+        if (existing) {
+            existing.push(i);
+        }
+        else {
+            clusterGroups.set(label, [i]);
+        }
+    }
+    // Sort cluster labels
+    const sortedLabels = Array.from(clusterGroups.keys()).sort((a, b) => a - b);
+    const clusterIdMap = new Map(); // label -> cluster UUID
+    // Use a transaction to store everything atomically
+    const storeTransaction = conn.transaction(() => {
+        for (let ci = 0; ci < sortedLabels.length; ci++) {
+            const label = sortedLabels[ci];
+            const docIndices = clusterGroups.get(label);
+            const centroid = centroids[ci];
+            const coherence = coherenceScores[ci];
+            const clusterId = uuidv4();
+            clusterIdMap.set(label, clusterId);
+            // Content hash from centroid + run_id
+            const contentHash = computeHash(JSON.stringify(centroid) + ':' + runId);
+            // Find any document's provenance_id as the source_id for the cluster provenance
+            // Use the first document in this cluster
+            const firstDocId = orderedDocIds[docIndices[0]];
+            const firstDoc = db.getDocument(firstDocId);
+            const sourceProvId = firstDoc?.provenance_id ?? null;
+            // Create CLUSTERING provenance record
+            const provId = tracker.createProvenance({
+                type: ProvenanceType.CLUSTERING,
+                source_type: 'CLUSTERING',
+                source_id: sourceProvId,
+                root_document_id: firstDoc?.provenance_id ?? runId,
+                content_hash: contentHash,
+                input_hash: computeHash(algorithmParamsJson + ':' + docIndices.length),
+                processor: 'clustering-service',
+                processor_version: '1.0.0',
+                processing_params: {
+                    algorithm: config.algorithm,
+                    run_id: runId,
+                    cluster_index: ci,
+                    document_count: docIndices.length,
+                },
+                processing_duration_ms: processingDurationMs,
+                processing_quality_score: coherence,
+            });
+            // Insert cluster record
+            const cluster = {
+                id: clusterId,
+                run_id: runId,
+                cluster_index: ci,
+                label: null,
+                description: null,
+                classification_tag: null,
+                document_count: docIndices.length,
+                centroid_json: JSON.stringify(centroid),
+                top_terms_json: null,
+                coherence_score: coherence,
+                algorithm: config.algorithm,
+                algorithm_params_json: algorithmParamsJson,
+                silhouette_score: workerResult.silhouette_score ?? null,
+                content_hash: contentHash,
+                provenance_id: provId,
+                created_at: now,
+                processing_duration_ms: processingDurationMs,
+            };
+            insertCluster(conn, cluster);
+            // Build result item
+            const itemDocIds = [];
+            const itemSimilarities = [];
+            const itemProbabilities = [];
+            for (const idx of docIndices) {
+                itemDocIds.push(orderedDocIds[idx]);
+                itemSimilarities.push(cosineSimilarity(docEmbeddings[idx].embedding, centroid));
+                itemProbabilities.push(probabilities[idx]);
+            }
+            clusterItems.push({
+                cluster_index: ci,
+                document_count: docIndices.length,
+                coherence_score: coherence,
+                centroid,
+                document_ids: itemDocIds,
+                similarities: itemSimilarities,
+                probabilities: itemProbabilities,
+            });
+        }
+        // Store document-cluster assignments (both clustered and noise documents)
+        for (let i = 0; i < labels.length; i++) {
+            const label = labels[i];
+            const isNoise = label === -1;
+            const clusterId = isNoise ? null : (clusterIdMap.get(label) ?? null);
+            const centroid = isNoise ? null : centroids[sortedLabels.indexOf(label)];
+            const similarity = centroid ? cosineSimilarity(docEmbeddings[i].embedding, centroid) : 0;
+            const dc = {
+                id: uuidv4(),
+                document_id: orderedDocIds[i],
+                cluster_id: clusterId,
+                run_id: runId,
+                similarity_to_centroid: Math.round(similarity * 1000000) / 1000000,
+                membership_probability: probabilities[i],
+                is_noise: isNoise,
+                assigned_at: now,
+            };
+            insertDocumentCluster(conn, dc);
+        }
+    });
+    storeTransaction();
+    const noiseDocIds = orderedDocIds.filter((_, i) => labels[i] === -1);
+    const totalDurationMs = Math.round(performance.now() - startTime);
+    console.error(`[CLUSTER] Done: ${workerResult.n_clusters} clusters, ${noiseDocIds.length} noise docs, ${totalDurationMs}ms`);
+    return {
+        run_id: runId,
+        algorithm: config.algorithm,
+        n_clusters: workerResult.n_clusters,
+        total_documents: docEmbeddings.length,
+        noise_document_ids: noiseDocIds,
+        silhouette_score: workerResult.silhouette_score ?? 0,
+        clusters: clusterItems,
+        processing_duration_ms: totalDurationMs,
+    };
+}
+//# sourceMappingURL=clustering-service.js.map

package/dist/services/clustering/clustering-service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"clustering-service.js","sourceRoot":"","sources":["../../../src/services/clustering/clustering-service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,MAAM,CAAC;AACpC,OAAO,EAAE,WAAW,EAAiC,MAAM,cAAc,CAAC;AAC1E,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AAIpC,OAAO,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAC9D,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAS5D,OAAO,EAAE,aAAa,EAAE,qBAAqB,EAAE,MAAM,2CAA2C,CAAC;AACjG,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAElD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAa/D,MAAM,eAAgB,SAAQ,KAAK;IAGf;IACA;IAHlB,YACE,OAAe,EACC,IAAyB,EACzB,OAAiC;QAEjD,KAAK,CAAC,OAAO,CAAC,CAAC;QAHC,SAAI,GAAJ,IAAI,CAAqB;QACzB,YAAO,GAAP,OAAO,CAA0B;QAGjD,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAC;IAChC,CAAC;CACF;AA6BD,kFAAkF;AAClF,UAAU;AACV,kFAAkF;AAElF,gCAAgC;AAChC,MAAM,iBAAiB,GAAG,OAAO,CAAC;AAElC,oCAAoC;AACpC,MAAM,iBAAiB,GAAG,MAAM,CAAC;AAEjC;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,yBAAyB,CACvC,IAAuB,EACvB,WAAsB;IAEtB,oFAAoF;IACpF,MAAM,SAAS,GAAG,WAAW,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC;IACxD,MAAM,YAAY,GAAG,SAAS;QAC5B,CAAC,CAAC,0BAA0B,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;QACpE,CAAC,CAAC,EAAE,CAAC;IAEP,MAAM,IAAI,GAAG,IAAI;SACd,OAAO,CACN;;;;kCAI4B,YAAY;;GAE3C,CACE;SACA,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC,CAGtC,CAAC;IAEH,uBAAuB;IACvB,MAAM,UAAU,GAAG,IAAI,GAAG,EAAoB,CAAC;IAC/C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACjD,IAAI,QAAQ,EAAE,CAAC;YACb,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC5B,CAAC;aAAM,CAAC;YACN,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAED,2CAA2C;IAC3C,MAAM,OAAO,GAAwB,EAAE,CAAC;IACxC,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,UAAU,EAAE,CAAC;QAC1C,MAAM,QAAQ,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;QACzC,OAAO,CAAC,IAAI,CAAC;YACX,WAAW,EAAE,KAAK;YAClB,SAAS,EAAE,QAAQ;YACnB,WAAW,EAAE,OAAO,CAAC,MAAM;SAC5B,CAAC,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;;GAKG;AACH,SAAS,cAAc,CAAC,OAAiB;IACvC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC;IAC/B,CAAC;IAED,MAAM,GAAG,GAAG,GAAG,CAAC;IAChB,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,yCAAyC;IAE5E,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;QAC9D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7B,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC;QACnB,CAAC;IACH,CAAC;IAED,OAAO;IACP,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IACzB,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC;IACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7B,MAAM,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IAED,eAAe;IACf,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7B,IAAI,IAAI,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IAChC,CAAC;IACD,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEvB,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;QACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7B,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;QACpB,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;;;;;;GAWG;AACH,KAAK,UAAU,mBAAmB,CAChC,UAAsB,EACtB,WAAqB,EACrB,MAAwB,EACxB,cAA2B;IAE3B,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,sCAAsC,CAAC,CAAC;IAEnF,MAAM,WAAW,GAA4B;QAC3C,UAAU;QACV,YAAY,EAAE,WAAW;QACzB,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;QACzC,kBAAkB,EAAE,MAAM,CAAC,kBAAkB;QAC7C,OAAO,EAAE,MAAM,CAAC,OAAO;KACxB,CAAC;IAEF,IAAI,cAAc,EAAE,CAAC;QACnB,WAAW,CAAC,eAAe,GAAG,cAAc,CAAC;IAC/C,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAE1C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,MAAM,OAAO,GAAuB;YAClC,IAAI,EAAE,MAAM;YACZ,aAAa,EAAE,CAAC,IAAI,CAAC;YACrB,IAAI,EAAE,EAAE;SACT,CAAC;QAEF,MAAM,KAAK,GAAG,IAAI,WAAW,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QACnD,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,YAAY,GAAyC,IAAI,CAAC;QAE9D,MAAM,OAAO,GAAG,GAAG,EAAE;YACnB,IAAI,YAAY,EAAE,CAAC;gBACjB,YAAY,CAAC,YAAY,CAAC,CAAC;gBAC3B,YAAY,GAAG,IAAI,CAAC;YACtB,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,IAAI,OAAO;gBAAE,OAAO;YACpB,IAAI,CAAC;gBACH,KAAK,CAAC,IAAI,EAAE,CAAC;YACf,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CACX,sDAAsD,EACtD,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CACvD,CAAC;gBACF,YAAY;YACd,CAAC;YACD,4DAA4D;YAC5D,YAAY,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC7B,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,OAAO,CAAC,KAAK,CACX,iFAAiF,KAAK,CAAC,YAAY,EAAE,GAAG,GAAG,CAC5G,CAAC;oBACF,IAAI,CAAC;wBACH,KAAK,CAAC,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;oBACtC,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,OAAO,CAAC,KAAK,CACX,sEAAsE,EACtE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CACvD,CAAC;oBACJ,CAAC;gBACH,CAAC;gBACD,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,OAAO,GAAG,IAAI,CAAC;oBACf,MAAM,CACJ,IAAI,eAAe,CACjB,mCAAmC,iBAAiB,6BAA6B,EACjF,gBAAgB,EAChB,EAAE,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,CACtC,CACF,CAAC;gBACJ,CAAC;YACH,CAAC,EAAE,IAAI,CAAC,CAAC;QACX,CAAC,EAAE,iBAAiB,CAAC,CAAC;QAEtB,MAAM,YAAY,GAAa,EAAE,CAAC;QAClC,KAAK,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAW,EAAE,EAAE;YAClC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACzB,CAAC,CAAC,CAAC;QAEH,KAAK,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,GAAW,EAAE,EAAE;YACjC,IAAI,MAAM,CAAC,MAAM,GAAG,iBAAiB,EAAE,CAAC;gBACtC,MAAM,IAAI,GAAG,GAAG,IAAI,CAAC;YACvB,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,MAAM,SAAS,GAAG,CAAC,GAAW,EAAE,EAAE;YAChC,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,OAAO,EAAE,CAAC;YACV,IAAI,OAAO;gBAAE,OAAO;YACpB,OAAO,GAAG,IAAI,CAAC;YAEf,IAAI,GAAG,EAAE,CAAC;gBACR,OAAO,CAAC,KAAK,CAAC,wBAAwB,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;gBACrD,IAAI,MAAM;oBAAE,OAAO,CAAC,KAAK,CAAC,yBAAyB,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;YAClF,CAAC;YAED,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACvC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;gBACnB,IAAI,GAAG,EAAE,CAAC;oBACR,MAAM,CACJ,IAAI,eAAe,CAAC,6BAA6B,GAAG,CAAC,OAAO,EAAE,EAAE,eAAe,EAAE;wBAC/E,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC;qBAClC,CAAC,CACH,CAAC;gBACJ,CAAC;qBAAM,CAAC;oBACN,MAAM,CACJ,IAAI,eAAe,CAAC,sCAAsC,EAAE,eAAe,EAAE;wBAC3E,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC;qBAClC,CAAC,CACH,CAAC;gBACJ,CAAC;gBACD,OAAO;YACT,CAAC;YAED,2BAA2B;YAC3B,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACxC,IAAI,MAAgC,CAAC;YACrC,KAAK,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC3C,IAAI,CAAC;oBACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAiB,CAAC;oBACrD,MAAM;gBACR,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,OAAO,CAAC,KAAK,CACX,yEAAyE,EACzE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CACvD,CAAC;oBACF,iCAAiC;gBACnC,CAAC;YACH,CAAC;YAED,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;gBACzB,yDAAyD;gBACzD,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;oBAClD,MAAM,CACJ,IAAI,eAAe,CACjB,8CAA8C,EAC9C,oBAAoB,EACpB,EAAE,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,CACtC,CACF,CAAC;oBACF,OAAO;gBACT,CAAC;gBACD,IAAI,OAAO,MAAM,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;oBACxC,MAAM,CACJ,IAAI,eAAe,CACjB,2DAA2D,EAC3D,oBAAoB,EACpB,EAAE,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,CACtC,CACF,CAAC;oBACF,OAAO;gBACT,CAAC;gBACD,IACE,MAAM,CAAC,OAAO;oBACd,MAAM,CAAC,gBAAgB,KAAK,SAAS;oBACrC,MAAM,CAAC,MAAM,KAAK,SAAS,EAC3B,CAAC;oBACD,MAAM,CACJ,IAAI,eAAe,CACjB,mFAAmF,EACnF,oBAAoB,EACpB,EAAE,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,CACtC,CACF,CAAC;oBACF,OAAO;gBACT,CAAC;gBACD,OAAO,CAAC,MAAM,CAAC,CAAC;YAClB,CAAC;iBAAM,CAAC;gBACN,MAAM,CACJ,IAAI,eAAe,CACjB,kDAAkD,EAClD,oBAAoB,EACpB,EAAE,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,CACtC,CACF,CAAC;YACJ,CAAC;QACH,CAAC,CAAC;QAEF,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClB,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACvB,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,CAA0B,EAAE,CAAW;IACtE,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACnC,CAAC;IACD,iDAAiD;IACjD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;AACvC,CAAC;AAED,kFAAkF;AAClF,2BAA2B;AAC3B,kFAAkF;AAElF;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,EAAmB,EACnB,OAAsB,EACtB,MAAwB,EACxB,WAAsB;IAEtB,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IACpC,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC;IACvB,MAAM,IAAI,GAAG,EAAE,CAAC,aAAa,EAAE,CAAC;IAEhC,4CAA4C;IAC5C,OAAO,CAAC,KAAK,CAAC,4CAA4C,CAAC,CAAC;IAC5D,MAAM,aAAa,GAAG,yBAAyB,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;IAEnE,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,eAAe,CACvB,qEAAqE,aAAa,CAAC,MAAM,EAAE,EAC3F,wBAAwB,EACxB,EAAE,KAAK,EAAE,aAAa,CAAC,MAAM,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,IAAI,KAAK,EAAE,CACzE,CAAC;IACJ,CAAC;IAED,OAAO,CAAC,KAAK,CAAC,aAAa,aAAa,CAAC,MAAM,4BAA4B,CAAC,CAAC;IAE7E,yCAAyC;IACzC,MAAM,aAAa,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;IAC9D,MAAM,eAAe,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;IAE1E,wCAAwC;IACxC,OAAO,CAAC,KAAK,CAAC,qBAAqB,MAAM,CAAC,SAAS,gBAAgB,CAAC,CAAC;IACrE,MAAM,YAAY,GAAG,MAAM,mBAAmB,CAC5C,eAAe,EACf,aAAa,EACb,MAAM,CACP,CAAC;IAEF,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;QAC1B,MAAM,IAAI,eAAe,CAAC,6BAA6B,YAAY,CAAC,KAAK,EAAE,EAAE,eAAe,EAAE;YAC5F,UAAU,EAAE,YAAY,CAAC,UAAU;YACnC,KAAK,EAAE,YAAY,CAAC,KAAK;SAC1B,CAAC,CAAC;IACL,CAAC;IAED,oCAAoC;IACpC,OAAO,CAAC,KAAK,CAAC,mBAAmB,YAAY,CAAC,UAAU,+BAA+B,CAAC,CAAC;IACzF,MAAM,OAAO,GAAG,oBAAoB,CAAC,EAAE,CAAC,CAAC;IACzC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACrC,MAAM,oBAAoB,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,CAAC;IACvE,MAAM,mBAAmB,GAAG,IAAI,CAAC,SAAS,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;QACzC,kBAAkB,EAAE,MAAM,CAAC,kBAAkB;QAC7C,OAAO,EAAE,MAAM,CAAC,OAAO;KACxB,CAAC,CAAC;IAEH,uDAAuD;IACvD,MAAM,YAAY,GAAwB,EAAE,CAAC;IAC7C,MAAM,MAAM,GAAG,YAAY,CAAC,MAAO,CAAC;IACpC,MAAM,aAAa,GAAG,YAAY,CAAC,aAAc,CAAC;IAClD,MAAM,SAAS,GAAG,YAAY,CAAC,SAAU,CAAC;IAC1C,MAAM,eAAe,GAAG,YAAY,CAAC,gBAAiB,CAAC;IAEvD,mCAAmC;IACnC,MAAM,aAAa,GAAG,IAAI,GAAG,EAAoB,CAAC,CAAC,uBAAuB;IAC1E,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACxB,IAAI,KAAK,KAAK,CAAC,CAAC;YAAE,SAAS,CAAC,aAAa;QACzC,MAAM,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC1C,IAAI,QAAQ,EAAE,CAAC;YACb,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACnB,CAAC;aAAM,CAAC;YACN,aAAa,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IAED,sBAAsB;IACtB,MAAM,YAAY,GAAG,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5E,MAAM,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAC,CAAC,wBAAwB;IAExE,mDAAmD;IACnD,MAAM,gBAAgB,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE;QAC7C,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,YAAY,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC;YAChD,MAAM,KAAK,GAAG,YAAY,CAAC,EAAE,CAAC,CAAC;YAC/B,MAAM,UAAU,GAAG,aAAa,CAAC,GAAG,CAAC,KAAK,CAAE,CAAC;YAC7C,MAAM,QAAQ,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC;YAC/B,MAAM,SAAS,GAAG,eAAe,CAAC,EAAE,CAAC,CAAC;YAEtC,MAAM,SAAS,GAAG,MAAM,EAAE,CAAC;YAC3B,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;YAEnC,sCAAsC;YACtC,MAAM,WAAW,GAAG,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,GAAG,GAAG,KAAK,CAAC,CAAC;YAExE,gFAAgF;YAChF,yCAAyC;YACzC,MAAM,UAAU,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;YAChD,MAAM,QAAQ,GAAG,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;YAC5C,MAAM,YAAY,GAAG,QAAQ,EAAE,aAAa,IAAI,IAAI,CAAC;YAErD,sCAAsC;YACtC,MAAM,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;gBACtC,IAAI,EAAE,cAAc,CAAC,UAAU;gBAC/B,WAAW,EAAE,YAA0B;gBACvC,SAAS,EAAE,YAAY;gBACvB,gBAAgB,EAAE,QAAQ,EAAE,aAAa,IAAI,KAAK;gBAClD,YAAY,EAAE,WAAW;gBACzB,UAAU,EAAE,WAAW,CAAC,mBAAmB,GAAG,GAAG,GAAG,UAAU,CAAC,MAAM,CAAC;gBACtE,SAAS,EAAE,oBAAoB;gBAC/B,iBAAiB,EAAE,OAAO;gBAC1B,iBAAiB,EAAE;oBACjB,SAAS,EAAE,MAAM,CAAC,SAAS;oBAC3B,MAAM,EAAE,KAAK;oBACb,aAAa,EAAE,EAAE;oBACjB,cAAc,EAAE,UAAU,CAAC,MAAM;iBAClC;gBACD,sBAAsB,EAAE,oBAAoB;gBAC5C,wBAAwB,EAAE,SAAS;aACpC,CAAC,CAAC;YAEH,wBAAwB;YACxB,MAAM,OAAO,GAAY;gBACvB,EAAE,EAAE,SAAS;gBACb,MAAM,EAAE,KAAK;gBACb,aAAa,EAAE,EAAE;gBACjB,KAAK,EAAE,IAAI;gBACX,WAAW,EAAE,IAAI;gBACjB,kBAAkB,EAAE,IAAI;gBACxB,cAAc,EAAE,UAAU,CAAC,MAAM;gBACjC,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC;gBACvC,cAAc,EAAE,IAAI;gBACpB,eAAe,EAAE,SAAS;gBAC1B,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,qBAAqB,EAAE,mBAAmB;gBAC1C,gBAAgB,EAAE,YAAY,CAAC,gBAAgB,IAAI,IAAI;gBACvD,YAAY,EAAE,WAAW;gBACzB,aAAa,EAAE,MAAM;gBACrB,UAAU,EAAE,GAAG;gBACf,sBAAsB,EAAE,oBAAoB;aAC7C,CAAC;YAEF,aAAa,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YAE7B,oBAAoB;YACpB,MAAM,UAAU,GAAa,EAAE,CAAC;YAChC,MAAM,gBAAgB,GAAa,EAAE,CAAC;YACtC,MAAM,iBAAiB,GAAa,EAAE,CAAC;YAEvC,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;gBAC7B,UAAU,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC;gBACpC,gBAAgB,CAAC,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC;gBAChF,iBAAiB,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC;YAC7C,CAAC;YAED,YAAY,CAAC,IAAI,CAAC;gBAChB,aAAa,EAAE,EAAE;gBACjB,cAAc,EAAE,UAAU,CAAC,MAAM;gBACjC,eAAe,EAAE,SAAS;gBAC1B,QAAQ;gBACR,YAAY,EAAE,UAAU;gBACxB,YAAY,EAAE,gBAAgB;gBAC9B,aAAa,EAAE,iBAAiB;aACjC,CAAC,CAAC;QACL,CAAC;QAED,0EAA0E;QAC1E,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACxB,MAAM,OAAO,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC;YAC7B,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC;YACrE,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;YACzE,MAAM,UAAU,GAAG,QAAQ,CAAC,CAAC,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAEzF,MAAM,EAAE,GAAoB;gBAC1B,EAAE,EAAE,MAAM,EAAE;gBACZ,WAAW,EAAE,aAAa,CAAC,CAAC,CAAC;gBAC7B,UAAU,EAAE,SAAS;gBACrB,MAAM,EAAE,KAAK;gBACb,sBAAsB,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,OAAO,CAAC,GAAG,OAAO;gBAClE,sBAAsB,EAAE,aAAa,CAAC,CAAC,CAAC;gBACxC,QAAQ,EAAE,OAAO;gBACjB,WAAW,EAAE,GAAG;aACjB,CAAC;YAEF,qBAAqB,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAClC,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,gBAAgB,EAAE,CAAC;IAEnB,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAErE,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,CAAC;IAClE,OAAO,CAAC,KAAK,CACX,mBAAmB,YAAY,CAAC,UAAU,cAAc,WAAW,CAAC,MAAM,gBAAgB,eAAe,IAAI,CAC9G,CAAC;IAEF,OAAO;QACL,MAAM,EAAE,KAAK;QACb,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,UAAU,EAAE,YAAY,CAAC,UAAW;QACpC,eAAe,EAAE,aAAa,CAAC,MAAM;QACrC,kBAAkB,EAAE,WAAW;QAC/B,gBAAgB,EAAE,YAAY,CAAC,gBAAgB,IAAI,CAAC;QACpD,QAAQ,EAAE,YAAY;QACtB,sBAAsB,EAAE,eAAe;KACxC,CAAC;AACJ,CAAC"}

package/dist/services/comparison/diff-service.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Document Comparison Diff Service
+ *
+ * Computes text and structural diffs between two OCR-processed documents.
+ * Uses the `diff` npm package (jsdiff) for text comparison.
+ *
+ * CRITICAL: NEVER use console.log() - stdout is JSON-RPC protocol.
+ */
+import type { TextDiffResult, StructuralDiff } from '../../models/comparison.js';
+/**
+ * Input shape for a document's structural metadata used in compareStructure()
+ */
+export interface StructuralDocInput {
+    page_count: number | null;
+    text_length: number;
+    quality_score: number | null;
+    ocr_mode: string;
+    chunk_count: number;
+}
+/**
+ * Compare structural metadata between two documents
+ *
+ * @param doc1 - First document structural metadata
+ * @param doc2 - Second document structural metadata
+ * @returns StructuralDiff with side-by-side metadata
+ */
+export declare function compareStructure(doc1: StructuralDocInput, doc2: StructuralDocInput): StructuralDiff;
+/**
+ * Compare two texts using line-level diff
+ *
+ * @param text1 - First document text
+ * @param text2 - Second document text
+ * @param maxOperations - Maximum operations to return (default 1000)
+ * @returns TextDiffResult with operations, counts, and similarity ratio
+ */
+export declare function compareText(text1: string, text2: string, maxOperations?: number): TextDiffResult;
+/**
+ * Generate a human-readable summary of the comparison
+ */
+export declare function generateSummary(textDiff: TextDiffResult | null, structuralDiff: StructuralDiff, doc1Name: string, doc2Name: string): string;
+//# sourceMappingURL=diff-service.d.ts.map

package/dist/services/comparison/diff-service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"diff-service.d.ts","sourceRoot":"","sources":["../../../src/services/comparison/diff-service.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,KAAK,EAEV,cAAc,EACd,cAAc,EACf,MAAM,4BAA4B,CAAC;AAEpC;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAC9B,IAAI,EAAE,kBAAkB,EACxB,IAAI,EAAE,kBAAkB,GACvB,cAAc,CAahB;AAED;;;;;;;GAOG;AACH,wBAAgB,WAAW,CACzB,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,aAAa,GAAE,MAAa,GAC3B,cAAc,CA+DhB;AAED;;GAEG;AACH,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,cAAc,GAAG,IAAI,EAC/B,cAAc,EAAE,cAAc,EAC9B,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,GACf,MAAM,CAwBR"}

package/dist/services/comparison/diff-service.js

@@ -0,0 +1,120 @@
+/**
+ * Document Comparison Diff Service
+ *
+ * Computes text and structural diffs between two OCR-processed documents.
+ * Uses the `diff` npm package (jsdiff) for text comparison.
+ *
+ * CRITICAL: NEVER use console.log() - stdout is JSON-RPC protocol.
+ */
+import { diffLines } from 'diff';
+/**
+ * Compare structural metadata between two documents
+ *
+ * @param doc1 - First document structural metadata
+ * @param doc2 - Second document structural metadata
+ * @returns StructuralDiff with side-by-side metadata
+ */
+export function compareStructure(doc1, doc2) {
+    return {
+        doc1_page_count: doc1.page_count,
+        doc2_page_count: doc2.page_count,
+        doc1_chunk_count: doc1.chunk_count,
+        doc2_chunk_count: doc2.chunk_count,
+        doc1_text_length: doc1.text_length,
+        doc2_text_length: doc2.text_length,
+        doc1_quality_score: doc1.quality_score,
+        doc2_quality_score: doc2.quality_score,
+        doc1_ocr_mode: doc1.ocr_mode,
+        doc2_ocr_mode: doc2.ocr_mode,
+    };
+}
+/**
+ * Compare two texts using line-level diff
+ *
+ * @param text1 - First document text
+ * @param text2 - Second document text
+ * @param maxOperations - Maximum operations to return (default 1000)
+ * @returns TextDiffResult with operations, counts, and similarity ratio
+ */
+export function compareText(text1, text2, maxOperations = 1000) {
+    const changes = diffLines(text1, text2);
+    let doc1Offset = 0;
+    let doc2Offset = 0;
+    let insertions = 0;
+    let deletions = 0;
+    let unchanged = 0;
+    const operations = [];
+    for (const change of changes) {
+        let type;
+        if (change.added) {
+            type = 'insert';
+        }
+        else if (change.removed) {
+            type = 'delete';
+        }
+        else {
+            type = 'equal';
+        }
+        operations.push({
+            type,
+            text: change.value,
+            doc1_offset: doc1Offset,
+            doc2_offset: doc2Offset,
+            line_count: change.count ?? 0,
+        });
+        if (change.added) {
+            insertions++;
+            doc2Offset += change.value.length;
+        }
+        else if (change.removed) {
+            deletions++;
+            doc1Offset += change.value.length;
+        }
+        else {
+            unchanged++;
+            doc1Offset += change.value.length;
+            doc2Offset += change.value.length;
+        }
+    }
+    const totalOps = operations.length;
+    const truncated = totalOps > maxOperations;
+    const finalOps = truncated ? operations.slice(0, maxOperations) : operations;
+    // Similarity = unchanged chars / total chars
+    const unchangedChars = operations
+        .filter((o) => o.type === 'equal')
+        .reduce((sum, o) => sum + o.text.length, 0);
+    const totalChars = text1.length + text2.length;
+    const similarityRatio = totalChars === 0 ? 1.0 : (2 * unchangedChars) / totalChars;
+    return {
+        operations: finalOps,
+        total_operations: totalOps,
+        truncated,
+        insertions,
+        deletions,
+        unchanged,
+        similarity_ratio: Math.round(similarityRatio * 10000) / 10000,
+        doc1_length: text1.length,
+        doc2_length: text2.length,
+    };
+}
+/**
+ * Generate a human-readable summary of the comparison
+ */
+export function generateSummary(textDiff, structuralDiff, doc1Name, doc2Name) {
+    const parts = [];
+    parts.push(`Comparison of "${doc1Name}" vs "${doc2Name}".`);
+    if (textDiff) {
+        const pct = Math.round(textDiff.similarity_ratio * 100);
+        parts.push(`Text similarity: ${pct}%.`);
+        parts.push(`${textDiff.insertions} insertions, ${textDiff.deletions} deletions, ${textDiff.unchanged} unchanged sections.`);
+        if (textDiff.truncated) {
+            parts.push(`(Diff truncated: showing ${textDiff.operations.length} of ${textDiff.total_operations} operations.)`);
+        }
+    }
+    const pageDiff = (structuralDiff.doc1_page_count ?? 0) - (structuralDiff.doc2_page_count ?? 0);
+    if (pageDiff !== 0) {
+        parts.push(`Page count difference: ${Math.abs(pageDiff)} pages.`);
+    }
+    return parts.join(' ');
+}
+//# sourceMappingURL=diff-service.js.map

package/dist/services/comparison/diff-service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"diff-service.js","sourceRoot":"","sources":["../../../src/services/comparison/diff-service.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,MAAM,CAAC;AAkBjC;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAC9B,IAAwB,EACxB,IAAwB;IAExB,OAAO;QACL,eAAe,EAAE,IAAI,CAAC,UAAU;QAChC,eAAe,EAAE,IAAI,CAAC,UAAU;QAChC,gBAAgB,EAAE,IAAI,CAAC,WAAW;QAClC,gBAAgB,EAAE,IAAI,CAAC,WAAW;QAClC,gBAAgB,EAAE,IAAI,CAAC,WAAW;QAClC,gBAAgB,EAAE,IAAI,CAAC,WAAW;QAClC,kBAAkB,EAAE,IAAI,CAAC,aAAa;QACtC,kBAAkB,EAAE,IAAI,CAAC,aAAa;QACtC,aAAa,EAAE,IAAI,CAAC,QAAQ;QAC5B,aAAa,EAAE,IAAI,CAAC,QAAQ;KAC7B,CAAC;AACJ,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,WAAW,CACzB,KAAa,EACb,KAAa,EACb,gBAAwB,IAAI;IAE5B,MAAM,OAAO,GAAG,SAAS,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IAExC,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,MAAM,UAAU,GAAwB,EAAE,CAAC;IAE3C,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,IAA+B,CAAC;QACpC,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YACjB,IAAI,GAAG,QAAQ,CAAC;QAClB,CAAC;aAAM,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YAC1B,IAAI,GAAG,QAAQ,CAAC;QAClB,CAAC;aAAM,CAAC;YACN,IAAI,GAAG,OAAO,CAAC;QACjB,CAAC;QAED,UAAU,CAAC,IAAI,CAAC;YACd,IAAI;YACJ,IAAI,EAAE,MAAM,CAAC,KAAK;YAClB,WAAW,EAAE,UAAU;YACvB,WAAW,EAAE,UAAU;YACvB,UAAU,EAAE,MAAM,CAAC,KAAK,IAAI,CAAC;SAC9B,CAAC,CAAC;QAEH,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YACjB,UAAU,EAAE,CAAC;YACb,UAAU,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC;QACpC,CAAC;aAAM,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YAC1B,SAAS,EAAE,CAAC;YACZ,UAAU,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC;QACpC,CAAC;aAAM,CAAC;YACN,SAAS,EAAE,CAAC;YACZ,UAAU,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC;YAClC,UAAU,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC;QACpC,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAG,UAAU,CAAC,MAAM,CAAC;IACnC,MAAM,SAAS,GAAG,QAAQ,GAAG,aAAa,CAAC;IAC3C,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;IAE7E,6CAA6C;IAC7C,MAAM,cAAc,GAAG,UAAU;SAC9B,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC;SACjC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC9C,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IAC/C,MAAM,eAAe,GAAG,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,cAAc,CAAC,GAAG,UAAU,CAAC;IAEnF,OAAO;QACL,UAAU,EAAE,QAAQ;QACpB,gBAAgB,EAAE,QAAQ;QAC1B,SAAS;QACT,UAAU;QACV,SAAS;QACT,SAAS;QACT,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,eAAe,GAAG,KAAK,CAAC,GAAG,KAAK;QAC7D,WAAW,EAAE,KAAK,CAAC,MAAM;QACzB,WAAW,EAAE,KAAK,CAAC,MAAM;KAC1B,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAC7B,QAA+B,EAC/B,cAA8B,EAC9B,QAAgB,EAChB,QAAgB;IAEhB,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,CAAC,IAAI,CAAC,kBAAkB,QAAQ,SAAS,QAAQ,IAAI,CAAC,CAAC;IAE5D,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,gBAAgB,GAAG,GAAG,CAAC,CAAC;QACxD,KAAK,CAAC,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,CAAC;QACxC,KAAK,CAAC,IAAI,CACR,GAAG,QAAQ,CAAC,UAAU,gBAAgB,QAAQ,CAAC,SAAS,eAAe,QAAQ,CAAC,SAAS,sBAAsB,CAChH,CAAC;QACF,IAAI,QAAQ,CAAC,SAAS,EAAE,CAAC;YACvB,KAAK,CAAC,IAAI,CACR,4BAA4B,QAAQ,CAAC,UAAU,CAAC,MAAM,OAAO,QAAQ,CAAC,gBAAgB,eAAe,CACtG,CAAC;QACJ,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAG,CAAC,cAAc,CAAC,eAAe,IAAI,CAAC,CAAC,GAAG,CAAC,cAAc,CAAC,eAAe,IAAI,CAAC,CAAC,CAAC;IAC/F,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;QACnB,KAAK,CAAC,IAAI,CAAC,0BAA0B,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;IACpE,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC"}