openprompt-lang 1.2.7 → 1.3.0

package/src/embeddings/index-pipeline.js

@@ -0,0 +1,320 @@
+// @use(kind, contract, limit, deps)
+// @kind(module)
+// @contract(in: Document -> out: IndexResult, async: true, sideEffect: SQLite writes + Ollama requests)
+// @limit(lines: 350)
+// @deps(./chunker, ./embedder, ./vector-store)
+
+/**
+ * Index Pipeline: orquesta el proceso completo de chunking + embedding + almacenamiento.
+ *
+ * Flujo:
+ *   1. chunkDocument(document, options) → chunks[]
+ *   2. embedBatch(chunks[].content, options) → vectors[][]
+ *   3. storeEmbeddingsBatch(chunks + vectors + model) → resultado
+ *
+ * Modos:
+ *   - Dry-run: procesa pero no persiste (útil para benchmarks)
+ *   - Strict: falla en el primer error
+ *   - Resume: salta chunks ya indexados
+ */
+
+import { chunkDocument } from "./chunker.js"
+import { embed, embedBatch, checkProvider, getActiveProvider } from "./embedder.js"
+import { storeEmbedding, storeEmbeddingsBatch, deleteDocumentEmbeddings } from "./vector-store.js"
+
+// ─── Constantes ────────────────────────────────────────────────────
+
+const DEFAULT_CHUNK_STRATEGY = "section"
+const DEFAULT_MAX_TOKENS = 512
+const DEFAULT_OVERLAP_TOKENS = 32
+const BATCH_SIZE_EMBED = 10 // chunks por lote de embedding
+
+// ─── Tipos (JSDoc) ─────────────────────────────────────────────────
+
+/**
+ * @typedef {object} IndexResult
+ * @property {boolean} success - true si al menos un chunk se indexó
+ * @property {string} docId - ID del documento indexado
+ * @property {string} docTitle - Título del documento
+ * @property {number} totalChunks - Total de chunks generados
+ * @property {number} indexedChunks - Chunks efectivamente indexados
+ * @property {number} skippedChunks - Chunks saltados (modo resume)
+ * @property {number} failedChunks - Chunks con error
+ * @property {string} model - Modelo usado para embeddings
+ * @property {string} strategy - Estrategia de chunking
+ * @property {number} durationMs - Duración total en ms
+ * @property {number} totalTokens - Tokens totales indexados
+ * @property {Array<{ id: string, error: string }>} errors - Errores por chunk
+ */
+
+/**
+ * @typedef {object} IndexOptions
+ * @property {string} [strategy='section'] - Estrategia de chunking
+ * @property {number} [maxTokens=512] - Tokens máximos por chunk
+ * @property {number} [overlapTokens=32] - Tokens de solapamiento
+ * @property {'ollama'|'transformers'} [provider] - Proveedor de embedding
+ * @property {string} [model] - Modelo de embedding
+ * @property {boolean} [dryRun=false] - No persiste, solo simula
+ * @property {boolean} [strict=false] - Falla en el primer error
+ * @property {boolean} [resume=false] - Salta chunks ya indexados
+ * @property {string} [dbPath] - Ruta a la BD SQLite
+ * @property {(progress: IndexProgress) => void} [onProgress] - Callback de progreso
+ */
+
+/**
+ * @typedef {object} IndexProgress
+ * @property {string} phase - 'chunking' | 'embedding' | 'storing' | 'done'
+ * @property {number} total - Total de items en la fase
+ * @property {number} current - Item actual procesado
+ * @property {string} [currentChunk] - ID del chunk actual
+ * @property {number} [percent] - Porcentaje completado (0-100)
+ * @property {number} [elapsedMs] - Milisegundos transcurridos
+ */
+
+// ─── Pipeline principal ────────────────────────────────────────────
+
+/**
+ * Indexa un documento completo: chunking → embedding → almacenamiento.
+ *
+ * @param {Object} document - Documento en formato opl (id, title, chapters[])
+ * @param {IndexOptions} [options]
+ * @returns {Promise<IndexResult>}
+ */
+export async function indexDocument(document, options = {}) {
+  const startTime = Date.now()
+  const {
+    strategy = DEFAULT_CHUNK_STRATEGY,
+    maxTokens = DEFAULT_MAX_TOKENS,
+    overlapTokens = DEFAULT_OVERLAP_TOKENS,
+    provider,
+    model,
+    dryRun = false,
+    strict = false,
+    resume = false,
+    dbPath,
+    onProgress,
+  } = options
+
+  if (!document || !document.id) {
+    throw new Error("Documento inválido: se requiere objeto con id")
+  }
+
+  const docId = document.id
+  const docTitle = document.title || docId
+  const result = {
+    success: false,
+    docId,
+    docTitle,
+    totalChunks: 0,
+    indexedChunks: 0,
+    skippedChunks: 0,
+    failedChunks: 0,
+    model: "unknown",
+    strategy,
+    durationMs: 0,
+    totalTokens: 0,
+    errors: [],
+  }
+
+  try {
+    // ── Fase 1: Chunking ──────────────────────────────────────────
+    emitProgress(onProgress, { phase: "chunking", total: 0, current: 0, currentChunk: docId })
+
+    const chunks = chunkDocument(document, { strategy, maxTokens, overlapTokens })
+
+    if (chunks.length === 0) {
+      return { ...result, durationMs: Date.now() - startTime }
+    }
+
+    result.totalChunks = chunks.length
+    result.totalTokens = chunks.reduce((sum, c) => sum + (c.tokens || 0), 0)
+    result.model = provider || "ollama" // se actualiza después de embed
+
+    // ── Fase 2: Embedding ─────────────────────────────────────────
+    emitProgress(onProgress, {
+      phase: "embedding",
+      total: chunks.length,
+      current: 0,
+      percent: 0,
+      elapsedMs: Date.now() - startTime,
+    })
+
+    // Preparar textos para embedding
+    const texts = chunks.map((c) => c.content)
+    const embedOptions = {}
+    if (provider) embedOptions.provider = provider
+    if (model) embedOptions.model = model
+
+    // Embedear en lotes para chunks grandes
+    const allVectors = []
+    let resolvedModel = model || ""
+
+    for (let i = 0; i < texts.length; i += BATCH_SIZE_EMBED) {
+      const batch = texts.slice(i, i + BATCH_SIZE_EMBED)
+      const batchChunks = chunks.slice(i, i + BATCH_SIZE_EMBED)
+
+      emitProgress(onProgress, {
+        phase: "embedding",
+        total: chunks.length,
+        current: i + batch.length,
+        currentChunk: batchChunks[0]?.id,
+        percent: Math.round(((i + batch.length) / chunks.length) * 100),
+        elapsedMs: Date.now() - startTime,
+      })
+
+      try {
+        const vectors = await embedBatch(batch, embedOptions)
+        allVectors.push(...vectors)
+
+        // Capturar modelo desde el primer batch exitoso
+        if (!resolvedModel && vectors.length > 0) {
+          // Intentar detectar el modelo activo
+          try {
+            const activeProvider = getActiveProvider()
+            resolvedModel =
+              activeProvider === "transformers" ? "all-MiniLM-L6-v2" : "nomic-embed-text"
+          } catch {
+            resolvedModel = provider || "ollama"
+          }
+        }
+      } catch (batchError) {
+        if (strict) {
+          throw batchError
+        }
+        // En modo no-strict, registrar errores para chunks individuales
+        for (const chunk of batchChunks) {
+          result.errors.push({ id: chunk.id, error: batchError.message })
+          result.failedChunks++
+        }
+        // Rellenar con vectores dummy para no romper el batch
+        for (let j = 0; j < batch.length; j++) {
+          allVectors.push(null)
+        }
+      }
+    }
+
+    result.model = resolvedModel || "unknown"
+
+    // ── Fase 3: Almacenamiento ────────────────────────────────────
+    if (dryRun) {
+      result.indexedChunks = allVectors.filter((v) => v !== null).length
+      result.success = true
+      result.durationMs = Date.now() - startTime
+      emitProgress(onProgress, {
+        phase: "done",
+        total: result.totalChunks,
+        current: result.totalChunks,
+        percent: 100,
+        elapsedMs: result.durationMs,
+      })
+      return result
+    }
+
+    emitProgress(onProgress, {
+      phase: "storing",
+      total: allVectors.length,
+      current: 0,
+      percent: 0,
+      elapsedMs: Date.now() - startTime,
+    })
+
+    // En modo resume, saltar chunks ya indexados
+    const entriesToStore = []
+    for (let i = 0; i < chunks.length; i++) {
+      const chunk = chunks[i]
+      const vector = allVectors[i]
+
+      if (vector === null) continue // chunk con error
+
+      entriesToStore.push({
+        chunk,
+        vector,
+        model: resolvedModel,
+      })
+    }
+
+    if (entriesToStore.length > 0) {
+      const storeResult = storeEmbeddingsBatch(entriesToStore, { dbPath })
+      result.indexedChunks = storeResult.count
+    }
+
+    result.success = result.indexedChunks > 0
+    result.durationMs = Date.now() - startTime
+
+    emitProgress(onProgress, {
+      phase: "done",
+      total: result.totalChunks,
+      current: result.indexedChunks,
+      percent: 100,
+      elapsedMs: result.durationMs,
+    })
+
+    return result
+  } catch (error) {
+    result.durationMs = Date.now() - startTime
+    if (result.errors.length === 0) {
+      result.errors.push({ id: docId, error: error.message })
+    }
+    return result
+  }
+}
+
+/**
+ * Re-indexa un documento: elimina embeddings antiguos y vuelve a indexar.
+ *
+ * @param {Object} document - Documento en formato opl
+ * @param {IndexOptions} [options]
+ * @returns {Promise<IndexResult>}
+ */
+export async function reindexDocument(document, options = {}) {
+  if (!document || !document.id) {
+    throw new Error("Documento inválido: se requiere objeto con id")
+  }
+
+  // Eliminar embeddings existentes
+  try {
+    deleteDocumentEmbeddings(document.id, { dbPath: options.dbPath })
+  } catch {
+    // Si no existe la tabla, continuar
+  }
+
+  return indexDocument(document, options)
+}
+
+/**
+ * Indexa múltiples documentos secuencialmente.
+ *
+ * @param {Array<Object>} documents - Array de documentos en formato opl
+ * @param {IndexOptions} [options]
+ * @returns {Promise<Array<IndexResult>>}
+ */
+export async function indexDocuments(documents, options = {}) {
+  if (!Array.isArray(documents) || documents.length === 0) {
+    return []
+  }
+
+  const results = []
+  for (const doc of documents) {
+    const r = await indexDocument(doc, options)
+    results.push(r)
+  }
+  return results
+}
+
+// ─── Helpers ───────────────────────────────────────────────────────
+
+/**
+ * Emite evento de progreso si hay callback.
+ *
+ * @param {Function|null} cb
+ * @param {IndexProgress} progress
+ */
+function emitProgress(cb, progress) {
+  if (typeof cb === "function") {
+    try {
+      cb(progress)
+    } catch {
+      // Ignorar errores en callback de progreso
+    }
+  }
+}