openprompt-lang 1.2.7 → 1.3.0

package/src/commands/opl-embeddings.js

@@ -0,0 +1,556 @@
+// @use(kind, contract, limit, deps)
+// @kind(feature)
+// @contract(in: subcommand, options -> out: void, sideEffect: indexing/status/remove operations)
+// @limit(lines: 300)
+// @deps(../embeddings/index-pipeline, ../embeddings/vector-store, ../embeddings/embedder)
+
+/**
+ * Comando CLI para gestionar embeddings vectoriales.
+ *
+ * Subcomandos:
+ *   opl embeddings index <docId>    → Indexar un documento
+ *   opl embeddings status           → Mostrar estado del índice
+ *   opl embeddings remove <docId>   → Eliminar embeddings de un documento
+ *   opl embeddings config           → Mostrar/configurar proveedor de embeddings
+ */
+
+import chalk from "chalk"
+import { indexDocument, reindexDocument } from "../embeddings/index-pipeline.js"
+import { getEmbeddingStats, deleteDocumentEmbeddings } from "../embeddings/vector-store.js"
+import { setActiveProvider, getActiveProvider, checkProvider } from "../embeddings/embedder.js"
+
+// ─── Constantes ────────────────────────────────────────────────────
+
+const CHUNK_STRATEGIES = ["section", "paragraph", "fixed"]
+const PROVIDERS = ["ollama", "transformers"]
+
+// ─── Handler principal ─────────────────────────────────────────────
+
+/**
+ * Ejecuta un subcomando de embeddings.
+ *
+ * @param {string} subcommand - Subcomando a ejecutar
+ * @param {Object} options - Opciones adicionales
+ */
+export async function embeddings(subcommand, options = {}) {
+  switch (subcommand) {
+    case "index":
+      return cmdIndex(options)
+    case "status":
+      return cmdStatus()
+    case "remove":
+      return cmdRemove(options)
+    case "config":
+      return cmdConfig(options)
+    default:
+      showHelp()
+  }
+}
+
+// ─── Subcomandos ───────────────────────────────────────────────────
+
+/**
+ * Indexa un documento en el vector store.
+ */
+async function cmdIndex(options) {
+  const docId = options.docId || options.args?.[0]
+  const strategy = options.strategy || "section"
+  const provider = options.provider || null
+  const model = options.model || null
+  const dryRun = !!options.dryRun
+  const resume = !!options.resume
+
+  if (!docId) {
+    console.log(
+      chalk.yellow("\n⚠️  Especifica un documento a indexar: opl embeddings index <docId>\n")
+    )
+    return
+  }
+
+  if (!CHUNK_STRATEGIES.includes(strategy)) {
+    console.log(
+      chalk.yellow(
+        `\n⚠️  Estrategia inválida: "${strategy}". Usa: ${CHUNK_STRATEGIES.join(", ")}\n`
+      )
+    )
+    return
+  }
+
+  console.log("")
+  console.log(chalk.cyan(`📦  Indexando documento: "${chalk.bold(docId)}"`))
+  console.log(chalk.gray(`  Estrategia: ${strategy}`))
+  if (provider) console.log(chalk.gray(`  Proveedor: ${provider}`))
+  if (model) console.log(chalk.gray(`  Modelo: ${model}`))
+  if (dryRun) console.log(chalk.gray(`  Modo: ${chalk.yellow("dry-run")} (no persiste)`))
+  if (resume) console.log(chalk.gray(`  Modo: ${chalk.yellow("resume")} (salta chunks existentes)`))
+  console.log("")
+
+  // Buscar el documento en el conocimiento
+  const doc = await findDocument(docId)
+  if (!doc) {
+    console.log(chalk.red(`  ✗ Documento "${docId}" no encontrado en el conocimiento.`))
+    console.log(chalk.gray("  Usa el comando: opl index para ver los documentos disponibles."))
+    console.log("")
+    return
+  }
+
+  // Intentar proveedor
+  if (provider) {
+    const status = await checkProvider(provider)
+    if (!status.available) {
+      console.log(chalk.yellow(`  ⚠  Proveedor "${provider}" no disponible: ${status.error}`))
+      console.log(chalk.gray("  Usa: opl embeddings config --provider transformers"))
+      console.log("")
+      return
+    }
+  }
+
+  const indexOptions = {
+    strategy,
+    provider,
+    model,
+    dryRun,
+    resume,
+    onProgress: (p) => {
+      if (p.phase === "done") return
+      const icon =
+        p.phase === "chunking"
+          ? "📄"
+          : p.phase === "embedding"
+            ? "🧠"
+            : p.phase === "storing"
+              ? "💾"
+              : "•"
+      const percent = p.percent !== undefined ? ` ${p.percent}%` : ""
+      console.log(
+        `  ${icon} ${p.phase.charAt(0).toUpperCase() + p.phase.slice(1)}: ${p.current}/${p.total}${percent}${p.currentChunk ? ` (${p.currentChunk})` : ""}`
+      )
+    },
+  }
+
+  const result = await indexDocument(doc, indexOptions)
+
+  console.log("")
+  if (result.success) {
+    console.log(chalk.green(`  ✅ Documento indexado: ${result.docId}`))
+  } else {
+    console.log(chalk.yellow(`  ⚠️  Indexación parcial: ${result.docId}`))
+  }
+  console.log(chalk.gray(`  ─────────────────────────────────`))
+  console.log(chalk.gray(`  Total chunks:    ${result.totalChunks}`))
+  console.log(chalk.gray(`  Indexados:       ${result.indexedChunks}`))
+  console.log(chalk.gray(`  Saltados:        ${result.skippedChunks}`))
+  console.log(chalk.gray(`  Fallos:          ${result.failedChunks}`))
+  console.log(chalk.gray(`  Duración:        ${result.durationMs}ms`))
+  console.log(chalk.gray(`  Estrategia:      ${result.strategy}`))
+  console.log(chalk.gray(`  Modelo:          ${result.model}`))
+  console.log(chalk.gray(`  Tokens totales:  ${result.totalTokens}`))
+  console.log("")
+
+  if (result.errors.length > 0) {
+    console.log(chalk.yellow("  Errores:"))
+    for (const err of result.errors.slice(0, 5)) {
+      console.log(chalk.gray(`    ${err.id}: ${err.error}`))
+    }
+    if (result.errors.length > 5) {
+      console.log(chalk.gray(`    ... y ${result.errors.length - 5} más`))
+    }
+    console.log("")
+  }
+}
+
+/**
+ * Muestra el estado del índice de embeddings.
+ */
+async function cmdStatus() {
+  console.log("")
+  console.log(chalk.cyan("📊  Estado del índice de embeddings"))
+  console.log("")
+
+  const stats = getEmbeddingStats()
+
+  if (stats.totalEmbeddings === 0) {
+    console.log(chalk.yellow("  No hay documentos indexados."))
+    console.log(chalk.gray("  Usa: opl embeddings index <docId>"))
+    console.log("")
+    return
+  }
+
+  console.log(chalk.gray(`  ┌─────────────────────────────────────────────┐`))
+  console.log(
+    chalk.gray("  │ ") +
+      chalk.white("Chunks indexados:".padEnd(25)) +
+      chalk.bold(String(stats.totalEmbeddings).padStart(18)) +
+      chalk.gray(" │")
+  )
+  console.log(
+    chalk.gray("  │ ") +
+      chalk.white("Documentos:".padEnd(25)) +
+      chalk.bold(String(stats.totalDocs).padStart(18)) +
+      chalk.gray(" │")
+  )
+  console.log(
+    chalk.gray("  │ ") +
+      chalk.white("Dimensión:".padEnd(25)) +
+      chalk.bold(String(stats.dimension).padStart(18)) +
+      chalk.gray(" │")
+  )
+  console.log(
+    chalk.gray("  │ ") +
+      chalk.white("Modelo:".padEnd(25)) +
+      chalk.bold((stats.model || "—").padStart(18)) +
+      chalk.gray(" │")
+  )
+  console.log(
+    chalk.gray("  │ ") +
+      chalk.white("Último indexado:".padEnd(25)) +
+      chalk.bold((stats.lastIndexed || "—").padStart(18)) +
+      chalk.gray(" │")
+  )
+  console.log(
+    chalk.gray("  │ ") +
+      chalk.white("Almacenamiento:".padEnd(25)) +
+      chalk.bold(formatBytes(stats.storageBytes).padStart(18)) +
+      chalk.gray(" │")
+  )
+  console.log(chalk.gray(`  └─────────────────────────────────────────────┘`))
+
+  // Mostrar proveedor activo
+  const activeProvider = getActiveProvider()
+  console.log("")
+  console.log(chalk.gray(`  Proveedor activo: ${activeProvider || "auto-detect"}`))
+  console.log("")
+}
+
+/**
+ * Elimina embeddings de un documento.
+ */
+async function cmdRemove(options) {
+  const docId = options.docId || options.args?.[0]
+
+  if (!docId) {
+    console.log(chalk.yellow("\n⚠️  Especifica un documento: opl embeddings remove <docId>\n"))
+    return
+  }
+
+  const result = deleteDocumentEmbeddings(docId)
+  console.log("")
+  if (result.deleted > 0) {
+    console.log(chalk.green(`  ✅ Eliminados ${result.deleted} chunks del documento "${docId}".`))
+  } else {
+    console.log(chalk.yellow(`  ⚠️  No se encontraron embeddings del documento "${docId}".`))
+  }
+  console.log("")
+}
+
+/**
+ * Muestra o configura el proveedor de embeddings.
+ */
+async function cmdConfig(options) {
+  const setProvider = options.provider || null
+
+  if (setProvider) {
+    if (!PROVIDERS.includes(setProvider)) {
+      console.log(
+        chalk.yellow(`\n⚠️  Proveedor inválido: "${setProvider}". Usa: ${PROVIDERS.join(", ")}\n`)
+      )
+      return
+    }
+
+    const status = await checkProvider(setProvider)
+    if (!status.available) {
+      console.log(chalk.yellow(`\n⚠️  Proveedor "${setProvider}" no disponible: ${status.error}\n`))
+      return
+    }
+
+    setActiveProvider(setProvider)
+    console.log(chalk.green(`\n✅  Proveedor cambiado a: ${setProvider}\n`))
+    return
+  }
+
+  // Mostrar configuración actual
+  console.log("")
+  console.log(chalk.cyan("⚙️   Configuración de embeddings"))
+  console.log("")
+
+  const activeProvider = getActiveProvider()
+  console.log(chalk.gray(`  Proveedor activo: ${activeProvider || "auto-detect"}`))
+  console.log("")
+
+  // Chequear disponibilidad de proveedores
+  for (const p of PROVIDERS) {
+    const status = await checkProvider(p)
+    const icon = status.available ? "✅" : "❌"
+    const detail = status.available ? "" : chalk.gray(` (${status.error})`)
+    console.log(`  ${icon} ${p.padEnd(14)}${detail}`)
+  }
+
+  console.log("")
+  console.log(
+    chalk.gray("  Para cambiar:    opl embeddings config --provider <ollama|transformers>")
+  )
+  console.log(chalk.gray("  Para indexar:    opl embeddings index <docId>"))
+  console.log("")
+}
+
+// ─── Helpers ───────────────────────────────────────────────────────
+
+/**
+ * Busca un documento en los archivos de conocimiento del proyecto.
+ * Intenta cargar el documento desde la estructura de directorios de conocimiento OPL.
+ *
+ * @param {string} docId - ID del documento a buscar
+ * @returns {Promise<Object|null>}
+ */
+async function findDocument(docId) {
+  const { existsSync, readdirSync, readFileSync } = await import("fs")
+  const { join } = await import("path")
+  const { getProjectKnowledgeDir } = await import("./knowledge-helpers.js")
+
+  const knowledgeDir = getProjectKnowledgeDir()
+  if (!knowledgeDir || !existsSync(knowledgeDir)) return null
+
+  // 1. Buscar en dominios (knowledge/<domain>/<docId>/)
+  const domains = readdirSync(knowledgeDir, { withFileTypes: true }).filter((d) => d.isDirectory())
+
+  for (const domain of domains) {
+    const domainPath = join(knowledgeDir, domain.name)
+    const entries = readdirSync(domainPath, { withFileTypes: true })
+
+    for (const entry of entries) {
+      if (!entry.isDirectory()) continue
+
+      if (entry.name === docId || entry.name.includes(docId)) {
+        const docPath = join(domainPath, entry.name)
+        const docFiles = readdirSync(docPath).filter(
+          (f) => f.endsWith(".json") || f.endsWith(".md")
+        )
+
+        for (const file of docFiles) {
+          if (file.endsWith(".json")) {
+            try {
+              const data = JSON.parse(readFileSync(join(docPath, file), "utf-8"))
+              return normalizeDocument(data, entry.name)
+            } catch {
+              /* continuar */
+            }
+          }
+        }
+
+        if (docFiles.length > 0) {
+          const chapters = buildChaptersFromMarkdown(docPath, docFiles, readFileSync, join)
+          if (chapters.length > 0) {
+            return { id: entry.name, title: entry.name.replace(/-/g, " "), chapters }
+          }
+        }
+      }
+    }
+  }
+
+  // 2. Buscar en processed/extractions/<docId>/ (documentos extraídos de PDFs)
+  const extractionsDir = join(knowledgeDir, "processed", "extractions")
+  if (existsSync(extractionsDir)) {
+    const extractions = readdirSync(extractionsDir, { withFileTypes: true }).filter((d) =>
+      d.isDirectory()
+    )
+    for (const entry of extractions) {
+      if (entry.name.includes(docId)) {
+        const docPath = join(extractionsDir, entry.name)
+        const files = readdirSync(docPath).filter((f) => f.endsWith(".md") || f.endsWith(".json"))
+
+        // Intentar construir desde full.md
+        const fullMd = files.find((f) => f === "full.md")
+        if (fullMd) {
+          const content = readFileSync(join(docPath, fullMd), "utf-8")
+          return buildDocumentFromFullMd(docId, content)
+        }
+
+        // Fallback: todos los .md como capítulos
+        const chapters = buildChaptersFromMarkdown(docPath, files, readFileSync, join)
+        if (chapters.length > 0) {
+          return { id: entry.name, title: entry.name.replace(/-/g, " "), chapters }
+        }
+      }
+    }
+  }
+
+  // 3. Buscar en sources/chapters/<docId>/ (capítulos de PDFs procesados)
+  const chaptersDir = join(knowledgeDir, "sources", "chapters")
+  if (existsSync(chaptersDir)) {
+    const chapterDocs = readdirSync(chaptersDir, { withFileTypes: true }).filter((d) =>
+      d.isDirectory()
+    )
+    for (const entry of chapterDocs) {
+      if (entry.name.includes(docId)) {
+        const docPath = join(chaptersDir, entry.name)
+        const files = readdirSync(docPath)
+          .filter((f) => f.endsWith(".md"))
+          .sort()
+        const chapters = buildChaptersFromMarkdown(docPath, files, readFileSync, join)
+        if (chapters.length > 0) {
+          return { id: entry.name, title: entry.name.replace(/-/g, " "), chapters }
+        }
+      }
+    }
+  }
+
+  return null
+}
+
+/**
+ * Construye capítulos desde archivos markdown en un directorio.
+ *
+ * @param {string} dirPath
+ * @param {string[]} files
+ * @param {Function} readFileSync - fs.readFileSync
+ * @param {Function} pathJoin - path.join
+ * @returns {Array<{ index: number, title: string, content: string }>}
+ */
+function buildChaptersFromMarkdown(dirPath, files, readFileSync, pathJoin) {
+  const chapters = []
+  for (const file of files.sort()) {
+    if (file.endsWith(".md")) {
+      const content = readFileSync(pathJoin(dirPath, file), "utf-8")
+      const title = file.replace(/\.md$/, "").replace(/^\d+-/, "").replace(/-/g, " ")
+      chapters.push({
+        index: chapters.length,
+        title: title.charAt(0).toUpperCase() + title.slice(1),
+        content,
+      })
+    }
+  }
+  return chapters
+}
+
+/**
+ * Construye un documento desde un full.md (documento extraído completo).
+ * Detecta capítulos por ## headings o # headings.
+ *
+ * @param {string} docId
+ * @param {string} content
+ * @returns {Object|null}
+ */
+function buildDocumentFromFullMd(docId, content) {
+  const lines = content.split("\n")
+  let title = ""
+  const chapters = []
+  let currentTitle = "Contenido"
+  let currentContent = []
+
+  for (const line of lines) {
+    // Primer # heading es el título del documento
+    const h1Match = line.match(/^# (.+)/)
+    if (h1Match && !title) {
+      title = h1Match[1]
+      continue
+    }
+
+    // ## headings marcan capítulos
+    const h2Match = line.match(/^## (.+)/)
+    if (h2Match) {
+      if (currentContent.length > 0) {
+        chapters.push({
+          index: chapters.length,
+          title: currentTitle,
+          content: currentContent.join("\n").trim(),
+        })
+        currentContent = []
+      }
+      currentTitle = h2Match[1]
+    } else {
+      currentContent.push(line)
+    }
+  }
+
+  // Último capítulo
+  if (currentContent.length > 0) {
+    chapters.push({
+      index: chapters.length,
+      title: currentTitle,
+      content: currentContent.join("\n").trim(),
+    })
+  }
+
+  if (chapters.length === 0) {
+    chapters.push({ index: 0, title: title || docId, content: content.trim() })
+  }
+
+  return {
+    id: docId,
+    title: title || docId,
+    chapters,
+  }
+}
+
+/**
+ * Normaliza un documento JSON al formato esperado por el pipeline.
+ *
+ * @param {Object} data
+ * @param {string} docId
+ * @returns {Object|null}
+ */
+function normalizeDocument(data, docId) {
+  if (!data) return null
+
+  // Si ya tiene el formato opl (id, chapters[])
+  if (data.chapters && Array.isArray(data.chapters)) {
+    return {
+      id: data.id || docId,
+      title: data.title || docId,
+      chapters: data.chapters.map((ch, i) => ({
+        index: ch.index ?? i,
+        title: ch.title || "",
+        content: typeof ch === "string" ? ch : ch.content || "",
+      })),
+    }
+  }
+
+  // Si tiene content directamente, crear un capítulo único
+  if (data.content) {
+    return {
+      id: data.id || docId,
+      title: data.title || docId,
+      chapters: [{ index: 0, title: data.title || docId, content: data.content }],
+    }
+  }
+
+  return null
+}
+
+/**
+ * Formatea bytes a representación legible.
+ *
+ * @param {number} bytes
+ * @returns {string}
+ */
+function formatBytes(bytes) {
+  if (bytes === 0) return "0 B"
+  const units = ["B", "KB", "MB", "GB"]
+  const i = Math.floor(Math.log(bytes) / Math.log(1024))
+  return `${(bytes / Math.pow(1024, i)).toFixed(1)} ${units[i]}`
+}
+
+/**
+ * Muestra ayuda del comando embeddings.
+ */
+function showHelp() {
+  console.log("")
+  console.log(chalk.cyan("🧠  Comando: opl embeddings"))
+  console.log("")
+  console.log(chalk.white("  Subcomandos:"))
+  console.log(chalk.gray("    index <docId>      Indexar un documento (chunk → embed → store)"))
+  console.log(chalk.gray("    status             Mostrar estado del índice de embeddings"))
+  console.log(chalk.gray("    remove <docId>     Eliminar embeddings de un documento"))
+  console.log(chalk.gray("    config             Ver/configurar proveedor de embeddings"))
+  console.log("")
+  console.log(chalk.gray("  Opciones para index:"))
+  console.log(
+    chalk.gray("    --strategy <s>     Estrategia: section, paragraph, fixed (default: section)")
+  )
+  console.log(
+    chalk.gray("    --provider <p>     Proveedor: ollama, transformers (default: auto-detect)")
+  )
+  console.log(chalk.gray("    --dry-run          Simular sin persistir"))
+  console.log(chalk.gray("    --resume           Saltar chunks ya indexados"))
+  console.log("")
+}

package/src/commands/opl-help.js

@@ -108,7 +108,25 @@ export async function showWelcome(options = {}) {
     `    ${chalk.cyan("opl validate")}       ${chalk.gray("→")}  ${chalk.white("Validar anotaciones del proyecto")}`
   )
   console.log(
-    `    ${chalk.cyan("opl workflow")}       ${chalk.gray("→")}  ${chalk.white("Workflow completo: discovery → spec → delivery → close")}`
+    `    ${chalk.cyan("opl workflow")}       ${chalk.gray("→")}  ${chalk.white("Workflow select | discovery | spec | delivery | close")}`
+  )
+  console.log(
+    `    ${chalk.cyan("opl workflow select")} ${chalk.gray("→")}  ${chalk.white("IA selecciona workflow óptimo según descripción de la tarea")}`
+  )
+  console.log(
+    `    ${chalk.cyan("opl epic")}           ${chalk.gray("→")}  ${chalk.white("Épicas: create | list | show | close | status")}`
+  )
+  console.log(
+    `    ${chalk.cyan("opl sprint")}         ${chalk.gray("→")}  ${chalk.white("Sprints: create | list | plan | close")}`
+  )
+  console.log(
+    `    ${chalk.cyan("opl ticket")}         ${chalk.gray("→")}  ${chalk.white("Tickets: create | list | close (bugs y tareas)")}`
+  )
+  console.log(
+    `    ${chalk.cyan("opl embeddings")}     ${chalk.gray("→")}  ${chalk.white("Indexar / status / remove / config (vectores semánticos)")}`
+  )
+  console.log(
+    `    ${chalk.cyan("opl webscrape")}      ${chalk.gray("→")}  ${chalk.white("Extraer contenido web → knowledge + embeddings")}`
   )
   console.log(
     `    ${chalk.cyan("opl teach")}          ${chalk.gray("→")}  ${chalk.white("Sistema de enseñanza: assess, study, project-guide, template")}`
@@ -137,13 +155,19 @@ export async function showWelcome(options = {}) {
     `    ${chalk.green("$")} ${chalk.cyan("opl system list")}           ${chalk.gray("—")}  ${chalk.white("Ver sistemas de conocimiento (agrupaciones)")}`
   )
   console.log(
-    `    ${chalk.green("$")} ${chalk.cyan("opl workflow discovery")}    ${chalk.gray("—")}  ${chalk.white("Arrancar un proyecto con el wizard")}`
+    `    ${chalk.green("$")} ${chalk.cyan("opl workflow select")}       ${chalk.gray("—")}  ${chalk.white("Seleccionar workflow óptimo para tu tarea")}`
+  )
+  console.log(
+    `    ${chalk.green("$")} ${chalk.cyan("opl workflow delivery")}     ${chalk.gray("—")}  ${chalk.white("Iniciar desarrollo con tickets")}`
   )
   console.log(`    ${chalk.green("$")} ${chalk.cyan("opl teach progress")}         ${chalk.gray("—")}  ${chalk.white("Ver tu progreso de aprendizaje consolidado")}
     ${chalk.green("$")} ${chalk.cyan("opl teach assess")}          ${chalk.gray("—")}  ${chalk.white("Diagnosticar tu nivel en un concepto")}`)
   console.log(
     `    ${chalk.green("$")} ${chalk.cyan("opl assess")}                ${chalk.gray("—")}  ${chalk.white("Evaluar qué tan listo para producción estás")}`
   )
+  console.log(
+    `    ${chalk.green("$")} ${chalk.cyan("opl embeddings status")}     ${chalk.gray("—")}  ${chalk.white("Ver estado de los vectores semánticos")}`
+  )
   console.log("")
 
   // Ayuda adicional