openprompt-lang 1.2.6 → 1.3.0

package/src/commands/opl-search.js

@@ -8,6 +8,8 @@ import { join } from "path"
 import chalk from "chalk"
 import { getProjectKnowledgeDir, getDomainsFromTaxonomy, getBookMeta } from "./knowledge-helpers.js"
 import { systemDetect, getSystemTags } from "./opl-system.js"
+import { hybridSearch, getEmbeddingStats } from "../embeddings/vector-store.js"
+import { embed } from "../embeddings/embedder.js"
 
 // ──────────────────────────────────────────────
 // Búsqueda por tags (rápida)
@@ -183,6 +185,66 @@ function searchSemantic(query, domains) {
   return results.sort((a, b) => b.matchCount - a.matchCount).slice(0, 15)
 }
 
+// ──────────────────────────────────────────────
+// Búsqueda vectorial (embeddings)
+// ──────────────────────────────────────────────
+
+async function searchVector(query, domains) {
+  try {
+    // 1. Generar embedding de la consulta
+    let queryVector
+    try {
+      queryVector = await embed(query)
+    } catch {
+      // Embedder no disponible
+      return { vectorResults: [], stats: null }
+    }
+
+    if (!queryVector || queryVector.length === 0) return { vectorResults: [], stats: null }
+
+    // 2. Buscar en vector store
+    const vectorResults = await hybridSearch(query, queryVector, { topK: 10 })
+
+    // 3. Mapear chunks a documentos
+    const docMap = new Map()
+    for (const vr of vectorResults) {
+      const key = vr.docId
+      if (!docMap.has(key)) {
+        docMap.set(key, {
+          domain: "knowledge",
+          id: vr.docId,
+          title: vr.docTitle || vr.docId,
+          type: "vector",
+          score: vr.score,
+          matchCount: 1,
+          chunks: [],
+        })
+      }
+      const entry = docMap.get(key)
+      entry.matchCount++
+      entry.chunks.push({
+        content: vr.content.slice(0, 120),
+        score: vr.score,
+        chapterTitle: vr.chapterTitle,
+      })
+      // Actualizar score al máximo
+      if (vr.score > entry.score) entry.score = vr.score
+    }
+
+    // 4. Obtener stats
+    const stats = getEmbeddingStats()
+
+    return {
+      vectorResults: Array.from(docMap.values())
+        .sort((a, b) => b.score - a.score)
+        .slice(0, 10),
+      stats,
+    }
+  } catch {
+    return { vectorResults: [], stats: null }
+  }
+}
+
 // ──────────────────────────────────────────────
 // Main search function
 // ──────────────────────────────────────────────
@@ -196,6 +258,9 @@ export async function search(query, options = {}) {
     console.log(chalk.gray("    --mode tags      → Búsqueda rápida por etiquetas"))
     console.log(chalk.gray("    --mode fulltext  → Búsqueda literal en texto"))
     console.log(chalk.gray("    --mode semantic  → Búsqueda por concepto/significado"))
+    console.log(
+      chalk.gray("    --mode vector    → Búsqueda semántica con embeddings (requiere Ollama)")
+    )
     console.log(chalk.gray("    --mode hybrid    → Combinación de todos (default)"))
     console.log("")
     return
@@ -208,6 +273,7 @@ export async function search(query, options = {}) {
 
   const domains = getDomainsFromTaxonomy()
   const results = []
+  let vectorStats = null
 
   if (mode === "tags" || mode === "hybrid") {
     const tagResults = searchByTags(query, domains)
@@ -224,6 +290,12 @@ export async function search(query, options = {}) {
     results.push(...semResults.map((r) => ({ ...r, mode: "semantic" })))
   }
 
+  if (mode === "vector" || mode === "hybrid") {
+    const { vectorResults, stats } = await searchVector(query, domains)
+    results.push(...vectorResults.map((r) => ({ ...r, mode: "vector" })))
+    vectorStats = stats
+  }
+
   // Deduplicar y ordenar
   const seen = new Set()
   const unique = results
@@ -240,14 +312,34 @@ export async function search(query, options = {}) {
     console.log(chalk.gray("  Sugerencias:"))
     console.log(chalk.gray("  - Prueba con términos más generales"))
     console.log(chalk.gray("  - Usa --mode semantic para buscar por concepto"))
+    console.log(chalk.gray("  - Usa --mode vector para búsqueda semántica con IA"))
     console.log(chalk.gray("  - Usa opl index para navegar manualmente"))
+
+    // Mostrar estado de embeddings
+    if (vectorStats) {
+      console.log("")
+      console.log(chalk.gray("  Estado de embeddings:"))
+      console.log(
+        chalk.gray(
+          `    📊 ${vectorStats.totalEmbeddings} chunks indexados de ${vectorStats.totalDocs} documentos`
+        )
+      )
+    }
+
     console.log("")
     return
   }
 
-  const modeIcons = { tags: "🏷️", fulltext: "📄", semantic: "🧠", hybrid: "🔀" }
+  const modeIcons = { tags: "🏷️", fulltext: "📄", semantic: "🧠", vector: "🔬", hybrid: "🔀" }
 
   console.log(chalk.bold(`  ${unique.length} resultado(s):`))
+  if (vectorStats && vectorStats.totalEmbeddings > 0) {
+    console.log(
+      chalk.gray(
+        `  📊 ${vectorStats.totalEmbeddings} chunks · ${vectorStats.totalDocs} docs · modelo: ${vectorStats.model || "—"}`
+      )
+    )
+  }
   console.log("")
 
   for (let i = 0; i < unique.length; i++) {
@@ -257,9 +349,14 @@ export async function search(query, options = {}) {
     const title = (r.title || "").length > 50 ? `${(r.title || "").slice(0, 47)}...` : r.title || ""
     const chapter = r.chapter ? chalk.gray(` → ${r.chapter.slice(0, 50)}`) : ""
     const terms = r.terms ? chalk.gray(` [${r.terms.join(", ")}]`) : ""
+    const score = r.score !== undefined ? chalk.gray(` sim: ${(r.score * 100).toFixed(0)}%`) : ""
 
-    console.log(`  ${chalk.cyan(`${num}.`)} ${icon} ${chalk.white(title)}`)
+    console.log(`  ${chalk.cyan(`${num}.`)} ${icon} ${chalk.white(title)}${score}`)
     console.log(`      ${chalk.gray("Dominio:")} ${r.domain}${chapter}${terms}`)
+    if (r.chunks && r.chunks.length > 0) {
+      const topChunk = r.chunks[0]
+      console.log(`      ${chalk.gray("Chunk:")} "${topChunk.content.slice(0, 80)}..."`)
+    }
     console.log(`      ${chalk.cyan(`opl read ${r.domain}/${r.id}`)}`)
     console.log("")
   }
@@ -303,6 +400,13 @@ export async function search(query, options = {}) {
       chalk.italic("Para navegar:") +
       chalk.gray(` opl index <dominio>                                │`)
   )
+  if (vectorStats) {
+    console.log(
+      chalk.gray("│ ") +
+        chalk.italic("Embeddings:") +
+        chalk.gray(` ${vectorStats.totalEmbeddings} chunks · ${vectorStats.model} │`)
+    )
+  }
   console.log(chalk.gray("└─────────────────────────────────────────────────────────────┘"))
   console.log("")
 }

package/src/commands/opl-webscrape.js

@@ -0,0 +1,390 @@
+// @use(kind, contract, limit, deps)
+// @kind(feature)
+// @contract(in: url:string -> out: void, sideEffect: fetch web + crea archivo knowledge + indexa embeddings)
+// @limit(lines: 350)
+// @deps(@mozilla/readability, turndown, node-fetch|undici, ../embeddings/index-pipeline)
+
+/**
+ * Web Scraper OPL — Extrae contenido web, lo limpia con Ollama y lo indexa.
+ *
+ * Flujo:
+ *   1. GET url → HTML
+ *   2. Readability.js → contenido limpio (título + body HTML)
+ *   3. Turndown → Markdown
+ *   4. (opcional) Ollama → limpieza + estructuración
+ *   5. Guardar en knowledge/<dominio>/<id>/
+ *   6. Indexar embeddings automáticamente
+ *
+ * Uso:
+ *   opl webscrape <url>
+ *   opl webscrape <url> --domain frontend
+ *   opl webscrape <url> --no-embed
+ *   opl webscrape <url> --clean (usa Ollama para limpiar)
+ */
+
+import chalk from "chalk"
+
+// ─── Constantes ────────────────────────────────────────────────────
+
+const DEFAULT_DOMAIN = "web"
+
+// ─── Handler principal ─────────────────────────────────────────────
+
+/**
+ * Ejecuta el web scraper.
+ *
+ * @param {string} url - URL a scrapear
+ * @param {Object} [options]
+ * @param {string} [options.domain='web'] - Dominio de conocimiento destino
+ * @param {boolean} [options.noEmbed] - Saltar indexación de embeddings
+ * @param {boolean} [options.clean] - Limpiar contenido con Ollama
+ */
+export async function webscrape(url, options = {}) {
+  if (!url) {
+    showHelp()
+    return
+  }
+
+  const domain = options.domain || DEFAULT_DOMAIN
+  const noEmbed = options.noEmbed === true
+  const doClean = options.clean === true
+
+  // Validar URL
+  let parsedUrl
+  try {
+    parsedUrl = new URL(url)
+  } catch {
+    console.log(chalk.red(`\n❌ URL inválida: "${url}"\n`))
+    return
+  }
+
+  console.log("")
+  console.log(chalk.cyan(`🌐  Web Scraper: "${chalk.bold(url)}"`))
+  console.log(chalk.gray(`  Dominio destino: ${domain}`))
+  console.log("")
+
+  try {
+    // ── Fase 1: Fetch ──────────────────────────────────────────
+    console.log(chalk.blue("  📥 Descargando página..."))
+
+    const response = await fetch(url, {
+      headers: {
+        "User-Agent": "Mozilla/5.0 (compatible; OPL-WebScraper/1.0; +https://openprompt-lang.dev)",
+        Accept: "text/html,application/xhtml+xml",
+      },
+      signal: AbortSignal.timeout(15000),
+    })
+
+    if (!response.ok) {
+      console.log(chalk.red(`  ❌ Error HTTP: ${response.status} ${response.statusText}`))
+      console.log("")
+      return
+    }
+
+    const html = await response.text()
+    console.log(chalk.green(`  ✅ Descargado: ${(html.length / 1024).toFixed(1)} KB`))
+
+    // ── Fase 2: Extraer con Readability ────────────────────────
+    console.log(chalk.blue("  📄 Extrayendo contenido..."))
+
+    let title, content
+    try {
+      const { JSDOM } = await import("jsdom")
+      const dom = new JSDOM(html, { url })
+      const reader = new (await import("@mozilla/readability")).Readability(dom.window.document)
+      const article = reader.parse()
+
+      if (!article || !article.content) {
+        console.log(chalk.yellow("  ⚠️  Readability no pudo extraer contenido estructurado."))
+        console.log(chalk.gray("  Usando raw <body> como fallback..."))
+        const body = dom.window.document.querySelector("body")
+        title = dom.window.document.title || new URL(url).hostname
+        content = body ? body.innerHTML : html
+      } else {
+        title = article.title
+        content = article.content
+      }
+    } catch (err) {
+      console.log(chalk.yellow(`  ⚠️  Readability falló: ${err.message}`))
+      console.log(chalk.gray("  Usando contenido raw HTML..."))
+      title = parsedUrl.hostname
+      content = html
+    }
+
+    console.log(chalk.green(`  ✅ Extraído: "${title}"`))
+
+    // ── Fase 3: Convertir a Markdown ───────────────────────────
+    console.log(chalk.blue("  📝 Convirtiendo a Markdown..."))
+
+    let markdown
+    try {
+      const TurndownService = (await import("turndown")).default
+      const turndown = new TurndownService({
+        headingStyle: "atx",
+        codeBlockStyle: "fenced",
+        emDelimiter: "*",
+      })
+      markdown = turndown.turndown(content)
+    } catch (err) {
+      console.log(chalk.yellow(`  ⚠️  Turndown falló: ${err.message}`))
+      markdown = content.replace(/<[^>]+>/g, "").trim()
+    }
+
+    const wordCount = markdown.split(/\s+/).filter(Boolean).length
+    console.log(chalk.green(`  ✅ Markdown: ${wordCount} palabras`))
+
+    // ── Fase 4 (opcional): Limpiar con Ollama ──────────────────
+    if (doClean) {
+      console.log(chalk.blue("  🧹 Limpiando con Ollama..."))
+      markdown = await cleanWithOllama(markdown, title)
+      console.log(chalk.green("  ✅ Contenido limpiado"))
+    }
+
+    // ── Fase 5: Guardar en knowledge ───────────────────────────
+    console.log(chalk.blue("  💾 Guardando en conocimiento..."))
+
+    const docId = slugify(title)
+    const saved = await saveToKnowledge(docId, domain, title, markdown, url)
+
+    if (!saved) {
+      console.log(chalk.red("  ❌ No se pudo guardar el documento."))
+      console.log("")
+      return
+    }
+
+    console.log(chalk.green(`  ✅ Guardado: knowledge/${domain}/${docId}/`))
+    console.log(chalk.gray(`     ${saved.chapterCount} capítulo(s)`))
+    console.log(chalk.gray(`     ${saved.filePath}`))
+
+    // ── Fase 6: Indexar embeddings ─────────────────────────────
+    if (!noEmbed) {
+      console.log(chalk.blue("  🧠 Indexando embeddings..."))
+      try {
+        const { indexDocument } = await import("../embeddings/index-pipeline.js")
+
+        const doc = {
+          id: docId,
+          title,
+          chapters: saved.chapters.map((ch, i) => ({
+            index: i,
+            title: ch.title,
+            content: ch.content,
+          })),
+        }
+
+        const result = await indexDocument(doc, { strategy: "section" })
+
+        if (result.success) {
+          console.log(
+            chalk.green(`  ✅ Embeddings: ${result.indexedChunks} chunks · ${result.durationMs}ms`)
+          )
+        } else {
+          console.log(
+            chalk.yellow(`  ⚠️  Embedding parcial: ${result.indexedChunks}/${result.totalChunks}`)
+          )
+        }
+      } catch (embedErr) {
+        console.log(chalk.gray(`  ⏭️   Embedding no disponible: ${embedErr.message}`))
+      }
+    }
+
+    console.log("")
+    console.log(chalk.cyan(`✅  Página scrapeada: "${title}"`))
+    console.log(chalk.gray(`   ${url}`))
+    console.log(chalk.gray(`   knowledge/${domain}/${docId}/`))
+    console.log("")
+  } catch (error) {
+    console.log(chalk.red(`\n❌ Error: ${error.message}\n`))
+  }
+}
+
+// ─── Helpers ───────────────────────────────────────────────────────
+
+/**
+ * Limpia y estructura contenido markdown usando Ollama.
+ *
+ * @param {string} markdown - Contenido markdown
+ * @param {string} title - Título del documento
+ * @returns {Promise<string>}
+ */
+async function cleanWithOllama(markdown, title) {
+  try {
+    const response = await fetch("http://localhost:11434/api/generate", {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({
+        model: "llama3.2",
+        prompt: `Limpia y estructura el siguiente contenido web en Markdown. 
+Elimina navegación, publicidad, barras laterales y elementos no relevantes.
+Mantén solo el contenido principal bien estructurado con encabezados.
+
+Título: ${title}
+
+Contenido:
+${markdown.slice(0, 12000)}`,
+        stream: false,
+        options: { temperature: 0.1, num_predict: 4096 },
+      }),
+      signal: AbortSignal.timeout(60000),
+    })
+
+    if (!response.ok) return markdown
+
+    const data = await response.json()
+    return data.response || markdown
+  } catch {
+    return markdown // fallback: devolver sin limpiar
+  }
+}
+
+/**
+ * Guarda el contenido scrapeado en la estructura knowledge/.
+ *
+ * @param {string} docId
+ * @param {string} domain
+ * @param {string} title
+ * @param {string} markdown
+ * @param {string} sourceUrl
+ * @returns {Promise<Object|null>}
+ */
+async function saveToKnowledge(docId, domain, title, markdown, sourceUrl) {
+  const { existsSync, mkdirSync, writeFileSync } = await import("fs")
+  const { join } = await import("path")
+  const { getProjectKnowledgeDir } = await import("./knowledge-helpers.js")
+
+  const knowledgeDir = getProjectKnowledgeDir()
+  if (!knowledgeDir) return null
+
+  const docDir = join(knowledgeDir, domain, docId)
+  mkdirSync(docDir, { recursive: true })
+
+  // Guardar metadata
+  const meta = {
+    id: docId,
+    title,
+    source: sourceUrl,
+    scrapedAt: new Date().toISOString(),
+    domain,
+  }
+  writeFileSync(join(docDir, "meta.json"), JSON.stringify(meta, null, 2), "utf-8")
+
+  // Guardar contenido completo
+  writeFileSync(join(docDir, "full.md"), markdown, "utf-8")
+
+  // Detectar capítulos por ## headings
+  const chapters = detectChapters(markdown, title)
+  const chDir = join(docDir, "chapters")
+  mkdirSync(chDir, { recursive: true })
+
+  for (const ch of chapters) {
+    const safeTitle = slugify(ch.title)
+    const fileName = `${String(ch.index).padStart(2, "0")}-${safeTitle}.md`
+    writeFileSync(join(chDir, fileName), ch.content, "utf-8")
+  }
+
+  return {
+    filePath: join(domain, docId),
+    chapterCount: chapters.length,
+    chapters,
+  }
+}
+
+/**
+ * Detecta capítulos en un texto markdown usando ## headings.
+ *
+ * @param {string} markdown
+ * @param {string} defaultTitle
+ * @returns {Array<{ index: number, title: string, content: string }>}
+ */
+function detectChapters(markdown, defaultTitle) {
+  const lines = markdown.split("\n")
+  const chapters = []
+  let currentTitle = defaultTitle || "Introducción"
+  let currentContent = []
+
+  for (const line of lines) {
+    const headingMatch = line.match(/^## (.+)/)
+    if (headingMatch) {
+      if (currentContent.length > 0 || chapters.length > 0) {
+        chapters.push({
+          index: chapters.length,
+          title: currentTitle,
+          content: currentContent.join("\n").trim(),
+        })
+        currentContent = []
+      }
+      currentTitle = headingMatch[1]
+    } else {
+      currentContent.push(line)
+    }
+  }
+
+  // Último capítulo
+  if (currentContent.length > 0) {
+    chapters.push({
+      index: chapters.length,
+      title: currentTitle,
+      content: currentContent.join("\n").trim(),
+    })
+  }
+
+  // Si no se encontraron headings, crear un capítulo único
+  if (chapters.length === 0) {
+    chapters.push({
+      index: 0,
+      title: defaultTitle || "Contenido",
+      content: markdown.trim(),
+    })
+  }
+
+  return chapters
+}
+
+/**
+ * Convierte un texto a slug para usar como ID/directorio.
+ *
+ * @param {string} text
+ * @returns {string}
+ */
+function slugify(text) {
+  return text
+    .toLowerCase()
+    .replace(/[^a-z0-9áéíóúüñ\s-]/g, "")
+    .replace(/\s+/g, "-")
+    .replace(/-+/g, "-")
+    .replace(/^-|-$/g, "")
+    .slice(0, 80)
+}
+
+/**
+ * Muestra la ayuda del comando.
+ */
+function showHelp() {
+  console.log("")
+  console.log(chalk.cyan("🌐  Comando: opl webscrape <url>"))
+  console.log("")
+  console.log(chalk.white("  Extrae contenido de una página web y lo indexa en el conocimiento."))
+  console.log("")
+  console.log(chalk.gray("  Opciones:"))
+  console.log(chalk.gray("    --domain <d>       Dominio de conocimiento (default: web)"))
+  console.log(chalk.gray("    --no-embed         No indexar embeddings automáticamente"))
+  console.log(chalk.gray("    --clean            Limpiar contenido con Ollama (requiere Ollama)"))
+  console.log(
+    chalk.gray("    --deep             🕸️  Deep scrape: navega categorías y subcategorías")
+  )
+  console.log(chalk.gray("    --max-depth <n>    Profundidad máxima para deep scrape (default: 2)"))
+  console.log(chalk.gray("    --concurrency <n>  Fetchs simultáneos (default: 3)"))
+  console.log(chalk.gray("    --verbose          Log detallado del proceso"))
+  console.log("")
+  console.log(chalk.gray("  Ejemplos:"))
+  console.log(chalk.gray("    opl webscrape https://react.dev/learn"))
+  console.log(chalk.gray("    opl webscrape https://freefrontend.com --deep --max-depth 2"))
+  console.log(
+    chalk.gray(
+      "    opl webscrape https://freefrontend.com/tailwind-code-examples --deep --concurrency 5"
+    )
+  )
+  console.log(chalk.gray("    opl webscrape https://example.com --domain frontend"))
+  console.log(chalk.gray("    opl webscrape https://example.com --clean"))
+  console.log("")
+}

package/src/commands/work-context.js

@@ -603,6 +603,15 @@ async function start(description) {
 
   writeJSON(SESSION_FILE, session)
   appendLog("session_start", description)
+
+  // Sincronizar con MCP shared state (workflow delivery → modo execute)
+  try {
+    const { syncWorkflowPhase } = await import("../mcp-shared-state.js")
+    syncWorkflowPhase("DELIVERY")
+  } catch {
+    /* silencioso */
+  }
+
   console.log(chalk.green(`\n✅ Sesión iniciada: ${session.session.id}\n`))
   console.log(`  ${chalk.bold("Tarea:")} ${description}`)
   console.log(`  ${chalk.bold("Rama:")}  ${branch}`)
@@ -824,6 +833,14 @@ async function close() {
   writeJSON(SESSION_FILE, session)
   appendLog("session_close", `Sesión cerrada: ${session.session.id}`)
 
+  // Sincronizar con MCP shared state (workflow close → modo plan de nuevo)
+  try {
+    const { syncWorkflowPhase } = await import("../mcp-shared-state.js")
+    syncWorkflowPhase("CLOSE_SESSION")
+  } catch {
+    /* silencioso */
+  }
+
   try {
     const { updateProjectContext } = await import("./project-context.js")
     await updateProjectContext(resolveBase())

package/src/commands/workflow/close/index.js

@@ -16,7 +16,7 @@ import {
 } from "../../../persistence/sqlite/queries.js"
 import { executeTransition, canTransition } from "../../../core/workflow/transitions.js"
 import { generateSessionDocs, generateIndexDoc } from "../../../docgen/session-docs.js"
-import { setPhase } from "../../../mcp-shared-state.js"
+import { setPhase, syncWorkflowPhase } from "../../../mcp-shared-state.js"
 import chalk from "chalk"
 
 /**
@@ -148,6 +148,7 @@ export async function closeWorkflow(options = {}) {
   setMetadata(db, "active_session", "")
   setMetadata(db, "active_ticket", "")
   setPhase("idle", process.cwd())
+  syncWorkflowPhase("CLOSE_SESSION")
 
   // ─── 3. Generar documentación viva ────────────────────────────────────
   const sessionFiles = generateSessionDocs({

package/src/commands/workflow/delivery/index.js

@@ -13,6 +13,7 @@ import {
   createTicket,
   createSession,
 } from "../../../persistence/sqlite/queries.js"
+import { syncWorkflowPhase } from "../../../mcp-shared-state.js"
 import { executeTransition, canTransition } from "../../../core/workflow/transitions.js"
 import { getPhase } from "../../../core/workflow/phases.js"
 import chalk from "chalk"
@@ -133,6 +134,9 @@ export async function delivery(options = {}) {
   setMetadata(db, "active_ticket", ticketId)
   setMetadata(db, "workflow_phase", "DELIVERY")
 
+  // Sincronizar con MCP shared state: DELIVERY → modo execute
+  syncWorkflowPhase("DELIVERY")
+
   conn.close()
 
   const phaseInfo = getPhase("DELIVERY")

package/src/commands/workflow/discovery/index.js

@@ -12,6 +12,7 @@ import {
   canTransition,
   getInitialContext,
 } from "../../../core/workflow/transitions.js"
+import { syncWorkflowPhase } from "../../../mcp-shared-state.js"
 import { runWizard, getWizardResponses } from "../../../wizard/orchestrator.js"
 import chalk from "chalk"
 
@@ -75,6 +76,9 @@ export async function discovery(options = {}) {
     setMetadata(db, "workflow_project_id", projectId)
   }
 
+  // Sincronizar con MCP shared state: DISCOVERY → modo plan
+  syncWorkflowPhase("DISCOVERY")
+
   conn.close()
 
   console.log(chalk.green(`\n✅ Discovery completado para: ${projectId}`))