openprompt-lang 1.2.6 → 1.3.0

package/src/core/webscrape/deep-scraper.js

@@ -0,0 +1,1027 @@
+// @use(kind, contract, limit, deps)
+// @kind(module)
+// @contract(in: url -> out: structured knowledge tree, sideEffect: multiple fetches + saves)
+// @limit(lines: 500)
+// @deps(../commands/knowledge-helpers, node-fetch, jsdom)
+
+/**
+ * Deep Scraper OPL — Navega sitios jerárquicos (catálogos, directorios)
+ * y extrae contenido estructurado con navegación recursiva.
+ *
+ * Arquitectura:
+ *   1. Detecta estructura del sitio (categorías, subcategorías, items)
+ *   2. Sigue enlaces recursivamente con profundidad configurable
+ *   3. Extrae metadatos de cada página (cards, listados, etc.)
+ *   4. Intenta obtener código fuente (CodePen, GitHub, etc.)
+ *   5. Guarda todo en knowledge/ con estructura jerárquica
+ *
+ * Uso:
+ *   import { deepScrape } from "./deep-scraper.js"
+ *   await deepScrape("https://freefrontend.com", { maxDepth: 3 })
+ */
+
+import chalk from "chalk"
+import { analyzePage, PAGE_TYPES as ANALYZER_PAGE_TYPES } from "./analyzer.js"
+
+// Modelo Ollama por defecto (llama3.2 es más ligero para PCs modestas)
+const DEFAULT_OLLAMA_MODEL = "llama3.2"
+
+// ─── Constantes ────────────────────────────────────────────────────
+
+const DEFAULT_MAX_DEPTH = 2
+const DEFAULT_CONCURRENCY = 3
+const REQUEST_TIMEOUT = 15000
+const USER_AGENT = "Mozilla/5.0 (compatible; OPL-DeepScraper/1.0; +https://openprompt-lang.dev)"
+
+// ─── Tipos de página detectables ────────────────────────────────────
+
+const PAGE_TYPES = {
+  CATALOG_HOME: "catalog-home", // freefrontend.com (tech categories)
+  CATEGORY_INDEX: "category-index", // /tailwind-code-examples/ (subcategories list)
+  ITEM_LIST: "item-list", // /tailwind-buttons/ (card grid)
+  ITEM_DETAIL: "item-detail", // Individual example page
+  ARTICLE: "article", // Blog post / article
+  UNKNOWN: "unknown",
+}
+
+// ─── Handler principal ────────────────────────────────────────────
+
+/**
+ * Ejecuta el deep scrape desde una URL raíz.
+ *
+ * @param {string} startUrl - URL inicial
+ * @param {Object} [options]
+ * @param {number} [options.maxDepth=2] - Profundidad máxima de navegación
+ * @param {number} [options.concurrency=3] - Fetchs simultáneos
+ * @param {string} [options.domain='web'] - Dominio OPL destino
+ * @param {boolean} [options.noEmbed] - Saltar embeddings
+ * @param {boolean} [options.verbose] - Log detallado
+ * @returns {Promise<{success: boolean, pagesScraped: number, itemsExtracted: number, tree: Object}>}
+ */
+export async function deepScrape(startUrl, options = {}) {
+  const maxDepth = options.maxDepth || DEFAULT_MAX_DEPTH
+  const concurrency = options.concurrency || DEFAULT_CONCURRENCY
+  const domain = options.domain || "web"
+  const verbose = options.verbose === true
+  const noEmbed = options.noEmbed === true
+  const useOllama = options.ollama !== false // default: true
+
+  console.log("")
+  console.log(chalk.cyan(`🕸️  Deep Scraper: "${chalk.bold(startUrl)}"`))
+  console.log(chalk.gray(`  Profundidad máxima: ${maxDepth}`))
+  console.log(chalk.gray(`  Concurrencia: ${concurrency}`))
+  if (useOllama) console.log(chalk.gray(`  🧠 Ollama: habilitado (llama3.2)`))
+  console.log("")
+
+  const visited = new Set()
+  const tree = {
+    url: startUrl,
+    type: null,
+    title: null,
+    children: [],
+    items: [],
+    stats: { pagesScraped: 0, itemsExtracted: 0 },
+  }
+
+  try {
+    await scrapeRecursive(startUrl, 0, maxDepth, visited, tree, {
+      domain,
+      verbose,
+      concurrency,
+      noEmbed,
+      useOllama,
+      ollamaModel: DEFAULT_OLLAMA_MODEL,
+    })
+
+    // Resultado final
+    console.log("")
+    console.log(chalk.cyan("═".repeat(50)))
+    console.log(chalk.green(`✅  Deep Scrape completado`))
+    console.log(chalk.gray(`  Páginas scrapeadas: ${tree.stats.pagesScraped}`))
+    console.log(chalk.gray(`  Items extraídos: ${tree.stats.itemsExtracted}`))
+    console.log(chalk.gray(`  Nivel de profundidad: ${maxDepth}`))
+    if (tree.stats.codepenLinks) {
+      console.log(chalk.gray(`  Enlaces a CodePen: ${tree.stats.codepenLinks}`))
+    }
+    console.log("")
+
+    return {
+      success: true,
+      pagesScraped: tree.stats.pagesScraped,
+      itemsExtracted: tree.stats.itemsExtracted,
+      tree,
+    }
+  } catch (error) {
+    console.log(chalk.red(`\n❌ Deep Scrape falló: ${error.message}\n`))
+    return {
+      success: false,
+      pagesScraped: tree.stats.pagesScraped,
+      itemsExtracted: tree.stats.itemsExtracted,
+      tree,
+      error: error.message,
+    }
+  }
+}
+
+// ─── Navegación recursiva ─────────────────────────────────────────
+
+/**
+ * Scrapea una URL y decide cómo continuar según el tipo de página.
+ *
+ * @param {string} url
+ * @param {number} depth - Profundidad actual
+ * @param {number} maxDepth - Profundidad máxima
+ * @param {Set} visited - URLs ya visitadas
+ * @param {Object} tree - Árbol de resultado
+ * @param {Object} opts - Opciones
+ */
+async function scrapeRecursive(url, depth, maxDepth, visited, tree, opts) {
+  if (visited.has(url) || depth > maxDepth) return
+  visited.add(url)
+
+  const { hostname, pathname } = new URL(url)
+  const isSameDomain = (link) => {
+    try {
+      return new URL(link, url).hostname === hostname
+    } catch {
+      return false
+    }
+  }
+
+  if (opts.verbose) {
+    console.log(chalk.gray(`  ${"  ".repeat(depth)}🔍 ${depth > 0 ? "└─ " : ""}${url}`))
+  }
+
+  // Fetch HTML
+  const html = await fetchPage(url)
+  if (!html) return
+
+  tree.stats.pagesScraped++
+
+  // Detectar tipo de página
+  let pageType
+  let ollamaPlan = null
+
+  // Solo usar Ollama para la primera página (depth === 0).
+  // Para subpáginas, usar heurísticas que son más rápidas.
+  if (opts.useOllama && depth === 0) {
+    // Usar Ollama para clasificar la página raíz
+    try {
+      ollamaPlan = await analyzePage(html, url, { model: opts.ollamaModel })
+      pageType = mapAnalyzerType(ollamaPlan.pageType)
+      if (opts.verbose) {
+        console.log(
+          chalk.gray(
+            `  ${"  ".repeat(depth)}🧠 Ollama: ${ollamaPlan.pageType} (${ollamaPlan.confidence}%) → ${pageType}`
+          )
+        )
+      }
+    } catch {
+      // Fallback a heurísticas si Ollama falla
+      pageType = detectPageType(url, html)
+    }
+  } else {
+    // Heurísticas rápidas para subpáginas
+    pageType = detectPageType(url, html)
+  }
+  if (!tree.type) tree.type = pageType
+
+  // Extraer título
+  const title = extractTitle(html, url)
+  if (!tree.title) tree.title = title
+
+  // Según el tipo de página, extraer diferente contenido
+  switch (pageType) {
+    case PAGE_TYPES.CATALOG_HOME:
+      await handleCatalogHome(
+        url,
+        html,
+        depth,
+        maxDepth,
+        visited,
+        tree,
+        opts,
+        isSameDomain,
+        ollamaPlan
+      )
+      break
+
+    case PAGE_TYPES.CATEGORY_INDEX:
+      await handleCategoryIndex(
+        url,
+        html,
+        depth,
+        maxDepth,
+        visited,
+        tree,
+        opts,
+        isSameDomain,
+        ollamaPlan
+      )
+      break
+
+    case PAGE_TYPES.ITEM_LIST:
+      await handleItemList(
+        url,
+        html,
+        depth,
+        maxDepth,
+        visited,
+        tree,
+        opts,
+        isSameDomain,
+        ollamaPlan
+      )
+      break
+
+    default:
+      // Unknown: solo extraer contenido con Readability
+      await extractGenericContent(url, html, tree, opts)
+      break
+  }
+}
+
+// ─── Mapeo de tipos de página ──────────────────────────────────────
+
+/**
+ * Mapea tipos del analyzer a tipos internos del deep scraper.
+ * El analyzer puede devolver strings como "catalog-home" o "item-list"
+ * que mapeamos a nuestros PAGE_TYPES.
+ */
+function mapAnalyzerType(analyzerType) {
+  const type = (analyzerType || "").toLowerCase().trim()
+  const mapping = {
+    "catalog-home": PAGE_TYPES.CATALOG_HOME,
+    "category-index": PAGE_TYPES.CATEGORY_INDEX,
+    "item-list": PAGE_TYPES.ITEM_LIST,
+    "item-detail": PAGE_TYPES.ITEM_DETAIL,
+    article: PAGE_TYPES.ARTICLE,
+    product: PAGE_TYPES.PRODUCT,
+    docs: PAGE_TYPES.DOCS,
+    portfolio: PAGE_TYPES.PORTFOLIO,
+    landing: PAGE_TYPES.LANDING,
+  }
+  return mapping[type] || PAGE_TYPES.UNKNOWN
+}
+
+// ─── Detectores de tipo de página ─────────────────────────────────
+
+/**
+ * Detecta el tipo de página basado en URL y contenido HTML.
+ */
+function detectPageType(url, html) {
+  const path = new URL(url).pathname.toLowerCase()
+
+  // Catalog homepage: tiene nav con categorías principales
+  if (path === "/" || path === "") {
+    if (hasCatalogStructure(html)) return PAGE_TYPES.CATALOG_HOME
+    return PAGE_TYPES.UNKNOWN
+  }
+
+  // Category index: /tailwind-code-examples/ - tiene subcategorías listadas
+  if (path.match(/\/[a-z-]+(?:code-examples|examples)\/?$/)) {
+    if (hasSubcategoryLinks(html)) return PAGE_TYPES.CATEGORY_INDEX
+    return PAGE_TYPES.ITEM_LIST
+  }
+
+  // Item list: /tailwind-buttons/ - tiene grid de cards
+  if (path.match(/\/tailwind-[a-z-]+\/?$/)) {
+    if (hasCardGrid(html)) return PAGE_TYPES.ITEM_LIST
+    return PAGE_TYPES.UNKNOWN
+  }
+
+  return PAGE_TYPES.UNKNOWN
+}
+
+/**
+ * Detecta si el HTML tiene estructura de catálogo (nav con tecnologías).
+ */
+function hasCatalogStructure(html) {
+  // Buscar sidebar con grupos de navegación (Languages, Frameworks)
+  return (
+    (html.includes("nav-title") || html.includes("nav-group")) &&
+    (html.includes("code-examples") || html.includes("nav-list"))
+  )
+}
+
+/**
+ * Detecta si el HTML contiene una lista de subcategorías.
+ */
+function hasSubcategoryLinks(html) {
+  const links = extractAllLinks(html)
+  const subcatCount = links.filter(
+    (l) => l.match(/\/[a-z]+-[a-z-]+\/?$/) && !l.includes("page/")
+  ).length
+  return subcatCount > 5
+}
+
+/**
+ * Detecta si el HTML contiene un grid de cards.
+ */
+function hasCardGrid(html) {
+  const cardCount = (html.match(/<div[^>]*class="?card[^"]*"?>/gi) || []).length
+  return cardCount > 2
+}
+
+// ─── Handlers por tipo de página ──────────────────────────────────
+
+/**
+ * Maneja páginas tipo CATALOG_HOME (freefrontend.com).
+ * Extrae categorías de tecnología del nav y las sigue.
+ */
+async function handleCatalogHome(
+  url,
+  html,
+  depth,
+  maxDepth,
+  visited,
+  tree,
+  opts,
+  isSameDomain,
+  ollamaPlan = null
+) {
+  // Extraer links de categorías del nav/sidebar
+  const links = extractCategoryLinks(html, url)
+
+  if (opts.verbose) {
+    console.log(chalk.gray(`  ${"  ".repeat(depth)}📂 Categorías encontradas: ${links.length}`))
+  }
+
+  const childNode = {
+    url,
+    type: PAGE_TYPES.CATALOG_HOME,
+    title: extractTitle(html, url),
+    children: [],
+    items: [],
+  }
+
+  // Seguir cada categoría (con concurrencia limitada)
+  const batchSize = opts.concurrency
+  for (let i = 0; i < links.length; i += batchSize) {
+    const batch = links.slice(i, i + batchSize)
+    await Promise.all(
+      batch.map(async (link) => {
+        const childTree = {
+          url: link,
+          children: [],
+          items: [],
+          stats: { pagesScraped: 0, itemsExtracted: 0 },
+        }
+        await scrapeRecursive(link, depth + 1, maxDepth, visited, childTree, opts)
+        if (childTree.stats.pagesScraped > 0 || childTree.stats.itemsExtracted > 0) {
+          childNode.children.push(childTree)
+          tree.stats.pagesScraped += childTree.stats.pagesScraped
+          tree.stats.itemsExtracted += childTree.stats.itemsExtracted
+        }
+      })
+    )
+  }
+
+  tree.children.push(childNode)
+}
+
+/**
+ * Maneja páginas tipo CATEGORY_INDEX (/tailwind-code-examples/).
+ * Extrae subcategorías y las sigue.
+ */
+async function handleCategoryIndex(
+  url,
+  html,
+  depth,
+  maxDepth,
+  visited,
+  tree,
+  opts,
+  isSameDomain,
+  ollamaPlan = null
+) {
+  let links = extractSubcategoryLinks(html, url)
+
+  // Si Ollama sugirió un filtro de links, usarlo para priorizar
+  if (ollamaPlan?.linkFilter && links.length > 0) {
+    const filter = ollamaPlan.linkFilter.toLowerCase()
+    const filtered = links.filter((l) => l.toLowerCase().includes(filter))
+    if (filtered.length > 0 && filtered.length < links.length) {
+      if (opts.verbose) {
+        console.log(
+          chalk.gray(
+            `  ${"  ".repeat(depth)}🧠 Ollama filter: "${filter}" → ${filtered.length}/${links.length} links`
+          )
+        )
+      }
+      links = filtered
+    }
+  }
+
+  // Si Ollama detectó links específicos, usarlos como prioridad
+  if (ollamaPlan?.links?.length > 0) {
+    const ollamaUrls = ollamaPlan.links
+      .filter((l) => l.type === "category" || l.type === "subcategory")
+      .map((l) => {
+        try {
+          return l.url.startsWith("http") ? l.url : new URL(l.url, url).href
+        } catch {
+          return null
+        }
+      })
+      .filter(Boolean)
+
+    if (ollamaUrls.length > 0) {
+      // Combinar: links de Ollama que también estén en los extraídos por heurísticas
+      const heuristicSet = new Set(links)
+      const ollamaMatched = ollamaUrls.filter((u) => heuristicSet.has(u))
+      if (ollamaMatched.length > 0) {
+        // Ollama acertó: usar solo los relevantes
+        links = ollamaMatched
+        if (opts.verbose) {
+          console.log(
+            chalk.gray(
+              `  ${"  ".repeat(depth)}🧠 Ollama links: ${ollamaMatched.length}/${ollamaUrls.length} matched`
+            )
+          )
+        }
+      }
+    }
+  }
+
+  if (opts.verbose) {
+    console.log(chalk.gray(`  ${"  ".repeat(depth)}📁 Subcategorías: ${links.length}`))
+  }
+
+  const childNode = {
+    url,
+    type: PAGE_TYPES.CATEGORY_INDEX,
+    title: extractTitle(html, url),
+    children: [],
+    items: [],
+  }
+
+  const batchSize = opts.concurrency
+  for (let i = 0; i < links.length; i += batchSize) {
+    const batch = links.slice(i, i + batchSize)
+    await Promise.all(
+      batch.map(async (link) => {
+        const childTree = {
+          url: link,
+          children: [],
+          items: [],
+          stats: { pagesScraped: 0, itemsExtracted: 0 },
+        }
+        await scrapeRecursive(link, depth + 1, maxDepth, visited, childTree, opts)
+        if (childTree.stats.pagesScraped > 0 || childTree.stats.itemsExtracted > 0) {
+          childNode.children.push(childTree)
+          tree.stats.pagesScraped += childTree.stats.pagesScraped
+          tree.stats.itemsExtracted += childTree.stats.itemsExtracted
+        }
+      })
+    )
+  }
+
+  tree.children.push(childNode)
+}
+
+/**
+ * Maneja páginas tipo ITEM_LIST (/tailwind-buttons/).
+ * Extrae todas las cards con su metadata.
+ */
+async function handleItemList(
+  url,
+  html,
+  depth,
+  maxDepth,
+  visited,
+  tree,
+  opts,
+  isSameDomain,
+  ollamaPlan = null
+) {
+  const cards = extractCards(html, url)
+
+  if (opts.verbose) {
+    console.log(chalk.gray(`  ${"  ".repeat(depth)}🎴 Cards: ${cards.length}`))
+  }
+
+  const childNode = {
+    url,
+    type: PAGE_TYPES.ITEM_LIST,
+    title: extractTitle(html, url),
+    children: [],
+    items: cards,
+  }
+
+  tree.children.push(childNode)
+  tree.stats.itemsExtracted += cards.length
+
+  // Guardar cada card como conocimiento
+  for (const card of cards) {
+    await saveCard(card, opts.domain)
+  }
+
+  // Contar CodePen links
+  const codepenCount = cards.filter((c) => c.codepenSlug).length
+  if (!tree.stats.codepenLinks) tree.stats.codepenLinks = 0
+  tree.stats.codepenLinks += codepenCount
+}
+
+// ─── Extractores de estructura ─────────────────────────────────────
+
+/**
+ * Extrae links de categorías del nav/sidebar del homepage.
+ */
+function extractCategoryLinks(html, baseUrl) {
+  const links = extractAllLinks(html)
+  // Filtrar solo links que sean categorías de tecnología
+  const techPatterns = [
+    "code-examples",
+    "html",
+    "css",
+    "javascript",
+    "js",
+    "bootstrap",
+    "tailwind",
+    "react",
+    "vue",
+    "angular",
+    "svelte",
+    "jquery",
+    "typescript",
+  ]
+
+  return [
+    ...new Set(
+      links.filter((l) => {
+        const path = l.replace(baseUrl.origin, "")
+        return techPatterns.some((p) => path.includes(p)) && !path.includes("page/")
+      })
+    ),
+  ].map((l) => (l.startsWith("http") ? l : new URL(l, baseUrl).href))
+}
+
+/**
+ * Extrae links de subcategorías de una página de categoría.
+ * Ej: /tailwind-code-examples/ → /tailwind-buttons/, /tailwind-cards/, etc.
+ */
+function extractSubcategoryLinks(html, baseUrl) {
+  const links = extractAllLinks(html)
+  const currentPath = new URL(baseUrl).pathname.replace(/\/$/, "")
+  const pathPrefix = currentPath.split("/").pop() || ""
+  const tech = pathPrefix.split("-code")[0] || pathPrefix
+
+  return [
+    ...new Set(
+      links.filter((l) => {
+        const path = typeof l === "string" ? l.replace(baseUrl.origin, "") : ""
+        // Debe ser un link a una subcategoría del mismo tipo
+        return (
+          path.includes(tech) &&
+          path !== currentPath && // excluir la misma página
+          !path.includes("page/") &&
+          !path.includes("code-examples") &&
+          path.split("/").filter(Boolean).length <= 3
+        )
+      })
+    ),
+  ]
+    .map((l) => (l.startsWith("http") ? l : new URL(l, baseUrl).href))
+    .slice(0, 100)
+}
+
+/**
+ * Extrae todas las URLs href de un HTML.
+ */
+function extractAllLinks(html) {
+  const hrefs = [...html.matchAll(/href=(\S+?)(?:\s|>)/g)]
+  return hrefs
+    .map((m) => {
+      let v = m[1]
+      if ((v.startsWith('"') && v.endsWith('"')) || (v.startsWith("'") && v.endsWith("'"))) {
+        v = v.slice(1, -1)
+      }
+      return v
+    })
+    .filter((h) => {
+      return (
+        h &&
+        !h.startsWith("#") &&
+        !h.startsWith("javascript") &&
+        !h.startsWith("mailto") &&
+        !h.startsWith("tel") &&
+        !h.includes("google") &&
+        !h.includes("facebook") &&
+        !h.includes("twitter") &&
+        !h.endsWith(".css") &&
+        !h.endsWith(".js") &&
+        !h.endsWith(".png") &&
+        !h.endsWith(".jpg") &&
+        !h.endsWith(".svg") &&
+        !h.endsWith(".ico") &&
+        !h.endsWith(".xml") &&
+        h.length > 2
+      )
+    })
+}
+
+// ─── Extractores de contenido ──────────────────────────────────────
+
+/**
+ * Extrae el título de una página HTML.
+ */
+function extractTitle(html, url) {
+  const titleMatch = html.match(/<title>([^<]+)<\/title>/i)
+  if (titleMatch) return titleMatch[1].trim()
+  return new URL(url).hostname
+}
+
+/**
+ * Extrae todas las cards de una página ITEM_LIST.
+ *
+ * freefrontend.com tiene 2 tipos de cards:
+ *
+ * Tipo A (CodePen popover, ~12 por página):
+ *   <div class=card>
+ *     <div class=card-header>
+ *       <h3>Título</h3>
+ *       <button popovertarget=XXX>demo & code</button>
+ *     </div>
+ *     <div class=card-content>...</div>
+ *     <div class=card-meta-info>...</div>
+ *   </div>
+ *   <div id=XXX popover><p class=codepen data-slug-hash=XXX></div>
+ *
+ * Tipo B (External link, ~18 por página, <div class=snippet-card>):
+ *   <div class=snippet-card>
+ *     <div class=card-header>
+ *       <h3><a href=https://...>Título<img class=external></a></h3>
+ *     </div>
+ *     <video/img ...>
+ *   </div>
+ */
+function extractCards(html, baseUrl) {
+  // ── Fase 1: Construir mapa popover ID → slug ──
+  const popoverMap = new Map()
+  const popoverRegex = /<div[^>]*\bid=(\S+?)[\s>][^>]*\bpopover\b[^>]*>([\s\S]{0,2000}?)<\/div>/gi
+  let pMatch
+  while ((pMatch = popoverRegex.exec(html)) !== null) {
+    const id = pMatch[1].replace(/"/g, "")
+    const slugMatch = pMatch[2].match(/data-slug-hash=([^\s>"']+)/)
+    if (slugMatch) {
+      popoverMap.set(id, slugMatch[1])
+    }
+  }
+
+  const cards = []
+
+  // ── Tipo A: cards con popover CodePen ──
+  // La estructura real tiene card-content ANIDADO:
+  //   <div class=card-header>...<button popovertarget=XXX>...</div>
+  //   <div class=card-content>
+  //     <div class=card-content>  ← anidado!
+  //       <p>Descripción</p>
+  //     </div>
+  //   </div>
+  //   <div class=card-meta-info>...</div>
+  //
+  // Estrategia: buscar card-header que contenga popovertarget,
+  // luego card-meta-info, y entre medio está el content.
+  const typeAPattern = /<div class=card-header>([\s\S]{0,800}?)<\/div>/gi
+  let aMatch
+  while ((aMatch = typeAPattern.exec(html)) !== null) {
+    const header = aMatch[1]
+    const headerStart = aMatch.index
+
+    // Solo procesar si tiene popovertarget (Tipo A)
+    const popoverTarget = (header.match(/popovertarget=(\S+?)(?:\s|>)/) || [])[1]
+    if (!popoverTarget) continue
+
+    const title = extractCardTitle(header)
+    const codepenSlug = popoverMap.get(popoverTarget) || null
+
+    // Buscar card-meta-info DESPUÉS del header (hasta ~5000 chars)
+    const afterHeader = html.slice(headerStart, headerStart + 5000)
+    const metaMatch = afterHeader.match(
+      /<div class=card-meta-info>([\s\S]{0,3000}?)<\/div>\s*<\/div>/
+    )
+    const meta = metaMatch ? metaMatch[0] : ""
+
+    // Buscar descripción: el <p> entre card-header y card-meta-info
+    const contentEnd = metaMatch
+      ? afterHeader.indexOf("<div class=card-meta-info>")
+      : afterHeader.length
+    const contentSection = afterHeader.slice(0, contentEnd)
+
+    // Extraer la descripción del <p> más largo
+    const desc = extractCardDescription(contentSection)
+
+    cards.push({
+      title,
+      description: desc,
+      technologies: extractTechnologies(meta),
+      difficulty: extractDifficulty(meta),
+      browserSupport: extractBrowserSupport(meta),
+      features: extractFeatures(meta),
+      requires: extractRequires(contentSection),
+      codepenSlug,
+      demoUrl: codepenSlug ? `https://codepen.io/anon/pen/${codepenSlug}` : null,
+      sourceUrl: baseUrl,
+      type: "codepen",
+    })
+  }
+
+  // ── Tipo B: snippet-cards (enlaces externos) ──
+  // Solo extraer tipo B si hay AL MENOS un card tipo A (significa que
+  // estamos en una página de subcategoría con componentes, no en una
+  // página de categorías que solo lista subcategorías)
+  const hasTypeA = cards.some((c) => c.type === "codepen")
+  if (!hasTypeA) return cards
+
+  // Patrón: <div class=snippet-card>...<div class=card-header>...<a href=...
+  const typeBPattern = /<div class=snippet-card>([\s\S]{0,3000}?)<\/div>\s*<\/div>/gi
+  let bMatch
+  while ((bMatch = typeBPattern.exec(html)) !== null) {
+    const section = bMatch[1]
+    const headerMatch = section.match(/<div class=card-header>([\s\S]{0,800}?)<\/div>/)
+    if (!headerMatch) continue
+
+    const header = headerMatch[1]
+    const title = extractCardTitle(header)
+
+    // Extraer enlace externo
+    const linkMatch = header.match(/href=(\S+?)(?:\s|>)/)
+    let demoUrl = null
+    if (linkMatch) {
+      let link = linkMatch[1]
+      if (
+        (link.startsWith('"') && link.endsWith('"')) ||
+        (link.startsWith("'") && link.endsWith("'"))
+      ) {
+        link = link.slice(1, -1)
+      }
+      if (link.startsWith("http")) {
+        demoUrl = link
+      }
+    }
+
+    // Extraer descripción del snippet-card
+    const desc = extractCardDescription(section)
+
+    // Extraer video/imagen
+    const videoMatch = section.match(/<video[^>]*aria-label="([^"]+)"/)
+    const imgMatch = section.match(/<img[^>]*alt="([^"]+)"/)
+    const mediaDesc = videoMatch ? videoMatch[1] : imgMatch ? imgMatch[1] : null
+
+    cards.push({
+      title,
+      description: desc || mediaDesc || "",
+      technologies: [],
+      difficulty: null,
+      browserSupport: [],
+      features: [],
+      requires: [],
+      codepenSlug: null,
+      demoUrl,
+      sourceUrl: baseUrl,
+      type: "external",
+    })
+  }
+
+  return cards
+}
+
+/**
+ * Extrae el título de una card desde el header.
+ */
+function extractCardTitle(header) {
+  const hMatch = header.match(/<h[234][^>]*>([^<]+)/)
+  return hMatch ? hMatch[1].trim() : "Untitled"
+}
+
+/**
+ * Extrae la descripción de una card.
+ * Busca el <p> más largo dentro del contenido.
+ */
+function extractCardDescription(content) {
+  const pTags = [...content.matchAll(/<p>([\s\S]{0,2000}?)<\/p>/gi)]
+  if (!pTags.length) return ""
+
+  // Tomar el <p> más largo (el principal, no snippets cortos)
+  const sorted = pTags
+    .map((m) => m[1])
+    .filter((t) => t.replace(/<[^>]+>/g, "").trim().length > 20)
+    .sort((a, b) => b.length - a.length)
+
+  const best = sorted[0] || ""
+  return best
+    .replace(/<[^>]+>/g, " ")
+    .replace(/&rsquo;/g, "'")
+    .replace(/&ldquo;/g, '"')
+    .replace(/&rdquo;/g, '"')
+    .replace(/&amp;/g, "&")
+    .replace(/&nbsp;/g, " ")
+    .replace(/\s+/g, " ")
+    .trim()
+}
+
+/**
+ * Extrae las tecnologías de la metadata.
+ */
+function extractTechnologies(meta) {
+  const techs = [...meta.matchAll(/<span class="meta-value tech-([^"]+)"[^>]*>([^<]+)<\/span>/gi)]
+  return techs.map((t) => t[2].trim())
+}
+
+/**
+ * Extrae la dificultad de la metadata.
+ */
+function extractDifficulty(meta) {
+  const diffMatch = meta.match(/Difficulty:<\/span>\s*\n?\s*<span[^>]*>([^<]+)/i)
+  return diffMatch ? diffMatch[1].trim() : null
+}
+
+/**
+ * Extrae browser support de la metadata.
+ */
+function extractBrowserSupport(meta) {
+  const browsers = [
+    ...meta.matchAll(/<span class="meta-value browser-tag">[\s\S]{0,200}?<span>([^<]+)<\/span>/gi),
+  ]
+  return browsers.map((b) => b[1].trim())
+}
+
+/**
+ * Extrae features de la metadata.
+ */
+function extractFeatures(meta) {
+  const features = [...meta.matchAll(/<span class=meta-value>([^<]+)<\/span>/g)]
+  // Filtrar technologies y otros valores no-features
+  const knownTechs = ["HTML", "CSS", "JavaScript", "TypeScript", "JS", "TS"]
+  const difficultyLevels = ["Beginner", "Intermediate", "Advanced"]
+  return features
+    .map((f) => f[1].trim())
+    .filter((f) => !knownTechs.includes(f) && !difficultyLevels.includes(f) && f.length > 2)
+    .slice(0, 10)
+}
+
+/**
+ * Extrae los requires de una card (tecnologías requeridas).
+ */
+function extractRequires(content) {
+  const reqMatch = content.match(/Requires?:\s*([^<]+)/i)
+  if (!reqMatch) return []
+  return reqMatch[1]
+    .split(",")
+    .map((r) => r.trim())
+    .filter(Boolean)
+}
+
+/**
+ * Extrae el CodePen slug hash del HTML (busca popovers con data-slug-hash).
+ * Busca en todo el HTML cercano al índice de la card.
+ */
+function extractCodePenSlug(html, cardIndex) {
+  // Buscar popover después de la card (dentro de ~5000 chars)
+  const afterCard = html.slice(cardIndex, cardIndex + 5000)
+  const slugMatch = afterCard.match(/data-slug-hash=([^\s>"']+)/)
+  return slugMatch ? slugMatch[1] : null
+}
+
+// ─── Fetch ─────────────────────────────────────────────────────────
+
+/**
+ * Fetch de una página HTML con timeout y User-Agent.
+ *
+ * @param {string} url
+ * @returns {Promise<string|null>}
+ */
+async function fetchPage(url) {
+  try {
+    const response = await fetch(url, {
+      headers: {
+        "User-Agent": USER_AGENT,
+        Accept: "text/html,application/xhtml+xml",
+      },
+      signal: AbortSignal.timeout(REQUEST_TIMEOUT),
+    })
+
+    if (!response.ok) {
+      console.log(chalk.yellow(`  ⚠️  HTTP ${response.status}: ${url}`))
+      return null
+    }
+
+    return await response.text()
+  } catch (error) {
+    console.log(chalk.yellow(`  ⚠️  Fetch error: ${url} - ${error.message}`))
+    return null
+  }
+}
+
+// ─── Guardado ──────────────────────────────────────────────────────
+
+/**
+ * Guarda una card extraída en la estructura knowledge/.
+ *
+ * @param {Object} card - Card data
+ * @param {string} domain - Dominio OPL
+ */
+async function saveCard(card, domain) {
+  try {
+    const { existsSync, mkdirSync, writeFileSync } = await import("fs")
+    const { join } = await import("path")
+    const { getProjectKnowledgeDir } = await import("../../commands/knowledge-helpers.js")
+
+    const knowledgeDir = getProjectKnowledgeDir()
+    if (!knowledgeDir) return
+
+    const slug = slugify(card.title)
+    const docDir = join(knowledgeDir, domain, "components", slug)
+    mkdirSync(docDir, { recursive: true })
+
+    // Guardar metadata
+    const meta = {
+      title: card.title,
+      description: card.description,
+      technologies: card.technologies,
+      difficulty: card.difficulty,
+      browserSupport: card.browserSupport,
+      features: card.features,
+      requires: card.requires,
+      codepen: card.demoUrl,
+      codepenSlug: card.codepenSlug,
+      source: card.sourceUrl,
+      scrapedAt: new Date().toISOString(),
+      type: "frontend-component",
+      domain,
+    }
+    writeFileSync(join(docDir, "meta.json"), JSON.stringify(meta, null, 2), "utf-8")
+
+    // Guardar descripción como markdown
+    const markdown = `# ${card.title}
+
+${card.description}
+
+## Metadata
+
+- **Tecnologías:** ${card.technologies.join(", ") || "N/A"}
+- **Dificultad:** ${card.difficulty || "N/A"}
+- **Features:** ${card.features.join(", ") || "N/A"}
+- **Browser Support:** ${card.browserSupport.join(", ") || "N/A"}
+
+## Demo
+
+${card.demoUrl || "No disponible"}
+`
+    writeFileSync(join(docDir, "page.md"), markdown, "utf-8")
+  } catch (error) {
+    // No crítico - solo log si verbose
+    if (error.message.includes("knowledge-helpers")) {
+      // knowledge dir not configured
+    }
+  }
+}
+
+/**
+ * Extracción genérica con Readability (fallback para páginas desconocidas).
+ */
+async function extractGenericContent(url, html, tree, opts) {
+  try {
+    const { JSDOM } = await import("jsdom")
+    const dom = new JSDOM(html, { url })
+    const { Readability } = await import("@mozilla/readability")
+    const reader = new Readability(dom.window.document)
+    const article = reader.parse()
+
+    if (article && article.content) {
+      const TurndownService = (await import("turndown")).default
+      const turndown = new TurndownService({
+        headingStyle: "atx",
+        codeBlockStyle: "fenced",
+      })
+      const markdown = turndown.turndown(article.content)
+
+      tree.children.push({
+        url,
+        type: PAGE_TYPES.ARTICLE,
+        title: article.title,
+        children: [],
+        items: [{ title: article.title, content: markdown, sourceUrl: url }],
+      })
+      tree.stats.itemsExtracted++
+    }
+  } catch {
+    // Readability no disponible, ignorar
+  }
+}
+
+// ─── Utilities ──────────────────────────────────────────────────────
+
+/**
+ * Slugify un texto para usar como ID/directorio.
+ */
+function slugify(text) {
+  return text
+    .toLowerCase()
+    .replace(/[^a-z0-9áéíóúüñ\s-]/g, "")
+    .replace(/\s+/g, "-")
+    .replace(/-+/g, "-")
+    .replace(/^-|-$/g, "")
+    .slice(0, 80)
+}