@geekbeer/minion 3.52.0 → 3.53.1

package/.env.example

@@ -17,3 +17,10 @@ MINION_ID=
 
 # Agent port (optional, default: 8080)
 AGENT_PORT=8080
+
+# Anthropic API key (optional, experimental, fallback only) —
+# POST /api/web/extract prefers the primary LLM plugin (see PUT /api/llm/config)
+# and only uses ANTHROPIC_API_KEY if no primary plugin is configured. Set via:
+#   curl -X PUT http://localhost:8080/api/secrets/ANTHROPIC_API_KEY \
+#     -H "Authorization: Bearer $API_TOKEN" -d '{"value": "sk-ant-..."}'
+ANTHROPIC_API_KEY=

package/core/db/migrations/20260508000000_page_recipes.js

@@ -0,0 +1,33 @@
+/**
+ * page_recipes — Web page extraction recipe cache (experimental, v3.53.0).
+ *
+ * Stores selectors learned from a first-time visit so subsequent visits to
+ * structurally similar pages skip the LLM round trip. Keyed by URL template
+ * (after normalization) + DOM fingerprint to tolerate A/B variants.
+ *
+ * Marked experimental: schema may change before the API stabilizes.
+ */
+
+module.exports = {
+  version: 20260508000000,
+  name: 'page_recipes',
+
+  up(db, { tableExists }) {
+    if (tableExists(db, 'page_recipes')) return
+
+    db.exec(`
+      CREATE TABLE page_recipes (
+        url_template TEXT NOT NULL,
+        dom_fingerprint TEXT NOT NULL,
+        selectors_json TEXT NOT NULL,
+        page_type TEXT,
+        hit_count INTEGER NOT NULL DEFAULT 0,
+        fail_count INTEGER NOT NULL DEFAULT 0,
+        last_verified_at TEXT,
+        created_at TEXT NOT NULL DEFAULT (datetime('now')),
+        PRIMARY KEY (url_template, dom_fingerprint)
+      );
+      CREATE INDEX idx_page_recipes_template ON page_recipes(url_template);
+    `)
+  },
+}

package/core/lib/web-extract/extractor.js

@@ -0,0 +1,142 @@
+/**
+ * Web extraction orchestrator (experimental — v3.53.0).
+ *
+ * Cold path: Playwright fetch -> Readability/Turndown clean -> Anthropic
+ *            Haiku selects fields -> store recipe -> verify by replaying
+ *            selectors against the same page.
+ *
+ * Hot path:  Playwright fetch -> fingerprint -> recipe lookup -> selector
+ *            replay. No LLM call.
+ *
+ * Self-heal: hot replays that come back empty bump fail_count; the recipe
+ *            is dropped after MAX_FAIL_COUNT and the next request retries
+ *            cold. A single in-request fall-through from hot -> cold is
+ *            allowed so callers don't see transient breakage.
+ */
+
+const { normalizeUrl } = require('./url-normalize')
+const { computeFingerprint } = require('./fingerprint')
+const { renderPage, extractWithSelectors } = require('./playwright-runner')
+const { cleanHtml } = require('./html-cleaner')
+const { generateRecipe } = require('./recipe-generator')
+const pageRecipeStore = require('../../stores/page-recipe-store')
+
+function isEmptyResult(data) {
+  if (!data || typeof data !== 'object') return true
+  const values = Object.values(data)
+  if (values.length === 0) return true
+  return values.every(v => {
+    if (v == null) return true
+    if (typeof v === 'string') return v.trim() === ''
+    if (Array.isArray(v)) return v.length === 0
+    return false
+  })
+}
+
+async function extract({ url, hint }) {
+  const { template, canonicalUrl } = normalizeUrl(url)
+
+  // Always render once up-front so we can compute the fingerprint regardless
+  // of cache state. Cold path reuses the HTML; hot path discards it.
+  const rendered = await renderPage(canonicalUrl)
+  const fingerprint = computeFingerprint(rendered.html)
+
+  const cached = pageRecipeStore.find({
+    urlTemplate: template,
+    domFingerprint: fingerprint,
+  })
+
+  if (cached) {
+    const data = await extractWithSelectors(canonicalUrl, cached.selectors)
+    if (!isEmptyResult(data)) {
+      pageRecipeStore.incrementHit({ urlTemplate: template, domFingerprint: fingerprint })
+      pageRecipeStore.setLastVerified({ urlTemplate: template, domFingerprint: fingerprint })
+      return shape({
+        url: canonicalUrl,
+        finalUrl: rendered.finalUrl,
+        statusCode: rendered.statusCode,
+        recipeMode: 'hot',
+        urlTemplate: template,
+        fingerprint,
+        pageType: cached.page_type,
+        selectors: cached.selectors,
+        data,
+        cleaned: null,
+      })
+    }
+    // Hot replay returned nothing — penalize and fall through to cold.
+    pageRecipeStore.incrementFail({ urlTemplate: template, domFingerprint: fingerprint })
+  }
+
+  // Cold path
+  const cleaned = cleanHtml(rendered.html, canonicalUrl)
+  const recipe = await generateRecipe({
+    url: canonicalUrl,
+    cleanedMarkdown: cleaned.contentMarkdown,
+    hint,
+  })
+
+  // Verify the recipe against this exact page before persisting.
+  const verifyData = await extractWithSelectors(canonicalUrl, recipe.selectors)
+  const verified = !isEmptyResult(verifyData)
+
+  if (verified) {
+    pageRecipeStore.upsert({
+      urlTemplate: template,
+      domFingerprint: fingerprint,
+      selectors: recipe.selectors,
+      pageType: recipe.pageType,
+    })
+    pageRecipeStore.incrementHit({ urlTemplate: template, domFingerprint: fingerprint })
+  }
+
+  return shape({
+    url: canonicalUrl,
+    finalUrl: rendered.finalUrl,
+    statusCode: rendered.statusCode,
+    recipeMode: 'cold',
+    urlTemplate: template,
+    fingerprint,
+    pageType: recipe.pageType,
+    selectors: recipe.selectors,
+    data: verified ? verifyData : recipe.extracted,
+    cleaned,
+    recipePersisted: verified,
+  })
+}
+
+function shape({ url, finalUrl, statusCode, recipeMode, urlTemplate, fingerprint, pageType, selectors, data, cleaned, recipePersisted }) {
+  const out = {
+    experimental: true,
+    url,
+    finalUrl,
+    statusCode,
+    recipeMode,
+    recipeId: `${urlTemplate}#${fingerprint}`,
+    pageType: pageType || null,
+    title: pickField(data, ['title', 'headline', 'name']) || cleaned?.title || null,
+    content: pickField(data, ['body', 'content', 'article', 'description']) || cleaned?.contentMarkdown || null,
+    structured: data || {},
+    selectors: selectors || {},
+  }
+  if (recipeMode === 'cold' && recipePersisted === false) {
+    out.warning = 'Recipe verification failed (selectors returned empty). Result reflects LLM extraction; recipe was not persisted.'
+  }
+  return out
+}
+
+function pickField(obj, candidates) {
+  if (!obj || typeof obj !== 'object') return null
+  for (const key of candidates) {
+    const v = obj[key]
+    if (v == null) continue
+    if (typeof v === 'string' && v.trim() !== '') return v
+    if (Array.isArray(v) && v.length > 0) return v.join('\n\n')
+  }
+  return null
+}
+
+module.exports = {
+  extract,
+  isEmptyResult,
+}

package/core/lib/web-extract/fingerprint.js

@@ -0,0 +1,63 @@
+/**
+ * Lightweight DOM structure fingerprint.
+ *
+ * Two pages with the same template URL but materially different layouts
+ * (A/B test, logged-in vs logged-out, mobile vs desktop served) need to
+ * use different recipes. We hash a minimal structural signature instead
+ * of the full HTML so the fingerprint stays stable against trivial copy
+ * changes but flips when block-level structure shifts.
+ *
+ * Signature inputs:
+ *   - Order of structural landmark tags (header/nav/main/article/...)
+ *   - Top 5 most frequent class names on <div> elements
+ */
+
+const crypto = require('crypto')
+
+const LANDMARK_TAGS = ['header', 'nav', 'main', 'article', 'section', 'aside', 'footer', 'form']
+
+function computeFingerprint(html) {
+  if (typeof html !== 'string' || html.length === 0) {
+    return 'empty'
+  }
+
+  let document
+  try {
+    const { parseHTML } = require('linkedom')
+    document = parseHTML(html).document
+  } catch (err) {
+    // If linkedom fails (extremely malformed HTML), fall back to a length bucket
+    return 'fallback-' + crypto.createHash('sha1').update(html.slice(0, 4096)).digest('hex').slice(0, 12)
+  }
+
+  // Landmark tag sequence (first occurrence only, in document order)
+  const seen = []
+  const seenSet = new Set()
+  const allEls = document.querySelectorAll(LANDMARK_TAGS.join(','))
+  for (const el of allEls) {
+    const tag = el.tagName.toLowerCase()
+    if (!seenSet.has(tag)) {
+      seenSet.add(tag)
+      seen.push(tag)
+    }
+  }
+
+  // Top 5 div classes by frequency
+  const classCounts = new Map()
+  const divs = document.querySelectorAll('div[class]')
+  for (const div of divs) {
+    const classes = (div.getAttribute('class') || '').split(/\s+/).filter(Boolean)
+    for (const c of classes) {
+      classCounts.set(c, (classCounts.get(c) || 0) + 1)
+    }
+  }
+  const topClasses = [...classCounts.entries()]
+    .sort((a, b) => b[1] - a[1] || (a[0] < b[0] ? -1 : 1))
+    .slice(0, 5)
+    .map(([c]) => c)
+
+  const signature = `tags:${seen.join(',')}|cls:${topClasses.join(',')}`
+  return crypto.createHash('sha1').update(signature).digest('hex').slice(0, 12)
+}
+
+module.exports = { computeFingerprint }

package/core/lib/web-extract/html-cleaner.js

@@ -0,0 +1,72 @@
+/**
+ * HTML → cleaned content (Readability) → Markdown (Turndown).
+ *
+ * The cleaned Markdown is the *only* page representation handed to the
+ * recipe-generation LLM. Keeping the input small and structured is what
+ * makes this experiment cheap enough to be worth running on every
+ * cold-cache miss.
+ */
+
+const MAX_MARKDOWN_LENGTH = 50_000
+
+function cleanHtml(html, url) {
+  let parsedDocument
+  try {
+    const { parseHTML } = require('linkedom')
+    parsedDocument = parseHTML(html).document
+  } catch (err) {
+    return {
+      title: null,
+      contentHtml: '',
+      contentMarkdown: '',
+      byline: null,
+      excerpt: null,
+      length: 0,
+    }
+  }
+
+  let article = null
+  try {
+    const { Readability } = require('@mozilla/readability')
+    article = new Readability(parsedDocument).parse()
+  } catch {
+    article = null
+  }
+
+  const contentHtml =
+    (article && article.content) ||
+    parsedDocument.body?.innerHTML ||
+    ''
+
+  let contentMarkdown = ''
+  try {
+    const TurndownService = require('turndown')
+    const td = new TurndownService({
+      headingStyle: 'atx',
+      codeBlockStyle: 'fenced',
+      bulletListMarker: '-',
+    })
+    td.remove(['script', 'style', 'noscript', 'iframe'])
+    contentMarkdown = td.turndown(contentHtml)
+  } catch {
+    contentMarkdown = ''
+  }
+
+  if (contentMarkdown.length > MAX_MARKDOWN_LENGTH) {
+    contentMarkdown = contentMarkdown.slice(0, MAX_MARKDOWN_LENGTH) + '\n\n[... truncated ...]'
+  }
+
+  return {
+    title: article?.title || parsedDocument.title || null,
+    contentHtml,
+    contentMarkdown,
+    byline: article?.byline || null,
+    excerpt: article?.excerpt || null,
+    length: article?.length || contentMarkdown.length,
+  }
+}
+
+module.exports = {
+  cleanHtml,
+  MAX_MARKDOWN_LENGTH,
+}

package/core/lib/web-extract/index.js

@@ -0,0 +1,21 @@
+/**
+ * Web extraction (experimental, v3.53.0).
+ *
+ * Public surface for core/routes/web.js. Internal modules:
+ *   - url-normalize.js     URL → template + canonical URL
+ *   - fingerprint.js       DOM structural hash
+ *   - playwright-runner.js headless fetch + selector replay
+ *   - html-cleaner.js      Readability + Turndown
+ *   - recipe-generator.js  Anthropic Haiku cold path
+ *   - extractor.js         orchestrator (hot/cold + self-heal)
+ */
+
+const { extract } = require('./extractor')
+const { normalizeUrl } = require('./url-normalize')
+const { computeFingerprint } = require('./fingerprint')
+
+module.exports = {
+  extract,
+  normalizeUrl,
+  computeFingerprint,
+}

package/core/lib/web-extract/playwright-runner.js

@@ -0,0 +1,129 @@
+/**
+ * Headless browser fetch + selector-based extraction.
+ *
+ * `playwright` is an optionalDependency: if it's missing (e.g. ARM host
+ * where the chromium binary failed to install), require() throws and the
+ * route layer surfaces a 503 "browser unavailable" error instead of
+ * crashing the agent.
+ *
+ * Each call spins up a fresh chromium instance. Pooling can come later
+ * once the API stabilizes — for the experimental MVP, simple is better.
+ */
+
+const DEFAULT_NAV_TIMEOUT_MS = 20_000
+const DEFAULT_EVAL_TIMEOUT_MS = 5_000
+const DEFAULT_USER_AGENT =
+  'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) ' +
+  'Chrome/124.0.0.0 Safari/537.36 MinionWebExtract/0.1'
+
+function loadChromium() {
+  let playwright
+  try {
+    playwright = require('playwright')
+  } catch (err) {
+    const e = new Error(
+      'playwright is not installed. Run `npx playwright install chromium` ' +
+      'on the minion host to enable POST /api/web/extract.'
+    )
+    e.code = 'PLAYWRIGHT_UNAVAILABLE'
+    throw e
+  }
+  return playwright.chromium
+}
+
+async function withPage(fn, opts = {}) {
+  const chromium = loadChromium()
+  const browser = await chromium.launch({
+    headless: true,
+    args: ['--no-sandbox', '--disable-dev-shm-usage'],
+  })
+  try {
+    const context = await browser.newContext({
+      userAgent: opts.userAgent || DEFAULT_USER_AGENT,
+      viewport: { width: 1280, height: 800 },
+    })
+    const page = await context.newPage()
+    return await fn(page)
+  } finally {
+    await browser.close().catch(() => {})
+  }
+}
+
+async function renderPage(url, opts = {}) {
+  return withPage(async page => {
+    const response = await page.goto(url, {
+      waitUntil: 'domcontentloaded',
+      timeout: opts.timeoutMs ?? DEFAULT_NAV_TIMEOUT_MS,
+    })
+    const html = await page.content()
+    return {
+      html,
+      finalUrl: page.url(),
+      statusCode: response?.status() ?? null,
+    }
+  }, opts)
+}
+
+/**
+ * Run a recipe against a freshly loaded page.
+ *
+ * `selectors` shape (each value is an object):
+ *   {
+ *     title: { selector: 'h1', attr: 'text' },
+ *     body:  { selector: 'article', attr: 'text' },
+ *     items: { selector: '.list-item .title', attr: 'text', multiple: true },
+ *     link:  { selector: 'a.permalink', attr: 'href' }
+ *   }
+ *
+ * `attr` defaults to 'text' (innerText). Special value 'html' returns
+ * innerHTML. Any other string is read as an HTML attribute.
+ */
+async function extractWithSelectors(url, selectors, opts = {}) {
+  return withPage(async page => {
+    await page.goto(url, {
+      waitUntil: 'domcontentloaded',
+      timeout: opts.timeoutMs ?? DEFAULT_NAV_TIMEOUT_MS,
+    })
+    return await page.evaluate(
+      ({ selectorMap, evalTimeoutMs }) => {
+        const start = Date.now()
+        const result = {}
+        for (const [field, spec] of Object.entries(selectorMap)) {
+          if (Date.now() - start > evalTimeoutMs) {
+            result[field] = null
+            continue
+          }
+          if (!spec || !spec.selector) {
+            result[field] = null
+            continue
+          }
+          const attr = spec.attr || 'text'
+          const readOne = (el) => {
+            if (!el) return null
+            if (attr === 'text') return (el.innerText || el.textContent || '').trim()
+            if (attr === 'html') return el.innerHTML
+            return el.getAttribute(attr)
+          }
+          try {
+            if (spec.multiple) {
+              const nodes = document.querySelectorAll(spec.selector)
+              result[field] = Array.from(nodes).map(readOne).filter(v => v != null && v !== '')
+            } else {
+              const node = document.querySelector(spec.selector)
+              result[field] = readOne(node)
+            }
+          } catch {
+            result[field] = null
+          }
+        }
+        return result
+      },
+      { selectorMap: selectors, evalTimeoutMs: opts.evalTimeoutMs ?? DEFAULT_EVAL_TIMEOUT_MS },
+    )
+  }, opts)
+}
+
+module.exports = {
+  renderPage,
+  extractWithSelectors,
+}