@geekbeer/minion 3.52.0 → 3.55.1

package/.env.example

@@ -17,3 +17,10 @@ MINION_ID=
 
 # Agent port (optional, default: 8080)
 AGENT_PORT=8080
+
+# Anthropic API key (optional, experimental, fallback only) —
+# POST /api/web/extract prefers the primary LLM plugin (see PUT /api/llm/config)
+# and only uses ANTHROPIC_API_KEY if no primary plugin is configured. Set via:
+#   curl -X PUT http://localhost:8080/api/secrets/ANTHROPIC_API_KEY \
+#     -H "Authorization: Bearer $API_TOKEN" -d '{"value": "sk-ant-..."}'
+ANTHROPIC_API_KEY=

package/core/db/migrations/20260508000000_page_recipes.js

@@ -0,0 +1,33 @@
+/**
+ * page_recipes — Web page extraction recipe cache (experimental, v3.53.0).
+ *
+ * Stores selectors learned from a first-time visit so subsequent visits to
+ * structurally similar pages skip the LLM round trip. Keyed by URL template
+ * (after normalization) + DOM fingerprint to tolerate A/B variants.
+ *
+ * Marked experimental: schema may change before the API stabilizes.
+ */
+
+module.exports = {
+  version: 20260508000000,
+  name: 'page_recipes',
+
+  up(db, { tableExists }) {
+    if (tableExists(db, 'page_recipes')) return
+
+    db.exec(`
+      CREATE TABLE page_recipes (
+        url_template TEXT NOT NULL,
+        dom_fingerprint TEXT NOT NULL,
+        selectors_json TEXT NOT NULL,
+        page_type TEXT,
+        hit_count INTEGER NOT NULL DEFAULT 0,
+        fail_count INTEGER NOT NULL DEFAULT 0,
+        last_verified_at TEXT,
+        created_at TEXT NOT NULL DEFAULT (datetime('now')),
+        PRIMARY KEY (url_template, dom_fingerprint)
+      );
+      CREATE INDEX idx_page_recipes_template ON page_recipes(url_template);
+    `)
+  },
+}

package/core/lib/dag-step-poller.js

@@ -119,12 +119,15 @@ async function pollOnce() {
 
 /**
  * Execute a single pending DAG node.
- * Routes to skill or transform execution based on node_type.
+ * Routes to skill / transform / script execution based on node_type.
  */
 async function executeNode(node) {
   if (node.node_type === 'transform') {
     return executeTransformNode(node)
   }
+  if (node.node_type === 'script') {
+    return executeScriptNode(node)
+  }
   return executeSkillNode(node)
 }
 
@@ -467,6 +470,247 @@ function appendContractTable(lines, fields) {
   }
 }
 
+/**
+ * Execute a script node:
+ * 1. Claim the node
+ * 2. Spawn python3 / node as a child process
+ * 3. Pipe input_data as JSON on stdin
+ * 4. Parse stdout as JSON → output_data, report completion
+ *
+ * Failure modes (all → status 'failed'):
+ * - Unsupported runtime
+ * - Non-zero exit code (stderr → error_message)
+ * - stdout is not parseable as JSON
+ * - Wall-clock timeout exceeded (process is SIGKILL'd)
+ *
+ * No tmux session is involved — this is a synchronous in-process child.
+ * The Output Contract on the outgoing edge is enforced by HQ on
+ * /node-complete, identical to skill / transform nodes.
+ */
+async function executeScriptNode(node) {
+  const {
+    node_execution_id,
+    node_id,
+    dag_workflow_name,
+    scope_path,
+    assigned_role,
+    input_data,
+    script_runtime,
+    script_source,
+    script_timeout_seconds,
+  } = node
+
+  console.log(
+    `[DagPoller] Executing script node "${node_id}" of DAG "${dag_workflow_name}" ` +
+    `(runtime: ${script_runtime}, scope: "${scope_path || 'root'}", role: ${assigned_role})`
+  )
+
+  try {
+    try {
+      await dagRequest('/claim-node', {
+        method: 'POST',
+        body: JSON.stringify({ node_execution_id }),
+      })
+    } catch (claimErr) {
+      if (claimErr.statusCode === 409) {
+        console.log(`[DagPoller] Script node ${node_id} already claimed, skipping`)
+        return
+      }
+      throw claimErr
+    }
+
+    if (!script_source || !script_source.trim()) {
+      await reportNodeComplete(node_execution_id, 'failed', null, 'Script node has empty script_source')
+      return
+    }
+
+    let interpreter
+    if (script_runtime === 'python') {
+      interpreter = 'python3'
+    } else if (script_runtime === 'node') {
+      interpreter = 'node'
+    } else {
+      await reportNodeComplete(
+        node_execution_id,
+        'failed',
+        null,
+        `Unsupported script_runtime: ${script_runtime} (allowed: python, node)`,
+      )
+      return
+    }
+
+    const timeoutMs = clampTimeout(script_timeout_seconds) * 1000
+
+    const result = await runScriptProcess(interpreter, script_source, input_data || {}, timeoutMs)
+
+    if (result.timedOut) {
+      await reportNodeComplete(
+        node_execution_id,
+        'failed',
+        null,
+        `Script timed out after ${timeoutMs / 1000}s. stderr: ${truncate(result.stderr, 2000)}`,
+      )
+      return
+    }
+
+    if (result.exitCode !== 0) {
+      await reportNodeComplete(
+        node_execution_id,
+        'failed',
+        null,
+        `Script exited with code ${result.exitCode}. stderr: ${truncate(result.stderr, 2000)}`,
+      )
+      return
+    }
+
+    let outputData
+    try {
+      outputData = JSON.parse(result.stdout)
+    } catch (parseErr) {
+      await reportNodeComplete(
+        node_execution_id,
+        'failed',
+        null,
+        `Script stdout is not valid JSON: ${parseErr.message}. stdout (first 500 chars): ${truncate(result.stdout, 500)}`,
+      )
+      return
+    }
+
+    if (typeof outputData !== 'object' || outputData === null || Array.isArray(outputData)) {
+      await reportNodeComplete(
+        node_execution_id,
+        'failed',
+        null,
+        `Script output_data must be a JSON object (got ${Array.isArray(outputData) ? 'array' : typeof outputData})`,
+      )
+      return
+    }
+
+    await reportNodeComplete(
+      node_execution_id,
+      'completed',
+      outputData,
+      buildScriptSummary(outputData, result.stderr),
+    )
+    console.log(`[DagPoller] Script node "${node_id}" completed`)
+
+  } catch (err) {
+    console.error(`[DagPoller] Failed to execute script node ${node_id}: ${err.message}`)
+    try {
+      await reportNodeComplete(node_execution_id, 'failed', null, err.message)
+    } catch {
+      // best-effort
+    }
+  }
+}
+
+/**
+ * Build the output_summary text shown in the HQ execution detail UI for a
+ * successful script node. The UI renders output_summary as a collapsible
+ * code block under "Output"; output_data itself is only visible from the
+ * Pin-fixture dialog. Always emit at least the JSON output so users can
+ * inspect what the script returned without round-tripping through fixtures.
+ */
+function buildScriptSummary(outputData, stderr) {
+  const lines = []
+  let serialized
+  try {
+    serialized = JSON.stringify(outputData, null, 2)
+  } catch {
+    serialized = '(failed to serialize output_data)'
+  }
+  lines.push('output_data:')
+  lines.push('```json')
+  lines.push(truncate(serialized, 8000))
+  lines.push('```')
+  if (stderr && stderr.trim()) {
+    lines.push('')
+    lines.push('stderr:')
+    lines.push(truncate(stderr, 2000))
+  }
+  return lines.join('\n')
+}
+
+function clampTimeout(seconds) {
+  const n = typeof seconds === 'number' && Number.isFinite(seconds) ? seconds : 60
+  if (n < 1) return 1
+  if (n > 600) return 600
+  return Math.floor(n)
+}
+
+function truncate(str, max) {
+  if (!str) return ''
+  return str.length > max ? str.slice(0, max) + '…(truncated)' : str
+}
+
+/**
+ * Spawn the interpreter, write the script to stdin via `-` / `-c` is not
+ * portable (python requires `-`, node requires `-e` with the source as an
+ * argument and no stdin). Instead we feed the script via stdin for both:
+ * - python3 reads source from stdin when invoked with `-`
+ * - node reads source from stdin when invoked with no args, but stdin is
+ *   then consumed by the script — so for node we must use `-e` with the
+ *   source as a flag value. To keep input_data on stdin for both, we
+ *   write the script to a temp file and pass the path as argv[1].
+ */
+function runScriptProcess(interpreter, source, inputData, timeoutMs) {
+  const { spawn } = require('child_process')
+  const fs = require('fs')
+  const os = require('os')
+  const path = require('path')
+  const crypto = require('crypto')
+
+  const ext = interpreter === 'python3' ? 'py' : 'js'
+  const tmpFile = path.join(
+    os.tmpdir(),
+    `dag-script-${crypto.randomBytes(8).toString('hex')}.${ext}`,
+  )
+  fs.writeFileSync(tmpFile, source, 'utf-8')
+
+  return new Promise(resolve => {
+    const child = spawn(interpreter, [tmpFile], {
+      stdio: ['pipe', 'pipe', 'pipe'],
+    })
+
+    let stdout = ''
+    let stderr = ''
+    let timedOut = false
+    let killed = false
+
+    const timer = setTimeout(() => {
+      timedOut = true
+      killed = true
+      try { child.kill('SIGKILL') } catch { /* noop */ }
+    }, timeoutMs)
+
+    child.stdout.on('data', chunk => { stdout += chunk.toString('utf-8') })
+    child.stderr.on('data', chunk => { stderr += chunk.toString('utf-8') })
+
+    child.on('error', err => {
+      clearTimeout(timer)
+      cleanup(tmpFile)
+      resolve({ exitCode: -1, stdout, stderr: stderr + `\nspawn error: ${err.message}`, timedOut })
+    })
+
+    child.on('close', code => {
+      clearTimeout(timer)
+      cleanup(tmpFile)
+      resolve({ exitCode: killed ? -1 : (code ?? -1), stdout, stderr, timedOut })
+    })
+
+    try {
+      child.stdin.write(JSON.stringify(inputData))
+      child.stdin.end()
+    } catch (writeErr) {
+      stderr += `\nstdin write error: ${writeErr.message}`
+    }
+  })
+}
+
+function cleanup(tmpFile) {
+  try { require('fs').unlinkSync(tmpFile) } catch { /* noop */ }
+}
+
 /**
  * Resolve skill_version_id to skill name via HQ API.
  */

package/core/lib/web-extract/extractor.js

@@ -0,0 +1,142 @@
+/**
+ * Web extraction orchestrator (experimental — v3.53.0).
+ *
+ * Cold path: Playwright fetch -> Readability/Turndown clean -> Anthropic
+ *            Haiku selects fields -> store recipe -> verify by replaying
+ *            selectors against the same page.
+ *
+ * Hot path:  Playwright fetch -> fingerprint -> recipe lookup -> selector
+ *            replay. No LLM call.
+ *
+ * Self-heal: hot replays that come back empty bump fail_count; the recipe
+ *            is dropped after MAX_FAIL_COUNT and the next request retries
+ *            cold. A single in-request fall-through from hot -> cold is
+ *            allowed so callers don't see transient breakage.
+ */
+
+const { normalizeUrl } = require('./url-normalize')
+const { computeFingerprint } = require('./fingerprint')
+const { renderPage, extractWithSelectors } = require('./playwright-runner')
+const { cleanHtml } = require('./html-cleaner')
+const { generateRecipe } = require('./recipe-generator')
+const pageRecipeStore = require('../../stores/page-recipe-store')
+
+function isEmptyResult(data) {
+  if (!data || typeof data !== 'object') return true
+  const values = Object.values(data)
+  if (values.length === 0) return true
+  return values.every(v => {
+    if (v == null) return true
+    if (typeof v === 'string') return v.trim() === ''
+    if (Array.isArray(v)) return v.length === 0
+    return false
+  })
+}
+
+async function extract({ url, hint }) {
+  const { template, canonicalUrl } = normalizeUrl(url)
+
+  // Always render once up-front so we can compute the fingerprint regardless
+  // of cache state. Cold path reuses the HTML; hot path discards it.
+  const rendered = await renderPage(canonicalUrl)
+  const fingerprint = computeFingerprint(rendered.html)
+
+  const cached = pageRecipeStore.find({
+    urlTemplate: template,
+    domFingerprint: fingerprint,
+  })
+
+  if (cached) {
+    const data = await extractWithSelectors(canonicalUrl, cached.selectors)
+    if (!isEmptyResult(data)) {
+      pageRecipeStore.incrementHit({ urlTemplate: template, domFingerprint: fingerprint })
+      pageRecipeStore.setLastVerified({ urlTemplate: template, domFingerprint: fingerprint })
+      return shape({
+        url: canonicalUrl,
+        finalUrl: rendered.finalUrl,
+        statusCode: rendered.statusCode,
+        recipeMode: 'hot',
+        urlTemplate: template,
+        fingerprint,
+        pageType: cached.page_type,
+        selectors: cached.selectors,
+        data,
+        cleaned: null,
+      })
+    }
+    // Hot replay returned nothing — penalize and fall through to cold.
+    pageRecipeStore.incrementFail({ urlTemplate: template, domFingerprint: fingerprint })
+  }
+
+  // Cold path
+  const cleaned = cleanHtml(rendered.html, canonicalUrl)
+  const recipe = await generateRecipe({
+    url: canonicalUrl,
+    cleanedMarkdown: cleaned.contentMarkdown,
+    hint,
+  })
+
+  // Verify the recipe against this exact page before persisting.
+  const verifyData = await extractWithSelectors(canonicalUrl, recipe.selectors)
+  const verified = !isEmptyResult(verifyData)
+
+  if (verified) {
+    pageRecipeStore.upsert({
+      urlTemplate: template,
+      domFingerprint: fingerprint,
+      selectors: recipe.selectors,
+      pageType: recipe.pageType,
+    })
+    pageRecipeStore.incrementHit({ urlTemplate: template, domFingerprint: fingerprint })
+  }
+
+  return shape({
+    url: canonicalUrl,
+    finalUrl: rendered.finalUrl,
+    statusCode: rendered.statusCode,
+    recipeMode: 'cold',
+    urlTemplate: template,
+    fingerprint,
+    pageType: recipe.pageType,
+    selectors: recipe.selectors,
+    data: verified ? verifyData : recipe.extracted,
+    cleaned,
+    recipePersisted: verified,
+  })
+}
+
+function shape({ url, finalUrl, statusCode, recipeMode, urlTemplate, fingerprint, pageType, selectors, data, cleaned, recipePersisted }) {
+  const out = {
+    experimental: true,
+    url,
+    finalUrl,
+    statusCode,
+    recipeMode,
+    recipeId: `${urlTemplate}#${fingerprint}`,
+    pageType: pageType || null,
+    title: pickField(data, ['title', 'headline', 'name']) || cleaned?.title || null,
+    content: pickField(data, ['body', 'content', 'article', 'description']) || cleaned?.contentMarkdown || null,
+    structured: data || {},
+    selectors: selectors || {},
+  }
+  if (recipeMode === 'cold' && recipePersisted === false) {
+    out.warning = 'Recipe verification failed (selectors returned empty). Result reflects LLM extraction; recipe was not persisted.'
+  }
+  return out
+}
+
+function pickField(obj, candidates) {
+  if (!obj || typeof obj !== 'object') return null
+  for (const key of candidates) {
+    const v = obj[key]
+    if (v == null) continue
+    if (typeof v === 'string' && v.trim() !== '') return v
+    if (Array.isArray(v) && v.length > 0) return v.join('\n\n')
+  }
+  return null
+}
+
+module.exports = {
+  extract,
+  isEmptyResult,
+}

package/core/lib/web-extract/fingerprint.js

@@ -0,0 +1,63 @@
+/**
+ * Lightweight DOM structure fingerprint.
+ *
+ * Two pages with the same template URL but materially different layouts
+ * (A/B test, logged-in vs logged-out, mobile vs desktop served) need to
+ * use different recipes. We hash a minimal structural signature instead
+ * of the full HTML so the fingerprint stays stable against trivial copy
+ * changes but flips when block-level structure shifts.
+ *
+ * Signature inputs:
+ *   - Order of structural landmark tags (header/nav/main/article/...)
+ *   - Top 5 most frequent class names on <div> elements
+ */
+
+const crypto = require('crypto')
+
+const LANDMARK_TAGS = ['header', 'nav', 'main', 'article', 'section', 'aside', 'footer', 'form']
+
+function computeFingerprint(html) {
+  if (typeof html !== 'string' || html.length === 0) {
+    return 'empty'
+  }
+
+  let document
+  try {
+    const { parseHTML } = require('linkedom')
+    document = parseHTML(html).document
+  } catch (err) {
+    // If linkedom fails (extremely malformed HTML), fall back to a length bucket
+    return 'fallback-' + crypto.createHash('sha1').update(html.slice(0, 4096)).digest('hex').slice(0, 12)
+  }
+
+  // Landmark tag sequence (first occurrence only, in document order)
+  const seen = []
+  const seenSet = new Set()
+  const allEls = document.querySelectorAll(LANDMARK_TAGS.join(','))
+  for (const el of allEls) {
+    const tag = el.tagName.toLowerCase()
+    if (!seenSet.has(tag)) {
+      seenSet.add(tag)
+      seen.push(tag)
+    }
+  }
+
+  // Top 5 div classes by frequency
+  const classCounts = new Map()
+  const divs = document.querySelectorAll('div[class]')
+  for (const div of divs) {
+    const classes = (div.getAttribute('class') || '').split(/\s+/).filter(Boolean)
+    for (const c of classes) {
+      classCounts.set(c, (classCounts.get(c) || 0) + 1)
+    }
+  }
+  const topClasses = [...classCounts.entries()]
+    .sort((a, b) => b[1] - a[1] || (a[0] < b[0] ? -1 : 1))
+    .slice(0, 5)
+    .map(([c]) => c)
+
+  const signature = `tags:${seen.join(',')}|cls:${topClasses.join(',')}`
+  return crypto.createHash('sha1').update(signature).digest('hex').slice(0, 12)
+}
+
+module.exports = { computeFingerprint }

package/core/lib/web-extract/html-cleaner.js

@@ -0,0 +1,72 @@
+/**
+ * HTML → cleaned content (Readability) → Markdown (Turndown).
+ *
+ * The cleaned Markdown is the *only* page representation handed to the
+ * recipe-generation LLM. Keeping the input small and structured is what
+ * makes this experiment cheap enough to be worth running on every
+ * cold-cache miss.
+ */
+
+const MAX_MARKDOWN_LENGTH = 50_000
+
+function cleanHtml(html, url) {
+  let parsedDocument
+  try {
+    const { parseHTML } = require('linkedom')
+    parsedDocument = parseHTML(html).document
+  } catch (err) {
+    return {
+      title: null,
+      contentHtml: '',
+      contentMarkdown: '',
+      byline: null,
+      excerpt: null,
+      length: 0,
+    }
+  }
+
+  let article = null
+  try {
+    const { Readability } = require('@mozilla/readability')
+    article = new Readability(parsedDocument).parse()
+  } catch {
+    article = null
+  }
+
+  const contentHtml =
+    (article && article.content) ||
+    parsedDocument.body?.innerHTML ||
+    ''
+
+  let contentMarkdown = ''
+  try {
+    const TurndownService = require('turndown')
+    const td = new TurndownService({
+      headingStyle: 'atx',
+      codeBlockStyle: 'fenced',
+      bulletListMarker: '-',
+    })
+    td.remove(['script', 'style', 'noscript', 'iframe'])
+    contentMarkdown = td.turndown(contentHtml)
+  } catch {
+    contentMarkdown = ''
+  }
+
+  if (contentMarkdown.length > MAX_MARKDOWN_LENGTH) {
+    contentMarkdown = contentMarkdown.slice(0, MAX_MARKDOWN_LENGTH) + '\n\n[... truncated ...]'
+  }
+
+  return {
+    title: article?.title || parsedDocument.title || null,
+    contentHtml,
+    contentMarkdown,
+    byline: article?.byline || null,
+    excerpt: article?.excerpt || null,
+    length: article?.length || contentMarkdown.length,
+  }
+}
+
+module.exports = {
+  cleanHtml,
+  MAX_MARKDOWN_LENGTH,
+}

package/core/lib/web-extract/index.js

@@ -0,0 +1,21 @@
+/**
+ * Web extraction (experimental, v3.53.0).
+ *
+ * Public surface for core/routes/web.js. Internal modules:
+ *   - url-normalize.js     URL → template + canonical URL
+ *   - fingerprint.js       DOM structural hash
+ *   - playwright-runner.js headless fetch + selector replay
+ *   - html-cleaner.js      Readability + Turndown
+ *   - recipe-generator.js  Anthropic Haiku cold path
+ *   - extractor.js         orchestrator (hot/cold + self-heal)
+ */
+
+const { extract } = require('./extractor')
+const { normalizeUrl } = require('./url-normalize')
+const { computeFingerprint } = require('./fingerprint')
+
+module.exports = {
+  extract,
+  normalizeUrl,
+  computeFingerprint,
+}