@geekbeer/minion 3.51.2 → 3.53.1

package/core/lib/web-extract/playwright-runner.js

@@ -0,0 +1,129 @@
+/**
+ * Headless browser fetch + selector-based extraction.
+ *
+ * `playwright` is an optionalDependency: if it's missing (e.g. ARM host
+ * where the chromium binary failed to install), require() throws and the
+ * route layer surfaces a 503 "browser unavailable" error instead of
+ * crashing the agent.
+ *
+ * Each call spins up a fresh chromium instance. Pooling can come later
+ * once the API stabilizes — for the experimental MVP, simple is better.
+ */
+
+const DEFAULT_NAV_TIMEOUT_MS = 20_000
+const DEFAULT_EVAL_TIMEOUT_MS = 5_000
+const DEFAULT_USER_AGENT =
+  'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) ' +
+  'Chrome/124.0.0.0 Safari/537.36 MinionWebExtract/0.1'
+
+function loadChromium() {
+  let playwright
+  try {
+    playwright = require('playwright')
+  } catch (err) {
+    const e = new Error(
+      'playwright is not installed. Run `npx playwright install chromium` ' +
+      'on the minion host to enable POST /api/web/extract.'
+    )
+    e.code = 'PLAYWRIGHT_UNAVAILABLE'
+    throw e
+  }
+  return playwright.chromium
+}
+
+async function withPage(fn, opts = {}) {
+  const chromium = loadChromium()
+  const browser = await chromium.launch({
+    headless: true,
+    args: ['--no-sandbox', '--disable-dev-shm-usage'],
+  })
+  try {
+    const context = await browser.newContext({
+      userAgent: opts.userAgent || DEFAULT_USER_AGENT,
+      viewport: { width: 1280, height: 800 },
+    })
+    const page = await context.newPage()
+    return await fn(page)
+  } finally {
+    await browser.close().catch(() => {})
+  }
+}
+
+async function renderPage(url, opts = {}) {
+  return withPage(async page => {
+    const response = await page.goto(url, {
+      waitUntil: 'domcontentloaded',
+      timeout: opts.timeoutMs ?? DEFAULT_NAV_TIMEOUT_MS,
+    })
+    const html = await page.content()
+    return {
+      html,
+      finalUrl: page.url(),
+      statusCode: response?.status() ?? null,
+    }
+  }, opts)
+}
+
+/**
+ * Run a recipe against a freshly loaded page.
+ *
+ * `selectors` shape (each value is an object):
+ *   {
+ *     title: { selector: 'h1', attr: 'text' },
+ *     body:  { selector: 'article', attr: 'text' },
+ *     items: { selector: '.list-item .title', attr: 'text', multiple: true },
+ *     link:  { selector: 'a.permalink', attr: 'href' }
+ *   }
+ *
+ * `attr` defaults to 'text' (innerText). Special value 'html' returns
+ * innerHTML. Any other string is read as an HTML attribute.
+ */
+async function extractWithSelectors(url, selectors, opts = {}) {
+  return withPage(async page => {
+    await page.goto(url, {
+      waitUntil: 'domcontentloaded',
+      timeout: opts.timeoutMs ?? DEFAULT_NAV_TIMEOUT_MS,
+    })
+    return await page.evaluate(
+      ({ selectorMap, evalTimeoutMs }) => {
+        const start = Date.now()
+        const result = {}
+        for (const [field, spec] of Object.entries(selectorMap)) {
+          if (Date.now() - start > evalTimeoutMs) {
+            result[field] = null
+            continue
+          }
+          if (!spec || !spec.selector) {
+            result[field] = null
+            continue
+          }
+          const attr = spec.attr || 'text'
+          const readOne = (el) => {
+            if (!el) return null
+            if (attr === 'text') return (el.innerText || el.textContent || '').trim()
+            if (attr === 'html') return el.innerHTML
+            return el.getAttribute(attr)
+          }
+          try {
+            if (spec.multiple) {
+              const nodes = document.querySelectorAll(spec.selector)
+              result[field] = Array.from(nodes).map(readOne).filter(v => v != null && v !== '')
+            } else {
+              const node = document.querySelector(spec.selector)
+              result[field] = readOne(node)
+            }
+          } catch {
+            result[field] = null
+          }
+        }
+        return result
+      },
+      { selectorMap: selectors, evalTimeoutMs: opts.evalTimeoutMs ?? DEFAULT_EVAL_TIMEOUT_MS },
+    )
+  }, opts)
+}
+
+module.exports = {
+  renderPage,
+  extractWithSelectors,
+}

package/core/lib/web-extract/recipe-generator.js

@@ -0,0 +1,247 @@
+/**
+ * Cold-path recipe generator.
+ *
+ * Resolution order (first available wins):
+ *   1. Primary LLM plugin (PUT /api/llm/config) — uses claude / gemini / codex
+ *      CLI subprocess via the same plugin contract that runQuickLlmCall does.
+ *      Prompt asks for plain JSON; we extract+parse it.
+ *   2. ANTHROPIC_API_KEY env var — direct Anthropic Messages API with
+ *      tool_use schema enforcement (same fetch pattern as
+ *      core/lib/revision-watcher.js).
+ *   3. Otherwise: throw LLM_UNAVAILABLE so the route layer can surface a 503.
+ *
+ * The model only ever sees cleaned Markdown — never the raw HTML — and only
+ * runs on cold-cache misses, so the cost is bounded.
+ *
+ * Returns:
+ *   {
+ *     pageType: 'article' | 'listing' | 'product' | 'profile' | 'form' | 'other',
+ *     selectors: { fieldName: { selector, attr?, multiple? }, ... },
+ *     extracted: { fieldName: <value already pulled from this page> }
+ *   }
+ */
+
+const { getActivePrimary } = require('../../llm-plugins/lib/active')
+
+const ANTHROPIC_MODEL = 'claude-haiku-4-5-20251001'
+const MAX_TOKENS = 2048
+const PLUGIN_TIMEOUT_MS = 60_000
+
+const TOOL_DESCRIPTION =
+  'Classify the page and propose CSS selectors for the most useful fields. ' +
+  'Also produce the extracted values directly so the caller can verify the recipe.'
+
+const ANTHROPIC_TOOLS = [{
+  name: 'page_extraction',
+  description: TOOL_DESCRIPTION,
+  input_schema: {
+    type: 'object',
+    required: ['page_type', 'selectors', 'extracted'],
+    properties: {
+      page_type: {
+        type: 'string',
+        enum: ['article', 'listing', 'product', 'profile', 'form', 'other'],
+        description: 'High-level classification of the page.',
+      },
+      selectors: {
+        type: 'object',
+        description:
+          'Map of fieldName -> { selector, attr?, multiple? }. Use plain CSS selectors. ' +
+          'attr defaults to "text" (innerText); use "html" or an HTML attribute name to override. ' +
+          'Set multiple=true for list fields. Aim for 3-8 fields covering the page\'s primary content.',
+        additionalProperties: {
+          type: 'object',
+          required: ['selector'],
+          properties: {
+            selector: { type: 'string' },
+            attr: { type: 'string' },
+            multiple: { type: 'boolean' },
+          },
+        },
+      },
+      extracted: {
+        type: 'object',
+        description:
+          'Values extracted from this specific page using the selectors above. ' +
+          'Strings or arrays of strings.',
+      },
+    },
+  },
+}]
+
+const SYSTEM_PROMPT = `You design CSS selector recipes for extracting structured content from web pages.
+
+Given a cleaned Markdown rendering of one page, you must:
+1. Classify the page (article / listing / product / profile / form / other).
+2. Propose 3-8 CSS selectors that capture the page's primary information.
+   - Prefer semantic selectors (article, h1, time[datetime], a[rel="author"]) over class names where possible.
+   - Use class-based selectors only when semantic ones are unavailable.
+   - Avoid fragile attribute selectors like data-react-* or auto-generated hashes.
+3. Fill the "extracted" object with the values pulled from this exact page so the caller can verify your recipe works.
+
+The same recipe will be reused for structurally similar pages, so think about what generalizes.`
+
+async function generateRecipe({ url, cleanedMarkdown, hint }) {
+  const primary = getActivePrimary()
+  if (primary) {
+    return await generateViaPlugin(primary, { url, cleanedMarkdown, hint })
+  }
+  if (process.env.ANTHROPIC_API_KEY) {
+    return await generateViaAnthropicDirect({ url, cleanedMarkdown, hint })
+  }
+  const e = new Error(
+    'No LLM available for cold-path recipe generation. Configure a primary LLM via ' +
+    'PUT /api/llm/config (recommended) or set ANTHROPIC_API_KEY as a fallback.'
+  )
+  e.code = 'LLM_UNAVAILABLE'
+  throw e
+}
+
+async function generateViaPlugin(plugin, { url, cleanedMarkdown, hint }) {
+  const prompt = buildTextPrompt({ url, cleanedMarkdown, hint })
+  const input = {
+    prompt,
+    timeoutMs: PLUGIN_TIMEOUT_MS,
+  }
+  // Claude Code CLI accepts model aliases like 'haiku'. Other plugins ignore this.
+  if (plugin.name === 'claude') input.model = 'haiku'
+
+  let output
+  try {
+    output = await plugin.invoke(input)
+  } catch (err) {
+    const e = new Error(`Primary LLM (${plugin.name}) invoke failed: ${err.message}`)
+    e.code = 'PRIMARY_LLM_FAILED'
+    throw e
+  }
+  if (output.error) {
+    const e = new Error(`Primary LLM (${plugin.name}) returned error: ${output.error.message}`)
+    e.code = 'PRIMARY_LLM_FAILED'
+    throw e
+  }
+
+  const json = extractJson(output.text || '')
+  if (!json) {
+    const e = new Error(
+      `Primary LLM (${plugin.name}) did not return parseable JSON. ` +
+      `Raw output (first 500 chars): ${(output.text || '').slice(0, 500)}`
+    )
+    e.code = 'PRIMARY_LLM_BAD_JSON'
+    throw e
+  }
+  return {
+    pageType: json.page_type || 'other',
+    selectors: json.selectors || {},
+    extracted: json.extracted || {},
+    source: `primary:${plugin.name}`,
+  }
+}
+
+async function generateViaAnthropicDirect({ url, cleanedMarkdown, hint }) {
+  const apiKey = process.env.ANTHROPIC_API_KEY
+  const userParts = [
+    `URL: ${url}`,
+    hint ? `Caller hint: ${hint}` : null,
+    '',
+    '--- Cleaned Markdown (Readability output) ---',
+    cleanedMarkdown || '(empty)',
+  ].filter(Boolean).join('\n')
+
+  const resp = await fetch('https://api.anthropic.com/v1/messages', {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+      'x-api-key': apiKey,
+      'anthropic-version': '2023-06-01',
+    },
+    body: JSON.stringify({
+      model: ANTHROPIC_MODEL,
+      max_tokens: MAX_TOKENS,
+      system: SYSTEM_PROMPT,
+      tools: ANTHROPIC_TOOLS,
+      tool_choice: { type: 'tool', name: 'page_extraction' },
+      messages: [{ role: 'user', content: userParts }],
+    }),
+  })
+
+  if (!resp.ok) {
+    const text = await resp.text()
+    throw new Error(`Anthropic API error: ${resp.status} ${text}`)
+  }
+
+  const data = await resp.json()
+  const toolUse = (data.content || []).find(block => block.type === 'tool_use' && block.name === 'page_extraction')
+  if (!toolUse || !toolUse.input) {
+    throw new Error('Anthropic API returned no tool_use block for page_extraction')
+  }
+
+  const { page_type, selectors, extracted } = toolUse.input
+  return {
+    pageType: page_type || 'other',
+    selectors: selectors || {},
+    extracted: extracted || {},
+    source: 'anthropic-direct',
+  }
+}
+
+function buildTextPrompt({ url, cleanedMarkdown, hint }) {
+  return [
+    SYSTEM_PROMPT,
+    '',
+    `URL: ${url}`,
+    hint ? `Caller hint: ${hint}` : null,
+    '',
+    'Output ONLY a JSON object — no prose, no explanations, no code fences.',
+    'The JSON must have exactly this shape:',
+    '',
+    '{',
+    '  "page_type": "article" | "listing" | "product" | "profile" | "form" | "other",',
+    '  "selectors": {',
+    '    "<fieldName>": { "selector": "<css>", "attr"?: "<text|html|attribute-name>", "multiple"?: <boolean> }',
+    '  },',
+    '  "extracted": { "<fieldName>": "<string or array of strings>" }',
+    '}',
+    '',
+    'Notes:',
+    '- attr defaults to "text" (innerText). Use "html" or an HTML attribute name to override.',
+    '- Set multiple=true for list fields (returns array).',
+    '- "extracted" must contain the values you actually read from THIS page using those selectors.',
+    '',
+    '--- Cleaned Markdown ---',
+    cleanedMarkdown || '(empty)',
+  ].filter(s => s !== null).join('\n')
+}
+
+/**
+ * Extract a JSON object from arbitrary LLM text. Handles three common shapes:
+ *   1. Raw JSON
+ *   2. Fenced code block (```json ... ``` or ``` ... ```)
+ *   3. Prose with embedded {...} — uses the outermost braces
+ */
+function extractJson(text) {
+  if (!text || typeof text !== 'string') return null
+  const trimmed = text.trim()
+  if (!trimmed) return null
+
+  try { return JSON.parse(trimmed) } catch {}
+
+  const fence = trimmed.match(/```(?:json)?\s*([\s\S]*?)\s*```/)
+  if (fence && fence[1]) {
+    try { return JSON.parse(fence[1].trim()) } catch {}
+  }
+
+  const first = trimmed.indexOf('{')
+  const last = trimmed.lastIndexOf('}')
+  if (first >= 0 && last > first) {
+    try { return JSON.parse(trimmed.slice(first, last + 1)) } catch {}
+  }
+
+  return null
+}
+
+module.exports = {
+  generateRecipe,
+  ANTHROPIC_MODEL,
+  // exported for tests
+  extractJson,
+}

package/core/lib/web-extract/url-normalize.js

@@ -0,0 +1,90 @@
+/**
+ * URL normalization for the page recipe cache.
+ *
+ * Two URLs that resolve to the same template + the same DOM fingerprint are
+ * treated as structurally identical and share an extraction recipe. The
+ * template captures the path/query *shape* (with IDs / slugs / pagination
+ * placeholdered out) so the cache hits across e.g.
+ *   /work/proposal/123456?utm_source=foo
+ *   /work/proposal/789012?utm_source=bar&fbclid=...
+ * Tracking params are stripped, remaining query keys are sorted, and known
+ * pagination params keep their key but lose their value.
+ */
+
+const STRIP_QUERY_PARAMS = new Set([
+  'utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
+  'fbclid', 'gclid', 'mc_cid', 'mc_eid', 'ref', 'referrer',
+])
+
+const PAGINATION_QUERY_PARAMS = new Set([
+  'page', 'p', 'pagenumber', 'pagenum', 'offset', 'start',
+])
+
+const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i
+const HEX32_RE = /^[0-9a-f]{32}$/i
+const ALL_DIGITS_RE = /^\d+$/
+const LONG_ALNUM_RE = /^(?=.*\d)(?=.*[a-zA-Z])[A-Za-z0-9]{20,}$/
+
+function placeholderForSegment(segment) {
+  if (!segment) return segment
+  if (ALL_DIGITS_RE.test(segment)) return ':id'
+  if (UUID_RE.test(segment) || HEX32_RE.test(segment)) return ':uuid'
+  if (LONG_ALNUM_RE.test(segment)) return ':slug'
+  return segment
+}
+
+function normalizeUrl(rawUrl) {
+  let parsed
+  try {
+    parsed = new URL(rawUrl)
+  } catch (err) {
+    throw new Error(`Invalid URL: ${rawUrl}`)
+  }
+
+  // Lowercase host (case-insensitive per RFC 3986)
+  const host = parsed.host.toLowerCase()
+
+  // Path: keep separators, placeholder each segment.
+  const pathSegments = parsed.pathname.split('/').map(seg => seg ? placeholderForSegment(seg) : seg)
+  const templatePath = pathSegments.join('/')
+
+  // Query: filter, sort, and placeholder pagination values.
+  const queryEntries = []
+  for (const [key, value] of parsed.searchParams.entries()) {
+    const lowerKey = key.toLowerCase()
+    if (STRIP_QUERY_PARAMS.has(lowerKey)) continue
+    if (PAGINATION_QUERY_PARAMS.has(lowerKey)) {
+      queryEntries.push([key, ':n'])
+    } else {
+      queryEntries.push([key, value])
+    }
+  }
+  queryEntries.sort((a, b) => a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0)
+
+  const templateQuery = queryEntries.length
+    ? '?' + queryEntries.map(([k, v]) => `${k}=${v}`).join('&')
+    : ''
+
+  const template = `${host}${templatePath}${templateQuery}`
+
+  // Canonical URL: original protocol + host + path, query reduced to non-tracking params
+  // (keeping their *real* values so the actual fetch stays correct).
+  const canonical = new URL(parsed.toString())
+  for (const param of [...canonical.searchParams.keys()]) {
+    if (STRIP_QUERY_PARAMS.has(param.toLowerCase())) {
+      canonical.searchParams.delete(param)
+    }
+  }
+  canonical.hash = ''
+
+  return {
+    template,
+    canonicalUrl: canonical.toString(),
+  }
+}
+
+module.exports = {
+  normalizeUrl,
+  STRIP_QUERY_PARAMS,
+  PAGINATION_QUERY_PARAMS,
+}

package/core/routes/admin.js

@@ -0,0 +1,49 @@
+/**
+ * Admin routes
+ *
+ * HQ-only operations pushed via Cloudflare Tunnel. Currently exposes the
+ * billing-driven freeze endpoint. Recovery is intentionally NOT exposed
+ * here — HQ recovers a frozen minion by sending the existing
+ * `restart-agent` command (commands.js), which terminates the process and
+ * lets the in-memory frozen flag clear naturally.
+ *
+ * Endpoints:
+ *   POST /api/admin/freeze   - Set in-memory frozen state
+ *   GET  /api/admin/state    - Diagnostic: report frozen state
+ */
+
+const { verifyToken } = require('../lib/auth')
+const frozenState = require('../lib/frozen-state')
+
+async function adminRoutes(fastify) {
+  fastify.post('/api/admin/freeze', async (request, reply) => {
+    if (!verifyToken(request)) {
+      reply.code(401)
+      return { success: false, error: 'Unauthorized' }
+    }
+
+    const body = request.body || {}
+    const reason = typeof body.reason === 'string' ? body.reason : 'past_due'
+
+    frozenState.setFrozen({ reason })
+
+    return {
+      success: true,
+      state: frozenState.getState(),
+    }
+  })
+
+  fastify.get('/api/admin/state', async (request, reply) => {
+    if (!verifyToken(request)) {
+      reply.code(401)
+      return { success: false, error: 'Unauthorized' }
+    }
+
+    return {
+      success: true,
+      state: frozenState.getState(),
+    }
+  })
+}
+
+module.exports = { adminRoutes }

package/core/routes/web.js

@@ -0,0 +1,94 @@
+/**
+ * Web Page Extraction (experimental — v3.53.0)
+ *
+ * Endpoints:
+ *   POST /api/web/extract       - Fetch a URL and return structured JSON
+ *   GET  /api/web/recipes       - List cached recipes (debug)
+ *   DELETE /api/web/recipes/:template/:fingerprint - Drop a recipe
+ *
+ * The extract endpoint runs Playwright + Readability + an Anthropic Haiku
+ * recipe-generation step in this minion process, returning only structured
+ * JSON to the caller. Designed to keep large DOM payloads off the main
+ * Claude Code chat session that issued the request.
+ *
+ * Marked experimental: response shape may change.
+ */
+
+const { verifyToken } = require('../lib/auth')
+const { extract } = require('../lib/web-extract')
+const pageRecipeStore = require('../stores/page-recipe-store')
+
+const REQUEST_TIMEOUT_MS = 60_000
+
+async function webRoutes(fastify) {
+  fastify.post('/api/web/extract', async (request, reply) => {
+    if (!verifyToken(request)) {
+      reply.code(401)
+      return { success: false, error: 'Unauthorized' }
+    }
+
+    const body = request.body || {}
+    const { url, hint } = body
+
+    if (!url || typeof url !== 'string') {
+      reply.code(400)
+      return { success: false, error: 'url (string) is required' }
+    }
+    try {
+      new URL(url)
+    } catch {
+      reply.code(400)
+      return { success: false, error: 'url is not a valid URL' }
+    }
+
+    try {
+      const result = await Promise.race([
+        extract({ url, hint: typeof hint === 'string' ? hint : null }),
+        new Promise((_, rej) => setTimeout(() => rej(new Error('extract timeout')), REQUEST_TIMEOUT_MS)),
+      ])
+      return { success: true, ...result }
+    } catch (err) {
+      if (
+        err.code === 'PLAYWRIGHT_UNAVAILABLE' ||
+        err.code === 'LLM_UNAVAILABLE' ||
+        err.code === 'ANTHROPIC_KEY_MISSING'
+      ) {
+        reply.code(503)
+        return { success: false, error: err.message, code: err.code }
+      }
+      if (err.code === 'PRIMARY_LLM_FAILED' || err.code === 'PRIMARY_LLM_BAD_JSON') {
+        reply.code(502)
+        return { success: false, error: err.message, code: err.code }
+      }
+      request.log.error({ err }, '[web/extract] failed')
+      reply.code(500)
+      return { success: false, error: err.message || String(err) }
+    }
+  })
+
+  // Debug helpers — list / delete cached recipes.
+  fastify.get('/api/web/recipes', async (request, reply) => {
+    if (!verifyToken(request)) {
+      reply.code(401)
+      return { success: false, error: 'Unauthorized' }
+    }
+    const recipes = pageRecipeStore.listAll({ limit: 200 })
+    return { success: true, experimental: true, recipes }
+  })
+
+  fastify.delete('/api/web/recipes', async (request, reply) => {
+    if (!verifyToken(request)) {
+      reply.code(401)
+      return { success: false, error: 'Unauthorized' }
+    }
+    const { template, fingerprint } = request.query || {}
+    if (!template || !fingerprint) {
+      reply.code(400)
+      return { success: false, error: 'template and fingerprint query params are required' }
+    }
+    const removed = pageRecipeStore.remove({ urlTemplate: template, domFingerprint: fingerprint })
+    return { success: true, removed }
+  })
+}
+
+module.exports = { webRoutes }