@geekbeer/minion 4.4.0 → 4.7.0

package/core/db/migrations/20260607000000_chat_runs.js

@@ -0,0 +1,48 @@
+/**
+ * Add `chat_runs` — the durability backbone for detached chat execution.
+ *
+ * A "run" is one Claude invocation owned by the chat-run-manager, NOT by the
+ * HTTP request that started it. The SSE connection is a mere subscriber: when
+ * it drops, the run keeps going. This table lets a reconnecting client (new
+ * tab, refresh, dropped network) discover the in-flight run for a workspace and
+ * resume tailing its event log from the last seen sequence number.
+ *
+ * Columns:
+ *  - run_id:             manager-minted UUID (also the NDJSON event-log filename)
+ *  - session_id:         resolved Claude CLI session id (null until reported)
+ *  - pending_session_id: local pending id used to persist the user message
+ *                        before the CLI session id is known
+ *  - workspace_id:       scope (null = minion-wide bucket)
+ *  - status:             running | done | error | aborted | interrupted
+ *  - last_seq:           highest event sequence appended (reconnect cursor hint)
+ *
+ * On boot, any row still marked `running` is stale (its in-process owner died
+ * with the previous server) and is swept to `interrupted` by the store.
+ */
+
+module.exports = {
+  version: 20260607000000,
+  name: 'chat_runs',
+
+  up(db, { tableExists }) {
+    if (tableExists(db, 'chat_runs')) return
+
+    db.exec(`
+      CREATE TABLE chat_runs (
+        run_id TEXT PRIMARY KEY,
+        session_id TEXT DEFAULT NULL,
+        pending_session_id TEXT DEFAULT NULL,
+        workspace_id TEXT DEFAULT NULL,
+        status TEXT NOT NULL DEFAULT 'running',
+        started_at INTEGER NOT NULL,
+        ended_at INTEGER DEFAULT NULL,
+        last_seq INTEGER NOT NULL DEFAULT 0
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_chat_runs_workspace
+        ON chat_runs(workspace_id, started_at DESC);
+      CREATE INDEX IF NOT EXISTS idx_chat_runs_status
+        ON chat_runs(status);
+    `)
+  },
+}

package/core/db/migrations/20260607120000_page_recipes_ready_selector.js

@@ -0,0 +1,22 @@
+/**
+ * page_recipes.ready_selector — SPA-aware wait support (v4.7.0).
+ *
+ * The cold path now asks the LLM for a `ready_selector`: a CSS selector for an
+ * element that only exists once the page's primary content has rendered. On hot
+ * replays the extractor waits for it before reading the DOM, so client-rendered
+ * (SPA) pages are captured after hydration instead of as an empty shell.
+ *
+ * Nullable: recipes learned before this column existed simply fall back to
+ * DOM-settle detection until they are regenerated.
+ */
+
+module.exports = {
+  version: 20260607120000,
+  name: 'page_recipes_ready_selector',
+
+  up(db, { tableExists, hasColumn }) {
+    if (!tableExists(db, 'page_recipes')) return
+    if (hasColumn(db, 'page_recipes', 'ready_selector')) return
+    db.exec('ALTER TABLE page_recipes ADD COLUMN ready_selector TEXT;')
+  },
+}

package/core/lib/chat-run-manager.js

@@ -0,0 +1,406 @@
+/**
+ * Chat Run Manager — detached execution backbone for chat.
+ *
+ * THE PROBLEM THIS SOLVES
+ * -----------------------
+ * Historically a chat message spawned the LLM as a child of the HTTP request
+ * handler and streamed its stdout straight into the SSE response. A
+ * `res.on('close')` handler then killed that child. So ANY break in the
+ * browser → HQ → minion connection chain (tab close, navigation, reverse-proxy
+ * idle timeout, network blip) silently SIGTERM'd the work mid-task.
+ *
+ * THE MODEL
+ * ---------
+ * A "run" is one LLM invocation owned by THIS manager, not by any HTTP request.
+ * The manager spawns the LLM (via a caller-supplied `invoke` executor),
+ * records every wire event into an append-only NDJSON log + an in-memory
+ * buffer, and notifies subscribers. The SSE endpoints are mere SUBSCRIBERS:
+ * when a connection drops they unsubscribe — the run keeps going. A
+ * reconnecting client (new tab / refresh / dropped network) finds the in-flight
+ * run via `getActiveRunId()` and resumes tailing from the last sequence number
+ * it saw. Only an explicit `/api/chat/abort` kills the process.
+ *
+ * SEPARATION OF CONCERNS
+ * ----------------------
+ * The manager is LLM-agnostic. The route supplies an `invoke(emit, activeRef)`
+ * executor that does the actual plugin.stream() / legacy spawn and:
+ *   - calls emit(wireEvent) for each delta / text / tool / result event
+ *   - sets activeRef.current = <child process> so abort can reach it
+ *   - returns { fullResponse, resolvedSessionId, turnCount } (or throws)
+ * The manager owns everything durable: run id, event log, pub/sub, session
+ * persistence/rekey, the terminal `done`/`error` event, abort, and eviction.
+ *
+ * DURABILITY vs MEMORY
+ * --------------------
+ * While a run is live (and during a post-completion TTL) its events live in
+ * memory and memory is authoritative for replay — the common reconnect (refresh
+ * during an active run) never touches disk. The NDJSON log is the fallback for
+ * reconnects that arrive after the run was evicted from memory. Phase 1 does NOT
+ * survive a minion process restart (the in-process owner dies with it); on boot
+ * any run still flagged `running` is swept to `interrupted` so clients stop
+ * waiting. (A future phase could relaunch under tmux like board tasks.)
+ */
+
+const crypto = require('crypto')
+const fs = require('fs')
+const path = require('path')
+
+const { DATA_DIR } = require('./platform')
+const { config } = require('../config')
+const chatStore = require('../stores/chat-store')
+
+// Keep a finished run inspectable for late reconnects before freeing its memory.
+const RUN_TTL_MS = 10 * 60 * 1000
+// Terminal event types that end a subscriber's stream.
+const TERMINAL_TYPES = new Set(['done', 'error', 'aborted'])
+// Prune NDJSON logs / run rows older than this on boot.
+const LOG_MAX_AGE_MS = 24 * 60 * 60 * 1000
+
+/** @type {Map<string, RunState>} */
+const registry = new Map()
+
+/**
+ * @typedef {Object} RunState
+ * @property {string} runId
+ * @property {string|null} sessionId          resume target (null for new sessions)
+ * @property {string|null} pendingSessionId   local id the user message was stored under
+ * @property {string|null} workspaceId
+ * @property {string} status                  running | done | error | aborted
+ * @property {Array<object>} events           in-memory mirror of logged events (seq-stamped)
+ * @property {number} seq                     highest sequence assigned
+ * @property {Set<function>} subscribers      live listeners
+ * @property {{ current: any }} activeRef      handle to the child process for abort
+ * @property {string} logPath
+ * @property {import('fs').WriteStream|null} logStream
+ * @property {boolean} aborting
+ */
+
+function resolveRunsDir() {
+  // Mirror the db module's writability fallback (DATA_DIR may be read-only in
+  // some deployments; HOME_DIR is always writable).
+  let base = DATA_DIR
+  try {
+    fs.accessSync(DATA_DIR, fs.constants.W_OK)
+  } catch {
+    base = path.join(config.HOME_DIR, '.minion')
+  }
+  return path.join(base, 'chat-runs')
+}
+
+const RUNS_DIR = resolveRunsDir()
+
+function logPathFor(runId) {
+  return path.join(RUNS_DIR, `${runId}.ndjson`)
+}
+
+/**
+ * One-time boot sweep: stale `running` rows belong to a dead previous process,
+ * and old logs/rows are pruned. Safe to call more than once.
+ */
+function init() {
+  try {
+    fs.mkdirSync(RUNS_DIR, { recursive: true })
+  } catch (err) {
+    console.error(`[ChatRun] failed to create runs dir: ${err.message}`)
+  }
+  try {
+    const swept = chatStore.markRunningInterrupted()
+    if (swept) console.log(`[ChatRun] swept ${swept} interrupted run(s) from previous boot`)
+  } catch (err) {
+    console.error(`[ChatRun] interrupted sweep failed: ${err.message}`)
+  }
+  try {
+    chatStore.pruneRuns(LOG_MAX_AGE_MS)
+  } catch (err) {
+    console.error(`[ChatRun] run prune failed: ${err.message}`)
+  }
+  pruneOldLogs()
+}
+
+function pruneOldLogs() {
+  let files
+  try {
+    files = fs.readdirSync(RUNS_DIR)
+  } catch {
+    return
+  }
+  const cutoff = Date.now() - LOG_MAX_AGE_MS
+  for (const f of files) {
+    if (!f.endsWith('.ndjson')) continue
+    const full = path.join(RUNS_DIR, f)
+    try {
+      if (fs.statSync(full).mtimeMs < cutoff) fs.unlinkSync(full)
+    } catch { /* ignore */ }
+  }
+}
+
+/**
+ * Start a detached run. Returns the run id immediately; the LLM keeps streaming
+ * in the background regardless of who (if anyone) is subscribed.
+ *
+ * @param {Object} params
+ * @param {string|null} params.sessionId
+ * @param {string|null} params.pendingSessionId
+ * @param {string|null} params.workspaceId
+ * @param {(emit: (event: object) => void, activeRef: { current: any }) => Promise<{ fullResponse: string, resolvedSessionId: string|null, turnCount: number }>} params.invoke
+ * @returns {string} runId
+ */
+function start({ sessionId = null, pendingSessionId = null, workspaceId = null, invoke }) {
+  const runId = crypto.randomUUID()
+  /** @type {RunState} */
+  const run = {
+    runId,
+    sessionId,
+    pendingSessionId,
+    workspaceId,
+    status: 'running',
+    events: [],
+    seq: 0,
+    subscribers: new Set(),
+    activeRef: { current: null },
+    logPath: logPathFor(runId),
+    logStream: null,
+    aborting: false,
+  }
+  registry.set(runId, run)
+
+  try {
+    run.logStream = fs.createWriteStream(run.logPath, { flags: 'w' })
+    run.logStream.on('error', err => console.error(`[ChatRun] log write error (${runId}): ${err.message}`))
+  } catch (err) {
+    console.error(`[ChatRun] failed to open log for ${runId}: ${err.message}`)
+  }
+
+  try {
+    chatStore.createRun({ runId, sessionId, pendingSessionId, workspaceId })
+  } catch (err) {
+    console.error(`[ChatRun] createRun failed: ${err.message}`)
+  }
+
+  const emit = event => appendEvent(run, event)
+
+  // Fire-and-forget. The executor owns the child process; we own durability.
+  Promise.resolve()
+    .then(() => invoke(emit, run.activeRef))
+    .then(result => finalize(run, result, null))
+    .catch(err => finalize(run, null, err))
+
+  console.log(`[ChatRun] started run ${runId} (ws=${workspaceId || 'none'}, resume=${sessionId || 'new'})`)
+  return runId
+}
+
+/**
+ * Append a wire event: stamp a sequence number, persist, mirror in memory,
+ * notify subscribers. Single-threaded JS guarantees subscribers attached
+ * between events never miss one (no await between push and notify).
+ */
+function appendEvent(run, event) {
+  run.seq += 1
+  const stamped = { seq: run.seq, ...event }
+  run.events.push(stamped)
+  if (run.logStream && !run.logStream.destroyed) {
+    try { run.logStream.write(JSON.stringify(stamped) + '\n') } catch { /* best-effort */ }
+  }
+  for (const fn of run.subscribers) {
+    try { fn(stamped) } catch (err) { console.error(`[ChatRun] subscriber error: ${err.message}`) }
+  }
+  return stamped
+}
+
+/**
+ * Persist the assistant message, rekey the session, emit the terminal event,
+ * and schedule eviction. Runs whether the invoke resolved or threw.
+ */
+async function finalize(run, result, err) {
+  const resolvedSessionId = (result && result.resolvedSessionId) || run.sessionId || null
+  const fullResponse = (result && result.fullResponse) || ''
+  const turnCount = (result && result.turnCount) || 0
+
+  // Rekey the pending session to the real CLI session id, and persist the
+  // assistant response — even a partial one from an aborted run. These are
+  // INDEPENDENT: a rekey failure must never block persisting the reply. If the
+  // rekey doesn't succeed, we keep the whole conversation under the pending id
+  // (messages stay together) rather than splitting it across two session rows.
+  let persistSessionId = resolvedSessionId || run.pendingSessionId
+  if (!run.sessionId && resolvedSessionId && run.pendingSessionId && run.pendingSessionId !== resolvedSessionId) {
+    try {
+      const rekeyed = chatStore.rekeySession(run.pendingSessionId, resolvedSessionId)
+      persistSessionId = rekeyed ? resolvedSessionId : run.pendingSessionId
+    } catch (e) {
+      console.error(`[ChatRun] rekey failed (${run.runId}): ${e.message}`)
+      persistSessionId = run.pendingSessionId
+    }
+  }
+  try {
+    if (fullResponse && persistSessionId) {
+      await chatStore.addMessage(persistSessionId, { role: 'assistant', content: fullResponse }, turnCount, run.workspaceId)
+    }
+  } catch (e) {
+    console.error(`[ChatRun] persist failed (${run.runId}): ${e.message}`)
+  }
+
+  if (err) {
+    console.error(`[ChatRun] run ${run.runId} errored: ${err.message}`)
+    appendEvent(run, { type: 'error', error: err.message, partial: !!fullResponse })
+  }
+
+  let totalTurnCount = turnCount
+  try {
+    const session = chatStore.load(run.workspaceId)
+    totalTurnCount = (session && session.turn_count) || turnCount
+  } catch (e) {
+    console.error(`[ChatRun] load for done event failed: ${e.message}`)
+  }
+
+  run.status = run.aborting ? 'aborted' : (err ? 'error' : 'done')
+  appendEvent(run, { type: 'done', session_id: resolvedSessionId, turn_count: totalTurnCount })
+
+  try {
+    chatStore.updateRun(run.runId, { status: run.status, sessionId: resolvedSessionId, lastSeq: run.seq })
+  } catch (e) {
+    console.error(`[ChatRun] updateRun failed: ${e.message}`)
+  }
+
+  if (run.logStream && !run.logStream.destroyed) {
+    try { run.logStream.end() } catch { /* ignore */ }
+  }
+  run.activeRef.current = null
+
+  console.log(`[ChatRun] run ${run.runId} ${run.status} (${fullResponse.length} chars, ${run.seq} events)`)
+
+  // Keep the run inspectable for late reconnects, then free its memory.
+  setTimeout(() => evict(run.runId), RUN_TTL_MS)
+}
+
+function evict(runId) {
+  const run = registry.get(runId)
+  if (!run) return
+  registry.delete(runId)
+}
+
+/**
+ * Subscribe to a run's events from a cursor. Replays everything with seq >
+ * fromSeq, then streams live events. The subscriber should close its own
+ * transport when it sees a terminal event (done/error/aborted).
+ *
+ * @param {string} runId
+ * @param {number} fromSeq    last sequence the client already has (0 = from start)
+ * @param {(event: object) => void} onEvent
+ * @returns {() => void} unsubscribe
+ */
+function subscribe(runId, fromSeq, onEvent) {
+  const from = Number(fromSeq) || 0
+  const run = registry.get(runId)
+
+  if (!run) {
+    // Evicted or owned by a previous process: replay from disk, no live tail.
+    for (const e of readEventsFromFile(runId)) {
+      if (e.seq > from) onEvent(e)
+    }
+    return () => {}
+  }
+
+  // In memory: replay buffered events, then attach for everything after. No
+  // await between the two, so the live listener cannot miss an event.
+  for (const e of run.events) {
+    if (e.seq > from) onEvent(e)
+  }
+  // Already terminal (run finished, still within TTL): the replay above
+  // included its `done`, and no further events will come — don't register a
+  // live listener that would never fire (and never be cleaned up).
+  if (run.status !== 'running') {
+    return () => {}
+  }
+  // Only forward events strictly newer than what we just replayed.
+  const replayedUpTo = run.seq
+  const guarded = e => { if (e.seq > replayedUpTo) onEvent(e) }
+  run.subscribers.add(guarded)
+  return () => run.subscribers.delete(guarded)
+}
+
+function readEventsFromFile(runId) {
+  try {
+    const raw = fs.readFileSync(logPathFor(runId), 'utf-8')
+    const out = []
+    for (const line of raw.split('\n')) {
+      if (!line.trim()) continue
+      try { out.push(JSON.parse(line)) } catch { /* skip malformed line */ }
+    }
+    return out
+  } catch {
+    return []
+  }
+}
+
+/**
+ * Abort a run: SIGTERM, then SIGKILL after a 2s grace period. The run's invoke
+ * promise then settles and finalize() persists whatever partial response was
+ * collected. No-op if the run isn't live.
+ * @param {string} runId
+ * @returns {boolean} true if a signal was sent
+ */
+function abort(runId) {
+  const run = registry.get(runId)
+  const child = run && run.activeRef.current
+  if (!child) return false
+  run.aborting = true
+  console.log(`[ChatRun] aborting run ${runId} (PID ${child.pid})`)
+  try { child.kill('SIGTERM') } catch { /* already dead */ }
+  const pid = child.pid
+  setTimeout(() => {
+    try {
+      if (run.activeRef.current && run.activeRef.current.pid === pid) {
+        run.activeRef.current.kill('SIGKILL')
+      }
+    } catch { /* already dead */ }
+  }, 2000)
+  return true
+}
+
+/**
+ * Most recent running run for a workspace — the reconnect target. Prefers the
+ * live registry, falls back to the durable index.
+ * @param {string|null} workspaceId
+ * @returns {string|null} runId
+ */
+function getActiveRunId(workspaceId) {
+  const wsKey = workspaceId || null
+  for (const run of registry.values()) {
+    if (run.status === 'running' && (run.workspaceId || null) === wsKey) return run.runId
+  }
+  try {
+    const row = chatStore.getActiveRun(workspaceId)
+    return row ? row.run_id : null
+  } catch {
+    return null
+  }
+}
+
+/**
+ * Snapshot of a run's status for the session endpoint. Returns null if unknown.
+ * @param {string} runId
+ */
+function getRunInfo(runId) {
+  const run = registry.get(runId)
+  if (run) {
+    return { run_id: run.runId, status: run.status, last_seq: run.seq, session_id: run.sessionId }
+  }
+  try {
+    const row = chatStore.getRun(runId)
+    if (!row) return null
+    return { run_id: row.run_id, status: row.status, last_seq: row.last_seq, session_id: row.session_id }
+  } catch {
+    return null
+  }
+}
+
+module.exports = {
+  init,
+  start,
+  subscribe,
+  abort,
+  getActiveRunId,
+  getRunInfo,
+  // exposed for tests
+  _registry: registry,
+}

package/core/lib/web-extract/extractor.js

@@ -16,7 +16,7 @@
 
 const { normalizeUrl } = require('./url-normalize')
 const { computeFingerprint } = require('./fingerprint')
-const { renderPage, extractWithSelectors } = require('./playwright-runner')
+const { renderPage, extractWithSelectors, normalizeScroll } = require('./playwright-runner')
 const { cleanHtml } = require('./html-cleaner')
 const { generateRecipe } = require('./recipe-generator')
 const pageRecipeStore = require('../../stores/page-recipe-store')
@@ -33,11 +33,13 @@ function isEmptyResult(data) {
   })
 }
 
-async function extract({ url, hint }) {
+async function extract({ url, hint, scroll }) {
   const { template, canonicalUrl } = normalizeUrl(url)
 
   // Always render once up-front so we can compute the fingerprint regardless
-  // of cache state. Cold path reuses the HTML; hot path discards it.
+  // of cache state. Cold path reuses the HTML; hot path discards it. No
+  // readySelector is known yet here, so renderPage falls back to DOM-settle
+  // detection — enough to let an SPA hydrate before we fingerprint/clean.
   const rendered = await renderPage(canonicalUrl)
   const fingerprint = computeFingerprint(rendered.html)
 
@@ -47,7 +49,11 @@ async function extract({ url, hint }) {
   })
 
   if (cached) {
-    const data = await extractWithSelectors(canonicalUrl, cached.selectors)
+    const scrollCfg = normalizeScroll(scroll, cached.selectors)
+    const { data, scrollInfo } = await extractWithSelectors(canonicalUrl, cached.selectors, {
+      readySelector: cached.ready_selector,
+      scroll: scrollCfg,
+    })
     if (!isEmptyResult(data)) {
       pageRecipeStore.incrementHit({ urlTemplate: template, domFingerprint: fingerprint })
       pageRecipeStore.setLastVerified({ urlTemplate: template, domFingerprint: fingerprint })
@@ -62,6 +68,7 @@ async function extract({ url, hint }) {
         selectors: cached.selectors,
         data,
         cleaned: null,
+        scrollInfo,
       })
     }
     // Hot replay returned nothing — penalize and fall through to cold.
@@ -76,8 +83,13 @@ async function extract({ url, hint }) {
     hint,
   })
 
-  // Verify the recipe against this exact page before persisting.
-  const verifyData = await extractWithSelectors(canonicalUrl, recipe.selectors)
+  // Verify the recipe against this exact page before persisting. Now that we
+  // have a readySelector, the verify render waits for real content.
+  const scrollCfg = normalizeScroll(scroll, recipe.selectors)
+  const { data: verifyData, scrollInfo } = await extractWithSelectors(canonicalUrl, recipe.selectors, {
+    readySelector: recipe.readySelector,
+    scroll: scrollCfg,
+  })
   const verified = !isEmptyResult(verifyData)
 
   if (verified) {
@@ -86,6 +98,7 @@ async function extract({ url, hint }) {
       domFingerprint: fingerprint,
       selectors: recipe.selectors,
       pageType: recipe.pageType,
+      readySelector: recipe.readySelector,
     })
     pageRecipeStore.incrementHit({ urlTemplate: template, domFingerprint: fingerprint })
   }
@@ -102,10 +115,11 @@ async function extract({ url, hint }) {
     data: verified ? verifyData : recipe.extracted,
     cleaned,
     recipePersisted: verified,
+    scrollInfo,
   })
 }
 
-function shape({ url, finalUrl, statusCode, recipeMode, urlTemplate, fingerprint, pageType, selectors, data, cleaned, recipePersisted }) {
+function shape({ url, finalUrl, statusCode, recipeMode, urlTemplate, fingerprint, pageType, selectors, data, cleaned, recipePersisted, scrollInfo }) {
   const out = {
     experimental: true,
     url,
@@ -119,6 +133,12 @@ function shape({ url, finalUrl, statusCode, recipeMode, urlTemplate, fingerprint
     structured: data || {},
     selectors: selectors || {},
   }
+  if (scrollInfo) {
+    out.scrollInfo = scrollInfo
+    if (scrollInfo.reachedTarget === false) {
+      out.warning = `Scroll stopped before reaching target (reason: ${scrollInfo.stoppedReason}, items: ${scrollInfo.items}). Raise scroll.maxScrolls / scroll.maxMs to collect more.`
+    }
+  }
   if (recipeMode === 'cold' && recipePersisted === false) {
     out.warning = 'Recipe verification failed (selectors returned empty). Result reflects LLM extraction; recipe was not persisted.'
   }