@geekbeer/minion 4.4.0 → 4.7.0

package/core/lib/web-extract/playwright-runner.js

@@ -8,14 +8,37 @@
  *
  * Each call spins up a fresh chromium instance. Pooling can come later
  * once the API stabilizes — for the experimental MVP, simple is better.
+ *
+ * Wait strategy (SPA-aware, v4.7.0):
+ *   `page.goto` resolves on `domcontentloaded`, which for a client-rendered
+ *   SPA fires *before* the framework has mounted and fetched its data. So
+ *   after navigation we additionally wait for the content to actually appear:
+ *     1. If a `readySelector` is known (from the recipe), wait for it.
+ *     2. Otherwise wait for the DOM to *settle* — i.e. no MutationObserver
+ *        events for `settleMs` — which is self-calibrating and works whether
+ *        the page renders 50 chars or 50,000.
+ *   `scroll` (optional) then drives infinite-scroll / lazy-load pages up to
+ *   caller-declared limits, with hard server-side caps.
  */
 
 const DEFAULT_NAV_TIMEOUT_MS = 20_000
 const DEFAULT_EVAL_TIMEOUT_MS = 5_000
+const DEFAULT_READY_TIMEOUT_MS = 8_000
+const DEFAULT_SETTLE_MS = 500
+const DEFAULT_SETTLE_MAX_MS = 8_000
 const DEFAULT_USER_AGENT =
   'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) ' +
   'Chrome/124.0.0.0 Safari/537.36 MinionWebExtract/0.1'
 
+// Scroll defaults (caller may override) and hard caps (server enforces).
+const SCROLL_DEFAULT_MAX_SCROLLS = 10
+const SCROLL_HARD_MAX_SCROLLS = 50
+const SCROLL_DEFAULT_MAX_MS = 15_000
+const SCROLL_HARD_MAX_MS = 45_000
+const SCROLL_DEFAULT_SETTLE_MS = 600
+const SCROLL_STABLE_ROUNDS = 2
+const SCROLL_STRATEGIES = ['count', 'untilStable', 'fixed']
+
 function loadChromium() {
   let playwright
   try {
@@ -49,12 +72,172 @@ async function withPage(fn, opts = {}) {
   }
 }
 
+/**
+ * Wait for the DOM to stop mutating. Resolves once no childList/characterData
+ * mutation has fired for `settleMs`, or after `settleMaxMs` regardless. Scoped
+ * to the main content landmark when present so ever-churning headers/ads/beacons
+ * don't keep it awake. Best-effort: any failure (e.g. mid-navigation) resolves.
+ */
+async function waitForSettle(page, opts = {}) {
+  const settleMs = opts.settleMs ?? DEFAULT_SETTLE_MS
+  const settleMaxMs = opts.settleMaxMs ?? DEFAULT_SETTLE_MAX_MS
+  try {
+    await page.evaluate(({ quiet, max }) => new Promise(resolve => {
+      const target = document.querySelector('main, article, [role="main"]') || document.body
+      if (!target) { resolve(); return }
+      let quietTimer = setTimeout(finish, quiet)
+      const hardCap = setTimeout(finish, max)
+      const obs = new MutationObserver(() => {
+        clearTimeout(quietTimer)
+        quietTimer = setTimeout(finish, quiet)
+      })
+      obs.observe(target, { childList: true, subtree: true, characterData: true })
+      function finish() {
+        clearTimeout(quietTimer)
+        clearTimeout(hardCap)
+        obs.disconnect()
+        resolve()
+      }
+    }), { quiet: settleMs, max: settleMaxMs })
+  } catch {
+    // navigation/teardown raced us — caller proceeds with whatever rendered.
+  }
+}
+
+/**
+ * After navigation, wait until the meaningful content is present:
+ * prefer the recipe's `readySelector`; fall back to DOM-settle detection.
+ */
+async function waitForReady(page, opts = {}) {
+  if (opts.readySelector) {
+    try {
+      await page.waitForSelector(opts.readySelector, {
+        state: 'visible',
+        timeout: opts.readyTimeoutMs ?? DEFAULT_READY_TIMEOUT_MS,
+      })
+      return
+    } catch {
+      // readySelector never showed — fall through to settle so we still
+      // capture whatever did render rather than returning the bare shell.
+    }
+  }
+  await waitForSettle(page, opts)
+}
+
+/**
+ * Normalize a caller-supplied scroll request into a clamped config, or null
+ * when no (valid) scroll was requested. `itemSelector` resolution order:
+ * explicit > the recipe's first `multiple: true` selector. With no item
+ * selector, 'count' is meaningless so we measure scrollHeight stability.
+ */
+function normalizeScroll(scroll, selectors) {
+  if (!scroll || typeof scroll !== 'object') return null
+  if (!SCROLL_STRATEGIES.includes(scroll.strategy)) return null
+
+  let itemSelector =
+    typeof scroll.itemSelector === 'string' && scroll.itemSelector.trim()
+      ? scroll.itemSelector.trim()
+      : null
+  if (!itemSelector && selectors && typeof selectors === 'object') {
+    for (const spec of Object.values(selectors)) {
+      if (spec && spec.multiple && typeof spec.selector === 'string' && spec.selector.trim()) {
+        itemSelector = spec.selector.trim()
+        break
+      }
+    }
+  }
+
+  const clampInt = (val, def, min, max) => {
+    const n = Number.isFinite(val) ? Math.floor(val) : def
+    return Math.max(min, Math.min(max, n))
+  }
+
+  return {
+    strategy: scroll.strategy,
+    itemSelector,
+    targetItems: clampInt(scroll.targetItems, 0, 0, 100_000),
+    maxScrolls: clampInt(scroll.maxScrolls, SCROLL_DEFAULT_MAX_SCROLLS, 1, SCROLL_HARD_MAX_SCROLLS),
+    maxMs: clampInt(scroll.maxMs, SCROLL_DEFAULT_MAX_MS, 500, SCROLL_HARD_MAX_MS),
+    settleMs: clampInt(scroll.settleMs, SCROLL_DEFAULT_SETTLE_MS, 100, 5_000),
+    times: clampInt(scroll.times, SCROLL_DEFAULT_MAX_SCROLLS, 1, SCROLL_HARD_MAX_SCROLLS),
+  }
+}
+
+/**
+ * Drive an infinite-scroll / lazy-load page within the caller's limits.
+ * Returns { scrolls, items, reachedTarget, stoppedReason } so the caller can
+ * tell whether it hit the target or was capped (never silently truncated).
+ */
+async function scrollToLoad(page, cfg) {
+  const start = Date.now()
+  const elapsed = () => Date.now() - start
+  const measure = async () => {
+    try {
+      if (cfg.itemSelector) {
+        return await page.evaluate(sel => document.querySelectorAll(sel).length, cfg.itemSelector)
+      }
+      return await page.evaluate(() => document.body.scrollHeight)
+    } catch {
+      return 0
+    }
+  }
+
+  let last = await measure()
+  let scrolls = 0
+  let stableRounds = 0
+  let stoppedReason = null
+
+  while (true) {
+    if (cfg.strategy === 'count' && cfg.itemSelector && cfg.targetItems > 0 && last >= cfg.targetItems) {
+      stoppedReason = 'reachedTarget'
+      break
+    }
+    if (cfg.strategy === 'fixed' && scrolls >= cfg.times) {
+      stoppedReason = 'fixedDone'
+      break
+    }
+    if (scrolls >= cfg.maxScrolls) { stoppedReason = 'maxScrolls'; break }
+    if (elapsed() >= cfg.maxMs) { stoppedReason = 'maxMs'; break }
+
+    await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight)).catch(() => {})
+    scrolls++
+    await waitForSettle(page, {
+      settleMs: cfg.settleMs,
+      settleMaxMs: Math.min(cfg.settleMs * 4, Math.max(cfg.maxMs - elapsed(), cfg.settleMs)),
+    })
+
+    const cur = await measure()
+    if (cfg.strategy !== 'fixed') {
+      // No growth this round counts toward "settled"; two flat rounds = done.
+      stableRounds = cur <= last ? stableRounds + 1 : 0
+      if (stableRounds >= SCROLL_STABLE_ROUNDS) {
+        last = cur
+        stoppedReason = 'stable'
+        break
+      }
+    }
+    last = cur
+  }
+
+  const items = cfg.itemSelector ? await measure() : null
+  return {
+    scrolls,
+    items,
+    reachedTarget:
+      cfg.strategy === 'count' && cfg.itemSelector && cfg.targetItems > 0
+        ? items >= cfg.targetItems
+        : null,
+    stoppedReason: stoppedReason || 'stable',
+  }
+}
+
 async function renderPage(url, opts = {}) {
   return withPage(async page => {
     const response = await page.goto(url, {
       waitUntil: 'domcontentloaded',
       timeout: opts.timeoutMs ?? DEFAULT_NAV_TIMEOUT_MS,
     })
+    await waitForReady(page, opts)
     const html = await page.content()
     return {
       html,
@@ -77,6 +260,11 @@ async function renderPage(url, opts = {}) {
  *
  * `attr` defaults to 'text' (innerText). Special value 'html' returns
  * innerHTML. Any other string is read as an HTML attribute.
+ *
+ * `opts.readySelector` waits for that element before extracting; `opts.scroll`
+ * (a config from `normalizeScroll`) drives lazy-load pages first.
+ *
+ * Returns `{ data, scrollInfo }` — scrollInfo is null when no scroll ran.
  */
 async function extractWithSelectors(url, selectors, opts = {}) {
   return withPage(async page => {
@@ -84,7 +272,14 @@ async function extractWithSelectors(url, selectors, opts = {}) {
       waitUntil: 'domcontentloaded',
       timeout: opts.timeoutMs ?? DEFAULT_NAV_TIMEOUT_MS,
     })
-    return await page.evaluate(
+    await waitForReady(page, opts)
+
+    let scrollInfo = null
+    if (opts.scroll) {
+      scrollInfo = await scrollToLoad(page, opts.scroll)
+    }
+
+    const data = await page.evaluate(
       ({ selectorMap, evalTimeoutMs }) => {
         const start = Date.now()
         const result = {}
@@ -120,10 +315,13 @@ async function extractWithSelectors(url, selectors, opts = {}) {
       },
       { selectorMap: selectors, evalTimeoutMs: opts.evalTimeoutMs ?? DEFAULT_EVAL_TIMEOUT_MS },
     )
+
+    return { data, scrollInfo }
   }, opts)
 }
 
 module.exports = {
   renderPage,
   extractWithSelectors,
+  normalizeScroll,
 }

package/core/lib/web-extract/recipe-generator.js

@@ -17,6 +17,7 @@
  *   {
  *     pageType: 'article' | 'listing' | 'product' | 'profile' | 'form' | 'other',
  *     selectors: { fieldName: { selector, attr?, multiple? }, ... },
+ *     readySelector: <css selector whose presence signals content is rendered>,
  *     extracted: { fieldName: <value already pulled from this page> }
  *   }
  */
@@ -59,6 +60,15 @@ const ANTHROPIC_TOOLS = [{
           },
         },
       },
+      ready_selector: {
+        type: 'string',
+        description:
+          'A single CSS selector for an element that exists ONLY once the primary ' +
+          'content has rendered (e.g. the article body, the first list item, a price). ' +
+          'On later visits the extractor waits for this element before reading the page, ' +
+          'so client-rendered (SPA) pages are captured after hydration rather than as an ' +
+          'empty shell. Pick a stable, semantic element; avoid spinners/skeletons.',
+      },
       extracted: {
         type: 'object',
         description:
@@ -77,7 +87,10 @@ Given a cleaned Markdown rendering of one page, you must:
    - Prefer semantic selectors (article, h1, time[datetime], a[rel="author"]) over class names where possible.
    - Use class-based selectors only when semantic ones are unavailable.
    - Avoid fragile attribute selectors like data-react-* or auto-generated hashes.
-3. Fill the "extracted" object with the values pulled from this exact page so the caller can verify your recipe works.
+3. Pick a "ready_selector": one CSS selector for an element that only exists once the
+   primary content has rendered (the article body, the first list item, the price, etc.).
+   Prefer a stable semantic element; never pick a loading spinner or skeleton placeholder.
+4. Fill the "extracted" object with the values pulled from this exact page so the caller can verify your recipe works.
 
 The same recipe will be reused for structurally similar pages, so think about what generalizes.`
 
@@ -132,6 +145,7 @@ async function generateViaPlugin(plugin, { url, cleanedMarkdown, hint }) {
   return {
     pageType: json.page_type || 'other',
     selectors: json.selectors || {},
+    readySelector: typeof json.ready_selector === 'string' ? json.ready_selector : null,
     extracted: json.extracted || {},
     source: `primary:${plugin.name}`,
   }
@@ -175,10 +189,11 @@ async function generateViaAnthropicDirect({ url, cleanedMarkdown, hint }) {
     throw new Error('Anthropic API returned no tool_use block for page_extraction')
   }
 
-  const { page_type, selectors, extracted } = toolUse.input
+  const { page_type, selectors, ready_selector, extracted } = toolUse.input
   return {
     pageType: page_type || 'other',
     selectors: selectors || {},
+    readySelector: typeof ready_selector === 'string' ? ready_selector : null,
     extracted: extracted || {},
     source: 'anthropic-direct',
   }
@@ -199,12 +214,14 @@ function buildTextPrompt({ url, cleanedMarkdown, hint }) {
     '  "selectors": {',
     '    "<fieldName>": { "selector": "<css>", "attr"?: "<text|html|attribute-name>", "multiple"?: <boolean> }',
     '  },',
+    '  "ready_selector": "<css selector that only exists once the main content has rendered>",',
     '  "extracted": { "<fieldName>": "<string or array of strings>" }',
     '}',
     '',
     'Notes:',
     '- attr defaults to "text" (innerText). Use "html" or an HTML attribute name to override.',
     '- Set multiple=true for list fields (returns array).',
+    '- "ready_selector" should target a stable content element (article body, first list item, price); never a spinner/skeleton.',
     '- "extracted" must contain the values you actually read from THIS page using those selectors.',
     '',
     '--- Cleaned Markdown ---',

package/core/routes/variables.js

@@ -108,6 +108,29 @@ function variableRoutes(fastify, _opts, done) {
     return { success: true, scopes: variableStore.listVariableScopes() }
   })
 
+  // Move a variable from one workspace scope to another. Body:
+  // { from_workspace_id, to_workspace_id } — omit/empty string targets the
+  // minion-wide bucket. The destination is never overwritten on conflict.
+  fastify.post('/api/variables/:key/move', async (request, reply) => {
+    if (!verifyToken(request)) {
+      return reply.code(401).send({ error: 'Unauthorized' })
+    }
+    const { key } = request.params
+    const from = typeof request.body?.from_workspace_id === 'string' ? request.body.from_workspace_id : ''
+    const to = typeof request.body?.to_workspace_id === 'string' ? request.body.to_workspace_id : ''
+    const result = variableStore.moveVariable(from, to, key)
+    if (result.status === 'same_scope') {
+      return reply.code(400).send({ error: 'Source and destination scopes are the same.' })
+    }
+    if (result.status === 'not_found') {
+      return reply.code(404).send({ error: `Variable not found: ${key}` })
+    }
+    if (result.status === 'conflict') {
+      return reply.code(409).send({ error: `A variable with the same key already exists in the destination scope: ${key}` })
+    }
+    return { success: true, key, from_workspace_id: result.from, to_workspace_id: result.to }
+  })
+
   // ─── Secrets (sensitive, workspace-scoped) ────────────────────────────
   //
   // Secrets are scoped per workspace. Pass ?workspace_id=<uuid> to target a
@@ -118,11 +141,6 @@ function variableRoutes(fastify, _opts, done) {
   // Values are never returned via the API by design — only key names. Secrets
   // never leave the minion: the HQ proxy is a pure pass-through.
 
-  function readWorkspaceId(request) {
-    const rawWs = request.query?.workspace_id
-    return (typeof rawWs === 'string') ? rawWs : ''
-  }
-
   fastify.get('/api/secrets', async (request, reply) => {
     if (!verifyToken(request)) {
       return reply.code(401).send({ error: 'Unauthorized' })
@@ -175,6 +193,30 @@ function variableRoutes(fastify, _opts, done) {
     return { success: true, scopes: variableStore.listSecretScopes() }
   })
 
+  // Move a secret from one workspace scope to another. Body:
+  // { from_workspace_id, to_workspace_id } — omit/empty string targets the
+  // minion-wide bucket. The value is moved within the minion (it never leaves);
+  // the destination is never overwritten on conflict.
+  fastify.post('/api/secrets/:key/move', async (request, reply) => {
+    if (!verifyToken(request)) {
+      return reply.code(401).send({ error: 'Unauthorized' })
+    }
+    const { key } = request.params
+    const from = typeof request.body?.from_workspace_id === 'string' ? request.body.from_workspace_id : ''
+    const to = typeof request.body?.to_workspace_id === 'string' ? request.body.to_workspace_id : ''
+    const result = variableStore.moveSecret(from, to, key)
+    if (result.status === 'same_scope') {
+      return reply.code(400).send({ error: 'Source and destination scopes are the same.' })
+    }
+    if (result.status === 'not_found') {
+      return reply.code(404).send({ error: `Secret not found: ${key}` })
+    }
+    if (result.status === 'conflict') {
+      return reply.code(409).send({ error: `A secret with the same key already exists in the destination scope: ${key}` })
+    }
+    return { success: true, key, from_workspace_id: result.from, to_workspace_id: result.to }
+  })
+
   done()
 }
 

package/core/routes/web.js

@@ -19,6 +19,9 @@ const { extract } = require('../lib/web-extract')
 const pageRecipeStore = require('../stores/page-recipe-store')
 
 const REQUEST_TIMEOUT_MS = 60_000
+// Scrolling adds an extra navigation plus an in-page scroll loop, so give
+// scroll-enabled requests a wider ceiling than the plain extract path.
+const SCROLL_REQUEST_TIMEOUT_MS = 120_000
 
 async function webRoutes(fastify) {
   fastify.post('/api/web/extract', async (request, reply) => {
@@ -28,7 +31,7 @@ async function webRoutes(fastify) {
     }
 
     const body = request.body || {}
-    const { url, hint } = body
+    const { url, hint, scroll } = body
 
     if (!url || typeof url !== 'string') {
       reply.code(400)
@@ -40,11 +43,17 @@ async function webRoutes(fastify) {
       reply.code(400)
       return { success: false, error: 'url is not a valid URL' }
     }
+    if (scroll != null && typeof scroll !== 'object') {
+      reply.code(400)
+      return { success: false, error: 'scroll must be an object when provided' }
+    }
+
+    const requestTimeoutMs = scroll ? SCROLL_REQUEST_TIMEOUT_MS : REQUEST_TIMEOUT_MS
 
     try {
       const result = await Promise.race([
-        extract({ url, hint: typeof hint === 'string' ? hint : null }),
-        new Promise((_, rej) => setTimeout(() => rej(new Error('extract timeout')), REQUEST_TIMEOUT_MS)),
+        extract({ url, hint: typeof hint === 'string' ? hint : null, scroll: scroll || null }),
+        new Promise((_, rej) => setTimeout(() => rej(new Error('extract timeout')), requestTimeoutMs)),
       ])
       return { success: true, ...result }
     } catch (err) {

package/core/stores/chat-store.js

@@ -200,8 +200,18 @@ function rekeySession(oldSessionId, newSessionId) {
         .run(Date.now(), oldSession.turn_count || 0, newSessionId)
       db.prepare('DELETE FROM chat_sessions WHERE session_id = ?').run(oldSessionId)
     } else {
-      db.prepare('UPDATE chat_sessions SET session_id = ? WHERE session_id = ?').run(newSessionId, oldSessionId)
+      // Insert the new parent row first, repoint the children, then drop the
+      // old parent. Updating chat_sessions.session_id in place would orphan the
+      // existing chat_messages mid-statement — the FK is ON UPDATE NO ACTION, so
+      // SQLite raises "FOREIGN KEY constraint failed" before the follow-up
+      // UPDATE can repoint them. (The final DELETE cascades to nothing because
+      // the messages were already moved.)
+      db.prepare(
+        'INSERT INTO chat_sessions (session_id, workspace_id, turn_count, created_at, updated_at) ' +
+        'SELECT ?, workspace_id, turn_count, created_at, updated_at FROM chat_sessions WHERE session_id = ?'
+      ).run(newSessionId, oldSessionId)
       db.prepare('UPDATE chat_messages SET session_id = ? WHERE session_id = ?').run(newSessionId, oldSessionId)
+      db.prepare('DELETE FROM chat_sessions WHERE session_id = ?').run(oldSessionId)
     }
     return true
   })
@@ -240,4 +250,111 @@ function deleteSession(sessionId) {
   return result.changes > 0
 }
 
-module.exports = { load, loadById, listSessions, save, addMessage, rekeySession, clear, deleteSession }
+// ---------------------------------------------------------------------------
+// Chat runs — durable index for detached chat execution (see chat-run-manager).
+// A run is owned by the run manager, not the HTTP request. These rows let a
+// reconnecting client find the in-flight run for a workspace and resume tailing.
+// ---------------------------------------------------------------------------
+
+/**
+ * Record the start of a detached run.
+ * @param {{ runId: string, sessionId?: string|null, pendingSessionId?: string|null, workspaceId?: string|null }} run
+ */
+function createRun({ runId, sessionId, pendingSessionId, workspaceId }) {
+  const db = getDb()
+  db.prepare(
+    `INSERT INTO chat_runs (run_id, session_id, pending_session_id, workspace_id, status, started_at, last_seq)
+     VALUES (?, ?, ?, ?, 'running', ?, 0)`
+  ).run(runId, sessionId || null, pendingSessionId || null, workspaceId || null, Date.now())
+}
+
+/**
+ * Update a run's progress/terminal state. Only provided fields are written.
+ * @param {string} runId
+ * @param {{ status?: string, sessionId?: string|null, lastSeq?: number }} patch
+ */
+function updateRun(runId, { status, sessionId, lastSeq } = {}) {
+  const db = getDb()
+  const sets = []
+  const vals = []
+  if (status !== undefined) {
+    sets.push('status = ?')
+    vals.push(status)
+    if (status !== 'running') {
+      sets.push('ended_at = ?')
+      vals.push(Date.now())
+    }
+  }
+  if (sessionId !== undefined) {
+    sets.push('session_id = ?')
+    vals.push(sessionId)
+  }
+  if (typeof lastSeq === 'number') {
+    sets.push('last_seq = ?')
+    vals.push(lastSeq)
+  }
+  if (!sets.length) return
+  vals.push(runId)
+  db.prepare(`UPDATE chat_runs SET ${sets.join(', ')} WHERE run_id = ?`).run(...vals)
+}
+
+/**
+ * Most recent still-running run for a workspace (reconnect target).
+ * @param {string|null} workspaceId
+ * @returns {object|null}
+ */
+function getActiveRun(workspaceId) {
+  const db = getDb()
+  if (workspaceId) {
+    return db.prepare(
+      `SELECT * FROM chat_runs WHERE workspace_id = ? AND status = 'running' ORDER BY started_at DESC LIMIT 1`
+    ).get(workspaceId) || null
+  }
+  return db.prepare(
+    `SELECT * FROM chat_runs WHERE workspace_id IS NULL AND status = 'running' ORDER BY started_at DESC LIMIT 1`
+  ).get() || null
+}
+
+/**
+ * Load a run row by id.
+ * @param {string} runId
+ * @returns {object|null}
+ */
+function getRun(runId) {
+  const db = getDb()
+  return db.prepare('SELECT * FROM chat_runs WHERE run_id = ?').get(runId) || null
+}
+
+/**
+ * Boot-time sweep: any run still flagged `running` belongs to a previous server
+ * process whose in-memory owner is gone. Mark them interrupted so clients stop
+ * waiting on a stream that will never resume.
+ * @returns {number} rows swept
+ */
+function markRunningInterrupted() {
+  const db = getDb()
+  const res = db.prepare(
+    `UPDATE chat_runs SET status = 'interrupted', ended_at = ? WHERE status = 'running'`
+  ).run(Date.now())
+  return res.changes || 0
+}
+
+/**
+ * Prune run rows older than maxAgeMs (terminal states only). Keeps the table
+ * from growing unbounded; the NDJSON event logs are pruned separately.
+ * @param {number} maxAgeMs
+ * @returns {number} rows removed
+ */
+function pruneRuns(maxAgeMs) {
+  const db = getDb()
+  const cutoff = Date.now() - maxAgeMs
+  const res = db.prepare(
+    `DELETE FROM chat_runs WHERE status != 'running' AND COALESCE(ended_at, started_at) < ?`
+  ).run(cutoff)
+  return res.changes || 0
+}
+
+module.exports = {
+  load, loadById, listSessions, save, addMessage, rekeySession, clear, deleteSession,
+  createRun, updateRun, getActiveRun, getRun, markRunningInterrupted, pruneRuns,
+}

package/core/stores/page-recipe-store.js

@@ -12,7 +12,7 @@ const MAX_FAIL_COUNT = 3
 function find({ urlTemplate, domFingerprint }) {
   const db = getDb()
   const row = db.prepare(`
-    SELECT url_template, dom_fingerprint, selectors_json, page_type,
+    SELECT url_template, dom_fingerprint, selectors_json, page_type, ready_selector,
            hit_count, fail_count, last_verified_at, created_at
     FROM page_recipes
     WHERE url_template = ? AND dom_fingerprint = ?
@@ -24,7 +24,7 @@ function find({ urlTemplate, domFingerprint }) {
 function findByTemplate(urlTemplate) {
   const db = getDb()
   const rows = db.prepare(`
-    SELECT url_template, dom_fingerprint, selectors_json, page_type,
+    SELECT url_template, dom_fingerprint, selectors_json, page_type, ready_selector,
            hit_count, fail_count, last_verified_at, created_at
     FROM page_recipes
     WHERE url_template = ?
@@ -33,19 +33,20 @@ function findByTemplate(urlTemplate) {
   return rows.map(parseRow)
 }
 
-function upsert({ urlTemplate, domFingerprint, selectors, pageType }) {
+function upsert({ urlTemplate, domFingerprint, selectors, pageType, readySelector }) {
   const db = getDb()
   const json = JSON.stringify(selectors || {})
   const now = new Date().toISOString()
   db.prepare(`
-    INSERT INTO page_recipes (url_template, dom_fingerprint, selectors_json, page_type, hit_count, fail_count, last_verified_at, created_at)
-    VALUES (?, ?, ?, ?, 0, 0, ?, ?)
+    INSERT INTO page_recipes (url_template, dom_fingerprint, selectors_json, page_type, ready_selector, hit_count, fail_count, last_verified_at, created_at)
+    VALUES (?, ?, ?, ?, ?, 0, 0, ?, ?)
     ON CONFLICT(url_template, dom_fingerprint) DO UPDATE SET
       selectors_json = excluded.selectors_json,
       page_type = excluded.page_type,
+      ready_selector = excluded.ready_selector,
       fail_count = 0,
       last_verified_at = excluded.last_verified_at
-  `).run(urlTemplate, domFingerprint, json, pageType || null, now, now)
+  `).run(urlTemplate, domFingerprint, json, pageType || null, readySelector || null, now, now)
   return find({ urlTemplate, domFingerprint })
 }
 
@@ -102,7 +103,7 @@ function remove({ urlTemplate, domFingerprint }) {
 function listAll({ limit = 100 } = {}) {
   const db = getDb()
   const rows = db.prepare(`
-    SELECT url_template, dom_fingerprint, selectors_json, page_type,
+    SELECT url_template, dom_fingerprint, selectors_json, page_type, ready_selector,
            hit_count, fail_count, last_verified_at, created_at
     FROM page_recipes
     ORDER BY last_verified_at DESC NULLS LAST, created_at DESC
@@ -123,6 +124,7 @@ function parseRow(row) {
     dom_fingerprint: row.dom_fingerprint,
     selectors,
     page_type: row.page_type,
+    ready_selector: row.ready_selector || null,
     hit_count: row.hit_count,
     fail_count: row.fail_count,
     last_verified_at: row.last_verified_at,