halo-agent 1.1.0

package/orchestrator.js

@@ -0,0 +1,1122 @@
+'use strict';
+
+/**
+ * Core state machine for auto-applying to a single job.
+ * States: ANALYZING -> FILLING -> PAGINATING -> REVIEWING -> SUBMITTING -> DONE | NEEDS_HUMAN
+ *
+ * The orchestrator never opens a fake browser — it always uses the user's real Chrome
+ * via CDP (connectOverCDP), so bot detection is not triggered.
+ */
+
+const os = require('os');
+const path = require('path');
+const fs = require('fs');
+const { fillFields, uploadFile, findNextButton, findSubmitButton, waitForStableDOM, snapshotFieldLabels } = require('./filler');
+const { detectCaptcha, solveCaptcha, injectCaptchaToken } = require('./captcha');
+const { visionFill, visionNavigateAndSubmit, visionFillSkipped } = require('./vision');
+
+// ATS types that need vision fallback due to shadow DOM / canvas
+const VISION_ATS = new Set(['workday', 'icims', 'taleo', 'sap', 'successfactors']);
+
+/**
+ * Create a fresh FormContext for tracking state across a multi-page application.
+ * The context accumulates what was filled, what was skipped, and what conditional
+ * fields appeared as a result of previous answers.
+ */
+function createFormContext() {
+  return {
+    pagesVisited: [],         // [{pageIndex, url, fieldsFound, fieldsFilled}]
+    answeredFields: new Map(), // label -> {value, pageIndex, source}
+    skippedFields: [],         // [{label, reason}]
+    conditionalTriggers: [],   // [{triggerLabel, triggerValue, appearedLabel}]
+    currentPageIndex: 0,
+  };
+}
+
+/**
+ * Run the full apply flow for one queued job item.
+ *
+ * @param {object} queueItem - from GET /apply-queue/next
+ * @param {object} chromeConn - from browser.connectToChrome()
+ * @param {object} config - from ~/.halo-agent/config.json
+ * @param {function} reportStatus - async fn(status, extra) to call HALO backend
+ */
+async function runJob(queueItem, chromeConn, config, reportStatus) {
+  const { id: queueId, job_id: jobId, apply_url, ats_type } = queueItem;
+  const apiKey = config.captchaApiKey || null;
+  const anthropicKey = config.anthropicApiKey || process.env.ANTHROPIC_API_KEY || null;
+  const typingSpeed = config.typingSpeed || 'normal';
+  const useVision = config.useVisionFallback !== false;
+
+  let page = null;
+  let tempResumeFile = null;
+
+  // Initialize cross-page context tracker
+  const ctx = createFormContext();
+
+  try {
+    // STEP 1: ANALYZING
+    await reportStatus('IN_PROGRESS', { status: 'IN_PROGRESS' });
+    console.log(`[orchestrator] Analyzing: ${queueItem.company} - ${queueItem.title}`);
+
+    // Fetch the full AEP from HALO backend
+    const aep = await fetchAEP(config.apiUrl, config.token, jobId);
+    if (!aep) throw new Error('Could not fetch application packet from HALO backend');
+
+    // Download resume PDF to temp file if available
+    if (aep.recommended_resume?.pdf_presigned_url) {
+      tempResumeFile = await downloadResume(aep.recommended_resume.pdf_presigned_url);
+    }
+
+    // Check for an existing checkpoint — if a previous run got past page 1
+    // before crashing, resume from where it stopped. The fillFields path uses
+    // ctx.answeredFields to skip already-completed labels, so restoring the
+    // map is enough to make the next page fill skip its predecessors' fields.
+    const checkpoint = await fetchCheckpoint(config, queueId);
+    const resuming = checkpoint && checkpoint.page_index > 0 && checkpoint.last_url;
+
+    // Open new tab to the application URL (or the checkpoint URL when resuming)
+    const url = resuming ? checkpoint.last_url : (apply_url || aep.job.apply_url || aep.job.url);
+    if (!url) throw new Error('No application URL for this job');
+    page = await chromeConn.newPage(url);
+    await waitForStableDOM(page, 3000); // let page fully render
+
+    if (resuming) {
+      const restoredCount = Object.keys(checkpoint.fields_filled || {}).length;
+      console.log(`[orchestrator] Resuming from checkpoint: page ${checkpoint.page_index}, ${restoredCount} fields already filled`);
+      for (const [label, value] of Object.entries(checkpoint.fields_filled || {})) {
+        ctx.answeredFields.set(label, { value, pageIndex: checkpoint.page_index, source: 'checkpoint' });
+      }
+      ctx.currentPageIndex = checkpoint.page_index + 1;
+      await reportStatus('IN_PROGRESS', {
+        step: 'FILLING',
+        step_detail: `Resuming page ${checkpoint.page_index + 1} (${restoredCount} fields restored)`,
+        fields_filled: restoredCount,
+      });
+    }
+
+    // If we landed on a job description page (not a form), click Apply.
+    // May return a new page object if Apply opened a new tab.
+    page = await clickApplyIfNeeded(page);
+
+    // STEP 2: Vision gate detection — handles modals, resume prompts, login walls, preamble screens.
+    // Runs before captcha check so the form is actually visible before we try to fill it.
+    const gateResult = await handlePageGate(page, aep, tempResumeFile, config);
+    if (!gateResult.handled && gateResult.gateType === 'login') {
+      throw new Error('Login wall encountered — cannot auto-proceed. Please log in to this ATS in Chrome and re-queue the job.');
+    }
+
+    // STEP 4: CAPTCHA CHECK before filling
+    await handleCaptchaIfPresent(page, apiKey, queueId, reportStatus, config);
+
+    // STEP 5: FILLING
+    console.log(`[orchestrator] Filling fields...`);
+    const useVisionForThis = useVision && VISION_ATS.has((ats_type || '').toLowerCase());
+
+    let fillResult;
+    if (useVisionForThis && anthropicKey) {
+      console.log(`[orchestrator] Using vision fallback for ${ats_type}`);
+      fillResult = await visionFill(page, aep, anthropicKey, { alreadyFilled: ctx.answeredFields });
+    } else {
+      fillResult = await fillFields(page, aep, { speed: typingSpeed, ctx, ats: ats_type });
+    }
+
+    // Fetch AI answers for any fields that scanPage identified as needing AI
+    const needsAI = fillResult.needsAI || [];
+    if (needsAI.length > 0) {
+      console.log(`[orchestrator] Fetching AI answers for ${needsAI.length} custom field(s)...`);
+      const previouslyAnswered = Array.from(ctx.answeredFields.entries()).map(([label, data]) => ({ label, value: data.value }));
+      for (const f of needsAI) {
+        try {
+          const ans = await fetchFieldAnswer(config.apiUrl, config.token, {
+            job_id: jobId,
+            field_label: f.label,
+            field_type: f.inputType === 'textarea' || f.tag === 'textarea' ? 'textarea' : f.inputType || 'text',
+            previously_answered: previouslyAnswered,
+          });
+          if (ans?.value) {
+            aep.field_answers = aep.field_answers || [];
+            aep.field_answers.push({ field_id: f.label, label: f.label, value: ans.value, confidence: ans.confidence || 0.75, source: 'on-demand' });
+            previouslyAnswered.push({ label: f.label, value: ans.value });
+            console.log(`[orchestrator] Got answer for "${f.label}": ${String(ans.value).slice(0, 80)}...`);
+          }
+        } catch (e) {
+          console.warn(`[orchestrator] Could not fetch answer for "${f.label}": ${e.message}`);
+        }
+      }
+      // Second fill pass with newly fetched answers
+      const retryResult = await fillFields(page, aep, { speed: typingSpeed, ctx, ats: ats_type });
+      fillResult.filled += retryResult.filled;
+      fillResult.skipped = retryResult.skipped;
+      console.log(`[orchestrator] Retry fill: +${retryResult.filled} fields filled`);
+    }
+
+    // If DOM filling still missed fields, use vision as precision fill
+    if (!useVisionForThis && anthropicKey && (fillResult.skipped > 2 || fillResult.failed > 0)) {
+      console.log(`[orchestrator] DOM fill left ${fillResult.skipped} skipped / ${fillResult.failed} failed — running vision precision fill...`);
+      const precisionResult = await visionFillSkipped(page, aep, anthropicKey, ctx.answeredFields);
+      if (precisionResult.filled > 0) {
+        fillResult.filled += precisionResult.filled;
+        console.log(`[orchestrator] Vision precision fill recovered ${precisionResult.filled} fields`);
+      }
+    }
+
+    // Upload resume if we have a temp file
+    if (tempResumeFile) {
+      const uploaded = await uploadFile(page, 'input[type="file"], [data-testid*="resume"], button:has-text("Upload")', tempResumeFile);
+      if (uploaded) fillResult.filled = (fillResult.filled || 0) + 1;
+    }
+
+    await reportStatus('IN_PROGRESS', {
+      fields_filled: fillResult.filled || 0,
+      step: 'FILLING',
+      step_detail: `Page 1 — ${fillResult.filled || 0} fields filled`,
+    });
+    // Persist progress so a crash before pagination doesn't lose page-1 work
+    await saveCheckpoint(config, queueId, buildCheckpoint(ctx, page, ats_type));
+
+    // STEP 4: PAGINATION (multi-page forms)
+    let paginationAttempts = 0;
+    let cumulativeFilled = fillResult.filled || 0;
+
+    while (paginationAttempts < 10) {
+      await handleCaptchaIfPresent(page, apiKey, queueId, reportStatus, config);
+
+      const submitBtn = await findSubmitButton(page);
+      if (submitBtn) break; // reached review/submit page
+
+      const nextBtn = await findNextButton(page);
+      if (!nextBtn) {
+        // No next and no submit — wait for dynamic forms to settle
+        await waitForStableDOM(page, 3000);
+        const retrySubmit = await findSubmitButton(page);
+        if (retrySubmit) break;
+        // Check if URL changed (some ATS redirect to confirmation)
+        const currentUrl = page.url();
+        const looksLikeConfirmation = /thank|confirm|success|applied|submitted/i.test(currentUrl);
+        if (looksLikeConfirmation) {
+          await reportStatus('DONE', { fields_filled: cumulativeFilled });
+          await clearCheckpoint(config, queueId);
+          console.log(`[orchestrator] Detected confirmation page by URL: ${currentUrl}`);
+          return;
+        }
+        console.log(`[orchestrator] No next/submit button found — stopping pagination at attempt ${paginationAttempts}`);
+        break;
+      }
+
+      console.log(`[orchestrator] Navigating to next page (attempt ${paginationAttempts + 1})...`);
+
+      // Snapshot visible fields BEFORE clicking next (for conditional field detection)
+      const fieldsBefore = await snapshotFieldLabels(page);
+
+      await nextBtn.click();
+      await page.waitForLoadState('domcontentloaded').catch(() => {});
+      await waitForStableDOM(page, 3000); // smart wait — stops as soon as form stabilizes
+
+      // Snapshot AFTER navigation to detect newly appeared conditional fields
+      const fieldsAfter = await snapshotFieldLabels(page);
+      const newConditionalFields = [...fieldsAfter].filter(label => label && !fieldsBefore.has(label));
+
+      if (newConditionalFields.length > 0) {
+        console.log(`[orchestrator] Detected ${newConditionalFields.length} conditional field(s): ${newConditionalFields.slice(0, 3).join(', ')}`);
+
+        // Fetch answers for conditional fields from backend with cross-page context
+        const previouslyAnswered = Array.from(ctx.answeredFields.entries()).map(([label, data]) => ({
+          label,
+          value: data.value,
+        }));
+
+        for (const condLabel of newConditionalFields) {
+          // Check if it's already in the AEP answers (may have been pre-generated)
+          const existing = (aep.field_answers || []).find(fa =>
+            fa.label && fa.label.toLowerCase().trim() === condLabel.toLowerCase().trim()
+          );
+          if (existing) continue; // already have the answer in AEP, fillFields will handle it
+
+          // Fetch a fresh answer from backend with context
+          try {
+            const condAnswer = await fetchFieldAnswer(config.apiUrl, config.token, {
+              job_id: jobId,
+              field_label: condLabel,
+              field_type: 'input',
+              previously_answered: previouslyAnswered,
+            });
+
+            if (condAnswer?.value) {
+              // Inject into AEP so fillFields can pick it up
+              aep.field_answers = aep.field_answers || [];
+              aep.field_answers.push({
+                field_id: condLabel,
+                label: condLabel,
+                value: condAnswer.value,
+                confidence: condAnswer.confidence || 0.75,
+                source: condAnswer.source || 'conditional',
+              });
+
+              // Track conditional trigger
+              ctx.conditionalTriggers.push({
+                appeared_label: condLabel,
+                page_index: ctx.currentPageIndex,
+              });
+            }
+          } catch (e) {
+            console.warn(`[orchestrator] Could not fetch conditional answer for "${condLabel}": ${e.message}`);
+          }
+        }
+      }
+
+      ctx.currentPageIndex++;
+
+      // Fill any new fields that appeared on this page
+      const pageResult = await fillFields(page, aep, { speed: typingSpeed, ctx, ats: ats_type });
+      cumulativeFilled += pageResult.filled || 0;
+
+      // Fetch AI answers for custom fields scanPage identified on this page
+      const pageNeedsAI = pageResult.needsAI || [];
+      if (pageNeedsAI.length > 0) {
+        const prevAnswered = Array.from(ctx.answeredFields.entries()).map(([l, d]) => ({ label: l, value: d.value }));
+        for (const f of pageNeedsAI) {
+          try {
+            const ans = await fetchFieldAnswer(config.apiUrl, config.token, { job_id: jobId, field_label: f.label, field_type: f.inputType === 'textarea' || f.tag === 'textarea' ? 'textarea' : f.inputType || 'text', previously_answered: prevAnswered });
+            if (ans?.value) {
+              aep.field_answers = aep.field_answers || [];
+              aep.field_answers.push({ field_id: f.label, label: f.label, value: ans.value, confidence: ans.confidence || 0.75, source: 'on-demand' });
+              prevAnswered.push({ label: f.label, value: ans.value });
+            }
+          } catch {}
+        }
+        const retryPage = await fillFields(page, aep, { speed: typingSpeed, ctx, ats: ats_type });
+        cumulativeFilled += retryPage.filled || 0;
+      }
+
+      // Precision vision fill for missed fields on this page too
+      if (anthropicKey && !useVisionForThis && (pageResult.skipped > 2 || pageResult.failed > 0)) {
+        const precisionResult = await visionFillSkipped(page, aep, anthropicKey, ctx.answeredFields);
+        cumulativeFilled += precisionResult.filled || 0;
+      }
+
+      await reportStatus('IN_PROGRESS', {
+        fields_filled: cumulativeFilled,
+        step: 'PAGINATING',
+        step_detail: `Page ${paginationAttempts + 2} — ${cumulativeFilled} fields filled`,
+      });
+      // Persist after each page so a crash mid-flow only loses the current page,
+      // not the whole application. Backend caps at 32KB; ctx.answeredFields is
+      // a label→value map so the payload grows linearly with form complexity.
+      await saveCheckpoint(config, queueId, buildCheckpoint(ctx, page, ats_type));
+      paginationAttempts++;
+    }
+
+    // STEP 5: REVIEWING
+    let submitBtn = await findSubmitButton(page);
+
+    if (!submitBtn && anthropicKey) {
+      // DOM couldn't find submit — hand off to Claude vision to navigate to submit page
+      console.log(`[orchestrator] Submit button not found via DOM. Trying vision fallback...`);
+      const visionResult = await visionNavigateAndSubmit(page, aep, anthropicKey, {
+        autoSubmit: config.autoSubmit || false,
+        alreadyFilled: ctx.answeredFields,
+      });
+
+      if (visionResult.submitted) {
+        // Vision already submitted — we're done
+        const confirmShot = await page.screenshot({ type: 'jpeg', quality: 70 }).catch(() => null);
+        const confirmKey = confirmShot ? await uploadScreenshot(config, confirmShot, `confirm_${queueId}.jpg`) : null;
+        await reportStatus('DONE', {
+          confirmation_screenshot_r2_key: confirmKey || null,
+          fields_filled: cumulativeFilled,
+        });
+        await clearCheckpoint(config, queueId);
+        console.log(`[orchestrator] Done via vision: ${queueItem.company} - ${queueItem.title}`);
+        return;
+      }
+
+      // Vision navigated to review page but didn't submit — try DOM submit one more time
+      submitBtn = await findSubmitButton(page);
+    }
+
+    if (!submitBtn) {
+      const pageTitle = await page.title().catch(() => 'unknown');
+      const pageUrl = page.url();
+      throw new Error(`Could not find submit button. Page: "${pageTitle}" (${pageUrl})`);
+    }
+
+    console.log(`[orchestrator] Reached review/submit page. Waiting for confirmation...`);
+
+    // Take a screenshot of the review page
+    const reviewScreenshot = await page.screenshot({ type: 'jpeg', quality: 70 });
+    const reviewKey = await uploadScreenshot(config, reviewScreenshot, `review_${queueId}.jpg`);
+
+    await reportStatus('REVIEWING', {
+      review_screenshot_r2_key: reviewKey || null,
+      step: 'REVIEWING',
+      step_detail: `${cumulativeFilled} fields filled · awaiting your confirm`,
+    });
+
+    // Wait for user to confirm submission from dashboard
+    // OR auto-submit if config.autoSubmit is true
+    if (config.autoSubmit || aep.agent_config?.auto_submit) {
+      const timeout = (aep.agent_config?.review_timeout_seconds || 30) * 1000;
+      await page.waitForTimeout(timeout);
+    } else {
+      await waitForSubmitConfirmation(config, queueId);
+    }
+
+    // STEP 6: SUBMITTING
+    console.log(`[orchestrator] Submitting application...`);
+    await reportStatus('IN_PROGRESS', {
+      step: 'SUBMITTING',
+      step_detail: 'Clicking submit',
+      fields_filled: cumulativeFilled,
+    });
+    await submitBtn.click();
+
+    // Wait for confirmation page
+    try {
+      await page.waitForURL(/thank|confirm|success|applied|submitted/i, { timeout: 15000 });
+    } catch {
+      await page.waitForTimeout(3000); // fallback wait
+    }
+
+    const confirmScreenshot = await page.screenshot({ type: 'jpeg', quality: 70 });
+    const confirmKey = await uploadScreenshot(config, confirmScreenshot, `confirm_${queueId}.jpg`);
+
+    await reportStatus('DONE', {
+      confirmation_screenshot_r2_key: confirmKey || null,
+      fields_filled: cumulativeFilled,
+    });
+    await clearCheckpoint(config, queueId);
+
+    console.log(`[orchestrator] Done: ${queueItem.company} - ${queueItem.title}`);
+
+    // Post fill session data to backend for learning loop
+    await postFillSession(config, {
+      job_id: jobId,
+      ats_type: ats_type || 'unknown',
+      pages_visited: ctx.currentPageIndex + 1,
+      fields_filled: cumulativeFilled,
+      skipped_fields: ctx.skippedFields,
+      conditional_fields: ctx.conditionalTriggers,
+    }).catch(() => {}); // non-critical
+
+  } catch (err) {
+    console.error(`[orchestrator] Error:`, err.message);
+    // Classify the failure into one of the InterventionPanel types so the user
+    // sees the right instructions rather than a raw stack trace.
+    const msg = (err.message || '').toLowerCase();
+    let interventionType = 'generic';
+    if (/timeout|timed.out|deadline/.test(msg)) interventionType = 'timeout';
+    else if (/login|sign.?in|auth.?required|please.+log.?in/.test(msg)) interventionType = 'login_wall';
+    await reportStatus('NEEDS_ATTENTION', {
+      needs_attention_reason: err.message,
+      intervention_type: interventionType,
+      step: 'NEEDS_ATTENTION',
+      step_detail: err.message?.slice(0, 120),
+    }).catch(() => {});
+  } finally {
+    // Clean up temp resume file
+    if (tempResumeFile) {
+      try { fs.unlinkSync(tempResumeFile); } catch {}
+    }
+    // Close the tab (leave other tabs untouched)
+    if (page) {
+      try { await page.close(); } catch {}
+    }
+  }
+}
+
+async function handleCaptchaIfPresent(page, apiKey, queueId, reportStatus, config) {
+  const captcha = await detectCaptcha(page);
+  if (!captcha.detected) return;
+
+  console.log(`[orchestrator] CAPTCHA detected: ${captcha.type}`);
+
+  if (apiKey && captcha.sitekey) {
+    console.log(`[orchestrator] Attempting CapSolver...`);
+    const token = await solveCaptcha(captcha, apiKey);
+    if (token) {
+      await injectCaptchaToken(page, token, captcha.type);
+      await reportStatus('IN_PROGRESS', { captcha_triggered: 1, captcha_solved: 1 });
+      console.log(`[orchestrator] CAPTCHA solved.`);
+      return;
+    }
+  }
+
+  // Could not solve — pause for human
+  await reportStatus('NEEDS_ATTENTION', {
+    captcha_triggered: 1,
+    captcha_solved: 0,
+    needs_attention_reason: `CAPTCHA appeared (${captcha.type}). Please solve it in Chrome and click "Resume" in the dashboard.`,
+    intervention_type: 'captcha',
+    step: 'CAPTCHA',
+    step_detail: `${captcha.type} CAPTCHA — awaiting your solve`,
+  });
+
+  // Wait for human to resolve (poll submit-confirmed endpoint as a resume signal)
+  for (let i = 0; i < 150; i++) { // up to 5 minutes
+    await new Promise(r => setTimeout(r, 2000));
+    // Check if user clicked "Resume" in dashboard (sets KV flag)
+    try {
+      const flagRes = await fetch(`${config.apiUrl}/apply-queue/captcha-resolved/${queueId}`, {
+        headers: { Authorization: `Bearer ${config.token}` },
+      });
+      if (flagRes.ok) {
+        const flagData = await flagRes.json();
+        if (flagData.resolved) {
+          await reportStatus('IN_PROGRESS', { captcha_triggered: 1, captcha_solved: 1 });
+          return;
+        }
+      }
+    } catch {}
+    // Also re-check if CAPTCHA is gone from the page (human solved it directly)
+    const stillThere = await detectCaptcha(page);
+    if (!stillThere.detected) {
+      await reportStatus('IN_PROGRESS', { captcha_triggered: 1, captcha_solved: 1 });
+      return;
+    }
+  }
+
+  throw new Error('CAPTCHA not solved within timeout. Application paused.');
+}
+
+async function fetchAEP(apiUrl, token, jobId) {
+  try {
+    const res = await fetch(`${apiUrl}/smartfill/agent-packet/${jobId}`, {
+      headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' },
+    });
+    if (!res.ok) return null;
+    return res.json();
+  } catch {
+    return null;
+  }
+}
+
+// ── Checkpoint helpers ───────────────────────────────────────────────────────
+// Persist what the agent has already filled so a crashed/restarted agent can
+// resume from the last completed page. Saved JSON is small (label→value map
+// for the current job's pages), bounded by the backend at 32KB.
+async function fetchCheckpoint(config, queueId) {
+  try {
+    const res = await fetch(`${config.apiUrl}/agent/queue/${queueId}/checkpoint`, {
+      headers: { Authorization: `Bearer ${config.token}` },
+    });
+    if (!res.ok) return null;
+    const data = await res.json();
+    return data.checkpoint || null;
+  } catch { return null; }
+}
+
+async function saveCheckpoint(config, queueId, checkpoint) {
+  try {
+    await fetch(`${config.apiUrl}/agent/queue/${queueId}/checkpoint`, {
+      method: 'PATCH',
+      headers: { Authorization: `Bearer ${config.token}`, 'Content-Type': 'application/json' },
+      body: JSON.stringify(checkpoint),
+    });
+  } catch { /* fire-and-forget; non-critical */ }
+}
+
+async function clearCheckpoint(config, queueId) {
+  try {
+    await fetch(`${config.apiUrl}/agent/queue/${queueId}/checkpoint`, {
+      method: 'DELETE',
+      headers: { Authorization: `Bearer ${config.token}` },
+    });
+  } catch { /* fire-and-forget */ }
+}
+
+// Build a checkpoint snapshot from the current FormContext. Flattens
+// answeredFields (a Map of label -> {value, ...}) into label -> value.
+function buildCheckpoint(ctx, page, atsType) {
+  const fields_filled = {};
+  for (const [label, data] of ctx.answeredFields.entries()) {
+    if (data && typeof data.value === 'string') {
+      fields_filled[label] = data.value;
+    }
+  }
+  return {
+    page_index: ctx.currentPageIndex,
+    fields_filled,
+    pages_remaining: null,
+    ats_type: atsType || null,
+    last_url: page ? page.url() : '',
+    saved_at: new Date().toISOString(),
+  };
+}
+
+async function fetchFieldAnswer(apiUrl, token, body) {
+  try {
+    const res = await fetch(`${apiUrl}/smartfill/field-answer`, {
+      method: 'POST',
+      headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' },
+      body: JSON.stringify(body),
+    });
+    if (!res.ok) return null;
+    return res.json();
+  } catch {
+    return null;
+  }
+}
+
+async function postFillSession(config, data) {
+  try {
+    await fetch(`${config.apiUrl}/smartfill/session`, {
+      method: 'POST',
+      headers: { Authorization: `Bearer ${config.token}`, 'Content-Type': 'application/json' },
+      body: JSON.stringify(data),
+    });
+  } catch {}
+}
+
+async function downloadResume(presignedUrl) {
+  try {
+    const res = await fetch(presignedUrl);
+    if (!res.ok) return null;
+    const buffer = Buffer.from(await res.arrayBuffer());
+    const tmpPath = path.join(os.tmpdir(), `halo_resume_${Date.now()}.pdf`);
+    fs.writeFileSync(tmpPath, buffer);
+    return tmpPath;
+  } catch {
+    return null;
+  }
+}
+
+async function uploadScreenshot(config, screenshotBuffer, filename) {
+  // Upload to HALO backend which stores in R2
+  try {
+    const formData = new FormData();
+    const blob = new Blob([screenshotBuffer], { type: 'image/jpeg' });
+    formData.append('file', blob, filename);
+    const res = await fetch(`${config.apiUrl}/agent/screenshot`, {
+      method: 'POST',
+      headers: { Authorization: `Bearer ${config.token}` },
+      body: formData,
+    });
+    if (!res.ok) return null;
+    const data = await res.json();
+    return data.r2_key || null;
+  } catch {
+    return null; // screenshots are nice-to-have, not critical
+  }
+}
+
+/**
+ * Vision-based intermediary step handler.
+ *
+ * After each navigation, takes a screenshot and sends it to the HALO backend
+ * which runs it through Workers AI (@cf/meta/llama-3.2-11b-vision-instruct) to classify
+ * any "gate" blocking the actual form (modal, resume upload, login wall, preamble, EEOC).
+ *
+ * The AI returns {gate_type, action, button_text, confidence} and the agent executes
+ * the appropriate action: click a button, upload the resume, or skip.
+ * Loops up to 5 times to handle stacked gates (e.g. modal then resume prompt).
+ */
+async function handlePageGate(page, aep, resumeFile, config) {
+  // Re-run up to 4 times — some gates are stacked (modal then resume upload)
+  for (let pass = 0; pass < 5; pass++) {
+    await waitForStableDOM(page, 2000);
+
+    // Take screenshot and POST to HALO backend for vision-based gate classification
+    let gate;
+    try {
+      const screenshot = await page.screenshot({ type: 'jpeg', quality: 70 });
+      const base64 = screenshot.toString('base64');
+      const pageUrl = page.url();
+
+      const res = await fetch(`${config.apiUrl}/smartfill/detect-gate`, {
+        method: 'POST',
+        headers: {
+          Authorization: `Bearer ${config.token}`,
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({ screenshot_base64: base64, page_url: pageUrl }),
+      });
+
+      if (!res.ok) {
+        console.warn(`[gate] detect-gate returned ${res.status} — skipping gate check`);
+        break;
+      }
+      gate = await res.json();
+    } catch (e) {
+      console.warn(`[gate] Detection request failed: ${e.message} — skipping gate check`);
+      break;
+    }
+
+    console.log(`[gate] Pass ${pass + 1}: ${gate.gate_type} (confidence: ${(gate.confidence || 0).toFixed(2)}, action: ${gate.action})`);
+
+    // No gate or low confidence — we're on the actual form
+    if (gate.gate_type === 'none' || (gate.confidence || 0) < 0.55) break;
+
+    // Login wall — cannot auto-proceed
+    if (gate.gate_type === 'login') {
+      console.warn('[gate] Login wall detected — cannot auto-proceed');
+      return { handled: false, gateType: 'login' };
+    }
+
+    // Click a button (modal choice, preamble, EEOC acknowledge)
+    if (gate.action === 'click' && gate.button_text) {
+      console.log(`[gate] Clicking "${gate.button_text}"`);
+      const pattern = new RegExp(escapeRegex(gate.button_text), 'i');
+      const clicked =
+        await page.getByRole('button', { name: pattern }).first().click({ timeout: 3000 }).then(() => true).catch(() => false) ||
+        await page.getByRole('link', { name: pattern }).first().click({ timeout: 2000 }).then(() => true).catch(() => false) ||
+        await page.locator('button, a, [role="button"]').filter({ hasText: pattern }).first().click({ timeout: 2000 }).then(() => true).catch(() => false);
+
+      if (!clicked) {
+        console.warn(`[gate] Could not find/click "${gate.button_text}" — stopping gate loop`);
+        break;
+      }
+      await waitForStableDOM(page, 2500);
+      continue;
+    }
+
+    // Upload resume (Ashby autofill bar, Lever drop zone, etc.)
+    if (gate.action === 'upload_resume' && resumeFile) {
+      console.log('[gate] Uploading resume to gate prompt');
+      const uploaded = await uploadResumeToPage(page, resumeFile);
+      if (uploaded) {
+        console.log('[gate] Resume uploaded successfully');
+        await waitForStableDOM(page, 3000);
+        continue;
+      } else {
+        console.warn('[gate] Resume upload failed — continuing to form fill anyway');
+        break;
+      }
+    }
+
+    // Skip / unknown action — stop looping
+    break;
+  }
+
+  return { handled: true, gateType: null };
+}
+
+/**
+ * Upload a resume file to the current page.
+ * Tries direct setInputFiles on hidden inputs first, then file chooser intercept.
+ */
+async function uploadResumeToPage(page, filePath) {
+  try {
+    // Unhide all file inputs and set files on the first one
+    const allInputs = page.locator('input[type="file"]');
+    const count = await allInputs.count();
+    for (let i = 0; i < count; i++) {
+      await allInputs.nth(i).evaluate(el => {
+        el.style.display = 'block';
+        el.style.opacity = '1';
+        el.style.visibility = 'visible';
+        el.style.width = '1px';
+        el.style.height = '1px';
+        el.style.position = 'fixed';
+        el.style.top = '0';
+      }).catch(() => {});
+    }
+    if (count > 0) {
+      await allInputs.first().setInputFiles(filePath);
+      return true;
+    }
+  } catch {}
+
+  // Fallback: trigger file chooser by clicking an upload button/label
+  try {
+    const uploadBtns = page.locator('button, a, label, [role="button"]').filter({ hasText: /upload|attach|browse/i });
+    const btnCount = await uploadBtns.count();
+    if (btnCount > 0) {
+      const [chooser] = await Promise.all([
+        page.waitForEvent('filechooser', { timeout: 5000 }),
+        uploadBtns.first().click(),
+      ]);
+      await chooser.setFiles(filePath);
+      return true;
+    }
+  } catch {}
+
+  return false;
+}
+
+function escapeRegex(str) {
+  return (str || '').replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+/**
+ * Detect if we're on a job listing/description page instead of an application form.
+ * If so, find and click the Apply button to get to the actual form.
+ * Returns the page to use going forward (may be a new tab if Apply opens one).
+ */
+async function clickApplyIfNeeded(page) {
+  // Check if there's a visible form input — if yes, we're already on an application form
+  const hasFormFields = await page.evaluate(() => {
+    const inputs = document.querySelectorAll('input[type="text"], input[type="email"], input[type="tel"], textarea');
+    return inputs.length > 0;
+  });
+  if (hasFormFields) return page; // already on form
+
+  // Look for an Apply button on the listing page
+  const applySelectors = [
+    '[data-automation-id="jobPostingApplyButton"]', // Workday — most specific first
+    '[data-testid="apply-button"]',
+    '.apply-button',
+    '#apply-button',
+    'button:has-text("Apply Now")',
+    'a:has-text("Apply Now")',
+    'button:has-text("Apply")',
+    'a:has-text("Apply")',
+  ];
+
+  for (const sel of applySelectors) {
+    try {
+      const el = page.locator(sel).first();
+      if (await el.isVisible({ timeout: 1000 })) {
+        console.log(`[orchestrator] On listing page — clicking Apply button (selector: ${sel})...`);
+
+        // Listen for a popup (new tab) BEFORE clicking
+        const popupPromise = page.context().waitForEvent('page', { timeout: 8000 }).catch(() => null);
+        await el.click();
+
+        const urlBefore = page.url();
+
+        // Check if a new tab opened
+        const newTab = await popupPromise;
+        if (newTab) {
+          console.log(`[orchestrator] Apply opened new tab: ${newTab.url()}`);
+          await newTab.waitForLoadState('domcontentloaded', { timeout: 15000 }).catch(() => {});
+          await waitForStableDOM(newTab, 3000);
+          console.log(`[orchestrator] New tab settled at: ${newTab.url()}`);
+          return newTab; // caller must use this page going forward
+        }
+
+        // Same-tab navigation
+        await page.waitForLoadState('domcontentloaded', { timeout: 15000 }).catch(() => {});
+        await waitForStableDOM(page, 3000);
+        const urlAfter = page.url();
+        console.log(`[orchestrator] Post-click URL: ${urlAfter} (was: ${urlBefore})`);
+        return page;
+      }
+    } catch {}
+  }
+
+  // No apply button found — log what's on the page for diagnosis
+  const pageUrl = page.url();
+  const pageTitle = await page.title().catch(() => 'unknown');
+  console.log(`[orchestrator] No Apply button found on "${pageTitle}" (${pageUrl}) — assuming already on form`);
+  return page;
+}
+
+/**
+ * Find visible textarea/input fields on the page that have no answer in the AEP yet.
+ * These are custom essay questions the AEP packet didn't pre-generate.
+ */
+async function getUnmatchedFields(page, aep) {
+  const { extractPageFields } = require('./filler');
+  try {
+    const pageFields = await extractPageFields(page);
+    const answeredLabels = new Set((aep.field_answers || []).map(fa => (fa.label || '').toLowerCase().trim()));
+
+    // Only look at textarea fields and long-text inputs (essay questions)
+    const candidates = pageFields.filter(f => {
+      if (f.tag !== 'textarea' && !(f.tag === 'input' && f.type === 'text')) return false;
+      if (!f.label || f.label.length < 10) return false; // skip unlabeled / short labels
+      const norm = f.label.toLowerCase().trim();
+      // Skip if already answered
+      if (answeredLabels.has(norm)) return false;
+      // Skip common profile fields that the profile_fill map handles
+      const profileKeywords = ['first name', 'last name', 'email', 'phone', 'linkedin', 'github', 'address', 'city', 'zip', 'postal', 'salary', 'url', 'website'];
+      if (profileKeywords.some(k => norm.includes(k))) return false;
+      return true;
+    });
+
+    return candidates.map(f => f.label);
+  } catch {
+    return [];
+  }
+}
+
+async function waitForSubmitConfirmation(config, queueId) {
+  console.log(`[orchestrator] Waiting for user to confirm submission from dashboard...`);
+  for (let i = 0; i < 180; i++) { // up to 15 minutes
+    await new Promise(r => setTimeout(r, 5000));
+    try {
+      const res = await fetch(`${config.apiUrl}/apply-queue/submit-confirmed/${queueId}`, {
+        headers: { Authorization: `Bearer ${config.token}` },
+      });
+      if (!res.ok) continue;
+      const data = await res.json();
+      if (data.confirmed) return;
+    } catch {}
+  }
+  throw new Error('Submission confirmation timed out. Application paused at review stage.');
+}
+
+/**
+ * Extension-triggered fill (Architecture 4 handoff).
+ *
+ * Unlike runJob(), this does NOT open a new tab — the user is already on the
+ * application page. We attach to that tab via CDP, run the full fill pipeline,
+ * then wait for the extension's confirm signal before submitting.
+ *
+ * @param {object} opts
+ *   fillId          — unique ID for this fill session (used for logging)
+ *   apply_url       — URL of the page the user is on (used to find the tab)
+ *   job_id          — HALO job ID (for AEP fetch and field-answer calls)
+ *   ats_type        — detected ATS type from extension (e.g. 'greenhouse')
+ *   packet          — optional pre-generated packet from extension session storage
+ *   extConfig       — { apiUrl, token } from extension (may differ from agent config)
+ *   chromeConn      — existing CDP connection { browser, context, newPage }
+ *   reportFillStatus — fn(state, extra) writes to in-memory session map
+ *   waitForConfirm  — Promise that resolves when user clicks Submit in extension overlay
+ */
+async function runExtensionFill({
+  fillId, apply_url, job_id, ats_type, packet,
+  extConfig, chromeConn, reportFillStatus, waitForConfirm,
+}) {
+  // Load agent config to get credentials + settings
+  const { loadConfig } = require('./config');
+  const config = loadConfig();
+
+  // Prefer extension-supplied credentials (user may be on a different account)
+  const apiUrl  = (extConfig && extConfig.apiUrl)  || config.apiUrl;
+  const token   = (extConfig && extConfig.token)   || config.token;
+  const anthropicKey = config.anthropicApiKey || process.env.ANTHROPIC_API_KEY || null;
+  const typingSpeed  = config.typingSpeed || 'normal';
+  const useVision    = config.useVisionFallback !== false;
+
+  const log = (msg) => console.log(`[ext-fill:${fillId}] ${msg}`);
+
+  reportFillStatus('ANALYZING', { message: 'Finding active Chrome tab...' });
+  log('Starting extension-triggered fill');
+
+  // Find the active tab that matches apply_url — attach to it rather than opening a new one
+  let page = null;
+  try {
+    const ctx = chromeConn.browser.contexts()[0];
+    const pages = ctx.pages();
+    // Match by URL prefix — handles redirects and hash changes
+    const applyHost = new URL(apply_url).hostname;
+    page = pages.find(p => {
+      try { return new URL(p.url()).hostname === applyHost; } catch { return false; }
+    }) || pages[pages.length - 1]; // fallback: most-recently-opened tab
+    if (!page) throw new Error('No matching Chrome tab found — make sure the application page is open');
+  } catch (e) {
+    throw new Error('Could not attach to application tab: ' + e.message);
+  }
+
+  log(`Attached to tab: ${page.url()}`);
+  reportFillStatus('ANALYZING', { message: 'Fetching your application packet...' });
+
+  // Fetch AEP — prefer pre-built packet from extension, fall back to backend fetch
+  let aep = null;
+  if (job_id) {
+    aep = await fetchAEP(apiUrl, token, job_id);
+  }
+  if (!aep && packet) {
+    // Build a minimal AEP from the extension's session-stored packet
+    aep = {
+      profile_fill: packet.profile || {},
+      field_answers: packet._raw_answers || [],
+      cover_letter: packet.cover_letter || '',
+      sponsorship_answer: packet.sponsorship_answer || '',
+      relocation_answer: packet.relocation_answer || '',
+      salary_answer: packet.salary_answer || '',
+      job: { id: job_id || 'ext', title: '', company: '', apply_url },
+      agent_config: { auto_submit: false, review_timeout_seconds: 30, typing_speed: typingSpeed },
+    };
+    // Hydrate field_answers from short_answers_json if present
+    if (packet.short_answers_json) {
+      try {
+        const sa = JSON.parse(packet.short_answers_json);
+        aep.field_answers = sa.map((a, i) => ({
+          field_id: `sa_${i}`, label: a.question, value: a.answer, confidence: 0.85, source: 'packet',
+        }));
+      } catch {}
+    }
+  }
+  if (!aep) throw new Error('No application packet available — generate one from the HALO dashboard first');
+
+  // Download resume if available
+  let tempResumeFile = null;
+  if (aep.recommended_resume?.pdf_presigned_url) {
+    tempResumeFile = await downloadResume(aep.recommended_resume.pdf_presigned_url);
+  }
+
+  const ctx = createFormContext();
+  const useVisionForThis = useVision && VISION_ATS.has((ats_type || '').toLowerCase());
+
+  // Wait for page to settle (user may have just navigated here)
+  reportFillStatus('ANALYZING', { message: 'Waiting for page to load...' });
+  await waitForStableDOM(page, 2000);
+
+  // Gate handling — same as runJob
+  reportFillStatus('ANALYZING', { message: 'Checking for page gates...' });
+  await handlePageGate(page, aep, tempResumeFile, { apiUrl, token, ...config });
+
+  // CAPTCHA check
+  const dummyQueueId = fillId;
+  const dummyReportStatus = async () => {};
+  await handleCaptchaIfPresent(page, config.captchaApiKey || null, dummyQueueId, dummyReportStatus, config);
+
+  // FILLING
+  reportFillStatus('FILLING', { message: 'Filling form fields...' });
+  log('Starting field fill');
+
+  let fillResult;
+  if (useVisionForThis && anthropicKey) {
+    log(`Vision fill for ${ats_type}`);
+    fillResult = await visionFill(page, aep, anthropicKey, { alreadyFilled: ctx.answeredFields });
+  } else {
+    fillResult = await fillFields(page, aep, { speed: typingSpeed, ctx, ats: ats_type });
+  }
+
+  reportFillStatus('FILLING', { fieldsFilled: fillResult.filled || 0, message: `Filled ${fillResult.filled || 0} fields...` });
+
+  // AI answers for custom fields
+  const needsAI = fillResult.needsAI || [];
+  if (needsAI.length > 0) {
+    reportFillStatus('FILLING', { message: `Getting AI answers for ${needsAI.length} custom question(s)...` });
+    const prevAnswered = Array.from(ctx.answeredFields.entries()).map(([label, d]) => ({ label, value: d.value }));
+    for (const f of needsAI) {
+      try {
+        const ans = await fetchFieldAnswer(apiUrl, token, {
+          job_id: aep.job?.id || job_id,
+          field_label: f.label,
+          field_type: f.inputType === 'textarea' || f.tag === 'textarea' ? 'textarea' : f.inputType || 'text',
+          previously_answered: prevAnswered,
+        });
+        if (ans?.value) {
+          aep.field_answers = aep.field_answers || [];
+          aep.field_answers.push({ field_id: f.label, label: f.label, value: ans.value, confidence: ans.confidence || 0.75, source: 'on-demand' });
+          prevAnswered.push({ label: f.label, value: ans.value });
+        }
+      } catch (e) { log(`AI answer failed for "${f.label}": ${e.message}`); }
+    }
+    const retry = await fillFields(page, aep, { speed: typingSpeed, ctx, ats: ats_type });
+    fillResult.filled += retry.filled;
+    fillResult.skipped = retry.skipped;
+  }
+
+  // Vision precision fill for anything DOM missed
+  if (!useVisionForThis && anthropicKey && (fillResult.skipped > 2 || fillResult.failed > 0)) {
+    reportFillStatus('FILLING', { message: 'Running vision precision fill for missed fields...' });
+    const precision = await visionFillSkipped(page, aep, anthropicKey, ctx.answeredFields);
+    fillResult.filled += precision.filled || 0;
+  }
+
+  if (tempResumeFile) {
+    const uploaded = await uploadFile(page, 'input[type="file"], [data-testid*="resume"], button:has-text("Upload")', tempResumeFile);
+    if (uploaded) fillResult.filled = (fillResult.filled || 0) + 1;
+  }
+
+  let cumulativeFilled = fillResult.filled || 0;
+  reportFillStatus('FILLING', { fieldsFilled: cumulativeFilled, message: `Filled ${cumulativeFilled} fields. Checking for more pages...` });
+
+  // PAGINATION — same logic as runJob
+  let paginationAttempts = 0;
+  while (paginationAttempts < 10) {
+    await handleCaptchaIfPresent(page, config.captchaApiKey || null, dummyQueueId, dummyReportStatus, config);
+
+    const submitBtn = await findSubmitButton(page);
+    if (submitBtn) break;
+
+    const nextBtn = await findNextButton(page);
+    if (!nextBtn) {
+      await waitForStableDOM(page, 3000);
+      if (await findSubmitButton(page)) break;
+      const confUrl = /thank|confirm|success|applied|submitted/i.test(page.url());
+      if (confUrl) {
+        reportFillStatus('DONE', { fieldsFilled: cumulativeFilled, message: 'Application submitted.' });
+        return;
+      }
+      break;
+    }
+
+    reportFillStatus('FILLING', { message: `Moving to next page (${paginationAttempts + 1})...` });
+    const fieldsBefore = await snapshotFieldLabels(page);
+    await nextBtn.click();
+    await page.waitForLoadState('domcontentloaded').catch(() => {});
+    await waitForStableDOM(page, 3000);
+
+    const fieldsAfter = await snapshotFieldLabels(page);
+    const newFields = [...fieldsAfter].filter(l => l && !fieldsBefore.has(l));
+    if (newFields.length > 0) {
+      const prevAnswered = Array.from(ctx.answeredFields.entries()).map(([l, d]) => ({ label: l, value: d.value }));
+      for (const condLabel of newFields) {
+        const existing = (aep.field_answers || []).find(fa => fa.label?.toLowerCase().trim() === condLabel.toLowerCase().trim());
+        if (existing) continue;
+        try {
+          const condAns = await fetchFieldAnswer(apiUrl, token, { job_id: aep.job?.id || job_id, field_label: condLabel, field_type: 'input', previously_answered: prevAnswered });
+          if (condAns?.value) {
+            aep.field_answers = aep.field_answers || [];
+            aep.field_answers.push({ field_id: condLabel, label: condLabel, value: condAns.value, confidence: condAns.confidence || 0.75, source: 'conditional' });
+          }
+        } catch {}
+      }
+    }
+
+    ctx.currentPageIndex++;
+    const pageResult = await fillFields(page, aep, { speed: typingSpeed, ctx, ats: ats_type });
+    cumulativeFilled += pageResult.filled || 0;
+
+    if (anthropicKey && !useVisionForThis && (pageResult.skipped > 2 || pageResult.failed > 0)) {
+      const pr = await visionFillSkipped(page, aep, anthropicKey, ctx.answeredFields);
+      cumulativeFilled += pr.filled || 0;
+    }
+
+    reportFillStatus('FILLING', { fieldsFilled: cumulativeFilled, message: `Filled ${cumulativeFilled} fields total...` });
+    paginationAttempts++;
+  }
+
+  // REVIEWING — take screenshot and send back to extension overlay
+  reportFillStatus('REVIEWING', { fieldsFilled: cumulativeFilled, message: 'Review your application and click Submit.' });
+  log('Reached review stage');
+
+  const reviewShot = await page.screenshot({ type: 'jpeg', quality: 65 }).catch(() => null);
+  if (reviewShot) {
+    // Send as base64 so extension can display it inline — no R2 needed for extension flow
+    const b64 = reviewShot.toString('base64');
+    reportFillStatus('REVIEWING', {
+      fieldsFilled: cumulativeFilled,
+      message: 'Review your application and click Submit in the HALO panel.',
+      reviewScreenshot: `data:image/jpeg;base64,${b64}`,
+    });
+  }
+
+  // Wait for user to confirm from extension overlay
+  log('Waiting for user confirm from extension...');
+  await waitForConfirm();
+
+  // SUBMITTING
+  reportFillStatus('SUBMITTING', { fieldsFilled: cumulativeFilled, message: 'Submitting...' });
+  log('Submit confirmed — clicking submit button');
+
+  let submitBtn = await findSubmitButton(page);
+  if (!submitBtn && anthropicKey) {
+    const vr = await visionNavigateAndSubmit(page, aep, anthropicKey, { autoSubmit: true, alreadyFilled: ctx.answeredFields });
+    if (vr.submitted) {
+      reportFillStatus('DONE', { fieldsFilled: cumulativeFilled, message: 'Application submitted!' });
+      log('Done (via vision submit)');
+      if (tempResumeFile) try { require('fs').unlinkSync(tempResumeFile); } catch {}
+      return;
+    }
+    submitBtn = await findSubmitButton(page);
+  }
+  if (!submitBtn) throw new Error('Could not find submit button');
+
+  await submitBtn.click();
+  try { await page.waitForURL(/thank|confirm|success|applied|submitted/i, { timeout: 15000 }); } catch {}
+
+  reportFillStatus('DONE', { fieldsFilled: cumulativeFilled, message: 'Application submitted!' });
+  log('Done');
+
+  // Learning loop session post (non-critical)
+  await postFillSession({ apiUrl, token }, {
+    job_id: aep.job?.id || job_id || 'ext',
+    ats_type: ats_type || 'unknown',
+    pages_visited: ctx.currentPageIndex + 1,
+    fields_filled: cumulativeFilled,
+    skipped_fields: ctx.skippedFields,
+    conditional_fields: ctx.conditionalTriggers,
+  }).catch(() => {});
+
+  if (tempResumeFile) try { require('fs').unlinkSync(tempResumeFile); } catch {}
+}
+
+module.exports = { runJob, runExtensionFill };