halo-agent 1.1.0

package/vision.js

@@ -0,0 +1,398 @@
+'use strict';
+
+/**
+ * Claude computer-use vision driver for HALO agent.
+ *
+ * Two entry points:
+ *
+ * 1. visionNavigateAndSubmit(page, aep, apiKey)
+ *    Targeted use: DOM filling already ran, but submit button not found.
+ *    Claude looks at the current page state and completes the navigation + submission.
+ *    Used as a fallback after fillFields() + pagination have run.
+ *
+ * 2. visionFill(page, aep, apiKey)
+ *    Full fallback: fills all visible fields from scratch via vision.
+ *    Used for ATS types that completely defeat DOM automation (Workday canvas, Taleo).
+ *
+ * Both use a proper action loop:
+ *   screenshot -> Claude decides action -> execute in Playwright -> screenshot -> repeat
+ * until Claude signals done or max iterations reached.
+ */
+
+const CLAUDE_MODEL = 'claude-opus-4-6';
+const MAX_ITERATIONS = 40;
+const SCREENSHOT_QUALITY = 75;
+
+/**
+ * Execute a single computer_use action returned by Claude.
+ * Returns a fresh screenshot buffer after the action.
+ */
+async function executeAction(page, action) {
+  const { action: type, coordinate, text, key, scroll_direction, scroll_distance } = action;
+
+  switch (type) {
+    case 'screenshot':
+      // Claude is checking its work — just return fresh screenshot, no interaction
+      break;
+
+    case 'left_click':
+      await page.mouse.click(coordinate[0], coordinate[1]);
+      await page.waitForTimeout(400);
+      break;
+
+    case 'double_click':
+      await page.mouse.dblclick(coordinate[0], coordinate[1]);
+      await page.waitForTimeout(400);
+      break;
+
+    case 'right_click':
+      await page.mouse.click(coordinate[0], coordinate[1], { button: 'right' });
+      await page.waitForTimeout(300);
+      break;
+
+    case 'mouse_move':
+      await page.mouse.move(coordinate[0], coordinate[1]);
+      break;
+
+    case 'left_click_drag':
+      await page.mouse.move(coordinate[0], coordinate[1]);
+      await page.mouse.down();
+      if (action.end_coordinate) {
+        await page.mouse.move(action.end_coordinate[0], action.end_coordinate[1]);
+      }
+      await page.mouse.up();
+      break;
+
+    case 'type':
+      await page.keyboard.type(text, { delay: 25 });
+      break;
+
+    case 'key':
+      // Handle key combos like "ctrl+a"
+      if (key.includes('+')) {
+        const parts = key.split('+');
+        const modifiers = parts.slice(0, -1);
+        const mainKey = parts[parts.length - 1];
+        for (const mod of modifiers) await page.keyboard.down(mod);
+        await page.keyboard.press(mainKey);
+        for (const mod of modifiers.reverse()) await page.keyboard.up(mod);
+      } else {
+        await page.keyboard.press(key);
+      }
+      await page.waitForTimeout(200);
+      break;
+
+    case 'scroll':
+      await page.mouse.move(coordinate[0], coordinate[1]);
+      const delta = (scroll_distance || 3) * 100;
+      await page.mouse.wheel(0, scroll_direction === 'up' ? -delta : delta);
+      await page.waitForTimeout(300);
+      break;
+
+    default:
+      console.warn(`[vision] Unknown action type: ${type}`);
+  }
+
+  // Wait for any triggered navigation or DOM updates
+  await page.waitForLoadState('domcontentloaded', { timeout: 5000 }).catch(() => {});
+
+  // Return fresh screenshot
+  return page.screenshot({ type: 'jpeg', quality: SCREENSHOT_QUALITY });
+}
+
+/**
+ * Core action loop. Sends messages to Claude and executes returned actions
+ * until Claude stops issuing tool calls (signals completion).
+ */
+async function runActionLoop(page, systemPrompt, userPrompt, apiKey, maxIter = MAX_ITERATIONS) {
+  // Get the current viewport size so Claude knows the coordinate space
+  const viewport = page.viewportSize() || { width: 1280, height: 800 };
+
+  const screenshot = await page.screenshot({ type: 'jpeg', quality: SCREENSHOT_QUALITY });
+  const base64 = screenshot.toString('base64');
+
+  const messages = [
+    {
+      role: 'user',
+      content: [
+        {
+          type: 'image',
+          source: { type: 'base64', media_type: 'image/jpeg', data: base64 },
+        },
+        { type: 'text', text: userPrompt },
+      ],
+    },
+  ];
+
+  let iteration = 0;
+  let actionsExecuted = 0;
+
+  while (iteration < maxIter) {
+    iteration++;
+
+    let response;
+    try {
+      const res = await fetch('https://api.anthropic.com/v1/messages', {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          'x-api-key': apiKey,
+          'anthropic-version': '2023-06-01',
+          'anthropic-beta': 'computer-use-2024-10-22',
+        },
+        body: JSON.stringify({
+          model: CLAUDE_MODEL,
+          max_tokens: 2048,
+          system: systemPrompt,
+          tools: [{
+            type: 'computer_20241022',
+            name: 'computer',
+            display_width_px: viewport.width,
+            display_height_px: viewport.height,
+          }],
+          messages,
+        }),
+      });
+
+      if (!res.ok) {
+        const errText = await res.text();
+        console.error('[vision] Claude API error:', res.status, errText.slice(0, 200));
+        break;
+      }
+
+      response = await res.json();
+    } catch (e) {
+      console.error('[vision] Fetch error:', e.message);
+      break;
+    }
+
+    // Check stop reason
+    if (response.stop_reason === 'end_turn') {
+      console.log(`[vision] Claude signaled completion after ${actionsExecuted} actions`);
+      break;
+    }
+
+    // Find tool use blocks
+    const toolUseBlocks = (response.content || []).filter(b => b.type === 'tool_use' && b.name === 'computer');
+    if (toolUseBlocks.length === 0) {
+      console.log(`[vision] No more tool calls — done after ${actionsExecuted} actions`);
+      break;
+    }
+
+    // Add assistant message to history
+    messages.push({ role: 'assistant', content: response.content });
+
+    // Execute each action and collect results
+    const toolResults = [];
+    for (const block of toolUseBlocks) {
+      const action = block.input;
+      console.log(`[vision] Action: ${action.action}${action.coordinate ? ` at [${action.coordinate}]` : ''}${action.text ? ` "${action.text.slice(0, 40)}"` : ''}`);
+
+      let newScreenshot;
+      try {
+        newScreenshot = await executeAction(page, action);
+        actionsExecuted++;
+      } catch (e) {
+        console.warn(`[vision] Action execution failed: ${e.message}`);
+        newScreenshot = await page.screenshot({ type: 'jpeg', quality: SCREENSHOT_QUALITY }).catch(() => screenshot);
+      }
+
+      toolResults.push({
+        type: 'tool_result',
+        tool_use_id: block.id,
+        content: [{
+          type: 'image',
+          source: {
+            type: 'base64',
+            media_type: 'image/jpeg',
+            data: newScreenshot.toString('base64'),
+          },
+        }],
+      });
+    }
+
+    // Add tool results to conversation so Claude sees the outcome
+    messages.push({ role: 'user', content: toolResults });
+  }
+
+  return { actionsExecuted, iterations: iteration };
+}
+
+/**
+ * Precision fill: DOM filling already ran but left some fields unfilled.
+ * Claude looks only at the skipped fields and fills just those.
+ * Called when fillResult.skipped > 2 or fillResult.failed > 0.
+ */
+async function visionFillSkipped(page, aep, apiKey, alreadyFilledMap = new Map()) {
+  if (!apiKey) {
+    return { success: false, filled: 0 };
+  }
+
+  const alreadyFilledLabels = [...alreadyFilledMap.keys()].join(', ') || 'none';
+  console.log('[vision] Precision fill for skipped fields...');
+
+  const systemPrompt = `You are controlling a web browser to fill a job application form.
+Some fields were already filled by an automated DOM filler. Your job is to fill only the REMAINING empty fields.
+
+Rules:
+- DO NOT re-fill fields that are already filled — they are listed below as "already filled"
+- Fill only the visible fields that appear empty
+- Click a field before typing, use clear+type pattern
+- For dropdowns: click to open, wait for options, click the correct option
+- For checkboxes/radio buttons: click to select the correct option
+- Do NOT click Next, Continue, or Submit
+- Work top to bottom, left to right
+- If all fields appear filled, take a screenshot and stop
+
+Already filled fields (DO NOT touch these): ${alreadyFilledLabels}`;
+
+  const answers = buildAnswerSummary(aep);
+  const userPrompt = `Fill the remaining empty fields with this candidate data:
+
+${answers}
+
+Skip any fields that are already filled. Focus only on empty fields.`;
+
+  try {
+    const result = await runActionLoop(page, systemPrompt, userPrompt, apiKey, 20);
+    return { success: true, filled: Math.max(0, Math.floor(result.actionsExecuted / 2)) };
+  } catch (e) {
+    console.error('[vision] Precision fill error:', e.message);
+    return { success: false, filled: 0 };
+  }
+}
+
+/**
+ * Targeted fallback: DOM filling already ran.
+ * Claude's job is just to find and click through to the submit confirmation.
+ * Used when findSubmitButton() returns null after full pagination.
+ */
+async function visionNavigateAndSubmit(page, aep, apiKey, { autoSubmit = false, alreadyFilled = new Map() } = {}) {
+  if (!apiKey) {
+    console.warn('[vision] No API key — vision fallback skipped');
+    return { success: false, reason: 'No API key' };
+  }
+
+  console.log('[vision] Starting targeted navigation loop to find and submit...');
+
+  const alreadyFilledLabels = [...alreadyFilled.keys()].join(', ') || 'none';
+
+  const systemPrompt = `You are controlling a web browser to complete a job application form that has already been partially filled.
+Your goal is to navigate to the final submission page and, if auto-submit is enabled, click the submit button.
+
+Rules:
+- Click Next/Continue buttons to advance through multi-page forms
+- If you see a review page with all filled answers, that is the target state — take a screenshot and stop
+- Only click Submit if autoSubmit is true (it is currently: ${autoSubmit})
+- If the page requires login or shows an error, stop and describe the issue
+- Do NOT re-fill fields that are already filled (already filled: ${alreadyFilledLabels})
+- Do NOT close the browser window or tab`;
+
+  const profile = aep?.profile_fill || {};
+  const userPrompt = `The form is already filled with data for ${profile.first_name} ${profile.last_name} applying to ${aep?.job?.company || 'this company'}.
+
+Navigate to the final submit/review page:
+1. If you see a Next or Continue button, click it
+2. If you see a review page summarizing the application, stop (this is the target state)
+3. If you see a Submit button and autoSubmit=${autoSubmit}, click it
+4. If the form is already on the submit page and ${autoSubmit} is false, just confirm you can see the submit button
+
+Current page: ${page.url()}`;
+
+  try {
+    const result = await runActionLoop(page, systemPrompt, userPrompt, apiKey, 20);
+    console.log(`[vision] Navigation complete. Actions: ${result.actionsExecuted}`);
+
+    // Check if we landed on a confirmation/thank-you page
+    const finalUrl = page.url();
+    const pageText = await page.textContent('body').catch(() => '');
+    const isDone = /thank you|application submitted|application received|we.ve received|confirmation/i.test(pageText)
+      || /thank|confirm|success|applied|submitted/i.test(finalUrl);
+
+    return { success: true, submitted: isDone, actionsExecuted: result.actionsExecuted };
+  } catch (e) {
+    console.error('[vision] Navigation loop error:', e.message);
+    return { success: false, reason: e.message };
+  }
+}
+
+/**
+ * Full vision fill: Claude fills the entire form from scratch.
+ * Last resort for ATS types that defeat all DOM automation.
+ */
+async function visionFill(page, aep, apiKey, { alreadyFilled = new Map() } = {}) {
+  if (!apiKey) {
+    console.warn('[vision] No API key — vision fallback skipped');
+    return { success: false, filled: 0 };
+  }
+
+  console.log('[vision] Starting full vision fill...');
+
+  const alreadyFilledLabels = [...alreadyFilled.keys()].join(', ') || 'none';
+
+  const systemPrompt = `You are controlling a web browser to fill out a job application form.
+Fill every visible input field, textarea, and dropdown using the candidate data provided.
+
+Rules:
+- Click a field before typing into it
+- For dropdowns: click to open, wait for options, click the correct option
+- For checkboxes/radio buttons: click to select the correct option
+- Do NOT click Next, Continue, or Submit — only fill visible fields
+- Skip fields that are already correctly filled: ${alreadyFilledLabels}
+- If a field has no matching data, leave it empty
+- Work systematically top to bottom, left to right`;
+
+  const answers = buildAnswerSummary(aep);
+  const userPrompt = `Fill all visible form fields with this candidate data:
+
+${answers}
+
+Fill each visible input field on the current page. Do not advance to the next page.`;
+
+  try {
+    const result = await runActionLoop(page, systemPrompt, userPrompt, apiKey, MAX_ITERATIONS);
+    return { success: true, filled: result.actionsExecuted };
+  } catch (e) {
+    console.error('[vision] Full fill error:', e.message);
+    return { success: false, filled: 0, reason: e.message };
+  }
+}
+
+function buildAnswerSummary(aep) {
+  const p = aep?.profile_fill || {};
+  const lines = [
+    `Full name: ${p.first_name} ${p.last_name}`,
+    `Email: ${p.email}`,
+    `Phone: ${p.phone}`,
+    p.linkedin && `LinkedIn: ${p.linkedin}`,
+    p.github && `GitHub: ${p.github}`,
+    p.portfolio && `Portfolio/Website: ${p.portfolio}`,
+    p.school && `Education: ${p.school}`,
+    p.gpa && `GPA: ${p.gpa}`,
+    p.sponsorship_text && `Visa/Sponsorship: ${p.sponsorship_text}`,
+    p.relocation_text && `Relocation: ${p.relocation_text}`,
+    p.salary && `Salary expectation: ${p.salary}`,
+    p.gender && `Gender: ${p.gender}`,
+    p.race && `Race/Ethnicity: ${p.race}`,
+    p.veteran && `Veteran status: ${p.veteran}`,
+    p.disability && `Disability: ${p.disability}`,
+  ].filter(Boolean);
+
+  if (aep?.cover_letter) {
+    lines.push(`\nCover Letter:\n${aep.cover_letter.slice(0, 600)}...`);
+  }
+
+  const fieldAnswers = (aep?.field_answers || []).slice(0, 10);
+  if (fieldAnswers.length > 0) {
+    lines.push('\nApplication-specific answers:');
+    for (const fa of fieldAnswers) {
+      if (fa.label && fa.value) {
+        lines.push(`  "${fa.label}": ${fa.value.slice(0, 300)}`);
+      }
+    }
+  }
+
+  return lines.join('\n');
+}
+
+module.exports = { visionFill, visionNavigateAndSubmit, visionFillSkipped };