jerob 1.0.0

package/plan/browser-agent/USAGE.md

@@ -0,0 +1,308 @@
+# Browser Agent System - Quick Start Guide
+
+## Overview
+
+The Browser Agent is an intelligent automation system that:
+- 🎯 Understands your goal
+- 📋 Plans automation steps
+- 🤖 Executes browser actions
+- ✅ Evaluates results
+- 🔄 Refines automatically (up to 5 iterations)
+
+## Quick Start
+
+### 1. Launch Browser Agent
+```bash
+jimmy jet
+# Select "Browser Agent" from menu
+```
+
+### 2. Enter Your Query
+```
+Enter query: Find the top 5 AI jobs on LinkedIn and extract their descriptions
+```
+
+### 3. Watch the Magic
+The system will:
+1. **Iteration 1**: Plan → Execute → Evaluate → Score
+2. **Iteration 2** (if needed): Refine plan → Execute → Evaluate → Score
+3. **Iteration 3+** (if needed): Continue refining...
+
+### 4. Review Results
+- Final score and completion status
+- Extracted data (if any)
+- Iteration logs
+- Option to save to JSON file
+
+## Example Queries
+
+### Data Extraction
+```
+"Extract all product names and prices from Amazon search results for laptops"
+"Get emails and phone numbers from the company website"
+"List all job titles and salary ranges from the careers page"
+```
+
+### Navigation & Interaction
+```
+"Sign up for newsletter with email: test@example.com"
+"Search for flights from NYC to LA and sort by price"
+"Add first 5 items to shopping cart and proceed to checkout"
+```
+
+### Information Gathering
+```
+"Find the latest 10 blog posts and extract their titles and dates"
+"Get contact information for all team members on the about page"
+"List all upcoming webinars with dates and registration links"
+```
+
+### Complex Tasks
+```
+"Compare prices of iPhone 15 across 3 different retailers"
+"Find user reviews for product XYZ and extract ratings"
+"Automate login and export order history"
+```
+
+## Understanding the Iteration Process
+
+### Iteration Feedback Loop
+
+```
+Query: "Find top 3 tech jobs with full descriptions"
+
+┌─ Iteration 1 ─────────────────────────────────────┐
+│ Plan:  Navigate → Search → Observe                │
+│ Score: 60/100 ❌ (Incomplete data)                 │
+│ Issues: Found 3 jobs but descriptions cut off     │
+└────────────────────────────────────────────────────┘
+                      │
+                      ▼ (Feedback: Fix description extraction)
+┌─ Iteration 2 ─────────────────────────────────────┐
+│ Plan:  Navigate → Search → Click → Extract         │
+│ Score: 85/100 ⚠️ (Mostly complete)                 │
+│ Issues: 2 jobs have full description, 1 missing   │
+└────────────────────────────────────────────────────┘
+                      │
+                      ▼ (Feedback: Get missing job details)
+┌─ Iteration 3 ─────────────────────────────────────┐
+│ Plan:  Navigate → Search → Extract All → Verify   │
+│ Score: 98/100 ✅ (Complete!)                      │
+│ Status: SATISFIED - All 3 jobs with descriptions  │
+└────────────────────────────────────────────────────┘
+```
+
+## Reading the Results
+
+### Console Output Example
+```
+═════════════════════════════════════════
+📊 Browser Agent Execution Summary
+═════════════════════════════════════════
+
+Query: Find top 3 AI jobs with descriptions
+Status: ✓ Succeeded
+Total Iterations: 3/5
+Final Score: 98/100
+Completeness: 99%
+Accuracy: 97%
+
+📦 Extracted Data:
+[
+  {
+    "title": "AI Engineer",
+    "company": "Tech Corp",
+    "salary": "$150k-$200k",
+    "description": "..."
+  },
+  ...
+]
+```
+
+### Saved JSON Structure
+```json
+{
+  "success": true,
+  "query": "Find top 3 AI jobs with descriptions",
+  "finalData": { ... },
+  "iterations": [
+    {
+      "iteration": 1,
+      "plan": { "goal": "...", "steps": [...] },
+      "execution": [
+        { "success": true, "action": "navigate", ... },
+        { "success": true, "action": "observe", ... },
+        { "success": false, "action": "extract", "error": "..." }
+      ],
+      "evaluation": {
+        "satisfied": false,
+        "score": 60,
+        "feedback": "...",
+        "completeness": 50,
+        "accuracy": 70,
+        "issues": ["..."]
+      },
+      "shouldContinue": true
+    },
+    { "iteration": 2, ... },
+    { "iteration": 3, ... }
+  ],
+  "totalIterations": 3,
+  "completedAt": "2026-06-02T10:30:45.123Z"
+}
+```
+
+## Supported Actions
+
+### Navigation
+- `navigate` - Go to URL
+- `scroll` - Scroll up/down/left/right
+
+### Interaction
+- `click` - Click on elements
+- `type` - Enter text
+
+### Extraction
+- `extract` - Get structured data with custom schema
+- `observe` - Analyze available actions on page
+
+### Timing
+- `wait` - Wait for specified milliseconds
+
+## Evaluation Metrics
+
+**Score (0-100)**
+- 0-30: Task not started/failed
+- 31-60: Partial progress
+- 61-80: Mostly complete
+- 81-100: Complete/satisfied
+
+**Completeness (0-100)**
+- Percentage of required data obtained
+
+**Accuracy (0-100)**
+- Correctness of extracted data
+
+**Satisfaction**
+- Default threshold: 80/100
+- Task marked as satisfied when score ≥ threshold
+
+## Tips for Better Results
+
+### 1. **Be Specific**
+```
+❌ "Find jobs"
+✅ "Find senior software engineer jobs in SF with salary > $200k"
+```
+
+### 2. **Include Context**
+```
+❌ "Extract prices"
+✅ "Extract iPhone 15 prices from Apple, Amazon, and Best Buy"
+```
+
+### 3. **Define Output Format**
+```
+❌ "Get company info"
+✅ "Extract company name, founded year, and employee count"
+```
+
+### 4. **Set Clear Success Criteria**
+```
+❌ "Search for flights"
+✅ "Find cheapest round-trip flights from NYC to LA under $300 for next week"
+```
+
+## Troubleshooting
+
+### Query Too Vague
+**Error**: Multiple iterations with low scores
+**Solution**: Be more specific about what you need
+
+### Navigation Fails
+**Error**: First iteration fails on navigate step
+**Solution**: Verify URL is correct and accessible
+
+### Data Not Extracting
+**Error**: Extract steps succeed but return empty data
+**Solution**: Try with screenshot tool first to see page structure
+
+### Browser Not Opening
+**Error**: Connection error on init
+**Solution**: Ensure Chrome is installed and `GOOGLE_GENERATIVE_AI_API_KEY` is set
+
+## System Configuration
+
+Edit `plan/browser-agent/orchestrator.ts`:
+
+```typescript
+const DEFAULT_CONFIG = {
+  maxIterations: 5,              // Increase for complex tasks
+  timeout: 120000,               // 2 minutes per execution
+  model: "google/gemini-3.1-flash-lite-preview",
+  evaluationThreshold: 80,       // Lower = accept sooner
+};
+```
+
+## Environment Setup
+
+**Required:**
+```bash
+export GOOGLE_GENERATIVE_AI_API_KEY="your-api-key-here"
+```
+
+**Optional:**
+```bash
+export DEBUG=true  # Enable verbose logging
+```
+
+## Integration Notes
+
+✅ Works alongside existing modes:
+- Agent Mode - File/folder operations
+- Plan Mode - Task planning
+- Ask Mode - General questions
+- Browser Agent Mode - Web automation (NEW)
+
+✅ No conflicts or breaking changes
+
+## Performance Tips
+
+1. **Reduce maxIterations** for quick results (default: 5)
+2. **Increase timeout** for slower sites (default: 120s)
+3. **Lower threshold** to accept "good enough" results faster
+4. **Use variables** for repeated values (email, password, etc.)
+
+## Advanced Usage
+
+### Manual Plan (Future Feature)
+```typescript
+const manualPlan = {
+  goal: "Extract pricing info",
+  steps: [
+    { id: 1, action: "navigate", value: "https://example.com" },
+    { id: 2, action: "click", value: "Search for laptops" },
+    { id: 3, action: "extract", description: "Get name and price" }
+  ]
+};
+
+const result = await executeBrowserPlan(manualPlan);
+```
+
+### Direct Executor Usage
+```typescript
+import { runBrowserAgentMode } from './plan/browser-agent';
+
+await runBrowserAgentMode();
+```
+
+## Support & Issues
+
+For issues or feature requests, check:
+1. Browser console for detailed error messages
+2. Saved JSON result files for iteration logs
+3. TypeScript types in `plan/browser-agent/types.ts`
+
+---
+
+**Happy automating!** 🚀

package/plan/browser-agent/evaluator.ts

@@ -0,0 +1,353 @@
+import { getAgentModel2 } from "../../config/ai.config";
+import type { EvaluationResult, ExecutionResult, BrowserPlan } from "./types";
+import { generateText } from "ai";
+import { z } from "zod";
+import { withLLMRetry } from "../../utils/llm-error";
+
+const NUMBER_WORDS: Record<string, number> = {
+  zero: 0,
+  one: 1,
+  two: 2,
+  three: 3,
+  four: 4,
+  five: 5,
+  six: 6,
+  seven: 7,
+  eight: 8,
+  nine: 9,
+  ten: 10,
+  eleven: 11,
+  twelve: 12,
+  thirteen: 13,
+  fourteen: 14,
+  fifteen: 15,
+  sixteen: 16,
+  seventeen: 17,
+  eighteen: 18,
+  nineteen: 19,
+  twenty: 20,
+  thirty: 30,
+  forty: 40,
+  fifty: 50,
+  sixty: 60,
+  seventy: 70,
+  eighty: 80,
+  ninety: 90,
+  hundred: 100,
+};
+
+function parseNumberLike(value: unknown): number | null {
+  if (typeof value === "number") {
+    return value;
+  }
+
+  if (typeof value !== "string") {
+    return null;
+  }
+
+  const normalized = value.trim().toLowerCase();
+  if (normalized.length === 0) {
+    return null;
+  }
+
+  const numeric = Number(normalized.replace(/,/g, ""));
+  if (!Number.isNaN(numeric)) {
+    return numeric;
+  }
+
+  const parts = normalized.replace(/-/g, " ").split(/\s+/);
+  let total = 0;
+  let valid = false;
+
+  for (const part of parts) {
+    if (NUMBER_WORDS[part] === undefined) {
+      return null;
+    }
+
+    const value = NUMBER_WORDS[part];
+    if (value === 100 && total !== 0) {
+      total *= value;
+    } else {
+      total += value;
+    }
+    valid = true;
+  }
+
+  return valid ? total : null;
+}
+
+const booleanLike = z.preprocess((value) => {
+  if (typeof value === "boolean") {
+    return value;
+  }
+  if (typeof value === "string") {
+    const normalized = value.trim().toLowerCase();
+    if (["true", "yes", "y"].includes(normalized)) {
+      return true;
+    }
+    if (["false", "no", "n"].includes(normalized)) {
+      return false;
+    }
+  }
+  return value;
+}, z.boolean());
+
+const percentageLike = z.preprocess((value) => {
+  const num = parseNumberLike(value);
+  return num;
+}, z.number().min(0).max(100));
+
+const issuesLike = z.preprocess((value) => {
+  if (typeof value === "string") {
+    const trimmed = value.trim();
+    if (trimmed.length === 0) {
+      return [];
+    }
+    return [trimmed];
+  }
+  return value;
+}, z.array(z.string()));
+
+const EvaluationSchema = z.object({
+  satisfied: booleanLike,
+  score: percentageLike,
+  feedback: z.string(),
+  completeness: percentageLike,
+  accuracy: percentageLike,
+  issues: issuesLike,
+}).strict();
+
+export async function evaluateExecutionResults(
+  query: string,
+  plan: BrowserPlan,
+  results: ExecutionResult[],
+  evaluationThreshold: number = 80
+): Promise<EvaluationResult> {
+  const model = getAgentModel2();
+
+  const successfulResults = results.filter((r) => r.success);
+  const failedSteps = results
+    .filter((r) => !r.success)
+    .map((r) => `Step ${r.stepNumber} (${r.action}): ${r.error}`);
+
+  // Check if this is an agent-based result (single entry with agentOutput)
+  const agentResult = results[0];
+  const agentOutput = agentResult?.agentOutput ?? "";
+  const agentData = agentResult?.data
+    ? JSON.stringify(agentResult.data, null, 2)
+    : "";
+
+  const executionSummary = `
+Plan Goal: ${plan.goal}
+Query: ${query}
+Agent Completed: ${agentResult?.success ? "Yes" : "No"}
+${agentResult?.error ? `Error: ${agentResult.error}` : ""}
+
+Agent Output:
+${agentOutput || "(no text output)"}
+
+${agentData ? `Extracted Data:\n${agentData}` : ""}
+
+Execution Details:
+${results.map((r) => `- Step ${r.stepNumber} (${r.action}): ${r.message}`).join("\n")}
+`;
+
+  const systemPrompt = `You are an evaluator for browser automation tasks. Assess whether the execution successfully completed the requested task.
+
+Respond with ONLY valid JSON (no markdown, no code blocks):
+{
+  "satisfied": boolean,
+  "score": number (0-100),
+  "feedback": "string",
+  "completeness": number (0-100),
+  "accuracy": number (0-100),
+  "issues": ["issue1", "issue2"]
+}`;
+
+    const userPrompt = `Evaluate this browser automation execution:
+
+  ${executionSummary}
+
+  Original Query: "${query}"`;
+
+  const response = await withLLMRetry(
+    () => generateText({
+      model,
+      system: systemPrompt,
+      prompt: userPrompt,
+      temperature: 0.3,
+    }),
+    { maxRetries: 2, context: "Evaluator" }
+  );
+
+  const extractJsonObjectFromText = (text: string): string | null => {
+    let jsonText = text.trim();
+
+    if (jsonText.startsWith("```json")) {
+      jsonText = jsonText.slice(7);
+    }
+    if (jsonText.startsWith("```")) {
+      jsonText = jsonText.slice(3);
+    }
+    if (jsonText.endsWith("```")) {
+      jsonText = jsonText.slice(0, -3);
+    }
+
+    jsonText = jsonText.trim();
+
+    const firstBraceIndex = jsonText.indexOf("{");
+    if (firstBraceIndex === -1) {
+      return null;
+    }
+
+    let depth = 0;
+    for (let i = firstBraceIndex; i < jsonText.length; i += 1) {
+      const char = jsonText[i];
+      if (char === "{") {
+        depth += 1;
+      } else if (char === "}") {
+        depth -= 1;
+        if (depth === 0) {
+          return jsonText.slice(firstBraceIndex, i + 1);
+        }
+      }
+    }
+
+    return null;
+  };
+
+  const normalizeLooseString = (value: string): string => {
+    let normalized = value.trim();
+    if ((normalized.startsWith('"') && normalized.endsWith('"')) || (normalized.startsWith("'") && normalized.endsWith("'"))) {
+      normalized = normalized.slice(1, -1);
+    }
+    return normalized.replace(/\s+/g, " ").trim();
+  };
+
+  const normalizeLooseBoolean = (value: string): boolean | null => {
+    const normalized = value.trim().toLowerCase();
+    if (["true", "yes", "y"].includes(normalized)) {
+      return true;
+    }
+    if (["false", "no", "n"].includes(normalized)) {
+      return false;
+    }
+    return null;
+  };
+
+  const normalizeLooseValue = (value: string): unknown => {
+    const normalized = value.trim();
+    const bool = normalizeLooseBoolean(normalized);
+    if (bool !== null) {
+      return bool;
+    }
+
+    const num = parseNumberLike(normalized);
+    if (num !== null) {
+      return num;
+    }
+
+    try {
+      if (normalized.startsWith("[") || normalized.startsWith("{")) {
+        const sanitized = normalized.replace(/'/g, '"');
+        return JSON.parse(sanitized);
+      }
+    } catch {
+      // fallback to string
+    }
+
+    return normalizeLooseString(normalized);
+  };
+
+  const parseLooseEvaluationResponse = (rawText: string): unknown => {
+    const jsonText = extractJsonObjectFromText(rawText);
+    if (!jsonText) {
+      throw new Error(
+        `Unable to locate a valid JSON object in the model response. Raw response: ${rawText}`
+      );
+    }
+
+    try {
+      return JSON.parse(jsonText);
+    } catch {
+      const result: Record<string, unknown> = {};
+      const pairs: Array<[string, string]> = [];
+      const pattern = /["']?(satisfied|score|feedback|completeness|accuracy|issues)["']?\s*:\s*([^,}\n]+|\[[^\]]*\]|\{[^}]*\})/gi;
+      let match: RegExpExecArray | null;
+      while (true) {
+        match = pattern.exec(jsonText);
+        if (!match) {
+          break;
+        }
+        pairs.push([match[1]!.toLowerCase(), match[2]!.trim()]);
+      }
+
+      for (const [key, rawValue] of pairs) {
+        if (key === "issues") {
+          const trimmed = rawValue.trim();
+          if (trimmed.startsWith("[")) {
+            try {
+              result.issues = JSON.parse(trimmed.replace(/'/g, '"'));
+              continue;
+            } catch {
+              // fallback
+            }
+          }
+          result.issues = [normalizeLooseString(trimmed)];
+          continue;
+        }
+
+        if (key === "feedback") {
+          result.feedback = normalizeLooseString(rawValue);
+          continue;
+        }
+
+        result[key] = normalizeLooseValue(rawValue);
+      }
+
+      return result;
+    }
+  };
+
+  try {
+    const rawText = response.text?.trim() ?? "";
+    const parsed = parseLooseEvaluationResponse(rawText);
+    const evaluation = EvaluationSchema.parse(parsed);
+
+    if (evaluation.score < evaluationThreshold) {
+      evaluation.satisfied = false;
+    }
+
+    return evaluation;
+  } catch (error) {
+    throw new Error(
+      `Failed to parse evaluation result: ${error instanceof Error ? error.message : String(error)}`
+    );
+  }
+}
+
+export function shouldContinueIterating(
+  evaluation: EvaluationResult,
+  iteration: number,
+  maxIterations: number
+): boolean {
+  // Stop if satisfied
+  if (evaluation.satisfied) {
+    return false;
+  }
+
+  // Stop if max iterations reached
+  if (iteration >= maxIterations) {
+    return false;
+  }
+
+  // Continue if score is improvable and iterations remain
+  return true;
+}
+
+export function extractFeedbackForNextIteration(
+  evaluation: EvaluationResult
+): string {
+  const issues = evaluation.issues.slice(0, 3).join("; ");
+  return `Previous iteration score: ${evaluation.score}/100. Issues to fix: ${issues}. Completeness: ${evaluation.completeness}%, Accuracy: ${evaluation.accuracy}%.`;
+}