@agentium/browser 3.1.1 → 3.1.2

package/dist/index.js

@@ -1,5 +1,139 @@
+// src/action-space.ts
+var DEFAULT_MAX_ACTION_CHOICES = 40;
+function searchUrl(query, engine = "duckduckgo") {
+  const q = encodeURIComponent(query.trim());
+  switch (engine) {
+    case "google":
+      return `https://www.google.com/search?q=${q}`;
+    case "bing":
+      return `https://www.bing.com/search?q=${q}`;
+    default:
+      return `https://html.duckduckgo.com/html/?q=${q}`;
+  }
+}
+function isSearchResultsUrl(url) {
+  try {
+    const u = new URL(url);
+    const host = u.hostname.replace(/^www\./, "");
+    if (host === "duckduckgo.com" || host.endsWith(".duckduckgo.com")) {
+      return u.searchParams.has("q") || u.pathname.includes("/html");
+    }
+    if (host === "google.com" || host.endsWith(".google.com")) return u.pathname.startsWith("/search");
+    if (host === "bing.com" || host.endsWith(".bing.com")) return u.pathname.startsWith("/search");
+    return false;
+  } catch {
+    return false;
+  }
+}
+function isBlockedPageUrl(url) {
+  return /418\.html|\/sorry\/|captcha|challenge|cf-browser-verification/i.test(url);
+}
+function isBotChallengeText(text) {
+  return /unfortunately, bots use|select all squares containing a duck|unusual traffic from your computer|are you a robot|verify you are human|detected unusual traffic/i.test(
+    text
+  );
+}
+var CHROME_LABEL = /^(email us|feedback|privacy|terms|settings|facebook|twitter|reddit|about|help|sign in|log in|login|all regions|any time|more results|next|previous|images|videos|news|maps|shopping|lite|duckduckgo|menu|home|cookies|advertising|github|wikipedia|instagram|youtube)$/i;
+function isResultTitle(label) {
+  const t = label.replace(/\s+/g, " ").trim();
+  if (t.length < 16) return false;
+  if (CHROME_LABEL.test(t)) return false;
+  if (/^(https?:|javascript:|mailto:)/i.test(t)) return false;
+  return true;
+}
+function looksLikeResultList(text) {
+  const lines = text.split("\n").map((l) => l.replace(/^\d+\.\s*/, "").trim()).filter(Boolean);
+  return lines.length >= 2 && lines.filter(isResultTitle).length >= 2;
+}
+function titlesFromElements(elements, max = 5) {
+  const seen = /* @__PURE__ */ new Set();
+  const labels = [];
+  for (const e of elements) {
+    if (e.tag !== "a" && e.role !== "link" && e.role !== "heading") continue;
+    const label = e.label?.replace(/\s+/g, " ").trim() ?? "";
+    if (!isResultTitle(label) || seen.has(label.toLowerCase())) continue;
+    seen.add(label.toLowerCase());
+    labels.push(label);
+  }
+  labels.sort((a, b) => b.length - a.length);
+  return labels.slice(0, max).map((t, i) => `${i + 1}. ${t}`).join("\n");
+}
+var SERP_TITLE_SELECTORS = ["a.result__a", "a.result-link", "#links .result__a", "h2 a", "h3 a", "a > h3"];
+function guessSearchQuery(task) {
+  const quoted = task.match(/["']([^"']{1,200})["']/);
+  if (quoted?.[1]) return quoted[1].trim();
+  return task.replace(/^(please\s+)?(search for|google|bing|find|look up)\s+/i, "").slice(0, 200).trim();
+}
+function buildActionSpace(elements, tabs = [], opts) {
+  const max = opts?.max ?? DEFAULT_MAX_ACTION_CHOICES;
+  const criteria = {
+    back: "Go to the previous page",
+    screenshot: "Take a fresh screenshot next step",
+    fail: "Give up \u2014 the task cannot be completed",
+    new_tab: "Open a blank new tab"
+  };
+  if (opts?.allowWait !== false) {
+    criteria.wait = "Wait for the page to settle. Use at most once.";
+  }
+  if (opts?.allowDone !== false) {
+    criteria.done = "The task is finished \u2014 return the result titles you already found";
+  }
+  if (opts?.allowSearch !== false) {
+    criteria.search = "Open a NEW web search. Do not pick this if results are already on screen.";
+  }
+  if ((opts?.pagesBelow ?? 1) > 0) {
+    criteria.scroll_down = "Scroll down one viewport";
+  }
+  if ((opts?.pagesAbove ?? 1) > 0) {
+    criteria.scroll_up = "Scroll up one viewport";
+  }
+  const slice = elements.slice(0, max);
+  for (const e of slice) {
+    const label = e.label?.trim() || e.role || e.tag;
+    criteria[`click_${e.index}`] = `Click [${e.index}] ${e.role}: ${label}`;
+    if (e.isInput) {
+      criteria[`type_${e.index}`] = `Type into [${e.index}] ${label}`;
+    }
+  }
+  for (const tab of tabs) {
+    if (!tab.active) {
+      criteria[`switch_tab_${tab.id}`] = `Switch to tab ${tab.id} (${tab.url || "blank"})`;
+    }
+    if (tabs.length > 1) {
+      criteria[`close_tab_${tab.id}`] = `Close tab ${tab.id}`;
+    }
+  }
+  return {
+    criteria,
+    toAction(label) {
+      return labelToAction(label);
+    }
+  };
+}
+function labelToAction(label, extras) {
+  if (label === "back") return { action: "back" };
+  if (label === "scroll_down") return { action: "scroll", direction: "down" };
+  if (label === "scroll_up") return { action: "scroll", direction: "up" };
+  if (label === "wait") return { action: "wait", ms: 1500 };
+  if (label === "screenshot") return { action: "screenshot" };
+  if (label === "done") return { action: "done", result: extras?.doneResult ?? "Done" };
+  if (label === "fail") return { action: "fail", reason: "Jev chose fail" };
+  if (label === "search") return { action: "search", query: extras?.searchQuery ?? "" };
+  if (label === "new_tab") return { action: "new_tab" };
+  if (label === "none") return void 0;
+  const click = /^click_(\d+)$/.exec(label);
+  if (click) return { action: "click", index: Number(click[1]) };
+  const type = /^type_(\d+)$/.exec(label);
+  if (type) return { action: "type", index: Number(type[1]), text: extras?.typeText ?? "" };
+  const sw = /^switch_tab_(.+)$/.exec(label);
+  if (sw) return { action: "switch_tab", tabId: sw[1] };
+  const cl = /^close_tab_(.+)$/.exec(label);
+  if (cl) return { action: "close_tab", tabId: cl[1] };
+  return void 0;
+}
+
 // src/browser-agent.ts
-import { EventBus, Logger, MemoryManager, RunContext } from "@agentium/core";
+import { choice, EventBus, jev, Logger, MemoryManager, RunContext } from "@agentium/core";
 import { z } from "zod";
 
 // src/stealth.ts
@@ -906,6 +1040,73 @@ var BrowserProvider = class {
       }
     }
   }
+  async visibleText(maxChars = 2e3) {
+    this.ensurePage();
+    return this.page.evaluate(
+      (n) => (globalThis.document?.body?.innerText ?? "").slice(0, n),
+      maxChars
+    );
+  }
+  /**
+   * Grep visible page text. No LLM. Used by `search_page`.
+   */
+  async searchPage(opts) {
+    this.ensurePage();
+    const pattern = opts.pattern;
+    const useRegex = !!opts.regex;
+    const caseSensitive = !!opts.caseSensitive;
+    const maxResults = Math.min(Math.max(opts.maxResults ?? 20, 1), 50);
+    const contextChars = opts.contextChars ?? 100;
+    return this.page.evaluate(
+      ({ pattern: pattern2, useRegex: useRegex2, caseSensitive: caseSensitive2, maxResults: maxResults2, contextChars: contextChars2 }) => {
+        const doc = globalThis.document;
+        const text = doc?.body?.innerText ?? "";
+        const src = String(pattern2);
+        const flags = caseSensitive2 ? "g" : "gi";
+        let re;
+        try {
+          re = useRegex2 ? new RegExp(src, flags) : new RegExp(src.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), flags);
+        } catch {
+          return [];
+        }
+        const matches = [];
+        let m = re.exec(text);
+        while (m && matches.length < Number(maxResults2)) {
+          const start = Math.max(0, m.index - Number(contextChars2));
+          const end = Math.min(text.length, m.index + m[0].length + Number(contextChars2));
+          matches.push({ match: m[0], context: text.slice(start, end), index: m.index });
+          if (!re.global) break;
+          m = re.exec(text);
+        }
+        return matches;
+      },
+      { pattern, useRegex, caseSensitive, maxResults, contextChars }
+    );
+  }
+  /**
+   * querySelectorAll over the page. No LLM. Used by `find_elements`.
+   */
+  async findElements(selector, opts) {
+    this.ensurePage();
+    const maxResults = Math.min(Math.max(opts?.maxResults ?? 50, 1), 80);
+    return this.page.evaluate(
+      ({ selector: selector2, maxResults: maxResults2 }) => {
+        const doc = globalThis.document;
+        let nodes;
+        try {
+          nodes = Array.from(doc.querySelectorAll(selector2)).slice(0, maxResults2);
+        } catch {
+          return [];
+        }
+        return nodes.map((el) => ({
+          tag: String(el.tagName ?? "").toLowerCase(),
+          text: String(el.textContent ?? "").trim().slice(0, 200),
+          href: el.href ? String(el.href) : void 0
+        }));
+      },
+      { selector, maxResults }
+    );
+  }
   listTabs() {
     const tabs = [];
     for (const [id, pg] of this.pages) {
@@ -1123,7 +1324,19 @@ function normalizeAction(action) {
     case "scroll":
       return `scroll:${action.direction}:${action.amount ?? "default"}${action.index != null ? `:${action.index}` : ""}`;
     case "navigate":
-      return `navigate:${action.url}`;
+      return `navigate:${action.url}${action.newTab ? ":new" : ""}`;
+    case "search":
+      return `search:${truncate(action.query, 40)}:${action.engine ?? "default"}`;
+    case "new_tab":
+      return `new_tab:${action.url ?? ""}`;
+    case "switch_tab":
+      return `switch_tab:${action.tabId}`;
+    case "close_tab":
+      return `close_tab:${action.tabId}`;
+    case "search_page":
+      return `search_page:${truncate(action.pattern, 40)}`;
+    case "find_elements":
+      return `find_elements:${truncate(action.selector, 40)}`;
     case "back":
       return "back";
     case "wait":
@@ -1230,8 +1443,18 @@ function buildSystemPrompt(viewport, extraInstructions, credentialKeys, options)
     `Set a file on an \`<input type="file">\` by index. \`path\` must be a path the runtime can read; the agent does NOT have a filesystem of its own.`,
     `\`{ "action": "upload_file", "index": <n>, "path": "/abs/path/to/file.pdf" }\``,
     ``,
-    `### navigate / back`,
-    `\`{ "action": "navigate", "url": "<full URL>" }\` \xB7 \`{ "action": "back" }\``,
+    `### navigate / back / search`,
+    `\`{ "action": "navigate", "url": "<full URL>" }\` \xB7 \`{ "action": "navigate", "url": "<full URL>", "newTab": true }\` \xB7 \`{ "action": "back" }\``,
+    `\`{ "action": "search", "query": "<query>", "engine": "duckduckgo"|"google"|"bing" }\` opens a search-engine results page (default engine: duckduckgo).`,
+    ``,
+    `### tabs`,
+    `Tab ids come from the **Open tabs** list in the observation.`,
+    `\`{ "action": "new_tab", "url": "<optional URL>" }\` \xB7 \`{ "action": "switch_tab", "tabId": "tab-2" }\` \xB7 \`{ "action": "close_tab", "tabId": "tab-1" }\``,
+    ``,
+    `### search_page / find_elements`,
+    `Zero-LLM inspect. Use these instead of \`extract\` when you just need to grep visible text or run a CSS selector.`,
+    `\`{ "action": "search_page", "pattern": "Total", "regex": false, "caseSensitive": false }\``,
+    `\`{ "action": "find_elements", "selector": "a.storylink" }\``,
     ``,
     `### wait / screenshot`,
     `\`{ "action": "wait", "ms": <ms \u2264 10000> }\` \xB7 \`{ "action": "screenshot" }\` (request a fresh image on the next step \u2014 only useful when vision mode is "auto").`,
@@ -1321,7 +1544,7 @@ function responseFormatLines(maxActions, useThinking) {
     `Only the "action" field is executed; the others help you self-correct over multiple steps.`
   ];
 }
-function buildUserMessage(task, pageUrl, pageTitle, stepIndex, actionHistory, domSnapshot, lastExtract, scroll, nudge, stepBudget) {
+function buildUserMessage(task, pageUrl, pageTitle, stepIndex, actionHistory, domSnapshot, lastExtract, scroll, nudge, stepBudget, tabs) {
   const lines = [];
   lines.push(`**Task:** ${task}`);
   lines.push(`**Current URL:** ${pageUrl}`);
@@ -1339,6 +1562,13 @@ function buildUserMessage(task, pageUrl, pageTitle, stepIndex, actionHistory, do
     if (scroll.pagesAbove === 0 && scroll.pagesBelow === 0) parts.push("fits in viewport");
     lines.push(`**Page stats:** ${parts.join(" \xB7 ")}`);
   }
+  if (tabs && tabs.length > 0) {
+    lines.push(`**Open tabs:**`);
+    for (const tab of tabs) {
+      const mark = tab.active ? " (active)" : "";
+      lines.push(`- ${tab.id}${mark}: ${tab.url || "blank"}`);
+    }
+  }
   if (domSnapshot !== void 0) {
     if (domSnapshot.trim().length === 0) {
       lines.push(``);
@@ -1383,7 +1613,19 @@ function summarizeAction(action) {
       if (typeof action.index === "number") return `Scrolled [${action.index}] into view`;
       return `Scrolled ${action.direction}${action.amount ? ` ${action.amount}px` : ""}`;
     case "navigate":
-      return `Navigated to ${action.url}`;
+      return action.newTab ? `Opened ${action.url} in a new tab` : `Navigated to ${action.url}`;
+    case "search":
+      return `Searched ${action.engine ?? "web"} for "${action.query}"`;
+    case "new_tab":
+      return action.url ? `Opened new tab: ${action.url}` : `Opened new tab`;
+    case "switch_tab":
+      return `Switched to ${action.tabId}`;
+    case "close_tab":
+      return `Closed ${action.tabId}`;
+    case "search_page":
+      return `Searched page for "${action.pattern}"`;
+    case "find_elements":
+      return `Found elements: ${action.selector}`;
     case "back":
       return `Went back to previous page`;
     case "wait":
@@ -1449,6 +1691,11 @@ var BrowserAgent = class {
   stealth;
   humanize;
   tools;
+  planner;
+  jevModel;
+  maxActionChoices;
+  searchEngine;
+  jevProvider = null;
   costTracker;
   memoryManager = null;
   logger;
@@ -1488,6 +1735,10 @@ var BrowserAgent = class {
     this.stealth = config.stealth;
     this.humanize = config.humanize;
     this.tools = config.tools ?? [];
+    this.planner = config.planner ?? "vision";
+    this.jevModel = config.jevModel ?? "jev-latest";
+    this.maxActionChoices = config.maxActionChoices ?? DEFAULT_MAX_ACTION_CHOICES;
+    this.searchEngine = config.searchEngine ?? "duckduckgo";
     this.costTracker = config.costTracker ?? null;
     this.eventBus = config.eventBus ?? new EventBus();
     this.logger = new Logger({
@@ -1573,11 +1824,14 @@ ${memoryContext}` : memoryContext;
         const pageInfo = await browser.getPageInfo();
         let domSnapshot;
         let scrollCtx;
+        let elements = [];
         if (this.useDOM) {
           const dom = await browser.extractDOM();
           domSnapshot = dom.text;
           scrollCtx = dom.scroll;
+          elements = dom.elements;
         }
+        const tabs = browser.listTabs();
         const pageAdvice = loop.recordPage({
           url: pageInfo.url,
           interactiveCount: scrollCtx?.totalInteractive ?? 0,
@@ -1611,34 +1865,65 @@ ${memoryContext}` : memoryContext;
           lastExtractResult,
           scrollCtx,
           nudge,
-          { current: step, max: maxSteps }
+          { current: step, max: maxSteps },
+          tabs
         );
-        const messages = this.buildMessages(systemPrompt, historyTurns, userText, wantVision ? screenshot : null);
-        this.logger.debug("Calling model", { step, url: pageInfo.url, vision: wantVision });
-        const { response, modelUsed } = await this.callModelWithFallback(messages, opts?.apiKey);
-        if (!response) {
-          consecutiveFailures++;
-          actionHistory.push(`(model call failed \u2014 retrying, ${consecutiveFailures}/${this.maxFailures})`);
-          if (consecutiveFailures > this.maxFailures) {
-            return await this.forceDone(browser, steps, startTime, opts, extractedContent, actionHistory, "model");
-          }
-          continue;
-        }
-        if (this.costTracker && response.usage) {
-          this.costTracker.track({
-            runId: sessionId,
-            agentName: this.name,
-            modelId: modelUsed.modelId,
-            usage: response.usage,
-            sessionId,
-            userId
+        this.logger.debug("Calling planner", { step, url: pageInfo.url, planner: this.planner, vision: wantVision });
+        let envelope = null;
+        let modelUsed = this.model;
+        if (this.planner === "jev") {
+          const planned = await this.planWithJev({
+            task,
+            url: pageInfo.url,
+            title: pageInfo.title,
+            elements,
+            tabs,
+            actionHistory,
+            lastExtract: lastExtractResult,
+            pagesBelow: scrollCtx?.pagesBelow,
+            pagesAbove: scrollCtx?.pagesAbove,
+            apiKey: opts?.apiKey
           });
+          envelope = planned.envelope;
+          modelUsed = planned.modelUsed;
+          if (this.costTracker && planned.usage) {
+            this.costTracker.track({
+              runId: sessionId,
+              agentName: this.name,
+              modelId: modelUsed.modelId,
+              usage: planned.usage,
+              sessionId,
+              userId
+            });
+          }
+        } else {
+          const messages = this.buildMessages(systemPrompt, historyTurns, userText, wantVision ? screenshot : null);
+          const { response, modelUsed: used } = await this.callModelWithFallback(messages, opts?.apiKey);
+          modelUsed = used;
+          if (!response) {
+            consecutiveFailures++;
+            actionHistory.push(`(model call failed \u2014 retrying, ${consecutiveFailures}/${this.maxFailures})`);
+            if (consecutiveFailures > this.maxFailures) {
+              return await this.forceDone(browser, steps, startTime, opts, extractedContent, actionHistory, "model");
+            }
+            continue;
+          }
+          if (this.costTracker && response.usage) {
+            this.costTracker.track({
+              runId: sessionId,
+              agentName: this.name,
+              modelId: modelUsed.modelId,
+              usage: response.usage,
+              sessionId,
+              userId
+            });
+          }
+          const raw = typeof response.message.content === "string" ? response.message.content : "";
+          envelope = this.parseEnvelope(raw);
         }
-        const raw = typeof response.message.content === "string" ? response.message.content : "";
-        const envelope = this.parseEnvelope(raw);
         if (!envelope) {
           consecutiveFailures++;
-          this.logger.warn("Failed to parse model response", { raw: raw.slice(0, 200), consecutiveFailures });
+          this.logger.warn("Failed to parse planner response", { planner: this.planner, consecutiveFailures });
           if (consecutiveFailures > this.maxFailures) {
             return await this.forceDone(browser, steps, startTime, opts, extractedContent, actionHistory, "parse");
           }
@@ -1693,7 +1978,9 @@ ${memoryContext}` : memoryContext;
               didNavigate = true;
               await this.navigationHealthCheck(browser, pageInfo.url);
             }
-            if (action.action === "extract" && exec?.output) lastExtractResult = exec.output;
+            if (exec?.output && (action.action === "extract" || action.action === "search" || action.action === "find_elements" || action.action === "search_page")) {
+              lastExtractResult = exec.output;
+            }
             if (action.action === "screenshot") lastActionWasScreenshot = true;
           } catch (e) {
             stepOk = false;
@@ -1773,6 +2060,7 @@ ${memoryContext}` : memoryContext;
   }
   // ── Private helpers ──────────────────────────────────────────────────
   shouldCaptureVision(step, lastActionWasScreenshot) {
+    if (this.planner === "jev" && this.useVision !== true) return false;
     if (this.useVision === false) return false;
     if (this.useVision === true) return true;
     if (step === 0) return true;
@@ -1872,6 +2160,119 @@ ${memoryContext}` : memoryContext;
     if (msg.includes("401") || msg.includes("402") || msg.includes("auth")) return true;
     return false;
   }
+  getJev() {
+    if (!this.jevProvider) this.jevProvider = jev(this.jevModel);
+    return this.jevProvider;
+  }
+  /**
+   * Ask Jev to pick one label from this frame's action space, then map it
+   * to a BrowserAction. Type/search strings come from a text model or the task.
+   */
+  async planWithJev(args) {
+    const alreadySearched = args.actionHistory.some((h) => /^Searched /.test(h));
+    const alreadyWaited = args.actionHistory.some((h) => /^Waited /.test(h));
+    const foundTitles = args.lastExtract ?? titlesFromElements(args.elements);
+    const space = buildActionSpace(args.elements, args.tabs, {
+      max: this.maxActionChoices,
+      pagesBelow: args.pagesBelow,
+      pagesAbove: args.pagesAbove,
+      allowSearch: !alreadySearched && !isSearchResultsUrl(args.url) && !isBlockedPageUrl(args.url),
+      allowDone: looksLikeResultList(foundTitles),
+      allowWait: !alreadyWaited
+    });
+    const provider = this.getJev();
+    try {
+      const response = await provider.generate(
+        [
+          {
+            role: "user",
+            content: JSON.stringify({
+              task: args.task,
+              url: args.url,
+              title: args.title,
+              lastActions: args.actionHistory.slice(-8),
+              lastExtract: args.lastExtract,
+              visibleTitles: titlesFromElements(args.elements, 8),
+              tabs: args.tabs,
+              hint: looksLikeResultList(foundTitles) ? "Result titles are in lastExtract. Pick done." : alreadySearched ? "A search already ran. Wait or click a result title. Do not search again." : void 0
+            })
+          }
+        ],
+        {
+          questions: {
+            action: choice(
+              "Which browser action next? Do not repeat lastActions. If search results are already visible, click a result or done.",
+              space.criteria
+            )
+          },
+          apiKey: args.apiKey
+        }
+      );
+      const raw = typeof response.message.content === "string" ? response.message.content : "";
+      let pick;
+      try {
+        const answers = JSON.parse(raw);
+        pick = typeof answers.action === "string" ? answers.action : answers.action?.choice;
+      } catch {
+        pick = void 0;
+      }
+      if (!pick) return { envelope: null, modelUsed: provider, usage: response.usage };
+      const extras = {
+        searchQuery: pick === "search" ? guessSearchQuery(args.task) : void 0,
+        typeText: pick.startsWith("type_") ? await this.inferTypeText(args.task, pick) : void 0,
+        doneResult: pick === "done" ? looksLikeResultList(args.lastExtract ?? "") ? args.lastExtract : titlesFromElements(args.elements) || args.lastExtract || "Done" : void 0
+      };
+      const action = labelToAction(pick, extras);
+      if (!action) return { envelope: null, modelUsed: provider, usage: response.usage };
+      return { envelope: { action, nextGoal: pick }, modelUsed: provider, usage: response.usage };
+    } catch (e) {
+      this.logger.warn("Jev planner failed", { error: e?.message });
+      return { envelope: null, modelUsed: provider };
+    }
+  }
+  async pageLooksBlocked(browser) {
+    const info = await browser.getPageInfo();
+    if (isBlockedPageUrl(info.url)) return true;
+    try {
+      return isBotChallengeText(await browser.visibleText());
+    } catch {
+      return false;
+    }
+  }
+  async scrapeSerpTitles(browser) {
+    for (const selector of SERP_TITLE_SELECTORS) {
+      const found = await browser.findElements(selector, { maxResults: 8 });
+      const labels = [];
+      const seen = /* @__PURE__ */ new Set();
+      for (const f of found) {
+        const text = f.text.replace(/\s+/g, " ").trim();
+        if (!isResultTitle(text) || seen.has(text.toLowerCase())) continue;
+        seen.add(text.toLowerCase());
+        labels.push(text);
+      }
+      if (labels.length >= 2) {
+        return labels.slice(0, 5).map((t, i) => `${i + 1}. ${t}`).join("\n");
+      }
+    }
+    return "";
+  }
+  async inferTypeText(task, pick) {
+    const model = this.pageExtractionLLM ?? (this.model.providerId === "jev" ? null : this.model);
+    if (!model) return guessSearchQuery(task);
+    try {
+      const response = await model.generate([
+        {
+          role: "user",
+          content: `Task: ${task}
+The next browser action is ${pick}. Reply with ONLY the text to type into that field. No quotes.`
+        }
+      ]);
+      const text = typeof response.message.content === "string" ? response.message.content.trim() : "";
+      return text.replace(/^["']|["']$/g, "").slice(0, 500);
+    } catch {
+      return guessSearchQuery(task);
+    }
+  }
   /**
    * Parse the model's raw response into an `AgentOutput`. Tolerant to
    * three shapes:
@@ -2064,9 +2465,69 @@ Last actions: ${tail || "(none)"}${extracts}`;
       }
       case "navigate": {
         this.assertDomainAllowed(action.url);
+        if (action.newTab) {
+          const tabId = await browser.newTab(action.url);
+          await browser.switchTab(tabId);
+          return { didNavigate: true, output: `Opened ${action.url} in ${tabId}` };
+        }
         await browser.navigate(action.url);
         return { didNavigate: true };
       }
+      case "search": {
+        const engine = action.engine ?? this.searchEngine;
+        const url = searchUrl(action.query, engine);
+        this.assertDomainAllowed(url);
+        await browser.navigate(url);
+        let used = engine;
+        if (await this.pageLooksBlocked(browser)) {
+          if (engine === "duckduckgo") {
+            const fallback = searchUrl(action.query, "bing");
+            this.assertDomainAllowed(fallback);
+            await browser.navigate(fallback);
+            used = "bing";
+          } else {
+            return {
+              didNavigate: true,
+              output: `Search blocked on ${engine}. Pick fail or navigate elsewhere.`
+            };
+          }
+        }
+        await browser.waitForStable(400);
+        const titles = await this.scrapeSerpTitles(browser);
+        return {
+          didNavigate: true,
+          output: titles || `Searched ${used} for "${action.query}"`
+        };
+      }
+      case "new_tab": {
+        if (action.url) this.assertDomainAllowed(action.url);
+        const tabId = await browser.newTab(action.url);
+        await browser.switchTab(tabId);
+        return { didNavigate: !!action.url, output: `Opened ${tabId}` };
+      }
+      case "switch_tab":
+        await browser.switchTab(action.tabId);
+        return { didNavigate: true, output: `Switched to ${action.tabId}` };
+      case "close_tab":
+        await browser.closeTab(action.tabId);
+        return { output: `Closed ${action.tabId}` };
+      case "search_page": {
+        const hits = await browser.searchPage({
+          pattern: action.pattern,
+          regex: action.regex,
+          caseSensitive: action.caseSensitive,
+          maxResults: action.maxResults
+        });
+        const lines = hits.map((h, i) => `${i + 1}. "${h.match}" \u2014 \u2026${h.context}\u2026`);
+        return { output: hits.length ? `search_page:
+${lines.join("\n")}` : "search_page: no matches" };
+      }
+      case "find_elements": {
+        const found = await browser.findElements(action.selector, { maxResults: action.maxResults });
+        const lines = found.map((el, i) => `${i + 1}. <${el.tag}> ${el.text}${el.href ? ` ${el.href}` : ""}`);
+        return { output: found.length ? `find_elements:
+${lines.join("\n")}` : "find_elements: none" };
+      }
       case "back":
         await browser.back();
         return { didNavigate: true };
@@ -2268,5 +2729,15 @@ var CredentialVault = class {
 export {
   BrowserAgent,
   BrowserProvider,
-  CredentialVault
+  CredentialVault,
+  buildActionSpace,
+  guessSearchQuery,
+  isBlockedPageUrl,
+  isBotChallengeText,
+  isResultTitle,
+  isSearchResultsUrl,
+  labelToAction,
+  looksLikeResultList,
+  searchUrl,
+  titlesFromElements
 };

package/dist/loop-detector.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"loop-detector.d.ts","sourceRoot":"","sources":["../src/loop-detector.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEhD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,MAAM,GAAG,UAAU,GAAG,OAAO,CAAC;AAElE,MAAM,WAAW,UAAU;IACzB,QAAQ,EAAE,YAAY,CAAC;IACvB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,gBAAgB,EAAE,MAAM,CAAC;IACzB,gEAAgE;IAChE,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,qBAAa,YAAY;IACvB,OAAO,CAAC,YAAY,CAAgB;IACpC,OAAO,CAAC,WAAW,CAAyB;IAC5C,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;IAC1C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAS;IACzC,8DAA8D;IAC9D,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAoD;IACrF,yEAAyE;IACzE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAoD;gBAEvE,IAAI,CAAC,EAAE;QACjB,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,gBAAgB,CAAC,EAAE;YAAE,IAAI,CAAC,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;YAAC,KAAK,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QACxE,cAAc,CAAC,EAAE;YAAE,IAAI,CAAC,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;YAAC,KAAK,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;KACvE;IAeD;;;OAGG;IACH,YAAY,CAAC,MAAM,EAAE,aAAa,GAAG,UAAU;IA8B/C;;;;OAIG;IACH,UAAU,CAAC,EAAE,EAAE,eAAe,GAAG,UAAU;IAmC3C,0DAA0D;IAC1D,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,UAAU,GAAG,UAAU;CAIzD;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,aAAa,GAAG,MAAM,CA8C7D;AAED;;;GAGG;AACH,wBAAgB,OAAO,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAOzC"}
+{"version":3,"file":"loop-detector.d.ts","sourceRoot":"","sources":["../src/loop-detector.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEhD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,MAAM,GAAG,UAAU,GAAG,OAAO,CAAC;AAElE,MAAM,WAAW,UAAU;IACzB,QAAQ,EAAE,YAAY,CAAC;IACvB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,gBAAgB,EAAE,MAAM,CAAC;IACzB,gEAAgE;IAChE,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,qBAAa,YAAY;IACvB,OAAO,CAAC,YAAY,CAAgB;IACpC,OAAO,CAAC,WAAW,CAAyB;IAC5C,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;IAC1C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAS;IACzC,8DAA8D;IAC9D,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAoD;IACrF,yEAAyE;IACzE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAoD;gBAEvE,IAAI,CAAC,EAAE;QACjB,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,gBAAgB,CAAC,EAAE;YAAE,IAAI,CAAC,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;YAAC,KAAK,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QACxE,cAAc,CAAC,EAAE;YAAE,IAAI,CAAC,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;YAAC,KAAK,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;KACvE;IAeD;;;OAGG;IACH,YAAY,CAAC,MAAM,EAAE,aAAa,GAAG,UAAU;IA8B/C;;;;OAIG;IACH,UAAU,CAAC,EAAE,EAAE,eAAe,GAAG,UAAU;IAmC3C,0DAA0D;IAC1D,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,UAAU,GAAG,UAAU;CAIzD;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,aAAa,GAAG,MAAM,CA0D7D;AAED;;;GAGG;AACH,wBAAgB,OAAO,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAOzC"}

package/dist/prompts.d.ts

@@ -1,4 +1,5 @@
 import type { ToolDef } from "@agentium/core";
+import type { TabInfo } from "./action-space.js";
 import type { DomScrollContext } from "./types.js";
 export declare function buildSystemPrompt(viewport: {
     width: number;
@@ -15,6 +16,6 @@ export declare function buildSystemPrompt(viewport: {
 export declare function buildUserMessage(task: string, pageUrl: string, pageTitle: string, stepIndex: number, actionHistory: string[], domSnapshot?: string, lastExtract?: string, scroll?: DomScrollContext, nudge?: string, stepBudget?: {
     current: number;
     max: number;
-}): string;
+}, tabs?: TabInfo[]): string;
 export declare function summarizeAction(action: Record<string, unknown>): string;
 //# sourceMappingURL=prompts.d.ts.map

package/dist/prompts.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"prompts.d.ts","sourceRoot":"","sources":["../src/prompts.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAEnD,wBAAgB,iBAAiB,CAC/B,QAAQ,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,EAC3C,iBAAiB,CAAC,EAAE,MAAM,EAC1B,cAAc,CAAC,EAAE,MAAM,EAAE,EACzB,OAAO,CAAC,EAAE;IACR,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,KAAK,CAAC,EAAE,OAAO,EAAE,CAAC;IAClB,SAAS,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IAC7B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB,GACA,MAAM,CAiIR;AA6CD,wBAAgB,gBAAgB,CAC9B,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EAAE,EACvB,WAAW,CAAC,EAAE,MAAM,EACpB,WAAW,CAAC,EAAE,MAAM,EACpB,MAAM,CAAC,EAAE,gBAAgB,EACzB,KAAK,CAAC,EAAE,MAAM,EACd,UAAU,CAAC,EAAE;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAC5C,MAAM,CA0DR;AAED,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CA4CvE"}
+{"version":3,"file":"prompts.d.ts","sourceRoot":"","sources":["../src/prompts.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AACjD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAEnD,wBAAgB,iBAAiB,CAC/B,QAAQ,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,EAC3C,iBAAiB,CAAC,EAAE,MAAM,EAC1B,cAAc,CAAC,EAAE,MAAM,EAAE,EACzB,OAAO,CAAC,EAAE;IACR,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,KAAK,CAAC,EAAE,OAAO,EAAE,CAAC;IAClB,SAAS,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IAC7B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB,GACA,MAAM,CA2IR;AA6CD,wBAAgB,gBAAgB,CAC9B,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EAAE,EACvB,WAAW,CAAC,EAAE,MAAM,EACpB,WAAW,CAAC,EAAE,MAAM,EACpB,MAAM,CAAC,EAAE,gBAAgB,EACzB,KAAK,CAAC,EAAE,MAAM,EACd,UAAU,CAAC,EAAE;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,EAC7C,IAAI,CAAC,EAAE,OAAO,EAAE,GACf,MAAM,CAkER;AAED,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAwDvE"}