@akshayram1/omnibrowser-agent 0.2.8 → 0.2.28

package/dist/popup.html

@@ -7,9 +7,10 @@
     <style>
       body { font-family: system-ui, sans-serif; margin: 12px; width: 340px; }
       h1 { font-size: 16px; margin: 0 0 8px; }
-      textarea, select, button { width: 100%; margin-top: 8px; }
+      textarea, select, input, button { width: 100%; margin-top: 8px; }
       textarea { min-height: 86px; }
       .row { display: grid; grid-template-columns: 1fr 1fr; gap: 8px; }
+      .hidden { display: none; }
       #status { margin-top: 10px; font-size: 12px; color: #444; white-space: pre-wrap; }
     </style>
   </head>
@@ -35,6 +36,11 @@
       </div>
     </div>
 
+    <div id="model-row" class="hidden">
+      <label for="modelId">WebLLM Model ID</label>
+      <input id="modelId" type="text" placeholder="Llama-3.2-1B-Instruct-q4f16_1-MLC" />
+    </div>
+
     <button id="start">Start</button>
     <button id="approve">Approve pending action</button>
     <button id="stop">Stop</button>

package/dist/popup.js

@@ -2,6 +2,8 @@
 var goal = document.getElementById("goal");
 var mode = document.getElementById("mode");
 var planner = document.getElementById("planner");
+var modelRow = document.getElementById("model-row");
+var modelId = document.getElementById("modelId");
 var status = document.getElementById("status");
 var start = document.getElementById("start");
 var approve = document.getElementById("approve");
@@ -13,16 +15,32 @@ async function withActiveTab(fn) {
   }
   return fn(tab.id);
 }
+function syncModelInputVisibility() {
+  const isWebLLM = planner.value === "webllm";
+  modelRow.classList.toggle("hidden", !isWebLLM);
+}
+planner.addEventListener("change", syncModelInputVisibility);
+syncModelInputVisibility();
 start.addEventListener("click", async () => {
   try {
     status.textContent = "Starting...";
+    const plannerKind = planner.value;
+    const plannerConfig = {
+      kind: plannerKind
+    };
+    if (plannerKind === "webllm") {
+      const trimmedModelId = modelId.value.trim();
+      if (trimmedModelId) {
+        plannerConfig.modelId = trimmedModelId;
+      }
+    }
     await withActiveTab(
       (tabId) => chrome.runtime.sendMessage({
         type: "START_AGENT",
         tabId,
         goal: goal.value.trim(),
         mode: mode.value,
-        planner: planner.value
+        planner: plannerConfig
       })
     );
     status.textContent = "Agent started";

package/dist/popup.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../src/popup/index.ts"],
-  "sourcesContent": ["import type { AgentMode, PlannerKind } from \"../shared/contracts\";\n\nconst goal = document.getElementById(\"goal\") as HTMLTextAreaElement;\nconst mode = document.getElementById(\"mode\") as HTMLSelectElement;\nconst planner = document.getElementById(\"planner\") as HTMLSelectElement;\nconst status = document.getElementById(\"status\") as HTMLDivElement;\n\nconst start = document.getElementById(\"start\") as HTMLButtonElement;\nconst approve = document.getElementById(\"approve\") as HTMLButtonElement;\nconst stop = document.getElementById(\"stop\") as HTMLButtonElement;\n\nasync function withActiveTab<T>(fn: (tabId: number) => Promise<T>) {\n  const [tab] = await chrome.tabs.query({ active: true, currentWindow: true });\n  if (!tab?.id) {\n    throw new Error(\"No active tab found\");\n  }\n  return fn(tab.id);\n}\n\nstart.addEventListener(\"click\", async () => {\n  try {\n    status.textContent = \"Starting...\";\n    await withActiveTab((tabId) =>\n      chrome.runtime.sendMessage({\n        type: \"START_AGENT\",\n        tabId,\n        goal: goal.value.trim(),\n        mode: mode.value as AgentMode,\n        planner: planner.value as PlannerKind\n      })\n    );\n    status.textContent = \"Agent started\";\n  } catch (error) {\n    status.textContent = `Error: ${String(error)}`;\n  }\n});\n\napprove.addEventListener(\"click\", async () => {\n  await withActiveTab((tabId) => chrome.runtime.sendMessage({ type: \"APPROVE_ACTION\", tabId }));\n  status.textContent = \"Approved pending action\";\n});\n\nstop.addEventListener(\"click\", async () => {\n  await withActiveTab((tabId) => chrome.runtime.sendMessage({ type: \"STOP_AGENT\", tabId }));\n  status.textContent = \"Stopped\";\n});\n\nchrome.runtime.sendMessage({ type: \"GET_STATUS\" }, (resp) => {\n  if (resp?.status) {\n    status.textContent = resp.status;\n  }\n});\n"],
-  "mappings": ";AAEA,IAAM,OAAO,SAAS,eAAe,MAAM;AAC3C,IAAM,OAAO,SAAS,eAAe,MAAM;AAC3C,IAAM,UAAU,SAAS,eAAe,SAAS;AACjD,IAAM,SAAS,SAAS,eAAe,QAAQ;AAE/C,IAAM,QAAQ,SAAS,eAAe,OAAO;AAC7C,IAAM,UAAU,SAAS,eAAe,SAAS;AACjD,IAAM,OAAO,SAAS,eAAe,MAAM;AAE3C,eAAe,cAAiB,IAAmC;AACjE,QAAM,CAAC,GAAG,IAAI,MAAM,OAAO,KAAK,MAAM,EAAE,QAAQ,MAAM,eAAe,KAAK,CAAC;AAC3E,MAAI,CAAC,KAAK,IAAI;AACZ,UAAM,IAAI,MAAM,qBAAqB;AAAA,EACvC;AACA,SAAO,GAAG,IAAI,EAAE;AAClB;AAEA,MAAM,iBAAiB,SAAS,YAAY;AAC1C,MAAI;AACF,WAAO,cAAc;AACrB,UAAM;AAAA,MAAc,CAAC,UACnB,OAAO,QAAQ,YAAY;AAAA,QACzB,MAAM;AAAA,QACN;AAAA,QACA,MAAM,KAAK,MAAM,KAAK;AAAA,QACtB,MAAM,KAAK;AAAA,QACX,SAAS,QAAQ;AAAA,MACnB,CAAC;AAAA,IACH;AACA,WAAO,cAAc;AAAA,EACvB,SAAS,OAAO;AACd,WAAO,cAAc,UAAU,OAAO,KAAK,CAAC;AAAA,EAC9C;AACF,CAAC;AAED,QAAQ,iBAAiB,SAAS,YAAY;AAC5C,QAAM,cAAc,CAAC,UAAU,OAAO,QAAQ,YAAY,EAAE,MAAM,kBAAkB,MAAM,CAAC,CAAC;AAC5F,SAAO,cAAc;AACvB,CAAC;AAED,KAAK,iBAAiB,SAAS,YAAY;AACzC,QAAM,cAAc,CAAC,UAAU,OAAO,QAAQ,YAAY,EAAE,MAAM,cAAc,MAAM,CAAC,CAAC;AACxF,SAAO,cAAc;AACvB,CAAC;AAED,OAAO,QAAQ,YAAY,EAAE,MAAM,aAAa,GAAG,CAAC,SAAS;AAC3D,MAAI,MAAM,QAAQ;AAChB,WAAO,cAAc,KAAK;AAAA,EAC5B;AACF,CAAC;",
+  "sourcesContent": ["import type { AgentMode, PlannerConfig, PlannerKind } from \"../shared/contracts\";\n\nconst goal = document.getElementById(\"goal\") as HTMLTextAreaElement;\nconst mode = document.getElementById(\"mode\") as HTMLSelectElement;\nconst planner = document.getElementById(\"planner\") as HTMLSelectElement;\nconst modelRow = document.getElementById(\"model-row\") as HTMLDivElement;\nconst modelId = document.getElementById(\"modelId\") as HTMLInputElement;\nconst status = document.getElementById(\"status\") as HTMLDivElement;\n\nconst start = document.getElementById(\"start\") as HTMLButtonElement;\nconst approve = document.getElementById(\"approve\") as HTMLButtonElement;\nconst stop = document.getElementById(\"stop\") as HTMLButtonElement;\n\nasync function withActiveTab<T>(fn: (tabId: number) => Promise<T>) {\n  const [tab] = await chrome.tabs.query({ active: true, currentWindow: true });\n  if (!tab?.id) {\n    throw new Error(\"No active tab found\");\n  }\n  return fn(tab.id);\n}\n\nfunction syncModelInputVisibility() {\n  const isWebLLM = planner.value === \"webllm\";\n  modelRow.classList.toggle(\"hidden\", !isWebLLM);\n}\n\nplanner.addEventListener(\"change\", syncModelInputVisibility);\nsyncModelInputVisibility();\n\nstart.addEventListener(\"click\", async () => {\n  try {\n    status.textContent = \"Starting...\";\n    const plannerKind = planner.value as PlannerKind;\n    const plannerConfig: PlannerConfig = {\n      kind: plannerKind\n    };\n\n    if (plannerKind === \"webllm\") {\n      const trimmedModelId = modelId.value.trim();\n      if (trimmedModelId) {\n        plannerConfig.modelId = trimmedModelId;\n      }\n    }\n\n    await withActiveTab((tabId) =>\n      chrome.runtime.sendMessage({\n        type: \"START_AGENT\",\n        tabId,\n        goal: goal.value.trim(),\n        mode: mode.value as AgentMode,\n        planner: plannerConfig\n      })\n    );\n    status.textContent = \"Agent started\";\n  } catch (error) {\n    status.textContent = `Error: ${String(error)}`;\n  }\n});\n\napprove.addEventListener(\"click\", async () => {\n  await withActiveTab((tabId) => chrome.runtime.sendMessage({ type: \"APPROVE_ACTION\", tabId }));\n  status.textContent = \"Approved pending action\";\n});\n\nstop.addEventListener(\"click\", async () => {\n  await withActiveTab((tabId) => chrome.runtime.sendMessage({ type: \"STOP_AGENT\", tabId }));\n  status.textContent = \"Stopped\";\n});\n\nchrome.runtime.sendMessage({ type: \"GET_STATUS\" }, (resp) => {\n  if (resp?.status) {\n    status.textContent = resp.status;\n  }\n});\n"],
+  "mappings": ";AAEA,IAAM,OAAO,SAAS,eAAe,MAAM;AAC3C,IAAM,OAAO,SAAS,eAAe,MAAM;AAC3C,IAAM,UAAU,SAAS,eAAe,SAAS;AACjD,IAAM,WAAW,SAAS,eAAe,WAAW;AACpD,IAAM,UAAU,SAAS,eAAe,SAAS;AACjD,IAAM,SAAS,SAAS,eAAe,QAAQ;AAE/C,IAAM,QAAQ,SAAS,eAAe,OAAO;AAC7C,IAAM,UAAU,SAAS,eAAe,SAAS;AACjD,IAAM,OAAO,SAAS,eAAe,MAAM;AAE3C,eAAe,cAAiB,IAAmC;AACjE,QAAM,CAAC,GAAG,IAAI,MAAM,OAAO,KAAK,MAAM,EAAE,QAAQ,MAAM,eAAe,KAAK,CAAC;AAC3E,MAAI,CAAC,KAAK,IAAI;AACZ,UAAM,IAAI,MAAM,qBAAqB;AAAA,EACvC;AACA,SAAO,GAAG,IAAI,EAAE;AAClB;AAEA,SAAS,2BAA2B;AAClC,QAAM,WAAW,QAAQ,UAAU;AACnC,WAAS,UAAU,OAAO,UAAU,CAAC,QAAQ;AAC/C;AAEA,QAAQ,iBAAiB,UAAU,wBAAwB;AAC3D,yBAAyB;AAEzB,MAAM,iBAAiB,SAAS,YAAY;AAC1C,MAAI;AACF,WAAO,cAAc;AACrB,UAAM,cAAc,QAAQ;AAC5B,UAAM,gBAA+B;AAAA,MACnC,MAAM;AAAA,IACR;AAEA,QAAI,gBAAgB,UAAU;AAC5B,YAAM,iBAAiB,QAAQ,MAAM,KAAK;AAC1C,UAAI,gBAAgB;AAClB,sBAAc,UAAU;AAAA,MAC1B;AAAA,IACF;AAEA,UAAM;AAAA,MAAc,CAAC,UACnB,OAAO,QAAQ,YAAY;AAAA,QACzB,MAAM;AAAA,QACN;AAAA,QACA,MAAM,KAAK,MAAM,KAAK;AAAA,QACtB,MAAM,KAAK;AAAA,QACX,SAAS;AAAA,MACX,CAAC;AAAA,IACH;AACA,WAAO,cAAc;AAAA,EACvB,SAAS,OAAO;AACd,WAAO,cAAc,UAAU,OAAO,KAAK,CAAC;AAAA,EAC9C;AACF,CAAC;AAED,QAAQ,iBAAiB,SAAS,YAAY;AAC5C,QAAM,cAAc,CAAC,UAAU,OAAO,QAAQ,YAAY,EAAE,MAAM,kBAAkB,MAAM,CAAC,CAAC;AAC5F,SAAO,cAAc;AACvB,CAAC;AAED,KAAK,iBAAiB,SAAS,YAAY;AACzC,QAAM,cAAc,CAAC,UAAU,OAAO,QAAQ,YAAY,EAAE,MAAM,cAAc,MAAM,CAAC,CAAC;AACxF,SAAO,cAAc;AACvB,CAAC;AAED,OAAO,QAAQ,YAAY,EAAE,MAAM,aAAa,GAAG,CAAC,SAAS;AAC3D,MAAI,MAAM,QAAQ;AAChB,WAAO,cAAc,KAAK;AAAA,EAC5B;AACF,CAAC;",
   "names": []
 }

package/dist/types/core/prompt.d.ts

@@ -0,0 +1,3 @@
+import type { PlannerInput } from "../shared/contracts";
+export declare function buildSystemPrompt(customPrompt?: string): string;
+export declare function buildUserMessage(input: PlannerInput): string;

package/dist/types/core/webllm-bridge.d.ts

@@ -0,0 +1,33 @@
+import type { PlannerInput, PlannerResult } from "../shared/contracts";
+export declare const INVALID_JSON_RETRY_MESSAGE = "Invalid JSON. Reply with only a valid JSON object.";
+type ChatRole = "system" | "user" | "assistant";
+type ChatMessage = {
+    role: ChatRole;
+    content: string;
+};
+type CompletionRequest = {
+    messages: ChatMessage[];
+    temperature?: number;
+    max_tokens?: number;
+    model?: string;
+};
+type CompletionResponse = {
+    choices?: Array<{
+        message?: {
+            content?: unknown;
+        };
+    }>;
+};
+export type WebLLMEngineLike = {
+    chat: {
+        completions: {
+            create(request: CompletionRequest): Promise<CompletionResponse>;
+        };
+    };
+};
+export type BrowserAgentWebLLMBridge = {
+    plan(input: PlannerInput, modelId?: string): Promise<PlannerResult>;
+    retryInvalidJson(input: PlannerInput, badOutput: string, modelId?: string): Promise<PlannerResult>;
+};
+export declare function createWebLLMBridge(engine: WebLLMEngineLike): BrowserAgentWebLLMBridge;
+export {};

package/dist/types/lib/index.d.ts

@@ -20,5 +20,7 @@ export declare class BrowserAgent {
     private delay;
 }
 export declare function createBrowserAgent(config: LibraryAgentConfig, events?: LibraryAgentEvents): BrowserAgent;
+export { createWebLLMBridge } from "../core/webllm-bridge";
 export { parseAction, parsePlannerResult } from "../shared/parse-action";
+export type { BrowserAgentWebLLMBridge, WebLLMEngineLike } from "../core/webllm-bridge";
 export type { AgentAction, AgentMode, AgentSession, ContentResult, LibraryAgentConfig, LibraryAgentEvents, PlannerConfig, PlannerInput, PlannerKind, PlannerResult, RiskLevel } from "../shared/contracts";

package/dist/types/shared/parse-action.d.ts

@@ -10,7 +10,8 @@ export declare function parseAction(raw: string): AgentAction;
  * Parse a full PlannerResult from raw LLM output.
  *
  * Accepts the reflection+action format:
- *   { "evaluation": "...", "memory": "...", "next_goal": "...", "action": { ... } }
+ *   { "evaluation": "...", "memory": "...", "nextGoal": "...", "action": { ... } }
+ * Also supports legacy `next_goal` key for backward compatibility.
  *
  * Also accepts a bare AgentAction for backward compatibility with simple bridges.
  */

package/docs/ARCHITECTURE.md

@@ -61,4 +61,4 @@ window.__browserAgentWebLLM = {
 - No persistent long-term memory yet
 - No task DSL/skills registry yet
 - Risk scoring is simple keyword heuristic
-- No robust selector healing yet
+- Selector healing is basic (attribute fallback + single-element shortcut)

package/docs/EMBEDDING.md

@@ -53,22 +53,11 @@ To use planner mode `webllm`, load the WebLLM engine and wire the bridge before
 
 ```ts
 import * as webllm from "@mlc-ai/web-llm";
+import { createBrowserAgent, createWebLLMBridge } from "@akshayram1/omnibrowser-agent";
 
 const engine = await webllm.CreateMLCEngine("Llama-3.2-1B-Instruct-q4f16_1-MLC");
 
-window.__browserAgentWebLLM = {
-  async plan(input, modelId) {
-    const resp = await engine.chat.completions.create({
-      messages: [
-        { role: "system", content: "Output only a JSON AgentAction object." },
-        { role: "user",   content: `Goal: ${input.goal}\nHistory: ${input.history.join(", ")}` }
-      ],
-      temperature: 0,
-      max_tokens: 100
-    });
-    return JSON.parse(resp.choices[0].message.content);
-  }
-};
+window.__browserAgentWebLLM = createWebLLMBridge(engine);
 
 const agent = createBrowserAgent({
   goal: "Fill the contact form",
@@ -82,4 +71,4 @@ await agent.start();
 
 - For production, mount this inside an authenticated app shell and add your own permission checks.
 - `human-approved` mode is recommended for CRM/finance/admin actions.
-- The WebLLM bridge is not bundled — bring your own engine instance and wire it to `window.__browserAgentWebLLM`.
+- Bring your own WebLLM engine instance, then wire `createWebLLMBridge(engine)` to `window.__browserAgentWebLLM`.

package/docs/ROADMAP.md

@@ -16,19 +16,14 @@
 
 ## v0.3
 
-- Site profile + policy engine (allowlist, blocked domains)
-- Selector healing and fallback strategy
-- Session memory and action replay log
-- Drupal CRM starter skills
-
-## v0.3
-
-- Long-term encrypted memory in IndexedDB
-- Goal decomposition planner (multi-step task graphs)
-- Multi-tab workflows
+- Expanded WebLLM model catalog (new 7B/8B options + compatibility matrix)
+- Improved model loading UX (recommended presets by speed/quality and device memory)
+- Enhanced default system prompts for safer, clearer multi-step planning
+- Prompt presets for common workflows (docs navigation, CRM form fill, task automation)
 
 ## v1.0
 
-- Stable plugin API for site skills
-- Validation/eval harness with benchmark tasks
-- Cross-browser packaging (Chromium + Firefox)
+- Advanced prompt orchestration (goal-aware system prompt routing and contextual guardrails)
+- Functionality expansion: richer action toolkit and stronger extraction/navigation reliability
+- Adaptive planner behaviour (model-aware retries, fallback strategies, and recovery flows)
+- Evaluation suite for prompt and model quality across benchmark browser tasks

package/docs/arch.md

@@ -90,7 +90,7 @@ These three modules are **shared** between the extension content script and the
 | `executor.ts` | Performs DOM actions and returns a result string |
 
 **`observer.ts` — `collectSnapshot()`**
-Queries all interactive elements (`a`, `button`, `input`, `textarea`, `select`, `[role=button]`, `[contenteditable]`), filters out invisible ones (hidden, `display:none`, zero dimensions), and prioritises in-viewport elements. Resolves accessible labels via `aria-labelledby`, `aria-label`, `for/id`, and wrapping `<label>`. Caps at 60 candidates. Returns `url`, `title`, `textPreview`, and `candidates[]`.
+Queries all interactive elements (`a`, `button`, `input`, `textarea`, `select`, `[role=button]`, `[contenteditable]`), filters out invisible ones (hidden, `display:none`, zero dimensions), and prioritises in-viewport elements. Resolves accessible labels via `aria-labelledby`, `aria-label`, `for/id`, and wrapping `<label>`. Generates stable CSS selectors preferring `name`, `placeholder`, and `aria-label` attributes over fragile `:nth-of-type()` indices. Caps at 60 candidates. Returns `url`, `title`, `textPreview`, and `candidates[]`.
 
 **`planner.ts` — `planNextAction()`**
 Two modes:
@@ -98,7 +98,7 @@ Two modes:
 - *WebLLM* — delegates to `window.__browserAgentWebLLM.plan()`. The bridge is external — you wire it in. Accepts both legacy `AgentAction` returns and the new `PlannerResult` (with `evaluation`, `memory`, `nextGoal` reflection fields).
 
 **`executor.ts` — `executeAction()`**
-Performs the action. Uses `InputEvent` with `bubbles: true` so React/Vue controlled inputs receive proper framework events. Verifies: element exists, is not disabled (for clicks), value updated (for type), extracted text is non-empty. Throws on failure so the retry loop can feed `lastError` back to the planner.
+Performs the action. Uses `InputEvent` with `bubbles: true` so React/Vue controlled inputs receive proper framework events. Verifies: element exists, is not disabled (for clicks), value updated (for type), extracted text is non-empty. Includes selector fallback: when a selector fails, tries to recover via tag+attribute matching or single-element shortcut before throwing. Throws on failure so the retry loop can feed `lastError` back to the planner.
 
 ---