browser-pilot 0.0.7 → 0.0.9

package/README.md

@@ -34,6 +34,7 @@ await browser.close();
 | Single-selector API (fragile) | Multi-selector by default: `['#primary', '.fallback']` |
 | No action batching (high latency) | Batch DSL: one call for entire sequences |
 | No AI-optimized snapshots | Built-in accessibility tree extraction |
+| No audio I/O for voice agents | Mic injection + output capture + Whisper transcription |
 
 ## Installation
 
@@ -246,6 +247,26 @@ await page.setLocale('fr-FR');
 
 Devices: `iPhone 14`, `iPhone 14 Pro Max`, `Pixel 7`, `iPad Pro 11`, `Desktop Chrome`, `Desktop Firefox`
 
+### Audio I/O
+
+```typescript
+// Set up audio input/output interception
+await page.setupAudio();
+
+// Play audio into the page's fake microphone
+await page.audioInput.play(wavBytes, { waitForEnd: true });
+
+// Capture audio output until silence
+const capture = await page.audioOutput.captureUntilSilence({ silenceTimeout: 5000 });
+
+// Full round-trip: play input → capture response
+const result = await page.audioRoundTrip({ input: wavBytes, silenceTimeout: 5000 });
+
+// Transcribe captured audio (requires OPENAI_API_KEY)
+import { transcribe } from 'browser-pilot';
+const { text } = await transcribe(capture);
+```
+
 ### Request Interception
 
 ```typescript
@@ -378,7 +399,7 @@ bp exec '[
   {"action":"fill","selector":"ref:e5","value":"laptop"},
   {"action":"click","selector":"ref:e12"},
   {"action":"snapshot"}
-]' --output json
+]' --format json
 ```
 
 Multi-selector fallbacks for robustness:
@@ -403,6 +424,44 @@ Output:
 
 Run `bp actions` for complete action reference.
 
+### Voice Agent Testing
+
+Test audio-based AI apps (voice assistants, phone agents) by injecting microphone input and capturing spoken responses.
+
+> **Full guide:** [Voice Agent Testing Guide](./docs/guides/voice-agent-testing.md)
+
+```bash
+export OPENAI_API_KEY=sk-...  # Required for --transcribe
+
+# Validate audio pipeline
+bp audio check -s my-session
+# Output: "READY for roundtrip" with agent AudioContext detected
+
+# Full round-trip: send audio prompt → wait for response → transcribe
+bp audio roundtrip -i prompt.wav --transcribe --silence-timeout 1500
+# Output: { "transcript": "Welcome! I'd be happy to help...", "latencyMs": 5200, ... }
+
+# Save response audio for manual review
+bp audio roundtrip -i prompt.wav -o response.wav --transcribe
+```
+
+**Important:** Audio overrides must be injected before the voice agent initializes. Use `bp audio check` to validate the pipeline. See the [full guide](./docs/guides/voice-agent-testing.md) for setup order and troubleshooting.
+
+Programmatic API:
+
+```typescript
+await page.setupAudio();
+
+const result = await page.audioRoundTrip({
+  input: audioBytes,
+  silenceTimeout: 1500,
+});
+
+import { transcribe } from 'browser-pilot';
+const { text } = await transcribe(result.audio);
+console.log(text); // "Welcome! I'd be happy to help..."
+```
+
 ### Recording Browser Actions
 
 Record human interactions to create automation recipes:
@@ -573,6 +632,7 @@ See the [docs](./docs) folder for detailed documentation:
 - [Multi-Selector Guide](./docs/guides/multi-selector.md)
 - [Batch Actions](./docs/guides/batch-actions.md)
 - [Snapshots](./docs/guides/snapshots.md)
+- [Voice Agent Testing](./docs/guides/voice-agent-testing.md)
 - [CLI Reference](./docs/cli.md)
 - [API Reference](./docs/api/page.md)
 

package/dist/actions.cjs

@@ -21,10 +21,24 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 var actions_exports = {};
 __export(actions_exports, {
   BatchExecutor: () => BatchExecutor,
-  addBatchToPage: () => addBatchToPage
+  addBatchToPage: () => addBatchToPage,
+  validateSteps: () => validateSteps
 });
 module.exports = __toCommonJS(actions_exports);
 
+// src/browser/types.ts
+var ElementNotFoundError = class extends Error {
+  selectors;
+  hints;
+  constructor(selectors, hints) {
+    const selectorList = Array.isArray(selectors) ? selectors : [selectors];
+    super(`Element not found: ${selectorList.join(", ")}`);
+    this.name = "ElementNotFoundError";
+    this.selectors = selectorList;
+    this.hints = hints;
+  }
+};
+
 // src/actions/executor.ts
 var DEFAULT_TIMEOUT = 3e4;
 var BatchExecutor = class {
@@ -56,13 +70,15 @@ var BatchExecutor = class {
         });
       } catch (error) {
         const errorMessage = error instanceof Error ? error.message : String(error);
+        const hints = error instanceof ElementNotFoundError ? error.hints : void 0;
         results.push({
           index: i,
           action: step.action,
           selector: step.selector,
           success: false,
           durationMs: Date.now() - stepStart,
-          error: errorMessage
+          error: errorMessage,
+          hints
         });
         if (onFail === "stop" && !step.optional) {
           return {
@@ -251,17 +267,41 @@ var BatchExecutor = class {
         await this.page.switchToMain();
         return {};
       }
-      default:
-        throw new Error(
-          `Unknown action: ${step.action}. Run 'bp actions' for available actions.`
-        );
+      default: {
+        const action = step.action;
+        const aliases = {
+          execute: "evaluate",
+          navigate: "goto",
+          input: "fill",
+          tap: "click",
+          go: "goto",
+          run: "evaluate",
+          capture: "screenshot",
+          inspect: "snapshot",
+          enter: "press",
+          open: "goto",
+          visit: "goto",
+          eval: "evaluate",
+          js: "evaluate",
+          snap: "snapshot",
+          frame: "switchFrame"
+        };
+        const suggestion = aliases[action.toLowerCase()];
+        const hint = suggestion ? ` Did you mean "${suggestion}"?` : "";
+        const valid = "goto, click, fill, type, select, check, uncheck, submit, press, focus, hover, scroll, wait, snapshot, screenshot, evaluate, text, switchFrame, switchToMain";
+        throw new Error(`Unknown action "${action}".${hint}
+
+Valid actions: ${valid}`);
+      }
     }
   }
   /**
-   * Get the first selector if multiple were provided
-   * (actual used selector tracking would need to be implemented in Page)
+   * Get the actual selector that matched the element.
+   * Uses the last matched selector tracked by Page, falls back to first selector if unavailable.
    */
   getUsedSelector(selector) {
+    const matched = this.page.getLastMatchedSelector();
+    if (matched) return matched;
     return Array.isArray(selector) ? selector[0] : selector;
   }
 };
@@ -271,8 +311,444 @@ function addBatchToPage(page) {
     batch: (steps, options) => executor.execute(steps, options)
   });
 }
+
+// src/actions/validate.ts
+function levenshtein(a, b) {
+  const m = a.length;
+  const n = b.length;
+  const dp = Array.from({ length: m + 1 }, () => Array(n + 1).fill(0));
+  for (let i = 0; i <= m; i++) dp[i][0] = i;
+  for (let j = 0; j <= n; j++) dp[0][j] = j;
+  for (let i = 1; i <= m; i++) {
+    for (let j = 1; j <= n; j++) {
+      dp[i][j] = a[i - 1] === b[j - 1] ? dp[i - 1][j - 1] : 1 + Math.min(dp[i - 1][j], dp[i][j - 1], dp[i - 1][j - 1]);
+    }
+  }
+  return dp[m][n];
+}
+var ACTION_ALIASES = {
+  execute: "evaluate",
+  navigate: "goto",
+  input: "fill",
+  tap: "click",
+  go: "goto",
+  run: "evaluate",
+  capture: "screenshot",
+  inspect: "snapshot",
+  enter: "press",
+  keypress: "press",
+  nav: "goto",
+  open: "goto",
+  visit: "goto",
+  browse: "goto",
+  load: "goto",
+  write: "fill",
+  set: "fill",
+  pick: "select",
+  choose: "select",
+  send: "press",
+  eval: "evaluate",
+  js: "evaluate",
+  script: "evaluate",
+  snap: "snapshot",
+  accessibility: "snapshot",
+  a11y: "snapshot",
+  image: "screenshot",
+  pic: "screenshot",
+  frame: "switchFrame",
+  iframe: "switchFrame"
+};
+var PROPERTY_ALIASES = {
+  expression: "value",
+  href: "url",
+  target: "selector",
+  element: "selector",
+  code: "value",
+  script: "value",
+  src: "url",
+  link: "url",
+  char: "key",
+  text: "value",
+  query: "selector",
+  el: "selector",
+  elem: "selector",
+  css: "selector",
+  xpath: "selector",
+  input: "value",
+  content: "value",
+  keys: "key",
+  button: "key",
+  address: "url",
+  page: "url",
+  path: "url"
+};
+var ACTION_RULES = {
+  goto: {
+    required: { url: { type: "string" } },
+    optional: {}
+  },
+  click: {
+    required: { selector: { type: "string|string[]" } },
+    optional: {
+      waitForNavigation: { type: "boolean" }
+    }
+  },
+  fill: {
+    required: { selector: { type: "string|string[]" }, value: { type: "string" } },
+    optional: {
+      clear: { type: "boolean" },
+      blur: { type: "boolean" }
+    }
+  },
+  type: {
+    required: { selector: { type: "string|string[]" }, value: { type: "string" } },
+    optional: {
+      delay: { type: "number" }
+    }
+  },
+  select: {
+    required: {},
+    optional: {
+      selector: { type: "string|string[]" },
+      value: { type: "string|string[]" },
+      trigger: { type: "string|string[]" },
+      option: { type: "string|string[]" },
+      match: { type: "string", enum: ["text", "value", "contains"] }
+    }
+  },
+  check: {
+    required: { selector: { type: "string|string[]" } },
+    optional: {}
+  },
+  uncheck: {
+    required: { selector: { type: "string|string[]" } },
+    optional: {}
+  },
+  submit: {
+    required: { selector: { type: "string|string[]" } },
+    optional: {
+      method: { type: "string", enum: ["enter", "click", "enter+click"] }
+    }
+  },
+  press: {
+    required: { key: { type: "string" } },
+    optional: {}
+  },
+  focus: {
+    required: { selector: { type: "string|string[]" } },
+    optional: {}
+  },
+  hover: {
+    required: { selector: { type: "string|string[]" } },
+    optional: {}
+  },
+  scroll: {
+    required: {},
+    optional: {
+      selector: { type: "string|string[]" },
+      x: { type: "number" },
+      y: { type: "number" },
+      direction: { type: "string", enum: ["up", "down", "left", "right"] },
+      amount: { type: "number" }
+    }
+  },
+  wait: {
+    required: {},
+    optional: {
+      selector: { type: "string|string[]" },
+      waitFor: {
+        type: "string",
+        enum: ["visible", "hidden", "attached", "detached", "navigation", "networkIdle"]
+      }
+    }
+  },
+  snapshot: {
+    required: {},
+    optional: {}
+  },
+  screenshot: {
+    required: {},
+    optional: {
+      format: { type: "string", enum: ["png", "jpeg", "webp"] },
+      quality: { type: "number" },
+      fullPage: { type: "boolean" }
+    }
+  },
+  evaluate: {
+    required: { value: { type: "string" } },
+    optional: {}
+  },
+  text: {
+    required: {},
+    optional: {
+      selector: { type: "string|string[]" }
+    }
+  },
+  switchFrame: {
+    required: { selector: { type: "string|string[]" } },
+    optional: {}
+  },
+  switchToMain: {
+    required: {},
+    optional: {}
+  }
+};
+var VALID_ACTIONS = Object.keys(ACTION_RULES);
+var VALID_ACTIONS_LIST = VALID_ACTIONS.join(", ");
+var KNOWN_STEP_FIELDS = /* @__PURE__ */ new Set([
+  "action",
+  "selector",
+  "url",
+  "value",
+  "key",
+  "waitFor",
+  "timeout",
+  "optional",
+  "method",
+  "clear",
+  "blur",
+  "delay",
+  "waitForNavigation",
+  "trigger",
+  "option",
+  "match",
+  "x",
+  "y",
+  "direction",
+  "amount",
+  "format",
+  "quality",
+  "fullPage"
+]);
+function resolveAction(name) {
+  if (VALID_ACTIONS.includes(name)) {
+    return { action: name };
+  }
+  const lower = name.toLowerCase();
+  if (ACTION_ALIASES[lower]) {
+    return {
+      action: ACTION_ALIASES[lower],
+      suggestion: `Did you mean "${ACTION_ALIASES[lower]}"?`
+    };
+  }
+  let best = null;
+  let bestDist = Infinity;
+  for (const valid of VALID_ACTIONS) {
+    const dist = levenshtein(lower, valid);
+    if (dist < bestDist) {
+      bestDist = dist;
+      best = valid;
+    }
+  }
+  if (best && bestDist <= 2) {
+    return { action: best, suggestion: `Did you mean "${best}"?` };
+  }
+  return null;
+}
+function suggestProperty(name) {
+  if (PROPERTY_ALIASES[name]) {
+    return PROPERTY_ALIASES[name];
+  }
+  let best = null;
+  let bestDist = Infinity;
+  for (const known of KNOWN_STEP_FIELDS) {
+    if (known === "action") continue;
+    const dist = levenshtein(name, known);
+    if (dist < bestDist) {
+      bestDist = dist;
+      best = known;
+    }
+  }
+  if (best && bestDist <= 2) {
+    return best;
+  }
+  return void 0;
+}
+function checkFieldType(value, rule) {
+  switch (rule.type) {
+    case "string":
+      if (typeof value !== "string") return `expected string, got ${typeof value}`;
+      if (rule.enum && !rule.enum.includes(value)) {
+        return `must be one of: ${rule.enum.join(", ")}`;
+      }
+      return null;
+    case "string|string[]":
+      if (typeof value !== "string" && !Array.isArray(value)) {
+        return `expected string or string[], got ${typeof value}`;
+      }
+      if (Array.isArray(value) && value.some((v) => typeof v !== "string")) {
+        return "array elements must be strings";
+      }
+      return null;
+    case "number":
+      if (typeof value !== "number") return `expected number, got ${typeof value}`;
+      return null;
+    case "boolean":
+      if (typeof value !== "boolean") return `expected boolean, got ${typeof value}`;
+      return null;
+  }
+}
+function validateSteps(steps) {
+  const errors = [];
+  for (let i = 0; i < steps.length; i++) {
+    const step = steps[i];
+    if (!step || typeof step !== "object" || Array.isArray(step)) {
+      errors.push({
+        stepIndex: i,
+        field: "step",
+        message: "step must be a JSON object."
+      });
+      continue;
+    }
+    const obj = step;
+    if (!("action" in obj)) {
+      errors.push({
+        stepIndex: i,
+        field: "action",
+        message: 'missing required "action" field.'
+      });
+      continue;
+    }
+    const actionName = obj["action"];
+    if (typeof actionName !== "string") {
+      errors.push({
+        stepIndex: i,
+        field: "action",
+        message: `"action" must be a string, got ${typeof actionName}.`
+      });
+      continue;
+    }
+    const resolved = resolveAction(actionName);
+    if (!resolved) {
+      errors.push({
+        stepIndex: i,
+        field: "action",
+        message: `unknown action "${actionName}".`,
+        suggestion: `Valid actions: ${VALID_ACTIONS_LIST}`
+      });
+      continue;
+    }
+    if (resolved.suggestion) {
+      errors.push({
+        stepIndex: i,
+        field: "action",
+        message: `unknown action "${actionName}". ${resolved.suggestion}`,
+        suggestion: resolved.suggestion
+      });
+      continue;
+    }
+    const action = resolved.action;
+    const rule = ACTION_RULES[action];
+    for (const key of Object.keys(obj)) {
+      if (key === "action") continue;
+      if (!KNOWN_STEP_FIELDS.has(key)) {
+        const suggestion = suggestProperty(key);
+        errors.push({
+          stepIndex: i,
+          field: key,
+          message: suggestion ? `unknown property "${key}". Did you mean "${suggestion}"?` : `unknown property "${key}".`,
+          suggestion: suggestion ? `Did you mean "${suggestion}"?` : void 0
+        });
+      }
+    }
+    for (const [field, fieldRule] of Object.entries(rule.required)) {
+      if (!(field in obj) || obj[field] === void 0) {
+        errors.push({
+          stepIndex: i,
+          field,
+          message: `missing required "${field}" (${fieldRule.type}).`
+        });
+      } else {
+        const typeErr = checkFieldType(obj[field], fieldRule);
+        if (typeErr) {
+          errors.push({
+            stepIndex: i,
+            field,
+            message: `"${field}" ${typeErr}.`
+          });
+        }
+      }
+    }
+    for (const [field, fieldRule] of Object.entries(rule.optional)) {
+      if (field in obj && obj[field] !== void 0) {
+        const typeErr = checkFieldType(obj[field], fieldRule);
+        if (typeErr) {
+          errors.push({
+            stepIndex: i,
+            field,
+            message: `"${field}" ${typeErr}.`
+          });
+        }
+      }
+    }
+    if ("timeout" in obj && obj["timeout"] !== void 0) {
+      if (typeof obj["timeout"] !== "number") {
+        errors.push({
+          stepIndex: i,
+          field: "timeout",
+          message: `"timeout" expected number, got ${typeof obj["timeout"]}.`
+        });
+      }
+    }
+    if ("optional" in obj && obj["optional"] !== void 0) {
+      if (typeof obj["optional"] !== "boolean") {
+        errors.push({
+          stepIndex: i,
+          field: "optional",
+          message: `"optional" expected boolean, got ${typeof obj["optional"]}.`
+        });
+      }
+    }
+    if (action === "select") {
+      const hasNative = "selector" in obj && "value" in obj;
+      const hasCustom = "trigger" in obj && "option" in obj && "value" in obj;
+      if (!hasNative && !hasCustom) {
+        errors.push({
+          stepIndex: i,
+          field: "selector",
+          message: "select requires either (selector + value) for native select, or (trigger + option + value) for custom select."
+        });
+      }
+    }
+  }
+  return {
+    valid: errors.length === 0,
+    errors,
+    formatted() {
+      if (errors.length === 0) return "";
+      const lines = [`Validation failed (${errors.length} error${errors.length > 1 ? "s" : ""}):`];
+      for (const err of errors) {
+        const stepLabel = err.field === "action" || err.field === "step" ? `Step ${err.stepIndex}` : `Step ${err.stepIndex}`;
+        lines.push("");
+        lines.push(`  ${stepLabel}: ${err.message}`);
+        if (err.suggestion && !err.message.includes(err.suggestion)) {
+          lines.push(`    ${err.suggestion}`);
+        }
+        const step = steps[err.stepIndex];
+        if (step && typeof step === "object") {
+          lines.push(`    Got: ${JSON.stringify(step)}`);
+        }
+      }
+      const hasEvaluateError = errors.some((err) => {
+        const step = steps[err.stepIndex];
+        return step && typeof step === "object" && step["action"] === "evaluate";
+      });
+      if (hasEvaluateError) {
+        lines.push("");
+        lines.push(
+          "Tip: For JavaScript evaluation, use 'bp eval' instead \u2014 no JSON wrapping needed:"
+        );
+        lines.push("  bp eval 'your.expression.here'");
+      }
+      lines.push("");
+      lines.push(`Valid actions: ${VALID_ACTIONS_LIST}`);
+      return lines.join("\n");
+    }
+  };
+}
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   BatchExecutor,
-  addBatchToPage
+  addBatchToPage,
+  validateSteps
 });

package/dist/actions.d.cts

@@ -1,5 +1,5 @@
-import { P as Page, S as Step, B as BatchOptions, b as BatchResult } from './types-TVlTA7nH.cjs';
-export { A as ActionType, c as StepResult } from './types-TVlTA7nH.cjs';
+import { y as Page, S as Step, B as BatchOptions, b as BatchResult } from './types-CYw-7vx1.cjs';
+export { A as ActionType, c as StepResult } from './types-CYw-7vx1.cjs';
 import './client-7Nqka5MV.cjs';
 
 /**
@@ -18,8 +18,8 @@ declare class BatchExecutor {
      */
     private executeStep;
     /**
-     * Get the first selector if multiple were provided
-     * (actual used selector tracking would need to be implemented in Page)
+     * Get the actual selector that matched the element.
+     * Uses the last matched selector tracked by Page, falls back to first selector if unavailable.
      */
     private getUsedSelector;
 }
@@ -30,4 +30,23 @@ declare function addBatchToPage(page: Page): Page & {
     batch: (steps: Step[], options?: BatchOptions) => Promise<BatchResult>;
 };
 
-export { BatchExecutor, BatchOptions, BatchResult, Step, addBatchToPage };
+/**
+ * Step validation for batch executor
+ *
+ * Validates steps before browser connection, catching malformed JSON
+ * from AI agents with actionable, specific feedback.
+ */
+interface ValidationError {
+    stepIndex: number;
+    field: string;
+    message: string;
+    suggestion?: string;
+}
+interface ValidationResult {
+    valid: boolean;
+    errors: ValidationError[];
+    formatted(): string;
+}
+declare function validateSteps(steps: unknown[]): ValidationResult;
+
+export { BatchExecutor, BatchOptions, BatchResult, Step, type ValidationError, type ValidationResult, addBatchToPage, validateSteps };