harnery 0.40.0 → 0.42.0

package/src/lazy-commands.ts

@@ -256,7 +256,17 @@ function selectedCommand(
     if (token === "--") return undefined;
     if (token.startsWith("-")) {
       const option = findRootOption(program, token);
-      if (option?.required || (option?.optional && !args[index + 1]?.startsWith("-"))) index++;
+      // `--format=json` carries its value inline, so the next token is the
+      // command, not the value. Skipping it here consumed the command name and
+      // the wrong bundle (or none) was materialized, which surfaced downstream
+      // as empty output or a bogus "too many arguments" error.
+      const inlineValue = token.includes("=");
+      if (
+        !inlineValue &&
+        (option?.required || (option?.optional && !args[index + 1]?.startsWith("-")))
+      ) {
+        index++;
+      }
       continue;
     }
     if (token === "help") return state.byName.get(args[index + 1] ?? "");

package/src/lib/agent-browser/client.ts

@@ -1,6 +1,7 @@
 import { type SpawnSyncOptions, spawnSync } from "node:child_process";
 import { writeFileSync } from "node:fs";
 import type { CookieJar, Cookie as JarCookie } from "../cookies/index.ts";
+import { PaceGate, pacePolicyFromEnv } from "../pace/index.ts";
 
 /**
  * Thin wrapper over Vercel Labs' `agent-browser` Rust CLI.
@@ -34,6 +35,11 @@ export interface AgentBrowserOptions {
    * agent-browser session. Defaults to a tmp file derived from the jar.
    */
   stateFilePath?: string;
+  /**
+   * Human-pace gate applied before every `open`. Unset resolves the default
+   * from the `HARNERY_PACE*` environment; `null` disables pacing.
+   */
+  pace?: PaceGate | null;
 }
 
 export interface ExecResult {
@@ -48,11 +54,21 @@ const DEFAULT_TIMEOUT_MS = 60_000;
 export class AgentBrowser {
   private readonly opts: AgentBrowserOptions;
   private cookiesSeeded = false;
+  private paceGate: PaceGate | null | undefined;
 
   constructor(opts: AgentBrowserOptions = {}) {
     this.opts = opts;
   }
 
+  /** Block for the human-pace slot before a page load. Synchronous like the rest of this client. */
+  private paceBefore(url: string): void {
+    if (this.paceGate === undefined) {
+      this.paceGate =
+        this.opts.pace === undefined ? new PaceGate(pacePolicyFromEnv()) : this.opts.pace;
+    }
+    this.paceGate?.beforeSync(url);
+  }
+
   /**
    * Run an `agent-browser` subcommand. Returns a structured result.
    * Throws only on spawn failure (e.g., binary not found); non-zero exit
@@ -132,6 +148,7 @@ export class AgentBrowser {
 
   open(url: string, timeoutMs?: number): ExecResult {
     this.seedCookies();
+    this.paceBefore(url);
     return this.execOrThrow(["open", url], timeoutMs);
   }
 
@@ -183,6 +200,7 @@ export class AgentBrowser {
       // Naive shell-like splitting; agent-browser's own argv parser handles
       // the actual command; we just split on whitespace for the wrapper.
       const args = trimmed.split(/\s+/);
+      if (args[0] === "open" && args[1]) this.paceBefore(args[1]);
       results.push(this.exec(args));
     }
     return results;

package/src/lib/browser/batch-steps.ts

@@ -0,0 +1,79 @@
+/**
+ * Split a `browse --batch` string into steps.
+ *
+ * Steps are separated by `;`. Two rules keep JavaScript intact:
+ *
+ * 1. A step that starts with the `eval` verb ends only at a `;` that sits
+ *    outside JavaScript quotes ('...', "...", `...`) and outside (), [] and {}.
+ *    So `eval (() => { const a = 1; return a })(); wait 500` is two steps.
+ * 2. Every other step ends at the next `;`, exactly as before. Selectors and
+ *    fill values may contain apostrophes or brackets that are not balanced
+ *    JavaScript, so they get no quote or bracket tracking.
+ *
+ * `\;` always inserts a literal `;` without ending the step, in any verb.
+ * A top-level `;` inside an eval (two bare statements) still ends the step:
+ * wrap multiple statements in a block or an arrow function.
+ */
+export function splitBatchSteps(input: string): string[] {
+  const steps: string[] = [];
+  let buf = "";
+  let stepStart = true;
+  let jsAware = false;
+  let quote: string | null = null;
+  let depth = 0;
+
+  const endStep = () => {
+    if (buf.trim()) steps.push(buf);
+    buf = "";
+    stepStart = true;
+    jsAware = false;
+    quote = null;
+    depth = 0;
+  };
+
+  for (let i = 0; i < input.length; i++) {
+    if (stepStart) {
+      // Decide the splitting rule once per step from its leading verb.
+      jsAware = /^\s*eval(\s|$)/i.test(input.slice(i));
+      stepStart = false;
+    }
+    const ch = input[i] as string;
+    if (ch === "\\" && input[i + 1] === ";") {
+      buf += ";";
+      i++;
+      continue;
+    }
+    if (jsAware) {
+      if (quote) {
+        buf += ch;
+        if (ch === "\\" && i + 1 < input.length) {
+          buf += input[i + 1];
+          i++;
+        } else if (ch === quote) {
+          quote = null;
+        }
+        continue;
+      }
+      if (ch === "'" || ch === '"' || ch === "`") {
+        quote = ch;
+        buf += ch;
+        continue;
+      }
+      if (ch === "(" || ch === "[" || ch === "{") depth++;
+      else if (ch === ")" || ch === "]" || ch === "}") depth = Math.max(0, depth - 1);
+      if (ch === ";" && depth === 0) {
+        endStep();
+        continue;
+      }
+      buf += ch;
+      continue;
+    }
+    if (ch === ";") {
+      endStep();
+      continue;
+    }
+    buf += ch;
+  }
+  if (buf.trim()) steps.push(buf);
+  return steps;
+}

package/src/lib/browser/client.ts

@@ -16,6 +16,7 @@ import {
   CookieStoreParseError,
   type Cookie as JarCookie,
 } from "../cookies/index.ts";
+import { PaceGate, pacePolicyFromEnv } from "../pace/index.ts";
 import { type AssertResult, type AssertSpec, buildAssertCheck } from "./asserts.js";
 import { stitchPngRows } from "./capture-fidelity.js";
 import {
@@ -33,6 +34,8 @@ import {
   type LayoutLintRequest,
   type LayoutLintResult,
 } from "./geometry.js";
+import { browserIdentity, installDocumentIdentity } from "./identity.js";
+import { AUTOMATION_DEFAULT_ARGS_TO_DROP, automationDisguiseArgs } from "./launch-args.js";
 import {
   buildClearLayoutAnnotationsScript,
   buildLayoutAnnotateScript,
@@ -91,7 +94,7 @@ export interface BrowserOptions {
   /** Cookie jar to seed/sync with. Pass `null` to skip jar entirely. */
   jar?: CookieJar | null;
   /** Viewport. Default 1280x800. */
-  viewport?: { width: number; height: number };
+  viewport?: { width: number; height: number } | null;
   /**
    * Device scale factor (DPR) for the context. Unset keeps Playwright's
    * default of 1, so screenshots are CSS-pixel sized. A higher value renders
@@ -128,6 +131,13 @@ export interface BrowserOptions {
    * via this callback (e.g., a Cloudflare-bypass header for specific zones).
    */
   extraHeaders?: (url: string) => Record<string, string>;
+  /**
+   * Human-pace gate applied before every page load: `navigate`, `reload`, and
+   * the session tab verbs. Unset resolves the default from the `HARNERY_PACE*`
+   * environment (on, 3 to 9 seconds between loads of one site). `null`
+   * disables pacing for this client.
+   */
+  pace?: PaceGate | null;
   /**
    * Extra Chromium command-line flags, passed through to Playwright's
    * `launchPersistentContext` `args`. Used for environment-specific
@@ -136,6 +146,28 @@ export interface BrowserOptions {
    * defaults only.
    */
   launchArgs?: string[];
+  /**
+   * Playwright browser channel. `"chrome"` launches the operator's installed
+   * Google Chrome instead of the bundled "Chrome for Testing" build. Unset or
+   * `"chromium"` keeps the bundled browser.
+   */
+  channel?: "chrome" | "chrome-beta" | "msedge" | "chromium";
+  /**
+   * Drop Playwright's `--enable-automation` default, disable Blink's
+   * AutomationControlled feature, and run with the Chromium sandbox on (no
+   * `--no-sandbox`), so pages do not see `navigator.webdriver`, the automation
+   * infobar, or the unsupported-flag banner. Headed sessions set this; sites
+   * that screen for automation otherwise refuse clicks a person makes in the
+   * window.
+   */
+  hideAutomation?: boolean;
+  /**
+   * User agent to present. Set by `browse` from `lib/browser/user-agent`
+   * (never Linux, never HeadlessChrome). Applied both as the Playwright
+   * context override and as `--user-agent` so workers and frames agree.
+   * Unset keeps the browser's native string.
+   */
+  userAgent?: string;
   /** Authenticated browser proxy passed directly to Playwright. */
   proxy?: { server: string; username?: string; password?: string };
 }
@@ -306,6 +338,7 @@ export class Browser {
   private consoleEvents: ConsoleEvent[] = [];
   private pageErrors: PageErrorEvent[] = [];
   private failedRequests: FailedRequest[] = [];
+  private identityReady = new WeakMap<Page, Promise<void>>();
 
   constructor(private opts: BrowserOptions = {}) {
     this.profileDir = opts.profileDir ?? DEFAULT_PROFILE;
@@ -326,20 +359,54 @@ export class Browser {
    */
   private async openOnce(): Promise<void> {
     const jarCookies = this.opts.jar?.list() ?? [];
+    const launchIdentity = this.opts.userAgent ? browserIdentity(this.opts.userAgent) : undefined;
+    const launchArgs = [
+      ...new Set([
+        ...(this.opts.headed && this.opts.viewport == null ? ["--window-size=1536,864"] : []),
+        ...(this.opts.launchArgs ?? []),
+        ...(this.opts.hideAutomation ? automationDisguiseArgs() : []),
+        ...(this.opts.userAgent ? [`--user-agent=${this.opts.userAgent}`] : []),
+      ]),
+    ];
 
     // Hold a launch slot only across the spawn itself, not the browser's life.
     await acquireLaunchSlot();
     try {
       this.context = await chromium.launchPersistentContext(this.profileDir, {
         headless: !this.opts.headed,
-        viewport: this.opts.viewport ?? { width: 1280, height: 800 },
+        // Service-worker-owned navigations bypass Playwright request routes.
+        ...(this.getPaceGate()?.policy.enabled ? { serviceWorkers: "block" as const } : {}),
+        viewport:
+          this.opts.viewport !== undefined
+            ? this.opts.viewport
+            : this.opts.headed
+              ? null
+              : { width: 1280, height: 800 },
         ...(this.opts.deviceScaleFactor !== undefined
           ? { deviceScaleFactor: this.opts.deviceScaleFactor }
           : {}),
         ...(this.opts.colorScheme ? { colorScheme: this.opts.colorScheme } : {}),
+        ...(this.opts.userAgent ? { userAgent: this.opts.userAgent } : {}),
+        ...(launchIdentity
+          ? {
+              extraHTTPHeaders: {
+                "sec-ch-ua-platform": JSON.stringify(launchIdentity.userAgentMetadata.platform),
+                "sec-ch-ua-mobile": "?0",
+                "sec-ch-ua": launchIdentity.userAgentMetadata.brands
+                  .map(
+                    ({ brand, version }) => `${JSON.stringify(brand)};v=${JSON.stringify(version)}`,
+                  )
+                  .join(", "),
+              },
+            }
+          : {}),
         ...(this.opts.launchTimeout !== undefined ? { timeout: this.opts.launchTimeout } : {}),
-        ...(this.opts.launchArgs && this.opts.launchArgs.length > 0
-          ? { args: this.opts.launchArgs }
+        ...(launchArgs.length > 0 ? { args: launchArgs } : {}),
+        ...(this.opts.channel && this.opts.channel !== "chromium"
+          ? { channel: this.opts.channel }
+          : {}),
+        ...(this.opts.hideAutomation
+          ? { ignoreDefaultArgs: [...AUTOMATION_DEFAULT_ARGS_TO_DROP], chromiumSandbox: true }
           : {}),
         ...(this.opts.proxy ? { proxy: this.opts.proxy } : {}),
         ...(this.opts.recordHarPath
@@ -350,18 +417,55 @@ export class Browser {
       releaseLaunchSlot();
     }
     this.context.setDefaultNavigationTimeout(this.opts.navigationTimeout ?? 30_000);
+    const identity = this.opts.userAgent
+      ? browserIdentity(this.opts.userAgent, this.context.browser()?.version())
+      : undefined;
+    if (identity) await this.context.addInitScript(installDocumentIdentity, identity);
 
     if (jarCookies.length > 0) {
       await this.context.addCookies(jarCookies.map(toPWCookie));
     }
 
-    // Caller-injected extraHeaders callback (e.g., for Cloudflare-bypass
-    // or custom auth headers). Per-request route handler so headers only
-    // attach when the callback returns non-empty.
+    // Gate actual document requests, including links, forms, script navigation,
+    // and the first request of a popup. Method-level waits miss those loads.
+    // Redirect hops are part of the original load, not additional reservations.
     const headersCb = this.opts.extraHeaders;
-    if (headersCb) {
+    if (this.getPaceGate()?.policy.enabled || headersCb || this.opts.userAgent) {
       await this.context.route("**/*", async (route, request) => {
-        const extra = headersCb(request.url());
+        let mainDocument = false;
+        if (request.isNavigationRequest() && !request.redirectedFrom()) {
+          try {
+            mainDocument = request.frame() === request.frame().page().mainFrame();
+          } catch {
+            // A popup's first request can precede creation of its Page/Frame.
+            mainDocument = true;
+          }
+        }
+        if (mainDocument) {
+          try {
+            await this.getPaceGate()?.before(request.url());
+          } catch {
+            await route.abort("failed");
+            return;
+          }
+        }
+        const extra = { ...headersCb?.(request.url()) };
+        // Popup request headers are already computed before its CDP session is
+        // available. Keep negotiated hints consistent on that first request too.
+        if (identity) {
+          const metadata = identity.userAgentMetadata;
+          extra["sec-ch-ua-platform"] = JSON.stringify(metadata.platform);
+          extra["sec-ch-ua-mobile"] = "?0";
+          extra["sec-ch-ua"] = metadata.brands
+            .map(({ brand, version }) => `${JSON.stringify(brand)};v=${JSON.stringify(version)}`)
+            .join(", ");
+          for (const [header, value] of Object.entries({
+            "sec-ch-ua-platform-version": metadata.platformVersion,
+            "sec-ch-ua-arch": metadata.architecture,
+            "sec-ch-ua-bitness": metadata.bitness,
+          }))
+            if (header in request.headers()) extra[header] = JSON.stringify(value);
+        }
         if (Object.keys(extra).length === 0) return route.continue();
         const headers = { ...request.headers(), ...extra };
         return route.continue({ headers });
@@ -370,6 +474,7 @@ export class Browser {
 
     const pages = this.context.pages();
     const firstPage = pages[0] ?? (await this.context.newPage());
+    await Promise.all(this.context.pages().map((page) => this.preparePageIdentity(page)));
     for (const page of this.context.pages()) this.trackPage(page, page === firstPage);
     this.context.on("page", (page) => this.trackPage(page, true));
   }
@@ -491,6 +596,10 @@ export class Browser {
         type: "unknown",
       });
       this.attachDiagnosticListeners(page);
+      void this.preparePageIdentity(page).catch((error) => {
+        if (!page.isClosed())
+          this.pageErrors.push({ message: `Browser identity setup failed: ${String(error)}` });
+      });
       page.on("domcontentloaded", () => this.recordDocumentNavigation(page));
       page.once("close", () => {
         this.pageRecency.delete(page);
@@ -501,6 +610,23 @@ export class Browser {
     if (makeActive) this.setActivePage(page);
   }
 
+  private preparePageIdentity(page: Page): Promise<void> {
+    if (!this.opts.userAgent || !this.context) return Promise.resolve();
+    let ready = this.identityReady.get(page);
+    if (!ready) {
+      const context = this.context;
+      const userAgent = this.opts.userAgent;
+      ready = (async () => {
+        const identity = browserIdentity(userAgent, context.browser()?.version());
+        if (!identity) return;
+        const session = await context.newCDPSession(page);
+        await session.send("Emulation.setUserAgentOverride", identity);
+      })();
+      this.identityReady.set(page, ready);
+    }
+    return ready;
+  }
+
   private setActivePage(page: Page): void {
     if (page.isClosed()) return;
     this.page = page;
@@ -569,8 +695,19 @@ export class Browser {
     };
   }
 
+  private paceGate: PaceGate | null | undefined;
+
+  private getPaceGate(): PaceGate | null {
+    if (this.paceGate === undefined) {
+      this.paceGate =
+        this.opts.pace === undefined ? new PaceGate(pacePolicyFromEnv()) : this.opts.pace;
+    }
+    return this.paceGate;
+  }
+
   async navigate(url: string): Promise<NavigateResult> {
     const page = this.currentPage;
+    await this.preparePageIdentity(page);
     const response = await page.goto(url, { waitUntil: this.opts.waitUntil ?? "load" });
     return {
       url: page.url(),
@@ -1381,6 +1518,7 @@ export class Browser {
     const page = await this.context.newPage();
     this.trackPage(page, true);
     try {
+      await this.preparePageIdentity(page);
       await page.goto(url, { waitUntil: this.opts.waitUntil ?? "load" });
       this.sessionRevision++;
       return await this.describeTab(page);