harnery 0.41.0 → 0.42.0

package/src/lib/browser/collect/scroll-plan.ts

@@ -0,0 +1,179 @@
+/**
+ * Seeded randomness and the human-like scroll schedule for `browse --collect`.
+ *
+ * Everything here is pure: given a seed and a pacing policy, the plan for each
+ * step (wheel notches, pauses, reading pauses, small scroll-backs) is fully
+ * reproducible, which is what the unit tests rely on.
+ */
+
+/** A random source returning floats in [0, 1). */
+export type Rng = () => number;
+
+/** mulberry32: a small, fast, well-distributed 32-bit seeded generator. */
+export function createRng(seed: number): Rng {
+  let a = seed >>> 0;
+  return () => {
+    a = (a + 0x6d2b79f5) >>> 0;
+    let t = a;
+    t = Math.imul(t ^ (t >>> 15), t | 1);
+    t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
+    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+  };
+}
+
+/** A fresh 32-bit seed for runs that did not pass one. */
+export function randomSeed(): number {
+  return Math.floor(Math.random() * 0x1_0000_0000) >>> 0;
+}
+
+export interface Range {
+  min: number;
+  max: number;
+}
+
+export function uniform(rng: Rng, range: Range): number {
+  return range.min + (range.max - range.min) * rng();
+}
+
+export interface ScrollPacing {
+  /** Scroll distance per step, as a fraction of the viewport height. */
+  step: Range;
+  /** Pause after each step, in ms. */
+  pauseMs: Range;
+  /** Probability that a step is followed by a longer reading pause instead. */
+  readChance: number;
+  /** Reading pause, in ms. */
+  readPauseMs: Range;
+  /** Probability that a step starts with a small scroll back up. */
+  backChance: number;
+  /** Scroll-back distance, as a fraction of the viewport height. */
+  back: Range;
+  /** Size of one wheel notch, in px. */
+  tickPx: Range;
+  /** Gap between wheel notches, in ms. */
+  tickGapMs: Range;
+}
+
+export const DEFAULT_SCROLL_PACING: ScrollPacing = {
+  step: { min: 0.6, max: 1.1 },
+  pauseMs: { min: 1_200, max: 3_500 },
+  readChance: 0.12,
+  readPauseMs: { min: 4_000, max: 9_000 },
+  backChance: 0.08,
+  back: { min: 0.1, max: 0.3 },
+  tickPx: { min: 80, max: 140 },
+  tickGapMs: { min: 12, max: 45 },
+};
+
+/** Upper bounds that keep every pause and step bounded. */
+export const PACING_LIMITS = {
+  maxStepFraction: 3,
+  maxPauseMs: 120_000,
+} as const;
+
+export interface WheelTick {
+  /** Positive scrolls down, negative scrolls up. */
+  deltaY: number;
+  /** Wait after this notch, in ms. */
+  gapMs: number;
+}
+
+export interface ScrollStepPlan {
+  /** Optional scroll-back notches run before the main step (negative deltas). */
+  back: WheelTick[];
+  /** Pause after the scroll-back, in ms (0 when there is no scroll-back). */
+  backPauseMs: number;
+  /** Main downward notches. */
+  ticks: WheelTick[];
+  /** Total downward distance, in px. */
+  deltaY: number;
+  /** Pause after the main step, in ms. */
+  pauseMs: number;
+  /** True when the pause is a longer reading pause. */
+  reading: boolean;
+}
+
+/** Split a distance into wheel notches the way a mouse wheel reports them. */
+export function wheelTicks(rng: Rng, pacing: ScrollPacing, distance: number): WheelTick[] {
+  const sign = distance < 0 ? -1 : 1;
+  let remaining = Math.round(Math.abs(distance));
+  const ticks: WheelTick[] = [];
+  while (remaining > 0) {
+    const size = Math.min(remaining, Math.max(1, Math.round(uniform(rng, pacing.tickPx))));
+    remaining -= size;
+    ticks.push({ deltaY: sign * size, gapMs: Math.round(uniform(rng, pacing.tickGapMs)) });
+  }
+  return ticks;
+}
+
+export function planScrollStep(
+  rng: Rng,
+  pacing: ScrollPacing,
+  viewportHeight: number,
+): ScrollStepPlan {
+  const height = Math.max(1, viewportHeight);
+  let back: WheelTick[] = [];
+  let backPauseMs = 0;
+  if (rng() < pacing.backChance) {
+    back = wheelTicks(rng, pacing, -uniform(rng, pacing.back) * height);
+    backPauseMs = Math.round(uniform(rng, { min: 400, max: 1_100 }));
+  }
+  const deltaY = Math.max(1, Math.round(uniform(rng, pacing.step) * height));
+  const ticks = wheelTicks(rng, pacing, deltaY);
+  const reading = rng() < pacing.readChance;
+  const pauseMs = Math.round(uniform(rng, reading ? pacing.readPauseMs : pacing.pauseMs));
+  return { back, backPauseMs, ticks, deltaY, pauseMs, reading };
+}
+
+/**
+ * Parse `min-max` (or a single value meaning min = max). Throws a message that
+ * names the flag when the value is malformed or out of bounds.
+ */
+export function parseRange(raw: string, flag: string, bounds: { min: number; max: number }): Range {
+  const text = raw.trim();
+  const m = /^(\d+(?:\.\d+)?)(?:\s*-\s*(\d+(?:\.\d+)?))?$/.exec(text);
+  if (!m) throw new Error(`${flag} must be <min>-<max> or a single number (got "${raw}").`);
+  const min = Number(m[1]);
+  const max = m[2] === undefined ? min : Number(m[2]);
+  if (min > max) throw new Error(`${flag}: min (${min}) must not exceed max (${max}).`);
+  if (min < bounds.min || max > bounds.max) {
+    throw new Error(`${flag} must stay within ${bounds.min}-${bounds.max} (got "${raw}").`);
+  }
+  return { min, max };
+}
+
+export function parseProbability(raw: string, flag: string): number {
+  const n = Number(raw);
+  if (!Number.isFinite(n) || n < 0 || n > 1) {
+    throw new Error(`${flag} must be a number from 0 to 1 (got "${raw}").`);
+  }
+  return n;
+}
+
+/**
+ * Machine-wide pacing defaults from the environment, in the same style as the
+ * page-load pace gate (`HARNERY_PACE_*`):
+ *
+ * - `HARNERY_COLLECT_PAUSE_MS`: pause after each step, `min-max` in ms.
+ * - `HARNERY_COLLECT_STEP`: step size as a viewport fraction, `min-max`.
+ *
+ * Command flags override these.
+ */
+export function scrollPacingFromEnv(env: NodeJS.ProcessEnv = process.env): ScrollPacing {
+  const pacing: ScrollPacing = { ...DEFAULT_SCROLL_PACING };
+  const pause = env.HARNERY_COLLECT_PAUSE_MS?.trim();
+  if (pause) {
+    pacing.pauseMs = parseRange(pause, "HARNERY_COLLECT_PAUSE_MS", {
+      min: 0,
+      max: PACING_LIMITS.maxPauseMs,
+    });
+  }
+  const step = env.HARNERY_COLLECT_STEP?.trim();
+  if (step) {
+    pacing.step = parseRange(step, "HARNERY_COLLECT_STEP", {
+      min: 0.05,
+      max: PACING_LIMITS.maxStepFraction,
+    });
+  }
+  return pacing;
+}

package/src/lib/browser/index.ts

@@ -185,6 +185,7 @@ export {
   type QaRunContext,
   type QaRunCritiqueOutcome,
   type QaRunCritiquePool,
+  type QaRunCritiqueTileTiming,
   type QaRunInteraction,
   type QaRunJob,
   type QaRunJobValidation,

package/src/lib/browser/page-review-judge.ts

@@ -25,6 +25,7 @@ import {
 } from "./page-review-pack.js";
 import type { QaContext } from "./qa-plan.js";
 import type { CritiqueReusePlan } from "./qa-reuse.js";
+import type { QaRunCritiqueTileTiming } from "./qa-run-contracts.js";
 import { loadQaSnapshot, type QaSnapshotStoreOptions } from "./qa-snapshot.js";
 
 export interface JudgePageReviewPackOptions {
@@ -62,6 +63,7 @@ export interface JudgedContext extends PageReviewCritiqueRecord {
 export interface JudgePageReviewPackResult {
   contexts: JudgedContext[];
   pool: { concurrency: number; wall_time_ms: number; provider: string };
+  tile_timings: QaRunCritiqueTileTiming[];
   tiles_total: number;
   tiles_reviewed: number;
   tiles_reused: number;
@@ -261,6 +263,7 @@ export async function judgePageReviewPack(
 
   let meta: Record<string, unknown> | undefined;
   let concurrency = 0;
+  const tileTimings: Array<QaRunCritiqueTileTiming | undefined> = new Array(work.length);
   if (provider && work.length > 0) {
     concurrency = Math.max(
       1,
@@ -277,12 +280,15 @@ export async function judgePageReviewPack(
         const entry = perContext[item.contextIndex];
         if (!entry) continue;
         entry.judged.add(item.tile.id);
+        const tileStarted = performance.now();
+        let tileOutcome: QaRunCritiqueTileTiming["outcome"] = "passed";
         try {
           const found = await provider({ url: item.url, rubric: item.rubric, tile: item.tile });
           for (const finding of found) {
             entry.findings.push({ ...finding, tile: item.tile.index, tile_id: item.tile.id });
           }
         } catch (err: unknown) {
+          tileOutcome = "failed";
           entry.findings.push({
             tile: item.tile.index,
             tile_id: item.tile.id,
@@ -290,6 +296,13 @@ export async function judgePageReviewPack(
             category: "provider-error",
             description: `critique provider failed on ${item.tile.label}: ${err instanceof Error ? err.message : String(err)}`,
           });
+        } finally {
+          tileTimings[i] = {
+            context_id: entry.record.id,
+            tile_id: item.tile.id,
+            duration_ms: Math.max(0, Math.round(performance.now() - tileStarted)),
+            outcome: tileOutcome,
+          };
         }
       }
     };
@@ -360,6 +373,9 @@ export async function judgePageReviewPack(
   return {
     contexts,
     pool: { concurrency, wall_time_ms: Date.now() - started, provider: label },
+    tile_timings: tileTimings.filter(
+      (timing): timing is QaRunCritiqueTileTiming => timing !== undefined,
+    ),
     tiles_total: tilesTotal,
     tiles_reviewed: tilesReviewed,
     tiles_reused: tilesReused,

package/src/lib/browser/qa-run-contracts.ts

@@ -145,6 +145,15 @@ export interface QaRunCritiqueLatency {
   p95: number;
 }
 
+/** Wall time for one tile's complete vision call, including provider retries
+ * and failover. Reused and unjudged tiles have no call and no timing entry. */
+export interface QaRunCritiqueTileTiming {
+  context_id: string;
+  tile_id: string;
+  duration_ms: number;
+  outcome: "passed" | "failed";
+}
+
 /** What share of the page the critique tiles covered, lifted from the browse
  * envelope. `capped` means the tiler dropped bands past its per-context
  * maximum; in signoff mode that is a blocker, in review mode a flag. Across
@@ -188,6 +197,8 @@ export interface QaRunCritiquePool {
   wall_time_ms: number;
   provider: string;
   latency_ms?: Record<string, QaRunCritiqueLatency>;
+  /** One entry per attempted tile, in context and tile order. */
+  tile_timings?: QaRunCritiqueTileTiming[];
 }
 
 /** Where the run's page review pack lives: the on-disk evidence an agent can

package/src/lib/browser/qa-run.ts

@@ -1397,6 +1397,7 @@ export async function runQaMatrix(options: QaRunMatrixOptions): Promise<QaRunRes
       wall_time_ms: judged.pool.wall_time_ms,
       provider: judged.pool.provider,
       ...(latency ? { latency_ms: latency } : {}),
+      tile_timings: judged.tile_timings,
     };
     for (const row of judged.contexts) {
       judgedById.set(row.context_id, row);

package/src/lib/browser/windows-chrome.ts

@@ -0,0 +1,227 @@
+import { spawnSync } from "node:child_process";
+import { existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import type { Cookie } from "../cookies/client.ts";
+
+/**
+ * Sign-in through the Windows host's Google Chrome when running under WSL.
+ *
+ * Some sign-in flows (X is the reported case) reject every browser running
+ * inside WSL, including a plain Linux Google Chrome with nothing attached:
+ * the sign-in request comes back "429 Too Many Requests" while the same
+ * account signs in from the Windows desktop browser without trouble. The
+ * Windows Chrome is the client those flows accept, so `browse --login
+ * --plain` under WSL opens it on a dedicated Windows-side profile. The
+ * person signs in there with nothing attached. Afterwards the profile is
+ * reopened headless for a moment, its cookies for the sign-in site are read
+ * through the DevTools protocol, and they are merged into the shared cookie
+ * store that later `browse` runs attach. The Windows profile cannot be shared
+ * with Linux Chromium directly because Windows encrypts its cookie database
+ * with keys only Windows can read.
+ *
+ * Under WSL2's default NAT networking the Windows loopback port is not
+ * reachable from Linux, so the DevTools conversation runs inside a short
+ * PowerShell script on the Windows side and prints the result to stdout.
+ */
+
+const WINDOWS_CHROME_CANDIDATES = [
+  "/mnt/c/Program Files/Google/Chrome/Application/chrome.exe",
+  "/mnt/c/Program Files (x86)/Google/Chrome/Application/chrome.exe",
+];
+
+/** Linux path of the Windows host's Google Chrome, or `undefined`. */
+export function windowsChromePath(
+  exists: (path: string) => boolean = existsSync,
+  localAppDataLinux?: string,
+): string | undefined {
+  const candidates = [...WINDOWS_CHROME_CANDIDATES];
+  if (localAppDataLinux)
+    candidates.push(join(localAppDataLinux, "Google", "Chrome", "Application", "chrome.exe"));
+  return candidates.find((candidate) => exists(candidate));
+}
+
+/** `%LOCALAPPDATA%` as a Windows path, read through cmd.exe interop. */
+export function windowsLocalAppData(): string | undefined {
+  const result = spawnSync("cmd.exe", ["/c", "echo %LOCALAPPDATA%"], {
+    encoding: "utf8",
+    timeout: 10_000,
+  });
+  const value = (result.stdout ?? "").replace(/\r/g, "").trim();
+  return result.status === 0 && /^[A-Za-z]:\\/.test(value) ? value : undefined;
+}
+
+/** Windows-side profile used only for `--login --plain` sign-ins under WSL. */
+export function windowsPlainProfile(localAppData: string): string {
+  return `${localAppData}\\harnery\\plain-login-profile`;
+}
+
+/**
+ * The registrable domain to harvest for a sign-in URL: the last two labels of
+ * the host (`x.com` from `https://x.com/i/flow/login`). Cookies are kept when
+ * their domain equals it or is a subdomain of it, so a sign-in that also sets
+ * cookies for an identity provider does not copy those into the store.
+ */
+export function harvestDomain(url: string): string {
+  const host = new URL(url).hostname.toLowerCase();
+  const labels = host.split(".").filter(Boolean);
+  return labels.length <= 2 ? host : labels.slice(-2).join(".");
+}
+
+export function cookieInDomain(cookieDomain: string, domain: string): boolean {
+  const bare = cookieDomain.replace(/^\./, "").toLowerCase();
+  return bare === domain || bare.endsWith(`.${domain}`);
+}
+
+/** Shape `Storage.getCookies` returns; extra fields are ignored. */
+interface CdpCookie {
+  name: string;
+  value: string;
+  domain: string;
+  path: string;
+  expires: number;
+  size?: number;
+  httpOnly: boolean;
+  secure: boolean;
+  session?: boolean;
+  sameSite?: string;
+}
+
+/** Keep the sign-in site's cookies and normalize them to the store's shape. */
+export function cookiesForDomain(cdpCookies: CdpCookie[], domain: string): Cookie[] {
+  return cdpCookies
+    .filter((cookie) => cookieInDomain(cookie.domain, domain))
+    .map((cookie) => ({
+      name: cookie.name,
+      value: cookie.value,
+      domain: cookie.domain,
+      path: cookie.path || "/",
+      expires: typeof cookie.expires === "number" ? cookie.expires : -1,
+      ...(cookie.size === undefined ? {} : { size: cookie.size }),
+      httpOnly: Boolean(cookie.httpOnly),
+      secure: Boolean(cookie.secure),
+      ...(cookie.session === undefined ? {} : { session: cookie.session }),
+      ...(cookie.sameSite ? { sameSite: cookie.sameSite } : {}),
+    }));
+}
+
+/**
+ * PowerShell that does the whole read on the Windows side: start Chrome
+ * headless on the profile, wait for the DevToolsActivePort file it writes,
+ * read every cookie with `Storage.getCookies`, write the raw response to
+ * stdout, and close the browser. WSL never touches the profile folder, which
+ * it may not be able to list. Every wait carries a deadline.
+ */
+export function harvestScript(): string {
+  return [
+    "param([string]$Chrome, [string]$Profile, [int]$StartupSeconds = 30)",
+    "$ErrorActionPreference = 'Stop'",
+    "$portFile = Join-Path $Profile 'DevToolsActivePort'",
+    "if (Test-Path $portFile) { Remove-Item $portFile -Force }",
+    "$args = @('--headless=new', '--no-first-run', '--no-default-browser-check', '--remote-debugging-port=0', \"--user-data-dir=$Profile\", 'about:blank')",
+    "$proc = Start-Process -FilePath $Chrome -ArgumentList $args -PassThru -WindowStyle Hidden",
+    "$deadline = (Get-Date).AddSeconds($StartupSeconds)",
+    "while (-not (Test-Path $portFile)) { if ((Get-Date) -gt $deadline) { throw 'Windows Chrome did not open its DevTools port in time.' }; if ($proc.HasExited) { throw 'Windows Chrome exited before opening its DevTools port.' }; Start-Sleep -Milliseconds 250 }",
+    "Start-Sleep -Milliseconds 200",
+    "$lines = Get-Content $portFile",
+    "$port = [int]$lines[0]; $path = $lines[1]",
+    "$ws = New-Object System.Net.WebSockets.ClientWebSocket",
+    "$cts = New-Object System.Threading.CancellationTokenSource 20000",
+    '$ws.ConnectAsync([Uri]("ws://127.0.0.1:$port$path"), $cts.Token).Wait()',
+    "function Send($obj) { $b = [Text.Encoding]::UTF8.GetBytes(($obj | ConvertTo-Json -Compress)); $ws.SendAsync([ArraySegment[byte]]$b, 'Text', $true, $cts.Token).Wait() }",
+    "function Recv() { $buf = New-Object byte[] 65536; $ms = New-Object IO.MemoryStream; do { $r = $ws.ReceiveAsync([ArraySegment[byte]]$buf, $cts.Token).Result; $ms.Write($buf, 0, $r.Count) } while (-not $r.EndOfMessage); [Text.Encoding]::UTF8.GetString($ms.ToArray()) }",
+    "Send @{ id = 1; method = 'Storage.getCookies' }",
+    "$out = $null",
+    "while ($null -eq $out) { $m = Recv; if ($m -match '\"id\":1[,}]') { $out = $m } }",
+    "[Console]::Out.Write($out)",
+    "try { Send @{ id = 2; method = 'Browser.close' } } catch {}",
+    "if (-not $proc.WaitForExit(10000)) { Stop-Process -Id $proc.Id -Force -ErrorAction SilentlyContinue }",
+  ].join("\r\n");
+}
+
+/**
+ * Reopen the Windows profile headless, read its cookies for `domain`, and
+ * close it. Returns the cookies in the store's shape.
+ */
+export function harvestWindowsChromeCookies(input: {
+  chromeWindows: string;
+  profileWindows: string;
+  domain: string;
+  startupTimeoutSeconds?: number;
+}): Cookie[] {
+  const scriptDir = join(tmpdir(), "harnery-windows-chrome");
+  mkdirSync(scriptDir, { recursive: true });
+  const scriptLinux = join(scriptDir, `harvest-cookies-${process.pid}.ps1`);
+  writeFileSync(scriptLinux, harvestScript(), { mode: 0o600 });
+  try {
+    const scriptWindows = spawnSync("wslpath", ["-w", scriptLinux], {
+      encoding: "utf8",
+    }).stdout.trim();
+    const result = spawnSync(
+      "powershell.exe",
+      [
+        "-NoProfile",
+        "-ExecutionPolicy",
+        "Bypass",
+        "-File",
+        scriptWindows,
+        "-Chrome",
+        input.chromeWindows,
+        "-Profile",
+        input.profileWindows,
+        "-StartupSeconds",
+        String(input.startupTimeoutSeconds ?? 30),
+      ],
+      { encoding: "utf8", timeout: 90_000, maxBuffer: 64 * 1024 * 1024 },
+    );
+    if (result.status !== 0) {
+      throw new Error(
+        `Reading cookies from Windows Chrome failed: ${(result.stderr || result.stdout || "no output").trim().slice(0, 400)}`,
+      );
+    }
+    const parsed = JSON.parse(result.stdout) as {
+      result?: { cookies?: CdpCookie[] };
+      error?: { message?: string };
+    };
+    if (!parsed.result?.cookies)
+      throw new Error(
+        `Windows Chrome returned no cookie list: ${parsed.error?.message ?? "unknown response"}`,
+      );
+    return cookiesForDomain(parsed.result.cookies, input.domain);
+  } finally {
+    rmSync(scriptLinux, { force: true });
+  }
+}
+
+/**
+ * Close any Chrome still running on `profileWindows`, for when the person ends
+ * the sign-in from the terminal while the window is open. The profile must be
+ * closed before its cookies can be read. Closing the window first lets Chrome
+ * write recent cookies to disk; a forced stop is the fallback after 15 s.
+ */
+export function stopWindowsChromeProfile(profileWindows: string): void {
+  const escaped = profileWindows.replace(/'/g, "''");
+  const select = `Get-CimInstance Win32_Process -Filter "Name='chrome.exe'" | Where-Object { $_.CommandLine -like '*${escaped}*' }`;
+  spawnSync(
+    "powershell.exe",
+    [
+      "-NoProfile",
+      "-Command",
+      [
+        `$ids = @(${select} | ForEach-Object { $_.ProcessId })`,
+        "foreach ($id in $ids) { $p = Get-Process -Id $id -ErrorAction SilentlyContinue; if ($p -and $p.MainWindowHandle -ne 0) { [void]$p.CloseMainWindow() } }",
+        "$deadline = (Get-Date).AddSeconds(15)",
+        `while ((Get-Date) -lt $deadline -and @(${select}).Count -gt 0) { Start-Sleep -Milliseconds 250 }`,
+        `${select} | ForEach-Object { Stop-Process -Id $_.ProcessId -Force -ErrorAction SilentlyContinue }`,
+      ].join("; "),
+    ],
+    { encoding: "utf8", timeout: 45_000 },
+  );
+}
+
+/** Windows form of a `/mnt/<drive>/...` path. */
+export function wslPathToWindows(linuxPath: string): string {
+  const match = /^\/mnt\/([a-z])\/(.*)$/.exec(linuxPath);
+  if (!match) throw new Error(`Not a /mnt/<drive> path: ${linuxPath}`);
+  return `${(match[1] as string).toUpperCase()}:\\${(match[2] ?? "").replace(/\//g, "\\")}`;
+}