@parcha/pi-browser 0.1.0-beta.18 → 0.1.0-beta.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/CHANGELOG.md +5 -0
  2. package/README.md +177 -39
  3. package/dist/coding-agent-local.js +2 -1
  4. package/dist/coding-agent.d.ts +42 -2
  5. package/dist/coding-agent.js +159 -21
  6. package/dist/core/clip.d.ts +31 -0
  7. package/dist/core/clip.js +49 -0
  8. package/dist/core/contract.d.ts +4 -1
  9. package/dist/core/contract.js +40 -7
  10. package/dist/core/custody.d.ts +79 -5
  11. package/dist/core/custody.js +66 -5
  12. package/dist/core/decisions.d.ts +4 -0
  13. package/dist/core/dial.d.ts +7 -0
  14. package/dist/core/dial.js +41 -0
  15. package/dist/core/downloads.d.ts +1 -0
  16. package/dist/core/downloads.js +17 -5
  17. package/dist/core/driving-guard.d.ts +13 -0
  18. package/dist/core/driving-guard.js +61 -0
  19. package/dist/core/driving.d.ts +61 -0
  20. package/dist/core/driving.js +156 -0
  21. package/dist/core/effects.js +33 -13
  22. package/dist/core/evidence.d.ts +3 -1
  23. package/dist/core/failures.d.ts +5 -1
  24. package/dist/core/failures.js +20 -4
  25. package/dist/core/files.d.ts +8 -0
  26. package/dist/core/files.js +74 -0
  27. package/dist/core/find.d.ts +3 -1
  28. package/dist/core/find.js +2 -1
  29. package/dist/core/guard.d.ts +9 -1
  30. package/dist/core/guard.js +7 -2
  31. package/dist/core/host.d.ts +49 -1
  32. package/dist/core/images.d.ts +10 -2
  33. package/dist/core/images.js +4 -4
  34. package/dist/core/lease.d.ts +1 -1
  35. package/dist/core/lease.js +2 -1
  36. package/dist/core/session-control.d.ts +2 -0
  37. package/dist/core/session-control.js +5 -0
  38. package/dist/core/tools.d.ts +70 -4
  39. package/dist/core/tools.js +288 -45
  40. package/dist/core/unanswered.d.ts +3 -0
  41. package/dist/core/unanswered.js +17 -0
  42. package/dist/core/usage.d.ts +2 -0
  43. package/dist/core/usage.js +6 -0
  44. package/dist/core/web.d.ts +1 -1
  45. package/dist/core/web.js +9 -7
  46. package/dist/driver/extension-worker.d.ts +4 -2
  47. package/dist/driver/extension-worker.js +16 -10
  48. package/dist/driver/page-driver.d.ts +8 -0
  49. package/dist/driver/page-driver.js +128 -0
  50. package/dist/driver/stagehand.d.ts +3 -1
  51. package/dist/driver/stagehand.js +40 -66
  52. package/dist/durable.d.ts +16 -4
  53. package/dist/durable.js +415 -82
  54. package/dist/index.d.ts +2 -0
  55. package/dist/index.js +2 -0
  56. package/dist/live-view/index.d.ts +4 -0
  57. package/dist/live-view/index.js +4 -0
  58. package/dist/live-view/protocol.d.ts +152 -0
  59. package/dist/live-view/protocol.js +84 -0
  60. package/dist/live-view/provider.d.ts +6 -0
  61. package/dist/live-view/provider.js +23 -0
  62. package/dist/live-view/server.d.ts +50 -0
  63. package/dist/live-view/server.js +342 -0
  64. package/dist/live-view/stream.d.ts +7 -0
  65. package/dist/live-view/stream.js +388 -0
  66. package/dist/live-view/token.d.ts +6 -0
  67. package/dist/live-view/token.js +13 -0
  68. package/dist/live-view/viewer.d.ts +6 -0
  69. package/dist/live-view/viewer.js +350 -0
  70. package/dist/live-view/ws.d.ts +19 -0
  71. package/dist/live-view/ws.js +176 -0
  72. package/dist/providers/assigned.d.ts +18 -0
  73. package/dist/providers/assigned.js +76 -0
  74. package/dist/providers/cdp-call.js +12 -2
  75. package/dist/providers/cdp.js +37 -11
  76. package/dist/recorder/follow.d.ts +10 -0
  77. package/dist/recorder/follow.js +27 -2
  78. package/dist/recorder/frames.d.ts +7 -0
  79. package/dist/recorder/frames.js +73 -5
  80. package/dist/store.d.ts +22 -3
  81. package/dist/store.js +75 -4
  82. package/dist/testing/backend.d.ts +26 -0
  83. package/dist/testing/backend.js +21 -3
  84. package/dist/testing/driver.d.ts +18 -1
  85. package/dist/testing/driver.js +4 -0
  86. package/dist/web.js +2 -2
  87. package/package.json +10 -2
@@ -37,14 +37,14 @@ export function imageSize(bytes) {
37
37
  export class ScreenshotBudgetError extends Error {
38
38
  name = "ScreenshotBudgetError";
39
39
  }
40
- /** Over budget, retakes as a viewport JPEG at CSS scale, quality 40, 25, 10; throws ScreenshotBudgetError. */
40
+ /** Over budget, retakes as a viewport JPEG at CSS scale, quality 40, 25, 10 (of the same clip, when there is one); throws ScreenshotBudgetError. */
41
41
  export async function captureWithinBudget(capture, requested, budget = { maxEdgePx: SCREENSHOT_MAX_EDGE_PX, maxBase64Bytes: SCREENSHOT_MAX_BASE64_BYTES }) {
42
42
  const ceiling = requested.type === "png" ? 40 : (requested.quality ?? 40);
43
43
  const ladder = [
44
- { fullPage: requested.fullPage ?? false, type: requested.type ?? "jpeg", ...(requested.type === "png" ? {} : { quality: requested.quality ?? 40 }), ...(requested.scale ? { scale: requested.scale } : {}) },
45
- ...[40, 25, 10].map((quality) => ({ fullPage: false, type: "jpeg", quality: Math.min(ceiling, quality), scale: "css" })),
44
+ { fullPage: requested.fullPage ?? false, type: requested.type ?? "jpeg", ...(requested.type === "png" ? {} : { quality: requested.quality ?? 40 }), ...(requested.scale ? { scale: requested.scale } : {}), ...(requested.clip ? { clip: requested.clip } : {}) },
45
+ ...[40, 25, 10].map((quality) => ({ fullPage: false, type: "jpeg", quality: Math.min(ceiling, quality), scale: "css", ...(requested.clip ? { clip: requested.clip } : {}) })),
46
46
  ];
47
- const key = (o) => `${o.fullPage}:${o.type}:${o.quality}:${o.scale}`;
47
+ const key = (o) => `${o.fullPage}:${o.type}:${o.quality}:${o.scale}:${o.clip ? `${o.clip.x},${o.clip.y},${o.clip.width},${o.clip.height}` : ""}`;
48
48
  const attempts = ladder.filter((o, i) => ladder.findIndex((other) => key(other) === key(o)) === i);
49
49
  let adjusted = null;
50
50
  for (const options of attempts) {
@@ -1,4 +1,4 @@
1
- export type LeaseState = "creating" | "live" | "releasing" | "released" | "lost";
1
+ export type LeaseState = "creating" | "live" | "held" | "releasing" | "released" | "lost";
2
2
  /** `gone`: the provider no longer knows the session, which counts as released. */
3
3
  export type ResourceStatus = "pending" | "running" | "stopped" | "gone";
4
4
  /** Written to documents, so never a secret: a provider keeps connect URLs and keys in its own memory, found again by
@@ -4,7 +4,8 @@ import { createHash } from "node:crypto";
4
4
  export const LEASE_TRANSITIONS = Object.freeze({
5
5
  none: ["creating"],
6
6
  creating: ["live", "released", "releasing"],
7
- live: ["releasing", "released", "lost"],
7
+ live: ["held", "releasing", "released", "lost"],
8
+ held: ["live", "releasing", "released", "lost"],
8
9
  releasing: ["released", "lost"],
9
10
  released: [],
10
11
  lost: [],
@@ -23,3 +23,5 @@ export declare function idleTracker<K = string>(options: {
23
23
  /** Every environment variable that holds a provider key: each redactor in either adapter scrubs their values. */
24
24
  export declare const PROVIDER_KEY_ENV: readonly string[];
25
25
  export declare const providerKeyValues: (env: NodeJS.ProcessEnv, host?: () => readonly string[]) => string[];
26
+ /** Runs one at a time, in arrival order: each starts once the one before it has settled, however that one ended. */
27
+ export declare function oneAtATime(): <T>(run: () => Promise<T>) => Promise<T>;
@@ -73,3 +73,8 @@ export function idleTracker(options) {
73
73
  /** Every environment variable that holds a provider key: each redactor in either adapter scrubs their values. */
74
74
  export const PROVIDER_KEY_ENV = Object.freeze(["BROWSERBASE_API_KEY", "BB_API_KEY", "KERNEL_API_KEY"]);
75
75
  export const providerKeyValues = (env, host) => [...PROVIDER_KEY_ENV.map((name) => env[name] ?? "").filter(Boolean), ...(host?.() ?? [])];
76
+ /** Runs one at a time, in arrival order: each starts once the one before it has settled, however that one ended. */
77
+ export function oneAtATime() {
78
+ let turn = Promise.resolve();
79
+ return (run) => { const mine = turn.then(run, run); turn = mine.catch(() => undefined); return mine; };
80
+ }
@@ -1,30 +1,67 @@
1
1
  import { type BrowserDriver, type CustodyPort, type ToolOutput } from "./custody.js";
2
2
  import { type EvidenceSink } from "./evidence.js";
3
3
  import type { Decisions } from "./decisions.js";
4
+ import type { WorkspaceFiles } from "./host.js";
4
5
  import { type RepeatGuard } from "./guard.js";
6
+ import { type Locate } from "./clip.js";
5
7
  import { type ScreenshotOptions } from "./images.js";
6
8
  import { type Redact } from "./redact.js";
7
9
  import type { Geo } from "./host.js";
10
+ /** One action of a run's batch, as the contract's schema admits it. */
11
+ export type RunAction = {
12
+ op: "click";
13
+ id: string;
14
+ } | {
15
+ op: "hover";
16
+ id: string;
17
+ } | {
18
+ op: "fill";
19
+ id: string;
20
+ value: string;
21
+ } | {
22
+ op: "type";
23
+ id: string;
24
+ text: string;
25
+ delay?: number;
26
+ } | {
27
+ op: "press";
28
+ id: string;
29
+ key: string;
30
+ } | {
31
+ op: "select";
32
+ id: string;
33
+ values: string | string[];
34
+ };
8
35
  export type PageDriver = BrowserDriver & {
9
36
  snapshot(options?: {
10
37
  includeIframes?: boolean;
11
38
  }): Promise<string>;
12
39
  run(input: {
13
40
  code?: string;
14
- actions?: unknown[];
41
+ actions?: RunAction[];
15
42
  }): Promise<unknown>;
16
43
  /** The page's URL, cheaply: no page content is read. */
17
44
  url(): Promise<string>;
45
+ /** Drops every snapshot's element ids, so an action by an old id fails as a stale snapshot (a person changed the page, whatever its URL). Optional:
46
+ * a driver without ids has none to drop. */
47
+ forgetSnapshots?(): void;
18
48
  screenshot(options: ScreenshotOptions): Promise<{
19
49
  data: string;
20
50
  mimeType: string;
21
51
  }>;
52
+ /** Where an element or the viewport is now (see clip.ts); an element is scrolled into view first. Absent: no region can be placed. */
53
+ locate?: Locate;
22
54
  page(): Promise<{
23
55
  url: string;
24
56
  title: string;
25
57
  text: string;
26
58
  html: string;
27
59
  }>;
60
+ setInputFiles?(selector: string, file: {
61
+ name: string;
62
+ mimeType: string;
63
+ buffer: Uint8Array;
64
+ }): Promise<void>;
28
65
  };
29
66
  export type ToolCall = {
30
67
  callId: string;
@@ -34,6 +71,8 @@ export type ToolCall = {
34
71
  redact: Redact;
35
72
  evidence: EvidenceSink;
36
73
  classifyPage?: Decisions["classifyPage"];
74
+ workspace?: string;
75
+ files?: WorkspaceFiles;
37
76
  };
38
77
  export type ToolImpl<D extends BrowserDriver = BrowserDriver, A = unknown> = (args: A, port: CustodyPort<D>, call: ToolCall) => Promise<ToolOutput>;
39
78
  /** Each browser tool's arguments, as its contract's schema admits them. */
@@ -43,13 +82,21 @@ export type BrowserToolArgs = {
43
82
  };
44
83
  run: {
45
84
  code?: string;
46
- actions?: unknown[];
85
+ actions?: RunAction[];
47
86
  };
48
87
  screenshot: {
49
88
  fullPage?: boolean;
50
89
  type?: "png" | "jpeg";
51
90
  quality?: number;
52
91
  ask?: string;
92
+ clip?: {
93
+ x: number;
94
+ y: number;
95
+ width: number;
96
+ height: number;
97
+ };
98
+ id?: string;
99
+ selector?: string;
53
100
  };
54
101
  browser_read: {
55
102
  what?: (typeof WHATS)[number];
@@ -60,13 +107,32 @@ export type BrowserToolArgs = {
60
107
  geolocation?: Geo;
61
108
  };
62
109
  browser_downloads: Record<string, never>;
110
+ browser_request_handoff: {
111
+ request_id: string;
112
+ instruction: string;
113
+ completion_condition: string;
114
+ };
115
+ browser_upload: {
116
+ path?: string;
117
+ selector?: string;
118
+ };
63
119
  };
64
120
  export type BrowserToolImpls<D extends BrowserDriver = BrowserDriver> = {
65
121
  [K in keyof BrowserToolArgs]: ToolImpl<D, BrowserToolArgs[K]>;
66
122
  };
67
123
  declare const WHATS: readonly ["markdown", "text", "html", "url", "title"];
124
+ /** A dropped connection the provider still runs is reconnected on a backoff, within one budget: 90 s or the call's own end,
125
+ * whichever comes first. A try starts only when a whole one still fits (Stagehand's 60 s init deadline plus the attach
126
+ * around it), so every try that starts ends inside the budget. */
127
+ type ReconnectTimings = {
128
+ budgetMs: number;
129
+ tryMs: number;
130
+ backoffMs: readonly number[];
131
+ };
68
132
  /** `guards` holds each conversation's repeat guard; a host that owns the conversation's lifecycle passes it to evict one. */
69
- export declare function pageTools({ guards }?: {
133
+ /** `reconnect`: the reconnect timings, for tests; hosts keep the defaults. */
134
+ export declare function pageTools({ guards, reconnect: timings }?: {
70
135
  guards?: Map<number, RepeatGuard>;
71
- }): Pick<BrowserToolImpls<PageDriver>, "snapshot" | "run" | "screenshot" | "browser_read">;
136
+ reconnect?: ReconnectTimings;
137
+ }): Pick<BrowserToolImpls<PageDriver>, "snapshot" | "run" | "screenshot" | "browser_read" | "browser_upload" | "browser_downloads">;
72
138
  export {};