autokap 1.0.7 → 1.0.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (278) hide show
  1. package/assets/cursors/macos.svg +4 -0
  2. package/assets/cursors/windows.svg +15 -0
  3. package/assets/skill/OPCODE-REFERENCE.md +607 -0
  4. package/assets/skill/README.md +39 -0
  5. package/assets/skill/SKILL.md +453 -468
  6. package/assets/skill/STUDIO-SKILL.md +476 -0
  7. package/assets/skill/references/examples.md +104 -0
  8. package/assets/skill/references/interactive-demo.md +225 -0
  9. package/assets/skill/references/mock-data.md +178 -0
  10. package/dist/action-verifier.d.ts +29 -0
  11. package/dist/action-verifier.js +133 -0
  12. package/dist/agent-action-recovery.d.ts +45 -0
  13. package/dist/agent-action-recovery.js +370 -0
  14. package/dist/agent-message-utils.d.ts +21 -0
  15. package/dist/agent-message-utils.js +77 -0
  16. package/dist/agent-url-utils.d.ts +30 -0
  17. package/dist/agent-url-utils.js +138 -0
  18. package/dist/agent.d.ts +92 -8
  19. package/dist/agent.js +2936 -781
  20. package/dist/ak-tree.d.ts +39 -0
  21. package/dist/ak-tree.js +368 -0
  22. package/dist/alt-text.d.ts +26 -0
  23. package/dist/alt-text.js +55 -0
  24. package/dist/auth-capture.d.ts +17 -0
  25. package/dist/auth-capture.js +164 -0
  26. package/dist/benchmark.d.ts +59 -0
  27. package/dist/benchmark.js +135 -0
  28. package/dist/browser-bar.d.ts +14 -6
  29. package/dist/browser-bar.js +145 -8
  30. package/dist/browser-pool.d.ts +7 -0
  31. package/dist/browser-pool.js +15 -5
  32. package/dist/browser-utils.d.ts +31 -0
  33. package/dist/browser-utils.js +97 -0
  34. package/dist/browser.d.ts +51 -1
  35. package/dist/browser.js +1481 -31
  36. package/dist/capture-alt-text.js +2 -1
  37. package/dist/capture-language-preflight.js +14 -0
  38. package/dist/capture-llm-page-identity.js +22 -10
  39. package/dist/capture-page-identity.d.ts +5 -7
  40. package/dist/capture-page-identity.js +211 -78
  41. package/dist/capture-preset-credentials.d.ts +50 -0
  42. package/dist/capture-preset-credentials.js +127 -0
  43. package/dist/capture-request-plan.d.ts +2 -2
  44. package/dist/capture-request-plan.js +64 -16
  45. package/dist/capture-run-optimizer.js +48 -33
  46. package/dist/capture-selector-memory.d.ts +5 -0
  47. package/dist/capture-selector-memory.js +18 -0
  48. package/dist/capture-strategy.d.ts +36 -0
  49. package/dist/capture-strategy.js +95 -0
  50. package/dist/capture-studio-sync.d.ts +1 -0
  51. package/dist/capture-studio-sync.js +9 -3
  52. package/dist/capture-surface-contract.d.ts +36 -0
  53. package/dist/capture-surface-contract.js +299 -0
  54. package/dist/capture-transition-engine.d.ts +28 -0
  55. package/dist/capture-transition-engine.js +292 -0
  56. package/dist/capture-variant-state.d.ts +2 -0
  57. package/dist/capture-variant-state.js +26 -0
  58. package/dist/capture-verification.d.ts +35 -0
  59. package/dist/capture-verification.js +95 -0
  60. package/dist/capture-viewport-lock.d.ts +48 -0
  61. package/dist/capture-viewport-lock.js +74 -0
  62. package/dist/circuit-breaker.d.ts +42 -0
  63. package/dist/circuit-breaker.js +119 -0
  64. package/dist/cli-config.d.ts +8 -1
  65. package/dist/cli-config.js +62 -6
  66. package/dist/cli-contract.d.ts +15 -0
  67. package/dist/cli-contract.js +167 -0
  68. package/dist/cli-runner-local.d.ts +12 -0
  69. package/dist/cli-runner-local.js +102 -0
  70. package/dist/cli-runner.d.ts +34 -0
  71. package/dist/cli-runner.js +433 -0
  72. package/dist/cli-utils.d.ts +0 -1
  73. package/dist/cli-utils.js +2 -5
  74. package/dist/cli.js +1005 -267
  75. package/dist/clip-orchestrator.js +9 -2
  76. package/dist/clip-postprocess.js +25 -16
  77. package/dist/cookie-dismiss.d.ts +2 -0
  78. package/dist/cookie-dismiss.js +48 -13
  79. package/dist/cost-logging.d.ts +8 -0
  80. package/dist/cost-logging.js +160 -46
  81. package/dist/cost-resolution-monitor.d.ts +16 -0
  82. package/dist/cost-resolution-monitor.js +34 -0
  83. package/dist/credential-templates.js +2 -2
  84. package/dist/cursor-overlay-script.d.ts +6 -0
  85. package/dist/cursor-overlay-script.js +169 -0
  86. package/dist/dom-css-purger.d.ts +65 -0
  87. package/dist/dom-css-purger.js +333 -0
  88. package/dist/dom-font-inliner.d.ts +45 -0
  89. package/dist/dom-font-inliner.js +148 -0
  90. package/dist/dom-patch-resolver.d.ts +52 -0
  91. package/dist/dom-patch-resolver.js +242 -0
  92. package/dist/dom-serializer.d.ts +82 -0
  93. package/dist/dom-serializer.js +378 -0
  94. package/dist/element-capture.d.ts +1 -41
  95. package/dist/element-capture.js +202 -446
  96. package/dist/env-validation.d.ts +5 -0
  97. package/dist/env-validation.js +29 -0
  98. package/dist/execution-schema.d.ts +4423 -0
  99. package/dist/execution-schema.js +507 -0
  100. package/dist/execution-types.d.ts +886 -0
  101. package/dist/execution-types.js +65 -0
  102. package/dist/fonts-loader.d.ts +14 -0
  103. package/dist/fonts-loader.js +55 -0
  104. package/dist/hybrid-navigator.js +12 -12
  105. package/dist/index.d.ts +9 -6
  106. package/dist/index.js +10 -4
  107. package/dist/legacy/agent-action-recovery.d.ts +45 -0
  108. package/dist/legacy/agent-action-recovery.js +370 -0
  109. package/dist/legacy/agent-message-utils.d.ts +21 -0
  110. package/dist/legacy/agent-message-utils.js +77 -0
  111. package/dist/legacy/agent-url-utils.d.ts +30 -0
  112. package/dist/legacy/agent-url-utils.js +138 -0
  113. package/dist/legacy/agent.d.ts +226 -0
  114. package/dist/legacy/agent.js +6666 -0
  115. package/dist/legacy/clip-orchestrator.d.ts +148 -0
  116. package/dist/legacy/clip-orchestrator.js +957 -0
  117. package/dist/legacy/credential-templates.d.ts +5 -0
  118. package/dist/legacy/credential-templates.js +60 -0
  119. package/dist/legacy/hybrid-navigator.d.ts +138 -0
  120. package/dist/legacy/hybrid-navigator.js +468 -0
  121. package/dist/legacy/llm-usage.d.ts +17 -0
  122. package/dist/legacy/llm-usage.js +45 -0
  123. package/dist/legacy/prompt-cache.d.ts +10 -0
  124. package/dist/legacy/prompt-cache.js +24 -0
  125. package/dist/legacy/prompts.d.ts +175 -0
  126. package/dist/legacy/prompts.js +1038 -0
  127. package/dist/legacy/tools.d.ts +4 -0
  128. package/dist/legacy/tools.js +216 -0
  129. package/dist/legacy/video-agent.d.ts +143 -0
  130. package/dist/legacy/video-agent.js +4788 -0
  131. package/dist/legacy/video-observation.d.ts +36 -0
  132. package/dist/legacy/video-observation.js +192 -0
  133. package/dist/legacy/video-planner.d.ts +12 -0
  134. package/dist/legacy/video-planner.js +501 -0
  135. package/dist/legacy/video-prompts.d.ts +37 -0
  136. package/dist/legacy/video-prompts.js +569 -0
  137. package/dist/legacy/video-tools.d.ts +3 -0
  138. package/dist/legacy/video-tools.js +59 -0
  139. package/dist/legacy/video-variant-state.d.ts +29 -0
  140. package/dist/legacy/video-variant-state.js +80 -0
  141. package/dist/legacy/vision-model.d.ts +17 -0
  142. package/dist/legacy/vision-model.js +74 -0
  143. package/dist/llm-healer.d.ts +63 -0
  144. package/dist/llm-healer.js +166 -0
  145. package/dist/llm-provider.d.ts +29 -0
  146. package/dist/llm-provider.js +80 -0
  147. package/dist/logger.d.ts +6 -2
  148. package/dist/logger.js +15 -1
  149. package/dist/mockup-html.js +35 -25
  150. package/dist/mockup.d.ts +95 -2
  151. package/dist/mockup.js +427 -166
  152. package/dist/mouse-animation.d.ts +2 -2
  153. package/dist/mouse-animation.js +34 -20
  154. package/dist/opcode-actions.d.ts +42 -0
  155. package/dist/opcode-actions.js +511 -0
  156. package/dist/opcode-runner.d.ts +51 -0
  157. package/dist/opcode-runner.js +770 -0
  158. package/dist/openrouter-client.d.ts +40 -0
  159. package/dist/openrouter-client.js +16 -0
  160. package/dist/overlay-engine.d.ts +24 -0
  161. package/dist/overlay-engine.js +176 -0
  162. package/dist/postcondition.d.ts +16 -0
  163. package/dist/postcondition.js +269 -0
  164. package/dist/program-patcher.d.ts +25 -0
  165. package/dist/program-patcher.js +44 -0
  166. package/dist/prompts.d.ts +13 -5
  167. package/dist/prompts.js +224 -351
  168. package/dist/provider-config.d.ts +12 -0
  169. package/dist/provider-config.js +15 -0
  170. package/dist/recovery-chain.d.ts +37 -0
  171. package/dist/recovery-chain.js +350 -0
  172. package/dist/remote-browser.d.ts +28 -4
  173. package/dist/remote-browser.js +60 -5
  174. package/dist/safari-browser-bar.d.ts +15 -0
  175. package/dist/safari-browser-bar.js +95 -0
  176. package/dist/safari-toolbar-asset.d.ts +15 -0
  177. package/dist/safari-toolbar-asset.js +12 -0
  178. package/dist/security.d.ts +2 -1
  179. package/dist/security.js +49 -10
  180. package/dist/selector-resolver.d.ts +34 -0
  181. package/dist/selector-resolver.js +181 -0
  182. package/dist/semantic-resolver.d.ts +35 -0
  183. package/dist/semantic-resolver.js +161 -0
  184. package/dist/server-capture-runtime.d.ts +5 -3
  185. package/dist/server-capture-runtime.js +42 -95
  186. package/dist/server-credit-usage.d.ts +2 -2
  187. package/dist/server-project-webhooks.d.ts +15 -1
  188. package/dist/server-project-webhooks.js +34 -8
  189. package/dist/server-screenshot-watermark.js +27 -5
  190. package/dist/session-profile.js +164 -1
  191. package/dist/sf-pro-symbols.d.ts +1 -0
  192. package/dist/sf-pro-symbols.js +55 -0
  193. package/dist/skill-packaging.d.ts +28 -0
  194. package/dist/skill-packaging.js +169 -0
  195. package/dist/smart-wait.d.ts +27 -0
  196. package/dist/smart-wait.js +81 -0
  197. package/dist/status-bar-render.d.ts +20 -0
  198. package/dist/status-bar-render.js +410 -0
  199. package/dist/status-bar.d.ts +9 -0
  200. package/dist/status-bar.js +298 -14
  201. package/dist/svg-browser-bar.d.ts +33 -0
  202. package/dist/svg-browser-bar.js +206 -0
  203. package/dist/svg-status-bar.d.ts +36 -0
  204. package/dist/svg-status-bar.js +597 -0
  205. package/dist/svg-text.d.ts +61 -0
  206. package/dist/svg-text.js +118 -0
  207. package/dist/tools.js +89 -451
  208. package/dist/types.d.ts +240 -5
  209. package/dist/types.js +23 -1
  210. package/dist/v2/action-verifier.d.ts +29 -0
  211. package/dist/v2/action-verifier.js +133 -0
  212. package/dist/v2/alt-text.d.ts +26 -0
  213. package/dist/v2/alt-text.js +55 -0
  214. package/dist/v2/benchmark.d.ts +59 -0
  215. package/dist/v2/benchmark.js +135 -0
  216. package/dist/v2/capture-strategy.d.ts +30 -0
  217. package/dist/v2/capture-strategy.js +67 -0
  218. package/dist/v2/capture-verification.d.ts +35 -0
  219. package/dist/v2/capture-verification.js +95 -0
  220. package/dist/v2/circuit-breaker.d.ts +42 -0
  221. package/dist/v2/circuit-breaker.js +119 -0
  222. package/dist/v2/cli-runner-local.d.ts +11 -0
  223. package/dist/v2/cli-runner-local.js +91 -0
  224. package/dist/v2/cli-runner.d.ts +34 -0
  225. package/dist/v2/cli-runner.js +300 -0
  226. package/dist/v2/compiler-prompts.d.ts +27 -0
  227. package/dist/v2/compiler-prompts.js +123 -0
  228. package/dist/v2/compiler.d.ts +37 -0
  229. package/dist/v2/compiler.js +147 -0
  230. package/dist/v2/explorer.d.ts +41 -0
  231. package/dist/v2/explorer.js +56 -0
  232. package/dist/v2/index.d.ts +37 -0
  233. package/dist/v2/index.js +31 -0
  234. package/dist/v2/llm-healer.d.ts +62 -0
  235. package/dist/v2/llm-healer.js +166 -0
  236. package/dist/v2/llm-provider.d.ts +29 -0
  237. package/dist/v2/llm-provider.js +80 -0
  238. package/dist/v2/opcode-runner.d.ts +47 -0
  239. package/dist/v2/opcode-runner.js +634 -0
  240. package/dist/v2/overlay-engine.d.ts +24 -0
  241. package/dist/v2/overlay-engine.js +150 -0
  242. package/dist/v2/postcondition.d.ts +16 -0
  243. package/dist/v2/postcondition.js +249 -0
  244. package/dist/v2/program-patcher.d.ts +25 -0
  245. package/dist/v2/program-patcher.js +44 -0
  246. package/dist/v2/recovery-chain.d.ts +30 -0
  247. package/dist/v2/recovery-chain.js +368 -0
  248. package/dist/v2/schema.d.ts +2580 -0
  249. package/dist/v2/schema.js +295 -0
  250. package/dist/v2/selector-resolver.d.ts +34 -0
  251. package/dist/v2/selector-resolver.js +181 -0
  252. package/dist/v2/semantic-resolver.d.ts +35 -0
  253. package/dist/v2/semantic-resolver.js +161 -0
  254. package/dist/v2/smart-wait.d.ts +27 -0
  255. package/dist/v2/smart-wait.js +81 -0
  256. package/dist/v2/types.d.ts +444 -0
  257. package/dist/v2/types.js +19 -0
  258. package/dist/v2/web-playwright-local.d.ts +69 -0
  259. package/dist/v2/web-playwright-local.js +392 -0
  260. package/dist/version.d.ts +1 -0
  261. package/dist/version.js +5 -0
  262. package/dist/video-agent.js +18 -13
  263. package/dist/video-planner.js +2 -1
  264. package/dist/video-prompts.js +3 -3
  265. package/dist/web-playwright-local.d.ts +126 -0
  266. package/dist/web-playwright-local.js +819 -0
  267. package/dist/ws-auth.js +4 -1
  268. package/dist/ws-broadcast.d.ts +34 -0
  269. package/dist/ws-broadcast.js +85 -0
  270. package/dist/ws-connection-limits.d.ts +12 -0
  271. package/dist/ws-connection-limits.js +44 -0
  272. package/dist/ws-handler-utils.d.ts +32 -0
  273. package/dist/ws-handler-utils.js +139 -0
  274. package/dist/ws-handler.js +294 -164
  275. package/dist/ws-metrics-server.d.ts +9 -0
  276. package/dist/ws-metrics-server.js +31 -0
  277. package/dist/ws-server.js +41 -1
  278. package/package.json +51 -34
@@ -0,0 +1,12 @@
1
+ /**
2
+ * Shared OpenRouter provider configuration.
3
+ *
4
+ * Zero Data Retention (ZDR) tells OpenRouter to only route requests to
5
+ * providers that guarantee they won't store or train on request data.
6
+ * Controlled by the ENABLE_ZDR environment variable (default: off).
7
+ */
8
+ export declare function isZdrEnabled(): boolean;
9
+ /** Build the `provider.zdr` fragment for OpenRouter requests. */
10
+ export declare function zdrParam(): {
11
+ zdr?: true;
12
+ };
@@ -0,0 +1,15 @@
1
+ /**
2
+ * Shared OpenRouter provider configuration.
3
+ *
4
+ * Zero Data Retention (ZDR) tells OpenRouter to only route requests to
5
+ * providers that guarantee they won't store or train on request data.
6
+ * Controlled by the ENABLE_ZDR environment variable (default: off).
7
+ */
8
+ export function isZdrEnabled() {
9
+ return process.env.ENABLE_ZDR === '1' || process.env.ENABLE_ZDR === 'true';
10
+ }
11
+ /** Build the `provider.zdr` fragment for OpenRouter requests. */
12
+ export function zdrParam() {
13
+ return isZdrEnabled() ? { zdr: true } : {};
14
+ }
15
+ //# sourceMappingURL=provider-config.js.map
@@ -0,0 +1,37 @@
1
+ /**
2
+ * Capture Agent — Recovery Chain
3
+ *
4
+ * Ordered recovery strategies when a deterministic opcode fails:
5
+ * 1. Deterministic retry (same opcode, fresh observation)
6
+ * 2. Selector memory lookup (known-good selectors from Supabase)
7
+ * 3. Alternative interaction (keyboard, JS dispatch, coordinates)
8
+ * 4. Targeted reload (reload page and retry)
9
+ * 5. LLM Healer (last resort)
10
+ */
11
+ import type { ExecutionOpcode, RuntimeAdapter } from './execution-types.js';
12
+ import type { RecoveryChain as IRecoveryChain, RecoveryAttemptResult, RecoveryAttemptOptions } from './opcode-runner.js';
13
+ import { type HealerLLMProvider } from './llm-healer.js';
14
+ export interface RecoveryChainOptions {
15
+ /** Selector memory: stepSignature -> known-good selectors */
16
+ selectorMemory?: Record<string, string[]>;
17
+ /** LLM provider for the healer (optional, disables healer if not provided) */
18
+ llmProvider?: HealerLLMProvider;
19
+ /** Max healer invocations per run. Default: 3 */
20
+ maxHealerInvocations?: number;
21
+ /** Full program steps for healer context */
22
+ programSteps?: ExecutionOpcode[];
23
+ /** Credentials for {{email}}/{{password}}/{{loginUrl}} substitution during recovery */
24
+ credentials?: {
25
+ email?: string;
26
+ password?: string;
27
+ loginUrl?: string;
28
+ };
29
+ }
30
+ export declare class RecoveryChainImpl implements IRecoveryChain {
31
+ private healer;
32
+ private selectorMemory;
33
+ private programSteps;
34
+ private credentials?;
35
+ constructor(options?: RecoveryChainOptions);
36
+ attempt(failedOpcode: ExecutionOpcode, opcodeIndex: number, adapter: RuntimeAdapter, options?: RecoveryAttemptOptions): Promise<RecoveryAttemptResult>;
37
+ }
@@ -0,0 +1,350 @@
1
+ /**
2
+ * Capture Agent — Recovery Chain
3
+ *
4
+ * Ordered recovery strategies when a deterministic opcode fails:
5
+ * 1. Deterministic retry (same opcode, fresh observation)
6
+ * 2. Selector memory lookup (known-good selectors from Supabase)
7
+ * 3. Alternative interaction (keyboard, JS dispatch, coordinates)
8
+ * 4. Targeted reload (reload page and retry)
9
+ * 5. LLM Healer (last resort)
10
+ */
11
+ import { resolveSelector } from './selector-resolver.js';
12
+ import { evaluatePostcondition } from './postcondition.js';
13
+ import { LLMHealer } from './llm-healer.js';
14
+ import { serializeAKTree } from './ak-tree.js';
15
+ import { executeOpcodeCoreAction, substituteCredentialPlaceholders } from './opcode-actions.js';
16
+ import { dismissAllOverlays } from './overlay-engine.js';
17
+ import { logger } from './logger.js';
18
+ export class RecoveryChainImpl {
19
+ healer;
20
+ selectorMemory;
21
+ programSteps;
22
+ credentials;
23
+ constructor(options = {}) {
24
+ this.selectorMemory = options.selectorMemory ?? {};
25
+ this.programSteps = options.programSteps ?? [];
26
+ this.credentials = options.credentials;
27
+ this.healer = options.llmProvider
28
+ ? new LLMHealer(options.llmProvider, { maxInvocations: options.maxHealerInvocations ?? 3 })
29
+ : null;
30
+ }
31
+ async attempt(failedOpcode, opcodeIndex, adapter, options = {}) {
32
+ const recovery = failedOpcode.recovery;
33
+ logger.debug(`[recovery ${opcodeIndex}] starting — remainingMs=${options.remainingTimeMs ?? '∞'}, flags=[retries=${recovery.retries},selMem=${recovery.useSelectorMemory},alt=${recovery.useAltInteraction},reload=${recovery.allowReload},healer=${recovery.allowHealer}]`);
34
+ // Strategy 1: Deterministic retry
35
+ const retryBudget = Math.max(0, Math.min(recovery.retries, options.maxDeterministicRetries ?? recovery.retries));
36
+ if (retryBudget > 0) {
37
+ logger.debug(`[recovery ${opcodeIndex}] strategy 1 (retry x${retryBudget})`);
38
+ const result = await retryOpcode(failedOpcode, adapter, retryBudget, options.remainingTimeMs, options.currentVariant, this.credentials);
39
+ logger.debug(`[recovery ${opcodeIndex}] strategy 1 → recovered=${result.recovered}, reason=${result.reason}`);
40
+ if (result.recovered)
41
+ return result;
42
+ }
43
+ // Strategy 2: Selector memory + alternate selectors
44
+ if (recovery.useSelectorMemory && hasSelector(failedOpcode)) {
45
+ logger.debug(`[recovery ${opcodeIndex}] strategy 2 (selector memory)`);
46
+ const result = await trySelectorAlternatives(failedOpcode, opcodeIndex, adapter, this.selectorMemory, this.credentials);
47
+ logger.debug(`[recovery ${opcodeIndex}] strategy 2 → recovered=${result.recovered}, reason=${result.reason}`);
48
+ if (result.recovered)
49
+ return result;
50
+ }
51
+ // Strategy 3: Alternative interaction methods
52
+ if (recovery.useAltInteraction && hasSelector(failedOpcode)) {
53
+ logger.debug(`[recovery ${opcodeIndex}] strategy 3 (alt interaction: keyboard, js_dispatch, coords)`);
54
+ const result = await tryAltInteraction(failedOpcode, adapter, this.credentials);
55
+ logger.debug(`[recovery ${opcodeIndex}] strategy 3 → recovered=${result.recovered}, reason=${result.reason}`);
56
+ if (result.recovered)
57
+ return result;
58
+ }
59
+ // Strategy 4: Targeted reload
60
+ if (recovery.allowReload) {
61
+ logger.debug(`[recovery ${opcodeIndex}] strategy 4 (reload)`);
62
+ const result = await tryReload(failedOpcode, adapter, options.currentVariant, this.credentials);
63
+ logger.debug(`[recovery ${opcodeIndex}] strategy 4 → recovered=${result.recovered}, reason=${result.reason}`);
64
+ if (result.recovered)
65
+ return result;
66
+ }
67
+ // Strategy 5: LLM Healer
68
+ if (recovery.allowHealer && this.healer && this.healer.remainingInvocations > 0) {
69
+ logger.debug(`[recovery ${opcodeIndex}] strategy 5 (LLM healer)`);
70
+ const result = await tryHealer(failedOpcode, opcodeIndex, adapter, this.healer, this.programSteps, options.currentVariant, this.credentials);
71
+ logger.debug(`[recovery ${opcodeIndex}] strategy 5 → recovered=${result.recovered}, reason=${result.reason}`);
72
+ return result;
73
+ }
74
+ logger.debug(`[recovery ${opcodeIndex}] all strategies exhausted`);
75
+ return { recovered: false, reason: 'all recovery strategies exhausted' };
76
+ }
77
+ }
78
+ // ── Strategy 1: Deterministic retry ─────────────────────────────────
79
+ async function retryOpcode(opcode, adapter, maxRetries, remainingTimeMs, currentVariant, credentials) {
80
+ const deadlineMs = typeof remainingTimeMs === 'number'
81
+ ? Date.now() + remainingTimeMs
82
+ : Number.POSITIVE_INFINITY;
83
+ for (let i = 0; i < maxRetries; i++) {
84
+ if (Date.now() >= deadlineMs) {
85
+ return { recovered: false, reason: 'retry budget exhausted by timeout' };
86
+ }
87
+ await sleep(500 * (i + 1)); // backoff
88
+ try {
89
+ await executeRawAction(opcode, adapter, currentVariant, credentials);
90
+ const postcondition = await evaluatePostcondition(adapter, clampPostconditionWait(opcode.postcondition, deadlineMs));
91
+ if (postcondition.passed) {
92
+ return { recovered: true, strategy: 'retry', reason: `retry ${i + 1}/${maxRetries} succeeded` };
93
+ }
94
+ }
95
+ catch {
96
+ // continue to next retry
97
+ }
98
+ }
99
+ return { recovered: false, reason: `${maxRetries} retries exhausted` };
100
+ }
101
+ // ── Strategy 2: Selector memory ─────────────────────────────────────
102
+ async function trySelectorAlternatives(opcode, opcodeIndex, adapter, selectorMemory, credentials) {
103
+ if (!hasSelector(opcode)) {
104
+ return { recovered: false, reason: 'opcode has no selector' };
105
+ }
106
+ const selectorOpcode = opcode;
107
+ const primarySelector = selectorOpcode.selector ?? '';
108
+ const stepSignature = `${opcode.kind}:${primarySelector}`;
109
+ const resolved = await resolveSelector(adapter, {
110
+ primary: primarySelector,
111
+ fingerprint: selectorOpcode.fingerprint,
112
+ alternates: selectorOpcode.selectorAlternates,
113
+ selectorMemory,
114
+ stepSignature,
115
+ description: opcode.description,
116
+ });
117
+ if (!resolved || resolved.strategy === 'primary') {
118
+ return { recovered: false, reason: 'no alternative selectors found' };
119
+ }
120
+ // Try the resolved selector
121
+ try {
122
+ await executeWithSelector(opcode, adapter, resolved.selector, credentials);
123
+ const postcondition = await evaluatePostcondition(adapter, opcode.postcondition);
124
+ if (postcondition.passed) {
125
+ return {
126
+ recovered: true,
127
+ strategy: 'selector_memory',
128
+ reason: `resolved via ${resolved.strategy}: "${resolved.selector}"`,
129
+ };
130
+ }
131
+ }
132
+ catch {
133
+ // fall through
134
+ }
135
+ return { recovered: false, reason: `resolved selector "${resolved.selector}" also failed` };
136
+ }
137
+ // ── Strategy 3: Alternative interaction ─────────────────────────────
138
+ async function tryAltInteraction(opcode, adapter, _credentials) {
139
+ if (opcode.kind !== 'CLICK' || !opcode.selector) {
140
+ return { recovered: false, reason: 'alt interaction only for CLICK opcodes with selectors' };
141
+ }
142
+ const clickOpcode = opcode;
143
+ const sel = clickOpcode.selector;
144
+ const methods = [
145
+ { name: 'keyboard', options: { useKeyboard: true } },
146
+ { name: 'js_dispatch', options: { useJsDispatch: true } },
147
+ ];
148
+ for (const method of methods) {
149
+ try {
150
+ await adapter.click(sel, method.options);
151
+ await sleep(500);
152
+ const postcondition = await evaluatePostcondition(adapter, opcode.postcondition);
153
+ if (postcondition.passed) {
154
+ return {
155
+ recovered: true,
156
+ strategy: 'alt_interaction',
157
+ reason: `${method.name} interaction succeeded`,
158
+ };
159
+ }
160
+ }
161
+ catch {
162
+ // continue to next method
163
+ }
164
+ }
165
+ // Try coordinate-based click from AKTree
166
+ try {
167
+ const tree = await adapter.getAKTree();
168
+ const node = findInteractiveNodeBySelector(tree, sel);
169
+ if (node) {
170
+ const x = node.bounds.x + node.bounds.w / 2;
171
+ const y = node.bounds.y + node.bounds.h / 2;
172
+ await adapter.click(sel, { coordinates: { x, y } });
173
+ await sleep(500);
174
+ const postcondition = await evaluatePostcondition(adapter, opcode.postcondition);
175
+ if (postcondition.passed) {
176
+ return {
177
+ recovered: true,
178
+ strategy: 'alt_interaction',
179
+ reason: `coordinate click at (${x}, ${y}) succeeded`,
180
+ };
181
+ }
182
+ }
183
+ }
184
+ catch {
185
+ // fall through
186
+ }
187
+ return { recovered: false, reason: 'all alternative interactions failed' };
188
+ }
189
+ // ── Strategy 4: Targeted reload ─────────────────────────────────────
190
+ async function tryReload(opcode, adapter, currentVariant, credentials) {
191
+ try {
192
+ if (adapter.reloadPage) {
193
+ await adapter.reloadPage();
194
+ }
195
+ else {
196
+ const url = await adapter.getCurrentUrl();
197
+ await adapter.navigate(url);
198
+ }
199
+ await sleep(1000);
200
+ await dismissAllOverlays(adapter);
201
+ await executeRawAction(opcode, adapter, currentVariant, credentials);
202
+ const postcondition = await evaluatePostcondition(adapter, opcode.postcondition);
203
+ if (postcondition.passed) {
204
+ return { recovered: true, strategy: 'reload', reason: 'reload + retry succeeded' };
205
+ }
206
+ }
207
+ catch {
208
+ // fall through
209
+ }
210
+ return { recovered: false, reason: 'reload + retry failed' };
211
+ }
212
+ // ── Strategy 5: LLM Healer ─────────────────────────────────────────
213
+ async function tryHealer(failedOpcode, opcodeIndex, adapter, healer, programSteps, currentVariant, credentials) {
214
+ try {
215
+ const tree = await adapter.getAKTree();
216
+ const akTreeSerialized = serializeAKTree(tree);
217
+ const currentUrl = await adapter.getCurrentUrl();
218
+ let screenshot;
219
+ try {
220
+ screenshot = await adapter.takeScreenshot();
221
+ }
222
+ catch { /* optional */ }
223
+ // Get surrounding opcodes for context
224
+ const start = Math.max(0, opcodeIndex - 3);
225
+ const end = Math.min(programSteps.length, opcodeIndex + 4);
226
+ const surroundingOpcodes = programSteps.slice(start, end).filter((_, i) => start + i !== opcodeIndex);
227
+ const context = {
228
+ failedOpcode,
229
+ opcodeIndex,
230
+ akTreeSerialized,
231
+ currentUrl,
232
+ screenshot,
233
+ surroundingOpcodes,
234
+ errorMessage: `opcode ${failedOpcode.kind} failed after all deterministic recovery strategies`,
235
+ };
236
+ const result = await healer.heal(context);
237
+ if (result.healed && result.patch) {
238
+ // Execute the replacement opcode(s) to verify they work
239
+ for (const replacementOp of result.patch.replacementOpcodes) {
240
+ try {
241
+ await executeRawAction(replacementOp, adapter, currentVariant, credentials);
242
+ }
243
+ catch (err) {
244
+ return {
245
+ recovered: false,
246
+ llmResult: result.llmResult,
247
+ reason: `healer patch failed during execution: ${err instanceof Error ? err.message : String(err)}`,
248
+ };
249
+ }
250
+ }
251
+ // Verify the last replacement's postcondition
252
+ const lastOp = result.patch.replacementOpcodes[result.patch.replacementOpcodes.length - 1];
253
+ const postcondition = await evaluatePostcondition(adapter, lastOp.postcondition);
254
+ if (postcondition.passed) {
255
+ return {
256
+ recovered: true,
257
+ strategy: 'healer',
258
+ patch: result.patch,
259
+ llmResult: result.llmResult,
260
+ reason: result.reason,
261
+ };
262
+ }
263
+ return {
264
+ recovered: false,
265
+ llmResult: result.llmResult,
266
+ reason: `healer patch executed but postcondition failed: ${postcondition.reason}`,
267
+ };
268
+ }
269
+ return { recovered: false, llmResult: result.llmResult, reason: result.reason };
270
+ }
271
+ catch (err) {
272
+ return {
273
+ recovered: false,
274
+ reason: `healer error: ${err instanceof Error ? err.message : String(err)}`,
275
+ };
276
+ }
277
+ }
278
+ // ── Shared helpers ──────────────────────────────────────────────────
279
+ function hasSelector(opcode) {
280
+ return 'selector' in opcode && typeof opcode.selector === 'string' && !!opcode.selector;
281
+ }
282
+ async function executeRawAction(opcode, adapter, currentVariant, credentials) {
283
+ const result = await executeOpcodeCoreAction(opcode, adapter, { currentVariant, credentials });
284
+ if (!result.success) {
285
+ throw new Error(result.error ?? `opcode ${opcode.kind} could not be replayed`);
286
+ }
287
+ }
288
+ async function executeWithSelector(opcode, adapter, newSelector, credentials) {
289
+ switch (opcode.kind) {
290
+ case 'CLICK':
291
+ await adapter.click(newSelector, opcode.button ? { button: opcode.button } : undefined);
292
+ break;
293
+ case 'TYPE': {
294
+ const text = substituteCredentialPlaceholders(opcode.text, credentials);
295
+ await adapter.type(newSelector, text, opcode.clearFirst);
296
+ break;
297
+ }
298
+ case 'HOVER':
299
+ if (!adapter.hover)
300
+ throw new Error('adapter does not support HOVER');
301
+ await adapter.hover(newSelector);
302
+ break;
303
+ case 'SELECT_OPTION':
304
+ if (!adapter.selectOption)
305
+ throw new Error('adapter does not support SELECT_OPTION');
306
+ await adapter.selectOption(newSelector, {
307
+ label: opcode.optionLabel,
308
+ value: opcode.optionValue,
309
+ index: opcode.optionIndex,
310
+ });
311
+ break;
312
+ case 'CHECK':
313
+ if (!adapter.check)
314
+ throw new Error('adapter does not support CHECK');
315
+ await adapter.check(newSelector, opcode.checked);
316
+ break;
317
+ case 'DOUBLE_CLICK':
318
+ if (!adapter.doubleClick)
319
+ throw new Error('adapter does not support DOUBLE_CLICK');
320
+ await adapter.doubleClick(newSelector);
321
+ break;
322
+ default:
323
+ throw new Error(`cannot swap selector for opcode kind: ${opcode.kind}`);
324
+ }
325
+ }
326
+ function findInteractiveNodeBySelector(tree, selector) {
327
+ function walk(node) {
328
+ if (node.visible && node.interactive && node.sourceRef.includes(selector.replace(/[[\]"#.]/g, ''))) {
329
+ return node;
330
+ }
331
+ for (const child of node.children) {
332
+ const found = walk(child);
333
+ if (found)
334
+ return found;
335
+ }
336
+ return null;
337
+ }
338
+ return walk(tree.root);
339
+ }
340
+ function sleep(ms) {
341
+ return new Promise(resolve => setTimeout(resolve, ms));
342
+ }
343
+ function clampPostconditionWait(spec, deadlineMs) {
344
+ const remainingTimeMs = Math.max(1, deadlineMs - Date.now());
345
+ return {
346
+ ...spec,
347
+ waitMs: Math.max(1, Math.min(spec.waitMs ?? remainingTimeMs, remainingTimeMs)),
348
+ };
349
+ }
350
+ //# sourceMappingURL=recovery-chain.js.map
@@ -5,7 +5,7 @@
5
5
  * Used by the WS server to control a Playwright browser running on the user's machine.
6
6
  */
7
7
  import type { WebSocket } from 'ws';
8
- import type { PageState, PageStateLite, InteractiveElement, BrowserStorageState, BrowserSessionStorageState, VideoPageSignals, OutscaleConfig, SelectorValidationResult } from './types.js';
8
+ import type { PageState, PageStateLite, InteractiveElement, BrowserStorageState, BrowserSessionStorageState, VideoPageSignals, OutscaleConfig, SelectorValidationResult, AKTree, AKNode, AKNodeRuntimeIndexEntry, FocusQuery } from './types.js';
9
9
  import type { BrowserObservation, BrowserReaction, BrowserSelectorProbe, BrowserStorageHintWriteParams, BrowserVerificationBundle, BrowserVideoVerificationBundle } from './browser.js';
10
10
  export interface WsCommand {
11
11
  id: string;
@@ -18,14 +18,15 @@ export interface WsResult {
18
18
  error?: string;
19
19
  }
20
20
  export interface WsEvent {
21
- type: 'done' | 'error' | 'progress';
21
+ type: 'done' | 'error' | 'progress' | 'live_url';
22
22
  message?: string;
23
+ url?: string;
23
24
  summary?: {
24
25
  successes: number;
25
26
  total: number;
26
27
  };
27
28
  }
28
- export declare const REMOTE_BROWSER_PROTOCOL_VERSION = 2;
29
+ export declare const REMOTE_BROWSER_PROTOCOL_VERSION = 3;
29
30
  export interface RemoteBrowserClientInfo {
30
31
  version?: string;
31
32
  protocolVersion?: number;
@@ -33,8 +34,9 @@ export interface RemoteBrowserClientInfo {
33
34
  export interface WsClientHello {
34
35
  type: 'client_hello';
35
36
  client: RemoteBrowserClientInfo;
37
+ apiKey?: string;
36
38
  }
37
- export declare function buildRemoteBrowserClientHello(version: string): WsClientHello;
39
+ export declare function buildRemoteBrowserClientHello(version: string, apiKey?: string): WsClientHello;
38
40
  export declare function getRemoteBrowserCompatibilityError(client: RemoteBrowserClientInfo | null | undefined): string | null;
39
41
  /**
40
42
  * Lightweight proxy for `browser.currentPage` — only exposes the methods
@@ -153,6 +155,26 @@ export declare class RemoteBrowser {
153
155
  }): Promise<void>;
154
156
  pressKey(key: string): Promise<void>;
155
157
  searchText(query: string): Promise<Array<Record<string, unknown>>>;
158
+ getAKTree(): Promise<AKTree>;
159
+ resolveAKNode(nodeId: string, options?: {
160
+ reparse?: boolean;
161
+ }): Promise<AKNodeRuntimeIndexEntry | null>;
162
+ tapNode(nodeId: string, options?: {
163
+ force?: boolean;
164
+ }): Promise<void>;
165
+ typeIntoNode(nodeId: string, text: string, options?: {
166
+ clearFirst?: boolean;
167
+ }): Promise<void>;
168
+ captureNode(nodeId?: string): Promise<Buffer>;
169
+ focusTree(query: FocusQuery): Promise<{
170
+ tree: AKTree;
171
+ serialized: string;
172
+ matches: AKNode[];
173
+ }>;
174
+ centerNodeInView(centerNodeId: string, options?: {
175
+ containerNodeId?: string;
176
+ offset?: number;
177
+ }): Promise<void>;
156
178
  waitForPageReaction(before?: BrowserObservation, options?: {
157
179
  timeoutMs?: number;
158
180
  settleMs?: number;
@@ -183,6 +205,8 @@ export declare class RemoteBrowser {
183
205
  observeNetworkRequests(url: string, waitMs?: number): Promise<Array<Record<string, unknown>>>;
184
206
  setupRouteInterception(mocks: Array<Record<string, unknown>>): Promise<void>;
185
207
  clearRouteInterception(): Promise<void>;
208
+ /** Explicit allowlist of callable RPC methods — defense-in-depth. */
209
+ private static readonly ALLOWED_METHODS;
186
210
  private call;
187
211
  private callAndSyncPageInfo;
188
212
  sendEvent(event: WsEvent): void;
@@ -4,14 +4,16 @@
4
4
  *
5
5
  * Used by the WS server to control a Playwright browser running on the user's machine.
6
6
  */
7
- export const REMOTE_BROWSER_PROTOCOL_VERSION = 2;
8
- export function buildRemoteBrowserClientHello(version) {
7
+ import { logger } from './logger.js';
8
+ export const REMOTE_BROWSER_PROTOCOL_VERSION = 3;
9
+ export function buildRemoteBrowserClientHello(version, apiKey) {
9
10
  return {
10
11
  type: 'client_hello',
11
12
  client: {
12
13
  version,
13
14
  protocolVersion: REMOTE_BROWSER_PROTOCOL_VERSION,
14
15
  },
16
+ ...(apiKey ? { apiKey } : {}),
15
17
  };
16
18
  }
17
19
  export function getRemoteBrowserCompatibilityError(client) {
@@ -80,7 +82,10 @@ export class RemoteBrowser {
80
82
  call.resolve(msg.result);
81
83
  }
82
84
  }
83
- catch { /* ignore malformed messages */ }
85
+ catch {
86
+ const raw = typeof data === 'string' ? data : data.toString('utf-8');
87
+ logger.warn(`[WS] Malformed message received (${raw.length} bytes)`);
88
+ }
84
89
  });
85
90
  }
86
91
  // ── Lifecycle ────────────────────────────────────────────────────
@@ -217,6 +222,29 @@ export class RemoteBrowser {
217
222
  async searchText(query) {
218
223
  return await this.call('searchText', { query });
219
224
  }
225
+ // ── AKTree ──────────────────────────────────────────────────────
226
+ async getAKTree() {
227
+ return await this.call('getAKTree', {});
228
+ }
229
+ async resolveAKNode(nodeId, options = {}) {
230
+ return await this.call('resolveAKNode', { nodeId, ...options });
231
+ }
232
+ async tapNode(nodeId, options) {
233
+ await this.callAndSyncPageInfo('tapNode', { nodeId, ...options });
234
+ }
235
+ async typeIntoNode(nodeId, text, options) {
236
+ await this.callAndSyncPageInfo('typeIntoNode', { nodeId, text, ...options });
237
+ }
238
+ async captureNode(nodeId) {
239
+ const r = await this.call('captureNode', { nodeId });
240
+ return Buffer.from(r.screenshot, 'base64');
241
+ }
242
+ async focusTree(query) {
243
+ return await this.call('focusTree', query);
244
+ }
245
+ async centerNodeInView(centerNodeId, options) {
246
+ await this.callAndSyncPageInfo('centerNodeInView', { centerOn: centerNodeId, ...options });
247
+ }
220
248
  // ── Reactions ────────────────────────────────────────────────────
221
249
  async waitForPageReaction(before, options) {
222
250
  const reaction = await this.call('waitForPageReaction', { before, ...options });
@@ -239,8 +267,7 @@ export class RemoteBrowser {
239
267
  await this.callAndSyncPageInfo('setLanguage', { lang });
240
268
  }
241
269
  async resizeViewport(width, height) {
242
- await this.call('resizeViewport', { width, height });
243
- this._pageProxy._cachedViewport = { width, height };
270
+ await this.callAndSyncPageInfo('resizeViewport', { width, height });
244
271
  }
245
272
  async forceLoadLazyImages(options) {
246
273
  await this.call('forceLoadLazyImages', options ?? {});
@@ -275,7 +302,35 @@ export class RemoteBrowser {
275
302
  await this.call('clearRouteInterception', {});
276
303
  }
277
304
  // ── RPC core ─────────────────────────────────────────────────────
305
+ /** Explicit allowlist of callable RPC methods — defense-in-depth. */
306
+ static ALLOWED_METHODS = new Set([
307
+ 'launch', 'close', 'closeContext', 'recreateContext',
308
+ 'setDeviceScaleFactor', 'navigateTo', 'addCookies',
309
+ 'getPageInfo', 'getPageTitle',
310
+ 'takeScreenshot', 'takeScreenshotForAI',
311
+ 'screenshotElement', 'screenshotRegion', 'screenshotBySelector', 'screenshotByRegion',
312
+ 'getPageState', 'getPageStateLite', 'getAccessibilityTree',
313
+ 'getInteractiveElements', 'getSimplifiedDOM',
314
+ 'captureObservation', 'capturePageSignals',
315
+ 'captureVerificationBundle', 'captureVideoVerificationBundle',
316
+ 'clickByIndex', 'clickBySelector', 'clickByCoordinates',
317
+ 'hoverByIndex', 'hoverBySelector', 'hoverByCoordinates',
318
+ 'typeText', 'selectOption', 'scroll', 'scrollElementIntoView',
319
+ 'safeExpand', 'pressKey', 'searchText',
320
+ 'getAKTree', 'resolveAKNode', 'tapNode', 'typeIntoNode',
321
+ 'captureNode', 'focusTree', 'centerNodeInView',
322
+ 'waitForPageReaction', 'wait', 'dismissOverlays',
323
+ 'setColorScheme', 'setLanguage', 'resizeViewport',
324
+ 'forceLoadLazyImages',
325
+ 'exportStorageState', 'exportSessionStorage',
326
+ 'reloadCurrentPage', 'writeStorageHintCandidate',
327
+ 'probeSelector', 'prepareSessionStorage',
328
+ 'observeNetworkRequests', 'setupRouteInterception', 'clearRouteInterception',
329
+ ]);
278
330
  call(method, params, timeoutMs = DEFAULT_TIMEOUT_MS) {
331
+ if (!RemoteBrowser.ALLOWED_METHODS.has(method)) {
332
+ return Promise.reject(new Error(`RemoteBrowser: method "${method}" is not allowed`));
333
+ }
279
334
  return new Promise((resolve, reject) => {
280
335
  const id = `cmd_${++this.idCounter}`;
281
336
  const timer = setTimeout(() => {
@@ -0,0 +1,15 @@
1
+ /**
2
+ * Safari macOS browser bar — uses the Figma-exported asset directly.
3
+ *
4
+ * Reference SVG: assets/frames/Safari tool bar.svg (1299×88, single-row toolbar).
5
+ * The asset is embedded as a string constant via `safari-toolbar-asset.ts`
6
+ * (auto-generated). All icons are baked as paths — only the URL inside
7
+ * the address pill is dynamic.
8
+ *
9
+ * Two outputs:
10
+ * - HTML (Playwright preview / React) — inline SVG inside a scaled wrapper
11
+ * - SVG (sharp / resvg server compositing)
12
+ */
13
+ import type { BrowserBarRenderOptions } from './browser-bar.js';
14
+ export declare function generateSafariBrowserBarHtml(options: BrowserBarRenderOptions): string;
15
+ export declare function generateSafariBrowserBarSvg(options: BrowserBarRenderOptions): string;