autokap 1.1.4 → 1.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (184) hide show
  1. package/dist/cli-runner.js +70 -15
  2. package/dist/cli.js +0 -0
  3. package/dist/clip-capture-loop.d.ts +12 -0
  4. package/dist/clip-capture-loop.js +26 -0
  5. package/dist/web-playwright-local.js +4 -1
  6. package/package.json +1 -1
  7. package/dist/agent-action-recovery.d.ts +0 -45
  8. package/dist/agent-action-recovery.js +0 -370
  9. package/dist/agent-message-utils.d.ts +0 -21
  10. package/dist/agent-message-utils.js +0 -77
  11. package/dist/agent-url-utils.d.ts +0 -30
  12. package/dist/agent-url-utils.js +0 -138
  13. package/dist/agent.d.ts +0 -226
  14. package/dist/agent.js +0 -6666
  15. package/dist/browser-utils.d.ts +0 -31
  16. package/dist/browser-utils.js +0 -97
  17. package/dist/capture-studio-sync.d.ts +0 -23
  18. package/dist/capture-studio-sync.js +0 -172
  19. package/dist/clip-begin-frame-recorder.d.ts +0 -44
  20. package/dist/clip-begin-frame-recorder.js +0 -250
  21. package/dist/clip-capture-backend.d.ts +0 -25
  22. package/dist/clip-capture-backend.js +0 -189
  23. package/dist/clip-frame-recorder.d.ts +0 -63
  24. package/dist/clip-frame-recorder.js +0 -305
  25. package/dist/clip-orchestrator.d.ts +0 -148
  26. package/dist/clip-orchestrator.js +0 -957
  27. package/dist/clip-runtime.d.ts +0 -18
  28. package/dist/clip-runtime.js +0 -67
  29. package/dist/clip-scale.d.ts +0 -10
  30. package/dist/clip-scale.js +0 -21
  31. package/dist/clip-screencast-recorder.d.ts +0 -42
  32. package/dist/clip-screencast-recorder.js +0 -242
  33. package/dist/clip-sidecar.d.ts +0 -54
  34. package/dist/clip-sidecar.js +0 -208
  35. package/dist/cost-resolution-monitor.d.ts +0 -16
  36. package/dist/cost-resolution-monitor.js +0 -34
  37. package/dist/credential-templates.d.ts +0 -5
  38. package/dist/credential-templates.js +0 -60
  39. package/dist/dom-css-purger.d.ts +0 -65
  40. package/dist/dom-css-purger.js +0 -333
  41. package/dist/dom-font-inliner.d.ts +0 -45
  42. package/dist/dom-font-inliner.js +0 -148
  43. package/dist/dom-patch-resolver.d.ts +0 -52
  44. package/dist/dom-patch-resolver.js +0 -242
  45. package/dist/dom-serializer.d.ts +0 -82
  46. package/dist/dom-serializer.js +0 -378
  47. package/dist/element-capture.d.ts +0 -13
  48. package/dist/element-capture.js +0 -522
  49. package/dist/fonts-loader.d.ts +0 -14
  50. package/dist/fonts-loader.js +0 -55
  51. package/dist/hybrid-navigator.d.ts +0 -138
  52. package/dist/hybrid-navigator.js +0 -468
  53. package/dist/legacy/agent-action-recovery.d.ts +0 -45
  54. package/dist/legacy/agent-action-recovery.js +0 -370
  55. package/dist/legacy/agent-message-utils.d.ts +0 -21
  56. package/dist/legacy/agent-message-utils.js +0 -77
  57. package/dist/legacy/agent-url-utils.d.ts +0 -30
  58. package/dist/legacy/agent-url-utils.js +0 -138
  59. package/dist/legacy/agent.d.ts +0 -226
  60. package/dist/legacy/agent.js +0 -6666
  61. package/dist/legacy/clip-orchestrator.d.ts +0 -148
  62. package/dist/legacy/clip-orchestrator.js +0 -957
  63. package/dist/legacy/credential-templates.d.ts +0 -5
  64. package/dist/legacy/credential-templates.js +0 -60
  65. package/dist/legacy/hybrid-navigator.d.ts +0 -138
  66. package/dist/legacy/hybrid-navigator.js +0 -468
  67. package/dist/legacy/llm-usage.d.ts +0 -17
  68. package/dist/legacy/llm-usage.js +0 -45
  69. package/dist/legacy/prompt-cache.d.ts +0 -10
  70. package/dist/legacy/prompt-cache.js +0 -24
  71. package/dist/legacy/prompts.d.ts +0 -175
  72. package/dist/legacy/prompts.js +0 -1038
  73. package/dist/legacy/tools.d.ts +0 -4
  74. package/dist/legacy/tools.js +0 -216
  75. package/dist/legacy/video-agent.d.ts +0 -143
  76. package/dist/legacy/video-agent.js +0 -4788
  77. package/dist/legacy/video-observation.d.ts +0 -36
  78. package/dist/legacy/video-observation.js +0 -192
  79. package/dist/legacy/video-planner.d.ts +0 -12
  80. package/dist/legacy/video-planner.js +0 -501
  81. package/dist/legacy/video-prompts.d.ts +0 -37
  82. package/dist/legacy/video-prompts.js +0 -569
  83. package/dist/legacy/video-tools.d.ts +0 -3
  84. package/dist/legacy/video-tools.js +0 -59
  85. package/dist/legacy/video-variant-state.d.ts +0 -29
  86. package/dist/legacy/video-variant-state.js +0 -80
  87. package/dist/legacy/vision-model.d.ts +0 -17
  88. package/dist/legacy/vision-model.js +0 -74
  89. package/dist/llm-usage.d.ts +0 -17
  90. package/dist/llm-usage.js +0 -45
  91. package/dist/overlay-utils.d.ts +0 -14
  92. package/dist/overlay-utils.js +0 -13
  93. package/dist/prompt-cache.d.ts +0 -10
  94. package/dist/prompt-cache.js +0 -24
  95. package/dist/prompts.d.ts +0 -175
  96. package/dist/prompts.js +0 -1038
  97. package/dist/remote-browser.d.ts +0 -215
  98. package/dist/remote-browser.js +0 -360
  99. package/dist/svg-browser-bar.d.ts +0 -33
  100. package/dist/svg-browser-bar.js +0 -206
  101. package/dist/svg-status-bar.d.ts +0 -36
  102. package/dist/svg-status-bar.js +0 -597
  103. package/dist/svg-text.d.ts +0 -61
  104. package/dist/svg-text.js +0 -118
  105. package/dist/tools.d.ts +0 -4
  106. package/dist/tools.js +0 -216
  107. package/dist/v2/action-verifier.d.ts +0 -29
  108. package/dist/v2/action-verifier.js +0 -133
  109. package/dist/v2/alt-text.d.ts +0 -26
  110. package/dist/v2/alt-text.js +0 -55
  111. package/dist/v2/benchmark.d.ts +0 -59
  112. package/dist/v2/benchmark.js +0 -135
  113. package/dist/v2/capture-strategy.d.ts +0 -30
  114. package/dist/v2/capture-strategy.js +0 -67
  115. package/dist/v2/capture-verification.d.ts +0 -35
  116. package/dist/v2/capture-verification.js +0 -95
  117. package/dist/v2/circuit-breaker.d.ts +0 -42
  118. package/dist/v2/circuit-breaker.js +0 -119
  119. package/dist/v2/cli-runner-local.d.ts +0 -11
  120. package/dist/v2/cli-runner-local.js +0 -91
  121. package/dist/v2/cli-runner.d.ts +0 -34
  122. package/dist/v2/cli-runner.js +0 -300
  123. package/dist/v2/compiler-prompts.d.ts +0 -27
  124. package/dist/v2/compiler-prompts.js +0 -123
  125. package/dist/v2/compiler.d.ts +0 -37
  126. package/dist/v2/compiler.js +0 -147
  127. package/dist/v2/explorer.d.ts +0 -41
  128. package/dist/v2/explorer.js +0 -56
  129. package/dist/v2/index.d.ts +0 -37
  130. package/dist/v2/index.js +0 -31
  131. package/dist/v2/llm-healer.d.ts +0 -62
  132. package/dist/v2/llm-healer.js +0 -166
  133. package/dist/v2/llm-provider.d.ts +0 -29
  134. package/dist/v2/llm-provider.js +0 -80
  135. package/dist/v2/opcode-runner.d.ts +0 -47
  136. package/dist/v2/opcode-runner.js +0 -634
  137. package/dist/v2/overlay-engine.d.ts +0 -24
  138. package/dist/v2/overlay-engine.js +0 -150
  139. package/dist/v2/postcondition.d.ts +0 -16
  140. package/dist/v2/postcondition.js +0 -249
  141. package/dist/v2/program-patcher.d.ts +0 -25
  142. package/dist/v2/program-patcher.js +0 -44
  143. package/dist/v2/recovery-chain.d.ts +0 -30
  144. package/dist/v2/recovery-chain.js +0 -368
  145. package/dist/v2/schema.d.ts +0 -2580
  146. package/dist/v2/schema.js +0 -295
  147. package/dist/v2/selector-resolver.d.ts +0 -34
  148. package/dist/v2/selector-resolver.js +0 -181
  149. package/dist/v2/semantic-resolver.d.ts +0 -35
  150. package/dist/v2/semantic-resolver.js +0 -161
  151. package/dist/v2/smart-wait.d.ts +0 -27
  152. package/dist/v2/smart-wait.js +0 -81
  153. package/dist/v2/types.d.ts +0 -444
  154. package/dist/v2/types.js +0 -19
  155. package/dist/v2/web-playwright-local.d.ts +0 -69
  156. package/dist/v2/web-playwright-local.js +0 -392
  157. package/dist/video-agent.d.ts +0 -143
  158. package/dist/video-agent.js +0 -4788
  159. package/dist/video-observation.d.ts +0 -36
  160. package/dist/video-observation.js +0 -192
  161. package/dist/video-planner.d.ts +0 -12
  162. package/dist/video-planner.js +0 -501
  163. package/dist/video-prompts.d.ts +0 -37
  164. package/dist/video-prompts.js +0 -554
  165. package/dist/video-tools.d.ts +0 -3
  166. package/dist/video-tools.js +0 -59
  167. package/dist/video-variant-state.d.ts +0 -29
  168. package/dist/video-variant-state.js +0 -80
  169. package/dist/vision-model.d.ts +0 -17
  170. package/dist/vision-model.js +0 -74
  171. package/dist/ws-auth.d.ts +0 -20
  172. package/dist/ws-auth.js +0 -70
  173. package/dist/ws-broadcast.d.ts +0 -34
  174. package/dist/ws-broadcast.js +0 -85
  175. package/dist/ws-connection-limits.d.ts +0 -12
  176. package/dist/ws-connection-limits.js +0 -44
  177. package/dist/ws-handler-utils.d.ts +0 -32
  178. package/dist/ws-handler-utils.js +0 -139
  179. package/dist/ws-handler.d.ts +0 -10
  180. package/dist/ws-handler.js +0 -1793
  181. package/dist/ws-metrics-server.d.ts +0 -9
  182. package/dist/ws-metrics-server.js +0 -31
  183. package/dist/ws-server.d.ts +0 -9
  184. package/dist/ws-server.js +0 -92
@@ -27,6 +27,8 @@ import { callLLM } from './llm-provider.js';
27
27
  import { APP_VERSION } from './version.js';
28
28
  import { normalizeAllowedOrigins, normalizeHttpOrigin, verifySignedExecutionProgramEnvelope, } from './program-signing.js';
29
29
  const MAX_CLIP_CAPTURE_DEVICE_SCALE_FACTOR = 1;
30
+ const FETCH_PROGRAM_MAX_ATTEMPTS = 4;
31
+ const FETCH_PROGRAM_RETRY_DELAYS_MS = [1000, 3000, 5000];
30
32
  const HEALER_SYSTEM_PROMPT = 'You repair failed deterministic browser opcodes. Respond only with JSON.';
31
33
  // ── Main entry point ────────────────────────────────────────────────
32
34
  export async function runCapture(options) {
@@ -184,22 +186,55 @@ export async function runCapture(options) {
184
186
  }
185
187
  // ── Server communication ────────────────────────────────────────────
186
188
  async function fetchProgram(config, presetId, environmentName) {
187
- try {
188
- const url = new URL(`${config.apiBaseUrl}/api/cli/programs/${presetId}`);
189
- if (environmentName) {
190
- url.searchParams.set('env', environmentName);
189
+ const url = new URL(`${config.apiBaseUrl}/api/cli/programs/${presetId}`);
190
+ if (environmentName) {
191
+ url.searchParams.set('env', environmentName);
192
+ }
193
+ const requestedUrl = url.toString();
194
+ for (let attempt = 1; attempt <= FETCH_PROGRAM_MAX_ATTEMPTS; attempt += 1) {
195
+ let response;
196
+ try {
197
+ response = await fetch(requestedUrl, {
198
+ headers: {
199
+ 'Authorization': `Bearer ${config.apiKey}`,
200
+ 'Content-Type': 'application/json',
201
+ [CLI_VERSION_HEADER]: APP_VERSION,
202
+ },
203
+ });
204
+ }
205
+ catch (err) {
206
+ const error = `failed to fetch program: ${err instanceof Error ? err.message : String(err)}`;
207
+ if (attempt < FETCH_PROGRAM_MAX_ATTEMPTS) {
208
+ await retryProgramFetch(attempt, error);
209
+ continue;
210
+ }
211
+ return { success: false, error };
191
212
  }
192
- const response = await fetch(url.toString(), {
193
- headers: {
194
- 'Authorization': `Bearer ${config.apiKey}`,
195
- 'Content-Type': 'application/json',
196
- [CLI_VERSION_HEADER]: APP_VERSION,
197
- },
198
- });
199
213
  if (!response.ok) {
200
- return { success: false, error: await formatServerError(response, url.toString()) };
214
+ const error = await formatServerError(response, requestedUrl);
215
+ if (shouldRetryProgramFetch(response.status, error) && attempt < FETCH_PROGRAM_MAX_ATTEMPTS) {
216
+ await retryProgramFetch(attempt, error);
217
+ continue;
218
+ }
219
+ return { success: false, error };
220
+ }
221
+ const contentType = response.headers?.get?.('content-type') ?? '';
222
+ let data;
223
+ try {
224
+ if (contentType.includes('text/html')) {
225
+ const htmlPreview = (await response.text()).replace(/\s+/g, ' ').trim().slice(0, 120);
226
+ throw new Error(`unexpected HTML response from ${requestedUrl}${htmlPreview ? `: ${htmlPreview}` : ''}`);
227
+ }
228
+ data = await response.json();
229
+ }
230
+ catch (err) {
231
+ const error = `failed to fetch program: ${err instanceof Error ? err.message : String(err)}`;
232
+ if (attempt < FETCH_PROGRAM_MAX_ATTEMPTS) {
233
+ await retryProgramFetch(attempt, error);
234
+ continue;
235
+ }
236
+ return { success: false, error };
201
237
  }
202
- const data = await response.json();
203
238
  const envelope = verifySignedExecutionProgramEnvelope({
204
239
  apiKey: config.apiKey,
205
240
  envelope: data,
@@ -210,9 +245,29 @@ async function fetchProgram(config, presetId, environmentName) {
210
245
  }
211
246
  return { success: true, program: envelope.program, security: envelope.security };
212
247
  }
213
- catch (err) {
214
- return { success: false, error: `failed to fetch program: ${err instanceof Error ? err.message : String(err)}` };
248
+ return { success: false, error: 'failed to fetch program: retry attempts exhausted' };
249
+ }
250
+ function shouldRetryProgramFetch(status, error) {
251
+ if ([408, 429, 500, 502, 503, 504].includes(status)) {
252
+ return true;
215
253
  }
254
+ return status >= 500 && error.includes('unexpected HTML response');
255
+ }
256
+ async function retryProgramFetch(attempt, error) {
257
+ const delayMs = getProgramFetchRetryDelayMs(attempt);
258
+ logger.warn(`[capture] Program fetch failed (attempt ${attempt}/${FETCH_PROGRAM_MAX_ATTEMPTS}): ${error}. Retrying in ${delayMs}ms...`);
259
+ if (delayMs > 0) {
260
+ await sleep(delayMs);
261
+ }
262
+ }
263
+ function getProgramFetchRetryDelayMs(attempt) {
264
+ if (process.env.NODE_ENV === 'test' || process.env.VITEST === 'true') {
265
+ return 0;
266
+ }
267
+ return FETCH_PROGRAM_RETRY_DELAYS_MS[Math.max(0, attempt - 1)] ?? FETCH_PROGRAM_RETRY_DELAYS_MS.at(-1) ?? 0;
268
+ }
269
+ function sleep(ms) {
270
+ return new Promise((resolve) => setTimeout(resolve, ms));
216
271
  }
217
272
  async function uploadResults(config, program, result) {
218
273
  const runId = randomUUID();
package/dist/cli.js CHANGED
File without changes
@@ -35,6 +35,10 @@ export interface ClipCaptureLoopOptions {
35
35
  export interface ClipCaptureLoopResult {
36
36
  framesDir: string;
37
37
  frameCount: number;
38
+ /** Configured capture attempt ceiling. Actual FPS may be lower when CDP is slow. */
39
+ targetFps: number;
40
+ /** Minimum idle time yielded after each CDP screenshot. */
41
+ minRestMs: number;
38
42
  /** (frameCount - 1) * 1000 / (lastTs - firstTs); 0 if < 2 frames. */
39
43
  measuredFps: number;
40
44
  actualDurationMs: number;
@@ -48,11 +52,18 @@ export interface ClipCaptureLoopResult {
48
52
  * loop keeps trying, producing bursts and gaps).
49
53
  */
50
54
  frameOffsetsMs: number[];
55
+ /** CDP Page.captureScreenshot wall time in milliseconds. */
56
+ captureTimingMs: {
57
+ p50: number;
58
+ p95: number;
59
+ max: number;
60
+ };
51
61
  }
52
62
  export declare class ClipCaptureLoop {
53
63
  private readonly page;
54
64
  private readonly framesDir;
55
65
  private readonly jpegQuality;
66
+ private readonly targetFps;
56
67
  private readonly targetFrameIntervalMs;
57
68
  private readonly minRestMs;
58
69
  private cdp;
@@ -60,6 +71,7 @@ export declare class ClipCaptureLoop {
60
71
  private loopPromise;
61
72
  private frames;
62
73
  private frameTimestamps;
74
+ private frameCaptureDurationsMs;
63
75
  private startedAt;
64
76
  private firstFrameAt;
65
77
  private lastFrameAt;
@@ -18,6 +18,7 @@ export class ClipCaptureLoop {
18
18
  page;
19
19
  framesDir;
20
20
  jpegQuality;
21
+ targetFps;
21
22
  targetFrameIntervalMs;
22
23
  minRestMs;
23
24
  cdp = null;
@@ -25,6 +26,7 @@ export class ClipCaptureLoop {
25
26
  loopPromise = null;
26
27
  frames = [];
27
28
  frameTimestamps = [];
29
+ frameCaptureDurationsMs = [];
28
30
  startedAt = 0;
29
31
  firstFrameAt = 0;
30
32
  lastFrameAt = 0;
@@ -33,6 +35,7 @@ export class ClipCaptureLoop {
33
35
  this.framesDir = opts.framesDir;
34
36
  this.jpegQuality = opts.jpegQuality ?? 80;
35
37
  const targetFps = Math.max(1, Math.min(30, opts.targetFps ?? (process.platform === 'linux' ? 8 : 15)));
38
+ this.targetFps = targetFps;
36
39
  this.targetFrameIntervalMs = 1000 / targetFps;
37
40
  this.minRestMs = Math.max(0, Math.min(250, opts.minRestMs ?? (process.platform === 'linux' ? 50 : 16)));
38
41
  }
@@ -69,16 +72,21 @@ export class ClipCaptureLoop {
69
72
  const trimStartMs = this.firstFrameAt > 0 ? Math.max(0, this.firstFrameAt - this.startedAt) : 0;
70
73
  // Snapshot offsets (ms from first frame) for VFR encoding downstream.
71
74
  const frameOffsetsMs = this.frameTimestamps.map(ts => ts - this.firstFrameAt);
75
+ const captureTimingMs = summarizeTiming(this.frameCaptureDurationsMs);
72
76
  // Release memory — the caller owns framesDir from here on.
73
77
  this.frames = [];
74
78
  this.frameTimestamps = [];
79
+ this.frameCaptureDurationsMs = [];
75
80
  return {
76
81
  framesDir: this.framesDir,
77
82
  frameCount,
83
+ targetFps: this.targetFps,
84
+ minRestMs: this.minRestMs,
78
85
  measuredFps,
79
86
  actualDurationMs,
80
87
  trimStartMs,
81
88
  frameOffsetsMs,
89
+ captureTimingMs,
82
90
  };
83
91
  }
84
92
  async loop() {
@@ -112,6 +120,7 @@ export class ClipCaptureLoop {
112
120
  this.frames.push(data);
113
121
  this.frameTimestamps.push(ts);
114
122
  const elapsed = performance.now() - frameStartedAt;
123
+ this.frameCaptureDurationsMs.push(elapsed);
115
124
  const restMs = Math.max(this.minRestMs, this.targetFrameIntervalMs - elapsed);
116
125
  if (restMs > 0 && this.running) {
117
126
  await new Promise(resolve => setTimeout(resolve, restMs));
@@ -119,4 +128,21 @@ export class ClipCaptureLoop {
119
128
  }
120
129
  }
121
130
  }
131
+ function summarizeTiming(values) {
132
+ if (values.length === 0)
133
+ return { p50: 0, p95: 0, max: 0 };
134
+ const sorted = [...values].sort((a, b) => a - b);
135
+ return {
136
+ p50: roundMs(percentile(sorted, 0.5)),
137
+ p95: roundMs(percentile(sorted, 0.95)),
138
+ max: roundMs(sorted[sorted.length - 1] ?? 0),
139
+ };
140
+ }
141
+ function percentile(sorted, ratio) {
142
+ const index = Math.min(sorted.length - 1, Math.max(0, Math.ceil(sorted.length * ratio) - 1));
143
+ return sorted[index] ?? 0;
144
+ }
145
+ function roundMs(value) {
146
+ return Math.round(value * 10) / 10;
147
+ }
122
148
  //# sourceMappingURL=clip-capture-loop.js.map
@@ -311,7 +311,10 @@ export class WebPlaywrightLocal {
311
311
  }
312
312
  const result = await this.recording.loop.stop();
313
313
  logger.info(`[capture] Clip frame capture: ${result.frameCount} frame(s), ` +
314
- `${result.measuredFps.toFixed(1)} fps over ${(result.actualDurationMs / 1000).toFixed(2)}s`);
314
+ `${result.measuredFps.toFixed(1)} fps over ${(result.actualDurationMs / 1000).toFixed(2)}s ` +
315
+ `(target ${result.targetFps} fps, rest >= ${result.minRestMs}ms; ` +
316
+ `CDP frame time p50=${result.captureTimingMs.p50}ms ` +
317
+ `p95=${result.captureTimingMs.p95}ms max=${result.captureTimingMs.max}ms)`);
315
318
  await this.browser.closeContext();
316
319
  await assembleMp4FromFrames({
317
320
  framesDir: this.recording.framesDir,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "autokap",
3
- "version": "1.1.4",
3
+ "version": "1.1.5",
4
4
  "description": "AI-powered CLI tool for capturing clean screenshots of websites",
5
5
  "type": "module",
6
6
  "main": "./dist/index.js",
@@ -1,45 +0,0 @@
1
- /**
2
- * Action recovery, guard inference, and login detection utilities for the capture agent.
3
- * Extracted from agent.ts — these are pure functions with no closure dependencies.
4
- */
5
- import type { ExecutedAction, ActionType } from './types.js';
6
- export declare const META_ACTIONS: Set<ActionType>;
7
- export declare const BOOTSTRAP_ACTIONS: Set<ActionType>;
8
- export declare const REPLAYABLE_ACTIONS: ReadonlyArray<ActionType>;
9
- export declare function isNoEffectAction(action: ExecutedAction): boolean;
10
- export declare function isBootstrapStabilizationAction(action: ExecutedAction): boolean;
11
- export declare function hasMeaningfulBrowserAction(actionHistory: ExecutedAction[]): boolean;
12
- export declare function buildRecoveryActionSignature(action: ExecutedAction): string;
13
- export declare function isExplicitLoginAction(action: ExecutedAction): boolean;
14
- export declare function hasRecentExplicitLoginAction(previousActions: ExecutedAction[]): boolean;
15
- export declare function isLoginAction(action: ExecutedAction, previousActions?: ExecutedAction[]): boolean;
16
- export declare function compactReplayActions(recordedActions: ExecutedAction[], params?: {
17
- currentUrl?: string;
18
- targetUrl?: string;
19
- currentViewport?: {
20
- width: number;
21
- height: number;
22
- } | null;
23
- isAuthenticated?: boolean;
24
- ignoreRecordedViewport?: boolean;
25
- }): ExecutedAction[];
26
- export declare function countRecentNoEffectActions(actionHistory: ExecutedAction[]): number;
27
- export declare function shouldTriggerRecovery(actionHistory: ExecutedAction[]): boolean;
28
- export declare function inferPrematureGiveUpCorrection(params: {
29
- reason: string;
30
- actionHistory: ExecutedAction[];
31
- lastVerificationFailure?: string;
32
- iteration: number;
33
- maxIterations: number;
34
- }): string | null;
35
- export declare function inferSearchScrollLoopGuard(params: {
36
- actionHistory: ExecutedAction[];
37
- action: ActionType;
38
- args: Record<string, unknown>;
39
- }): string | null;
40
- export declare function inferRepeatedActionGuard(params: {
41
- actionHistory: ExecutedAction[];
42
- action: ActionType;
43
- args: Record<string, unknown>;
44
- currentUrl?: string;
45
- }): string | null;
@@ -1,370 +0,0 @@
1
- /**
2
- * Action recovery, guard inference, and login detection utilities for the capture agent.
3
- * Extracted from agent.ts — these are pure functions with no closure dependencies.
4
- */
5
- import { urlsRoughlyMatch, buildClickGuardSignature, } from './agent-url-utils.js';
6
- // -- Constants --
7
- export const META_ACTIONS = new Set(['note', 'begin_subgoal', 'focus', 'capture', 'analyze_screenshot']);
8
- export const BOOTSTRAP_ACTIONS = new Set(['wait']);
9
- const REPEAT_GUARD_ACTIONS = new Set([
10
- 'tap',
11
- 'press_key',
12
- ]);
13
- export const REPLAYABLE_ACTIONS = [
14
- 'navigate_to', 'tap', 'type', 'scroll',
15
- 'press_key', 'wait',
16
- ];
17
- const LOGIN_URL_RE = /\b(login|log-in|signin|sign-in|auth|session|connexion|connect)\b/i;
18
- const LOGIN_FIELD_RE = /\b(password|mot de passe|passcode|otp|verification.?code)\b/i;
19
- const LOGIN_CLICK_RE = /\b(login|log-in|signin|sign-in|auth|session|connexion|connect|password|mot de passe|passcode|otp|verification.?code|email|e-mail)\b/i;
20
- const AUTH_SUBMIT_RE = /\b(continue|next|submit|verify|unlock|access|enter|continuer|suivant|soumettre|verifier|v[eé]rifier|acc[eé]der|entrer)\b/i;
21
- const HARD_GIVE_UP_RE = /\b(5xx|500\b|404\b|page not found|not found|http error|server error|js crash|javascript crash|browser crashed|connection refused|dns|net::|ssl|certificate|blank page|white page|no content|infinite spinner)\b/i;
22
- const RECOVERABLE_GIVE_UP_RE = /\b(verification|ready_to_capture|dialog|modal|overlay|gallery|editor|route|navigation|assistant|conversation|wrong page|duplicate capture|capture target)\b/i;
23
- // -- Internal helpers --
24
- export function isNoEffectAction(action) {
25
- return (action.success === false
26
- || action.stateChanged === false
27
- || action.outcome === 'No visible state change detected after the action.');
28
- }
29
- export function isBootstrapStabilizationAction(action) {
30
- if (BOOTSTRAP_ACTIONS.has(action.action))
31
- return true;
32
- return action.action === 'press_key' && String(action.params.key || '').toLowerCase() === 'escape';
33
- }
34
- export function hasMeaningfulBrowserAction(actionHistory) {
35
- return actionHistory.some((action) => !META_ACTIONS.has(action.action) && !isBootstrapStabilizationAction(action));
36
- }
37
- export function buildRecoveryActionSignature(action) {
38
- const parts = [action.action];
39
- if (typeof action.params.nodeId === 'string' && action.params.nodeId.trim()) {
40
- parts.push(`nodeId:${action.params.nodeId.trim()}`);
41
- }
42
- else if (typeof action.params.selector === 'string' && action.params.selector.trim()) {
43
- parts.push(`selector:${action.params.selector.trim()}`);
44
- }
45
- else if (action.params.index !== undefined) {
46
- parts.push(`index:${String(action.params.index)}`);
47
- }
48
- else if (typeof action.params.url === 'string' && action.params.url.trim()) {
49
- parts.push(`url:${action.params.url.trim().slice(0, 160)}`);
50
- }
51
- else if (action.params.x !== undefined && action.params.y !== undefined) {
52
- parts.push(`xy:${String(action.params.x)},${String(action.params.y)}`);
53
- }
54
- if (typeof action.params.text === 'string' && action.params.text.trim()) {
55
- parts.push(`text:${action.params.text.trim().slice(0, 80)}`);
56
- }
57
- if (typeof action.params.key === 'string' && action.params.key.trim()) {
58
- parts.push(`key:${action.params.key.trim()}`);
59
- }
60
- return parts.join('|');
61
- }
62
- function getMeaningfulBrowserActions(actionHistory) {
63
- return actionHistory.filter(action => !META_ACTIONS.has(action.action) && !isBootstrapStabilizationAction(action));
64
- }
65
- function countDistinctActionSignatures(actionHistory) {
66
- return new Set(getMeaningfulBrowserActions(actionHistory).map(action => buildRecoveryActionSignature(action))).size;
67
- }
68
- // -- Login detection --
69
- export function isExplicitLoginAction(action) {
70
- if (action.action === 'navigate_to' && typeof action.params.url === 'string' && LOGIN_URL_RE.test(action.params.url)) {
71
- return true;
72
- }
73
- if ((action.action === 'type_text' || action.action === 'type') && typeof action.params.selector === 'string' && LOGIN_FIELD_RE.test(action.params.selector)) {
74
- return true;
75
- }
76
- if ((action.action === 'type_text' || action.action === 'type') && typeof action.params.text === 'string' && /\{\{credential\./i.test(action.params.text)) {
77
- return true;
78
- }
79
- if (action.action === 'click' || action.action === 'tap') {
80
- const haystack = [
81
- action.params.selector,
82
- action.params.elementLabel,
83
- action.params.href,
84
- action.params.nodeId,
85
- ]
86
- .filter((value) => typeof value === 'string')
87
- .join(' ');
88
- if (LOGIN_CLICK_RE.test(haystack)) {
89
- return true;
90
- }
91
- }
92
- return false;
93
- }
94
- export function hasRecentExplicitLoginAction(previousActions) {
95
- return previousActions.slice(-3).some(isExplicitLoginAction);
96
- }
97
- export function isLoginAction(action, previousActions = []) {
98
- if (isExplicitLoginAction(action)) {
99
- return true;
100
- }
101
- if (action.action === 'click' || action.action === 'tap') {
102
- const haystack = [
103
- action.params.selector,
104
- action.params.elementLabel,
105
- action.params.href,
106
- action.params.nodeId,
107
- ]
108
- .filter((value) => typeof value === 'string')
109
- .join(' ');
110
- if (AUTH_SUBMIT_RE.test(haystack) && hasRecentExplicitLoginAction(previousActions)) {
111
- return true;
112
- }
113
- }
114
- if (action.action === 'press_key'
115
- && action.params.key === 'Enter'
116
- && hasRecentExplicitLoginAction(previousActions)) {
117
- return true;
118
- }
119
- return false;
120
- }
121
- export function compactReplayActions(recordedActions, params = {}) {
122
- let replayable = recordedActions.filter(a => REPLAYABLE_ACTIONS.includes(a.action));
123
- if (params.ignoreRecordedViewport) {
124
- replayable = replayable.filter((action) => action.action !== 'resize_viewport');
125
- }
126
- if (params.isAuthenticated) {
127
- const authAwareReplayable = replayable;
128
- replayable = authAwareReplayable.filter((action, index) => !isLoginAction(action, authAwareReplayable.slice(0, index)));
129
- }
130
- let startIndex = 0;
131
- while (startIndex < replayable.length) {
132
- const action = replayable[startIndex];
133
- if (action.action === 'wait') {
134
- startIndex += 1;
135
- continue;
136
- }
137
- if (action.action === 'dismiss_overlays') {
138
- startIndex += 1;
139
- continue;
140
- }
141
- if (action.action === 'resize_viewport'
142
- && params.currentViewport
143
- && Number(action.params.width) === params.currentViewport.width
144
- && Number(action.params.height) === params.currentViewport.height) {
145
- startIndex += 1;
146
- continue;
147
- }
148
- if (action.action === 'navigate_to'
149
- && typeof action.params.url === 'string'
150
- && (urlsRoughlyMatch(action.params.url, params.currentUrl)
151
- || urlsRoughlyMatch(action.params.url, params.targetUrl))) {
152
- startIndex += 1;
153
- continue;
154
- }
155
- break;
156
- }
157
- return replayable.slice(startIndex);
158
- }
159
- // -- Recovery --
160
- export function countRecentNoEffectActions(actionHistory) {
161
- let count = 0;
162
- for (let index = actionHistory.length - 1; index >= 0; index -= 1) {
163
- const action = actionHistory[index];
164
- if (META_ACTIONS.has(action.action))
165
- continue;
166
- if (isNoEffectAction(action)) {
167
- count += 1;
168
- continue;
169
- }
170
- break;
171
- }
172
- return count;
173
- }
174
- export function shouldTriggerRecovery(actionHistory) {
175
- const browserActions = actionHistory.filter(a => !META_ACTIONS.has(a.action));
176
- if (browserActions.length < 2)
177
- return false;
178
- if (!hasMeaningfulBrowserAction(browserActions))
179
- return false;
180
- const last = browserActions[browserActions.length - 1];
181
- const previous = browserActions[browserActions.length - 2];
182
- const sameFailureSignature = !last.success
183
- && !previous.success
184
- && buildRecoveryActionSignature(last) === buildRecoveryActionSignature(previous)
185
- && String(last.error || '').slice(0, 120) === String(previous.error || '').slice(0, 120);
186
- if (browserActions.length >= 4) {
187
- const [a, b, c, d] = browserActions.slice(-4);
188
- const sigA = buildRecoveryActionSignature(a);
189
- const sigB = buildRecoveryActionSignature(b);
190
- const sigC = buildRecoveryActionSignature(c);
191
- const sigD = buildRecoveryActionSignature(d);
192
- if ([a, b, c, d].every(isNoEffectAction)
193
- && sigA === sigC
194
- && sigB === sigD
195
- && sigA !== sigB) {
196
- return true;
197
- }
198
- }
199
- return sameFailureSignature || countRecentNoEffectActions(actionHistory) >= 2;
200
- }
201
- // -- Guard inference --
202
- export function inferPrematureGiveUpCorrection(params) {
203
- const reason = params.reason.trim();
204
- const lastVerificationFailure = params.lastVerificationFailure?.trim();
205
- if (HARD_GIVE_UP_RE.test(reason) || (lastVerificationFailure && HARD_GIVE_UP_RE.test(lastVerificationFailure))) {
206
- return null;
207
- }
208
- const recentCaptureFailures = params.actionHistory
209
- .filter(a => a.action === 'ready_to_capture' && !a.success && a.error)
210
- .slice(-6);
211
- if (recentCaptureFailures.length >= 3) {
212
- const distinctReasons = new Set(recentCaptureFailures.map(a => (a.error || '').replace(/^Verification failed:\s*/i, '').trim().slice(0, 120)));
213
- if (distinctReasons.size >= 2 && distinctReasons.size <= 3) {
214
- return null;
215
- }
216
- }
217
- if (countRecentNoEffectActions(params.actionHistory) >= 8) {
218
- return null;
219
- }
220
- if (lastVerificationFailure && RECOVERABLE_GIVE_UP_RE.test(lastVerificationFailure)) {
221
- return `Do not give up yet. The last verification failure is still recoverable: ${lastVerificationFailure}. Try a materially different navigation or repair step first.`;
222
- }
223
- const hasTriedSearch = params.actionHistory.some(a => a.action === 'search_text');
224
- const hasTriedNavigate = params.actionHistory.some(a => a.action === 'navigate_to');
225
- if (!hasTriedSearch && !hasTriedNavigate) {
226
- return 'Do not give up yet. You have not tried search_text or navigate_to. Use search_text to find the element by text, or navigate_to to go directly to the target URL.';
227
- }
228
- const meaningfulActions = getMeaningfulBrowserActions(params.actionHistory);
229
- const distinctActionCount = countDistinctActionSignatures(params.actionHistory);
230
- const hasTriedEnoughStrategies = meaningfulActions.length >= 4 && distinctActionCount >= 3;
231
- const nearingBudget = params.iteration >= Math.max(6, params.maxIterations - 2);
232
- if (!hasTriedEnoughStrategies && !nearingBudget) {
233
- return 'Do not give up yet. You have not tried enough materially different actions. Change strategy before giving up.';
234
- }
235
- if (/click.*no effect|no.*effect.*click|clicking.*no/i.test(reason) && !hasTriedNavigate) {
236
- return 'Clicking has no effect, but you have not tried navigate_to. Use navigate_to to go directly to the target URL instead of clicking.';
237
- }
238
- if (RECOVERABLE_GIVE_UP_RE.test(reason) && !nearingBudget) {
239
- return 'Do not give up yet. The current issue still looks recoverable. Try a different navigation, search, or repair approach first.';
240
- }
241
- return null;
242
- }
243
- function normalizeSearchLoopQuery(value) {
244
- return typeof value === 'string'
245
- ? value.trim().toLowerCase().replace(/\s+/g, ' ')
246
- : '';
247
- }
248
- function getTrailingSearchScrollActions(actionHistory) {
249
- const trailing = [];
250
- for (let index = actionHistory.length - 1; index >= 0; index -= 1) {
251
- const action = actionHistory[index];
252
- if (META_ACTIONS.has(action.action))
253
- continue;
254
- if (action.action !== 'search_text' && action.action !== 'scroll')
255
- break;
256
- trailing.unshift(action);
257
- }
258
- return trailing;
259
- }
260
- export function inferSearchScrollLoopGuard(params) {
261
- if (params.action !== 'search_text' && params.action !== 'scroll')
262
- return null;
263
- const candidate = {
264
- iteration: 0,
265
- action: params.action,
266
- params: params.args,
267
- success: params.action === 'scroll',
268
- stateChanged: params.action === 'scroll',
269
- };
270
- const trailingLoopActions = [...getTrailingSearchScrollActions(params.actionHistory), candidate];
271
- if (trailingLoopActions.length < 4)
272
- return null;
273
- if (!trailingLoopActions.every((action) => action.action === 'search_text' || action.action === 'scroll')) {
274
- return null;
275
- }
276
- const repeatedQueries = new Map();
277
- for (const action of trailingLoopActions) {
278
- if (action.action !== 'search_text')
279
- continue;
280
- const query = normalizeSearchLoopQuery(action.params.query);
281
- if (!query)
282
- continue;
283
- repeatedQueries.set(query, (repeatedQueries.get(query) ?? 0) + 1);
284
- }
285
- const repeatedQueryEntry = Array.from(repeatedQueries.entries())
286
- .sort((a, b) => b[1] - a[1])[0];
287
- if (!repeatedQueryEntry || repeatedQueryEntry[1] < 2)
288
- return null;
289
- const [repeatedQuery, repeatedCount] = repeatedQueryEntry;
290
- if (params.action === 'search_text') {
291
- const candidateQuery = normalizeSearchLoopQuery(params.args.query);
292
- if (candidateQuery !== repeatedQuery || repeatedCount < 3)
293
- return null;
294
- return `BLOCKED: You have already searched for "${repeatedQuery}" repeatedly and only scrolled around without interacting. Stop looping on the same query. Try a different search term, navigate_to the target page, or interact with a concrete result instead.`;
295
- }
296
- return `BLOCKED: You are stuck in a search_text/scroll loop around "${repeatedQuery}" without acting on a result. Stop scrolling blindly. Try a different query, navigate_to the correct page, or interact with a specific matching control.`;
297
- }
298
- export function inferRepeatedActionGuard(params) {
299
- if (params.action === 'navigate_to' && typeof params.args.url === 'string' && params.currentUrl) {
300
- const targetUrl = params.args.url.replace(/\/$/, '');
301
- const recentNavigations = params.actionHistory
302
- .filter(a => !META_ACTIONS.has(a.action) && a.success !== false)
303
- .slice(-5);
304
- if (urlsRoughlyMatch(params.currentUrl, targetUrl)) {
305
- return 'WARNING: You are already on this URL. No navigation needed. Focus on interacting with the current page instead.';
306
- }
307
- const recentAllActions = params.actionHistory
308
- .filter(a => !META_ACTIONS.has(a.action))
309
- .slice(-4);
310
- const hasRecentNoEffect = recentAllActions.some(a => isNoEffectAction(a));
311
- if (!hasRecentNoEffect) {
312
- const recentUrls = recentNavigations
313
- .filter(a => typeof a.params.href === 'string' || typeof a.params.url === 'string')
314
- .slice(-3)
315
- .map(a => String(a.params.href || a.params.url || '').replace(/\/$/, ''));
316
- const wasRecentlyOnTarget = recentUrls.some(url => urlsRoughlyMatch(url, targetUrl));
317
- if (wasRecentlyOnTarget && !urlsRoughlyMatch(params.currentUrl, targetUrl)) {
318
- return `WARNING: You just navigated away from ${targetUrl} — going back suggests you are confused about the goal. Re-read the <task>/<goal> and <variant_manifest> carefully. If you need a modal, open it from the current page instead of navigating back.`;
319
- }
320
- }
321
- }
322
- if (!REPEAT_GUARD_ACTIONS.has(params.action))
323
- return null;
324
- const recentBrowserActions = params.actionHistory
325
- .filter(action => !META_ACTIONS.has(action.action))
326
- .slice(-2);
327
- if (recentBrowserActions.length < 2)
328
- return null;
329
- const candidateSignature = buildRecoveryActionSignature({
330
- iteration: 0,
331
- action: params.action,
332
- params: params.args,
333
- success: false,
334
- });
335
- const candidateClickSignature = params.action === 'click'
336
- ? buildClickGuardSignature(params.args, params.currentUrl)
337
- : null;
338
- const repeatedNoEffect = recentBrowserActions.every(action => isNoEffectAction(action)
339
- && (candidateClickSignature && action.action === 'click'
340
- ? buildClickGuardSignature(action.params, params.currentUrl) === candidateClickSignature
341
- : buildRecoveryActionSignature(action) === candidateSignature));
342
- if (repeatedNoEffect) {
343
- return 'BLOCKED: The previous attempts on this same target had no effect. Try a different control, search_text, scrolling, or a repair step instead of repeating it.';
344
- }
345
- const recentInteractions = params.actionHistory
346
- .filter(action => REPEAT_GUARD_ACTIONS.has(action.action)
347
- && !META_ACTIONS.has(action.action)
348
- && !(typeof action.error === 'string' && action.error.startsWith('BLOCKED:')))
349
- .slice(-4);
350
- if (recentInteractions.length >= 4 && recentInteractions.every(a => isNoEffectAction(a))) {
351
- return 'BLOCKED: The last 4 interaction actions all failed or had no visible effect. You are stuck. Step back and reconsider: verify the current URL, check if you are on the right page, try navigate_to to reach the correct page, or call give_up if this capture is impossible.';
352
- }
353
- if (params.action === 'click') {
354
- const sig = buildClickGuardSignature(params.args, params.currentUrl);
355
- if (sig) {
356
- const candidateTarget = String((typeof params.args.elementLabel === 'string' && params.args.elementLabel.trim())
357
- ? params.args.elementLabel
358
- : params.args.selector ?? params.args.index ?? 'this target');
359
- const recentClicks = params.actionHistory
360
- .filter(a => a.action === 'click' && !META_ACTIONS.has(a.action))
361
- .slice(-6);
362
- const sameTargetCount = recentClicks.filter(a => buildClickGuardSignature(a.params, params.currentUrl) === sig).length;
363
- if (sameTargetCount >= 3) {
364
- return `BLOCKED: You have clicked "${candidateTarget}" ${sameTargetCount} times recently without progress. This is likely a toggle/multi-select control. Press Escape to close any open dropdown, then look for a DIFFERENT button (e.g., an edit/settings icon) to achieve your goal.`;
365
- }
366
- }
367
- }
368
- return null;
369
- }
370
- //# sourceMappingURL=agent-action-recovery.js.map