autokap 1.0.7 → 1.0.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (301) hide show
  1. package/assets/cursors/macos.svg +4 -0
  2. package/assets/cursors/windows.svg +15 -0
  3. package/assets/skill/OPCODE-REFERENCE.md +635 -0
  4. package/assets/skill/README.md +39 -0
  5. package/assets/skill/SKILL.md +454 -468
  6. package/assets/skill/STUDIO-SKILL.md +476 -0
  7. package/assets/skill/references/examples.md +104 -0
  8. package/assets/skill/references/interactive-demo.md +225 -0
  9. package/assets/skill/references/mock-data.md +178 -0
  10. package/dist/action-verifier.d.ts +29 -0
  11. package/dist/action-verifier.js +133 -0
  12. package/dist/agent-action-recovery.d.ts +45 -0
  13. package/dist/agent-action-recovery.js +370 -0
  14. package/dist/agent-message-utils.d.ts +21 -0
  15. package/dist/agent-message-utils.js +77 -0
  16. package/dist/agent-url-utils.d.ts +30 -0
  17. package/dist/agent-url-utils.js +138 -0
  18. package/dist/agent.d.ts +92 -8
  19. package/dist/agent.js +2936 -781
  20. package/dist/ak-tree.d.ts +39 -0
  21. package/dist/ak-tree.js +368 -0
  22. package/dist/alt-text.d.ts +26 -0
  23. package/dist/alt-text.js +55 -0
  24. package/dist/auth-capture.d.ts +17 -0
  25. package/dist/auth-capture.js +197 -0
  26. package/dist/benchmark.d.ts +59 -0
  27. package/dist/benchmark.js +135 -0
  28. package/dist/billing-operation-logging.d.ts +3 -1
  29. package/dist/billing-operation-logging.js +4 -0
  30. package/dist/browser-bar.d.ts +14 -6
  31. package/dist/browser-bar.js +145 -8
  32. package/dist/browser-pool.d.ts +7 -0
  33. package/dist/browser-pool.js +15 -5
  34. package/dist/browser-utils.d.ts +31 -0
  35. package/dist/browser-utils.js +97 -0
  36. package/dist/browser.d.ts +61 -11
  37. package/dist/browser.js +1513 -59
  38. package/dist/capture-alt-text.js +2 -1
  39. package/dist/capture-encryption.d.ts +3 -1
  40. package/dist/capture-encryption.js +21 -6
  41. package/dist/capture-language-preflight.js +14 -0
  42. package/dist/capture-llm-page-identity.js +22 -10
  43. package/dist/capture-page-identity.d.ts +5 -7
  44. package/dist/capture-page-identity.js +211 -78
  45. package/dist/capture-preset-credentials.d.ts +50 -0
  46. package/dist/capture-preset-credentials.js +127 -0
  47. package/dist/capture-request-plan.d.ts +2 -2
  48. package/dist/capture-request-plan.js +64 -16
  49. package/dist/capture-run-optimizer.js +48 -33
  50. package/dist/capture-selector-memory.d.ts +5 -0
  51. package/dist/capture-selector-memory.js +18 -0
  52. package/dist/capture-strategy.d.ts +36 -0
  53. package/dist/capture-strategy.js +96 -0
  54. package/dist/capture-studio-sync.d.ts +1 -0
  55. package/dist/capture-studio-sync.js +9 -3
  56. package/dist/capture-surface-contract.d.ts +36 -0
  57. package/dist/capture-surface-contract.js +299 -0
  58. package/dist/capture-transition-engine.d.ts +28 -0
  59. package/dist/capture-transition-engine.js +292 -0
  60. package/dist/capture-variant-state.d.ts +2 -0
  61. package/dist/capture-variant-state.js +26 -0
  62. package/dist/capture-verification.d.ts +35 -0
  63. package/dist/capture-verification.js +95 -0
  64. package/dist/capture-viewport-lock.d.ts +48 -0
  65. package/dist/capture-viewport-lock.js +74 -0
  66. package/dist/circuit-breaker.d.ts +42 -0
  67. package/dist/circuit-breaker.js +119 -0
  68. package/dist/cli-config.d.ts +9 -1
  69. package/dist/cli-config.js +112 -7
  70. package/dist/cli-contract.d.ts +19 -0
  71. package/dist/cli-contract.js +173 -0
  72. package/dist/cli-runner-local.d.ts +12 -0
  73. package/dist/cli-runner-local.js +115 -0
  74. package/dist/cli-runner.d.ts +34 -0
  75. package/dist/cli-runner.js +580 -0
  76. package/dist/cli-utils.d.ts +0 -1
  77. package/dist/cli-utils.js +2 -5
  78. package/dist/cli.js +1011 -267
  79. package/dist/clip-begin-frame-recorder.d.ts +44 -0
  80. package/dist/clip-begin-frame-recorder.js +250 -0
  81. package/dist/clip-capture-backend.d.ts +25 -0
  82. package/dist/clip-capture-backend.js +189 -0
  83. package/dist/clip-capture-loop.d.ts +61 -0
  84. package/dist/clip-capture-loop.js +111 -0
  85. package/dist/clip-frame-recorder.d.ts +63 -0
  86. package/dist/clip-frame-recorder.js +305 -0
  87. package/dist/clip-orchestrator.js +9 -2
  88. package/dist/clip-postprocess.d.ts +31 -2
  89. package/dist/clip-postprocess.js +194 -68
  90. package/dist/clip-runtime.d.ts +18 -0
  91. package/dist/clip-runtime.js +67 -0
  92. package/dist/clip-scale.d.ts +10 -0
  93. package/dist/clip-scale.js +21 -0
  94. package/dist/clip-screencast-recorder.d.ts +42 -0
  95. package/dist/clip-screencast-recorder.js +242 -0
  96. package/dist/clip-sidecar.d.ts +54 -0
  97. package/dist/clip-sidecar.js +208 -0
  98. package/dist/cookie-dismiss.d.ts +2 -0
  99. package/dist/cookie-dismiss.js +48 -13
  100. package/dist/cost-logging.d.ts +8 -0
  101. package/dist/cost-logging.js +160 -46
  102. package/dist/cost-resolution-monitor.d.ts +16 -0
  103. package/dist/cost-resolution-monitor.js +34 -0
  104. package/dist/credential-templates.js +2 -2
  105. package/dist/cursor-overlay-script.d.ts +6 -0
  106. package/dist/cursor-overlay-script.js +169 -0
  107. package/dist/dom-css-purger.d.ts +65 -0
  108. package/dist/dom-css-purger.js +333 -0
  109. package/dist/dom-font-inliner.d.ts +45 -0
  110. package/dist/dom-font-inliner.js +148 -0
  111. package/dist/dom-patch-resolver.d.ts +52 -0
  112. package/dist/dom-patch-resolver.js +242 -0
  113. package/dist/dom-serializer.d.ts +82 -0
  114. package/dist/dom-serializer.js +378 -0
  115. package/dist/element-capture.d.ts +1 -41
  116. package/dist/element-capture.js +202 -446
  117. package/dist/env-validation.d.ts +5 -0
  118. package/dist/env-validation.js +63 -0
  119. package/dist/execution-schema.d.ts +4753 -0
  120. package/dist/execution-schema.js +563 -0
  121. package/dist/execution-types.d.ts +936 -0
  122. package/dist/execution-types.js +66 -0
  123. package/dist/fonts-loader.d.ts +14 -0
  124. package/dist/fonts-loader.js +55 -0
  125. package/dist/hybrid-navigator.js +12 -12
  126. package/dist/index.d.ts +11 -6
  127. package/dist/index.js +11 -4
  128. package/dist/legacy/agent-action-recovery.d.ts +45 -0
  129. package/dist/legacy/agent-action-recovery.js +370 -0
  130. package/dist/legacy/agent-message-utils.d.ts +21 -0
  131. package/dist/legacy/agent-message-utils.js +77 -0
  132. package/dist/legacy/agent-url-utils.d.ts +30 -0
  133. package/dist/legacy/agent-url-utils.js +138 -0
  134. package/dist/legacy/agent.d.ts +226 -0
  135. package/dist/legacy/agent.js +6666 -0
  136. package/dist/legacy/clip-orchestrator.d.ts +148 -0
  137. package/dist/legacy/clip-orchestrator.js +957 -0
  138. package/dist/legacy/credential-templates.d.ts +5 -0
  139. package/dist/legacy/credential-templates.js +60 -0
  140. package/dist/legacy/hybrid-navigator.d.ts +138 -0
  141. package/dist/legacy/hybrid-navigator.js +468 -0
  142. package/dist/legacy/llm-usage.d.ts +17 -0
  143. package/dist/legacy/llm-usage.js +45 -0
  144. package/dist/legacy/prompt-cache.d.ts +10 -0
  145. package/dist/legacy/prompt-cache.js +24 -0
  146. package/dist/legacy/prompts.d.ts +175 -0
  147. package/dist/legacy/prompts.js +1038 -0
  148. package/dist/legacy/tools.d.ts +4 -0
  149. package/dist/legacy/tools.js +216 -0
  150. package/dist/legacy/video-agent.d.ts +143 -0
  151. package/dist/legacy/video-agent.js +4788 -0
  152. package/dist/legacy/video-observation.d.ts +36 -0
  153. package/dist/legacy/video-observation.js +192 -0
  154. package/dist/legacy/video-planner.d.ts +12 -0
  155. package/dist/legacy/video-planner.js +501 -0
  156. package/dist/legacy/video-prompts.d.ts +37 -0
  157. package/dist/legacy/video-prompts.js +569 -0
  158. package/dist/legacy/video-tools.d.ts +3 -0
  159. package/dist/legacy/video-tools.js +59 -0
  160. package/dist/legacy/video-variant-state.d.ts +29 -0
  161. package/dist/legacy/video-variant-state.js +80 -0
  162. package/dist/legacy/vision-model.d.ts +17 -0
  163. package/dist/legacy/vision-model.js +74 -0
  164. package/dist/llm-healer.d.ts +55 -0
  165. package/dist/llm-healer.js +213 -0
  166. package/dist/llm-provider.d.ts +29 -0
  167. package/dist/llm-provider.js +83 -0
  168. package/dist/logger.d.ts +6 -2
  169. package/dist/logger.js +15 -1
  170. package/dist/mockup-html.js +35 -25
  171. package/dist/mockup.d.ts +95 -2
  172. package/dist/mockup.js +427 -166
  173. package/dist/mouse-animation.d.ts +2 -2
  174. package/dist/mouse-animation.js +34 -20
  175. package/dist/opcode-actions.d.ts +42 -0
  176. package/dist/opcode-actions.js +524 -0
  177. package/dist/opcode-runner.d.ts +51 -0
  178. package/dist/opcode-runner.js +779 -0
  179. package/dist/openrouter-client.d.ts +40 -0
  180. package/dist/openrouter-client.js +16 -0
  181. package/dist/overlay-engine.d.ts +24 -0
  182. package/dist/overlay-engine.js +176 -0
  183. package/dist/postcondition.d.ts +16 -0
  184. package/dist/postcondition.js +269 -0
  185. package/dist/program-patcher.d.ts +25 -0
  186. package/dist/program-patcher.js +44 -0
  187. package/dist/program-signing.d.ts +1094 -0
  188. package/dist/program-signing.js +140 -0
  189. package/dist/prompts.d.ts +13 -5
  190. package/dist/prompts.js +224 -351
  191. package/dist/provider-config.d.ts +17 -0
  192. package/dist/provider-config.js +42 -0
  193. package/dist/recovery-chain.d.ts +37 -0
  194. package/dist/recovery-chain.js +374 -0
  195. package/dist/remote-browser.d.ts +28 -4
  196. package/dist/remote-browser.js +60 -5
  197. package/dist/safari-browser-bar.d.ts +15 -0
  198. package/dist/safari-browser-bar.js +95 -0
  199. package/dist/safari-toolbar-asset.d.ts +15 -0
  200. package/dist/safari-toolbar-asset.js +12 -0
  201. package/dist/security.d.ts +2 -1
  202. package/dist/security.js +49 -10
  203. package/dist/selector-resolver.d.ts +34 -0
  204. package/dist/selector-resolver.js +181 -0
  205. package/dist/semantic-resolver.d.ts +35 -0
  206. package/dist/semantic-resolver.js +161 -0
  207. package/dist/server-capture-runtime.d.ts +5 -3
  208. package/dist/server-capture-runtime.js +42 -95
  209. package/dist/server-credit-usage.d.ts +2 -2
  210. package/dist/server-project-webhooks.d.ts +15 -1
  211. package/dist/server-project-webhooks.js +34 -8
  212. package/dist/server-screenshot-watermark.js +27 -5
  213. package/dist/session-profile.js +164 -1
  214. package/dist/sf-pro-symbols.d.ts +1 -0
  215. package/dist/sf-pro-symbols.js +55 -0
  216. package/dist/skill-packaging.d.ts +28 -0
  217. package/dist/skill-packaging.js +169 -0
  218. package/dist/smart-wait.d.ts +27 -0
  219. package/dist/smart-wait.js +81 -0
  220. package/dist/status-bar-render.d.ts +20 -0
  221. package/dist/status-bar-render.js +410 -0
  222. package/dist/status-bar.d.ts +9 -0
  223. package/dist/status-bar.js +298 -14
  224. package/dist/svg-browser-bar.d.ts +33 -0
  225. package/dist/svg-browser-bar.js +206 -0
  226. package/dist/svg-status-bar.d.ts +36 -0
  227. package/dist/svg-status-bar.js +597 -0
  228. package/dist/svg-text.d.ts +61 -0
  229. package/dist/svg-text.js +118 -0
  230. package/dist/tools.js +89 -451
  231. package/dist/types.d.ts +248 -7
  232. package/dist/types.js +23 -1
  233. package/dist/v2/action-verifier.d.ts +29 -0
  234. package/dist/v2/action-verifier.js +133 -0
  235. package/dist/v2/alt-text.d.ts +26 -0
  236. package/dist/v2/alt-text.js +55 -0
  237. package/dist/v2/benchmark.d.ts +59 -0
  238. package/dist/v2/benchmark.js +135 -0
  239. package/dist/v2/capture-strategy.d.ts +30 -0
  240. package/dist/v2/capture-strategy.js +67 -0
  241. package/dist/v2/capture-verification.d.ts +35 -0
  242. package/dist/v2/capture-verification.js +95 -0
  243. package/dist/v2/circuit-breaker.d.ts +42 -0
  244. package/dist/v2/circuit-breaker.js +119 -0
  245. package/dist/v2/cli-runner-local.d.ts +11 -0
  246. package/dist/v2/cli-runner-local.js +91 -0
  247. package/dist/v2/cli-runner.d.ts +34 -0
  248. package/dist/v2/cli-runner.js +300 -0
  249. package/dist/v2/compiler-prompts.d.ts +27 -0
  250. package/dist/v2/compiler-prompts.js +123 -0
  251. package/dist/v2/compiler.d.ts +37 -0
  252. package/dist/v2/compiler.js +147 -0
  253. package/dist/v2/explorer.d.ts +41 -0
  254. package/dist/v2/explorer.js +56 -0
  255. package/dist/v2/index.d.ts +37 -0
  256. package/dist/v2/index.js +31 -0
  257. package/dist/v2/llm-healer.d.ts +62 -0
  258. package/dist/v2/llm-healer.js +166 -0
  259. package/dist/v2/llm-provider.d.ts +29 -0
  260. package/dist/v2/llm-provider.js +80 -0
  261. package/dist/v2/opcode-runner.d.ts +47 -0
  262. package/dist/v2/opcode-runner.js +634 -0
  263. package/dist/v2/overlay-engine.d.ts +24 -0
  264. package/dist/v2/overlay-engine.js +150 -0
  265. package/dist/v2/postcondition.d.ts +16 -0
  266. package/dist/v2/postcondition.js +249 -0
  267. package/dist/v2/program-patcher.d.ts +25 -0
  268. package/dist/v2/program-patcher.js +44 -0
  269. package/dist/v2/recovery-chain.d.ts +30 -0
  270. package/dist/v2/recovery-chain.js +368 -0
  271. package/dist/v2/schema.d.ts +2580 -0
  272. package/dist/v2/schema.js +295 -0
  273. package/dist/v2/selector-resolver.d.ts +34 -0
  274. package/dist/v2/selector-resolver.js +181 -0
  275. package/dist/v2/semantic-resolver.d.ts +35 -0
  276. package/dist/v2/semantic-resolver.js +161 -0
  277. package/dist/v2/smart-wait.d.ts +27 -0
  278. package/dist/v2/smart-wait.js +81 -0
  279. package/dist/v2/types.d.ts +444 -0
  280. package/dist/v2/types.js +19 -0
  281. package/dist/v2/web-playwright-local.d.ts +69 -0
  282. package/dist/v2/web-playwright-local.js +392 -0
  283. package/dist/version.d.ts +1 -0
  284. package/dist/version.js +5 -0
  285. package/dist/video-agent.js +18 -13
  286. package/dist/video-planner.js +2 -1
  287. package/dist/video-prompts.js +3 -3
  288. package/dist/web-playwright-local.d.ts +156 -0
  289. package/dist/web-playwright-local.js +989 -0
  290. package/dist/ws-auth.js +4 -1
  291. package/dist/ws-broadcast.d.ts +34 -0
  292. package/dist/ws-broadcast.js +85 -0
  293. package/dist/ws-connection-limits.d.ts +12 -0
  294. package/dist/ws-connection-limits.js +44 -0
  295. package/dist/ws-handler-utils.d.ts +32 -0
  296. package/dist/ws-handler-utils.js +139 -0
  297. package/dist/ws-handler.js +294 -164
  298. package/dist/ws-metrics-server.d.ts +9 -0
  299. package/dist/ws-metrics-server.js +31 -0
  300. package/dist/ws-server.js +41 -1
  301. package/package.json +62 -35
@@ -0,0 +1,29 @@
1
+ import { Browser } from '../browser.js';
2
+ import type { VideoPageSignals } from '../types.js';
3
+ export interface VariantStateDetection {
4
+ lang: {
5
+ requested: string | undefined;
6
+ detected: string | null;
7
+ active: boolean;
8
+ ambiguous: boolean;
9
+ };
10
+ theme: {
11
+ requested: 'light' | 'dark' | undefined;
12
+ detected: 'light' | 'dark' | null;
13
+ active: boolean;
14
+ ambiguous: boolean;
15
+ };
16
+ pageSignals: VideoPageSignals;
17
+ }
18
+ export declare function scoreLocaleSignals(signals: VideoPageSignals, requestedLang?: string): {
19
+ score: number;
20
+ reasons: string[];
21
+ ambiguous: boolean;
22
+ };
23
+ export declare function evaluateRequestedThemeState(signals: VideoPageSignals, requestedTheme?: 'light' | 'dark'): {
24
+ detected: 'light' | 'dark' | null;
25
+ active: boolean;
26
+ ambiguous: boolean;
27
+ reason: string;
28
+ };
29
+ export declare function detectVariantStateDeterministic(browser: Browser, requestedLang?: string, requestedTheme?: 'light' | 'dark'): Promise<VariantStateDetection>;
@@ -0,0 +1,80 @@
1
+ import { evaluateRequestedLanguageState, evaluateRequestedThemeState as evaluateRequestedThemeStateRich, } from '../session-profile.js';
2
+ function normalizeConfidenceScore(confidence) {
3
+ switch (confidence) {
4
+ case 'high':
5
+ return 1;
6
+ case 'medium':
7
+ return 0.78;
8
+ case 'low':
9
+ default:
10
+ return 0.58;
11
+ }
12
+ }
13
+ export function scoreLocaleSignals(signals, requestedLang) {
14
+ if (!requestedLang)
15
+ return { score: 1, reasons: [], ambiguous: false };
16
+ const languageState = evaluateRequestedLanguageState({
17
+ currentUrl: signals.url,
18
+ requestedLang,
19
+ signals,
20
+ });
21
+ if (languageState.active) {
22
+ return {
23
+ score: normalizeConfidenceScore(languageState.confidence),
24
+ reasons: languageState.reasons,
25
+ ambiguous: false,
26
+ };
27
+ }
28
+ if (languageState.ambiguous) {
29
+ return {
30
+ score: languageState.confidence === 'high' ? 0.45 : languageState.confidence === 'medium' ? 0.32 : 0.18,
31
+ reasons: languageState.reasons,
32
+ ambiguous: true,
33
+ };
34
+ }
35
+ return {
36
+ score: 0,
37
+ reasons: languageState.reasons,
38
+ ambiguous: false,
39
+ };
40
+ }
41
+ export function evaluateRequestedThemeState(signals, requestedTheme) {
42
+ const themeState = evaluateRequestedThemeStateRich({
43
+ requestedTheme,
44
+ signals,
45
+ });
46
+ return {
47
+ detected: themeState.detected,
48
+ active: themeState.active,
49
+ ambiguous: themeState.ambiguous,
50
+ reason: themeState.reasons.join('; ') || 'no_theme_signal',
51
+ };
52
+ }
53
+ export async function detectVariantStateDeterministic(browser, requestedLang, requestedTheme) {
54
+ const signals = await browser.capturePageSignals();
55
+ const languageState = evaluateRequestedLanguageState({
56
+ currentUrl: signals.url,
57
+ requestedLang,
58
+ signals,
59
+ });
60
+ const themeState = evaluateRequestedThemeStateRich({
61
+ requestedTheme,
62
+ signals,
63
+ });
64
+ return {
65
+ lang: {
66
+ requested: requestedLang,
67
+ detected: languageState.detected,
68
+ active: requestedLang ? languageState.active : true,
69
+ ambiguous: requestedLang ? languageState.ambiguous : false,
70
+ },
71
+ theme: {
72
+ requested: requestedTheme,
73
+ detected: themeState.detected,
74
+ active: requestedTheme ? themeState.active : true,
75
+ ambiguous: requestedTheme ? themeState.ambiguous : false,
76
+ },
77
+ pageSignals: signals,
78
+ };
79
+ }
80
+ //# sourceMappingURL=video-variant-state.js.map
@@ -0,0 +1,17 @@
1
+ export declare class VisionModelUnsupportedError extends Error {
2
+ requestedModel: string;
3
+ fallbackModel?: string;
4
+ constructor(requestedModel: string, fallbackModel?: string);
5
+ }
6
+ export declare function isImageInputUnsupportedError(err: unknown): boolean;
7
+ export declare function buildVisionModelUnavailableMessage(requestedModel: string, fallbackModel?: string): string;
8
+ export declare function callVisionCapableModel<T>(params: {
9
+ primaryModel: string;
10
+ fallbackModel?: string;
11
+ callModel: (model: string) => Promise<T>;
12
+ onFallbackActivated?: (model: string, reason: string) => void;
13
+ }): Promise<{
14
+ result: T;
15
+ model: string;
16
+ fellBack: boolean;
17
+ }>;
@@ -0,0 +1,74 @@
1
+ export class VisionModelUnsupportedError extends Error {
2
+ requestedModel;
3
+ fallbackModel;
4
+ constructor(requestedModel, fallbackModel) {
5
+ super(buildVisionModelUnavailableMessage(requestedModel, fallbackModel));
6
+ this.name = 'VisionModelUnsupportedError';
7
+ this.requestedModel = requestedModel;
8
+ this.fallbackModel = fallbackModel;
9
+ }
10
+ }
11
+ function extractNestedErrorMessage(err) {
12
+ if (!err || typeof err !== 'object')
13
+ return '';
14
+ const errorRecord = err;
15
+ const directMessage = typeof errorRecord.message === 'string' ? errorRecord.message : '';
16
+ const nestedError = errorRecord.error;
17
+ const nestedMessage = nestedError && typeof nestedError === 'object' && typeof nestedError.message === 'string'
18
+ ? nestedError.message
19
+ : '';
20
+ if (nestedMessage && nestedMessage === directMessage)
21
+ return directMessage;
22
+ return [directMessage, nestedMessage].filter(Boolean).join(' ');
23
+ }
24
+ export function isImageInputUnsupportedError(err) {
25
+ if (!err || typeof err !== 'object')
26
+ return false;
27
+ const errorRecord = err;
28
+ const status = typeof errorRecord.status === 'number' ? errorRecord.status : undefined;
29
+ const message = extractNestedErrorMessage(err).toLowerCase();
30
+ if (!message)
31
+ return false;
32
+ const mentionsImageEndpoint = message.includes('no endpoints found that support image input')
33
+ || message.includes('no providers that support image input')
34
+ || (message.includes('image input') && message.includes('no') && message.includes('support'))
35
+ || (message.includes('image input') && message.includes('endpoint') && message.includes('no'));
36
+ if (!mentionsImageEndpoint)
37
+ return false;
38
+ return status === undefined || status === 400 || status === 404 || status === 422;
39
+ }
40
+ export function buildVisionModelUnavailableMessage(requestedModel, fallbackModel) {
41
+ const fallbackHint = fallbackModel && fallbackModel !== requestedModel
42
+ ? ` Automatic fallback to "${fallbackModel}" also failed or was unavailable.`
43
+ : '';
44
+ return `The model "${requestedModel}" has no available endpoint that supports image input.${fallbackHint} Configure a vision-capable default/fallback model or retry later if the provider is degraded.`;
45
+ }
46
+ export async function callVisionCapableModel(params) {
47
+ const { primaryModel, fallbackModel, callModel, onFallbackActivated } = params;
48
+ let fallbackReason = '';
49
+ try {
50
+ const result = await callModel(primaryModel);
51
+ return { result, model: primaryModel, fellBack: false };
52
+ }
53
+ catch (err) {
54
+ if (!isImageInputUnsupportedError(err)) {
55
+ throw err;
56
+ }
57
+ fallbackReason = extractNestedErrorMessage(err) || 'image input unsupported';
58
+ if (!fallbackModel || fallbackModel === primaryModel) {
59
+ throw new VisionModelUnsupportedError(primaryModel, fallbackModel);
60
+ }
61
+ }
62
+ try {
63
+ const result = await callModel(fallbackModel);
64
+ onFallbackActivated?.(fallbackModel, fallbackReason);
65
+ return { result, model: fallbackModel, fellBack: true };
66
+ }
67
+ catch (fallbackErr) {
68
+ if (isImageInputUnsupportedError(fallbackErr)) {
69
+ throw new VisionModelUnsupportedError(primaryModel, fallbackModel);
70
+ }
71
+ throw fallbackErr;
72
+ }
73
+ }
74
+ //# sourceMappingURL=vision-model.js.map
@@ -0,0 +1,55 @@
1
+ /**
2
+ * Capture Agent — LLM Healer
3
+ *
4
+ * Last-resort recovery: when all deterministic strategies fail,
5
+ * the healer asks an LLM to analyze the current page state and
6
+ * produce a bounded selector patch.
7
+ */
8
+ import type { ExecutionOpcode, HealerPatch } from './execution-types.js';
9
+ import type { LLMCallResult } from './llm-provider.js';
10
+ export interface HealerContext {
11
+ /** The failed opcode */
12
+ failedOpcode: ExecutionOpcode;
13
+ /** Index in the program */
14
+ opcodeIndex: number;
15
+ /** Serialized AKTree of the current page */
16
+ akTreeSerialized: string;
17
+ /** Current page URL */
18
+ currentUrl: string;
19
+ /** Screenshot buffer (for multimodal models) */
20
+ screenshot?: Buffer;
21
+ /** The 3 opcodes before and after the failed one (for context) */
22
+ surroundingOpcodes: ExecutionOpcode[];
23
+ /** Error message from the failed attempt */
24
+ errorMessage: string;
25
+ }
26
+ export interface HealerResult {
27
+ healed: boolean;
28
+ patch?: HealerPatch;
29
+ reason: string;
30
+ /** Raw LLM usage for this healing attempt */
31
+ llmResult?: LLMCallResult;
32
+ }
33
+ export interface HealerLLMProvider {
34
+ /**
35
+ * Call the LLM with a healing prompt. Returns raw JSON string.
36
+ * The provider handles model selection, API keys, etc.
37
+ */
38
+ call(prompt: string, screenshot?: Buffer): Promise<{
39
+ response: string;
40
+ llmResult: LLMCallResult;
41
+ }>;
42
+ }
43
+ /**
44
+ * The LLM Healer — a constrained agent that repairs failed selectors.
45
+ */
46
+ export declare class LLMHealer {
47
+ private llmProvider;
48
+ private invocationCount;
49
+ private maxInvocations;
50
+ constructor(llmProvider: HealerLLMProvider, options?: {
51
+ maxInvocations?: number;
52
+ });
53
+ get remainingInvocations(): number;
54
+ heal(context: HealerContext): Promise<HealerResult>;
55
+ }
@@ -0,0 +1,213 @@
1
+ /**
2
+ * Capture Agent — LLM Healer
3
+ *
4
+ * Last-resort recovery: when all deterministic strategies fail,
5
+ * the healer asks an LLM to analyze the current page state and
6
+ * produce a bounded selector patch.
7
+ */
8
+ /**
9
+ * The LLM Healer — a constrained agent that repairs failed selectors.
10
+ */
11
+ export class LLMHealer {
12
+ llmProvider;
13
+ invocationCount = 0;
14
+ maxInvocations;
15
+ constructor(llmProvider, options) {
16
+ this.llmProvider = llmProvider;
17
+ this.maxInvocations = options?.maxInvocations ?? 3;
18
+ }
19
+ get remainingInvocations() {
20
+ return Math.max(0, this.maxInvocations - this.invocationCount);
21
+ }
22
+ async heal(context) {
23
+ if (this.invocationCount >= this.maxInvocations) {
24
+ return { healed: false, reason: `healer budget exhausted (${this.maxInvocations}/${this.maxInvocations} used)` };
25
+ }
26
+ this.invocationCount++;
27
+ try {
28
+ const prompt = buildHealerPrompt(context);
29
+ const { response, llmResult } = await this.llmProvider.call(prompt, context.screenshot);
30
+ const parsed = parseHealerResponse(response);
31
+ if (!parsed) {
32
+ return { healed: false, reason: 'healer produced invalid response', llmResult };
33
+ }
34
+ if (parsed.cannotHeal) {
35
+ return { healed: false, reason: parsed.reason ?? 'healer cannot resolve this failure', llmResult };
36
+ }
37
+ const replacementOpcode = buildReplacementOpcode(context.failedOpcode, parsed);
38
+ if (!replacementOpcode) {
39
+ return { healed: false, reason: 'healer could not produce a bounded selector patch', llmResult };
40
+ }
41
+ const patch = {
42
+ opcodeIndex: context.opcodeIndex,
43
+ originalOpcode: context.failedOpcode,
44
+ replacementOpcodes: [replacementOpcode],
45
+ patchType: 'selector_patch',
46
+ interactionMode: parsed.interactionMode,
47
+ reason: parsed.reason ?? 'healer selector repair',
48
+ patchedAt: new Date().toISOString(),
49
+ };
50
+ return { healed: true, patch, reason: patch.reason, llmResult };
51
+ }
52
+ catch (err) {
53
+ return {
54
+ healed: false,
55
+ reason: `healer error: ${err instanceof Error ? err.message : String(err)}`,
56
+ };
57
+ }
58
+ }
59
+ }
60
+ function buildHealerPrompt(context) {
61
+ const { failedOpcode, errorMessage, akTreeSerialized, currentUrl, surroundingOpcodes } = context;
62
+ const selector = hasPatchableSelector(failedOpcode) ? failedOpcode.selector : null;
63
+ const surroundingDesc = surroundingOpcodes
64
+ .map((op, i) => ` ${i + 1}. ${op.kind}: ${sanitizePromptText(op.description)}`)
65
+ .join('\n');
66
+ return `You are a web automation repair agent. An opcode in a deterministic browser automation program has failed. Your job is to repair the selector only.
67
+
68
+ ## Failed Opcode
69
+ Kind: ${failedOpcode.kind}
70
+ Description: ${sanitizePromptText(failedOpcode.description)}
71
+ Selector: ${selector ?? '<unsupported>'}
72
+
73
+ ## Error
74
+ <<<ERROR>>>
75
+ ${sanitizePromptText(errorMessage)}
76
+ <<<END_ERROR>>>
77
+
78
+ ## Current Page
79
+ URL: ${sanitizeUrlForPrompt(currentUrl)}
80
+
81
+ ## AKTree (current page structure)
82
+ <<<AKTREE>>>
83
+ ${sanitizePromptText(akTreeSerialized).slice(0, 8000)}
84
+ <<<END_AKTREE>>>
85
+
86
+ ## Surrounding Opcodes (context)
87
+ ${surroundingDesc}
88
+
89
+ ## Rules
90
+ 1. You MUST NOT emit opcodes or navigation targets.
91
+ 2. You may only return a selector patch for the existing opcode.
92
+ 3. You may optionally return a bounded interaction mode: default, keyboard, js_dispatch, or coordinates.
93
+ 4. Use selectors you can actually see in the AKTree above.
94
+ 5. If the opcode kind is unsupported or there is no reliable selector, return cannotHeal=true.
95
+
96
+ ## Response Format
97
+ Respond with a JSON object:
98
+ {
99
+ "cannotHeal": false,
100
+ "reason": "explanation of what you changed",
101
+ "selector": "[data-ak='submit']",
102
+ "selectorAlternates": ["button[type='submit']"],
103
+ "interactionMode": "default"
104
+ }
105
+
106
+ Or if you cannot fix it:
107
+ {
108
+ "cannotHeal": true,
109
+ "reason": "explanation of why this cannot be fixed"
110
+ }
111
+
112
+ Respond ONLY with the JSON object, no markdown, no explanation outside JSON.`;
113
+ }
114
+ function parseHealerResponse(response) {
115
+ try {
116
+ const jsonStr = response
117
+ .replace(/```json\s*/g, '')
118
+ .replace(/```\s*/g, '')
119
+ .trim();
120
+ const parsed = JSON.parse(jsonStr);
121
+ if (!parsed || typeof parsed !== 'object')
122
+ return null;
123
+ if (parsed.cannotHeal) {
124
+ return {
125
+ cannotHeal: true,
126
+ reason: typeof parsed.reason === 'string' ? parsed.reason : null,
127
+ selector: null,
128
+ selectorAlternates: [],
129
+ interactionMode: 'default',
130
+ };
131
+ }
132
+ const selector = typeof parsed.selector === 'string' && parsed.selector.trim()
133
+ ? parsed.selector.trim()
134
+ : null;
135
+ if (!selector)
136
+ return null;
137
+ return {
138
+ cannotHeal: false,
139
+ reason: typeof parsed.reason === 'string' ? parsed.reason : null,
140
+ selector,
141
+ selectorAlternates: Array.isArray(parsed.selectorAlternates)
142
+ ? parsed.selectorAlternates
143
+ .filter((entry) => typeof entry === 'string' && entry.trim().length > 0)
144
+ .map((entry) => entry.trim())
145
+ .slice(0, 5)
146
+ : [],
147
+ interactionMode: parsed.interactionMode === 'keyboard'
148
+ || parsed.interactionMode === 'js_dispatch'
149
+ || parsed.interactionMode === 'coordinates'
150
+ ? parsed.interactionMode
151
+ : 'default',
152
+ };
153
+ }
154
+ catch {
155
+ return null;
156
+ }
157
+ }
158
+ function hasPatchableSelector(opcode) {
159
+ return ((opcode.kind === 'CLICK'
160
+ || opcode.kind === 'TYPE'
161
+ || opcode.kind === 'WAIT_FOR'
162
+ || opcode.kind === 'HOVER'
163
+ || opcode.kind === 'SELECT_OPTION'
164
+ || opcode.kind === 'CHECK'
165
+ || opcode.kind === 'DOUBLE_CLICK')
166
+ && typeof opcode.selector === 'string'
167
+ && opcode.selector.trim().length > 0);
168
+ }
169
+ function buildReplacementOpcode(failedOpcode, parsed) {
170
+ if (!hasPatchableSelector(failedOpcode) || !parsed.selector) {
171
+ return null;
172
+ }
173
+ switch (failedOpcode.kind) {
174
+ case 'CLICK':
175
+ case 'TYPE':
176
+ case 'WAIT_FOR':
177
+ case 'HOVER':
178
+ case 'SELECT_OPTION':
179
+ case 'CHECK':
180
+ case 'DOUBLE_CLICK':
181
+ return {
182
+ ...failedOpcode,
183
+ selector: parsed.selector,
184
+ ...('selectorAlternates' in failedOpcode
185
+ ? { selectorAlternates: parsed.selectorAlternates }
186
+ : {}),
187
+ };
188
+ default:
189
+ return null;
190
+ }
191
+ }
192
+ function sanitizePromptText(value) {
193
+ return value
194
+ .replace(/\{\{(email|password|loginUrl)\}\}/g, '<credential-placeholder>')
195
+ .replace(/data:[^"' )]+/gi, '<data-url>')
196
+ .replace(/https?:\/\/[^\s"'<>]+/gi, (match) => sanitizeUrlForPrompt(match))
197
+ .replace(/[^\x09\x0a\x0d\x20-\x7e]/g, ' ')
198
+ .slice(0, 12_000);
199
+ }
200
+ function sanitizeUrlForPrompt(value) {
201
+ try {
202
+ const parsed = new URL(value);
203
+ parsed.username = '';
204
+ parsed.password = '';
205
+ parsed.search = parsed.search ? '?<redacted>' : '';
206
+ parsed.hash = parsed.hash ? '#<redacted>' : '';
207
+ return parsed.toString();
208
+ }
209
+ catch {
210
+ return value.slice(0, 512);
211
+ }
212
+ }
213
+ //# sourceMappingURL=llm-healer.js.map
@@ -0,0 +1,29 @@
1
+ /**
2
+ * Capture Agent — LLM Provider
3
+ *
4
+ * Thin wrapper around OpenAI SDK for OpenRouter calls.
5
+ * Used by: capture verification, alt text generation, healer.
6
+ */
7
+ export interface LLMCallResult {
8
+ text: string;
9
+ model: string;
10
+ promptTokens: number;
11
+ completionTokens: number;
12
+ costEur: number;
13
+ /** OpenRouter generation ID for async cost resolution */
14
+ generationId: string | null;
15
+ }
16
+ export interface LLMProviderConfig {
17
+ apiKey: string;
18
+ /** Model to use. Default: google/gemini-2.5-flash */
19
+ model?: string;
20
+ /** Max tokens for response. Default: 200 */
21
+ maxTokens?: number;
22
+ /** Timeout in ms. Default: 15000 */
23
+ timeoutMs?: number;
24
+ }
25
+ /**
26
+ * Makes a single LLM call with optional image input.
27
+ * Returns the text response and usage metadata.
28
+ */
29
+ export declare function callLLM(config: LLMProviderConfig, systemPrompt: string, userContent: string, image?: Buffer): Promise<LLMCallResult>;
@@ -0,0 +1,83 @@
1
+ /**
2
+ * Capture Agent — LLM Provider
3
+ *
4
+ * Thin wrapper around OpenAI SDK for OpenRouter calls.
5
+ * Used by: capture verification, alt text generation, healer.
6
+ */
7
+ import OpenAI from 'openai';
8
+ import { createOpenRouterCompletion } from './openrouter-client.js';
9
+ import { assertZdrCompatibleModel, zdrParam } from './provider-config.js';
10
+ const OPENROUTER_BASE_URL = 'https://openrouter.ai/api/v1';
11
+ const DEFAULT_MODEL = 'google/gemini-2.5-flash';
12
+ /**
13
+ * Makes a single LLM call with optional image input.
14
+ * Returns the text response and usage metadata.
15
+ */
16
+ export async function callLLM(config, systemPrompt, userContent, image) {
17
+ const client = new OpenAI({
18
+ baseURL: OPENROUTER_BASE_URL,
19
+ apiKey: config.apiKey,
20
+ });
21
+ const model = config.model ?? DEFAULT_MODEL;
22
+ const maxTokens = config.maxTokens ?? 200;
23
+ assertZdrCompatibleModel(model);
24
+ const messages = [
25
+ { role: 'system', content: systemPrompt },
26
+ ];
27
+ if (image) {
28
+ const base64 = image.toString('base64');
29
+ messages.push({
30
+ role: 'user',
31
+ content: [
32
+ { type: 'text', text: userContent },
33
+ {
34
+ type: 'image_url',
35
+ image_url: { url: `data:image/png;base64,${base64}`, detail: 'low' },
36
+ },
37
+ ],
38
+ });
39
+ }
40
+ else {
41
+ messages.push({ role: 'user', content: userContent });
42
+ }
43
+ const params = {
44
+ model,
45
+ messages,
46
+ max_tokens: maxTokens,
47
+ temperature: 0.1,
48
+ provider: Object.keys(zdrParam()).length > 0 ? zdrParam() : undefined,
49
+ };
50
+ const controller = new AbortController();
51
+ const timer = setTimeout(() => controller.abort(), config.timeoutMs ?? 15000);
52
+ try {
53
+ const response = await createOpenRouterCompletion(client, params, { signal: controller.signal });
54
+ clearTimeout(timer);
55
+ const text = response.choices?.[0]?.message?.content?.trim() ?? '';
56
+ const usage = response.usage;
57
+ return {
58
+ text,
59
+ model: response.model ?? model,
60
+ promptTokens: usage?.prompt_tokens ?? 0,
61
+ completionTokens: usage?.completion_tokens ?? 0,
62
+ costEur: estimateCost(model, usage?.prompt_tokens ?? 0, usage?.completion_tokens ?? 0),
63
+ generationId: response.id ?? null,
64
+ };
65
+ }
66
+ catch (err) {
67
+ clearTimeout(timer);
68
+ throw err;
69
+ }
70
+ }
71
+ /**
72
+ * Rough cost estimation based on known model pricing.
73
+ * Actual cost is resolved asynchronously via OpenRouter generation API.
74
+ */
75
+ function estimateCost(model, promptTokens, completionTokens) {
76
+ // Gemini 2.5 Flash pricing (approximate)
77
+ if (model.includes('gemini')) {
78
+ return (promptTokens * 0.15 + completionTokens * 0.6) / 1_000_000;
79
+ }
80
+ // Fallback: conservative estimate
81
+ return (promptTokens * 0.5 + completionTokens * 1.5) / 1_000_000;
82
+ }
83
+ //# sourceMappingURL=llm-provider.js.map
package/dist/logger.d.ts CHANGED
@@ -1,4 +1,4 @@
1
- export type LogLevel = 'info' | 'action' | 'success' | 'error' | 'ai';
1
+ export type LogLevel = 'info' | 'warn' | 'action' | 'success' | 'error' | 'ai';
2
2
  export type LogKind = 'reasoning' | 'action' | 'system' | 'guidance';
3
3
  export interface LogEntry {
4
4
  level: LogLevel;
@@ -25,6 +25,8 @@ interface LoggerContext {
25
25
  onScreenshot?: OnScreenshotCallback | null;
26
26
  onReasoningDelta?: OnReasoningDeltaCallback | null;
27
27
  }
28
+ export declare function setDebugEnabled(enabled: boolean): void;
29
+ export declare function isDebugEnabled(): boolean;
28
30
  export declare function runWithLoggerCallbacks<T>(callbacks: LoggerContext, fn: () => Promise<T>): Promise<T>;
29
31
  export declare function setOnLog(cb: OnLogCallback | null): void;
30
32
  export declare function setOnScreenshot(cb: OnScreenshotCallback | null): void;
@@ -33,8 +35,10 @@ export declare function emitReasoningDelta(delta: string, messageId: string): vo
33
35
  export declare const logger: {
34
36
  /** User-visible informational log — shown in both console and UI. */
35
37
  info(msg: string): void;
36
- /** Internal debug log — console only, NOT emitted to the UI callback. */
38
+ /** Internal debug log — only printed when --debug is enabled. Console only, NOT emitted to the UI callback. */
37
39
  debug(msg: string): void;
40
+ /** Security / rate-limit warnings — emitted to both console and UI. */
41
+ warn(msg: string): void;
38
42
  action(iteration: number, maxIter: number, action: string, params: Record<string, unknown>, context?: {
39
43
  lang?: string;
40
44
  theme?: string;
package/dist/logger.js CHANGED
@@ -2,7 +2,14 @@ import { AsyncLocalStorage } from 'node:async_hooks';
2
2
  import chalk from 'chalk';
3
3
  let _onLog = null;
4
4
  let _onScreenshot = null;
5
+ let _debugEnabled = false;
5
6
  const _loggerContext = new AsyncLocalStorage();
7
+ export function setDebugEnabled(enabled) {
8
+ _debugEnabled = enabled;
9
+ }
10
+ export function isDebugEnabled() {
11
+ return _debugEnabled;
12
+ }
6
13
  export async function runWithLoggerCallbacks(callbacks, fn) {
7
14
  return _loggerContext.run(callbacks, fn);
8
15
  }
@@ -42,10 +49,17 @@ export const logger = {
42
49
  console.log(chalk.cyan('[INFO]'), msg);
43
50
  emit({ level: 'info', message: msg, timestamp: Date.now(), kind: 'system' });
44
51
  },
45
- /** Internal debug log — console only, NOT emitted to the UI callback. */
52
+ /** Internal debug log — only printed when --debug is enabled. Console only, NOT emitted to the UI callback. */
46
53
  debug(msg) {
54
+ if (!_debugEnabled)
55
+ return;
47
56
  console.log(chalk.gray('[DEBUG]'), msg);
48
57
  },
58
+ /** Security / rate-limit warnings — emitted to both console and UI. */
59
+ warn(msg) {
60
+ console.log(chalk.yellow('[WARN]'), msg);
61
+ emit({ level: 'warn', message: msg, timestamp: Date.now(), kind: 'system' });
62
+ },
49
63
  action(iteration, maxIter, action, params, context) {
50
64
  const paramsStr = Object.entries(params)
51
65
  .map(([k, v]) => `${k}=${JSON.stringify(v)}`)