autokap 1.0.7 → 1.0.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (301) hide show
  1. package/assets/cursors/macos.svg +4 -0
  2. package/assets/cursors/windows.svg +15 -0
  3. package/assets/skill/OPCODE-REFERENCE.md +635 -0
  4. package/assets/skill/README.md +39 -0
  5. package/assets/skill/SKILL.md +454 -468
  6. package/assets/skill/STUDIO-SKILL.md +476 -0
  7. package/assets/skill/references/examples.md +104 -0
  8. package/assets/skill/references/interactive-demo.md +225 -0
  9. package/assets/skill/references/mock-data.md +178 -0
  10. package/dist/action-verifier.d.ts +29 -0
  11. package/dist/action-verifier.js +133 -0
  12. package/dist/agent-action-recovery.d.ts +45 -0
  13. package/dist/agent-action-recovery.js +370 -0
  14. package/dist/agent-message-utils.d.ts +21 -0
  15. package/dist/agent-message-utils.js +77 -0
  16. package/dist/agent-url-utils.d.ts +30 -0
  17. package/dist/agent-url-utils.js +138 -0
  18. package/dist/agent.d.ts +92 -8
  19. package/dist/agent.js +2936 -781
  20. package/dist/ak-tree.d.ts +39 -0
  21. package/dist/ak-tree.js +368 -0
  22. package/dist/alt-text.d.ts +26 -0
  23. package/dist/alt-text.js +55 -0
  24. package/dist/auth-capture.d.ts +17 -0
  25. package/dist/auth-capture.js +197 -0
  26. package/dist/benchmark.d.ts +59 -0
  27. package/dist/benchmark.js +135 -0
  28. package/dist/billing-operation-logging.d.ts +3 -1
  29. package/dist/billing-operation-logging.js +4 -0
  30. package/dist/browser-bar.d.ts +14 -6
  31. package/dist/browser-bar.js +145 -8
  32. package/dist/browser-pool.d.ts +7 -0
  33. package/dist/browser-pool.js +15 -5
  34. package/dist/browser-utils.d.ts +31 -0
  35. package/dist/browser-utils.js +97 -0
  36. package/dist/browser.d.ts +61 -11
  37. package/dist/browser.js +1513 -59
  38. package/dist/capture-alt-text.js +2 -1
  39. package/dist/capture-encryption.d.ts +3 -1
  40. package/dist/capture-encryption.js +21 -6
  41. package/dist/capture-language-preflight.js +14 -0
  42. package/dist/capture-llm-page-identity.js +22 -10
  43. package/dist/capture-page-identity.d.ts +5 -7
  44. package/dist/capture-page-identity.js +211 -78
  45. package/dist/capture-preset-credentials.d.ts +50 -0
  46. package/dist/capture-preset-credentials.js +127 -0
  47. package/dist/capture-request-plan.d.ts +2 -2
  48. package/dist/capture-request-plan.js +64 -16
  49. package/dist/capture-run-optimizer.js +48 -33
  50. package/dist/capture-selector-memory.d.ts +5 -0
  51. package/dist/capture-selector-memory.js +18 -0
  52. package/dist/capture-strategy.d.ts +36 -0
  53. package/dist/capture-strategy.js +96 -0
  54. package/dist/capture-studio-sync.d.ts +1 -0
  55. package/dist/capture-studio-sync.js +9 -3
  56. package/dist/capture-surface-contract.d.ts +36 -0
  57. package/dist/capture-surface-contract.js +299 -0
  58. package/dist/capture-transition-engine.d.ts +28 -0
  59. package/dist/capture-transition-engine.js +292 -0
  60. package/dist/capture-variant-state.d.ts +2 -0
  61. package/dist/capture-variant-state.js +26 -0
  62. package/dist/capture-verification.d.ts +35 -0
  63. package/dist/capture-verification.js +95 -0
  64. package/dist/capture-viewport-lock.d.ts +48 -0
  65. package/dist/capture-viewport-lock.js +74 -0
  66. package/dist/circuit-breaker.d.ts +42 -0
  67. package/dist/circuit-breaker.js +119 -0
  68. package/dist/cli-config.d.ts +9 -1
  69. package/dist/cli-config.js +112 -7
  70. package/dist/cli-contract.d.ts +19 -0
  71. package/dist/cli-contract.js +173 -0
  72. package/dist/cli-runner-local.d.ts +12 -0
  73. package/dist/cli-runner-local.js +115 -0
  74. package/dist/cli-runner.d.ts +34 -0
  75. package/dist/cli-runner.js +580 -0
  76. package/dist/cli-utils.d.ts +0 -1
  77. package/dist/cli-utils.js +2 -5
  78. package/dist/cli.js +1011 -267
  79. package/dist/clip-begin-frame-recorder.d.ts +44 -0
  80. package/dist/clip-begin-frame-recorder.js +250 -0
  81. package/dist/clip-capture-backend.d.ts +25 -0
  82. package/dist/clip-capture-backend.js +189 -0
  83. package/dist/clip-capture-loop.d.ts +61 -0
  84. package/dist/clip-capture-loop.js +111 -0
  85. package/dist/clip-frame-recorder.d.ts +63 -0
  86. package/dist/clip-frame-recorder.js +305 -0
  87. package/dist/clip-orchestrator.js +9 -2
  88. package/dist/clip-postprocess.d.ts +31 -2
  89. package/dist/clip-postprocess.js +194 -68
  90. package/dist/clip-runtime.d.ts +18 -0
  91. package/dist/clip-runtime.js +67 -0
  92. package/dist/clip-scale.d.ts +10 -0
  93. package/dist/clip-scale.js +21 -0
  94. package/dist/clip-screencast-recorder.d.ts +42 -0
  95. package/dist/clip-screencast-recorder.js +242 -0
  96. package/dist/clip-sidecar.d.ts +54 -0
  97. package/dist/clip-sidecar.js +208 -0
  98. package/dist/cookie-dismiss.d.ts +2 -0
  99. package/dist/cookie-dismiss.js +48 -13
  100. package/dist/cost-logging.d.ts +8 -0
  101. package/dist/cost-logging.js +160 -46
  102. package/dist/cost-resolution-monitor.d.ts +16 -0
  103. package/dist/cost-resolution-monitor.js +34 -0
  104. package/dist/credential-templates.js +2 -2
  105. package/dist/cursor-overlay-script.d.ts +6 -0
  106. package/dist/cursor-overlay-script.js +169 -0
  107. package/dist/dom-css-purger.d.ts +65 -0
  108. package/dist/dom-css-purger.js +333 -0
  109. package/dist/dom-font-inliner.d.ts +45 -0
  110. package/dist/dom-font-inliner.js +148 -0
  111. package/dist/dom-patch-resolver.d.ts +52 -0
  112. package/dist/dom-patch-resolver.js +242 -0
  113. package/dist/dom-serializer.d.ts +82 -0
  114. package/dist/dom-serializer.js +378 -0
  115. package/dist/element-capture.d.ts +1 -41
  116. package/dist/element-capture.js +202 -446
  117. package/dist/env-validation.d.ts +5 -0
  118. package/dist/env-validation.js +63 -0
  119. package/dist/execution-schema.d.ts +4753 -0
  120. package/dist/execution-schema.js +563 -0
  121. package/dist/execution-types.d.ts +936 -0
  122. package/dist/execution-types.js +66 -0
  123. package/dist/fonts-loader.d.ts +14 -0
  124. package/dist/fonts-loader.js +55 -0
  125. package/dist/hybrid-navigator.js +12 -12
  126. package/dist/index.d.ts +11 -6
  127. package/dist/index.js +11 -4
  128. package/dist/legacy/agent-action-recovery.d.ts +45 -0
  129. package/dist/legacy/agent-action-recovery.js +370 -0
  130. package/dist/legacy/agent-message-utils.d.ts +21 -0
  131. package/dist/legacy/agent-message-utils.js +77 -0
  132. package/dist/legacy/agent-url-utils.d.ts +30 -0
  133. package/dist/legacy/agent-url-utils.js +138 -0
  134. package/dist/legacy/agent.d.ts +226 -0
  135. package/dist/legacy/agent.js +6666 -0
  136. package/dist/legacy/clip-orchestrator.d.ts +148 -0
  137. package/dist/legacy/clip-orchestrator.js +957 -0
  138. package/dist/legacy/credential-templates.d.ts +5 -0
  139. package/dist/legacy/credential-templates.js +60 -0
  140. package/dist/legacy/hybrid-navigator.d.ts +138 -0
  141. package/dist/legacy/hybrid-navigator.js +468 -0
  142. package/dist/legacy/llm-usage.d.ts +17 -0
  143. package/dist/legacy/llm-usage.js +45 -0
  144. package/dist/legacy/prompt-cache.d.ts +10 -0
  145. package/dist/legacy/prompt-cache.js +24 -0
  146. package/dist/legacy/prompts.d.ts +175 -0
  147. package/dist/legacy/prompts.js +1038 -0
  148. package/dist/legacy/tools.d.ts +4 -0
  149. package/dist/legacy/tools.js +216 -0
  150. package/dist/legacy/video-agent.d.ts +143 -0
  151. package/dist/legacy/video-agent.js +4788 -0
  152. package/dist/legacy/video-observation.d.ts +36 -0
  153. package/dist/legacy/video-observation.js +192 -0
  154. package/dist/legacy/video-planner.d.ts +12 -0
  155. package/dist/legacy/video-planner.js +501 -0
  156. package/dist/legacy/video-prompts.d.ts +37 -0
  157. package/dist/legacy/video-prompts.js +569 -0
  158. package/dist/legacy/video-tools.d.ts +3 -0
  159. package/dist/legacy/video-tools.js +59 -0
  160. package/dist/legacy/video-variant-state.d.ts +29 -0
  161. package/dist/legacy/video-variant-state.js +80 -0
  162. package/dist/legacy/vision-model.d.ts +17 -0
  163. package/dist/legacy/vision-model.js +74 -0
  164. package/dist/llm-healer.d.ts +55 -0
  165. package/dist/llm-healer.js +213 -0
  166. package/dist/llm-provider.d.ts +29 -0
  167. package/dist/llm-provider.js +83 -0
  168. package/dist/logger.d.ts +6 -2
  169. package/dist/logger.js +15 -1
  170. package/dist/mockup-html.js +35 -25
  171. package/dist/mockup.d.ts +95 -2
  172. package/dist/mockup.js +427 -166
  173. package/dist/mouse-animation.d.ts +2 -2
  174. package/dist/mouse-animation.js +34 -20
  175. package/dist/opcode-actions.d.ts +42 -0
  176. package/dist/opcode-actions.js +524 -0
  177. package/dist/opcode-runner.d.ts +51 -0
  178. package/dist/opcode-runner.js +779 -0
  179. package/dist/openrouter-client.d.ts +40 -0
  180. package/dist/openrouter-client.js +16 -0
  181. package/dist/overlay-engine.d.ts +24 -0
  182. package/dist/overlay-engine.js +176 -0
  183. package/dist/postcondition.d.ts +16 -0
  184. package/dist/postcondition.js +269 -0
  185. package/dist/program-patcher.d.ts +25 -0
  186. package/dist/program-patcher.js +44 -0
  187. package/dist/program-signing.d.ts +1094 -0
  188. package/dist/program-signing.js +140 -0
  189. package/dist/prompts.d.ts +13 -5
  190. package/dist/prompts.js +224 -351
  191. package/dist/provider-config.d.ts +17 -0
  192. package/dist/provider-config.js +42 -0
  193. package/dist/recovery-chain.d.ts +37 -0
  194. package/dist/recovery-chain.js +374 -0
  195. package/dist/remote-browser.d.ts +28 -4
  196. package/dist/remote-browser.js +60 -5
  197. package/dist/safari-browser-bar.d.ts +15 -0
  198. package/dist/safari-browser-bar.js +95 -0
  199. package/dist/safari-toolbar-asset.d.ts +15 -0
  200. package/dist/safari-toolbar-asset.js +12 -0
  201. package/dist/security.d.ts +2 -1
  202. package/dist/security.js +49 -10
  203. package/dist/selector-resolver.d.ts +34 -0
  204. package/dist/selector-resolver.js +181 -0
  205. package/dist/semantic-resolver.d.ts +35 -0
  206. package/dist/semantic-resolver.js +161 -0
  207. package/dist/server-capture-runtime.d.ts +5 -3
  208. package/dist/server-capture-runtime.js +42 -95
  209. package/dist/server-credit-usage.d.ts +2 -2
  210. package/dist/server-project-webhooks.d.ts +15 -1
  211. package/dist/server-project-webhooks.js +34 -8
  212. package/dist/server-screenshot-watermark.js +27 -5
  213. package/dist/session-profile.js +164 -1
  214. package/dist/sf-pro-symbols.d.ts +1 -0
  215. package/dist/sf-pro-symbols.js +55 -0
  216. package/dist/skill-packaging.d.ts +28 -0
  217. package/dist/skill-packaging.js +169 -0
  218. package/dist/smart-wait.d.ts +27 -0
  219. package/dist/smart-wait.js +81 -0
  220. package/dist/status-bar-render.d.ts +20 -0
  221. package/dist/status-bar-render.js +410 -0
  222. package/dist/status-bar.d.ts +9 -0
  223. package/dist/status-bar.js +298 -14
  224. package/dist/svg-browser-bar.d.ts +33 -0
  225. package/dist/svg-browser-bar.js +206 -0
  226. package/dist/svg-status-bar.d.ts +36 -0
  227. package/dist/svg-status-bar.js +597 -0
  228. package/dist/svg-text.d.ts +61 -0
  229. package/dist/svg-text.js +118 -0
  230. package/dist/tools.js +89 -451
  231. package/dist/types.d.ts +248 -7
  232. package/dist/types.js +23 -1
  233. package/dist/v2/action-verifier.d.ts +29 -0
  234. package/dist/v2/action-verifier.js +133 -0
  235. package/dist/v2/alt-text.d.ts +26 -0
  236. package/dist/v2/alt-text.js +55 -0
  237. package/dist/v2/benchmark.d.ts +59 -0
  238. package/dist/v2/benchmark.js +135 -0
  239. package/dist/v2/capture-strategy.d.ts +30 -0
  240. package/dist/v2/capture-strategy.js +67 -0
  241. package/dist/v2/capture-verification.d.ts +35 -0
  242. package/dist/v2/capture-verification.js +95 -0
  243. package/dist/v2/circuit-breaker.d.ts +42 -0
  244. package/dist/v2/circuit-breaker.js +119 -0
  245. package/dist/v2/cli-runner-local.d.ts +11 -0
  246. package/dist/v2/cli-runner-local.js +91 -0
  247. package/dist/v2/cli-runner.d.ts +34 -0
  248. package/dist/v2/cli-runner.js +300 -0
  249. package/dist/v2/compiler-prompts.d.ts +27 -0
  250. package/dist/v2/compiler-prompts.js +123 -0
  251. package/dist/v2/compiler.d.ts +37 -0
  252. package/dist/v2/compiler.js +147 -0
  253. package/dist/v2/explorer.d.ts +41 -0
  254. package/dist/v2/explorer.js +56 -0
  255. package/dist/v2/index.d.ts +37 -0
  256. package/dist/v2/index.js +31 -0
  257. package/dist/v2/llm-healer.d.ts +62 -0
  258. package/dist/v2/llm-healer.js +166 -0
  259. package/dist/v2/llm-provider.d.ts +29 -0
  260. package/dist/v2/llm-provider.js +80 -0
  261. package/dist/v2/opcode-runner.d.ts +47 -0
  262. package/dist/v2/opcode-runner.js +634 -0
  263. package/dist/v2/overlay-engine.d.ts +24 -0
  264. package/dist/v2/overlay-engine.js +150 -0
  265. package/dist/v2/postcondition.d.ts +16 -0
  266. package/dist/v2/postcondition.js +249 -0
  267. package/dist/v2/program-patcher.d.ts +25 -0
  268. package/dist/v2/program-patcher.js +44 -0
  269. package/dist/v2/recovery-chain.d.ts +30 -0
  270. package/dist/v2/recovery-chain.js +368 -0
  271. package/dist/v2/schema.d.ts +2580 -0
  272. package/dist/v2/schema.js +295 -0
  273. package/dist/v2/selector-resolver.d.ts +34 -0
  274. package/dist/v2/selector-resolver.js +181 -0
  275. package/dist/v2/semantic-resolver.d.ts +35 -0
  276. package/dist/v2/semantic-resolver.js +161 -0
  277. package/dist/v2/smart-wait.d.ts +27 -0
  278. package/dist/v2/smart-wait.js +81 -0
  279. package/dist/v2/types.d.ts +444 -0
  280. package/dist/v2/types.js +19 -0
  281. package/dist/v2/web-playwright-local.d.ts +69 -0
  282. package/dist/v2/web-playwright-local.js +392 -0
  283. package/dist/version.d.ts +1 -0
  284. package/dist/version.js +5 -0
  285. package/dist/video-agent.js +18 -13
  286. package/dist/video-planner.js +2 -1
  287. package/dist/video-prompts.js +3 -3
  288. package/dist/web-playwright-local.d.ts +156 -0
  289. package/dist/web-playwright-local.js +989 -0
  290. package/dist/ws-auth.js +4 -1
  291. package/dist/ws-broadcast.d.ts +34 -0
  292. package/dist/ws-broadcast.js +85 -0
  293. package/dist/ws-connection-limits.d.ts +12 -0
  294. package/dist/ws-connection-limits.js +44 -0
  295. package/dist/ws-handler-utils.d.ts +32 -0
  296. package/dist/ws-handler-utils.js +139 -0
  297. package/dist/ws-handler.js +294 -164
  298. package/dist/ws-metrics-server.d.ts +9 -0
  299. package/dist/ws-metrics-server.js +31 -0
  300. package/dist/ws-server.js +41 -1
  301. package/package.json +62 -35
@@ -0,0 +1,147 @@
1
+ /**
2
+ * V2 Capture Agent — Preset Compiler
3
+ *
4
+ * Transforms natural-language presets into ExecutionProgramV2.
5
+ * Flow:
6
+ * 1. Parse preset metadata (URL, variants, auth, media mode)
7
+ * 2. (Optional) Explore the target site to capture real AKTree selectors
8
+ * 3. LLM generates opcode sequence grounded in real DOM
9
+ * 4. Validate output against Zod schema
10
+ * 5. Return compiled program with fingerprint
11
+ */
12
+ import { DEFAULT_RECOVERY_POLICY } from './types.js';
13
+ import { ExecutionOpcodeSchema, ExecutionProgramV2Schema } from './schema.js';
14
+ import { buildCompilerSystemPrompt, buildCompilerUserPrompt } from './compiler-prompts.js';
15
+ // ── Main compile function ───────────────────────────────────────────
16
+ export async function compilePreset(input, llmProvider) {
17
+ const systemPrompt = buildCompilerSystemPrompt();
18
+ const context = {
19
+ presetPrompt: input.prompt,
20
+ baseUrl: input.baseUrl,
21
+ mediaMode: input.mediaMode,
22
+ variants: input.variants,
23
+ requiresAuth: input.preconditions.auth === 'authenticated',
24
+ navigationInstructions: input.navigationInstructions,
25
+ akTreeSnapshot: input.akTreeSnapshot,
26
+ exploredUrl: input.exploredUrl,
27
+ };
28
+ const userPrompt = buildCompilerUserPrompt(context);
29
+ try {
30
+ const { response, costEur } = await llmProvider.call(systemPrompt, userPrompt);
31
+ // Parse opcodes from LLM response
32
+ const opcodes = parseOpcodeResponse(response);
33
+ if (!opcodes) {
34
+ return { success: false, error: 'LLM response is not a valid JSON array of opcodes', costEur };
35
+ }
36
+ // Validate each opcode
37
+ const validatedOpcodes = [];
38
+ for (let i = 0; i < opcodes.length; i++) {
39
+ const enriched = enrichOpcode(opcodes[i]);
40
+ const result = ExecutionOpcodeSchema.safeParse(enriched);
41
+ if (!result.success) {
42
+ return {
43
+ success: false,
44
+ error: `opcode ${i} validation failed: ${result.error}`,
45
+ costEur,
46
+ };
47
+ }
48
+ validatedOpcodes.push(result.data);
49
+ }
50
+ if (validatedOpcodes.length === 0) {
51
+ return { success: false, error: 'LLM produced zero opcodes', costEur };
52
+ }
53
+ // Build the program
54
+ const program = {
55
+ presetId: input.presetId,
56
+ programVersion: 1,
57
+ mediaMode: input.mediaMode,
58
+ baseUrl: input.baseUrl,
59
+ variants: input.variants,
60
+ preconditions: input.preconditions,
61
+ steps: validatedOpcodes,
62
+ artifactPlan: {
63
+ mediaMode: input.mediaMode,
64
+ ...(input.mediaMode === 'clip' ? { format: { clipFormat: 'gif' } } : {}),
65
+ },
66
+ compileFingerprint: computeFingerprint(input),
67
+ compiledAt: new Date().toISOString(),
68
+ compiledWith: 'gemini-3-flash',
69
+ };
70
+ // Final program-level validation
71
+ const programResult = ExecutionProgramV2Schema.safeParse(program);
72
+ if (!programResult.success) {
73
+ return {
74
+ success: false,
75
+ error: `program validation failed: ${programResult.error}`,
76
+ costEur,
77
+ };
78
+ }
79
+ return { success: true, program: programResult.data, costEur };
80
+ }
81
+ catch (err) {
82
+ return {
83
+ success: false,
84
+ error: `compiler error: ${err instanceof Error ? err.message : String(err)}`,
85
+ costEur: 0,
86
+ };
87
+ }
88
+ }
89
+ // ── Response parsing ────────────────────────────────────────────────
90
+ function parseOpcodeResponse(response) {
91
+ try {
92
+ // Strip markdown code fences if present
93
+ let cleaned = response.trim();
94
+ if (cleaned.startsWith('```')) {
95
+ cleaned = cleaned.replace(/^```(?:json)?\s*/, '').replace(/```\s*$/, '').trim();
96
+ }
97
+ const parsed = JSON.parse(cleaned);
98
+ if (Array.isArray(parsed))
99
+ return parsed;
100
+ // Some models wrap in { steps: [...] } or { opcodes: [...] }
101
+ if (parsed && typeof parsed === 'object') {
102
+ if (Array.isArray(parsed.steps))
103
+ return parsed.steps;
104
+ if (Array.isArray(parsed.opcodes))
105
+ return parsed.opcodes;
106
+ }
107
+ return null;
108
+ }
109
+ catch {
110
+ return null;
111
+ }
112
+ }
113
+ /**
114
+ * Enriches a raw opcode from LLM with default values for fields
115
+ * the LLM might omit.
116
+ */
117
+ function enrichOpcode(raw) {
118
+ return {
119
+ ...raw,
120
+ description: raw.description || `${raw.kind} step`,
121
+ postcondition: raw.postcondition ?? { type: 'always' },
122
+ recovery: raw.recovery ?? { ...DEFAULT_RECOVERY_POLICY },
123
+ timeoutMs: raw.timeoutMs ?? 15000,
124
+ maxFailures: raw.maxFailures ?? 3,
125
+ // Type-specific defaults
126
+ ...(raw.kind === 'TYPE' && raw.clearFirst === undefined ? { clearFirst: true } : {}),
127
+ ...(raw.kind === 'ASSERT_SURFACE' && raw.matchAll === undefined ? { matchAll: true } : {}),
128
+ ...(raw.kind === 'WAIT_FOR' && !raw.state ? { state: 'visible' } : {}),
129
+ };
130
+ }
131
+ // ── Fingerprint ─────────────────────────────────────────────────────
132
+ function computeFingerprint(input) {
133
+ const content = JSON.stringify({
134
+ prompt: input.prompt,
135
+ baseUrl: input.baseUrl,
136
+ mediaMode: input.mediaMode,
137
+ navigation: input.navigationInstructions,
138
+ auth: input.preconditions.auth,
139
+ });
140
+ // Simple hash
141
+ let hash = 0;
142
+ for (let i = 0; i < content.length; i++) {
143
+ hash = ((hash << 5) - hash + content.charCodeAt(i)) | 0;
144
+ }
145
+ return Math.abs(hash).toString(36);
146
+ }
147
+ //# sourceMappingURL=compiler.js.map
@@ -0,0 +1,41 @@
1
+ /**
2
+ * V2 Capture Agent — Site Explorer
3
+ *
4
+ * Deterministic exploration of the target site to capture AKTree snapshots.
5
+ * These snapshots ground the compiler's opcode generation in real DOM structure,
6
+ * preventing hallucinated selectors.
7
+ *
8
+ * The explorer uses the Browser class directly — no LLM calls.
9
+ */
10
+ import type { Browser } from '../browser.js';
11
+ import type { AKTree } from '../types.js';
12
+ export interface ExplorationResult {
13
+ /** Serialized AKTree at the explored URL */
14
+ akTreeSerialized: string;
15
+ /** Raw AKTree object */
16
+ akTree: AKTree;
17
+ /** URL at time of exploration */
18
+ url: string;
19
+ /** Whether overlays were dismissed */
20
+ overlaysDismissed: boolean;
21
+ }
22
+ export interface ExploreOptions {
23
+ /** Target URL to explore */
24
+ url: string;
25
+ /** Dismiss overlays before capturing AKTree. Default: true */
26
+ dismissOverlays?: boolean;
27
+ /** Max time to wait for page load (ms). Default: 15000 */
28
+ loadTimeoutMs?: number;
29
+ }
30
+ /**
31
+ * Explores a single URL: navigates, dismisses overlays, and captures AKTree.
32
+ * Uses an existing Browser instance — does not create or close it.
33
+ */
34
+ export declare function explorePage(browser: Browser, options: ExploreOptions): Promise<ExplorationResult>;
35
+ /**
36
+ * Explores multiple URLs on the same site.
37
+ * Useful for presets that involve navigation across pages.
38
+ */
39
+ export declare function explorePages(browser: Browser, urls: string[], options?: {
40
+ dismissOverlays?: boolean;
41
+ }): Promise<ExplorationResult[]>;
@@ -0,0 +1,56 @@
1
+ /**
2
+ * V2 Capture Agent — Site Explorer
3
+ *
4
+ * Deterministic exploration of the target site to capture AKTree snapshots.
5
+ * These snapshots ground the compiler's opcode generation in real DOM structure,
6
+ * preventing hallucinated selectors.
7
+ *
8
+ * The explorer uses the Browser class directly — no LLM calls.
9
+ */
10
+ import { serializeAKTree } from '../ak-tree.js';
11
+ /**
12
+ * Explores a single URL: navigates, dismisses overlays, and captures AKTree.
13
+ * Uses an existing Browser instance — does not create or close it.
14
+ */
15
+ export async function explorePage(browser, options) {
16
+ const { url, dismissOverlays: shouldDismiss = true, loadTimeoutMs = 15000 } = options;
17
+ // Navigate to the target URL
18
+ await browser.navigateTo(url);
19
+ // Dismiss overlays if requested
20
+ let overlaysDismissed = false;
21
+ if (shouldDismiss) {
22
+ try {
23
+ const result = await browser.dismissOverlays();
24
+ overlaysDismissed = result.dismissed;
25
+ }
26
+ catch {
27
+ // Non-fatal: proceed without dismissal
28
+ }
29
+ }
30
+ // Capture AKTree
31
+ const akTree = await browser.getAKTree();
32
+ const akTreeSerialized = serializeAKTree(akTree);
33
+ const currentUrl = (await browser.currentPage).url();
34
+ return {
35
+ akTreeSerialized,
36
+ akTree,
37
+ url: currentUrl,
38
+ overlaysDismissed,
39
+ };
40
+ }
41
+ /**
42
+ * Explores multiple URLs on the same site.
43
+ * Useful for presets that involve navigation across pages.
44
+ */
45
+ export async function explorePages(browser, urls, options) {
46
+ const results = [];
47
+ for (const url of urls) {
48
+ const result = await explorePage(browser, {
49
+ url,
50
+ dismissOverlays: options?.dismissOverlays ?? true,
51
+ });
52
+ results.push(result);
53
+ }
54
+ return results;
55
+ }
56
+ //# sourceMappingURL=explorer.js.map
@@ -0,0 +1,37 @@
1
+ /**
2
+ * Capture Agent — Public API
3
+ *
4
+ * Architecture: deterministic opcode execution with selectors set by the user's AI assistant.
5
+ * AutoKap AI is limited to: overlay dismissal + capture verification.
6
+ */
7
+ export type { ExecutionProgram, ExecutionOpcode, OpcodeKind, SemanticTarget, PostconditionSpec, PostconditionType, RecoveryPolicy, VariantSpec, PreconditionSpec, ArtifactSpec, MediaMode, CircuitBreakerConfig, HealerPatch, OpcodeResult, OpcodeResultStatus, VariantResult, ArtifactResult, RunResult, RunTelemetry, LLMStepUsage, LLMStepType, RuntimeAdapter, ClickOptions, WaitCondition, RecordingOptions, RecordingResult, } from './types.js';
8
+ export { DEFAULT_RECOVERY_POLICY, DEFAULT_CIRCUIT_BREAKER } from './types.js';
9
+ export { ExecutionProgramSchema, ExecutionOpcodeSchema, parseProgram, parseOpcode, safeParseProgramResult, } from './schema.js';
10
+ export { WebPlaywrightLocal } from './web-playwright-local.js';
11
+ export { evaluatePostcondition } from './postcondition.js';
12
+ export { ActionVerifier } from './action-verifier.js';
13
+ export { CircuitBreaker } from './circuit-breaker.js';
14
+ export { executeProgram, NoOpRecoveryChain } from './opcode-runner.js';
15
+ export type { RecoveryChain, RecoveryAttemptResult, RunOptions, ProgressEvent } from './opcode-runner.js';
16
+ export { resolveSelector } from './selector-resolver.js';
17
+ export type { ResolvedSelector, SelectorResolverOptions } from './selector-resolver.js';
18
+ export { resolveTarget } from './semantic-resolver.js';
19
+ export type { ResolvedTarget, ResolveOptions } from './semantic-resolver.js';
20
+ export { RecoveryChainImpl } from './recovery-chain.js';
21
+ export { LLMHealer } from './llm-healer.js';
22
+ export type { HealerLLMProvider, HealerContext, HealerResult } from './llm-healer.js';
23
+ export { applyPatch, applyAllPatches, cloneProgram } from './program-patcher.js';
24
+ export { createCaptureStrategy, ScreenshotStrategy, ClipStrategy } from './capture-strategy.js';
25
+ export type { CaptureStrategy } from './capture-strategy.js';
26
+ export { dismissAllOverlays } from './overlay-engine.js';
27
+ export type { OverlayDismissResult } from './overlay-engine.js';
28
+ export { callLLM } from './llm-provider.js';
29
+ export type { LLMProviderConfig, LLMCallResult } from './llm-provider.js';
30
+ export { verifyCaptureQuality } from './capture-verification.js';
31
+ export type { VerificationResult, VerificationContext } from './capture-verification.js';
32
+ export { generateAltText } from './alt-text.js';
33
+ export type { AltTextResult, AltTextContext } from './alt-text.js';
34
+ export { smartWaitForStability } from './smart-wait.js';
35
+ export type { SmartWaitResult } from './smart-wait.js';
36
+ export { runBenchmark, formatBenchmarkSummary } from './benchmark.js';
37
+ export type { BenchmarkPreset, BenchmarkRunResult, BenchmarkSummary } from './benchmark.js';
@@ -0,0 +1,31 @@
1
+ /**
2
+ * Capture Agent — Public API
3
+ *
4
+ * Architecture: deterministic opcode execution with selectors set by the user's AI assistant.
5
+ * AutoKap AI is limited to: overlay dismissal + capture verification.
6
+ */
7
+ export { DEFAULT_RECOVERY_POLICY, DEFAULT_CIRCUIT_BREAKER } from './types.js';
8
+ export { ExecutionProgramSchema, ExecutionOpcodeSchema, parseProgram, parseOpcode, safeParseProgramResult, } from './schema.js';
9
+ // ── Runtime ─────────────────────────────────────────────────────────
10
+ export { WebPlaywrightLocal } from './web-playwright-local.js';
11
+ export { evaluatePostcondition } from './postcondition.js';
12
+ export { ActionVerifier } from './action-verifier.js';
13
+ export { CircuitBreaker } from './circuit-breaker.js';
14
+ export { executeProgram, NoOpRecoveryChain } from './opcode-runner.js';
15
+ // ── Recovery & Healing ──────────────────────────────────────────────
16
+ export { resolveSelector } from './selector-resolver.js';
17
+ export { resolveTarget } from './semantic-resolver.js';
18
+ export { RecoveryChainImpl } from './recovery-chain.js';
19
+ export { LLMHealer } from './llm-healer.js';
20
+ export { applyPatch, applyAllPatches, cloneProgram } from './program-patcher.js';
21
+ // ── Capture & Overlays ──────────────────────────────────────────────
22
+ export { createCaptureStrategy, ScreenshotStrategy, ClipStrategy } from './capture-strategy.js';
23
+ export { dismissAllOverlays } from './overlay-engine.js';
24
+ // ── AI Services (verification, alt text, smart wait) ────────────────
25
+ export { callLLM } from './llm-provider.js';
26
+ export { verifyCaptureQuality } from './capture-verification.js';
27
+ export { generateAltText } from './alt-text.js';
28
+ export { smartWaitForStability } from './smart-wait.js';
29
+ // ── Benchmark ───────────────────────────────────────────────────────
30
+ export { runBenchmark, formatBenchmarkSummary } from './benchmark.js';
31
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1,62 @@
1
+ /**
2
+ * Capture Agent — LLM Healer
3
+ *
4
+ * Last-resort recovery: when all deterministic strategies fail,
5
+ * the healer asks an LLM to analyze the current page state and
6
+ * produce a replacement opcode.
7
+ *
8
+ * Constraints:
9
+ * - Cannot change the preset's intention
10
+ * - Cannot change variant order
11
+ * - Cannot invent new capture targets
12
+ * - Cannot skip the failed opcode's postcondition
13
+ * - Max 1 LLM call per healing attempt
14
+ * - Max 3 healing attempts per run
15
+ */
16
+ import type { ExecutionOpcode, HealerPatch } from './types.js';
17
+ export interface HealerContext {
18
+ /** The failed opcode */
19
+ failedOpcode: ExecutionOpcode;
20
+ /** Index in the program */
21
+ opcodeIndex: number;
22
+ /** Serialized AKTree of the current page */
23
+ akTreeSerialized: string;
24
+ /** Current page URL */
25
+ currentUrl: string;
26
+ /** Screenshot buffer (for multimodal models) */
27
+ screenshot?: Buffer;
28
+ /** The 3 opcodes before and after the failed one (for context) */
29
+ surroundingOpcodes: ExecutionOpcode[];
30
+ /** Error message from the failed attempt */
31
+ errorMessage: string;
32
+ }
33
+ export interface HealerResult {
34
+ healed: boolean;
35
+ patch?: HealerPatch;
36
+ reason: string;
37
+ /** LLM cost for this healing attempt */
38
+ costEur?: number;
39
+ }
40
+ export interface HealerLLMProvider {
41
+ /**
42
+ * Call the LLM with a healing prompt. Returns raw JSON string.
43
+ * The provider handles model selection, API keys, etc.
44
+ */
45
+ call(prompt: string, screenshot?: Buffer): Promise<{
46
+ response: string;
47
+ costEur: number;
48
+ }>;
49
+ }
50
+ /**
51
+ * The LLM Healer — a constrained agent that repairs failed opcodes.
52
+ */
53
+ export declare class LLMHealer {
54
+ private llmProvider;
55
+ private invocationCount;
56
+ private maxInvocations;
57
+ constructor(llmProvider: HealerLLMProvider, options?: {
58
+ maxInvocations?: number;
59
+ });
60
+ get remainingInvocations(): number;
61
+ heal(context: HealerContext): Promise<HealerResult>;
62
+ }
@@ -0,0 +1,166 @@
1
+ /**
2
+ * Capture Agent — LLM Healer
3
+ *
4
+ * Last-resort recovery: when all deterministic strategies fail,
5
+ * the healer asks an LLM to analyze the current page state and
6
+ * produce a replacement opcode.
7
+ *
8
+ * Constraints:
9
+ * - Cannot change the preset's intention
10
+ * - Cannot change variant order
11
+ * - Cannot invent new capture targets
12
+ * - Cannot skip the failed opcode's postcondition
13
+ * - Max 1 LLM call per healing attempt
14
+ * - Max 3 healing attempts per run
15
+ */
16
+ import { ExecutionOpcodeSchema } from './schema.js';
17
+ /**
18
+ * The LLM Healer — a constrained agent that repairs failed opcodes.
19
+ */
20
+ export class LLMHealer {
21
+ llmProvider;
22
+ invocationCount = 0;
23
+ maxInvocations;
24
+ constructor(llmProvider, options) {
25
+ this.llmProvider = llmProvider;
26
+ this.maxInvocations = options?.maxInvocations ?? 3;
27
+ }
28
+ get remainingInvocations() {
29
+ return Math.max(0, this.maxInvocations - this.invocationCount);
30
+ }
31
+ async heal(context) {
32
+ if (this.invocationCount >= this.maxInvocations) {
33
+ return { healed: false, reason: `healer budget exhausted (${this.maxInvocations}/${this.maxInvocations} used)` };
34
+ }
35
+ this.invocationCount++;
36
+ try {
37
+ const prompt = buildHealerPrompt(context);
38
+ const { response, costEur } = await this.llmProvider.call(prompt, context.screenshot);
39
+ // Parse the LLM response
40
+ const parsed = parseHealerResponse(response, context);
41
+ if (!parsed) {
42
+ return { healed: false, reason: 'healer produced invalid response', costEur };
43
+ }
44
+ if (parsed.cannotHeal) {
45
+ return { healed: false, reason: parsed.reason ?? 'healer cannot resolve this failure', costEur };
46
+ }
47
+ // Validate replacement opcodes against schema
48
+ const validOpcodes = [];
49
+ for (const opcode of parsed.replacementOpcodes) {
50
+ const result = ExecutionOpcodeSchema.safeParse(opcode);
51
+ if (!result.success) {
52
+ return { healed: false, reason: `healer produced invalid opcode: ${result.error}`, costEur };
53
+ }
54
+ validOpcodes.push(result.data);
55
+ }
56
+ if (validOpcodes.length === 0 || validOpcodes.length > 3) {
57
+ return { healed: false, reason: `healer produced ${validOpcodes.length} opcodes (expected 1-3)`, costEur };
58
+ }
59
+ const patch = {
60
+ opcodeIndex: context.opcodeIndex,
61
+ originalOpcode: context.failedOpcode,
62
+ replacementOpcodes: validOpcodes,
63
+ reason: parsed.reason ?? 'healer repair',
64
+ patchedAt: new Date().toISOString(),
65
+ };
66
+ return { healed: true, patch, reason: parsed.reason ?? 'healer repair applied', costEur };
67
+ }
68
+ catch (err) {
69
+ return {
70
+ healed: false,
71
+ reason: `healer error: ${err instanceof Error ? err.message : String(err)}`,
72
+ };
73
+ }
74
+ }
75
+ }
76
+ // ── Prompt construction ─────────────────────────────────────────────
77
+ function buildHealerPrompt(context) {
78
+ const { failedOpcode, errorMessage, akTreeSerialized, currentUrl, surroundingOpcodes } = context;
79
+ const surroundingDesc = surroundingOpcodes
80
+ .map((op, i) => ` ${i + 1}. ${op.kind}: ${op.description}`)
81
+ .join('\n');
82
+ return `You are a web automation repair agent. An opcode in a deterministic browser automation program has failed. Your job is to analyze the current page state and produce a replacement opcode that achieves the same goal.
83
+
84
+ ## Failed Opcode
85
+ Kind: ${failedOpcode.kind}
86
+ Description: ${failedOpcode.description}
87
+ ${failedOpcode.kind === 'CLICK' ? `Selector: ${failedOpcode.selector}` : ''}
88
+ ${failedOpcode.kind === 'TYPE' ? `Selector: ${failedOpcode.selector}, Text: ${failedOpcode.text}` : ''}
89
+ ${failedOpcode.kind === 'WAIT_FOR' ? `Selector: ${failedOpcode.selector}` : ''}
90
+
91
+ ## Error
92
+ ${errorMessage}
93
+
94
+ ## Current Page
95
+ URL: ${currentUrl}
96
+
97
+ ## AKTree (current page structure)
98
+ ${akTreeSerialized.slice(0, 8000)}
99
+
100
+ ## Surrounding Opcodes (context)
101
+ ${surroundingDesc}
102
+
103
+ ## Rules
104
+ 1. You MUST produce the same kind of opcode (${failedOpcode.kind}) or a sequence of 1-3 opcodes that achieve the same postcondition.
105
+ 2. You CANNOT change the intention — only fix the selector or interaction method.
106
+ 3. You CANNOT invent new navigation targets or capture actions.
107
+ 4. Every opcode must have a valid postcondition.
108
+ 5. Use selectors you can actually see in the AKTree above.
109
+
110
+ ## Response Format
111
+ Respond with a JSON object:
112
+ {
113
+ "cannotHeal": false,
114
+ "reason": "explanation of what you changed",
115
+ "replacementOpcodes": [
116
+ { "kind": "${failedOpcode.kind}", "description": "...", "selector": "...", "postcondition": {...}, "recovery": {...}, "timeoutMs": 15000, "maxFailures": 3, ... }
117
+ ]
118
+ }
119
+
120
+ Or if you cannot fix it:
121
+ {
122
+ "cannotHeal": true,
123
+ "reason": "explanation of why this cannot be fixed"
124
+ }
125
+
126
+ Respond ONLY with the JSON object, no markdown, no explanation outside JSON.`;
127
+ }
128
+ function parseHealerResponse(response, context) {
129
+ try {
130
+ // Extract JSON from response (handle markdown code blocks)
131
+ const jsonStr = response
132
+ .replace(/```json\s*/g, '')
133
+ .replace(/```\s*/g, '')
134
+ .trim();
135
+ const parsed = JSON.parse(jsonStr);
136
+ if (typeof parsed !== 'object' || parsed === null)
137
+ return null;
138
+ if (parsed.cannotHeal) {
139
+ return {
140
+ cannotHeal: true,
141
+ reason: parsed.reason ?? null,
142
+ replacementOpcodes: [],
143
+ };
144
+ }
145
+ if (!Array.isArray(parsed.replacementOpcodes))
146
+ return null;
147
+ // Inject the original postcondition and recovery into replacement opcodes
148
+ // (healer cannot skip postconditions)
149
+ const enriched = parsed.replacementOpcodes.map((op) => ({
150
+ ...op,
151
+ postcondition: op.postcondition ?? context.failedOpcode.postcondition,
152
+ recovery: op.recovery ?? context.failedOpcode.recovery,
153
+ timeoutMs: op.timeoutMs ?? context.failedOpcode.timeoutMs,
154
+ maxFailures: op.maxFailures ?? context.failedOpcode.maxFailures,
155
+ }));
156
+ return {
157
+ cannotHeal: false,
158
+ reason: parsed.reason ?? null,
159
+ replacementOpcodes: enriched,
160
+ };
161
+ }
162
+ catch {
163
+ return null;
164
+ }
165
+ }
166
+ //# sourceMappingURL=llm-healer.js.map
@@ -0,0 +1,29 @@
1
+ /**
2
+ * Capture Agent — LLM Provider
3
+ *
4
+ * Thin wrapper around OpenAI SDK for OpenRouter calls.
5
+ * Used by: capture verification, alt text generation, healer.
6
+ */
7
+ export interface LLMCallResult {
8
+ text: string;
9
+ model: string;
10
+ promptTokens: number;
11
+ completionTokens: number;
12
+ costEur: number;
13
+ /** OpenRouter generation ID for async cost resolution */
14
+ generationId: string | null;
15
+ }
16
+ export interface LLMProviderConfig {
17
+ apiKey: string;
18
+ /** Model to use. Default: google/gemini-2.5-flash */
19
+ model?: string;
20
+ /** Max tokens for response. Default: 200 */
21
+ maxTokens?: number;
22
+ /** Timeout in ms. Default: 15000 */
23
+ timeoutMs?: number;
24
+ }
25
+ /**
26
+ * Makes a single LLM call with optional image input.
27
+ * Returns the text response and usage metadata.
28
+ */
29
+ export declare function callLLM(config: LLMProviderConfig, systemPrompt: string, userContent: string, image?: Buffer): Promise<LLMCallResult>;
@@ -0,0 +1,80 @@
1
+ /**
2
+ * Capture Agent — LLM Provider
3
+ *
4
+ * Thin wrapper around OpenAI SDK for OpenRouter calls.
5
+ * Used by: capture verification, alt text generation, healer.
6
+ */
7
+ import OpenAI from 'openai';
8
+ import { createOpenRouterCompletion } from '../openrouter-client.js';
9
+ const OPENROUTER_BASE_URL = 'https://openrouter.ai/api/v1';
10
+ const DEFAULT_MODEL = 'google/gemini-2.5-flash';
11
+ /**
12
+ * Makes a single LLM call with optional image input.
13
+ * Returns the text response and usage metadata.
14
+ */
15
+ export async function callLLM(config, systemPrompt, userContent, image) {
16
+ const client = new OpenAI({
17
+ baseURL: OPENROUTER_BASE_URL,
18
+ apiKey: config.apiKey,
19
+ });
20
+ const model = config.model ?? DEFAULT_MODEL;
21
+ const maxTokens = config.maxTokens ?? 200;
22
+ const messages = [
23
+ { role: 'system', content: systemPrompt },
24
+ ];
25
+ if (image) {
26
+ const base64 = image.toString('base64');
27
+ messages.push({
28
+ role: 'user',
29
+ content: [
30
+ { type: 'text', text: userContent },
31
+ {
32
+ type: 'image_url',
33
+ image_url: { url: `data:image/png;base64,${base64}`, detail: 'low' },
34
+ },
35
+ ],
36
+ });
37
+ }
38
+ else {
39
+ messages.push({ role: 'user', content: userContent });
40
+ }
41
+ const params = {
42
+ model,
43
+ messages,
44
+ max_tokens: maxTokens,
45
+ temperature: 0.1,
46
+ };
47
+ const controller = new AbortController();
48
+ const timer = setTimeout(() => controller.abort(), config.timeoutMs ?? 15000);
49
+ try {
50
+ const response = await createOpenRouterCompletion(client, params, { signal: controller.signal });
51
+ clearTimeout(timer);
52
+ const text = response.choices?.[0]?.message?.content?.trim() ?? '';
53
+ const usage = response.usage;
54
+ return {
55
+ text,
56
+ model: response.model ?? model,
57
+ promptTokens: usage?.prompt_tokens ?? 0,
58
+ completionTokens: usage?.completion_tokens ?? 0,
59
+ costEur: estimateCost(model, usage?.prompt_tokens ?? 0, usage?.completion_tokens ?? 0),
60
+ generationId: response.id ?? null,
61
+ };
62
+ }
63
+ catch (err) {
64
+ clearTimeout(timer);
65
+ throw err;
66
+ }
67
+ }
68
+ /**
69
+ * Rough cost estimation based on known model pricing.
70
+ * Actual cost is resolved asynchronously via OpenRouter generation API.
71
+ */
72
+ function estimateCost(model, promptTokens, completionTokens) {
73
+ // Gemini 2.5 Flash pricing (approximate)
74
+ if (model.includes('gemini')) {
75
+ return (promptTokens * 0.15 + completionTokens * 0.6) / 1_000_000;
76
+ }
77
+ // Fallback: conservative estimate
78
+ return (promptTokens * 0.5 + completionTokens * 1.5) / 1_000_000;
79
+ }
80
+ //# sourceMappingURL=llm-provider.js.map