autokap 1.0.6 → 1.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/chrome/ios-statusbar-comparison-reference.jpg +0 -0
- package/assets/chrome/ios-statusbar-dark-reference.jpg +0 -0
- package/assets/chrome/ios-statusbar-light-reference.jpg +0 -0
- package/assets/cursors/macos.svg +4 -0
- package/assets/cursors/windows.svg +15 -0
- package/assets/devices/ipad-pro-11-m4.json +52 -0
- package/assets/devices/iphone-16-pro.json +53 -0
- package/assets/devices/macbook-air-13.json +45 -0
- package/assets/frames/MacBook Air 13.svg +242 -0
- package/assets/frames/Status bar - iPhone.png +0 -0
- Menu bar- iPad.png +0 -0
- package/assets/frames/iPad Pro M4 11_.png +0 -0
- package/assets/frames/iPhone 16 Pro.png +0 -0
- package/assets/icons/Cellular Connection.svg +3 -0
- package/assets/icons/Union.svg +6 -0
- package/assets/icons/Wifi.svg +3 -0
- package/assets/icons/battery.svg +5 -0
- package/assets/icons/battery_charging.svg +8 -0
- package/assets/skill/OPCODE-REFERENCE.md +607 -0
- package/assets/skill/README.md +39 -0
- package/assets/skill/SKILL.md +453 -468
- package/assets/skill/STUDIO-SKILL.md +476 -0
- package/assets/skill/references/examples.md +104 -0
- package/assets/skill/references/interactive-demo.md +225 -0
- package/assets/skill/references/mock-data.md +178 -0
- package/dist/abort.d.ts +5 -0
- package/dist/abort.js +44 -0
- package/dist/action-verifier.d.ts +29 -0
- package/dist/action-verifier.js +133 -0
- package/dist/agent-action-recovery.d.ts +45 -0
- package/dist/agent-action-recovery.js +370 -0
- package/dist/agent-message-utils.d.ts +21 -0
- package/dist/agent-message-utils.js +77 -0
- package/dist/agent-url-utils.d.ts +30 -0
- package/dist/agent-url-utils.js +138 -0
- package/dist/agent.d.ts +226 -0
- package/dist/agent.js +6666 -0
- package/dist/ak-tree.d.ts +39 -0
- package/dist/ak-tree.js +368 -0
- package/dist/alt-text.d.ts +26 -0
- package/dist/alt-text.js +55 -0
- package/dist/auth-capture.d.ts +17 -0
- package/dist/auth-capture.js +164 -0
- package/dist/benchmark.d.ts +59 -0
- package/dist/benchmark.js +135 -0
- package/dist/billing-operation-logging.d.ts +38 -0
- package/dist/billing-operation-logging.js +248 -0
- package/dist/browser-bar.d.ts +48 -0
- package/dist/browser-bar.js +284 -0
- package/dist/browser-pool.d.ts +7 -0
- package/dist/browser-pool.js +15 -5
- package/dist/browser-utils.d.ts +31 -0
- package/dist/browser-utils.js +97 -0
- package/dist/browser.d.ts +76 -1
- package/dist/browser.js +1657 -39
- package/dist/capture-alt-text.d.ts +12 -0
- package/dist/capture-alt-text.js +52 -0
- package/dist/capture-encryption.d.ts +10 -0
- package/dist/capture-encryption.js +41 -0
- package/dist/capture-language-preflight.d.ts +41 -0
- package/dist/capture-language-preflight.js +300 -0
- package/dist/capture-llm-page-identity.d.ts +15 -0
- package/dist/capture-llm-page-identity.js +128 -0
- package/dist/capture-model-resolution.d.ts +9 -0
- package/dist/capture-model-resolution.js +21 -0
- package/dist/capture-page-identity.d.ts +7 -0
- package/dist/capture-page-identity.js +352 -0
- package/dist/capture-preset-credentials.d.ts +62 -0
- package/dist/capture-preset-credentials.js +184 -0
- package/dist/capture-request-plan.d.ts +58 -0
- package/dist/capture-request-plan.js +264 -0
- package/dist/capture-run-optimizer.d.ts +139 -0
- package/dist/capture-run-optimizer.js +863 -0
- package/dist/capture-selector-memory.d.ts +31 -0
- package/dist/capture-selector-memory.js +345 -0
- package/dist/capture-session-profile-encryption.d.ts +2 -0
- package/dist/capture-session-profile-encryption.js +22 -0
- package/dist/capture-step-timeout.d.ts +10 -0
- package/dist/capture-step-timeout.js +30 -0
- package/dist/capture-strategy.d.ts +36 -0
- package/dist/capture-strategy.js +95 -0
- package/dist/capture-studio-sync.d.ts +23 -0
- package/dist/capture-studio-sync.js +172 -0
- package/dist/capture-surface-contract.d.ts +36 -0
- package/dist/capture-surface-contract.js +299 -0
- package/dist/capture-transition-engine.d.ts +28 -0
- package/dist/capture-transition-engine.js +292 -0
- package/dist/capture-variant-state.d.ts +56 -0
- package/dist/capture-variant-state.js +182 -0
- package/dist/capture-verification.d.ts +35 -0
- package/dist/capture-verification.js +95 -0
- package/dist/capture-viewport-lock.d.ts +48 -0
- package/dist/capture-viewport-lock.js +74 -0
- package/dist/circuit-breaker.d.ts +42 -0
- package/dist/circuit-breaker.js +119 -0
- package/dist/cli-config.d.ts +8 -1
- package/dist/cli-config.js +62 -6
- package/dist/cli-contract.d.ts +15 -0
- package/dist/cli-contract.js +167 -0
- package/dist/cli-runner-local.d.ts +12 -0
- package/dist/cli-runner-local.js +102 -0
- package/dist/cli-runner.d.ts +34 -0
- package/dist/cli-runner.js +433 -0
- package/dist/cli-utils.d.ts +0 -1
- package/dist/cli-utils.js +2 -5
- package/dist/cli.js +1005 -252
- package/dist/clip-orchestrator.d.ts +148 -0
- package/dist/clip-orchestrator.js +957 -0
- package/dist/clip-postprocess.d.ts +42 -0
- package/dist/clip-postprocess.js +201 -0
- package/dist/cookie-dismiss.d.ts +2 -0
- package/dist/cookie-dismiss.js +48 -13
- package/dist/cost-logging.d.ts +35 -0
- package/dist/cost-logging.js +242 -0
- package/dist/cost-resolution-monitor.d.ts +16 -0
- package/dist/cost-resolution-monitor.js +34 -0
- package/dist/credential-templates.d.ts +5 -0
- package/dist/credential-templates.js +60 -0
- package/dist/cursor-overlay-script.d.ts +6 -0
- package/dist/cursor-overlay-script.js +169 -0
- package/dist/dom-css-purger.d.ts +65 -0
- package/dist/dom-css-purger.js +333 -0
- package/dist/dom-font-inliner.d.ts +45 -0
- package/dist/dom-font-inliner.js +148 -0
- package/dist/dom-patch-resolver.d.ts +52 -0
- package/dist/dom-patch-resolver.js +242 -0
- package/dist/dom-serializer.d.ts +82 -0
- package/dist/dom-serializer.js +378 -0
- package/dist/element-capture.d.ts +13 -0
- package/dist/element-capture.js +522 -0
- package/dist/env-validation.d.ts +5 -0
- package/dist/env-validation.js +29 -0
- package/dist/execution-schema.d.ts +4423 -0
- package/dist/execution-schema.js +507 -0
- package/dist/execution-types.d.ts +886 -0
- package/dist/execution-types.js +65 -0
- package/dist/fonts-loader.d.ts +14 -0
- package/dist/fonts-loader.js +55 -0
- package/dist/hybrid-navigator.d.ts +138 -0
- package/dist/hybrid-navigator.js +468 -0
- package/dist/index.d.ts +18 -0
- package/dist/index.js +17 -0
- package/dist/legacy/agent-action-recovery.d.ts +45 -0
- package/dist/legacy/agent-action-recovery.js +370 -0
- package/dist/legacy/agent-message-utils.d.ts +21 -0
- package/dist/legacy/agent-message-utils.js +77 -0
- package/dist/legacy/agent-url-utils.d.ts +30 -0
- package/dist/legacy/agent-url-utils.js +138 -0
- package/dist/legacy/agent.d.ts +226 -0
- package/dist/legacy/agent.js +6666 -0
- package/dist/legacy/clip-orchestrator.d.ts +148 -0
- package/dist/legacy/clip-orchestrator.js +957 -0
- package/dist/legacy/credential-templates.d.ts +5 -0
- package/dist/legacy/credential-templates.js +60 -0
- package/dist/legacy/hybrid-navigator.d.ts +138 -0
- package/dist/legacy/hybrid-navigator.js +468 -0
- package/dist/legacy/llm-usage.d.ts +17 -0
- package/dist/legacy/llm-usage.js +45 -0
- package/dist/legacy/prompt-cache.d.ts +10 -0
- package/dist/legacy/prompt-cache.js +24 -0
- package/dist/legacy/prompts.d.ts +175 -0
- package/dist/legacy/prompts.js +1038 -0
- package/dist/legacy/tools.d.ts +4 -0
- package/dist/legacy/tools.js +216 -0
- package/dist/legacy/video-agent.d.ts +143 -0
- package/dist/legacy/video-agent.js +4788 -0
- package/dist/legacy/video-observation.d.ts +36 -0
- package/dist/legacy/video-observation.js +192 -0
- package/dist/legacy/video-planner.d.ts +12 -0
- package/dist/legacy/video-planner.js +501 -0
- package/dist/legacy/video-prompts.d.ts +37 -0
- package/dist/legacy/video-prompts.js +569 -0
- package/dist/legacy/video-tools.d.ts +3 -0
- package/dist/legacy/video-tools.js +59 -0
- package/dist/legacy/video-variant-state.d.ts +29 -0
- package/dist/legacy/video-variant-state.js +80 -0
- package/dist/legacy/vision-model.d.ts +17 -0
- package/dist/legacy/vision-model.js +74 -0
- package/dist/llm-healer.d.ts +63 -0
- package/dist/llm-healer.js +166 -0
- package/dist/llm-provider.d.ts +29 -0
- package/dist/llm-provider.js +80 -0
- package/dist/llm-usage.d.ts +17 -0
- package/dist/llm-usage.js +45 -0
- package/dist/logger.d.ts +6 -2
- package/dist/logger.js +15 -1
- package/dist/mockup-html.d.ts +119 -0
- package/dist/mockup-html.js +263 -0
- package/dist/mockup.d.ts +187 -0
- package/dist/mockup.js +869 -0
- package/dist/mouse-animation.d.ts +46 -0
- package/dist/mouse-animation.js +114 -0
- package/dist/opcode-actions.d.ts +42 -0
- package/dist/opcode-actions.js +511 -0
- package/dist/opcode-runner.d.ts +51 -0
- package/dist/opcode-runner.js +770 -0
- package/dist/openrouter-client.d.ts +40 -0
- package/dist/openrouter-client.js +16 -0
- package/dist/overlay-engine.d.ts +24 -0
- package/dist/overlay-engine.js +176 -0
- package/dist/overlay-utils.d.ts +14 -0
- package/dist/overlay-utils.js +13 -0
- package/dist/postcondition.d.ts +16 -0
- package/dist/postcondition.js +269 -0
- package/dist/posthog.d.ts +4 -0
- package/dist/posthog.js +26 -0
- package/dist/program-patcher.d.ts +25 -0
- package/dist/program-patcher.js +44 -0
- package/dist/prompt-cache.d.ts +10 -0
- package/dist/prompt-cache.js +24 -0
- package/dist/prompts.d.ts +175 -0
- package/dist/prompts.js +1038 -0
- package/dist/provider-config.d.ts +12 -0
- package/dist/provider-config.js +15 -0
- package/dist/recovery-chain.d.ts +37 -0
- package/dist/recovery-chain.js +350 -0
- package/dist/remote-browser.d.ts +215 -0
- package/dist/remote-browser.js +360 -0
- package/dist/safari-browser-bar.d.ts +15 -0
- package/dist/safari-browser-bar.js +95 -0
- package/dist/safari-toolbar-asset.d.ts +15 -0
- package/dist/safari-toolbar-asset.js +12 -0
- package/dist/security.d.ts +21 -0
- package/dist/security.js +608 -0
- package/dist/selector-resolver.d.ts +34 -0
- package/dist/selector-resolver.js +181 -0
- package/dist/semantic-resolver.d.ts +35 -0
- package/dist/semantic-resolver.js +161 -0
- package/dist/server-capture-runtime.d.ts +125 -0
- package/dist/server-capture-runtime.js +585 -0
- package/dist/server-credit-usage.d.ts +12 -0
- package/dist/server-credit-usage.js +41 -0
- package/dist/server-posthog.d.ts +2 -0
- package/dist/server-posthog.js +16 -0
- package/dist/server-project-webhooks.d.ts +59 -0
- package/dist/server-project-webhooks.js +123 -0
- package/dist/server-screenshot-watermark.d.ts +7 -0
- package/dist/server-screenshot-watermark.js +60 -0
- package/dist/session-profile.d.ts +86 -0
- package/dist/session-profile.js +1536 -0
- package/dist/sf-pro-fonts.d.ts +4 -0
- package/dist/sf-pro-fonts.js +7 -0
- package/dist/sf-pro-symbols.d.ts +1 -0
- package/dist/sf-pro-symbols.js +55 -0
- package/dist/skill-packaging.d.ts +28 -0
- package/dist/skill-packaging.js +169 -0
- package/dist/smart-wait.d.ts +27 -0
- package/dist/smart-wait.js +81 -0
- package/dist/status-bar-l10n.d.ts +14 -0
- package/dist/status-bar-l10n.js +177 -0
- package/dist/status-bar-render.d.ts +20 -0
- package/dist/status-bar-render.js +410 -0
- package/dist/status-bar.d.ts +53 -0
- package/dist/status-bar.js +620 -0
- package/dist/svg-browser-bar.d.ts +33 -0
- package/dist/svg-browser-bar.js +206 -0
- package/dist/svg-status-bar.d.ts +36 -0
- package/dist/svg-status-bar.js +597 -0
- package/dist/svg-text.d.ts +61 -0
- package/dist/svg-text.js +118 -0
- package/dist/tools.d.ts +4 -0
- package/dist/tools.js +216 -0
- package/dist/types.d.ts +240 -5
- package/dist/types.js +23 -1
- package/dist/v2/action-verifier.d.ts +29 -0
- package/dist/v2/action-verifier.js +133 -0
- package/dist/v2/alt-text.d.ts +26 -0
- package/dist/v2/alt-text.js +55 -0
- package/dist/v2/benchmark.d.ts +59 -0
- package/dist/v2/benchmark.js +135 -0
- package/dist/v2/capture-strategy.d.ts +30 -0
- package/dist/v2/capture-strategy.js +67 -0
- package/dist/v2/capture-verification.d.ts +35 -0
- package/dist/v2/capture-verification.js +95 -0
- package/dist/v2/circuit-breaker.d.ts +42 -0
- package/dist/v2/circuit-breaker.js +119 -0
- package/dist/v2/cli-runner-local.d.ts +11 -0
- package/dist/v2/cli-runner-local.js +91 -0
- package/dist/v2/cli-runner.d.ts +34 -0
- package/dist/v2/cli-runner.js +300 -0
- package/dist/v2/compiler-prompts.d.ts +27 -0
- package/dist/v2/compiler-prompts.js +123 -0
- package/dist/v2/compiler.d.ts +37 -0
- package/dist/v2/compiler.js +147 -0
- package/dist/v2/explorer.d.ts +41 -0
- package/dist/v2/explorer.js +56 -0
- package/dist/v2/index.d.ts +37 -0
- package/dist/v2/index.js +31 -0
- package/dist/v2/llm-healer.d.ts +62 -0
- package/dist/v2/llm-healer.js +166 -0
- package/dist/v2/llm-provider.d.ts +29 -0
- package/dist/v2/llm-provider.js +80 -0
- package/dist/v2/opcode-runner.d.ts +47 -0
- package/dist/v2/opcode-runner.js +634 -0
- package/dist/v2/overlay-engine.d.ts +24 -0
- package/dist/v2/overlay-engine.js +150 -0
- package/dist/v2/postcondition.d.ts +16 -0
- package/dist/v2/postcondition.js +249 -0
- package/dist/v2/program-patcher.d.ts +25 -0
- package/dist/v2/program-patcher.js +44 -0
- package/dist/v2/recovery-chain.d.ts +30 -0
- package/dist/v2/recovery-chain.js +368 -0
- package/dist/v2/schema.d.ts +2580 -0
- package/dist/v2/schema.js +295 -0
- package/dist/v2/selector-resolver.d.ts +34 -0
- package/dist/v2/selector-resolver.js +181 -0
- package/dist/v2/semantic-resolver.d.ts +35 -0
- package/dist/v2/semantic-resolver.js +161 -0
- package/dist/v2/smart-wait.d.ts +27 -0
- package/dist/v2/smart-wait.js +81 -0
- package/dist/v2/types.d.ts +444 -0
- package/dist/v2/types.js +19 -0
- package/dist/v2/web-playwright-local.d.ts +69 -0
- package/dist/v2/web-playwright-local.js +392 -0
- package/dist/version.d.ts +1 -0
- package/dist/version.js +5 -0
- package/dist/video-agent.d.ts +143 -0
- package/dist/video-agent.js +4788 -0
- package/dist/video-observation.d.ts +36 -0
- package/dist/video-observation.js +192 -0
- package/dist/video-planner.d.ts +12 -0
- package/dist/video-planner.js +501 -0
- package/dist/video-prompts.d.ts +37 -0
- package/dist/video-prompts.js +554 -0
- package/dist/video-tools.d.ts +3 -0
- package/dist/video-tools.js +59 -0
- package/dist/video-variant-state.d.ts +29 -0
- package/dist/video-variant-state.js +80 -0
- package/dist/vision-model.d.ts +17 -0
- package/dist/vision-model.js +74 -0
- package/dist/web-playwright-local.d.ts +126 -0
- package/dist/web-playwright-local.js +819 -0
- package/dist/ws-auth.d.ts +20 -0
- package/dist/ws-auth.js +70 -0
- package/dist/ws-broadcast.d.ts +34 -0
- package/dist/ws-broadcast.js +85 -0
- package/dist/ws-connection-limits.d.ts +12 -0
- package/dist/ws-connection-limits.js +44 -0
- package/dist/ws-handler-utils.d.ts +32 -0
- package/dist/ws-handler-utils.js +139 -0
- package/dist/ws-handler.d.ts +10 -0
- package/dist/ws-handler.js +1793 -0
- package/dist/ws-metrics-server.d.ts +9 -0
- package/dist/ws-metrics-server.js +31 -0
- package/dist/ws-server.d.ts +9 -0
- package/dist/ws-server.js +92 -0
- package/package.json +142 -71
|
@@ -0,0 +1,300 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Capture Agent — CLI Runner
|
|
3
|
+
*
|
|
4
|
+
* Entry point for V2 execution from the CLI.
|
|
5
|
+
* Flow:
|
|
6
|
+
* 1. Authenticate via stored API key
|
|
7
|
+
* 2. Fetch compiled program from server
|
|
8
|
+
* 3. Validate program against Zod schema
|
|
9
|
+
* 4. Launch local Playwright browser per variant
|
|
10
|
+
* 5. Execute program via opcode-runner
|
|
11
|
+
* 6. Upload artifacts + telemetry to server
|
|
12
|
+
*/
|
|
13
|
+
import fs from 'node:fs/promises';
|
|
14
|
+
import os from 'node:os';
|
|
15
|
+
import path from 'node:path';
|
|
16
|
+
import { randomUUID } from 'node:crypto';
|
|
17
|
+
import { Browser } from '../browser.js';
|
|
18
|
+
import { requireConfig } from '../cli-config.js';
|
|
19
|
+
import { WebPlaywrightLocal } from './web-playwright-local.js';
|
|
20
|
+
import { executeProgram } from './opcode-runner.js';
|
|
21
|
+
import { RecoveryChainImpl } from './recovery-chain.js';
|
|
22
|
+
import { parseProgram } from './schema.js';
|
|
23
|
+
import { buildCursorOverlayScript } from '../legacy/video-prompts.js';
|
|
24
|
+
import { logger } from '../logger.js';
|
|
25
|
+
// ── Main entry point ────────────────────────────────────────────────
|
|
26
|
+
export async function runCapture(options) {
|
|
27
|
+
const config = await requireConfig();
|
|
28
|
+
// Step 1: Get the compiled program
|
|
29
|
+
let program;
|
|
30
|
+
if (options.program) {
|
|
31
|
+
program = options.program;
|
|
32
|
+
}
|
|
33
|
+
else {
|
|
34
|
+
const fetched = await fetchProgram(config, options.presetId);
|
|
35
|
+
if (!fetched.success) {
|
|
36
|
+
return { success: false, error: fetched.error };
|
|
37
|
+
}
|
|
38
|
+
program = fetched.program;
|
|
39
|
+
}
|
|
40
|
+
// Step 2: Validate the program
|
|
41
|
+
try {
|
|
42
|
+
program = parseProgram(program);
|
|
43
|
+
}
|
|
44
|
+
catch (err) {
|
|
45
|
+
return { success: false, error: `program validation failed: ${err instanceof Error ? err.message : String(err)}` };
|
|
46
|
+
}
|
|
47
|
+
// Persist against the canonical preset identifier selected by the CLI,
|
|
48
|
+
// even if the compiled program carries a human-readable preset slug/name.
|
|
49
|
+
if (program.presetId !== options.presetId) {
|
|
50
|
+
program = {
|
|
51
|
+
...program,
|
|
52
|
+
presetId: options.presetId,
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
logger.info(`[capture] Running preset "${options.presetId}" — ${program.steps.length} opcodes, ${program.variants.length} variant(s)`);
|
|
56
|
+
// Step 3: Set up recovery chain
|
|
57
|
+
const recoveryChain = new RecoveryChainImpl({
|
|
58
|
+
selectorMemory: options.selectorMemory,
|
|
59
|
+
programSteps: program.steps,
|
|
60
|
+
// LLM provider will be wired in when the healer is integrated with OpenRouter
|
|
61
|
+
});
|
|
62
|
+
const llmConfig = resolveCliLLMConfig();
|
|
63
|
+
// Step 4: Execute the program
|
|
64
|
+
const runOptions = {
|
|
65
|
+
recoveryChain,
|
|
66
|
+
abortSignal: options.abortSignal,
|
|
67
|
+
llmConfig,
|
|
68
|
+
presetName: program.presetId,
|
|
69
|
+
onProgress: (event) => {
|
|
70
|
+
logProgress(event);
|
|
71
|
+
options.onProgress?.(event);
|
|
72
|
+
},
|
|
73
|
+
};
|
|
74
|
+
const createAdapter = async (variant) => {
|
|
75
|
+
const browserOptions = {
|
|
76
|
+
headed: options.headed ?? false,
|
|
77
|
+
viewport: variant.viewport,
|
|
78
|
+
deviceScaleFactor: variant.deviceScaleFactor,
|
|
79
|
+
lang: variant.locale,
|
|
80
|
+
colorScheme: variant.theme,
|
|
81
|
+
storageState: program.preconditions.storageState,
|
|
82
|
+
};
|
|
83
|
+
const recordable = program.mediaMode === 'clip';
|
|
84
|
+
let recordingDir;
|
|
85
|
+
let browser;
|
|
86
|
+
if (recordable) {
|
|
87
|
+
recordingDir = await fs.mkdtemp(path.join(os.tmpdir(), `autokap-${program.mediaMode}-`));
|
|
88
|
+
browser = await Browser.forVideoRecording(browserOptions, recordingDir, buildCursorOverlayScript(program.artifactPlan.cursorTheme ?? 'minimal'));
|
|
89
|
+
}
|
|
90
|
+
else {
|
|
91
|
+
browser = await Browser.fromPool(browserOptions);
|
|
92
|
+
await browser.launch();
|
|
93
|
+
}
|
|
94
|
+
await applyPreconditions(browser, program);
|
|
95
|
+
return new WebPlaywrightLocal(browser, recordingDir);
|
|
96
|
+
};
|
|
97
|
+
const runResult = await executeProgram(program, createAdapter, runOptions);
|
|
98
|
+
if (runResult.success) {
|
|
99
|
+
logger.info(`[capture] Run completed successfully — ${runResult.telemetry.totalOpcodes} opcodes, ${runResult.telemetry.recoveredOpcodes} recovered, ${runResult.totalDurationMs}ms`);
|
|
100
|
+
}
|
|
101
|
+
else {
|
|
102
|
+
logger.error(`[capture] Run failed: ${runResult.error}`);
|
|
103
|
+
}
|
|
104
|
+
try {
|
|
105
|
+
await uploadResults(config, program, runResult);
|
|
106
|
+
}
|
|
107
|
+
catch (err) {
|
|
108
|
+
logger.error(`[capture] Failed to upload results: ${err instanceof Error ? err.message : String(err)}`);
|
|
109
|
+
}
|
|
110
|
+
return { success: runResult.success, runResult };
|
|
111
|
+
}
|
|
112
|
+
// ── Server communication ────────────────────────────────────────────
|
|
113
|
+
async function fetchProgram(config, presetId) {
|
|
114
|
+
try {
|
|
115
|
+
const url = `${config.apiBaseUrl}/api/v2/programs/${presetId}`;
|
|
116
|
+
const response = await fetch(url, {
|
|
117
|
+
headers: {
|
|
118
|
+
'Authorization': `Bearer ${config.apiKey}`,
|
|
119
|
+
'Content-Type': 'application/json',
|
|
120
|
+
},
|
|
121
|
+
});
|
|
122
|
+
if (!response.ok) {
|
|
123
|
+
return { success: false, error: `server returned ${response.status}: ${await response.text()}` };
|
|
124
|
+
}
|
|
125
|
+
const data = await response.json();
|
|
126
|
+
return { success: true, program: data };
|
|
127
|
+
}
|
|
128
|
+
catch (err) {
|
|
129
|
+
return { success: false, error: `failed to fetch program: ${err instanceof Error ? err.message : String(err)}` };
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
async function uploadResults(config, program, result) {
|
|
133
|
+
const runId = randomUUID();
|
|
134
|
+
// Upload artifacts
|
|
135
|
+
for (const variant of result.variantResults) {
|
|
136
|
+
const variantSpec = program.variants.find((entry) => entry.id === variant.variantId);
|
|
137
|
+
for (const artifact of variant.artifacts) {
|
|
138
|
+
const formData = new FormData();
|
|
139
|
+
const filename = buildArtifactFilename(program.presetId, variant.variantId, artifact);
|
|
140
|
+
formData.append('file', new Blob([new Uint8Array(artifact.buffer)], { type: artifact.mimeType }), filename);
|
|
141
|
+
formData.append('presetId', program.presetId);
|
|
142
|
+
formData.append('programVersion', String(program.programVersion));
|
|
143
|
+
formData.append('compileFingerprint', program.compileFingerprint);
|
|
144
|
+
formData.append('runId', runId);
|
|
145
|
+
formData.append('variantId', variant.variantId);
|
|
146
|
+
formData.append('targetId', variantSpec?.targetId ?? variant.variantId);
|
|
147
|
+
formData.append('targetLabel', variantSpec?.targetLabel ?? variantSpec?.deviceFrame ?? variant.variantId);
|
|
148
|
+
formData.append('mediaMode', artifact.mediaMode);
|
|
149
|
+
formData.append('mimeType', artifact.mimeType);
|
|
150
|
+
formData.append('captureType', artifact.captureType ?? 'fullpage');
|
|
151
|
+
formData.append('captureUrl', artifact.captureUrl ?? program.baseUrl);
|
|
152
|
+
formData.append('lang', variantSpec?.locale ?? 'en');
|
|
153
|
+
formData.append('theme', variantSpec?.theme ?? 'light');
|
|
154
|
+
if (variantSpec?.deviceFrame) {
|
|
155
|
+
formData.append('deviceFrame', variantSpec.deviceFrame);
|
|
156
|
+
}
|
|
157
|
+
formData.append('viewport', JSON.stringify(variantSpec?.viewport ?? null));
|
|
158
|
+
formData.append('artifactPlan', JSON.stringify(program.artifactPlan));
|
|
159
|
+
if (artifact.altText) {
|
|
160
|
+
formData.append('altText', artifact.altText);
|
|
161
|
+
}
|
|
162
|
+
if (artifact.elementSelector) {
|
|
163
|
+
formData.append('elementSelector', artifact.elementSelector);
|
|
164
|
+
}
|
|
165
|
+
if (artifact.captureId) {
|
|
166
|
+
formData.append('captureId', artifact.captureId);
|
|
167
|
+
}
|
|
168
|
+
if (artifact.captureName) {
|
|
169
|
+
formData.append('captureName', artifact.captureName);
|
|
170
|
+
}
|
|
171
|
+
if (artifact.clipId) {
|
|
172
|
+
formData.append('clipId', artifact.clipId);
|
|
173
|
+
}
|
|
174
|
+
if (artifact.clipName) {
|
|
175
|
+
formData.append('clipName', artifact.clipName);
|
|
176
|
+
}
|
|
177
|
+
if (artifact.stepDescription) {
|
|
178
|
+
formData.append('stepDescription', artifact.stepDescription);
|
|
179
|
+
}
|
|
180
|
+
if (typeof artifact.stepIndex === 'number') {
|
|
181
|
+
formData.append('stepIndex', String(artifact.stepIndex));
|
|
182
|
+
}
|
|
183
|
+
if (typeof artifact.durationMs === 'number') {
|
|
184
|
+
formData.append('durationMs', String(artifact.durationMs));
|
|
185
|
+
}
|
|
186
|
+
if (typeof artifact.trimStartMs === 'number') {
|
|
187
|
+
formData.append('trimStartMs', String(artifact.trimStartMs));
|
|
188
|
+
}
|
|
189
|
+
const response = await fetch(`${config.apiBaseUrl}/api/v2/artifacts`, {
|
|
190
|
+
method: 'POST',
|
|
191
|
+
headers: { 'Authorization': `Bearer ${config.apiKey}` },
|
|
192
|
+
body: formData,
|
|
193
|
+
});
|
|
194
|
+
if (!response.ok) {
|
|
195
|
+
throw new Error(`artifact upload failed for ${variant.variantId}: ${response.status} ${await response.text()}`);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
// Upload telemetry + healer patches
|
|
200
|
+
const telemetryResponse = await fetch(`${config.apiBaseUrl}/api/v2/telemetry`, {
|
|
201
|
+
method: 'POST',
|
|
202
|
+
headers: {
|
|
203
|
+
'Authorization': `Bearer ${config.apiKey}`,
|
|
204
|
+
'Content-Type': 'application/json',
|
|
205
|
+
},
|
|
206
|
+
body: JSON.stringify({
|
|
207
|
+
runId,
|
|
208
|
+
presetId: program.presetId,
|
|
209
|
+
programVersion: program.programVersion,
|
|
210
|
+
compileFingerprint: program.compileFingerprint,
|
|
211
|
+
success: result.success,
|
|
212
|
+
mediaMode: program.mediaMode,
|
|
213
|
+
telemetry: result.telemetry,
|
|
214
|
+
healerPatches: result.healerPatches,
|
|
215
|
+
runResult: result,
|
|
216
|
+
totalDurationMs: result.totalDurationMs,
|
|
217
|
+
variantSummaries: result.variantResults.map(v => ({
|
|
218
|
+
variantId: v.variantId,
|
|
219
|
+
success: v.success,
|
|
220
|
+
opcodeCount: v.opcodeResults.length,
|
|
221
|
+
artifactCount: v.artifacts.length,
|
|
222
|
+
durationMs: v.durationMs,
|
|
223
|
+
error: v.error,
|
|
224
|
+
})),
|
|
225
|
+
}),
|
|
226
|
+
});
|
|
227
|
+
if (!telemetryResponse.ok) {
|
|
228
|
+
throw new Error(`telemetry upload failed: ${telemetryResponse.status} ${await telemetryResponse.text()}`);
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
async function applyPreconditions(browser, program) {
|
|
232
|
+
const { preconditions } = program;
|
|
233
|
+
const hasConcreteSessionState = Boolean(preconditions.storageState
|
|
234
|
+
|| (preconditions.cookies && preconditions.cookies.length > 0)
|
|
235
|
+
|| (preconditions.sessionStorage && Object.keys(preconditions.sessionStorage).length > 0));
|
|
236
|
+
if (preconditions.auth === 'authenticated' && !hasConcreteSessionState) {
|
|
237
|
+
throw new Error('authenticated precondition requires storageState, cookies, or sessionStorage');
|
|
238
|
+
}
|
|
239
|
+
if (preconditions.credentialsId && !hasConcreteSessionState) {
|
|
240
|
+
throw new Error(`credentialsId "${preconditions.credentialsId}" is not resolved by the CLI runtime; provide storageState/cookies/sessionStorage in the program`);
|
|
241
|
+
}
|
|
242
|
+
if (preconditions.cookies && preconditions.cookies.length > 0) {
|
|
243
|
+
await browser.addCookies(preconditions.cookies);
|
|
244
|
+
}
|
|
245
|
+
if (preconditions.sessionStorage && Object.keys(preconditions.sessionStorage).length > 0) {
|
|
246
|
+
await browser.prepareSessionStorage(preconditions.sessionStorage, { replace: false });
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
function resolveCliLLMConfig() {
|
|
250
|
+
const apiKey = process.env.OPENROUTER_API_KEY?.trim();
|
|
251
|
+
if (!apiKey)
|
|
252
|
+
return undefined;
|
|
253
|
+
return {
|
|
254
|
+
apiKey,
|
|
255
|
+
model: process.env.AUTOKAP_V2_LLM_MODEL?.trim() || undefined,
|
|
256
|
+
};
|
|
257
|
+
}
|
|
258
|
+
function buildArtifactFilename(presetId, variantId, artifact) {
|
|
259
|
+
const ext = artifact.mimeType === 'image/jpeg'
|
|
260
|
+
? 'jpg'
|
|
261
|
+
: artifact.mimeType === 'image/png'
|
|
262
|
+
? 'png'
|
|
263
|
+
: artifact.mimeType.includes('gif')
|
|
264
|
+
? 'gif'
|
|
265
|
+
: artifact.mimeType.includes('mp4')
|
|
266
|
+
? 'mp4'
|
|
267
|
+
: 'webm';
|
|
268
|
+
const stepToken = typeof artifact.stepIndex === 'number' ? `-${artifact.stepIndex}` : '';
|
|
269
|
+
return `${presetId}-${variantId}${stepToken}.${ext}`;
|
|
270
|
+
}
|
|
271
|
+
// ── Progress logging ────────────────────────────────────────────────
|
|
272
|
+
function logProgress(event) {
|
|
273
|
+
const prefix = `[capture][${event.variantId}]`;
|
|
274
|
+
switch (event.type) {
|
|
275
|
+
case 'variant_start':
|
|
276
|
+
logger.info(`${prefix} Starting variant`);
|
|
277
|
+
break;
|
|
278
|
+
case 'variant_end':
|
|
279
|
+
logger.info(`${prefix} ${event.status === 'ok' ? 'Completed' : 'Failed'}: ${event.message}`);
|
|
280
|
+
break;
|
|
281
|
+
case 'opcode_start':
|
|
282
|
+
logger.debug(`${prefix}[${event.opcodeIndex}] ${event.opcodeKind}: ${event.message}`);
|
|
283
|
+
break;
|
|
284
|
+
case 'opcode_end':
|
|
285
|
+
if (event.status === 'failed') {
|
|
286
|
+
logger.error(`${prefix}[${event.opcodeIndex}] FAILED: ${event.message}`);
|
|
287
|
+
}
|
|
288
|
+
else if (event.status === 'recovered') {
|
|
289
|
+
logger.warn(`${prefix}[${event.opcodeIndex}] RECOVERED: ${event.message}`);
|
|
290
|
+
}
|
|
291
|
+
break;
|
|
292
|
+
case 'recovery':
|
|
293
|
+
logger.warn(`${prefix}[${event.opcodeIndex}] Recovery: ${event.message}`);
|
|
294
|
+
break;
|
|
295
|
+
case 'breaker_trip':
|
|
296
|
+
logger.error(`${prefix} Circuit breaker tripped: ${event.message}`);
|
|
297
|
+
break;
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
//# sourceMappingURL=cli-runner.js.map
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* V2 Capture Agent — Compiler Prompts
|
|
3
|
+
*
|
|
4
|
+
* System and user prompts for compiling natural-language presets
|
|
5
|
+
* into ExecutionProgramV2 opcodes.
|
|
6
|
+
*/
|
|
7
|
+
import type { MediaMode, VariantSpec } from './types.js';
|
|
8
|
+
export interface CompilerPromptContext {
|
|
9
|
+
/** User-authored preset prompt (natural language) */
|
|
10
|
+
presetPrompt: string;
|
|
11
|
+
/** Base URL of the target site */
|
|
12
|
+
baseUrl: string;
|
|
13
|
+
/** Media mode */
|
|
14
|
+
mediaMode: MediaMode;
|
|
15
|
+
/** Variant specifications */
|
|
16
|
+
variants: VariantSpec[];
|
|
17
|
+
/** AKTree snapshot from exploration (optional) */
|
|
18
|
+
akTreeSnapshot?: string;
|
|
19
|
+
/** Page URL at time of exploration */
|
|
20
|
+
exploredUrl?: string;
|
|
21
|
+
/** Whether auth is required */
|
|
22
|
+
requiresAuth: boolean;
|
|
23
|
+
/** Navigation instructions (if separate from capture instructions) */
|
|
24
|
+
navigationInstructions?: string;
|
|
25
|
+
}
|
|
26
|
+
export declare function buildCompilerSystemPrompt(): string;
|
|
27
|
+
export declare function buildCompilerUserPrompt(context: CompilerPromptContext): string;
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* V2 Capture Agent — Compiler Prompts
|
|
3
|
+
*
|
|
4
|
+
* System and user prompts for compiling natural-language presets
|
|
5
|
+
* into ExecutionProgramV2 opcodes.
|
|
6
|
+
*/
|
|
7
|
+
export function buildCompilerSystemPrompt() {
|
|
8
|
+
return `You are a web automation compiler. You transform natural-language instructions into a deterministic sequence of browser opcodes.
|
|
9
|
+
|
|
10
|
+
## Opcode Reference
|
|
11
|
+
|
|
12
|
+
Each opcode is a JSON object with these common fields:
|
|
13
|
+
- kind: the opcode type (see below)
|
|
14
|
+
- description: human-readable description of what this step does
|
|
15
|
+
- postcondition: { type, pattern?, selector?, text?, threshold?, waitMs? }
|
|
16
|
+
- recovery: { retries: 2, useSelectorMemory: true, useAltInteraction: true, allowReload: false, allowHealer: true }
|
|
17
|
+
- timeoutMs: max time in ms (default 15000)
|
|
18
|
+
- maxFailures: max recovery attempts (default 3)
|
|
19
|
+
|
|
20
|
+
### Available Opcodes
|
|
21
|
+
|
|
22
|
+
NAVIGATE { url: string }
|
|
23
|
+
Navigate to a URL. Postcondition: route_matches.
|
|
24
|
+
|
|
25
|
+
DISMISS_OVERLAYS {}
|
|
26
|
+
Dismiss cookie banners, popups, chat widgets. Postcondition: overlay_dismissed.
|
|
27
|
+
|
|
28
|
+
ASSERT_ROUTE { urlPattern: string }
|
|
29
|
+
Verify the current URL matches a pattern. Postcondition: route_matches.
|
|
30
|
+
|
|
31
|
+
ASSERT_SURFACE { selectors: string[], matchAll: boolean }
|
|
32
|
+
Verify elements are visible. Postcondition: element_visible.
|
|
33
|
+
|
|
34
|
+
CLICK { selector: string, fingerprint?: string, selectorAlternates?: string[] }
|
|
35
|
+
Click an element. Postcondition: what CHANGED as a result (not "element was clicked").
|
|
36
|
+
|
|
37
|
+
TYPE { selector: string, text: string, clearFirst: boolean }
|
|
38
|
+
Type text into a field. Postcondition: text_contains or any_change.
|
|
39
|
+
|
|
40
|
+
PRESS_KEY { key: string }
|
|
41
|
+
Press a keyboard key. Postcondition: any_change.
|
|
42
|
+
|
|
43
|
+
WAIT_FOR { selector: string, state: "visible"|"attached" }
|
|
44
|
+
Wait for an element. Postcondition: element_visible.
|
|
45
|
+
|
|
46
|
+
SET_LOCALE { locale: string, method: "browser_context"|"ui_interaction"|"storage", selector?: string }
|
|
47
|
+
Set the page locale. Postcondition: always.
|
|
48
|
+
|
|
49
|
+
SET_THEME { theme: "light"|"dark", method: "color_scheme"|"ui_interaction"|"storage", selector?: string }
|
|
50
|
+
Set the color scheme. Postcondition: always.
|
|
51
|
+
|
|
52
|
+
SCROLL { direction: "up"|"down"|"left"|"right", amount?: number, targetSelector?: string }
|
|
53
|
+
Scroll the page. Postcondition: element_visible (for the target) or always.
|
|
54
|
+
|
|
55
|
+
CAPTURE_SCREENSHOT { elementSelector?: string }
|
|
56
|
+
Take a screenshot. Postcondition: always.
|
|
57
|
+
|
|
58
|
+
BEGIN_CLIP {} / END_CLIP {}
|
|
59
|
+
Start/stop clip recording. Postcondition: always.
|
|
60
|
+
|
|
61
|
+
BEGIN_VIDEO {} / END_VIDEO {}
|
|
62
|
+
Start/stop video recording. Postcondition: always.
|
|
63
|
+
|
|
64
|
+
### Postcondition Types
|
|
65
|
+
- route_matches: URL pathname matches glob pattern. e.g. { type: "route_matches", pattern: "/dashboard*" }
|
|
66
|
+
- element_visible: Element matching selector is visible. e.g. { type: "element_visible", selector: "h1" }
|
|
67
|
+
- element_absent: Element is NOT visible. e.g. { type: "element_absent", selector: ".modal" }
|
|
68
|
+
- text_contains: Element text includes string. e.g. { type: "text_contains", selector: "#name", text: "John" }
|
|
69
|
+
- overlay_dismissed: No blocking overlays. e.g. { type: "overlay_dismissed" }
|
|
70
|
+
- screenshot_stable: Page visually stable. e.g. { type: "screenshot_stable", threshold: 0.01 }
|
|
71
|
+
- any_change: Any detectable change (soft). e.g. { type: "any_change" }
|
|
72
|
+
- always: Always passes. e.g. { type: "always" }
|
|
73
|
+
|
|
74
|
+
## Critical Rules
|
|
75
|
+
|
|
76
|
+
1. Every CLICK/TYPE must have a postcondition that describes WHAT CHANGED, not just "element was clicked".
|
|
77
|
+
Bad: { type: "always" } after a click
|
|
78
|
+
Good: { type: "element_visible", selector: ".dropdown-menu" } after clicking a dropdown trigger
|
|
79
|
+
|
|
80
|
+
2. Always start with NAVIGATE + DISMISS_OVERLAYS.
|
|
81
|
+
|
|
82
|
+
3. Use real CSS selectors from the AKTree when provided. Prefer data-testid, id, or semantic selectors over fragile class-based ones.
|
|
83
|
+
|
|
84
|
+
4. For multi-step flows (login, modal navigation), add WAIT_FOR between steps to ensure the page has transitioned.
|
|
85
|
+
|
|
86
|
+
5. For clips/videos, place BEGIN_CLIP/END_CLIP or BEGIN_VIDEO/END_VIDEO around the interaction portion only (not navigation).
|
|
87
|
+
|
|
88
|
+
6. Include selectorAlternates when you can infer multiple valid selectors for the same element.
|
|
89
|
+
|
|
90
|
+
7. Set waitMs on postconditions that involve page transitions (login, navigation) to at least 10000.
|
|
91
|
+
|
|
92
|
+
## Response Format
|
|
93
|
+
|
|
94
|
+
Return ONLY a JSON array of opcodes. No markdown, no explanation.`;
|
|
95
|
+
}
|
|
96
|
+
export function buildCompilerUserPrompt(context) {
|
|
97
|
+
const parts = [];
|
|
98
|
+
parts.push(`## Preset Instructions\n${context.presetPrompt}`);
|
|
99
|
+
parts.push(`\n## Target\nURL: ${context.baseUrl}\nMedia: ${context.mediaMode}`);
|
|
100
|
+
if (context.navigationInstructions) {
|
|
101
|
+
parts.push(`\n## Navigation Instructions\n${context.navigationInstructions}`);
|
|
102
|
+
}
|
|
103
|
+
if (context.requiresAuth) {
|
|
104
|
+
parts.push(`\n## Authentication\nThis preset requires authentication. Use TYPE opcodes with placeholder text "{{email}}" and "{{password}}" for credential fields. The runtime will substitute real credentials.`);
|
|
105
|
+
}
|
|
106
|
+
if (context.variants.length > 0) {
|
|
107
|
+
const variantDesc = context.variants.map(v => {
|
|
108
|
+
const parts = [`${v.viewport.width}x${v.viewport.height}`];
|
|
109
|
+
if (v.locale)
|
|
110
|
+
parts.push(`locale: ${v.locale}`);
|
|
111
|
+
if (v.theme)
|
|
112
|
+
parts.push(`theme: ${v.theme}`);
|
|
113
|
+
return parts.join(', ');
|
|
114
|
+
}).join('\n ');
|
|
115
|
+
parts.push(`\n## Variants\nThe program will be executed once per variant. Locale/theme switching is handled by the runtime via SET_LOCALE/SET_THEME opcodes.\n ${variantDesc}`);
|
|
116
|
+
}
|
|
117
|
+
if (context.akTreeSnapshot) {
|
|
118
|
+
parts.push(`\n## Live AKTree (from site exploration)\nURL at exploration: ${context.exploredUrl ?? context.baseUrl}\n\n${context.akTreeSnapshot.slice(0, 12000)}`);
|
|
119
|
+
}
|
|
120
|
+
parts.push(`\nCompile these instructions into a JSON array of opcodes. Use real selectors from the AKTree when available.`);
|
|
121
|
+
return parts.join('\n');
|
|
122
|
+
}
|
|
123
|
+
//# sourceMappingURL=compiler-prompts.js.map
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* V2 Capture Agent — Preset Compiler
|
|
3
|
+
*
|
|
4
|
+
* Transforms natural-language presets into ExecutionProgramV2.
|
|
5
|
+
* Flow:
|
|
6
|
+
* 1. Parse preset metadata (URL, variants, auth, media mode)
|
|
7
|
+
* 2. (Optional) Explore the target site to capture real AKTree selectors
|
|
8
|
+
* 3. LLM generates opcode sequence grounded in real DOM
|
|
9
|
+
* 4. Validate output against Zod schema
|
|
10
|
+
* 5. Return compiled program with fingerprint
|
|
11
|
+
*/
|
|
12
|
+
import type { ExecutionProgramV2, VariantSpec, MediaMode, PreconditionSpec } from './types.js';
|
|
13
|
+
export interface CompilerLLMProvider {
|
|
14
|
+
call(systemPrompt: string, userPrompt: string): Promise<{
|
|
15
|
+
response: string;
|
|
16
|
+
costEur: number;
|
|
17
|
+
}>;
|
|
18
|
+
}
|
|
19
|
+
export interface PresetInput {
|
|
20
|
+
presetId: string;
|
|
21
|
+
prompt: string;
|
|
22
|
+
baseUrl: string;
|
|
23
|
+
mediaMode: MediaMode;
|
|
24
|
+
variants: VariantSpec[];
|
|
25
|
+
preconditions: PreconditionSpec;
|
|
26
|
+
navigationInstructions?: string;
|
|
27
|
+
/** Pre-captured AKTree from exploration */
|
|
28
|
+
akTreeSnapshot?: string;
|
|
29
|
+
exploredUrl?: string;
|
|
30
|
+
}
|
|
31
|
+
export interface CompilerResult {
|
|
32
|
+
success: boolean;
|
|
33
|
+
program?: ExecutionProgramV2;
|
|
34
|
+
error?: string;
|
|
35
|
+
costEur: number;
|
|
36
|
+
}
|
|
37
|
+
export declare function compilePreset(input: PresetInput, llmProvider: CompilerLLMProvider): Promise<CompilerResult>;
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* V2 Capture Agent — Preset Compiler
|
|
3
|
+
*
|
|
4
|
+
* Transforms natural-language presets into ExecutionProgramV2.
|
|
5
|
+
* Flow:
|
|
6
|
+
* 1. Parse preset metadata (URL, variants, auth, media mode)
|
|
7
|
+
* 2. (Optional) Explore the target site to capture real AKTree selectors
|
|
8
|
+
* 3. LLM generates opcode sequence grounded in real DOM
|
|
9
|
+
* 4. Validate output against Zod schema
|
|
10
|
+
* 5. Return compiled program with fingerprint
|
|
11
|
+
*/
|
|
12
|
+
import { DEFAULT_RECOVERY_POLICY } from './types.js';
|
|
13
|
+
import { ExecutionOpcodeSchema, ExecutionProgramV2Schema } from './schema.js';
|
|
14
|
+
import { buildCompilerSystemPrompt, buildCompilerUserPrompt } from './compiler-prompts.js';
|
|
15
|
+
// ── Main compile function ───────────────────────────────────────────
|
|
16
|
+
export async function compilePreset(input, llmProvider) {
|
|
17
|
+
const systemPrompt = buildCompilerSystemPrompt();
|
|
18
|
+
const context = {
|
|
19
|
+
presetPrompt: input.prompt,
|
|
20
|
+
baseUrl: input.baseUrl,
|
|
21
|
+
mediaMode: input.mediaMode,
|
|
22
|
+
variants: input.variants,
|
|
23
|
+
requiresAuth: input.preconditions.auth === 'authenticated',
|
|
24
|
+
navigationInstructions: input.navigationInstructions,
|
|
25
|
+
akTreeSnapshot: input.akTreeSnapshot,
|
|
26
|
+
exploredUrl: input.exploredUrl,
|
|
27
|
+
};
|
|
28
|
+
const userPrompt = buildCompilerUserPrompt(context);
|
|
29
|
+
try {
|
|
30
|
+
const { response, costEur } = await llmProvider.call(systemPrompt, userPrompt);
|
|
31
|
+
// Parse opcodes from LLM response
|
|
32
|
+
const opcodes = parseOpcodeResponse(response);
|
|
33
|
+
if (!opcodes) {
|
|
34
|
+
return { success: false, error: 'LLM response is not a valid JSON array of opcodes', costEur };
|
|
35
|
+
}
|
|
36
|
+
// Validate each opcode
|
|
37
|
+
const validatedOpcodes = [];
|
|
38
|
+
for (let i = 0; i < opcodes.length; i++) {
|
|
39
|
+
const enriched = enrichOpcode(opcodes[i]);
|
|
40
|
+
const result = ExecutionOpcodeSchema.safeParse(enriched);
|
|
41
|
+
if (!result.success) {
|
|
42
|
+
return {
|
|
43
|
+
success: false,
|
|
44
|
+
error: `opcode ${i} validation failed: ${result.error}`,
|
|
45
|
+
costEur,
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
validatedOpcodes.push(result.data);
|
|
49
|
+
}
|
|
50
|
+
if (validatedOpcodes.length === 0) {
|
|
51
|
+
return { success: false, error: 'LLM produced zero opcodes', costEur };
|
|
52
|
+
}
|
|
53
|
+
// Build the program
|
|
54
|
+
const program = {
|
|
55
|
+
presetId: input.presetId,
|
|
56
|
+
programVersion: 1,
|
|
57
|
+
mediaMode: input.mediaMode,
|
|
58
|
+
baseUrl: input.baseUrl,
|
|
59
|
+
variants: input.variants,
|
|
60
|
+
preconditions: input.preconditions,
|
|
61
|
+
steps: validatedOpcodes,
|
|
62
|
+
artifactPlan: {
|
|
63
|
+
mediaMode: input.mediaMode,
|
|
64
|
+
...(input.mediaMode === 'clip' ? { format: { clipFormat: 'gif' } } : {}),
|
|
65
|
+
},
|
|
66
|
+
compileFingerprint: computeFingerprint(input),
|
|
67
|
+
compiledAt: new Date().toISOString(),
|
|
68
|
+
compiledWith: 'gemini-3-flash',
|
|
69
|
+
};
|
|
70
|
+
// Final program-level validation
|
|
71
|
+
const programResult = ExecutionProgramV2Schema.safeParse(program);
|
|
72
|
+
if (!programResult.success) {
|
|
73
|
+
return {
|
|
74
|
+
success: false,
|
|
75
|
+
error: `program validation failed: ${programResult.error}`,
|
|
76
|
+
costEur,
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
return { success: true, program: programResult.data, costEur };
|
|
80
|
+
}
|
|
81
|
+
catch (err) {
|
|
82
|
+
return {
|
|
83
|
+
success: false,
|
|
84
|
+
error: `compiler error: ${err instanceof Error ? err.message : String(err)}`,
|
|
85
|
+
costEur: 0,
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
// ── Response parsing ────────────────────────────────────────────────
|
|
90
|
+
function parseOpcodeResponse(response) {
|
|
91
|
+
try {
|
|
92
|
+
// Strip markdown code fences if present
|
|
93
|
+
let cleaned = response.trim();
|
|
94
|
+
if (cleaned.startsWith('```')) {
|
|
95
|
+
cleaned = cleaned.replace(/^```(?:json)?\s*/, '').replace(/```\s*$/, '').trim();
|
|
96
|
+
}
|
|
97
|
+
const parsed = JSON.parse(cleaned);
|
|
98
|
+
if (Array.isArray(parsed))
|
|
99
|
+
return parsed;
|
|
100
|
+
// Some models wrap in { steps: [...] } or { opcodes: [...] }
|
|
101
|
+
if (parsed && typeof parsed === 'object') {
|
|
102
|
+
if (Array.isArray(parsed.steps))
|
|
103
|
+
return parsed.steps;
|
|
104
|
+
if (Array.isArray(parsed.opcodes))
|
|
105
|
+
return parsed.opcodes;
|
|
106
|
+
}
|
|
107
|
+
return null;
|
|
108
|
+
}
|
|
109
|
+
catch {
|
|
110
|
+
return null;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* Enriches a raw opcode from LLM with default values for fields
|
|
115
|
+
* the LLM might omit.
|
|
116
|
+
*/
|
|
117
|
+
function enrichOpcode(raw) {
|
|
118
|
+
return {
|
|
119
|
+
...raw,
|
|
120
|
+
description: raw.description || `${raw.kind} step`,
|
|
121
|
+
postcondition: raw.postcondition ?? { type: 'always' },
|
|
122
|
+
recovery: raw.recovery ?? { ...DEFAULT_RECOVERY_POLICY },
|
|
123
|
+
timeoutMs: raw.timeoutMs ?? 15000,
|
|
124
|
+
maxFailures: raw.maxFailures ?? 3,
|
|
125
|
+
// Type-specific defaults
|
|
126
|
+
...(raw.kind === 'TYPE' && raw.clearFirst === undefined ? { clearFirst: true } : {}),
|
|
127
|
+
...(raw.kind === 'ASSERT_SURFACE' && raw.matchAll === undefined ? { matchAll: true } : {}),
|
|
128
|
+
...(raw.kind === 'WAIT_FOR' && !raw.state ? { state: 'visible' } : {}),
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
// ── Fingerprint ─────────────────────────────────────────────────────
|
|
132
|
+
function computeFingerprint(input) {
|
|
133
|
+
const content = JSON.stringify({
|
|
134
|
+
prompt: input.prompt,
|
|
135
|
+
baseUrl: input.baseUrl,
|
|
136
|
+
mediaMode: input.mediaMode,
|
|
137
|
+
navigation: input.navigationInstructions,
|
|
138
|
+
auth: input.preconditions.auth,
|
|
139
|
+
});
|
|
140
|
+
// Simple hash
|
|
141
|
+
let hash = 0;
|
|
142
|
+
for (let i = 0; i < content.length; i++) {
|
|
143
|
+
hash = ((hash << 5) - hash + content.charCodeAt(i)) | 0;
|
|
144
|
+
}
|
|
145
|
+
return Math.abs(hash).toString(36);
|
|
146
|
+
}
|
|
147
|
+
//# sourceMappingURL=compiler.js.map
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* V2 Capture Agent — Site Explorer
|
|
3
|
+
*
|
|
4
|
+
* Deterministic exploration of the target site to capture AKTree snapshots.
|
|
5
|
+
* These snapshots ground the compiler's opcode generation in real DOM structure,
|
|
6
|
+
* preventing hallucinated selectors.
|
|
7
|
+
*
|
|
8
|
+
* The explorer uses the Browser class directly — no LLM calls.
|
|
9
|
+
*/
|
|
10
|
+
import type { Browser } from '../browser.js';
|
|
11
|
+
import type { AKTree } from '../types.js';
|
|
12
|
+
export interface ExplorationResult {
|
|
13
|
+
/** Serialized AKTree at the explored URL */
|
|
14
|
+
akTreeSerialized: string;
|
|
15
|
+
/** Raw AKTree object */
|
|
16
|
+
akTree: AKTree;
|
|
17
|
+
/** URL at time of exploration */
|
|
18
|
+
url: string;
|
|
19
|
+
/** Whether overlays were dismissed */
|
|
20
|
+
overlaysDismissed: boolean;
|
|
21
|
+
}
|
|
22
|
+
export interface ExploreOptions {
|
|
23
|
+
/** Target URL to explore */
|
|
24
|
+
url: string;
|
|
25
|
+
/** Dismiss overlays before capturing AKTree. Default: true */
|
|
26
|
+
dismissOverlays?: boolean;
|
|
27
|
+
/** Max time to wait for page load (ms). Default: 15000 */
|
|
28
|
+
loadTimeoutMs?: number;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Explores a single URL: navigates, dismisses overlays, and captures AKTree.
|
|
32
|
+
* Uses an existing Browser instance — does not create or close it.
|
|
33
|
+
*/
|
|
34
|
+
export declare function explorePage(browser: Browser, options: ExploreOptions): Promise<ExplorationResult>;
|
|
35
|
+
/**
|
|
36
|
+
* Explores multiple URLs on the same site.
|
|
37
|
+
* Useful for presets that involve navigation across pages.
|
|
38
|
+
*/
|
|
39
|
+
export declare function explorePages(browser: Browser, urls: string[], options?: {
|
|
40
|
+
dismissOverlays?: boolean;
|
|
41
|
+
}): Promise<ExplorationResult[]>;
|