autokap 1.0.6 → 1.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/chrome/ios-statusbar-comparison-reference.jpg +0 -0
- package/assets/chrome/ios-statusbar-dark-reference.jpg +0 -0
- package/assets/chrome/ios-statusbar-light-reference.jpg +0 -0
- package/assets/cursors/macos.svg +4 -0
- package/assets/cursors/windows.svg +15 -0
- package/assets/devices/ipad-pro-11-m4.json +52 -0
- package/assets/devices/iphone-16-pro.json +53 -0
- package/assets/devices/macbook-air-13.json +45 -0
- package/assets/frames/MacBook Air 13.svg +242 -0
- package/assets/frames/Status bar - iPhone.png +0 -0
- Menu bar- iPad.png +0 -0
- package/assets/frames/iPad Pro M4 11_.png +0 -0
- package/assets/frames/iPhone 16 Pro.png +0 -0
- package/assets/icons/Cellular Connection.svg +3 -0
- package/assets/icons/Union.svg +6 -0
- package/assets/icons/Wifi.svg +3 -0
- package/assets/icons/battery.svg +5 -0
- package/assets/icons/battery_charging.svg +8 -0
- package/assets/skill/OPCODE-REFERENCE.md +607 -0
- package/assets/skill/README.md +39 -0
- package/assets/skill/SKILL.md +453 -468
- package/assets/skill/STUDIO-SKILL.md +476 -0
- package/assets/skill/references/examples.md +104 -0
- package/assets/skill/references/interactive-demo.md +225 -0
- package/assets/skill/references/mock-data.md +178 -0
- package/dist/abort.d.ts +5 -0
- package/dist/abort.js +44 -0
- package/dist/action-verifier.d.ts +29 -0
- package/dist/action-verifier.js +133 -0
- package/dist/agent-action-recovery.d.ts +45 -0
- package/dist/agent-action-recovery.js +370 -0
- package/dist/agent-message-utils.d.ts +21 -0
- package/dist/agent-message-utils.js +77 -0
- package/dist/agent-url-utils.d.ts +30 -0
- package/dist/agent-url-utils.js +138 -0
- package/dist/agent.d.ts +226 -0
- package/dist/agent.js +6666 -0
- package/dist/ak-tree.d.ts +39 -0
- package/dist/ak-tree.js +368 -0
- package/dist/alt-text.d.ts +26 -0
- package/dist/alt-text.js +55 -0
- package/dist/auth-capture.d.ts +17 -0
- package/dist/auth-capture.js +164 -0
- package/dist/benchmark.d.ts +59 -0
- package/dist/benchmark.js +135 -0
- package/dist/billing-operation-logging.d.ts +38 -0
- package/dist/billing-operation-logging.js +248 -0
- package/dist/browser-bar.d.ts +48 -0
- package/dist/browser-bar.js +284 -0
- package/dist/browser-pool.d.ts +7 -0
- package/dist/browser-pool.js +15 -5
- package/dist/browser-utils.d.ts +31 -0
- package/dist/browser-utils.js +97 -0
- package/dist/browser.d.ts +76 -1
- package/dist/browser.js +1657 -39
- package/dist/capture-alt-text.d.ts +12 -0
- package/dist/capture-alt-text.js +52 -0
- package/dist/capture-encryption.d.ts +10 -0
- package/dist/capture-encryption.js +41 -0
- package/dist/capture-language-preflight.d.ts +41 -0
- package/dist/capture-language-preflight.js +300 -0
- package/dist/capture-llm-page-identity.d.ts +15 -0
- package/dist/capture-llm-page-identity.js +128 -0
- package/dist/capture-model-resolution.d.ts +9 -0
- package/dist/capture-model-resolution.js +21 -0
- package/dist/capture-page-identity.d.ts +7 -0
- package/dist/capture-page-identity.js +352 -0
- package/dist/capture-preset-credentials.d.ts +62 -0
- package/dist/capture-preset-credentials.js +184 -0
- package/dist/capture-request-plan.d.ts +58 -0
- package/dist/capture-request-plan.js +264 -0
- package/dist/capture-run-optimizer.d.ts +139 -0
- package/dist/capture-run-optimizer.js +863 -0
- package/dist/capture-selector-memory.d.ts +31 -0
- package/dist/capture-selector-memory.js +345 -0
- package/dist/capture-session-profile-encryption.d.ts +2 -0
- package/dist/capture-session-profile-encryption.js +22 -0
- package/dist/capture-step-timeout.d.ts +10 -0
- package/dist/capture-step-timeout.js +30 -0
- package/dist/capture-strategy.d.ts +36 -0
- package/dist/capture-strategy.js +95 -0
- package/dist/capture-studio-sync.d.ts +23 -0
- package/dist/capture-studio-sync.js +172 -0
- package/dist/capture-surface-contract.d.ts +36 -0
- package/dist/capture-surface-contract.js +299 -0
- package/dist/capture-transition-engine.d.ts +28 -0
- package/dist/capture-transition-engine.js +292 -0
- package/dist/capture-variant-state.d.ts +56 -0
- package/dist/capture-variant-state.js +182 -0
- package/dist/capture-verification.d.ts +35 -0
- package/dist/capture-verification.js +95 -0
- package/dist/capture-viewport-lock.d.ts +48 -0
- package/dist/capture-viewport-lock.js +74 -0
- package/dist/circuit-breaker.d.ts +42 -0
- package/dist/circuit-breaker.js +119 -0
- package/dist/cli-config.d.ts +8 -1
- package/dist/cli-config.js +62 -6
- package/dist/cli-contract.d.ts +15 -0
- package/dist/cli-contract.js +167 -0
- package/dist/cli-runner-local.d.ts +12 -0
- package/dist/cli-runner-local.js +102 -0
- package/dist/cli-runner.d.ts +34 -0
- package/dist/cli-runner.js +433 -0
- package/dist/cli-utils.d.ts +0 -1
- package/dist/cli-utils.js +2 -5
- package/dist/cli.js +1005 -252
- package/dist/clip-orchestrator.d.ts +148 -0
- package/dist/clip-orchestrator.js +957 -0
- package/dist/clip-postprocess.d.ts +42 -0
- package/dist/clip-postprocess.js +201 -0
- package/dist/cookie-dismiss.d.ts +2 -0
- package/dist/cookie-dismiss.js +48 -13
- package/dist/cost-logging.d.ts +35 -0
- package/dist/cost-logging.js +242 -0
- package/dist/cost-resolution-monitor.d.ts +16 -0
- package/dist/cost-resolution-monitor.js +34 -0
- package/dist/credential-templates.d.ts +5 -0
- package/dist/credential-templates.js +60 -0
- package/dist/cursor-overlay-script.d.ts +6 -0
- package/dist/cursor-overlay-script.js +169 -0
- package/dist/dom-css-purger.d.ts +65 -0
- package/dist/dom-css-purger.js +333 -0
- package/dist/dom-font-inliner.d.ts +45 -0
- package/dist/dom-font-inliner.js +148 -0
- package/dist/dom-patch-resolver.d.ts +52 -0
- package/dist/dom-patch-resolver.js +242 -0
- package/dist/dom-serializer.d.ts +82 -0
- package/dist/dom-serializer.js +378 -0
- package/dist/element-capture.d.ts +13 -0
- package/dist/element-capture.js +522 -0
- package/dist/env-validation.d.ts +5 -0
- package/dist/env-validation.js +29 -0
- package/dist/execution-schema.d.ts +4423 -0
- package/dist/execution-schema.js +507 -0
- package/dist/execution-types.d.ts +886 -0
- package/dist/execution-types.js +65 -0
- package/dist/fonts-loader.d.ts +14 -0
- package/dist/fonts-loader.js +55 -0
- package/dist/hybrid-navigator.d.ts +138 -0
- package/dist/hybrid-navigator.js +468 -0
- package/dist/index.d.ts +18 -0
- package/dist/index.js +17 -0
- package/dist/legacy/agent-action-recovery.d.ts +45 -0
- package/dist/legacy/agent-action-recovery.js +370 -0
- package/dist/legacy/agent-message-utils.d.ts +21 -0
- package/dist/legacy/agent-message-utils.js +77 -0
- package/dist/legacy/agent-url-utils.d.ts +30 -0
- package/dist/legacy/agent-url-utils.js +138 -0
- package/dist/legacy/agent.d.ts +226 -0
- package/dist/legacy/agent.js +6666 -0
- package/dist/legacy/clip-orchestrator.d.ts +148 -0
- package/dist/legacy/clip-orchestrator.js +957 -0
- package/dist/legacy/credential-templates.d.ts +5 -0
- package/dist/legacy/credential-templates.js +60 -0
- package/dist/legacy/hybrid-navigator.d.ts +138 -0
- package/dist/legacy/hybrid-navigator.js +468 -0
- package/dist/legacy/llm-usage.d.ts +17 -0
- package/dist/legacy/llm-usage.js +45 -0
- package/dist/legacy/prompt-cache.d.ts +10 -0
- package/dist/legacy/prompt-cache.js +24 -0
- package/dist/legacy/prompts.d.ts +175 -0
- package/dist/legacy/prompts.js +1038 -0
- package/dist/legacy/tools.d.ts +4 -0
- package/dist/legacy/tools.js +216 -0
- package/dist/legacy/video-agent.d.ts +143 -0
- package/dist/legacy/video-agent.js +4788 -0
- package/dist/legacy/video-observation.d.ts +36 -0
- package/dist/legacy/video-observation.js +192 -0
- package/dist/legacy/video-planner.d.ts +12 -0
- package/dist/legacy/video-planner.js +501 -0
- package/dist/legacy/video-prompts.d.ts +37 -0
- package/dist/legacy/video-prompts.js +569 -0
- package/dist/legacy/video-tools.d.ts +3 -0
- package/dist/legacy/video-tools.js +59 -0
- package/dist/legacy/video-variant-state.d.ts +29 -0
- package/dist/legacy/video-variant-state.js +80 -0
- package/dist/legacy/vision-model.d.ts +17 -0
- package/dist/legacy/vision-model.js +74 -0
- package/dist/llm-healer.d.ts +63 -0
- package/dist/llm-healer.js +166 -0
- package/dist/llm-provider.d.ts +29 -0
- package/dist/llm-provider.js +80 -0
- package/dist/llm-usage.d.ts +17 -0
- package/dist/llm-usage.js +45 -0
- package/dist/logger.d.ts +6 -2
- package/dist/logger.js +15 -1
- package/dist/mockup-html.d.ts +119 -0
- package/dist/mockup-html.js +263 -0
- package/dist/mockup.d.ts +187 -0
- package/dist/mockup.js +869 -0
- package/dist/mouse-animation.d.ts +46 -0
- package/dist/mouse-animation.js +114 -0
- package/dist/opcode-actions.d.ts +42 -0
- package/dist/opcode-actions.js +511 -0
- package/dist/opcode-runner.d.ts +51 -0
- package/dist/opcode-runner.js +770 -0
- package/dist/openrouter-client.d.ts +40 -0
- package/dist/openrouter-client.js +16 -0
- package/dist/overlay-engine.d.ts +24 -0
- package/dist/overlay-engine.js +176 -0
- package/dist/overlay-utils.d.ts +14 -0
- package/dist/overlay-utils.js +13 -0
- package/dist/postcondition.d.ts +16 -0
- package/dist/postcondition.js +269 -0
- package/dist/posthog.d.ts +4 -0
- package/dist/posthog.js +26 -0
- package/dist/program-patcher.d.ts +25 -0
- package/dist/program-patcher.js +44 -0
- package/dist/prompt-cache.d.ts +10 -0
- package/dist/prompt-cache.js +24 -0
- package/dist/prompts.d.ts +175 -0
- package/dist/prompts.js +1038 -0
- package/dist/provider-config.d.ts +12 -0
- package/dist/provider-config.js +15 -0
- package/dist/recovery-chain.d.ts +37 -0
- package/dist/recovery-chain.js +350 -0
- package/dist/remote-browser.d.ts +215 -0
- package/dist/remote-browser.js +360 -0
- package/dist/safari-browser-bar.d.ts +15 -0
- package/dist/safari-browser-bar.js +95 -0
- package/dist/safari-toolbar-asset.d.ts +15 -0
- package/dist/safari-toolbar-asset.js +12 -0
- package/dist/security.d.ts +21 -0
- package/dist/security.js +608 -0
- package/dist/selector-resolver.d.ts +34 -0
- package/dist/selector-resolver.js +181 -0
- package/dist/semantic-resolver.d.ts +35 -0
- package/dist/semantic-resolver.js +161 -0
- package/dist/server-capture-runtime.d.ts +125 -0
- package/dist/server-capture-runtime.js +585 -0
- package/dist/server-credit-usage.d.ts +12 -0
- package/dist/server-credit-usage.js +41 -0
- package/dist/server-posthog.d.ts +2 -0
- package/dist/server-posthog.js +16 -0
- package/dist/server-project-webhooks.d.ts +59 -0
- package/dist/server-project-webhooks.js +123 -0
- package/dist/server-screenshot-watermark.d.ts +7 -0
- package/dist/server-screenshot-watermark.js +60 -0
- package/dist/session-profile.d.ts +86 -0
- package/dist/session-profile.js +1536 -0
- package/dist/sf-pro-fonts.d.ts +4 -0
- package/dist/sf-pro-fonts.js +7 -0
- package/dist/sf-pro-symbols.d.ts +1 -0
- package/dist/sf-pro-symbols.js +55 -0
- package/dist/skill-packaging.d.ts +28 -0
- package/dist/skill-packaging.js +169 -0
- package/dist/smart-wait.d.ts +27 -0
- package/dist/smart-wait.js +81 -0
- package/dist/status-bar-l10n.d.ts +14 -0
- package/dist/status-bar-l10n.js +177 -0
- package/dist/status-bar-render.d.ts +20 -0
- package/dist/status-bar-render.js +410 -0
- package/dist/status-bar.d.ts +53 -0
- package/dist/status-bar.js +620 -0
- package/dist/svg-browser-bar.d.ts +33 -0
- package/dist/svg-browser-bar.js +206 -0
- package/dist/svg-status-bar.d.ts +36 -0
- package/dist/svg-status-bar.js +597 -0
- package/dist/svg-text.d.ts +61 -0
- package/dist/svg-text.js +118 -0
- package/dist/tools.d.ts +4 -0
- package/dist/tools.js +216 -0
- package/dist/types.d.ts +240 -5
- package/dist/types.js +23 -1
- package/dist/v2/action-verifier.d.ts +29 -0
- package/dist/v2/action-verifier.js +133 -0
- package/dist/v2/alt-text.d.ts +26 -0
- package/dist/v2/alt-text.js +55 -0
- package/dist/v2/benchmark.d.ts +59 -0
- package/dist/v2/benchmark.js +135 -0
- package/dist/v2/capture-strategy.d.ts +30 -0
- package/dist/v2/capture-strategy.js +67 -0
- package/dist/v2/capture-verification.d.ts +35 -0
- package/dist/v2/capture-verification.js +95 -0
- package/dist/v2/circuit-breaker.d.ts +42 -0
- package/dist/v2/circuit-breaker.js +119 -0
- package/dist/v2/cli-runner-local.d.ts +11 -0
- package/dist/v2/cli-runner-local.js +91 -0
- package/dist/v2/cli-runner.d.ts +34 -0
- package/dist/v2/cli-runner.js +300 -0
- package/dist/v2/compiler-prompts.d.ts +27 -0
- package/dist/v2/compiler-prompts.js +123 -0
- package/dist/v2/compiler.d.ts +37 -0
- package/dist/v2/compiler.js +147 -0
- package/dist/v2/explorer.d.ts +41 -0
- package/dist/v2/explorer.js +56 -0
- package/dist/v2/index.d.ts +37 -0
- package/dist/v2/index.js +31 -0
- package/dist/v2/llm-healer.d.ts +62 -0
- package/dist/v2/llm-healer.js +166 -0
- package/dist/v2/llm-provider.d.ts +29 -0
- package/dist/v2/llm-provider.js +80 -0
- package/dist/v2/opcode-runner.d.ts +47 -0
- package/dist/v2/opcode-runner.js +634 -0
- package/dist/v2/overlay-engine.d.ts +24 -0
- package/dist/v2/overlay-engine.js +150 -0
- package/dist/v2/postcondition.d.ts +16 -0
- package/dist/v2/postcondition.js +249 -0
- package/dist/v2/program-patcher.d.ts +25 -0
- package/dist/v2/program-patcher.js +44 -0
- package/dist/v2/recovery-chain.d.ts +30 -0
- package/dist/v2/recovery-chain.js +368 -0
- package/dist/v2/schema.d.ts +2580 -0
- package/dist/v2/schema.js +295 -0
- package/dist/v2/selector-resolver.d.ts +34 -0
- package/dist/v2/selector-resolver.js +181 -0
- package/dist/v2/semantic-resolver.d.ts +35 -0
- package/dist/v2/semantic-resolver.js +161 -0
- package/dist/v2/smart-wait.d.ts +27 -0
- package/dist/v2/smart-wait.js +81 -0
- package/dist/v2/types.d.ts +444 -0
- package/dist/v2/types.js +19 -0
- package/dist/v2/web-playwright-local.d.ts +69 -0
- package/dist/v2/web-playwright-local.js +392 -0
- package/dist/version.d.ts +1 -0
- package/dist/version.js +5 -0
- package/dist/video-agent.d.ts +143 -0
- package/dist/video-agent.js +4788 -0
- package/dist/video-observation.d.ts +36 -0
- package/dist/video-observation.js +192 -0
- package/dist/video-planner.d.ts +12 -0
- package/dist/video-planner.js +501 -0
- package/dist/video-prompts.d.ts +37 -0
- package/dist/video-prompts.js +554 -0
- package/dist/video-tools.d.ts +3 -0
- package/dist/video-tools.js +59 -0
- package/dist/video-variant-state.d.ts +29 -0
- package/dist/video-variant-state.js +80 -0
- package/dist/vision-model.d.ts +17 -0
- package/dist/vision-model.js +74 -0
- package/dist/web-playwright-local.d.ts +126 -0
- package/dist/web-playwright-local.js +819 -0
- package/dist/ws-auth.d.ts +20 -0
- package/dist/ws-auth.js +70 -0
- package/dist/ws-broadcast.d.ts +34 -0
- package/dist/ws-broadcast.js +85 -0
- package/dist/ws-connection-limits.d.ts +12 -0
- package/dist/ws-connection-limits.js +44 -0
- package/dist/ws-handler-utils.d.ts +32 -0
- package/dist/ws-handler-utils.js +139 -0
- package/dist/ws-handler.d.ts +10 -0
- package/dist/ws-handler.js +1793 -0
- package/dist/ws-metrics-server.d.ts +9 -0
- package/dist/ws-metrics-server.js +31 -0
- package/dist/ws-server.d.ts +9 -0
- package/dist/ws-server.js +92 -0
- package/package.json +142 -71
package/dist/types.d.ts
CHANGED
|
@@ -39,6 +39,67 @@ export interface CapturePageIdentity {
|
|
|
39
39
|
dialogTarget: boolean;
|
|
40
40
|
dedicatedRoute: boolean;
|
|
41
41
|
}
|
|
42
|
+
export type CaptureExpectedSurfaceKind = "gallery" | "dialog_selection" | "dialog_configuration" | "editor_route" | "detail_route" | "route" | "unknown";
|
|
43
|
+
export type CaptureDialogPolicy = "must_be_open" | "must_be_closed" | "allow_either";
|
|
44
|
+
export type CaptureRoutePolicy = "exact_or_descendant" | "same_origin" | "ignore";
|
|
45
|
+
export type CaptureReusePolicy = "verified_only" | "same_surface_only" | "never";
|
|
46
|
+
export type CaptureVariantPolicy = "must_match" | "prefer_match" | "ignore";
|
|
47
|
+
export interface CapturePageContract {
|
|
48
|
+
expectedSurfaceKind: CaptureExpectedSurfaceKind;
|
|
49
|
+
dialogPolicy: CaptureDialogPolicy;
|
|
50
|
+
routePolicy: CaptureRoutePolicy;
|
|
51
|
+
reusePolicy: CaptureReusePolicy;
|
|
52
|
+
variantPolicy: CaptureVariantPolicy;
|
|
53
|
+
}
|
|
54
|
+
export type CaptureSessionTrust = "verified" | "degraded" | "dirty";
|
|
55
|
+
export interface CaptureCheckpointEvidence {
|
|
56
|
+
urlMatched: boolean;
|
|
57
|
+
dialogMatched: boolean;
|
|
58
|
+
authMatched: boolean;
|
|
59
|
+
variantMatched: boolean;
|
|
60
|
+
strongSurfaceObserved: boolean;
|
|
61
|
+
surfaceMatched?: boolean;
|
|
62
|
+
strongSurfaceSignature?: string | null;
|
|
63
|
+
primarySurface?: string | null;
|
|
64
|
+
overlaySurface?: string | null;
|
|
65
|
+
navigationSurface?: string | null;
|
|
66
|
+
configurationSurface?: string | null;
|
|
67
|
+
reasons: string[];
|
|
68
|
+
}
|
|
69
|
+
export interface CaptureTransitionDecision {
|
|
70
|
+
mode: CaptureSessionTrust;
|
|
71
|
+
contract: CapturePageContract;
|
|
72
|
+
evidence: CaptureCheckpointEvidence;
|
|
73
|
+
reasons: string[];
|
|
74
|
+
}
|
|
75
|
+
export interface CaptureBaselineResult extends CaptureTransitionDecision {
|
|
76
|
+
shouldNavigateToCanonical: boolean;
|
|
77
|
+
shouldRecreateContext: boolean;
|
|
78
|
+
allowFastPath: boolean;
|
|
79
|
+
}
|
|
80
|
+
export type CaptureSuccessVerdict = 'candidate_match' | 'blocked' | 'terminal_match';
|
|
81
|
+
export type VariantVerdict = 'matched' | 'mismatched' | 'ambiguous' | 'not_requested';
|
|
82
|
+
export type CaptureStage = 'session_bootstrap' | 'page_reachability' | 'variant_hygiene' | 'capture_readiness' | 'persist_and_handoff';
|
|
83
|
+
export type RegressionCause = 'auth_mismatch' | 'route_mismatch' | 'dialog_mismatch' | 'variant_hygiene_pending' | 'surface_too_weak' | 'repair_handoff' | 'stability_pending' | 'duplicate_candidate' | 'unknown';
|
|
84
|
+
export interface BlockerSet {
|
|
85
|
+
stage: CaptureStage;
|
|
86
|
+
reasons: string[];
|
|
87
|
+
}
|
|
88
|
+
export interface StageGate {
|
|
89
|
+
stage: CaptureStage;
|
|
90
|
+
blockers: BlockerSet;
|
|
91
|
+
regressionCause?: RegressionCause | null;
|
|
92
|
+
}
|
|
93
|
+
export interface ObservedTaskState {
|
|
94
|
+
authState: SessionProfileAuthState | 'ambiguous';
|
|
95
|
+
pageIdentity: CapturePageIdentity | null;
|
|
96
|
+
routeState: 'matched' | 'same_origin' | 'mismatched' | 'unknown';
|
|
97
|
+
dialogState: 'open' | 'closed' | 'ambiguous' | 'unknown';
|
|
98
|
+
variantState: VariantVerdict;
|
|
99
|
+
loadingState: 'stable' | 'loading' | 'unstable' | 'unknown';
|
|
100
|
+
surfaceConfidence: SessionValidationDiagnosticConfidence;
|
|
101
|
+
goalSatisfaction: CaptureSuccessVerdict;
|
|
102
|
+
}
|
|
42
103
|
export interface VariantValidatedCapture {
|
|
43
104
|
pageId: string;
|
|
44
105
|
prompt: string;
|
|
@@ -97,6 +158,14 @@ export interface CaptureCheckpoint {
|
|
|
97
158
|
canonicalUrl?: string | null;
|
|
98
159
|
resumeUrl?: string | null;
|
|
99
160
|
pageIdentity?: CapturePageIdentity | null;
|
|
161
|
+
dialogExpected?: boolean;
|
|
162
|
+
identityKind?: CapturePageIdentityKind | null;
|
|
163
|
+
surfaceSignature?: string | null;
|
|
164
|
+
strongSurfaceSignature?: string | null;
|
|
165
|
+
pageFingerprint?: string | null;
|
|
166
|
+
sessionTrust?: CaptureSessionTrust | null;
|
|
167
|
+
checkpointEvidence?: CaptureCheckpointEvidence | null;
|
|
168
|
+
checkpointType?: 'route' | 'dialog_open' | 'config_open' | 'content_ready' | 'gallery_detail' | 'editor_ready' | 'unknown';
|
|
100
169
|
verifiedAt: string;
|
|
101
170
|
}
|
|
102
171
|
export interface VariantCaptureRepairRecord {
|
|
@@ -135,6 +204,8 @@ export interface VariantCaptureManifest {
|
|
|
135
204
|
expectedPageIds: string[];
|
|
136
205
|
currentPageId?: string | null;
|
|
137
206
|
currentPageIdentity?: CapturePageIdentity | null;
|
|
207
|
+
promptFingerprint?: string | null;
|
|
208
|
+
pagePromptFingerprints?: Record<string, string>;
|
|
138
209
|
completedPages: string[];
|
|
139
210
|
remainingPages: string[];
|
|
140
211
|
previousValidatedCaptures: VariantValidatedCapture[];
|
|
@@ -157,6 +228,9 @@ export interface CaptureHandoffContext {
|
|
|
157
228
|
selectorHints?: string[];
|
|
158
229
|
navigationHints?: string[];
|
|
159
230
|
selectorMemory?: Record<string, string[]>;
|
|
231
|
+
trust?: CaptureSessionTrust;
|
|
232
|
+
strongCheckpointId?: string | null;
|
|
233
|
+
derivedFromRepair?: boolean;
|
|
160
234
|
}
|
|
161
235
|
export interface BrowserOptions {
|
|
162
236
|
headed: boolean;
|
|
@@ -503,7 +577,7 @@ export interface ClipOptions {
|
|
|
503
577
|
/** Usage metadata from a single OpenRouter API call */
|
|
504
578
|
export interface StepUsage {
|
|
505
579
|
stepNumber: number;
|
|
506
|
-
stepType: 'agent_iteration' | 'verification' | 'element_capture' | 'video_planning' | 'video_variant_classification' | 'video_step_verification' | 'video_step_fix' | 'assistant_chat' | 'mock_data_generation' | 'page_identity_classification';
|
|
580
|
+
stepType: 'agent_iteration' | 'verification' | 'element_capture' | 'video_planning' | 'video_variant_classification' | 'video_step_verification' | 'video_step_fix' | 'assistant_chat' | 'mock_data_generation' | 'page_identity_classification' | 'capture_verification' | 'alt_text_generation' | 'healer_invocation';
|
|
507
581
|
generationId: string | null;
|
|
508
582
|
modelRequested: string;
|
|
509
583
|
modelUsed: string | null;
|
|
@@ -532,16 +606,41 @@ export interface StepUsage {
|
|
|
532
606
|
sessionProfileReused?: boolean;
|
|
533
607
|
actionReplayUsed?: boolean;
|
|
534
608
|
}
|
|
609
|
+
export type AgentLane = 'fast_nav' | 'full_nav' | 'repair' | 'verification';
|
|
610
|
+
export type ActionExpectedEffect = 'route_change' | 'dialog_open' | 'menu_open' | 'panel_expand' | 'selection_applied' | 'content_identity_change' | 'no_effect_ok';
|
|
611
|
+
export type VerificationReadinessState = 'ready' | 'not_ready_actionable' | 'unstable_retryable';
|
|
612
|
+
export type ReplayFailureCode = 'wrong_route' | 'wrong_identity' | 'missing_dialog' | 'unstable_loading' | 'anchor_unresolved' | 'late_replay_divergence' | 'duplicate_verification_skipped' | 'execution_failed' | 'verification_failed';
|
|
613
|
+
export interface AgentRunTelemetry {
|
|
614
|
+
plannerCallCount: number;
|
|
615
|
+
visionCallCount: number;
|
|
616
|
+
verificationCallCount: number;
|
|
617
|
+
imageUploadCount: number;
|
|
618
|
+
verificationCacheHits: number;
|
|
619
|
+
replayTrimmedActionCount: number;
|
|
620
|
+
noEffectFailureCount: number;
|
|
621
|
+
unstableRetryableCount: number;
|
|
622
|
+
replayCheckpointMatchCount: number;
|
|
623
|
+
replayCheckpointMissCount: number;
|
|
624
|
+
replaySkipReason?: string | null;
|
|
625
|
+
replayFailReason?: string | null;
|
|
626
|
+
replayFailureCode?: ReplayFailureCode | null;
|
|
627
|
+
phaseTimingsMs: Record<string, number>;
|
|
628
|
+
reasoningEffortUsed?: 'low' | 'medium' | 'high' | 'xhigh' | 'off';
|
|
629
|
+
usedLitePageState?: boolean;
|
|
630
|
+
}
|
|
535
631
|
export interface VerificationResult {
|
|
536
632
|
verified: boolean;
|
|
633
|
+
successVerdict?: CaptureSuccessVerdict;
|
|
537
634
|
reason?: string;
|
|
538
|
-
usage: StepUsage
|
|
635
|
+
usage: StepUsage[];
|
|
539
636
|
fatal?: boolean;
|
|
540
637
|
matchedPageId?: string | null;
|
|
541
638
|
duplicateOfPageId?: string | null;
|
|
542
639
|
blockingReason?: string;
|
|
543
640
|
pageFingerprint?: string | null;
|
|
641
|
+
checkpointEvidence?: CaptureCheckpointEvidence | null;
|
|
544
642
|
mode?: 'deterministic' | 'vision' | 'text_fallback' | 'bailout';
|
|
643
|
+
readinessState?: VerificationReadinessState;
|
|
545
644
|
}
|
|
546
645
|
export interface SelectorValidationResult {
|
|
547
646
|
matchCount: number;
|
|
@@ -599,6 +698,7 @@ export interface AgentContextEntry {
|
|
|
599
698
|
export interface AgentConfig {
|
|
600
699
|
url: string;
|
|
601
700
|
prompt: string;
|
|
701
|
+
promptFingerprint?: string;
|
|
602
702
|
dark: boolean;
|
|
603
703
|
langs: string[];
|
|
604
704
|
outputDir: string;
|
|
@@ -633,8 +733,11 @@ export interface AgentConfig {
|
|
|
633
733
|
}>;
|
|
634
734
|
/** Language for the agent's reasoning/thinking output (e.g. "fr", "en") */
|
|
635
735
|
reasoningLocale?: string;
|
|
636
|
-
/** OpenRouter reasoning effort for compatible models
|
|
637
|
-
reasoningEffort?: 'low' | 'medium' | 'high' | 'off';
|
|
736
|
+
/** OpenRouter reasoning effort for compatible models. "off" disables. Default: "medium". */
|
|
737
|
+
reasoningEffort?: 'low' | 'medium' | 'high' | 'xhigh' | 'off';
|
|
738
|
+
/** Set of model IDs whose OpenRouter `supported_parameters` includes "reasoning". Resolved at capture start. */
|
|
739
|
+
reasoningCapableModels?: Set<string>;
|
|
740
|
+
lane?: AgentLane;
|
|
638
741
|
/** Past errors and user corrections, compressed into actionable hints. */
|
|
639
742
|
runHints?: AgentRunHint[];
|
|
640
743
|
/** Known selectors from previous successful runs, keyed by step signature */
|
|
@@ -679,6 +782,9 @@ export interface AgentConfig {
|
|
|
679
782
|
* Returns null if no pause is pending.
|
|
680
783
|
*/
|
|
681
784
|
guidanceCallback?: () => Promise<string | null>;
|
|
785
|
+
verificationCache?: Map<string, VerificationResult>;
|
|
786
|
+
runtimeTelemetry?: AgentRunTelemetry;
|
|
787
|
+
uploadImageMode?: 'prefer_data_uri' | 'prefer_upload';
|
|
682
788
|
/**
|
|
683
789
|
* Optional callback to upload a screenshot buffer to external storage and return a public URL.
|
|
684
790
|
* When provided, image messages use HTTPS URLs instead of base64 data URIs — improving
|
|
@@ -710,9 +816,119 @@ export interface InteractiveElement {
|
|
|
710
816
|
visible: boolean;
|
|
711
817
|
visibilityState: 'full' | 'partial' | 'offscreen';
|
|
712
818
|
}
|
|
819
|
+
export type AKType = 'button' | 'input' | 'link' | 'text' | 'image' | 'heading' | 'list' | 'table' | 'container' | 'icon' | 'toggle' | 'select' | 'checkbox' | 'radio' | 'slider' | 'tab' | 'modal' | 'nav' | 'form' | 'video' | 'audio' | 'iframe';
|
|
820
|
+
export type SemanticPattern = 'cookie-banner' | 'modal' | 'navbar' | 'footer' | 'hero' | 'sidebar' | 'dropdown' | 'toast' | 'tooltip' | 'form' | 'card' | 'pricing-table' | 'cta-group';
|
|
821
|
+
export type SemanticTrait = 'floating' | 'overlay' | 'sticky' | 'fixed' | 'above-fold' | 'below-fold' | 'full-width' | 'scrollable';
|
|
822
|
+
export interface AKBounds {
|
|
823
|
+
x: number;
|
|
824
|
+
y: number;
|
|
825
|
+
w: number;
|
|
826
|
+
h: number;
|
|
827
|
+
}
|
|
828
|
+
export interface ScrollState {
|
|
829
|
+
scrollTop: number;
|
|
830
|
+
scrollLeft: number;
|
|
831
|
+
scrollHeight: number;
|
|
832
|
+
scrollWidth: number;
|
|
833
|
+
clientHeight: number;
|
|
834
|
+
clientWidth: number;
|
|
835
|
+
overflowY: boolean;
|
|
836
|
+
overflowX: boolean;
|
|
837
|
+
}
|
|
838
|
+
export interface OverlayInfo {
|
|
839
|
+
nodeId: string;
|
|
840
|
+
zIndex: number;
|
|
841
|
+
coveragePercent: number;
|
|
842
|
+
blocksInteraction: boolean;
|
|
843
|
+
}
|
|
844
|
+
export interface OverlayScopeSummary {
|
|
845
|
+
nodeId: string;
|
|
846
|
+
kind: 'dialog' | 'menu' | 'popover' | 'dropdown' | 'overlay';
|
|
847
|
+
label: string;
|
|
848
|
+
interactiveCount: number;
|
|
849
|
+
subjectTokenOverlap: number;
|
|
850
|
+
visibleLabels: string[];
|
|
851
|
+
summary: string;
|
|
852
|
+
}
|
|
853
|
+
export interface AKNode {
|
|
854
|
+
id: string;
|
|
855
|
+
type: AKType;
|
|
856
|
+
label: string;
|
|
857
|
+
value?: string;
|
|
858
|
+
bounds: AKBounds;
|
|
859
|
+
visible: boolean;
|
|
860
|
+
interactive: boolean;
|
|
861
|
+
state: {
|
|
862
|
+
disabled: boolean;
|
|
863
|
+
focused: boolean;
|
|
864
|
+
checked?: boolean;
|
|
865
|
+
expanded?: boolean;
|
|
866
|
+
selected?: boolean;
|
|
867
|
+
};
|
|
868
|
+
style: {
|
|
869
|
+
bgColor?: string;
|
|
870
|
+
fgColor?: string;
|
|
871
|
+
fontSize?: number;
|
|
872
|
+
opacity?: number;
|
|
873
|
+
};
|
|
874
|
+
semantic: {
|
|
875
|
+
pattern?: SemanticPattern;
|
|
876
|
+
confidence: 'high' | 'medium';
|
|
877
|
+
traits: SemanticTrait[];
|
|
878
|
+
};
|
|
879
|
+
scroll?: ScrollState;
|
|
880
|
+
attributes: Record<string, string>;
|
|
881
|
+
children: AKNode[];
|
|
882
|
+
sourceRef: string;
|
|
883
|
+
}
|
|
884
|
+
export interface AKPageState {
|
|
885
|
+
url: string;
|
|
886
|
+
title: string;
|
|
887
|
+
viewport: {
|
|
888
|
+
width: number;
|
|
889
|
+
height: number;
|
|
890
|
+
};
|
|
891
|
+
scroll: ScrollState;
|
|
892
|
+
}
|
|
893
|
+
export interface AKTree {
|
|
894
|
+
root: AKNode;
|
|
895
|
+
page: AKPageState;
|
|
896
|
+
overlays: OverlayInfo[];
|
|
897
|
+
}
|
|
898
|
+
export interface FocusQuery {
|
|
899
|
+
type?: AKType[];
|
|
900
|
+
semantic?: SemanticPattern[];
|
|
901
|
+
trait?: SemanticTrait[];
|
|
902
|
+
interactive?: boolean;
|
|
903
|
+
visible?: boolean;
|
|
904
|
+
within?: string;
|
|
905
|
+
labelContains?: string;
|
|
906
|
+
maxDepth?: number;
|
|
907
|
+
includeAncestors?: boolean;
|
|
908
|
+
}
|
|
909
|
+
export interface AKNodeRuntimeIndexEntry {
|
|
910
|
+
id: string;
|
|
911
|
+
sourceRef: string;
|
|
912
|
+
bounds: AKBounds;
|
|
913
|
+
type: AKType;
|
|
914
|
+
label: string;
|
|
915
|
+
interactive: boolean;
|
|
916
|
+
visible: boolean;
|
|
917
|
+
value?: string;
|
|
918
|
+
attributes: Record<string, string>;
|
|
919
|
+
state: AKNode['state'];
|
|
920
|
+
semantic: AKNode['semantic'];
|
|
921
|
+
scroll?: ScrollState;
|
|
922
|
+
}
|
|
713
923
|
export interface PageState {
|
|
714
924
|
cleanScreenshot: Buffer;
|
|
715
925
|
screenshot: Buffer;
|
|
926
|
+
akTree: AKTree;
|
|
927
|
+
serializedAKTree: string;
|
|
928
|
+
/**
|
|
929
|
+
* Legacy observation fields retained temporarily for internal adapter paths.
|
|
930
|
+
* New screenshot-agent logic must read `akTree` / `serializedAKTree` instead.
|
|
931
|
+
*/
|
|
716
932
|
accessibilityTree: string;
|
|
717
933
|
interactiveElements: InteractiveElement[];
|
|
718
934
|
/** Simplified DOM: clean HTML structure without scripts/styles/classes, budget-capped. */
|
|
@@ -725,6 +941,8 @@ export interface PageState {
|
|
|
725
941
|
}
|
|
726
942
|
/** Lightweight page state without screenshots — used when vision is not needed. */
|
|
727
943
|
export interface PageStateLite {
|
|
944
|
+
akTree: AKTree;
|
|
945
|
+
serializedAKTree: string;
|
|
728
946
|
accessibilityTree: string;
|
|
729
947
|
interactiveElements: InteractiveElement[];
|
|
730
948
|
simplifiedDOM: string;
|
|
@@ -734,7 +952,7 @@ export interface PageStateLite {
|
|
|
734
952
|
viewportHeight: number;
|
|
735
953
|
};
|
|
736
954
|
}
|
|
737
|
-
export type ActionType = 'click' | 'type_text' | 'select_option' | 'scroll' | 'press_key' | 'wait' | 'dismiss_overlays' | 'search_text' | 'resize_viewport' | 'navigate_to' | 'analyze_screenshot' | 'take_screenshot' | 'ready_to_capture' | 'give_up' | 'note' | 'begin_subgoal' | 'capture_by_selector' | 'hover' | 'safe_expand' | 'scroll_to_element';
|
|
955
|
+
export type ActionType = 'tap' | 'type' | 'click' | 'type_text' | 'select_option' | 'scroll' | 'press_key' | 'wait' | 'dismiss_overlays' | 'search_text' | 'resize_viewport' | 'navigate_to' | 'analyze_screenshot' | 'capture' | 'focus' | 'take_screenshot' | 'ready_to_capture' | 'give_up' | 'note' | 'begin_subgoal' | 'capture_by_selector' | 'hover' | 'safe_expand' | 'scroll_to_element';
|
|
738
956
|
export interface ExecutedAction {
|
|
739
957
|
iteration: number;
|
|
740
958
|
action: ActionType;
|
|
@@ -743,6 +961,11 @@ export interface ExecutedAction {
|
|
|
743
961
|
error?: string;
|
|
744
962
|
outcome?: string;
|
|
745
963
|
stateChanged?: boolean;
|
|
964
|
+
expectedEffect?: ActionExpectedEffect;
|
|
965
|
+
observedEffects?: string[];
|
|
966
|
+
effectConfirmed?: boolean;
|
|
967
|
+
noEffectReason?: string | null;
|
|
968
|
+
failureClass?: string | null;
|
|
746
969
|
origin?: CaptureActionOrigin;
|
|
747
970
|
phase?: CaptureRunPhase;
|
|
748
971
|
checkpointId?: string | null;
|
|
@@ -769,6 +992,8 @@ export interface DiagnosticState {
|
|
|
769
992
|
}
|
|
770
993
|
export interface AgentResult {
|
|
771
994
|
success: boolean;
|
|
995
|
+
captureCompleted?: boolean;
|
|
996
|
+
captureNodeId?: string | null;
|
|
772
997
|
screenshotPath: string | null;
|
|
773
998
|
screenshots: WorkflowScreenshot[];
|
|
774
999
|
iterations: number;
|
|
@@ -780,7 +1005,17 @@ export interface AgentResult {
|
|
|
780
1005
|
deterministicRecoveryUsed?: boolean;
|
|
781
1006
|
evaluatorUsed?: boolean;
|
|
782
1007
|
verification?: VerificationResult;
|
|
1008
|
+
telemetry?: AgentRunTelemetry;
|
|
783
1009
|
}
|
|
1010
|
+
export declare function isStrongCheckpointVerified(evidence?: CaptureCheckpointEvidence | null): boolean;
|
|
1011
|
+
export declare function isTerminalVerificationSuccess(verification?: VerificationResult | null): verification is VerificationResult & {
|
|
1012
|
+
verified: true;
|
|
1013
|
+
successVerdict?: 'terminal_match';
|
|
1014
|
+
};
|
|
1015
|
+
export declare function isTerminalAgentResultSuccess(agentResult?: Pick<AgentResult, 'success' | 'verification' | 'captureCompleted'> | null): agentResult is Pick<AgentResult, 'success' | 'verification' | 'captureCompleted'> & {
|
|
1016
|
+
success: true;
|
|
1017
|
+
captureCompleted?: true;
|
|
1018
|
+
};
|
|
784
1019
|
export interface CaptureManifest {
|
|
785
1020
|
url: string;
|
|
786
1021
|
prompt: string;
|
package/dist/types.js
CHANGED
|
@@ -1,2 +1,24 @@
|
|
|
1
|
-
export {
|
|
1
|
+
export function isStrongCheckpointVerified(evidence) {
|
|
2
|
+
return !!evidence
|
|
3
|
+
&& evidence.urlMatched
|
|
4
|
+
&& evidence.dialogMatched
|
|
5
|
+
&& evidence.authMatched
|
|
6
|
+
&& evidence.variantMatched
|
|
7
|
+
&& (evidence.surfaceMatched ?? true)
|
|
8
|
+
&& evidence.strongSurfaceObserved;
|
|
9
|
+
}
|
|
10
|
+
export function isTerminalVerificationSuccess(verification) {
|
|
11
|
+
if (!verification?.verified)
|
|
12
|
+
return false;
|
|
13
|
+
if ((verification.successVerdict ?? 'terminal_match') !== 'terminal_match')
|
|
14
|
+
return false;
|
|
15
|
+
return isStrongCheckpointVerified(verification.checkpointEvidence ?? null);
|
|
16
|
+
}
|
|
17
|
+
export function isTerminalAgentResultSuccess(agentResult) {
|
|
18
|
+
if (!agentResult?.success)
|
|
19
|
+
return false;
|
|
20
|
+
if (agentResult.captureCompleted)
|
|
21
|
+
return true;
|
|
22
|
+
return isTerminalVerificationSuccess(agentResult.verification ?? null);
|
|
23
|
+
}
|
|
2
24
|
//# sourceMappingURL=types.js.map
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Capture Agent — Action Verifier
|
|
3
|
+
*
|
|
4
|
+
* Post-action validation: diff AKTree before/after an action
|
|
5
|
+
* to detect whether the action had a real effect or was intercepted.
|
|
6
|
+
*/
|
|
7
|
+
import type { RuntimeAdapter } from './types.js';
|
|
8
|
+
export interface ActionVerification {
|
|
9
|
+
/** Whether the action caused a detectable change */
|
|
10
|
+
hadEffect: boolean;
|
|
11
|
+
/** What changed */
|
|
12
|
+
changes: ActionChange[];
|
|
13
|
+
/** Summary for logging */
|
|
14
|
+
summary: string;
|
|
15
|
+
}
|
|
16
|
+
export type ActionChangeKind = 'url_changed' | 'tree_structure_changed' | 'node_appeared' | 'node_disappeared' | 'overlay_changed' | 'no_change';
|
|
17
|
+
export interface ActionChange {
|
|
18
|
+
kind: ActionChangeKind;
|
|
19
|
+
detail: string;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Captures AKTree state before and after an action to detect changes.
|
|
23
|
+
*/
|
|
24
|
+
export declare class ActionVerifier {
|
|
25
|
+
private beforeTree;
|
|
26
|
+
private beforeUrl;
|
|
27
|
+
captureBeforeState(adapter: RuntimeAdapter): Promise<void>;
|
|
28
|
+
verifyAfterAction(adapter: RuntimeAdapter): Promise<ActionVerification>;
|
|
29
|
+
}
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Capture Agent — Action Verifier
|
|
3
|
+
*
|
|
4
|
+
* Post-action validation: diff AKTree before/after an action
|
|
5
|
+
* to detect whether the action had a real effect or was intercepted.
|
|
6
|
+
*/
|
|
7
|
+
/**
|
|
8
|
+
* Captures AKTree state before and after an action to detect changes.
|
|
9
|
+
*/
|
|
10
|
+
export class ActionVerifier {
|
|
11
|
+
beforeTree = null;
|
|
12
|
+
beforeUrl = null;
|
|
13
|
+
async captureBeforeState(adapter) {
|
|
14
|
+
try {
|
|
15
|
+
this.beforeTree = await adapter.getAKTree();
|
|
16
|
+
}
|
|
17
|
+
catch {
|
|
18
|
+
this.beforeTree = null;
|
|
19
|
+
}
|
|
20
|
+
try {
|
|
21
|
+
this.beforeUrl = await adapter.getCurrentUrl();
|
|
22
|
+
}
|
|
23
|
+
catch {
|
|
24
|
+
this.beforeUrl = null;
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
async verifyAfterAction(adapter) {
|
|
28
|
+
if (!this.beforeTree || !this.beforeUrl) {
|
|
29
|
+
return {
|
|
30
|
+
hadEffect: true, // assume effect if no before state
|
|
31
|
+
changes: [],
|
|
32
|
+
summary: 'no before state captured, assuming action had effect',
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
let afterUrl;
|
|
36
|
+
let afterTree;
|
|
37
|
+
try {
|
|
38
|
+
[afterUrl, afterTree] = await Promise.all([
|
|
39
|
+
adapter.getCurrentUrl(),
|
|
40
|
+
adapter.getAKTree(),
|
|
41
|
+
]);
|
|
42
|
+
}
|
|
43
|
+
catch {
|
|
44
|
+
this.beforeTree = null;
|
|
45
|
+
this.beforeUrl = null;
|
|
46
|
+
return {
|
|
47
|
+
hadEffect: true,
|
|
48
|
+
changes: [],
|
|
49
|
+
summary: 'after state unavailable, skipping AKTree effect verification',
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
const changes = [];
|
|
53
|
+
// 1. URL change
|
|
54
|
+
if (afterUrl !== this.beforeUrl) {
|
|
55
|
+
changes.push({
|
|
56
|
+
kind: 'url_changed',
|
|
57
|
+
detail: `${this.beforeUrl} -> ${afterUrl}`,
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
// 2. Overlay changes
|
|
61
|
+
const beforeOverlayCount = this.beforeTree.overlays.length;
|
|
62
|
+
const afterOverlayCount = afterTree.overlays.length;
|
|
63
|
+
if (beforeOverlayCount !== afterOverlayCount) {
|
|
64
|
+
changes.push({
|
|
65
|
+
kind: 'overlay_changed',
|
|
66
|
+
detail: `overlays: ${beforeOverlayCount} -> ${afterOverlayCount}`,
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
// 3. Tree structure diff (node count at top level)
|
|
70
|
+
const beforeNodes = countVisibleNodes(this.beforeTree.root);
|
|
71
|
+
const afterNodes = countVisibleNodes(afterTree.root);
|
|
72
|
+
if (Math.abs(beforeNodes - afterNodes) > 2) {
|
|
73
|
+
changes.push({
|
|
74
|
+
kind: 'tree_structure_changed',
|
|
75
|
+
detail: `visible nodes: ${beforeNodes} -> ${afterNodes}`,
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
// 4. Check for new visible interactive nodes (e.g., modal appeared)
|
|
79
|
+
const beforeIds = collectVisibleInteractiveIds(this.beforeTree.root);
|
|
80
|
+
const afterIds = collectVisibleInteractiveIds(afterTree.root);
|
|
81
|
+
const appeared = afterIds.filter(id => !beforeIds.includes(id));
|
|
82
|
+
const disappeared = beforeIds.filter(id => !afterIds.includes(id));
|
|
83
|
+
if (appeared.length > 0) {
|
|
84
|
+
changes.push({
|
|
85
|
+
kind: 'node_appeared',
|
|
86
|
+
detail: `${appeared.length} new interactive node(s)`,
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
if (disappeared.length > 0) {
|
|
90
|
+
changes.push({
|
|
91
|
+
kind: 'node_disappeared',
|
|
92
|
+
detail: `${disappeared.length} interactive node(s) gone`,
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
// 5. Check page title change
|
|
96
|
+
if (this.beforeTree.page.title !== afterTree.page.title) {
|
|
97
|
+
changes.push({
|
|
98
|
+
kind: 'tree_structure_changed',
|
|
99
|
+
detail: `title: "${this.beforeTree.page.title}" -> "${afterTree.page.title}"`,
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
// If no changes detected at all
|
|
103
|
+
if (changes.length === 0) {
|
|
104
|
+
changes.push({ kind: 'no_change', detail: 'no detectable changes in URL, tree, or overlays' });
|
|
105
|
+
}
|
|
106
|
+
const hadEffect = changes.some(c => c.kind !== 'no_change');
|
|
107
|
+
const summary = changes.map(c => `${c.kind}: ${c.detail}`).join('; ');
|
|
108
|
+
// Reset state
|
|
109
|
+
this.beforeTree = null;
|
|
110
|
+
this.beforeUrl = null;
|
|
111
|
+
return { hadEffect, changes, summary };
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
// ── Helpers ─────────────────────────────────────────────────────────
|
|
115
|
+
function countVisibleNodes(node) {
|
|
116
|
+
let count = node.visible ? 1 : 0;
|
|
117
|
+
for (const child of node.children) {
|
|
118
|
+
count += countVisibleNodes(child);
|
|
119
|
+
}
|
|
120
|
+
return count;
|
|
121
|
+
}
|
|
122
|
+
function collectVisibleInteractiveIds(node) {
|
|
123
|
+
const ids = [];
|
|
124
|
+
function walk(n) {
|
|
125
|
+
if (n.visible && n.interactive) {
|
|
126
|
+
ids.push(n.id);
|
|
127
|
+
}
|
|
128
|
+
n.children.forEach(walk);
|
|
129
|
+
}
|
|
130
|
+
walk(node);
|
|
131
|
+
return ids;
|
|
132
|
+
}
|
|
133
|
+
//# sourceMappingURL=action-verifier.js.map
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Capture Agent — Alt Text Generation
|
|
3
|
+
*
|
|
4
|
+
* Generates accessible alt text for captured screenshots.
|
|
5
|
+
* Adapts to the variant's locale for localized descriptions.
|
|
6
|
+
* Cost: ~0.001 EUR per generation.
|
|
7
|
+
*/
|
|
8
|
+
import { type LLMProviderConfig, type LLMCallResult } from './llm-provider.js';
|
|
9
|
+
export interface AltTextResult {
|
|
10
|
+
altText: string;
|
|
11
|
+
llmResult?: LLMCallResult;
|
|
12
|
+
}
|
|
13
|
+
export interface AltTextContext {
|
|
14
|
+
/** What the screenshot captures */
|
|
15
|
+
description: string;
|
|
16
|
+
/** Target URL */
|
|
17
|
+
url: string;
|
|
18
|
+
/** Locale for the alt text language */
|
|
19
|
+
locale?: string;
|
|
20
|
+
/** Page/preset name */
|
|
21
|
+
presetName?: string;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Generates alt text for a screenshot, localized to the variant's language.
|
|
25
|
+
*/
|
|
26
|
+
export declare function generateAltText(screenshot: Buffer, context: AltTextContext, llmConfig: LLMProviderConfig): Promise<AltTextResult>;
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Capture Agent — Alt Text Generation
|
|
3
|
+
*
|
|
4
|
+
* Generates accessible alt text for captured screenshots.
|
|
5
|
+
* Adapts to the variant's locale for localized descriptions.
|
|
6
|
+
* Cost: ~0.001 EUR per generation.
|
|
7
|
+
*/
|
|
8
|
+
import { callLLM } from './llm-provider.js';
|
|
9
|
+
const SYSTEM_PROMPT = `You generate concise alt text for web page screenshots.
|
|
10
|
+
|
|
11
|
+
Rules:
|
|
12
|
+
- Maximum 150 characters
|
|
13
|
+
- Describe what is VISIBLE, not what the page is about
|
|
14
|
+
- Use the language specified by the locale
|
|
15
|
+
- Start with the main visual element (e.g., "Dashboard showing 5 projects")
|
|
16
|
+
- Do not wrap in quotes
|
|
17
|
+
- Do not start with "Screenshot of" or "Image of"
|
|
18
|
+
|
|
19
|
+
Respond with ONLY the alt text, nothing else.`;
|
|
20
|
+
/**
|
|
21
|
+
* Generates alt text for a screenshot, localized to the variant's language.
|
|
22
|
+
*/
|
|
23
|
+
export async function generateAltText(screenshot, context, llmConfig) {
|
|
24
|
+
const locale = context.locale ?? 'en';
|
|
25
|
+
const userPrompt = [
|
|
26
|
+
`Generate alt text in ${localeToLanguage(locale)} for this screenshot.`,
|
|
27
|
+
`Page: ${context.presetName ?? context.url}`,
|
|
28
|
+
`Content: ${context.description}`,
|
|
29
|
+
].join('\n');
|
|
30
|
+
try {
|
|
31
|
+
const result = await callLLM({ ...llmConfig, maxTokens: 100, timeoutMs: 10000 }, SYSTEM_PROMPT, userPrompt, screenshot);
|
|
32
|
+
// Clean up: remove quotes, trim
|
|
33
|
+
let altText = result.text.replace(/^["']|["']$/g, '').trim();
|
|
34
|
+
if (altText.length > 150)
|
|
35
|
+
altText = altText.slice(0, 147) + '...';
|
|
36
|
+
return { altText, llmResult: result };
|
|
37
|
+
}
|
|
38
|
+
catch {
|
|
39
|
+
// Fallback: generate from context without LLM
|
|
40
|
+
return {
|
|
41
|
+
altText: `${context.presetName ?? 'Page'} — ${context.description}`.slice(0, 150),
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
function localeToLanguage(locale) {
|
|
46
|
+
const map = {
|
|
47
|
+
en: 'English', fr: 'French', de: 'German', es: 'Spanish',
|
|
48
|
+
pt: 'Portuguese', it: 'Italian', nl: 'Dutch', ja: 'Japanese',
|
|
49
|
+
ko: 'Korean', zh: 'Chinese', ar: 'Arabic', ru: 'Russian',
|
|
50
|
+
pl: 'Polish', sv: 'Swedish', da: 'Danish', fi: 'Finnish',
|
|
51
|
+
nb: 'Norwegian', tr: 'Turkish', cs: 'Czech', ro: 'Romanian',
|
|
52
|
+
};
|
|
53
|
+
return map[locale.split('-')[0]] ?? locale;
|
|
54
|
+
}
|
|
55
|
+
//# sourceMappingURL=alt-text.js.map
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Capture Agent — Benchmark Harness
|
|
3
|
+
*
|
|
4
|
+
* Runs the same preset through V2, comparing:
|
|
5
|
+
* - Total LLM cost
|
|
6
|
+
* - Total wall-clock time
|
|
7
|
+
* - Success rate
|
|
8
|
+
* - Recovery rate (how often healing was needed)
|
|
9
|
+
*
|
|
10
|
+
* Used to prove V2 >= V1 before cutover.
|
|
11
|
+
*/
|
|
12
|
+
import type { ExecutionProgram, RunResult } from './types.js';
|
|
13
|
+
export interface BenchmarkPreset {
|
|
14
|
+
name: string;
|
|
15
|
+
presetId: string;
|
|
16
|
+
program: ExecutionProgram;
|
|
17
|
+
/** Expected behavior description for manual verification */
|
|
18
|
+
expectedBehavior: string;
|
|
19
|
+
/** Difficulty tier */
|
|
20
|
+
tier: 'simple' | 'multi_step' | 'auth_required' | 'clip' | 'video';
|
|
21
|
+
}
|
|
22
|
+
export interface BenchmarkRunResult {
|
|
23
|
+
presetName: string;
|
|
24
|
+
tier: string;
|
|
25
|
+
success: boolean;
|
|
26
|
+
totalDurationMs: number;
|
|
27
|
+
llmCostEur: number;
|
|
28
|
+
llmCallCount: number;
|
|
29
|
+
totalOpcodes: number;
|
|
30
|
+
recoveredOpcodes: number;
|
|
31
|
+
failedOpcodes: number;
|
|
32
|
+
healerInvocations: number;
|
|
33
|
+
circuitBreakerTrips: number;
|
|
34
|
+
variantCount: number;
|
|
35
|
+
successfulVariants: number;
|
|
36
|
+
error?: string;
|
|
37
|
+
}
|
|
38
|
+
export interface BenchmarkSummary {
|
|
39
|
+
totalPresets: number;
|
|
40
|
+
successCount: number;
|
|
41
|
+
failCount: number;
|
|
42
|
+
successRate: number;
|
|
43
|
+
averageDurationMs: number;
|
|
44
|
+
averageLlmCostEur: number;
|
|
45
|
+
totalLlmCostEur: number;
|
|
46
|
+
deterministicRate: number;
|
|
47
|
+
recoveryRate: number;
|
|
48
|
+
results: BenchmarkRunResult[];
|
|
49
|
+
byTier: Record<string, {
|
|
50
|
+
total: number;
|
|
51
|
+
success: number;
|
|
52
|
+
rate: number;
|
|
53
|
+
}>;
|
|
54
|
+
}
|
|
55
|
+
export type RunFn = (program: ExecutionProgram) => Promise<RunResult>;
|
|
56
|
+
export declare function runBenchmark(presets: BenchmarkPreset[], runCapture: RunFn, options?: {
|
|
57
|
+
runs?: number;
|
|
58
|
+
}): Promise<BenchmarkSummary>;
|
|
59
|
+
export declare function formatBenchmarkSummary(summary: BenchmarkSummary): string;
|