autokap 1.0.6 → 1.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/chrome/ios-statusbar-comparison-reference.jpg +0 -0
- package/assets/chrome/ios-statusbar-dark-reference.jpg +0 -0
- package/assets/chrome/ios-statusbar-light-reference.jpg +0 -0
- package/assets/cursors/macos.svg +4 -0
- package/assets/cursors/windows.svg +15 -0
- package/assets/devices/ipad-pro-11-m4.json +52 -0
- package/assets/devices/iphone-16-pro.json +53 -0
- package/assets/devices/macbook-air-13.json +45 -0
- package/assets/frames/MacBook Air 13.svg +242 -0
- package/assets/frames/Status bar - iPhone.png +0 -0
- Menu bar- iPad.png +0 -0
- package/assets/frames/iPad Pro M4 11_.png +0 -0
- package/assets/frames/iPhone 16 Pro.png +0 -0
- package/assets/icons/Cellular Connection.svg +3 -0
- package/assets/icons/Union.svg +6 -0
- package/assets/icons/Wifi.svg +3 -0
- package/assets/icons/battery.svg +5 -0
- package/assets/icons/battery_charging.svg +8 -0
- package/assets/skill/OPCODE-REFERENCE.md +607 -0
- package/assets/skill/README.md +39 -0
- package/assets/skill/SKILL.md +453 -468
- package/assets/skill/STUDIO-SKILL.md +476 -0
- package/assets/skill/references/examples.md +104 -0
- package/assets/skill/references/interactive-demo.md +225 -0
- package/assets/skill/references/mock-data.md +178 -0
- package/dist/abort.d.ts +5 -0
- package/dist/abort.js +44 -0
- package/dist/action-verifier.d.ts +29 -0
- package/dist/action-verifier.js +133 -0
- package/dist/agent-action-recovery.d.ts +45 -0
- package/dist/agent-action-recovery.js +370 -0
- package/dist/agent-message-utils.d.ts +21 -0
- package/dist/agent-message-utils.js +77 -0
- package/dist/agent-url-utils.d.ts +30 -0
- package/dist/agent-url-utils.js +138 -0
- package/dist/agent.d.ts +226 -0
- package/dist/agent.js +6666 -0
- package/dist/ak-tree.d.ts +39 -0
- package/dist/ak-tree.js +368 -0
- package/dist/alt-text.d.ts +26 -0
- package/dist/alt-text.js +55 -0
- package/dist/auth-capture.d.ts +17 -0
- package/dist/auth-capture.js +164 -0
- package/dist/benchmark.d.ts +59 -0
- package/dist/benchmark.js +135 -0
- package/dist/billing-operation-logging.d.ts +38 -0
- package/dist/billing-operation-logging.js +248 -0
- package/dist/browser-bar.d.ts +48 -0
- package/dist/browser-bar.js +284 -0
- package/dist/browser-pool.d.ts +7 -0
- package/dist/browser-pool.js +15 -5
- package/dist/browser-utils.d.ts +31 -0
- package/dist/browser-utils.js +97 -0
- package/dist/browser.d.ts +76 -1
- package/dist/browser.js +1657 -39
- package/dist/capture-alt-text.d.ts +12 -0
- package/dist/capture-alt-text.js +52 -0
- package/dist/capture-encryption.d.ts +10 -0
- package/dist/capture-encryption.js +41 -0
- package/dist/capture-language-preflight.d.ts +41 -0
- package/dist/capture-language-preflight.js +300 -0
- package/dist/capture-llm-page-identity.d.ts +15 -0
- package/dist/capture-llm-page-identity.js +128 -0
- package/dist/capture-model-resolution.d.ts +9 -0
- package/dist/capture-model-resolution.js +21 -0
- package/dist/capture-page-identity.d.ts +7 -0
- package/dist/capture-page-identity.js +352 -0
- package/dist/capture-preset-credentials.d.ts +62 -0
- package/dist/capture-preset-credentials.js +184 -0
- package/dist/capture-request-plan.d.ts +58 -0
- package/dist/capture-request-plan.js +264 -0
- package/dist/capture-run-optimizer.d.ts +139 -0
- package/dist/capture-run-optimizer.js +863 -0
- package/dist/capture-selector-memory.d.ts +31 -0
- package/dist/capture-selector-memory.js +345 -0
- package/dist/capture-session-profile-encryption.d.ts +2 -0
- package/dist/capture-session-profile-encryption.js +22 -0
- package/dist/capture-step-timeout.d.ts +10 -0
- package/dist/capture-step-timeout.js +30 -0
- package/dist/capture-strategy.d.ts +36 -0
- package/dist/capture-strategy.js +95 -0
- package/dist/capture-studio-sync.d.ts +23 -0
- package/dist/capture-studio-sync.js +172 -0
- package/dist/capture-surface-contract.d.ts +36 -0
- package/dist/capture-surface-contract.js +299 -0
- package/dist/capture-transition-engine.d.ts +28 -0
- package/dist/capture-transition-engine.js +292 -0
- package/dist/capture-variant-state.d.ts +56 -0
- package/dist/capture-variant-state.js +182 -0
- package/dist/capture-verification.d.ts +35 -0
- package/dist/capture-verification.js +95 -0
- package/dist/capture-viewport-lock.d.ts +48 -0
- package/dist/capture-viewport-lock.js +74 -0
- package/dist/circuit-breaker.d.ts +42 -0
- package/dist/circuit-breaker.js +119 -0
- package/dist/cli-config.d.ts +8 -1
- package/dist/cli-config.js +62 -6
- package/dist/cli-contract.d.ts +15 -0
- package/dist/cli-contract.js +167 -0
- package/dist/cli-runner-local.d.ts +12 -0
- package/dist/cli-runner-local.js +102 -0
- package/dist/cli-runner.d.ts +34 -0
- package/dist/cli-runner.js +433 -0
- package/dist/cli-utils.d.ts +0 -1
- package/dist/cli-utils.js +2 -5
- package/dist/cli.js +1005 -252
- package/dist/clip-orchestrator.d.ts +148 -0
- package/dist/clip-orchestrator.js +957 -0
- package/dist/clip-postprocess.d.ts +42 -0
- package/dist/clip-postprocess.js +201 -0
- package/dist/cookie-dismiss.d.ts +2 -0
- package/dist/cookie-dismiss.js +48 -13
- package/dist/cost-logging.d.ts +35 -0
- package/dist/cost-logging.js +242 -0
- package/dist/cost-resolution-monitor.d.ts +16 -0
- package/dist/cost-resolution-monitor.js +34 -0
- package/dist/credential-templates.d.ts +5 -0
- package/dist/credential-templates.js +60 -0
- package/dist/cursor-overlay-script.d.ts +6 -0
- package/dist/cursor-overlay-script.js +169 -0
- package/dist/dom-css-purger.d.ts +65 -0
- package/dist/dom-css-purger.js +333 -0
- package/dist/dom-font-inliner.d.ts +45 -0
- package/dist/dom-font-inliner.js +148 -0
- package/dist/dom-patch-resolver.d.ts +52 -0
- package/dist/dom-patch-resolver.js +242 -0
- package/dist/dom-serializer.d.ts +82 -0
- package/dist/dom-serializer.js +378 -0
- package/dist/element-capture.d.ts +13 -0
- package/dist/element-capture.js +522 -0
- package/dist/env-validation.d.ts +5 -0
- package/dist/env-validation.js +29 -0
- package/dist/execution-schema.d.ts +4423 -0
- package/dist/execution-schema.js +507 -0
- package/dist/execution-types.d.ts +886 -0
- package/dist/execution-types.js +65 -0
- package/dist/fonts-loader.d.ts +14 -0
- package/dist/fonts-loader.js +55 -0
- package/dist/hybrid-navigator.d.ts +138 -0
- package/dist/hybrid-navigator.js +468 -0
- package/dist/index.d.ts +18 -0
- package/dist/index.js +17 -0
- package/dist/legacy/agent-action-recovery.d.ts +45 -0
- package/dist/legacy/agent-action-recovery.js +370 -0
- package/dist/legacy/agent-message-utils.d.ts +21 -0
- package/dist/legacy/agent-message-utils.js +77 -0
- package/dist/legacy/agent-url-utils.d.ts +30 -0
- package/dist/legacy/agent-url-utils.js +138 -0
- package/dist/legacy/agent.d.ts +226 -0
- package/dist/legacy/agent.js +6666 -0
- package/dist/legacy/clip-orchestrator.d.ts +148 -0
- package/dist/legacy/clip-orchestrator.js +957 -0
- package/dist/legacy/credential-templates.d.ts +5 -0
- package/dist/legacy/credential-templates.js +60 -0
- package/dist/legacy/hybrid-navigator.d.ts +138 -0
- package/dist/legacy/hybrid-navigator.js +468 -0
- package/dist/legacy/llm-usage.d.ts +17 -0
- package/dist/legacy/llm-usage.js +45 -0
- package/dist/legacy/prompt-cache.d.ts +10 -0
- package/dist/legacy/prompt-cache.js +24 -0
- package/dist/legacy/prompts.d.ts +175 -0
- package/dist/legacy/prompts.js +1038 -0
- package/dist/legacy/tools.d.ts +4 -0
- package/dist/legacy/tools.js +216 -0
- package/dist/legacy/video-agent.d.ts +143 -0
- package/dist/legacy/video-agent.js +4788 -0
- package/dist/legacy/video-observation.d.ts +36 -0
- package/dist/legacy/video-observation.js +192 -0
- package/dist/legacy/video-planner.d.ts +12 -0
- package/dist/legacy/video-planner.js +501 -0
- package/dist/legacy/video-prompts.d.ts +37 -0
- package/dist/legacy/video-prompts.js +569 -0
- package/dist/legacy/video-tools.d.ts +3 -0
- package/dist/legacy/video-tools.js +59 -0
- package/dist/legacy/video-variant-state.d.ts +29 -0
- package/dist/legacy/video-variant-state.js +80 -0
- package/dist/legacy/vision-model.d.ts +17 -0
- package/dist/legacy/vision-model.js +74 -0
- package/dist/llm-healer.d.ts +63 -0
- package/dist/llm-healer.js +166 -0
- package/dist/llm-provider.d.ts +29 -0
- package/dist/llm-provider.js +80 -0
- package/dist/llm-usage.d.ts +17 -0
- package/dist/llm-usage.js +45 -0
- package/dist/logger.d.ts +6 -2
- package/dist/logger.js +15 -1
- package/dist/mockup-html.d.ts +119 -0
- package/dist/mockup-html.js +263 -0
- package/dist/mockup.d.ts +187 -0
- package/dist/mockup.js +869 -0
- package/dist/mouse-animation.d.ts +46 -0
- package/dist/mouse-animation.js +114 -0
- package/dist/opcode-actions.d.ts +42 -0
- package/dist/opcode-actions.js +511 -0
- package/dist/opcode-runner.d.ts +51 -0
- package/dist/opcode-runner.js +770 -0
- package/dist/openrouter-client.d.ts +40 -0
- package/dist/openrouter-client.js +16 -0
- package/dist/overlay-engine.d.ts +24 -0
- package/dist/overlay-engine.js +176 -0
- package/dist/overlay-utils.d.ts +14 -0
- package/dist/overlay-utils.js +13 -0
- package/dist/postcondition.d.ts +16 -0
- package/dist/postcondition.js +269 -0
- package/dist/posthog.d.ts +4 -0
- package/dist/posthog.js +26 -0
- package/dist/program-patcher.d.ts +25 -0
- package/dist/program-patcher.js +44 -0
- package/dist/prompt-cache.d.ts +10 -0
- package/dist/prompt-cache.js +24 -0
- package/dist/prompts.d.ts +175 -0
- package/dist/prompts.js +1038 -0
- package/dist/provider-config.d.ts +12 -0
- package/dist/provider-config.js +15 -0
- package/dist/recovery-chain.d.ts +37 -0
- package/dist/recovery-chain.js +350 -0
- package/dist/remote-browser.d.ts +215 -0
- package/dist/remote-browser.js +360 -0
- package/dist/safari-browser-bar.d.ts +15 -0
- package/dist/safari-browser-bar.js +95 -0
- package/dist/safari-toolbar-asset.d.ts +15 -0
- package/dist/safari-toolbar-asset.js +12 -0
- package/dist/security.d.ts +21 -0
- package/dist/security.js +608 -0
- package/dist/selector-resolver.d.ts +34 -0
- package/dist/selector-resolver.js +181 -0
- package/dist/semantic-resolver.d.ts +35 -0
- package/dist/semantic-resolver.js +161 -0
- package/dist/server-capture-runtime.d.ts +125 -0
- package/dist/server-capture-runtime.js +585 -0
- package/dist/server-credit-usage.d.ts +12 -0
- package/dist/server-credit-usage.js +41 -0
- package/dist/server-posthog.d.ts +2 -0
- package/dist/server-posthog.js +16 -0
- package/dist/server-project-webhooks.d.ts +59 -0
- package/dist/server-project-webhooks.js +123 -0
- package/dist/server-screenshot-watermark.d.ts +7 -0
- package/dist/server-screenshot-watermark.js +60 -0
- package/dist/session-profile.d.ts +86 -0
- package/dist/session-profile.js +1536 -0
- package/dist/sf-pro-fonts.d.ts +4 -0
- package/dist/sf-pro-fonts.js +7 -0
- package/dist/sf-pro-symbols.d.ts +1 -0
- package/dist/sf-pro-symbols.js +55 -0
- package/dist/skill-packaging.d.ts +28 -0
- package/dist/skill-packaging.js +169 -0
- package/dist/smart-wait.d.ts +27 -0
- package/dist/smart-wait.js +81 -0
- package/dist/status-bar-l10n.d.ts +14 -0
- package/dist/status-bar-l10n.js +177 -0
- package/dist/status-bar-render.d.ts +20 -0
- package/dist/status-bar-render.js +410 -0
- package/dist/status-bar.d.ts +53 -0
- package/dist/status-bar.js +620 -0
- package/dist/svg-browser-bar.d.ts +33 -0
- package/dist/svg-browser-bar.js +206 -0
- package/dist/svg-status-bar.d.ts +36 -0
- package/dist/svg-status-bar.js +597 -0
- package/dist/svg-text.d.ts +61 -0
- package/dist/svg-text.js +118 -0
- package/dist/tools.d.ts +4 -0
- package/dist/tools.js +216 -0
- package/dist/types.d.ts +240 -5
- package/dist/types.js +23 -1
- package/dist/v2/action-verifier.d.ts +29 -0
- package/dist/v2/action-verifier.js +133 -0
- package/dist/v2/alt-text.d.ts +26 -0
- package/dist/v2/alt-text.js +55 -0
- package/dist/v2/benchmark.d.ts +59 -0
- package/dist/v2/benchmark.js +135 -0
- package/dist/v2/capture-strategy.d.ts +30 -0
- package/dist/v2/capture-strategy.js +67 -0
- package/dist/v2/capture-verification.d.ts +35 -0
- package/dist/v2/capture-verification.js +95 -0
- package/dist/v2/circuit-breaker.d.ts +42 -0
- package/dist/v2/circuit-breaker.js +119 -0
- package/dist/v2/cli-runner-local.d.ts +11 -0
- package/dist/v2/cli-runner-local.js +91 -0
- package/dist/v2/cli-runner.d.ts +34 -0
- package/dist/v2/cli-runner.js +300 -0
- package/dist/v2/compiler-prompts.d.ts +27 -0
- package/dist/v2/compiler-prompts.js +123 -0
- package/dist/v2/compiler.d.ts +37 -0
- package/dist/v2/compiler.js +147 -0
- package/dist/v2/explorer.d.ts +41 -0
- package/dist/v2/explorer.js +56 -0
- package/dist/v2/index.d.ts +37 -0
- package/dist/v2/index.js +31 -0
- package/dist/v2/llm-healer.d.ts +62 -0
- package/dist/v2/llm-healer.js +166 -0
- package/dist/v2/llm-provider.d.ts +29 -0
- package/dist/v2/llm-provider.js +80 -0
- package/dist/v2/opcode-runner.d.ts +47 -0
- package/dist/v2/opcode-runner.js +634 -0
- package/dist/v2/overlay-engine.d.ts +24 -0
- package/dist/v2/overlay-engine.js +150 -0
- package/dist/v2/postcondition.d.ts +16 -0
- package/dist/v2/postcondition.js +249 -0
- package/dist/v2/program-patcher.d.ts +25 -0
- package/dist/v2/program-patcher.js +44 -0
- package/dist/v2/recovery-chain.d.ts +30 -0
- package/dist/v2/recovery-chain.js +368 -0
- package/dist/v2/schema.d.ts +2580 -0
- package/dist/v2/schema.js +295 -0
- package/dist/v2/selector-resolver.d.ts +34 -0
- package/dist/v2/selector-resolver.js +181 -0
- package/dist/v2/semantic-resolver.d.ts +35 -0
- package/dist/v2/semantic-resolver.js +161 -0
- package/dist/v2/smart-wait.d.ts +27 -0
- package/dist/v2/smart-wait.js +81 -0
- package/dist/v2/types.d.ts +444 -0
- package/dist/v2/types.js +19 -0
- package/dist/v2/web-playwright-local.d.ts +69 -0
- package/dist/v2/web-playwright-local.js +392 -0
- package/dist/version.d.ts +1 -0
- package/dist/version.js +5 -0
- package/dist/video-agent.d.ts +143 -0
- package/dist/video-agent.js +4788 -0
- package/dist/video-observation.d.ts +36 -0
- package/dist/video-observation.js +192 -0
- package/dist/video-planner.d.ts +12 -0
- package/dist/video-planner.js +501 -0
- package/dist/video-prompts.d.ts +37 -0
- package/dist/video-prompts.js +554 -0
- package/dist/video-tools.d.ts +3 -0
- package/dist/video-tools.js +59 -0
- package/dist/video-variant-state.d.ts +29 -0
- package/dist/video-variant-state.js +80 -0
- package/dist/vision-model.d.ts +17 -0
- package/dist/vision-model.js +74 -0
- package/dist/web-playwright-local.d.ts +126 -0
- package/dist/web-playwright-local.js +819 -0
- package/dist/ws-auth.d.ts +20 -0
- package/dist/ws-auth.js +70 -0
- package/dist/ws-broadcast.d.ts +34 -0
- package/dist/ws-broadcast.js +85 -0
- package/dist/ws-connection-limits.d.ts +12 -0
- package/dist/ws-connection-limits.js +44 -0
- package/dist/ws-handler-utils.d.ts +32 -0
- package/dist/ws-handler-utils.js +139 -0
- package/dist/ws-handler.d.ts +10 -0
- package/dist/ws-handler.js +1793 -0
- package/dist/ws-metrics-server.d.ts +9 -0
- package/dist/ws-metrics-server.js +31 -0
- package/dist/ws-server.d.ts +9 -0
- package/dist/ws-server.js +92 -0
- package/package.json +142 -71
|
@@ -0,0 +1,886 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Capture Agent — IR Types
|
|
3
|
+
*
|
|
4
|
+
* All types for the compiled execution model:
|
|
5
|
+
* preset (natural language) -> ExecutionProgram (typed IR) -> deterministic runtime
|
|
6
|
+
*/
|
|
7
|
+
import type { AKTree, BrowserStorageState, BrowserSessionStorageState, VideoCursorTheme, VideoPageSignals } from './types.js';
|
|
8
|
+
/** Sentinel value that resolves to the current variant's locale or theme at runtime */
|
|
9
|
+
export declare const VARIANT_PLACEHOLDER: "$variant";
|
|
10
|
+
export declare const OPCODE_KINDS: readonly ["NAVIGATE", "DISMISS_OVERLAYS", "ASSERT_ROUTE", "ASSERT_SURFACE", "CLICK", "TYPE", "PRESS_KEY", "WAIT_FOR", "SET_LOCALE", "SET_THEME", "SCROLL", "CAPTURE_SCREENSHOT", "CAPTURE_DOM", "CAPTURE_FRAGMENT", "BEGIN_CLIP", "END_CLIP", "HOVER", "SELECT_OPTION", "CHECK", "DOUBLE_CLICK", "CLONE_ELEMENT", "INJECT_MOCK_DATA", "REMOVE_ELEMENT", "SET_ATTRIBUTE"];
|
|
11
|
+
export type OpcodeKind = (typeof OPCODE_KINDS)[number];
|
|
12
|
+
/**
|
|
13
|
+
* Soft opcodes are non-blocking — if their action or postcondition fails at
|
|
14
|
+
* runtime, the runner returns `status: 'skipped'` instead of aborting the
|
|
15
|
+
* variant. The circuit breaker is not ticked. Used by mock data injection so
|
|
16
|
+
* a missing template selector logs a warning and the capture proceeds.
|
|
17
|
+
*/
|
|
18
|
+
export declare const SOFT_OPCODE_KINDS: ReadonlySet<OpcodeKind>;
|
|
19
|
+
export declare function isSoftOpcodeKind(kind: OpcodeKind): boolean;
|
|
20
|
+
/** A single fillable field within a mock data group's templated item. */
|
|
21
|
+
export interface MockDataSlot {
|
|
22
|
+
/** Stable identifier referenced by INJECT_MOCK_DATA.slotMappings[].slot */
|
|
23
|
+
name: string;
|
|
24
|
+
/** Human-readable description shown in the Phase 3 editing UI */
|
|
25
|
+
description: string;
|
|
26
|
+
/**
|
|
27
|
+
* Optional hint about how the Phase 3 editing UI should render the input.
|
|
28
|
+
* Canonical values recognized by the dashboard editor are `text`,
|
|
29
|
+
* `number`, `image`, `color`, `date`, `url`. Any other string is allowed
|
|
30
|
+
* and the editor falls back to a plain text input.
|
|
31
|
+
*/
|
|
32
|
+
hint?: string;
|
|
33
|
+
}
|
|
34
|
+
/** One concrete row of values, keyed by slot name. */
|
|
35
|
+
export type MockDataRow = Record<string, string>;
|
|
36
|
+
/** A mock data group describes one fillable area of the UI (e.g. a table body). */
|
|
37
|
+
export interface MockDataGroup {
|
|
38
|
+
/** Stable identifier referenced by INJECT_MOCK_DATA.groupName. Kebab-case. */
|
|
39
|
+
name: string;
|
|
40
|
+
/** Human-readable description for the editing UI */
|
|
41
|
+
description: string;
|
|
42
|
+
/** Schema of each item in this group */
|
|
43
|
+
slots: MockDataSlot[];
|
|
44
|
+
/** Default seed values — one row per item the user wants rendered */
|
|
45
|
+
defaultValues: MockDataRow[];
|
|
46
|
+
/**
|
|
47
|
+
* If true, after cloning the runtime removes every existing child of the
|
|
48
|
+
* container that is not a clone. Used to wipe placeholders / real items so
|
|
49
|
+
* the mocked dataset is the sole content. Default: false.
|
|
50
|
+
*/
|
|
51
|
+
replaceExisting?: boolean;
|
|
52
|
+
}
|
|
53
|
+
export type PostconditionType = 'route_matches' | 'element_visible' | 'element_absent' | 'text_contains' | 'overlay_dismissed' | 'screenshot_stable' | 'any_change' | 'always';
|
|
54
|
+
export interface PostconditionSpec {
|
|
55
|
+
type: PostconditionType;
|
|
56
|
+
/** URL pattern (glob) for route_matches */
|
|
57
|
+
pattern?: string;
|
|
58
|
+
/** CSS/AKTree selector for element_visible, element_absent, text_contains */
|
|
59
|
+
selector?: string;
|
|
60
|
+
/** Expected text substring for text_contains */
|
|
61
|
+
text?: string;
|
|
62
|
+
/** Pixel diff threshold (0-1) for screenshot_stable. Default: 0.01 */
|
|
63
|
+
threshold?: number;
|
|
64
|
+
/** Max wait time (ms) for the postcondition to become true. Default: 5000 */
|
|
65
|
+
waitMs?: number;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Describes an element semantically, without requiring a CSS selector.
|
|
69
|
+
* The runtime resolves the target using Playwright semantic locators
|
|
70
|
+
* (getByRole, getByText, getByLabel) then falls back to AKTree fuzzy matching.
|
|
71
|
+
*
|
|
72
|
+
* At least one of text/role/label should be provided.
|
|
73
|
+
*/
|
|
74
|
+
export interface SemanticTarget {
|
|
75
|
+
/** Visible text content of the element (exact or partial match) */
|
|
76
|
+
text?: string;
|
|
77
|
+
/** ARIA role: "button", "link", "textbox", "checkbox", "tab", "menuitem", etc. */
|
|
78
|
+
role?: string;
|
|
79
|
+
/** Accessible name: aria-label, associated label text, or title attribute */
|
|
80
|
+
label?: string;
|
|
81
|
+
/** Nearby text or heading for disambiguation (e.g. "in the pricing section") */
|
|
82
|
+
near?: string;
|
|
83
|
+
/** Placeholder text (for inputs) */
|
|
84
|
+
placeholder?: string;
|
|
85
|
+
/** Whether the text match should be exact. Default: false (substring match) */
|
|
86
|
+
exact?: boolean;
|
|
87
|
+
}
|
|
88
|
+
export interface RecoveryPolicy {
|
|
89
|
+
/** Number of deterministic retries before escalating. Default: 2 */
|
|
90
|
+
retries: number;
|
|
91
|
+
/** Try selector memory alternatives. Default: true */
|
|
92
|
+
useSelectorMemory: boolean;
|
|
93
|
+
/** Try alternative interaction methods (keyboard, JS, coords). Default: true */
|
|
94
|
+
useAltInteraction: boolean;
|
|
95
|
+
/** Reload the page and retry from last checkpoint. Default: false */
|
|
96
|
+
allowReload: boolean;
|
|
97
|
+
/** Allow LLM healer as last resort. Default: true */
|
|
98
|
+
allowHealer: boolean;
|
|
99
|
+
}
|
|
100
|
+
export declare const DEFAULT_RECOVERY_POLICY: RecoveryPolicy;
|
|
101
|
+
interface OpcodeBase {
|
|
102
|
+
kind: OpcodeKind;
|
|
103
|
+
/** Human-readable description of what this opcode does */
|
|
104
|
+
description: string;
|
|
105
|
+
postcondition: PostconditionSpec;
|
|
106
|
+
recovery: RecoveryPolicy;
|
|
107
|
+
/** Max time (ms) for this opcode to complete, including recovery. Default: 15000 */
|
|
108
|
+
timeoutMs: number;
|
|
109
|
+
/** Max recovery attempts before this opcode is considered failed. Default: 3 */
|
|
110
|
+
maxFailures: number;
|
|
111
|
+
}
|
|
112
|
+
export interface NavigateOpcode extends OpcodeBase {
|
|
113
|
+
kind: 'NAVIGATE';
|
|
114
|
+
url: string;
|
|
115
|
+
}
|
|
116
|
+
export interface DismissOverlaysOpcode extends OpcodeBase {
|
|
117
|
+
kind: 'DISMISS_OVERLAYS';
|
|
118
|
+
}
|
|
119
|
+
export interface AssertRouteOpcode extends OpcodeBase {
|
|
120
|
+
kind: 'ASSERT_ROUTE';
|
|
121
|
+
/** URL pattern (glob or regex) to assert */
|
|
122
|
+
urlPattern: string;
|
|
123
|
+
}
|
|
124
|
+
export interface AssertSurfaceOpcode extends OpcodeBase {
|
|
125
|
+
kind: 'ASSERT_SURFACE';
|
|
126
|
+
/** Selector(s) that must be visible */
|
|
127
|
+
selectors: string[];
|
|
128
|
+
/** If true, ALL selectors must match. If false, ANY one suffices. Default: true */
|
|
129
|
+
matchAll: boolean;
|
|
130
|
+
}
|
|
131
|
+
export interface ClickOpcode extends OpcodeBase {
|
|
132
|
+
kind: 'CLICK';
|
|
133
|
+
/** CSS selector — typically a data-ak attribute set by the user's AI assistant */
|
|
134
|
+
selector: string;
|
|
135
|
+
/** Semantic target fallback — used by recovery chain when selector fails */
|
|
136
|
+
target?: SemanticTarget;
|
|
137
|
+
/** Mouse button: 'right' for context menus, 'middle' for new tab. Default: left click */
|
|
138
|
+
button?: 'right' | 'middle';
|
|
139
|
+
/** AKTree fingerprint for fuzzy matching fallback */
|
|
140
|
+
fingerprint?: string;
|
|
141
|
+
/** Alternative selectors to try in order */
|
|
142
|
+
selectorAlternates?: string[];
|
|
143
|
+
}
|
|
144
|
+
export interface TypeOpcode extends OpcodeBase {
|
|
145
|
+
kind: 'TYPE';
|
|
146
|
+
/** CSS selector — typically a data-ak attribute set by the user's AI assistant */
|
|
147
|
+
selector: string;
|
|
148
|
+
/** Semantic target fallback */
|
|
149
|
+
target?: SemanticTarget;
|
|
150
|
+
text: string;
|
|
151
|
+
/** Locale-keyed text overrides. When the current variant has a matching locale key,
|
|
152
|
+
* that value is used instead of `text`. Supports credential placeholders. */
|
|
153
|
+
textByLocale?: Record<string, string>;
|
|
154
|
+
/** Clear existing value before typing. Default: true */
|
|
155
|
+
clearFirst: boolean;
|
|
156
|
+
fingerprint?: string;
|
|
157
|
+
selectorAlternates?: string[];
|
|
158
|
+
}
|
|
159
|
+
export interface PressKeyOpcode extends OpcodeBase {
|
|
160
|
+
kind: 'PRESS_KEY';
|
|
161
|
+
key: string;
|
|
162
|
+
}
|
|
163
|
+
export interface WaitForOpcode extends OpcodeBase {
|
|
164
|
+
kind: 'WAIT_FOR';
|
|
165
|
+
/** CSS selector to wait for — required unless target is provided */
|
|
166
|
+
selector?: string;
|
|
167
|
+
/** Semantic target fallback — used when selector is not available */
|
|
168
|
+
target?: SemanticTarget;
|
|
169
|
+
/** 'visible' = element visible in viewport, 'attached' = exists in DOM. Default: 'visible' */
|
|
170
|
+
state: 'visible' | 'attached';
|
|
171
|
+
}
|
|
172
|
+
export interface SetLocaleOpcode extends OpcodeBase {
|
|
173
|
+
kind: 'SET_LOCALE';
|
|
174
|
+
/** Target BCP-47 locale (e.g. "fr", "en-US") */
|
|
175
|
+
locale: string;
|
|
176
|
+
/** Strategy to set locale */
|
|
177
|
+
method: 'browser_context' | 'ui_interaction' | 'storage';
|
|
178
|
+
/** Selector to interact with (for ui_interaction) */
|
|
179
|
+
selector?: string;
|
|
180
|
+
/** Storage key/value pairs to set (for storage method) */
|
|
181
|
+
storageHints?: {
|
|
182
|
+
storage: 'localStorage' | 'sessionStorage' | 'cookie';
|
|
183
|
+
key: string;
|
|
184
|
+
value: string;
|
|
185
|
+
}[];
|
|
186
|
+
}
|
|
187
|
+
export interface SetThemeOpcode extends OpcodeBase {
|
|
188
|
+
kind: 'SET_THEME';
|
|
189
|
+
theme: 'light' | 'dark' | '$variant';
|
|
190
|
+
method: 'color_scheme' | 'ui_interaction' | 'storage';
|
|
191
|
+
selector?: string;
|
|
192
|
+
storageHints?: {
|
|
193
|
+
storage: 'localStorage' | 'sessionStorage' | 'cookie';
|
|
194
|
+
key: string;
|
|
195
|
+
value: string;
|
|
196
|
+
}[];
|
|
197
|
+
}
|
|
198
|
+
export interface ScrollOpcode extends OpcodeBase {
|
|
199
|
+
kind: 'SCROLL';
|
|
200
|
+
direction: 'up' | 'down' | 'left' | 'right';
|
|
201
|
+
/** Pixels to scroll. Default: viewport height */
|
|
202
|
+
amount?: number;
|
|
203
|
+
/** Optional target selector to scroll into view */
|
|
204
|
+
targetSelector?: string;
|
|
205
|
+
/** Semantic target to scroll into view (used when targetSelector is absent) */
|
|
206
|
+
target?: SemanticTarget;
|
|
207
|
+
}
|
|
208
|
+
export interface CaptureScreenshotOpcode extends OpcodeBase {
|
|
209
|
+
kind: 'CAPTURE_SCREENSHOT';
|
|
210
|
+
/** Stable preset capture identifier (page id / element name) for Studio and dev links */
|
|
211
|
+
captureId?: string;
|
|
212
|
+
/** Human-readable preset capture name */
|
|
213
|
+
captureName?: string;
|
|
214
|
+
/** Optional element selector for element-level capture */
|
|
215
|
+
elementSelector?: string;
|
|
216
|
+
}
|
|
217
|
+
/**
|
|
218
|
+
* Capture the full serialized DOM of the current page as an interactive demo
|
|
219
|
+
* state. Produces an HTML artifact (no PNG) that the demo player can replay
|
|
220
|
+
* client-side. Used by AUT-121 Interactive Demos preset type.
|
|
221
|
+
*
|
|
222
|
+
* The runner runs `smartWaitForStability` before serializing, sanitizes the
|
|
223
|
+
* HTML (strips `<script>`, `on*` attributes, third-party iframes), and pushes
|
|
224
|
+
* an `ArtifactResult` with `mediaMode: 'dom'`.
|
|
225
|
+
*/
|
|
226
|
+
export interface CaptureDomOpcode extends OpcodeBase {
|
|
227
|
+
kind: 'CAPTURE_DOM';
|
|
228
|
+
/**
|
|
229
|
+
* Stable identifier for this state (kebab-case, e.g. `dashboard`,
|
|
230
|
+
* `dashboard-chat-open`). Referenced by the interaction script's
|
|
231
|
+
* `state` / `target` fields and used as the storage filename.
|
|
232
|
+
*/
|
|
233
|
+
stateName: string;
|
|
234
|
+
/**
|
|
235
|
+
* Optional CSS selector of the subtree to serialize instead of the full
|
|
236
|
+
* page. Use this when the interactive demo should focus on one specific
|
|
237
|
+
* surface (modal, dashboard shell, pricing calculator, editor panel, ...).
|
|
238
|
+
*
|
|
239
|
+
* The adapter keeps the current `<head>` and `<body>` attributes so global
|
|
240
|
+
* styles/themes still apply, but rewrites the body to contain only the
|
|
241
|
+
* selected branch (including its ancestor shell) and sizes the captured
|
|
242
|
+
* viewport to that subtree's bounding box.
|
|
243
|
+
*/
|
|
244
|
+
selector?: string;
|
|
245
|
+
}
|
|
246
|
+
/**
|
|
247
|
+
* Capture a single subtree of the current page as a reusable fragment
|
|
248
|
+
* (modal, popover, dropdown, tooltip, sheet, etc.). Phase 5 of AUT-121.
|
|
249
|
+
*
|
|
250
|
+
* Fragments are mounted/unmounted by the player on top of a base state via
|
|
251
|
+
* `show_fragment` / `hide_fragment` actions, so a single capture can power
|
|
252
|
+
* many local interactions without swapping the entire DOM.
|
|
253
|
+
*
|
|
254
|
+
* The runner optionally clicks `triggerSelector` first (typically a hidden
|
|
255
|
+
* `data-ak-fragment-trigger="..."` button exposed by the user's app for
|
|
256
|
+
* fragments that aren't reachable through visible UI), waits for `selector`
|
|
257
|
+
* to be present, then serializes the matching subtree's `outerHTML`.
|
|
258
|
+
*/
|
|
259
|
+
export interface CaptureFragmentOpcode extends OpcodeBase {
|
|
260
|
+
kind: 'CAPTURE_FRAGMENT';
|
|
261
|
+
/**
|
|
262
|
+
* Stable kebab-case identifier for the fragment, unique within
|
|
263
|
+
* `parentState`. Mirrors `data-ak-fragment="<name>"` in the source code.
|
|
264
|
+
*
|
|
265
|
+
* Multiple `CAPTURE_FRAGMENT` opcodes can target the same `fragmentName`
|
|
266
|
+
* with different `variantName` values to capture multiple versions of
|
|
267
|
+
* the same subtree (e.g. "default" / "purple" / "text-bottom"). The
|
|
268
|
+
* player swaps variants in place at runtime via
|
|
269
|
+
* `api.fragment.show(name, variantName)`.
|
|
270
|
+
*/
|
|
271
|
+
fragmentName: string;
|
|
272
|
+
/**
|
|
273
|
+
* Variant identifier for this capture, kebab-case. Defaults to `'default'`.
|
|
274
|
+
* Use distinct variant names when capturing the same fragment in different
|
|
275
|
+
* states (e.g. with a different background applied). The player can swap
|
|
276
|
+
* between variants in place without losing scroll/focus/sibling layout.
|
|
277
|
+
*/
|
|
278
|
+
variantName?: string;
|
|
279
|
+
/**
|
|
280
|
+
* The captured base state (`CAPTURE_DOM.stateName`) this fragment lives
|
|
281
|
+
* on top of. Used to scope the fragment in storage and at runtime.
|
|
282
|
+
*/
|
|
283
|
+
parentState: string;
|
|
284
|
+
/** CSS selector of the subtree wrapper. Typically `[data-ak-fragment="<name>"]`. */
|
|
285
|
+
selector: string;
|
|
286
|
+
/**
|
|
287
|
+
* Optional hidden trigger button selector (mirrors the mock data
|
|
288
|
+
* pattern). If provided, the runner clicks it via `clickHidden()` before
|
|
289
|
+
* waiting for `selector` to appear — useful for fragments that aren't
|
|
290
|
+
* reachable via existing UI buttons.
|
|
291
|
+
*/
|
|
292
|
+
triggerSelector?: string;
|
|
293
|
+
/**
|
|
294
|
+
* How the player should mount the fragment back onto the base state at
|
|
295
|
+
* runtime. Default: `inline` (mount in place of the original wrapper).
|
|
296
|
+
* - `overlay`: position absolute at the top-left of the iframe body
|
|
297
|
+
* - `inline`: mount in place of the original wrapper
|
|
298
|
+
* - `portal:<target>`: mount inside a `[data-ak-mount-target="<target>"]` element
|
|
299
|
+
*/
|
|
300
|
+
mountStrategy?: 'overlay' | 'inline' | string;
|
|
301
|
+
/**
|
|
302
|
+
* Optional override for the base-state element where the fragment
|
|
303
|
+
* should be mounted (CSS selector). When omitted, the player uses the
|
|
304
|
+
* fragment's source location (the element matching `selector` in the
|
|
305
|
+
* captured base state).
|
|
306
|
+
*/
|
|
307
|
+
mountTargetSelector?: string;
|
|
308
|
+
}
|
|
309
|
+
export interface BeginClipOpcode extends OpcodeBase {
|
|
310
|
+
kind: 'BEGIN_CLIP';
|
|
311
|
+
/** Stable preset clip identifier for Studio and dev links */
|
|
312
|
+
clipId?: string;
|
|
313
|
+
/** Human-readable preset clip name */
|
|
314
|
+
clipName?: string;
|
|
315
|
+
}
|
|
316
|
+
export interface EndClipOpcode extends OpcodeBase {
|
|
317
|
+
kind: 'END_CLIP';
|
|
318
|
+
/** Stable preset clip identifier for Studio and dev links */
|
|
319
|
+
clipId?: string;
|
|
320
|
+
/** Human-readable preset clip name */
|
|
321
|
+
clipName?: string;
|
|
322
|
+
}
|
|
323
|
+
export interface HoverOpcode extends OpcodeBase {
|
|
324
|
+
kind: 'HOVER';
|
|
325
|
+
selector: string;
|
|
326
|
+
target?: SemanticTarget;
|
|
327
|
+
fingerprint?: string;
|
|
328
|
+
selectorAlternates?: string[];
|
|
329
|
+
}
|
|
330
|
+
export interface SelectOptionOpcode extends OpcodeBase {
|
|
331
|
+
kind: 'SELECT_OPTION';
|
|
332
|
+
selector: string;
|
|
333
|
+
target?: SemanticTarget;
|
|
334
|
+
/** Select by visible label text */
|
|
335
|
+
optionLabel?: string;
|
|
336
|
+
/** Select by option value attribute */
|
|
337
|
+
optionValue?: string;
|
|
338
|
+
/** Select by zero-based index */
|
|
339
|
+
optionIndex?: number;
|
|
340
|
+
fingerprint?: string;
|
|
341
|
+
selectorAlternates?: string[];
|
|
342
|
+
}
|
|
343
|
+
export interface CheckOpcode extends OpcodeBase {
|
|
344
|
+
kind: 'CHECK';
|
|
345
|
+
selector: string;
|
|
346
|
+
target?: SemanticTarget;
|
|
347
|
+
/** Desired state: true = checked, false = unchecked */
|
|
348
|
+
checked: boolean;
|
|
349
|
+
fingerprint?: string;
|
|
350
|
+
selectorAlternates?: string[];
|
|
351
|
+
}
|
|
352
|
+
export interface DoubleClickOpcode extends OpcodeBase {
|
|
353
|
+
kind: 'DOUBLE_CLICK';
|
|
354
|
+
selector: string;
|
|
355
|
+
target?: SemanticTarget;
|
|
356
|
+
fingerprint?: string;
|
|
357
|
+
selectorAlternates?: string[];
|
|
358
|
+
}
|
|
359
|
+
/**
|
|
360
|
+
* Duplicate a template element N times into a container. **Soft / non-blocking.**
|
|
361
|
+
* If the source or container selector is missing, the opcode is skipped and
|
|
362
|
+
* the variant continues without aborting.
|
|
363
|
+
*/
|
|
364
|
+
export interface CloneElementOpcode extends OpcodeBase {
|
|
365
|
+
kind: 'CLONE_ELEMENT';
|
|
366
|
+
/** CSS selector of the element to clone */
|
|
367
|
+
sourceSelector: string;
|
|
368
|
+
/** CSS selector of the container that receives the clones */
|
|
369
|
+
containerSelector: string;
|
|
370
|
+
/** Number of clones to produce (1-500) */
|
|
371
|
+
count: number;
|
|
372
|
+
/** If true, the source template is removed after cloning. Default: false */
|
|
373
|
+
removeSource?: boolean;
|
|
374
|
+
}
|
|
375
|
+
/**
|
|
376
|
+
* Orchestrator opcode that fills a fillable area with mock data. Carries
|
|
377
|
+
* fields for **both** delivery mechanisms simultaneously, and the runtime
|
|
378
|
+
* applies whichever the opcode declares — typically both:
|
|
379
|
+
*
|
|
380
|
+
* - **Clone** (DOM cloning): looks up the group, clones a template element
|
|
381
|
+
* once per `defaultValues` row, writes each slot value into the cloned
|
|
382
|
+
* descendants. Provides instant visual feedback even before the app
|
|
383
|
+
* re-renders. Works for tables, lists, card grids — anything whose children
|
|
384
|
+
* are pure markup the runtime can mutate.
|
|
385
|
+
*
|
|
386
|
+
* - **Trigger** (hidden button + input): writes the JSON-encoded
|
|
387
|
+
* `defaultValues` into a hidden input the user's app exposed, then clicks a
|
|
388
|
+
* hidden trigger button. The user's `onClick` handler reads the input value
|
|
389
|
+
* and re-renders its own component with the seed data. Survives React
|
|
390
|
+
* re-renders that would otherwise clobber the cloned DOM, and works for
|
|
391
|
+
* programmatic widgets (Chart.js, Recharts, D3, maps) where the runtime
|
|
392
|
+
* can't mutate the DOM directly.
|
|
393
|
+
*
|
|
394
|
+
* The two mechanisms are complementary, not mutually exclusive: an opcode
|
|
395
|
+
* should typically declare BOTH so the runtime applies clone first (instant
|
|
396
|
+
* paint) and then trigger (state-driven, survives re-renders). At least one
|
|
397
|
+
* mechanism's required fields must be present.
|
|
398
|
+
*
|
|
399
|
+
* **Soft / non-blocking** — if neither mechanism produces a successful
|
|
400
|
+
* application, the opcode is skipped and the variant continues with
|
|
401
|
+
* `mockDataGroupResults[groupName] = 'skipped'`. If at least one succeeds,
|
|
402
|
+
* the group is recorded as `'applied'`.
|
|
403
|
+
*/
|
|
404
|
+
export interface InjectMockDataOpcode extends OpcodeBase {
|
|
405
|
+
kind: 'INJECT_MOCK_DATA';
|
|
406
|
+
/** Name of the MockDataGroup in ExecutionProgram.mockDataGroups */
|
|
407
|
+
groupName: string;
|
|
408
|
+
/** CSS selector of the container that holds the cloned items */
|
|
409
|
+
containerSelector?: string;
|
|
410
|
+
/** CSS selector of the template element. Default: first child of container. */
|
|
411
|
+
templateSelector?: string;
|
|
412
|
+
/** Override the row count. If absent, uses defaultValues.length. */
|
|
413
|
+
count?: number;
|
|
414
|
+
/** If true, the template element is removed after cloning (empty-state replacement) */
|
|
415
|
+
removeTemplate?: boolean;
|
|
416
|
+
/** Maps each slot name to a target inside a cloned item */
|
|
417
|
+
slotMappings?: Array<{
|
|
418
|
+
/** Slot name from the MockDataGroup */
|
|
419
|
+
slot: string;
|
|
420
|
+
/** CSS selector RELATIVE to the cloned item */
|
|
421
|
+
selector: string;
|
|
422
|
+
/** If present, set this attribute. If absent, set textContent. */
|
|
423
|
+
attribute?: string;
|
|
424
|
+
}>;
|
|
425
|
+
/**
|
|
426
|
+
* CSS selector of the hidden input that receives the JSON-encoded
|
|
427
|
+
* `defaultValues` array. The user's app reads this value when the trigger
|
|
428
|
+
* fires. Use a unique `data-ak-fill-input="<group-name>"` attribute.
|
|
429
|
+
*/
|
|
430
|
+
inputSelector?: string;
|
|
431
|
+
/**
|
|
432
|
+
* CSS selector of the hidden button that the runner clicks programmatically
|
|
433
|
+
* (via `element.click()`, bypassing visibility checks). The button's
|
|
434
|
+
* `onClick` handler reads the input value and re-renders the widget. Use a
|
|
435
|
+
* unique `data-ak-fill-trigger="<group-name>"` attribute.
|
|
436
|
+
*/
|
|
437
|
+
triggerSelector?: string;
|
|
438
|
+
}
|
|
439
|
+
/** Remove all elements matching a selector. **Soft / non-blocking.** */
|
|
440
|
+
export interface RemoveElementOpcode extends OpcodeBase {
|
|
441
|
+
kind: 'REMOVE_ELEMENT';
|
|
442
|
+
/** CSS selector — all matches are removed */
|
|
443
|
+
selector: string;
|
|
444
|
+
}
|
|
445
|
+
/** Set an attribute on the first matching element. **Soft / non-blocking.** */
|
|
446
|
+
export interface SetAttributeOpcode extends OpcodeBase {
|
|
447
|
+
kind: 'SET_ATTRIBUTE';
|
|
448
|
+
/** CSS selector of the target element (first match only) */
|
|
449
|
+
selector: string;
|
|
450
|
+
/** Attribute name (e.g. "src", "href", "data-state") */
|
|
451
|
+
attribute: string;
|
|
452
|
+
/** Attribute value */
|
|
453
|
+
value: string;
|
|
454
|
+
}
|
|
455
|
+
export type ExecutionOpcode = NavigateOpcode | DismissOverlaysOpcode | AssertRouteOpcode | AssertSurfaceOpcode | ClickOpcode | TypeOpcode | PressKeyOpcode | WaitForOpcode | SetLocaleOpcode | SetThemeOpcode | ScrollOpcode | CaptureScreenshotOpcode | CaptureDomOpcode | CaptureFragmentOpcode | BeginClipOpcode | EndClipOpcode | HoverOpcode | SelectOptionOpcode | CheckOpcode | DoubleClickOpcode | CloneElementOpcode | InjectMockDataOpcode | RemoveElementOpcode | SetAttributeOpcode;
|
|
456
|
+
export interface VariantSpec {
|
|
457
|
+
id: string;
|
|
458
|
+
viewport: {
|
|
459
|
+
width: number;
|
|
460
|
+
height: number;
|
|
461
|
+
};
|
|
462
|
+
deviceScaleFactor?: number;
|
|
463
|
+
locale?: string;
|
|
464
|
+
theme?: 'light' | 'dark';
|
|
465
|
+
/** Stable target identifier from the preset config */
|
|
466
|
+
targetId?: string;
|
|
467
|
+
/** Human-readable target label from the preset config */
|
|
468
|
+
targetLabel?: string;
|
|
469
|
+
/** Device frame label (e.g. "iPhone 15 Pro") for mockup rendering */
|
|
470
|
+
deviceFrame?: string;
|
|
471
|
+
}
|
|
472
|
+
export interface PreconditionSpec {
|
|
473
|
+
/**
|
|
474
|
+
* Auth bootstrap is auto-detected from what is configured on the preset:
|
|
475
|
+
* - If `cookies` / `storageState` / `sessionStorage` are present, the runtime
|
|
476
|
+
* injects them into the browser context before any opcode runs.
|
|
477
|
+
* - If `credentials` (email/password) are present, they are substituted
|
|
478
|
+
* into `{{email}}` / `{{password}}` placeholders inside the program's
|
|
479
|
+
* `TYPE` opcodes (the program performs the UI login itself).
|
|
480
|
+
* - If neither is present, the run is anonymous.
|
|
481
|
+
*
|
|
482
|
+
* Both mechanisms can coexist (e.g. seed cookies AND fall back to UI login).
|
|
483
|
+
*
|
|
484
|
+
* Note: the AI assistant must HARDCODE the login URL directly in the
|
|
485
|
+
* `NAVIGATE` opcode. The legacy `{{loginUrl}}` placeholder is deprecated
|
|
486
|
+
* because the field is optional and substituting an empty string crashes
|
|
487
|
+
* Playwright with a confusing "Cannot navigate to invalid URL" error.
|
|
488
|
+
*/
|
|
489
|
+
/** Credentials reference ID (resolved at runtime from CLI config) */
|
|
490
|
+
credentialsId?: string;
|
|
491
|
+
/**
|
|
492
|
+
* Plain login credentials (decrypted server-side and bridged into the
|
|
493
|
+
* program at fetch time). Used to substitute `{{email}}` / `{{password}}`
|
|
494
|
+
* placeholders in TYPE opcodes. **Do not use `{{loginUrl}}` in new
|
|
495
|
+
* programs** — it's kept for backwards compat with older presets but
|
|
496
|
+
* deprecated; the agent should hardcode the login URL in NAVIGATE.
|
|
497
|
+
*/
|
|
498
|
+
credentials?: {
|
|
499
|
+
email?: string;
|
|
500
|
+
password?: string;
|
|
501
|
+
/** @deprecated Hardcode the login URL directly in NAVIGATE opcodes instead. */
|
|
502
|
+
loginUrl?: string;
|
|
503
|
+
};
|
|
504
|
+
/** Pre-loaded storage state */
|
|
505
|
+
storageState?: BrowserStorageState;
|
|
506
|
+
sessionStorage?: BrowserSessionStorageState;
|
|
507
|
+
/** Cookies to set before execution */
|
|
508
|
+
cookies?: Array<{
|
|
509
|
+
name: string;
|
|
510
|
+
value: string;
|
|
511
|
+
domain: string;
|
|
512
|
+
path?: string;
|
|
513
|
+
}>;
|
|
514
|
+
}
|
|
515
|
+
export declare const MEDIA_MODES: readonly ["screenshot", "clip", "dom"];
|
|
516
|
+
export type MediaMode = (typeof MEDIA_MODES)[number];
|
|
517
|
+
export interface ArtifactSpec {
|
|
518
|
+
mediaMode: MediaMode;
|
|
519
|
+
/** Output format preferences */
|
|
520
|
+
format?: {
|
|
521
|
+
/** For clips: 'gif' | 'mp4' | 'both'. Default: 'gif' */
|
|
522
|
+
clipFormat?: 'gif' | 'mp4' | 'both';
|
|
523
|
+
/** For screenshots: 'png' | 'jpeg'. Default: 'png' */
|
|
524
|
+
screenshotFormat?: 'png' | 'jpeg';
|
|
525
|
+
};
|
|
526
|
+
/** Cursor style for clip recordings. Default: 'minimal'. */
|
|
527
|
+
cursorTheme?: VideoCursorTheme;
|
|
528
|
+
/** Max clip duration in seconds. Clips are trimmed if they exceed this. Default: 8. */
|
|
529
|
+
maxClipDurationSec?: number;
|
|
530
|
+
/** Whether to apply device frame mockup. Default: false */
|
|
531
|
+
applyMockup?: boolean;
|
|
532
|
+
/** Whether to add status bar. Default: false */
|
|
533
|
+
applyStatusBar?: boolean;
|
|
534
|
+
/** Options specific to DOM captures (Interactive Demos) */
|
|
535
|
+
domOptions?: {
|
|
536
|
+
/** Whether to sanitize the DOM (strip <script>, on*, third-party iframes). Default: true */
|
|
537
|
+
sanitize?: boolean;
|
|
538
|
+
};
|
|
539
|
+
}
|
|
540
|
+
export interface ExecutionProgram {
|
|
541
|
+
presetId: string;
|
|
542
|
+
programVersion: number;
|
|
543
|
+
mediaMode: MediaMode;
|
|
544
|
+
baseUrl: string;
|
|
545
|
+
/** Server-resolved concurrency cap for this run, derived from the owner's plan. */
|
|
546
|
+
maxParallelCaptures?: number;
|
|
547
|
+
variants: VariantSpec[];
|
|
548
|
+
preconditions: PreconditionSpec;
|
|
549
|
+
steps: ExecutionOpcode[];
|
|
550
|
+
artifactPlan: ArtifactSpec;
|
|
551
|
+
/** Global output scale (device pixel ratio). Applied to all variants that don't set their own deviceScaleFactor. */
|
|
552
|
+
outputScale?: number;
|
|
553
|
+
/** Hash of the source preset content, used to detect recompilation needs */
|
|
554
|
+
compileFingerprint: string;
|
|
555
|
+
/** Canonical fingerprint of langs+themes at compilation (e.g. "langs:en,fr|themes:dark,light"). Used for staleness detection. */
|
|
556
|
+
variantFingerprint?: string;
|
|
557
|
+
/** ISO timestamp of compilation */
|
|
558
|
+
compiledAt: string;
|
|
559
|
+
/** Model used for compilation */
|
|
560
|
+
compiledWith?: string;
|
|
561
|
+
/**
|
|
562
|
+
* Mock data groups available to INJECT_MOCK_DATA opcodes.
|
|
563
|
+
* Compiled from PresetConfig.mockDataInjection.groups by the server.
|
|
564
|
+
*/
|
|
565
|
+
mockDataGroups?: MockDataGroup[];
|
|
566
|
+
}
|
|
567
|
+
export interface CircuitBreakerConfig {
|
|
568
|
+
/** Max recovery attempts per opcode. Default: 3 */
|
|
569
|
+
maxPerOpcode: number;
|
|
570
|
+
/** Max total failures per page navigation. Default: 5 */
|
|
571
|
+
maxPerPage: number;
|
|
572
|
+
/** Max total failures per run. Default: 10 */
|
|
573
|
+
maxPerRun: number;
|
|
574
|
+
}
|
|
575
|
+
export declare const DEFAULT_CIRCUIT_BREAKER: CircuitBreakerConfig;
|
|
576
|
+
export interface HealerPatch {
|
|
577
|
+
/** Index of the opcode in the program that was patched */
|
|
578
|
+
opcodeIndex: number;
|
|
579
|
+
/** The original opcode before patching */
|
|
580
|
+
originalOpcode: ExecutionOpcode;
|
|
581
|
+
/** The replacement opcode(s). Usually 1, max 3. */
|
|
582
|
+
replacementOpcodes: ExecutionOpcode[];
|
|
583
|
+
/** Why the healer made this change */
|
|
584
|
+
reason: string;
|
|
585
|
+
/** Timestamp */
|
|
586
|
+
patchedAt: string;
|
|
587
|
+
}
|
|
588
|
+
export type OpcodeResultStatus = 'ok' | 'recovered' | 'skipped' | 'failed';
|
|
589
|
+
export interface OpcodeResult {
|
|
590
|
+
opcodeIndex: number;
|
|
591
|
+
kind: OpcodeKind;
|
|
592
|
+
status: OpcodeResultStatus;
|
|
593
|
+
/** Time taken for this opcode (ms) */
|
|
594
|
+
durationMs: number;
|
|
595
|
+
/** Number of recovery attempts used */
|
|
596
|
+
recoveryAttempts: number;
|
|
597
|
+
/** Recovery strategy that succeeded, if any */
|
|
598
|
+
recoveryStrategy?: 'retry' | 'selector_memory' | 'alt_interaction' | 'reload' | 'healer';
|
|
599
|
+
/** Error message if failed */
|
|
600
|
+
error?: string;
|
|
601
|
+
}
|
|
602
|
+
export interface VariantResult {
|
|
603
|
+
variantId: string;
|
|
604
|
+
success: boolean;
|
|
605
|
+
opcodeResults: OpcodeResult[];
|
|
606
|
+
/** Total duration for this variant (ms) */
|
|
607
|
+
durationMs: number;
|
|
608
|
+
/** Artifact buffers produced */
|
|
609
|
+
artifacts: ArtifactResult[];
|
|
610
|
+
error?: string;
|
|
611
|
+
}
|
|
612
|
+
export interface ArtifactResult {
|
|
613
|
+
mediaMode: MediaMode;
|
|
614
|
+
buffer: Buffer;
|
|
615
|
+
mimeType: string;
|
|
616
|
+
durationMs?: number;
|
|
617
|
+
trimStartMs?: number;
|
|
618
|
+
/** For screenshots: viewport dimensions of the capture */
|
|
619
|
+
dimensions?: {
|
|
620
|
+
width: number;
|
|
621
|
+
height: number;
|
|
622
|
+
};
|
|
623
|
+
/** Generated alt text for accessibility */
|
|
624
|
+
altText?: string;
|
|
625
|
+
/** Final URL at the time the artifact was produced */
|
|
626
|
+
captureUrl?: string;
|
|
627
|
+
/** Step index that produced the artifact */
|
|
628
|
+
stepIndex?: number;
|
|
629
|
+
/** Human-readable label for the artifact */
|
|
630
|
+
stepDescription?: string;
|
|
631
|
+
/** Variant metadata used by persistence and gallery views */
|
|
632
|
+
variantId?: string;
|
|
633
|
+
/** Stable preset capture identifier for screenshots */
|
|
634
|
+
captureId?: string;
|
|
635
|
+
/** Human-readable preset capture name for screenshots */
|
|
636
|
+
captureName?: string;
|
|
637
|
+
/** Stable preset clip identifier for clips */
|
|
638
|
+
clipId?: string;
|
|
639
|
+
/** Human-readable preset clip name for clips */
|
|
640
|
+
clipName?: string;
|
|
641
|
+
captureType?: 'fullpage' | 'element';
|
|
642
|
+
elementSelector?: string;
|
|
643
|
+
/** Favicon extracted from the captured page */
|
|
644
|
+
tabIconData?: Buffer;
|
|
645
|
+
tabIconMimeType?: string;
|
|
646
|
+
/** Stable identifier of the captured demo state (kebab-case, from CaptureDomOpcode.stateName) */
|
|
647
|
+
stateName?: string;
|
|
648
|
+
/** Sanitized HTML body (also stored as buffer for upload) */
|
|
649
|
+
domHtml?: string;
|
|
650
|
+
/** Asset URLs referenced by the captured DOM (img, source, link rel="stylesheet"). Phase 3 will resolve these via CAS. */
|
|
651
|
+
domAssetUrls?: string[];
|
|
652
|
+
/** Byte size of the sanitized HTML, for telemetry */
|
|
653
|
+
domHtmlBytes?: number;
|
|
654
|
+
/** PNG thumbnail screenshot taken at the time of DOM capture */
|
|
655
|
+
domThumbnailBuffer?: Buffer;
|
|
656
|
+
/** Stable kebab-case identifier of the captured fragment, from CaptureFragmentOpcode.fragmentName */
|
|
657
|
+
fragmentName?: string;
|
|
658
|
+
/**
|
|
659
|
+
* Variant of this fragment capture (Phase 8). Same `fragmentName` can be
|
|
660
|
+
* captured under multiple variants — e.g. "default", "purple-bg",
|
|
661
|
+
* "text-bottom" — and the player swaps between them in place. Defaults to
|
|
662
|
+
* `'default'` when the opcode does not specify one.
|
|
663
|
+
*/
|
|
664
|
+
fragmentVariantName?: string;
|
|
665
|
+
/** Parent base state this fragment belongs to */
|
|
666
|
+
parentStateName?: string;
|
|
667
|
+
/** Where the player should mount the fragment at runtime */
|
|
668
|
+
mountStrategy?: string;
|
|
669
|
+
/** Optional override for the base-state element to mount into (CSS selector) */
|
|
670
|
+
mountTargetSelector?: string;
|
|
671
|
+
}
|
|
672
|
+
export type LLMStepType = 'capture_verification' | 'alt_text_generation' | 'healer_invocation';
|
|
673
|
+
export interface LLMStepUsage {
|
|
674
|
+
stepType: LLMStepType;
|
|
675
|
+
generationId: string | null;
|
|
676
|
+
model: string;
|
|
677
|
+
promptTokens: number;
|
|
678
|
+
completionTokens: number;
|
|
679
|
+
}
|
|
680
|
+
export interface RunTelemetry {
|
|
681
|
+
/** Total LLM calls (compilation excluded) */
|
|
682
|
+
llmCallCount: number;
|
|
683
|
+
/** Total LLM cost in EUR */
|
|
684
|
+
llmCostEur: number;
|
|
685
|
+
/** Detailed per-call LLM usage for cost logging */
|
|
686
|
+
llmStepUsages: LLMStepUsage[];
|
|
687
|
+
/** Total opcodes executed across all variants */
|
|
688
|
+
totalOpcodes: number;
|
|
689
|
+
/** Opcodes that needed recovery */
|
|
690
|
+
recoveredOpcodes: number;
|
|
691
|
+
/** Opcodes that failed even after recovery */
|
|
692
|
+
failedOpcodes: number;
|
|
693
|
+
/** Soft opcodes that were skipped (e.g. mock data injection failures) */
|
|
694
|
+
skippedOpcodes: number;
|
|
695
|
+
/** Number of healer invocations */
|
|
696
|
+
healerInvocations: number;
|
|
697
|
+
/** Circuit breaker trips */
|
|
698
|
+
circuitBreakerTrips: number;
|
|
699
|
+
/** Per-group mock data application result */
|
|
700
|
+
mockDataGroupResults?: Record<string, 'applied' | 'skipped'>;
|
|
701
|
+
}
|
|
702
|
+
export interface RunResult {
|
|
703
|
+
programId: string;
|
|
704
|
+
success: boolean;
|
|
705
|
+
variantResults: VariantResult[];
|
|
706
|
+
telemetry: RunTelemetry;
|
|
707
|
+
/** Healer patches accumulated during the run. Only propagate to server if success=true */
|
|
708
|
+
healerPatches: HealerPatch[];
|
|
709
|
+
/** Total run duration (ms) */
|
|
710
|
+
totalDurationMs: number;
|
|
711
|
+
error?: string;
|
|
712
|
+
}
|
|
713
|
+
export interface WaitCondition {
|
|
714
|
+
selector: string;
|
|
715
|
+
state: 'visible' | 'attached';
|
|
716
|
+
timeoutMs: number;
|
|
717
|
+
}
|
|
718
|
+
export interface ClickOptions {
|
|
719
|
+
/** Force click even if element is covered */
|
|
720
|
+
force?: boolean;
|
|
721
|
+
/** Use keyboard (Tab + Enter) instead of mouse */
|
|
722
|
+
useKeyboard?: boolean;
|
|
723
|
+
/** Use JS dispatch instead of Playwright click */
|
|
724
|
+
useJsDispatch?: boolean;
|
|
725
|
+
/** Click by coordinates from bounding box */
|
|
726
|
+
coordinates?: {
|
|
727
|
+
x: number;
|
|
728
|
+
y: number;
|
|
729
|
+
};
|
|
730
|
+
/** Mouse button. Default: 'left' */
|
|
731
|
+
button?: 'left' | 'right' | 'middle';
|
|
732
|
+
}
|
|
733
|
+
export interface RecordingOptions {
|
|
734
|
+
mediaMode: 'clip';
|
|
735
|
+
}
|
|
736
|
+
export interface RecordingResult {
|
|
737
|
+
buffer: Buffer;
|
|
738
|
+
durationMs: number;
|
|
739
|
+
mimeType: string;
|
|
740
|
+
trimStartMs?: number;
|
|
741
|
+
}
|
|
742
|
+
export interface RuntimeAdapter {
|
|
743
|
+
navigate(url: string): Promise<void>;
|
|
744
|
+
getCurrentUrl(): Promise<string>;
|
|
745
|
+
getAKTree(): Promise<AKTree>;
|
|
746
|
+
getPageSignals(): Promise<VideoPageSignals>;
|
|
747
|
+
click(selector: string, options?: ClickOptions): Promise<void>;
|
|
748
|
+
type(selector: string, text: string, clearFirst?: boolean): Promise<void>;
|
|
749
|
+
pressKey(key: string): Promise<void>;
|
|
750
|
+
scroll(direction: 'up' | 'down' | 'left' | 'right', amount?: number): Promise<void>;
|
|
751
|
+
scrollIntoView(selector: string): Promise<void>;
|
|
752
|
+
waitFor(condition: WaitCondition): Promise<boolean>;
|
|
753
|
+
dismissOverlays(): Promise<{
|
|
754
|
+
dismissed: boolean;
|
|
755
|
+
method: string | null;
|
|
756
|
+
}>;
|
|
757
|
+
takeScreenshot(): Promise<Buffer>;
|
|
758
|
+
takeElementScreenshot?(selector: string): Promise<Buffer>;
|
|
759
|
+
takeCleanScreenshot(): Promise<Buffer>;
|
|
760
|
+
/**
|
|
761
|
+
* Serialize the current page DOM into a sanitized HTML string for the
|
|
762
|
+
* Interactive Demo pipeline. Returns the HTML and the asset URLs found
|
|
763
|
+
* during sanitization (img/source/link). Optional — adapters that don't
|
|
764
|
+
* implement this cannot run `CAPTURE_DOM` opcodes.
|
|
765
|
+
*/
|
|
766
|
+
serializeDom?(selector?: string): Promise<{
|
|
767
|
+
html: string;
|
|
768
|
+
assetUrls: string[];
|
|
769
|
+
viewport: {
|
|
770
|
+
width: number;
|
|
771
|
+
height: number;
|
|
772
|
+
};
|
|
773
|
+
capturedAt: string;
|
|
774
|
+
}>;
|
|
775
|
+
/**
|
|
776
|
+
* Serialize a single subtree of the page (matched by `selector`) as a
|
|
777
|
+
* sanitized fragment HTML string. Optional — adapters that don't
|
|
778
|
+
* implement this cannot run `CAPTURE_FRAGMENT` opcodes. Phase 5 of
|
|
779
|
+
* AUT-121 Interactive Demos.
|
|
780
|
+
*/
|
|
781
|
+
serializeFragment?(selector: string): Promise<{
|
|
782
|
+
html: string;
|
|
783
|
+
assetUrls: string[];
|
|
784
|
+
capturedAt: string;
|
|
785
|
+
}>;
|
|
786
|
+
beginRecording(options: RecordingOptions): Promise<void>;
|
|
787
|
+
endRecording(): Promise<RecordingResult>;
|
|
788
|
+
setLocale(locale: string): Promise<void>;
|
|
789
|
+
setColorScheme(scheme: 'light' | 'dark'): Promise<void>;
|
|
790
|
+
reloadPage?(): Promise<void>;
|
|
791
|
+
writeStorageHint?(params: {
|
|
792
|
+
storage: 'localStorage' | 'sessionStorage' | 'cookie';
|
|
793
|
+
key: string;
|
|
794
|
+
value: string;
|
|
795
|
+
kind: 'locale' | 'theme';
|
|
796
|
+
}): Promise<boolean>;
|
|
797
|
+
/** Extract the page's favicon as a buffer. Returns null if unavailable. */
|
|
798
|
+
extractFavicon?(): Promise<{
|
|
799
|
+
buffer: Buffer;
|
|
800
|
+
mimeType: string;
|
|
801
|
+
} | null>;
|
|
802
|
+
close(): Promise<void>;
|
|
803
|
+
/** Click an element by semantic target. Falls back to selector if target not found. */
|
|
804
|
+
clickByTarget?(opts: {
|
|
805
|
+
selector?: string;
|
|
806
|
+
target?: SemanticTarget;
|
|
807
|
+
selectorAlternates?: string[];
|
|
808
|
+
}): Promise<void>;
|
|
809
|
+
/** Type into an element by semantic target. */
|
|
810
|
+
typeByTarget?(opts: {
|
|
811
|
+
selector?: string;
|
|
812
|
+
target?: SemanticTarget;
|
|
813
|
+
selectorAlternates?: string[];
|
|
814
|
+
}, text: string, clearFirst?: boolean): Promise<void>;
|
|
815
|
+
/** Wait for an element by semantic target. */
|
|
816
|
+
waitForTarget?(opts: {
|
|
817
|
+
selector?: string;
|
|
818
|
+
target?: SemanticTarget;
|
|
819
|
+
selectorAlternates?: string[];
|
|
820
|
+
}, timeoutMs?: number): Promise<boolean>;
|
|
821
|
+
/** Scroll an element into view by semantic target. */
|
|
822
|
+
scrollIntoViewByTarget?(opts: {
|
|
823
|
+
selector?: string;
|
|
824
|
+
target?: SemanticTarget;
|
|
825
|
+
selectorAlternates?: string[];
|
|
826
|
+
}): Promise<void>;
|
|
827
|
+
hover?(selector: string): Promise<void>;
|
|
828
|
+
hoverByTarget?(opts: {
|
|
829
|
+
selector?: string;
|
|
830
|
+
target?: SemanticTarget;
|
|
831
|
+
selectorAlternates?: string[];
|
|
832
|
+
}): Promise<void>;
|
|
833
|
+
selectOption?(selector: string, option: {
|
|
834
|
+
label?: string;
|
|
835
|
+
value?: string;
|
|
836
|
+
index?: number;
|
|
837
|
+
}): Promise<void>;
|
|
838
|
+
check?(selector: string, checked: boolean): Promise<void>;
|
|
839
|
+
doubleClick?(selector: string): Promise<void>;
|
|
840
|
+
/** Clone an element N times into a container. Throws if either selector misses. */
|
|
841
|
+
cloneElement?(opts: {
|
|
842
|
+
sourceSelector: string;
|
|
843
|
+
containerSelector: string;
|
|
844
|
+
count: number;
|
|
845
|
+
removeSource?: boolean;
|
|
846
|
+
}): Promise<{
|
|
847
|
+
clonedCount: number;
|
|
848
|
+
}>;
|
|
849
|
+
/** Set an attribute on the first matching element. Throws if the selector misses. */
|
|
850
|
+
setAttribute?(opts: {
|
|
851
|
+
selector: string;
|
|
852
|
+
attribute: string;
|
|
853
|
+
value: string;
|
|
854
|
+
}): Promise<void>;
|
|
855
|
+
/** Set the textContent of the first matching element. Throws if the selector misses. */
|
|
856
|
+
setTextContent?(opts: {
|
|
857
|
+
selector: string;
|
|
858
|
+
text: string;
|
|
859
|
+
}): Promise<void>;
|
|
860
|
+
/** Remove all matching elements. Throws only if no elements match. */
|
|
861
|
+
removeElement?(opts: {
|
|
862
|
+
selector: string;
|
|
863
|
+
}): Promise<{
|
|
864
|
+
removedCount: number;
|
|
865
|
+
}>;
|
|
866
|
+
/**
|
|
867
|
+
* Set the `value` of an input/textarea/select via the React-aware native
|
|
868
|
+
* setter, then dispatch `input` and `change` events so React (and other
|
|
869
|
+
* controlled-component frameworks) detect the change. Throws if the selector
|
|
870
|
+
* misses. Used by INJECT_MOCK_DATA strategy='trigger'.
|
|
871
|
+
*/
|
|
872
|
+
setInputValue?(opts: {
|
|
873
|
+
selector: string;
|
|
874
|
+
value: string;
|
|
875
|
+
}): Promise<void>;
|
|
876
|
+
/**
|
|
877
|
+
* Click an element via JavaScript (`element.click()`), bypassing visibility
|
|
878
|
+
* and enabled checks. Useful for clicking hidden trigger buttons that
|
|
879
|
+
* `display:none` or `pointer-events:none` would block. Throws if the
|
|
880
|
+
* selector misses. Used by INJECT_MOCK_DATA strategy='trigger'.
|
|
881
|
+
*/
|
|
882
|
+
clickHidden?(opts: {
|
|
883
|
+
selector: string;
|
|
884
|
+
}): Promise<void>;
|
|
885
|
+
}
|
|
886
|
+
export {};
|