autokap 1.0.6 → 1.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/chrome/ios-statusbar-comparison-reference.jpg +0 -0
- package/assets/chrome/ios-statusbar-dark-reference.jpg +0 -0
- package/assets/chrome/ios-statusbar-light-reference.jpg +0 -0
- package/assets/cursors/macos.svg +4 -0
- package/assets/cursors/windows.svg +15 -0
- package/assets/devices/ipad-pro-11-m4.json +52 -0
- package/assets/devices/iphone-16-pro.json +53 -0
- package/assets/devices/macbook-air-13.json +45 -0
- package/assets/frames/MacBook Air 13.svg +242 -0
- package/assets/frames/Status bar - iPhone.png +0 -0
- Menu bar- iPad.png +0 -0
- package/assets/frames/iPad Pro M4 11_.png +0 -0
- package/assets/frames/iPhone 16 Pro.png +0 -0
- package/assets/icons/Cellular Connection.svg +3 -0
- package/assets/icons/Union.svg +6 -0
- package/assets/icons/Wifi.svg +3 -0
- package/assets/icons/battery.svg +5 -0
- package/assets/icons/battery_charging.svg +8 -0
- package/assets/skill/OPCODE-REFERENCE.md +607 -0
- package/assets/skill/README.md +39 -0
- package/assets/skill/SKILL.md +453 -468
- package/assets/skill/STUDIO-SKILL.md +476 -0
- package/assets/skill/references/examples.md +104 -0
- package/assets/skill/references/interactive-demo.md +225 -0
- package/assets/skill/references/mock-data.md +178 -0
- package/dist/abort.d.ts +5 -0
- package/dist/abort.js +44 -0
- package/dist/action-verifier.d.ts +29 -0
- package/dist/action-verifier.js +133 -0
- package/dist/agent-action-recovery.d.ts +45 -0
- package/dist/agent-action-recovery.js +370 -0
- package/dist/agent-message-utils.d.ts +21 -0
- package/dist/agent-message-utils.js +77 -0
- package/dist/agent-url-utils.d.ts +30 -0
- package/dist/agent-url-utils.js +138 -0
- package/dist/agent.d.ts +226 -0
- package/dist/agent.js +6666 -0
- package/dist/ak-tree.d.ts +39 -0
- package/dist/ak-tree.js +368 -0
- package/dist/alt-text.d.ts +26 -0
- package/dist/alt-text.js +55 -0
- package/dist/auth-capture.d.ts +17 -0
- package/dist/auth-capture.js +164 -0
- package/dist/benchmark.d.ts +59 -0
- package/dist/benchmark.js +135 -0
- package/dist/billing-operation-logging.d.ts +38 -0
- package/dist/billing-operation-logging.js +248 -0
- package/dist/browser-bar.d.ts +48 -0
- package/dist/browser-bar.js +284 -0
- package/dist/browser-pool.d.ts +7 -0
- package/dist/browser-pool.js +15 -5
- package/dist/browser-utils.d.ts +31 -0
- package/dist/browser-utils.js +97 -0
- package/dist/browser.d.ts +76 -1
- package/dist/browser.js +1657 -39
- package/dist/capture-alt-text.d.ts +12 -0
- package/dist/capture-alt-text.js +52 -0
- package/dist/capture-encryption.d.ts +10 -0
- package/dist/capture-encryption.js +41 -0
- package/dist/capture-language-preflight.d.ts +41 -0
- package/dist/capture-language-preflight.js +300 -0
- package/dist/capture-llm-page-identity.d.ts +15 -0
- package/dist/capture-llm-page-identity.js +128 -0
- package/dist/capture-model-resolution.d.ts +9 -0
- package/dist/capture-model-resolution.js +21 -0
- package/dist/capture-page-identity.d.ts +7 -0
- package/dist/capture-page-identity.js +352 -0
- package/dist/capture-preset-credentials.d.ts +62 -0
- package/dist/capture-preset-credentials.js +184 -0
- package/dist/capture-request-plan.d.ts +58 -0
- package/dist/capture-request-plan.js +264 -0
- package/dist/capture-run-optimizer.d.ts +139 -0
- package/dist/capture-run-optimizer.js +863 -0
- package/dist/capture-selector-memory.d.ts +31 -0
- package/dist/capture-selector-memory.js +345 -0
- package/dist/capture-session-profile-encryption.d.ts +2 -0
- package/dist/capture-session-profile-encryption.js +22 -0
- package/dist/capture-step-timeout.d.ts +10 -0
- package/dist/capture-step-timeout.js +30 -0
- package/dist/capture-strategy.d.ts +36 -0
- package/dist/capture-strategy.js +95 -0
- package/dist/capture-studio-sync.d.ts +23 -0
- package/dist/capture-studio-sync.js +172 -0
- package/dist/capture-surface-contract.d.ts +36 -0
- package/dist/capture-surface-contract.js +299 -0
- package/dist/capture-transition-engine.d.ts +28 -0
- package/dist/capture-transition-engine.js +292 -0
- package/dist/capture-variant-state.d.ts +56 -0
- package/dist/capture-variant-state.js +182 -0
- package/dist/capture-verification.d.ts +35 -0
- package/dist/capture-verification.js +95 -0
- package/dist/capture-viewport-lock.d.ts +48 -0
- package/dist/capture-viewport-lock.js +74 -0
- package/dist/circuit-breaker.d.ts +42 -0
- package/dist/circuit-breaker.js +119 -0
- package/dist/cli-config.d.ts +8 -1
- package/dist/cli-config.js +62 -6
- package/dist/cli-contract.d.ts +15 -0
- package/dist/cli-contract.js +167 -0
- package/dist/cli-runner-local.d.ts +12 -0
- package/dist/cli-runner-local.js +102 -0
- package/dist/cli-runner.d.ts +34 -0
- package/dist/cli-runner.js +433 -0
- package/dist/cli-utils.d.ts +0 -1
- package/dist/cli-utils.js +2 -5
- package/dist/cli.js +1005 -252
- package/dist/clip-orchestrator.d.ts +148 -0
- package/dist/clip-orchestrator.js +957 -0
- package/dist/clip-postprocess.d.ts +42 -0
- package/dist/clip-postprocess.js +201 -0
- package/dist/cookie-dismiss.d.ts +2 -0
- package/dist/cookie-dismiss.js +48 -13
- package/dist/cost-logging.d.ts +35 -0
- package/dist/cost-logging.js +242 -0
- package/dist/cost-resolution-monitor.d.ts +16 -0
- package/dist/cost-resolution-monitor.js +34 -0
- package/dist/credential-templates.d.ts +5 -0
- package/dist/credential-templates.js +60 -0
- package/dist/cursor-overlay-script.d.ts +6 -0
- package/dist/cursor-overlay-script.js +169 -0
- package/dist/dom-css-purger.d.ts +65 -0
- package/dist/dom-css-purger.js +333 -0
- package/dist/dom-font-inliner.d.ts +45 -0
- package/dist/dom-font-inliner.js +148 -0
- package/dist/dom-patch-resolver.d.ts +52 -0
- package/dist/dom-patch-resolver.js +242 -0
- package/dist/dom-serializer.d.ts +82 -0
- package/dist/dom-serializer.js +378 -0
- package/dist/element-capture.d.ts +13 -0
- package/dist/element-capture.js +522 -0
- package/dist/env-validation.d.ts +5 -0
- package/dist/env-validation.js +29 -0
- package/dist/execution-schema.d.ts +4423 -0
- package/dist/execution-schema.js +507 -0
- package/dist/execution-types.d.ts +886 -0
- package/dist/execution-types.js +65 -0
- package/dist/fonts-loader.d.ts +14 -0
- package/dist/fonts-loader.js +55 -0
- package/dist/hybrid-navigator.d.ts +138 -0
- package/dist/hybrid-navigator.js +468 -0
- package/dist/index.d.ts +18 -0
- package/dist/index.js +17 -0
- package/dist/legacy/agent-action-recovery.d.ts +45 -0
- package/dist/legacy/agent-action-recovery.js +370 -0
- package/dist/legacy/agent-message-utils.d.ts +21 -0
- package/dist/legacy/agent-message-utils.js +77 -0
- package/dist/legacy/agent-url-utils.d.ts +30 -0
- package/dist/legacy/agent-url-utils.js +138 -0
- package/dist/legacy/agent.d.ts +226 -0
- package/dist/legacy/agent.js +6666 -0
- package/dist/legacy/clip-orchestrator.d.ts +148 -0
- package/dist/legacy/clip-orchestrator.js +957 -0
- package/dist/legacy/credential-templates.d.ts +5 -0
- package/dist/legacy/credential-templates.js +60 -0
- package/dist/legacy/hybrid-navigator.d.ts +138 -0
- package/dist/legacy/hybrid-navigator.js +468 -0
- package/dist/legacy/llm-usage.d.ts +17 -0
- package/dist/legacy/llm-usage.js +45 -0
- package/dist/legacy/prompt-cache.d.ts +10 -0
- package/dist/legacy/prompt-cache.js +24 -0
- package/dist/legacy/prompts.d.ts +175 -0
- package/dist/legacy/prompts.js +1038 -0
- package/dist/legacy/tools.d.ts +4 -0
- package/dist/legacy/tools.js +216 -0
- package/dist/legacy/video-agent.d.ts +143 -0
- package/dist/legacy/video-agent.js +4788 -0
- package/dist/legacy/video-observation.d.ts +36 -0
- package/dist/legacy/video-observation.js +192 -0
- package/dist/legacy/video-planner.d.ts +12 -0
- package/dist/legacy/video-planner.js +501 -0
- package/dist/legacy/video-prompts.d.ts +37 -0
- package/dist/legacy/video-prompts.js +569 -0
- package/dist/legacy/video-tools.d.ts +3 -0
- package/dist/legacy/video-tools.js +59 -0
- package/dist/legacy/video-variant-state.d.ts +29 -0
- package/dist/legacy/video-variant-state.js +80 -0
- package/dist/legacy/vision-model.d.ts +17 -0
- package/dist/legacy/vision-model.js +74 -0
- package/dist/llm-healer.d.ts +63 -0
- package/dist/llm-healer.js +166 -0
- package/dist/llm-provider.d.ts +29 -0
- package/dist/llm-provider.js +80 -0
- package/dist/llm-usage.d.ts +17 -0
- package/dist/llm-usage.js +45 -0
- package/dist/logger.d.ts +6 -2
- package/dist/logger.js +15 -1
- package/dist/mockup-html.d.ts +119 -0
- package/dist/mockup-html.js +263 -0
- package/dist/mockup.d.ts +187 -0
- package/dist/mockup.js +869 -0
- package/dist/mouse-animation.d.ts +46 -0
- package/dist/mouse-animation.js +114 -0
- package/dist/opcode-actions.d.ts +42 -0
- package/dist/opcode-actions.js +511 -0
- package/dist/opcode-runner.d.ts +51 -0
- package/dist/opcode-runner.js +770 -0
- package/dist/openrouter-client.d.ts +40 -0
- package/dist/openrouter-client.js +16 -0
- package/dist/overlay-engine.d.ts +24 -0
- package/dist/overlay-engine.js +176 -0
- package/dist/overlay-utils.d.ts +14 -0
- package/dist/overlay-utils.js +13 -0
- package/dist/postcondition.d.ts +16 -0
- package/dist/postcondition.js +269 -0
- package/dist/posthog.d.ts +4 -0
- package/dist/posthog.js +26 -0
- package/dist/program-patcher.d.ts +25 -0
- package/dist/program-patcher.js +44 -0
- package/dist/prompt-cache.d.ts +10 -0
- package/dist/prompt-cache.js +24 -0
- package/dist/prompts.d.ts +175 -0
- package/dist/prompts.js +1038 -0
- package/dist/provider-config.d.ts +12 -0
- package/dist/provider-config.js +15 -0
- package/dist/recovery-chain.d.ts +37 -0
- package/dist/recovery-chain.js +350 -0
- package/dist/remote-browser.d.ts +215 -0
- package/dist/remote-browser.js +360 -0
- package/dist/safari-browser-bar.d.ts +15 -0
- package/dist/safari-browser-bar.js +95 -0
- package/dist/safari-toolbar-asset.d.ts +15 -0
- package/dist/safari-toolbar-asset.js +12 -0
- package/dist/security.d.ts +21 -0
- package/dist/security.js +608 -0
- package/dist/selector-resolver.d.ts +34 -0
- package/dist/selector-resolver.js +181 -0
- package/dist/semantic-resolver.d.ts +35 -0
- package/dist/semantic-resolver.js +161 -0
- package/dist/server-capture-runtime.d.ts +125 -0
- package/dist/server-capture-runtime.js +585 -0
- package/dist/server-credit-usage.d.ts +12 -0
- package/dist/server-credit-usage.js +41 -0
- package/dist/server-posthog.d.ts +2 -0
- package/dist/server-posthog.js +16 -0
- package/dist/server-project-webhooks.d.ts +59 -0
- package/dist/server-project-webhooks.js +123 -0
- package/dist/server-screenshot-watermark.d.ts +7 -0
- package/dist/server-screenshot-watermark.js +60 -0
- package/dist/session-profile.d.ts +86 -0
- package/dist/session-profile.js +1536 -0
- package/dist/sf-pro-fonts.d.ts +4 -0
- package/dist/sf-pro-fonts.js +7 -0
- package/dist/sf-pro-symbols.d.ts +1 -0
- package/dist/sf-pro-symbols.js +55 -0
- package/dist/skill-packaging.d.ts +28 -0
- package/dist/skill-packaging.js +169 -0
- package/dist/smart-wait.d.ts +27 -0
- package/dist/smart-wait.js +81 -0
- package/dist/status-bar-l10n.d.ts +14 -0
- package/dist/status-bar-l10n.js +177 -0
- package/dist/status-bar-render.d.ts +20 -0
- package/dist/status-bar-render.js +410 -0
- package/dist/status-bar.d.ts +53 -0
- package/dist/status-bar.js +620 -0
- package/dist/svg-browser-bar.d.ts +33 -0
- package/dist/svg-browser-bar.js +206 -0
- package/dist/svg-status-bar.d.ts +36 -0
- package/dist/svg-status-bar.js +597 -0
- package/dist/svg-text.d.ts +61 -0
- package/dist/svg-text.js +118 -0
- package/dist/tools.d.ts +4 -0
- package/dist/tools.js +216 -0
- package/dist/types.d.ts +240 -5
- package/dist/types.js +23 -1
- package/dist/v2/action-verifier.d.ts +29 -0
- package/dist/v2/action-verifier.js +133 -0
- package/dist/v2/alt-text.d.ts +26 -0
- package/dist/v2/alt-text.js +55 -0
- package/dist/v2/benchmark.d.ts +59 -0
- package/dist/v2/benchmark.js +135 -0
- package/dist/v2/capture-strategy.d.ts +30 -0
- package/dist/v2/capture-strategy.js +67 -0
- package/dist/v2/capture-verification.d.ts +35 -0
- package/dist/v2/capture-verification.js +95 -0
- package/dist/v2/circuit-breaker.d.ts +42 -0
- package/dist/v2/circuit-breaker.js +119 -0
- package/dist/v2/cli-runner-local.d.ts +11 -0
- package/dist/v2/cli-runner-local.js +91 -0
- package/dist/v2/cli-runner.d.ts +34 -0
- package/dist/v2/cli-runner.js +300 -0
- package/dist/v2/compiler-prompts.d.ts +27 -0
- package/dist/v2/compiler-prompts.js +123 -0
- package/dist/v2/compiler.d.ts +37 -0
- package/dist/v2/compiler.js +147 -0
- package/dist/v2/explorer.d.ts +41 -0
- package/dist/v2/explorer.js +56 -0
- package/dist/v2/index.d.ts +37 -0
- package/dist/v2/index.js +31 -0
- package/dist/v2/llm-healer.d.ts +62 -0
- package/dist/v2/llm-healer.js +166 -0
- package/dist/v2/llm-provider.d.ts +29 -0
- package/dist/v2/llm-provider.js +80 -0
- package/dist/v2/opcode-runner.d.ts +47 -0
- package/dist/v2/opcode-runner.js +634 -0
- package/dist/v2/overlay-engine.d.ts +24 -0
- package/dist/v2/overlay-engine.js +150 -0
- package/dist/v2/postcondition.d.ts +16 -0
- package/dist/v2/postcondition.js +249 -0
- package/dist/v2/program-patcher.d.ts +25 -0
- package/dist/v2/program-patcher.js +44 -0
- package/dist/v2/recovery-chain.d.ts +30 -0
- package/dist/v2/recovery-chain.js +368 -0
- package/dist/v2/schema.d.ts +2580 -0
- package/dist/v2/schema.js +295 -0
- package/dist/v2/selector-resolver.d.ts +34 -0
- package/dist/v2/selector-resolver.js +181 -0
- package/dist/v2/semantic-resolver.d.ts +35 -0
- package/dist/v2/semantic-resolver.js +161 -0
- package/dist/v2/smart-wait.d.ts +27 -0
- package/dist/v2/smart-wait.js +81 -0
- package/dist/v2/types.d.ts +444 -0
- package/dist/v2/types.js +19 -0
- package/dist/v2/web-playwright-local.d.ts +69 -0
- package/dist/v2/web-playwright-local.js +392 -0
- package/dist/version.d.ts +1 -0
- package/dist/version.js +5 -0
- package/dist/video-agent.d.ts +143 -0
- package/dist/video-agent.js +4788 -0
- package/dist/video-observation.d.ts +36 -0
- package/dist/video-observation.js +192 -0
- package/dist/video-planner.d.ts +12 -0
- package/dist/video-planner.js +501 -0
- package/dist/video-prompts.d.ts +37 -0
- package/dist/video-prompts.js +554 -0
- package/dist/video-tools.d.ts +3 -0
- package/dist/video-tools.js +59 -0
- package/dist/video-variant-state.d.ts +29 -0
- package/dist/video-variant-state.js +80 -0
- package/dist/vision-model.d.ts +17 -0
- package/dist/vision-model.js +74 -0
- package/dist/web-playwright-local.d.ts +126 -0
- package/dist/web-playwright-local.js +819 -0
- package/dist/ws-auth.d.ts +20 -0
- package/dist/ws-auth.js +70 -0
- package/dist/ws-broadcast.d.ts +34 -0
- package/dist/ws-broadcast.js +85 -0
- package/dist/ws-connection-limits.d.ts +12 -0
- package/dist/ws-connection-limits.js +44 -0
- package/dist/ws-handler-utils.d.ts +32 -0
- package/dist/ws-handler-utils.js +139 -0
- package/dist/ws-handler.d.ts +10 -0
- package/dist/ws-handler.js +1793 -0
- package/dist/ws-metrics-server.d.ts +9 -0
- package/dist/ws-metrics-server.js +31 -0
- package/dist/ws-server.d.ts +9 -0
- package/dist/ws-server.js +92 -0
- package/package.json +142 -71
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { Browser } from '../browser.js';
|
|
2
|
+
import type { VideoPageSignals } from '../types.js';
|
|
3
|
+
export interface VariantStateDetection {
|
|
4
|
+
lang: {
|
|
5
|
+
requested: string | undefined;
|
|
6
|
+
detected: string | null;
|
|
7
|
+
active: boolean;
|
|
8
|
+
ambiguous: boolean;
|
|
9
|
+
};
|
|
10
|
+
theme: {
|
|
11
|
+
requested: 'light' | 'dark' | undefined;
|
|
12
|
+
detected: 'light' | 'dark' | null;
|
|
13
|
+
active: boolean;
|
|
14
|
+
ambiguous: boolean;
|
|
15
|
+
};
|
|
16
|
+
pageSignals: VideoPageSignals;
|
|
17
|
+
}
|
|
18
|
+
export declare function scoreLocaleSignals(signals: VideoPageSignals, requestedLang?: string): {
|
|
19
|
+
score: number;
|
|
20
|
+
reasons: string[];
|
|
21
|
+
ambiguous: boolean;
|
|
22
|
+
};
|
|
23
|
+
export declare function evaluateRequestedThemeState(signals: VideoPageSignals, requestedTheme?: 'light' | 'dark'): {
|
|
24
|
+
detected: 'light' | 'dark' | null;
|
|
25
|
+
active: boolean;
|
|
26
|
+
ambiguous: boolean;
|
|
27
|
+
reason: string;
|
|
28
|
+
};
|
|
29
|
+
export declare function detectVariantStateDeterministic(browser: Browser, requestedLang?: string, requestedTheme?: 'light' | 'dark'): Promise<VariantStateDetection>;
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import { evaluateRequestedLanguageState, evaluateRequestedThemeState as evaluateRequestedThemeStateRich, } from '../session-profile.js';
|
|
2
|
+
function normalizeConfidenceScore(confidence) {
|
|
3
|
+
switch (confidence) {
|
|
4
|
+
case 'high':
|
|
5
|
+
return 1;
|
|
6
|
+
case 'medium':
|
|
7
|
+
return 0.78;
|
|
8
|
+
case 'low':
|
|
9
|
+
default:
|
|
10
|
+
return 0.58;
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
export function scoreLocaleSignals(signals, requestedLang) {
|
|
14
|
+
if (!requestedLang)
|
|
15
|
+
return { score: 1, reasons: [], ambiguous: false };
|
|
16
|
+
const languageState = evaluateRequestedLanguageState({
|
|
17
|
+
currentUrl: signals.url,
|
|
18
|
+
requestedLang,
|
|
19
|
+
signals,
|
|
20
|
+
});
|
|
21
|
+
if (languageState.active) {
|
|
22
|
+
return {
|
|
23
|
+
score: normalizeConfidenceScore(languageState.confidence),
|
|
24
|
+
reasons: languageState.reasons,
|
|
25
|
+
ambiguous: false,
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
if (languageState.ambiguous) {
|
|
29
|
+
return {
|
|
30
|
+
score: languageState.confidence === 'high' ? 0.45 : languageState.confidence === 'medium' ? 0.32 : 0.18,
|
|
31
|
+
reasons: languageState.reasons,
|
|
32
|
+
ambiguous: true,
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
return {
|
|
36
|
+
score: 0,
|
|
37
|
+
reasons: languageState.reasons,
|
|
38
|
+
ambiguous: false,
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
export function evaluateRequestedThemeState(signals, requestedTheme) {
|
|
42
|
+
const themeState = evaluateRequestedThemeStateRich({
|
|
43
|
+
requestedTheme,
|
|
44
|
+
signals,
|
|
45
|
+
});
|
|
46
|
+
return {
|
|
47
|
+
detected: themeState.detected,
|
|
48
|
+
active: themeState.active,
|
|
49
|
+
ambiguous: themeState.ambiguous,
|
|
50
|
+
reason: themeState.reasons.join('; ') || 'no_theme_signal',
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
export async function detectVariantStateDeterministic(browser, requestedLang, requestedTheme) {
|
|
54
|
+
const signals = await browser.capturePageSignals();
|
|
55
|
+
const languageState = evaluateRequestedLanguageState({
|
|
56
|
+
currentUrl: signals.url,
|
|
57
|
+
requestedLang,
|
|
58
|
+
signals,
|
|
59
|
+
});
|
|
60
|
+
const themeState = evaluateRequestedThemeStateRich({
|
|
61
|
+
requestedTheme,
|
|
62
|
+
signals,
|
|
63
|
+
});
|
|
64
|
+
return {
|
|
65
|
+
lang: {
|
|
66
|
+
requested: requestedLang,
|
|
67
|
+
detected: languageState.detected,
|
|
68
|
+
active: requestedLang ? languageState.active : true,
|
|
69
|
+
ambiguous: requestedLang ? languageState.ambiguous : false,
|
|
70
|
+
},
|
|
71
|
+
theme: {
|
|
72
|
+
requested: requestedTheme,
|
|
73
|
+
detected: themeState.detected,
|
|
74
|
+
active: requestedTheme ? themeState.active : true,
|
|
75
|
+
ambiguous: requestedTheme ? themeState.ambiguous : false,
|
|
76
|
+
},
|
|
77
|
+
pageSignals: signals,
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
//# sourceMappingURL=video-variant-state.js.map
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
export declare class VisionModelUnsupportedError extends Error {
|
|
2
|
+
requestedModel: string;
|
|
3
|
+
fallbackModel?: string;
|
|
4
|
+
constructor(requestedModel: string, fallbackModel?: string);
|
|
5
|
+
}
|
|
6
|
+
export declare function isImageInputUnsupportedError(err: unknown): boolean;
|
|
7
|
+
export declare function buildVisionModelUnavailableMessage(requestedModel: string, fallbackModel?: string): string;
|
|
8
|
+
export declare function callVisionCapableModel<T>(params: {
|
|
9
|
+
primaryModel: string;
|
|
10
|
+
fallbackModel?: string;
|
|
11
|
+
callModel: (model: string) => Promise<T>;
|
|
12
|
+
onFallbackActivated?: (model: string, reason: string) => void;
|
|
13
|
+
}): Promise<{
|
|
14
|
+
result: T;
|
|
15
|
+
model: string;
|
|
16
|
+
fellBack: boolean;
|
|
17
|
+
}>;
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
export class VisionModelUnsupportedError extends Error {
|
|
2
|
+
requestedModel;
|
|
3
|
+
fallbackModel;
|
|
4
|
+
constructor(requestedModel, fallbackModel) {
|
|
5
|
+
super(buildVisionModelUnavailableMessage(requestedModel, fallbackModel));
|
|
6
|
+
this.name = 'VisionModelUnsupportedError';
|
|
7
|
+
this.requestedModel = requestedModel;
|
|
8
|
+
this.fallbackModel = fallbackModel;
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
function extractNestedErrorMessage(err) {
|
|
12
|
+
if (!err || typeof err !== 'object')
|
|
13
|
+
return '';
|
|
14
|
+
const errorRecord = err;
|
|
15
|
+
const directMessage = typeof errorRecord.message === 'string' ? errorRecord.message : '';
|
|
16
|
+
const nestedError = errorRecord.error;
|
|
17
|
+
const nestedMessage = nestedError && typeof nestedError === 'object' && typeof nestedError.message === 'string'
|
|
18
|
+
? nestedError.message
|
|
19
|
+
: '';
|
|
20
|
+
if (nestedMessage && nestedMessage === directMessage)
|
|
21
|
+
return directMessage;
|
|
22
|
+
return [directMessage, nestedMessage].filter(Boolean).join(' ');
|
|
23
|
+
}
|
|
24
|
+
export function isImageInputUnsupportedError(err) {
|
|
25
|
+
if (!err || typeof err !== 'object')
|
|
26
|
+
return false;
|
|
27
|
+
const errorRecord = err;
|
|
28
|
+
const status = typeof errorRecord.status === 'number' ? errorRecord.status : undefined;
|
|
29
|
+
const message = extractNestedErrorMessage(err).toLowerCase();
|
|
30
|
+
if (!message)
|
|
31
|
+
return false;
|
|
32
|
+
const mentionsImageEndpoint = message.includes('no endpoints found that support image input')
|
|
33
|
+
|| message.includes('no providers that support image input')
|
|
34
|
+
|| (message.includes('image input') && message.includes('no') && message.includes('support'))
|
|
35
|
+
|| (message.includes('image input') && message.includes('endpoint') && message.includes('no'));
|
|
36
|
+
if (!mentionsImageEndpoint)
|
|
37
|
+
return false;
|
|
38
|
+
return status === undefined || status === 400 || status === 404 || status === 422;
|
|
39
|
+
}
|
|
40
|
+
export function buildVisionModelUnavailableMessage(requestedModel, fallbackModel) {
|
|
41
|
+
const fallbackHint = fallbackModel && fallbackModel !== requestedModel
|
|
42
|
+
? ` Automatic fallback to "${fallbackModel}" also failed or was unavailable.`
|
|
43
|
+
: '';
|
|
44
|
+
return `The model "${requestedModel}" has no available endpoint that supports image input.${fallbackHint} Configure a vision-capable default/fallback model or retry later if the provider is degraded.`;
|
|
45
|
+
}
|
|
46
|
+
export async function callVisionCapableModel(params) {
|
|
47
|
+
const { primaryModel, fallbackModel, callModel, onFallbackActivated } = params;
|
|
48
|
+
let fallbackReason = '';
|
|
49
|
+
try {
|
|
50
|
+
const result = await callModel(primaryModel);
|
|
51
|
+
return { result, model: primaryModel, fellBack: false };
|
|
52
|
+
}
|
|
53
|
+
catch (err) {
|
|
54
|
+
if (!isImageInputUnsupportedError(err)) {
|
|
55
|
+
throw err;
|
|
56
|
+
}
|
|
57
|
+
fallbackReason = extractNestedErrorMessage(err) || 'image input unsupported';
|
|
58
|
+
if (!fallbackModel || fallbackModel === primaryModel) {
|
|
59
|
+
throw new VisionModelUnsupportedError(primaryModel, fallbackModel);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
try {
|
|
63
|
+
const result = await callModel(fallbackModel);
|
|
64
|
+
onFallbackActivated?.(fallbackModel, fallbackReason);
|
|
65
|
+
return { result, model: fallbackModel, fellBack: true };
|
|
66
|
+
}
|
|
67
|
+
catch (fallbackErr) {
|
|
68
|
+
if (isImageInputUnsupportedError(fallbackErr)) {
|
|
69
|
+
throw new VisionModelUnsupportedError(primaryModel, fallbackModel);
|
|
70
|
+
}
|
|
71
|
+
throw fallbackErr;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
//# sourceMappingURL=vision-model.js.map
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Capture Agent — LLM Healer
|
|
3
|
+
*
|
|
4
|
+
* Last-resort recovery: when all deterministic strategies fail,
|
|
5
|
+
* the healer asks an LLM to analyze the current page state and
|
|
6
|
+
* produce a replacement opcode.
|
|
7
|
+
*
|
|
8
|
+
* Constraints:
|
|
9
|
+
* - Cannot change the preset's intention
|
|
10
|
+
* - Cannot change variant order
|
|
11
|
+
* - Cannot invent new capture targets
|
|
12
|
+
* - Cannot skip the failed opcode's postcondition
|
|
13
|
+
* - Max 1 LLM call per healing attempt
|
|
14
|
+
* - Max 3 healing attempts per run
|
|
15
|
+
*/
|
|
16
|
+
import type { ExecutionOpcode, HealerPatch } from './execution-types.js';
|
|
17
|
+
import type { LLMCallResult } from './llm-provider.js';
|
|
18
|
+
export interface HealerContext {
|
|
19
|
+
/** The failed opcode */
|
|
20
|
+
failedOpcode: ExecutionOpcode;
|
|
21
|
+
/** Index in the program */
|
|
22
|
+
opcodeIndex: number;
|
|
23
|
+
/** Serialized AKTree of the current page */
|
|
24
|
+
akTreeSerialized: string;
|
|
25
|
+
/** Current page URL */
|
|
26
|
+
currentUrl: string;
|
|
27
|
+
/** Screenshot buffer (for multimodal models) */
|
|
28
|
+
screenshot?: Buffer;
|
|
29
|
+
/** The 3 opcodes before and after the failed one (for context) */
|
|
30
|
+
surroundingOpcodes: ExecutionOpcode[];
|
|
31
|
+
/** Error message from the failed attempt */
|
|
32
|
+
errorMessage: string;
|
|
33
|
+
}
|
|
34
|
+
export interface HealerResult {
|
|
35
|
+
healed: boolean;
|
|
36
|
+
patch?: HealerPatch;
|
|
37
|
+
reason: string;
|
|
38
|
+
/** Raw LLM usage for this healing attempt */
|
|
39
|
+
llmResult?: LLMCallResult;
|
|
40
|
+
}
|
|
41
|
+
export interface HealerLLMProvider {
|
|
42
|
+
/**
|
|
43
|
+
* Call the LLM with a healing prompt. Returns raw JSON string.
|
|
44
|
+
* The provider handles model selection, API keys, etc.
|
|
45
|
+
*/
|
|
46
|
+
call(prompt: string, screenshot?: Buffer): Promise<{
|
|
47
|
+
response: string;
|
|
48
|
+
llmResult: LLMCallResult;
|
|
49
|
+
}>;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* The LLM Healer — a constrained agent that repairs failed opcodes.
|
|
53
|
+
*/
|
|
54
|
+
export declare class LLMHealer {
|
|
55
|
+
private llmProvider;
|
|
56
|
+
private invocationCount;
|
|
57
|
+
private maxInvocations;
|
|
58
|
+
constructor(llmProvider: HealerLLMProvider, options?: {
|
|
59
|
+
maxInvocations?: number;
|
|
60
|
+
});
|
|
61
|
+
get remainingInvocations(): number;
|
|
62
|
+
heal(context: HealerContext): Promise<HealerResult>;
|
|
63
|
+
}
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Capture Agent — LLM Healer
|
|
3
|
+
*
|
|
4
|
+
* Last-resort recovery: when all deterministic strategies fail,
|
|
5
|
+
* the healer asks an LLM to analyze the current page state and
|
|
6
|
+
* produce a replacement opcode.
|
|
7
|
+
*
|
|
8
|
+
* Constraints:
|
|
9
|
+
* - Cannot change the preset's intention
|
|
10
|
+
* - Cannot change variant order
|
|
11
|
+
* - Cannot invent new capture targets
|
|
12
|
+
* - Cannot skip the failed opcode's postcondition
|
|
13
|
+
* - Max 1 LLM call per healing attempt
|
|
14
|
+
* - Max 3 healing attempts per run
|
|
15
|
+
*/
|
|
16
|
+
import { ExecutionOpcodeSchema } from './execution-schema.js';
|
|
17
|
+
/**
|
|
18
|
+
* The LLM Healer — a constrained agent that repairs failed opcodes.
|
|
19
|
+
*/
|
|
20
|
+
export class LLMHealer {
|
|
21
|
+
llmProvider;
|
|
22
|
+
invocationCount = 0;
|
|
23
|
+
maxInvocations;
|
|
24
|
+
constructor(llmProvider, options) {
|
|
25
|
+
this.llmProvider = llmProvider;
|
|
26
|
+
this.maxInvocations = options?.maxInvocations ?? 3;
|
|
27
|
+
}
|
|
28
|
+
get remainingInvocations() {
|
|
29
|
+
return Math.max(0, this.maxInvocations - this.invocationCount);
|
|
30
|
+
}
|
|
31
|
+
async heal(context) {
|
|
32
|
+
if (this.invocationCount >= this.maxInvocations) {
|
|
33
|
+
return { healed: false, reason: `healer budget exhausted (${this.maxInvocations}/${this.maxInvocations} used)` };
|
|
34
|
+
}
|
|
35
|
+
this.invocationCount++;
|
|
36
|
+
try {
|
|
37
|
+
const prompt = buildHealerPrompt(context);
|
|
38
|
+
const { response, llmResult } = await this.llmProvider.call(prompt, context.screenshot);
|
|
39
|
+
// Parse the LLM response
|
|
40
|
+
const parsed = parseHealerResponse(response, context);
|
|
41
|
+
if (!parsed) {
|
|
42
|
+
return { healed: false, reason: 'healer produced invalid response', llmResult };
|
|
43
|
+
}
|
|
44
|
+
if (parsed.cannotHeal) {
|
|
45
|
+
return { healed: false, reason: parsed.reason ?? 'healer cannot resolve this failure', llmResult };
|
|
46
|
+
}
|
|
47
|
+
// Validate replacement opcodes against schema
|
|
48
|
+
const validOpcodes = [];
|
|
49
|
+
for (const opcode of parsed.replacementOpcodes) {
|
|
50
|
+
const result = ExecutionOpcodeSchema.safeParse(opcode);
|
|
51
|
+
if (!result.success) {
|
|
52
|
+
return { healed: false, reason: `healer produced invalid opcode: ${result.error}`, llmResult };
|
|
53
|
+
}
|
|
54
|
+
validOpcodes.push(result.data);
|
|
55
|
+
}
|
|
56
|
+
if (validOpcodes.length === 0 || validOpcodes.length > 3) {
|
|
57
|
+
return { healed: false, reason: `healer produced ${validOpcodes.length} opcodes (expected 1-3)`, llmResult };
|
|
58
|
+
}
|
|
59
|
+
const patch = {
|
|
60
|
+
opcodeIndex: context.opcodeIndex,
|
|
61
|
+
originalOpcode: context.failedOpcode,
|
|
62
|
+
replacementOpcodes: validOpcodes,
|
|
63
|
+
reason: parsed.reason ?? 'healer repair',
|
|
64
|
+
patchedAt: new Date().toISOString(),
|
|
65
|
+
};
|
|
66
|
+
return { healed: true, patch, reason: parsed.reason ?? 'healer repair applied', llmResult };
|
|
67
|
+
}
|
|
68
|
+
catch (err) {
|
|
69
|
+
return {
|
|
70
|
+
healed: false,
|
|
71
|
+
reason: `healer error: ${err instanceof Error ? err.message : String(err)}`,
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
// ── Prompt construction ─────────────────────────────────────────────
|
|
77
|
+
function buildHealerPrompt(context) {
|
|
78
|
+
const { failedOpcode, errorMessage, akTreeSerialized, currentUrl, surroundingOpcodes } = context;
|
|
79
|
+
const surroundingDesc = surroundingOpcodes
|
|
80
|
+
.map((op, i) => ` ${i + 1}. ${op.kind}: ${op.description}`)
|
|
81
|
+
.join('\n');
|
|
82
|
+
return `You are a web automation repair agent. An opcode in a deterministic browser automation program has failed. Your job is to analyze the current page state and produce a replacement opcode that achieves the same goal.
|
|
83
|
+
|
|
84
|
+
## Failed Opcode
|
|
85
|
+
Kind: ${failedOpcode.kind}
|
|
86
|
+
Description: ${failedOpcode.description}
|
|
87
|
+
${failedOpcode.kind === 'CLICK' ? `Selector: ${failedOpcode.selector}` : ''}
|
|
88
|
+
${failedOpcode.kind === 'TYPE' ? `Selector: ${failedOpcode.selector}, Text: ${failedOpcode.text}` : ''}
|
|
89
|
+
${failedOpcode.kind === 'WAIT_FOR' ? `Selector: ${failedOpcode.selector}` : ''}
|
|
90
|
+
|
|
91
|
+
## Error
|
|
92
|
+
${errorMessage}
|
|
93
|
+
|
|
94
|
+
## Current Page
|
|
95
|
+
URL: ${currentUrl}
|
|
96
|
+
|
|
97
|
+
## AKTree (current page structure)
|
|
98
|
+
${akTreeSerialized.slice(0, 8000)}
|
|
99
|
+
|
|
100
|
+
## Surrounding Opcodes (context)
|
|
101
|
+
${surroundingDesc}
|
|
102
|
+
|
|
103
|
+
## Rules
|
|
104
|
+
1. You MUST produce the same kind of opcode (${failedOpcode.kind}) or a sequence of 1-3 opcodes that achieve the same postcondition.
|
|
105
|
+
2. You CANNOT change the intention — only fix the selector or interaction method.
|
|
106
|
+
3. You CANNOT invent new navigation targets or capture actions.
|
|
107
|
+
4. Every opcode must have a valid postcondition.
|
|
108
|
+
5. Use selectors you can actually see in the AKTree above.
|
|
109
|
+
|
|
110
|
+
## Response Format
|
|
111
|
+
Respond with a JSON object:
|
|
112
|
+
{
|
|
113
|
+
"cannotHeal": false,
|
|
114
|
+
"reason": "explanation of what you changed",
|
|
115
|
+
"replacementOpcodes": [
|
|
116
|
+
{ "kind": "${failedOpcode.kind}", "description": "...", "selector": "...", "postcondition": {...}, "recovery": {...}, "timeoutMs": 15000, "maxFailures": 3, ... }
|
|
117
|
+
]
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
Or if you cannot fix it:
|
|
121
|
+
{
|
|
122
|
+
"cannotHeal": true,
|
|
123
|
+
"reason": "explanation of why this cannot be fixed"
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
Respond ONLY with the JSON object, no markdown, no explanation outside JSON.`;
|
|
127
|
+
}
|
|
128
|
+
function parseHealerResponse(response, context) {
|
|
129
|
+
try {
|
|
130
|
+
// Extract JSON from response (handle markdown code blocks)
|
|
131
|
+
const jsonStr = response
|
|
132
|
+
.replace(/```json\s*/g, '')
|
|
133
|
+
.replace(/```\s*/g, '')
|
|
134
|
+
.trim();
|
|
135
|
+
const parsed = JSON.parse(jsonStr);
|
|
136
|
+
if (typeof parsed !== 'object' || parsed === null)
|
|
137
|
+
return null;
|
|
138
|
+
if (parsed.cannotHeal) {
|
|
139
|
+
return {
|
|
140
|
+
cannotHeal: true,
|
|
141
|
+
reason: parsed.reason ?? null,
|
|
142
|
+
replacementOpcodes: [],
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
if (!Array.isArray(parsed.replacementOpcodes))
|
|
146
|
+
return null;
|
|
147
|
+
// Inject the original postcondition and recovery into replacement opcodes
|
|
148
|
+
// (healer cannot skip postconditions)
|
|
149
|
+
const enriched = parsed.replacementOpcodes.map((op) => ({
|
|
150
|
+
...op,
|
|
151
|
+
postcondition: op.postcondition ?? context.failedOpcode.postcondition,
|
|
152
|
+
recovery: op.recovery ?? context.failedOpcode.recovery,
|
|
153
|
+
timeoutMs: op.timeoutMs ?? context.failedOpcode.timeoutMs,
|
|
154
|
+
maxFailures: op.maxFailures ?? context.failedOpcode.maxFailures,
|
|
155
|
+
}));
|
|
156
|
+
return {
|
|
157
|
+
cannotHeal: false,
|
|
158
|
+
reason: parsed.reason ?? null,
|
|
159
|
+
replacementOpcodes: enriched,
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
catch {
|
|
163
|
+
return null;
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
//# sourceMappingURL=llm-healer.js.map
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Capture Agent — LLM Provider
|
|
3
|
+
*
|
|
4
|
+
* Thin wrapper around OpenAI SDK for OpenRouter calls.
|
|
5
|
+
* Used by: capture verification, alt text generation, healer.
|
|
6
|
+
*/
|
|
7
|
+
export interface LLMCallResult {
|
|
8
|
+
text: string;
|
|
9
|
+
model: string;
|
|
10
|
+
promptTokens: number;
|
|
11
|
+
completionTokens: number;
|
|
12
|
+
costEur: number;
|
|
13
|
+
/** OpenRouter generation ID for async cost resolution */
|
|
14
|
+
generationId: string | null;
|
|
15
|
+
}
|
|
16
|
+
export interface LLMProviderConfig {
|
|
17
|
+
apiKey: string;
|
|
18
|
+
/** Model to use. Default: google/gemini-2.5-flash */
|
|
19
|
+
model?: string;
|
|
20
|
+
/** Max tokens for response. Default: 200 */
|
|
21
|
+
maxTokens?: number;
|
|
22
|
+
/** Timeout in ms. Default: 15000 */
|
|
23
|
+
timeoutMs?: number;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Makes a single LLM call with optional image input.
|
|
27
|
+
* Returns the text response and usage metadata.
|
|
28
|
+
*/
|
|
29
|
+
export declare function callLLM(config: LLMProviderConfig, systemPrompt: string, userContent: string, image?: Buffer): Promise<LLMCallResult>;
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Capture Agent — LLM Provider
|
|
3
|
+
*
|
|
4
|
+
* Thin wrapper around OpenAI SDK for OpenRouter calls.
|
|
5
|
+
* Used by: capture verification, alt text generation, healer.
|
|
6
|
+
*/
|
|
7
|
+
import OpenAI from 'openai';
|
|
8
|
+
import { createOpenRouterCompletion } from './openrouter-client.js';
|
|
9
|
+
const OPENROUTER_BASE_URL = 'https://openrouter.ai/api/v1';
|
|
10
|
+
const DEFAULT_MODEL = 'google/gemini-2.5-flash';
|
|
11
|
+
/**
|
|
12
|
+
* Makes a single LLM call with optional image input.
|
|
13
|
+
* Returns the text response and usage metadata.
|
|
14
|
+
*/
|
|
15
|
+
export async function callLLM(config, systemPrompt, userContent, image) {
|
|
16
|
+
const client = new OpenAI({
|
|
17
|
+
baseURL: OPENROUTER_BASE_URL,
|
|
18
|
+
apiKey: config.apiKey,
|
|
19
|
+
});
|
|
20
|
+
const model = config.model ?? DEFAULT_MODEL;
|
|
21
|
+
const maxTokens = config.maxTokens ?? 200;
|
|
22
|
+
const messages = [
|
|
23
|
+
{ role: 'system', content: systemPrompt },
|
|
24
|
+
];
|
|
25
|
+
if (image) {
|
|
26
|
+
const base64 = image.toString('base64');
|
|
27
|
+
messages.push({
|
|
28
|
+
role: 'user',
|
|
29
|
+
content: [
|
|
30
|
+
{ type: 'text', text: userContent },
|
|
31
|
+
{
|
|
32
|
+
type: 'image_url',
|
|
33
|
+
image_url: { url: `data:image/png;base64,${base64}`, detail: 'low' },
|
|
34
|
+
},
|
|
35
|
+
],
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
else {
|
|
39
|
+
messages.push({ role: 'user', content: userContent });
|
|
40
|
+
}
|
|
41
|
+
const params = {
|
|
42
|
+
model,
|
|
43
|
+
messages,
|
|
44
|
+
max_tokens: maxTokens,
|
|
45
|
+
temperature: 0.1,
|
|
46
|
+
};
|
|
47
|
+
const controller = new AbortController();
|
|
48
|
+
const timer = setTimeout(() => controller.abort(), config.timeoutMs ?? 15000);
|
|
49
|
+
try {
|
|
50
|
+
const response = await createOpenRouterCompletion(client, params, { signal: controller.signal });
|
|
51
|
+
clearTimeout(timer);
|
|
52
|
+
const text = response.choices?.[0]?.message?.content?.trim() ?? '';
|
|
53
|
+
const usage = response.usage;
|
|
54
|
+
return {
|
|
55
|
+
text,
|
|
56
|
+
model: response.model ?? model,
|
|
57
|
+
promptTokens: usage?.prompt_tokens ?? 0,
|
|
58
|
+
completionTokens: usage?.completion_tokens ?? 0,
|
|
59
|
+
costEur: estimateCost(model, usage?.prompt_tokens ?? 0, usage?.completion_tokens ?? 0),
|
|
60
|
+
generationId: response.id ?? null,
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
catch (err) {
|
|
64
|
+
clearTimeout(timer);
|
|
65
|
+
throw err;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Rough cost estimation based on known model pricing.
|
|
70
|
+
* Actual cost is resolved asynchronously via OpenRouter generation API.
|
|
71
|
+
*/
|
|
72
|
+
function estimateCost(model, promptTokens, completionTokens) {
|
|
73
|
+
// Gemini 2.5 Flash pricing (approximate)
|
|
74
|
+
if (model.includes('gemini')) {
|
|
75
|
+
return (promptTokens * 0.15 + completionTokens * 0.6) / 1_000_000;
|
|
76
|
+
}
|
|
77
|
+
// Fallback: conservative estimate
|
|
78
|
+
return (promptTokens * 0.5 + completionTokens * 1.5) / 1_000_000;
|
|
79
|
+
}
|
|
80
|
+
//# sourceMappingURL=llm-provider.js.map
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { ChatCompletion } from 'openai/resources/chat/completions';
|
|
2
|
+
import type { StepUsage } from './types.js';
|
|
3
|
+
export type LlmUsageSnapshot = Pick<StepUsage, 'generationId' | 'modelUsed' | 'promptTokens' | 'completionTokens' | 'totalTokens' | 'cacheReadTokens' | 'cacheWriteTokens' | 'reasoningTokens'>;
|
|
4
|
+
export declare function extractLlmUsageSnapshot(response: ChatCompletion): LlmUsageSnapshot;
|
|
5
|
+
export declare function buildStepUsageFromSnapshot(snapshot: LlmUsageSnapshot, params: {
|
|
6
|
+
stepNumber: number;
|
|
7
|
+
stepType: StepUsage['stepType'];
|
|
8
|
+
modelRequested: string;
|
|
9
|
+
imagesInPrompt: number;
|
|
10
|
+
}): StepUsage;
|
|
11
|
+
export declare function normalizeLlmUsageSnapshot(snapshot: Partial<LlmUsageSnapshot>): LlmUsageSnapshot;
|
|
12
|
+
export declare function extractStepUsage(response: ChatCompletion, params: {
|
|
13
|
+
stepNumber: number;
|
|
14
|
+
stepType: StepUsage['stepType'];
|
|
15
|
+
modelRequested: string;
|
|
16
|
+
imagesInPrompt: number;
|
|
17
|
+
}): StepUsage;
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
export function extractLlmUsageSnapshot(response) {
|
|
2
|
+
const usageDetails = response.usage;
|
|
3
|
+
const promptDetails = usageDetails?.prompt_tokens_details;
|
|
4
|
+
const completionDetails = usageDetails?.completion_tokens_details;
|
|
5
|
+
return {
|
|
6
|
+
generationId: response.id ?? null,
|
|
7
|
+
modelUsed: response.model ?? null,
|
|
8
|
+
promptTokens: response.usage?.prompt_tokens ?? null,
|
|
9
|
+
completionTokens: response.usage?.completion_tokens ?? null,
|
|
10
|
+
totalTokens: response.usage?.total_tokens ?? null,
|
|
11
|
+
cacheReadTokens: promptDetails?.cached_tokens
|
|
12
|
+
?? usageDetails?.cache_read_input_tokens
|
|
13
|
+
?? null,
|
|
14
|
+
cacheWriteTokens: promptDetails?.cache_creation_input_tokens
|
|
15
|
+
?? usageDetails?.cache_write_input_tokens
|
|
16
|
+
?? null,
|
|
17
|
+
reasoningTokens: completionDetails?.reasoning_tokens
|
|
18
|
+
?? null,
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
export function buildStepUsageFromSnapshot(snapshot, params) {
|
|
22
|
+
return {
|
|
23
|
+
stepNumber: params.stepNumber,
|
|
24
|
+
stepType: params.stepType,
|
|
25
|
+
modelRequested: params.modelRequested,
|
|
26
|
+
imagesInPrompt: params.imagesInPrompt,
|
|
27
|
+
...snapshot,
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
export function normalizeLlmUsageSnapshot(snapshot) {
|
|
31
|
+
return {
|
|
32
|
+
generationId: snapshot.generationId ?? null,
|
|
33
|
+
modelUsed: snapshot.modelUsed ?? null,
|
|
34
|
+
promptTokens: snapshot.promptTokens ?? null,
|
|
35
|
+
completionTokens: snapshot.completionTokens ?? null,
|
|
36
|
+
totalTokens: snapshot.totalTokens ?? null,
|
|
37
|
+
cacheReadTokens: snapshot.cacheReadTokens ?? null,
|
|
38
|
+
cacheWriteTokens: snapshot.cacheWriteTokens ?? null,
|
|
39
|
+
reasoningTokens: snapshot.reasoningTokens ?? null,
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
export function extractStepUsage(response, params) {
|
|
43
|
+
return buildStepUsageFromSnapshot(extractLlmUsageSnapshot(response), params);
|
|
44
|
+
}
|
|
45
|
+
//# sourceMappingURL=llm-usage.js.map
|
package/dist/logger.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export type LogLevel = 'info' | 'action' | 'success' | 'error' | 'ai';
|
|
1
|
+
export type LogLevel = 'info' | 'warn' | 'action' | 'success' | 'error' | 'ai';
|
|
2
2
|
export type LogKind = 'reasoning' | 'action' | 'system' | 'guidance';
|
|
3
3
|
export interface LogEntry {
|
|
4
4
|
level: LogLevel;
|
|
@@ -25,6 +25,8 @@ interface LoggerContext {
|
|
|
25
25
|
onScreenshot?: OnScreenshotCallback | null;
|
|
26
26
|
onReasoningDelta?: OnReasoningDeltaCallback | null;
|
|
27
27
|
}
|
|
28
|
+
export declare function setDebugEnabled(enabled: boolean): void;
|
|
29
|
+
export declare function isDebugEnabled(): boolean;
|
|
28
30
|
export declare function runWithLoggerCallbacks<T>(callbacks: LoggerContext, fn: () => Promise<T>): Promise<T>;
|
|
29
31
|
export declare function setOnLog(cb: OnLogCallback | null): void;
|
|
30
32
|
export declare function setOnScreenshot(cb: OnScreenshotCallback | null): void;
|
|
@@ -33,8 +35,10 @@ export declare function emitReasoningDelta(delta: string, messageId: string): vo
|
|
|
33
35
|
export declare const logger: {
|
|
34
36
|
/** User-visible informational log — shown in both console and UI. */
|
|
35
37
|
info(msg: string): void;
|
|
36
|
-
/** Internal debug log —
|
|
38
|
+
/** Internal debug log — only printed when --debug is enabled. Console only, NOT emitted to the UI callback. */
|
|
37
39
|
debug(msg: string): void;
|
|
40
|
+
/** Security / rate-limit warnings — emitted to both console and UI. */
|
|
41
|
+
warn(msg: string): void;
|
|
38
42
|
action(iteration: number, maxIter: number, action: string, params: Record<string, unknown>, context?: {
|
|
39
43
|
lang?: string;
|
|
40
44
|
theme?: string;
|
package/dist/logger.js
CHANGED
|
@@ -2,7 +2,14 @@ import { AsyncLocalStorage } from 'node:async_hooks';
|
|
|
2
2
|
import chalk from 'chalk';
|
|
3
3
|
let _onLog = null;
|
|
4
4
|
let _onScreenshot = null;
|
|
5
|
+
let _debugEnabled = false;
|
|
5
6
|
const _loggerContext = new AsyncLocalStorage();
|
|
7
|
+
export function setDebugEnabled(enabled) {
|
|
8
|
+
_debugEnabled = enabled;
|
|
9
|
+
}
|
|
10
|
+
export function isDebugEnabled() {
|
|
11
|
+
return _debugEnabled;
|
|
12
|
+
}
|
|
6
13
|
export async function runWithLoggerCallbacks(callbacks, fn) {
|
|
7
14
|
return _loggerContext.run(callbacks, fn);
|
|
8
15
|
}
|
|
@@ -42,10 +49,17 @@ export const logger = {
|
|
|
42
49
|
console.log(chalk.cyan('[INFO]'), msg);
|
|
43
50
|
emit({ level: 'info', message: msg, timestamp: Date.now(), kind: 'system' });
|
|
44
51
|
},
|
|
45
|
-
/** Internal debug log —
|
|
52
|
+
/** Internal debug log — only printed when --debug is enabled. Console only, NOT emitted to the UI callback. */
|
|
46
53
|
debug(msg) {
|
|
54
|
+
if (!_debugEnabled)
|
|
55
|
+
return;
|
|
47
56
|
console.log(chalk.gray('[DEBUG]'), msg);
|
|
48
57
|
},
|
|
58
|
+
/** Security / rate-limit warnings — emitted to both console and UI. */
|
|
59
|
+
warn(msg) {
|
|
60
|
+
console.log(chalk.yellow('[WARN]'), msg);
|
|
61
|
+
emit({ level: 'warn', message: msg, timestamp: Date.now(), kind: 'system' });
|
|
62
|
+
},
|
|
49
63
|
action(iteration, maxIter, action, params, context) {
|
|
50
64
|
const paramsStr = Object.entries(params)
|
|
51
65
|
.map(([k, v]) => `${k}=${JSON.stringify(v)}`)
|