explorbot 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +94 -0
- package/README.md +267 -0
- package/assets/sample-files/sample.docx +0 -0
- package/assets/sample-files/sample.mp3 +0 -0
- package/assets/sample-files/sample.mp4 +0 -0
- package/assets/sample-files/sample.pdf +21 -0
- package/assets/sample-files/sample.png +0 -0
- package/assets/sample-files/sample.xlsx +0 -0
- package/assets/sample-files/sample.zip +0 -0
- package/dist/assets/sample-files/sample.docx +0 -0
- package/dist/assets/sample-files/sample.mp3 +0 -0
- package/dist/assets/sample-files/sample.mp4 +0 -0
- package/dist/assets/sample-files/sample.pdf +21 -0
- package/dist/assets/sample-files/sample.png +0 -0
- package/dist/assets/sample-files/sample.xlsx +0 -0
- package/dist/assets/sample-files/sample.zip +0 -0
- package/dist/bin/explorbot-cli.js +683 -0
- package/dist/bin/explorbot-cli.js.map +1 -0
- package/dist/boat/api-tester/bin/apibot-cli.js +5 -0
- package/dist/boat/api-tester/bin/apibot-cli.js.map +1 -0
- package/dist/boat/api-tester/example/apibot.config.js +31 -0
- package/dist/boat/api-tester/example/apibot.config.js.map +1 -0
- package/dist/boat/api-tester/src/ai/chief/styles.js +13 -0
- package/dist/boat/api-tester/src/ai/chief/styles.js.map +1 -0
- package/dist/boat/api-tester/src/ai/chief.js +301 -0
- package/dist/boat/api-tester/src/ai/chief.js.map +1 -0
- package/dist/boat/api-tester/src/ai/curler-tools.js +263 -0
- package/dist/boat/api-tester/src/ai/curler-tools.js.map +1 -0
- package/dist/boat/api-tester/src/ai/curler.js +271 -0
- package/dist/boat/api-tester/src/ai/curler.js.map +1 -0
- package/dist/boat/api-tester/src/api-client.js +26 -0
- package/dist/boat/api-tester/src/api-client.js.map +1 -0
- package/dist/boat/api-tester/src/apibot.js +166 -0
- package/dist/boat/api-tester/src/apibot.js.map +1 -0
- package/dist/boat/api-tester/src/cli.js +262 -0
- package/dist/boat/api-tester/src/cli.js.map +1 -0
- package/dist/boat/api-tester/src/config.js +159 -0
- package/dist/boat/api-tester/src/config.js.map +1 -0
- package/dist/prompts/audit-rules.md +124 -0
- package/dist/rules/chief/general.md +11 -0
- package/dist/rules/chief/styles/curious.md +12 -0
- package/dist/rules/chief/styles/hacker.md +19 -0
- package/dist/rules/chief/styles/normal.md +11 -0
- package/dist/rules/chief/styles/psycho.md +17 -0
- package/dist/rules/navigator/multiple-locator.md +47 -0
- package/dist/rules/navigator/output.md +69 -0
- package/dist/rules/navigator/verification-actions.md +122 -0
- package/dist/rules/navigator/verification-output.md +53 -0
- package/dist/rules/planner/styles/curious.md +39 -0
- package/dist/rules/planner/styles/normal.md +21 -0
- package/dist/rules/planner/styles/psycho.md +14 -0
- package/dist/rules/researcher/list-element.md +11 -0
- package/dist/rules/researcher/screenshot-ui-map.md +30 -0
- package/dist/rules/researcher/section-ui-map.md +18 -0
- package/dist/rules/researcher/ui-map-table.md +18 -0
- package/dist/src/action-result.js +574 -0
- package/dist/src/action-result.js.map +1 -0
- package/dist/src/action.js +388 -0
- package/dist/src/action.js.map +1 -0
- package/dist/src/activity.js +86 -0
- package/dist/src/activity.js.map +1 -0
- package/dist/src/ai/agent.js +2 -0
- package/dist/src/ai/agent.js.map +1 -0
- package/dist/src/ai/bosun.js +443 -0
- package/dist/src/ai/bosun.js.map +1 -0
- package/dist/src/ai/captain/idle-mode.js +102 -0
- package/dist/src/ai/captain/idle-mode.js.map +1 -0
- package/dist/src/ai/captain/mixin.js +11 -0
- package/dist/src/ai/captain/mixin.js.map +1 -0
- package/dist/src/ai/captain/test-mode.js +251 -0
- package/dist/src/ai/captain/test-mode.js.map +1 -0
- package/dist/src/ai/captain/web-mode.js +124 -0
- package/dist/src/ai/captain/web-mode.js.map +1 -0
- package/dist/src/ai/captain.js +442 -0
- package/dist/src/ai/captain.js.map +1 -0
- package/dist/src/ai/conversation.js +176 -0
- package/dist/src/ai/conversation.js.map +1 -0
- package/dist/src/ai/experience-compactor.js +232 -0
- package/dist/src/ai/experience-compactor.js.map +1 -0
- package/dist/src/ai/fisherman-tools.js +154 -0
- package/dist/src/ai/fisherman-tools.js.map +1 -0
- package/dist/src/ai/fisherman.js +184 -0
- package/dist/src/ai/fisherman.js.map +1 -0
- package/dist/src/ai/historian.js +384 -0
- package/dist/src/ai/historian.js.map +1 -0
- package/dist/src/ai/navigator.js +493 -0
- package/dist/src/ai/navigator.js.map +1 -0
- package/dist/src/ai/pilot.js +684 -0
- package/dist/src/ai/pilot.js.map +1 -0
- package/dist/src/ai/planner/session-dedup.js +28 -0
- package/dist/src/ai/planner/session-dedup.js.map +1 -0
- package/dist/src/ai/planner/styles.js +15 -0
- package/dist/src/ai/planner/styles.js.map +1 -0
- package/dist/src/ai/planner/subpages.js +118 -0
- package/dist/src/ai/planner/subpages.js.map +1 -0
- package/dist/src/ai/planner.js +486 -0
- package/dist/src/ai/planner.js.map +1 -0
- package/dist/src/ai/provider.js +540 -0
- package/dist/src/ai/provider.js.map +1 -0
- package/dist/src/ai/quartermaster.js +210 -0
- package/dist/src/ai/quartermaster.js.map +1 -0
- package/dist/src/ai/researcher/cache.js +95 -0
- package/dist/src/ai/researcher/cache.js.map +1 -0
- package/dist/src/ai/researcher/coordinates.js +210 -0
- package/dist/src/ai/researcher/coordinates.js.map +1 -0
- package/dist/src/ai/researcher/deep-analysis.js +364 -0
- package/dist/src/ai/researcher/deep-analysis.js.map +1 -0
- package/dist/src/ai/researcher/fingerprint-worker.js +46 -0
- package/dist/src/ai/researcher/fingerprint-worker.js.map +1 -0
- package/dist/src/ai/researcher/focus.js +37 -0
- package/dist/src/ai/researcher/focus.js.map +1 -0
- package/dist/src/ai/researcher/locators.js +242 -0
- package/dist/src/ai/researcher/locators.js.map +1 -0
- package/dist/src/ai/researcher/mixin.js +3 -0
- package/dist/src/ai/researcher/mixin.js.map +1 -0
- package/dist/src/ai/researcher/parser.js +160 -0
- package/dist/src/ai/researcher/parser.js.map +1 -0
- package/dist/src/ai/researcher/research-result.js +110 -0
- package/dist/src/ai/researcher/research-result.js.map +1 -0
- package/dist/src/ai/researcher.js +776 -0
- package/dist/src/ai/researcher.js.map +1 -0
- package/dist/src/ai/rules.js +368 -0
- package/dist/src/ai/rules.js.map +1 -0
- package/dist/src/ai/task-agent.js +110 -0
- package/dist/src/ai/task-agent.js.map +1 -0
- package/dist/src/ai/tester.js +840 -0
- package/dist/src/ai/tester.js.map +1 -0
- package/dist/src/ai/tools.js +980 -0
- package/dist/src/ai/tools.js.map +1 -0
- package/dist/src/api/api-client.js +91 -0
- package/dist/src/api/api-client.js.map +1 -0
- package/dist/src/api/request-result.js +177 -0
- package/dist/src/api/request-result.js.map +1 -0
- package/dist/src/api/request-store.js +109 -0
- package/dist/src/api/request-store.js.map +1 -0
- package/dist/src/api/spec-reader.js +148 -0
- package/dist/src/api/spec-reader.js.map +1 -0
- package/dist/src/api/xhr-capture.js +91 -0
- package/dist/src/api/xhr-capture.js.map +1 -0
- package/dist/src/browser-server.js +67 -0
- package/dist/src/browser-server.js.map +1 -0
- package/dist/src/command-handler.js +363 -0
- package/dist/src/command-handler.js.map +1 -0
- package/dist/src/commands/add-rule-command.js +52 -0
- package/dist/src/commands/add-rule-command.js.map +1 -0
- package/dist/src/commands/base-command.js +14 -0
- package/dist/src/commands/base-command.js.map +1 -0
- package/dist/src/commands/clean-command.js +67 -0
- package/dist/src/commands/clean-command.js.map +1 -0
- package/dist/src/commands/context-aria-command.js +18 -0
- package/dist/src/commands/context-aria-command.js.map +1 -0
- package/dist/src/commands/context-command.js +57 -0
- package/dist/src/commands/context-command.js.map +1 -0
- package/dist/src/commands/context-data-command.js +25 -0
- package/dist/src/commands/context-data-command.js.map +1 -0
- package/dist/src/commands/context-experience-command.js +41 -0
- package/dist/src/commands/context-experience-command.js.map +1 -0
- package/dist/src/commands/context-html-command.js +26 -0
- package/dist/src/commands/context-html-command.js.map +1 -0
- package/dist/src/commands/context-knowledge-command.js +36 -0
- package/dist/src/commands/context-knowledge-command.js.map +1 -0
- package/dist/src/commands/debug-command.js +12 -0
- package/dist/src/commands/debug-command.js.map +1 -0
- package/dist/src/commands/drill-command.js +29 -0
- package/dist/src/commands/drill-command.js.map +1 -0
- package/dist/src/commands/exit-command.js +26 -0
- package/dist/src/commands/exit-command.js.map +1 -0
- package/dist/src/commands/explore-command.js +124 -0
- package/dist/src/commands/explore-command.js.map +1 -0
- package/dist/src/commands/freesail-command.js +84 -0
- package/dist/src/commands/freesail-command.js.map +1 -0
- package/dist/src/commands/help-command.js +7 -0
- package/dist/src/commands/help-command.js.map +1 -0
- package/dist/src/commands/index.js +63 -0
- package/dist/src/commands/index.js.map +1 -0
- package/dist/src/commands/knows-command.js +54 -0
- package/dist/src/commands/knows-command.js.map +1 -0
- package/dist/src/commands/learn-command.js +35 -0
- package/dist/src/commands/learn-command.js.map +1 -0
- package/dist/src/commands/navigate-command.js +16 -0
- package/dist/src/commands/navigate-command.js.map +1 -0
- package/dist/src/commands/path-command.js +70 -0
- package/dist/src/commands/path-command.js.map +1 -0
- package/dist/src/commands/plan-clear-command.js +13 -0
- package/dist/src/commands/plan-clear-command.js.map +1 -0
- package/dist/src/commands/plan-command.js +36 -0
- package/dist/src/commands/plan-command.js.map +1 -0
- package/dist/src/commands/plan-edit-command.js +8 -0
- package/dist/src/commands/plan-edit-command.js.map +1 -0
- package/dist/src/commands/plan-load-command.js +16 -0
- package/dist/src/commands/plan-load-command.js.map +1 -0
- package/dist/src/commands/plan-reload-command.js +23 -0
- package/dist/src/commands/plan-reload-command.js.map +1 -0
- package/dist/src/commands/plan-save-command.js +22 -0
- package/dist/src/commands/plan-save-command.js.map +1 -0
- package/dist/src/commands/research-command.js +38 -0
- package/dist/src/commands/research-command.js.map +1 -0
- package/dist/src/commands/start-command.js +12 -0
- package/dist/src/commands/start-command.js.map +1 -0
- package/dist/src/commands/status-command.js +19 -0
- package/dist/src/commands/status-command.js.map +1 -0
- package/dist/src/commands/test-command.js +85 -0
- package/dist/src/commands/test-command.js.map +1 -0
- package/dist/src/components/ActivityPane.js +55 -0
- package/dist/src/components/ActivityPane.js.map +1 -0
- package/dist/src/components/AddKnowledge.js +122 -0
- package/dist/src/components/AddKnowledge.js.map +1 -0
- package/dist/src/components/AddRule.js +117 -0
- package/dist/src/components/AddRule.js.map +1 -0
- package/dist/src/components/App.js +313 -0
- package/dist/src/components/App.js.map +1 -0
- package/dist/src/components/Autocomplete.js +43 -0
- package/dist/src/components/Autocomplete.js.map +1 -0
- package/dist/src/components/InputPane.js +207 -0
- package/dist/src/components/InputPane.js.map +1 -0
- package/dist/src/components/InputReadline.js +598 -0
- package/dist/src/components/InputReadline.js.map +1 -0
- package/dist/src/components/LogPane.js +123 -0
- package/dist/src/components/LogPane.js.map +1 -0
- package/dist/src/components/PlanEditor.js +126 -0
- package/dist/src/components/PlanEditor.js.map +1 -0
- package/dist/src/components/PlanPane.js +51 -0
- package/dist/src/components/PlanPane.js.map +1 -0
- package/dist/src/components/SessionTimer.js +26 -0
- package/dist/src/components/SessionTimer.js.map +1 -0
- package/dist/src/components/StateTransitionPane.js +107 -0
- package/dist/src/components/StateTransitionPane.js.map +1 -0
- package/dist/src/components/StatusPane.js +37 -0
- package/dist/src/components/StatusPane.js.map +1 -0
- package/dist/src/components/TaskPane.js +96 -0
- package/dist/src/components/TaskPane.js.map +1 -0
- package/dist/src/components/Welcome.js +52 -0
- package/dist/src/components/Welcome.js.map +1 -0
- package/dist/src/components/WelcomeChecklist.js +96 -0
- package/dist/src/components/WelcomeChecklist.js.map +1 -0
- package/dist/src/components/WelcomeCommands.js +61 -0
- package/dist/src/components/WelcomeCommands.js.map +1 -0
- package/dist/src/components/autocomplete-store.js +22 -0
- package/dist/src/components/autocomplete-store.js.map +1 -0
- package/dist/src/components/parse-keypress.js +174 -0
- package/dist/src/components/parse-keypress.js.map +1 -0
- package/dist/src/config.js +249 -0
- package/dist/src/config.js.map +1 -0
- package/dist/src/execution-controller.js +92 -0
- package/dist/src/execution-controller.js.map +1 -0
- package/dist/src/experience-tracker.js +294 -0
- package/dist/src/experience-tracker.js.map +1 -0
- package/dist/src/explorbot.js +348 -0
- package/dist/src/explorbot.js.map +1 -0
- package/dist/src/explorer.js +611 -0
- package/dist/src/explorer.js.map +1 -0
- package/dist/src/index.js +56 -0
- package/dist/src/index.js.map +1 -0
- package/dist/src/knowledge-tracker.js +184 -0
- package/dist/src/knowledge-tracker.js.map +1 -0
- package/dist/src/observability.js +126 -0
- package/dist/src/observability.js.map +1 -0
- package/dist/src/reporter.js +185 -0
- package/dist/src/reporter.js.map +1 -0
- package/dist/src/state-manager.js +427 -0
- package/dist/src/state-manager.js.map +1 -0
- package/dist/src/stats.js +44 -0
- package/dist/src/stats.js.map +1 -0
- package/dist/src/test-plan.js +343 -0
- package/dist/src/test-plan.js.map +1 -0
- package/dist/src/utils/aria.js +588 -0
- package/dist/src/utils/aria.js.map +1 -0
- package/dist/src/utils/code-extractor.js +21 -0
- package/dist/src/utils/code-extractor.js.map +1 -0
- package/dist/src/utils/context-formatter.js +205 -0
- package/dist/src/utils/context-formatter.js.map +1 -0
- package/dist/src/utils/error-page.js +19 -0
- package/dist/src/utils/error-page.js.map +1 -0
- package/dist/src/utils/expandable.js +35 -0
- package/dist/src/utils/expandable.js.map +1 -0
- package/dist/src/utils/hooks-runner.js +77 -0
- package/dist/src/utils/hooks-runner.js.map +1 -0
- package/dist/src/utils/html-diff.js +734 -0
- package/dist/src/utils/html-diff.js.map +1 -0
- package/dist/src/utils/html.js +1163 -0
- package/dist/src/utils/html.js.map +1 -0
- package/dist/src/utils/logger.js +465 -0
- package/dist/src/utils/logger.js.map +1 -0
- package/dist/src/utils/loop.js +126 -0
- package/dist/src/utils/loop.js.map +1 -0
- package/dist/src/utils/markdown-parser.js +117 -0
- package/dist/src/utils/markdown-parser.js.map +1 -0
- package/dist/src/utils/markdown-query.js +393 -0
- package/dist/src/utils/markdown-query.js.map +1 -0
- package/dist/src/utils/markdown-terminal.js +40 -0
- package/dist/src/utils/markdown-terminal.js.map +1 -0
- package/dist/src/utils/research-parser.js +2 -0
- package/dist/src/utils/research-parser.js.map +1 -0
- package/dist/src/utils/retry.js +55 -0
- package/dist/src/utils/retry.js.map +1 -0
- package/dist/src/utils/rules-loader.js +104 -0
- package/dist/src/utils/rules-loader.js.map +1 -0
- package/dist/src/utils/strings.js +14 -0
- package/dist/src/utils/strings.js.map +1 -0
- package/dist/src/utils/test-plan-markdown.js +301 -0
- package/dist/src/utils/test-plan-markdown.js.map +1 -0
- package/dist/src/utils/throttle.js +16 -0
- package/dist/src/utils/throttle.js.map +1 -0
- package/dist/src/utils/unique-names.js +13 -0
- package/dist/src/utils/unique-names.js.map +1 -0
- package/dist/src/utils/url-matcher.js +48 -0
- package/dist/src/utils/url-matcher.js.map +1 -0
- package/dist/src/utils/web-element.js +131 -0
- package/dist/src/utils/web-element.js.map +1 -0
- package/dist/src/utils/xpath.js +110 -0
- package/dist/src/utils/xpath.js.map +1 -0
- package/package.json +119 -0
- package/prompts/audit-rules.md +124 -0
- package/rules/chief/general.md +11 -0
- package/rules/chief/styles/curious.md +12 -0
- package/rules/chief/styles/hacker.md +19 -0
- package/rules/chief/styles/normal.md +11 -0
- package/rules/chief/styles/psycho.md +17 -0
- package/rules/navigator/multiple-locator.md +47 -0
- package/rules/navigator/output.md +69 -0
- package/rules/navigator/verification-actions.md +122 -0
- package/rules/navigator/verification-output.md +53 -0
- package/rules/planner/styles/curious.md +39 -0
- package/rules/planner/styles/normal.md +21 -0
- package/rules/planner/styles/psycho.md +14 -0
- package/rules/researcher/list-element.md +11 -0
- package/rules/researcher/screenshot-ui-map.md +30 -0
- package/rules/researcher/section-ui-map.md +18 -0
- package/rules/researcher/ui-map-table.md +18 -0
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { z } from 'zod';
|
|
4
|
+
import { ConfigParser } from "../config.js";
|
|
5
|
+
import { createDebug, tag } from "../utils/logger.js";
|
|
6
|
+
import { CODECEPT_TOOLS } from "./tools.js";
|
|
7
|
+
const debugLog = createDebug('explorbot:quartermaster');
|
|
8
|
+
export class Quartermaster {
|
|
9
|
+
provider;
|
|
10
|
+
model;
|
|
11
|
+
outputDir;
|
|
12
|
+
pageAnalyses = new Map();
|
|
13
|
+
unsubscribe = null;
|
|
14
|
+
playwrightHelper = null;
|
|
15
|
+
pendingAnalyses = [];
|
|
16
|
+
constructor(provider, options) {
|
|
17
|
+
this.provider = provider;
|
|
18
|
+
this.model = options?.model;
|
|
19
|
+
const configParser = ConfigParser.getInstance();
|
|
20
|
+
this.outputDir = join(configParser.getOutputDir(), 'a11y');
|
|
21
|
+
}
|
|
22
|
+
start(playwrightHelper, stateManager) {
|
|
23
|
+
this.playwrightHelper = playwrightHelper;
|
|
24
|
+
this.ensureDirectory();
|
|
25
|
+
this.unsubscribe = stateManager.onStateChange((event) => {
|
|
26
|
+
this.onPageChange(event);
|
|
27
|
+
});
|
|
28
|
+
debugLog('Quartermaster started, listening for page changes');
|
|
29
|
+
}
|
|
30
|
+
stop() {
|
|
31
|
+
if (this.unsubscribe) {
|
|
32
|
+
this.unsubscribe();
|
|
33
|
+
this.unsubscribe = null;
|
|
34
|
+
}
|
|
35
|
+
debugLog('Quartermaster stopped');
|
|
36
|
+
}
|
|
37
|
+
ensureDirectory() {
|
|
38
|
+
if (!existsSync(this.outputDir)) {
|
|
39
|
+
mkdirSync(this.outputDir, { recursive: true });
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
onPageChange(event) {
|
|
43
|
+
const state = event.toState;
|
|
44
|
+
if (!state?.hash)
|
|
45
|
+
return;
|
|
46
|
+
if (this.pageAnalyses.has(state.hash))
|
|
47
|
+
return;
|
|
48
|
+
const analysisPromise = this.runAxeAnalysis(state).catch((err) => {
|
|
49
|
+
debugLog('Axe analysis failed:', err);
|
|
50
|
+
});
|
|
51
|
+
this.pendingAnalyses.push(analysisPromise);
|
|
52
|
+
}
|
|
53
|
+
async runAxeAnalysis(state) {
|
|
54
|
+
const page = this.playwrightHelper?.page;
|
|
55
|
+
if (!page || !state.hash) {
|
|
56
|
+
debugLog('No page available for axe analysis');
|
|
57
|
+
return;
|
|
58
|
+
}
|
|
59
|
+
try {
|
|
60
|
+
const { AxeBuilder } = await import('@axe-core/playwright');
|
|
61
|
+
const results = await new AxeBuilder({ page }).analyze();
|
|
62
|
+
const violations = results.violations.map((v) => ({
|
|
63
|
+
id: v.id,
|
|
64
|
+
impact: v.impact,
|
|
65
|
+
description: v.description,
|
|
66
|
+
helpUrl: v.helpUrl,
|
|
67
|
+
nodes: v.nodes.map((n) => ({
|
|
68
|
+
html: n.html,
|
|
69
|
+
target: n.target,
|
|
70
|
+
})),
|
|
71
|
+
}));
|
|
72
|
+
this.pageAnalyses.set(state.hash, {
|
|
73
|
+
url: state.url,
|
|
74
|
+
stateHash: state.hash,
|
|
75
|
+
timestamp: new Date().toISOString(),
|
|
76
|
+
axeViolations: violations,
|
|
77
|
+
});
|
|
78
|
+
debugLog(`Axe analysis complete for ${state.url}: ${violations.length} violations`);
|
|
79
|
+
}
|
|
80
|
+
catch (error) {
|
|
81
|
+
debugLog('Axe analysis error:', error);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
async analyzeSession(task, initialState, conversation) {
|
|
85
|
+
await Promise.all(this.pendingAnalyses);
|
|
86
|
+
this.pendingAnalyses = [];
|
|
87
|
+
try {
|
|
88
|
+
const stateHash = initialState.getStateHash();
|
|
89
|
+
const pageAnalysis = this.pageAnalyses.get(stateHash);
|
|
90
|
+
const toolExecutions = conversation.getToolExecutions();
|
|
91
|
+
const codeceptExecutions = toolExecutions.filter((e) => CODECEPT_TOOLS.includes(e.toolName));
|
|
92
|
+
if (codeceptExecutions.length === 0 && !pageAnalysis?.axeViolations.length) {
|
|
93
|
+
debugLog('No interactions or violations to analyze');
|
|
94
|
+
return null;
|
|
95
|
+
}
|
|
96
|
+
const axeViolations = pageAnalysis?.axeViolations || [];
|
|
97
|
+
const semanticIssues = await this.generateSemanticAnalysis(axeViolations, codeceptExecutions, initialState);
|
|
98
|
+
if (axeViolations.length === 0 && semanticIssues.length === 0) {
|
|
99
|
+
debugLog('No issues found');
|
|
100
|
+
return null;
|
|
101
|
+
}
|
|
102
|
+
const report = {
|
|
103
|
+
scenario: task.description,
|
|
104
|
+
timestamp: new Date().toISOString(),
|
|
105
|
+
url: initialState.url || '',
|
|
106
|
+
axeViolations,
|
|
107
|
+
semanticIssues,
|
|
108
|
+
};
|
|
109
|
+
this.saveReport(stateHash, report);
|
|
110
|
+
this.addNotesToTask(task, report);
|
|
111
|
+
tag('substep').log(`Quartermaster: ${axeViolations.length} technical + ${semanticIssues.length} semantic issues`);
|
|
112
|
+
return report;
|
|
113
|
+
}
|
|
114
|
+
catch (error) {
|
|
115
|
+
debugLog('Quartermaster analysis failed:', error);
|
|
116
|
+
return null;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
async generateSemanticAnalysis(axeViolations, executions, initialState) {
|
|
120
|
+
const failedExecs = executions.filter((e) => !e.wasSuccessful);
|
|
121
|
+
if (failedExecs.length === 0 && axeViolations.length === 0)
|
|
122
|
+
return [];
|
|
123
|
+
const axeSummary = axeViolations
|
|
124
|
+
.slice(0, 10)
|
|
125
|
+
.map((v) => `[${v.impact}] ${v.id}: ${v.description}`)
|
|
126
|
+
.join('\n');
|
|
127
|
+
const failedActions = failedExecs
|
|
128
|
+
.slice(0, 10)
|
|
129
|
+
.map((e) => {
|
|
130
|
+
const locator = e.input?.locator || e.output?.locator || '';
|
|
131
|
+
const error = e.output?.message || 'Failed';
|
|
132
|
+
return `${e.toolName}("${locator}"): ${error}`;
|
|
133
|
+
})
|
|
134
|
+
.join('\n');
|
|
135
|
+
const ariaSnapshot = initialState.getCompactARIA().slice(0, 3000);
|
|
136
|
+
const schema = z.object({
|
|
137
|
+
issues: z.array(z.object({
|
|
138
|
+
type: z.enum(['unclear_intention', 'confusing_naming', 'hard_to_interact']).describe('Type of semantic issue'),
|
|
139
|
+
element: z.string().describe('Short element identifier: tag with key attributes, e.g. <button class="nav-toggle"> or <a href="/login" role="link">'),
|
|
140
|
+
issue: z.string().describe('What is confusing or problematic'),
|
|
141
|
+
suggestion: z.string().describe('Actionable improvement suggestion'),
|
|
142
|
+
})),
|
|
143
|
+
});
|
|
144
|
+
const prompt = `Analyze this page for semantic UX issues that automated tools cannot detect.
|
|
145
|
+
|
|
146
|
+
## Technical Violations (axe-core)
|
|
147
|
+
${axeSummary || 'None'}
|
|
148
|
+
|
|
149
|
+
## Failed Agent Interactions
|
|
150
|
+
${failedActions || 'None'}
|
|
151
|
+
|
|
152
|
+
## Page Structure (ARIA)
|
|
153
|
+
${ariaSnapshot}
|
|
154
|
+
|
|
155
|
+
Focus on issues that require human judgment:
|
|
156
|
+
- **unclear_intention**: Button/link text doesn't match actual behavior
|
|
157
|
+
- **confusing_naming**: Ambiguous labels, inconsistent terminology
|
|
158
|
+
- **hard_to_interact**: Controls requiring non-obvious sequences
|
|
159
|
+
|
|
160
|
+
Provide 0-5 high-signal issues. Skip obvious technical violations already covered by axe-core.
|
|
161
|
+
Focus on what would confuse a real user or caused the agent to make mistakes.`;
|
|
162
|
+
const response = await this.provider.generateObject([
|
|
163
|
+
{ role: 'system', content: 'You are a UX expert analyzing pages for semantic issues that confuse users.' },
|
|
164
|
+
{ role: 'user', content: prompt },
|
|
165
|
+
], schema, this.model);
|
|
166
|
+
return response?.object?.issues || [];
|
|
167
|
+
}
|
|
168
|
+
addNotesToTask(task, report) {
|
|
169
|
+
const criticalViolations = report.axeViolations.filter((v) => v.impact === 'critical' || v.impact === 'serious');
|
|
170
|
+
for (const v of criticalViolations.slice(0, 3)) {
|
|
171
|
+
const nodeHtml = v.nodes[0]?.html.slice(0, 100) || '';
|
|
172
|
+
task.addNote(`🔴 A11Y [${v.impact}] ${v.id}: ${v.description} — ${nodeHtml}`);
|
|
173
|
+
}
|
|
174
|
+
for (const issue of report.semanticIssues.slice(0, 3)) {
|
|
175
|
+
task.addNote(`💡 UX [${issue.type}] ${issue.element}: ${issue.suggestion}`);
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
saveReport(stateHash, report) {
|
|
179
|
+
const filePath = join(this.outputDir, `${stateHash}.md`);
|
|
180
|
+
const content = this.formatReportMarkdown(report);
|
|
181
|
+
writeFileSync(filePath, content, 'utf8');
|
|
182
|
+
debugLog(`Saved a11y report to ${filePath}`);
|
|
183
|
+
}
|
|
184
|
+
formatReportMarkdown(report) {
|
|
185
|
+
let content = `## A11Y Analysis: ${report.url}\n\n`;
|
|
186
|
+
content += `**Scenario**: ${report.scenario}\n`;
|
|
187
|
+
content += `**Date**: ${report.timestamp}\n\n`;
|
|
188
|
+
if (report.axeViolations.length > 0) {
|
|
189
|
+
content += '### Technical Violations (axe-core)\n\n';
|
|
190
|
+
for (const v of report.axeViolations) {
|
|
191
|
+
content += `- [${v.impact}] **${v.id}**: ${v.description}\n`;
|
|
192
|
+
content += ` [Learn more](${v.helpUrl})\n`;
|
|
193
|
+
for (const node of v.nodes.slice(0, 2)) {
|
|
194
|
+
content += ` - \`${node.target.join(' > ')}\` — \`${node.html.slice(0, 100)}\`\n`;
|
|
195
|
+
}
|
|
196
|
+
content += '\n';
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
if (report.semanticIssues.length > 0) {
|
|
200
|
+
content += '### Semantic Issues (LLM analysis)\n\n';
|
|
201
|
+
for (const issue of report.semanticIssues) {
|
|
202
|
+
content += `- **${issue.type}** - ${issue.element}\n`;
|
|
203
|
+
content += ` Issue: ${issue.issue}\n`;
|
|
204
|
+
content += ` → ${issue.suggestion}\n\n`;
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
return content;
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
//# sourceMappingURL=quartermaster.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"quartermaster.js","sourceRoot":"","sources":["../../../src/ai/quartermaster.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC/D,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAG5C,OAAO,EAAE,WAAW,EAAE,GAAG,EAAE,MAAM,oBAAoB,CAAC;AAGtD,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAE5C,MAAM,QAAQ,GAAG,WAAW,CAAC,yBAAyB,CAAC,CAAC;AAgCxD,MAAM,OAAO,aAAa;IAChB,QAAQ,CAAW;IACnB,KAAK,CAAO;IACZ,SAAS,CAAS;IAClB,YAAY,GAA8B,IAAI,GAAG,EAAE,CAAC;IACpD,WAAW,GAAwB,IAAI,CAAC;IACxC,gBAAgB,GAAQ,IAAI,CAAC;IAC7B,eAAe,GAAoB,EAAE,CAAC;IAE9C,YAAY,QAAkB,EAAE,OAAyB;QACvD,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,KAAK,GAAG,OAAO,EAAE,KAAK,CAAC;QAE5B,MAAM,YAAY,GAAG,YAAY,CAAC,WAAW,EAAE,CAAC;QAChD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE,EAAE,MAAM,CAAC,CAAC;IAC7D,CAAC;IAED,KAAK,CAAC,gBAAqB,EAAE,YAA0B;QACrD,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QACzC,IAAI,CAAC,eAAe,EAAE,CAAC;QAEvB,IAAI,CAAC,WAAW,GAAG,YAAY,CAAC,aAAa,CAAC,CAAC,KAAK,EAAE,EAAE;YACtD,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;QAC3B,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,mDAAmD,CAAC,CAAC;IAChE,CAAC;IAED,IAAI;QACF,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAC1B,CAAC;QACD,QAAQ,CAAC,uBAAuB,CAAC,CAAC;IACpC,CAAC;IAEO,eAAe;QACrB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YAChC,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACjD,CAAC;IACH,CAAC;IAEO,YAAY,CAAC,KAAsB;QACzC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,KAAK,EAAE,IAAI;YAAE,OAAO;QACzB,IAAI,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC;YAAE,OAAO;QAE9C,MAAM,eAAe,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YAC/D,QAAQ,CAAC,sBAAsB,EAAE,GAAG,CAAC,CAAC;QACxC,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAC7C,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,KAAmB;QAC9C,MAAM,IAAI,GAAG,IAAI,CAAC,gBAAgB,EAAE,IAAI,CAAC;QACzC,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YACzB,QAAQ,CAAC,oCAAoC,CAAC,CAAC;YAC/C,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAAC;YAC5D,MAAM,OAAO,GAAG,MAAM,IAAI,UAAU,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;YAEzD,MAAM,UAAU,GAAmB,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAChE,EAAE,EAAE,CAAC,CAAC,EAAE;gBACR,MAAM,EAAE,CAAC,CAAC,MAAgC;gBAC1C,WAAW,EAAE,CAAC,CAAC,WAAW;gBAC1B,OAAO,EAAE,CAAC,CAAC,OAAO;gBAClB,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBACzB,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,MAAM,EAAE,CAAC,CAAC,MAAkB;iBAC7B,CAAC,CAAC;aACJ,CAAC,CAAC,CAAC;YAEJ,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE;gBAChC,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,SAAS,EAAE,KAAK,CAAC,IAAI;gBACrB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,aAAa,EAAE,UAAU;aAC1B,CAAC,CAAC;YAEH,QAAQ,CAAC,6BAA6B,KAAK,CAAC,GAAG,KAAK,UAAU,CAAC,MAAM,aAAa,CAAC,CAAC;QACtF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,QAAQ,CAAC,qBAAqB,EAAE,KAAK,CAAC,CAAC;QACzC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,IAAU,EAAE,YAA0B,EAAE,YAA0B;QACrF,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QACxC,IAAI,CAAC,eAAe,GAAG,EAAE,CAAC;QAE1B,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,YAAY,CAAC,YAAY,EAAE,CAAC;YAC9C,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAEtD,MAAM,cAAc,GAAG,YAAY,CAAC,iBAAiB,EAAE,CAAC;YACxD,MAAM,kBAAkB,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAe,CAAC,CAAC,CAAC;YAEpG,IAAI,kBAAkB,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,YAAY,EAAE,aAAa,CAAC,MAAM,EAAE,CAAC;gBAC3E,QAAQ,CAAC,0CAA0C,CAAC,CAAC;gBACrD,OAAO,IAAI,CAAC;YACd,CAAC;YAED,MAAM,aAAa,GAAG,YAAY,EAAE,aAAa,IAAI,EAAE,CAAC;YACxD,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,aAAa,EAAE,kBAAkB,EAAE,YAAY,CAAC,CAAC;YAE5G,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC9D,QAAQ,CAAC,iBAAiB,CAAC,CAAC;gBAC5B,OAAO,IAAI,CAAC;YACd,CAAC;YAED,MAAM,MAAM,GAAmB;gBAC7B,QAAQ,EAAE,IAAI,CAAC,WAAW;gBAC1B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,GAAG,EAAE,YAAY,CAAC,GAAG,IAAI,EAAE;gBAC3B,aAAa;gBACb,cAAc;aACf,CAAC;YAEF,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YACnC,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAElC,GAAG,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,kBAAkB,aAAa,CAAC,MAAM,gBAAgB,cAAc,CAAC,MAAM,kBAAkB,CAAC,CAAC;YAClH,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,QAAQ,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;YAClD,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,wBAAwB,CAAC,aAA6B,EAAE,UAA2B,EAAE,YAA0B;QAC3H,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC;QAC/D,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEtE,MAAM,UAAU,GAAG,aAAa;aAC7B,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;aACZ,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC;aACrD,IAAI,CAAC,IAAI,CAAC,CAAC;QAEd,MAAM,aAAa,GAAG,WAAW;aAC9B,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;aACZ,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YACT,MAAM,OAAO,GAAG,CAAC,CAAC,KAAK,EAAE,OAAO,IAAI,CAAC,CAAC,MAAM,EAAE,OAAO,IAAI,EAAE,CAAC;YAC5D,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,EAAE,OAAO,IAAI,QAAQ,CAAC;YAC5C,OAAO,GAAG,CAAC,CAAC,QAAQ,KAAK,OAAO,OAAO,KAAK,EAAE,CAAC;QACjD,CAAC,CAAC;aACD,IAAI,CAAC,IAAI,CAAC,CAAC;QAEd,MAAM,YAAY,GAAG,YAAY,CAAC,cAAc,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QAElE,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;YACtB,MAAM,EAAE,CAAC,CAAC,KAAK,CACb,CAAC,CAAC,MAAM,CAAC;gBACP,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,mBAAmB,EAAE,kBAAkB,EAAE,kBAAkB,CAAC,CAAC,CAAC,QAAQ,CAAC,wBAAwB,CAAC;gBAC9G,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,sHAAsH,CAAC;gBACpJ,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,kCAAkC,CAAC;gBAC9D,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,mCAAmC,CAAC;aACrE,CAAC,CACH;SACF,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG;;;EAGjB,UAAU,IAAI,MAAM;;;EAGpB,aAAa,IAAI,MAAM;;;EAGvB,YAAY;;;;;;;;8EAQgE,CAAC;QAE3E,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,cAAc,CACjD;YACE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,6EAA6E,EAAE;YAC1G,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE;SAClC,EACD,MAAM,EACN,IAAI,CAAC,KAAK,CACX,CAAC;QAEF,OAAO,QAAQ,EAAE,MAAM,EAAE,MAAM,IAAI,EAAE,CAAC;IACxC,CAAC;IAEO,cAAc,CAAC,IAAU,EAAE,MAAsB;QACvD,MAAM,kBAAkB,GAAG,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,UAAU,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;QACjH,KAAK,MAAM,CAAC,IAAI,kBAAkB,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;YAC/C,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC;YACtD,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,WAAW,MAAM,QAAQ,EAAE,CAAC,CAAC;QAChF,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;YACtD,IAAI,CAAC,OAAO,CAAC,UAAU,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,OAAO,KAAK,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC;QAC9E,CAAC;IACH,CAAC;IAEO,UAAU,CAAC,SAAiB,EAAE,MAAsB;QAC1D,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,SAAS,KAAK,CAAC,CAAC;QACzD,MAAM,OAAO,GAAG,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC;QAClD,aAAa,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QACzC,QAAQ,CAAC,wBAAwB,QAAQ,EAAE,CAAC,CAAC;IAC/C,CAAC;IAEO,oBAAoB,CAAC,MAAsB;QACjD,IAAI,OAAO,GAAG,qBAAqB,MAAM,CAAC,GAAG,MAAM,CAAC;QACpD,OAAO,IAAI,iBAAiB,MAAM,CAAC,QAAQ,IAAI,CAAC;QAChD,OAAO,IAAI,aAAa,MAAM,CAAC,SAAS,MAAM,CAAC;QAE/C,IAAI,MAAM,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpC,OAAO,IAAI,yCAAyC,CAAC;YACrD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;gBACrC,OAAO,IAAI,MAAM,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,WAAW,IAAI,CAAC;gBAC7D,OAAO,IAAI,kBAAkB,CAAC,CAAC,OAAO,KAAK,CAAC;gBAC5C,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;oBACvC,OAAO,IAAI,SAAS,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC;gBACrF,CAAC;gBACD,OAAO,IAAI,IAAI,CAAC;YAClB,CAAC;QACH,CAAC;QAED,IAAI,MAAM,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrC,OAAO,IAAI,wCAAwC,CAAC;YACpD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC;gBAC1C,OAAO,IAAI,OAAO,KAAK,CAAC,IAAI,QAAQ,KAAK,CAAC,OAAO,IAAI,CAAC;gBACtD,OAAO,IAAI,YAAY,KAAK,CAAC,KAAK,IAAI,CAAC;gBACvC,OAAO,IAAI,OAAO,KAAK,CAAC,UAAU,MAAM,CAAC;YAC3C,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;CACF"}
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { outputPath } from "../../config.js";
|
|
4
|
+
import { computeHtmlFingerprint } from "../../utils/html-diff.js";
|
|
5
|
+
import { debugLog } from "./mixin.js";
|
|
6
|
+
const CACHE_TTL_MS = 6 * 60 * 60 * 1000; // 6 hours
|
|
7
|
+
const FINGERPRINT_MAX_AGE_MS = 60 * 60 * 1000; // 1 hour
|
|
8
|
+
const FINGERPRINT_WORKER_TIMEOUT_MS = 10_000;
|
|
9
|
+
const SIMILARITY_THRESHOLD = 90;
|
|
10
|
+
const memoryCache = {};
|
|
11
|
+
const memoryCacheTimestamps = {};
|
|
12
|
+
let fingerprintWorker = null;
|
|
13
|
+
function getStatesDir() {
|
|
14
|
+
return outputPath('states');
|
|
15
|
+
}
|
|
16
|
+
function getFingerprintWorker() {
|
|
17
|
+
if (!fingerprintWorker) {
|
|
18
|
+
const ext = import.meta.url.endsWith('.ts') ? '.ts' : '.js';
|
|
19
|
+
fingerprintWorker = new Worker(new URL(`./fingerprint-worker${ext}`, import.meta.url).href);
|
|
20
|
+
}
|
|
21
|
+
return fingerprintWorker;
|
|
22
|
+
}
|
|
23
|
+
export function getCachedResearch(hash) {
|
|
24
|
+
if (!hash)
|
|
25
|
+
return '';
|
|
26
|
+
const now = Date.now();
|
|
27
|
+
const timestamp = memoryCacheTimestamps[hash];
|
|
28
|
+
if (timestamp && now - timestamp <= CACHE_TTL_MS) {
|
|
29
|
+
return memoryCache[hash] || '';
|
|
30
|
+
}
|
|
31
|
+
const researchFile = outputPath('research', `${hash}.md`);
|
|
32
|
+
if (!existsSync(researchFile))
|
|
33
|
+
return '';
|
|
34
|
+
const stats = statSync(researchFile);
|
|
35
|
+
if (now - stats.mtimeMs > CACHE_TTL_MS)
|
|
36
|
+
return '';
|
|
37
|
+
const cached = readFileSync(researchFile, 'utf8');
|
|
38
|
+
memoryCache[hash] = cached;
|
|
39
|
+
memoryCacheTimestamps[hash] = now;
|
|
40
|
+
return cached;
|
|
41
|
+
}
|
|
42
|
+
export function saveResearch(hash, text, combinedHtml) {
|
|
43
|
+
const researchDir = outputPath('research');
|
|
44
|
+
const researchFile = join(researchDir, `${hash}.md`);
|
|
45
|
+
if (!existsSync(researchDir))
|
|
46
|
+
mkdirSync(researchDir, { recursive: true });
|
|
47
|
+
writeFileSync(researchFile, text);
|
|
48
|
+
memoryCache[hash] = text;
|
|
49
|
+
memoryCacheTimestamps[hash] = Date.now();
|
|
50
|
+
debugLog(`Research saved to ${researchFile}`);
|
|
51
|
+
if (combinedHtml) {
|
|
52
|
+
const statesDir = getStatesDir();
|
|
53
|
+
if (!existsSync(statesDir))
|
|
54
|
+
mkdirSync(statesDir, { recursive: true });
|
|
55
|
+
const fingerprint = computeHtmlFingerprint(combinedHtml);
|
|
56
|
+
const fingerprintFile = join(statesDir, `${hash}.fingerprint`);
|
|
57
|
+
writeFileSync(fingerprintFile, fingerprint.join('\n'));
|
|
58
|
+
debugLog(`Fingerprint saved to ${fingerprintFile}`);
|
|
59
|
+
}
|
|
60
|
+
return researchFile;
|
|
61
|
+
}
|
|
62
|
+
export function findSimilarResearch(combinedHtml) {
|
|
63
|
+
const statesDir = getStatesDir();
|
|
64
|
+
if (!existsSync(statesDir))
|
|
65
|
+
return Promise.resolve(null);
|
|
66
|
+
const worker = getFingerprintWorker();
|
|
67
|
+
return new Promise((resolve) => {
|
|
68
|
+
const timeout = setTimeout(() => {
|
|
69
|
+
debugLog('Fingerprint worker timed out');
|
|
70
|
+
resolve(null);
|
|
71
|
+
}, FINGERPRINT_WORKER_TIMEOUT_MS);
|
|
72
|
+
worker.onmessage = (event) => {
|
|
73
|
+
clearTimeout(timeout);
|
|
74
|
+
const { matchHash, similarity } = event.data;
|
|
75
|
+
if (!matchHash) {
|
|
76
|
+
resolve(null);
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
79
|
+
debugLog(`Similar research found: ${matchHash} (${similarity}% similar)`);
|
|
80
|
+
const research = getCachedResearch(matchHash);
|
|
81
|
+
if (research) {
|
|
82
|
+
resolve(research);
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
85
|
+
resolve(null);
|
|
86
|
+
};
|
|
87
|
+
worker.postMessage({
|
|
88
|
+
html: combinedHtml,
|
|
89
|
+
statesDir,
|
|
90
|
+
maxAgeMs: FINGERPRINT_MAX_AGE_MS,
|
|
91
|
+
threshold: SIMILARITY_THRESHOLD,
|
|
92
|
+
});
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
//# sourceMappingURL=cache.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cache.js","sourceRoot":"","sources":["../../../../src/ai/researcher/cache.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACvF,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEtC,MAAM,YAAY,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,UAAU;AACnD,MAAM,sBAAsB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,SAAS;AACxD,MAAM,6BAA6B,GAAG,MAAM,CAAC;AAC7C,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAEhC,MAAM,WAAW,GAA2B,EAAE,CAAC;AAC/C,MAAM,qBAAqB,GAA2B,EAAE,CAAC;AAEzD,IAAI,iBAAiB,GAAkB,IAAI,CAAC;AAE5C,SAAS,YAAY;IACnB,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC;AAC9B,CAAC;AAED,SAAS,oBAAoB;IAC3B,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;QAC5D,iBAAiB,GAAG,IAAI,MAAM,CAAC,IAAI,GAAG,CAAC,uBAAuB,GAAG,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;IAC9F,CAAC;IACD,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,IAAY;IAC5C,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,CAAC;IACrB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,SAAS,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAC;IAC9C,IAAI,SAAS,IAAI,GAAG,GAAG,SAAS,IAAI,YAAY,EAAE,CAAC;QACjD,OAAO,WAAW,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;IACjC,CAAC;IACD,MAAM,YAAY,GAAG,UAAU,CAAC,UAAU,EAAE,GAAG,IAAI,KAAK,CAAC,CAAC;IAC1D,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC;QAAE,OAAO,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,QAAQ,CAAC,YAAY,CAAC,CAAC;IACrC,IAAI,GAAG,GAAG,KAAK,CAAC,OAAO,GAAG,YAAY;QAAE,OAAO,EAAE,CAAC;IAClD,MAAM,MAAM,GAAG,YAAY,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IAClD,WAAW,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC;IAC3B,qBAAqB,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC;IAClC,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAY,EAAE,IAAY,EAAE,YAAqB;IAC5E,MAAM,WAAW,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;IAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,WAAW,EAAE,GAAG,IAAI,KAAK,CAAC,CAAC;IACrD,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC;QAAE,SAAS,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1E,aAAa,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;IAClC,WAAW,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IACzB,qBAAqB,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzC,QAAQ,CAAC,qBAAqB,YAAY,EAAE,CAAC,CAAC;IAE9C,IAAI,YAAY,EAAE,CAAC;QACjB,MAAM,SAAS,GAAG,YAAY,EAAE,CAAC;QACjC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;YAAE,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACtE,MAAM,WAAW,GAAG,sBAAsB,CAAC,YAAY,CAAC,CAAC;QACzD,MAAM,eAAe,GAAG,IAAI,CAAC,SAAS,EAAE,GAAG,IAAI,cAAc,CAAC,CAAC;QAC/D,aAAa,CAAC,eAAe,EAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QACvD,QAAQ,CAAC,wBAAwB,eAAe,EAAE,CAAC,CAAC;IACtD,CAAC;IAED,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,YAAoB;IACtD,MAAM,SAAS,GAAG,YAAY,EAAE,CAAC;IACjC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAEzD,MAAM,MAAM,GAAG,oBAAoB,EAAE,CAAC;IAEtC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE;YAC9B,QAAQ,CAAC,8BAA8B,CAAC,CAAC;YACzC,OAAO,CAAC,IAAI,CAAC,CAAC;QAChB,CAAC,EAAE,6BAA6B,CAAC,CAAC;QAElC,MAAM,CAAC,SAAS,GAAG,CAAC,KAAmB,EAAE,EAAE;YACzC,YAAY,CAAC,OAAO,CAAC,CAAC;YACtB,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,KAAK,CAAC,IAAwD,CAAC;YACjG,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,CAAC;gBACd,OAAO;YACT,CAAC;YAED,QAAQ,CAAC,2BAA2B,SAAS,KAAK,UAAU,YAAY,CAAC,CAAC;YAC1E,MAAM,QAAQ,GAAG,iBAAiB,CAAC,SAAS,CAAC,CAAC;YAC9C,IAAI,QAAQ,EAAE,CAAC;gBACb,OAAO,CAAC,QAAQ,CAAC,CAAC;gBAClB,OAAO;YACT,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,CAAC;QAChB,CAAC,CAAC;QAEF,MAAM,CAAC,WAAW,CAAC;YACjB,IAAI,EAAE,YAAY;YAClB,SAAS;YACT,QAAQ,EAAE,sBAAsB;YAChC,SAAS,EAAE,oBAAoB;SAChC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
import dedent from 'dedent';
|
|
2
|
+
import { tag } from '../../utils/logger.js';
|
|
3
|
+
import { mdq } from "../../utils/markdown-query.js";
|
|
4
|
+
import { WebElement } from "../../utils/web-element.js";
|
|
5
|
+
import { debugLog } from "./mixin.js";
|
|
6
|
+
import { parseResearchSections } from "./parser.js";
|
|
7
|
+
export async function visuallyAnnotateContainers(page, containers) {
|
|
8
|
+
return page.evaluate((ctrs) => {
|
|
9
|
+
const containerColors = ['#9b59b6', '#16a085', '#c0392b', '#2980b9'];
|
|
10
|
+
const drawnContainers = [];
|
|
11
|
+
for (let i = 0; i < ctrs.length; i++) {
|
|
12
|
+
let el;
|
|
13
|
+
try {
|
|
14
|
+
el = document.querySelector(ctrs[i].css);
|
|
15
|
+
}
|
|
16
|
+
catch {
|
|
17
|
+
continue;
|
|
18
|
+
}
|
|
19
|
+
if (!el)
|
|
20
|
+
continue;
|
|
21
|
+
const rect = el.getBoundingClientRect();
|
|
22
|
+
if (rect.width === 0 && rect.height === 0)
|
|
23
|
+
continue;
|
|
24
|
+
const color = containerColors[i % containerColors.length];
|
|
25
|
+
const box = document.createElement('div');
|
|
26
|
+
box.setAttribute('data-explorbot-annotation', 'true');
|
|
27
|
+
box.style.cssText = `position:absolute;left:${rect.left + window.scrollX}px;top:${rect.top + window.scrollY}px;width:${rect.width}px;height:${rect.height}px;border:1px dashed ${color};z-index:99998;pointer-events:none;`;
|
|
28
|
+
document.body.appendChild(box);
|
|
29
|
+
drawnContainers.push({ label: ctrs[i].label, color });
|
|
30
|
+
}
|
|
31
|
+
if (drawnContainers.length > 0) {
|
|
32
|
+
const legend = document.createElement('div');
|
|
33
|
+
legend.setAttribute('data-explorbot-annotation', 'true');
|
|
34
|
+
legend.style.cssText = 'position:fixed;right:10px;bottom:10px;background:white;color:black;font-size:14px;font-family:sans-serif;padding:10px 14px;z-index:100001;pointer-events:none;border:3px solid #e63946;border-radius:6px;line-height:22px;';
|
|
35
|
+
const title = '<div style="font-weight:bold;margin-bottom:4px;color:#e63946;">Legend</div>';
|
|
36
|
+
const items = drawnContainers.map((c) => `<div><span style="display:inline-block;width:24px;border-top:3px dashed ${c.color};margin-right:8px;vertical-align:middle;"></span>${c.label}</div>`).join('');
|
|
37
|
+
legend.innerHTML = title + items;
|
|
38
|
+
document.body.appendChild(legend);
|
|
39
|
+
}
|
|
40
|
+
const colors = ['#e63946', '#2a9d8f', '#e9c46a', '#264653', '#f4a261', '#7b2cbf', '#0077b6', '#d62828'];
|
|
41
|
+
const elements = document.querySelectorAll('[data-explorbot-eidx]');
|
|
42
|
+
let count = 0;
|
|
43
|
+
for (const el of elements) {
|
|
44
|
+
const eidx = el.getAttribute('data-explorbot-eidx');
|
|
45
|
+
if (!eidx)
|
|
46
|
+
continue;
|
|
47
|
+
const rect = el.getBoundingClientRect();
|
|
48
|
+
if (rect.width === 0 && rect.height === 0)
|
|
49
|
+
continue;
|
|
50
|
+
const color = colors[count % colors.length];
|
|
51
|
+
const box = document.createElement('div');
|
|
52
|
+
box.setAttribute('data-explorbot-annotation', 'true');
|
|
53
|
+
box.style.cssText = `position:absolute;left:${rect.left + window.scrollX}px;top:${rect.top + window.scrollY}px;width:${rect.width}px;height:${rect.height}px;border:2px solid ${color};z-index:99999;pointer-events:none;`;
|
|
54
|
+
const label = document.createElement('div');
|
|
55
|
+
label.textContent = eidx;
|
|
56
|
+
label.style.cssText = `position:absolute;top:-14px;right:-2px;background:${color};color:white;font-size:10px;padding:0 3px;line-height:14px;font-family:monospace;z-index:100000;pointer-events:none;`;
|
|
57
|
+
box.appendChild(label);
|
|
58
|
+
document.body.appendChild(box);
|
|
59
|
+
count++;
|
|
60
|
+
}
|
|
61
|
+
return count;
|
|
62
|
+
}, containers);
|
|
63
|
+
}
|
|
64
|
+
export function WithCoordinates(Base) {
|
|
65
|
+
return class extends Base {
|
|
66
|
+
analyzeScreenshotForVisualProps() {
|
|
67
|
+
return this._analyzeScreenshotForVisualProps();
|
|
68
|
+
}
|
|
69
|
+
async visuallyAnnotateElements(opts) {
|
|
70
|
+
return visuallyAnnotateContainers(this.explorer.playwrightHelper.page, opts?.containers || []);
|
|
71
|
+
}
|
|
72
|
+
async _analyzeScreenshotForVisualProps() {
|
|
73
|
+
const elements = new Map();
|
|
74
|
+
const emptyResult = { elements, pagePurpose: null, primaryActions: null, focusedSection: null };
|
|
75
|
+
if (!this.actionResult)
|
|
76
|
+
return emptyResult;
|
|
77
|
+
const image = this.actionResult.screenshot;
|
|
78
|
+
if (!image)
|
|
79
|
+
return emptyResult;
|
|
80
|
+
tag('step').log('Analyzing annotated screenshot for visual properties');
|
|
81
|
+
const prompt = dedent `
|
|
82
|
+
This screenshot has two types of annotations:
|
|
83
|
+
- **Section containers**: dashed bordered boxes (no labels on them). A legend at the bottom-left maps dashed line colors to section names. Ignore containers for this task.
|
|
84
|
+
- **Interactive elements**: solid bordered boxes with eidx numbers in the top-right corner above the box. Adjacent elements use different colors.
|
|
85
|
+
|
|
86
|
+
For each interactive element (solid border, eidx number), report:
|
|
87
|
+
| eidx | Coordinates | Color | Icon |
|
|
88
|
+
|
|
89
|
+
Column definitions:
|
|
90
|
+
- eidx: the number shown in the colored label above the top-right corner of each solid-bordered box
|
|
91
|
+
- Coordinates: (X, Y) center point of the element
|
|
92
|
+
- Color: accent color if distinctive (red, green, blue, orange, yellow, purple, gray), otherwise -
|
|
93
|
+
- Icon: one-word icon description (plus, x, trash, pencil, gear, search, hamburger, ellipsis, chevron, star, check, filter), otherwise -
|
|
94
|
+
|
|
95
|
+
Ignore the legend block and dashed container borders.
|
|
96
|
+
|
|
97
|
+
Then add:
|
|
98
|
+
|
|
99
|
+
## Page Purpose
|
|
100
|
+
One sentence: what this page is for from the user's perspective.
|
|
101
|
+
|
|
102
|
+
## Primary Actions
|
|
103
|
+
List 3-5 most prominent interactive elements (accent buttons, CTAs, main form actions).
|
|
104
|
+
Format: - action description
|
|
105
|
+
|
|
106
|
+
## Focused Section
|
|
107
|
+
Which section from the legend appears to be the user's primary focus area?
|
|
108
|
+
Look for: modal overlays, drawers, panels with shadows or elevated z-index, highlighted/active areas.
|
|
109
|
+
If a dialog or overlay is visible, that is the focused section.
|
|
110
|
+
Otherwise, pick the section with the most prominent interactive content.
|
|
111
|
+
Reply with the exact section name from the legend. If no legend is shown, reply with "-".
|
|
112
|
+
`;
|
|
113
|
+
try {
|
|
114
|
+
const aiResult = await this.provider.processImage(prompt, image.toString('base64'));
|
|
115
|
+
const text = aiResult.text || '';
|
|
116
|
+
const rows = mdq(text).query('table').toJson();
|
|
117
|
+
for (const row of rows) {
|
|
118
|
+
const eidx = Number.parseInt(row.eidx, 10);
|
|
119
|
+
if (Number.isNaN(eidx))
|
|
120
|
+
continue;
|
|
121
|
+
const val = (v) => (v && v !== '-' ? v : null);
|
|
122
|
+
elements.set(eidx, {
|
|
123
|
+
coordinates: val(row.Coordinates),
|
|
124
|
+
color: val(row.Color),
|
|
125
|
+
icon: val(row.Icon),
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
const pagePurposeSection = mdq(text).query('section2("Page Purpose")').text().trim();
|
|
129
|
+
const primaryActionsSection = mdq(text).query('section2("Primary Actions")').text().trim();
|
|
130
|
+
const focusedSectionRaw = mdq(text).query('section2("Focused Section")').text().trim();
|
|
131
|
+
const focusedSection = focusedSectionRaw && focusedSectionRaw !== '-' ? focusedSectionRaw.split('\n')[0].trim() : null;
|
|
132
|
+
debugLog(`Parsed visual props for ${elements.size} elements`);
|
|
133
|
+
return {
|
|
134
|
+
elements,
|
|
135
|
+
pagePurpose: pagePurposeSection || null,
|
|
136
|
+
primaryActions: primaryActionsSection
|
|
137
|
+
? primaryActionsSection
|
|
138
|
+
.split('\n')
|
|
139
|
+
.filter((l) => l.trim().startsWith('-'))
|
|
140
|
+
.map((l) => l.trim())
|
|
141
|
+
: null,
|
|
142
|
+
focusedSection,
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
catch (err) {
|
|
146
|
+
debugLog(`Screenshot visual analysis failed: ${err instanceof Error ? err.message : err}`);
|
|
147
|
+
}
|
|
148
|
+
debugLog(`Parsed visual props for ${elements.size} elements`);
|
|
149
|
+
return emptyResult;
|
|
150
|
+
}
|
|
151
|
+
async mergeVisualData(result, visualData) {
|
|
152
|
+
const sections = parseResearchSections(result.text);
|
|
153
|
+
let merged = 0;
|
|
154
|
+
for (const section of sections) {
|
|
155
|
+
let sectionMerged = false;
|
|
156
|
+
for (const el of section.elements) {
|
|
157
|
+
let eidx = el.eidx || null;
|
|
158
|
+
if (!eidx) {
|
|
159
|
+
const locator = el.css || el.xpath || (el.aria ? `role=${el.aria.role}[name="${el.aria.text}"]` : null);
|
|
160
|
+
if (locator)
|
|
161
|
+
eidx = await this.explorer.getEidxByLocator(locator, section.containerCss);
|
|
162
|
+
}
|
|
163
|
+
if (!eidx)
|
|
164
|
+
continue;
|
|
165
|
+
const vis = visualData.get(eidx);
|
|
166
|
+
if (!vis)
|
|
167
|
+
continue;
|
|
168
|
+
Object.assign(el, Object.fromEntries(Object.entries(vis).filter(([, v]) => v)));
|
|
169
|
+
sectionMerged = true;
|
|
170
|
+
merged++;
|
|
171
|
+
}
|
|
172
|
+
if (sectionMerged)
|
|
173
|
+
result.rebuildSectionInText(section);
|
|
174
|
+
}
|
|
175
|
+
debugLog(`Merged visual props for ${merged} elements`);
|
|
176
|
+
}
|
|
177
|
+
async backfillCoordinates(result) {
|
|
178
|
+
const page = this.explorer.playwrightHelper.page;
|
|
179
|
+
const sections = parseResearchSections(result.text);
|
|
180
|
+
const eidxWithoutCoords = [];
|
|
181
|
+
for (const section of sections) {
|
|
182
|
+
for (const el of section.elements) {
|
|
183
|
+
if (el.eidx && !el.coordinates)
|
|
184
|
+
eidxWithoutCoords.push(el.eidx);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
if (eidxWithoutCoords.length === 0)
|
|
188
|
+
return;
|
|
189
|
+
const webElements = await WebElement.fromEidxList(page, eidxWithoutCoords);
|
|
190
|
+
if (webElements.length === 0)
|
|
191
|
+
return;
|
|
192
|
+
const rectMap = new Map(webElements.map((w) => [w.eidx, w]));
|
|
193
|
+
for (const section of sections) {
|
|
194
|
+
let changed = false;
|
|
195
|
+
for (const el of section.elements) {
|
|
196
|
+
if (el.eidx && !el.coordinates) {
|
|
197
|
+
const w = rectMap.get(el.eidx);
|
|
198
|
+
if (w) {
|
|
199
|
+
el.coordinates = w.coordinates;
|
|
200
|
+
changed = true;
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
if (changed)
|
|
205
|
+
result.rebuildSectionInText(section);
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
};
|
|
209
|
+
}
|
|
210
|
+
//# sourceMappingURL=coordinates.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"coordinates.js","sourceRoot":"","sources":["../../../../src/ai/researcher/coordinates.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAI5B,OAAO,EAAE,GAAG,EAAE,MAAM,uBAAuB,CAAC;AAC5C,OAAO,EAAE,GAAG,EAAE,MAAM,+BAA+B,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAExD,OAAO,EAAoB,QAAQ,EAAE,MAAM,YAAY,CAAC;AACxD,OAAO,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAGpD,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,IAAU,EAAE,UAAiD;IAC5G,OAAO,IAAI,CAAC,QAAQ,CAAC,CAAC,IAA2C,EAAE,EAAE;QACnE,MAAM,eAAe,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;QACrE,MAAM,eAAe,GAA4C,EAAE,CAAC;QACpE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,IAAI,EAAkB,CAAC;YACvB,IAAI,CAAC;gBACH,EAAE,GAAG,QAAQ,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YAC3C,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS;YACX,CAAC;YACD,IAAI,CAAC,EAAE;gBAAE,SAAS;YAClB,MAAM,IAAI,GAAG,EAAE,CAAC,qBAAqB,EAAE,CAAC;YACxC,IAAI,IAAI,CAAC,KAAK,KAAK,CAAC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAEpD,MAAM,KAAK,GAAG,eAAe,CAAC,CAAC,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;YAC1D,MAAM,GAAG,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;YAC1C,GAAG,CAAC,YAAY,CAAC,2BAA2B,EAAE,MAAM,CAAC,CAAC;YACtD,GAAG,CAAC,KAAK,CAAC,OAAO,GAAG,0BAA0B,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,OAAO,UAAU,IAAI,CAAC,GAAG,GAAG,MAAM,CAAC,OAAO,YAAY,IAAI,CAAC,KAAK,aAAa,IAAI,CAAC,MAAM,wBAAwB,KAAK,qCAAqC,CAAC;YAC5N,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;YAC/B,eAAe,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;QACxD,CAAC;QAED,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,MAAM,MAAM,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;YAC7C,MAAM,CAAC,YAAY,CAAC,2BAA2B,EAAE,MAAM,CAAC,CAAC;YACzD,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,4NAA4N,CAAC;YACpP,MAAM,KAAK,GAAG,6EAA6E,CAAC;YAC5F,MAAM,KAAK,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,2EAA2E,CAAC,CAAC,KAAK,oDAAoD,CAAC,CAAC,KAAK,QAAQ,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACzM,MAAM,CAAC,SAAS,GAAG,KAAK,GAAG,KAAK,CAAC;YACjC,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QACpC,CAAC;QAED,MAAM,MAAM,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;QACxG,MAAM,QAAQ,GAAG,QAAQ,CAAC,gBAAgB,CAAC,uBAAuB,CAAC,CAAC;QACpE,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;YAC1B,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,qBAAqB,CAAC,CAAC;YACpD,IAAI,CAAC,IAAI;gBAAE,SAAS;YACpB,MAAM,IAAI,GAAG,EAAE,CAAC,qBAAqB,EAAE,CAAC;YACxC,IAAI,IAAI,CAAC,KAAK,KAAK,CAAC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAEpD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;YAE5C,MAAM,GAAG,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;YAC1C,GAAG,CAAC,YAAY,CAAC,2BAA2B,EAAE,MAAM,CAAC,CAAC;YACtD,GAAG,CAAC,KAAK,CAAC,OAAO,GAAG,0BAA0B,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,OAAO,UAAU,IAAI,CAAC,GAAG,GAAG,MAAM,CAAC,OAAO,YAAY,IAAI,CAAC,KAAK,aAAa,IAAI,CAAC,MAAM,uBAAuB,KAAK,qCAAqC,CAAC;YAE3N,MAAM,KAAK,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;YAC5C,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC;YACzB,KAAK,CAAC,KAAK,CAAC,OAAO,GAAG,qDAAqD,KAAK,sHAAsH,CAAC;YACvM,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;YAEvB,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;YAC/B,KAAK,EAAE,CAAC;QACV,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC,EAAE,UAAU,CAAC,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,eAAe,CAAwB,IAAO;IAC5D,OAAO,KAAM,SAAQ,IAAI;QAKvB,+BAA+B;YAC7B,OAAO,IAAI,CAAC,gCAAgC,EAAE,CAAC;QACjD,CAAC;QAED,KAAK,CAAC,wBAAwB,CAAC,IAA6D;YAC1F,OAAO,0BAA0B,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,IAAI,EAAE,IAAI,EAAE,UAAU,IAAI,EAAE,CAAC,CAAC;QACjG,CAAC;QAEO,KAAK,CAAC,gCAAgC;YAC5C,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAqF,CAAC;YAC9G,MAAM,WAAW,GAAyB,EAAE,QAAQ,EAAE,WAAW,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC;YACtH,IAAI,CAAC,IAAI,CAAC,YAAY;gBAAE,OAAO,WAAW,CAAC;YAE3C,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC;YAC3C,IAAI,CAAC,KAAK;gBAAE,OAAO,WAAW,CAAC;YAC/B,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,sDAAsD,CAAC,CAAC;YAExE,MAAM,MAAM,GAAG,MAAM,CAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+BpB,CAAC;YAEF,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,MAAM,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;gBACpF,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAC;gBACjC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC;gBAC/C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;oBACvB,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;oBAC3C,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;wBAAE,SAAS;oBACjC,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;oBACvD,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE;wBACjB,WAAW,EAAE,GAAG,CAAC,GAAG,CAAC,WAAW,CAAC;wBACjC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC;wBACrB,IAAI,EAAE,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC;qBACpB,CAAC,CAAC;gBACL,CAAC;gBAED,MAAM,kBAAkB,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;gBACrF,MAAM,qBAAqB,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;gBAC3F,MAAM,iBAAiB,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;gBACvF,MAAM,cAAc,GAAG,iBAAiB,IAAI,iBAAiB,KAAK,GAAG,CAAC,CAAC,CAAC,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;gBAEvH,QAAQ,CAAC,2BAA2B,QAAQ,CAAC,IAAI,WAAW,CAAC,CAAC;gBAC9D,OAAO;oBACL,QAAQ;oBACR,WAAW,EAAE,kBAAkB,IAAI,IAAI;oBACvC,cAAc,EAAE,qBAAqB;wBACnC,CAAC,CAAC,qBAAqB;6BAClB,KAAK,CAAC,IAAI,CAAC;6BACX,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;6BACvC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;wBACzB,CAAC,CAAC,IAAI;oBACR,cAAc;iBACf,CAAC;YACJ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,QAAQ,CAAC,sCAAsC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;YAC7F,CAAC;YAED,QAAQ,CAAC,2BAA2B,QAAQ,CAAC,IAAI,WAAW,CAAC,CAAC;YAC9D,OAAO,WAAW,CAAC;QACrB,CAAC;QAED,KAAK,CAAC,eAAe,CAAC,MAAsB,EAAE,UAAkG;YAC9I,MAAM,QAAQ,GAAG,qBAAqB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACpD,IAAI,MAAM,GAAG,CAAC,CAAC;YAEf,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC/B,IAAI,aAAa,GAAG,KAAK,CAAC;gBAC1B,KAAK,MAAM,EAAE,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;oBAClC,IAAI,IAAI,GAAG,EAAE,CAAC,IAAI,IAAI,IAAI,CAAC;oBAC3B,IAAI,CAAC,IAAI,EAAE,CAAC;wBACV,MAAM,OAAO,GAAG,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,KAAK,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAC,IAAI,UAAU,EAAE,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;wBACxG,IAAI,OAAO;4BAAE,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;oBAC1F,CAAC;oBACD,IAAI,CAAC,IAAI;wBAAE,SAAS;oBAEpB,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;oBACjC,IAAI,CAAC,GAAG;wBAAE,SAAS;oBACnB,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;oBAChF,aAAa,GAAG,IAAI,CAAC;oBACrB,MAAM,EAAE,CAAC;gBACX,CAAC;gBACD,IAAI,aAAa;oBAAE,MAAM,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC;YAC1D,CAAC;YACD,QAAQ,CAAC,2BAA2B,MAAM,WAAW,CAAC,CAAC;QACzD,CAAC;QAED,KAAK,CAAC,mBAAmB,CAAC,MAAsB;YAC9C,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,IAAI,CAAC;YACjD,MAAM,QAAQ,GAAG,qBAAqB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACpD,MAAM,iBAAiB,GAAa,EAAE,CAAC;YACvC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC/B,KAAK,MAAM,EAAE,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;oBAClC,IAAI,EAAE,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC,WAAW;wBAAE,iBAAiB,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;gBAClE,CAAC;YACH,CAAC;YACD,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO;YAE3C,MAAM,WAAW,GAAG,MAAM,UAAU,CAAC,YAAY,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;YAC3E,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO;YAErC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9D,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC/B,IAAI,OAAO,GAAG,KAAK,CAAC;gBACpB,KAAK,MAAM,EAAE,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;oBAClC,IAAI,EAAE,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC;wBAC/B,MAAM,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;wBAC/B,IAAI,CAAC,EAAE,CAAC;4BACN,EAAE,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW,CAAC;4BAC/B,OAAO,GAAG,IAAI,CAAC;wBACjB,CAAC;oBACH,CAAC;gBACH,CAAC;gBACD,IAAI,OAAO;oBAAE,MAAM,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC;YACpD,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC"}
|