explorbot 0.4.10 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -1
- package/bin/mdq.ts +18 -0
- package/boat/api-tester/src/ai/chief.ts +72 -0
- package/boat/api-tester/src/api-client.ts +37 -0
- package/boat/prima/src/prima.ts +41 -2
- package/dist/bin/mdq.js +19 -0
- package/dist/boat/api-tester/src/ai/chief.js +69 -0
- package/dist/boat/api-tester/src/api-client.js +26 -0
- package/dist/boat/prima/src/prima.js +42 -2
- package/dist/package.json +3 -2
- package/dist/src/ai/agent.d.ts +3 -1
- package/dist/src/ai/judge-provider.d.ts +17 -0
- package/dist/src/ai/judge-provider.js +56 -0
- package/dist/src/ai/judge-tool.d.ts +2 -0
- package/dist/src/ai/judge-tool.js +33 -0
- package/dist/src/ai/judge.d.ts +28 -0
- package/dist/src/ai/judge.js +71 -0
- package/dist/src/ai/navigator.d.ts +3 -0
- package/dist/src/ai/navigator.js +13 -4
- package/dist/src/ai/pilot.d.ts +4 -0
- package/dist/src/ai/pilot.js +57 -5
- package/dist/src/ai/planner.js +9 -6
- package/dist/src/ai/provider.d.ts +4 -1
- package/dist/src/ai/provider.js +52 -7
- package/dist/src/ai/researcher/deep-analysis.js +2 -2
- package/dist/src/ai/researcher/locators.js +2 -2
- package/dist/src/ai/researcher/pagination.js +1 -1
- package/dist/src/ai/researcher/research-result.js +2 -2
- package/dist/src/ai/researcher.js +1 -1
- package/dist/src/ai/task-agent.d.ts +2 -0
- package/dist/src/ai/task-agent.js +3 -1
- package/dist/src/ai/tester.js +19 -15
- package/dist/src/ai/tools.d.ts +4 -3
- package/dist/src/ai/tools.js +35 -7
- package/dist/src/api/request-result.js +2 -1
- package/dist/src/command-handler.d.ts +1 -0
- package/dist/src/command-handler.js +24 -3
- package/dist/src/commands/base-command.d.ts +5 -0
- package/dist/src/commands/base-command.js +3 -0
- package/dist/src/commands/explore-command.d.ts +2 -1
- package/dist/src/commands/explore-command.js +12 -1
- package/dist/src/commands/freesail-command.js +8 -2
- package/dist/src/commands/init-command.js +1 -1
- package/dist/src/commands/navigate-command.d.ts +2 -1
- package/dist/src/commands/navigate-command.js +6 -0
- package/dist/src/commands/plan-load-command.d.ts +2 -1
- package/dist/src/commands/plan-load-command.js +4 -0
- package/dist/src/commands/plans-command.d.ts +3 -9
- package/dist/src/commands/plans-command.js +11 -21
- package/dist/src/commands/rerun-command.d.ts +2 -1
- package/dist/src/commands/rerun-command.js +5 -1
- package/dist/src/commands/research-command.d.ts +2 -1
- package/dist/src/commands/research-command.js +6 -0
- package/dist/src/commands/test-command.d.ts +2 -1
- package/dist/src/commands/test-command.js +4 -1
- package/dist/src/components/Autocomplete.js +26 -12
- package/dist/src/components/InputReadline.js +10 -1
- package/dist/src/config.d.ts +6 -0
- package/dist/src/experience-tracker.js +4 -3
- package/dist/src/explorbot.d.ts +3 -0
- package/dist/src/explorbot.js +8 -0
- package/dist/src/knowledge-tracker.js +1 -1
- package/dist/src/state-manager.d.ts +2 -0
- package/dist/src/state-manager.js +16 -0
- package/dist/src/test-plan.d.ts +11 -0
- package/dist/src/test-plan.js +54 -2
- package/dist/src/utils/aria-ref.js +1 -1
- package/dist/src/utils/logger.js +8 -2
- package/dist/src/utils/markdown-query.d.ts +1 -48
- package/dist/src/utils/markdown-query.js +1 -444
- package/dist/src/utils/mdq/cli.d.ts +6 -0
- package/dist/src/utils/mdq/cli.js +122 -0
- package/dist/src/utils/mdq/edit.d.ts +24 -0
- package/dist/src/utils/mdq/edit.js +147 -0
- package/dist/src/utils/mdq/query.d.ts +118 -0
- package/dist/src/utils/mdq/query.js +451 -0
- package/dist/src/utils/strings.d.ts +1 -0
- package/dist/src/utils/strings.js +7 -0
- package/dist/src/utils/test-files.d.ts +1 -0
- package/dist/src/utils/test-files.js +5 -2
- package/docs/api-testing/planning.md +1 -1
- package/docs/superpowers/plans/2026-09-15-mdq-package.md +130 -94
- package/docs/superpowers/specs/2026-09-18-judge-decision-model-design.md +79 -0
- package/package.json +3 -2
- package/src/ai/agent.ts +3 -1
- package/src/ai/judge-provider.ts +62 -0
- package/src/ai/judge-tool.ts +35 -0
- package/src/ai/judge.ts +75 -0
- package/src/ai/navigator.ts +15 -4
- package/src/ai/pilot.ts +58 -5
- package/src/ai/planner.ts +9 -6
- package/src/ai/provider.ts +51 -7
- package/src/ai/researcher/deep-analysis.ts +2 -2
- package/src/ai/researcher/locators.ts +2 -2
- package/src/ai/researcher/pagination.ts +1 -1
- package/src/ai/researcher/research-result.ts +2 -2
- package/src/ai/researcher.ts +1 -1
- package/src/ai/task-agent.ts +4 -1
- package/src/ai/tester.ts +19 -16
- package/src/ai/tools.ts +42 -7
- package/src/api/request-result.ts +2 -1
- package/src/command-handler.ts +28 -3
- package/src/commands/base-command.ts +9 -0
- package/src/commands/explore-command.ts +15 -2
- package/src/commands/freesail-command.ts +9 -2
- package/src/commands/init-command.ts +1 -1
- package/src/commands/navigate-command.ts +8 -1
- package/src/commands/plan-load-command.ts +6 -1
- package/src/commands/plans-command.ts +13 -29
- package/src/commands/rerun-command.ts +7 -2
- package/src/commands/research-command.ts +8 -1
- package/src/commands/test-command.ts +6 -2
- package/src/components/Autocomplete.tsx +39 -10
- package/src/components/InputReadline.tsx +10 -1
- package/src/config.ts +1 -0
- package/src/experience-tracker.ts +4 -3
- package/src/explorbot.ts +8 -0
- package/src/knowledge-tracker.ts +1 -1
- package/src/state-manager.ts +16 -0
- package/src/test-plan.ts +67 -2
- package/src/utils/aria-ref.ts +1 -1
- package/src/utils/logger.ts +6 -1
- package/src/utils/markdown-query.ts +1 -519
- package/src/utils/mdq/cli.ts +118 -0
- package/src/utils/mdq/edit.ts +158 -0
- package/src/utils/mdq/query.ts +556 -0
- package/src/utils/strings.ts +7 -0
- package/src/utils/test-files.ts +5 -2
|
@@ -7,6 +7,7 @@ import type { StateManager } from '../state-manager.js';
|
|
|
7
7
|
import { HooksRunner } from '../utils/hooks-runner.js';
|
|
8
8
|
import type { AgentDeps, ToolDeps } from './agent.js';
|
|
9
9
|
import { Historian } from './historian.js';
|
|
10
|
+
import type { Judge } from './judge.js';
|
|
10
11
|
import type { Navigator } from './navigator.js';
|
|
11
12
|
import type { Provider } from './provider.js';
|
|
12
13
|
import { Quartermaster } from './quartermaster.js';
|
|
@@ -17,6 +18,7 @@ export declare abstract class TaskAgent {
|
|
|
17
18
|
config: ExplorbotConfig;
|
|
18
19
|
stateManager: StateManager;
|
|
19
20
|
knowledgeTracker: KnowledgeTracker;
|
|
21
|
+
judge?: Judge;
|
|
20
22
|
hooksRunner: HooksRunner;
|
|
21
23
|
consecutiveFailures: number;
|
|
22
24
|
consecutiveEmptyResults: number;
|
|
@@ -16,6 +16,7 @@ export class TaskAgent {
|
|
|
16
16
|
config;
|
|
17
17
|
stateManager;
|
|
18
18
|
knowledgeTracker;
|
|
19
|
+
judge;
|
|
19
20
|
hooksRunner;
|
|
20
21
|
consecutiveFailures = 0;
|
|
21
22
|
consecutiveEmptyResults = 0;
|
|
@@ -31,6 +32,7 @@ export class TaskAgent {
|
|
|
31
32
|
this.config = deps.config;
|
|
32
33
|
this.stateManager = deps.stateManager;
|
|
33
34
|
this.knowledgeTracker = deps.knowledgeTracker;
|
|
35
|
+
this.judge = deps.judge;
|
|
34
36
|
this.hooksRunner = new HooksRunner(deps.explorer, deps.config);
|
|
35
37
|
}
|
|
36
38
|
setHistorian(historian) {
|
|
@@ -40,7 +42,7 @@ export class TaskAgent {
|
|
|
40
42
|
this._quartermaster = quartermaster;
|
|
41
43
|
}
|
|
42
44
|
get toolDeps() {
|
|
43
|
-
return { explorer: this.explorer, stateManager: this.stateManager, ai: this.provider };
|
|
45
|
+
return { explorer: this.explorer, stateManager: this.stateManager, ai: this.provider, judge: this.judge };
|
|
44
46
|
}
|
|
45
47
|
getExperienceTracker() {
|
|
46
48
|
return this.stateManager.getExperienceTracker();
|
package/dist/src/ai/tester.js
CHANGED
|
@@ -112,8 +112,8 @@ export class Tester extends TaskAgent {
|
|
|
112
112
|
let initialState = ActionResult.fromState(state);
|
|
113
113
|
const currentUrl = state.fullUrl || state.url;
|
|
114
114
|
let startOnCurrentPage = opts.startOnCurrentPage;
|
|
115
|
-
if (
|
|
116
|
-
debugLog(`
|
|
115
|
+
if (!startOnCurrentPage && task.startUrl && normalizeUrl(currentUrl) !== normalizeUrl(task.startUrl)) {
|
|
116
|
+
debugLog(`Opening test start URL ${task.startUrl} before building context (was at ${currentUrl})`);
|
|
117
117
|
try {
|
|
118
118
|
await this.explorer.visit(task.startUrl);
|
|
119
119
|
state = this.stateManager.getCurrentState();
|
|
@@ -123,7 +123,7 @@ export class Tester extends TaskAgent {
|
|
|
123
123
|
startOnCurrentPage = true;
|
|
124
124
|
}
|
|
125
125
|
catch (error) {
|
|
126
|
-
debugLog(`Could not
|
|
126
|
+
debugLog(`Could not open test start URL: ${compactErrorMessage(error)}`);
|
|
127
127
|
}
|
|
128
128
|
}
|
|
129
129
|
if (isErrorPage(initialState)) {
|
|
@@ -135,13 +135,6 @@ export class Tester extends TaskAgent {
|
|
|
135
135
|
const conversation = this.provider.startConversation(this.getSystemMessage(), 'tester');
|
|
136
136
|
conversation.markLastMessageCacheable();
|
|
137
137
|
this.currentConversation = conversation;
|
|
138
|
-
const scenarioBlock = this.buildScenarioBlock(task, initialState);
|
|
139
|
-
conversation.addUserText(scenarioBlock);
|
|
140
|
-
conversation.markLastMessageCacheable();
|
|
141
|
-
conversation.protectPrefix(conversation.messages.length);
|
|
142
|
-
const pageContext = await this.reinjectContextIfNeeded(1, initialState);
|
|
143
|
-
if (pageContext)
|
|
144
|
-
conversation.addUserText(pageContext);
|
|
145
138
|
return await Observability.run(`test: ${task.scenario}`, {
|
|
146
139
|
sessionId: task.sessionName,
|
|
147
140
|
tags: ['tester'],
|
|
@@ -154,16 +147,15 @@ export class Tester extends TaskAgent {
|
|
|
154
147
|
}
|
|
155
148
|
async runTestSession(task, initialState, conversation, handlers, opts) {
|
|
156
149
|
const { offFailedRequest } = handlers;
|
|
150
|
+
let plan = '';
|
|
157
151
|
if (this.pilot) {
|
|
158
152
|
try {
|
|
159
|
-
|
|
153
|
+
await this.researcher.research(initialState).catch(this.skipResearch);
|
|
154
|
+
plan = await this.pilot.planTest(task, initialState);
|
|
160
155
|
if (task.hasFinished) {
|
|
161
156
|
offFailedRequest?.();
|
|
162
157
|
return { success: task.isSuccessful };
|
|
163
158
|
}
|
|
164
|
-
if (plan) {
|
|
165
|
-
conversation.addUserText(`Pilot's test plan:\n${plan}\n\nFollow this plan while executing the test.`);
|
|
166
|
-
}
|
|
167
159
|
}
|
|
168
160
|
catch (err) {
|
|
169
161
|
const message = err instanceof Error ? err.message : String(err);
|
|
@@ -183,7 +175,7 @@ export class Tester extends TaskAgent {
|
|
|
183
175
|
}
|
|
184
176
|
if (opts.startOnCurrentPage)
|
|
185
177
|
debugLog(`Starting on the page already open at ${task.startUrl}`);
|
|
186
|
-
if (!opts.startOnCurrentPage) {
|
|
178
|
+
if (!opts.startOnCurrentPage || task.preparedData.length > 0) {
|
|
187
179
|
debugLog(`Navigating to ${task.startUrl}`);
|
|
188
180
|
try {
|
|
189
181
|
await this.explorer.visit(task.startUrl);
|
|
@@ -208,6 +200,15 @@ export class Tester extends TaskAgent {
|
|
|
208
200
|
}
|
|
209
201
|
const currentUrl = startState?.url || task.startUrl || '';
|
|
210
202
|
await this.hooksRunner.runBeforeHook('tester', currentUrl);
|
|
203
|
+
const testerState = this.getCurrentState();
|
|
204
|
+
conversation.addUserText(this.buildScenarioBlock(task, testerState));
|
|
205
|
+
conversation.markLastMessageCacheable();
|
|
206
|
+
conversation.protectPrefix(conversation.messages.length);
|
|
207
|
+
const pageContext = await this.reinjectContextIfNeeded(1, testerState);
|
|
208
|
+
if (pageContext)
|
|
209
|
+
conversation.addUserText(pageContext);
|
|
210
|
+
if (plan)
|
|
211
|
+
conversation.addUserText(`Pilot's test plan:\n${plan}\n\nFollow this plan while executing the test.`);
|
|
211
212
|
const offStateChange = this.stateManager.onStateChange((event) => {
|
|
212
213
|
if (task.hasFinished)
|
|
213
214
|
return;
|
|
@@ -836,6 +837,9 @@ export class Tester extends TaskAgent {
|
|
|
836
837
|
`;
|
|
837
838
|
}
|
|
838
839
|
buildDeletionScope(task) {
|
|
840
|
+
if (task.preparedData.length > 0) {
|
|
841
|
+
return `When deleting items, ONLY delete the data prepared for this test: ${task.preparedData.join('; ')}.`;
|
|
842
|
+
}
|
|
839
843
|
const deletableItems = this.getDeletableSessionNames(task);
|
|
840
844
|
if (deletableItems.length > 0) {
|
|
841
845
|
return `When deleting items, ONLY delete items whose title contains one of these session names: ${deletableItems.join(', ')}. These were created by previous tests.`;
|
package/dist/src/ai/tools.d.ts
CHANGED
|
@@ -3,6 +3,7 @@ import type { ExecutedStep } from '../action.js';
|
|
|
3
3
|
import { type ExperienceTracker } from '../experience-tracker.js';
|
|
4
4
|
import { type Task } from '../test-plan.js';
|
|
5
5
|
import type { ToolDeps } from './agent.js';
|
|
6
|
+
import { type Judge } from './judge.js';
|
|
6
7
|
import { Navigator } from './navigator.js';
|
|
7
8
|
import { Researcher } from './researcher.js';
|
|
8
9
|
interface AgentToolDeps extends ToolDeps {
|
|
@@ -12,7 +13,7 @@ interface AgentToolDeps extends ToolDeps {
|
|
|
12
13
|
withExperience?: boolean;
|
|
13
14
|
}
|
|
14
15
|
export declare const ASSERTION_TOOLS: readonly ["verify"];
|
|
15
|
-
export declare function createCodeceptJSTools({ explorer, stateManager }: ToolDeps, task: Task): {
|
|
16
|
+
export declare function createCodeceptJSTools({ explorer, stateManager, judge }: ToolDeps, task: Task): {
|
|
16
17
|
click: import("@ai-sdk/provider-utils").ExecutableTool<import("ai").Tool<{
|
|
17
18
|
commands: any;
|
|
18
19
|
explanation: any;
|
|
@@ -55,7 +56,7 @@ export declare function createLearnExperienceTool({ getExperienceTracker, getSta
|
|
|
55
56
|
} | {
|
|
56
57
|
error: string;
|
|
57
58
|
}, import("@ai-sdk/provider-utils").Context>>;
|
|
58
|
-
export declare function createAgentTools({ explorer, stateManager, ai, researcher, navigator, supervisor, withExperience }: AgentToolDeps): any;
|
|
59
|
+
export declare function createAgentTools({ explorer, stateManager, ai, judge, researcher, navigator, supervisor, withExperience }: AgentToolDeps): any;
|
|
59
60
|
export declare function commitNote(activeNote: any, result: TestResult, toolResult: any, action: any): Promise<void>;
|
|
60
61
|
export declare function successToolResult(action: string, data?: Record<string, any>, source?: {
|
|
61
62
|
playwrightGroupId?: string | null;
|
|
@@ -64,7 +65,7 @@ export declare function successToolResult(action: string, data?: Record<string,
|
|
|
64
65
|
export declare function isMajorPageChange(pageDiff: PageDiff): boolean;
|
|
65
66
|
export declare function hasFailedRequest(pageDiff: PageDiff): boolean;
|
|
66
67
|
export declare function formatExecutedSteps(steps: ExecutedStep[], requestedCount?: number): string;
|
|
67
|
-
export declare function failedToolResult(action: string, message: string, data?: Record<string, any>, error?: Error | null): Promise<Record<string, any>>;
|
|
68
|
+
export declare function failedToolResult(action: string, message: string, data?: Record<string, any>, error?: Error | null, judge?: Judge, intent?: string): Promise<Record<string, any>>;
|
|
68
69
|
export declare function withdrawVisionTools(tools: Record<string, any>): void;
|
|
69
70
|
export declare function clickFailureSuggestion(attempts: Array<{
|
|
70
71
|
error?: string;
|
package/dist/src/ai/tools.js
CHANGED
|
@@ -13,11 +13,13 @@ import { createDebug, tag } from '../utils/logger.js';
|
|
|
13
13
|
import { pause } from '../utils/loop.js';
|
|
14
14
|
import { compactErrorMessage, normalizeInlineText, truncate } from "../utils/strings.js";
|
|
15
15
|
import { WebElement } from "../utils/web-element.js";
|
|
16
|
+
import { createJudgeTool } from "./judge-tool.js";
|
|
17
|
+
import { JUDGE_PAGE_CAP, UNDECIDED } from "./judge.js";
|
|
16
18
|
import { sectionContextRule } from "./rules.js";
|
|
17
19
|
import { isInteractive } from "./task-agent.js";
|
|
18
20
|
const debugLog = createDebug('explorbot:tools');
|
|
19
21
|
export const ASSERTION_TOOLS = ['verify'];
|
|
20
|
-
export function createCodeceptJSTools({ explorer, stateManager }, task) {
|
|
22
|
+
export function createCodeceptJSTools({ explorer, stateManager, judge }, task) {
|
|
21
23
|
return {
|
|
22
24
|
click: tool({
|
|
23
25
|
description: dedent `
|
|
@@ -125,7 +127,7 @@ export function createCodeceptJSTools({ explorer, stateManager }, task) {
|
|
|
125
127
|
...toolResult,
|
|
126
128
|
attempts,
|
|
127
129
|
suggestion,
|
|
128
|
-
}, ambiguityError || action.lastError);
|
|
130
|
+
}, ambiguityError || action.lastError, judge, explanation);
|
|
129
131
|
},
|
|
130
132
|
}),
|
|
131
133
|
hover: tool({
|
|
@@ -189,7 +191,7 @@ export function createCodeceptJSTools({ explorer, stateManager }, task) {
|
|
|
189
191
|
...toolResult,
|
|
190
192
|
attempts,
|
|
191
193
|
suggestion: 'Use xpathCheck() to locate the row/card/tree node, or visualClick() if the hover target is only visually identifiable.',
|
|
192
|
-
}, action.lastError);
|
|
194
|
+
}, action.lastError, judge, explanation);
|
|
193
195
|
},
|
|
194
196
|
}),
|
|
195
197
|
pressKey: tool({
|
|
@@ -312,7 +314,7 @@ export function createCodeceptJSTools({ explorer, stateManager }, task) {
|
|
|
312
314
|
- Performing multiple form actions in a single batch
|
|
313
315
|
- Complex interactions requiring sequential commands
|
|
314
316
|
- Reaching items further down a list (I.scrollTo)
|
|
315
|
-
- Reloading the page to prove a change outlived it (I.
|
|
317
|
+
- Reloading the page to prove a change outlived it (I.refreshPage)
|
|
316
318
|
|
|
317
319
|
Example - filling a form with context (PREFERRED):
|
|
318
320
|
I.fillField('Username', 'John', '.login-form')
|
|
@@ -363,7 +365,7 @@ export function createCodeceptJSTools({ explorer, stateManager }, task) {
|
|
|
363
365
|
code: codeBlock,
|
|
364
366
|
attempts: action.executedSteps,
|
|
365
367
|
suggestion: formSuggestion,
|
|
366
|
-
}, action.lastError);
|
|
368
|
+
}, action.lastError, judge, explanation);
|
|
367
369
|
}
|
|
368
370
|
if (!hasObservablePageChange(toolResult)) {
|
|
369
371
|
activeNote.commit(TestResult.FAILED);
|
|
@@ -498,7 +500,7 @@ export function createLearnExperienceTool({ getExperienceTracker, getState }) {
|
|
|
498
500
|
},
|
|
499
501
|
});
|
|
500
502
|
}
|
|
501
|
-
export function createAgentTools({ explorer, stateManager, ai, researcher, navigator, supervisor, withExperience }) {
|
|
503
|
+
export function createAgentTools({ explorer, stateManager, ai, judge, researcher, navigator, supervisor, withExperience }) {
|
|
502
504
|
const tools = {
|
|
503
505
|
see: tool({
|
|
504
506
|
description: dedent `
|
|
@@ -619,6 +621,12 @@ export function createAgentTools({ explorer, stateManager, ai, researcher, navig
|
|
|
619
621
|
}, { assertionSteps: result.assertionSteps });
|
|
620
622
|
}
|
|
621
623
|
if (result.inexpressible) {
|
|
624
|
+
if (result.judged?.approved) {
|
|
625
|
+
return failedToolResult('verify', `No assertion could express this claim, but the page appears to confirm it: ${assertion}`, {
|
|
626
|
+
inexpressible: true,
|
|
627
|
+
suggestion: 'This is a judgement about the page, not an assertion that ran in the browser — a hint, not proof. Restate the claim in terms of what is visible or of a control state to get a real assertion.',
|
|
628
|
+
});
|
|
629
|
+
}
|
|
622
630
|
return failedToolResult('verify', `No assertion could express this claim: ${assertion}`, {
|
|
623
631
|
inexpressible: true,
|
|
624
632
|
suggestion: 'This is not evidence the page is wrong — the claim could not be turned into an assertion. Restate it in terms of what is visible or of a control state, or check it with see().',
|
|
@@ -994,6 +1002,19 @@ export function createAgentTools({ explorer, stateManager, ai, researcher, navig
|
|
|
994
1002
|
},
|
|
995
1003
|
});
|
|
996
1004
|
}
|
|
1005
|
+
const buildJudgeState = async () => {
|
|
1006
|
+
const activeTest = explorer.activeTest;
|
|
1007
|
+
const state = stateManager.getCurrentState();
|
|
1008
|
+
const result = state ? ActionResult.fromState(state) : null;
|
|
1009
|
+
return {
|
|
1010
|
+
task: activeTest?.scenario || '',
|
|
1011
|
+
page: cap(result?.getCompactARIA(), JUDGE_PAGE_CAP),
|
|
1012
|
+
recentActions: Object.values(activeTest?.steps || {})
|
|
1013
|
+
.slice(-JUDGE_RECENT_ACTIONS_LIMIT)
|
|
1014
|
+
.map((step) => step.text),
|
|
1015
|
+
};
|
|
1016
|
+
};
|
|
1017
|
+
Object.assign(tools, createJudgeTool({ explorer, stateManager, ai, judge }, buildJudgeState));
|
|
997
1018
|
withdrawVisionTools(tools);
|
|
998
1019
|
return tools;
|
|
999
1020
|
}
|
|
@@ -1003,6 +1024,7 @@ const NAVIGATED_SUGGESTION = 'The action left the page. Elements are never compa
|
|
|
1003
1024
|
const ARIA_OUTPUT_CAP = 4000;
|
|
1004
1025
|
const HTML_OUTPUT_CAP = 6000;
|
|
1005
1026
|
const ANALYSIS_OUTPUT_CAP = 2000;
|
|
1027
|
+
const JUDGE_RECENT_ACTIONS_LIMIT = 8;
|
|
1006
1028
|
function cap(text, max) {
|
|
1007
1029
|
if (!text)
|
|
1008
1030
|
return '';
|
|
@@ -1115,7 +1137,7 @@ function hasObservablePageChange(data) {
|
|
|
1115
1137
|
return true;
|
|
1116
1138
|
return Array.isArray(data.pageDiff.htmlParts) && data.pageDiff.htmlParts.length > 0;
|
|
1117
1139
|
}
|
|
1118
|
-
export async function failedToolResult(action, message, data, error) {
|
|
1140
|
+
export async function failedToolResult(action, message, data, error, judge, intent) {
|
|
1119
1141
|
const result = { success: false, action, message, ...data };
|
|
1120
1142
|
if (data?.pageDiff) {
|
|
1121
1143
|
result.suggestion = data.suggestion ? `${data.suggestion} ${PAGE_DIFF_SUGGESTION}` : PAGE_DIFF_SUGGESTION;
|
|
@@ -1126,6 +1148,12 @@ export async function failedToolResult(action, message, data, error) {
|
|
|
1126
1148
|
result.suggestion = getMultipleElementsSuggestion();
|
|
1127
1149
|
result.multipleElementsDetected = true;
|
|
1128
1150
|
result.elements = formatElementList(matched);
|
|
1151
|
+
const labels = (matched || []).map((element) => `${element.text || 'no text'} — ${element.html}`);
|
|
1152
|
+
const pick = await judge?.decide('Which listed element does the intent name?', [...labels, UNDECIDED], { intent });
|
|
1153
|
+
if (!pick?.value)
|
|
1154
|
+
return result;
|
|
1155
|
+
const index = labels.indexOf(pick.value) + 1;
|
|
1156
|
+
result.suggestion = `Element ${index} is the one meant. Repeat the action with step.opts({ elementIndex: ${index} }) as the last argument.`;
|
|
1129
1157
|
return result;
|
|
1130
1158
|
}
|
|
1131
1159
|
const notFoundSuggestion = getNotFoundSuggestion(message);
|
|
@@ -162,7 +162,8 @@ export class RequestResult {
|
|
|
162
162
|
if (Array.isArray(data))
|
|
163
163
|
return {};
|
|
164
164
|
const id = data.id ?? data._id ?? data.uuid;
|
|
165
|
-
const
|
|
165
|
+
const attributes = data.attributes || {};
|
|
166
|
+
const title = data.name ?? data.title ?? data.label ?? attributes.name ?? attributes.title ?? attributes.label;
|
|
166
167
|
const result = {};
|
|
167
168
|
if (id !== undefined)
|
|
168
169
|
result.id = id;
|
|
@@ -207,6 +207,28 @@ export class CommandHandler {
|
|
|
207
207
|
const parsed = parseCommand(query);
|
|
208
208
|
const exactCommand = parsed ? this.findCommand(parsed.name) : undefined;
|
|
209
209
|
const hasArguments = commandEnd !== -1 && input.slice(commandEnd + 1).trim().length > 0;
|
|
210
|
+
const argumentHint = !insideCommand && exactCommand && !hasArguments && exactCommand.options.length > 0 ? exactCommand.options.map((option) => option.flags).join(' ') : undefined;
|
|
211
|
+
if (!insideCommand && exactCommand) {
|
|
212
|
+
const argumentQuery = input.slice(commandEnd + 1).trim();
|
|
213
|
+
const argumentEntries = exactCommand
|
|
214
|
+
.completeArguments()
|
|
215
|
+
.filter((completion) => completion.value !== argumentQuery)
|
|
216
|
+
.map((completion) => ({
|
|
217
|
+
aliases: [],
|
|
218
|
+
canonical: completion.value,
|
|
219
|
+
description: '',
|
|
220
|
+
value: completion.display || completion.value,
|
|
221
|
+
}));
|
|
222
|
+
const argumentSuggestions = this.rankSuggestions(argumentQuery, argumentEntries);
|
|
223
|
+
return {
|
|
224
|
+
suggestions: argumentSuggestions,
|
|
225
|
+
replaceFrom: commandEnd + 1,
|
|
226
|
+
replaceTo: input.length,
|
|
227
|
+
visible: argumentSuggestions.length > 0,
|
|
228
|
+
argumentHint,
|
|
229
|
+
completesArgument: true,
|
|
230
|
+
};
|
|
231
|
+
}
|
|
210
232
|
const commandEntries = this.getSlashCommandEntries();
|
|
211
233
|
let suggestions = [];
|
|
212
234
|
if (insideCommand) {
|
|
@@ -222,9 +244,8 @@ export class CommandHandler {
|
|
|
222
244
|
suggestions = this.rankSuggestions(query, commandEntries);
|
|
223
245
|
}
|
|
224
246
|
}
|
|
225
|
-
const argumentHint = !insideCommand && exactCommand && !hasArguments && exactCommand.options.length > 0 ? exactCommand.options.map((option) => option.flags).join(' ') : undefined;
|
|
226
247
|
return {
|
|
227
|
-
suggestions,
|
|
248
|
+
suggestions: suggestions.map((suggestion) => ({ ...suggestion, value: `${suggestion.value} ` })),
|
|
228
249
|
replaceFrom: 0,
|
|
229
250
|
replaceTo,
|
|
230
251
|
visible: insideCommand && suggestions.length > 0,
|
|
@@ -305,7 +326,7 @@ export class CommandHandler {
|
|
|
305
326
|
rankSuggestions(query, entries) {
|
|
306
327
|
if (!query) {
|
|
307
328
|
return entries.slice(0, 20).map((entry) => ({
|
|
308
|
-
value: entry.
|
|
329
|
+
value: entry.canonical,
|
|
309
330
|
display: entry.value,
|
|
310
331
|
description: entry.description,
|
|
311
332
|
argumentHint: entry.argumentHint,
|
|
@@ -18,9 +18,14 @@ export declare abstract class BaseCommand<T = ExplorBot> {
|
|
|
18
18
|
constructor(explorBot: T);
|
|
19
19
|
abstract execute(args: string): Promise<void>;
|
|
20
20
|
matches(commandName: string): boolean;
|
|
21
|
+
completeArguments(): ArgumentCompletion[];
|
|
21
22
|
printSuggestions(): void;
|
|
22
23
|
parseArgs(args: string): {
|
|
23
24
|
opts: Record<string, string | boolean>;
|
|
24
25
|
args: string[];
|
|
25
26
|
};
|
|
26
27
|
}
|
|
28
|
+
export interface ArgumentCompletion {
|
|
29
|
+
value: string;
|
|
30
|
+
display?: string;
|
|
31
|
+
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { type Plan, type Test } from '../test-plan.js';
|
|
2
|
-
import { BaseCommand, type Suggestion } from './base-command.js';
|
|
2
|
+
import { type ArgumentCompletion, BaseCommand, type Suggestion } from './base-command.js';
|
|
3
3
|
export declare const DEADLINE_RESERVE_MS: number;
|
|
4
4
|
export declare const DEADLINE_TEST_ALLOWANCE_MS: number;
|
|
5
5
|
export declare class ExploreCommand extends BaseCommand {
|
|
@@ -22,6 +22,7 @@ export declare class ExploreCommand extends BaseCommand {
|
|
|
22
22
|
priorityFilter?: Set<string>;
|
|
23
23
|
getCurrentPageUrl(): string | undefined;
|
|
24
24
|
execute(args: string): Promise<void>;
|
|
25
|
+
completeArguments(): ArgumentCompletion[];
|
|
25
26
|
originLabel(test: Test): string;
|
|
26
27
|
printPreview(label: string, tests: Test[]): void;
|
|
27
28
|
runFreshMode(mainUrl: string | undefined, feature: string | undefined, styles?: string[]): Promise<void>;
|
|
@@ -17,7 +17,7 @@ export const DEADLINE_RESERVE_MS = 3 * 60_000;
|
|
|
17
17
|
export const DEADLINE_TEST_ALLOWANCE_MS = 5 * 60_000;
|
|
18
18
|
export class ExploreCommand extends BaseCommand {
|
|
19
19
|
name = 'explore';
|
|
20
|
-
description = 'Start web exploration';
|
|
20
|
+
description = 'Start web exploration on the current page, or on the URL given as the first argument';
|
|
21
21
|
options = [
|
|
22
22
|
{ flags: '--max-tests <number>', description: 'Maximum number of tests to run' },
|
|
23
23
|
{ flags: '--max-duration <minutes>', description: 'Wall-clock budget in minutes; wraps up the session before the limit is hit' },
|
|
@@ -55,6 +55,11 @@ export class ExploreCommand extends BaseCommand {
|
|
|
55
55
|
if (this.hardDeadlineAt == null && this.maxDurationMinutes != null) {
|
|
56
56
|
this.hardDeadlineAt = Date.now() + this.maxDurationMinutes * 60_000 - DEADLINE_RESERVE_MS;
|
|
57
57
|
}
|
|
58
|
+
const [target] = remaining;
|
|
59
|
+
if (target?.startsWith('/') || target?.startsWith('http://') || target?.startsWith('https://')) {
|
|
60
|
+
remaining.shift();
|
|
61
|
+
await this.explorBot.visit(target);
|
|
62
|
+
}
|
|
58
63
|
const feature = opts.focus || remaining.join(' ') || undefined;
|
|
59
64
|
const cfg = this.parseConfigure(opts.configure);
|
|
60
65
|
if (cfg.priorities)
|
|
@@ -96,6 +101,12 @@ export class ExploreCommand extends BaseCommand {
|
|
|
96
101
|
await this.explorBot.printSessionAnalysis();
|
|
97
102
|
}
|
|
98
103
|
}
|
|
104
|
+
completeArguments() {
|
|
105
|
+
return this.explorBot
|
|
106
|
+
.stateManager()
|
|
107
|
+
.getKnownUrls()
|
|
108
|
+
.map((url) => ({ value: url }));
|
|
109
|
+
}
|
|
99
110
|
originLabel(test) {
|
|
100
111
|
return this.oldTestRefs.has(test) ? 'OLD' : 'NEW';
|
|
101
112
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { Planner } from '../ai/planner.js';
|
|
2
2
|
import { Researcher } from '../ai/researcher.js';
|
|
3
|
+
import { normalizeUrl } from '../state-manager.js';
|
|
3
4
|
import { Stats } from '../stats.js';
|
|
4
5
|
import { tag } from '../utils/logger.js';
|
|
5
6
|
import { loop } from '../utils/loop.js';
|
|
@@ -65,11 +66,16 @@ export class FreesailCommand extends BaseCommand {
|
|
|
65
66
|
const suggestion = await navigator.freeSail({ strategy, scope, visitedUrls });
|
|
66
67
|
if (!suggestion) {
|
|
67
68
|
tag('info').log('No navigation suggestion available');
|
|
68
|
-
|
|
69
|
+
ctx.stop();
|
|
69
70
|
}
|
|
70
71
|
if (scope && !suggestion.target.startsWith(scope)) {
|
|
71
72
|
tag('warning').log(`Suggestion ${suggestion.target} is outside scope ${scope}, skipping`);
|
|
72
|
-
|
|
73
|
+
ctx.stop();
|
|
74
|
+
}
|
|
75
|
+
const currentUrl = stateManager.getCurrentState()?.url;
|
|
76
|
+
if (currentUrl && normalizeUrl(suggestion.target) === normalizeUrl(currentUrl)) {
|
|
77
|
+
tag('info').log(`No new navigation target available after ${currentUrl}; stopping exploration`);
|
|
78
|
+
ctx.stop();
|
|
73
79
|
}
|
|
74
80
|
tag('info').log(`Navigating to: ${suggestion.target} - ${suggestion.reason}`);
|
|
75
81
|
await this.explorBot.openTab();
|
|
@@ -145,7 +145,7 @@ export function runInitCommand(options) {
|
|
|
145
145
|
log('2. Set AI models config file');
|
|
146
146
|
log('3. Set web application URL in the config file');
|
|
147
147
|
log('4. Add initial knowledge (how to authorize to the application, etc.)');
|
|
148
|
-
tag('substep').log(chalk.yellow(`${getCliName()} learn * 'to authorize use these credentials: admin@example.com / secret123'`));
|
|
148
|
+
tag('substep').log(chalk.yellow(`${getCliName()} learn '*' 'to authorize use these credentials: admin@example.com / secret123'`));
|
|
149
149
|
tag('substep').log('You can use ${env.LOGIN} and ${env.PASSWORD} to reference environment variables.');
|
|
150
150
|
log('5. Launch application on a relative URL');
|
|
151
151
|
tag('substep').log(chalk.yellow(`${getCliName()} start /dashboard`));
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import { BaseCommand, type Suggestion } from './base-command.js';
|
|
1
|
+
import { type ArgumentCompletion, BaseCommand, type Suggestion } from './base-command.js';
|
|
2
2
|
export declare class NavigateCommand extends BaseCommand {
|
|
3
3
|
name: string;
|
|
4
4
|
description: string;
|
|
5
5
|
suggestions: Suggestion[];
|
|
6
6
|
execute(args: string): Promise<void>;
|
|
7
|
+
completeArguments(): ArgumentCompletion[];
|
|
7
8
|
}
|
|
@@ -15,4 +15,10 @@ export class NavigateCommand extends BaseCommand {
|
|
|
15
15
|
await this.explorBot.visit(destination);
|
|
16
16
|
tag('success').log(`Navigation requested: ${destination}`);
|
|
17
17
|
}
|
|
18
|
+
completeArguments() {
|
|
19
|
+
return this.explorBot
|
|
20
|
+
.stateManager()
|
|
21
|
+
.getKnownUrls()
|
|
22
|
+
.map((url) => ({ value: url }));
|
|
23
|
+
}
|
|
18
24
|
}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import { BaseCommand, type Suggestion } from './base-command.js';
|
|
1
|
+
import { type ArgumentCompletion, BaseCommand, type Suggestion } from './base-command.js';
|
|
2
2
|
export declare class PlanLoadCommand extends BaseCommand {
|
|
3
3
|
name: string;
|
|
4
4
|
description: string;
|
|
5
5
|
suggestions: Suggestion[];
|
|
6
6
|
execute(args: string): Promise<void>;
|
|
7
|
+
completeArguments(): ArgumentCompletion[];
|
|
7
8
|
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { Plan } from '../test-plan.js';
|
|
1
2
|
import { tag } from '../utils/logger.js';
|
|
2
3
|
import { BaseCommand } from './base-command.js';
|
|
3
4
|
export class PlanLoadCommand extends BaseCommand {
|
|
@@ -15,4 +16,7 @@ export class PlanLoadCommand extends BaseCommand {
|
|
|
15
16
|
const plan = this.explorBot.loadPlan(filename);
|
|
16
17
|
tag('success').log(`Plan loaded: ${plan.title} with ${plan.tests.length} tests`);
|
|
17
18
|
}
|
|
19
|
+
completeArguments() {
|
|
20
|
+
return Plan.listFiles(this.explorBot.getPlansDir()).map((file) => ({ value: file.name, display: file.label }));
|
|
21
|
+
}
|
|
18
22
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { Plan } from '../test-plan.js';
|
|
2
|
-
import { BaseCommand } from './base-command.js';
|
|
1
|
+
import { Plan, type PlanFile } from '../test-plan.js';
|
|
2
|
+
import { type ArgumentCompletion, BaseCommand } from './base-command.js';
|
|
3
3
|
export declare class PlansCommand extends BaseCommand {
|
|
4
4
|
name: string;
|
|
5
5
|
description: string;
|
|
@@ -8,7 +8,7 @@ export declare class PlansCommand extends BaseCommand {
|
|
|
8
8
|
description: string;
|
|
9
9
|
}[];
|
|
10
10
|
execute(args: string): Promise<void>;
|
|
11
|
-
|
|
11
|
+
completeArguments(): ArgumentCompletion[];
|
|
12
12
|
printPlans(files: PlanFile[]): void;
|
|
13
13
|
printPlanDetails(plan: Plan, file: PlanFile): void;
|
|
14
14
|
resolvePlanFile(target: string, files: PlanFile[]): {
|
|
@@ -16,9 +16,3 @@ export declare class PlansCommand extends BaseCommand {
|
|
|
16
16
|
file: PlanFile;
|
|
17
17
|
};
|
|
18
18
|
}
|
|
19
|
-
interface PlanFile {
|
|
20
|
-
name: string;
|
|
21
|
-
path: string;
|
|
22
|
-
modifiedAt: number;
|
|
23
|
-
}
|
|
24
|
-
export {};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { statSync } from 'node:fs';
|
|
2
2
|
import path from 'node:path';
|
|
3
3
|
import { Plan } from '../test-plan.js';
|
|
4
4
|
import { getCliName } from '../utils/cli-name.js';
|
|
@@ -11,7 +11,7 @@ export class PlansCommand extends BaseCommand {
|
|
|
11
11
|
options = [{ flags: '--from-plan <file>', description: 'Plan file to show' }];
|
|
12
12
|
async execute(args) {
|
|
13
13
|
const { opts, args: remaining } = this.parseArgs(args);
|
|
14
|
-
const files = this.
|
|
14
|
+
const files = Plan.listFiles(this.explorBot.getPlansDir());
|
|
15
15
|
const target = String(opts.fromPlan || remaining[0] || '').trim();
|
|
16
16
|
if (!target) {
|
|
17
17
|
this.printPlans(files);
|
|
@@ -20,22 +20,8 @@ export class PlansCommand extends BaseCommand {
|
|
|
20
20
|
const { plan, file } = this.resolvePlanFile(target, files);
|
|
21
21
|
this.printPlanDetails(plan, file);
|
|
22
22
|
}
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
if (!existsSync(plansDir))
|
|
26
|
-
return [];
|
|
27
|
-
return readdirSync(plansDir)
|
|
28
|
-
.filter((file) => file.endsWith('.md'))
|
|
29
|
-
.map((file) => {
|
|
30
|
-
const filePath = path.join(plansDir, file);
|
|
31
|
-
const stat = statSync(filePath);
|
|
32
|
-
return {
|
|
33
|
-
name: file,
|
|
34
|
-
path: filePath,
|
|
35
|
-
modifiedAt: stat.mtimeMs,
|
|
36
|
-
};
|
|
37
|
-
})
|
|
38
|
-
.sort((left, right) => right.modifiedAt - left.modifiedAt);
|
|
23
|
+
completeArguments() {
|
|
24
|
+
return Plan.listFiles(this.explorBot.getPlansDir()).map((file) => ({ value: file.name, display: file.label }));
|
|
39
25
|
}
|
|
40
26
|
printPlans(files) {
|
|
41
27
|
if (files.length === 0) {
|
|
@@ -45,8 +31,7 @@ export class PlansCommand extends BaseCommand {
|
|
|
45
31
|
tag('info').log('Saved plans:');
|
|
46
32
|
for (let i = 0; i < files.length; i++) {
|
|
47
33
|
const file = files[i];
|
|
48
|
-
|
|
49
|
-
tag('info').log(`${i + 1}. ${plan.title} (${plan.tests.length} tests) - ${file.name}`);
|
|
34
|
+
tag('info').log(`${i + 1}. ${file.title} (${file.testCount} tests) - ${file.name}`);
|
|
50
35
|
}
|
|
51
36
|
tag('info').log('');
|
|
52
37
|
tag('info').log(`View plan tests: ${getCliName()} plans <number>`);
|
|
@@ -73,10 +58,15 @@ export class PlansCommand extends BaseCommand {
|
|
|
73
58
|
if (!plan?.filePath) {
|
|
74
59
|
throw new Error(`Plan file not found: ${target}`);
|
|
75
60
|
}
|
|
61
|
+
const name = path.basename(plan.filePath);
|
|
76
62
|
const file = {
|
|
77
|
-
name
|
|
63
|
+
name,
|
|
78
64
|
path: plan.filePath,
|
|
79
65
|
modifiedAt: statSync(plan.filePath).mtimeMs,
|
|
66
|
+
title: plan.title,
|
|
67
|
+
url: plan.startUrl || '',
|
|
68
|
+
testCount: plan.tests.length,
|
|
69
|
+
label: name,
|
|
80
70
|
};
|
|
81
71
|
return { plan, file };
|
|
82
72
|
}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import { BaseCommand } from './base-command.js';
|
|
1
|
+
import { type ArgumentCompletion, BaseCommand } from './base-command.js';
|
|
2
2
|
export declare class RerunCommand extends BaseCommand {
|
|
3
3
|
name: string;
|
|
4
4
|
description: string;
|
|
5
5
|
tuiEnabled: boolean;
|
|
6
6
|
execute(args: string): Promise<void>;
|
|
7
|
+
completeArguments(): ArgumentCompletion[];
|
|
7
8
|
}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { existsSync } from 'node:fs';
|
|
2
|
-
import { resolve } from 'node:path';
|
|
2
|
+
import { basename, resolve } from 'node:path';
|
|
3
3
|
import { ConfigParser } from "../config.js";
|
|
4
4
|
import { tag } from "../utils/logger.js";
|
|
5
|
+
import { listTestFiles } from "../utils/test-files.js";
|
|
5
6
|
import { BaseCommand } from './base-command.js';
|
|
6
7
|
export class RerunCommand extends BaseCommand {
|
|
7
8
|
name = 'rerun';
|
|
@@ -26,6 +27,9 @@ export class RerunCommand extends BaseCommand {
|
|
|
26
27
|
const testIndices = indexArg ? parseTestIndices(indexArg) : undefined;
|
|
27
28
|
await this.explorBot.agentRerunner().rerun(filePath, { testIndices });
|
|
28
29
|
}
|
|
30
|
+
completeArguments() {
|
|
31
|
+
return listTestFiles(ConfigParser.getInstance().getTestsDir()).map((file) => ({ value: basename(file) }));
|
|
32
|
+
}
|
|
29
33
|
}
|
|
30
34
|
function parseTestIndices(input) {
|
|
31
35
|
if (input === '*' || input === 'all')
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { BaseCommand, type Suggestion } from './base-command.js';
|
|
1
|
+
import { type ArgumentCompletion, BaseCommand, type Suggestion } from './base-command.js';
|
|
2
2
|
export declare class ResearchCommand extends BaseCommand {
|
|
3
3
|
name: string;
|
|
4
4
|
description: string;
|
|
@@ -8,4 +8,5 @@ export declare class ResearchCommand extends BaseCommand {
|
|
|
8
8
|
description: string;
|
|
9
9
|
}[];
|
|
10
10
|
execute(args: string): Promise<void>;
|
|
11
|
+
completeArguments(): ArgumentCompletion[];
|
|
11
12
|
}
|