explorbot 0.4.4 β 0.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/boat/prima/src/prima.ts +8 -3
- package/dist/boat/prima/src/prima.js +10 -3
- package/dist/package.json +1 -1
- package/dist/src/ai/fisherman/tools.js +7 -1
- package/dist/src/ai/fisherman.js +2 -1
- package/dist/src/ai/pilot.d.ts +0 -1
- package/dist/src/ai/pilot.js +8 -24
- package/dist/src/ai/planner.d.ts +4 -0
- package/dist/src/ai/planner.js +28 -0
- package/dist/src/ai/provider.js +3 -1
- package/dist/src/ai/rules.js +8 -7
- package/dist/src/ai/scout/tools.d.ts +17 -0
- package/dist/src/ai/scout/tools.js +130 -0
- package/dist/src/ai/scout.d.ts +21 -0
- package/dist/src/ai/scout.js +150 -0
- package/dist/src/ai/tools.js +56 -30
- package/dist/src/application-spec.d.ts +3 -0
- package/dist/src/application-spec.js +21 -5
- package/dist/src/config.d.ts +6 -1
- package/dist/src/explorbot.d.ts +3 -0
- package/dist/src/explorbot.js +33 -0
- package/dist/src/knowledge-tracker.d.ts +1 -0
- package/dist/src/knowledge-tracker.js +3 -0
- package/dist/src/utils/aria-ref.d.ts +16 -0
- package/dist/src/utils/aria-ref.js +47 -0
- package/dist/src/utils/aria.js +3 -3
- package/dist/src/utils/web-annotate.js +3 -15
- package/dist/src/utils/web-element.d.ts +0 -2
- package/dist/src/utils/web-element.js +0 -8
- package/docs/reference/configuration.md +28 -1
- package/docs/web-testing/agents.md +9 -1
- package/docs/web-testing/planner.md +5 -0
- package/docs/workflow/application-spec.md +4 -0
- package/package.json +1 -1
- package/src/ai/fisherman/tools.ts +8 -1
- package/src/ai/fisherman.ts +2 -1
- package/src/ai/pilot.ts +8 -25
- package/src/ai/planner.ts +33 -0
- package/src/ai/provider.ts +2 -1
- package/src/ai/rules.ts +8 -7
- package/src/ai/scout/tools.ts +150 -0
- package/src/ai/scout.ts +173 -0
- package/src/ai/tools.ts +69 -36
- package/src/application-spec.ts +22 -4
- package/src/config.ts +7 -0
- package/src/explorbot.ts +36 -0
- package/src/knowledge-tracker.ts +4 -0
- package/src/utils/aria-ref.ts +61 -0
- package/src/utils/aria.ts +3 -3
- package/src/utils/web-annotate.ts +3 -15
- package/src/utils/web-element.ts +0 -9
package/boat/prima/src/prima.ts
CHANGED
|
@@ -20,6 +20,7 @@ import { Reporter } from '../../../src/reporter.ts';
|
|
|
20
20
|
import type { WebPageState } from '../../../src/state-manager.ts';
|
|
21
21
|
import { Stats } from '../../../src/stats.ts';
|
|
22
22
|
import { Task, Test, TestResult } from '../../../src/test-plan.ts';
|
|
23
|
+
import { ariaRefSnapshot } from '../../../src/utils/aria-ref.ts';
|
|
23
24
|
import { compactAriaSnapshot } from '../../../src/utils/aria.ts';
|
|
24
25
|
import { browserErrorMessage } from '../../../src/utils/browser-errors.ts';
|
|
25
26
|
import { pluralize } from '../../../src/utils/logger.ts';
|
|
@@ -324,11 +325,15 @@ export class Prima {
|
|
|
324
325
|
return null;
|
|
325
326
|
});
|
|
326
327
|
|
|
327
|
-
|
|
328
|
-
|
|
328
|
+
const stillOpen = ledger.filter((entry) => entry.status === 'open').length;
|
|
329
329
|
for (const execution of invoked?.toolExecutions || []) {
|
|
330
330
|
this.applyLedgerReport(execution, ledger, trace);
|
|
331
331
|
}
|
|
332
|
+
|
|
333
|
+
let unsettled = '';
|
|
334
|
+
if (settleError) unsettled = browserErrorMessage(settleError);
|
|
335
|
+
if (invoked && ledger.filter((entry) => entry.status === 'open').length === stillOpen) unsettled = 'the model was asked to report every remaining instruction and reported none';
|
|
336
|
+
if (unsettled) trace.push({ label: 'settling which instructions were satisfied', ok: false, proof: unsettled });
|
|
332
337
|
}
|
|
333
338
|
|
|
334
339
|
private ledgerProgress(ledger: LedgerEntry[]): string {
|
|
@@ -916,7 +921,7 @@ export class Prima {
|
|
|
916
921
|
}
|
|
917
922
|
|
|
918
923
|
private async refAriaSnapshot(result: ActionResult): Promise<string | null> {
|
|
919
|
-
const snapshot = await Promise.resolve(this.bot.getExplorer()?.withPage?.(
|
|
924
|
+
const snapshot = await Promise.resolve(this.bot.getExplorer()?.withPage?.(ariaRefSnapshot)).catch(() => null);
|
|
920
925
|
return snapshot || result.ariaSnapshot;
|
|
921
926
|
}
|
|
922
927
|
|
|
@@ -18,6 +18,7 @@ import { findSiteWith, listSites } from "../../../src/global-config.js";
|
|
|
18
18
|
import { Reporter } from "../../../src/reporter.js";
|
|
19
19
|
import { Stats } from "../../../src/stats.js";
|
|
20
20
|
import { Task, Test, TestResult } from "../../../src/test-plan.js";
|
|
21
|
+
import { ariaRefSnapshot } from "../../../src/utils/aria-ref.js";
|
|
21
22
|
import { compactAriaSnapshot } from "../../../src/utils/aria.js";
|
|
22
23
|
import { browserErrorMessage } from "../../../src/utils/browser-errors.js";
|
|
23
24
|
import { pluralize } from "../../../src/utils/logger.js";
|
|
@@ -295,11 +296,17 @@ export class Prima {
|
|
|
295
296
|
settleError = error;
|
|
296
297
|
return null;
|
|
297
298
|
});
|
|
298
|
-
|
|
299
|
-
trace.push({ label: 'settling which instructions were satisfied', ok: false, proof: browserErrorMessage(settleError) });
|
|
299
|
+
const stillOpen = ledger.filter((entry) => entry.status === 'open').length;
|
|
300
300
|
for (const execution of invoked?.toolExecutions || []) {
|
|
301
301
|
this.applyLedgerReport(execution, ledger, trace);
|
|
302
302
|
}
|
|
303
|
+
let unsettled = '';
|
|
304
|
+
if (settleError)
|
|
305
|
+
unsettled = browserErrorMessage(settleError);
|
|
306
|
+
if (invoked && ledger.filter((entry) => entry.status === 'open').length === stillOpen)
|
|
307
|
+
unsettled = 'the model was asked to report every remaining instruction and reported none';
|
|
308
|
+
if (unsettled)
|
|
309
|
+
trace.push({ label: 'settling which instructions were satisfied', ok: false, proof: unsettled });
|
|
303
310
|
}
|
|
304
311
|
ledgerProgress(ledger) {
|
|
305
312
|
return ledger
|
|
@@ -858,7 +865,7 @@ export class Prima {
|
|
|
858
865
|
return path.join(path.basename(dir), name);
|
|
859
866
|
}
|
|
860
867
|
async refAriaSnapshot(result) {
|
|
861
|
-
const snapshot = await Promise.resolve(this.bot.getExplorer()?.withPage?.(
|
|
868
|
+
const snapshot = await Promise.resolve(this.bot.getExplorer()?.withPage?.(ariaRefSnapshot)).catch(() => null);
|
|
862
869
|
return snapshot || result.ariaSnapshot;
|
|
863
870
|
}
|
|
864
871
|
executedCodes(code) {
|
package/dist/package.json
CHANGED
|
@@ -3,8 +3,10 @@ import dedent from 'dedent';
|
|
|
3
3
|
import { z } from 'zod';
|
|
4
4
|
import { extractEndpointDefinition } from "../../api/spec-reader.js";
|
|
5
5
|
import { tag } from "../../utils/logger.js";
|
|
6
|
+
import { truncate } from "../../utils/strings.js";
|
|
6
7
|
import { isDynamicSegment } from "../../utils/url-matcher.js";
|
|
7
8
|
const BODY_PREVIEW_LIMIT = 2000;
|
|
9
|
+
const READS_IN_ANSWER = 3;
|
|
8
10
|
export function createFishermanTools(apiClient, requestStore, haul, opts) {
|
|
9
11
|
const readOnly = opts.readOnly === true;
|
|
10
12
|
let finished = false;
|
|
@@ -213,7 +215,7 @@ export function createAskApiTool(fisherman, task) {
|
|
|
213
215
|
return { answered: false, reason: result.summary || 'The API could not answer this question' };
|
|
214
216
|
}
|
|
215
217
|
task.addNote(`Asked API: ${question} β ${result.summary}`);
|
|
216
|
-
tag('success').log(`Ask API: ${result.summary}`);
|
|
218
|
+
tag('success').log(`Ask API: ${truncate(result.summary, 200)}`);
|
|
217
219
|
return { answered: true, answer: result.summary };
|
|
218
220
|
},
|
|
219
221
|
}),
|
|
@@ -252,6 +254,10 @@ function synthesizeResult(haul, declaredDone, readOnly) {
|
|
|
252
254
|
succeeded = haul.successfulReads();
|
|
253
255
|
successLabel = 'successful reads';
|
|
254
256
|
}
|
|
257
|
+
if (readOnly && succeeded.length > 0) {
|
|
258
|
+
const bodies = succeeded.slice(-READS_IN_ANSWER).map((read) => `${read.toEndpoint()} β ${read.rawResponseBody.substring(0, BODY_PREVIEW_LIMIT)}`);
|
|
259
|
+
return { success: true, summary: bodies.join('\n\n'), created: [], failed: [] };
|
|
260
|
+
}
|
|
255
261
|
let summary = `Stopped before finishing: ${made.length} requests, ${succeeded.length} ${successLabel}, ${failures.length} failed`;
|
|
256
262
|
const lastFailure = failures[failures.length - 1];
|
|
257
263
|
if (lastFailure)
|
package/dist/src/ai/fisherman.js
CHANGED
|
@@ -2,6 +2,7 @@ import dedent from 'dedent';
|
|
|
2
2
|
import { isFailedRequest } from "../api/request-store.js";
|
|
3
3
|
import { listAllEndpoints } from "../api/spec-reader.js";
|
|
4
4
|
import { createDebug, tag } from "../utils/logger.js";
|
|
5
|
+
import { truncate } from "../utils/strings.js";
|
|
5
6
|
const debugLog = createDebug('explorbot:fisherman');
|
|
6
7
|
import { loop } from "../utils/loop.js";
|
|
7
8
|
import { RequestHaul } from "./fisherman/request-haul.js";
|
|
@@ -106,7 +107,7 @@ export class Fisherman {
|
|
|
106
107
|
`);
|
|
107
108
|
await this.runSession(conversation, tools, { haul, isFinished, finishFromText, label: `fisherman lookup: ${question.slice(0, 50)}` });
|
|
108
109
|
const result = getResult();
|
|
109
|
-
tag('info').log(`Fisherman answer: ${result.summary}`);
|
|
110
|
+
tag('info').log(`Fisherman answer: ${truncate(result.summary, 200)}`);
|
|
110
111
|
return result;
|
|
111
112
|
}
|
|
112
113
|
async runSession(conversation, tools, opts) {
|
package/dist/src/ai/pilot.d.ts
CHANGED
|
@@ -97,7 +97,6 @@ export declare class Pilot implements Agent {
|
|
|
97
97
|
hasSuccessfulCheckEvidence(currentState: ActionResult, testerConversation: Conversation): boolean;
|
|
98
98
|
formatSuccessfulAssertions(currentState: ActionResult, testerConversation: Conversation): string;
|
|
99
99
|
formatActions(toolCalls: any[]): string;
|
|
100
|
-
buildDeletionScope(task: Test): string;
|
|
101
100
|
getSystemPrompt(task: Test, initialState: ActionResult): string;
|
|
102
101
|
}
|
|
103
102
|
export type SettledStatus = 'passed' | 'failed' | 'unverified' | 'contradiction';
|
package/dist/src/ai/pilot.js
CHANGED
|
@@ -316,28 +316,28 @@ export class Pilot {
|
|
|
316
316
|
return dedent `
|
|
317
317
|
SCENARIO: ${task.scenario}
|
|
318
318
|
|
|
319
|
-
${this.buildDeletionScope(task)}
|
|
320
|
-
|
|
321
319
|
EXPECTED RESULTS (milestones):
|
|
322
320
|
${task.expected.map((e) => `- ${e}`).join('\n')}
|
|
323
321
|
`;
|
|
324
322
|
}
|
|
325
323
|
buildResetSystemPrompt(task) {
|
|
326
324
|
return dedent `
|
|
327
|
-
You are Pilot β decide whether a reset is legitimate. Reset
|
|
328
|
-
iteration's work
|
|
329
|
-
|
|
325
|
+
You are Pilot β decide whether a reset is legitimate. Reset only re-navigates to the start URL:
|
|
326
|
+
it writes nothing, though it abandons this iteration's work and server-side side effects persist.
|
|
327
|
+
The hazard is the tester REDOING a completed flow afterwards β duplicate data and infinite loops.
|
|
330
328
|
|
|
331
329
|
${this.buildSharedEvidenceRules()}
|
|
332
330
|
|
|
333
331
|
DECISION:
|
|
334
|
-
- "allow": current page cannot host the scenario, irrecoverable error,
|
|
335
|
-
|
|
332
|
+
- "allow": current page cannot host the scenario, irrecoverable error, no path back, or an
|
|
333
|
+
expectation requires the outcome to survive a reload or a return to the start page and no
|
|
334
|
+
reset has been taken yet this run β there the reset IS the check, not a redo.
|
|
335
|
+
- "continue": the outcome the scenario needs is already observable on the CURRENT page β verify/finish instead. Provide guidance.
|
|
336
336
|
- "fail": resetCount >= 2 and underlying situation hasn't changed; same flow tried twice with same failure mode.
|
|
337
337
|
- "skipped": feature doesn't exist on this app or prerequisites can't be met.
|
|
338
338
|
|
|
339
339
|
PRIORITY:
|
|
340
|
-
1) Successful side effects in session_log β
|
|
340
|
+
1) Successful side effects in session_log β allow reset only to re-observe them, never to repeat them.
|
|
341
341
|
2) resetCount β each prior reset raises the bar.
|
|
342
342
|
3) Tester's stated reason β weigh against evidence, don't trust blindly.
|
|
343
343
|
|
|
@@ -999,22 +999,6 @@ export class Pilot {
|
|
|
999
999
|
})
|
|
1000
1000
|
.join('\n\n');
|
|
1001
1001
|
}
|
|
1002
|
-
buildDeletionScope(task) {
|
|
1003
|
-
const deletableItems = task.plan
|
|
1004
|
-
? task.plan
|
|
1005
|
-
.listTests()
|
|
1006
|
-
.filter((t) => t.isSuccessful && t.sessionName)
|
|
1007
|
-
.map((t) => t.sessionName)
|
|
1008
|
-
: [];
|
|
1009
|
-
const scenarioLower = task.scenario.toLowerCase();
|
|
1010
|
-
if (deletableItems.length > 0) {
|
|
1011
|
-
return `For deletion scenarios, items can only be deleted if their title contains: ${deletableItems.join(', ')}`;
|
|
1012
|
-
}
|
|
1013
|
-
if (scenarioLower.includes('delete') || scenarioLower.includes('remove')) {
|
|
1014
|
-
return 'No items available for deletion β test should create an item first';
|
|
1015
|
-
}
|
|
1016
|
-
return '';
|
|
1017
|
-
}
|
|
1018
1002
|
getSystemPrompt(task, initialState) {
|
|
1019
1003
|
const interactive = isInteractive();
|
|
1020
1004
|
const stepsText = task.plannedSteps.length > 0 ? task.plannedSteps.map((s, i) => `${i + 1}. ${s}`).join('\n') : 'No planned steps';
|
package/dist/src/ai/planner.d.ts
CHANGED
|
@@ -9,6 +9,7 @@ import { Conversation } from './conversation.js';
|
|
|
9
9
|
import type { Fisherman } from './fisherman.js';
|
|
10
10
|
import type { Provider } from './provider.js';
|
|
11
11
|
import { type Researcher } from './researcher.js';
|
|
12
|
+
import type { Scout } from './scout.js';
|
|
12
13
|
declare const PlannerBase: {
|
|
13
14
|
new (...args: any[]): {
|
|
14
15
|
currentPlan: Plan | null;
|
|
@@ -59,9 +60,12 @@ export declare class Planner extends PlannerBase implements Agent {
|
|
|
59
60
|
lastSuite: Suite | null;
|
|
60
61
|
researcher: Researcher;
|
|
61
62
|
fisherman: Fisherman | null;
|
|
63
|
+
scout: Scout | null;
|
|
62
64
|
constructor(deps: AgentDeps, researcher: Researcher);
|
|
63
65
|
setFisherman(fisherman: Fisherman): void;
|
|
66
|
+
setScout(scout: Scout): void;
|
|
64
67
|
get sectionOrder(): string[];
|
|
68
|
+
get docsWeight(): number;
|
|
65
69
|
getDefaultStartUrl(state: {
|
|
66
70
|
url: string;
|
|
67
71
|
fullUrl?: string;
|
package/dist/src/ai/planner.js
CHANGED
|
@@ -50,6 +50,7 @@ export class Planner extends PlannerBase {
|
|
|
50
50
|
lastSuite = null;
|
|
51
51
|
researcher;
|
|
52
52
|
fisherman = null;
|
|
53
|
+
scout = null;
|
|
53
54
|
constructor(deps, researcher) {
|
|
54
55
|
super();
|
|
55
56
|
this.explorer = deps.explorer;
|
|
@@ -62,9 +63,16 @@ export class Planner extends PlannerBase {
|
|
|
62
63
|
setFisherman(fisherman) {
|
|
63
64
|
this.fisherman = fisherman;
|
|
64
65
|
}
|
|
66
|
+
setScout(scout) {
|
|
67
|
+
this.scout = scout;
|
|
68
|
+
}
|
|
65
69
|
get sectionOrder() {
|
|
66
70
|
return ConfigParser.getInstance().getConfig().ai?.agents?.researcher?.sections || Object.keys(POSSIBLE_SECTIONS);
|
|
67
71
|
}
|
|
72
|
+
get docsWeight() {
|
|
73
|
+
const value = ConfigParser.getInstance().getConfig().ai?.agents?.planner?.docsWeight ?? 70;
|
|
74
|
+
return Math.max(0, Math.min(100, value));
|
|
75
|
+
}
|
|
68
76
|
getDefaultStartUrl(state) {
|
|
69
77
|
return state.fullUrl || state.url;
|
|
70
78
|
}
|
|
@@ -291,6 +299,7 @@ export class Planner extends PlannerBase {
|
|
|
291
299
|
const conversation = new Conversation([], model);
|
|
292
300
|
conversation.autoTrimTag('page_research', 20000);
|
|
293
301
|
conversation.autoTrimTag('tested_scenarios', 10000);
|
|
302
|
+
conversation.autoTrimTag('docs_context', 8000);
|
|
294
303
|
conversation.addUserText(this.getSystemMessage(feature));
|
|
295
304
|
const planningPrompt = dedent `
|
|
296
305
|
<task>
|
|
@@ -345,6 +354,10 @@ export class Planner extends PlannerBase {
|
|
|
345
354
|
const research = await this.researcher.research(currentState || state, {
|
|
346
355
|
deep: true,
|
|
347
356
|
});
|
|
357
|
+
let docsPromise = null;
|
|
358
|
+
if (this.scout && this.docsWeight > 0) {
|
|
359
|
+
docsPromise = this.scout.collectDocs({ url: state.url, title: state.title, feature, excludeUrls: this.knowledgeTracker.applicationSpecUrls(state) });
|
|
360
|
+
}
|
|
348
361
|
let plannerResearch = mdq(research).query('code').replace('');
|
|
349
362
|
plannerResearch = mdq(plannerResearch)
|
|
350
363
|
.query('table')
|
|
@@ -376,6 +389,21 @@ export class Planner extends PlannerBase {
|
|
|
376
389
|
if (applicationContext) {
|
|
377
390
|
conversation.addUserText(applicationContext);
|
|
378
391
|
}
|
|
392
|
+
if (docsPromise) {
|
|
393
|
+
const docs = await docsPromise;
|
|
394
|
+
if (docs) {
|
|
395
|
+
conversation.addUserText(dedent `
|
|
396
|
+
<docs_context>
|
|
397
|
+
Documentation retrieved from the collected corpus by the Scout agent.
|
|
398
|
+
Ground scenarios in these documented capabilities where they apply; treat them as supporting context, not a script.
|
|
399
|
+
|
|
400
|
+
Aim for roughly ${this.docsWeight}% of the scenarios to exercise behavior documented above; the remainder may explore beyond the documentation.
|
|
401
|
+
|
|
402
|
+
${docs}
|
|
403
|
+
</docs_context>
|
|
404
|
+
`);
|
|
405
|
+
}
|
|
406
|
+
}
|
|
379
407
|
conversation.addUserText(dedent `
|
|
380
408
|
${this.buildApproach(style)}
|
|
381
409
|
|
package/dist/src/ai/provider.js
CHANGED
|
@@ -366,7 +366,9 @@ export class Provider {
|
|
|
366
366
|
const modelName = getModelName(model);
|
|
367
367
|
setActivity(`π€ Asking ${modelName} with dynamic tools`, 'ai');
|
|
368
368
|
promptLog(`Using model: ${modelName}`);
|
|
369
|
-
|
|
369
|
+
let toolsWithCommentary = tools;
|
|
370
|
+
if (!tools?.commentary && options.toolChoice !== 'required')
|
|
371
|
+
toolsWithCommentary = { ...tools, commentary: createHarmonyChannelFallbackTool() };
|
|
370
372
|
const toolNames = Object.keys(toolsWithCommentary || {});
|
|
371
373
|
tag('debug').log(`Tools enabled: [${toolNames.join(', ')}]`);
|
|
372
374
|
promptLog('Available tools:', toolNames);
|
package/dist/src/ai/rules.js
CHANGED
|
@@ -6,8 +6,9 @@ const locatorPriorityRule = dedent `
|
|
|
6
6
|
|
|
7
7
|
1. ARIA locators (first choice) - target browser's accessibility tree, most reliable
|
|
8
8
|
Use JSON format: { "role": "button", "text": "Login" }
|
|
9
|
-
Copy role and text VERBATIM from the ARIA snapshot
|
|
10
|
-
|
|
9
|
+
Copy role and text VERBATIM from the ARIA snapshot, UI map, or the page diff that
|
|
10
|
+
reported the element β never guess the pair; a guessed role can silently match a
|
|
11
|
+
different element with the same text. If named nowhere, use text or CSS instead.
|
|
11
12
|
|
|
12
13
|
2. Text locators (second choice) - exact visible text, use only when unique on the page
|
|
13
14
|
Example: 'Login', 'Submit', 'Username'
|
|
@@ -225,10 +226,10 @@ export const unexpectedPopupRule = dedent `
|
|
|
225
226
|
If buttons are disabled unexpectedly, check if a popup is blocking interaction or if required form fields are empty.
|
|
226
227
|
|
|
227
228
|
Dismiss strategy (try in order):
|
|
228
|
-
1. I.
|
|
229
|
-
2. I.
|
|
230
|
-
3. I.click('
|
|
231
|
-
4. I.
|
|
229
|
+
1. I.pressKey('Escape') β press Escape to dismiss
|
|
230
|
+
2. I.click('Cancel') β click Cancel button if present
|
|
231
|
+
3. I.click({ role: 'button', text: 'Close' }) β click X/close button if present
|
|
232
|
+
4. I.clickXY(0, 0) via form() tool and check if page diff changed
|
|
232
233
|
</unexpected_popup_rule>
|
|
233
234
|
`;
|
|
234
235
|
export const sectionContextRule = dedent `
|
|
@@ -318,7 +319,7 @@ export const actionRule = dedent `
|
|
|
318
319
|
Prefer text/ARIA locators with context over complex CSS/XPath selectors.
|
|
319
320
|
For inline create/edit flows, after filling a field verify it contains the value, then confirm using the nearest explicit button/link, an adjacent icon-only confirm control in the same row/form, or Enter if the field remains focused.
|
|
320
321
|
If locator doesn't work, try CSS or XPath locators.
|
|
321
|
-
If nothing works, use
|
|
322
|
+
If nothing works, use visualClick() β it locates the target in a screenshot before clicking it.
|
|
322
323
|
|
|
323
324
|
When a click result reports several matches, pick one from its numbered list by position rather than guessing a new locator.
|
|
324
325
|
Reuse the same locator with step.opts({ elementIndex: N }) as the LAST argument. N is the "Element N" number.
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
export declare function loadScoutCorpus(dirs: string[]): ScoutCorpus;
|
|
2
|
+
export declare function excludeCorpusUrls(corpus: ScoutCorpus, urls: string[]): ScoutCorpus;
|
|
3
|
+
export declare function createScoutTools(corpus: ScoutCorpus): Promise<{
|
|
4
|
+
tools: Record<string, any>;
|
|
5
|
+
scanner: "rg" | "grep";
|
|
6
|
+
getResult: () => string;
|
|
7
|
+
finishFromText: (text?: string) => void;
|
|
8
|
+
}>;
|
|
9
|
+
export interface ScoutCorpus {
|
|
10
|
+
dirs: string[];
|
|
11
|
+
files: ScoutCorpusFile[];
|
|
12
|
+
excludedPaths: string[];
|
|
13
|
+
}
|
|
14
|
+
export interface ScoutCorpusFile {
|
|
15
|
+
path: string;
|
|
16
|
+
url?: string;
|
|
17
|
+
}
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
import { readFileSync } from 'node:fs';
|
|
2
|
+
import { resolve } from 'node:path';
|
|
3
|
+
import { tool } from 'ai';
|
|
4
|
+
import { createBashTool } from 'bash-tool';
|
|
5
|
+
import dedent from 'dedent';
|
|
6
|
+
import { z } from 'zod';
|
|
7
|
+
import { ConfigParser } from "../../config.js";
|
|
8
|
+
import { tag } from "../../utils/logger.js";
|
|
9
|
+
import { loadMarkdownFiles } from "../../utils/markdown-files.js";
|
|
10
|
+
import { readCaptainFile } from "../captain/file-tools.js";
|
|
11
|
+
const MAX_FILES = 500;
|
|
12
|
+
const MAX_FINDINGS = 6000;
|
|
13
|
+
let cachedScanner = null;
|
|
14
|
+
export function loadScoutCorpus(dirs) {
|
|
15
|
+
const files = [];
|
|
16
|
+
for (const dir of dirs) {
|
|
17
|
+
if (files.length >= MAX_FILES) {
|
|
18
|
+
tag('warning').log(`Scout corpus capped at ${MAX_FILES} files β remaining directories skipped`);
|
|
19
|
+
break;
|
|
20
|
+
}
|
|
21
|
+
for (const file of loadMarkdownFiles(dir, { recursive: true })) {
|
|
22
|
+
if (files.length >= MAX_FILES)
|
|
23
|
+
break;
|
|
24
|
+
const entry = { path: file.filePath };
|
|
25
|
+
if (typeof file.data.url === 'string')
|
|
26
|
+
entry.url = file.data.url;
|
|
27
|
+
files.push(entry);
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
return { dirs, files, excludedPaths: [] };
|
|
31
|
+
}
|
|
32
|
+
export function excludeCorpusUrls(corpus, urls) {
|
|
33
|
+
if (urls.length === 0)
|
|
34
|
+
return corpus;
|
|
35
|
+
const excludedUrls = new Set(urls);
|
|
36
|
+
const files = [];
|
|
37
|
+
const excludedPaths = [...corpus.excludedPaths];
|
|
38
|
+
for (const file of corpus.files) {
|
|
39
|
+
if (file.url && excludedUrls.has(file.url)) {
|
|
40
|
+
excludedPaths.push(file.path);
|
|
41
|
+
continue;
|
|
42
|
+
}
|
|
43
|
+
files.push(file);
|
|
44
|
+
}
|
|
45
|
+
return { dirs: corpus.dirs, files, excludedPaths };
|
|
46
|
+
}
|
|
47
|
+
export async function createScoutTools(corpus) {
|
|
48
|
+
const scanner = await detectScanner();
|
|
49
|
+
const projectRoot = ConfigParser.getInstance().getProjectRoot();
|
|
50
|
+
let result = '';
|
|
51
|
+
let searchedOrRead = false;
|
|
52
|
+
const getResult = () => result;
|
|
53
|
+
const finishFromText = (text) => {
|
|
54
|
+
if (text && searchedOrRead)
|
|
55
|
+
result = text.slice(0, MAX_FINDINGS);
|
|
56
|
+
};
|
|
57
|
+
const files = {};
|
|
58
|
+
const readableFiles = new Set();
|
|
59
|
+
for (const file of corpus.files) {
|
|
60
|
+
files[toPosix(file.path)] = readFileSync(file.path, 'utf8');
|
|
61
|
+
readableFiles.add(resolve(file.path));
|
|
62
|
+
}
|
|
63
|
+
const toolkit = await createBashTool({
|
|
64
|
+
destination: '/',
|
|
65
|
+
files,
|
|
66
|
+
maxOutputLength: 20000,
|
|
67
|
+
});
|
|
68
|
+
const bashExecute = toolkit.bash.execute;
|
|
69
|
+
const bash = {
|
|
70
|
+
...toolkit.bash,
|
|
71
|
+
execute: async (input) => {
|
|
72
|
+
tag('step').log(`Scout: bash ${input.command}`);
|
|
73
|
+
searchedOrRead = true;
|
|
74
|
+
return bashExecute?.(input);
|
|
75
|
+
},
|
|
76
|
+
};
|
|
77
|
+
const tools = {
|
|
78
|
+
bash,
|
|
79
|
+
readFile: tool({
|
|
80
|
+
description: dedent `
|
|
81
|
+
Read one documentation file from the corpus.
|
|
82
|
+
Pass the exact path returned by a search result.
|
|
83
|
+
`,
|
|
84
|
+
inputSchema: z.object({
|
|
85
|
+
path: z.string().describe('File path from a search result'),
|
|
86
|
+
startLine: z.number().optional().describe('First line to read, 1-based. Negative values count from the end of the file'),
|
|
87
|
+
endLine: z.number().optional().describe('Last line to read, 1-based and inclusive. Negative values count from the end of the file'),
|
|
88
|
+
maxChars: z.number().optional().describe('Maximum characters to return, default 12000'),
|
|
89
|
+
}),
|
|
90
|
+
execute: async (input) => {
|
|
91
|
+
tag('step').log(`Scout: read ${input.path}`);
|
|
92
|
+
const output = readCaptainFile(projectRoot, input, corpus.dirs);
|
|
93
|
+
if (!output.success)
|
|
94
|
+
return output;
|
|
95
|
+
const resolvedPath = resolve(projectRoot || process.cwd(), output.path);
|
|
96
|
+
if (!readableFiles.has(resolvedPath)) {
|
|
97
|
+
return { success: false, message: 'File is outside the Scout corpus' };
|
|
98
|
+
}
|
|
99
|
+
searchedOrRead = true;
|
|
100
|
+
return output;
|
|
101
|
+
},
|
|
102
|
+
}),
|
|
103
|
+
};
|
|
104
|
+
return { tools, scanner, getResult, finishFromText };
|
|
105
|
+
}
|
|
106
|
+
async function detectScanner() {
|
|
107
|
+
if (cachedScanner)
|
|
108
|
+
return cachedScanner;
|
|
109
|
+
if (await binaryRuns('rg')) {
|
|
110
|
+
cachedScanner = 'rg';
|
|
111
|
+
return cachedScanner;
|
|
112
|
+
}
|
|
113
|
+
if (await binaryRuns('grep')) {
|
|
114
|
+
cachedScanner = 'grep';
|
|
115
|
+
return cachedScanner;
|
|
116
|
+
}
|
|
117
|
+
throw new Error('Scout requires ripgrep or grep on PATH β neither was found');
|
|
118
|
+
}
|
|
119
|
+
async function binaryRuns(binary) {
|
|
120
|
+
try {
|
|
121
|
+
const proc = Bun.spawn([binary, '--version'], { stdout: 'ignore', stderr: 'ignore', stdin: 'ignore' });
|
|
122
|
+
return (await proc.exited) === 0;
|
|
123
|
+
}
|
|
124
|
+
catch {
|
|
125
|
+
return false;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
function toPosix(path) {
|
|
129
|
+
return path.split('\\').join('/');
|
|
130
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { Agent } from './agent.js';
|
|
2
|
+
import type { Provider } from './provider.js';
|
|
3
|
+
import { type ScoutCorpus } from './scout/tools.js';
|
|
4
|
+
export declare class Scout implements Agent {
|
|
5
|
+
provider: Provider;
|
|
6
|
+
corpus: ScoutCorpus;
|
|
7
|
+
emoji: string;
|
|
8
|
+
cache: Map<string, string>;
|
|
9
|
+
constructor(provider: Provider, corpus: ScoutCorpus);
|
|
10
|
+
isAvailable(): boolean;
|
|
11
|
+
collectDocs(query: ScoutQuery): Promise<string>;
|
|
12
|
+
runSession(corpus: ScoutCorpus, query: ScoutQuery): Promise<string | null>;
|
|
13
|
+
buildSystemPrompt(toolNames: string[], corpus: ScoutCorpus, query: ScoutQuery, scanner: 'rg' | 'grep'): string;
|
|
14
|
+
buildTaskPrompt(query: ScoutQuery): string;
|
|
15
|
+
}
|
|
16
|
+
export interface ScoutQuery {
|
|
17
|
+
url?: string;
|
|
18
|
+
title?: string;
|
|
19
|
+
feature?: string;
|
|
20
|
+
excludeUrls: string[];
|
|
21
|
+
}
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
import dedent from 'dedent';
|
|
2
|
+
import { tag } from "../utils/logger.js";
|
|
3
|
+
import { loop } from "../utils/loop.js";
|
|
4
|
+
import { createScoutTools, excludeCorpusUrls } from "./scout/tools.js";
|
|
5
|
+
const MAX_ITERATIONS = 3;
|
|
6
|
+
const MAX_TOOL_ROUNDTRIPS = 5;
|
|
7
|
+
const CACHE_LIMIT = 40;
|
|
8
|
+
const URL_LISTING_LIMIT = 40;
|
|
9
|
+
export class Scout {
|
|
10
|
+
provider;
|
|
11
|
+
corpus;
|
|
12
|
+
emoji = 'π';
|
|
13
|
+
cache = new Map();
|
|
14
|
+
constructor(provider, corpus) {
|
|
15
|
+
this.provider = provider;
|
|
16
|
+
this.corpus = corpus;
|
|
17
|
+
}
|
|
18
|
+
isAvailable() {
|
|
19
|
+
return this.corpus.files.length > 0;
|
|
20
|
+
}
|
|
21
|
+
async collectDocs(query) {
|
|
22
|
+
if (!this.isAvailable())
|
|
23
|
+
return '';
|
|
24
|
+
const cacheKey = `${query.url || ''}|${query.feature || ''}|${query.excludeUrls.join(',')}`;
|
|
25
|
+
const cached = this.cache.get(cacheKey);
|
|
26
|
+
if (cached !== undefined)
|
|
27
|
+
return cached;
|
|
28
|
+
const corpus = excludeCorpusUrls(this.corpus, query.excludeUrls);
|
|
29
|
+
if (corpus.files.length === 0)
|
|
30
|
+
return '';
|
|
31
|
+
const result = await this.runSession(corpus, query);
|
|
32
|
+
if (result === null)
|
|
33
|
+
return '';
|
|
34
|
+
if (this.cache.size > CACHE_LIMIT)
|
|
35
|
+
this.cache.clear();
|
|
36
|
+
this.cache.set(cacheKey, result);
|
|
37
|
+
return result;
|
|
38
|
+
}
|
|
39
|
+
async runSession(corpus, query) {
|
|
40
|
+
const { tools, scanner, getResult, finishFromText } = await createScoutTools(corpus);
|
|
41
|
+
const conversation = this.provider.startConversation(this.buildSystemPrompt(Object.keys(tools), corpus, query, scanner), 'scout', this.provider.getAgenticModel('scout'));
|
|
42
|
+
conversation.addUserText(this.buildTaskPrompt(query));
|
|
43
|
+
tag('info').log(`Scout: collecting documentation for ${query.feature || query.url || 'the current page'}`);
|
|
44
|
+
let failed = false;
|
|
45
|
+
await loop(async ({ stop, iteration }) => {
|
|
46
|
+
const invokeResult = await this.provider.invokeConversation(conversation, tools, {
|
|
47
|
+
maxToolRoundtrips: MAX_TOOL_ROUNDTRIPS,
|
|
48
|
+
agentName: 'scout',
|
|
49
|
+
});
|
|
50
|
+
if (!invokeResult?.toolExecutions?.length) {
|
|
51
|
+
finishFromText(invokeResult?.response?.text);
|
|
52
|
+
stop();
|
|
53
|
+
return;
|
|
54
|
+
}
|
|
55
|
+
if (iteration >= MAX_ITERATIONS) {
|
|
56
|
+
const final = await this.provider.invokeConversation(conversation, undefined, { agentName: 'scout' });
|
|
57
|
+
finishFromText(final?.response?.text);
|
|
58
|
+
stop();
|
|
59
|
+
}
|
|
60
|
+
}, {
|
|
61
|
+
maxAttempts: MAX_ITERATIONS,
|
|
62
|
+
observability: { name: `scout: ${query.feature || query.url || 'docs'}`, agent: 'scout' },
|
|
63
|
+
catch: async ({ error, stop }) => {
|
|
64
|
+
failed = true;
|
|
65
|
+
tag('warning').log(`Scout error: ${error.message}`);
|
|
66
|
+
stop();
|
|
67
|
+
},
|
|
68
|
+
});
|
|
69
|
+
if (failed)
|
|
70
|
+
return null;
|
|
71
|
+
const digest = getResult();
|
|
72
|
+
if (digest) {
|
|
73
|
+
const preview = digest.slice(0, 600);
|
|
74
|
+
const ellipsis = digest.length > 600 ? 'β¦' : '';
|
|
75
|
+
tag('info').log(`Scout digest:\n${preview}${ellipsis}`);
|
|
76
|
+
}
|
|
77
|
+
return digest;
|
|
78
|
+
}
|
|
79
|
+
buildSystemPrompt(toolNames, corpus, query, scanner) {
|
|
80
|
+
const urls = corpus.files.map((file) => file.url).filter(Boolean);
|
|
81
|
+
const urlless = corpus.files.filter((file) => !file.url);
|
|
82
|
+
let pagesListing = '';
|
|
83
|
+
if (urls.length > 0) {
|
|
84
|
+
const listing = urls
|
|
85
|
+
.slice(0, URL_LISTING_LIMIT)
|
|
86
|
+
.map((url) => `- ${url}`)
|
|
87
|
+
.join('\n');
|
|
88
|
+
pagesListing = `Documented pages:\n${listing}`;
|
|
89
|
+
const remaining = urls.length - URL_LISTING_LIMIT;
|
|
90
|
+
if (remaining > 0)
|
|
91
|
+
pagesListing += `\nβ¦and ${remaining} more β find them with ${scanner}`;
|
|
92
|
+
}
|
|
93
|
+
if (urlless.length > 0) {
|
|
94
|
+
const listing = urlless
|
|
95
|
+
.slice(0, URL_LISTING_LIMIT)
|
|
96
|
+
.map((file) => `- ${toPosix(file.path)}`)
|
|
97
|
+
.join('\n');
|
|
98
|
+
pagesListing += `\nFiles with no page URL (hand-written docs):\n${listing}`;
|
|
99
|
+
}
|
|
100
|
+
const prompt = dedent `
|
|
101
|
+
You are Scout β a documentation retrieval agent. You find collected documentation relevant to a testing focus and report it for test planning.
|
|
102
|
+
|
|
103
|
+
You never see the application itself. The documentation corpus is your only source of truth.
|
|
104
|
+
|
|
105
|
+
CORPUS:
|
|
106
|
+
${corpus.files.length} markdown files under:
|
|
107
|
+
- ${corpus.dirs.map(toPosix).join('\n- ')}
|
|
108
|
+
${pagesListing}
|
|
109
|
+
|
|
110
|
+
These pages are already provided to the planner in full β do not re-report them:
|
|
111
|
+
${query.excludeUrls.map((url) => `- ${url}`).join('\n') || '- none'}
|
|
112
|
+
|
|
113
|
+
AVAILABLE TOOLS:
|
|
114
|
+
${toolNames.join(', ')}.
|
|
115
|
+
Use tool names exactly as listed. Do not invent aliases or combined names.
|
|
116
|
+
Match each tool input schema exactly. Do not invent parameter names or pass extra fields.
|
|
117
|
+
|
|
118
|
+
SCANNER:
|
|
119
|
+
${scanner} is the search command. Scan the working directory through bash() β explore freely, pipelines, globs and repeated searches are fine. Read files with readFile().
|
|
120
|
+
|
|
121
|
+
WORKFLOW:
|
|
122
|
+
1. Scan with ${scanner} using plain prose words from the focus β feature names, page purposes, capabilities
|
|
123
|
+
2. Read the files whose hits look most relevant
|
|
124
|
+
3. Report the digest as your final message β no tool call is needed to finish
|
|
125
|
+
|
|
126
|
+
RULES:
|
|
127
|
+
- Report only what the documentation states. Never fill gaps with assumptions about the application
|
|
128
|
+
- Keep verified capabilities and unverified possibilities distinguishable, the way the documentation marks them
|
|
129
|
+
- Name the page URL each item belongs to, so scenarios anchor to real routes
|
|
130
|
+
- Explore briefly: a few scans and reads are enough, then report
|
|
131
|
+
- A short accurate digest beats a long loose one; reporting that nothing relevant exists is a valid answer
|
|
132
|
+
`;
|
|
133
|
+
const customPrompt = this.provider.getSystemPromptForAgent('scout');
|
|
134
|
+
if (customPrompt)
|
|
135
|
+
return `${prompt}\n\n${customPrompt}`;
|
|
136
|
+
return prompt;
|
|
137
|
+
}
|
|
138
|
+
buildTaskPrompt(query) {
|
|
139
|
+
return dedent `
|
|
140
|
+
Page URL: ${query.url || 'Unknown'}
|
|
141
|
+
Page title: ${query.title || 'Unknown'}
|
|
142
|
+
Focus: ${query.feature || 'the page as a whole'}
|
|
143
|
+
|
|
144
|
+
Report the documented capabilities, states and transitions a test planner could turn into scenarios.
|
|
145
|
+
`;
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
function toPosix(path) {
|
|
149
|
+
return path.split('\\').join('/');
|
|
150
|
+
}
|