explorbot 0.4.10 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. package/README.md +4 -1
  2. package/bin/mdq.ts +18 -0
  3. package/boat/api-tester/src/ai/chief.ts +72 -0
  4. package/boat/api-tester/src/api-client.ts +37 -0
  5. package/boat/prima/src/prima.ts +41 -2
  6. package/dist/bin/mdq.js +19 -0
  7. package/dist/boat/api-tester/src/ai/chief.js +69 -0
  8. package/dist/boat/api-tester/src/api-client.js +26 -0
  9. package/dist/boat/prima/src/prima.js +42 -2
  10. package/dist/package.json +3 -2
  11. package/dist/src/ai/agent.d.ts +3 -1
  12. package/dist/src/ai/judge-provider.d.ts +17 -0
  13. package/dist/src/ai/judge-provider.js +56 -0
  14. package/dist/src/ai/judge-tool.d.ts +2 -0
  15. package/dist/src/ai/judge-tool.js +33 -0
  16. package/dist/src/ai/judge.d.ts +28 -0
  17. package/dist/src/ai/judge.js +71 -0
  18. package/dist/src/ai/navigator.d.ts +3 -0
  19. package/dist/src/ai/navigator.js +13 -4
  20. package/dist/src/ai/pilot.d.ts +4 -0
  21. package/dist/src/ai/pilot.js +57 -5
  22. package/dist/src/ai/planner.js +9 -6
  23. package/dist/src/ai/provider.d.ts +4 -1
  24. package/dist/src/ai/provider.js +52 -7
  25. package/dist/src/ai/researcher/deep-analysis.js +2 -2
  26. package/dist/src/ai/researcher/locators.js +2 -2
  27. package/dist/src/ai/researcher/pagination.js +1 -1
  28. package/dist/src/ai/researcher/research-result.js +2 -2
  29. package/dist/src/ai/researcher.js +1 -1
  30. package/dist/src/ai/task-agent.d.ts +2 -0
  31. package/dist/src/ai/task-agent.js +3 -1
  32. package/dist/src/ai/tester.js +19 -15
  33. package/dist/src/ai/tools.d.ts +4 -3
  34. package/dist/src/ai/tools.js +35 -7
  35. package/dist/src/api/request-result.js +2 -1
  36. package/dist/src/command-handler.d.ts +1 -0
  37. package/dist/src/command-handler.js +24 -3
  38. package/dist/src/commands/base-command.d.ts +5 -0
  39. package/dist/src/commands/base-command.js +3 -0
  40. package/dist/src/commands/explore-command.d.ts +2 -1
  41. package/dist/src/commands/explore-command.js +12 -1
  42. package/dist/src/commands/freesail-command.js +8 -2
  43. package/dist/src/commands/init-command.js +1 -1
  44. package/dist/src/commands/navigate-command.d.ts +2 -1
  45. package/dist/src/commands/navigate-command.js +6 -0
  46. package/dist/src/commands/plan-load-command.d.ts +2 -1
  47. package/dist/src/commands/plan-load-command.js +4 -0
  48. package/dist/src/commands/plans-command.d.ts +3 -9
  49. package/dist/src/commands/plans-command.js +11 -21
  50. package/dist/src/commands/rerun-command.d.ts +2 -1
  51. package/dist/src/commands/rerun-command.js +5 -1
  52. package/dist/src/commands/research-command.d.ts +2 -1
  53. package/dist/src/commands/research-command.js +6 -0
  54. package/dist/src/commands/test-command.d.ts +2 -1
  55. package/dist/src/commands/test-command.js +4 -1
  56. package/dist/src/components/Autocomplete.js +26 -12
  57. package/dist/src/components/InputReadline.js +10 -1
  58. package/dist/src/config.d.ts +6 -0
  59. package/dist/src/experience-tracker.js +4 -3
  60. package/dist/src/explorbot.d.ts +3 -0
  61. package/dist/src/explorbot.js +8 -0
  62. package/dist/src/knowledge-tracker.js +1 -1
  63. package/dist/src/state-manager.d.ts +2 -0
  64. package/dist/src/state-manager.js +16 -0
  65. package/dist/src/test-plan.d.ts +11 -0
  66. package/dist/src/test-plan.js +54 -2
  67. package/dist/src/utils/aria-ref.js +1 -1
  68. package/dist/src/utils/logger.js +8 -2
  69. package/dist/src/utils/markdown-query.d.ts +1 -48
  70. package/dist/src/utils/markdown-query.js +1 -444
  71. package/dist/src/utils/mdq/cli.d.ts +6 -0
  72. package/dist/src/utils/mdq/cli.js +122 -0
  73. package/dist/src/utils/mdq/edit.d.ts +24 -0
  74. package/dist/src/utils/mdq/edit.js +147 -0
  75. package/dist/src/utils/mdq/query.d.ts +118 -0
  76. package/dist/src/utils/mdq/query.js +451 -0
  77. package/dist/src/utils/strings.d.ts +1 -0
  78. package/dist/src/utils/strings.js +7 -0
  79. package/dist/src/utils/test-files.d.ts +1 -0
  80. package/dist/src/utils/test-files.js +5 -2
  81. package/docs/api-testing/planning.md +1 -1
  82. package/docs/superpowers/plans/2026-09-15-mdq-package.md +130 -94
  83. package/docs/superpowers/specs/2026-09-18-judge-decision-model-design.md +79 -0
  84. package/package.json +3 -2
  85. package/src/ai/agent.ts +3 -1
  86. package/src/ai/judge-provider.ts +62 -0
  87. package/src/ai/judge-tool.ts +35 -0
  88. package/src/ai/judge.ts +75 -0
  89. package/src/ai/navigator.ts +15 -4
  90. package/src/ai/pilot.ts +58 -5
  91. package/src/ai/planner.ts +9 -6
  92. package/src/ai/provider.ts +51 -7
  93. package/src/ai/researcher/deep-analysis.ts +2 -2
  94. package/src/ai/researcher/locators.ts +2 -2
  95. package/src/ai/researcher/pagination.ts +1 -1
  96. package/src/ai/researcher/research-result.ts +2 -2
  97. package/src/ai/researcher.ts +1 -1
  98. package/src/ai/task-agent.ts +4 -1
  99. package/src/ai/tester.ts +19 -16
  100. package/src/ai/tools.ts +42 -7
  101. package/src/api/request-result.ts +2 -1
  102. package/src/command-handler.ts +28 -3
  103. package/src/commands/base-command.ts +9 -0
  104. package/src/commands/explore-command.ts +15 -2
  105. package/src/commands/freesail-command.ts +9 -2
  106. package/src/commands/init-command.ts +1 -1
  107. package/src/commands/navigate-command.ts +8 -1
  108. package/src/commands/plan-load-command.ts +6 -1
  109. package/src/commands/plans-command.ts +13 -29
  110. package/src/commands/rerun-command.ts +7 -2
  111. package/src/commands/research-command.ts +8 -1
  112. package/src/commands/test-command.ts +6 -2
  113. package/src/components/Autocomplete.tsx +39 -10
  114. package/src/components/InputReadline.tsx +10 -1
  115. package/src/config.ts +1 -0
  116. package/src/experience-tracker.ts +4 -3
  117. package/src/explorbot.ts +8 -0
  118. package/src/knowledge-tracker.ts +1 -1
  119. package/src/state-manager.ts +16 -0
  120. package/src/test-plan.ts +67 -2
  121. package/src/utils/aria-ref.ts +1 -1
  122. package/src/utils/logger.ts +6 -1
  123. package/src/utils/markdown-query.ts +1 -519
  124. package/src/utils/mdq/cli.ts +118 -0
  125. package/src/utils/mdq/edit.ts +158 -0
  126. package/src/utils/mdq/query.ts +556 -0
  127. package/src/utils/strings.ts +7 -0
  128. package/src/utils/test-files.ts +5 -2
@@ -3,7 +3,7 @@ import { LangfuseSpanProcessor } from '@langfuse/otel';
3
3
  import { NodeSDK } from '@opentelemetry/sdk-node';
4
4
  import { AsyncLocalStorage } from 'node:async_hooks';
5
5
  import dedent from 'dedent';
6
- import { APICallError, generateObject, generateText, isStepCount, registerTelemetry, tool } from 'ai';
6
+ import { APICallError, NoObjectGeneratedError, asSchema, extractJsonMiddleware, generateObject, generateText, isStepCount, parsePartialJson, registerTelemetry, tool, wrapLanguageModel } from 'ai';
7
7
  import type { ModelMessage } from 'ai';
8
8
  import { z } from 'zod';
9
9
  import { clearActivity, setActivity } from '../activity.ts';
@@ -60,11 +60,11 @@ function extractCachedTokens(usage: any): number {
60
60
  return usage?.inputTokenDetails?.cacheReadTokens ?? 0;
61
61
  }
62
62
 
63
- function abortAfterIdle(ms: number, cancel: { cancelled: boolean }, controller: AbortController): Promise<never> {
63
+ function abortAfterIdle(ms: number, cancel: { cancelled: boolean }, controller: AbortController, busy: { tools: number }): Promise<never> {
64
64
  return new Promise((_, reject) => {
65
65
  const tick = () => {
66
66
  if (cancel.cancelled) return;
67
- if (executionController.isAwaitingInput()) {
67
+ if (executionController.isAwaitingInput() || busy.tools > 0) {
68
68
  setTimeout(tick, ms);
69
69
  return;
70
70
  }
@@ -87,7 +87,7 @@ export class Provider {
87
87
  private otelSdk: NodeSDK | null = null;
88
88
  private defaultRetryOptions: RetryOptions = {
89
89
  maxAttempts: 3,
90
- baseDelay: 10,
90
+ baseDelay: 1000,
91
91
  maxDelay: 10000,
92
92
  retryCondition: (error: Error) => {
93
93
  return (
@@ -251,12 +251,12 @@ export class Provider {
251
251
  });
252
252
  }
253
253
 
254
- private async raceWithIdleTimeout<T>(fn: (signal: AbortSignal) => Promise<T>, timeoutMs: number): Promise<T> {
254
+ private async raceWithIdleTimeout<T>(fn: (signal: AbortSignal) => Promise<T>, timeoutMs: number, busy: { tools: number } = { tools: 0 }): Promise<T> {
255
255
  const cancel = { cancelled: false };
256
256
  const controller = new AbortController();
257
257
  const combinedSignal = combinedAbortSignal(controller);
258
258
  try {
259
- return await Promise.race([fn(combinedSignal), abortAfterIdle(timeoutMs, cancel, controller)]);
259
+ return await Promise.race([fn(combinedSignal), abortAfterIdle(timeoutMs, cancel, controller, busy)]);
260
260
  } finally {
261
261
  cancel.cancelled = true;
262
262
  }
@@ -269,6 +269,24 @@ export class Provider {
269
269
  return retry(reduced.messages, { ...options, _contextRetryLevel: reduced.nextLevel });
270
270
  }
271
271
 
272
+ private async recoverWithPlainJson(messages: ModelMessage[], schema: any, model: any, options: any): Promise<any> {
273
+ const target = asSchema(schema);
274
+ tag('warning').log(`${getModelName(model)} returned no structured output, asking for plain JSON instead`);
275
+ const instruction = dedent`
276
+ Respond with a single JSON object that matches this JSON Schema, and nothing else:
277
+ ${JSON.stringify(await target.jsonSchema)}
278
+ `;
279
+ const response = await this.chat([...messages, { role: 'user', content: instruction }], wrapLanguageModel({ model, middleware: extractJsonMiddleware() }), options);
280
+
281
+ const parsed = await parsePartialJson(response.text);
282
+ if (parsed.state !== 'successful-parse') throw new AiError('No object generated: plain JSON fallback returned no parsable JSON');
283
+ const validated = await target.validate?.(parsed.value);
284
+ if (validated && !validated.success) throw new AiError(`No object generated: plain JSON fallback did not match the schema: ${validated.error.message}`);
285
+
286
+ responseLog(parsed.value);
287
+ return { ...response, object: validated?.value ?? parsed.value };
288
+ }
289
+
272
290
  private initLangfuse() {
273
291
  const { enabled, publicKey, secretKey, baseUrl } = this.config.langfuse || {};
274
292
 
@@ -411,6 +429,8 @@ export class Provider {
411
429
 
412
430
  async generateWithTools(messages: ModelMessage[], model: any, tools: any, options: any = {}): Promise<any> {
413
431
  const modelName = getModelName(model);
432
+ const busy = { tools: 0 };
433
+ tools = withIdleExemption(tools, busy);
414
434
  setActivity(`🤖 Asking ${modelName} with dynamic tools`, 'ai');
415
435
  promptLog(`Using model: ${modelName}`);
416
436
 
@@ -437,7 +457,7 @@ export class Provider {
437
457
  const onStepEnd = (step: any) => {
438
458
  stepMessages.push(...(step.response?.messages || []));
439
459
  };
440
- const result = (await this.raceWithIdleTimeout((signal) => generateText({ messages: attemptMessages, ...config, abortSignal: signal, onStepEnd }), config.timeout || 30000).catch((error) => {
460
+ const result = (await this.raceWithIdleTimeout((signal) => generateText({ messages: attemptMessages, ...config, abortSignal: signal, onStepEnd }), config.timeout || 30000, busy).catch((error) => {
441
461
  if (stepMessages.length > 0) {
442
462
  tag('warning').log(`Keeping ${stepMessages.length} messages from tool steps that already ran before the failure`);
443
463
  executedStepMessages.push(...stepMessages);
@@ -527,6 +547,7 @@ export class Provider {
527
547
  if (Provider.isContextLengthError(error)) {
528
548
  return this.recoverFromContextLength(error, messages, options, (m, o) => this.generateObject(m, schema, model, o));
529
549
  }
550
+ if (NoObjectGeneratedError.isInstance(error)) return this.recoverWithPlainJson(messages, schema, modelToUse, options);
530
551
  throw new AiError(error.message || error.toString());
531
552
  }
532
553
  }
@@ -769,6 +790,29 @@ function repairHarmonyChannel({ toolCall, tools }: ToolCallRepairOptions): any |
769
790
  return { ...toolCall, toolName: NARRATION_TOOL, input };
770
791
  }
771
792
 
793
+ function withIdleExemption(tools: any, busy: { tools: number }): any {
794
+ if (!tools) return tools;
795
+ const wrapped: any = {};
796
+ for (const [name, definition] of Object.entries<any>(tools)) {
797
+ if (typeof definition?.execute !== 'function') {
798
+ wrapped[name] = definition;
799
+ continue;
800
+ }
801
+ wrapped[name] = {
802
+ ...definition,
803
+ execute: async (...args: any[]) => {
804
+ busy.tools++;
805
+ try {
806
+ return await definition.execute(...args);
807
+ } finally {
808
+ busy.tools--;
809
+ }
810
+ },
811
+ };
812
+ }
813
+ return wrapped;
814
+ }
815
+
772
816
  export { AiError, Provider as AIProvider };
773
817
 
774
818
  type ToolCallRepairOptions = { toolCall: any; tools: any };
@@ -126,7 +126,7 @@ export function WithDeepAnalysis<T extends Constructor>(Base: T) {
126
126
  let updated: string;
127
127
  if (extQuery.count() > 0) {
128
128
  const existing = extQuery.text().trimEnd();
129
- updated = extQuery.replace(`${existing}\n\n${sectionMarkdown}\n`);
129
+ updated = extQuery.replace(`${existing}\n\n${sectionMarkdown}\n`).toString();
130
130
  } else {
131
131
  updated = `${cached.trimEnd()}\n\n# Extended Research\n\n${sectionMarkdown}\n`;
132
132
  }
@@ -539,7 +539,7 @@ export function WithDeepAnalysis<T extends Constructor>(Base: T) {
539
539
  if (heading.count() === 0) heading = mdq(sectionMarkdown).query('h2[0]');
540
540
  if (heading.count() === 0) return sectionMarkdown;
541
541
 
542
- return heading.replace(`${heading.text().trimEnd()}\n\n> Container: '${containerCss}'\n\n`);
542
+ return heading.replace(`${heading.text().trimEnd()}\n\n> Container: '${containerCss}'\n\n`).toString();
543
543
  }
544
544
 
545
545
  private _deduplicateExpandedSections(sections: string[]): string[] {
@@ -304,9 +304,9 @@ export function WithLocators<T extends Constructor>(Base: T) {
304
304
  if (sectionQuery.count() === 0) sectionQuery = mdq(result.text).query(`section3(~"${escaped}")`);
305
305
 
306
306
  if (newCss) {
307
- result.text = sectionQuery.query('blockquote[0]').setKeyValue('Container', `'${newCss}'`);
307
+ result.text = sectionQuery.query('blockquote[0]').setKeyValue('Container', `'${newCss}'`).toString();
308
308
  } else {
309
- result.text = sectionQuery.query('blockquote[0]').replace('');
309
+ result.text = sectionQuery.query('blockquote[0]').replace('').toString();
310
310
  result.text = result.text.replace(`${FOCUSED_MARKER}\n`, '');
311
311
  }
312
312
 
@@ -58,7 +58,7 @@ export function WithPagination<T extends Constructor>(Base: T) {
58
58
  let sectionQuery = mdq(result.text).query(`section2(~"${escaped}")`);
59
59
  if (sectionQuery.count() === 0) sectionQuery = mdq(result.text).query(`section3(~"${escaped}")`);
60
60
  if (sectionQuery.count() === 0) return;
61
- result.text = sectionQuery.query('blockquote[0]').setKeyValue('Pagination', strategy);
61
+ result.text = sectionQuery.query('blockquote[0]').setKeyValue('Pagination', strategy).toString();
62
62
  }
63
63
  };
64
64
  }
@@ -52,9 +52,9 @@ export class ResearchResult {
52
52
  const escaped = section.name.replace(/"/g, '\\"');
53
53
  let sectionQuery = mdq(this.text).query(`section2(~"${escaped}")`);
54
54
  if (sectionQuery.count() === 0) sectionQuery = mdq(this.text).query(`section3(~"${escaped}")`);
55
- const updated = sectionQuery.query('table').replace(`${newTable.trimEnd()}\n`);
55
+ const updated = sectionQuery.query('table').replace(`${newTable.trimEnd()}\n`).toString();
56
56
  if (updated === this.text) return;
57
- section.rawMarkdown = mdq(section.rawMarkdown).query('table').replace(`${newTable.trimEnd()}\n`);
57
+ section.rawMarkdown = mdq(section.rawMarkdown).query('table').replace(`${newTable.trimEnd()}\n`).toString();
58
58
  this.text = updated;
59
59
  }
60
60
 
@@ -313,7 +313,7 @@ export class Researcher extends ResearcherBase implements Agent {
313
313
  researchFile = saveResearch(researchState, result.text, combinedHtml);
314
314
  }
315
315
 
316
- const summaryText = mdq(result.text).query('section2(/^summary/)').query('paragraph[0]').text().trim();
316
+ const summaryText = mdq(result.text).query('section2(/^summary/i)').query('paragraph[0]').text().trim();
317
317
  const summaryLine = summaryText.split('\n')[0]?.trim().slice(0, 200);
318
318
  if (summaryLine) this.experienceTracker.updateSummary(this.actionResult!, summaryLine);
319
319
 
@@ -8,6 +8,7 @@ import type { StateManager } from '../state-manager.ts';
8
8
  import { HooksRunner } from '../utils/hooks-runner.ts';
9
9
  import type { AgentDeps, ToolDeps } from './agent.ts';
10
10
  import { Historian } from './historian.js';
11
+ import type { Judge } from './judge.ts';
11
12
  import type { Navigator } from './navigator.js';
12
13
  import type { Provider } from './provider.js';
13
14
  import { Quartermaster } from './quartermaster.js';
@@ -29,6 +30,7 @@ export abstract class TaskAgent {
29
30
  config!: ExplorbotConfig;
30
31
  stateManager!: StateManager;
31
32
  knowledgeTracker!: KnowledgeTracker;
33
+ protected judge?: Judge;
32
34
  protected hooksRunner!: HooksRunner;
33
35
  protected consecutiveFailures = 0;
34
36
  protected consecutiveEmptyResults = 0;
@@ -45,6 +47,7 @@ export abstract class TaskAgent {
45
47
  this.config = deps.config;
46
48
  this.stateManager = deps.stateManager;
47
49
  this.knowledgeTracker = deps.knowledgeTracker;
50
+ this.judge = deps.judge;
48
51
  this.hooksRunner = new HooksRunner(deps.explorer, deps.config);
49
52
  }
50
53
 
@@ -59,7 +62,7 @@ export abstract class TaskAgent {
59
62
  protected abstract getNavigator(): Navigator;
60
63
 
61
64
  protected get toolDeps(): ToolDeps {
62
- return { explorer: this.explorer, stateManager: this.stateManager, ai: this.provider };
65
+ return { explorer: this.explorer, stateManager: this.stateManager, ai: this.provider, judge: this.judge };
63
66
  }
64
67
 
65
68
  protected getExperienceTracker(): ExperienceTracker {
package/src/ai/tester.ts CHANGED
@@ -136,8 +136,8 @@ export class Tester extends TaskAgent implements Agent {
136
136
  let initialState = ActionResult.fromState(state);
137
137
  const currentUrl = state.fullUrl || state.url;
138
138
  let startOnCurrentPage = opts.startOnCurrentPage;
139
- if (isErrorPage(initialState) && !startOnCurrentPage && task.startUrl && normalizeUrl(currentUrl) !== normalizeUrl(task.startUrl)) {
140
- debugLog(`Recovering from error page at ${currentUrl} by navigating to ${task.startUrl}`);
139
+ if (!startOnCurrentPage && task.startUrl && normalizeUrl(currentUrl) !== normalizeUrl(task.startUrl)) {
140
+ debugLog(`Opening test start URL ${task.startUrl} before building context (was at ${currentUrl})`);
141
141
  try {
142
142
  await this.explorer.visit(task.startUrl);
143
143
  state = this.stateManager.getCurrentState();
@@ -145,7 +145,7 @@ export class Tester extends TaskAgent implements Agent {
145
145
  initialState = ActionResult.fromState(state);
146
146
  startOnCurrentPage = true;
147
147
  } catch (error) {
148
- debugLog(`Could not recover from error page: ${compactErrorMessage(error)}`);
148
+ debugLog(`Could not open test start URL: ${compactErrorMessage(error)}`);
149
149
  }
150
150
  }
151
151
  if (isErrorPage(initialState)) {
@@ -159,14 +159,6 @@ export class Tester extends TaskAgent implements Agent {
159
159
  conversation.markLastMessageCacheable();
160
160
  this.currentConversation = conversation;
161
161
 
162
- const scenarioBlock = this.buildScenarioBlock(task, initialState);
163
- conversation.addUserText(scenarioBlock);
164
- conversation.markLastMessageCacheable();
165
- conversation.protectPrefix(conversation.messages.length);
166
-
167
- const pageContext = await this.reinjectContextIfNeeded(1, initialState);
168
- if (pageContext) conversation.addUserText(pageContext);
169
-
170
162
  return await Observability.run(
171
163
  `test: ${task.scenario}`,
172
164
  {
@@ -185,16 +177,15 @@ export class Tester extends TaskAgent implements Agent {
185
177
  private async runTestSession(task: Test, initialState: ActionResult, conversation: Conversation, handlers: TestSessionHandlers, opts: TestOptions): Promise<{ success: boolean }> {
186
178
  const { offFailedRequest } = handlers;
187
179
 
180
+ let plan = '';
188
181
  if (this.pilot) {
189
182
  try {
190
- const plan = await this.pilot.planTest(task, initialState);
183
+ await this.researcher.research(initialState).catch(this.skipResearch);
184
+ plan = await this.pilot.planTest(task, initialState);
191
185
  if (task.hasFinished) {
192
186
  offFailedRequest?.();
193
187
  return { success: task.isSuccessful };
194
188
  }
195
- if (plan) {
196
- conversation.addUserText(`Pilot's test plan:\n${plan}\n\nFollow this plan while executing the test.`);
197
- }
198
189
  } catch (err) {
199
190
  const message = err instanceof Error ? err.message : String(err);
200
191
  tag('error').log(`Pilot planning failed: ${message}`);
@@ -216,7 +207,7 @@ export class Tester extends TaskAgent implements Agent {
216
207
 
217
208
  if (opts.startOnCurrentPage) debugLog(`Starting on the page already open at ${task.startUrl}`);
218
209
 
219
- if (!opts.startOnCurrentPage) {
210
+ if (!opts.startOnCurrentPage || task.preparedData.length > 0) {
220
211
  debugLog(`Navigating to ${task.startUrl}`);
221
212
  try {
222
213
  await this.explorer.visit(task.startUrl!);
@@ -242,6 +233,15 @@ export class Tester extends TaskAgent implements Agent {
242
233
  const currentUrl = startState?.url || task.startUrl || '';
243
234
  await this.hooksRunner.runBeforeHook('tester', currentUrl);
244
235
 
236
+ const testerState = this.getCurrentState();
237
+ conversation.addUserText(this.buildScenarioBlock(task, testerState));
238
+ conversation.markLastMessageCacheable();
239
+ conversation.protectPrefix(conversation.messages.length);
240
+
241
+ const pageContext = await this.reinjectContextIfNeeded(1, testerState);
242
+ if (pageContext) conversation.addUserText(pageContext);
243
+ if (plan) conversation.addUserText(`Pilot's test plan:\n${plan}\n\nFollow this plan while executing the test.`);
244
+
245
245
  const offStateChange = this.stateManager.onStateChange((event: StateTransition) => {
246
246
  if (task.hasFinished) return;
247
247
  if (event.toState?.url === event.fromState?.url) return;
@@ -912,6 +912,9 @@ export class Tester extends TaskAgent implements Agent {
912
912
  }
913
913
 
914
914
  private buildDeletionScope(task: Test): string {
915
+ if (task.preparedData.length > 0) {
916
+ return `When deleting items, ONLY delete the data prepared for this test: ${task.preparedData.join('; ')}.`;
917
+ }
915
918
  const deletableItems = this.getDeletableSessionNames(task);
916
919
  if (deletableItems.length > 0) {
917
920
  return `When deleting items, ONLY delete items whose title contains one of these session names: ${deletableItems.join(', ')}. These were created by previous tests.`;
package/src/ai/tools.ts CHANGED
@@ -15,6 +15,8 @@ import { pause } from '../utils/loop.js';
15
15
  import { compactErrorMessage, normalizeInlineText, truncate } from '../utils/strings.ts';
16
16
  import { WebElement } from '../utils/web-element.ts';
17
17
  import type { ToolDeps } from './agent.ts';
18
+ import { createJudgeTool } from './judge-tool.ts';
19
+ import { JUDGE_PAGE_CAP, type Judge, UNDECIDED } from './judge.ts';
18
20
  import { Navigator } from './navigator.ts';
19
21
  import { Researcher } from './researcher.ts';
20
22
  import { sectionContextRule } from './rules.ts';
@@ -31,7 +33,7 @@ interface AgentToolDeps extends ToolDeps {
31
33
 
32
34
  export const ASSERTION_TOOLS = ['verify'] as const;
33
35
 
34
- export function createCodeceptJSTools({ explorer, stateManager }: ToolDeps, task: Task) {
36
+ export function createCodeceptJSTools({ explorer, stateManager, judge }: ToolDeps, task: Task) {
35
37
  return {
36
38
  click: tool({
37
39
  description: dedent`
@@ -156,7 +158,9 @@ export function createCodeceptJSTools({ explorer, stateManager }: ToolDeps, task
156
158
  attempts,
157
159
  suggestion,
158
160
  },
159
- ambiguityError || action.lastError
161
+ ambiguityError || action.lastError,
162
+ judge,
163
+ explanation
160
164
  );
161
165
  },
162
166
  }),
@@ -232,7 +236,9 @@ export function createCodeceptJSTools({ explorer, stateManager }: ToolDeps, task
232
236
  attempts,
233
237
  suggestion: 'Use xpathCheck() to locate the row/card/tree node, or visualClick() if the hover target is only visually identifiable.',
234
238
  },
235
- action.lastError
239
+ action.lastError,
240
+ judge,
241
+ explanation
236
242
  );
237
243
  },
238
244
  }),
@@ -375,7 +381,7 @@ export function createCodeceptJSTools({ explorer, stateManager }: ToolDeps, task
375
381
  - Performing multiple form actions in a single batch
376
382
  - Complex interactions requiring sequential commands
377
383
  - Reaching items further down a list (I.scrollTo)
378
- - Reloading the page to prove a change outlived it (I.reloadPage)
384
+ - Reloading the page to prove a change outlived it (I.refreshPage)
379
385
 
380
386
  Example - filling a form with context (PREFERRED):
381
387
  I.fillField('Username', 'John', '.login-form')
@@ -437,7 +443,9 @@ export function createCodeceptJSTools({ explorer, stateManager }: ToolDeps, task
437
443
  attempts: action.executedSteps,
438
444
  suggestion: formSuggestion,
439
445
  },
440
- action.lastError
446
+ action.lastError,
447
+ judge,
448
+ explanation
441
449
  );
442
450
  }
443
451
 
@@ -587,7 +595,7 @@ export function createLearnExperienceTool({ getExperienceTracker, getState }: {
587
595
  });
588
596
  }
589
597
 
590
- export function createAgentTools({ explorer, stateManager, ai, researcher, navigator, supervisor, withExperience }: AgentToolDeps): any {
598
+ export function createAgentTools({ explorer, stateManager, ai, judge, researcher, navigator, supervisor, withExperience }: AgentToolDeps): any {
591
599
  const tools: Record<string, any> = {
592
600
  see: tool({
593
601
  description: dedent`
@@ -724,6 +732,13 @@ export function createAgentTools({ explorer, stateManager, ai, researcher, navig
724
732
  }
725
733
 
726
734
  if (result.inexpressible) {
735
+ if (result.judged?.approved) {
736
+ return failedToolResult('verify', `No assertion could express this claim, but the page appears to confirm it: ${assertion}`, {
737
+ inexpressible: true,
738
+ suggestion: 'This is a judgement about the page, not an assertion that ran in the browser — a hint, not proof. Restate the claim in terms of what is visible or of a control state to get a real assertion.',
739
+ });
740
+ }
741
+
727
742
  return failedToolResult('verify', `No assertion could express this claim: ${assertion}`, {
728
743
  inexpressible: true,
729
744
  suggestion: 'This is not evidence the page is wrong — the claim could not be turned into an assertion. Restate it in terms of what is visible or of a control state, or check it with see().',
@@ -1143,6 +1158,20 @@ export function createAgentTools({ explorer, stateManager, ai, researcher, navig
1143
1158
  });
1144
1159
  }
1145
1160
 
1161
+ const buildJudgeState = async () => {
1162
+ const activeTest = explorer.activeTest;
1163
+ const state = stateManager.getCurrentState();
1164
+ const result = state ? ActionResult.fromState(state) : null;
1165
+ return {
1166
+ task: activeTest?.scenario || '',
1167
+ page: cap(result?.getCompactARIA(), JUDGE_PAGE_CAP),
1168
+ recentActions: Object.values(activeTest?.steps || {})
1169
+ .slice(-JUDGE_RECENT_ACTIONS_LIMIT)
1170
+ .map((step) => step.text),
1171
+ };
1172
+ };
1173
+ Object.assign(tools, createJudgeTool({ explorer, stateManager, ai, judge }, buildJudgeState));
1174
+
1146
1175
  withdrawVisionTools(tools);
1147
1176
 
1148
1177
  return tools;
@@ -1158,6 +1187,7 @@ const NAVIGATED_SUGGESTION = 'The action left the page. Elements are never compa
1158
1187
  const ARIA_OUTPUT_CAP = 4000;
1159
1188
  const HTML_OUTPUT_CAP = 6000;
1160
1189
  const ANALYSIS_OUTPUT_CAP = 2000;
1190
+ const JUDGE_RECENT_ACTIONS_LIMIT = 8;
1161
1191
 
1162
1192
  function cap(text: string | undefined | null, max: number): string {
1163
1193
  if (!text) return '';
@@ -1267,7 +1297,7 @@ function hasObservablePageChange(data?: Record<string, any>): boolean {
1267
1297
  return Array.isArray(data.pageDiff.htmlParts) && data.pageDiff.htmlParts.length > 0;
1268
1298
  }
1269
1299
 
1270
- export async function failedToolResult(action: string, message: string, data?: Record<string, any>, error?: Error | null) {
1300
+ export async function failedToolResult(action: string, message: string, data?: Record<string, any>, error?: Error | null, judge?: Judge, intent?: string) {
1271
1301
  const result: Record<string, any> = { success: false, action, message, ...data };
1272
1302
  if (data?.pageDiff) {
1273
1303
  result.suggestion = data.suggestion ? `${data.suggestion} ${PAGE_DIFF_SUGGESTION}` : PAGE_DIFF_SUGGESTION;
@@ -1279,6 +1309,11 @@ export async function failedToolResult(action: string, message: string, data?: R
1279
1309
  result.suggestion = getMultipleElementsSuggestion();
1280
1310
  result.multipleElementsDetected = true;
1281
1311
  result.elements = formatElementList(matched);
1312
+ const labels = (matched || []).map((element) => `${element.text || 'no text'} — ${element.html}`);
1313
+ const pick = await judge?.decide('Which listed element does the intent name?', [...labels, UNDECIDED], { intent });
1314
+ if (!pick?.value) return result;
1315
+ const index = labels.indexOf(pick.value) + 1;
1316
+ result.suggestion = `Element ${index} is the one meant. Repeat the action with step.opts({ elementIndex: ${index} }) as the last argument.`;
1282
1317
  return result;
1283
1318
  }
1284
1319
 
@@ -199,7 +199,8 @@ export class RequestResult {
199
199
  if (Array.isArray(data)) return {};
200
200
 
201
201
  const id = data.id ?? data._id ?? data.uuid;
202
- const title = data.name ?? data.title ?? data.label;
202
+ const attributes = data.attributes || {};
203
+ const title = data.name ?? data.title ?? data.label ?? attributes.name ?? attributes.title ?? attributes.label;
203
204
 
204
205
  const result: { id?: string | number; title?: string } = {};
205
206
  if (id !== undefined) result.id = id;
@@ -31,6 +31,7 @@ export interface CommandAutocomplete {
31
31
  replaceTo: number;
32
32
  visible: boolean;
33
33
  argumentHint?: string;
34
+ completesArgument?: boolean;
34
35
  }
35
36
 
36
37
  function parseCommand(input: string): ParsedCommand | null {
@@ -266,6 +267,31 @@ export class CommandHandler implements InputManager {
266
267
  const parsed = parseCommand(query);
267
268
  const exactCommand = parsed ? this.findCommand(parsed.name) : undefined;
268
269
  const hasArguments = commandEnd !== -1 && input.slice(commandEnd + 1).trim().length > 0;
270
+ const argumentHint = !insideCommand && exactCommand && !hasArguments && exactCommand.options.length > 0 ? exactCommand.options.map((option) => option.flags).join(' ') : undefined;
271
+
272
+ if (!insideCommand && exactCommand) {
273
+ const argumentQuery = input.slice(commandEnd + 1).trim();
274
+ const argumentEntries = exactCommand
275
+ .completeArguments()
276
+ .filter((completion) => completion.value !== argumentQuery)
277
+ .map((completion) => ({
278
+ aliases: [],
279
+ canonical: completion.value,
280
+ description: '',
281
+ value: completion.display || completion.value,
282
+ }));
283
+ const argumentSuggestions = this.rankSuggestions(argumentQuery, argumentEntries);
284
+
285
+ return {
286
+ suggestions: argumentSuggestions,
287
+ replaceFrom: commandEnd + 1,
288
+ replaceTo: input.length,
289
+ visible: argumentSuggestions.length > 0,
290
+ argumentHint,
291
+ completesArgument: true,
292
+ };
293
+ }
294
+
269
295
  const commandEntries = this.getSlashCommandEntries();
270
296
  let suggestions: CommandAutocompleteSuggestion[] = [];
271
297
  if (insideCommand) {
@@ -280,10 +306,9 @@ export class CommandHandler implements InputManager {
280
306
  suggestions = this.rankSuggestions(query, commandEntries);
281
307
  }
282
308
  }
283
- const argumentHint = !insideCommand && exactCommand && !hasArguments && exactCommand.options.length > 0 ? exactCommand.options.map((option) => option.flags).join(' ') : undefined;
284
309
 
285
310
  return {
286
- suggestions,
311
+ suggestions: suggestions.map((suggestion) => ({ ...suggestion, value: `${suggestion.value} ` })),
287
312
  replaceFrom: 0,
288
313
  replaceTo,
289
314
  visible: insideCommand && suggestions.length > 0,
@@ -378,7 +403,7 @@ export class CommandHandler implements InputManager {
378
403
  private rankSuggestions(query: string, entries: AutocompleteEntry[]): CommandAutocompleteSuggestion[] {
379
404
  if (!query) {
380
405
  return entries.slice(0, 20).map((entry) => ({
381
- value: entry.value,
406
+ value: entry.canonical,
382
407
  display: entry.value,
383
408
  description: entry.description,
384
409
  argumentHint: entry.argumentHint,
@@ -35,6 +35,10 @@ export abstract class BaseCommand<T = ExplorBot> {
35
35
  return this.name === commandName || this.aliases.includes(commandName);
36
36
  }
37
37
 
38
+ completeArguments(): ArgumentCompletion[] {
39
+ return [];
40
+ }
41
+
38
42
  printSuggestions(): void {
39
43
  if (this.suggestions.length === 0) return;
40
44
  const prefix = isInteractive() ? '/' : `${getCliName()} `;
@@ -63,3 +67,8 @@ export abstract class BaseCommand<T = ExplorBot> {
63
67
  return { opts: cmd.opts(), args: cmd.args };
64
68
  }
65
69
  }
70
+
71
+ export interface ArgumentCompletion {
72
+ value: string;
73
+ display?: string;
74
+ }
@@ -10,7 +10,7 @@ import { ErrorPageError, getStateErrorPageError } from '../utils/error-page.ts';
10
10
  import { tag } from '../utils/logger.js';
11
11
  import { type NextStepSection, printNextSteps, relativeToCwd } from '../utils/next-steps.ts';
12
12
  import { safeFilename } from '../utils/strings.ts';
13
- import { BaseCommand, type Suggestion } from './base-command.js';
13
+ import { type ArgumentCompletion, BaseCommand, type Suggestion } from './base-command.js';
14
14
 
15
15
  const MAX_SUB_PAGE_ATTEMPTS = 30;
16
16
  const PRIORITY_ORDER: Record<string, number> = { critical: 0, important: 1, high: 2, normal: 3, low: 4 };
@@ -19,7 +19,7 @@ export const DEADLINE_TEST_ALLOWANCE_MS = 5 * 60_000;
19
19
 
20
20
  export class ExploreCommand extends BaseCommand {
21
21
  name = 'explore';
22
- description = 'Start web exploration';
22
+ description = 'Start web exploration on the current page, or on the URL given as the first argument';
23
23
  options = [
24
24
  { flags: '--max-tests <number>', description: 'Maximum number of tests to run' },
25
25
  { flags: '--max-duration <minutes>', description: 'Wall-clock budget in minutes; wraps up the session before the limit is hit' },
@@ -61,6 +61,12 @@ export class ExploreCommand extends BaseCommand {
61
61
  this.hardDeadlineAt = Date.now() + this.maxDurationMinutes * 60_000 - DEADLINE_RESERVE_MS;
62
62
  }
63
63
 
64
+ const [target] = remaining;
65
+ if (target?.startsWith('/') || target?.startsWith('http://') || target?.startsWith('https://')) {
66
+ remaining.shift();
67
+ await this.explorBot.visit(target);
68
+ }
69
+
64
70
  const feature = (opts.focus as string) || remaining.join(' ') || undefined;
65
71
  const cfg = this.parseConfigure(opts.configure as string | undefined);
66
72
  if (cfg.priorities) this.priorityFilter = new Set(cfg.priorities);
@@ -97,6 +103,13 @@ export class ExploreCommand extends BaseCommand {
97
103
  }
98
104
  }
99
105
 
106
+ completeArguments(): ArgumentCompletion[] {
107
+ return this.explorBot
108
+ .stateManager()
109
+ .getKnownUrls()
110
+ .map((url) => ({ value: url }));
111
+ }
112
+
100
113
  private originLabel(test: Test): string {
101
114
  return this.oldTestRefs.has(test) ? 'OLD' : 'NEW';
102
115
  }
@@ -1,5 +1,6 @@
1
1
  import { Planner } from '../ai/planner.js';
2
2
  import { Researcher } from '../ai/researcher.js';
3
+ import { normalizeUrl } from '../state-manager.js';
3
4
  import { Stats } from '../stats.js';
4
5
  import { tag } from '../utils/logger.js';
5
6
  import { loop } from '../utils/loop.js';
@@ -67,12 +68,18 @@ export class FreesailCommand extends BaseCommand {
67
68
  const suggestion = await navigator.freeSail({ strategy, scope, visitedUrls });
68
69
  if (!suggestion) {
69
70
  tag('info').log('No navigation suggestion available');
70
- return;
71
+ ctx.stop();
71
72
  }
72
73
 
73
74
  if (scope && !suggestion.target.startsWith(scope)) {
74
75
  tag('warning').log(`Suggestion ${suggestion.target} is outside scope ${scope}, skipping`);
75
- return;
76
+ ctx.stop();
77
+ }
78
+
79
+ const currentUrl = stateManager.getCurrentState()?.url;
80
+ if (currentUrl && normalizeUrl(suggestion.target) === normalizeUrl(currentUrl)) {
81
+ tag('info').log(`No new navigation target available after ${currentUrl}; stopping exploration`);
82
+ ctx.stop();
76
83
  }
77
84
 
78
85
  tag('info').log(`Navigating to: ${suggestion.target} - ${suggestion.reason}`);
@@ -163,7 +163,7 @@ export function runInitCommand(options: InitCommandOptions): void {
163
163
  log('2. Set AI models config file');
164
164
  log('3. Set web application URL in the config file');
165
165
  log('4. Add initial knowledge (how to authorize to the application, etc.)');
166
- tag('substep').log(chalk.yellow(`${getCliName()} learn * 'to authorize use these credentials: admin@example.com / secret123'`));
166
+ tag('substep').log(chalk.yellow(`${getCliName()} learn '*' 'to authorize use these credentials: admin@example.com / secret123'`));
167
167
  tag('substep').log('You can use ${env.LOGIN} and ${env.PASSWORD} to reference environment variables.');
168
168
 
169
169
  log('5. Launch application on a relative URL');
@@ -1,5 +1,5 @@
1
1
  import { tag } from '../utils/logger.js';
2
- import { BaseCommand, type Suggestion } from './base-command.js';
2
+ import { type ArgumentCompletion, BaseCommand, type Suggestion } from './base-command.js';
3
3
 
4
4
  export class NavigateCommand extends BaseCommand {
5
5
  name = 'navigate';
@@ -18,4 +18,11 @@ export class NavigateCommand extends BaseCommand {
18
18
  await this.explorBot.visit(destination);
19
19
  tag('success').log(`Navigation requested: ${destination}`);
20
20
  }
21
+
22
+ completeArguments(): ArgumentCompletion[] {
23
+ return this.explorBot
24
+ .stateManager()
25
+ .getKnownUrls()
26
+ .map((url) => ({ value: url }));
27
+ }
21
28
  }
@@ -1,5 +1,6 @@
1
+ import { Plan } from '../test-plan.js';
1
2
  import { tag } from '../utils/logger.js';
2
- import { BaseCommand, type Suggestion } from './base-command.js';
3
+ import { type ArgumentCompletion, BaseCommand, type Suggestion } from './base-command.js';
3
4
 
4
5
  export class PlanLoadCommand extends BaseCommand {
5
6
  name = 'plan:load';
@@ -18,4 +19,8 @@ export class PlanLoadCommand extends BaseCommand {
18
19
  const plan = this.explorBot.loadPlan(filename);
19
20
  tag('success').log(`Plan loaded: ${plan.title} with ${plan.tests.length} tests`);
20
21
  }
22
+
23
+ completeArguments(): ArgumentCompletion[] {
24
+ return Plan.listFiles(this.explorBot.getPlansDir()).map((file) => ({ value: file.name, display: file.label }));
25
+ }
21
26
  }