explorbot 0.3.4 → 0.4.0

package/src/ai/historian/playwright.ts

@@ -98,7 +98,7 @@ export function WithPlaywright<T extends Constructor>(Base: T) {
       lines.push('');
       lines.push(`test.describe('${escapeString(plan.title)}', () => {`);
 
-      const startUrl = plan.url || plan.tests[0]?.startUrl;
+      const startUrl = plan.startUrl;
       if (startUrl) {
         lines.push('  test.beforeEach(async ({ page }) => {');
         lines.push(`    await page.goto('${escapeString(startUrl)}');`);

package/src/ai/pilot.ts

@@ -765,6 +765,7 @@ export class Pilot implements Agent {
             const parts = [c.type];
             if (c.title) parts.push(`"${c.title}"`);
             if (c.id) parts.push(`(id: ${c.id})`);
+            if (c.request) parts.push(`via ${c.request}`);
             return parts.join(' ');
           });
           const stepText = `Precondition: created ${items.join(', ')}`;
@@ -827,7 +828,13 @@ export class Pilot implements Agent {
 
     const focusArea = state.overlay;
     if (focusArea.detected) {
-      lines.push(`modal: ${focusArea.name || focusArea.type}`);
+      let line = `modal: ${focusArea.name || focusArea.type}`;
+      if (focusArea.root) line += ` (root: ${focusArea.root})`;
+      lines.push(line);
+    } else if (focusArea.present) {
+      let line = `region: ${focusArea.name || 'unnamed'} (inline`;
+      if (focusArea.root) line += `, root: ${focusArea.root}`;
+      lines.push(`${line})`);
     } else {
       lines.push('modal: none');
     }
@@ -1114,6 +1121,8 @@ export class Pilot implements Agent {
       state), instruct Tester to verify() and finish(). If goal was already true at the start, propose
       different input data so the test is meaningful. If Tester repeats the same successful action, STOP.
 
+      If needed you should pick the exact item the scenario should act on (from the page, or precondition() one) and pass it to tester
+
       Action classification: GOAL-ADVANCING actions mutate the scenario's subject data (create/edit/delete/submit/verify).
       VIEW-ONLY actions toggle filters/tabs/sort/collapse without changing data. One VIEW-ONLY to reveal a
       target is fine; ≥2 consecutive VIEW-ONLY actions with no GOAL-ADVANCING action in between is thrashing
@@ -1131,6 +1140,7 @@ export class Pilot implements Agent {
       Diagnostic patterns (use <state>, executed/element/skipped fields, ariaDiff):
       - Click failed + button in "disabled buttons" → required field missing. Instruct fill first.
       - "modal: none" but Tester targets a modal → modal closed; re-trigger.
+      - "region:" in <state> → a large area appeared in place without navigation (subview, wizard step, panel). Direct Tester to act inside it; the rest of the page is still usable.
       - Action SUCCESS but ariaDiff empty → may have worked without visible DOM change; check result message.
       - MultipleElementsFound → xpathCheck() to identify the right one, then precise locator or visualClick().
       - Wrong page (settings vs feature) → getVisitedStates() then back() or reset(). Don't try breadcrumbs (SPA back-nav is unreliable).

package/src/ai/planner.ts

@@ -33,7 +33,7 @@ const TasksSchema = z.object({
   scenarios: z
     .array(
       z.object({
-        scenario: z.string().describe('A single sentence describing what to test'),
+        scenario: z.string().describe('A single sentence describing the behavior to test.'),
         priority: z.enum(['critical', 'important', 'high', 'normal', 'low']).describe('Priority of the task based on business importance'),
         startUrl: z.string().nullable().describe('Start URL for the test if different from plan URL. Use only stable feature/list/detail pages, not transient create/edit/modal URLs unless the scenario specifically starts inside that form.'),
         steps: z.array(z.string()).describe('List of steps to perform for this scenario. Each step should be a specific action (e.g., "Open the form", "Enter required data", "Submit the form"). Keep steps atomic and actionable.'),

package/src/ai/provider.ts

@@ -2,7 +2,8 @@ import { OpenTelemetry } from '@ai-sdk/otel';
 import { LangfuseSpanProcessor } from '@langfuse/otel';
 import { NodeSDK } from '@opentelemetry/sdk-node';
 import { AsyncLocalStorage } from 'node:async_hooks';
-import { generateObject, generateText, isStepCount, registerTelemetry, tool } from 'ai';
+import dedent from 'dedent';
+import { APICallError, generateObject, generateText, isStepCount, registerTelemetry, tool } from 'ai';
 import type { ModelMessage } from 'ai';
 import { z } from 'zod';
 import { clearActivity, setActivity } from '../activity.ts';
@@ -423,10 +424,29 @@ export class Provider {
     const stopConditions: any[] = [isStepCount(maxRoundtrips)];
     if (extraStop) stopConditions.push(extraStop);
     const config = this.buildGenerateConfig({ tools: toolsWithCommentary, maxOutputTokens: 16384, toolChoice: 'auto', experimental_repairToolCall: repairToolCall }, { stopWhen: stopConditions, model }, options);
+    let attemptMessages = messages;
+    let invalidRequestFeedbackAdded = false;
+    const executedStepMessages: ModelMessage[] = [];
     try {
       const response = await this.withModelRequestSlot(() =>
         withRetry(async () => {
-          const result = (await this.raceWithIdleTimeout((signal) => generateText({ messages, ...config, abortSignal: signal }), config.timeout || 30000)) as any;
+          const stepMessages: ModelMessage[] = [];
+          const onStepEnd = (step: any) => {
+            stepMessages.push(...(step.response?.messages || []));
+          };
+          const result = (await this.raceWithIdleTimeout((signal) => generateText({ messages: attemptMessages, ...config, abortSignal: signal, onStepEnd }), config.timeout || 30000).catch((error) => {
+            if (stepMessages.length > 0) {
+              tag('warning').log(`Keeping ${stepMessages.length} messages from tool steps that already ran before the failure`);
+              executedStepMessages.push(...stepMessages);
+              attemptMessages = [...attemptMessages, ...stepMessages];
+            }
+            if (!invalidRequestFeedbackAdded) {
+              const amended = withInvalidRequestFeedback(attemptMessages, error);
+              invalidRequestFeedbackAdded = amended !== attemptMessages;
+              attemptMessages = amended;
+            }
+            throw error;
+          })) as any;
           this.recordUsage(options.agentName || 'unknown', modelName, result.usage);
           const hasToolCall = (result.toolCalls?.length || 0) > 0;
           if (!result.text && !hasToolCall && result.finishReason === 'length') {
@@ -438,6 +458,8 @@ export class Provider {
 
       clearActivity();
 
+      withExecutedSteps(response, executedStepMessages);
+
       // Log tool usage summary
       if (response.toolCalls && response.toolCalls.length > 0) {
         responseLog(response.toolCalls);
@@ -452,12 +474,13 @@ export class Provider {
     } catch (error: any) {
       clearActivity();
       if (error?.message?.includes('Tool choice is required')) {
-        return { text: '', toolCalls: [], toolResults: [], response: { messages: [] }, usage: null };
+        return { text: '', toolCalls: [], toolResults: [], responseMessages: executedStepMessages, usage: null };
       }
       if (error?.name === 'AbortError') throw error;
       if (error instanceof ContextLengthError) throw error;
       if (Provider.isContextLengthError(error)) {
-        return this.recoverFromContextLength(error, messages, options, (m, o) => this.generateWithTools(m, model, tools, o));
+        const recovered = await this.recoverFromContextLength(error, attemptMessages, options, (m, o) => this.generateWithTools(m, model, tools, o));
+        return withExecutedSteps(recovered, executedStepMessages);
       }
       if (error.constructor?.name === 'AI_APICallError') {
         responseLog(error.message);
@@ -693,6 +716,27 @@ function repairToolCall(options: ToolCallRepairOptions): any | null {
   return repairHarmonyChannel(options);
 }
 
+function withExecutedSteps(result: any, executed: ModelMessage[]): any {
+  if (executed.length === 0) return result;
+  return Object.defineProperty(result, 'responseMessages', { value: [...executed, ...(result.responseMessages || [])], configurable: true, enumerable: true });
+}
+
+function withInvalidRequestFeedback(messages: ModelMessage[], error: unknown): ModelMessage[] {
+  if (!(error instanceof APICallError) || error.statusCode !== 400) return messages;
+  tag('warning').log('Provider rejected the request as invalid — relaying its reason before the retry');
+  return [
+    ...messages,
+    {
+      role: 'user',
+      content: dedent`
+        The previous request was rejected by the provider as invalid:
+        "${error.message}"
+        Fix what it describes and re-issue the request.
+      `,
+    },
+  ];
+}
+
 function repairChannelMarker({ toolCall, tools }: ToolCallRepairOptions): any | null {
   const markerIndex = toolCall.toolName.indexOf('<|channel|>');
   if (markerIndex <= 0) return null;

package/src/ai/researcher/deep-analysis.ts

@@ -89,8 +89,7 @@ export function WithDeepAnalysis<T extends Constructor>(Base: T) {
 
     async researchOverlay(current: ActionResult, previous: ActionResult, pageStateHash: string): Promise<string | null> {
       const focusArea = current.overlay;
-      if (!focusArea.detected || !focusArea.name) return null;
-      if (focusArea.type !== 'dialog' && focusArea.type !== 'modal') return null;
+      if (!focusArea.present || !focusArea.name) return null;
 
       const cached = getCachedResearch(pageStateHash);
       if (!cached) return null;

package/src/ai/researcher.ts

@@ -76,7 +76,7 @@ export class Researcher extends ResearcherBase implements Agent {
   }
 
   static getCachedResearch(state: WebPageState): string {
-    return getCachedResearch(state.hash || '');
+    return getCachedResearch(ActionResult.fromState(state).baseHash);
   }
 
   getSystemMessage(): string {
@@ -96,7 +96,7 @@ export class Researcher extends ResearcherBase implements Agent {
     const maxRetries = (this.config.ai?.agents?.researcher as any)?.retries ?? 2;
     let retriesLeft = opts._retriesLeft ?? maxRetries;
     this.actionResult = ActionResult.fromState(state);
-    const stateHash = state.hash || this.actionResult.getStateHash();
+    const stateHash = this.actionResult.baseHash;
     const researchState = { ...state, hash: stateHash };
 
     if (!force && stateHash) {
@@ -268,7 +268,7 @@ export class Researcher extends ResearcherBase implements Agent {
 
       if (!interrupted() && deep) {
         try {
-          await this.performDeepAnalysis(state, result);
+          await this.performDeepAnalysis(researchState, result);
         } catch (err) {
           tag('warning').log(`Deep analysis failed, continuing with best-effort research: ${err instanceof Error ? err.message : err}`);
         }

package/src/ai/tester.ts

@@ -63,6 +63,8 @@ export class Tester extends TaskAgent implements Agent {
   private seenUiMapUrls = new Set<string>();
   private lastAnalyzedStateHash: string | null = null;
   private stalledIterations = 0;
+  private previousRegionPresent: boolean | null = null;
+  private regionTransitioned = false;
   private readonly MAX_STALLED_ITERATIONS = 3;
 
   private skipResearch = (err: Error): string => {
@@ -117,6 +119,8 @@ export class Tester extends TaskAgent implements Agent {
     this.seenUiMapUrls.clear();
     this.lastAnalyzedStateHash = null;
     this.stalledIterations = 0;
+    this.previousRegionPresent = null;
+    this.regionTransitioned = false;
     this.stateManager.clearHistory();
     this.resetFailureCount();
     this.pilot?.reset();
@@ -246,6 +250,7 @@ export class Tester extends TaskAgent implements Agent {
     const codeceptjsTools = createCodeceptJSTools(this.toolDeps, task);
     let assertionPerformed = false;
     let extensions = 0;
+    let deadlineReached = false;
     let shouldContinue = true;
 
     while (shouldContinue) {
@@ -253,6 +258,12 @@ export class Tester extends TaskAgent implements Agent {
 
       await loop(
         async ({ stop, pause, iteration, userInput }) => {
+          if (opts.deadline != null && Date.now() >= opts.deadline) {
+            deadlineReached = true;
+            task.addNote('Time budget reached. Stopped');
+            stop();
+            return;
+          }
           debugLog('iteration', iteration);
           if (!(await this.explorer.recover()).ok) {
             task.addNote('Browser page is unavailable');
@@ -420,6 +431,7 @@ export class Tester extends TaskAgent implements Agent {
       );
 
       if (task.hasFinished) break;
+      if (deadlineReached) break;
 
       if (!(await this.explorer.recover()).ok) break;
 
@@ -460,6 +472,10 @@ export class Tester extends TaskAgent implements Agent {
   }
 
   private shouldAnalyzeProgress(iteration: number, currentState: ActionResult): boolean {
+    if (this.regionTransitioned) {
+      this.regionTransitioned = false;
+      return true;
+    }
     if (this.consecutiveFailures >= 3) return true;
     if (this.consecutiveEmptyResults >= 2) return true;
     if (iteration % this.progressCheckInterval !== 0) return false;
@@ -530,6 +546,12 @@ export class Tester extends TaskAgent implements Agent {
     const currentStateHash = currentState.hash;
 
     const isNewUrl = this.previousUrl !== currentUrl;
+    const isNewState = !isNewUrl && this.previousStateHash !== null && this.previousStateHash !== currentStateHash;
+
+    if (this.previousRegionPresent !== null && this.previousRegionPresent !== currentState.overlay.present) {
+      this.regionTransitioned = true;
+    }
+    this.previousRegionPresent = currentState.overlay.present;
 
     this.previousUrl = currentUrl;
     this.previousStateHash = currentStateHash;
@@ -557,9 +579,11 @@ export class Tester extends TaskAgent implements Agent {
 
     if (focusArea.detected) {
       const areaName = focusArea.name ? ` "${focusArea.name}"` : '';
+      let rootHint = '';
+      if (focusArea.root) rootHint = `\nIts content lives inside \`${focusArea.root}\` — scope locators to it.`;
       context += dedent`
         <focus_scope>
-        A ${focusArea.type}${areaName} is currently open above the page.
+        A ${focusArea.type}${areaName} is currently open above the page.${rootHint}
         Scope all interactions to elements inside this ${focusArea.type}.
         Page navigation, filters, and tabs that exist outside it are not actionable while it is open and may share names or roles with elements inside it — prefer the locator inside the ${focusArea.type}.
         Use <page_aria> to confirm the element you target is actually inside the ${focusArea.type}.
@@ -567,6 +591,18 @@ export class Tester extends TaskAgent implements Agent {
       `;
     }
 
+    if (!focusArea.detected && focusArea.present && isNewState) {
+      let rootHint = '';
+      if (focusArea.root) rootHint = `\nIt lives inside \`${focusArea.root}\`.`;
+      context += dedent`
+        <area_of_interest>
+        A large new area "${focusArea.name || 'unnamed area'}" appeared on this page without navigation.${rootHint}
+        The scenario most likely continues inside this area — prefer its elements for your next actions.
+        The rest of the page (navigation, menus, filters) is still interactive and remains available.
+        </area_of_interest>
+      `;
+    }
+
     if (currentState.isInsideIframe) {
       const iframeInfo = currentState.iframeURL || 'iframe context active';
       context += dedent`
@@ -589,7 +625,7 @@ export class Tester extends TaskAgent implements Agent {
       if (!alreadySeenUiMap) {
         research = await this.researcher.research(currentState).catch(this.skipResearch);
       }
-      this.pageStateHash = currentStateHash;
+      this.pageStateHash = currentState.baseHash;
       this.pageActionResult = currentState;
       let uiMapSection = '';
       if (research) {
@@ -627,7 +663,7 @@ export class Tester extends TaskAgent implements Agent {
       return context;
     }
 
-    if (focusArea.detected && focusArea.name && this.pageStateHash && this.pageActionResult) {
+    if (focusArea.present && focusArea.name && this.pageStateHash && this.pageActionResult) {
       const overlaySection = await this.researcher.researchOverlay(currentState, this.pageActionResult, this.pageStateHash).catch(this.skipResearch);
       if (overlaySection) {
         context += dedent`
@@ -1172,4 +1208,5 @@ interface TestSessionHandlers {
 
 export interface TestOptions {
   startOnCurrentPage?: boolean;
+  deadline?: number;
 }

package/src/ai/tools.ts

@@ -8,8 +8,9 @@ import { Stats } from '../stats.ts';
 import { type Task, TestResult } from '../test-plan.js';
 import { LARGE_ARIA_CHANGE_THRESHOLD } from '../utils/aria.ts';
 import { isFatalBrowserError } from '../utils/browser-errors.ts';
+import { cleanHtmlSnippet } from '../utils/html.ts';
 import { createDebug, tag } from '../utils/logger.js';
-import { compactErrorMessage } from '../utils/strings.ts';
+import { compactErrorMessage, normalizeInlineText, truncate } from '../utils/strings.ts';
 import { pause } from '../utils/loop.js';
 import { WebElement } from '../utils/web-element.ts';
 import type { ToolDeps } from './agent.ts';
@@ -1332,20 +1333,23 @@ export function clickFailureSuggestion(attempts: Array<{ error?: string }>): str
 }
 
 const MAX_DISAMBIGUATE_ELEMENTS = 10;
+const MAX_DISAMBIGUATE_TEXT = 80;
+const MAX_DISAMBIGUATE_HTML = 300;
 const MULTIPLE_ELEMENTS_PATTERN = 'multiple elements';
 
-async function extractWebElements(error: Error | null | undefined): Promise<Array<{ xpath: string; html: string }> | null> {
+async function extractWebElements(error: Error | null | undefined): Promise<Array<{ xpath: string; html: string; text: string }> | null> {
   if (!error || error.name !== 'MultipleElementsFound') return null;
 
-  const elements = (error as any).webElements as Array<{ toAbsoluteXPath: () => Promise<string>; toSimplifiedHTML: () => Promise<string> }> | undefined;
+  const elements = (error as any).webElements as Array<{ toAbsoluteXPath: () => Promise<string>; toOuterHTML: () => Promise<string>; getText: () => Promise<string | null> }> | undefined;
   if (!elements?.length) return null;
 
-  const result: Array<{ xpath: string; html: string }> = [];
+  const result: Array<{ xpath: string; html: string; text: string }> = [];
   for (let i = 0; i < Math.min(elements.length, MAX_DISAMBIGUATE_ELEMENTS); i++) {
     try {
       const xpath = await elements[i].toAbsoluteXPath();
-      const html = await elements[i].toSimplifiedHTML();
-      result.push({ xpath, html });
+      const html = truncate(cleanHtmlSnippet(await elements[i].toOuterHTML()), MAX_DISAMBIGUATE_HTML);
+      const text = truncate(normalizeInlineText((await elements[i].getText()) || ''), MAX_DISAMBIGUATE_TEXT);
+      result.push({ xpath, html, text });
     } catch (e) {
       debugLog('Failed to get details for element %d: %s', i, e);
     }
@@ -1353,17 +1357,21 @@ async function extractWebElements(error: Error | null | undefined): Promise<Arra
   return result.length > 0 ? result : null;
 }
 
-async function formatMatchedElements(error: Error | null | undefined): Promise<string | null> {
+function formatElementList(details: Array<{ xpath: string; html: string; text: string }>): string {
+  return details.map((el, i) => `Element ${i + 1}:\nText: "${el.text}"\nXPath: ${el.xpath}\nHTML: ${el.html}`).join('\n\n');
+}
+
+export async function formatMatchedElements(error: Error | null | undefined): Promise<string | null> {
   const details = await extractWebElements(error);
   if (!details) return 'Could not fetch element details. Repeat the action to get better info.';
-  return details.map((el, i) => `Element ${i + 1}\nXPath: ${el.xpath}\nHTML: ${el.html}`).join('\n\n');
+  return formatElementList(details);
 }
 
 async function disambiguateElements(error: Error | null | undefined, explanation: string, provider: AIProvider): Promise<{ position: number; xpath: string } | null> {
   const elementDetails = await extractWebElements(error);
   if (!elementDetails) return null;
 
-  const elementList = elementDetails.map((el, i) => `Element ${i + 1}:\nXPath: ${el.xpath}\nHTML: ${el.html}`).join('\n\n');
+  const elementList = formatElementList(elementDetails);
 
   const schema = z.object({
     position: z.number().nullable().describe('1-based position of the correct element, or null if none match'),

package/src/api/request-result.ts

@@ -85,6 +85,10 @@ export class RequestResult {
     }
   }
 
+  get isWrite(): boolean {
+    return ['POST', 'PUT', 'PATCH', 'DELETE'].includes(this.method);
+  }
+
   save(outputDir: string): void {
     const requestsDir = path.join(outputDir, 'requests');
     if (!existsSync(requestsDir)) {
@@ -169,7 +173,7 @@ export class RequestResult {
       statusText: meta.statusText || '',
       responseHeaders: meta.responseHeaders || {},
       timing: Number.parseInt(meta.timing) || 0,
-      timestamp: new Date(meta.timestamp || Date.now()),
+      timestamp: new Date(meta.timestamp || 0),
     });
 
     result.requestFile = requestFile;
@@ -178,8 +182,12 @@ export class RequestResult {
     return result;
   }
 
+  toEndpoint(): string {
+    return `${this.method} ${this.path}`;
+  }
+
   toSummary(): string {
-    return `${this.method} ${this.path} → ${this.status} (${this.timing}ms)`;
+    return `${this.toEndpoint()} → ${this.status} (${this.timing}ms)`;
   }
 
   extractIdAndTitle(): { id?: string | number; title?: string } {

package/src/api/request-store.ts

@@ -1,8 +1,9 @@
 import { existsSync, readdirSync } from 'node:fs';
 import path from 'node:path';
+import { isDynamicSegment } from '../utils/url-matcher.ts';
 import { RequestResult } from './request-result.ts';
 
-const AUTH_HEADERS = ['authorization', 'cookie', 'x-api-key', 'x-csrf-token'];
+const AUTH_HEADERS = ['authorization', 'x-api-key', 'x-csrf-token'];
 
 export class RequestStore {
   private capturedRequests: RequestResult[] = [];
@@ -10,6 +11,7 @@ export class RequestStore {
   private failedRequests: RequestResult[] = [];
   private onFailedListeners: Array<(r: RequestResult) => void> = [];
   private outputDir: string;
+  private sessionStartedAt = new Date();
 
   constructor(outputDir: string) {
     this.outputDir = outputDir;
@@ -77,13 +79,15 @@ export class RequestStore {
     return this.madeRequests.filter((r) => r.status === status);
   }
 
-  toEndpointList(): string {
+  toEndpointList(scopePath?: string): string {
+    let requests = this.capturedRequests;
+    if (scopePath) requests = this.getWriteRequestsForScope(scopePath);
+
     const seen = new Set<string>();
     const lines: string[] = [];
 
-    for (const req of this.capturedRequests) {
-      const normalized = normalizePathPattern(req.path);
-      const key = `${req.method} ${normalized}`;
+    for (const req of requests) {
+      const key = `${req.method} ${normalizePathPattern(req.path)}`;
       if (seen.has(key)) continue;
       seen.add(key);
       lines.push(key);
@@ -94,23 +98,42 @@ export class RequestStore {
 
   extractAuthHeaders(): Record<string, string> {
     const headers: Record<string, string> = {};
+    const sessionCaptures = this.capturedRequests.filter((r) => r.timestamp >= this.sessionStartedAt).sort((a, b) => b.timestamp.getTime() - a.timestamp.getTime());
 
-    for (let i = this.capturedRequests.length - 1; i >= 0; i--) {
-      const req = this.capturedRequests[i];
+    for (const req of sessionCaptures) {
       for (const [key, value] of Object.entries(req.requestHeaders)) {
         if (AUTH_HEADERS.includes(key.toLowerCase()) && !headers[key]) {
           headers[key] = value;
         }
       }
-      if (AUTH_HEADERS.every((h) => Object.keys(headers).some((k) => k.toLowerCase() === h))) break;
     }
 
     return headers;
   }
 
-  findCapturedRequest(method: string, pathPrefix: string): RequestResult | undefined {
+  findCapturedRequest(method: string, searchPath: string): RequestResult | undefined {
     const upper = method.toUpperCase();
-    return this.capturedRequests.find((r) => r.method === upper && r.path.startsWith(pathPrefix));
+    const search = normalizePathPattern(searchPath).split('/').filter(Boolean);
+
+    let best: RequestResult | undefined;
+    let bestScore = -1;
+
+    for (const req of this.capturedRequests) {
+      if (req.method !== upper) continue;
+      const segments = normalizePathPattern(req.path).split('/').filter(Boolean);
+      if (segments.length < search.length) continue;
+      if (!search.every((segment, i) => segment === segments[i])) continue;
+
+      let score = 0;
+      if (segments.length === search.length) score += 4;
+      if (req.status < 400) score += 2;
+      if (score < bestScore) continue;
+      if (score === bestScore && best && req.timestamp <= best.timestamp) continue;
+      best = req;
+      bestScore = score;
+    }
+
+    return best;
   }
 
   toLog(): string {
@@ -122,7 +145,7 @@ export class RequestStore {
     if (!existsSync(requestsDir)) return;
 
     const existingIds = new Set(this.capturedRequests.map((r) => r.id));
-    const files = readdirSync(requestsDir).filter((f) => f.endsWith('.request.yaml'));
+    const files = readdirSync(requestsDir).filter((f) => f.startsWith('xhr_') && f.endsWith('.request.yaml'));
 
     for (const file of files) {
       try {
@@ -137,7 +160,28 @@ export class RequestStore {
 
   getWriteRequestsForScope(scopePath: string): RequestResult[] {
     const writeMethods = new Set(['POST', 'PUT', 'PATCH', 'DELETE']);
-    return this.capturedRequests.filter((r) => writeMethods.has(r.method) && r.path.startsWith(scopePath));
+    const writes = this.capturedRequests.filter((r) => writeMethods.has(r.method));
+    const scopeSegments = scopePath.split('/').filter(Boolean);
+    if (scopeSegments.length === 0) return writes;
+
+    let scoped: RequestResult[] = [];
+    let fewest = Number.POSITIVE_INFINITY;
+    let ambiguous = false;
+    for (const segment of scopeSegments) {
+      if (isDynamicSegment(segment)) continue;
+      const matches = writes.filter((r) => r.path.split('/').includes(segment));
+      if (matches.length === 0 || matches.length > fewest) continue;
+      if (matches.length === fewest) {
+        if (!scoped.every((r, i) => r.id === matches[i].id)) ambiguous = true;
+        continue;
+      }
+      scoped = matches;
+      fewest = matches.length;
+      ambiguous = false;
+    }
+    if (ambiguous) return [];
+
+    return scoped;
   }
 
   clear(): void {
@@ -148,5 +192,8 @@ export class RequestStore {
 }
 
 function normalizePathPattern(urlPath: string): string {
-  return urlPath.replace(/\/[0-9a-f]{24}\b/g, '/{id}').replace(/\/\d+\b/g, '/{id}');
+  return urlPath
+    .split('/')
+    .map((segment) => (segment && isDynamicSegment(segment) ? '{id}' : segment))
+    .join('/');
 }

package/src/commands/explore-command.ts

@@ -14,12 +14,15 @@ import { BaseCommand, type Suggestion } from './base-command.js';
 
 const MAX_SUB_PAGE_ATTEMPTS = 30;
 const PRIORITY_ORDER: Record<string, number> = { critical: 0, important: 1, high: 2, normal: 3, low: 4 };
+export const DEADLINE_RESERVE_MS = 3 * 60_000;
+export const DEADLINE_TEST_ALLOWANCE_MS = 5 * 60_000;
 
 export class ExploreCommand extends BaseCommand {
   name = 'explore';
   description = 'Start web exploration';
   options = [
     { flags: '--max-tests <number>', description: 'Maximum number of tests to run' },
+    { flags: '--max-duration <minutes>', description: 'Wall-clock budget in minutes; wraps up the session before the limit is hit' },
     { flags: '--focus <feature>', description: 'Focus area for exploration' },
     { flags: '--configure <spec>', description: 'Reuse spec: keys new|from|style|subpages|pick_by|priority, e.g. "new:25%;pick_by=random;priority=critical,high"' },
     { flags: '--dry-run', description: 'Mark picked tests as skipped without executing or generating new ones' },
@@ -31,8 +34,11 @@ export class ExploreCommand extends BaseCommand {
   ];
 
   maxTests?: number;
+  maxDurationMinutes?: number;
+  hardDeadlineAt?: number;
   dryRun = false;
   private testsRun = 0;
+  private deadlineLogged = false;
   private completedPlans: Plan[] = [];
   private failedSubPages = new Set<string>();
   private oldTestRefs = new Set<Test>();
@@ -48,6 +54,12 @@ export class ExploreCommand extends BaseCommand {
     if (opts.maxTests) {
       this.maxTests = Number.parseInt(opts.maxTests as string, 10);
     }
+    if (opts.maxDuration) {
+      this.maxDurationMinutes = Number.parseInt(opts.maxDuration as string, 10);
+    }
+    if (this.hardDeadlineAt == null && this.maxDurationMinutes != null) {
+      this.hardDeadlineAt = Date.now() + this.maxDurationMinutes * 60_000 - DEADLINE_RESERVE_MS;
+    }
 
     const feature = (opts.focus as string) || remaining.join(' ') || undefined;
     const cfg = this.parseConfigure(opts.configure as string | undefined);
@@ -526,7 +538,18 @@ export class ExploreCommand extends BaseCommand {
   }
 
   private isLimitReached(): boolean {
-    return this.maxTests != null && this.testsRun >= this.maxTests;
+    if (this.maxTests != null && this.testsRun >= this.maxTests) return true;
+    return this.isDeadlineReached();
+  }
+
+  private isDeadlineReached(): boolean {
+    if (this.hardDeadlineAt == null) return false;
+    if (Date.now() < this.hardDeadlineAt - DEADLINE_TEST_ALLOWANCE_MS) return false;
+    if (!this.deadlineLogged) {
+      this.deadlineLogged = true;
+      tag('info').log(`Time budget reached after ${this.testsRun} test(s): stopping new work and finishing the session`);
+    }
+    return true;
   }
 
   private async runPendingTests(): Promise<void> {
@@ -548,7 +571,7 @@ export class ExploreCommand extends BaseCommand {
       test.start();
       test.finish(TestResult.SKIPPED);
     } else {
-      await this.explorBot.agentTester().test(test);
+      await this.explorBot.agentTester().test(test, { deadline: this.hardDeadlineAt });
     }
     this.testsRun++;
   }

package/src/commands/freesail-command.ts

@@ -4,7 +4,7 @@ import { Stats } from '../stats.js';
 import { tag } from '../utils/logger.js';
 import { loop } from '../utils/loop.js';
 import { BaseCommand } from './base-command.js';
-import { ExploreCommand } from './explore-command.js';
+import { DEADLINE_RESERVE_MS, DEADLINE_TEST_ALLOWANCE_MS, ExploreCommand } from './explore-command.js';
 
 export class FreesailCommand extends BaseCommand {
   name = 'freesail';
@@ -16,6 +16,7 @@ export class FreesailCommand extends BaseCommand {
     { flags: '--shallow', description: 'Use shallow navigation strategy' },
     { flags: '--scope <url>', description: 'Limit navigation to URLs starting with this prefix' },
     { flags: '--max-tests <number>', description: 'Maximum number of tests to run' },
+    { flags: '--max-duration <number>', description: 'Wall-clock budget in minutes for the whole run' },
   ];
 
   async execute(args: string): Promise<void> {
@@ -26,6 +27,9 @@ export class FreesailCommand extends BaseCommand {
     if (opts.shallow) strategy = 'shallow';
     const scope = opts.scope as string | undefined;
     const maxTests = opts.maxTests ? Number.parseInt(opts.maxTests as string, 10) : undefined;
+    const maxDuration = opts.maxDuration ? Number.parseInt(opts.maxDuration as string, 10) : undefined;
+    let hardDeadlineAt: number | undefined;
+    if (maxDuration != null) hardDeadlineAt = Date.now() + maxDuration * 60_000 - DEADLINE_RESERVE_MS;
 
     await this.explorBot.visitInitialState();
 
@@ -34,6 +38,7 @@ export class FreesailCommand extends BaseCommand {
     await loop(
       async (ctx) => {
         if (maxTests != null && testsRun >= maxTests) ctx.stop();
+        if (hardDeadlineAt != null && Date.now() >= hardDeadlineAt - DEADLINE_TEST_ALLOWANCE_MS) ctx.stop();
 
         const stateManager = this.explorBot.stateManager();
         const state = stateManager.getCurrentState();
@@ -48,6 +53,7 @@ export class FreesailCommand extends BaseCommand {
         } else {
           const exploreCmd = new ExploreCommand(this.explorBot);
           if (maxTests != null) exploreCmd.maxTests = maxTests - testsRun;
+          if (hardDeadlineAt != null) exploreCmd.hardDeadlineAt = hardDeadlineAt;
           await exploreCmd.execute('');
 
           const plan = this.explorBot.getCurrentPlan();