explorbot 0.4.7 → 0.4.9

package/boat/api-tester/src/ai/chief.ts

@@ -261,7 +261,9 @@ export class Chief extends ChiefBase {
         - Use real enum values discovered in the data
         - Each test MUST use DIFFERENT data — never reuse the same field values across tests
         - For "create" tests: base payload on a real record but change field values to create new unique data
-        - For "update" tests: pick a real existing ID and modify specific fields
+        - Treat records and IDs from sample_data as read-only. Never update, patch, delete, archive, or otherwise mutate them
+        - For update/delete tests: the same scenario must first create its own target, then mutate only that target
+        - For negative or unsupported-method tests that could mutate data if accepted: create a scenario-owned target first; if that setup is impossible, do not send the destructive request
         - For tests needing parent references: use real _id field values from sample_data
       `);
     }

package/boat/api-tester/src/ai/curler.ts

@@ -284,6 +284,10 @@ export class Curler {
       - Record important findings as you go
       - Be precise about what you expect vs what you observe
       - If a test requires data from another endpoint, use schemaFor to look it up before guessing
+      - Treat existing records, sample data, and IDs supplied by the plan as read-only
+      - Before PUT, PATCH, DELETE, archive, or another destructive request, create the target inside the current scenario and mutate only that target
+      - This also applies when testing an unsupported method: the server may unexpectedly accept it, so never probe destructively against pre-existing data
+      - If a scenario-owned target cannot be created, use stop rather than risking existing data
       </rules>
     `;
   }

package/boat/api-tester/src/cli.ts

@@ -40,7 +40,7 @@ export function createApiCommands(name = 'api'): Command {
       if (runOptions.endpoint && URL.canParse(runOptions.endpoint)) runOptions.baseEndpoint ||= runOptions.endpoint;
       try {
         const config = await parser.loadConfig(runOptions);
-        console.log(ConfigCommand.render(config, { configPath: parser.getConfigPath(), root: parser.getProjectRoot(), json: options.json }));
+        console.log(ConfigCommand.render(config, { configPath: parser.getConfigPath(), siteConfigPath: parser.getSiteConfigPath(), root: parser.getProjectRoot(), json: options.json }));
       } catch (error) {
         console.error(error instanceof Error ? error.message : 'Unknown error');
         process.exit(1);

package/boat/api-tester/src/config.ts

@@ -18,7 +18,8 @@ import {
   resolveOutputRoot,
   setOutputDir,
 } from '../../../src/config.ts';
-import { type SiteRecord, findGlobalConfig, globalEnvPath, isGlobalConfigPath, registerSite, resolveSiteTarget } from '../../../src/global-config.ts';
+import { type SiteRecord, findGlobalConfig, globalEnvPath, isGlobalConfigPath, loadSiteConfig, registerSite, resolveSiteTarget } from '../../../src/global-config.ts';
+import { deepMerge } from '../../../src/utils/merge.ts';
 
 export type { AIConfig };
 
@@ -58,6 +59,7 @@ export class ApibotConfigParser {
   private config: ApibotConfig | null = null;
   private configPath: string | null = null;
   private site: SiteRecord | null = null;
+  private siteConfigPath: string | null = null;
 
   private constructor() {}
 
@@ -111,20 +113,22 @@ export class ApibotConfigParser {
         };
       }
 
-      this.config = this.mergeWithDefaults(loadedConfig);
-      this.applyEnvSpec(this.config.api);
-      this.applyEnvHeaders(this.config.api);
-      if (options?.baseEndpoint) this.config.api.baseEndpoint = options.baseEndpoint.replace(/\/$/, '');
-      await resolveConfigModels(this.config.ai);
-      resolveLangfuse(this.config.ai);
-      this.configPath = resolvedPath;
+      let config = this.mergeWithDefaults(loadedConfig);
+      this.applyEnvSpec(config.api);
+      this.applyEnvHeaders(config.api);
+      if (options?.baseEndpoint) config.api.baseEndpoint = options.baseEndpoint.replace(/\/$/, '');
+      await resolveConfigModels(config.ai);
+      resolveLangfuse(config.ai);
       this.site = null;
+      this.siteConfigPath = null;
 
       if (isGlobalConfigPath(resolvedPath)) {
-        this.enterGlobalMode(this.config, options?.endpoint);
+        config = await this.enterGlobalMode(config, options?.endpoint);
       }
 
-      this.validateConfig(this.config);
+      this.validateConfig(config);
+      this.config = config;
+      this.configPath = resolvedPath;
       setOutputDir(this.getOutputDir());
 
       return this.config;
@@ -144,6 +148,10 @@ export class ApibotConfigParser {
     return this.configPath;
   }
 
+  getSiteConfigPath(): string | null {
+    return this.siteConfigPath;
+  }
+
   getOutputDir(): string {
     const config = this.getConfig();
     return path.join(this.getProjectRoot(), config.dirs?.output || 'output');
@@ -207,17 +215,29 @@ export class ApibotConfigParser {
     api.headers = { ...api.headers, ...parseHeaders(process.env.EXPLORBOT_API_HEADERS) };
   }
 
-  private enterGlobalMode(config: ApibotConfig, endpoint?: string): void {
+  private async enterGlobalMode(config: ApibotConfig, endpoint?: string): Promise<ApibotConfig> {
     const site = resolveSiteTarget(endpoint);
     this.site = registerSite(site.baseUrl);
 
+    const { path, config: siteConfig } = await loadSiteConfig(this.site.dir, site.baseUrl);
+    this.siteConfigPath = path;
+
+    const overrides: Partial<ApibotConfig> = {};
+    if (siteConfig.ai) overrides.ai = siteConfig.ai;
+    if (siteConfig.api) overrides.api = siteConfig.api;
+
+    const merged = deepMerge(config, overrides);
+    await resolveConfigModels(merged.ai);
+    resolveLangfuse(merged.ai);
+
     let baseEndpoint = site.baseUrl;
     const envUrl = process.env.EXPLORBOT_URL;
     if (envUrl && URL.parse(envUrl)?.origin === site.baseUrl) baseEndpoint = envUrl.replace(/\/$/, '');
 
-    config.dirs = { output: 'output', knowledge: 'knowledge' };
-    config.api = { ...config.api, baseEndpoint };
+    merged.dirs = { output: 'output', knowledge: 'knowledge' };
+    merged.api = { ...merged.api, baseEndpoint };
     materializeKnowledge(this.site.dir);
+    return merged;
   }
 
   private async loadEnvConfig(): Promise<ApibotConfig> {
@@ -310,19 +330,7 @@ export class ApibotConfigParser {
   }
 
   private mergeWithDefaults(config: Partial<ApibotConfig>): ApibotConfig {
-    return this.deepMerge({ dirs: { output: 'output' }, api: {} }, config);
-  }
-
-  private deepMerge(target: any, source: any): any {
-    const result = { ...target };
-    for (const key in source) {
-      if (source[key] && typeof source[key] === 'object' && !Array.isArray(source[key]) && source[key].constructor === Object) {
-        result[key] = this.deepMerge(result[key] || {}, source[key]);
-      } else {
-        result[key] = source[key];
-      }
-    }
-    return result;
+    return deepMerge({ dirs: { output: 'output' }, api: {} }, config);
   }
 }
 

package/dist/boat/api-tester/src/ai/chief.js

@@ -230,7 +230,9 @@ export class Chief extends ChiefBase {
         - Use real enum values discovered in the data
         - Each test MUST use DIFFERENT data — never reuse the same field values across tests
         - For "create" tests: base payload on a real record but change field values to create new unique data
-        - For "update" tests: pick a real existing ID and modify specific fields
+        - Treat records and IDs from sample_data as read-only. Never update, patch, delete, archive, or otherwise mutate them
+        - For update/delete tests: the same scenario must first create its own target, then mutate only that target
+        - For negative or unsupported-method tests that could mutate data if accepted: create a scenario-owned target first; if that setup is impossible, do not send the destructive request
         - For tests needing parent references: use real _id field values from sample_data
       `);
         }

package/dist/boat/api-tester/src/ai/curler.js

@@ -243,6 +243,10 @@ export class Curler {
       - Record important findings as you go
       - Be precise about what you expect vs what you observe
       - If a test requires data from another endpoint, use schemaFor to look it up before guessing
+      - Treat existing records, sample data, and IDs supplied by the plan as read-only
+      - Before PUT, PATCH, DELETE, archive, or another destructive request, create the target inside the current scenario and mutate only that target
+      - This also applies when testing an unsupported method: the server may unexpectedly accept it, so never probe destructively against pre-existing data
+      - If a scenario-owned target cannot be created, use stop rather than risking existing data
       </rules>
     `;
     }

package/dist/boat/api-tester/src/cli.js

@@ -38,7 +38,7 @@ export function createApiCommands(name = 'api') {
             runOptions.baseEndpoint ||= runOptions.endpoint;
         try {
             const config = await parser.loadConfig(runOptions);
-            console.log(ConfigCommand.render(config, { configPath: parser.getConfigPath(), root: parser.getProjectRoot(), json: options.json }));
+            console.log(ConfigCommand.render(config, { configPath: parser.getConfigPath(), siteConfigPath: parser.getSiteConfigPath(), root: parser.getProjectRoot(), json: options.json }));
         }
         catch (error) {
             console.error(error instanceof Error ? error.message : 'Unknown error');

package/dist/boat/api-tester/src/config.js

@@ -11,7 +11,8 @@ import path, { resolve } from 'node:path';
 import { pathToFileURL } from 'node:url';
 import { parseEnv } from 'node:util';
 import { ConfigMissingError, EXPLORBOT_CONFIG_PATHS, createModel, envConfigRequested, materializeKnowledge, missingConfigMessage, resolveConfigModels, resolveLangfuse, resolveModel, resolveOutputRoot, setOutputDir, } from "../../../src/config.js";
-import { findGlobalConfig, globalEnvPath, isGlobalConfigPath, registerSite, resolveSiteTarget } from "../../../src/global-config.js";
+import { findGlobalConfig, globalEnvPath, isGlobalConfigPath, loadSiteConfig, registerSite, resolveSiteTarget } from "../../../src/global-config.js";
+import { deepMerge } from "../../../src/utils/merge.js";
 function isAbsoluteEndpoint(value) {
     return !!value && (value.startsWith('http://') || value.startsWith('https://'));
 }
@@ -30,6 +31,7 @@ export class ApibotConfigParser {
     config = null;
     configPath = null;
     site = null;
+    siteConfigPath = null;
     constructor() { }
     static getInstance() {
         if (!ApibotConfigParser.instance) {
@@ -76,19 +78,21 @@ export class ApibotConfigParser {
                     dirs: loadedConfig.dirs,
                 };
             }
-            this.config = this.mergeWithDefaults(loadedConfig);
-            this.applyEnvSpec(this.config.api);
-            this.applyEnvHeaders(this.config.api);
+            let config = this.mergeWithDefaults(loadedConfig);
+            this.applyEnvSpec(config.api);
+            this.applyEnvHeaders(config.api);
             if (options?.baseEndpoint)
-                this.config.api.baseEndpoint = options.baseEndpoint.replace(/\/$/, '');
-            await resolveConfigModels(this.config.ai);
-            resolveLangfuse(this.config.ai);
-            this.configPath = resolvedPath;
+                config.api.baseEndpoint = options.baseEndpoint.replace(/\/$/, '');
+            await resolveConfigModels(config.ai);
+            resolveLangfuse(config.ai);
             this.site = null;
+            this.siteConfigPath = null;
             if (isGlobalConfigPath(resolvedPath)) {
-                this.enterGlobalMode(this.config, options?.endpoint);
+                config = await this.enterGlobalMode(config, options?.endpoint);
             }
-            this.validateConfig(this.config);
+            this.validateConfig(config);
+            this.config = config;
+            this.configPath = resolvedPath;
             setOutputDir(this.getOutputDir());
             return this.config;
         }
@@ -106,6 +110,9 @@ export class ApibotConfigParser {
     getConfigPath() {
         return this.configPath;
     }
+    getSiteConfigPath() {
+        return this.siteConfigPath;
+    }
     getOutputDir() {
         const config = this.getConfig();
         return path.join(this.getProjectRoot(), config.dirs?.output || 'output');
@@ -168,16 +175,27 @@ export class ApibotConfigParser {
             return;
         api.headers = { ...api.headers, ...parseHeaders(process.env.EXPLORBOT_API_HEADERS) };
     }
-    enterGlobalMode(config, endpoint) {
+    async enterGlobalMode(config, endpoint) {
         const site = resolveSiteTarget(endpoint);
         this.site = registerSite(site.baseUrl);
+        const { path, config: siteConfig } = await loadSiteConfig(this.site.dir, site.baseUrl);
+        this.siteConfigPath = path;
+        const overrides = {};
+        if (siteConfig.ai)
+            overrides.ai = siteConfig.ai;
+        if (siteConfig.api)
+            overrides.api = siteConfig.api;
+        const merged = deepMerge(config, overrides);
+        await resolveConfigModels(merged.ai);
+        resolveLangfuse(merged.ai);
         let baseEndpoint = site.baseUrl;
         const envUrl = process.env.EXPLORBOT_URL;
         if (envUrl && URL.parse(envUrl)?.origin === site.baseUrl)
             baseEndpoint = envUrl.replace(/\/$/, '');
-        config.dirs = { output: 'output', knowledge: 'knowledge' };
-        config.api = { ...config.api, baseEndpoint };
+        merged.dirs = { output: 'output', knowledge: 'knowledge' };
+        merged.api = { ...merged.api, baseEndpoint };
         materializeKnowledge(this.site.dir);
+        return merged;
     }
     async loadEnvConfig() {
         const provider = process.env.EXPLORBOT_AI_PROVIDER;
@@ -261,18 +279,6 @@ export class ApibotConfigParser {
         }
     }
     mergeWithDefaults(config) {
-        return this.deepMerge({ dirs: { output: 'output' }, api: {} }, config);
-    }
-    deepMerge(target, source) {
-        const result = { ...target };
-        for (const key in source) {
-            if (source[key] && typeof source[key] === 'object' && !Array.isArray(source[key]) && source[key].constructor === Object) {
-                result[key] = this.deepMerge(result[key] || {}, source[key]);
-            }
-            else {
-                result[key] = source[key];
-            }
-        }
-        return result;
+        return deepMerge({ dirs: { output: 'output' }, api: {} }, config);
     }
 }

package/dist/package.json

@@ -1,6 +1,6 @@
 {
   "name": "explorbot",
-  "version": "0.4.7",
+  "version": "0.4.9",
   "description": "CLI app built with React Ink, CodeceptJS, and Playwright",
   "license": "Elastic-2.0",
   "type": "module",

package/dist/rules/chief/general.md

@@ -2,6 +2,8 @@
 - Steps should specify exact HTTP methods, paths, and key payload details
 - Expected outcomes should be specific and verifiable (status codes, response fields, error messages)
 - For CRUD operations, each test should handle its own setup and teardown
+- Treat existing records and IDs discovered from the API, knowledge, or sample data as read-only
+- A scenario that updates, patches, deletes, archives, or otherwise mutates a record must create that target inside the same scenario first; omit the scenario if safe setup is impossible
 - Expect standard REST conventions: 200 OK, 201 Created, 204 No Content, 400 Bad Request, 404 Not Found, 422 Unprocessable Entity
 - NEVER propose scenarios that test the same thing. "Create a basic suite" and "Successful creation of a simple suite" are DUPLICATES. Each scenario must test a DISTINCT behavior or aspect.
 - Before finalizing, review all scenarios and remove any that overlap in what they actually verify.

package/dist/rules/researcher/pagination.md

@@ -2,5 +2,6 @@
 When a section is a list that continues beyond what is shown, add one line under its `> Container:` line:
 `> Pagination: controls` — it has page numbers (1, 2, 3), prev/next arrows, or a "load more" button.
 `> Pagination: infinite` — it has none of those and loads more as it is scrolled.
+Omit the line when the items already shown are the whole collection.
 Sorting, filtering and switching tabs are not pagination — omit the line then.
 </pagination>

package/dist/src/action.js

@@ -552,10 +552,11 @@ export const attachStepLogger = (target, assertionsTarget) => {
         }
         tag('step').log(step);
     };
-    codeceptjs.event.dispatcher.on(codeceptjs.event.step.passed, listener);
+    const onPassed = (step) => listener(step);
+    codeceptjs.event.dispatcher.on(codeceptjs.event.step.passed, onPassed);
     codeceptjs.event.dispatcher.on(codeceptjs.event.step.failed, listener);
     return () => {
-        codeceptjs.event.dispatcher.off(codeceptjs.event.step.passed, listener);
+        codeceptjs.event.dispatcher.off(codeceptjs.event.step.passed, onPassed);
         codeceptjs.event.dispatcher.off(codeceptjs.event.step.failed, listener);
     };
 };

package/dist/src/ai/fisherman/tools.js

@@ -196,13 +196,19 @@ export function createAskApiTool(fisherman, task) {
     return {
         askApi: tool({
             description: dedent `
-        Ask what data already exists, changing nothing.
-        Ask a question about existing records: which ones are there, what they are called, whether a particular one exists.
-        Use it before precondition() to see whether suitable data is already available, and whenever a step needs the exact name or id of a record that is already there.
+        Read the app's data over the API, changing nothing.
+        Answers questions about records: which ones are there, what they are called, whether a particular one exists.
+
+        Use it to:
+        - check whether suitable data already exists, before precondition() creates any
+        - get the exact name or id of a record a step must act on
+        - find out whether an action was stored, when the app reported success but the page does not show the result
+        - find out whether data exists at all, when a list or dropdown is empty
+
         It never creates, edits or deletes anything — precondition() does that.
       `,
             inputSchema: z.object({
-                question: z.string().describe('What to find out about data that already exists'),
+                question: z.string().describe('What to find out about the data'),
             }),
             execute: async ({ question }) => {
                 tag('info').log(`Ask API: ${question}`);

package/dist/src/ai/navigator.js

@@ -630,7 +630,7 @@ class Navigator {
         const cachedVerification = actionResult.getVerification(message);
         if (cachedVerification !== null) {
             tag('operation').log(`Reusing cached verification: ${cachedVerification ? 'PASS' : 'FAIL'}`);
-            return { verified: cachedVerification, successfulCodes: [], assertionSteps: [], totalAttempted: 0 };
+            return { verified: cachedVerification, inexpressible: false, results: [], successfulCodes: [], assertionSteps: [], totalAttempted: 0 };
         }
         const knowledge = this.knowledgeTracker.renderRelevantContext(actionResult);
         let experience = '';
@@ -741,9 +741,6 @@ class Navigator {
                 observability: {
                     agent: 'navigator',
                 },
-                catch: async (error) => {
-                    debugLog(error);
-                },
             });
         }
         finally {

package/dist/src/ai/pilot.js

@@ -126,14 +126,6 @@ export class Pilot {
       ${sessionLog || 'No actions recorded'}
       </session_log>
 
-      Decide and commit. "continue" extends the loop and burns iterations — choose it only when
-      evidence is genuinely insufficient to call pass/fail, not as a safety hedge.
-      - "pass" if final state proves the SCENARIO GOAL is accomplished. Set requestVerification.
-      - "fail" if scenario was attempted but goal not achieved.
-      - "skipped" if scenario is irrelevant/inapplicable, OR systematic infrastructure failures.
-      - "continue" only when a concrete missing piece of evidence (a verify/see) would change your verdict.
-      - Mixed evidence + final state shows success → pass. Mixed + final state unclear → continue with guidance.
-
       When deciding "pass", you MUST also set requestVerification to a one-sentence natural-language
       claim about the current page (e.g., "New item Foo is visible in the items list"). NOT
       code — do not write I.*, expect(), .then(), or any JavaScript. Choose the strongest single
@@ -352,7 +344,7 @@ export class Pilot {
     buildVerdictSystemPrompt(task) {
         return dedent `
       You are Pilot — final decision maker for test pass/fail. Review the evidence and commit to a
-      verdict; "continue" only when evidence is genuinely insufficient.
+      verdict.
 
       ${capabilityGroundingRule}
 
@@ -366,10 +358,11 @@ export class Pilot {
         DOM assertion can't be made.
         Do not pass when Tester achieved only a related navigation/filter/tab/status outcome instead of the
         requested action, workflow, or entity detail goal.
-      - "fail": scenario was attempted but the goal was not achieved.
+      - "fail": goal not achieved and no further step toward it is available on the current page.
       - "skipped": scenario is irrelevant to the app, OR systematic infrastructure failures (LLM errors,
         crashes) prevented testing. NOT for "test failed to interact" — that's "fail" or "continue".
-      - "continue": tester hasn't completed the goal; provide concrete guidance (which tool, what to check).
+      - "continue": goal incomplete but the control for the NEXT step is present on the current page, or a
+        concrete missing check would change your verdict. Guidance must name that step.
         If a verify() asserted a state that was ALREADY TRUE before the test, it proves nothing — reject.
 
       reason field: one short sentence, maximum 120 characters. Do NOT restate the decision
@@ -929,14 +922,14 @@ export class Pilot {
         const lines = [];
         for (const [assertion, passed] of Object.entries(currentState.verifications ?? {})) {
             if (passed)
-                lines.push(`state verification (passed): ${assertion}`);
+                lines.push(`verify: ${assertion}`);
         }
         for (const exec of testerConversation.getToolExecutions()) {
             if (!EVIDENCE_TOOLS.includes(exec.toolName) || !exec.wasSuccessful)
                 continue;
             const description = exec.input?.assertion || exec.input?.request || truncateJson(exec.input);
-            const result = exec.output?.message || exec.output?.analysis || exec.output?.result;
-            lines.push(`CHECK ${exec.toolName} (executed successfully): ${description}${result ? ` -> ${result}` : ''}`);
+            const analysis = exec.output?.analysis;
+            lines.push(`${exec.toolName}: ${description}${analysis ? ` -> ${analysis}` : ''}`);
         }
         return [...new Set(lines)].join('\n');
     }
@@ -1030,7 +1023,9 @@ export class Pilot {
       ${interactive ? '- Use askUser() only as last resort.' : ''}
 
       Diagnostic patterns (use <state>, executed/element/skipped fields, ariaDiff):
-      - Click failed + button in "disabled buttons" → required field missing. Instruct fill first.
+      - Scenario's target control in "disabled buttons" → a precondition is unmet; identify which before acting.
+        Other disabled controls often name the unsatisfied constraint; "active form" marks [required] fields.
+        Aim Tester at the constraint the page names, not the one the scenario assumed — note the difference in PROGRESS.
       - "overlay: none" but Tester targets an overlay → overlay closed; re-trigger.
       - "region:" in <state> → a large area appeared in place without navigation (subview, wizard step, panel). Direct Tester to act inside it; the rest of the page is still usable.
       - Action SUCCESS but ariaDiff empty → may have worked without visible DOM change; check result message.
@@ -1052,14 +1047,20 @@ export class Pilot {
       Tester tools: click, pressKey, form, see, verify, interact, context, research, xpathCheck,
       visualClick, back, getVisitedStates, reset, stop, finish, record.
       Use tool names exactly as listed. Do not invent combined names or aliases.
+      Reloading is not a tool: to re-read a page from the server, instruct Tester to run I.reloadPage() through form.
 
       ${capabilityGroundingRule}
 
       YOUR Pilot-only tools, both over the API:
 
-      askApi(question) — ask what data already exists. It changes nothing. Use it to check whether
-      suitable data is already there before creating any, and to get the exact name or id of an existing
-      record a step must act on.
+      askApi(question) — read the app's data over the API. It changes nothing. Use when:
+
+      - Before precondition() — check whether suitable data already exists.
+      - A step needs the exact name or id of an existing record.
+      - The app reported success but the page does not show the result — ask whether it was stored.
+      - A list or dropdown is empty — ask whether the data exists at all.
+
+      The page is not the only witness. A record missing from the page may still exist.
 
       precondition(description) — create FRESH disposable test data. Never request users. Use when:
 

package/dist/src/ai/planner/session-dedup.d.ts

@@ -1,4 +1,4 @@
-import type { Plan } from '../../test-plan.js';
+import { type Plan, type Test } from '../../test-plan.js';
 import type { Constructor } from '../researcher/mixin.js';
 export declare function WithSessionDedup<T extends Constructor>(Base: T): {
     new (...args: any[]): {
@@ -9,4 +9,5 @@ export declare function WithSessionDedup<T extends Constructor>(Base: T): {
         getPreviousSessionScenariosExcluding(plan: Plan): Set<string>;
     };
 } & T;
+export declare function formatSessionTest(plan: Plan, test: Test): string;
 export declare function clearSessionDedup(): void;

package/dist/src/ai/planner/session-dedup.js

@@ -1,3 +1,4 @@
+import { TestResult } from "../../test-plan.js";
 const previousPlans = [];
 export function WithSessionDedup(Base) {
     return class extends Base {
@@ -12,7 +13,7 @@ export function WithSessionDedup(Base) {
                 if (plan === this.currentPlan)
                     continue;
                 for (const test of plan.tests) {
-                    lines.push(`${plan.url || '/'} | ${test.style || 'default'} | ${test.scenario}`);
+                    lines.push(formatSessionTest(plan, test));
                 }
             }
             return lines.join('\n');
@@ -25,6 +26,22 @@ export function WithSessionDedup(Base) {
         }
     };
 }
+export function formatSessionTest(plan, test) {
+    const lastNote = Object.values(test.notes)
+        .filter((note) => note.message)
+        .pop();
+    let outcome = test.result;
+    if (!outcome)
+        outcome = 'pending';
+    if (!test.result && lastNote)
+        outcome = 'unfinished';
+    const line = `${plan.url || '/'} | ${test.style || 'default'} | ${outcome} | ${test.scenario}`;
+    if (!lastNote)
+        return line;
+    if (outcome !== TestResult.FAILED && outcome !== 'unfinished')
+        return line;
+    return `${line} — ${lastNote.message.slice(0, 140)}`;
+}
 export function clearSessionDedup() {
     previousPlans.length = 0;
 }

package/dist/src/ai/planner.js

@@ -316,8 +316,10 @@ export class Planner extends PlannerBase {
       You can suggest scenarios that can be tested only through web interface.
       You can't test emails, database, SMS, or any external services.
       Suggest scenarios that can be potentially verified by UI.
-      Focus on error or success messages as outcome.
-      Focus on URL page change or data persistency after page reload.
+      Prefer outcomes grounded in observed interface behavior.
+      Every expected outcome must be verifiable through the web interface.
+      If a page or subpage has not been observed, describe the expected visible result generically instead of inventing interface details.
+      Persistency after a reload counts only when the persisted state can be confirmed through the interface.
       If there are subpages (pages with same URL path) plan testing of those subpages as well
       Plan CRUD operations in order: create, read, update, delete.
       Do not invent specific route names, success messages, validation texts, badge counts, or welcome messages unless they are visible in research, visited pages, or prior observed flows.
@@ -328,7 +330,7 @@ export class Planner extends PlannerBase {
       If a scenario needs existing records, recipients, results, notifications, or other target data, propose it only when that data is visible, API preconditions can create it, or the scenario itself creates the record as its setup.
       If the page appears read-only, degraded, demo-limited, maintenance-like, or lacks write controls, prefer read-only scenarios such as opening panels, inspecting visible lists, filtering, searching, or verifying current state.
       Do not assume hidden data exists just because a control is present.
-      For scenarios that act on existing items or search/filter by existing values, use only item names or values visible in research, visited pages, or prior observed flows.
+      Do not put record IDs or unique record names in test plans. Describe which record is needed and let Pilot choose it during execution; name a specific record only when research shows a small, complete list of available records.
       If the list is empty or no concrete item names are visible, do not invent "known" or "existing" items. Prefer empty-state, no-match search, clear-search, or read-only list behavior scenarios.
       Search, filter, sorting, tab, and list scenarios must start from a stable page where those controls are visible; avoid transient create/edit/new URLs unless the scenario tests that form.
       For option values and list items, use only visible or previously observed data; do not add create/update/delete setup unless the user explicitly requests that workflow.
@@ -540,7 +542,9 @@ export class Planner extends PlannerBase {
         const sessionTests = this.getSessionTestsSummary();
         if (sessionTests) {
             conversation.addUserText(dedent `
-        Tests already planned in this session across all pages. DO NOT duplicate any of these:
+        Tests already planned in this session across all pages, with how each one ended. DO NOT duplicate any of these.
+        A failed test means the app or the harness could not do what it tried: do not re-propose the same behavior on another page unless you can name what makes it work this time.
+        A failed or unfinished test carries the last thing it observed after the dash — read it before deciding that the behavior is worth trying again.
 
         <session_tests>
         ${sessionTests}

package/dist/src/ai/provider.js

@@ -32,6 +32,11 @@ function createHarmonyChannelFallbackTool() {
         execute: async () => ({ message: 'Noted. Continue with your next action.' }),
     });
 }
+function withHarmonyChannelFallback(tools) {
+    if (tools?.commentary)
+        return tools;
+    return { ...tools, commentary: createHarmonyChannelFallbackTool() };
+}
 let telemetryRegistered = false;
 let beforeExitFlushHooked = false;
 let activeOtelSdk = null;
@@ -367,8 +372,8 @@ export class Provider {
         setActivity(`🤖 Asking ${modelName} with dynamic tools`, 'ai');
         promptLog(`Using model: ${modelName}`);
         let toolsWithCommentary = tools;
-        if (!tools?.commentary && options.toolChoice !== 'required')
-            toolsWithCommentary = { ...tools, commentary: createHarmonyChannelFallbackTool() };
+        if (options.toolChoice !== 'required')
+            toolsWithCommentary = withHarmonyChannelFallback(tools);
         const toolNames = Object.keys(toolsWithCommentary || {});
         tag('debug').log(`Tools enabled: [${toolNames.join(', ')}]`);
         promptLog('Available tools:', toolNames);
@@ -378,9 +383,10 @@ export class Provider {
         const stopConditions = [isStepCount(maxRoundtrips)];
         if (extraStop)
             stopConditions.push(extraStop);
-        const config = this.buildGenerateConfig({ tools: toolsWithCommentary, maxOutputTokens: 16384, toolChoice: 'auto', experimental_repairToolCall: repairToolCall }, { stopWhen: stopConditions, model }, options);
+        let config = this.buildGenerateConfig({ tools: toolsWithCommentary, maxOutputTokens: 16384, toolChoice: 'auto', experimental_repairToolCall: repairToolCall }, { stopWhen: stopConditions, model }, options);
         let attemptMessages = messages;
         let invalidRequestFeedbackAdded = false;
+        let requiredToolChoiceRelaxed = false;
         const executedStepMessages = [];
         try {
             let response = await this.withModelRequestSlot(() => withRetry(async () => {
@@ -399,6 +405,11 @@ export class Provider {
                         invalidRequestFeedbackAdded = amended !== attemptMessages;
                         attemptMessages = amended;
                     }
+                    if (!requiredToolChoiceRelaxed && isRequiredToolChoiceError(error)) {
+                        requiredToolChoiceRelaxed = true;
+                        config = { ...config, tools: withHarmonyChannelFallback(tools), toolChoice: 'auto' };
+                        tag('warning').log('Provider rejected required tool choice — retrying with automatic tool choice and channel fallback');
+                    }
                     throw error;
                 }));
                 this.recordUsage(options.agentName || 'unknown', modelName, result.usage);
@@ -422,7 +433,7 @@ export class Provider {
         }
         catch (error) {
             clearActivity();
-            if (error?.message?.includes('Tool choice is required')) {
+            if (isRequiredToolChoiceError(error)) {
                 return { text: '', toolCalls: [], toolResults: [], responseMessages: executedStepMessages, usage: null };
             }
             if (error?.name === 'AbortError')
@@ -659,6 +670,9 @@ function withInvalidRequestFeedback(messages, error) {
         },
     ];
 }
+function isRequiredToolChoiceError(error) {
+    return error instanceof Error && error.message.includes('Tool choice is required');
+}
 function repairChannelMarker({ toolCall, tools }) {
     const markerIndex = toolCall.toolName.indexOf('<|channel|>');
     if (markerIndex <= 0)

package/dist/src/ai/rules.js

@@ -177,6 +177,7 @@ export const capabilityGroundingRule = dedent `
   When a scenario depends on a named action, menu item, status, option, workflow, or feature,
   that capability must be visible or explicitly confirmed in the current research/page context
   for the same target entity type.
+  Ground on the scenario's outcome, not a planned step's control label — a missing label is not a missing capability.
 
   Do not transfer capabilities between similar entities, rows, lists, detail pages, or menus.
   Do not replace a requested action with a synonym or related action unless the UI explicitly