explorbot 0.5.8 → 0.5.10

package/dist/rules/quartermaster/ux/motion.md

@@ -0,0 +1,5 @@
+<motion priority="MEDIUM">
+Judge only what measurements report:
+- Transitions on layout properties, which make content jump while they run.
+- Animations that keep running when the user asks for reduced motion.
+</motion>

package/dist/rules/quartermaster/ux/navigation.md

@@ -0,0 +1,7 @@
+<navigation priority="HIGH">
+- The current location is marked in the navigation that leads to it.
+- Back behaves predictably; a state worth sharing or returning to has its own URL.
+- Navigation offers few top-level choices; grouping beats a long flat list.
+- Destructive or session-ending items are set apart from ordinary destinations.
+- Modals do not carry primary navigation flows.
+</navigation>

package/dist/rules/quartermaster/ux/readability.md

@@ -0,0 +1,6 @@
+<readability priority="MEDIUM">
+Judge only what measurements report:
+- Body text below the minimum size.
+- Multi-line text with cramped line-height.
+- Truncated text that hides meaning and offers no way to read it in full.
+</readability>

package/dist/rules/quartermaster/ux/responsive.md

@@ -0,0 +1,7 @@
+<responsive priority="HIGH">
+- Compare breakpoints: the focus area must stay usable at every width it was measured at.
+- No horizontal scroll; content fits the viewport width. Zoom must never be disabled.
+- When navigation does not fit, it collapses behind one named, reachable control; nothing primary disappears without a way back to it.
+- On narrow screens core content comes first; secondary panels fold, they do not push the task below the fold.
+- Fixed or sticky bars never cover the content or controls beneath them.
+</responsive>

package/dist/src/ai/judge.d.ts

@@ -5,7 +5,8 @@ export declare const JUDGE_PAGE_CAP = 12000;
 export declare class Decision {
     readonly value: string | null;
     readonly confidence: number;
-    constructor(value: string | null, confidence: number);
+    readonly answered: boolean;
+    constructor(value: string | null, confidence: number, answered?: boolean);
     get approved(): boolean;
     get rejected(): boolean;
 }
@@ -23,6 +24,7 @@ export declare class Judge {
     static fromConfig(config: AIConfig['decisionModel']): Judge | null;
     get toolEnabled(): boolean;
     decide(question: string, options: string[] | boolean | null, state: unknown): Promise<Decision>;
+    assert(statement: string, state: unknown): Promise<boolean | null>;
     pick(question: string, options: string[], state: unknown): Promise<number | null>;
     consult(question: string, options: string[] | boolean | null, state: unknown): Promise<Decision>;
     request(question: string, options: string[] | boolean | null, state: unknown): Promise<Decision>;

package/dist/src/ai/judge.js

@@ -10,9 +10,11 @@ export const JUDGE_PAGE_CAP = 12000;
 export class Decision {
     value;
     confidence;
-    constructor(value, confidence) {
+    answered;
+    constructor(value, confidence, answered = true) {
         this.value = value;
         this.confidence = confidence;
+        this.answered = answered;
     }
     get approved() {
         return this.value !== null;
@@ -43,9 +45,17 @@ export class Judge {
     }
     async decide(question, options, state) {
         if (!this.enabled.direct)
-            return new Decision(null, 0);
+            return new Decision(null, 0, false);
         return this.consult(question, options, state);
     }
+    async assert(statement, state) {
+        const decision = await this.decide(statement, null, state);
+        if (decision.approved)
+            return true;
+        if (decision.answered && decision.confidence < 1 - this.threshold)
+            return false;
+        return null;
+    }
     async pick(question, options, state) {
         if (new Set(options).size < options.length)
             return null;
@@ -75,7 +85,7 @@ export class Judge {
             instructions = `${question} ${UNDECIDED_HINT}`;
         const answer = await this.provider.decide(state, instructions, list).catch((error) => this.recordFailure(error));
         if (!answer)
-            return new Decision(null, 0);
+            return new Decision(null, 0, false);
         Observability.getSpan()?.setAttribute('langfuse.observation.output', JSON.stringify(answer));
         if (answer.probability <= this.threshold)
             return new Decision(null, answer.probability);

package/dist/src/ai/navigator.js

@@ -763,8 +763,9 @@ class Navigator {
             page?.setDefaultTimeout(originalTimeout);
         }
         const totalAttempted = Math.min(codeBlocks.length, this.verifyAttempts);
-        const majorityNeeded = Math.floor(totalAttempted / 2) + 1;
-        let verified = successfulCodes.length >= majorityNeeded;
+        let verified = successfulCodes.length > 0 && !failures;
+        if (successfulCodes.length && failures)
+            verified = await this.provider.oracle(message, { assertions: results.map(({ code, passed, error }) => ({ code, passed, error })) }, { agentName: 'navigator', telemetry: { functionId: 'navigator.verifyState' } });
         if (alreadyVerified)
             verified = true;
         const inexpressible = !alreadyVerified && totalAttempted === 0;

package/dist/src/ai/pilot.js

@@ -553,8 +553,10 @@ export class Pilot {
         const schema = z.object({
             outcomes: z.array(z.object({
                 expectation: z.string().describe('The expected outcome, repeated exactly as it was given'),
-                status: z.enum(['passed', 'failed', 'unverified', 'contradiction']).describe('passed = the evidence shows it happened to the thing it names, failed = the evidence shows it did not, unverified = the run never established either way, contradiction = the picture and the run disagree'),
-                evidence: z.string().nullable().describe('What settled it. For a contradiction, what each side shows. Null when there is nothing to add'),
+                status: z
+                    .enum(['passed', 'failed', 'unverified', 'contradiction'])
+                    .describe('passed = the evidence shows it happened to the thing it names, failed = the evidence shows the application did not do it, unverified = the run never established either way, contradiction = the picture and the run disagree'),
+                evidence: z.string().nullable().describe('What settled it. For unverified, what stopped the run from checking it. For a contradiction, what each side shows. Null when there is nothing to add'),
             })),
         });
         let pageEvidence = '';
@@ -593,10 +595,14 @@ export class Pilot {
       The log is written in the tester's own words, so an outcome can be satisfied by a step that describes it
       differently. Judge by what the steps show happened, not by whether the wording matches.
       Wording may differ, the subject may not. An outcome refers to what the scenario names or chooses; it holds
-      only when that same thing appears. A different route or a different thing reaching a similar state is "failed",
-      and so is an outcome about something the run never managed to choose or create.
+      only when that same thing appears. A different route or a different thing reaching a similar state is "failed".
+      An outcome about something the run never managed to choose or create never passes: it is "failed" when the
+      application refused it, "unverified" when the run's own trouble got in the way.
       Choose "unverified" only when the evidence neither shows the outcome happening nor shows it failing —
       that is a statement about the run, not about the application.
+      "failed" is a statement about the application. When the run's own trouble — an earlier step it could not
+      get past, a control it could not reach or operate — left the outcome untested, it is "unverified", and
+      the evidence names that blocker.
     `;
         const settle = (content, model) => this.provider
             .generateObject([{ role: 'user', content }], schema, model, {

package/dist/src/ai/planner.js

@@ -18,6 +18,7 @@ import { WithSubPages, getPlannedByStateHash, getRegisteredPlan, registerPlan }
 import { POSSIBLE_SECTIONS } from "./researcher.js";
 import { findSimilarStateHash } from "./researcher/cache.js";
 import { hasFocusedSection } from "./researcher/focus.js";
+import { RulesLoader } from "../utils/rules-loader.js";
 import { capabilityGroundingRule, dataProtectionRules, fileUploadRule } from "./rules.js";
 const debugLog = createDebug('explorbot:planner');
 const TasksSchema = z.object({
@@ -33,6 +34,7 @@ const TasksSchema = z.object({
             .describe('List of expected outcomes that can be verified. Each outcome should be simple, specific, and easy to check (e.g., "Success message appears", "URL changes to /dashboard", "Form field shows error"). Keep outcomes atomic - do not combine multiple checks into one.'),
     }))
         .describe('List of testing scenarios'),
+    notPlannableReason: z.string().nullish().describe('When scenarios is empty, one sentence saying why this page has nothing to test; otherwise null.'),
 });
 const PlannerBase = WithSessionDedup(WithSubPages(Object));
 export class Planner extends PlannerBase {
@@ -178,6 +180,9 @@ export class Planner extends PlannerBase {
             if (!aiResult?.object?.scenarios) {
                 throw new Error('No tasks were created successfully');
             }
+            if (!this.currentPlan && aiResult.object.scenarios.length === 0 && aiResult.object.notPlannableReason) {
+                tag('warning').log(`Page is not plannable: ${aiResult.object.notPlannableReason}`);
+            }
             const defaultStartUrl = this.getDefaultStartUrl(state);
             const fromPlanning = aiResult.object.scenarios.map((s) => new Test(s.scenario, s.priority, s.expectedOutcomes, s.startUrl || defaultStartUrl, s.steps || []));
             return { tests: fromPlanning, planName: aiResult.object.planName };
@@ -303,11 +308,14 @@ export class Planner extends PlannerBase {
         conversation.autoTrimTag('tested_scenarios', 10000);
         conversation.autoTrimTag('docs_context', 8000);
         conversation.addUserText(this.getSystemMessage(feature));
+        let ignoredPagesRule = '';
+        if (!feature)
+            ignoredPagesRule = `\n${RulesLoader.loadRules('planner', ['ignored-pages'], state.url)}`;
         const planningPrompt = dedent `
       <task>
       Based on the page research, create ${this.MIN_TASKS}-${this.MAX_TASKS} exploratory testing scenarios.
       For each scenario provide specific steps and expected outcomes.
-      Exception: if the page reports the requested resource is missing, shows a failure state, or holds no content and no controls, return an empty scenarios list. Never invent tests for a page with nothing to exercise.
+      Exception: if the page reports the requested resource is missing, shows a failure state, or holds no content and no controls, return an empty scenarios list. Never invent tests for a page with nothing to exercise.${ignoredPagesRule}
       </task>
 
       <rules>

package/dist/src/ai/provider.d.ts

@@ -3,6 +3,7 @@ import type { ModelMessage } from 'ai';
 import { type AIConfig } from '../config.js';
 import { type RetryOptions } from '../utils/retry.js';
 import { Conversation } from './conversation.js';
+import type { Judge } from './judge.js';
 declare class AiError extends Error {
 }
 export declare class ContextLengthError extends Error {
@@ -10,6 +11,7 @@ export declare class ContextLengthError extends Error {
 export declare function flushTelemetry(): Promise<void>;
 export declare class Provider {
     config: AIConfig;
+    judge: Judge | null;
     telemetryEnabled: boolean;
     otelSdk: NodeSDK | null;
     defaultRetryOptions: RetryOptions;
@@ -20,6 +22,8 @@ export declare class Provider {
     validateConnection(): Promise<void>;
     stop(): Promise<void>;
     getModelForAgent(agentName?: string): any;
+    setJudge(judge: Judge | null): void;
+    oracle(statement: string, state: unknown, options?: any): Promise<boolean>;
     getAgenticModel(agentName?: string): any;
     getVisionModel(): any;
     getConfiguredModels(): Record<string, string>;

package/dist/src/ai/provider.js

@@ -2,11 +2,11 @@ import { AsyncLocalStorage } from 'node:async_hooks';
 import { OpenTelemetry } from '@ai-sdk/otel';
 import { LangfuseSpanProcessor } from '@langfuse/otel';
 import { NodeSDK } from '@opentelemetry/sdk-node';
-import { APICallError, NoObjectGeneratedError, asSchema, extractJsonMiddleware, generateObject, generateText, isStepCount, parsePartialJson, registerTelemetry, tool, wrapLanguageModel } from 'ai';
+import { APICallError, NoObjectGeneratedError, NoOutputGeneratedError, Output, asSchema, extractJsonMiddleware, generateText, isStepCount, parsePartialJson, registerTelemetry, tool, wrapLanguageModel } from 'ai';
 import dedent from 'dedent';
 import { z } from 'zod';
 import { clearActivity, setActivity } from "../activity.js";
-import { configuredModels, modelName as getModelName } from '../config.js';
+import { PROVIDERS, configuredModels, modelName as getModelName, modelProvider } from '../config.js';
 import { executionController } from "../execution-controller.js";
 import { Observability } from "../observability.js";
 import { Stats } from "../stats.js";
@@ -75,6 +75,7 @@ function combinedAbortSignal(controller) {
 }
 export class Provider {
     config;
+    judge = null;
     telemetryEnabled = false;
     otelSdk = null;
     defaultRetryOptions = {
@@ -89,6 +90,7 @@ export class Provider {
                 error.message.includes('AI request timeout') ||
                 error.message.includes('schema') ||
                 error.message.includes('No object generated') ||
+                error.message.includes('No output generated') ||
                 error.message.includes('No response text') ||
                 error.message.includes('validate JSON')) &&
                 !error.message.includes('output truncated at maxTokens'));
@@ -112,6 +114,13 @@ export class Provider {
             });
         }
         catch (error) {
+            const envKey = PROVIDERS[modelProvider(this.config.model)]?.envKey;
+            const apiKey = envKey && process.env[envKey];
+            if (envKey && !apiKey)
+                throw new AiError(`AI connection failed: ${envKey} is not set. ${error.message}`);
+            if (apiKey && [401, 403].includes(error.statusCode)) {
+                throw new AiError(`AI connection failed: ${error.message}. ${envKey}=${apiKey.slice(0, 5)}**${apiKey.slice(-3)} was rejected, check the key`);
+            }
             throw new AiError(`AI connection failed: ${error.message}`);
         }
     }
@@ -125,6 +134,18 @@ export class Provider {
         const agentConfig = this.config.agents?.[agentName];
         return agentConfig?.model || this.config.model;
     }
+    setJudge(judge) {
+        this.judge = judge;
+    }
+    async oracle(statement, state, options = {}) {
+        const judged = await this.judge?.assert(statement, state);
+        if (typeof judged === 'boolean')
+            return judged;
+        const schema = z.object({ holds: z.boolean().describe('whether the statement is true') });
+        const content = `${statement}\n\n${JSON.stringify(state, null, 2)}`;
+        const response = await this.generateObject([{ role: 'user', content }], schema, this.getAgenticModel(options.agentName), options).catch(() => null);
+        return response?.object?.holds === true;
+    }
     getAgenticModel(agentName) {
         if (agentName) {
             const agentConfig = this.config.agents?.[agentName];
@@ -482,11 +503,12 @@ export class Provider {
         const modelName = getModelName(modelToUse);
         setActivity(`🤖 Asking ${modelName} for structured output`, 'ai');
         promptLog(`Using model: ${modelName}`);
-        const config = this.buildGenerateConfig({ schema }, { model: modelToUse }, options);
         try {
+            const { timeout, ...config } = this.buildGenerateConfig({ output: Output.object({ schema }) }, { model: modelToUse }, options);
             promptLog(messages[messages.length - 1].content);
             const response = await this.withModelRequestSlot(() => withRetry(async () => {
-                return (await this.raceWithIdleTimeout((signal) => generateObject({ messages, ...config, abortSignal: signal }), config.timeout || 30000));
+                const result = (await this.raceWithIdleTimeout((signal) => generateText({ messages, ...config, abortSignal: signal }), timeout || 30000));
+                return { object: result.output, usage: result.usage };
             }, this.getRetryOptions(options)));
             clearActivity();
             responseLog(response.object);
@@ -502,7 +524,7 @@ export class Provider {
             if (Provider.isContextLengthError(error)) {
                 return this.recoverFromContextLength(error, messages, options, (m, o) => this.generateObject(m, schema, model, o));
             }
-            if (NoObjectGeneratedError.isInstance(error))
+            if (NoObjectGeneratedError.isInstance(error) || NoOutputGeneratedError.isInstance(error))
                 return this.recoverWithPlainJson(messages, schema, modelToUse, options);
             throw new AiError(error.message || error.toString());
         }

package/dist/src/ai/quartermaster.d.ts

@@ -1,7 +1,9 @@
+import type { Locator, Page } from 'playwright';
 import type { ActionResult } from '../action-result.js';
 import type Explorer from '../explorer.js';
 import type { StateManager, StateTransition, WebPageState } from '../state-manager.js';
 import type { Task } from '../test-plan.js';
+import { type UxMeasurements } from '../utils/ux-probe.js';
 import type { Conversation, ToolExecution } from './conversation.js';
 import type { Provider } from './provider.js';
 export declare class Quartermaster {
@@ -10,20 +12,46 @@ export declare class Quartermaster {
     outputDir: string;
     pageAnalyses: Map<string, PageAnalysis>;
     unsubscribe: (() => void) | null;
-    explorer: Explorer | null;
+    explorer: Explorer;
     pendingAnalyses: Promise<void>[];
-    constructor(provider: Provider, options?: {
+    constructor(provider: Provider, explorer: Explorer, options?: {
         model?: any;
     });
-    start(explorer: Explorer, stateManager: StateManager): void;
+    start(stateManager: StateManager): void;
     stop(): void;
     onPageChange(event: StateTransition): void;
     runAxeAnalysis(state: WebPageState): Promise<void>;
     analyzeSession(task: Task, initialState: ActionResult, conversation: Conversation): Promise<AnalysisReport | null>;
+    auditUx(result: ActionResult, context: UxAuditContext): Promise<string>;
     generateSemanticAnalysis(axeViolations: AxeViolation[], executions: ToolExecution[], initialState: ActionResult): Promise<SemanticIssue[]>;
     addNotesToTask(task: Task, report: AnalysisReport): void;
     saveReport(stateHash: string, report: AnalysisReport): void;
     formatReportMarkdown(report: AnalysisReport): string;
+    axeViolations(page: Page, include?: string): Promise<AxeViolation[]>;
+    markUxFocus(result: ActionResult, context: UxAuditContext): Promise<UxFocus>;
+    pickUxFocus(context: UxAuditContext, mark: (locator: Locator) => Promise<boolean>): Promise<UxFocus | null>;
+    collectUxEvidence(page: Page): Promise<UxEvidence>;
+    animationsIgnoringReducedMotion(page: Page): Promise<number>;
+    captureUxBreakpoint(page: Page): Promise<UxBreakpoint>;
+    writeUxReport(result: ActionResult, focus: UxFocus, evidence: UxEvidence, context: UxAuditContext): Promise<string>;
+}
+export interface UxAuditContext {
+    focus?: string;
+    commands: string[];
+}
+interface UxFocus {
+    area: string;
+    reason: string;
+}
+interface UxBreakpoint {
+    measurements: UxMeasurements;
+    aria: string;
+    screenshot?: string;
+}
+interface UxEvidence {
+    breakpoints: UxBreakpoint[];
+    axe: AxeViolation[];
+    notes: string[];
 }
 interface AxeViolation {
     id: string;