explorbot 0.5.8 → 0.5.10

package/dist/src/utils/ux-probe.js

@@ -0,0 +1,155 @@
+import { HTML_SELECTORS, HTML_VISIBILITY_LIMITS } from "./html.js";
+export const UX_FOCUS_ATTR = 'data-explorbot-ux';
+const UX_LIMITS = {
+    touchTarget: 44,
+    pointerTarget: 24,
+    touchWidth: 1024,
+    targetGap: 8,
+    minFontSize: 12,
+    minLineHeight: 1.3,
+    maxEntries: 8,
+    minOpacity: HTML_VISIBILITY_LIMITS.minOpacity,
+};
+export function uxProbeArgs(scope) {
+    return { scope, controlsSelector: HTML_SELECTORS.interactiveControl, limits: UX_LIMITS };
+}
+export function renderUxMeasurements(measurements) {
+    const lines = [`viewport: ${measurements.width}px wide`];
+    for (const [check, entries] of Object.entries(measurements.checks)) {
+        if (!entries.length)
+            continue;
+        lines.push(`${check}:`, ...entries.map((entry) => `  - ${entry}`));
+    }
+    if (lines.length === 1)
+        lines.push('no measurable problems');
+    return lines.join('\n');
+}
+// Passed to page.evaluate as a function — must stay self-contained with no outer-scope references.
+export function measureUx({ scope, controlsSelector, limits }) {
+    const root = document.querySelector(scope) || document.body;
+    const width = window.innerWidth;
+    const touch = width <= limits.touchWidth;
+    const visibleWidth = document.documentElement.clientWidth;
+    const checks = {
+        'horizontal overflow': [],
+        'zoom and viewport meta': [],
+        'small targets': [],
+        'crowded targets': [],
+        'placeholder-only labels': [],
+        'small text': [],
+        'cramped line-height': [],
+        'truncated text without full text': [],
+        'layout-property transitions': [],
+    };
+    const add = (check, entry) => {
+        if (checks[check].length >= limits.maxEntries)
+            return;
+        if (checks[check].includes(entry))
+            return;
+        checks[check].push(entry);
+    };
+    const describe = (el) => {
+        const role = el.getAttribute('role') || el.tagName.toLowerCase();
+        const name = (el.getAttribute('aria-label') || el.innerText || el.getAttribute('placeholder') || el.getAttribute('title') || '').trim().replace(/\s+/g, ' ');
+        if (!name)
+            return `${role} (unnamed)`;
+        return `${role} "${name.slice(0, 40)}"`;
+    };
+    const visible = (el) => {
+        const rect = el.getBoundingClientRect();
+        if (!rect.width || !rect.height)
+            return false;
+        const style = window.getComputedStyle(el);
+        return style.visibility !== 'hidden' && style.display !== 'none' && Number(style.opacity) > limits.minOpacity;
+    };
+    if (root === document.body || root.matches('main, [role="main"]')) {
+        const docWidth = document.documentElement.scrollWidth;
+        if (docWidth > visibleWidth + 1)
+            add('horizontal overflow', `page is ${docWidth}px wide in a ${visibleWidth}px visible area`);
+        const meta = document.querySelector('meta[name="viewport"]')?.getAttribute('content');
+        if (!meta)
+            add('zoom and viewport meta', 'no viewport meta tag');
+        const metaRules = (meta || '').toLowerCase().replace(/\s/g, '');
+        if (metaRules.includes('user-scalable=no') || metaRules.includes('user-scalable=0'))
+            add('zoom and viewport meta', 'zoom disabled by user-scalable');
+        const maxScale = metaRules.split(',').find((part) => part.startsWith('maximum-scale='));
+        if (maxScale && Number(maxScale.split('=')[1]) <= 1)
+            add('zoom and viewport meta', `zoom capped by ${maxScale}`);
+    }
+    const minTarget = touch ? limits.touchTarget : limits.pointerTarget;
+    const controls = Array.from(root.querySelectorAll(controlsSelector)).filter(visible);
+    const boxes = controls.map((el) => ({ el, rect: el.getBoundingClientRect() }));
+    for (const { el, rect } of boxes) {
+        if (el.tagName === 'A' && window.getComputedStyle(el).display === 'inline')
+            continue;
+        if (rect.width >= minTarget && rect.height >= minTarget)
+            continue;
+        add('small targets', `${describe(el)} is ${Math.round(rect.width)}×${Math.round(rect.height)}px, below ${minTarget}px`);
+    }
+    if (touch) {
+        for (let i = 0; i < boxes.length; i++) {
+            for (let j = i + 1; j < boxes.length; j++) {
+                const a = boxes[i];
+                const b = boxes[j];
+                if (a.el.contains(b.el) || b.el.contains(a.el))
+                    continue;
+                const gapX = Math.max(b.rect.left - a.rect.right, a.rect.left - b.rect.right);
+                const gapY = Math.max(b.rect.top - a.rect.bottom, a.rect.top - b.rect.bottom);
+                const gap = Math.max(gapX, gapY);
+                if (gap >= limits.targetGap)
+                    continue;
+                if (a.rect.height >= limits.touchTarget && b.rect.height >= limits.touchTarget && a.rect.width >= limits.touchTarget && b.rect.width >= limits.touchTarget)
+                    continue;
+                add('crowded targets', `${describe(a.el)} and ${describe(b.el)} are ${Math.max(0, Math.round(gap))}px apart`);
+            }
+        }
+    }
+    for (const el of Array.from(root.querySelectorAll('input[placeholder], textarea[placeholder]'))) {
+        if (!visible(el))
+            continue;
+        if (el.getAttribute('aria-label') || el.getAttribute('aria-labelledby') || el.getAttribute('title'))
+            continue;
+        if (el.labels?.length)
+            continue;
+        add('placeholder-only labels', describe(el));
+    }
+    for (const el of Array.from(root.querySelectorAll('*'))) {
+        if (!visible(el))
+            continue;
+        const rect = el.getBoundingClientRect();
+        const style = window.getComputedStyle(el);
+        const overflows = rect.right > visibleWidth + 1 && rect.left < visibleWidth;
+        if (overflows && !(el.parentElement && el.parentElement.getBoundingClientRect().right > visibleWidth + 1))
+            add('horizontal overflow', `${describe(el)} ends at ${Math.round(rect.right)}px`);
+        const animated = style.transitionProperty.split(',').map((property) => property.trim());
+        const layout = animated.filter((property) => ['width', 'height', 'top', 'left', 'right', 'bottom', 'margin', 'padding'].includes(property));
+        const transitions = style.transitionDuration.split(',').some((duration) => Number.parseFloat(duration) > 0);
+        if (transitions && layout.length)
+            add('layout-property transitions', `${describe(el)} transitions ${layout.join(', ')}`);
+        const ownText = Array.from(el.childNodes)
+            .filter((node) => node.nodeType === Node.TEXT_NODE)
+            .map((node) => node.textContent || '')
+            .join('')
+            .trim();
+        if (!ownText)
+            continue;
+        const fontSize = Number.parseFloat(style.fontSize);
+        if (fontSize < limits.minFontSize)
+            add('small text', `"${ownText.slice(0, 40)}" at ${fontSize}px`);
+        const lineHeight = Number.parseFloat(style.lineHeight);
+        const multiline = rect.height > fontSize * 2.5;
+        if (multiline && lineHeight && lineHeight / fontSize < limits.minLineHeight)
+            add('cramped line-height', `"${ownText.slice(0, 40)}" at ${(lineHeight / fontSize).toFixed(2)}`);
+        const htmlEl = el;
+        if (style.textOverflow === 'ellipsis' && htmlEl.scrollWidth > htmlEl.clientWidth && !el.getAttribute('title'))
+            add('truncated text without full text', `"${ownText.slice(0, 40)}"`);
+    }
+    return { width, checks };
+}
+export function countRunningAnimations(scope) {
+    const root = document.querySelector(scope) || document.body;
+    return document.getAnimations().filter((animation) => {
+        const target = animation.effect?.target;
+        return animation.playState === 'running' && target && root.contains(target);
+    }).length;
+}

package/docs/reference/commands.md

@@ -93,6 +93,17 @@ npx explorbot explore /pay --knowledge 'Test card 4111 1111 1111 1111, any futur
 
 Like `--ws`, it is a program-level option rather than a per-command one: it works on every command — including `api`, `docs` and `prima` — and can go anywhere on the line. It is listed under `npx explorbot --help` rather than in each command's own help.
 
+### `--auth`
+
+Passes login details to the run in free form — credentials, a token, whatever the app asks for. It joins the run's knowledge on every page and API endpoint, stays in memory, and is never written to disk. `EXPLORBOT_AUTH` does the same from the environment; the flag wins when both are set.
+
+```bash
+npx explorbot explore /dashboard --auth 'login: admin@example.com password: secret123'
+npx explorbot api explore /orders --auth 'Bearer: eyJhbGciOi...'
+```
+
+Like `--knowledge`, it works on every command of `explorbot`, `api`, `docs` and `prima`, and can go anywhere on the line.
+
 ### `--session`
 
 Saves browser state (cookies, localStorage, sessionStorage) to a JSON file. The next run restores the session, so you skip login and setup steps.
@@ -129,6 +140,7 @@ EXPLORBOT_AI_PROVIDER=openrouter \
 | `EXPLORBOT_EPHEMERAL` | Keep no state between runs — output goes to a fresh temp directory instead of the site dir |
 | `EXPLORBOT_KNOWLEDGE` | Inline knowledge text, applied to every page |
 | `EXPLORBOT_KNOWLEDGE_FILE` | Path to a knowledge markdown file |
+| `EXPLORBOT_AUTH` | Auth details in free form (credentials, a token), kept in memory for the run and never written to disk; same as --auth |
 | `EXPLORBOT_SPEC` | Docbot application spec directory or index.md, used as page knowledge |
 | `EXPLORBOT_API_SPEC` | OpenAPI spec path for the API boat |
 | `EXPLORBOT_API_HEADERS` | Headers sent with every API request, one "Name: value" per line |

package/docs/workflow/agentic-usage.md

@@ -57,6 +57,7 @@ No `init`, no config file, no project directory, no model IDs to look up. These
 | `EXPLORBOT_EPHEMERAL` | no | Keep no state between runs — output goes to a fresh temp directory instead of the site dir |
 | `EXPLORBOT_KNOWLEDGE` | no | Inline knowledge text, applied to every page |
 | `EXPLORBOT_KNOWLEDGE_FILE` | no | Path to a knowledge markdown file |
+| `EXPLORBOT_AUTH` | no | Auth details in free form (credentials, a token), kept in memory for the run and never written to disk; same as --auth |
 | `EXPLORBOT_SPEC` | no | Docbot application spec directory or index.md, used as page knowledge |
 | `EXPLORBOT_API_SPEC` | no | OpenAPI spec path for the API boat |
 | `EXPLORBOT_API_HEADERS` | no | Headers sent with every API request, one "Name: value" per line |
@@ -133,6 +134,12 @@ EXPLORBOT_AI_PROVIDER=openrouter \
   npx explorbot explore /admin/users
 ```
 
+`EXPLORBOT_AUTH` takes the same free-form credentials but keeps them in memory only — nothing lands in the knowledge directory. `--auth` is its flag twin:
+
+```bash
+EXPLORBOT_AUTH="login: admin@example.com password: secret123" npx explorbot explore /admin/users
+```
+
 `EXPLORBOT_KNOWLEDGE_FILE` points at a markdown file the agent wrote. Its frontmatter is preserved, so it can target specific URLs — see [Knowledge](./knowledge.md) for the format:
 
 ```bash

package/docs/workflow/knowledge.md

@@ -88,6 +88,12 @@ Send X-Api-Key: ${env.API_KEY} on every request'
 
 For runs driven from the environment — config-free, or on the global configuration — `EXPLORBOT_KNOWLEDGE` does the same job for the length of one run. See [Agentic usage](./agentic-usage.md).
 
+For credentials alone, `--auth` (or `EXPLORBOT_AUTH`) takes free-form text, applies it everywhere, and never writes it to disk:
+
+```bash
+npx explorbot explore /dashboard --auth 'login: admin@example.com password: secret123'
+```
+
 ## URL Patterns
 
 | Pattern | Matches |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "explorbot",
-  "version": "0.5.8",
+  "version": "0.5.10",
   "description": "CLI app built with React Ink, CodeceptJS, and Playwright",
   "license": "Elastic-2.0",
   "type": "module",

package/rules/navigator/verification-actions.md

@@ -55,9 +55,11 @@ Checks that page title contains expected text.
 I.seeInSource(<text>)
 
 Checks that page source contains expected text (including hidden elements).
+Pass plain text only — never markup, tags, attributes or class names. The HTML you are shown is rewritten
+and will not match the live source character for character. Use it only for text that no visible element carries.
 
 <example>
-  I.seeInSource('<div class="hidden">');
+  I.seeInSource('Draft saved');
 </example>
 
 ### I.dontSee
@@ -99,10 +101,10 @@ Checks that an input field does NOT contain the specified value.
 
 I.dontSeeInSource(<text>)
 
-Checks that page source does NOT contain expected text.
+Checks that page source does NOT contain expected text. Plain text only, same as I.seeInSource.
 
 <example>
-  I.dontSeeInSource('error-class');
+  I.dontSeeInSource('Session expired');
 </example>
 
 ### Asserting the state of a control

package/rules/planner/ignored-pages.md

@@ -0,0 +1,6 @@
+<ignored_pages>
+Do not plan tests for these pages, return an empty scenarios list:
+- sign in
+- sign up
+- password recovery
+</ignored_pages>

package/rules/quartermaster/ux/accessibility.md

@@ -0,0 +1,8 @@
+<accessibility priority="CRITICAL">
+- Every control has an accessible name; icon-only controls included. Confirm against the ARIA tree, not the picture.
+- Interactive elements are reachable by keyboard in visual order, with a visible focus indicator.
+- Headings form a sequential outline without skipped levels.
+- Meaning never rests on color alone: state, error and status also carry text or an icon.
+- Dialogs and multi-step flows offer a visible way out.
+- Contrast is reported only from axe results, never judged by eye.
+</accessibility>

package/rules/quartermaster/ux/charts.md

@@ -0,0 +1,6 @@
+<charts priority="LOW">
+Apply only when a chart or data visualisation is inside the focus area:
+- Series are identified by a legend or direct labels, not by color alone.
+- Exact values are reachable without hover.
+- An empty or failed dataset shows a message, not a blank frame.
+</charts>

package/rules/quartermaster/ux/copy.md

@@ -0,0 +1,9 @@
+<copy priority="HIGH">
+Judge what a text does for the user, not how it is phrased:
+- Intent: a label or heading must tell what happens or what the area is for; a text that names no outcome fails.
+- Duplication: two texts in the focus area that say the same thing, or a text that repeats the visible label beside it.
+- Relevance: a text that explains nothing the interface does not already show, or concerns something outside the user's task here.
+- Filler: generic, promotional or self-congratulatory text that could sit unchanged on any product.
+- Overload: more text than the task needs, competing with the controls for attention.
+Quote the text and state which check it fails. Never propose replacement wording.
+</copy>

package/rules/quartermaster/ux/forms.md

@@ -0,0 +1,7 @@
+<forms priority="MEDIUM">
+- Each input has a visible label that stays when the user types; a placeholder alone is not a label.
+- Errors appear next to the input they concern and state cause and recovery.
+- Required inputs are marked before submission, not discovered after it.
+- Long or complex forms reveal options progressively instead of all at once.
+- A completed submit confirms its result.
+</forms>

package/rules/quartermaster/ux/interaction.md

@@ -0,0 +1,7 @@
+<interaction priority="CRITICAL">
+- Targets meet the size and spacing limits in measurements: touch limits at touch widths, pointer limits above.
+- Every action is reachable by click or tap; hover may reveal, never be the only way in.
+- An action that waits on the server shows it is working and blocks a second submit.
+- Destructive or irreversible actions ask for confirmation or offer undo.
+- Disabled controls look disabled and say, or make evident, why.
+</interaction>

package/rules/quartermaster/ux/motion.md

@@ -0,0 +1,5 @@
+<motion priority="MEDIUM">
+Judge only what measurements report:
+- Transitions on layout properties, which make content jump while they run.
+- Animations that keep running when the user asks for reduced motion.
+</motion>

package/rules/quartermaster/ux/navigation.md

@@ -0,0 +1,7 @@
+<navigation priority="HIGH">
+- The current location is marked in the navigation that leads to it.
+- Back behaves predictably; a state worth sharing or returning to has its own URL.
+- Navigation offers few top-level choices; grouping beats a long flat list.
+- Destructive or session-ending items are set apart from ordinary destinations.
+- Modals do not carry primary navigation flows.
+</navigation>

package/rules/quartermaster/ux/readability.md

@@ -0,0 +1,6 @@
+<readability priority="MEDIUM">
+Judge only what measurements report:
+- Body text below the minimum size.
+- Multi-line text with cramped line-height.
+- Truncated text that hides meaning and offers no way to read it in full.
+</readability>

package/rules/quartermaster/ux/responsive.md

@@ -0,0 +1,7 @@
+<responsive priority="HIGH">
+- Compare breakpoints: the focus area must stay usable at every width it was measured at.
+- No horizontal scroll; content fits the viewport width. Zoom must never be disabled.
+- When navigation does not fit, it collapses behind one named, reachable control; nothing primary disappears without a way back to it.
+- On narrow screens core content comes first; secondary panels fold, they do not push the task below the fold.
+- Fixed or sticky bars never cover the content or controls beneath them.
+</responsive>

package/src/ai/judge.ts

@@ -15,7 +15,8 @@ export const JUDGE_PAGE_CAP = 12000;
 export class Decision {
   constructor(
     readonly value: string | null,
-    readonly confidence: number
+    readonly confidence: number,
+    readonly answered = true
   ) {}
 
   get approved(): boolean {
@@ -46,10 +47,17 @@ export class Judge {
   }
 
   async decide(question: string, options: string[] | boolean | null, state: unknown): Promise<Decision> {
-    if (!this.enabled.direct) return new Decision(null, 0);
+    if (!this.enabled.direct) return new Decision(null, 0, false);
     return this.consult(question, options, state);
   }
 
+  async assert(statement: string, state: unknown): Promise<boolean | null> {
+    const decision = await this.decide(statement, null, state);
+    if (decision.approved) return true;
+    if (decision.answered && decision.confidence < 1 - this.threshold) return false;
+    return null;
+  }
+
   async pick(question: string, options: string[], state: unknown): Promise<number | null> {
     if (new Set(options).size < options.length) return null;
     const decision = await this.decide(question, [...options, UNDECIDED], state);
@@ -76,7 +84,7 @@ export class Judge {
     if (list?.includes(UNDECIDED)) instructions = `${question} ${UNDECIDED_HINT}`;
 
     const answer = await this.provider.decide(state, instructions, list).catch((error: unknown) => this.recordFailure(error));
-    if (!answer) return new Decision(null, 0);
+    if (!answer) return new Decision(null, 0, false);
     Observability.getSpan()?.setAttribute('langfuse.observation.output', JSON.stringify(answer));
     if (answer.probability <= this.threshold) return new Decision(null, answer.probability);
     if (answer.value === UNDECIDED) return new Decision(null, answer.probability);

package/src/ai/navigator.ts

@@ -857,8 +857,8 @@ class Navigator implements Agent {
     }
 
     const totalAttempted = Math.min(codeBlocks.length, this.verifyAttempts);
-    const majorityNeeded = Math.floor(totalAttempted / 2) + 1;
-    let verified = successfulCodes.length >= majorityNeeded;
+    let verified = successfulCodes.length > 0 && !failures;
+    if (successfulCodes.length && failures) verified = await this.provider.oracle(message, { assertions: results.map(({ code, passed, error }) => ({ code, passed, error })) }, { agentName: 'navigator', telemetry: { functionId: 'navigator.verifyState' } });
     if (alreadyVerified) verified = true;
 
     const inexpressible = !alreadyVerified && totalAttempted === 0;

package/src/ai/pilot.ts

@@ -631,8 +631,10 @@ export class Pilot implements Agent {
       outcomes: z.array(
         z.object({
           expectation: z.string().describe('The expected outcome, repeated exactly as it was given'),
-          status: z.enum(['passed', 'failed', 'unverified', 'contradiction']).describe('passed = the evidence shows it happened to the thing it names, failed = the evidence shows it did not, unverified = the run never established either way, contradiction = the picture and the run disagree'),
-          evidence: z.string().nullable().describe('What settled it. For a contradiction, what each side shows. Null when there is nothing to add'),
+          status: z
+            .enum(['passed', 'failed', 'unverified', 'contradiction'])
+            .describe('passed = the evidence shows it happened to the thing it names, failed = the evidence shows the application did not do it, unverified = the run never established either way, contradiction = the picture and the run disagree'),
+          evidence: z.string().nullable().describe('What settled it. For unverified, what stopped the run from checking it. For a contradiction, what each side shows. Null when there is nothing to add'),
         })
       ),
     });
@@ -674,10 +676,14 @@ export class Pilot implements Agent {
       The log is written in the tester's own words, so an outcome can be satisfied by a step that describes it
       differently. Judge by what the steps show happened, not by whether the wording matches.
       Wording may differ, the subject may not. An outcome refers to what the scenario names or chooses; it holds
-      only when that same thing appears. A different route or a different thing reaching a similar state is "failed",
-      and so is an outcome about something the run never managed to choose or create.
+      only when that same thing appears. A different route or a different thing reaching a similar state is "failed".
+      An outcome about something the run never managed to choose or create never passes: it is "failed" when the
+      application refused it, "unverified" when the run's own trouble got in the way.
       Choose "unverified" only when the evidence neither shows the outcome happening nor shows it failing —
       that is a statement about the run, not about the application.
+      "failed" is a statement about the application. When the run's own trouble — an earlier step it could not
+      get past, a control it could not reach or operate — left the outcome untested, it is "unverified", and
+      the evidence names that blocker.
     `;
 
     const settle = (content: any, model: any) =>

package/src/ai/planner.ts

@@ -24,6 +24,7 @@ import type { Provider } from './provider.js';
 import { POSSIBLE_SECTIONS, type Researcher } from './researcher.ts';
 import { findSimilarStateHash } from './researcher/cache.ts';
 import { hasFocusedSection } from './researcher/focus.ts';
+import { RulesLoader } from '../utils/rules-loader.ts';
 import { capabilityGroundingRule, dataProtectionRules, fileUploadRule } from './rules.ts';
 import type { Scout } from './scout.ts';
 
@@ -44,6 +45,7 @@ const TasksSchema = z.object({
       })
     )
     .describe('List of testing scenarios'),
+  notPlannableReason: z.string().nullish().describe('When scenarios is empty, one sentence saying why this page has nothing to test; otherwise null.'),
 });
 
 const PlannerBase = WithSessionDedup(WithSubPages(Object as unknown as new (...args: any[]) => object));
@@ -210,6 +212,10 @@ export class Planner extends PlannerBase implements Agent {
         throw new Error('No tasks were created successfully');
       }
 
+      if (!this.currentPlan && aiResult.object.scenarios.length === 0 && aiResult.object.notPlannableReason) {
+        tag('warning').log(`Page is not plannable: ${aiResult.object.notPlannableReason}`);
+      }
+
       const defaultStartUrl = this.getDefaultStartUrl(state);
       const fromPlanning = aiResult.object.scenarios.map((s: any) => new Test(s.scenario, s.priority, s.expectedOutcomes, s.startUrl || defaultStartUrl, s.steps || []));
 
@@ -345,11 +351,14 @@ export class Planner extends PlannerBase implements Agent {
 
     conversation.addUserText(this.getSystemMessage(feature));
 
+    let ignoredPagesRule = '';
+    if (!feature) ignoredPagesRule = `\n${RulesLoader.loadRules('planner', ['ignored-pages'], state.url)}`;
+
     const planningPrompt = dedent`
       <task>
       Based on the page research, create ${this.MIN_TASKS}-${this.MAX_TASKS} exploratory testing scenarios.
       For each scenario provide specific steps and expected outcomes.
-      Exception: if the page reports the requested resource is missing, shows a failure state, or holds no content and no controls, return an empty scenarios list. Never invent tests for a page with nothing to exercise.
+      Exception: if the page reports the requested resource is missing, shows a failure state, or holds no content and no controls, return an empty scenarios list. Never invent tests for a page with nothing to exercise.${ignoredPagesRule}
       </task>
 
       <rules>

package/src/ai/provider.ts

@@ -2,12 +2,12 @@ import { AsyncLocalStorage } from 'node:async_hooks';
 import { OpenTelemetry } from '@ai-sdk/otel';
 import { LangfuseSpanProcessor } from '@langfuse/otel';
 import { NodeSDK } from '@opentelemetry/sdk-node';
-import { APICallError, NoObjectGeneratedError, asSchema, extractJsonMiddleware, generateObject, generateText, isStepCount, parsePartialJson, registerTelemetry, tool, wrapLanguageModel } from 'ai';
+import { APICallError, NoObjectGeneratedError, NoOutputGeneratedError, Output, asSchema, extractJsonMiddleware, generateText, isStepCount, parsePartialJson, registerTelemetry, tool, wrapLanguageModel } from 'ai';
 import type { ModelMessage } from 'ai';
 import dedent from 'dedent';
 import { z } from 'zod';
 import { clearActivity, setActivity } from '../activity.ts';
-import { type AIConfig, configuredModels, modelName as getModelName } from '../config.js';
+import { type AIConfig, PROVIDERS, configuredModels, modelName as getModelName, modelProvider } from '../config.js';
 import { executionController } from '../execution-controller.ts';
 import { Observability } from '../observability.ts';
 import { Stats } from '../stats.ts';
@@ -16,6 +16,7 @@ import { createDebug, tag } from '../utils/logger.js';
 import { type RetryOptions, withRetry } from '../utils/retry.js';
 import { RulesLoader } from '../utils/rules-loader.ts';
 import { Conversation, NARRATION_TOOL, toToolExecution } from './conversation.js';
+import type { Judge } from './judge.ts';
 
 const debugLog = createDebug('explorbot:provider');
 const promptLog = createDebug('explorbot:provider:out');
@@ -84,6 +85,7 @@ function combinedAbortSignal(controller: AbortController): AbortSignal {
 
 export class Provider {
   private config: AIConfig;
+  private judge: Judge | null = null;
   private telemetryEnabled = false;
   private otelSdk: NodeSDK | null = null;
   private defaultRetryOptions: RetryOptions = {
@@ -99,6 +101,7 @@ export class Provider {
           error.message.includes('AI request timeout') ||
           error.message.includes('schema') ||
           error.message.includes('No object generated') ||
+          error.message.includes('No output generated') ||
           error.message.includes('No response text') ||
           error.message.includes('validate JSON')) &&
         !error.message.includes('output truncated at maxTokens')
@@ -125,6 +128,12 @@ export class Provider {
         prompt: 'hi',
       });
     } catch (error: any) {
+      const envKey = PROVIDERS[modelProvider(this.config.model)]?.envKey;
+      const apiKey = envKey && process.env[envKey];
+      if (envKey && !apiKey) throw new AiError(`AI connection failed: ${envKey} is not set. ${error.message}`);
+      if (apiKey && [401, 403].includes(error.statusCode)) {
+        throw new AiError(`AI connection failed: ${error.message}. ${envKey}=${apiKey.slice(0, 5)}**${apiKey.slice(-3)} was rejected, check the key`);
+      }
       throw new AiError(`AI connection failed: ${error.message}`);
     }
   }
@@ -142,6 +151,20 @@ export class Provider {
     return agentConfig?.model || this.config.model;
   }
 
+  setJudge(judge: Judge | null): void {
+    this.judge = judge;
+  }
+
+  async oracle(statement: string, state: unknown, options: any = {}): Promise<boolean> {
+    const judged = await this.judge?.assert(statement, state);
+    if (typeof judged === 'boolean') return judged;
+
+    const schema = z.object({ holds: z.boolean().describe('whether the statement is true') });
+    const content = `${statement}\n\n${JSON.stringify(state, null, 2)}`;
+    const response = await this.generateObject([{ role: 'user' as const, content }], schema, this.getAgenticModel(options.agentName), options).catch(() => null);
+    return response?.object?.holds === true;
+  }
+
   getAgenticModel(agentName?: string): any {
     if (agentName) {
       const agentConfig = this.config.agents?.[agentName as keyof typeof this.config.agents];
@@ -530,13 +553,13 @@ export class Provider {
     setActivity(`🤖 Asking ${modelName} for structured output`, 'ai');
     promptLog(`Using model: ${modelName}`);
 
-    const config = this.buildGenerateConfig({ schema }, { model: modelToUse }, options);
-
     try {
+      const { timeout, ...config } = this.buildGenerateConfig({ output: Output.object({ schema }) }, { model: modelToUse }, options);
       promptLog(messages[messages.length - 1].content);
       const response = await this.withModelRequestSlot(() =>
         withRetry(async () => {
-          return (await this.raceWithIdleTimeout((signal) => generateObject({ messages, ...config, abortSignal: signal }), config.timeout || 30000)) as any;
+          const result = (await this.raceWithIdleTimeout((signal) => generateText({ messages, ...config, abortSignal: signal }), timeout || 30000)) as any;
+          return { object: result.output, usage: result.usage };
         }, this.getRetryOptions(options))
       );
 
@@ -553,7 +576,7 @@ export class Provider {
       if (Provider.isContextLengthError(error)) {
         return this.recoverFromContextLength(error, messages, options, (m, o) => this.generateObject(m, schema, model, o));
       }
-      if (NoObjectGeneratedError.isInstance(error)) return this.recoverWithPlainJson(messages, schema, modelToUse, options);
+      if (NoObjectGeneratedError.isInstance(error) || NoOutputGeneratedError.isInstance(error)) return this.recoverWithPlainJson(messages, schema, modelToUse, options);
       throw new AiError(error.message || error.toString());
     }
   }