explorbot 0.1.13 → 0.1.15

package/dist/src/reporter.js

@@ -87,7 +87,7 @@ export class Reporter {
             this.client = new Client({ apiKey: process.env.TESTOMATIO || '', title: this.buildTitle() });
             const timeoutMs = Number(process.env.TESTOMATIO_TIMEOUT_MS || '15000');
             const timeoutPromise = new Promise((resolve) => setTimeout(() => resolve('timeout'), timeoutMs));
-            const result = await Promise.race([this.client.createRun().then(() => 'success'), timeoutPromise]);
+            const result = await Promise.race([this.client.createRun({ configuration: { exploratory: true } }).then(() => 'success'), timeoutPromise]);
             if (result === 'timeout') {
                 debugLog('Reporter run creation timed out');
                 return;
@@ -117,6 +117,7 @@ export class Reporter {
             message: note.message,
             status: note.status,
             screenshot: note.screenshot,
+            log: note.log,
         }))
             .sort((a, b) => a.startTime - b.startTime);
         const stepEntries = Object.entries(test.steps)
@@ -148,8 +149,16 @@ export class Reporter {
             if (noteEntry.screenshot) {
                 step.artifacts = [outputPath('states', noteEntry.screenshot)];
             }
+            if (noteEntry.log) {
+                step.log = noteEntry.log;
+            }
             steps.push(step);
         }
+        const verificationStep = this.buildVerificationStep(test, lastScreenshotFile);
+        if (verificationStep) {
+            steps.push(verificationStep);
+            return steps;
+        }
         if (lastScreenshotFile && steps.length > 0) {
             const lastStep = steps[steps.length - 1];
             const screenshotPath = outputPath('states', lastScreenshotFile);
@@ -162,6 +171,37 @@ export class Reporter {
         }
         return steps;
     }
+    buildVerificationStep(test, lastScreenshotFile) {
+        const v = test.verification;
+        if (!v)
+            return undefined;
+        const subSteps = [];
+        if (v.message)
+            subSteps.push({ category: 'framework', title: v.message, duration: 0 });
+        if (v.url) {
+            subSteps.push({
+                category: 'framework',
+                title: v.pageLabel ? `Navigated to ${v.pageLabel}` : 'Final page',
+                log: v.url,
+                duration: 0,
+            });
+        }
+        for (const detail of v.details) {
+            subSteps.push({ category: 'framework', title: detail, duration: 0 });
+        }
+        const screenshotFile = v.screenshot || lastScreenshotFile;
+        const step = {
+            category: 'user',
+            title: 'Verification',
+            duration: 0,
+            status: v.status || 'none',
+            steps: subSteps.length > 0 ? subSteps : undefined,
+        };
+        if (screenshotFile) {
+            step.artifacts = [outputPath('states', screenshotFile)];
+        }
+        return step;
+    }
     async reportTest(test, meta) {
         await this.startRun();
         if (!this.isRunStarted) {

package/dist/src/stats.js

@@ -10,11 +10,12 @@ export class Stats {
     static models = {};
     static recordTokens(_agent, model, usage) {
         if (!Stats.models[model]) {
-            Stats.models[model] = { input: 0, output: 0, total: 0 };
+            Stats.models[model] = { input: 0, output: 0, total: 0, cached: 0 };
         }
         Stats.models[model].input += usage.input;
         Stats.models[model].output += usage.output;
         Stats.models[model].total += usage.total;
+        Stats.models[model].cached = (Stats.models[model].cached ?? 0) + (usage.cached ?? 0);
     }
     static getElapsedTime() {
         const elapsed = Date.now() - Stats.startTime;

package/dist/src/test-plan.js

@@ -17,6 +17,7 @@ export class ActiveNote {
     message;
     status;
     screenshot;
+    log;
     constructor(task, message, status) {
         this.task = task;
         this.startTime = performance.now();
@@ -41,6 +42,7 @@ export class Task {
     steps;
     states;
     startUrl;
+    verification;
     timestampCounter = 0;
     activeNote;
     constructor(description, startUrl = '') {
@@ -67,6 +69,7 @@ export class Task {
             startTime: activeNote.getStartTime(),
             endTime,
             screenshot: activeNote.screenshot,
+            log: activeNote.log,
         };
         this.activeNote = undefined;
     }
@@ -80,13 +83,30 @@ export class Task {
             .map((n) => `- ${n}`)
             .join('\n');
     }
-    addNote(message, status = null, screenshot) {
-        const isDuplicate = Object.values(this.notes).some((note) => note.message === message && note.status === status);
+    addNote(message, status = null, screenshot, log) {
+        const isDuplicate = Object.values(this.notes).some((note) => note.message === message && note.status === status && note.log === log);
         if (isDuplicate)
             return;
         const now = performance.now();
         const timestamp = `${now}_${this.timestampCounter++}`;
-        this.notes[timestamp] = { message, status, startTime: now, endTime: now, screenshot };
+        this.notes[timestamp] = { message, status, startTime: now, endTime: now, screenshot, log };
+    }
+    addUrlNote(state, prevState) {
+        const fullUrl = state.fullUrl || state.url;
+        if (!fullUrl)
+            return;
+        let label;
+        if (state.title && state.title !== prevState?.title)
+            label = state.title;
+        else if (state.h1 && state.h1 !== prevState?.h1)
+            label = state.h1;
+        else if (state.h2 && state.h2 !== prevState?.h2)
+            label = state.h2;
+        else
+            label = state.title || state.h1 || state.h2;
+        if (!label)
+            return;
+        this.addNote(`Navigated to ${label}`, TestResult.PASSED, state.screenshotFile, fullUrl);
     }
     addState(state) {
         this.states.push(state);
@@ -95,6 +115,30 @@ export class Task {
         const timestamp = `${performance.now()}_${this.timestampCounter++}`;
         this.steps[timestamp] = { text, duration, status, error, log, artifacts, noteStartTime: this.activeNote?.getStartTime() };
     }
+    setActiveNoteScreenshot(screenshotFile) {
+        if (!this.activeNote || !screenshotFile)
+            return;
+        this.activeNote.screenshot = screenshotFile;
+    }
+    setVerification(message, status, state) {
+        this.verification ||= { message: '', status: null, details: [] };
+        this.verification.message = message;
+        this.verification.status = status;
+        if (!state)
+            return;
+        if (state.screenshotFile)
+            this.verification.screenshot = state.screenshotFile;
+        const fullUrl = state.fullUrl || state.url;
+        if (fullUrl)
+            this.verification.url = fullUrl;
+        this.verification.pageLabel = state.title || state.h1 || state.h2 || undefined;
+    }
+    addVerificationDetail(detail) {
+        if (!detail)
+            return;
+        this.verification ||= { message: '', status: null, details: [] };
+        this.verification.details.push(detail);
+    }
     getLog() {
         const merged = {};
         for (const [key, stepData] of Object.entries(this.steps)) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "explorbot",
-  "version": "0.1.13",
+  "version": "0.1.15",
   "description": "CLI app built with React Ink, CodeceptJS, and Playwright",
   "license": "Elastic-2.0",
   "type": "module",
@@ -67,6 +67,7 @@
     "@ai-sdk/openai": "^3.0",
     "@axe-core/playwright": "^4.11.0",
     "@codeceptjs/reflection": "^0.5.2",
+    "@faker-js/faker": "^10.4.0",
     "@inkjs/ui": "^2.0.0",
     "@langfuse/otel": "^4.5.1",
     "@openrouter/ai-sdk-provider": "^2.3.3",
@@ -78,7 +79,7 @@
     "@opentelemetry/sdk-trace-base": "^2.2.0",
     "@opentelemetry/semantic-conventions": "^1.38.0",
     "@scalar/openapi-parser": "^0.25.6",
-    "@testomatio/reporter": "^2.7.9-beta.2-markdown",
+    "@testomatio/reporter": "^2.7.9-beta.3-markdown",
     "ai": "^6.0.6",
     "axe-core": "^4.11.1",
     "bash-tool": "^1.3.15",

package/src/action.ts

@@ -1,5 +1,6 @@
 import fs from 'node:fs';
 import { join } from 'node:path';
+import { faker } from '@faker-js/faker';
 import { context, trace } from '@opentelemetry/api';
 import { highlight } from 'cli-highlight';
 import { container, recorder } from 'codeceptjs';
@@ -255,8 +256,8 @@ class Action {
         await asyncFn(page);
         await sleep(this.config.action?.delay || 500);
       } else {
-        const codeFunction = new Function('I', 'tryTo', 'retryTo', 'within', 'hopeThat', 'step', sanitizedCode);
-        codeFunction(this.actor, tryTo, retryTo, within, hopeThat, step);
+        const codeFunction = new Function('I', 'tryTo', 'retryTo', 'within', 'hopeThat', 'step', 'faker', sanitizedCode);
+        codeFunction(this.actor, tryTo, retryTo, within, hopeThat, step, faker);
         await recorder.add(() => sleep(this.config.action?.delay || 500));
         await recorder.promise();
       }

package/src/ai/conversation.ts

@@ -19,6 +19,7 @@ export class Conversation {
   messages: ModelMessage[];
   model: any;
   telemetryFunctionId?: string;
+  protectedPrefixCount = 0;
   private autoTrimRules: Map<string, number>;
 
   constructor(messages: ModelMessage[] = [], model?: any, telemetryFunctionId?: string) {
@@ -29,6 +30,10 @@ export class Conversation {
     this.autoTrimRules = new Map();
   }
 
+  protectPrefix(count: number): void {
+    this.protectedPrefixCount = count;
+  }
+
   addUserText(text: string): void {
     this.messages.push({
       role: 'user',
@@ -85,9 +90,11 @@ export class Conversation {
     const escapedTag = tagName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
     const regex = new RegExp(`<${escapedTag}>[\\s\\S]*?<\\/${escapedTag}>`, 'g');
     const replacementText = `<${tagName}>${replacement}</${tagName}>`;
+    const start = this.protectedPrefixCount;
 
     if (keepLast === 0) {
-      for (const message of this.messages) {
+      for (let i = start; i < this.messages.length; i++) {
+        const message = this.messages[i];
         if (typeof message.content === 'string') {
           message.content = message.content.replace(regex, replacementText);
         }
@@ -96,7 +103,7 @@ export class Conversation {
     }
 
     const allMatches: Array<{ messageIndex: number; startIndex: number; endIndex: number }> = [];
-    for (let i = 0; i < this.messages.length; i++) {
+    for (let i = start; i < this.messages.length; i++) {
       const message = this.messages[i];
       if (typeof message.content === 'string') {
         const matches = [...message.content.matchAll(new RegExp(`<${escapedTag}>[\\s\\S]*?<\\/${escapedTag}>`, 'g'))];
@@ -112,7 +119,7 @@ export class Conversation {
     const keepMatches = allMatches.slice(-keepCount);
     const keepSet = new Set(keepMatches.map((m) => `${m.messageIndex}:${m.startIndex}`));
 
-    for (let i = 0; i < this.messages.length; i++) {
+    for (let i = start; i < this.messages.length; i++) {
       const message = this.messages[i];
       if (typeof message.content === 'string') {
         const matches = [...message.content.matchAll(new RegExp(`<${escapedTag}>[\\s\\S]*?<\\/${escapedTag}>`, 'g'))];
@@ -137,7 +144,7 @@ export class Conversation {
 
   compactToolResults(keepLastN: number): void {
     const toolMessageIndexes: number[] = [];
-    for (let i = 0; i < this.messages.length; i++) {
+    for (let i = this.protectedPrefixCount; i < this.messages.length; i++) {
       if (this.messages[i].role === 'tool') toolMessageIndexes.push(i);
     }
     const compactUpTo = toolMessageIndexes.length - Math.max(0, keepLastN);
@@ -169,6 +176,16 @@ export class Conversation {
     }
   }
 
+  markLastMessageCacheable(): void {
+    const last = this.messages[this.messages.length - 1];
+    if (!last) return;
+    (last as any).providerOptions = {
+      ...(last as any).providerOptions,
+      anthropic: { cacheControl: { type: 'ephemeral' } },
+      bedrock: { cachePoint: { type: 'default' } },
+    };
+  }
+
   hasTag(tagName: string, lastN?: number): boolean {
     const escapedTag = tagName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
     const regex = new RegExp(`<${escapedTag}>`, 'g');

package/src/ai/historian/utils.ts

@@ -1,7 +1,14 @@
+import { isDynamicId } from '../../utils/xpath.ts';
 import type { ToolExecution } from '../conversation.ts';
 
 export function isNonReusableCode(code: string): boolean {
-  return /\bI\.clickXY\s*\(/.test(code);
+  if (/\bI\.clickXY\s*\(/.test(code)) return true;
+
+  for (const m of code.matchAll(/#([A-Za-z_][\w-]*)/g)) {
+    if (isDynamicId(m[1])) return true;
+  }
+
+  return false;
 }
 
 export function escapeString(str: string): string {