explorbot 0.4.8 → 0.4.10

package/src/commands/config-command.ts

@@ -15,7 +15,7 @@ export class ConfigCommand extends BaseCommand {
 
   async execute(): Promise<void> {
     const parser = ConfigParser.getInstance();
-    tag('info').log(ConfigCommand.render(this.explorBot.getConfig(), { configPath: parser.getConfigPath(), root: parser.getProjectRoot() }));
+    tag('info').log(ConfigCommand.render(this.explorBot.getConfig(), { configPath: parser.getConfigPath(), siteConfigPath: parser.getSiteConfigPath(), root: parser.getProjectRoot() }));
   }
 
   static async summary(options: { config?: string; path?: string; url?: string; json?: boolean } = {}): Promise<string> {
@@ -28,13 +28,16 @@ export class ConfigCommand extends BaseCommand {
       return load(site.url);
     });
 
-    return ConfigCommand.render(config, { configPath: parser.getConfigPath(), root: parser.getProjectRoot(), json: options.json });
+    return ConfigCommand.render(config, { configPath: parser.getConfigPath(), siteConfigPath: parser.getSiteConfigPath(), root: parser.getProjectRoot(), json: options.json });
   }
 
   static data(config: SummarizedConfig, options: ConfigSummaryOptions = {}): ConfigData {
     let configPath = '';
     if (options.configPath && existsSync(options.configPath)) configPath = options.configPath;
 
+    let siteConfigPath = '';
+    if (options.siteConfigPath && existsSync(options.siteConfigPath)) siteConfigPath = options.siteConfigPath;
+
     const dirs: Record<string, string> = {};
     if (options.root) {
       for (const [name, dir] of Object.entries({ output: 'output', ...config.dirs })) {
@@ -60,6 +63,7 @@ export class ConfigCommand extends BaseCommand {
 
     return {
       config: configPath,
+      siteConfig: siteConfigPath,
       url: config.playwright?.url || config.web?.url || config.api?.baseEndpoint || '',
       browser: config.playwright?.browser || '',
       headless: !config.playwright?.show,
@@ -82,6 +86,7 @@ export class ConfigCommand extends BaseCommand {
     const section = (title: string, entries: [string, string][]) => lines.push(...renderSection(title, entries));
 
     const general: [string, string][] = [['config', data.config || 'EXPLORBOT_* environment variables']];
+    if (data.siteConfig) general.push(['site config', data.siteConfig]);
     if (data.url) general.push(['url', data.url]);
     if (data.browser) {
       let window = 'visible';
@@ -118,12 +123,14 @@ export class ConfigCommand extends BaseCommand {
 
 interface ConfigSummaryOptions {
   configPath?: string | null;
+  siteConfigPath?: string | null;
   root?: string;
   json?: boolean;
 }
 
 export interface ConfigData {
   config: string;
+  siteConfig: string;
   url: string;
   browser: string;
   headless: boolean;

package/src/commands/init-command.ts

@@ -7,10 +7,7 @@ import { getCliName } from '../utils/cli-name.ts';
 import { log, tag } from '../utils/logger.js';
 import { relativeToCwd } from '../utils/next-steps.ts';
 
-function defaultConfigTemplate(provider: string, esm: boolean): string {
-  let moduleExport = 'module.exports = config;';
-  if (esm) moduleExport = 'export default config;';
-
+function defaultConfigTemplate(provider: string): string {
   return `// 'provider/model-id' uses a bundled provider.
 // It is also possible to import provider as a module from Vercel AI SDK.
 // https://github.com/testomatio/explorbot/blob/main/docs/basics/providers.md
@@ -35,7 +32,7 @@ ${modelLines(provider)}
   },
 };
 
-${moduleExport}
+export default config;
 `;
 }
 
@@ -144,8 +141,7 @@ export function runInitCommand(options: InitCommandOptions): void {
       process.exit(1);
     }
 
-    const esm = extname(outPath) !== '.js' || isModuleProject(dirname(outPath));
-    writeFileSync(outPath, defaultConfigTemplate(provider, esm), 'utf8');
+    writeFileSync(outPath, defaultConfigTemplate(provider), 'utf8');
     log(`Created config file: ${relativeToCwd(outPath)}`);
 
     const envPath = resolve(process.cwd(), '.env');
@@ -271,6 +267,8 @@ function globalConfigTemplate(provider: string): string {
   const { envKey } = PROVIDERS[provider];
 
   return `// Global Explorbot configuration — used by every directory without its own explorbot.config.js.
+// Settings shared by every site. Each site extends them in
+// ~/.explorbot/sites/<host>/explorbot.config.js, written on its first run.
 // Models are written as 'provider/model-id' so they resolve without a local node_modules.
 // The key is read from ${envKey} in ~/.explorbot/.env
 // Model ids are snapshotted from the recommendations of this Explorbot version.
@@ -288,29 +286,10 @@ ${modelLines(provider)}
   },
 };
 
-module.exports = config;
+export default config;
 `;
 }
 
-function isModuleProject(configDir: string): boolean {
-  let currentDir = resolve(configDir);
-
-  while (true) {
-    const packagePath = join(currentDir, 'package.json');
-    if (existsSync(packagePath)) {
-      try {
-        return JSON.parse(readFileSync(packagePath, 'utf8')).type === 'module';
-      } catch {
-        return false;
-      }
-    }
-
-    const parentDir = dirname(currentDir);
-    if (parentDir === currentDir) return false;
-    currentDir = parentDir;
-  }
-}
-
 function writeEnvKey(key: string, value: string): void {
   const envPath = globalEnvPath();
   let content = '# AI provider API keys';

package/src/commands/sites-command.ts

@@ -1,4 +1,4 @@
-import { listSites, sitesDir } from '../global-config.js';
+import { findSiteConfig, listSites, sitesDir } from '../global-config.js';
 import { getCliName } from '../utils/cli-name.js';
 import { tag } from '../utils/logger.js';
 import { BaseCommand } from './base-command.js';
@@ -20,6 +20,11 @@ export class SitesCommand extends BaseCommand {
     tag('info').log(`Registered sites (${sites.length}):`);
     for (const site of sites) {
       tag('info').log(`  ${site.folder.padEnd(width)}  ${site.url}  last run ${site.lastRunAt.slice(0, 16).replace('T', ' ')}`);
+
+      let config = 'inherits global config';
+      const sitePath = findSiteConfig(site.dir);
+      if (sitePath) config = sitePath;
+      tag('info').log(`  ${' '.repeat(width)}  ${config}`);
     }
     tag('info').log('');
     tag('info').log(`Stored in ${sitesDir()}`);

package/src/config.ts

@@ -5,8 +5,9 @@ import { pathToFileURL } from 'node:url';
 import { parseEnv } from 'node:util';
 import dedent from 'dedent';
 import matter from 'gray-matter';
-import { type SiteRecord, findGlobalConfig, globalEnvPath, isGlobalConfigPath, registerSite, resolveSiteTarget } from './global-config.js';
+import { EXPLORBOT_CONFIG_PATHS, type SiteRecord, findGlobalConfig, globalEnvPath, isGlobalConfigPath, loadSiteConfig, registerSite, resolveSiteTarget } from './global-config.js';
 import { getCliName } from './utils/cli-name.js';
+import { deepMerge } from './utils/merge.js';
 import { log, tag } from './utils/logger.js';
 
 export const PROVIDERS: Record<string, ProviderInfo> = {
@@ -268,7 +269,7 @@ const config: ExplorbotConfig = {
 
 type RuleEntry = string | Record<string, string>;
 
-export const EXPLORBOT_CONFIG_PATHS = ['explorbot.config.js', 'explorbot.config.mjs', 'explorbot.config.ts'];
+export { EXPLORBOT_CONFIG_PATHS };
 
 export const EXPLORBOT_ENV_VARS: EnvVar[] = [
   { name: 'EXPLORBOT_AI_PROVIDER', required: true, description: 'Provider name; fills every model role from its recommended models. Turns on config-free mode' },
@@ -323,6 +324,7 @@ export class ConfigParser {
   private runtimeTarget: string | null = null;
   private site: SiteRecord | null = null;
   private siteStartPath = '/';
+  private siteConfigPath: string | null = null;
 
   private constructor() {}
 
@@ -407,16 +409,18 @@ export class ConfigParser {
         log(`Configuration built from EXPLORBOT_* environment variables. Output: ${outputRoot}`);
       }
 
-      this.config = this.resolveConfig(loadedConfig as ExplorbotConfig, options);
-      await resolveConfigModels(this.config.ai);
-      this.runtimeTarget = target;
-      this.configPath = sourcePath;
+      let config = this.resolveConfig(loadedConfig as ExplorbotConfig, options);
+      await resolveConfigModels(config.ai);
       this.site = null;
+      this.siteConfigPath = null;
 
       if (resolvedPath && isGlobalConfigPath(resolvedPath)) {
-        this.enterGlobalMode(this.config, target);
+        config = await this.enterGlobalMode(config, target);
       }
 
+      this.config = config;
+      this.runtimeTarget = target;
+      this.configPath = sourcePath;
       this.applyEnvSpec(this.config);
 
       // Restore original directory after successful config load
@@ -473,6 +477,10 @@ export class ConfigParser {
     return this.site;
   }
 
+  public getSiteConfigPath(): string | null {
+    return this.siteConfigPath;
+  }
+
   public resolveTargetPath(target?: string): string {
     if (!this.site) {
       const configured = this.config?.playwright?.url || this.config?.web?.url;
@@ -512,6 +520,7 @@ export class ConfigParser {
       ConfigParser.instance.runtimeTarget = null;
       ConfigParser.instance.site = null;
       ConfigParser.instance.siteStartPath = '/';
+      ConfigParser.instance.siteConfigPath = null;
     }
   }
 
@@ -563,16 +572,24 @@ export class ConfigParser {
     }
   }
 
-  private enterGlobalMode(config: ExplorbotConfig, target: string | null): void {
+  private async enterGlobalMode(config: ExplorbotConfig, target: string | null): Promise<ExplorbotConfig> {
     const site = resolveSiteTarget(target || undefined, config.web?.url || config.playwright?.url);
     this.site = registerSite(site.baseUrl);
     this.siteStartPath = site.path;
 
-    config.dirs = { knowledge: 'knowledge', experience: 'experience', output: 'output' };
-    config.playwright = { ...config.playwright, browser: config.playwright?.browser || 'chromium', url: site.baseUrl };
+    const { path: sitePath, config: siteConfig } = await loadSiteConfig(this.site.dir, site.baseUrl);
+    this.siteConfigPath = sitePath;
+
+    const merged = deepMerge(config, siteConfig);
+    await resolveConfigModels(merged.ai);
+    resolveLangfuse(merged.ai);
+
+    merged.dirs = { knowledge: 'knowledge', experience: 'experience', output: 'output' };
+    merged.playwright = { ...merged.playwright, browser: merged.playwright?.browser || 'chromium', url: site.baseUrl };
     materializeKnowledge(this.site.dir);
 
     log(`Global mode: ${site.baseUrl} stored in ${this.site.dir}`);
+    return merged;
   }
 
   private applyEnvSpec(config: ExplorbotConfig): void {
@@ -720,21 +737,7 @@ export class ConfigParser {
       },
     };
 
-    return this.deepMerge(defaults, config);
-  }
-
-  private deepMerge(target: any, source: any): any {
-    const result = { ...target };
-
-    for (const key in source) {
-      if (source[key] && typeof source[key] === 'object' && !Array.isArray(source[key]) && source[key].constructor === Object) {
-        result[key] = this.deepMerge(result[key] || {}, source[key]);
-      } else {
-        result[key] = source[key];
-      }
-    }
-
-    return result;
+    return deepMerge(defaults, config);
   }
 
   public ensureDirectory(path: string): void {

package/src/explorer.ts

@@ -21,6 +21,7 @@ import { Test, TestResult } from './test-plan.ts';
 import { BrowserRecoveryError, browserErrorMessage, isFatalBrowserError, isNavigationTransitionError } from './utils/browser-errors.ts';
 import { createDebug, log, tag } from './utils/logger.js';
 import { sleep, waitForPageReadiness } from './utils/page-readiness.ts';
+import { isInternalStep } from './utils/step-analyzer.ts';
 
 declare global {
   namespace NodeJS {
@@ -206,8 +207,7 @@ class Explorer {
 
     const stepHandler = (step: any, status?: string, error?: string, log?: string) => {
       if (!step.toCode) return;
-      if (step?.name?.startsWith('grab')) return;
-      if (step?.name?.startsWith('save')) return;
+      if (isInternalStep(step)) return;
 
       test.addStep(step.toCode(), step.duration, status, error, log);
 

package/src/global-config.ts

@@ -1,10 +1,14 @@
 import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
 import os from 'node:os';
-import { join } from 'node:path';
+import { join, resolve } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import dedent from 'dedent';
 
 const GLOBAL_CONFIG_NAMES = ['config.js', 'config.mjs', 'config.ts'];
 const SITE_DIRS = ['knowledge', 'experience', 'output'];
 
+export const EXPLORBOT_CONFIG_PATHS = ['explorbot.config.js', 'explorbot.config.mjs', 'explorbot.config.ts'];
+
 export function globalDir(): string {
   return join(os.homedir(), '.explorbot');
 }
@@ -18,11 +22,7 @@ export function globalConfigPath(): string {
 }
 
 export function findGlobalConfig(): string | null {
-  for (const name of GLOBAL_CONFIG_NAMES) {
-    const fullPath = join(globalDir(), name);
-    if (existsSync(fullPath)) return fullPath;
-  }
-  return null;
+  return firstExisting(globalDir(), GLOBAL_CONFIG_NAMES);
 }
 
 export function isGlobalConfigPath(configPath: string): boolean {
@@ -47,6 +47,52 @@ export function listSites(): SiteRecord[] {
     .sort((a, b) => b.lastRunAt.localeCompare(a.lastRunAt));
 }
 
+export function findSiteConfig(dir: string): string | null {
+  return firstExisting(dir, EXPLORBOT_CONFIG_PATHS);
+}
+
+function ensureSiteConfig(dir: string, baseUrl: string): string {
+  const existing = findSiteConfig(dir);
+  if (existing) return existing;
+
+  const path = join(dir, EXPLORBOT_CONFIG_PATHS[0]);
+  writeFileSync(path, siteConfigTemplate(baseUrl), 'utf8');
+  return path;
+}
+
+export async function loadSiteConfig(dir: string, baseUrl: string): Promise<{ path: string; config: any }> {
+  const path = ensureSiteConfig(dir, baseUrl);
+  const module = await import(pathToFileURL(resolve(path)).href);
+  const config = module.default || module;
+  validateSiteConfig(config, path, baseUrl);
+  return { path, config };
+}
+
+function validateSiteConfig(config: any, configPath: string, baseUrl: string): void {
+  const url = config?.web?.url;
+  if (!url) {
+    throw new Error(dedent`
+      Site config is missing web.url.
+        ${configPath}
+
+      Add it so the config states which site it configures:
+        web: { url: '${baseUrl}' },
+    `);
+  }
+
+  const declared = URL.parse(url)?.origin;
+  if (declared === baseUrl) return;
+
+  throw new Error(dedent`
+    Site config declares a different site.
+      ${configPath}
+      web.url:  ${url}
+      site:     ${baseUrl}
+
+    Fix web.url, or explore ${url} to register it as its own site.
+  `);
+}
+
 export function findSiteWith(subpath: string): SiteRecord | undefined {
   return listSites().find((site) => existsSync(join(site.dir, subpath)));
 }
@@ -107,6 +153,35 @@ export function resolveSiteTarget(target?: string, defaultBaseUrl?: string): Sit
   return { baseUrl: site.url, path };
 }
 
+function firstExisting(dir: string, names: string[]): string | null {
+  for (const name of names) {
+    const fullPath = join(dir, name);
+    if (existsSync(fullPath)) return fullPath;
+  }
+  return null;
+}
+
+function siteConfigTemplate(baseUrl: string): string {
+  return `// Config for ${baseUrl}
+// Extends ~/.explorbot/config.js — set only what differs.
+const config = {
+  web: {
+    url: '${baseUrl}',
+  },
+
+  // ai: {
+  //   model: 'openrouter/openai/gpt-oss-120b',
+  // },
+
+  // playwright: {
+  //   show: true,
+  // },
+};
+
+export default config;
+`;
+}
+
 function readSite(folder: string): SiteRecord | null {
   const dir = join(sitesDir(), folder);
   const metaPath = join(dir, 'site.json');

package/src/reporter.ts

@@ -138,6 +138,7 @@ export class Reporter {
 
   protected combineStepsAndNotes(test: Test, lastScreenshotFile?: string): Step[] {
     const noteEntries = Object.entries(test.notes)
+      .filter(([, note]) => !note.observation)
       .map(([timestampKey, note]) => ({
         startTime: note.startTime,
         endTime: note.endTime,
@@ -272,9 +273,12 @@ export class Reporter {
         description: test.description,
         code: test.generatedCode || '',
         steps,
-        logs: Object.values(test.steps)
-          .map((stepData) => stepData.text)
-          .join('\n'),
+        logs: [
+          ...Object.values(test.steps).map((stepData) => stepData.text),
+          ...Object.values(test.notes)
+            .filter((note) => note.observation)
+            .map((note) => note.message),
+        ].join('\n'),
         files: Object.values(test.artifacts) || [],
         message: test.summary || this.extractLastNoteMessage(test) || '',
         meta,
@@ -341,7 +345,7 @@ export class Reporter {
   }
 
   private extractLastNoteMessage(test: Test): string {
-    const notes = Object.values(test.notes);
+    const notes = Object.values(test.notes).filter((note) => !note.observation);
     if (notes.length === 0) return '';
     return notes[notes.length - 1].message;
   }

package/src/utils/html.ts

@@ -1420,6 +1420,12 @@ function cleanElement(element: parse5TreeAdapter.Element): void {
     'aria-labelledby',
     'aria-describedby',
     'aria-owns',
+    'aria-checked',
+    'aria-expanded',
+    'aria-selected',
+    'aria-pressed',
+    'aria-current',
+    'aria-disabled',
     'role',
     'title',
     'href',

package/src/utils/logger.ts

@@ -215,7 +215,7 @@ class SpanDestination implements LogDestination {
     if (!step?.toCode) {
       return;
     }
-    const stepName = step?.name ? `I.${step.name}` : 'I.step';
+    const stepName = step?.title ? `I.${step.title}` : 'I.step';
     const stepInput = typeof step?.toCode === 'function' ? step.toCode() : entry.content;
     const errorFromStep = step?.error;
     const errorMessage =

package/src/utils/merge.ts

@@ -0,0 +1,13 @@
+export function deepMerge(target: any, source: any): any {
+  const result = { ...target };
+
+  for (const key in source) {
+    if (source[key] && typeof source[key] === 'object' && !Array.isArray(source[key]) && source[key].constructor === Object) {
+      result[key] = deepMerge(result[key] || {}, source[key]);
+      continue;
+    }
+    result[key] = source[key];
+  }
+
+  return result;
+}

package/src/utils/step-analyzer.ts

@@ -5,6 +5,8 @@ import { isDynamicId } from './xpath.ts';
 export const CODECEPT_TOOLS = ['click', 'hover', 'pressKey', 'form'] as const;
 export type CodeceptToolName = (typeof CODECEPT_TOOLS)[number];
 
+const INTERNAL_STEP_PREFIXES = ['grab', 'save'];
+
 const CODECEPT_FORM_COMMANDS: readonly string[] = ['I.fillField', 'I.type', 'I.selectOption', 'I.attachFile', 'I.checkOption', 'I.uncheckOption'];
 
 export function isCodeceptToolName(toolName: string): toolName is CodeceptToolName {
@@ -71,3 +73,9 @@ export function mergeUniqueStepsByCode(primary: SessionStep[], secondary: Sessio
   }
   return merged;
 }
+
+export function isInternalStep(step: { title?: string }): boolean {
+  const title = step?.title;
+  if (!title) return false;
+  return INTERNAL_STEP_PREFIXES.some((prefix) => title.startsWith(prefix));
+}

package/src/utils/url-matcher.ts

@@ -132,3 +132,10 @@ export function matchesNavigationUrl(expected: string, current: string): boolean
     .filter(Boolean);
   return recordSegments.length > 0 && recordSegments.every(isDynamicSegment);
 }
+
+export function isSameHostFamily(urlA: string, urlB: string): boolean {
+  const hostA = URL.parse(urlA)?.host.toLowerCase();
+  const hostB = URL.parse(urlB)?.host.toLowerCase();
+  if (!hostA || !hostB) return false;
+  return hostA === hostB || hostA.endsWith(`.${hostB}`) || hostB.endsWith(`.${hostA}`);
+}