explorbot 0.5.10 → 0.6.0

package/src/utils/error-page.ts

@@ -1,6 +1,7 @@
+import dedent from 'dedent';
 import { ActionResult } from '../action-result.js';
 import type { WebPageState } from '../state-manager.js';
-import { isBodyEmpty } from './html.js';
+import { isBodyEmpty, sanitizeHtmlString } from './html.js';
 import { extractStatePath } from './url-matcher.js';
 
 const HTTP_ERRORS = ['400 Bad Request', '401 Unauthorized', '403 Forbidden', '404 Not Found', '405 Method Not Allowed', '408 Request Timeout', '500 Internal Server Error', '502 Bad Gateway', '503 Service Unavailable', '504 Gateway Timeout'];
@@ -33,7 +34,7 @@ export function detectPageCondition(actionResult: ActionResult): PageCondition {
 
   if (!actionResult.html || isBodyEmpty(actionResult.html)) return 'loading';
 
-  const bodyMatch = actionResult.html.match(/<body[^>]*>([\s\S]*?)<\/body>/i);
+  const bodyMatch = sanitizeHtmlString(actionResult.html).match(/<body[^>]*>([\s\S]*?)<\/body>/i);
   if (bodyMatch && bodyMatch[1].trim().length < SMALL_PAGE_THRESHOLD) return 'loading';
 
   return 'ok';
@@ -67,3 +68,64 @@ export class ErrorPageError extends Error {
     this.name = 'ErrorPageError';
   }
 }
+
+export class BlockedError extends Error {
+  constructor(
+    message: string,
+    public readonly url: string,
+    public readonly blocker: string,
+    public readonly request: string
+  ) {
+    super(message);
+    this.name = 'BlockedError';
+  }
+}
+
+export class PlanningBlockedError extends BlockedError {
+  constructor(
+    url: string,
+    prerequisite: string,
+    public readonly feature?: string,
+    public readonly style?: string
+  ) {
+    let focus = '';
+    if (feature) focus = ` focused on "${feature}"`;
+    let styleNote = '';
+    if (style) styleNote = ` in ${style} style`;
+    super(
+      `Planner could not plan ${url}: ${prerequisite}`,
+      url,
+      prerequisite,
+      dedent`
+        Planner was asked to plan tests for ${url}${focus}${styleNote} but stopped: ${prerequisite}.
+        Look at the current page to find out what blocks planning, resolve it, then call plan() to re-plan this page. If it cannot be resolved, finish without planning.
+      `
+    );
+    this.name = 'PlanningBlockedError';
+  }
+}
+
+export class NavigationBlockedError extends BlockedError {
+  constructor(url: string, currentUrl: string, reason: string, tried: string[]) {
+    const lines = [`Navigator was asked to open ${url} but stopped on ${currentUrl}: ${reason}.`];
+    if (tried.length) lines.push(`Already tried: ${tried.join('; ')}.`);
+    lines.push(`Look at the current page to find out what keeps it from ${url}, resolve it, then open ${url}. If it cannot be resolved, finish without opening it.`);
+    super(`Navigation to ${url} failed: ${reason}`, url, reason, lines.join('\n'));
+    this.name = 'NavigationBlockedError';
+  }
+}
+
+export class TestStartBlockedError extends BlockedError {
+  constructor(scenario: string, url: string, currentUrl: string) {
+    super(
+      `Test "${scenario}" could not start: ${currentUrl} is not the page it starts on`,
+      url,
+      'not the start page of this test',
+      dedent`
+        Tester was asked to start the test "${scenario}" at ${url}, but the page open at ${currentUrl} is not where this test can start.
+        Look at the current page to find out what keeps the test from starting, resolve it, then open ${url}. When you finish, the test runs again from ${url}. If it cannot be resolved, finish and the test is skipped.
+      `
+    );
+    this.name = 'TestStartBlockedError';
+  }
+}

package/src/utils/html-diff.ts

@@ -1,6 +1,5 @@
 import { parse, serialize } from 'parse5';
 import type * as parse5TreeAdapter from 'parse5/lib/tree-adapters/default';
-import type { HtmlConfig } from '../config.ts';
 import { isTailwindClass, isTrashClass, minifyHtml } from './html.ts';
 import { isDynamicId, isGenericClass } from './xpath.ts';
 
@@ -203,9 +202,9 @@ export function computeHtmlFingerprint(html: string): string[] {
 /**
  * Compares two HTML documents and returns differences along with a diff subtree.
  */
-export async function htmlDiff(originalHtml: string, modifiedHtml: string, htmlConfig?: HtmlConfig): Promise<HtmlDiffResult> {
-  const originalDocument = parseDocument(originalHtml, htmlConfig);
-  const modifiedDocument = parseDocument(modifiedHtml, htmlConfig);
+export async function htmlDiff(originalHtml: string, modifiedHtml: string): Promise<HtmlDiffResult> {
+  const originalDocument = parseDocument(originalHtml);
+  const modifiedDocument = parseDocument(modifiedHtml);
 
   const originalRoot = getRootNodeForFlatten(originalDocument);
   const modifiedRoot = getRootNodeForFlatten(modifiedDocument);
@@ -308,7 +307,7 @@ function isLiveRegion(element: ElementNode): boolean {
  * Parse HTML into a document, wrapping fragments with html/body for consistency.
  * Uses custom sanitization that removes iframes for diff purposes.
  */
-function parseDocument(html: string, htmlConfig?: HtmlConfig): DocumentNode {
+function parseDocument(html: string): DocumentNode {
   const document = parse(html);
   const documentTitle = getDocumentTitle(document);
   sanitizeDocumentTreeForDiff(document);
@@ -432,42 +431,6 @@ const attributesDiffer = (current: ElementNode, previous: ElementNode): boolean
   return false;
 };
 
-const getDirectTextValues = (element: ElementNode): string[] => {
-  if (!element.childNodes || element.childNodes.length === 0) {
-    return [];
-  }
-
-  const values: string[] = [];
-
-  for (const child of element.childNodes) {
-    if (child.nodeName === '#text') {
-      const value = (child as parse5TreeAdapter.TextNode).value.trim();
-      if (value.length > 0) {
-        values.push(value);
-      }
-    }
-  }
-
-  return values;
-};
-
-const directTextDiffer = (current: ElementNode, previous: ElementNode): boolean => {
-  const currentText = getDirectTextValues(current);
-  const previousText = getDirectTextValues(previous);
-
-  if (currentText.length !== previousText.length) {
-    return true;
-  }
-
-  for (let i = 0; i < currentText.length; i++) {
-    if (currentText[i] !== previousText[i]) {
-      return true;
-    }
-  }
-
-  return false;
-};
-
 function filterContainerClasses(classes: string[]): string[] {
   return classes
     .filter((cls) => !/\d/.test(cls))
@@ -792,21 +755,6 @@ function findHtmlElement(document: DocumentNode): ElementNode | null {
   return null;
 }
 
-function findHeadElement(document: DocumentNode): ElementNode | null {
-  const html = findHtmlElement(document);
-  if (!html || !html.childNodes) {
-    return null;
-  }
-
-  for (const node of html.childNodes) {
-    if ('tagName' in node && node.tagName && node.tagName.toLowerCase() === 'head') {
-      return node as ElementNode;
-    }
-  }
-
-  return null;
-}
-
 function findBodyElement(document: DocumentNode): ElementNode | null {
   const html = findHtmlElement(document);
   if (!html || !html.childNodes) {

package/src/utils/html.ts

@@ -1,4 +1,3 @@
-import dedent from 'dedent';
 import { minify } from 'html-minifier-next';
 import { parse, parseFragment, serialize } from 'parse5';
 import type * as parse5TreeAdapter from 'parse5/lib/tree-adapters/default';

package/src/utils/logger.ts

@@ -4,8 +4,8 @@ import { type Span, context, trace } from '@opentelemetry/api';
 import chalk from 'chalk';
 import debug from 'debug';
 import dedent from 'dedent';
-import { marked } from 'marked';
 import stripAnsi from 'strip-ansi';
+import wrapAnsi from 'wrap-ansi';
 import { ConfigParser } from '../config.js';
 import { Observability } from '../observability.ts';
 import { RecentStepFilter } from './log-filters.ts';
@@ -81,6 +81,11 @@ class ConsoleDestination implements LogDestination {
   private forceEnabled = false;
   private quiet = false;
   private recentSteps = new RecentStepFilter();
+  private width: number;
+
+  constructor() {
+    this.width = process.stdout.columns || 80;
+  }
 
   isEnabled(): boolean {
     if (this.quiet) return false;
@@ -113,8 +118,8 @@ class ConsoleDestination implements LogDestination {
     if (entry.type === 'multiline' || entry.type === 'details') {
       const cleaned = stripAnsi(dedent(entry.content));
       const parsed = parseMarkdownToTerminal(cleaned);
-      content = parsed;
-      content = `${chalk.gray(content)}\n`;
+
+      content = `${chalk.dim('\n ┌'.padEnd(this.width, '─'))}\n${this.prefixLines(parsed, ' │ ', chalk.dim, false)}\n`;
     } else if (entry.type === 'success') {
       content = `${chalk.green(content)}\n`;
     } else if (entry.type === 'error') {
@@ -122,14 +127,32 @@ class ConsoleDestination implements LogDestination {
     } else if (entry.type === 'warning') {
       content = chalk.yellow(content);
     } else if (entry.type === 'step') {
-      content = chalk.gray(`   ${content}`);
+      content = this.prefixLines(content, '   ', chalk.gray);
     } else if (entry.type === 'operation') {
-      content = chalk.gray(`   · ${content}`);
+      content = this.prefixLines(content, '   · ', chalk.gray);
     } else if (entry.type === 'substep') {
-      content = chalk.gray(`   > ${content}`);
+      content = this.prefixLines(content, '   > ', chalk.gray);
     }
     console.log(content);
   }
+
+  private prefixLines(content: string, prefix: string, style: (s: string) => string, strip = true): string {
+    const prefixWidth = stripAnsi(prefix).length;
+    const available = Math.max(20, this.width - prefixWidth);
+    const lines = content
+      .split('\n')
+      .filter((line) => stripAnsi(line).trim() !== '')
+      .flatMap((line) => {
+        const text = strip ? stripAnsi(line) : line;
+        const wrapped = wrapAnsi(text, available, { hard: true });
+        return wrapped.split('\n').map((part) => style(`${prefix}${part}`));
+      });
+    if (lines.length > 10) {
+      const remaining = lines.length - 10;
+      return [...lines.slice(0, 10), chalk.gray(` │ --- +${remaining} lines ---`)].join('\n');
+    }
+    return lines.join('\n');
+  }
 }
 
 class DebugDestination implements LogDestination {
@@ -160,16 +183,11 @@ class DebugDestination implements LogDestination {
 class FileDestination implements LogDestination {
   private initialized = false;
   private logFilePath: string | null = null;
-  private verboseMode = true;
 
   isEnabled(): boolean {
     return true;
   }
 
-  setVerboseMode(enabled: boolean): void {
-    this.verboseMode = enabled;
-  }
-
   write(entry: TaggedLogEntry): void {
     if (entry.type === 'html') return;
     if (entry.type === 'multiline') return;
@@ -404,7 +422,7 @@ class Logger {
   private truncateTagContent(content: string, tagName: string): string {
     const escapedTag = tagName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
     const tagRegex = new RegExp(`<${escapedTag}>([\\s\\S]*?)<\\/${escapedTag}>`, 'g');
-    return content.replace(tagRegex, (match, innerContent) => {
+    return content.replace(tagRegex, (_, innerContent) => {
       const trimmed = innerContent.trim();
       const totalLength = trimmed.length;
       if (totalLength <= 100) {

package/src/utils/markdown-terminal.ts

@@ -24,10 +24,9 @@ if (renderer) {
     const originalListitem = renderer.listitem;
     renderer.listitem = function (text: any) {
       if (typeof text === 'object') {
-        const item = text;
         const parser = this.parser;
         const originalParse = parser.parse.bind(parser);
-        parser.parse = (tokens: any, loose?: boolean) => parser.parseInline(tokens);
+        parser.parse = (tokens: any) => parser.parseInline(tokens);
         const result = originalListitem.call(this, text);
         parser.parse = originalParse;
         return result;

package/src/utils/next-steps.ts

@@ -1,4 +1,6 @@
 import path from 'node:path';
+import chalk from 'chalk';
+import figures from 'figures';
 import { tag } from './logger.js';
 
 export interface NextStepCommand {
@@ -20,27 +22,27 @@ export function relativeToCwd(absPath: string): string {
 export function printNextSteps(sections: NextStepSection[]): void {
   if (sections.length === 0) return;
 
-  const blocks: string[] = [];
+  const lines: string[] = [''];
+  let isDefault = true;
   for (const section of sections) {
-    const lines: string[] = [];
     const headerPath = section.path ? relativeToCwd(section.path) : '';
-    lines.push(headerPath ? `${section.label}: ${headerPath}` : section.label);
-
-    const commands = section.commands || [];
-    if (commands.length > 0) {
-      const labeled = commands.filter((c) => c.label);
-      const maxLabel = labeled.length > 0 ? Math.max(...labeled.map((c) => c.label.length)) : 0;
-      for (const cmd of commands) {
-        if (!cmd.label) {
-          lines.push(`  ${cmd.command}`);
-          continue;
-        }
-        const padded = `${cmd.label}:`.padEnd(maxLabel + 2);
-        lines.push(`  ${padded} ${cmd.command}`);
+    lines.push(headerPath ? `${chalk.bold(section.label)}  ${chalk.gray(headerPath)}` : `${chalk.bold(section.label)}`);
+
+    for (const cmd of section.commands || []) {
+      if (cmd.label) lines.push(BLANK);
+      if (isDefault) {
+        if (cmd.label) lines.push(chalk.bold.yellow(`${figures.pointer} ${cmd.label}`));
+        lines.push(chalk.bold(`    ${cmd.command}`));
+        isDefault = false;
+        continue;
       }
+      if (cmd.label) lines.push(chalk.dim(`  ${cmd.label}`));
+      lines.push(`  ${cmd.command}`);
     }
-    blocks.push(lines.join('\n'));
+    lines.push(BLANK);
   }
 
-  tag('multiline').log(blocks.join('\n\n'), { maxLines: 18 });
+  tag('info').log(lines.join('\n'));
 }
+
+const BLANK = ' ';

package/src/utils/page-readiness.ts

@@ -1,11 +1,33 @@
-export async function waitForPageReadiness(page: any, options: PageReadinessOptions = {}): Promise<void> {
-  if (!page) return;
+export async function waitForPageReadiness(page: any, options: PageReadinessOptions = {}): Promise<boolean> {
+  if (!page) return false;
 
   const timeout = options.timeout ?? 6000;
   await page.waitForLoadState?.('domcontentloaded', { timeout })?.catch(() => {});
 
   await Promise.race([waitForNetworkIdle(page, timeout), waitForDomQuiet(page, timeout), waitForVisibleSpinnersHidden(page, options.spinnerSelectors || [], timeout), sleep(timeout)]).catch(() => {});
-  await waitForPageBodyContent(page, timeout);
+  return waitForPageBodyContent(page, timeout);
+}
+
+export function waitForDomChange(page: any, timeout: number): Promise<boolean> {
+  if (!page?.evaluate) return Promise.resolve(false);
+
+  return page
+    .evaluate(
+      (ms: number) =>
+        new Promise<boolean>((resolve) => {
+          const observer = new MutationObserver(() => {
+            observer.disconnect();
+            resolve(true);
+          });
+          observer.observe(document, { subtree: true, childList: true, attributes: true, characterData: true });
+          setTimeout(() => {
+            observer.disconnect();
+            resolve(false);
+          }, ms);
+        }),
+      timeout
+    )
+    .catch(() => true);
 }
 
 const DOM_QUIET_MS = 350;
@@ -56,20 +78,24 @@ async function waitForVisibleSpinnersHidden(page: any, selectors: string[], time
   return Promise.all(visibleSpinners.map((locator) => locator.waitFor({ state: 'hidden', timeout }).catch(() => {}))).then(() => {});
 }
 
-function waitForPageBodyContent(page: any, timeout: number): Promise<void> {
-  if (!page?.waitForFunction) return Promise.resolve();
+function waitForPageBodyContent(page: any, timeout: number): Promise<boolean> {
+  if (!page?.waitForFunction) return Promise.resolve(true);
 
   return page
     .waitForFunction(
       () => {
         const body = document.body;
         if (!body) return false;
-        return body.children.length > 0 || body.textContent?.trim().length > 0;
+        if (body.innerText.trim().length > 0) return true;
+        return !!body.querySelector('input, button, select, textarea, img, svg, canvas, iframe, video');
       },
       undefined,
       { timeout }
     )
-    .catch(() => {});
+    .then(
+      () => true,
+      () => false
+    );
 }
 
 export function sleep(ms: number): Promise<void> {

package/src/utils/test-plan-markdown.ts

@@ -3,6 +3,7 @@ import { type Note, Plan, Test } from '../test-plan.ts';
 import { mdq } from './markdown-query.ts';
 
 const NOISE_PREFIXES = ['Test started', 'Finish requested:', 'Session name:'];
+const CURRENT_PAGE = 'Current page';
 
 function isNoiseNote(message: string): boolean {
   return NOISE_PREFIXES.some((prefix) => message.startsWith(prefix));
@@ -118,7 +119,10 @@ export function parsePlanFromMarkdown(filePath: string): Plan {
 }
 
 export function parsePlansFromMarkdown(filePath: string): Plan[] {
-  const content = readFileSync(filePath, 'utf-8');
+  return parsePlansFromMarkdownText(readFileSync(filePath, 'utf-8'));
+}
+
+export function parsePlansFromMarkdownText(content: string): Plan[] {
   const lines = content.split('\n');
 
   const plans: Plan[] = [];
@@ -193,7 +197,7 @@ export function parsePlansFromMarkdown(filePath: string): Plan[] {
     }
 
     if (currentTest && inRequirements && line && !line.startsWith('##')) {
-      currentTest.startUrl = line;
+      if (line !== CURRENT_PAGE) currentTest.startUrl = line;
       continue;
     }
 
@@ -217,7 +221,7 @@ export function parsePlansFromMarkdown(filePath: string): Plan[] {
     if (suiteStart === -1) continue;
     const nextSuite = content.indexOf('<!-- suite -->', suiteStart + 1);
     const suiteContent = nextSuite === -1 ? content.slice(suiteStart) : content.slice(suiteStart, nextSuite);
-    const firstItem = mdq(suiteContent).query('section[0] item[0]').text().trim();
+    const firstItem = mdq(suiteContent).query('section[0] item[0]').nodes()[0]?.text.trim() || '';
     const urlMatch = firstItem.match(/^URL:\s*(.+)/);
     if (urlMatch) {
       plan.url = urlMatch[1].replace(/\*\*|`|\*|_|~~?/g, '').trim();
@@ -230,6 +234,34 @@ export function parsePlansFromMarkdown(filePath: string): Plan[] {
   return plans;
 }
 
+export function findPlanMarkdownProblems(markdown: string): string[] {
+  const doc = mdq(markdown);
+  const problems: string[] = [];
+  const headings = doc.query('h1').nodes();
+  if (!doc.query('comment(/^suite/)').exists() || headings.length === 0) {
+    problems.push('Missing <!-- suite --> comment followed by the plan name heading');
+  }
+
+  const scenarioCount = Math.max(headings.length - 1, 0);
+  const testComments = doc.query('comment(/^test/)').count();
+  if (testComments !== scenarioCount) {
+    problems.push(`Found ${scenarioCount} scenario headings but ${testComments} <!-- test --> comments`);
+  }
+
+  for (const section of doc.query('section1').each().slice(1)) {
+    const title = section.query('heading').nodes()[0]?.text;
+    if (!section.query('section2("Steps") item').exists()) problems.push(`Scenario "${title}" has no ## Steps items`);
+    if (!section.query('section2("Expected") item').exists()) problems.push(`Scenario "${title}" has no ## Expected items`);
+  }
+  return problems;
+}
+
+export function readPlanMarkdownComment(markdown: string, key: string): string | null {
+  const body = mdq(markdown).query(`comment(/^${key}:/)`).nodes()[0]?.text;
+  if (!body) return null;
+  return body.slice(key.length + 1).trim() || null;
+}
+
 function formatPlanSuite(plan: Plan): string {
   let content = `<!-- suite -->\n# ${plan.title}\n\n`;
 
@@ -245,7 +277,7 @@ function formatPlanSuite(plan: Plan): string {
     content += `<!-- test\npriority: ${test.priority}\n-->\n`;
     content += `# ${test.scenario}\n\n`;
     content += '## Requirements\n';
-    content += `${test.startUrl || 'Current page'}\n\n`;
+    content += `${test.startUrl || CURRENT_PAGE}\n\n`;
 
     if (test.plannedSteps.length > 0) {
       content += '## Steps\n';

package/src/utils/url-matcher.ts

@@ -44,6 +44,12 @@ export function isSamePageFamily(urlA: string, urlB: string): boolean {
   return true;
 }
 
+export function isSamePageIgnoringQuery(urlA: string, urlB: string): boolean {
+  const a = new URL(urlA, 'http://localhost');
+  const b = new URL(urlB, 'http://localhost');
+  return a.pathname === b.pathname && a.hash === b.hash;
+}
+
 export function generalizeSegment(segment: string): string {
   if (/^\d+$/.test(segment)) return '\\d+';
   if (/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i.test(segment)) return '[a-f0-9-]+';