@letsrunit/executor 0.10.0 → 0.11.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@letsrunit/executor",
-  "version": "0.10.0",
+  "version": "0.11.0",
   "description": "High-level test execution orchestrator for letsrunit",
   "keywords": [
     "testing",
@@ -41,13 +41,15 @@
   },
   "packageManager": "yarn@4.10.3",
   "dependencies": {
-    "@letsrunit/ai": "0.10.0",
-    "@letsrunit/controller": "0.10.0",
-    "@letsrunit/gherkin": "0.10.0",
-    "@letsrunit/journal": "0.10.0",
-    "@letsrunit/mailbox": "0.10.0",
-    "@letsrunit/playwright": "0.10.0",
-    "@letsrunit/utils": "0.10.0"
+    "@letsrunit/ai": "0.11.0",
+    "@letsrunit/controller": "0.11.0",
+    "@letsrunit/gherkin": "0.11.0",
+    "@letsrunit/journal": "0.11.0",
+    "@letsrunit/mailbox": "0.11.0",
+    "@letsrunit/playwright": "0.11.0",
+    "@letsrunit/store": "0.11.0",
+    "@letsrunit/utils": "0.11.0",
+    "zod": "^4.3.5"
   },
   "devDependencies": {
     "@types/node": "^25.0.9",

package/src/ai/describe-page.ts

@@ -1,8 +1,7 @@
 import { generate } from '@letsrunit/ai';
 import type { PageInfo } from '@letsrunit/playwright';
+import { extractPageInfo, scrubHtml } from '@letsrunit/playwright';
 import { stringify as toYaml } from 'yaml';
-import { extractPageInfo } from '../../../playwright/src/page-info';
-import { scrubHtml } from '../../../playwright/src/scrub-html';
 
 const PROMPT = `Convert raw HTML into compact Markdown using a limited set of custom blocks. The output feeds must be consistent.
 

package/src/ai/detect-page-changes.ts

@@ -1,6 +1,6 @@
 import { generate } from '@letsrunit/ai';
 import { Journal } from '@letsrunit/journal';
-import { unifiedHtmlDiff } from '../../../playwright/src/unified-html-diff';
+import { unifiedHtmlDiff } from '@letsrunit/playwright';
 import { locatorRules } from './locator-rules';
 
 const PROMPT = `You analyze a diff of two HTML files. Your task is to detect the most significant user-visible changes and output 0 to 3 Playwright Gherkin steps using only these step definitions:

package/src/ai/explain-failure.ts

@@ -0,0 +1,28 @@
+import { generate } from '@letsrunit/ai';
+import { z } from 'zod';
+
+const PROMPT = `You are analyzing a failed browser test to provide useful information to the user
+
+Classify the failure as:
+- "test" when the failed step is caused by an intentional user-visible change in the application, and the product still appears to work as expected.
+- "code" when the failed step is caused by broken or missing functionality, or when the user can no longer complete the expected flow.
+
+The advise should match the classification. If the classification is to update the test, don't advise to update the code and visa-versa.
+The advise must be a single concrete action to solve the failure without examples or alternatives.
+`;
+
+const ResponseSchema = z.object({
+  update: z
+    .enum(['test', 'code'])
+    .describe('Classification: "test" if the test is outdated OR "code" if it is likely a regression'),
+  reason: z
+    .string()
+    .describe('One short sentence describing only the observed user-visible change'),
+  advice: z.string().describe('One short imperative sentence with a single action'),
+});
+
+export type ExplainFailureResponse = z.infer<typeof ResponseSchema>;
+
+export async function explainFailure(input: string): Promise<ExplainFailureResponse> {
+  return await generate(PROMPT, input, { model: 'medium', schema: ResponseSchema });
+}

package/src/explain.ts

@@ -0,0 +1,213 @@
+import { executableScenarioIds } from '@letsrunit/gherkin';
+import { unifiedHtmlDiff } from '@letsrunit/playwright';
+import { findArtifacts, findLastRun, findLastTest, type LastRunTest, openStore } from '@letsrunit/store';
+import { statusSymbol } from '@letsrunit/utils';
+import { readFileSync } from 'node:fs';
+import { dirname, isAbsolute, join } from 'node:path';
+import { explainFailure, type ExplainFailureResponse } from './ai/explain-failure';
+
+export interface ExplainOptions {
+  dbPath?: string;
+  artifactsDir?: string;
+}
+
+export interface ExplainScenario {
+  scenarioId: string;
+  scenarioName: string;
+  featurePath: string;
+  steps: string;
+  failureMessage: string;
+  update: 'test' | 'code';
+  updateMessage: string;
+  reason: string;
+  advice: string;
+}
+
+export interface ExplainScenarioError {
+  scenarioId: string;
+  scenarioName: string;
+  featurePath: string;
+  error: string;
+}
+
+export interface ExplainResult {
+  hasRun: boolean;
+  totalFailed: number;
+  explanations: ExplainScenario[];
+  errors: ExplainScenarioError[];
+}
+
+function defaultDbPath(): string {
+  return join(process.cwd(), '.letsrunit', 'letsrunit.db');
+}
+
+function resolveArtifactsDir(dbPath: string, artifactsDir?: string): string {
+  if (artifactsDir) return artifactsDir;
+  return join(dirname(dbPath), 'artifacts');
+}
+
+function resolveFeaturePath(path: string): string {
+  return isAbsolute(path) ? path : join(process.cwd(), path);
+}
+
+function isFailed(test: LastRunTest): boolean {
+  return test.failedStepIndex !== null || test.status === 'failed';
+}
+
+function colorize(status: string, text: string): string {
+  switch (status) {
+    case 'success':
+    case 'skipped':
+      return `\x1b[90m${text}\x1b[0m`;
+    case 'failure':
+      return `\x1b[31m${text}\x1b[0m`;
+    default:
+      return text;
+  }
+}
+
+function renderScenarioSteps(test: LastRunTest, failedStepIndex: number): string {
+  return test.steps
+    .map((step) => {
+      const status = step.index < failedStepIndex ? 'success' : step.index === failedStepIndex ? 'failure' : 'skipped';
+      const symbol = statusSymbol(status);
+      return colorize(status, `${symbol} ${step.text}`);
+    })
+    .join('\n');
+}
+
+function pickLatestHtmlFilename(filenames: Array<{ filename: string }>): string | null {
+  const html = filenames.filter((artifact) => artifact.filename.endsWith('.html'));
+  if (html.length === 0) return null;
+  return html.at(-1)?.filename ?? null;
+}
+
+function asHeading(update: ExplainFailureResponse['update']): string {
+  return update === 'test' ? 'Test outdated' : 'Possible regression';
+}
+
+function buildPrompt(test: LastRunTest, renderedSteps: string, failureMessage: string, diff: string): string {
+  return [
+    `Feature: ${test.featureName}`,
+    `Scenario: ${test.scenarioName}`,
+    '',
+    renderedSteps,
+    '',
+    'Failure message:',
+    failureMessage,
+    '',
+    'HTML diff (baseline -> failed):',
+    diff,
+  ].join('\n');
+}
+
+function parseFeatureIds(featureCache: Map<string, Set<string>>, featurePath: string): Set<string> {
+  const cached = featureCache.get(featurePath);
+  if (cached) return cached;
+
+  const source = readFileSync(resolveFeaturePath(featurePath), 'utf-8');
+  const ids = executableScenarioIds(source, featurePath);
+  featureCache.set(featurePath, ids);
+  return ids;
+}
+
+function createError(test: LastRunTest, error: string): ExplainScenarioError {
+  return {
+    scenarioId: test.scenarioId,
+    scenarioName: test.scenarioName,
+    featurePath: test.featurePath,
+    error,
+  };
+}
+
+export default async function explain(options: ExplainOptions = {}): Promise<ExplainResult> {
+  const dbPath = options.dbPath ?? defaultDbPath();
+  const artifactsDir = resolveArtifactsDir(dbPath, options.artifactsDir);
+  const db = openStore(dbPath);
+
+  try {
+    const run = findLastRun(db);
+    if (!run) {
+      return { hasRun: false, totalFailed: 0, explanations: [], errors: [] };
+    }
+
+    const featureCache = new Map<string, Set<string>>();
+    const failedTests = run.tests.filter((test) => isFailed(test));
+    const explanations: ExplainScenario[] = [];
+    const errors: ExplainScenarioError[] = [];
+
+    for (const test of failedTests) {
+      try {
+        if (test.failedStepIndex === null) {
+          errors.push(createError(test, 'Missing failed step index for failed scenario'));
+          continue;
+        }
+
+        const knownScenarioIds = parseFeatureIds(featureCache, test.featurePath);
+        if (!knownScenarioIds.has(test.scenarioId)) {
+          errors.push(createError(test, 'Scenario has changed in feature file since the last run'));
+          continue;
+        }
+
+        const failedStep = test.steps.find((step) => step.index === test.failedStepIndex);
+        if (!failedStep) {
+          errors.push(createError(test, `Failed step at index ${test.failedStepIndex} was not found`));
+          continue;
+        }
+
+        const baseline = findLastTest(db, test.scenarioId, 'passed');
+        if (!baseline) {
+          errors.push(createError(test, 'No prior successful baseline found for this scenario'));
+          continue;
+        }
+
+        const failedArtifacts = findArtifacts(db, test.id, failedStep.id);
+        const baselineArtifacts = findArtifacts(db, baseline.id, failedStep.id);
+
+        const failedHtmlFile = pickLatestHtmlFilename(failedArtifacts);
+        const baselineHtmlFile = pickLatestHtmlFilename(baselineArtifacts);
+
+        if (!failedHtmlFile || !baselineHtmlFile) {
+          errors.push(createError(test, 'Missing HTML snapshot artifact for failed step'));
+          continue;
+        }
+
+        const failedHtml = readFileSync(join(artifactsDir, failedHtmlFile), 'utf-8');
+        const baselineHtml = readFileSync(join(artifactsDir, baselineHtmlFile), 'utf-8');
+
+        const diff = await unifiedHtmlDiff(
+          { html: baselineHtml, url: 'about:blank' },
+          { html: failedHtml, url: 'about:blank' },
+        );
+
+        const renderedSteps = renderScenarioSteps(test, test.failedStepIndex);
+        const failureMessage = test.error ?? 'Step failed';
+        const prompt = buildPrompt(test, renderedSteps, failureMessage, diff);
+        const ai = await explainFailure(prompt);
+
+        explanations.push({
+          scenarioId: test.scenarioId,
+          scenarioName: test.scenarioName,
+          featurePath: test.featurePath,
+          steps: renderedSteps,
+          failureMessage,
+          update: ai.update,
+          updateMessage: asHeading(ai.update),
+          reason: ai.reason,
+          advice: ai.advice,
+        });
+      } catch (error) {
+        errors.push(createError(test, (error as Error).message));
+      }
+    }
+
+    return {
+      hasRun: true,
+      totalFailed: failedTests.length,
+      explanations,
+      errors,
+    };
+  } finally {
+    db.close();
+  }
+}

package/src/index.ts

@@ -1,4 +1,5 @@
 export { refineSuggestion } from './ai/refine-suggestion';
+export { default as explain, type ExplainResult } from './explain';
 export { default as explore, type PreparedAction as Action } from './explore';
 export { default as generate } from './generate';
 export { default as run } from './run';