@yasserkhanorg/e2e-agents 0.3.3 → 0.3.4

package/dist/esm/agent/report.js

@@ -42,6 +42,12 @@ export function writeReport(appRoot, config, data) {
     const markdownLines = [];
     markdownLines.push(`# ${data.mode === 'impact' ? 'Impact Analysis' : 'Gap Analysis'} Report`);
     markdownLines.push('');
+    if (data.runMetadata) {
+        markdownLines.push(`Run ID: ${data.runMetadata.runId}`);
+        markdownLines.push(`Run window: ${data.runMetadata.startedAt} -> ${data.runMetadata.completedAt}`);
+        markdownLines.push(`Run duration (ms): ${data.runMetadata.durationMs}`);
+        markdownLines.push(`Since ref: ${data.runMetadata.sinceRef}`);
+    }
     markdownLines.push(`Framework: ${data.framework}`);
     markdownLines.push(`Test Patterns: ${data.testPatterns.join(', ') || 'None'}`);
     if (data.flowCatalog) {
@@ -100,6 +106,9 @@ export function writeReport(appRoot, config, data) {
             if (result.error) {
                 markdownLines.push(`  Error: ${result.error}`);
             }
+            if (result.failureCategory || result.failureCode) {
+                markdownLines.push(`  Failure taxonomy: category=${result.failureCategory || 'unknown'} code=${result.failureCode || 'unknown'}`);
+            }
         }
         if (data.pipeline.warnings.length > 0) {
             markdownLines.push('Pipeline warnings:');

package/dist/esm/agent/runner.js

@@ -114,11 +114,14 @@ function buildRecommendedTestsFromCoverage(flows, coverage) {
         const flow = flowMap.get(entry.flowId);
         const flagSummary = formatFlags(flow?.flags || []);
         for (const test of entry.coveredBy) {
-            if (!testNotes.has(test)) {
-                testNotes.set(test, new Set());
+            const normalizedTest = normalizePath(test)
+                .replace(/^\.\//, '')
+                .replace(/^e2e-tests\/playwright\//, '');
+            if (!testNotes.has(normalizedTest)) {
+                testNotes.set(normalizedTest, new Set());
             }
             if (flagSummary !== 'none') {
-                testNotes.get(test)?.add(flagSummary);
+                testNotes.get(normalizedTest)?.add(flagSummary);
             }
         }
     }
@@ -190,12 +193,24 @@ function classifyImpactModelConfidence(flowMapping, testMapping, dependencyGraph
     }
     return 'low';
 }
+function createRunId(mode) {
+    const ciRunId = process.env.GITHUB_RUN_ID;
+    const entropy = Math.random().toString(36).slice(2, 8);
+    const ts = Date.now().toString(36);
+    if (ciRunId) {
+        return `${mode}-gh-${ciRunId}-${ts}-${entropy}`;
+    }
+    return `${mode}-local-${ts}-${entropy}`;
+}
 export async function runImpact(_config, _options) {
     ensureAppRoot(_config.path);
     if (_config.testsRoot) {
         ensureAppRoot(_config.testsRoot);
     }
     const deadline = Date.now() + _config.timeLimitMinutes * 60 * 1000;
+    const runStartedAt = new Date().toISOString();
+    const runStartedTs = Date.now();
+    const runId = createRunId('impact');
     const warnings = [];
     const testsRoot = _config.testsRoot || _config.path;
     const frameworkDetection = detectFramework(testsRoot, _config.framework);
@@ -268,7 +283,7 @@ export async function runImpact(_config, _options) {
                 coverageMap.set(entry.flowId, entry.coveredBy);
             }
             gaps = computeGaps(flows, coverageMap);
-            recommendedTests = buildRecommendedTestsWithFlags(flows, testsByFlow);
+            recommendedTests = buildRecommendedTestsFromCoverage(flows, coverage);
         }
         else {
             const traceability = mapTraceabilityToFlows(testsRoot, _config.impact.traceability, flows);
@@ -320,6 +335,15 @@ export async function runImpact(_config, _options) {
     const reportRoot = testsRoot;
     const report = writeReport(reportRoot, _config, {
         mode: 'impact',
+        runMetadata: {
+            runId,
+            startedAt: runStartedAt,
+            completedAt: new Date().toISOString(),
+            durationMs: Date.now() - runStartedTs,
+            sinceRef: _config.git.since,
+            appPath: _config.path,
+            testsRoot,
+        },
         changedFiles,
         flows: sortFlows(flows),
         coverage,
@@ -366,6 +390,9 @@ export async function runGap(_config, _options) {
         ensureAppRoot(_config.testsRoot);
     }
     const deadline = Date.now() + _config.timeLimitMinutes * 60 * 1000;
+    const runStartedAt = new Date().toISOString();
+    const runStartedTs = Date.now();
+    const runId = createRunId('gap');
     const warnings = [];
     const testsRoot = _config.testsRoot || _config.path;
     const frameworkDetection = detectFramework(testsRoot, _config.framework);
@@ -450,7 +477,7 @@ export async function runGap(_config, _options) {
                 coverageMap.set(entry.flowId, entry.coveredBy);
             }
             gaps = computeGaps(flows, coverageMap);
-            recommendedTests = buildRecommendedTestsWithFlags(flows, testsByFlow);
+            recommendedTests = buildRecommendedTestsFromCoverage(flows, coverage);
         }
         else {
             const traceability = mapTraceabilityToFlows(testsRoot, _config.impact.traceability, flows);
@@ -502,6 +529,15 @@ export async function runGap(_config, _options) {
     const reportRoot = testsRoot;
     const report = writeReport(reportRoot, _config, {
         mode: 'gap',
+        runMetadata: {
+            runId,
+            startedAt: runStartedAt,
+            completedAt: new Date().toISOString(),
+            durationMs: Date.now() - runStartedTs,
+            sinceRef: _config.git.since,
+            appPath: _config.path,
+            testsRoot,
+        },
         changedFiles,
         flows: sortFlows(flows),
         coverage,

package/dist/esm/api.js

@@ -4,7 +4,7 @@ import { existsSync, readFileSync } from 'fs';
 import { join } from 'path';
 import { resolveConfig } from './agent/config.js';
 import { runGap, runImpact } from './agent/runner.js';
-import { attachDeveloperActions, buildPlanFromImpactReport, renderCiSummaryMarkdown, writeCiSummary, writePlanReport, } from './agent/plan.js';
+import { appendPlanMetrics, attachDeveloperActions, buildPlanFromImpactReport, renderCiSummaryMarkdown, writeCiSummary, writePlanReport, } from './agent/plan.js';
 import { applyOperationalInsights } from './agent/operational_insights.js';
 import { finalizeGeneratedTests } from './agent/handoff.js';
 import { ingestTraceabilityInput, } from './agent/traceability_ingest.js';
@@ -62,6 +62,7 @@ export async function recommendTests(options = {}) {
     const planPath = writePlanReport(reportRoot, plan);
     const ciSummaryMarkdown = renderCiSummaryMarkdown(plan);
     const ciSummaryPath = writeCiSummary(reportRoot, ciSummaryMarkdown);
+    appendPlanMetrics(reportRoot, plan);
     return {
         report,
         reportPath: impactPath,

package/dist/esm/cli.js

@@ -7,7 +7,7 @@ import { resolveConfig } from './agent/config.js';
 import { AnthropicProvider } from './anthropic_provider.js';
 import { LLMProviderError } from './provider_interface.js';
 import { runGap, runImpact } from './agent/runner.js';
-import { attachDeveloperActions, buildPlanFromImpactReport, renderCiSummaryMarkdown, writeCiSummary, writePlanReport } from './agent/plan.js';
+import { appendPlanMetrics, attachDeveloperActions, buildPlanFromImpactReport, renderCiSummaryMarkdown, writeCiSummary, writePlanReport, } from './agent/plan.js';
 import { applyOperationalInsights } from './agent/operational_insights.js';
 import { appendFeedbackAndRecompute } from './agent/feedback.js';
 import { finalizeGeneratedTests } from './agent/handoff.js';
@@ -87,6 +87,7 @@ function printUsage() {
         '  --pipeline-base-url   Base URL for Playwright runs',
         '  --pipeline-browser    Browser: chrome|chromium|firefox|webkit',
         '  --pipeline-headless   Run in headless mode',
+        '  --pipeline-headed     Run in headed mode',
         '  --pipeline-project    Playwright project name',
         '  --pipeline-parallel   Enable parallel mode in generator',
         '  --pipeline-dry-run    Do not execute pipeline (report only)',
@@ -101,6 +102,8 @@ function printUsage() {
         '  --policy-safe-merge-confidence <n> Confidence needed for safe-to-merge',
         '  --policy-force-full-on-warnings <n> Escalate to full at warning count',
         '  --policy-risky-patterns <globs>     Comma-separated risky file globs',
+        '  --policy-enforcement-mode <mode>    advisory | warn | block',
+        '  --policy-block-actions <actions>    Comma-separated CI actions to block/warn',
         '  --ci-comment-path <path>            Write CI markdown summary',
         '  --github-output <path>              Write GitHub Actions outputs',
         '  --fail-on-must-add-tests            Exit non-zero on must-add-tests decision',
@@ -244,6 +247,10 @@ function parseArgs(argv) {
             parsed.pipelineHeadless = true;
             continue;
         }
+        if (arg === '--pipeline-headed') {
+            parsed.pipelineHeadless = false;
+            continue;
+        }
         if (arg === '--pipeline-project' && next) {
             parsed.pipelineProject = next;
             i += 1;
@@ -320,6 +327,21 @@ function parseArgs(argv) {
             i += 1;
             continue;
         }
+        if (arg === '--policy-enforcement-mode' && next) {
+            if (next === 'advisory' || next === 'warn' || next === 'block') {
+                parsed.policyEnforcementMode = next;
+            }
+            i += 1;
+            continue;
+        }
+        if (arg === '--policy-block-actions' && next) {
+            parsed.policyBlockActions = next
+                .split(',')
+                .map((value) => value.trim())
+                .filter((value) => (value === 'run-now' || value === 'must-add-tests' || value === 'safe-to-merge'));
+            i += 1;
+            continue;
+        }
         if (arg === '--ci-comment-path' && next) {
             parsed.ciCommentPath = next;
             i += 1;
@@ -794,12 +816,16 @@ async function main() {
         policy: args.policyMinConfidence !== undefined ||
             args.policySafeMergeConfidence !== undefined ||
             args.policyWarningsThreshold !== undefined ||
-            (args.policyRiskyPatterns && args.policyRiskyPatterns.length > 0)
+            (args.policyRiskyPatterns && args.policyRiskyPatterns.length > 0) ||
+            args.policyEnforcementMode !== undefined ||
+            (args.policyBlockActions && args.policyBlockActions.length > 0)
             ? {
                 minConfidenceForTargeted: args.policyMinConfidence,
                 safeMergeMinConfidence: args.policySafeMergeConfidence,
                 forceFullOnWarningsAtOrAbove: args.policyWarningsThreshold,
                 riskyFilePatterns: args.policyRiskyPatterns,
+                enforcementMode: args.policyEnforcementMode,
+                blockOnActions: args.policyBlockActions,
             }
             : undefined,
     });
@@ -828,24 +854,33 @@ async function main() {
         const planPath = writePlanReport(reportRoot, plan);
         const summaryMarkdown = renderCiSummaryMarkdown(plan);
         const summaryPath = writeCiSummary(reportRoot, summaryMarkdown, args.ciCommentPath);
+        const metrics = appendPlanMetrics(reportRoot, plan);
         const ghaOutput = args.githubOutputPath || process.env.GITHUB_OUTPUT;
         if (ghaOutput) {
             appendFileSync(ghaOutput, `run_set=${plan.runSet}\n`);
             appendFileSync(ghaOutput, `action=${plan.decision.action}\n`);
             appendFileSync(ghaOutput, `confidence=${plan.confidence}\n`);
+            appendFileSync(ghaOutput, `enforcement_mode=${plan.enforcement.mode}\n`);
+            appendFileSync(ghaOutput, `enforcement_should_fail=${plan.enforcement.shouldFail}\n`);
             appendFileSync(ghaOutput, `recommended_tests_count=${plan.recommendedTests.length}\n`);
             appendFileSync(ghaOutput, `required_new_tests_count=${plan.requiredNewTests.length}\n`);
             appendFileSync(ghaOutput, `plan_path=${planPath}\n`);
             appendFileSync(ghaOutput, `summary_path=${summaryPath}\n`);
+            appendFileSync(ghaOutput, `metrics_events_path=${metrics.eventsPath}\n`);
+            appendFileSync(ghaOutput, `metrics_summary_path=${metrics.summaryPath}\n`);
         }
         // eslint-disable-next-line no-console
         console.log(`Suggested run set: ${plan.runSet} (confidence ${plan.confidence})`);
         // eslint-disable-next-line no-console
         console.log(`Decision: ${plan.decision.action} - ${plan.decision.summary}`);
         // eslint-disable-next-line no-console
+        console.log(`Enforcement: ${plan.enforcement.mode} (shouldFail=${plan.enforcement.shouldFail})`);
+        // eslint-disable-next-line no-console
         console.log(`Plan data: ${planPath}`);
         // eslint-disable-next-line no-console
         console.log(`CI summary: ${summaryPath}`);
+        // eslint-disable-next-line no-console
+        console.log(`Plan metrics: ${metrics.summaryPath}`);
         if (plan.nextActions) {
             // eslint-disable-next-line no-console
             console.log(`Next action (run existing): ${plan.nextActions.runRecommendedTests || plan.nextActions.runSmokeSuite}`);
@@ -854,7 +889,8 @@ async function main() {
             // eslint-disable-next-line no-console
             console.log(`Next action (heal): ${plan.nextActions.healGeneratedTests}`);
         }
-        if (args.failOnMustAddTests && plan.decision.action === 'must-add-tests') {
+        const failOnLegacyFlag = args.failOnMustAddTests && plan.decision.action === 'must-add-tests';
+        if (failOnLegacyFlag || plan.enforcement.shouldFail) {
             process.exit(2);
         }
         return;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yasserkhanorg/e2e-agents",
-  "version": "0.3.3",
+  "version": "0.3.4",
   "description": "Pluggable LLM provider library for AI-powered test automation. Use Claude, Ollama, or your own LLM. Integrate with Playwright, Jest, or any test framework. MCP server for test agents, cost tracking, and hybrid provider mode.",
   "main": "dist/index.js",
   "module": "dist/esm/index.js",

package/schemas/impact.schema.json

@@ -18,6 +18,28 @@
     "mode": {
       "const": "impact"
     },
+    "runMetadata": {
+      "type": "object",
+      "required": [
+        "runId",
+        "startedAt",
+        "completedAt",
+        "durationMs",
+        "sinceRef",
+        "appPath",
+        "testsRoot"
+      ],
+      "properties": {
+        "runId": {"type": "string"},
+        "startedAt": {"type": "string", "format": "date-time"},
+        "completedAt": {"type": "string", "format": "date-time"},
+        "durationMs": {"type": "number"},
+        "sinceRef": {"type": "string"},
+        "appPath": {"type": "string"},
+        "testsRoot": {"type": "string"}
+      },
+      "additionalProperties": false
+    },
     "changedFiles": {
       "type": "array",
       "items": {
@@ -370,6 +392,21 @@
               },
               "error": {
                 "type": "string"
+              },
+              "failureCategory": {
+                "enum": [
+                  "config",
+                  "environment",
+                  "generation",
+                  "validation",
+                  "runtime",
+                  "quality",
+                  "path-safety",
+                  "unknown"
+                ]
+              },
+              "failureCode": {
+                "type": "string"
               }
             },
             "additionalProperties": false

package/schemas/plan.schema.json

@@ -5,6 +5,7 @@
   "type": "object",
   "required": [
     "schemaVersion",
+    "runId",
     "generatedAt",
     "source",
     "runSet",
@@ -14,12 +15,19 @@
     "requiredNewTests",
     "policy",
     "decision",
+    "enforcement",
     "metrics"
   ],
   "properties": {
     "schemaVersion": {
       "const": "1.0.0"
     },
+    "runId": {
+      "type": "string"
+    },
+    "sourceRunId": {
+      "type": "string"
+    },
     "generatedAt": {
       "type": "string",
       "format": "date-time"
@@ -108,6 +116,15 @@
               "items": {
                 "type": "string"
               }
+            },
+            "enforcementMode": {
+              "enum": ["advisory", "warn", "block"]
+            },
+            "blockOnActions": {
+              "type": "array",
+              "items": {
+                "enum": ["run-now", "must-add-tests", "safe-to-merge"]
+              }
             }
           },
           "additionalProperties": false
@@ -139,6 +156,37 @@
       },
       "additionalProperties": false
     },
+    "enforcement": {
+      "type": "object",
+      "required": [
+        "mode",
+        "blockOnActions",
+        "matchedAction",
+        "shouldFail",
+        "summary"
+      ],
+      "properties": {
+        "mode": {
+          "enum": ["advisory", "warn", "block"]
+        },
+        "blockOnActions": {
+          "type": "array",
+          "items": {
+            "enum": ["run-now", "must-add-tests", "safe-to-merge"]
+          }
+        },
+        "matchedAction": {
+          "type": "boolean"
+        },
+        "shouldFail": {
+          "type": "boolean"
+        },
+        "summary": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false
+    },
     "insights": {
       "type": "object",
       "properties": {