@vijaypjavvadi/bdd2pw 3.8.1 → 3.10.0

package/dist/llm/telemetry.js

@@ -0,0 +1,230 @@
+"use strict";
+/**
+ * v3.9.0 — LLM telemetry collector.
+ *
+ * Subscribes to the existing `LLMLogEvent` stream that AnthropicLLMClient
+ * emits during `generateBinding` / `generateBatchBindings`, accumulates
+ * per-call stats, and exposes an aggregated summary suitable for writing
+ * to `<repo>/artefacts/llm-stats.json`.
+ *
+ * Why this exists: v3.5.0 introduced per-scenario batching which can cut
+ * Anthropic spend by ~75% on unmatched-heavy scaffolds. Until this
+ * release, the savings were invisible — operators had no per-scaffold
+ * record of cache hit rate, batch size distribution, token counts, or
+ * estimated cost. The sidecar makes the ROI measurable per run.
+ *
+ * Design constraints:
+ *   - Zero overhead when disabled. The collector is constructed
+ *     conditionally; the log-event fan-out is a single `if` check at
+ *     the bdd2pw orchestrator layer.
+ *   - Per-call detail PLUS aggregates. Operators want both "this
+ *     scaffold cost $0.07" and "the batch of 5 unmatched steps in
+ *     scenario X took 2.3s and consumed 6700 input tokens".
+ *   - Cost is an ESTIMATE. Provider prices change; the sidecar
+ *     records the per-million rates used so a reviewer can re-derive
+ *     the number against current pricing.
+ *   - Backwards compatible: no change to LLMClient interface, no
+ *     change to existing log consumers. The collector is a regular
+ *     `log: (e) => {...}` subscriber.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LLMTelemetry = exports.DEFAULT_PRICING = void 0;
+/**
+ * Per-million-token pricing snapshot used to estimate cost. Prices
+ * change; the sidecar records what we used so a reviewer can re-derive.
+ *
+ * Update when you know providers raised / lowered prices, OR pass an
+ * override via `LLMTelemetry`'s constructor.
+ */
+exports.DEFAULT_PRICING = {
+    // Anthropic Claude Sonnet 4.6 — Q4 2025 published pricing.
+    "claude-sonnet-4-6": { inputUsdPerMillion: 3, outputUsdPerMillion: 15 },
+    // Older Sonnet generation (still used by some clients) — same pricing.
+    "claude-3-7-sonnet-latest": {
+        inputUsdPerMillion: 3,
+        outputUsdPerMillion: 15,
+    },
+    // Anthropic Opus — when used.
+    "claude-opus-4-6": { inputUsdPerMillion: 15, outputUsdPerMillion: 75 },
+    // Anthropic Haiku — when used.
+    "claude-haiku-4-5-20251001": {
+        inputUsdPerMillion: 0.8,
+        outputUsdPerMillion: 4,
+    },
+};
+/**
+ * Telemetry collector. One instance per scaffold call. Subscribe via
+ * `.handleEvent` (typed to match the existing `LLMLogEvent` shape);
+ * read aggregates via `.toSummary()`; write to disk via
+ * `.writeSidecar(path)`.
+ */
+class LLMTelemetry {
+    constructor(scaffoldId, version, 
+    /** Override DEFAULT_PRICING when known per model. */
+    pricingOverride) {
+        this.scaffoldId = scaffoldId;
+        this.version = version;
+        this.pricingOverride = pricingOverride;
+        this.callsCounter = 0;
+        /**
+         * cache_miss events fire BEFORE provider_call_start in
+         * AnthropicLLMClient.generateBatchBindings (cache lookup is step 1,
+         * provider call is step 4). We accumulate misses here and snapshot
+         * the counter into the in-flight record when the call starts, so
+         * batchSize reports correctly regardless of event ordering.
+         */
+        this.pendingMisses = 0;
+        /**
+         * The call we're currently accumulating into. provider_call_start
+         * opens a record (capturing pendingMisses as batchSize); provider_call_done
+         * fills in latency + tokens and appends to completedCalls.
+         */
+        this.inFlight = null;
+        this.completedCalls = [];
+        this.cacheHits = 0;
+        this.cacheMisses = 0;
+        this.bindingsGenerated = 0;
+        this.callsAttempted = 0;
+        this.modelName = "unknown";
+        /**
+         * Subscribe this method to the AnthropicLLMClient log callback:
+         *
+         *     new AnthropicLLMClient({ ..., log: telemetry.handleEvent }, ...)
+         *
+         * Or compose with the existing pino logger by fanning out:
+         *
+         *     log: (e) => { pinoLog(e); telemetry.handleEvent(e); }
+         *
+         * Multiple events arrive per call. The collector reads the shape
+         * to advance its internal state machine.
+         */
+        this.handleEvent = (event) => {
+            switch (event.kind) {
+                case "cache_hit":
+                    this.cacheHits += 1;
+                    this.bindingsGenerated += 1;
+                    break;
+                case "cache_miss":
+                    this.cacheMisses += 1;
+                    // Real flow: cache_miss fires BEFORE provider_call_start
+                    // (cache lookup is step 1, provider call is step 4). Buffer
+                    // the misses in `pendingMisses` and snapshot them into the
+                    // record at start time. Also support the case where misses
+                    // fire AFTER start (older code paths / future refactors) by
+                    // adding to inFlight.batchSize too.
+                    if (this.inFlight)
+                        this.inFlight.batchSize += 1;
+                    else
+                        this.pendingMisses += 1;
+                    break;
+                case "provider_call_start":
+                    this.callsAttempted += 1;
+                    this.inFlight = {
+                        model: event.model,
+                        latencyMs: 0,
+                        inputTokens: 0,
+                        outputTokens: 0,
+                        batchSize: this.pendingMisses,
+                        startTime: Date.now(),
+                    };
+                    // Consume the buffered misses so a subsequent call doesn't
+                    // double-count them.
+                    this.pendingMisses = 0;
+                    // The pricing snapshot tracks whichever model we see first.
+                    // Most scaffolds use one model start-to-finish.
+                    this.modelName = event.model;
+                    const pricing = (this.pricingOverride && this.pricingOverride[event.model]) ??
+                        exports.DEFAULT_PRICING[event.model];
+                    if (pricing)
+                        this.modelPricing = pricing;
+                    break;
+                case "provider_call_done":
+                    if (this.inFlight) {
+                        this.inFlight.latencyMs = event.latencyMs;
+                        this.inFlight.inputTokens = event.inputTokens;
+                        this.inFlight.outputTokens = event.outputTokens;
+                        // batchSize may have been incremented by cache_miss events
+                        // between start and done. If somehow 0, default to 1 (single-
+                        // step generateBinding case where the orchestrator goes
+                        // through the per-step path that doesn't emit cache_miss in
+                        // the start→done window).
+                        if (this.inFlight.batchSize === 0)
+                            this.inFlight.batchSize = 1;
+                        this.callsCounter += 1;
+                        this.completedCalls.push({
+                            index: this.callsCounter,
+                            model: this.inFlight.model,
+                            inputTokens: this.inFlight.inputTokens,
+                            outputTokens: this.inFlight.outputTokens,
+                            latencyMs: this.inFlight.latencyMs,
+                            fromCache: false,
+                            batchSize: this.inFlight.batchSize,
+                        });
+                        this.inFlight = null;
+                    }
+                    break;
+                case "binding_parsed":
+                    this.bindingsGenerated += 1;
+                    break;
+                // error events don't affect aggregates; they're already
+                // captured by the surrounding scaffold log.
+                default:
+                    break;
+            }
+        };
+    }
+    toSummary() {
+        const latencies = this.completedCalls
+            .map((c) => c.latencyMs)
+            .sort((a, b) => a - b);
+        const inputTokens = this.completedCalls.reduce((acc, c) => acc + c.inputTokens, 0);
+        const outputTokens = this.completedCalls.reduce((acc, c) => acc + c.outputTokens, 0);
+        const pricing = this.modelPricing ?? {
+            inputUsdPerMillion: 0,
+            outputUsdPerMillion: 0,
+        };
+        const estimatedCostUsd = (inputTokens / 1000000) * pricing.inputUsdPerMillion +
+            (outputTokens / 1000000) * pricing.outputUsdPerMillion;
+        const totalCacheChecks = this.cacheHits + this.cacheMisses;
+        return {
+            version: this.version,
+            scaffoldId: this.scaffoldId,
+            generatedAt: new Date().toISOString(),
+            totals: {
+                callsAttempted: this.callsAttempted,
+                callsSuccessful: this.callsCounter,
+                bindingsGenerated: this.bindingsGenerated,
+                cacheHits: this.cacheHits,
+                cacheMisses: this.cacheMisses,
+                cacheHitRate: totalCacheChecks === 0 ? 0 : this.cacheHits / totalCacheChecks,
+                inputTokens,
+                outputTokens,
+                estimatedCostUsd: Number(estimatedCostUsd.toFixed(6)),
+            },
+            calls: this.completedCalls,
+            latencyMs: {
+                p50: percentile(latencies, 0.5),
+                p95: percentile(latencies, 0.95),
+                min: latencies.length ? latencies[0] : 0,
+                max: latencies.length ? latencies[latencies.length - 1] : 0,
+            },
+            pricing: {
+                model: this.modelName,
+                inputUsdPerMillion: pricing.inputUsdPerMillion,
+                outputUsdPerMillion: pricing.outputUsdPerMillion,
+            },
+        };
+    }
+}
+exports.LLMTelemetry = LLMTelemetry;
+/**
+ * Index-based percentile. Returns 0 for an empty array. Sorted input.
+ * Picks the ceiling index so p95 of [10, 20, 30, 40, 50] is 50, not 40.
+ */
+function percentile(sortedAsc, p) {
+    if (sortedAsc.length === 0)
+        return 0;
+    const idx = Math.ceil(sortedAsc.length * p) - 1;
+    return sortedAsc[Math.max(0, Math.min(idx, sortedAsc.length - 1))];
+}
+//# sourceMappingURL=telemetry.js.map

package/dist/llm/telemetry.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"telemetry.js","sourceRoot":"","sources":["../../src/llm/telemetry.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;;;AA8DH;;;;;;GAMG;AACU,QAAA,eAAe,GAGxB;IACF,2DAA2D;IAC3D,mBAAmB,EAAE,EAAE,kBAAkB,EAAE,CAAC,EAAE,mBAAmB,EAAE,EAAE,EAAE;IACvE,uEAAuE;IACvE,0BAA0B,EAAE;QAC1B,kBAAkB,EAAE,CAAC;QACrB,mBAAmB,EAAE,EAAE;KACxB;IACD,8BAA8B;IAC9B,iBAAiB,EAAE,EAAE,kBAAkB,EAAE,EAAE,EAAE,mBAAmB,EAAE,EAAE,EAAE;IACtE,+BAA+B;IAC/B,2BAA2B,EAAE;QAC3B,kBAAkB,EAAE,GAAG;QACvB,mBAAmB,EAAE,CAAC;KACvB;CACF,CAAC;AAEF;;;;;GAKG;AACH,MAAa,YAAY;IAiCvB,YACmB,UAAkB,EAClB,OAAe;IAChC,qDAAqD;IACpC,eAGhB;QANgB,eAAU,GAAV,UAAU,CAAQ;QAClB,YAAO,GAAP,OAAO,CAAQ;QAEf,oBAAe,GAAf,eAAe,CAG/B;QAvCK,iBAAY,GAAG,CAAC,CAAC;QACzB;;;;;;WAMG;QACK,kBAAa,GAAG,CAAC,CAAC;QAC1B;;;;WAIG;QACK,aAAQ,GAOL,IAAI,CAAC;QACR,mBAAc,GAAoB,EAAE,CAAC;QACrC,cAAS,GAAG,CAAC,CAAC;QACd,gBAAW,GAAG,CAAC,CAAC;QAChB,sBAAiB,GAAG,CAAC,CAAC;QACtB,mBAAc,GAAG,CAAC,CAAC;QAInB,cAAS,GAAG,SAAS,CAAC;QAY9B;;;;;;;;;;;WAWG;QACH,gBAAW,GAAG,CAAC,KAAkB,EAAQ,EAAE;YACzC,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;gBACnB,KAAK,WAAW;oBACd,IAAI,CAAC,SAAS,IAAI,CAAC,CAAC;oBACpB,IAAI,CAAC,iBAAiB,IAAI,CAAC,CAAC;oBAC5B,MAAM;gBACR,KAAK,YAAY;oBACf,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;oBACtB,yDAAyD;oBACzD,4DAA4D;oBAC5D,2DAA2D;oBAC3D,2DAA2D;oBAC3D,4DAA4D;oBAC5D,oCAAoC;oBACpC,IAAI,IAAI,CAAC,QAAQ;wBAAE,IAAI,CAAC,QAAQ,CAAC,SAAS,IAAI,CAAC,CAAC;;wBAC3C,IAAI,CAAC,aAAa,IAAI,CAAC,CAAC;oBAC7B,MAAM;gBACR,KAAK,qBAAqB;oBACxB,IAAI,CAAC,cAAc,IAAI,CAAC,CAAC;oBACzB,IAAI,CAAC,QAAQ,GAAG;wBACd,KAAK,EAAE,KAAK,CAAC,KAAK;wBAClB,SAAS,EAAE,CAAC;wBACZ,WAAW,EAAE,CAAC;wBACd,YAAY,EAAE,CAAC;wBACf,SAAS,EAAE,IAAI,CAAC,aAAa;wBAC7B,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;qBACtB,CAAC;oBACF,2DAA2D;oBAC3D,qBAAqB;oBACrB,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC;oBACvB,4DAA4D;oBAC5D,gDAAgD;oBAChD,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC;oBAC7B,MAAM,OAAO,GACX,CAAC,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;wBAC3D,uBAAe,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;oBAC/B,IAAI,OAAO;wBAAE,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC;oBACzC,MAAM;gBACR,KAAK,oBAAoB;oBACvB,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;wBAClB,IAAI,CAAC,QAAQ,CAAC,SAAS,GAAG,KAAK,CAAC,SAAS,CAAC;wBAC1C,IAAI,CAAC,QAAQ,CAAC,WAAW,GAAG,KAAK,CAAC,WAAW,CAAC;wBAC9C,IAAI,CAAC,QAAQ,CAAC,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC;wBAChD,2DAA2D;wBAC3D,8DAA8D;wBAC9D,wDAAwD;wBACxD,4DAA4D;wBAC5D,0BAA0B;wBAC1B,IAAI,IAAI,CAAC,QAAQ,CAAC,SAAS,KAAK,CAAC;4BAAE,IAAI,CAAC,QAAQ,CAAC,SAAS,GAAG,CAAC,CAAC;wBAC/D,IAAI,CAAC,YAAY,IAAI,CAAC,CAAC;wBACvB,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC;4BACvB,KAAK,EAAE,IAAI,CAAC,YAAY;4BACxB,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK;4BAC1B,WAAW,EAAE,IAAI,CAAC,QAAQ,CAAC,WAAW;4BACtC,YAAY,EAAE,IAAI,CAAC,QAAQ,CAAC,YAAY;4BACxC,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,SAAS;4BAClC,SAAS,EAAE,KAAK;4BAChB,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,SAAS;yBACnC,CAAC,CAAC;wBACH,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;oBACvB,CAAC;oBACD,MAAM;gBACR,KAAK,gBAAgB;oBACnB,IAAI,CAAC,iBAAiB,IAAI,CAAC,CAAC;oBAC5B,MAAM;gBACR,wDAAwD;gBACxD,4CAA4C;gBAC5C;oBACE,MAAM;YACV,CAAC;QACH,CAAC,CAAC;IApFC,CAAC;IAsFJ,SAAS;QACP,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc;aAClC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;aACvB,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACzB,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,MAAM,CAC5C,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,WAAW,EAC/B,CAAC,CACF,CAAC;QACF,MAAM,YAAY,GAAG,IAAI,CAAC,cAAc,CAAC,MAAM,CAC7C,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,YAAY,EAChC,CAAC,CACF,CAAC;QACF,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,IAAI;YACnC,kBAAkB,EAAE,CAAC;YACrB,mBAAmB,EAAE,CAAC;SACvB,CAAC;QACF,MAAM,gBAAgB,GACpB,CAAC,WAAW,GAAG,OAAS,CAAC,GAAG,OAAO,CAAC,kBAAkB;YACtD,CAAC,YAAY,GAAG,OAAS,CAAC,GAAG,OAAO,CAAC,mBAAmB,CAAC;QAC3D,MAAM,gBAAgB,GAAG,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC;QAC3D,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACrC,MAAM,EAAE;gBACN,cAAc,EAAE,IAAI,CAAC,cAAc;gBACnC,eAAe,EAAE,IAAI,CAAC,YAAY;gBAClC,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;gBACzC,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,YAAY,EACV,gBAAgB,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,GAAG,gBAAgB;gBAChE,WAAW;gBACX,YAAY;gBACZ,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;aACtD;YACD,KAAK,EAAE,IAAI,CAAC,cAAc;YAC1B,SAAS,EAAE;gBACT,GAAG,EAAE,UAAU,CAAC,SAAS,EAAE,GAAG,CAAC;gBAC/B,GAAG,EAAE,UAAU,CAAC,SAAS,EAAE,IAAI,CAAC;gBAChC,GAAG,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACxC,GAAG,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;aAC5D;YACD,OAAO,EAAE;gBACP,KAAK,EAAE,IAAI,CAAC,SAAS;gBACrB,kBAAkB,EAAE,OAAO,CAAC,kBAAkB;gBAC9C,mBAAmB,EAAE,OAAO,CAAC,mBAAmB;aACjD;SACF,CAAC;IACJ,CAAC;CACF;AAjLD,oCAiLC;AAED;;;GAGG;AACH,SAAS,UAAU,CAAC,SAAmB,EAAE,CAAS;IAChD,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACrC,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;IAChD,OAAO,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;AACrE,CAAC"}

package/dist/reports/healStats.d.ts

@@ -0,0 +1,132 @@
+/**
+ * v3.10.0 — heal-events.jsonl → heal-stats.json analyzer.
+ *
+ * Same flavor as v3.9.0's `LLMTelemetry` sidecar but for the
+ * self-healing pipeline. Where v3.9.0 measures the LLM batching ROI
+ * per scaffold, v3.10.0 measures the self-healing ROI per TEST RUN.
+ *
+ * Input:  `<repo>/artefacts/heal-events.jsonl` — one JSON event per
+ *         line, written by the runtime `healOrThrow` helper from
+ *         `templates/heal.ts.tmpl`. Event kinds:
+ *           - `register`            — locator constructed
+ *           - `heal_attempt`        — action failed; asking the heal
+ *                                     service for a candidate
+ *           - `healed`              — candidate worked; subsequent
+ *                                     calls use it transparently
+ *           - `heal_unavailable`    — no candidate / candidate failed
+ *
+ * Output: `<repo>/artefacts/heal-stats.json` — stable shape:
+ *
+ *   {
+ *     "version": "3.10.0",
+ *     "generatedAt": "2026-...",
+ *     "source": "<events file>",
+ *     "totals": {
+ *       "registrations": 47,
+ *       "healAttempts": 6,
+ *       "healed": 4,
+ *       "healUnavailable": 2,
+ *       "healRate": 0.6667,         // healed / healAttempts
+ *       "uniqueFields": 23,
+ *       "uniquePages": 3
+ *     },
+ *     "topFailingFields": [
+ *       { "page": "LoginPage", "name": "submitButton", "attempts": 3, "healed": 2 },
+ *       ...
+ *     ],
+ *     "topErrors": [
+ *       { "error": "Timeout 30000ms exceeded", "count": 4 },
+ *       ...
+ *     ],
+ *     "topCandidates": [
+ *       { "selector": "[data-testid='submit']", "promotions": 2,
+ *         "averageConfidence": 0.87 },
+ *       ...
+ *     ],
+ *     "retryLatencyMs": { "p50": 1240, "p95": 3010, "min": 880, "max": 3010 },
+ *     "perScenario": [
+ *       { "scenario": "User can log in", "attempts": 1, "healed": 1 },
+ *       ...
+ *     ]
+ *   }
+ *
+ * Why a separate command (not part of scaffold): scaffold GENERATES the
+ * test repo; the heal events are produced when the user RUNS the
+ * generated tests. So this analyzer is a post-test command, run from
+ * CI after `npx playwright test`. Same pattern as `bdd2pw propose-rules`
+ * which runs after the LLM has populated `candidate-rules.jsonl`.
+ */
+export interface HealStatsOptions {
+    /** Path to heal-events.jsonl OR a repo dir containing it. */
+    inputPath: string;
+    /** Override output path. Default: <input dir>/heal-stats.json. */
+    outputPath?: string;
+    /** How many top entries to keep in each ranked list. Default 10. */
+    topN?: number;
+}
+export interface HealStatsResult {
+    outputPath: string;
+    totalEvents: number;
+}
+/** One event line in heal-events.jsonl. Shape tracks `templates/heal.ts.tmpl`. */
+interface HealEvent {
+    ts: string;
+    event: "register" | "heal_attempt" | "healed" | "heal_unavailable";
+    page: string;
+    name: string;
+    scenario_name?: string | null;
+    method?: string;
+    original?: string | null;
+    healed?: string | null;
+    confidence?: number | null;
+    error?: string | null;
+}
+export interface HealStats {
+    version: string;
+    generatedAt: string;
+    source: string;
+    totals: {
+        registrations: number;
+        healAttempts: number;
+        healed: number;
+        healUnavailable: number;
+        healRate: number;
+        uniqueFields: number;
+        uniquePages: number;
+    };
+    topFailingFields: {
+        page: string;
+        name: string;
+        attempts: number;
+        healed: number;
+    }[];
+    topErrors: {
+        error: string;
+        count: number;
+    }[];
+    topCandidates: {
+        selector: string;
+        promotions: number;
+        averageConfidence: number | null;
+    }[];
+    retryLatencyMs: {
+        p50: number;
+        p95: number;
+        min: number;
+        max: number;
+    };
+    perScenario: {
+        scenario: string;
+        attempts: number;
+        healed: number;
+    }[];
+}
+/** Public entry point. */
+export declare function analyseHealStats(opts: HealStatsOptions): Promise<HealStatsResult>;
+/**
+ * Compute a HealStats summary from an in-memory event list. Pure
+ * function — no I/O. Used both by the public CLI entry and the unit
+ * tests.
+ */
+export declare function aggregate(events: HealEvent[], source: string, topN: number): HealStats;
+export {};

package/dist/reports/healStats.js

@@ -0,0 +1,303 @@
+"use strict";
+/**
+ * v3.10.0 — heal-events.jsonl → heal-stats.json analyzer.
+ *
+ * Same flavor as v3.9.0's `LLMTelemetry` sidecar but for the
+ * self-healing pipeline. Where v3.9.0 measures the LLM batching ROI
+ * per scaffold, v3.10.0 measures the self-healing ROI per TEST RUN.
+ *
+ * Input:  `<repo>/artefacts/heal-events.jsonl` — one JSON event per
+ *         line, written by the runtime `healOrThrow` helper from
+ *         `templates/heal.ts.tmpl`. Event kinds:
+ *           - `register`            — locator constructed
+ *           - `heal_attempt`        — action failed; asking the heal
+ *                                     service for a candidate
+ *           - `healed`              — candidate worked; subsequent
+ *                                     calls use it transparently
+ *           - `heal_unavailable`    — no candidate / candidate failed
+ *
+ * Output: `<repo>/artefacts/heal-stats.json` — stable shape:
+ *
+ *   {
+ *     "version": "3.10.0",
+ *     "generatedAt": "2026-...",
+ *     "source": "<events file>",
+ *     "totals": {
+ *       "registrations": 47,
+ *       "healAttempts": 6,
+ *       "healed": 4,
+ *       "healUnavailable": 2,
+ *       "healRate": 0.6667,         // healed / healAttempts
+ *       "uniqueFields": 23,
+ *       "uniquePages": 3
+ *     },
+ *     "topFailingFields": [
+ *       { "page": "LoginPage", "name": "submitButton", "attempts": 3, "healed": 2 },
+ *       ...
+ *     ],
+ *     "topErrors": [
+ *       { "error": "Timeout 30000ms exceeded", "count": 4 },
+ *       ...
+ *     ],
+ *     "topCandidates": [
+ *       { "selector": "[data-testid='submit']", "promotions": 2,
+ *         "averageConfidence": 0.87 },
+ *       ...
+ *     ],
+ *     "retryLatencyMs": { "p50": 1240, "p95": 3010, "min": 880, "max": 3010 },
+ *     "perScenario": [
+ *       { "scenario": "User can log in", "attempts": 1, "healed": 1 },
+ *       ...
+ *     ]
+ *   }
+ *
+ * Why a separate command (not part of scaffold): scaffold GENERATES the
+ * test repo; the heal events are produced when the user RUNS the
+ * generated tests. So this analyzer is a post-test command, run from
+ * CI after `npx playwright test`. Same pattern as `bdd2pw propose-rules`
+ * which runs after the LLM has populated `candidate-rules.jsonl`.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.analyseHealStats = analyseHealStats;
+exports.aggregate = aggregate;
+const fs = __importStar(require("fs-extra"));
+const path = __importStar(require("path"));
+const PACKAGE_VERSION = "3.10.0";
+/** Public entry point. */
+async function analyseHealStats(opts) {
+    const jsonlPath = await resolveJsonlPath(opts.inputPath);
+    const events = await readEvents(jsonlPath);
+    const summary = aggregate(events, jsonlPath, opts.topN ?? 10);
+    const outputPath = opts.outputPath ?? path.join(path.dirname(jsonlPath), "heal-stats.json");
+    await fs.ensureDir(path.dirname(outputPath));
+    await fs.writeFile(outputPath, JSON.stringify(summary, null, 2), "utf8");
+    return { outputPath, totalEvents: events.length };
+}
+/* ─── pure aggregation (testable without fs) ──────────────────────── */
+/**
+ * Compute a HealStats summary from an in-memory event list. Pure
+ * function — no I/O. Used both by the public CLI entry and the unit
+ * tests.
+ */
+function aggregate(events, source, topN) {
+    const fieldKey = (e) => `${e.page}::${e.name}`;
+    const fieldAttempts = new Map();
+    const errorCounts = new Map();
+    const candidateStats = new Map();
+    const scenarioStats = new Map();
+    const uniquePages = new Set();
+    const uniqueFields = new Set();
+    const retryLatencies = [];
+    /**
+     * Match heal_attempt events with their matching `healed` /
+     * `heal_unavailable` outcome by (page, name, method). The keying
+     * assumes a heal attempt resolves before another one fires for the
+     * same field+method combo, which holds in practice — actions run
+     * serially per test.
+     */
+    const openAttempts = new Map();
+    const attemptKey = (e) => `${e.page}::${e.name}::${e.method ?? ""}`;
+    let registrations = 0;
+    let healAttempts = 0;
+    let healed = 0;
+    let healUnavailable = 0;
+    for (const e of events) {
+        uniquePages.add(e.page);
+        uniqueFields.add(fieldKey(e));
+        if (e.event === "register") {
+            registrations += 1;
+            continue;
+        }
+        if (e.event === "heal_attempt") {
+            healAttempts += 1;
+            const field = fieldAttempts.get(fieldKey(e)) ?? {
+                page: e.page,
+                name: e.name,
+                attempts: 0,
+                healed: 0,
+            };
+            field.attempts += 1;
+            fieldAttempts.set(fieldKey(e), field);
+            if (e.error) {
+                const msg = truncateError(e.error);
+                errorCounts.set(msg, (errorCounts.get(msg) ?? 0) + 1);
+            }
+            const scenario = e.scenario_name ?? "(no scenario context)";
+            const ss = scenarioStats.get(scenario) ?? { attempts: 0, healed: 0 };
+            ss.attempts += 1;
+            scenarioStats.set(scenario, ss);
+            openAttempts.set(attemptKey(e), Date.parse(e.ts));
+            continue;
+        }
+        if (e.event === "healed") {
+            healed += 1;
+            const field = fieldAttempts.get(fieldKey(e)) ?? {
+                page: e.page,
+                name: e.name,
+                attempts: 0,
+                healed: 0,
+            };
+            field.healed += 1;
+            fieldAttempts.set(fieldKey(e), field);
+            if (e.healed) {
+                const cs = candidateStats.get(e.healed) ?? {
+                    promotions: 0,
+                    confidenceSum: 0,
+                    confidenceCount: 0,
+                };
+                cs.promotions += 1;
+                if (typeof e.confidence === "number") {
+                    cs.confidenceSum += e.confidence;
+                    cs.confidenceCount += 1;
+                }
+                candidateStats.set(e.healed, cs);
+            }
+            const scenario = e.scenario_name ?? "(no scenario context)";
+            const ss = scenarioStats.get(scenario) ?? { attempts: 0, healed: 0 };
+            ss.healed += 1;
+            scenarioStats.set(scenario, ss);
+            const startTs = openAttempts.get(attemptKey(e));
+            if (startTs !== undefined) {
+                retryLatencies.push(Date.parse(e.ts) - startTs);
+                openAttempts.delete(attemptKey(e));
+            }
+            continue;
+        }
+        if (e.event === "heal_unavailable") {
+            healUnavailable += 1;
+            openAttempts.delete(attemptKey(e));
+            continue;
+        }
+    }
+    const sortedFailingFields = Array.from(fieldAttempts.values())
+        .sort((a, b) => b.attempts - a.attempts)
+        .slice(0, topN);
+    const sortedErrors = Array.from(errorCounts.entries())
+        .map(([error, count]) => ({ error, count }))
+        .sort((a, b) => b.count - a.count)
+        .slice(0, topN);
+    const sortedCandidates = Array.from(candidateStats.entries())
+        .map(([selector, s]) => ({
+        selector,
+        promotions: s.promotions,
+        averageConfidence: s.confidenceCount > 0 ? s.confidenceSum / s.confidenceCount : null,
+    }))
+        .sort((a, b) => b.promotions - a.promotions)
+        .slice(0, topN);
+    const sortedScenarios = Array.from(scenarioStats.entries())
+        .map(([scenario, s]) => ({ scenario, ...s }))
+        .sort((a, b) => b.attempts - a.attempts)
+        .slice(0, topN);
+    const latenciesSorted = [...retryLatencies].sort((a, b) => a - b);
+    return {
+        version: PACKAGE_VERSION,
+        generatedAt: new Date().toISOString(),
+        source,
+        totals: {
+            registrations,
+            healAttempts,
+            healed,
+            healUnavailable,
+            healRate: healAttempts === 0 ? 0 : healed / healAttempts,
+            uniqueFields: uniqueFields.size,
+            uniquePages: uniquePages.size,
+        },
+        topFailingFields: sortedFailingFields,
+        topErrors: sortedErrors,
+        topCandidates: sortedCandidates,
+        retryLatencyMs: {
+            p50: percentile(latenciesSorted, 0.5),
+            p95: percentile(latenciesSorted, 0.95),
+            min: latenciesSorted.length ? latenciesSorted[0] : 0,
+            max: latenciesSorted.length
+                ? latenciesSorted[latenciesSorted.length - 1]
+                : 0,
+        },
+        perScenario: sortedScenarios,
+    };
+}
+function percentile(sortedAsc, p) {
+    if (sortedAsc.length === 0)
+        return 0;
+    const idx = Math.ceil(sortedAsc.length * p) - 1;
+    return sortedAsc[Math.max(0, Math.min(idx, sortedAsc.length - 1))];
+}
+/**
+ * Normalise an error message so different timestamps / IDs collapse to
+ * the same bucket: lowercase, strip leading "Error: " / "TimeoutError",
+ * collapse runs of digits.
+ */
+function truncateError(s) {
+    return s
+        .toLowerCase()
+        .replace(/^(?:error|timeouterror|protocolerror):\s*/i, "")
+        .replace(/\b\d{3,}\b/g, "<n>")
+        .slice(0, 120);
+}
+/* ─── filesystem glue ─────────────────────────────────────────────── */
+async function resolveJsonlPath(inputPath) {
+    const stat = await fs.stat(inputPath).catch(() => undefined);
+    if (stat?.isFile())
+        return inputPath;
+    if (stat?.isDirectory()) {
+        const direct = path.join(inputPath, "heal-events.jsonl");
+        if (await fs.pathExists(direct))
+            return direct;
+        const nested = path.join(inputPath, "artefacts", "heal-events.jsonl");
+        if (await fs.pathExists(nested))
+            return nested;
+        return nested; // analyser will handle 'no events' gracefully
+    }
+    return path.join(inputPath, "artefacts", "heal-events.jsonl");
+}
+async function readEvents(jsonlPath) {
+    if (!(await fs.pathExists(jsonlPath)))
+        return [];
+    const raw = await fs.readFile(jsonlPath, "utf8");
+    const out = [];
+    for (const line of raw.split(/\r?\n/)) {
+        if (!line.trim())
+            continue;
+        try {
+            out.push(JSON.parse(line));
+        }
+        catch {
+            /* skip malformed line */
+        }
+    }
+    return out;
+}
+//# sourceMappingURL=healStats.js.map