@vijaypjavvadi/bdd2pw 3.8.0 → 3.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +81 -0
- package/README.md +396 -332
- package/dist/index.js +29 -1
- package/dist/index.js.map +1 -1
- package/dist/llm/index.d.ts +1 -0
- package/dist/llm/index.js +4 -1
- package/dist/llm/index.js.map +1 -1
- package/dist/llm/prompt.js +66 -66
- package/dist/llm/telemetry.d.ts +150 -0
- package/dist/llm/telemetry.js +230 -0
- package/dist/llm/telemetry.js.map +1 -0
- package/dist/types.d.ts +11 -0
- package/package.json +1 -1
- package/templates/heal.ts.tmpl +452 -452
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* v3.9.0 — LLM telemetry collector.
|
|
4
|
+
*
|
|
5
|
+
* Subscribes to the existing `LLMLogEvent` stream that AnthropicLLMClient
|
|
6
|
+
* emits during `generateBinding` / `generateBatchBindings`, accumulates
|
|
7
|
+
* per-call stats, and exposes an aggregated summary suitable for writing
|
|
8
|
+
* to `<repo>/artefacts/llm-stats.json`.
|
|
9
|
+
*
|
|
10
|
+
* Why this exists: v3.5.0 introduced per-scenario batching which can cut
|
|
11
|
+
* Anthropic spend by ~75% on unmatched-heavy scaffolds. Until this
|
|
12
|
+
* release, the savings were invisible — operators had no per-scaffold
|
|
13
|
+
* record of cache hit rate, batch size distribution, token counts, or
|
|
14
|
+
* estimated cost. The sidecar makes the ROI measurable per run.
|
|
15
|
+
*
|
|
16
|
+
* Design constraints:
|
|
17
|
+
* - Zero overhead when disabled. The collector is constructed
|
|
18
|
+
* conditionally; the log-event fan-out is a single `if` check at
|
|
19
|
+
* the bdd2pw orchestrator layer.
|
|
20
|
+
* - Per-call detail PLUS aggregates. Operators want both "this
|
|
21
|
+
* scaffold cost $0.07" and "the batch of 5 unmatched steps in
|
|
22
|
+
* scenario X took 2.3s and consumed 6700 input tokens".
|
|
23
|
+
* - Cost is an ESTIMATE. Provider prices change; the sidecar
|
|
24
|
+
* records the per-million rates used so a reviewer can re-derive
|
|
25
|
+
* the number against current pricing.
|
|
26
|
+
* - Backwards compatible: no change to LLMClient interface, no
|
|
27
|
+
* change to existing log consumers. The collector is a regular
|
|
28
|
+
* `log: (e) => {...}` subscriber.
|
|
29
|
+
*/
|
|
30
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
31
|
+
exports.LLMTelemetry = exports.DEFAULT_PRICING = void 0;
|
|
32
|
+
/**
|
|
33
|
+
* Per-million-token pricing snapshot used to estimate cost. Prices
|
|
34
|
+
* change; the sidecar records what we used so a reviewer can re-derive.
|
|
35
|
+
*
|
|
36
|
+
* Update when you know providers raised / lowered prices, OR pass an
|
|
37
|
+
* override via `LLMTelemetry`'s constructor.
|
|
38
|
+
*/
|
|
39
|
+
exports.DEFAULT_PRICING = {
|
|
40
|
+
// Anthropic Claude Sonnet 4.6 — Q4 2025 published pricing.
|
|
41
|
+
"claude-sonnet-4-6": { inputUsdPerMillion: 3, outputUsdPerMillion: 15 },
|
|
42
|
+
// Older Sonnet generation (still used by some clients) — same pricing.
|
|
43
|
+
"claude-3-7-sonnet-latest": {
|
|
44
|
+
inputUsdPerMillion: 3,
|
|
45
|
+
outputUsdPerMillion: 15,
|
|
46
|
+
},
|
|
47
|
+
// Anthropic Opus — when used.
|
|
48
|
+
"claude-opus-4-6": { inputUsdPerMillion: 15, outputUsdPerMillion: 75 },
|
|
49
|
+
// Anthropic Haiku — when used.
|
|
50
|
+
"claude-haiku-4-5-20251001": {
|
|
51
|
+
inputUsdPerMillion: 0.8,
|
|
52
|
+
outputUsdPerMillion: 4,
|
|
53
|
+
},
|
|
54
|
+
};
|
|
55
|
+
/**
|
|
56
|
+
* Telemetry collector. One instance per scaffold call. Subscribe via
|
|
57
|
+
* `.handleEvent` (typed to match the existing `LLMLogEvent` shape);
|
|
58
|
+
* read aggregates via `.toSummary()`; write to disk via
|
|
59
|
+
* `.writeSidecar(path)`.
|
|
60
|
+
*/
|
|
61
|
+
class LLMTelemetry {
|
|
62
|
+
constructor(scaffoldId, version,
|
|
63
|
+
/** Override DEFAULT_PRICING when known per model. */
|
|
64
|
+
pricingOverride) {
|
|
65
|
+
this.scaffoldId = scaffoldId;
|
|
66
|
+
this.version = version;
|
|
67
|
+
this.pricingOverride = pricingOverride;
|
|
68
|
+
this.callsCounter = 0;
|
|
69
|
+
/**
|
|
70
|
+
* cache_miss events fire BEFORE provider_call_start in
|
|
71
|
+
* AnthropicLLMClient.generateBatchBindings (cache lookup is step 1,
|
|
72
|
+
* provider call is step 4). We accumulate misses here and snapshot
|
|
73
|
+
* the counter into the in-flight record when the call starts, so
|
|
74
|
+
* batchSize reports correctly regardless of event ordering.
|
|
75
|
+
*/
|
|
76
|
+
this.pendingMisses = 0;
|
|
77
|
+
/**
|
|
78
|
+
* The call we're currently accumulating into. provider_call_start
|
|
79
|
+
* opens a record (capturing pendingMisses as batchSize); provider_call_done
|
|
80
|
+
* fills in latency + tokens and appends to completedCalls.
|
|
81
|
+
*/
|
|
82
|
+
this.inFlight = null;
|
|
83
|
+
this.completedCalls = [];
|
|
84
|
+
this.cacheHits = 0;
|
|
85
|
+
this.cacheMisses = 0;
|
|
86
|
+
this.bindingsGenerated = 0;
|
|
87
|
+
this.callsAttempted = 0;
|
|
88
|
+
this.modelName = "unknown";
|
|
89
|
+
/**
|
|
90
|
+
* Subscribe this method to the AnthropicLLMClient log callback:
|
|
91
|
+
*
|
|
92
|
+
* new AnthropicLLMClient({ ..., log: telemetry.handleEvent }, ...)
|
|
93
|
+
*
|
|
94
|
+
* Or compose with the existing pino logger by fanning out:
|
|
95
|
+
*
|
|
96
|
+
* log: (e) => { pinoLog(e); telemetry.handleEvent(e); }
|
|
97
|
+
*
|
|
98
|
+
* Multiple events arrive per call. The collector reads the shape
|
|
99
|
+
* to advance its internal state machine.
|
|
100
|
+
*/
|
|
101
|
+
this.handleEvent = (event) => {
|
|
102
|
+
switch (event.kind) {
|
|
103
|
+
case "cache_hit":
|
|
104
|
+
this.cacheHits += 1;
|
|
105
|
+
this.bindingsGenerated += 1;
|
|
106
|
+
break;
|
|
107
|
+
case "cache_miss":
|
|
108
|
+
this.cacheMisses += 1;
|
|
109
|
+
// Real flow: cache_miss fires BEFORE provider_call_start
|
|
110
|
+
// (cache lookup is step 1, provider call is step 4). Buffer
|
|
111
|
+
// the misses in `pendingMisses` and snapshot them into the
|
|
112
|
+
// record at start time. Also support the case where misses
|
|
113
|
+
// fire AFTER start (older code paths / future refactors) by
|
|
114
|
+
// adding to inFlight.batchSize too.
|
|
115
|
+
if (this.inFlight)
|
|
116
|
+
this.inFlight.batchSize += 1;
|
|
117
|
+
else
|
|
118
|
+
this.pendingMisses += 1;
|
|
119
|
+
break;
|
|
120
|
+
case "provider_call_start":
|
|
121
|
+
this.callsAttempted += 1;
|
|
122
|
+
this.inFlight = {
|
|
123
|
+
model: event.model,
|
|
124
|
+
latencyMs: 0,
|
|
125
|
+
inputTokens: 0,
|
|
126
|
+
outputTokens: 0,
|
|
127
|
+
batchSize: this.pendingMisses,
|
|
128
|
+
startTime: Date.now(),
|
|
129
|
+
};
|
|
130
|
+
// Consume the buffered misses so a subsequent call doesn't
|
|
131
|
+
// double-count them.
|
|
132
|
+
this.pendingMisses = 0;
|
|
133
|
+
// The pricing snapshot tracks whichever model we see first.
|
|
134
|
+
// Most scaffolds use one model start-to-finish.
|
|
135
|
+
this.modelName = event.model;
|
|
136
|
+
const pricing = (this.pricingOverride && this.pricingOverride[event.model]) ??
|
|
137
|
+
exports.DEFAULT_PRICING[event.model];
|
|
138
|
+
if (pricing)
|
|
139
|
+
this.modelPricing = pricing;
|
|
140
|
+
break;
|
|
141
|
+
case "provider_call_done":
|
|
142
|
+
if (this.inFlight) {
|
|
143
|
+
this.inFlight.latencyMs = event.latencyMs;
|
|
144
|
+
this.inFlight.inputTokens = event.inputTokens;
|
|
145
|
+
this.inFlight.outputTokens = event.outputTokens;
|
|
146
|
+
// batchSize may have been incremented by cache_miss events
|
|
147
|
+
// between start and done. If somehow 0, default to 1 (single-
|
|
148
|
+
// step generateBinding case where the orchestrator goes
|
|
149
|
+
// through the per-step path that doesn't emit cache_miss in
|
|
150
|
+
// the start→done window).
|
|
151
|
+
if (this.inFlight.batchSize === 0)
|
|
152
|
+
this.inFlight.batchSize = 1;
|
|
153
|
+
this.callsCounter += 1;
|
|
154
|
+
this.completedCalls.push({
|
|
155
|
+
index: this.callsCounter,
|
|
156
|
+
model: this.inFlight.model,
|
|
157
|
+
inputTokens: this.inFlight.inputTokens,
|
|
158
|
+
outputTokens: this.inFlight.outputTokens,
|
|
159
|
+
latencyMs: this.inFlight.latencyMs,
|
|
160
|
+
fromCache: false,
|
|
161
|
+
batchSize: this.inFlight.batchSize,
|
|
162
|
+
});
|
|
163
|
+
this.inFlight = null;
|
|
164
|
+
}
|
|
165
|
+
break;
|
|
166
|
+
case "binding_parsed":
|
|
167
|
+
this.bindingsGenerated += 1;
|
|
168
|
+
break;
|
|
169
|
+
// error events don't affect aggregates; they're already
|
|
170
|
+
// captured by the surrounding scaffold log.
|
|
171
|
+
default:
|
|
172
|
+
break;
|
|
173
|
+
}
|
|
174
|
+
};
|
|
175
|
+
}
|
|
176
|
+
toSummary() {
|
|
177
|
+
const latencies = this.completedCalls
|
|
178
|
+
.map((c) => c.latencyMs)
|
|
179
|
+
.sort((a, b) => a - b);
|
|
180
|
+
const inputTokens = this.completedCalls.reduce((acc, c) => acc + c.inputTokens, 0);
|
|
181
|
+
const outputTokens = this.completedCalls.reduce((acc, c) => acc + c.outputTokens, 0);
|
|
182
|
+
const pricing = this.modelPricing ?? {
|
|
183
|
+
inputUsdPerMillion: 0,
|
|
184
|
+
outputUsdPerMillion: 0,
|
|
185
|
+
};
|
|
186
|
+
const estimatedCostUsd = (inputTokens / 1000000) * pricing.inputUsdPerMillion +
|
|
187
|
+
(outputTokens / 1000000) * pricing.outputUsdPerMillion;
|
|
188
|
+
const totalCacheChecks = this.cacheHits + this.cacheMisses;
|
|
189
|
+
return {
|
|
190
|
+
version: this.version,
|
|
191
|
+
scaffoldId: this.scaffoldId,
|
|
192
|
+
generatedAt: new Date().toISOString(),
|
|
193
|
+
totals: {
|
|
194
|
+
callsAttempted: this.callsAttempted,
|
|
195
|
+
callsSuccessful: this.callsCounter,
|
|
196
|
+
bindingsGenerated: this.bindingsGenerated,
|
|
197
|
+
cacheHits: this.cacheHits,
|
|
198
|
+
cacheMisses: this.cacheMisses,
|
|
199
|
+
cacheHitRate: totalCacheChecks === 0 ? 0 : this.cacheHits / totalCacheChecks,
|
|
200
|
+
inputTokens,
|
|
201
|
+
outputTokens,
|
|
202
|
+
estimatedCostUsd: Number(estimatedCostUsd.toFixed(6)),
|
|
203
|
+
},
|
|
204
|
+
calls: this.completedCalls,
|
|
205
|
+
latencyMs: {
|
|
206
|
+
p50: percentile(latencies, 0.5),
|
|
207
|
+
p95: percentile(latencies, 0.95),
|
|
208
|
+
min: latencies.length ? latencies[0] : 0,
|
|
209
|
+
max: latencies.length ? latencies[latencies.length - 1] : 0,
|
|
210
|
+
},
|
|
211
|
+
pricing: {
|
|
212
|
+
model: this.modelName,
|
|
213
|
+
inputUsdPerMillion: pricing.inputUsdPerMillion,
|
|
214
|
+
outputUsdPerMillion: pricing.outputUsdPerMillion,
|
|
215
|
+
},
|
|
216
|
+
};
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
exports.LLMTelemetry = LLMTelemetry;
|
|
220
|
+
/**
|
|
221
|
+
* Index-based percentile. Returns 0 for an empty array. Sorted input.
|
|
222
|
+
* Picks the ceiling index so p95 of [10, 20, 30, 40, 50] is 50, not 40.
|
|
223
|
+
*/
|
|
224
|
+
function percentile(sortedAsc, p) {
|
|
225
|
+
if (sortedAsc.length === 0)
|
|
226
|
+
return 0;
|
|
227
|
+
const idx = Math.ceil(sortedAsc.length * p) - 1;
|
|
228
|
+
return sortedAsc[Math.max(0, Math.min(idx, sortedAsc.length - 1))];
|
|
229
|
+
}
|
|
230
|
+
//# sourceMappingURL=telemetry.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"telemetry.js","sourceRoot":"","sources":["../../src/llm/telemetry.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;;;AA8DH;;;;;;GAMG;AACU,QAAA,eAAe,GAGxB;IACF,2DAA2D;IAC3D,mBAAmB,EAAE,EAAE,kBAAkB,EAAE,CAAC,EAAE,mBAAmB,EAAE,EAAE,EAAE;IACvE,uEAAuE;IACvE,0BAA0B,EAAE;QAC1B,kBAAkB,EAAE,CAAC;QACrB,mBAAmB,EAAE,EAAE;KACxB;IACD,8BAA8B;IAC9B,iBAAiB,EAAE,EAAE,kBAAkB,EAAE,EAAE,EAAE,mBAAmB,EAAE,EAAE,EAAE;IACtE,+BAA+B;IAC/B,2BAA2B,EAAE;QAC3B,kBAAkB,EAAE,GAAG;QACvB,mBAAmB,EAAE,CAAC;KACvB;CACF,CAAC;AAEF;;;;;GAKG;AACH,MAAa,YAAY;IAiCvB,YACmB,UAAkB,EAClB,OAAe;IAChC,qDAAqD;IACpC,eAGhB;QANgB,eAAU,GAAV,UAAU,CAAQ;QAClB,YAAO,GAAP,OAAO,CAAQ;QAEf,oBAAe,GAAf,eAAe,CAG/B;QAvCK,iBAAY,GAAG,CAAC,CAAC;QACzB;;;;;;WAMG;QACK,kBAAa,GAAG,CAAC,CAAC;QAC1B;;;;WAIG;QACK,aAAQ,GAOL,IAAI,CAAC;QACR,mBAAc,GAAoB,EAAE,CAAC;QACrC,cAAS,GAAG,CAAC,CAAC;QACd,gBAAW,GAAG,CAAC,CAAC;QAChB,sBAAiB,GAAG,CAAC,CAAC;QACtB,mBAAc,GAAG,CAAC,CAAC;QAInB,cAAS,GAAG,SAAS,CAAC;QAY9B;;;;;;;;;;;WAWG;QACH,gBAAW,GAAG,CAAC,KAAkB,EAAQ,EAAE;YACzC,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;gBACnB,KAAK,WAAW;oBACd,IAAI,CAAC,SAAS,IAAI,CAAC,CAAC;oBACpB,IAAI,CAAC,iBAAiB,IAAI,CAAC,CAAC;oBAC5B,MAAM;gBACR,KAAK,YAAY;oBACf,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;oBACtB,yDAAyD;oBACzD,4DAA4D;oBAC5D,2DAA2D;oBAC3D,2DAA2D;oBAC3D,4DAA4D;oBAC5D,oCAAoC;oBACpC,IAAI,IAAI,CAAC,QAAQ;wBAAE,IAAI,CAAC,QAAQ,CAAC,SAAS,IAAI,CAAC,CAAC;;wBAC3C,IAAI,CAAC,aAAa,IAAI,CAAC,CAAC;oBAC7B,MAAM;gBACR,KAAK,qBAAqB;oBACxB,IAAI,CAAC,cAAc,IAAI,CAAC,CAAC;oBACzB,IAAI,CAAC,QAAQ,GAAG;wBACd,KAAK,EAAE,KAAK,CAAC,KAAK;wBAClB,SAAS,EAAE,CAAC;wBACZ,WAAW,EAAE,CAAC;wBACd,YAAY,EAAE,CAAC;wBACf,SAAS,EAAE,IAAI,CAAC,aAAa;wBAC7B,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;qBACtB,CAAC;oBACF,2DAA2D;oBAC3D,qBAAqB;oBACrB,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC;oBACvB,4DAA4D;oBAC5D,gDAAgD;oBAChD,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC;oBAC7B,MAAM,OAAO,GACX,CAAC,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;wBAC3D,uBAAe,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;oBAC/B,IAAI,OAAO;wBAAE,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC;oBACzC,MAAM;gBACR,KAAK,oBAAoB;oBACvB,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;wBAClB,IAAI,CAAC,QAAQ,CAAC,SAAS,GAAG,KAAK,CAAC,SAAS,CAAC;wBAC1C,IAAI,CAAC,QAAQ,CAAC,WAAW,GAAG,KAAK,CAAC,WAAW,CAAC;wBAC9C,IAAI,CAAC,QAAQ,CAAC,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC;wBAChD,2DAA2D;wBAC3D,8DAA8D;wBAC9D,wDAAwD;wBACxD,4DAA4D;wBAC5D,0BAA0B;wBAC1B,IAAI,IAAI,CAAC,QAAQ,CAAC,SAAS,KAAK,CAAC;4BAAE,IAAI,CAAC,QAAQ,CAAC,SAAS,GAAG,CAAC,CAAC;wBAC/D,IAAI,CAAC,YAAY,IAAI,CAAC,CAAC;wBACvB,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC;4BACvB,KAAK,EAAE,IAAI,CAAC,YAAY;4BACxB,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK;4BAC1B,WAAW,EAAE,IAAI,CAAC,QAAQ,CAAC,WAAW;4BACtC,YAAY,EAAE,IAAI,CAAC,QAAQ,CAAC,YAAY;4BACxC,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,SAAS;4BAClC,SAAS,EAAE,KAAK;4BAChB,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,SAAS;yBACnC,CAAC,CAAC;wBACH,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;oBACvB,CAAC;oBACD,MAAM;gBACR,KAAK,gBAAgB;oBACnB,IAAI,CAAC,iBAAiB,IAAI,CAAC,CAAC;oBAC5B,MAAM;gBACR,wDAAwD;gBACxD,4CAA4C;gBAC5C;oBACE,MAAM;YACV,CAAC;QACH,CAAC,CAAC;IApFC,CAAC;IAsFJ,SAAS;QACP,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc;aAClC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;aACvB,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACzB,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,MAAM,CAC5C,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,WAAW,EAC/B,CAAC,CACF,CAAC;QACF,MAAM,YAAY,GAAG,IAAI,CAAC,cAAc,CAAC,MAAM,CAC7C,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,YAAY,EAChC,CAAC,CACF,CAAC;QACF,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,IAAI;YACnC,kBAAkB,EAAE,CAAC;YACrB,mBAAmB,EAAE,CAAC;SACvB,CAAC;QACF,MAAM,gBAAgB,GACpB,CAAC,WAAW,GAAG,OAAS,CAAC,GAAG,OAAO,CAAC,kBAAkB;YACtD,CAAC,YAAY,GAAG,OAAS,CAAC,GAAG,OAAO,CAAC,mBAAmB,CAAC;QAC3D,MAAM,gBAAgB,GAAG,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC;QAC3D,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACrC,MAAM,EAAE;gBACN,cAAc,EAAE,IAAI,CAAC,cAAc;gBACnC,eAAe,EAAE,IAAI,CAAC,YAAY;gBAClC,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;gBACzC,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,YAAY,EACV,gBAAgB,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,GAAG,gBAAgB;gBAChE,WAAW;gBACX,YAAY;gBACZ,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;aACtD;YACD,KAAK,EAAE,IAAI,CAAC,cAAc;YAC1B,SAAS,EAAE;gBACT,GAAG,EAAE,UAAU,CAAC,SAAS,EAAE,GAAG,CAAC;gBAC/B,GAAG,EAAE,UAAU,CAAC,SAAS,EAAE,IAAI,CAAC;gBAChC,GAAG,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACxC,GAAG,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;aAC5D;YACD,OAAO,EAAE;gBACP,KAAK,EAAE,IAAI,CAAC,SAAS;gBACrB,kBAAkB,EAAE,OAAO,CAAC,kBAAkB;gBAC9C,mBAAmB,EAAE,OAAO,CAAC,mBAAmB;aACjD;SACF,CAAC;IACJ,CAAC;CACF;AAjLD,oCAiLC;AAED;;;GAGG;AACH,SAAS,UAAU,CAAC,SAAmB,EAAE,CAAS;IAChD,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACrC,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;IAChD,OAAO,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;AACrE,CAAC"}
|
package/dist/types.d.ts
CHANGED
|
@@ -283,6 +283,17 @@ export interface ScaffoldOptions {
|
|
|
283
283
|
* pollute BDD_REVIEW.md for users who don't need it.
|
|
284
284
|
*/
|
|
285
285
|
diagnostics?: boolean;
|
|
286
|
+
/**
|
|
287
|
+
* v3.9.0 — opt-in LLM telemetry sidecar. When true, scaffold()
|
|
288
|
+
* writes `<repo>/artefacts/llm-stats.json` with per-call detail
|
|
289
|
+
* (batch size, latency, token counts, cache state) and aggregates
|
|
290
|
+
* (cache hit rate, latency p50/p95, estimated cost). Makes the
|
|
291
|
+
* v3.5 batching ROI measurable per scaffold.
|
|
292
|
+
*
|
|
293
|
+
* Off by default — operators opt in when they want to track spend
|
|
294
|
+
* or debug latency issues.
|
|
295
|
+
*/
|
|
296
|
+
llmStats?: boolean;
|
|
286
297
|
}
|
|
287
298
|
export interface ScaffoldResult {
|
|
288
299
|
filesWritten: string[];
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@vijaypjavvadi/bdd2pw",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.9.0",
|
|
4
4
|
"description": "Scaffold runnable Playwright TypeScript tests from Gherkin .feature files. Detects existing Page Objects, scans live pages via Microsoft Playwright MCP, emits POMs + specs, ready for execution. CLI + HTTP service. Optional governed LLM fallback (Anthropic / OpenAI / Gemini).",
|
|
5
5
|
"bin": {
|
|
6
6
|
"bdd2pw": "dist/cli.js"
|