canary-test-cli 7.0.0 → 7.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/engine/analysis/cli.js +116 -54
- package/dist/engine/analysis/engine.js +34 -16
- package/dist/engine/analysis/reports.js +5 -4
- package/dist/engine/cli-commands.js +249 -41
- package/dist/engine/cli-common.js +15 -24
- package/dist/engine/cli.core.js +37 -11
- package/dist/engine/cli.js +2 -2
- package/dist/engine/company-knowledge-cli.js +2 -2
- package/dist/engine/core/adoption.js +408 -0
- package/dist/engine/core/framework-probes.js +7 -7
- package/dist/engine/core/fs-glob.js +2 -2
- package/dist/engine/core/gate-result.js +17 -0
- package/dist/engine/core/migrator.js +9 -17
- package/dist/engine/core/pattern-matcher.js +23 -5
- package/dist/engine/core/persona.js +421 -0
- package/dist/engine/core/promotion-verdict.js +261 -0
- package/dist/engine/core/reporter.js +1 -9
- package/dist/engine/core/skill-examples.js +292 -0
- package/dist/engine/core/skill-surfaces.js +307 -0
- package/dist/engine/core/static-linter.js +310 -38
- package/dist/engine/core/ticket-updater.js +1 -7
- package/dist/engine/core/vacuity-scanner.js +556 -0
- package/dist/engine/core/workflow-discovery.js +2 -8
- package/dist/engine/core/workspace-detect.js +7 -6
- package/dist/engine/data/personas/registry.json +36 -0
- package/dist/engine/guardian/adjudication.js +5 -5
- package/dist/engine/guardian/analysis-emit.js +13 -27
- package/dist/engine/guardian/cli.js +30 -43
- package/dist/engine/guardian/coverage.js +1 -1
- package/dist/engine/guardian/diff-coverage/heuristic-tier.js +1 -1
- package/dist/engine/guardian/diff-coverage/orchestrator.js +2 -2
- package/dist/engine/guardian/pr-check.js +5 -15
- package/dist/engine/guardian/pr-comment.js +4 -3
- package/dist/engine/history/cli.js +210 -6
- package/dist/engine/history/ndjson-store.js +9 -5
- package/dist/engine/history/record.js +34 -5
- package/dist/engine/history/run-recorder.js +165 -0
- package/dist/engine/history/schema.js +25 -7
- package/dist/engine/history/store.js +9 -0
- package/dist/engine/mcp-server.js +35 -13
- package/dist/engine/skills-cli.js +133 -11
- package/dist/engine/util/ensure-ascii.js +37 -0
- package/dist/engine/workflow-cli.js +6 -6
- package/dist/gate-result.d.ts +11 -0
- package/dist/gate-result.js +18 -0
- package/dist/uninstall.js +12 -5
- package/package.json +1 -1
|
@@ -0,0 +1,421 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Personas as a first-class engine concept (issue #462).
|
|
3
|
+
*
|
|
4
|
+
* Audience adaptation used to be hand-rolled per skill: each one restated some
|
|
5
|
+
* variant of "if tester, use simpler words" in its own prose, with its own
|
|
6
|
+
* vocabulary and its own inference rules. Nothing shared a definition, so tone
|
|
7
|
+
* drifted between skills and an overlay had nothing to override. The drift was
|
|
8
|
+
* measurable — `canary-edge-case-discovery` documented
|
|
9
|
+
* `--level sdet|junior|manual` while {@link
|
|
10
|
+
* import('./environment-detect.js').detectUserLevel} returned
|
|
11
|
+
* `sdet|manual|unknown`, and no code read either.
|
|
12
|
+
*
|
|
13
|
+
* This module replaces that with one definition skills **consult**:
|
|
14
|
+
*
|
|
15
|
+
* - a **registry** (`ts/src/data/personas/registry.json`) naming each
|
|
16
|
+
* audience, how much explanation it wants, which output formats suit it,
|
|
17
|
+
* and whether choices should be annotated with their reasoning;
|
|
18
|
+
* - a pure **resolver** that picks one from an explicit choice, a detected
|
|
19
|
+
* signal, or a fallback, and reports **which** and **why**;
|
|
20
|
+
* - **overlay extension** through the same `precedence` arbitration the
|
|
21
|
+
* overlay contract already defines for skill-name collisions (#333).
|
|
22
|
+
*
|
|
23
|
+
* Three deliberate boundaries, each of which was a decision rather than an
|
|
24
|
+
* omission:
|
|
25
|
+
*
|
|
26
|
+
* **Voice is not a persona field.** `voice/discovery.md` already resolves a
|
|
27
|
+
* named voice profile from its own project config, scoped by its own globs.
|
|
28
|
+
* Audience depth and character voice are orthogonal axes — a senior SDET may
|
|
29
|
+
* want a voiced-but-terse report while a manual tester wants the same voice
|
|
30
|
+
* with more explanation — so collapsing them into one field would make half
|
|
31
|
+
* the combinations inexpressible.
|
|
32
|
+
*
|
|
33
|
+
* **The fallback is explanatory, not degraded.** Most users will never
|
|
34
|
+
* configure this, so the no-signal answer has to be a good answer. Erring
|
|
35
|
+
* explanatory is the repo owner's stated default on #341 and #342: over-
|
|
36
|
+
* explaining is a mild annoyance, under-explaining silently fails a manual
|
|
37
|
+
* tester. It is *not* the maximum-explanation persona either, which would
|
|
38
|
+
* assert an audience nobody claimed.
|
|
39
|
+
*
|
|
40
|
+
* **Nothing here reads the environment.** Like `environment-detect.ts`, every
|
|
41
|
+
* input arrives as an argument, so the resolver is deterministic and the
|
|
42
|
+
* decision of where a signal comes from stays with the caller.
|
|
43
|
+
*/
|
|
44
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
45
|
+
import { dirname, join, resolve } from 'node:path';
|
|
46
|
+
import { fileURLToPath } from 'node:url';
|
|
47
|
+
import { uncertainDetectionMessage } from './detection.js';
|
|
48
|
+
import { listOverlays, registryPrecedence, resolveOverlay, } from './overlays.js';
|
|
49
|
+
/** How much explanation a persona wants around the same finding. */
|
|
50
|
+
const DEPTHS = ['terse', 'brief', 'guided'];
|
|
51
|
+
/**
|
|
52
|
+
* Confidence floor applied when a registry declares none.
|
|
53
|
+
*
|
|
54
|
+
* Note what the detector's confidence actually is: `|sdet - manual| / total`,
|
|
55
|
+
* a *margin* between two tallies, not a probability. One open `.ts` file with
|
|
56
|
+
* no opposing signal scores 1.0. So this floor screens out genuine ties and
|
|
57
|
+
* near-ties and nothing else; {@link DEFAULT_MIN_DETECTION_SIGNALS} is the
|
|
58
|
+
* screen that does the real discriminating.
|
|
59
|
+
*/
|
|
60
|
+
const DEFAULT_MIN_DETECTION_CONFIDENCE = 0.5;
|
|
61
|
+
/**
|
|
62
|
+
* Independent signals required before a detected level is trusted.
|
|
63
|
+
*
|
|
64
|
+
* A confidence floor cannot carry this decision on its own: because confidence
|
|
65
|
+
* is a margin, one unopposed observation arrives at a perfect 1.0, and a floor
|
|
66
|
+
* that screens ties but not single observations is confident-and-wrong — the
|
|
67
|
+
* failure mode this repo keeps rooting out. Two signals of genuinely different
|
|
68
|
+
* kinds is the screen that discriminates.
|
|
69
|
+
*
|
|
70
|
+
* "Independent" is per {@link independentSignalKinds}: distinct *kinds* of
|
|
71
|
+
* evidence, not a raw count. Ten open TypeScript files are one observation
|
|
72
|
+
* restated ten times.
|
|
73
|
+
*/
|
|
74
|
+
const DEFAULT_MIN_DETECTION_SIGNALS = 2;
|
|
75
|
+
// ---------------------------------------------------------------------------
|
|
76
|
+
// Parsing
|
|
77
|
+
// ---------------------------------------------------------------------------
|
|
78
|
+
function isRecord(v) {
|
|
79
|
+
return typeof v === 'object' && v !== null && !Array.isArray(v);
|
|
80
|
+
}
|
|
81
|
+
function isDepth(v) {
|
|
82
|
+
return DEPTHS.includes(v);
|
|
83
|
+
}
|
|
84
|
+
function stringArray(v) {
|
|
85
|
+
if (!Array.isArray(v))
|
|
86
|
+
return null;
|
|
87
|
+
return v.every((x) => typeof x === 'string') ? [...v] : null;
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Validate one raw persona entry, or return null.
|
|
91
|
+
*
|
|
92
|
+
* Dropping a malformed entry rather than throwing is deliberate for the
|
|
93
|
+
* *overlay* path: one bad downstream file must not take the engine's own
|
|
94
|
+
* personas down with it. The count is observable through {@link personaIds},
|
|
95
|
+
* so a dropped entry shows up as a missing id rather than as silence.
|
|
96
|
+
*/
|
|
97
|
+
function parsePersona(raw) {
|
|
98
|
+
if (!isRecord(raw))
|
|
99
|
+
return null;
|
|
100
|
+
const { id, label, audience, depth, reasoning } = raw;
|
|
101
|
+
const formats = stringArray(raw['formats']);
|
|
102
|
+
if (typeof id !== 'string' || id.trim() === '')
|
|
103
|
+
return null;
|
|
104
|
+
if (typeof label !== 'string' || typeof audience !== 'string')
|
|
105
|
+
return null;
|
|
106
|
+
if (!isDepth(depth) || formats === null)
|
|
107
|
+
return null;
|
|
108
|
+
if (typeof reasoning !== 'boolean')
|
|
109
|
+
return null;
|
|
110
|
+
return { id: id.trim(), label, audience, depth, formats, reasoning };
|
|
111
|
+
}
|
|
112
|
+
function parsePersonas(raw) {
|
|
113
|
+
if (!Array.isArray(raw))
|
|
114
|
+
return [];
|
|
115
|
+
const out = [];
|
|
116
|
+
for (const entry of raw) {
|
|
117
|
+
const parsed = parsePersona(entry);
|
|
118
|
+
if (parsed)
|
|
119
|
+
out.push(parsed);
|
|
120
|
+
}
|
|
121
|
+
return out;
|
|
122
|
+
}
|
|
123
|
+
function parseDetectionMap(raw) {
|
|
124
|
+
if (!isRecord(raw))
|
|
125
|
+
return {};
|
|
126
|
+
const out = {};
|
|
127
|
+
for (const [level, target] of Object.entries(raw)) {
|
|
128
|
+
if (typeof target === 'string')
|
|
129
|
+
out[level] = target;
|
|
130
|
+
}
|
|
131
|
+
return out;
|
|
132
|
+
}
|
|
133
|
+
/** Default registry path: `<module dir>/../data/personas/registry.json`. */
|
|
134
|
+
function defaultPersonaRegistryPath() {
|
|
135
|
+
const here = dirname(fileURLToPath(import.meta.url));
|
|
136
|
+
// here = <core> -> sibling data/ dir (src/data under vitest, dist/data once
|
|
137
|
+
// built and copied by ts/scripts/copy-data.mjs).
|
|
138
|
+
return resolve(here, '..', 'data', 'personas', 'registry.json');
|
|
139
|
+
}
|
|
140
|
+
/**
|
|
141
|
+
* Read and validate a persona registry.
|
|
142
|
+
*
|
|
143
|
+
* Throws — rather than degrading to an empty registry — because the engine's
|
|
144
|
+
* own registry going missing is a build/packaging fault, and a silently empty
|
|
145
|
+
* one would make every resolution return the same fallback while reporting
|
|
146
|
+
* nothing. Overlay registries take the forgiving path instead; see
|
|
147
|
+
* {@link readOverlayPersonaLayers}.
|
|
148
|
+
*/
|
|
149
|
+
function loadPersonaRegistry(registryPath = defaultPersonaRegistryPath()) {
|
|
150
|
+
let text;
|
|
151
|
+
try {
|
|
152
|
+
text = readFileSync(registryPath, 'utf-8');
|
|
153
|
+
}
|
|
154
|
+
catch (err) {
|
|
155
|
+
// `String(err)` rather than `err.message`: both throwers here raise real
|
|
156
|
+
// Errors, so an `instanceof` guard would only add an unreachable branch.
|
|
157
|
+
throw new Error(`could not read persona registry ${registryPath}: ${String(err)}`);
|
|
158
|
+
}
|
|
159
|
+
let raw;
|
|
160
|
+
try {
|
|
161
|
+
raw = JSON.parse(text);
|
|
162
|
+
}
|
|
163
|
+
catch (err) {
|
|
164
|
+
throw new Error(`persona registry ${registryPath} is not JSON: ${String(err)}`);
|
|
165
|
+
}
|
|
166
|
+
if (!isRecord(raw)) {
|
|
167
|
+
throw new Error(`persona registry ${registryPath} must be a JSON object, not an array` +
|
|
168
|
+
' or scalar');
|
|
169
|
+
}
|
|
170
|
+
const version = typeof raw['version'] === 'number' ? raw['version'] : 1;
|
|
171
|
+
const fallback = typeof raw['fallback'] === 'string' ? raw['fallback'] : '';
|
|
172
|
+
const floor = raw['minDetectionConfidence'];
|
|
173
|
+
const signalFloor = raw['minDetectionSignals'];
|
|
174
|
+
return {
|
|
175
|
+
version,
|
|
176
|
+
fallback,
|
|
177
|
+
minDetectionConfidence: typeof floor === 'number' ? floor : DEFAULT_MIN_DETECTION_CONFIDENCE,
|
|
178
|
+
minDetectionSignals: typeof signalFloor === 'number'
|
|
179
|
+
? signalFloor
|
|
180
|
+
: DEFAULT_MIN_DETECTION_SIGNALS,
|
|
181
|
+
detectionMap: parseDetectionMap(raw['detectionMap']),
|
|
182
|
+
personas: parsePersonas(raw['personas']),
|
|
183
|
+
};
|
|
184
|
+
}
|
|
185
|
+
// ---------------------------------------------------------------------------
|
|
186
|
+
// Lookup
|
|
187
|
+
// ---------------------------------------------------------------------------
|
|
188
|
+
/**
|
|
189
|
+
* The distinct *kinds* of evidence in a detector signal list.
|
|
190
|
+
*
|
|
191
|
+
* `detectUserLevel` emits signals shaped `"<kind>: <detail>"` — for example
|
|
192
|
+
* `"code/test file open: a.ts"`, `"project manifest present: package.json"` —
|
|
193
|
+
* plus one kindless string (`"cwd path suggests manual testing"`), which is
|
|
194
|
+
* treated as its own kind. Grouping on the part before the first `": "` is what
|
|
195
|
+
* makes ten open TypeScript files count once: they are one observation restated,
|
|
196
|
+
* and a raw length check would have read them as ten independent facts.
|
|
197
|
+
*
|
|
198
|
+
* Splitting on `": "` rather than `":"` keeps a Windows path (`C:\src\a.ts`)
|
|
199
|
+
* from being mistaken for a separator.
|
|
200
|
+
*/
|
|
201
|
+
function independentSignalKinds(signals) {
|
|
202
|
+
const kinds = new Set();
|
|
203
|
+
for (const signal of signals) {
|
|
204
|
+
const at = signal.indexOf(': ');
|
|
205
|
+
kinds.add(at === -1 ? signal : signal.slice(0, at));
|
|
206
|
+
}
|
|
207
|
+
return [...kinds];
|
|
208
|
+
}
|
|
209
|
+
/** Ids in registry order — the discoverable persona vocabulary. */
|
|
210
|
+
function personaIds(registry) {
|
|
211
|
+
return registry.personas.map((p) => p.id);
|
|
212
|
+
}
|
|
213
|
+
/** The persona with this id (case-insensitive), or null. */
|
|
214
|
+
function findPersona(registry, id) {
|
|
215
|
+
if (typeof id !== 'string')
|
|
216
|
+
return null;
|
|
217
|
+
const want = id.trim().toLowerCase();
|
|
218
|
+
if (want === '')
|
|
219
|
+
return null;
|
|
220
|
+
return registry.personas.find((p) => p.id.toLowerCase() === want) ?? null;
|
|
221
|
+
}
|
|
222
|
+
// ---------------------------------------------------------------------------
|
|
223
|
+
// Resolution
|
|
224
|
+
// ---------------------------------------------------------------------------
|
|
225
|
+
function fallbackPersona(registry) {
|
|
226
|
+
const named = findPersona(registry, registry.fallback);
|
|
227
|
+
if (named)
|
|
228
|
+
return named;
|
|
229
|
+
const first = registry.personas[0];
|
|
230
|
+
if (first)
|
|
231
|
+
return first;
|
|
232
|
+
// Never invent one. An empty registry is the denominator-zero case: the
|
|
233
|
+
// caller asked for an audience and there is no vocabulary to answer from.
|
|
234
|
+
throw new Error('persona registry defines no personas, so no persona can be resolved');
|
|
235
|
+
}
|
|
236
|
+
/**
|
|
237
|
+
* Choose a persona and say where the choice came from.
|
|
238
|
+
*
|
|
239
|
+
* Precedence, highest first: an explicit id, then a detected level whose
|
|
240
|
+
* confidence clears the registry floor, then the fallback. An explicit id that
|
|
241
|
+
* names nothing does **not** suppress the rest of the cascade — a typo is a
|
|
242
|
+
* mistake, not an instruction to ignore the signal — but it is always named in
|
|
243
|
+
* the reason, so the mistake surfaces instead of being swallowed.
|
|
244
|
+
*/
|
|
245
|
+
export function resolvePersona(input = {}) {
|
|
246
|
+
// `exactOptionalPropertyTypes` forbids passing an explicit `undefined`, so
|
|
247
|
+
// omit the key rather than forwarding it.
|
|
248
|
+
const registry = input.registry ??
|
|
249
|
+
effectivePersonaRegistry(input.home === undefined ? {} : { home: input.home });
|
|
250
|
+
const signals = [...(input.detected?.signals ?? [])];
|
|
251
|
+
const notes = [];
|
|
252
|
+
const explicit = input.explicit;
|
|
253
|
+
if (typeof explicit === 'string' && explicit.trim() !== '') {
|
|
254
|
+
const chosen = findPersona(registry, explicit);
|
|
255
|
+
if (chosen) {
|
|
256
|
+
return {
|
|
257
|
+
persona: chosen,
|
|
258
|
+
source: 'explicit',
|
|
259
|
+
reason: `explicit persona '${chosen.id}'`,
|
|
260
|
+
signals,
|
|
261
|
+
};
|
|
262
|
+
}
|
|
263
|
+
notes.push(uncertainDetectionMessage('persona', {
|
|
264
|
+
reason: `'${explicit.trim()}' is not a known persona`,
|
|
265
|
+
candidates: personaIds(registry),
|
|
266
|
+
}));
|
|
267
|
+
}
|
|
268
|
+
const detected = input.detected;
|
|
269
|
+
if (detected) {
|
|
270
|
+
const floor = registry.minDetectionConfidence;
|
|
271
|
+
const signalFloor = registry.minDetectionSignals;
|
|
272
|
+
const kinds = independentSignalKinds(signals);
|
|
273
|
+
const target = registry.detectionMap[detected.level];
|
|
274
|
+
const mapped = findPersona(registry, target);
|
|
275
|
+
if (!mapped) {
|
|
276
|
+
// Reported ahead of the signal count on purpose: an unmapped level is a
|
|
277
|
+
// vocabulary problem, and "open more files" would be the wrong next step.
|
|
278
|
+
notes.push(`detected user level '${detected.level}' maps to no persona, so the` +
|
|
279
|
+
' fallback applies');
|
|
280
|
+
}
|
|
281
|
+
else if (kinds.length < signalFloor) {
|
|
282
|
+
// Named as a count rather than as "no signal": a reader has to be able to
|
|
283
|
+
// tell "I looked and found too little" from "I found nothing", because
|
|
284
|
+
// only one of those is fixed by giving the detector more to look at.
|
|
285
|
+
notes.push(`detected '${detected.level}' from ${kinds.length} independent ` +
|
|
286
|
+
`signal(s) (${signalFloor} required), so the fallback applies`);
|
|
287
|
+
}
|
|
288
|
+
else if (detected.confidence < floor) {
|
|
289
|
+
notes.push(`detected '${detected.level}' at confidence ` +
|
|
290
|
+
`${detected.confidence} is below the ${floor} floor, so the` +
|
|
291
|
+
' fallback applies');
|
|
292
|
+
}
|
|
293
|
+
else {
|
|
294
|
+
notes.push(`detected user level '${detected.level}' at confidence ` +
|
|
295
|
+
`${detected.confidence}`);
|
|
296
|
+
return {
|
|
297
|
+
persona: mapped,
|
|
298
|
+
source: 'detected',
|
|
299
|
+
reason: notes.join('; '),
|
|
300
|
+
signals,
|
|
301
|
+
};
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
const persona = fallbackPersona(registry);
|
|
305
|
+
notes.push(`fell back to '${persona.id}'`);
|
|
306
|
+
return { persona, source: 'fallback', reason: notes.join('; '), signals };
|
|
307
|
+
}
|
|
308
|
+
/**
|
|
309
|
+
* A JSON-serialisable view for embedding in an MCP response.
|
|
310
|
+
*
|
|
311
|
+
* Flat rather than nested, and it carries `source`, `reason`, and `signals`
|
|
312
|
+
* beside the definition: a consumer that adapts its output owes the user an
|
|
313
|
+
* answer to "why did you decide I was that?".
|
|
314
|
+
*/
|
|
315
|
+
export function personaToDict(resolved) {
|
|
316
|
+
const { persona, source, reason, signals } = resolved;
|
|
317
|
+
return {
|
|
318
|
+
id: persona.id,
|
|
319
|
+
label: persona.label,
|
|
320
|
+
audience: persona.audience,
|
|
321
|
+
depth: persona.depth,
|
|
322
|
+
formats: [...persona.formats],
|
|
323
|
+
reasoning: persona.reasoning,
|
|
324
|
+
source,
|
|
325
|
+
reason,
|
|
326
|
+
signals: [...signals],
|
|
327
|
+
};
|
|
328
|
+
}
|
|
329
|
+
// ---------------------------------------------------------------------------
|
|
330
|
+
// Overlay extension (#333 precedence)
|
|
331
|
+
// ---------------------------------------------------------------------------
|
|
332
|
+
/**
|
|
333
|
+
* Read each installed overlay's `.canary/personas.json`.
|
|
334
|
+
*
|
|
335
|
+
* Forgiving by design: a missing file means the overlay simply has no opinion,
|
|
336
|
+
* and an unreadable or malformed one is skipped rather than fatal, because a
|
|
337
|
+
* downstream overlay must not be able to break the engine's own vocabulary.
|
|
338
|
+
*
|
|
339
|
+
* The clone path comes from `resolveOverlay` rather than being rebuilt here, so
|
|
340
|
+
* `~/.canary/overlays/<name>` stays defined in exactly one place.
|
|
341
|
+
*/
|
|
342
|
+
function readOverlayPersonaLayers(home) {
|
|
343
|
+
const precedence = registryPrecedence(home);
|
|
344
|
+
const layers = [];
|
|
345
|
+
for (const overlay of listOverlays(home)) {
|
|
346
|
+
const path = join(resolveOverlay(overlay, home), '.canary', 'personas.json');
|
|
347
|
+
if (!existsSync(path))
|
|
348
|
+
continue;
|
|
349
|
+
let raw;
|
|
350
|
+
try {
|
|
351
|
+
raw = JSON.parse(readFileSync(path, 'utf-8'));
|
|
352
|
+
}
|
|
353
|
+
catch {
|
|
354
|
+
continue;
|
|
355
|
+
}
|
|
356
|
+
if (!isRecord(raw))
|
|
357
|
+
continue;
|
|
358
|
+
const fallback = raw['fallback'];
|
|
359
|
+
layers.push({
|
|
360
|
+
overlay,
|
|
361
|
+
precedence: precedence[overlay] ?? 0,
|
|
362
|
+
personas: parsePersonas(raw['personas']),
|
|
363
|
+
fallback: typeof fallback === 'string' ? fallback : null,
|
|
364
|
+
});
|
|
365
|
+
}
|
|
366
|
+
return layers;
|
|
367
|
+
}
|
|
368
|
+
/**
|
|
369
|
+
* Fold overlay layers onto the base registry.
|
|
370
|
+
*
|
|
371
|
+
* Layers are applied in ascending `(precedence, overlay-name)` order and the
|
|
372
|
+
* last writer wins — byte-for-byte the rule `skill-registry.ts` uses for
|
|
373
|
+
* skill-name collisions, so a downstream operator who has already reasoned
|
|
374
|
+
* about which overlay wins a skill does not have to learn a second model for
|
|
375
|
+
* personas. An id already in the base is replaced **in place**, so extending
|
|
376
|
+
* the vocabulary never reorders it.
|
|
377
|
+
*/
|
|
378
|
+
function mergePersonaRegistries(base, layers) {
|
|
379
|
+
const ordered = [...layers].sort((a, b) => a.precedence - b.precedence || a.overlay.localeCompare(b.overlay));
|
|
380
|
+
const personas = [...base.personas];
|
|
381
|
+
let fallback = base.fallback;
|
|
382
|
+
for (const layer of ordered) {
|
|
383
|
+
for (const persona of layer.personas) {
|
|
384
|
+
const at = personas.findIndex((p) => p.id.toLowerCase() === persona.id.toLowerCase());
|
|
385
|
+
if (at >= 0)
|
|
386
|
+
personas[at] = persona;
|
|
387
|
+
else
|
|
388
|
+
personas.push(persona);
|
|
389
|
+
}
|
|
390
|
+
if (layer.fallback !== null)
|
|
391
|
+
fallback = layer.fallback;
|
|
392
|
+
}
|
|
393
|
+
const merged = { ...base, personas, fallback };
|
|
394
|
+
// A fallback naming nothing is worse than no opinion: it would silently
|
|
395
|
+
// demote resolution to "first persona in the list".
|
|
396
|
+
if (!findPersona(merged, merged.fallback))
|
|
397
|
+
merged.fallback = base.fallback;
|
|
398
|
+
return merged;
|
|
399
|
+
}
|
|
400
|
+
/**
|
|
401
|
+
* The registry a resolution actually sees: shipped, plus every overlay.
|
|
402
|
+
*
|
|
403
|
+
* This is the function that makes overlay extension real rather than
|
|
404
|
+
* decorative. {@link resolvePersona} calls it whenever the caller does not
|
|
405
|
+
* hand over a registry outright, so a downstream `.canary/personas.json` reaches
|
|
406
|
+
* every consumer without each one remembering to merge — which is the mistake
|
|
407
|
+
* #341 is a record of: a signal that was produced correctly and read by nobody.
|
|
408
|
+
*
|
|
409
|
+
* Re-read per call rather than cached. `analyze_file` already scans the project
|
|
410
|
+
* tree, so one small JSON read plus a directory listing is not the cost worth
|
|
411
|
+
* trading a stale-config bug for.
|
|
412
|
+
*
|
|
413
|
+
* `registryPath` points the base registry somewhere other than the shipped
|
|
414
|
+
* file. It is the whole public seam onto the loader: the parsing and merging
|
|
415
|
+
* helpers below are intentionally module-private, so this is how both a caller
|
|
416
|
+
* with its own registry and a test exercising a malformed one get in.
|
|
417
|
+
*/
|
|
418
|
+
export function effectivePersonaRegistry(options = {}) {
|
|
419
|
+
return mergePersonaRegistries(loadPersonaRegistry(options.registryPath), readOverlayPersonaLayers(options.home));
|
|
420
|
+
}
|
|
421
|
+
//# sourceMappingURL=persona.js.map
|
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The structured verdict `canary-promote-test` gates on (#477).
|
|
3
|
+
*
|
|
4
|
+
* ## Why this exists now, when the issue says it is blocked
|
|
5
|
+
*
|
|
6
|
+
* #477 was parked on the emit side: `harness:test-craft` runs an 8-axis per-test
|
|
7
|
+
* LLM critique with no machine-readable output, so there was nothing to consume
|
|
8
|
+
* and building a consumer against an unspecified shape meant building it twice.
|
|
9
|
+
* That is still true of the LLM critique. It is no longer true of the gate:
|
|
10
|
+
* #605 (soundness) and #612 (vacuity) emit structured, per-test, DETERMINISTIC
|
|
11
|
+
* verdicts, and those are the ones that were ever going to be allowed to block.
|
|
12
|
+
*
|
|
13
|
+
* ## The three decisions #477 asked for
|
|
14
|
+
*
|
|
15
|
+
* **Which axes gate.** Deterministic defects gate; style reports.
|
|
16
|
+
*
|
|
17
|
+
* | Axis | Rules | Gates | Why |
|
|
18
|
+
* | ----------------- | ---------------------------- | ----- | ---------------------------------------------------------------- |
|
|
19
|
+
* | `soundness` | `SOUND-001/002/003` | yes | Pins a value no correct implementation must produce |
|
|
20
|
+
* | `assertions` | `LINT-006` | yes | A test that asserts nothing always passes |
|
|
21
|
+
* | `flakiness` | `FLAKE-001/002` | yes | The issue's own named blocker; both are `critical` |
|
|
22
|
+
* | `vacuity` | `VAC-001/003`, annotated 002 | yes | Deterministic, or the author declared the target themselves |
|
|
23
|
+
* | `selectors` | `LINT-001/002/003` | no | Brittle, not wrong; a reviewer's call |
|
|
24
|
+
* | `maintainability` | `LINT-005`, `FLAKE-003/004` | no | Style and softer signals |
|
|
25
|
+
*
|
|
26
|
+
* Gating on all of them would block nearly every promotion, which is exactly the
|
|
27
|
+
* outcome #477 predicted for a naive 8-axis gate.
|
|
28
|
+
*
|
|
29
|
+
* **What happens with no verdict.** `abstain`, exit 3, and say so. Promotion
|
|
30
|
+
* falls back to today's manual review. It must never become silently stricter
|
|
31
|
+
* (a `block` nobody can act on) or silently looser (a `promote` over a file the
|
|
32
|
+
* scanner could not read) -- and two distinct zeros are guarded: a file no
|
|
33
|
+
* ruleset parses, and a parseable file holding no tests.
|
|
34
|
+
*
|
|
35
|
+
* **Whether an LLM judgement may block.** No. Everything that gates in this repo
|
|
36
|
+
* is deterministic, and this change does not spend that. The decision is
|
|
37
|
+
* structural rather than documentary: {@link VerdictSource} admits one value, so
|
|
38
|
+
* there is no field an LLM verdict can arrive in and quietly acquire authority.
|
|
39
|
+
* `harness:test-craft` stays what `canary-promote-test` already calls it -- an
|
|
40
|
+
* optional deeper audit for a human.
|
|
41
|
+
*
|
|
42
|
+
* ## The fidelity ladder
|
|
43
|
+
*
|
|
44
|
+
* `VAC-002` is inference, and #477's real anxiety was making a heuristic
|
|
45
|
+
* load-bearing on a promotion gate. So the rung decides the authority, mirroring
|
|
46
|
+
* the guardian's `coverage-verified > graph-verified > heuristic`:
|
|
47
|
+
* `annotated` (the author named the target) blocks; `import-inferred` reports.
|
|
48
|
+
*/
|
|
49
|
+
import { EXIT_ABSTAINED, errnoCode, gateOutcome, } from './gate-result.js';
|
|
50
|
+
import { StaticLinter, UnsupportedTestFileError, frameworkForPath, } from './static-linter.js';
|
|
51
|
+
import { scanVacuity, } from './vacuity-scanner.js';
|
|
52
|
+
/**
|
|
53
|
+
* Which rules land on which axis, and whether that axis gates.
|
|
54
|
+
*
|
|
55
|
+
* A rule matching NO row here is invisible to the verdict: not gating, not
|
|
56
|
+
* advisory, not printed. `LINT-004` -- an unawaited Playwright action, the
|
|
57
|
+
* linter's other `critical` and the canonical false-green defect -- was omitted
|
|
58
|
+
* from the first cut and walked straight through the gate built to stop it.
|
|
59
|
+
* `test-signal-review-findings.test.ts` now asserts that every rule a real lint
|
|
60
|
+
* produces lands somewhere, because the omission is otherwise silent.
|
|
61
|
+
*/
|
|
62
|
+
const AXES = [
|
|
63
|
+
{ axis: 'soundness', gating: true, rules: /^SOUND-/ },
|
|
64
|
+
{ axis: 'assertions', gating: true, rules: /^LINT-006$/ },
|
|
65
|
+
// LINT-004 sits here rather than in its own axis because an unawaited action
|
|
66
|
+
// IS the classic race: the assertion runs before the action lands.
|
|
67
|
+
{ axis: 'flakiness', gating: true, rules: /^FLAKE-00[12]$|^LINT-004$/ },
|
|
68
|
+
{ axis: 'vacuity', gating: true, rules: /^VAC-/ },
|
|
69
|
+
{ axis: 'selectors', gating: false, rules: /^LINT-00[123]$/ },
|
|
70
|
+
{
|
|
71
|
+
axis: 'maintainability',
|
|
72
|
+
gating: false,
|
|
73
|
+
rules: /^LINT-005$|^FLAKE-00[34]$/,
|
|
74
|
+
},
|
|
75
|
+
];
|
|
76
|
+
/**
|
|
77
|
+
* Whether a finding on a gating axis may actually block.
|
|
78
|
+
*
|
|
79
|
+
* The one place the fidelity ladder is spent. `VAC-002` at `import-inferred` is
|
|
80
|
+
* an inference about which symbol a test was meant to exercise; blocking a
|
|
81
|
+
* promotion on that would make a heuristic load-bearing, which is the specific
|
|
82
|
+
* thing #477 asked not to do by accident. At `annotated` fidelity the author
|
|
83
|
+
* wrote the target down, so the finding is a contradiction of a stated contract
|
|
84
|
+
* and blocks.
|
|
85
|
+
*/
|
|
86
|
+
function mayBlock(f) {
|
|
87
|
+
if (f.rule !== 'VAC-002')
|
|
88
|
+
return true;
|
|
89
|
+
return f.fidelity === 'annotated';
|
|
90
|
+
}
|
|
91
|
+
function normalizeLint(f) {
|
|
92
|
+
return {
|
|
93
|
+
rule: f.rule,
|
|
94
|
+
line: f.line,
|
|
95
|
+
severity: f.severity,
|
|
96
|
+
message: f.message,
|
|
97
|
+
suggestion: f.suggestion,
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
function normalizeVacuity(f) {
|
|
101
|
+
const out = {
|
|
102
|
+
rule: f.rule,
|
|
103
|
+
line: f.line,
|
|
104
|
+
severity: f.severity,
|
|
105
|
+
message: `${f.test}: ${f.message}`,
|
|
106
|
+
suggestion: f.suggestion,
|
|
107
|
+
};
|
|
108
|
+
if (f.fidelity)
|
|
109
|
+
out.fidelity = f.fidelity;
|
|
110
|
+
return out;
|
|
111
|
+
}
|
|
112
|
+
const ABSTAIN_REMEDY = 'No verdict could be produced, so promotion falls back to manual review ' +
|
|
113
|
+
'(canary-promote-test Phase 1) -- it has NOT been approved. ' +
|
|
114
|
+
'Point at a single generated test file that contains at least one test.';
|
|
115
|
+
function abstained(file, skipped, axes) {
|
|
116
|
+
const outcome = gateOutcome({ checked: 0, findings: [], skipped }, 'gate');
|
|
117
|
+
return {
|
|
118
|
+
file,
|
|
119
|
+
decision: 'abstain',
|
|
120
|
+
source: 'deterministic',
|
|
121
|
+
checked: 0,
|
|
122
|
+
axes,
|
|
123
|
+
blocked: [],
|
|
124
|
+
skipped,
|
|
125
|
+
summaryLine: outcome.summaryLine,
|
|
126
|
+
remedy: ABSTAIN_REMEDY,
|
|
127
|
+
exitCode: EXIT_ABSTAINED,
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
function emptyAxes() {
|
|
131
|
+
return AXES.map((a) => ({ axis: a.axis, gating: a.gating, findings: [] }));
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* Produce the promotion verdict for one generated test file.
|
|
135
|
+
*
|
|
136
|
+
* Deliberately single-file: promotion is a per-file decision, and a directory
|
|
137
|
+
* roll-up would let one clean file's verdict read as cover for a sibling's.
|
|
138
|
+
*/
|
|
139
|
+
export function promotionVerdict(path) {
|
|
140
|
+
if (frameworkForPath(path) === null) {
|
|
141
|
+
return abstained(path, [
|
|
142
|
+
{
|
|
143
|
+
name: path,
|
|
144
|
+
reason: 'no ruleset parses this extension, so a clean result would be meaningless',
|
|
145
|
+
},
|
|
146
|
+
], emptyAxes());
|
|
147
|
+
}
|
|
148
|
+
const lint = lintOrAbstain(path);
|
|
149
|
+
if (!Array.isArray(lint))
|
|
150
|
+
return lint;
|
|
151
|
+
const vacuity = scanVacuity(path);
|
|
152
|
+
const axes = groupIntoAxes([
|
|
153
|
+
...lint.map(normalizeLint),
|
|
154
|
+
...vacuity.findings.map(normalizeVacuity),
|
|
155
|
+
]);
|
|
156
|
+
const skipped = vacuity.skipped ?? [];
|
|
157
|
+
// A parseable file with no tests is another zero, and it must not read as a
|
|
158
|
+
// pass: promotion would let an empty file into the committed suite. Note that
|
|
159
|
+
// `lint` can still be non-empty here (a stray `Date.now()` outside any test),
|
|
160
|
+
// so the check is on the TEST count, not on the finding count.
|
|
161
|
+
if (vacuity.checked === 0) {
|
|
162
|
+
return abstained(path, [
|
|
163
|
+
...skipped,
|
|
164
|
+
{
|
|
165
|
+
name: path,
|
|
166
|
+
reason: 'file holds no test declarations, so there is nothing to promote',
|
|
167
|
+
},
|
|
168
|
+
], axes);
|
|
169
|
+
}
|
|
170
|
+
return decide(path, axes, skipped, vacuity.checked);
|
|
171
|
+
}
|
|
172
|
+
/**
|
|
173
|
+
* The lint findings, or a ready-made abstention when the linter refused.
|
|
174
|
+
*
|
|
175
|
+
* Two distinct refusals, and BOTH have to become an abstention rather than an
|
|
176
|
+
* exception. Letting a read error propagate meant the CLI printed a raw ENOENT
|
|
177
|
+
* stack and exited 0 -- a promotion gate that could not open the draft,
|
|
178
|
+
* reporting success. Anything without an `errno`-style `code` still throws,
|
|
179
|
+
* because swallowing an unknown fault is how a scanner learns to go quiet.
|
|
180
|
+
*/
|
|
181
|
+
function lintOrAbstain(path) {
|
|
182
|
+
try {
|
|
183
|
+
return new StaticLinter().lint(path);
|
|
184
|
+
}
|
|
185
|
+
catch (e) {
|
|
186
|
+
if (e instanceof UnsupportedTestFileError) {
|
|
187
|
+
return abstained(path, [{ name: path, reason: e.message }], emptyAxes());
|
|
188
|
+
}
|
|
189
|
+
// ERRNO-shaped only. Keying off "has a string `code`" swept in every Node
|
|
190
|
+
// PROGRAMMER error too -- `ERR_INVALID_ARG_TYPE`, `ERR_STRING_TOO_LONG` --
|
|
191
|
+
// so a genuine defect inside the linter was reported as a clean ABSTAIN with
|
|
192
|
+
// a misleading reason instead of surfacing. An unknown fault must still
|
|
193
|
+
// throw; that is the difference between degrading honestly and going quiet.
|
|
194
|
+
const code = errnoCode(e);
|
|
195
|
+
if (code === null)
|
|
196
|
+
throw e;
|
|
197
|
+
return abstained(path, [{ name: path, reason: `could not be read (${code})` }], emptyAxes());
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
/** Every axis, in registry order, each carrying its findings sorted by line. */
|
|
201
|
+
function groupIntoAxes(all) {
|
|
202
|
+
return AXES.map((spec) => ({
|
|
203
|
+
axis: spec.axis,
|
|
204
|
+
gating: spec.gating,
|
|
205
|
+
findings: all
|
|
206
|
+
.filter((f) => spec.rules.test(f.rule))
|
|
207
|
+
.sort((a, b) => a.line - b.line),
|
|
208
|
+
}));
|
|
209
|
+
}
|
|
210
|
+
/** Turn evaluated axes into the promote/block decision and its copy. */
|
|
211
|
+
function decide(path, axes, skipped, checked) {
|
|
212
|
+
const blocked = [
|
|
213
|
+
...new Set(axes
|
|
214
|
+
.filter((a) => a.gating)
|
|
215
|
+
.flatMap((a) => a.findings)
|
|
216
|
+
.filter(mayBlock)
|
|
217
|
+
.map((f) => f.rule)),
|
|
218
|
+
];
|
|
219
|
+
// `gateOutcome` owns the summary line so promotion reports its denominator in
|
|
220
|
+
// the same shape as every other gate, rather than inventing a private format.
|
|
221
|
+
const outcome = gateOutcome({ checked, findings: blocked, skipped }, 'gate', {
|
|
222
|
+
noun: 'test(s)',
|
|
223
|
+
});
|
|
224
|
+
const decision = blocked.length > 0 ? 'block' : 'promote';
|
|
225
|
+
const advisoryCount = axes
|
|
226
|
+
.filter((a) => !a.gating)
|
|
227
|
+
.reduce((n, a) => n + a.findings.length, 0);
|
|
228
|
+
return {
|
|
229
|
+
file: path,
|
|
230
|
+
decision,
|
|
231
|
+
source: 'deterministic',
|
|
232
|
+
checked,
|
|
233
|
+
axes,
|
|
234
|
+
blocked,
|
|
235
|
+
skipped,
|
|
236
|
+
summaryLine: outcome.summaryLine,
|
|
237
|
+
remedy: remedyFor(decision, blocked, advisoryCount, skipped.length),
|
|
238
|
+
exitCode: decision === 'block' ? 1 : 0,
|
|
239
|
+
};
|
|
240
|
+
}
|
|
241
|
+
function remedyFor(decision, blocked, advisoryCount, skipCount) {
|
|
242
|
+
if (decision === 'block') {
|
|
243
|
+
return (`Blocked on ${blocked.join(', ')}. Fix the test or regenerate it ` +
|
|
244
|
+
'with a sharper prompt -- do not hand-patch a generated draft ' +
|
|
245
|
+
'(canary-promote-test Phase 1).');
|
|
246
|
+
}
|
|
247
|
+
const parts = ['Promotable.'];
|
|
248
|
+
if (advisoryCount > 0) {
|
|
249
|
+
parts.push(`${advisoryCount} advisory finding(s) are a reviewer's call, not a blocker.`);
|
|
250
|
+
}
|
|
251
|
+
// A `promote` whose rules went dark must not read as an unqualified pass.
|
|
252
|
+
// `checked` legitimately counts the tests VAC-001 did run on, so the
|
|
253
|
+
// denominator is right -- but a reader seeing only "promotable" would never
|
|
254
|
+
// learn that two of the three vacuity rules could not be evaluated at all.
|
|
255
|
+
if (skipCount > 0) {
|
|
256
|
+
parts.push(`${skipCount} check(s) could not run on this file -- see the skip list; ` +
|
|
257
|
+
'those rules did NOT pass, they abstained.');
|
|
258
|
+
}
|
|
259
|
+
return advisoryCount > 0 || skipCount > 0 ? parts.join(' ') : '';
|
|
260
|
+
}
|
|
261
|
+
//# sourceMappingURL=promotion-verdict.js.map
|