canary-test-cli 6.3.0 → 6.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/canary-mcp.js +52 -0
- package/dist/doctor.d.ts +51 -3
- package/dist/doctor.js +76 -9
- package/dist/engine/analysis/cli.js +69 -6
- package/dist/engine/cli-commands.js +34 -1
- package/dist/engine/core/feedback.js +32 -18
- package/dist/engine/core/gate-result.js +80 -0
- package/dist/engine/core/migrator.js +83 -10
- package/dist/engine/core/skill-registry.js +95 -30
- package/dist/engine/guardian/adjudication.js +364 -0
- package/dist/engine/guardian/analysis-emit.js +2 -0
- package/dist/engine/guardian/cli.js +282 -15
- package/dist/engine/guardian/hard-gate.js +15 -2
- package/dist/engine/guardian/pr-check.js +5 -12
- package/dist/engine/history/cli.js +67 -0
- package/dist/engine/history/ndjson-store.js +4 -0
- package/dist/engine/history/store.js +3 -0
- package/dist/gate-result.d.ts +67 -0
- package/dist/gate-result.js +73 -0
- package/dist/overlay-commands.js +17 -1
- package/dist/overlay-lint.d.ts +6 -1
- package/dist/overlay-lint.js +53 -68
- package/dist/skill-frontmatter.d.ts +24 -0
- package/dist/skill-frontmatter.js +89 -0
- package/package.json +5 -2
|
@@ -43,6 +43,7 @@ import { basename, isAbsolute, join, relative, resolve, sep } from 'node:path';
|
|
|
43
43
|
import { readJsonWithWarning } from './config-validation.js';
|
|
44
44
|
import { uncertainDetectionMessage } from './detection.js';
|
|
45
45
|
import { FrameworkRegistry } from './framework-registry.js';
|
|
46
|
+
import { EXIT_ABSTAINED, gateOutcome } from './gate-result.js';
|
|
46
47
|
import { Scaffolder, scaffoldableFrameworks, TEMPLATES } from './scaffolder.js';
|
|
47
48
|
import { SkillRegistry } from './skill-registry.js';
|
|
48
49
|
// ---------------------------------------------------------------------------
|
|
@@ -594,8 +595,34 @@ export class FreshnessReport {
|
|
|
594
595
|
get in_sync() {
|
|
595
596
|
return !this.has_drift && !this.has_local_edits;
|
|
596
597
|
}
|
|
597
|
-
/**
|
|
598
|
+
/**
|
|
599
|
+
* The freshness gate as a {@link GateResult}: denominator = skills
|
|
600
|
+
* verified, findings = drift + local edits. Feeds the shared abstention
|
|
601
|
+
* helper (#508) so "verified zero skills" can never render as a pass.
|
|
602
|
+
*/
|
|
603
|
+
gateResult() {
|
|
604
|
+
return {
|
|
605
|
+
checked: this.results.length,
|
|
606
|
+
findings: [...this.stale, ...this.local_edits],
|
|
607
|
+
};
|
|
608
|
+
}
|
|
609
|
+
/**
|
|
610
|
+
* A gate that verified zero skills has abstained, not passed (#503): the
|
|
611
|
+
* shape matched nothing, so nothing was checked and "in sync" would be a
|
|
612
|
+
* silent false pass -- the #456 class. Reported as its own exit code and
|
|
613
|
+
* flagged in every output surface. Delegates to the shared helper (#508).
|
|
614
|
+
*/
|
|
615
|
+
get abstained() {
|
|
616
|
+
return gateOutcome(this.gateResult(), 'gate').abstained;
|
|
617
|
+
}
|
|
618
|
+
/**
|
|
619
|
+
* 0 in sync, 1 drift, 2 local edits (safety refusal wins), 3 abstained.
|
|
620
|
+
* The abstention path comes from the shared helper; the 1/2 mapping is
|
|
621
|
+
* this surface's own contract (local edits outrank drift).
|
|
622
|
+
*/
|
|
598
623
|
exit_code() {
|
|
624
|
+
if (this.abstained)
|
|
625
|
+
return EXIT_ABSTAINED;
|
|
599
626
|
if (this.has_local_edits)
|
|
600
627
|
return 2;
|
|
601
628
|
if (this.has_drift)
|
|
@@ -609,6 +636,8 @@ export class FreshnessReport {
|
|
|
609
636
|
in_sync: this.in_sync,
|
|
610
637
|
has_drift: this.has_drift,
|
|
611
638
|
has_local_edits: this.has_local_edits,
|
|
639
|
+
checked: this.results.length,
|
|
640
|
+
abstained: this.abstained,
|
|
612
641
|
exit_code: this.exit_code(),
|
|
613
642
|
skills: this.results.map((r) => ({
|
|
614
643
|
skill_name: r.skill_name,
|
|
@@ -628,6 +657,13 @@ export class FreshnessReport {
|
|
|
628
657
|
];
|
|
629
658
|
if (this.results.length === 0) {
|
|
630
659
|
lines.push("_No overlay skills match this project's shape._", '');
|
|
660
|
+
lines.push(`${WARN} **Abstained** ${EMDASH} the gate verified zero skills, so this is not a pass.`, '');
|
|
661
|
+
if (this.shape === 'unknown') {
|
|
662
|
+
lines.push('The shape could not be detected. Set `canary_shape` in', '`.canary/company.json` or pass `--framework <name>`.', '');
|
|
663
|
+
}
|
|
664
|
+
else {
|
|
665
|
+
lines.push(`The overlay ships no skills with \`deploy_to\` covering \`${this.shape}\`.`, "Check the overlay's `deploy_to` lists or the resolved `canary_shape`.", '');
|
|
666
|
+
}
|
|
631
667
|
lines.push(...workflowMarkdown(this.workflows, false));
|
|
632
668
|
return lines.join('\n');
|
|
633
669
|
}
|
|
@@ -719,6 +755,17 @@ export class MigrationReport {
|
|
|
719
755
|
this.installed_workflows = init.installed_workflows ?? [];
|
|
720
756
|
this.config_warnings = init.config_warnings ?? [];
|
|
721
757
|
}
|
|
758
|
+
/**
|
|
759
|
+
* The dry run's denominator (#504): config files that would be created,
|
|
760
|
+
* skills that would deploy, workflows that would install. Zero means the
|
|
761
|
+
* dry run has nothing to apply -- an advisory abstention, not a
|
|
762
|
+
* completed migration.
|
|
763
|
+
*/
|
|
764
|
+
get would_migrate_count() {
|
|
765
|
+
return (this.would_create.length +
|
|
766
|
+
this.deployed_skills.filter((r) => r.status === 'dry_run').length +
|
|
767
|
+
this.installed_workflows.filter((r) => r.status === 'dry_run').length);
|
|
768
|
+
}
|
|
722
769
|
to_markdown() {
|
|
723
770
|
const lines = ['# Canary Migration Report', ''];
|
|
724
771
|
if (this.dry_run) {
|
|
@@ -822,6 +869,28 @@ export class MigrationReport {
|
|
|
822
869
|
lines.push(`- ${item}`);
|
|
823
870
|
lines.push('');
|
|
824
871
|
}
|
|
872
|
+
else if (this.dry_run) {
|
|
873
|
+
// #504 abstention half: a dry run never completed anything. Zero
|
|
874
|
+
// pending work is an advisory abstention (D3) -- gateOutcome is the
|
|
875
|
+
// only summary-line path AND the only decision point (no local
|
|
876
|
+
// n === 0 arithmetic), so the refusal is structural.
|
|
877
|
+
const n = this.would_migrate_count;
|
|
878
|
+
const outcome = gateOutcome({ checked: n, findings: [] }, 'advisory', {
|
|
879
|
+
noun: 'item(s)',
|
|
880
|
+
});
|
|
881
|
+
lines.push('## Status', '');
|
|
882
|
+
if (outcome.abstained) {
|
|
883
|
+
lines.push(outcome.summaryLine, '', 'This dry run would migrate zero item(s) ' +
|
|
884
|
+
EMDASH +
|
|
885
|
+
' the project already carries everything this migration would ' +
|
|
886
|
+
'produce. If you expected changes, check `--from <overlay>` and ' +
|
|
887
|
+
'the detected framework/shape above.', '');
|
|
888
|
+
}
|
|
889
|
+
else {
|
|
890
|
+
lines.push(`Dry run ${EMDASH} would migrate ${n} item(s). ` +
|
|
891
|
+
'Re-run with `--apply` to write them.', '');
|
|
892
|
+
}
|
|
893
|
+
}
|
|
825
894
|
else {
|
|
826
895
|
lines.push('## Status', '', 'Migration complete. Run `canary recommend "<test description>"` to verify framework detection.', '');
|
|
827
896
|
}
|
|
@@ -1260,19 +1329,23 @@ export class HarnessMigrator {
|
|
|
1260
1329
|
this.installWorkflows(shape, overlayPath, projectRoot, true, false));
|
|
1261
1330
|
}
|
|
1262
1331
|
detectFramework(root, config) {
|
|
1263
|
-
//
|
|
1332
|
+
// Explicit override in .canary/company.json ("canary_shape" field) is
|
|
1333
|
+
// user intent: it wins over every probe tier's shape, including a total
|
|
1334
|
+
// probe miss (#502 — monorepos often have no root framework config).
|
|
1335
|
+
// Framework detection still runs so framework-dependent behavior keeps
|
|
1336
|
+
// working when a probe does match.
|
|
1264
1337
|
const rawShape = config['canary_shape'];
|
|
1265
1338
|
const explicitShape = (rawShape == null ? '' : String(rawShape))
|
|
1266
1339
|
.trim()
|
|
1267
1340
|
.toLowerCase();
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1341
|
+
const [framework, shape, source, confidence] = this.probeFramework(root, config);
|
|
1342
|
+
if (!explicitShape)
|
|
1343
|
+
return [framework, shape, source, confidence];
|
|
1344
|
+
return framework === null
|
|
1345
|
+
? [null, explicitShape, 'canary_shape (.canary/company.json)', 'explicit']
|
|
1346
|
+
: [framework, explicitShape, source, confidence];
|
|
1347
|
+
}
|
|
1348
|
+
probeFramework(root, config) {
|
|
1276
1349
|
// 1. Dedicated config file (highest confidence).
|
|
1277
1350
|
for (const [filename, framework, shape, confidence] of _CONFIG_PROBES) {
|
|
1278
1351
|
if (existsSync(join(root, filename))) {
|
|
@@ -339,7 +339,7 @@ export class SkillRegistry {
|
|
|
339
339
|
catch {
|
|
340
340
|
return null;
|
|
341
341
|
}
|
|
342
|
-
const fm = SkillRegistry.
|
|
342
|
+
const { frontmatter: fm, errors } = SkillRegistry.parseFrontmatterWithDiagnostics(text);
|
|
343
343
|
const stem = basename(path, extname(path));
|
|
344
344
|
const name = pyTruthy(fm['name']) ? fm['name'] : stem;
|
|
345
345
|
return new SkillInfo({
|
|
@@ -351,7 +351,7 @@ export class SkillRegistry {
|
|
|
351
351
|
entry: SkillRegistry.scalar(fm['entry']),
|
|
352
352
|
deploy_to: SkillRegistry.parseDeployTo(fm),
|
|
353
353
|
requires: SkillRegistry.parseStrList(fm, 'requires'),
|
|
354
|
-
error: SkillRegistry.
|
|
354
|
+
error: SkillRegistry.discoveryError(fm, errors),
|
|
355
355
|
});
|
|
356
356
|
}
|
|
357
357
|
// Public (Python `_parse_nested` is underscore-private but used cross-module):
|
|
@@ -365,7 +365,7 @@ export class SkillRegistry {
|
|
|
365
365
|
catch {
|
|
366
366
|
return null;
|
|
367
367
|
}
|
|
368
|
-
const fm = SkillRegistry.
|
|
368
|
+
const { frontmatter: fm, errors } = SkillRegistry.parseFrontmatterWithDiagnostics(text);
|
|
369
369
|
const name = pyTruthy(fm['name']) ? fm['name'] : dirName;
|
|
370
370
|
// Python: `fm.get("description") or self._blockquote_tagline(text)`.
|
|
371
371
|
const description = pyTruthy(fm['description'])
|
|
@@ -380,7 +380,7 @@ export class SkillRegistry {
|
|
|
380
380
|
entry: SkillRegistry.scalar(fm['entry']),
|
|
381
381
|
deploy_to: SkillRegistry.parseDeployTo(fm),
|
|
382
382
|
requires: SkillRegistry.parseStrList(fm, 'requires'),
|
|
383
|
-
error: SkillRegistry.
|
|
383
|
+
error: SkillRegistry.discoveryError(fm, errors),
|
|
384
384
|
});
|
|
385
385
|
}
|
|
386
386
|
/** Python `dict.get(key, default)`: default only on a missing key. */
|
|
@@ -408,39 +408,104 @@ export class SkillRegistry {
|
|
|
408
408
|
return [];
|
|
409
409
|
}
|
|
410
410
|
/**
|
|
411
|
-
* Tiny YAML-subset parser: top-level scalar and
|
|
412
|
-
* delimiters.
|
|
413
|
-
*
|
|
411
|
+
* Tiny YAML-subset parser: top-level scalar and list fields between `---`
|
|
412
|
+
* delimiters. Convenience wrapper over
|
|
413
|
+
* {@link parseFrontmatterWithDiagnostics} that drops the diagnostics.
|
|
414
414
|
*/
|
|
415
415
|
static parseFrontmatter(text) {
|
|
416
|
-
|
|
416
|
+
return SkillRegistry.parseFrontmatterWithDiagnostics(text).frontmatter;
|
|
417
|
+
}
|
|
418
|
+
/**
|
|
419
|
+
* YAML-subset parser with parse diagnostics (#501). The historical
|
|
420
|
+
* one-line-per-key subset read formatter-emitted YAML — wrapped flow lists,
|
|
421
|
+
* block sequences, indented scalar continuations — as silently EMPTY, so
|
|
422
|
+
* `migrate` skipped declared `deploy_to`/`install_workflows` entries while
|
|
423
|
+
* everything stayed green. Those shapes now parse, and a list-shaped value
|
|
424
|
+
* that still cannot be read (an unterminated `[`) is a recorded error,
|
|
425
|
+
* never a silent empty list. Still a deliberate subset: no nested mappings,
|
|
426
|
+
* no quoting, and top-level lines without a colon are skipped (pinned).
|
|
427
|
+
* Mirrored by npm/src/skill-frontmatter.ts for `overlay lint` — keep in sync.
|
|
428
|
+
*/
|
|
429
|
+
static parseFrontmatterWithDiagnostics(text) {
|
|
430
|
+
const frontmatter = {};
|
|
431
|
+
const errors = [];
|
|
417
432
|
if (!text.startsWith('---'))
|
|
418
|
-
return
|
|
419
|
-
const
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
continue;
|
|
426
|
-
if (!line.includes(':'))
|
|
427
|
-
continue;
|
|
433
|
+
return { frontmatter, errors };
|
|
434
|
+
const rest = text.split('\n').slice(1);
|
|
435
|
+
const end = rest.findIndex((l) => l.trim() === '---');
|
|
436
|
+
const body = (end === -1 ? rest : rest.slice(0, end)).filter((l) => !l.trim().startsWith('#'));
|
|
437
|
+
let i = 0;
|
|
438
|
+
while (i < body.length) {
|
|
439
|
+
const line = body[i];
|
|
428
440
|
const idx = line.indexOf(':'); // Python str.partition -> first colon.
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
.filter((item) => item);
|
|
441
|
+
i++;
|
|
442
|
+
// A line is a key only when top-level, non-blank, and colon-bearing.
|
|
443
|
+
if (!line.trim() || /^\s/.test(line) || idx === -1)
|
|
444
|
+
continue;
|
|
445
|
+
const cont = []; // indented continuation lines for this key
|
|
446
|
+
while (i < body.length && /^\s+\S/.test(body[i])) {
|
|
447
|
+
cont.push(body[i].trim());
|
|
448
|
+
i++;
|
|
438
449
|
}
|
|
439
|
-
|
|
440
|
-
|
|
450
|
+
SkillRegistry.assignFrontmatterValue(frontmatter, errors, line.slice(0, idx).trim(), line.slice(idx + 1).trim(), cont);
|
|
451
|
+
}
|
|
452
|
+
return { frontmatter, errors };
|
|
453
|
+
}
|
|
454
|
+
/**
|
|
455
|
+
* Assign one entry from its inline value plus indented continuations: flow
|
|
456
|
+
* lists (inline, wrapped mid-list, or entirely on a continuation line —
|
|
457
|
+
* prettier's rewrite), block sequences, and folded plain scalars.
|
|
458
|
+
*/
|
|
459
|
+
static assignFrontmatterValue(fm, errors, key, inline, cont) {
|
|
460
|
+
const flow = inline.startsWith('[')
|
|
461
|
+
? [inline, ...cont]
|
|
462
|
+
: inline === '' && cont[0]?.startsWith('[')
|
|
463
|
+
? cont
|
|
464
|
+
: null;
|
|
465
|
+
if (flow !== null) {
|
|
466
|
+
const joined = flow.join(' ').trim();
|
|
467
|
+
if (!joined.endsWith(']')) {
|
|
468
|
+
errors.push(`\`${key}\`: unterminated flow list (no closing \`]\`): ${joined}`);
|
|
469
|
+
fm[key] = [];
|
|
470
|
+
return;
|
|
441
471
|
}
|
|
472
|
+
fm[key] = joined
|
|
473
|
+
.slice(1, -1)
|
|
474
|
+
.split(',')
|
|
475
|
+
.map((s) => s.trim())
|
|
476
|
+
.filter(Boolean);
|
|
477
|
+
}
|
|
478
|
+
else if (inline === '' && /^-( |$)/.test(cont[0] ?? '')) {
|
|
479
|
+
const items = SkillRegistry.blockListItems(cont);
|
|
480
|
+
if (items.length === 0)
|
|
481
|
+
errors.push(`\`${key}\`: block list has no parseable items`);
|
|
482
|
+
fm[key] = items;
|
|
483
|
+
}
|
|
484
|
+
else {
|
|
485
|
+
// Scalar; indented continuation lines fold in (plain multiline YAML).
|
|
486
|
+
fm[key] = [inline, ...cont].join(' ').trim();
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
/** Block-sequence items; a dash-less line folds into the item above it. */
|
|
490
|
+
static blockListItems(cont) {
|
|
491
|
+
const items = [];
|
|
492
|
+
for (const c of cont) {
|
|
493
|
+
if (c.startsWith('- '))
|
|
494
|
+
items.push(c.slice(2).trim());
|
|
495
|
+
else if (c !== '-' && items.length > 0)
|
|
496
|
+
items[items.length - 1] = `${items[items.length - 1]} ${c}`.trim();
|
|
497
|
+
}
|
|
498
|
+
return items.filter(Boolean);
|
|
499
|
+
}
|
|
500
|
+
/**
|
|
501
|
+
* Frontmatter parse diagnostics + executable-field validation combined into
|
|
502
|
+
* the `SkillInfo.error` channel; parse errors win (#501: loud, never silent).
|
|
503
|
+
*/
|
|
504
|
+
static discoveryError(fm, parseErrors) {
|
|
505
|
+
if (parseErrors.length > 0) {
|
|
506
|
+
return `frontmatter parse error: ${parseErrors.join('; ')}`;
|
|
442
507
|
}
|
|
443
|
-
return
|
|
508
|
+
return SkillRegistry.validateExecutableFields(fm);
|
|
444
509
|
}
|
|
445
510
|
/** Return an error string if the cli/entry combination is invalid. */
|
|
446
511
|
static validateExecutableFields(fm) {
|
|
@@ -0,0 +1,364 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Finding adjudication collection — the precision the hard gate depends on
|
|
3
|
+
* (#490).
|
|
4
|
+
*
|
|
5
|
+
* `pr-check.ts` documents the soft→hard promotion contract as
|
|
6
|
+
* `precision = TP / (TP + FP)` fed by reviewer adjudication — but until this
|
|
7
|
+
* module nothing collected adjudications, so no repo could ever earn the hard
|
|
8
|
+
* gate. Reviewers already give the lowest-friction feedback available: a 👍
|
|
9
|
+
* (true positive) or 👎 (false positive) reaction on the guardian's sticky
|
|
10
|
+
* comment. This module reads those reactions back off the comment the guardian
|
|
11
|
+
* already upserts by marker, and persists a per-PR adjudication record to the
|
|
12
|
+
* existing `.harness/analyses/` channel (no new store — see
|
|
13
|
+
* {@link module:./analysis-emit}).
|
|
14
|
+
*
|
|
15
|
+
* Granularity (per the #490 design sketch): **whole-comment first**. One sticky
|
|
16
|
+
* comment carries N findings, so a reaction adjudicates the *run*, not one
|
|
17
|
+
* finding — except when the comment shows exactly one active finding, in which
|
|
18
|
+
* case the reaction is attributable to that finding's path. Per-finding
|
|
19
|
+
* comments were rejected as a worse artifact (N comments per PR).
|
|
20
|
+
*
|
|
21
|
+
* Zero-denominator discipline: a precision computed over 0 adjudicated
|
|
22
|
+
* findings is **unknown**, never 100%. {@link summarizePrecision} returns
|
|
23
|
+
* `precision: null` and {@link renderPrecision} says so in words. Most
|
|
24
|
+
* reviewers react to neither — the sample is small and self-selected, and every
|
|
25
|
+
* rendered surface states the sample size rather than presenting the number as
|
|
26
|
+
* ground truth.
|
|
27
|
+
*
|
|
28
|
+
* SC-11 boundary: deterministic HTTP/filesystem behind seams — no agent/LLM
|
|
29
|
+
* import. Network lives ONLY in {@link RestReactionsClient}; every unit test
|
|
30
|
+
* uses {@link FakeReactionsClient}.
|
|
31
|
+
*/
|
|
32
|
+
import { mkdirSync, readFileSync, readdirSync, renameSync, statSync, unlinkSync, writeFileSync, } from 'node:fs';
|
|
33
|
+
import { randomBytes } from 'node:crypto';
|
|
34
|
+
import { dirname, join } from 'node:path';
|
|
35
|
+
import { STICKY_MARKER, findSticky } from './pr-comment.js';
|
|
36
|
+
/** Schema tag for adjudication records (independent of the findings schema). */
|
|
37
|
+
export const ADJUDICATION_SCHEMA_VERSION = '1.0';
|
|
38
|
+
/**
|
|
39
|
+
* Record `source` + filename prefix. Deliberately namespaced UNDER the
|
|
40
|
+
* `canary-pr-guardian-` prefix (harness's `AnalysisArchive` reads every
|
|
41
|
+
* `*.json` in `.harness/analyses/`) while never colliding with a pr-check
|
|
42
|
+
* findings record: those are `canary-pr-guardian-<sanitized-ref>.json` and a
|
|
43
|
+
* ref is sanitized from a git ref / `pr-<n>`, never `adjudication-pr-<n>`.
|
|
44
|
+
*/
|
|
45
|
+
export const ADJUDICATION_SOURCE = 'canary-pr-guardian-adjudication';
|
|
46
|
+
/** GitHub reaction contents that carry an adjudication verdict. */
|
|
47
|
+
const THUMBS_UP = '+1';
|
|
48
|
+
const THUMBS_DOWN = '-1';
|
|
49
|
+
// Loud notices carry an em-dash as output data; escaped per the ASCII-source rule.
|
|
50
|
+
const EM_DASH = '\u{2014}';
|
|
51
|
+
/** In-memory {@link ReactionsClient} for unit tests — no network. */
|
|
52
|
+
export class FakeReactionsClient {
|
|
53
|
+
comments;
|
|
54
|
+
reactionsByComment;
|
|
55
|
+
constructor(init = {}) {
|
|
56
|
+
this.comments = init.comments ?? [];
|
|
57
|
+
this.reactionsByComment = new Map(Object.entries(init.reactions ?? {}).map(([id, rows]) => [
|
|
58
|
+
Number(id),
|
|
59
|
+
rows,
|
|
60
|
+
]));
|
|
61
|
+
}
|
|
62
|
+
async listComments() {
|
|
63
|
+
return this.comments;
|
|
64
|
+
}
|
|
65
|
+
async listReactions(commentId) {
|
|
66
|
+
return this.reactionsByComment.get(commentId) ?? [];
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Thin real {@link ReactionsClient} over the GitHub REST API (`fetch`).
|
|
71
|
+
* Network lives ONLY here; no unit test exercises this class. Both endpoints
|
|
72
|
+
* are reads, so a fork's read-only token is sufficient.
|
|
73
|
+
*/
|
|
74
|
+
export class RestReactionsClient {
|
|
75
|
+
repo;
|
|
76
|
+
prNumber;
|
|
77
|
+
token;
|
|
78
|
+
static API = 'https://api.github.com';
|
|
79
|
+
constructor(repo, prNumber, token) {
|
|
80
|
+
this.repo = repo;
|
|
81
|
+
this.prNumber = prNumber;
|
|
82
|
+
this.token = token;
|
|
83
|
+
}
|
|
84
|
+
async get(url) {
|
|
85
|
+
const resp = await fetch(url, {
|
|
86
|
+
method: 'GET',
|
|
87
|
+
headers: {
|
|
88
|
+
Authorization: `Bearer ${this.token}`,
|
|
89
|
+
Accept: 'application/vnd.github+json',
|
|
90
|
+
'X-GitHub-Api-Version': '2022-11-28',
|
|
91
|
+
'User-Agent': 'canary-pr-guardian',
|
|
92
|
+
},
|
|
93
|
+
});
|
|
94
|
+
if (!resp.ok) {
|
|
95
|
+
throw new Error(`GitHub API ${resp.status}: ${url}`);
|
|
96
|
+
}
|
|
97
|
+
return resp.json();
|
|
98
|
+
}
|
|
99
|
+
async listComments() {
|
|
100
|
+
const url = `${RestReactionsClient.API}/repos/${this.repo}/issues/${this.prNumber}/comments`;
|
|
101
|
+
const result = await this.get(url);
|
|
102
|
+
return Array.isArray(result) ? result : [];
|
|
103
|
+
}
|
|
104
|
+
async listReactions(commentId) {
|
|
105
|
+
const url = `${RestReactionsClient.API}/repos/${this.repo}/issues/comments/${commentId}/reactions`;
|
|
106
|
+
const result = await this.get(url);
|
|
107
|
+
if (!Array.isArray(result))
|
|
108
|
+
return [];
|
|
109
|
+
const rows = [];
|
|
110
|
+
for (const raw of result) {
|
|
111
|
+
if (typeof raw !== 'object' || raw === null)
|
|
112
|
+
continue;
|
|
113
|
+
const rec = raw;
|
|
114
|
+
const content = typeof rec.content === 'string' ? rec.content : '';
|
|
115
|
+
const user = typeof rec.user?.login === 'string' ? rec.user.login : 'unknown';
|
|
116
|
+
if (content)
|
|
117
|
+
rows.push({ user, content });
|
|
118
|
+
}
|
|
119
|
+
return rows;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* Tally verdict reactions: one vote per user, bots excluded (PURE).
|
|
124
|
+
*
|
|
125
|
+
* - Only `+1`/`-1` carry a verdict; every other content is ignored.
|
|
126
|
+
* - Logins ending in `[bot]` are excluded so the guardian's own automation (or
|
|
127
|
+
* any other bot) can never inflate its own precision.
|
|
128
|
+
* - A user who reacted both 👍 and 👎 is contradictory: counted as `ambiguous`
|
|
129
|
+
* and excluded from both TP and FP rather than guessed at.
|
|
130
|
+
*/
|
|
131
|
+
export function tallyAdjudications(reactions) {
|
|
132
|
+
const up = new Set();
|
|
133
|
+
const down = new Set();
|
|
134
|
+
for (const reaction of reactions) {
|
|
135
|
+
if (reaction.user.endsWith('[bot]'))
|
|
136
|
+
continue;
|
|
137
|
+
if (reaction.content === THUMBS_UP)
|
|
138
|
+
up.add(reaction.user);
|
|
139
|
+
else if (reaction.content === THUMBS_DOWN)
|
|
140
|
+
down.add(reaction.user);
|
|
141
|
+
}
|
|
142
|
+
let ambiguous = 0;
|
|
143
|
+
for (const user of up) {
|
|
144
|
+
if (down.has(user))
|
|
145
|
+
ambiguous += 1;
|
|
146
|
+
}
|
|
147
|
+
return {
|
|
148
|
+
tp: up.size - ambiguous,
|
|
149
|
+
fp: down.size - ambiguous,
|
|
150
|
+
ambiguous,
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
// A findings-table row in the sticky comment: `| <icon> <sev> | `path`... |`.
|
|
154
|
+
// The header row's second cell is ` File ` and the separator's is ` --- `,
|
|
155
|
+
// neither of which starts with a backtick, so anchoring on the second cell's
|
|
156
|
+
// leading backtick selects exactly the finding rows. Paths never contain `|`
|
|
157
|
+
// or backticks (see `fileLabel` in pr-check.ts), so the naive anchor is safe.
|
|
158
|
+
const FINDING_ROW_RE = /^\|[^|]*\|\s*`([^`]+)`/;
|
|
159
|
+
/**
|
|
160
|
+
* Extract the file paths of the ACTIVE findings shown in a sticky-comment body
|
|
161
|
+
* (PURE). Reads the rendered table `render(fmt='comment')` emitted — this is
|
|
162
|
+
* deliberately parsing the exact body reviewers reacted to, not the current
|
|
163
|
+
* finding set, so a reaction is attributed to what the reviewer actually saw.
|
|
164
|
+
* Returns `[]` for a no-gaps body (no table).
|
|
165
|
+
*/
|
|
166
|
+
export function activeFindingPaths(commentBody) {
|
|
167
|
+
const paths = [];
|
|
168
|
+
for (const line of commentBody.split(/\r\n|\r|\n/)) {
|
|
169
|
+
const match = FINDING_ROW_RE.exec(line);
|
|
170
|
+
if (match)
|
|
171
|
+
paths.push(match[1]);
|
|
172
|
+
}
|
|
173
|
+
return paths;
|
|
174
|
+
}
|
|
175
|
+
/** ISO-8601 UTC timestamp with a `+00:00` offset (matches analysis-emit). */
|
|
176
|
+
function isoUtcNow() {
|
|
177
|
+
return new Date().toISOString().replace('Z', '+00:00');
|
|
178
|
+
}
|
|
179
|
+
/** Build the v1.0 adjudication record (PURE given `collectedAt`). */
|
|
180
|
+
export function buildAdjudicationRecord(init) {
|
|
181
|
+
const findingPaths = activeFindingPaths(init.commentBody);
|
|
182
|
+
const single = findingPaths.length === 1;
|
|
183
|
+
return {
|
|
184
|
+
schemaVersion: ADJUDICATION_SCHEMA_VERSION,
|
|
185
|
+
source: ADJUDICATION_SOURCE,
|
|
186
|
+
repo: init.repo,
|
|
187
|
+
prNumber: init.prNumber,
|
|
188
|
+
commentId: init.commentId,
|
|
189
|
+
granularity: single ? 'finding' : 'run',
|
|
190
|
+
attributedPath: single ? findingPaths[0] : null,
|
|
191
|
+
findingPaths,
|
|
192
|
+
tp: init.tally.tp,
|
|
193
|
+
fp: init.tally.fp,
|
|
194
|
+
ambiguous: init.tally.ambiguous,
|
|
195
|
+
collectedAt: init.collectedAt ?? isoUtcNow(),
|
|
196
|
+
};
|
|
197
|
+
}
|
|
198
|
+
/** `canary-pr-guardian-adjudication-pr-<n>.json` under the analyses dir. */
|
|
199
|
+
export function adjudicationFilename(prNumber) {
|
|
200
|
+
return `${ADJUDICATION_SOURCE}-pr-${prNumber}.json`;
|
|
201
|
+
}
|
|
202
|
+
/** True iff the harness home (`dirname(analysesDir)`) exists. */
|
|
203
|
+
function channelAvailable(analysesDir) {
|
|
204
|
+
try {
|
|
205
|
+
return statSync(dirname(analysesDir)).isDirectory();
|
|
206
|
+
}
|
|
207
|
+
catch {
|
|
208
|
+
return false;
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
/**
|
|
212
|
+
* Read the sticky comment's reactions and persist the PR's adjudication record.
|
|
213
|
+
*
|
|
214
|
+
* Idempotent per PR: the record is the LATEST reaction state, overwritten in
|
|
215
|
+
* place on each collection (reactions live on the comment, which the guardian
|
|
216
|
+
* upserts rather than re-creates, so they accumulate monotonically). Records
|
|
217
|
+
* for different PRs never collide — the store is append-only across PRs.
|
|
218
|
+
*
|
|
219
|
+
* Never throws for an expected shape: a missing comment, zero reactions, or an
|
|
220
|
+
* unavailable channel each return a distinct non-`collected` result so the
|
|
221
|
+
* caller can report honestly instead of crashing the gate.
|
|
222
|
+
*/
|
|
223
|
+
export async function collectAdjudications(client, args) {
|
|
224
|
+
const sticky = findSticky(await client.listComments(), args.marker ?? STICKY_MARKER);
|
|
225
|
+
if (sticky === null) {
|
|
226
|
+
return { action: 'no-comment', path: null, record: null, notice: null };
|
|
227
|
+
}
|
|
228
|
+
const tally = tallyAdjudications(await client.listReactions(sticky.id));
|
|
229
|
+
if (tally.tp + tally.fp + tally.ambiguous === 0) {
|
|
230
|
+
return { action: 'no-reactions', path: null, record: null, notice: null };
|
|
231
|
+
}
|
|
232
|
+
const record = buildAdjudicationRecord({
|
|
233
|
+
repo: args.repo,
|
|
234
|
+
prNumber: args.prNumber,
|
|
235
|
+
commentId: sticky.id,
|
|
236
|
+
commentBody: sticky.body,
|
|
237
|
+
tally,
|
|
238
|
+
collectedAt: args.collectedAt,
|
|
239
|
+
});
|
|
240
|
+
if (!channelAvailable(args.analysesDir)) {
|
|
241
|
+
return {
|
|
242
|
+
action: 'unavailable',
|
|
243
|
+
path: null,
|
|
244
|
+
record,
|
|
245
|
+
notice: 'guardian: harness analyses channel unavailable (.harness/ absent) ' +
|
|
246
|
+
`${EM_DASH} adjudication not persisted`,
|
|
247
|
+
};
|
|
248
|
+
}
|
|
249
|
+
const target = join(args.analysesDir, adjudicationFilename(args.prNumber));
|
|
250
|
+
try {
|
|
251
|
+
mkdirSync(args.analysesDir, { recursive: true });
|
|
252
|
+
// Atomic write (same-dir temp + rename), matching analysis-emit: a torn
|
|
253
|
+
// record would poison every later precision summary.
|
|
254
|
+
const tmp = join(args.analysesDir, `.tmp-${randomBytes(8).toString('hex')}.json`);
|
|
255
|
+
writeFileSync(tmp, JSON.stringify(record, null, 2), 'utf-8');
|
|
256
|
+
try {
|
|
257
|
+
renameSync(tmp, target);
|
|
258
|
+
}
|
|
259
|
+
catch (err) {
|
|
260
|
+
try {
|
|
261
|
+
unlinkSync(tmp);
|
|
262
|
+
}
|
|
263
|
+
catch {
|
|
264
|
+
// best-effort cleanup
|
|
265
|
+
}
|
|
266
|
+
throw err;
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
catch (exc) {
|
|
270
|
+
const message = exc instanceof Error ? exc.message : String(exc);
|
|
271
|
+
return {
|
|
272
|
+
action: 'unavailable',
|
|
273
|
+
path: null,
|
|
274
|
+
record,
|
|
275
|
+
notice: `guardian: adjudication write failed (${message}) ${EM_DASH} ` +
|
|
276
|
+
'adjudication not persisted',
|
|
277
|
+
};
|
|
278
|
+
}
|
|
279
|
+
return { action: 'collected', path: target, record, notice: null };
|
|
280
|
+
}
|
|
281
|
+
/**
|
|
282
|
+
* Load every adjudication record under `analysesDir` (best-effort).
|
|
283
|
+
*
|
|
284
|
+
* Reads only `canary-pr-guardian-adjudication-*.json`; pr-check findings
|
|
285
|
+
* records and harness's own records are never touched. A malformed or
|
|
286
|
+
* wrong-`source` file is skipped, never fatal — one corrupt record must not
|
|
287
|
+
* take down the precision report.
|
|
288
|
+
*/
|
|
289
|
+
export function loadAdjudicationRecords(analysesDir) {
|
|
290
|
+
let names;
|
|
291
|
+
try {
|
|
292
|
+
names = readdirSync(analysesDir);
|
|
293
|
+
}
|
|
294
|
+
catch {
|
|
295
|
+
return [];
|
|
296
|
+
}
|
|
297
|
+
const records = [];
|
|
298
|
+
for (const name of names.sort()) {
|
|
299
|
+
if (!name.startsWith(`${ADJUDICATION_SOURCE}-`) || !name.endsWith('.json'))
|
|
300
|
+
continue;
|
|
301
|
+
try {
|
|
302
|
+
const raw = JSON.parse(readFileSync(join(analysesDir, name), 'utf-8'));
|
|
303
|
+
if (raw !== null &&
|
|
304
|
+
typeof raw === 'object' &&
|
|
305
|
+
raw.source === ADJUDICATION_SOURCE &&
|
|
306
|
+
typeof raw.tp === 'number' &&
|
|
307
|
+
typeof raw.fp === 'number') {
|
|
308
|
+
records.push(raw);
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
catch {
|
|
312
|
+
// skip malformed record
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
return records;
|
|
316
|
+
}
|
|
317
|
+
/** Aggregate records into the precision summary (PURE). */
|
|
318
|
+
export function summarizePrecision(records) {
|
|
319
|
+
let tp = 0;
|
|
320
|
+
let fp = 0;
|
|
321
|
+
let ambiguous = 0;
|
|
322
|
+
let prCount = 0;
|
|
323
|
+
for (const record of records) {
|
|
324
|
+
tp += record.tp;
|
|
325
|
+
fp += record.fp;
|
|
326
|
+
ambiguous += record.ambiguous ?? 0;
|
|
327
|
+
if (record.tp + record.fp > 0)
|
|
328
|
+
prCount += 1;
|
|
329
|
+
}
|
|
330
|
+
const adjudicated = tp + fp;
|
|
331
|
+
return {
|
|
332
|
+
adjudicated,
|
|
333
|
+
tp,
|
|
334
|
+
fp,
|
|
335
|
+
ambiguous,
|
|
336
|
+
prCount,
|
|
337
|
+
precision: adjudicated === 0 ? null : tp / adjudicated,
|
|
338
|
+
};
|
|
339
|
+
}
|
|
340
|
+
/**
|
|
341
|
+
* Render the precision summary as human text (PURE).
|
|
342
|
+
*
|
|
343
|
+
* Zero-denominator discipline: with no adjudications the FIRST word after the
|
|
344
|
+
* label is `unknown` — the report never implies 100% (or any number) from an
|
|
345
|
+
* empty sample. With data, the sample size and its self-selected nature ride
|
|
346
|
+
* alongside the number on the same line.
|
|
347
|
+
*/
|
|
348
|
+
export function renderPrecision(summary) {
|
|
349
|
+
if (summary.precision === null) {
|
|
350
|
+
return (`guardian precision: unknown ${EM_DASH} no adjudications yet ` +
|
|
351
|
+
`(0 reviewer verdicts collected). React with a thumbs-up (finding was ` +
|
|
352
|
+
`right) or thumbs-down (false positive) on the guardian's PR comment.`);
|
|
353
|
+
}
|
|
354
|
+
const pct = (summary.precision * 100).toFixed(1).replace(/\.0$/, '');
|
|
355
|
+
const ambiguousNote = summary.ambiguous > 0
|
|
356
|
+
? ` ${summary.ambiguous} contradictory verdict(s) excluded.`
|
|
357
|
+
: '';
|
|
358
|
+
return (`guardian precision: ${pct}% (${summary.tp} true / ${summary.fp} false ` +
|
|
359
|
+
`positive${summary.adjudicated === 1 ? '' : 's'}, n=${summary.adjudicated} ` +
|
|
360
|
+
`across ${summary.prCount} PR(s)).${ambiguousNote} Sample is ` +
|
|
361
|
+
`self-selected (reviewers who chose to react) ${EM_DASH} a signal, not ` +
|
|
362
|
+
`ground truth.`);
|
|
363
|
+
}
|
|
364
|
+
//# sourceMappingURL=adjudication.js.map
|