canary-test-cli 7.2.0 → 8.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agents/skills/README.md +23 -4
- package/agents/skills/claude-code/canary-batwoman/SKILL.md +119 -0
- package/agents/skills/claude-code/canary-cassandra/SKILL.md +23 -16
- package/agents/skills/claude-code/canary-cassandra/scripts/cli.mjs +3 -1
- package/agents/skills/claude-code/canary-ci-ready/SKILL.md +20 -3
- package/agents/skills/claude-code/canary-fleet-health/SKILL.md +1 -0
- package/agents/skills/claude-code/canary-pr-guardian/SKILL.md +15 -0
- package/agents/skills/claude-code/canary-screech/SKILL.md +109 -0
- package/agents/skills/claude-code/canary-screech/scripts/blast.mjs +125 -0
- package/agents/skills/claude-code/canary-screech/scripts/cli.mjs +128 -0
- package/agents/skills/claude-code/canary-screech/scripts/cluster.mjs +97 -0
- package/agents/skills/claude-code/canary-screech/scripts/history.mjs +73 -0
- package/agents/skills/claude-code/canary-screech/scripts/redness.mjs +94 -0
- package/agents/skills/lib/parse-args.mjs +200 -139
- package/dist/engine/analysis/batwoman/audit.js +39 -0
- package/dist/engine/analysis/batwoman/closure.js +159 -0
- package/dist/engine/analysis/batwoman/gh-history.js +119 -0
- package/dist/engine/analysis/batwoman/probes.js +195 -0
- package/dist/engine/analysis/batwoman/registry.js +142 -0
- package/dist/engine/analysis/batwoman/render.js +194 -0
- package/dist/engine/analysis/batwoman/run-window.js +122 -0
- package/dist/engine/analysis/batwoman/text.js +84 -0
- package/dist/engine/analysis/batwoman/triggers.js +122 -0
- package/dist/engine/analysis/batwoman/verdict.js +64 -0
- package/dist/engine/analysis/cli.js +47 -14
- package/dist/engine/analysis/gh-flaky/gh-run-attempts.js +206 -0
- package/dist/engine/batwoman-cli.js +119 -0
- package/dist/engine/ci-ready-cli.js +71 -0
- package/dist/engine/cli-commands.js +46 -7
- package/dist/engine/cli.core.js +16 -0
- package/dist/engine/company-knowledge-cli.js +10 -2
- package/dist/engine/core/ci-ready.js +112 -0
- package/dist/engine/core/company-knowledge.js +8 -0
- package/dist/engine/core/migrator.js +147 -20
- package/dist/engine/core/permission-matrix.js +219 -0
- package/dist/engine/core/quality-scorer.js +13 -18
- package/dist/engine/core/scaling-curve.js +143 -0
- package/dist/engine/core/string-literals.js +3 -1
- package/dist/engine/core/vacuity-scanner.js +151 -6
- package/dist/engine/core/workflow-discovery.js +41 -23
- package/dist/engine/guardian/adjudication-github.js +136 -0
- package/dist/engine/guardian/adjudication.js +119 -340
- package/dist/engine/guardian/cli.js +180 -264
- package/dist/engine/guardian/coverage.js +2 -1
- package/dist/engine/guardian/diff-coverage/coverage-delta.js +162 -0
- package/dist/engine/guardian/diff-coverage/formats/cobertura.js +45 -1
- package/dist/engine/guardian/diff-coverage/orchestrator.js +25 -21
- package/dist/engine/guardian/diff-coverage/paths.js +5 -9
- package/dist/engine/guardian/diff-coverage/report-tier.js +88 -12
- package/dist/engine/guardian/diff-extractor.js +31 -32
- package/dist/engine/guardian/pr-check.js +262 -430
- package/dist/engine/guardian/pr-comment.js +35 -58
- package/dist/engine/guardian/weak-test.js +236 -0
- package/dist/engine/mcp-server.js +67 -4
- package/dist/engine/permission-matrix-cli.js +51 -0
- package/dist/engine/scaling-curve-cli.js +147 -0
- package/dist/engine/skills-cli.js +48 -32
- package/dist/engine/workflow-cli.js +85 -65
- package/package.json +1 -1
|
@@ -37,7 +37,7 @@
|
|
|
37
37
|
* - Python truthiness (`""`/`None`/`{}`/`[]` falsy) via {@link pyTruthy}.
|
|
38
38
|
*/
|
|
39
39
|
import { createHash } from 'node:crypto';
|
|
40
|
-
import { cpSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync, } from 'node:fs';
|
|
40
|
+
import { cpSync, existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, rmSync, writeFileSync, } from 'node:fs';
|
|
41
41
|
import { homedir } from 'node:os';
|
|
42
42
|
import { basename, isAbsolute, join, relative, resolve, sep } from 'node:path';
|
|
43
43
|
import { readJsonWithWarning } from './config-validation.js';
|
|
@@ -236,12 +236,34 @@ function parseWorkflowEntry(entry) {
|
|
|
236
236
|
function resolveTemplatePath(skillDir, rel) {
|
|
237
237
|
if (!rel || isAbsolute(rel) || /^[A-Za-z]:/.test(rel))
|
|
238
238
|
return null;
|
|
239
|
-
const base = resolve(skillDir);
|
|
239
|
+
const base = realPathIfPresent(resolve(skillDir));
|
|
240
240
|
const full = resolve(base, rel);
|
|
241
|
-
|
|
241
|
+
// Containment is judged on the REAL path: `resolve` normalizes `..` but
|
|
242
|
+
// never follows a symlink, so a lexically-contained `templates/x.yml` that
|
|
243
|
+
// is a link to /etc/passwd would otherwise pass and have its target's bytes
|
|
244
|
+
// written into the consumer's `.github/workflows/`. This mirrors
|
|
245
|
+
// `SkillRegistry.resolveCliPath`, which guards the same class the same way.
|
|
246
|
+
const real = realPathIfPresent(full);
|
|
247
|
+
if (real !== base && !real.startsWith(base + sep))
|
|
242
248
|
return null;
|
|
249
|
+
// The declared path (resolved against the real skill dir) is returned rather
|
|
250
|
+
// than its own realpath: an intra-skill symlink is legitimate, and
|
|
251
|
+
// `basename(src)` is what names the installed workflow file.
|
|
243
252
|
return full;
|
|
244
253
|
}
|
|
254
|
+
/**
|
|
255
|
+
* *path* with symlinks resolved, or *path* itself when it does not exist --
|
|
256
|
+
* `realpathSync` throws on a missing path, and a declared-but-unshipped
|
|
257
|
+
* template must still reach the `missing` report rather than `invalid`.
|
|
258
|
+
*/
|
|
259
|
+
function realPathIfPresent(path) {
|
|
260
|
+
try {
|
|
261
|
+
return realpathSync(path);
|
|
262
|
+
}
|
|
263
|
+
catch {
|
|
264
|
+
return path;
|
|
265
|
+
}
|
|
266
|
+
}
|
|
245
267
|
/**
|
|
246
268
|
* Is the deployed copy of a skill the consumer's rather than the overlay's?
|
|
247
269
|
*
|
|
@@ -330,6 +352,77 @@ function findWorkspaceSuites(root, framework) {
|
|
|
330
352
|
}
|
|
331
353
|
return suites;
|
|
332
354
|
}
|
|
355
|
+
/**
|
|
356
|
+
* The distinct `framework/shape` pairs the workspace packages declare, in the
|
|
357
|
+
* order the (path-sorted) findings first mention them.
|
|
358
|
+
*
|
|
359
|
+
* The scalar resolution collapses more than one pair to `unknown` (see
|
|
360
|
+
* `resolveFromWorkspace`). Reporting that bare `unknown` without also
|
|
361
|
+
* reporting *what* was found is the residue of #504 part 1: the walk observed
|
|
362
|
+
* two real suites and the report claimed nothing matched.
|
|
363
|
+
*/
|
|
364
|
+
function workspacePairs(ws) {
|
|
365
|
+
return [...new Set(ws.findings.map((f) => `${f.framework}/${f.shape}`))];
|
|
366
|
+
}
|
|
367
|
+
/** How many distinct package directories carry at least one finding. */
|
|
368
|
+
function packagesWithFindings(ws) {
|
|
369
|
+
return new Set(ws.findings.map((f) => f.dir)).size;
|
|
370
|
+
}
|
|
371
|
+
/** The suites a workspace walk found, rendered as `` `dir/` (framework) ``. */
|
|
372
|
+
function namedWorkspaceSuites(ws) {
|
|
373
|
+
return ws.findings.map((f) => `\`${f.dir}/\` (${f.framework})`).join(', ');
|
|
374
|
+
}
|
|
375
|
+
/**
|
|
376
|
+
* The follow-ups for a run that resolved no framework: the uncertain-detection
|
|
377
|
+
* message, plus -- only when the walk actually found suites -- the per-package
|
|
378
|
+
* route, which needs no override at all (#504 part 1).
|
|
379
|
+
*/
|
|
380
|
+
function unresolvedFrameworkFollowups(ws) {
|
|
381
|
+
const out = [
|
|
382
|
+
uncertainDetectionMessage('test framework', {
|
|
383
|
+
reason: unresolvedFrameworkReason(ws),
|
|
384
|
+
candidates: KNOWN_FRAMEWORKS,
|
|
385
|
+
overrideHint: '`canary migrate --framework <name>`',
|
|
386
|
+
}),
|
|
387
|
+
];
|
|
388
|
+
if (ws !== null && ws.findings.length > 0) {
|
|
389
|
+
out.push(`Or re-run \`canary migrate\` from inside a package ${EMDASH} ` +
|
|
390
|
+
`${namedWorkspaceSuites(ws)}.`);
|
|
391
|
+
}
|
|
392
|
+
return out;
|
|
393
|
+
}
|
|
394
|
+
/**
|
|
395
|
+
* Why no single framework resolved, in the terms of what was actually probed.
|
|
396
|
+
*
|
|
397
|
+
* A workspace whose packages disagree is a different failure from a repo where
|
|
398
|
+
* nothing matched, and telling the user "no config file, dependency, or
|
|
399
|
+
* language marker matched" when two package configs *did* match is a claim the
|
|
400
|
+
* run's own evidence contradicts (#504 part 1).
|
|
401
|
+
*/
|
|
402
|
+
function unresolvedFrameworkReason(ws) {
|
|
403
|
+
const nothingMatched = 'no config file, dependency, or language marker matched a known framework';
|
|
404
|
+
if (ws === null || ws.findings.length === 0)
|
|
405
|
+
return nothingMatched;
|
|
406
|
+
const pairs = workspacePairs(ws);
|
|
407
|
+
const n = packagesWithFindings(ws);
|
|
408
|
+
const combos = pairs.length === 1 ? 'combination' : 'combinations';
|
|
409
|
+
return (`this workspace's packages declare ${pairs.length} different ` +
|
|
410
|
+
`framework/shape ${combos} across ${n} ` +
|
|
411
|
+
`package${n === 1 ? '' : 's'} (${pairs.join(', ')}), ` +
|
|
412
|
+
'so no single framework applies at the root');
|
|
413
|
+
}
|
|
414
|
+
/**
|
|
415
|
+
* The shape the workspace packages already resolved for *framework*, when they
|
|
416
|
+
* agree on one. Null when no package declares it, or when they disagree --
|
|
417
|
+
* either way the caller falls back to the root-only probe rather than picking
|
|
418
|
+
* a winner out of a tie (#504).
|
|
419
|
+
*/
|
|
420
|
+
function workspaceShapeForFramework(ws, framework) {
|
|
421
|
+
if (ws === null)
|
|
422
|
+
return null;
|
|
423
|
+
const shapes = new Set(ws.findings.filter((f) => f.framework === framework).map((f) => f.shape));
|
|
424
|
+
return shapes.size === 1 ? [...shapes][0] : null;
|
|
425
|
+
}
|
|
333
426
|
/**
|
|
334
427
|
* The shape implied by an explicit `--framework` override (#504 (2)).
|
|
335
428
|
*
|
|
@@ -753,11 +846,51 @@ export class MigrationReport {
|
|
|
753
846
|
const p = ws.scanned === 1 ? '1 package' : `${ws.scanned} packages`;
|
|
754
847
|
notes.push(`${p} scanned, none carries a recognizable test config.`);
|
|
755
848
|
}
|
|
849
|
+
else if (ws.findings.length > 0) {
|
|
850
|
+
// The other half of the denominator: what the walk *did* observe. The
|
|
851
|
+
// scalar above collapses disagreeing packages to `unknown`, so without
|
|
852
|
+
// these lines a repo with two real suites reads as a repo with none
|
|
853
|
+
// (#504 part 1).
|
|
854
|
+
const withFindings = packagesWithFindings(ws);
|
|
855
|
+
const p = ws.scanned === 1 ? '1 package' : `${ws.scanned} packages`;
|
|
856
|
+
notes.push(`${ws.manager} workspace ${EMDASH} ${p} scanned, ` +
|
|
857
|
+
`${withFindings} carrying a test config.`);
|
|
858
|
+
for (const f of ws.findings) {
|
|
859
|
+
notes.push(`\`${f.dir}/\` ${EMDASH} ${f.framework} / ${f.shape} ` +
|
|
860
|
+
`(\`${f.source}\`)`);
|
|
861
|
+
}
|
|
862
|
+
}
|
|
756
863
|
for (const dir of ws.unreadable) {
|
|
757
864
|
notes.push(`\`${dir}/\` could not be read and was not scanned.`);
|
|
758
865
|
}
|
|
759
866
|
return notes;
|
|
760
867
|
}
|
|
868
|
+
/**
|
|
869
|
+
* Why a dry run proposes nothing. Three different situations reach this
|
|
870
|
+
* line and they are not interchangeable: a suite already exists for this
|
|
871
|
+
* framework, the workspace packages disagree so no root framework applies,
|
|
872
|
+
* or the project genuinely carries everything already. Rendering the third
|
|
873
|
+
* sentence for the second case tells a monorepo with no Canary config that
|
|
874
|
+
* it has all of it (#504 parts 1 and 3).
|
|
875
|
+
*/
|
|
876
|
+
nothingToCreateNote() {
|
|
877
|
+
if (this.existing_suites.length > 0) {
|
|
878
|
+
const names = this.existing_suites.map((s) => `\`${s.dir}/\``);
|
|
879
|
+
return (`_Nothing ${EMDASH} ${names.join(', ')} already ` +
|
|
880
|
+
`carries a ${this.framework} suite. To scaffold a second one, ` +
|
|
881
|
+
're-run `canary migrate` from inside that package._');
|
|
882
|
+
}
|
|
883
|
+
const ws = this.workspace;
|
|
884
|
+
if (this.framework === 'unknown' && ws !== null && ws.findings.length > 0) {
|
|
885
|
+
return (`_Nothing ${EMDASH} this workspace's packages declare more than ` +
|
|
886
|
+
`one framework: ${namedWorkspaceSuites(ws)}. Re-run ` +
|
|
887
|
+
'`canary migrate` from inside a package, or pass ' +
|
|
888
|
+
'`--framework <name>` to pick one._');
|
|
889
|
+
}
|
|
890
|
+
return ('_Nothing new ' +
|
|
891
|
+
EMDASH +
|
|
892
|
+
' project already has all Canary config files._');
|
|
893
|
+
}
|
|
761
894
|
/**
|
|
762
895
|
* The dry run's denominator (#504): config files that would be created,
|
|
763
896
|
* skills that would deploy, workflows that would install. Zero means the
|
|
@@ -819,23 +952,15 @@ export class MigrationReport {
|
|
|
819
952
|
lines.push(`- \`${f}\``);
|
|
820
953
|
lines.push('');
|
|
821
954
|
}
|
|
955
|
+
lines.push('## Would Create', '');
|
|
822
956
|
if (this.would_create.length > 0) {
|
|
823
|
-
lines.push('## Would Create', '');
|
|
824
957
|
for (const f of this.would_create)
|
|
825
958
|
lines.push(`- \`${f}\``);
|
|
826
|
-
lines.push('');
|
|
827
|
-
}
|
|
828
|
-
else if (this.existing_suites.length > 0) {
|
|
829
|
-
const names = this.existing_suites.map((s) => `\`${s.dir}/\``);
|
|
830
|
-
lines.push('## Would Create', '', `_Nothing ${EMDASH} ${names.join(', ')} already ` +
|
|
831
|
-
`carries a ${this.framework} suite. To scaffold a second one, ` +
|
|
832
|
-
're-run `canary migrate` from inside that package._', '');
|
|
833
959
|
}
|
|
834
960
|
else {
|
|
835
|
-
lines.push(
|
|
836
|
-
EMDASH +
|
|
837
|
-
' project already has all Canary config files._', '');
|
|
961
|
+
lines.push(this.nothingToCreateNote());
|
|
838
962
|
}
|
|
963
|
+
lines.push('');
|
|
839
964
|
if (this.skipped_configs.length > 0) {
|
|
840
965
|
lines.push('## Already Present (will not be touched)', '');
|
|
841
966
|
for (const f of this.skipped_configs)
|
|
@@ -1027,7 +1152,13 @@ export class HarnessMigrator {
|
|
|
1027
1152
|
.trim()
|
|
1028
1153
|
.toLowerCase();
|
|
1029
1154
|
const overrideShape = pyTruthy(framework) && explicitShape === ''
|
|
1030
|
-
?
|
|
1155
|
+
? // A workspace package that already declares this framework has
|
|
1156
|
+
// already had its shape refined against its OWN spec files. The
|
|
1157
|
+
// root-only refinement cannot see them -- in the reported repo the
|
|
1158
|
+
// specs live in `apps/web-e2e/tests/`, so refining at the root reads
|
|
1159
|
+
// zero spec files and falls back to the table default (#504).
|
|
1160
|
+
(workspaceShapeForFramework(ctx.workspace, framework) ??
|
|
1161
|
+
shapeForFrameworkOverride(framework, projectRoot))
|
|
1031
1162
|
: null;
|
|
1032
1163
|
const shape = overrideShape ?? ctx.detected_shape;
|
|
1033
1164
|
// The set that actually drives deployment. Re-derived from the *effective*
|
|
@@ -1036,11 +1167,7 @@ export class HarnessMigrator {
|
|
|
1036
1167
|
const deployShapes = unionShapes(shape, ctx.workspace);
|
|
1037
1168
|
const followups = [];
|
|
1038
1169
|
if (effectiveFramework === null) {
|
|
1039
|
-
followups.push(
|
|
1040
|
-
reason: 'no config file, dependency, or language marker matched a known framework',
|
|
1041
|
-
candidates: KNOWN_FRAMEWORKS,
|
|
1042
|
-
overrideHint: '`canary migrate --framework <name>`',
|
|
1043
|
-
}));
|
|
1170
|
+
followups.push(...unresolvedFrameworkFollowups(ctx.workspace));
|
|
1044
1171
|
// Issue #295 point 3: a detection miss must not block skill deployment --
|
|
1045
1172
|
// nor, for the same reason, the workflow install (#459). The guardian
|
|
1046
1173
|
// workflow is exactly what an unrecognised repo most needs.
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Permission matrix (#857): a human-declared allow/deny grid, expanded across
|
|
3
|
+
* every acting tenant x target tenant, rendered as server-direct API tests.
|
|
4
|
+
*
|
|
5
|
+
* The grid is declared, never inferred. Inferring expectations from current
|
|
6
|
+
* behaviour would encode today's authz bugs as "expected". So a cell with no
|
|
7
|
+
* declaration is `undeclared` -- counted and named, never allowed or denied by
|
|
8
|
+
* default -- because a default is exactly where the cells nobody thought about
|
|
9
|
+
* would hide.
|
|
10
|
+
*/
|
|
11
|
+
import { load } from 'js-yaml';
|
|
12
|
+
const VALUES = ['allow', 'deny', 'own-tenant'];
|
|
13
|
+
// Backtick, `$` and backslash are excluded from paths because the path is
|
|
14
|
+
// spliced into a generated template literal.
|
|
15
|
+
const ENDPOINT = /^([A-Z]+)\s+(\/[^\s`$\\]*)$/;
|
|
16
|
+
function stringList(v, key) {
|
|
17
|
+
if (!Array.isArray(v) ||
|
|
18
|
+
v.length === 0 ||
|
|
19
|
+
!v.every((x) => typeof x === 'string')) {
|
|
20
|
+
throw new Error(`model needs a non-empty "${key}" list of names`);
|
|
21
|
+
}
|
|
22
|
+
return v;
|
|
23
|
+
}
|
|
24
|
+
export function parseMatrix(text) {
|
|
25
|
+
const raw = (load(text) ?? {});
|
|
26
|
+
const roles = stringList(raw['roles'], 'roles');
|
|
27
|
+
const tenants = stringList(raw['tenants'], 'tenants');
|
|
28
|
+
const endpoints = raw['endpoints'];
|
|
29
|
+
if (endpoints === null ||
|
|
30
|
+
typeof endpoints !== 'object' ||
|
|
31
|
+
Array.isArray(endpoints)) {
|
|
32
|
+
throw new Error('model needs an "endpoints" map of "METHOD /path" -> cells');
|
|
33
|
+
}
|
|
34
|
+
for (const key of Object.keys(endpoints)) {
|
|
35
|
+
if (!ENDPOINT.test(key)) {
|
|
36
|
+
throw new Error(`endpoint "${key}" must be written as "METHOD /path"`);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
return {
|
|
40
|
+
roles,
|
|
41
|
+
tenants,
|
|
42
|
+
endpoints: endpoints,
|
|
43
|
+
existenceProbes: probeList(raw['existence_probes'], endpoints),
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* A probe on an undeclared endpoint is a model error, not a skip: the author
|
|
48
|
+
* asked for a check, and quietly not running it would read as "no oracle".
|
|
49
|
+
*/
|
|
50
|
+
function probeList(v, endpoints) {
|
|
51
|
+
if (v === undefined || v === null)
|
|
52
|
+
return [];
|
|
53
|
+
if (!Array.isArray(v) || !v.every((x) => typeof x === 'string')) {
|
|
54
|
+
throw new Error('"existence_probes" must be a list of "METHOD /path"');
|
|
55
|
+
}
|
|
56
|
+
for (const p of v) {
|
|
57
|
+
if (!Object.hasOwn(endpoints, p)) {
|
|
58
|
+
throw new Error(`existence probe "${p}" is not in endpoints`);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
return v;
|
|
62
|
+
}
|
|
63
|
+
/** The declared value for one cell, or `null` (recorded in `undeclared`). */
|
|
64
|
+
function declaredValue(row, endpoint, role, undeclared) {
|
|
65
|
+
const declared = (row ?? {})[role];
|
|
66
|
+
if (declared === undefined) {
|
|
67
|
+
undeclared.push(`${endpoint} × ${role}`);
|
|
68
|
+
return null;
|
|
69
|
+
}
|
|
70
|
+
if (typeof declared !== 'string' || !VALUES.includes(declared)) {
|
|
71
|
+
undeclared.push(`${endpoint} × ${role} (unknown value "${String(declared)}")`);
|
|
72
|
+
return null;
|
|
73
|
+
}
|
|
74
|
+
return declared;
|
|
75
|
+
}
|
|
76
|
+
function expectFor(value, actingTenant, targetTenant) {
|
|
77
|
+
if (value === null)
|
|
78
|
+
return 'undeclared';
|
|
79
|
+
if (value !== 'own-tenant')
|
|
80
|
+
return value;
|
|
81
|
+
return actingTenant === targetTenant ? 'allow' : 'deny';
|
|
82
|
+
}
|
|
83
|
+
export function expandMatrix(model) {
|
|
84
|
+
const cells = [];
|
|
85
|
+
const undeclared = [];
|
|
86
|
+
const pairs = model.tenants.flatMap((a) => model.tenants.map((t) => [a, t]));
|
|
87
|
+
for (const [endpoint, row] of Object.entries(model.endpoints)) {
|
|
88
|
+
const [, method, path] = ENDPOINT.exec(endpoint);
|
|
89
|
+
for (const role of model.roles) {
|
|
90
|
+
const value = declaredValue(row, endpoint, role, undeclared);
|
|
91
|
+
for (const [actingTenant, targetTenant] of pairs) {
|
|
92
|
+
cells.push({
|
|
93
|
+
endpoint,
|
|
94
|
+
method: method,
|
|
95
|
+
path: path,
|
|
96
|
+
role,
|
|
97
|
+
actingTenant,
|
|
98
|
+
targetTenant,
|
|
99
|
+
expect: expectFor(value, actingTenant, targetTenant),
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
return { cells, undeclared };
|
|
105
|
+
}
|
|
106
|
+
/** Env-var-safe form of a role or tenant name. */
|
|
107
|
+
function envName(s) {
|
|
108
|
+
return s.toUpperCase().replace(/[^A-Z0-9]/g, '_');
|
|
109
|
+
}
|
|
110
|
+
const HEADER = [
|
|
111
|
+
'// Generated by `canary permission-matrix` (#857). Do not edit by hand:',
|
|
112
|
+
'// change the declared model and regenerate.',
|
|
113
|
+
'//',
|
|
114
|
+
'// Credentials and fixtures come from the environment, never this file:',
|
|
115
|
+
'// CANARY_TOKEN_<ROLE>_<TENANT> bearer token for that role in that tenant',
|
|
116
|
+
'// CANARY_ID_<TENANT> a synthetic record owned by that tenant',
|
|
117
|
+
"import { test, expect } from '@playwright/test';",
|
|
118
|
+
'',
|
|
119
|
+
'function env(name: string): string {',
|
|
120
|
+
' const v = process.env[name];',
|
|
121
|
+
' // A missing credential must fail, not skip: a skipped deny cell reads as',
|
|
122
|
+
' // "no leak" in the report.',
|
|
123
|
+
' if (!v) throw new Error(`${name} is not set`);',
|
|
124
|
+
' return v;',
|
|
125
|
+
'}',
|
|
126
|
+
'',
|
|
127
|
+
'const DENIED = [401, 403, 404];',
|
|
128
|
+
'',
|
|
129
|
+
];
|
|
130
|
+
const VERB = {
|
|
131
|
+
allow: 'allowed',
|
|
132
|
+
deny: 'denied',
|
|
133
|
+
undeclared: 'UNDECLARED',
|
|
134
|
+
};
|
|
135
|
+
function renderCell(c) {
|
|
136
|
+
const title = JSON.stringify(`${c.role}@${c.actingTenant} -> ${c.targetTenant}: ${c.endpoint} is ${VERB[c.expect]}`);
|
|
137
|
+
if (c.expect === 'undeclared') {
|
|
138
|
+
return [
|
|
139
|
+
`test.fixme(${title}, () => {`,
|
|
140
|
+
' // UNDECLARED: the model has no expectation for this cell. Declare it.',
|
|
141
|
+
'});',
|
|
142
|
+
'',
|
|
143
|
+
];
|
|
144
|
+
}
|
|
145
|
+
return [
|
|
146
|
+
`test(${title}, async ({ request }) => {`,
|
|
147
|
+
` const res = await request.fetch(\`${urlFor(c, targetId(c))}\`, {`,
|
|
148
|
+
` method: '${c.method}',`,
|
|
149
|
+
` headers: ${authHeader(c)},`,
|
|
150
|
+
' });',
|
|
151
|
+
c.expect === 'allow'
|
|
152
|
+
? ' expect(res.status()).toBeLessThan(400);'
|
|
153
|
+
: ' expect(DENIED).toContain(res.status());',
|
|
154
|
+
'});',
|
|
155
|
+
'',
|
|
156
|
+
];
|
|
157
|
+
}
|
|
158
|
+
function targetId(c) {
|
|
159
|
+
return `\${env('CANARY_ID_${envName(c.targetTenant)}')}`;
|
|
160
|
+
}
|
|
161
|
+
function urlFor(c, idExpr) {
|
|
162
|
+
return c.path.replace(/\{[^}]+\}/g, idExpr);
|
|
163
|
+
}
|
|
164
|
+
function authHeader(c) {
|
|
165
|
+
return `{ Authorization: \`Bearer \${env('CANARY_TOKEN_${envName(c.role)}_${envName(c.actingTenant)}')}\` }`;
|
|
166
|
+
}
|
|
167
|
+
// Emitted only when a probe exists, so a model without probes generates the
|
|
168
|
+
// same file it always did.
|
|
169
|
+
const SHAPE_HELPER = [
|
|
170
|
+
'// Existence probes also need CANARY_ABSENT_ID: an id that exists in no',
|
|
171
|
+
'// tenant, compared against each denied real record.',
|
|
172
|
+
'//',
|
|
173
|
+
'// What a caller can observe about a response body: sorted top-level keys',
|
|
174
|
+
'// for a JSON object, otherwise the raw text. Values are deliberately not',
|
|
175
|
+
'// compared -- two "not found" bodies may differ in a request id.',
|
|
176
|
+
'async function shape(res: { text(): Promise<string> }): Promise<unknown> {',
|
|
177
|
+
' const text = await res.text();',
|
|
178
|
+
' try {',
|
|
179
|
+
' const body: unknown = JSON.parse(text);',
|
|
180
|
+
" return body !== null && typeof body === 'object' && !Array.isArray(body)",
|
|
181
|
+
' ? Object.keys(body).sort()',
|
|
182
|
+
' : typeof body;',
|
|
183
|
+
' } catch {',
|
|
184
|
+
' return text;',
|
|
185
|
+
' }',
|
|
186
|
+
'}',
|
|
187
|
+
'',
|
|
188
|
+
];
|
|
189
|
+
/**
|
|
190
|
+
* A denied caller must get the same answer for a real record as for a
|
|
191
|
+
* missing one; otherwise the endpoint confirms the record exists.
|
|
192
|
+
*/
|
|
193
|
+
function renderProbe(c) {
|
|
194
|
+
const title = JSON.stringify(`${c.role}@${c.actingTenant} -> ${c.targetTenant}: ${c.endpoint} existence is not revealed`);
|
|
195
|
+
const opts = `{ method: '${c.method}', headers: ${authHeader(c)} }`;
|
|
196
|
+
return [
|
|
197
|
+
`test(${title}, async ({ request }) => {`,
|
|
198
|
+
` const present = await request.fetch(\`${urlFor(c, targetId(c))}\`, ${opts});`,
|
|
199
|
+
` const absent = await request.fetch(\`${urlFor(c, "${env('CANARY_ABSENT_ID')}")}\`, ${opts});`,
|
|
200
|
+
' expect(absent.status()).toBe(present.status());',
|
|
201
|
+
' expect(await shape(absent)).toEqual(await shape(present));',
|
|
202
|
+
'});',
|
|
203
|
+
'',
|
|
204
|
+
];
|
|
205
|
+
}
|
|
206
|
+
/** Denied cells of probed endpoints: allowed callers may know a record exists. */
|
|
207
|
+
export function probedCells(cells, existenceProbes) {
|
|
208
|
+
return cells.filter((c) => c.expect === 'deny' && existenceProbes.includes(c.endpoint));
|
|
209
|
+
}
|
|
210
|
+
export function renderPlaywright(cells, existenceProbes = []) {
|
|
211
|
+
const probed = probedCells(cells, existenceProbes);
|
|
212
|
+
return [
|
|
213
|
+
...HEADER,
|
|
214
|
+
...(probed.length > 0 ? SHAPE_HELPER : []),
|
|
215
|
+
...cells.flatMap(renderCell),
|
|
216
|
+
...probed.flatMap(renderProbe),
|
|
217
|
+
].join('\n');
|
|
218
|
+
}
|
|
219
|
+
//# sourceMappingURL=permission-matrix.js.map
|
|
@@ -33,9 +33,9 @@ const ASSERTIONS = {
|
|
|
33
33
|
// never gates — a false "asserts nothing" on a correct test is what teaches
|
|
34
34
|
// a reader to ignore the finding.
|
|
35
35
|
playwright: /\bexpect\w*\s*\(|\btoBeVisible\b|\btoHaveText\b|\btoHaveTitle\b|\btoHaveURL\b|\btoBeEnabled\b|\btoBeDisabled\b|\btoBeChecked\b|\btoHaveValue\b|\btoHaveCount\b/g,
|
|
36
|
-
// Plus
|
|
37
|
-
// `assert(
|
|
38
|
-
vitest: /\bexpect\w*\s*\(|\btoBe\s*\(|\btoEqual\s*\(|\btoThrow\b|\btoContain\s*\(|\btoBeNull\b|\btoBeUndefined\b|\btoMatchObject\b|\bassert\s*\(|\bassert\.\w+|\.should\b/g,
|
|
36
|
+
// Plus node:assert / `assert.equal(...)`, chai `x.should`, and any
|
|
37
|
+
// `assert*(` helper call, matching the pytest pattern (#929).
|
|
38
|
+
vitest: /\bexpect\w*\s*\(|\btoBe\s*\(|\btoEqual\s*\(|\btoThrow\b|\btoContain\s*\(|\btoBeNull\b|\btoBeUndefined\b|\btoMatchObject\b|\bassert\w*\s*\(|\bassert\.\w+|\.should\b/g,
|
|
39
39
|
k6: /\bcheck\s*\(|'[^']+'\s*:\s*\([^)]*\)\s*=>/g,
|
|
40
40
|
};
|
|
41
41
|
const NEGATIVE_KW = /\b(error|invalid|empty|null|undefined|throws|raises|exception|fail|missing|negative|reject|4\d{2}|5\d{2}|boundary|edge)\b/i;
|
|
@@ -178,24 +178,19 @@ function magicDetails(findings) {
|
|
|
178
178
|
return ['No magic numbers detected'];
|
|
179
179
|
return [`${findings.length} magic number(s) detected`, ...findings];
|
|
180
180
|
}
|
|
181
|
+
/** True iff `code` contains an assertion token for `framework`. */
|
|
182
|
+
export function hasAssertion(code, framework) {
|
|
183
|
+
const fw = framework.toLowerCase();
|
|
184
|
+
return countMatches(ASSERTIONS[fw] ?? ASSERTIONS['pytest'], code) > 0;
|
|
185
|
+
}
|
|
181
186
|
/**
|
|
182
|
-
* True iff `code` defines
|
|
183
|
-
*
|
|
184
|
-
*
|
|
185
|
-
* Faithful port of `is_assertion_free_test`. The high-precision "weak test"
|
|
186
|
-
* signal the PR guardian consumes: it requires BOTH a test function (so a
|
|
187
|
-
* non-test helper added to a test file isn't flagged) AND zero assertions. A
|
|
188
|
-
* snapshot or table-driven test still matches an assertion pattern (`expect(` /
|
|
189
|
-
* `assert`), so it is not flagged — keeping false positives, and the trust cost
|
|
190
|
-
* of them, low. Reuses the module's {@link TEST_FN}/{@link ASSERTIONS} maps.
|
|
187
|
+
* True iff `code` defines a test function but contains no assertions. Port of
|
|
188
|
+
* `is_assertion_free_test`: requiring BOTH keeps a non-test helper unflagged,
|
|
189
|
+
* and a snapshot or table-driven test still matches an assertion pattern.
|
|
191
190
|
*/
|
|
192
191
|
export function isAssertionFreeTest(code, framework) {
|
|
193
|
-
const
|
|
194
|
-
|
|
195
|
-
const assertPat = ASSERTIONS[fw] ?? ASSERTIONS['pytest'];
|
|
196
|
-
const hasTest = countMatches(fnPat, code) > 0;
|
|
197
|
-
const hasAssertion = countMatches(assertPat, code) > 0;
|
|
198
|
-
return hasTest && !hasAssertion;
|
|
192
|
+
const fnPat = TEST_FN[framework.toLowerCase()] ?? TEST_FN['pytest'];
|
|
193
|
+
return countMatches(fnPat, code) > 0 && !hasAssertion(code, framework);
|
|
199
194
|
}
|
|
200
195
|
export class QualityScorer {
|
|
201
196
|
/** Score `code` (a source string) for the given framework. */
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Scaling-curve probe (#856): how does cost grow as input grows?
|
|
3
|
+
*
|
|
4
|
+
* Fits the log-log slope of a cost metric against input size. A fixed-size load
|
|
5
|
+
* test cannot see an O(n·m) matcher; the exponent can, before the surge does.
|
|
6
|
+
*
|
|
7
|
+
* Pure: points in, verdict out. The Phase 2 runner only produces points, so it
|
|
8
|
+
* can never disagree with the abstention rules here.
|
|
9
|
+
*
|
|
10
|
+
* Every abstention rule is a way "linear ✓" could be printed off data that does
|
|
11
|
+
* not support it (ADR 0009): too few sizes, too narrow a span, noisy samples,
|
|
12
|
+
* or a curve the power law does not fit.
|
|
13
|
+
*/
|
|
14
|
+
const MIN_SIZES = 4;
|
|
15
|
+
const MIN_SPAN = 8;
|
|
16
|
+
const MAX_CV = 0.25;
|
|
17
|
+
const MIN_R2 = 0.9;
|
|
18
|
+
const KNEE_DELTA = 0.5;
|
|
19
|
+
const LINEAR_MAX = 1.15;
|
|
20
|
+
const SUPERLINEAR_MAX = 1.6;
|
|
21
|
+
function median(xs) {
|
|
22
|
+
const s = [...xs].sort((a, b) => a - b);
|
|
23
|
+
const mid = s.length >> 1;
|
|
24
|
+
return s.length % 2 ? s[mid] : (s[mid - 1] + s[mid]) / 2;
|
|
25
|
+
}
|
|
26
|
+
function cv(xs) {
|
|
27
|
+
const mean = xs.reduce((a, b) => a + b, 0) / xs.length;
|
|
28
|
+
const sd = Math.sqrt(xs.reduce((a, x) => a + (x - mean) ** 2, 0) / xs.length);
|
|
29
|
+
return sd / mean;
|
|
30
|
+
}
|
|
31
|
+
/** Least-squares slope, intercept and R² of y on x. */
|
|
32
|
+
function fit(xs, ys) {
|
|
33
|
+
const n = xs.length;
|
|
34
|
+
const mx = xs.reduce((a, b) => a + b, 0) / n;
|
|
35
|
+
const my = ys.reduce((a, b) => a + b, 0) / n;
|
|
36
|
+
let sxy = 0;
|
|
37
|
+
let sxx = 0;
|
|
38
|
+
let syy = 0;
|
|
39
|
+
for (let i = 0; i < n; i += 1) {
|
|
40
|
+
sxy += (xs[i] - mx) * (ys[i] - my);
|
|
41
|
+
sxx += (xs[i] - mx) ** 2;
|
|
42
|
+
syy += (ys[i] - my) ** 2;
|
|
43
|
+
}
|
|
44
|
+
const b = sxy / sxx;
|
|
45
|
+
// A perfectly flat curve is a perfect (b = 0) fit, not a 0/0.
|
|
46
|
+
const r2 = syy === 0 ? 1 : (sxy * sxy) / (sxx * syy);
|
|
47
|
+
return { b, a: my - b * mx, r2 };
|
|
48
|
+
}
|
|
49
|
+
export function fitScaling(raw, opts = {}) {
|
|
50
|
+
const reasons = [];
|
|
51
|
+
const bad = raw.filter((p) => !(Number.isFinite(p.size) && p.size > 0) ||
|
|
52
|
+
!(Number.isFinite(p.value) && p.value > 0));
|
|
53
|
+
if (bad.length > 0) {
|
|
54
|
+
reasons.push(`${bad.length} point(s) with a size or value that is not a positive finite number`);
|
|
55
|
+
}
|
|
56
|
+
const good = raw.filter((p) => !bad.includes(p));
|
|
57
|
+
const bySize = new Map();
|
|
58
|
+
for (const p of good) {
|
|
59
|
+
const vs = bySize.get(p.size);
|
|
60
|
+
if (vs)
|
|
61
|
+
vs.push(p.value);
|
|
62
|
+
else
|
|
63
|
+
bySize.set(p.size, [p.value]);
|
|
64
|
+
}
|
|
65
|
+
const points = [...bySize]
|
|
66
|
+
.sort(([a], [b]) => a - b)
|
|
67
|
+
.map(([size, vs]) => ({ size, value: median(vs), samples: vs.length }));
|
|
68
|
+
if (points.length < MIN_SIZES) {
|
|
69
|
+
reasons.push(`${points.length} distinct size(s); need at least ${MIN_SIZES} distinct sizes`);
|
|
70
|
+
}
|
|
71
|
+
else {
|
|
72
|
+
const span = points.at(-1).size / points[0].size;
|
|
73
|
+
if (span < MIN_SPAN) {
|
|
74
|
+
reasons.push(`size span is ${span.toFixed(1)}x; need at least ${MIN_SPAN}x`);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
for (const [size, vs] of bySize) {
|
|
78
|
+
if (vs.length >= 3 && cv(vs) > MAX_CV) {
|
|
79
|
+
reasons.push(`size ${size} is noisy: coefficient of variation ${cv(vs).toFixed(2)} > ${MAX_CV}`);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
let result = null;
|
|
83
|
+
if (points.length >= 2) {
|
|
84
|
+
result = fit(points.map((p) => Math.log(p.size)), points.map((p) => Math.log(p.value)));
|
|
85
|
+
if (result.r2 < MIN_R2) {
|
|
86
|
+
reasons.push(`power-law fit R² ${result.r2.toFixed(2)} < ${MIN_R2}: the curve is not one exponent`);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
if (reasons.length > 0 || result === null) {
|
|
90
|
+
return { verdict: 'INSUFFICIENT_DATA', reasons, points };
|
|
91
|
+
}
|
|
92
|
+
const slopes = points
|
|
93
|
+
.slice(1)
|
|
94
|
+
.map((p, i) => Math.log(p.value / points[i].value) /
|
|
95
|
+
Math.log(p.size / points[i].size));
|
|
96
|
+
const kneeAt = slopes.findIndex((s) => s >= slopes[0] + KNEE_DELTA);
|
|
97
|
+
const verdict = result.b <= LINEAR_MAX
|
|
98
|
+
? 'LINEAR_OR_BETTER'
|
|
99
|
+
: result.b <= SUPERLINEAR_MAX
|
|
100
|
+
? 'SUPERLINEAR'
|
|
101
|
+
: 'STRONGLY_SUPERLINEAR';
|
|
102
|
+
const out = {
|
|
103
|
+
verdict,
|
|
104
|
+
exponent: result.b,
|
|
105
|
+
r2: result.r2,
|
|
106
|
+
knee: kneeAt === -1 ? null : points[kneeAt].size,
|
|
107
|
+
points,
|
|
108
|
+
};
|
|
109
|
+
if (opts.target !== undefined) {
|
|
110
|
+
out.extrapolation = {
|
|
111
|
+
size: opts.target,
|
|
112
|
+
value: Math.exp(result.a + result.b * Math.log(opts.target)),
|
|
113
|
+
multipleOfLargest: opts.target / points.at(-1).size,
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
return out;
|
|
117
|
+
}
|
|
118
|
+
/** Read `{metric, points}` JSON or a `size,value` CSV. Unreadable input throws. */
|
|
119
|
+
export function parsePoints(text) {
|
|
120
|
+
const trimmed = text.trim();
|
|
121
|
+
if (trimmed.startsWith('{')) {
|
|
122
|
+
const j = JSON.parse(trimmed);
|
|
123
|
+
if (!Array.isArray(j.points)) {
|
|
124
|
+
throw new Error('JSON input needs a "points" array of {size, value}');
|
|
125
|
+
}
|
|
126
|
+
return j.metric === undefined
|
|
127
|
+
? { points: j.points }
|
|
128
|
+
: { metric: j.metric, points: j.points };
|
|
129
|
+
}
|
|
130
|
+
const [header, ...rows] = trimmed.split(/\r?\n/);
|
|
131
|
+
if (header?.replace(/\s/g, '').toLowerCase() !== 'size,value') {
|
|
132
|
+
throw new Error('expected JSON {"points": [...]} or CSV with a size,value header');
|
|
133
|
+
}
|
|
134
|
+
return {
|
|
135
|
+
points: rows
|
|
136
|
+
.filter((r) => r.trim())
|
|
137
|
+
.map((r) => {
|
|
138
|
+
const [size, value] = r.split(',').map(Number);
|
|
139
|
+
return { size: size, value: value };
|
|
140
|
+
}),
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
//# sourceMappingURL=scaling-curve.js.map
|
|
@@ -59,7 +59,9 @@ export function blankStringContent(code, options = {}) {
|
|
|
59
59
|
const spans = literalContentSpans(code, options.python === true);
|
|
60
60
|
if (spans.length === 0)
|
|
61
61
|
return code;
|
|
62
|
-
|
|
62
|
+
// `split('')`, not `[...code]`: spans are UTF-16 offsets, and spreading by
|
|
63
|
+
// code point would collapse each surrogate pair and shift every offset (#861).
|
|
64
|
+
const out = code.split('');
|
|
63
65
|
for (const [start, end] of spans) {
|
|
64
66
|
for (let i = start; i < end; i += 1) {
|
|
65
67
|
// Newlines survive so line numbering is unchanged; everything else goes.
|