@qulib/core 0.8.2 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -13
- package/bin/qulib.js +2 -3
- package/dist/__tests__/playwright-available.d.ts +32 -0
- package/dist/__tests__/playwright-available.d.ts.map +1 -0
- package/dist/__tests__/playwright-available.js +35 -0
- package/dist/adapters/ci-results-adapter.d.ts +67 -0
- package/dist/adapters/ci-results-adapter.d.ts.map +1 -0
- package/dist/adapters/ci-results-adapter.js +143 -0
- package/dist/adapters/cypress-e2e-adapter.d.ts.map +1 -1
- package/dist/adapters/cypress-e2e-adapter.js +25 -2
- package/dist/adapters/playwright-adapter.d.ts.map +1 -1
- package/dist/adapters/playwright-adapter.js +25 -2
- package/dist/adapters/pr-metadata-adapter.d.ts +75 -0
- package/dist/adapters/pr-metadata-adapter.d.ts.map +1 -0
- package/dist/adapters/pr-metadata-adapter.js +146 -0
- package/dist/adapters/validate-specs.d.ts +55 -0
- package/dist/adapters/validate-specs.d.ts.map +1 -0
- package/dist/adapters/validate-specs.js +67 -0
- package/dist/baseline/baseline.d.ts +54 -0
- package/dist/baseline/baseline.d.ts.map +1 -0
- package/dist/baseline/baseline.js +252 -0
- package/dist/baseline/baseline.schema.d.ts +233 -0
- package/dist/baseline/baseline.schema.d.ts.map +1 -0
- package/dist/baseline/baseline.schema.js +59 -0
- package/dist/cli/analyze-diff-run.d.ts +77 -0
- package/dist/cli/analyze-diff-run.d.ts.map +1 -0
- package/dist/cli/analyze-diff-run.js +266 -0
- package/dist/cli/baseline-run.d.ts +55 -0
- package/dist/cli/baseline-run.d.ts.map +1 -0
- package/dist/cli/baseline-run.js +259 -0
- package/dist/cli/confidence-run.d.ts +16 -0
- package/dist/cli/confidence-run.d.ts.map +1 -0
- package/dist/cli/confidence-run.js +162 -0
- package/dist/cli/index.d.ts +11 -1
- package/dist/cli/index.d.ts.map +1 -1
- package/dist/cli/index.js +84 -4
- package/dist/cli/scaffold-run.d.ts +86 -0
- package/dist/cli/scaffold-run.d.ts.map +1 -0
- package/dist/cli/scaffold-run.js +232 -0
- package/dist/cli/score-automation-run.d.ts +25 -0
- package/dist/cli/score-automation-run.d.ts.map +1 -0
- package/dist/cli/score-automation-run.js +127 -0
- package/dist/examples/notquality-dogfood/fixture.d.ts +166 -0
- package/dist/examples/notquality-dogfood/fixture.d.ts.map +1 -0
- package/dist/examples/notquality-dogfood/fixture.js +174 -0
- package/dist/examples/notquality-dogfood/run.d.ts +34 -0
- package/dist/examples/notquality-dogfood/run.d.ts.map +1 -0
- package/dist/examples/notquality-dogfood/run.js +139 -0
- package/dist/index.d.ts +18 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +15 -0
- package/dist/recipes/a11y.d.ts +36 -0
- package/dist/recipes/a11y.d.ts.map +1 -0
- package/dist/recipes/a11y.js +118 -0
- package/dist/recipes/auth.d.ts +38 -0
- package/dist/recipes/auth.d.ts.map +1 -0
- package/dist/recipes/auth.js +156 -0
- package/dist/recipes/index.d.ts +26 -0
- package/dist/recipes/index.d.ts.map +1 -0
- package/dist/recipes/index.js +41 -0
- package/dist/recipes/nav.d.ts +34 -0
- package/dist/recipes/nav.d.ts.map +1 -0
- package/dist/recipes/nav.js +128 -0
- package/dist/recipes/seed.d.ts +34 -0
- package/dist/recipes/seed.d.ts.map +1 -0
- package/dist/recipes/seed.js +87 -0
- package/dist/reporters/heatmap.d.ts +55 -0
- package/dist/reporters/heatmap.d.ts.map +1 -0
- package/dist/reporters/heatmap.js +146 -0
- package/dist/reporters/markdown-reporter.d.ts.map +1 -1
- package/dist/reporters/markdown-reporter.js +4 -1
- package/dist/scaffold-tests.d.ts +21 -0
- package/dist/scaffold-tests.d.ts.map +1 -1
- package/dist/scaffold-tests.js +12 -2
- package/dist/schemas/confidence.schema.d.ts +526 -0
- package/dist/schemas/confidence.schema.d.ts.map +1 -0
- package/dist/schemas/confidence.schema.js +161 -0
- package/dist/schemas/config.schema.d.ts.map +1 -1
- package/dist/schemas/config.schema.js +6 -1
- package/dist/schemas/index.d.ts +3 -0
- package/dist/schemas/index.d.ts.map +1 -1
- package/dist/schemas/index.js +3 -0
- package/dist/schemas/recipe.schema.d.ts +66 -0
- package/dist/schemas/recipe.schema.d.ts.map +1 -0
- package/dist/schemas/recipe.schema.js +45 -0
- package/dist/schemas/views.schema.d.ts +234 -0
- package/dist/schemas/views.schema.d.ts.map +1 -0
- package/dist/schemas/views.schema.js +82 -0
- package/dist/tools/scoring/confidence-from-qulib.d.ts +34 -0
- package/dist/tools/scoring/confidence-from-qulib.d.ts.map +1 -0
- package/dist/tools/scoring/confidence-from-qulib.js +206 -0
- package/dist/tools/scoring/confidence-views.d.ts +40 -0
- package/dist/tools/scoring/confidence-views.d.ts.map +1 -0
- package/dist/tools/scoring/confidence-views.js +163 -0
- package/dist/tools/scoring/confidence.d.ts +32 -0
- package/dist/tools/scoring/confidence.d.ts.map +1 -0
- package/dist/tools/scoring/confidence.js +180 -0
- package/dist/tools/scoring/levels.d.ts +15 -0
- package/dist/tools/scoring/levels.d.ts.map +1 -0
- package/dist/tools/scoring/levels.js +21 -0
- package/package.json +18 -8
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* PR-metadata evidence adapter (P4 — evidence collectors).
|
|
3
|
+
*
|
|
4
|
+
* Maps a pull-request review/checks payload (as returned by `gh pr view --json
|
|
5
|
+
* reviewDecision,statusCheckRollup,mergeable,number,url,additions,deletions`)
|
|
6
|
+
* into an `EvidenceItem` for `computeReleaseConfidence`, using the
|
|
7
|
+
* `deploy-metadata` source kind — the closest P3-reserved kind for
|
|
8
|
+
* PR-level ship-readiness.
|
|
9
|
+
*
|
|
10
|
+
* Design:
|
|
11
|
+
* - Pure function. The caller fetches the `gh` JSON; this adapter scores it.
|
|
12
|
+
* - Applicability:
|
|
13
|
+
* `applicable` — a PR exists and review/check state is readable
|
|
14
|
+
* `not_applicable` — no PR for this ref (direct push, script, etc.)
|
|
15
|
+
* `unknown` — PR exists but checks are still pending / state ambiguous
|
|
16
|
+
* - Score formula (0..100):
|
|
17
|
+
* Base 60 points: PR is open and merge-ready (no conflicts)
|
|
18
|
+
* +20: reviewDecision === 'APPROVED'
|
|
19
|
+
* +20: all status checks pass (statusCheckRollup every entry state === 'SUCCESS')
|
|
20
|
+
* Deductions:
|
|
21
|
+
* -10: any failing status check (per failing check, capped at 20)
|
|
22
|
+
* -15: reviewDecision === 'CHANGES_REQUESTED'
|
|
23
|
+
* - The adapter NEVER fabricates a PR number or URL; if absent from the payload
|
|
24
|
+
* the evidence strings omit them (WAVE-GUARDRAILS: no fabricated data).
|
|
25
|
+
*/
|
|
26
|
+
const PR_META_WEIGHT = 0.07; // conservative; real weight rebalanced by the aggregator
|
|
27
|
+
/**
|
|
28
|
+
* Produce a `deploy-metadata` EvidenceItem from a PR metadata payload.
|
|
29
|
+
* Returns `not_applicable` when `noPr` is true, `unknown` when checks are
|
|
30
|
+
* pending or state is ambiguous, and `applicable` with a real score otherwise.
|
|
31
|
+
*/
|
|
32
|
+
export function prMetadataToEvidence(input, collectedAt) {
|
|
33
|
+
const now = collectedAt ?? input.collectedAt ?? new Date().toISOString();
|
|
34
|
+
const source = 'deploy-metadata';
|
|
35
|
+
const weight = PR_META_WEIGHT;
|
|
36
|
+
// Explicit no-PR case.
|
|
37
|
+
if (input.noPr) {
|
|
38
|
+
return {
|
|
39
|
+
source,
|
|
40
|
+
score: 0,
|
|
41
|
+
weight,
|
|
42
|
+
applicability: 'not_applicable',
|
|
43
|
+
blocking: false,
|
|
44
|
+
evidence: ['No pull request exists for this ref (direct push or pre-PR state).'],
|
|
45
|
+
recommendations: [],
|
|
46
|
+
reason: 'No PR for this ref — PR review and check signal not applicable.',
|
|
47
|
+
collectedAt: now,
|
|
48
|
+
collector: { tool: 'qulib.pr-metadata-adapter' },
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
const prLabel = input.number != null ? `PR #${input.number}` : 'PR';
|
|
52
|
+
const prRef = input.url ?? null;
|
|
53
|
+
// Checks-pending / UNKNOWN mergeable → unknown applicability (cannot score honestly).
|
|
54
|
+
const allChecks = input.statusCheckRollup ?? [];
|
|
55
|
+
const pending = allChecks.filter((c) => c.state === 'PENDING');
|
|
56
|
+
if ((pending.length > 0 && allChecks.length > 0 && pending.length === allChecks.length) ||
|
|
57
|
+
input.mergeable === 'UNKNOWN') {
|
|
58
|
+
const evidence = [
|
|
59
|
+
`${prLabel}: status checks still pending (${pending.length}/${allChecks.length}).`,
|
|
60
|
+
...(prRef ? [`${prRef}`] : []),
|
|
61
|
+
];
|
|
62
|
+
return {
|
|
63
|
+
source,
|
|
64
|
+
score: 0,
|
|
65
|
+
weight,
|
|
66
|
+
applicability: 'unknown',
|
|
67
|
+
blocking: false,
|
|
68
|
+
evidence,
|
|
69
|
+
recommendations: ['Wait for all status checks to complete before shipping.'],
|
|
70
|
+
reason: `${prLabel} checks are still pending — cannot score PR readiness honestly.`,
|
|
71
|
+
collectedAt: now,
|
|
72
|
+
collector: { tool: 'qulib.pr-metadata-adapter', inputRef: prRef ?? undefined },
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
// Score computation.
|
|
76
|
+
let score = 60; // base: a PR exists and is evaluable
|
|
77
|
+
// Mergeability.
|
|
78
|
+
if (input.mergeable === 'CONFLICTING') {
|
|
79
|
+
score -= 30; // merge conflicts are a hard deduction
|
|
80
|
+
}
|
|
81
|
+
// Review decision.
|
|
82
|
+
const rd = input.reviewDecision;
|
|
83
|
+
if (rd === 'APPROVED') {
|
|
84
|
+
score += 20;
|
|
85
|
+
}
|
|
86
|
+
else if (rd === 'CHANGES_REQUESTED') {
|
|
87
|
+
score -= 15;
|
|
88
|
+
}
|
|
89
|
+
// REVIEW_REQUIRED or null/undefined: no bonus, no deduction (reviewer not yet assigned)
|
|
90
|
+
// Status checks.
|
|
91
|
+
const passing = allChecks.filter((c) => c.state === 'SUCCESS' || c.state === 'NEUTRAL' || c.state === 'SKIPPED');
|
|
92
|
+
const failing = allChecks.filter((c) => c.state === 'FAILURE' || c.state === 'ERROR');
|
|
93
|
+
if (allChecks.length > 0 && failing.length === 0) {
|
|
94
|
+
score += 20; // all checks green
|
|
95
|
+
}
|
|
96
|
+
// Deduct per failing check (capped at −20).
|
|
97
|
+
score -= Math.min(20, failing.length * 10);
|
|
98
|
+
score = Math.max(0, Math.min(100, score));
|
|
99
|
+
// Build evidence strings — never fabricated.
|
|
100
|
+
const evidence = [];
|
|
101
|
+
const reviewStr = rd === 'APPROVED'
|
|
102
|
+
? 'APPROVED'
|
|
103
|
+
: rd === 'CHANGES_REQUESTED'
|
|
104
|
+
? 'CHANGES_REQUESTED'
|
|
105
|
+
: rd === 'REVIEW_REQUIRED'
|
|
106
|
+
? 'REVIEW_REQUIRED'
|
|
107
|
+
: 'no review yet';
|
|
108
|
+
const mergeStr = input.mergeable === 'MERGEABLE'
|
|
109
|
+
? 'mergeable'
|
|
110
|
+
: input.mergeable === 'CONFLICTING'
|
|
111
|
+
? 'CONFLICTING (merge conflicts)'
|
|
112
|
+
: 'merge state unknown';
|
|
113
|
+
evidence.push(`${prLabel}: review=${reviewStr}, ${mergeStr}.`);
|
|
114
|
+
if (allChecks.length > 0) {
|
|
115
|
+
evidence.push(`Status checks: ${passing.length} passed, ${failing.length} failed, ${pending.length} pending of ${allChecks.length} total.`);
|
|
116
|
+
for (const f of failing.slice(0, 3)) {
|
|
117
|
+
evidence.push(` Check FAILED: ${f.name ?? 'unnamed'}${f.targetUrl ? ` (${f.targetUrl})` : ''}`);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
else {
|
|
121
|
+
evidence.push('No status checks configured for this PR.');
|
|
122
|
+
}
|
|
123
|
+
if (prRef)
|
|
124
|
+
evidence.push(prRef);
|
|
125
|
+
// Recommendations.
|
|
126
|
+
const recommendations = [];
|
|
127
|
+
if (rd === 'CHANGES_REQUESTED')
|
|
128
|
+
recommendations.push('Address review comments before shipping.');
|
|
129
|
+
if (rd === 'REVIEW_REQUIRED' || rd == null)
|
|
130
|
+
recommendations.push('Request and obtain PR approval before shipping.');
|
|
131
|
+
if (failing.length > 0)
|
|
132
|
+
recommendations.push(`Fix ${failing.length} failing status check(s) before merging.`);
|
|
133
|
+
if (input.mergeable === 'CONFLICTING')
|
|
134
|
+
recommendations.push('Resolve merge conflicts before shipping.');
|
|
135
|
+
return {
|
|
136
|
+
source,
|
|
137
|
+
score,
|
|
138
|
+
weight,
|
|
139
|
+
applicability: 'applicable',
|
|
140
|
+
blocking: false,
|
|
141
|
+
evidence,
|
|
142
|
+
recommendations,
|
|
143
|
+
collectedAt: now,
|
|
144
|
+
collector: { tool: 'qulib.pr-metadata-adapter', inputRef: prRef ?? undefined },
|
|
145
|
+
};
|
|
146
|
+
}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import type { GeneratedTest } from '../schemas/gap-analysis.schema.js';
|
|
2
|
+
/**
|
|
3
|
+
* Per-spec validation outcome from the scaffold dry-run.
|
|
4
|
+
*
|
|
5
|
+
* `valid: false` means the TypeScript compiler reported one or more *syntactic*
|
|
6
|
+
* errors when transpiling the generated spec as a standalone module — i.e. the
|
|
7
|
+
* generator emitted code that will not parse, so it would fail the moment a
|
|
8
|
+
* developer ran the test suite. `errors` carries the flattened compiler
|
|
9
|
+
* messages so the failure is actionable, not just a boolean.
|
|
10
|
+
*/
|
|
11
|
+
export interface SpecValidation {
|
|
12
|
+
scenarioId: string;
|
|
13
|
+
filename: string;
|
|
14
|
+
outputPath: string;
|
|
15
|
+
valid: boolean;
|
|
16
|
+
errors: string[];
|
|
17
|
+
}
|
|
18
|
+
/** Aggregate result of validating every generated spec in a scaffold run. */
|
|
19
|
+
export interface SpecValidationReport {
|
|
20
|
+
ok: boolean;
|
|
21
|
+
total: number;
|
|
22
|
+
invalidCount: number;
|
|
23
|
+
results: SpecValidation[];
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Dry-run a single generated spec through the TypeScript compiler.
|
|
27
|
+
*
|
|
28
|
+
* `transpileModule` performs single-file syntax transformation only — no type
|
|
29
|
+
* checking, no module resolution — so it is the right tool to answer the one
|
|
30
|
+
* question the scaffold must not get wrong: *does the string we are about to
|
|
31
|
+
* write to disk actually parse as TypeScript?* A clean run (zero error-category
|
|
32
|
+
* diagnostics AND non-empty output) means the spec is syntactically valid; any
|
|
33
|
+
* error diagnostic means the generator produced broken code.
|
|
34
|
+
*
|
|
35
|
+
* We deliberately do NOT resolve `@playwright/test` / Cypress globals here:
|
|
36
|
+
* those are type-level concerns that require the consumer's node_modules. The
|
|
37
|
+
* gap this closes is *parse/compile-shape*, which is generator-local and cheap
|
|
38
|
+
* to check at scaffold time.
|
|
39
|
+
*/
|
|
40
|
+
export declare function validateSpecCode(code: string): {
|
|
41
|
+
valid: boolean;
|
|
42
|
+
errors: string[];
|
|
43
|
+
};
|
|
44
|
+
/** Validate one GeneratedTest, returning a structured per-spec outcome. */
|
|
45
|
+
export declare function validateGeneratedTest(test: GeneratedTest): SpecValidation;
|
|
46
|
+
/**
|
|
47
|
+
* Validate every generated spec in a scaffold run.
|
|
48
|
+
*
|
|
49
|
+
* `ok` is true only when *all* specs parse. This is the gate the CLI consults
|
|
50
|
+
* to decide its exit code: a scaffold that writes a spec which cannot parse is a
|
|
51
|
+
* silent correctness failure, and the whole point of the dry-run is to catch it
|
|
52
|
+
* before the developer does.
|
|
53
|
+
*/
|
|
54
|
+
export declare function validateGeneratedTests(tests: GeneratedTest[]): SpecValidationReport;
|
|
55
|
+
//# sourceMappingURL=validate-specs.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"validate-specs.d.ts","sourceRoot":"","sources":["../../src/adapters/validate-specs.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,mCAAmC,CAAC;AAEvE;;;;;;;;GAQG;AACH,MAAM,WAAW,cAAc;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,6EAA6E;AAC7E,MAAM,WAAW,oBAAoB;IACnC,EAAE,EAAE,OAAO,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,cAAc,EAAE,CAAC;CAC3B;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG;IAAE,KAAK,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,MAAM,EAAE,CAAA;CAAE,CA2BnF;AAED,2EAA2E;AAC3E,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,aAAa,GAAG,cAAc,CASzE;AAED;;;;;;;GAOG;AACH,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,aAAa,EAAE,GAAG,oBAAoB,CASnF"}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import ts from 'typescript';
|
|
2
|
+
/**
|
|
3
|
+
* Dry-run a single generated spec through the TypeScript compiler.
|
|
4
|
+
*
|
|
5
|
+
* `transpileModule` performs single-file syntax transformation only — no type
|
|
6
|
+
* checking, no module resolution — so it is the right tool to answer the one
|
|
7
|
+
* question the scaffold must not get wrong: *does the string we are about to
|
|
8
|
+
* write to disk actually parse as TypeScript?* A clean run (zero error-category
|
|
9
|
+
* diagnostics AND non-empty output) means the spec is syntactically valid; any
|
|
10
|
+
* error diagnostic means the generator produced broken code.
|
|
11
|
+
*
|
|
12
|
+
* We deliberately do NOT resolve `@playwright/test` / Cypress globals here:
|
|
13
|
+
* those are type-level concerns that require the consumer's node_modules. The
|
|
14
|
+
* gap this closes is *parse/compile-shape*, which is generator-local and cheap
|
|
15
|
+
* to check at scaffold time.
|
|
16
|
+
*/
|
|
17
|
+
export function validateSpecCode(code) {
|
|
18
|
+
const result = ts.transpileModule(code, {
|
|
19
|
+
compilerOptions: {
|
|
20
|
+
module: ts.ModuleKind.ESNext,
|
|
21
|
+
target: ts.ScriptTarget.ES2020,
|
|
22
|
+
// isolatedModules keeps this a pure syntax pass and surfaces
|
|
23
|
+
// single-file-illegal constructs, matching how a bundler would see it.
|
|
24
|
+
isolatedModules: true,
|
|
25
|
+
},
|
|
26
|
+
reportDiagnostics: true,
|
|
27
|
+
});
|
|
28
|
+
const errorDiagnostics = (result.diagnostics ?? []).filter((d) => d.category === ts.DiagnosticCategory.Error);
|
|
29
|
+
const errors = errorDiagnostics.map((d) => ts.flattenDiagnosticMessageText(d.messageText, '\n'));
|
|
30
|
+
// A clean transpile yields output text. Empty output with no diagnostics would
|
|
31
|
+
// be suspicious, so treat it as invalid too rather than silently passing.
|
|
32
|
+
const producedOutput = result.outputText.trim().length > 0;
|
|
33
|
+
const valid = errors.length === 0 && producedOutput;
|
|
34
|
+
if (!valid && errors.length === 0) {
|
|
35
|
+
errors.push('transpile produced no output for a non-empty spec');
|
|
36
|
+
}
|
|
37
|
+
return { valid, errors };
|
|
38
|
+
}
|
|
39
|
+
/** Validate one GeneratedTest, returning a structured per-spec outcome. */
|
|
40
|
+
export function validateGeneratedTest(test) {
|
|
41
|
+
const { valid, errors } = validateSpecCode(test.code);
|
|
42
|
+
return {
|
|
43
|
+
scenarioId: test.scenarioId,
|
|
44
|
+
filename: test.filename,
|
|
45
|
+
outputPath: test.outputPath,
|
|
46
|
+
valid,
|
|
47
|
+
errors,
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Validate every generated spec in a scaffold run.
|
|
52
|
+
*
|
|
53
|
+
* `ok` is true only when *all* specs parse. This is the gate the CLI consults
|
|
54
|
+
* to decide its exit code: a scaffold that writes a spec which cannot parse is a
|
|
55
|
+
* silent correctness failure, and the whole point of the dry-run is to catch it
|
|
56
|
+
* before the developer does.
|
|
57
|
+
*/
|
|
58
|
+
export function validateGeneratedTests(tests) {
|
|
59
|
+
const results = tests.map(validateGeneratedTest);
|
|
60
|
+
const invalidCount = results.filter((r) => !r.valid).length;
|
|
61
|
+
return {
|
|
62
|
+
ok: invalidCount === 0,
|
|
63
|
+
total: results.length,
|
|
64
|
+
invalidCount,
|
|
65
|
+
results,
|
|
66
|
+
};
|
|
67
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import type { GapAnalysis } from '../schemas/gap-analysis.schema.js';
|
|
2
|
+
import { type BaselineSnapshot, type BaselineDelta } from './baseline.schema.js';
|
|
3
|
+
/**
|
|
4
|
+
* Produce a filesystem-safe slug from a URL.
|
|
5
|
+
* e.g. "https://my-app.vercel.app/admin" → "my-app_vercel_app__admin"
|
|
6
|
+
*/
|
|
7
|
+
export declare function slugifyUrl(url: string): string;
|
|
8
|
+
/**
|
|
9
|
+
* Return the default root for baseline storage: `<cwd>/.qulib-baselines`.
|
|
10
|
+
* Callers may supply an explicit `baseDir` to override.
|
|
11
|
+
*/
|
|
12
|
+
export declare function defaultBaselineRoot(): string;
|
|
13
|
+
/**
|
|
14
|
+
* Save a baseline snapshot derived from the given `GapAnalysis` result.
|
|
15
|
+
*
|
|
16
|
+
* @returns The saved snapshot.
|
|
17
|
+
*/
|
|
18
|
+
export declare function saveBaseline(analysis: GapAnalysis, url: string, options?: {
|
|
19
|
+
baseDir?: string;
|
|
20
|
+
label?: string;
|
|
21
|
+
}): Promise<BaselineSnapshot>;
|
|
22
|
+
/**
|
|
23
|
+
* Load a specific baseline snapshot by its `id`.
|
|
24
|
+
*
|
|
25
|
+
* @throws If the file does not exist or fails schema validation.
|
|
26
|
+
*/
|
|
27
|
+
export declare function loadBaseline(id: string, options?: {
|
|
28
|
+
baseDir?: string;
|
|
29
|
+
}): Promise<BaselineSnapshot>;
|
|
30
|
+
/**
|
|
31
|
+
* List all saved baselines for the given URL, sorted newest-first.
|
|
32
|
+
*
|
|
33
|
+
* Returns an empty array if no baselines exist yet.
|
|
34
|
+
*/
|
|
35
|
+
export declare function listBaselines(url: string, options?: {
|
|
36
|
+
baseDir?: string;
|
|
37
|
+
}): Promise<BaselineSnapshot[]>;
|
|
38
|
+
/**
|
|
39
|
+
* Delete a specific baseline snapshot by its `id`.
|
|
40
|
+
*
|
|
41
|
+
* @throws If the file does not exist.
|
|
42
|
+
*/
|
|
43
|
+
export declare function deleteBaseline(id: string, options?: {
|
|
44
|
+
baseDir?: string;
|
|
45
|
+
}): Promise<void>;
|
|
46
|
+
/**
|
|
47
|
+
* Compare two baseline snapshots and return a structured delta report.
|
|
48
|
+
*
|
|
49
|
+
* - `newGaps`: problems present in `current` but not in `prior`.
|
|
50
|
+
* - `resolvedGaps`: problems present in `prior` but no longer in `current`.
|
|
51
|
+
* - `severityChanges`: same problem (matching key) with a different severity.
|
|
52
|
+
*/
|
|
53
|
+
export declare function compareBaselines(prior: BaselineSnapshot, current: BaselineSnapshot): BaselineDelta;
|
|
54
|
+
//# sourceMappingURL=baseline.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"baseline.d.ts","sourceRoot":"","sources":["../../src/baseline/baseline.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,WAAW,EAAO,MAAM,mCAAmC,CAAC;AAC1E,OAAO,EAGL,KAAK,gBAAgB,EACrB,KAAK,aAAa,EAEnB,MAAM,sBAAsB,CAAC;AAY9B;;;GAGG;AACH,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAO9C;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,IAAI,MAAM,CAE5C;AAeD;;;;GAIG;AACH,wBAAsB,YAAY,CAChC,QAAQ,EAAE,WAAW,EACrB,GAAG,EAAE,MAAM,EACX,OAAO,GAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAO,GACjD,OAAO,CAAC,gBAAgB,CAAC,CA0B3B;AAED;;;;GAIG;AACH,wBAAsB,YAAY,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,GAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAuB5G;AAED;;;;GAIG;AACH,wBAAsB,aAAa,CACjC,GAAG,EAAE,MAAM,EACX,OAAO,GAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAA;CAAO,GACjC,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAoC7B;AAED;;;;GAIG;AACH,wBAAsB,cAAc,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,GAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAclG;AAcD;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,gBAAgB,EAAE,OAAO,EAAE,gBAAgB,GAAG,aAAa,CAiFlG"}
|
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
import { readFile, writeFile, mkdir, readdir, unlink } from 'node:fs/promises';
|
|
2
|
+
import { existsSync } from 'node:fs';
|
|
3
|
+
import { join, extname } from 'node:path';
|
|
4
|
+
import { BaselineSnapshotSchema, BaselineDeltaSchema, } from './baseline.schema.js';
|
|
5
|
+
const BASELINE_DIR_NAME = '.qulib-baselines';
|
|
6
|
+
/**
|
|
7
|
+
* Resolve the directory where baselines for the given URL are stored.
|
|
8
|
+
* Each URL gets its own subdirectory under baseDir, keyed by a stable slug.
|
|
9
|
+
*/
|
|
10
|
+
function resolveBaselineDir(baseDir, urlSlug) {
|
|
11
|
+
return join(baseDir, urlSlug);
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Produce a filesystem-safe slug from a URL.
|
|
15
|
+
* e.g. "https://my-app.vercel.app/admin" → "my-app_vercel_app__admin"
|
|
16
|
+
*/
|
|
17
|
+
export function slugifyUrl(url) {
|
|
18
|
+
return url
|
|
19
|
+
.replace(/^https?:\/\//, '')
|
|
20
|
+
.replace(/[^a-zA-Z0-9._-]/g, '_')
|
|
21
|
+
.replace(/__+/g, '_')
|
|
22
|
+
.replace(/^_|_$/g, '')
|
|
23
|
+
.slice(0, 80);
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Return the default root for baseline storage: `<cwd>/.qulib-baselines`.
|
|
27
|
+
* Callers may supply an explicit `baseDir` to override.
|
|
28
|
+
*/
|
|
29
|
+
export function defaultBaselineRoot() {
|
|
30
|
+
return join(process.cwd(), BASELINE_DIR_NAME);
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Convert a `GapAnalysis` result into the compact `BaselineGap[]` shape.
|
|
34
|
+
* Drops fields not needed for delta comparison (id, description, recommendation).
|
|
35
|
+
*/
|
|
36
|
+
function toBaselineGaps(gaps) {
|
|
37
|
+
return gaps.map((g) => ({
|
|
38
|
+
path: g.path,
|
|
39
|
+
severity: g.severity,
|
|
40
|
+
category: g.category,
|
|
41
|
+
reason: g.reason,
|
|
42
|
+
}));
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Save a baseline snapshot derived from the given `GapAnalysis` result.
|
|
46
|
+
*
|
|
47
|
+
* @returns The saved snapshot.
|
|
48
|
+
*/
|
|
49
|
+
export async function saveBaseline(analysis, url, options = {}) {
|
|
50
|
+
const baseDir = options.baseDir ?? defaultBaselineRoot();
|
|
51
|
+
const urlSlug = slugifyUrl(url);
|
|
52
|
+
const dir = resolveBaselineDir(baseDir, urlSlug);
|
|
53
|
+
await mkdir(dir, { recursive: true });
|
|
54
|
+
const now = new Date();
|
|
55
|
+
// Include milliseconds (23 chars: "2024-01-01T00-00-00-000") so rapid successive saves
|
|
56
|
+
// do not collide on the same filename within the same second.
|
|
57
|
+
const timestamp = now.toISOString().replace(/[:.]/g, '-').slice(0, 23);
|
|
58
|
+
const id = `${urlSlug}__${timestamp}`;
|
|
59
|
+
const filename = `${id}.json`;
|
|
60
|
+
const snapshot = {
|
|
61
|
+
id,
|
|
62
|
+
url,
|
|
63
|
+
savedAt: now.toISOString(),
|
|
64
|
+
releaseConfidence: analysis.releaseConfidence ?? 0,
|
|
65
|
+
gapCount: analysis.gaps.length,
|
|
66
|
+
gaps: toBaselineGaps(analysis.gaps),
|
|
67
|
+
...(options.label !== undefined ? { label: options.label } : {}),
|
|
68
|
+
};
|
|
69
|
+
const validated = BaselineSnapshotSchema.parse(snapshot);
|
|
70
|
+
await writeFile(join(dir, filename), JSON.stringify(validated, null, 2), 'utf8');
|
|
71
|
+
return validated;
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Load a specific baseline snapshot by its `id`.
|
|
75
|
+
*
|
|
76
|
+
* @throws If the file does not exist or fails schema validation.
|
|
77
|
+
*/
|
|
78
|
+
export async function loadBaseline(id, options = {}) {
|
|
79
|
+
const baseDir = options.baseDir ?? defaultBaselineRoot();
|
|
80
|
+
// id encodes the urlSlug: <urlSlug>__<timestamp>
|
|
81
|
+
const doubleUnderIndex = id.lastIndexOf('__');
|
|
82
|
+
if (doubleUnderIndex < 0) {
|
|
83
|
+
throw new Error(`Invalid baseline id (no __ separator): ${id}`);
|
|
84
|
+
}
|
|
85
|
+
const urlSlug = id.slice(0, doubleUnderIndex);
|
|
86
|
+
const dir = resolveBaselineDir(baseDir, urlSlug);
|
|
87
|
+
const filepath = join(dir, `${id}.json`);
|
|
88
|
+
if (!existsSync(filepath)) {
|
|
89
|
+
throw new Error(`Baseline not found: ${id} (${filepath})`);
|
|
90
|
+
}
|
|
91
|
+
const raw = await readFile(filepath, 'utf8');
|
|
92
|
+
let parsed;
|
|
93
|
+
try {
|
|
94
|
+
parsed = JSON.parse(raw);
|
|
95
|
+
}
|
|
96
|
+
catch {
|
|
97
|
+
throw new Error(`Baseline file is not valid JSON: ${filepath}`);
|
|
98
|
+
}
|
|
99
|
+
return BaselineSnapshotSchema.parse(parsed);
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* List all saved baselines for the given URL, sorted newest-first.
|
|
103
|
+
*
|
|
104
|
+
* Returns an empty array if no baselines exist yet.
|
|
105
|
+
*/
|
|
106
|
+
export async function listBaselines(url, options = {}) {
|
|
107
|
+
const baseDir = options.baseDir ?? defaultBaselineRoot();
|
|
108
|
+
const urlSlug = slugifyUrl(url);
|
|
109
|
+
const dir = resolveBaselineDir(baseDir, urlSlug);
|
|
110
|
+
if (!existsSync(dir)) {
|
|
111
|
+
return [];
|
|
112
|
+
}
|
|
113
|
+
let entries;
|
|
114
|
+
try {
|
|
115
|
+
entries = await readdir(dir);
|
|
116
|
+
}
|
|
117
|
+
catch {
|
|
118
|
+
return [];
|
|
119
|
+
}
|
|
120
|
+
const snapshots = [];
|
|
121
|
+
for (const entry of entries) {
|
|
122
|
+
if (extname(entry) !== '.json')
|
|
123
|
+
continue;
|
|
124
|
+
const filepath = join(dir, entry);
|
|
125
|
+
const raw = await readFile(filepath, 'utf8');
|
|
126
|
+
let parsed;
|
|
127
|
+
try {
|
|
128
|
+
parsed = JSON.parse(raw);
|
|
129
|
+
}
|
|
130
|
+
catch {
|
|
131
|
+
continue;
|
|
132
|
+
}
|
|
133
|
+
const result = BaselineSnapshotSchema.safeParse(parsed);
|
|
134
|
+
if (result.success) {
|
|
135
|
+
snapshots.push(result.data);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
// Newest first
|
|
139
|
+
snapshots.sort((a, b) => new Date(b.savedAt).getTime() - new Date(a.savedAt).getTime());
|
|
140
|
+
return snapshots;
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* Delete a specific baseline snapshot by its `id`.
|
|
144
|
+
*
|
|
145
|
+
* @throws If the file does not exist.
|
|
146
|
+
*/
|
|
147
|
+
export async function deleteBaseline(id, options = {}) {
|
|
148
|
+
const baseDir = options.baseDir ?? defaultBaselineRoot();
|
|
149
|
+
const doubleUnderIndex = id.lastIndexOf('__');
|
|
150
|
+
if (doubleUnderIndex < 0) {
|
|
151
|
+
throw new Error(`Invalid baseline id (no __ separator): ${id}`);
|
|
152
|
+
}
|
|
153
|
+
const urlSlug = id.slice(0, doubleUnderIndex);
|
|
154
|
+
const dir = resolveBaselineDir(baseDir, urlSlug);
|
|
155
|
+
const filepath = join(dir, `${id}.json`);
|
|
156
|
+
if (!existsSync(filepath)) {
|
|
157
|
+
throw new Error(`Baseline not found: ${id}`);
|
|
158
|
+
}
|
|
159
|
+
await unlink(filepath);
|
|
160
|
+
}
|
|
161
|
+
// ---------------------------------------------------------------------------
|
|
162
|
+
// Delta detection
|
|
163
|
+
// ---------------------------------------------------------------------------
|
|
164
|
+
/**
|
|
165
|
+
* Stable key used to match gaps across snapshots for delta detection.
|
|
166
|
+
* Two gaps are "the same problem" when they share path + category.
|
|
167
|
+
*/
|
|
168
|
+
function gapKey(g) {
|
|
169
|
+
return `${g.path}|||${g.category}`;
|
|
170
|
+
}
|
|
171
|
+
/**
|
|
172
|
+
* Compare two baseline snapshots and return a structured delta report.
|
|
173
|
+
*
|
|
174
|
+
* - `newGaps`: problems present in `current` but not in `prior`.
|
|
175
|
+
* - `resolvedGaps`: problems present in `prior` but no longer in `current`.
|
|
176
|
+
* - `severityChanges`: same problem (matching key) with a different severity.
|
|
177
|
+
*/
|
|
178
|
+
export function compareBaselines(prior, current) {
|
|
179
|
+
const priorMap = new Map();
|
|
180
|
+
for (const g of prior.gaps) {
|
|
181
|
+
priorMap.set(gapKey(g), g);
|
|
182
|
+
}
|
|
183
|
+
const currentMap = new Map();
|
|
184
|
+
for (const g of current.gaps) {
|
|
185
|
+
currentMap.set(gapKey(g), g);
|
|
186
|
+
}
|
|
187
|
+
const severityOrder = {
|
|
188
|
+
critical: 4,
|
|
189
|
+
high: 3,
|
|
190
|
+
medium: 2,
|
|
191
|
+
low: 1,
|
|
192
|
+
};
|
|
193
|
+
const newGaps = [];
|
|
194
|
+
const resolvedGaps = [];
|
|
195
|
+
const severityChanges = [];
|
|
196
|
+
// Gaps in current that were not in prior → new
|
|
197
|
+
for (const [key, g] of currentMap) {
|
|
198
|
+
if (!priorMap.has(key)) {
|
|
199
|
+
newGaps.push({ path: g.path, category: g.category, severity: g.severity, reason: g.reason, status: 'new' });
|
|
200
|
+
}
|
|
201
|
+
else {
|
|
202
|
+
const prev = priorMap.get(key);
|
|
203
|
+
if (prev.severity !== g.severity) {
|
|
204
|
+
const prevOrd = severityOrder[prev.severity];
|
|
205
|
+
const currOrd = severityOrder[g.severity];
|
|
206
|
+
severityChanges.push({
|
|
207
|
+
path: g.path,
|
|
208
|
+
category: g.category,
|
|
209
|
+
severity: g.severity,
|
|
210
|
+
reason: g.reason,
|
|
211
|
+
status: currOrd > prevOrd ? 'severity-increased' : 'severity-decreased',
|
|
212
|
+
});
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
// Gaps in prior that are no longer in current → resolved
|
|
217
|
+
for (const [key, g] of priorMap) {
|
|
218
|
+
if (!currentMap.has(key)) {
|
|
219
|
+
resolvedGaps.push({
|
|
220
|
+
path: g.path,
|
|
221
|
+
category: g.category,
|
|
222
|
+
severity: g.severity,
|
|
223
|
+
reason: g.reason,
|
|
224
|
+
status: 'resolved',
|
|
225
|
+
});
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
const confidenceDelta = current.releaseConfidence - prior.releaseConfidence;
|
|
229
|
+
const direction = confidenceDelta > 0 ? 'improved' : confidenceDelta < 0 ? 'regressed' : 'unchanged';
|
|
230
|
+
const summary = [
|
|
231
|
+
`Confidence ${direction} (${prior.releaseConfidence} → ${current.releaseConfidence})`,
|
|
232
|
+
newGaps.length > 0 ? `${newGaps.length} new gap(s)` : '',
|
|
233
|
+
resolvedGaps.length > 0 ? `${resolvedGaps.length} resolved gap(s)` : '',
|
|
234
|
+
severityChanges.length > 0 ? `${severityChanges.length} severity change(s)` : '',
|
|
235
|
+
]
|
|
236
|
+
.filter(Boolean)
|
|
237
|
+
.join(', ');
|
|
238
|
+
const delta = {
|
|
239
|
+
fromId: prior.id,
|
|
240
|
+
toId: current.id,
|
|
241
|
+
fromSavedAt: prior.savedAt,
|
|
242
|
+
toSavedAt: current.savedAt,
|
|
243
|
+
fromReleaseConfidence: prior.releaseConfidence,
|
|
244
|
+
toReleaseConfidence: current.releaseConfidence,
|
|
245
|
+
confidenceDelta,
|
|
246
|
+
newGaps,
|
|
247
|
+
resolvedGaps,
|
|
248
|
+
severityChanges,
|
|
249
|
+
summary,
|
|
250
|
+
};
|
|
251
|
+
return BaselineDeltaSchema.parse(delta);
|
|
252
|
+
}
|