@qulib/core 0.9.0 → 0.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -11
- package/dist/baseline/baseline.schema.d.ts +26 -26
- package/dist/baseline/baseline.schema.d.ts.map +1 -1
- package/dist/baseline/baseline.schema.js +1 -0
- package/dist/cli/analyze-diff-run.d.ts +77 -0
- package/dist/cli/analyze-diff-run.d.ts.map +1 -0
- package/dist/cli/analyze-diff-run.js +266 -0
- package/dist/cli/baseline-run.d.ts +55 -0
- package/dist/cli/baseline-run.d.ts.map +1 -0
- package/dist/cli/baseline-run.js +259 -0
- package/dist/cli/confidence-run.d.ts.map +1 -1
- package/dist/cli/confidence-run.js +10 -6
- package/dist/cli/index.js +4 -0
- package/dist/cli/score-automation-run.d.ts.map +1 -1
- package/dist/cli/score-automation-run.js +5 -1
- package/dist/index.d.ts +5 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -0
- package/dist/phases/think.d.ts.map +1 -1
- package/dist/phases/think.js +4 -1
- package/dist/reporters/heatmap.d.ts +55 -0
- package/dist/reporters/heatmap.d.ts.map +1 -0
- package/dist/reporters/heatmap.js +148 -0
- package/dist/reporters/markdown-reporter.d.ts.map +1 -1
- package/dist/reporters/markdown-reporter.js +4 -1
- package/dist/schemas/confidence.schema.d.ts +2 -2
- package/dist/schemas/config.schema.d.ts.map +1 -1
- package/dist/schemas/config.schema.js +6 -1
- package/dist/schemas/gap-analysis.schema.d.ts +8 -8
- package/dist/schemas/gap-analysis.schema.js +1 -1
- package/dist/schemas/golden-manifest.schema.d.ts +137 -0
- package/dist/schemas/golden-manifest.schema.d.ts.map +1 -0
- package/dist/schemas/golden-manifest.schema.js +25 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.d.ts.map +1 -1
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/public-surface.schema.d.ts +15 -5
- package/dist/schemas/public-surface.schema.d.ts.map +1 -1
- package/dist/schemas/route-inventory.schema.d.ts +20 -0
- package/dist/schemas/route-inventory.schema.d.ts.map +1 -1
- package/dist/schemas/route-inventory.schema.js +4 -0
- package/dist/schemas/views.schema.d.ts +1 -1
- package/dist/tools/scoring/confidence.d.ts.map +1 -1
- package/dist/tools/scoring/confidence.js +140 -14
- package/dist/tools/scoring/prompt-leakage.d.ts +29 -0
- package/dist/tools/scoring/prompt-leakage.d.ts.map +1 -0
- package/dist/tools/scoring/prompt-leakage.js +256 -0
- package/package.json +8 -4
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import type { Command } from 'commander';
|
|
2
|
+
import type { BaselineSnapshot, BaselineDelta } from '../baseline/baseline.schema.js';
|
|
3
|
+
import { type GapAnalysis } from '../schemas/gap-analysis.schema.js';
|
|
4
|
+
/**
|
|
5
|
+
* Read and validate a GapAnalysis from a report.json written by `qulib analyze`.
|
|
6
|
+
* report.json is exactly a serialized GapAnalysis (see reporters/json-reporter.ts),
|
|
7
|
+
* so we parse it through GapAnalysisSchema — fail loudly on a malformed/foreign file
|
|
8
|
+
* rather than baselining garbage.
|
|
9
|
+
*/
|
|
10
|
+
export declare function loadGapAnalysisFromReport(reportPath: string, cwd?: string): Promise<GapAnalysis>;
|
|
11
|
+
export interface BaselineSaveOptions {
|
|
12
|
+
url: string;
|
|
13
|
+
fromReport?: string;
|
|
14
|
+
label?: string;
|
|
15
|
+
dir?: string;
|
|
16
|
+
json?: boolean;
|
|
17
|
+
}
|
|
18
|
+
/** Render a saved snapshot as a short human line. */
|
|
19
|
+
export declare function formatSavedSnapshot(snap: BaselineSnapshot): string;
|
|
20
|
+
/**
|
|
21
|
+
* Core of `baseline save`, factored out so node:test can drive it without a process.
|
|
22
|
+
* Either --from-report (deterministic, reads a real on-disk report.json) or a live
|
|
23
|
+
* crawl of --url. --from-report is preferred for CI/agents that already ran analyze.
|
|
24
|
+
*/
|
|
25
|
+
export declare function runBaselineSave(options: BaselineSaveOptions, out?: (line: string) => void): Promise<BaselineSnapshot>;
|
|
26
|
+
export interface BaselineListOptions {
|
|
27
|
+
url: string;
|
|
28
|
+
dir?: string;
|
|
29
|
+
json?: boolean;
|
|
30
|
+
}
|
|
31
|
+
/** Render a list of snapshots as a human table. */
|
|
32
|
+
export declare function formatBaselineList(url: string, snaps: BaselineSnapshot[]): string;
|
|
33
|
+
/** Core of `baseline list`, factored out for direct testing. */
|
|
34
|
+
export declare function runBaselineList(options: BaselineListOptions, out?: (line: string) => void): Promise<BaselineSnapshot[]>;
|
|
35
|
+
export interface BaselineCompareOptions {
|
|
36
|
+
/** Compare two explicit baseline ids. Takes precedence over --url. */
|
|
37
|
+
from?: string;
|
|
38
|
+
to?: string;
|
|
39
|
+
/** Or: compare the two most-recent baselines for this URL. */
|
|
40
|
+
url?: string;
|
|
41
|
+
dir?: string;
|
|
42
|
+
json?: boolean;
|
|
43
|
+
}
|
|
44
|
+
/** Render a delta as a human report. */
|
|
45
|
+
export declare function formatBaselineDelta(delta: BaselineDelta): string;
|
|
46
|
+
/**
|
|
47
|
+
* Core of `baseline compare`, factored out for direct testing.
|
|
48
|
+
* Resolution order:
|
|
49
|
+
* 1. --from and --to explicit ids → load both, compare.
|
|
50
|
+
* 2. --url → take the two most-recent baselines (prior = older, current = newest).
|
|
51
|
+
* Fails clearly when fewer than two baselines are available — never invents a delta.
|
|
52
|
+
*/
|
|
53
|
+
export declare function runBaselineCompare(options: BaselineCompareOptions, out?: (line: string) => void): Promise<BaselineDelta>;
|
|
54
|
+
export declare function registerBaselineCommand(program: Command): void;
|
|
55
|
+
//# sourceMappingURL=baseline-run.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"baseline-run.d.ts","sourceRoot":"","sources":["../../src/cli/baseline-run.ts"],"names":[],"mappings":"AA4BA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAQzC,OAAO,KAAK,EAAE,gBAAgB,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AACtF,OAAO,EAAqB,KAAK,WAAW,EAAE,MAAM,mCAAmC,CAAC;AA0BxF;;;;;GAKG;AACH,wBAAsB,yBAAyB,CAC7C,UAAU,EAAE,MAAM,EAClB,GAAG,GAAE,MAAsB,GAC1B,OAAO,CAAC,WAAW,CAAC,CAsBtB;AAED,MAAM,WAAW,mBAAmB;IAClC,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAED,qDAAqD;AACrD,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,gBAAgB,GAAG,MAAM,CAQlE;AAED;;;;GAIG;AACH,wBAAsB,eAAe,CACnC,OAAO,EAAE,mBAAmB,EAC5B,GAAG,GAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAkC,GACxD,OAAO,CAAC,gBAAgB,CAAC,CA6B3B;AAED,MAAM,WAAW,mBAAmB;IAClC,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAED,mDAAmD;AACnD,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,gBAAgB,EAAE,GAAG,MAAM,CAUjF;AAED,gEAAgE;AAChE,wBAAsB,eAAe,CACnC,OAAO,EAAE,mBAAmB,EAC5B,GAAG,GAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAkC,GACxD,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAW7B;AAED,MAAM,WAAW,sBAAsB;IACrC,sEAAsE;IACtE,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,8DAA8D;IAC9D,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAED,wCAAwC;AACxC,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,aAAa,GAAG,MAAM,CAiBhE;AAED;;;;;;GAMG;AACH,wBAAsB,kBAAkB,CACtC,OAAO,EAAE,sBAAsB,EAC/B,GAAG,GAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAkC,GACxD,OAAO,CAAC,aAAa,CAAC,CAmCxB;AAED,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAwD9D"}
|
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `qulib baseline` — capture and compare quality-gap baselines over time.
|
|
3
|
+
*
|
|
4
|
+
* Surfaces the previously dead-ended baseline module (../baseline/baseline.js) as a
|
|
5
|
+
* first-class CLI. Until now save/load/list/compare existed only as a programmatic API
|
|
6
|
+
* with no consumer; this exposes them so an agent or CI can snapshot a release's gaps
|
|
7
|
+
* and detect drift (new / resolved / severity-changed gaps + confidence delta) between
|
|
8
|
+
* runs.
|
|
9
|
+
*
|
|
10
|
+
* Subcommands:
|
|
11
|
+
* baseline save — snapshot a GapAnalysis for a URL. Source the analysis either by
|
|
12
|
+
* crawling live (--url alone) or, deterministically, from an
|
|
13
|
+
* existing report.json written by `qulib analyze` (--from-report).
|
|
14
|
+
* baseline list — list saved baselines for a URL, newest-first.
|
|
15
|
+
* baseline compare — diff two baselines (explicit ids, or the two most-recent for a URL).
|
|
16
|
+
*
|
|
17
|
+
* Storage honesty (root design principle — no false confidence):
|
|
18
|
+
* Baselines persist under <cwd>/.qulib-baselines/<url-slug>/ unless --dir overrides.
|
|
19
|
+
* `compare` with fewer than two baselines fails with a clear, actionable message
|
|
20
|
+
* rather than fabricating a delta against nothing.
|
|
21
|
+
*
|
|
22
|
+
* This file owns the `baseline` command group end-to-end and is registered from
|
|
23
|
+
* cli/index.ts via `registerBaselineCommand(program)`, so this command's build never
|
|
24
|
+
* edits the body of index.ts beyond a single additive registration line (mirrors the
|
|
25
|
+
* score-automation / scaffold / confidence convention).
|
|
26
|
+
*/
|
|
27
|
+
import { resolve } from 'node:path';
|
|
28
|
+
import { readFile } from 'node:fs/promises';
|
|
29
|
+
import { z } from 'zod';
|
|
30
|
+
import { saveBaseline, listBaselines, loadBaseline, compareBaselines, } from '../baseline/baseline.js';
|
|
31
|
+
import { GapAnalysisSchema } from '../schemas/gap-analysis.schema.js';
|
|
32
|
+
import { analyzeApp } from '../analyze.js';
|
|
33
|
+
const UrlSchema = z.string().url();
|
|
34
|
+
/** Harness config used when `save --url` crawls live (no repo, read-only, no LLM). */
|
|
35
|
+
function liveCrawlConfig() {
|
|
36
|
+
return {
|
|
37
|
+
maxPagesToScan: 10,
|
|
38
|
+
maxDepth: 3,
|
|
39
|
+
minPagesForConfidence: 3,
|
|
40
|
+
timeoutMs: 30000,
|
|
41
|
+
retryCount: 0,
|
|
42
|
+
llmTokenBudget: 4096,
|
|
43
|
+
testGenerationLimit: 5,
|
|
44
|
+
enableLlmScenarios: false,
|
|
45
|
+
readOnlyMode: true,
|
|
46
|
+
requireHumanReview: false,
|
|
47
|
+
failOnConsoleError: false,
|
|
48
|
+
explorer: 'playwright',
|
|
49
|
+
defaultAdapter: 'playwright',
|
|
50
|
+
adapters: ['playwright'],
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Read and validate a GapAnalysis from a report.json written by `qulib analyze`.
|
|
55
|
+
* report.json is exactly a serialized GapAnalysis (see reporters/json-reporter.ts),
|
|
56
|
+
* so we parse it through GapAnalysisSchema — fail loudly on a malformed/foreign file
|
|
57
|
+
* rather than baselining garbage.
|
|
58
|
+
*/
|
|
59
|
+
export async function loadGapAnalysisFromReport(reportPath, cwd = process.cwd()) {
|
|
60
|
+
const abs = resolve(cwd, reportPath);
|
|
61
|
+
let raw;
|
|
62
|
+
try {
|
|
63
|
+
raw = await readFile(abs, 'utf8');
|
|
64
|
+
}
|
|
65
|
+
catch {
|
|
66
|
+
throw new Error(`--from-report path could not be read: ${abs}`);
|
|
67
|
+
}
|
|
68
|
+
let parsed;
|
|
69
|
+
try {
|
|
70
|
+
parsed = JSON.parse(raw);
|
|
71
|
+
}
|
|
72
|
+
catch {
|
|
73
|
+
throw new Error(`--from-report file is not valid JSON: ${abs}`);
|
|
74
|
+
}
|
|
75
|
+
const result = GapAnalysisSchema.safeParse(parsed);
|
|
76
|
+
if (!result.success) {
|
|
77
|
+
throw new Error(`--from-report file is not a valid qulib report.json (GapAnalysis): ${abs}\n` +
|
|
78
|
+
result.error.issues.map((i) => ` • ${i.path.join('.')}: ${i.message}`).join('\n'));
|
|
79
|
+
}
|
|
80
|
+
return result.data;
|
|
81
|
+
}
|
|
82
|
+
/** Render a saved snapshot as a short human line. */
|
|
83
|
+
export function formatSavedSnapshot(snap) {
|
|
84
|
+
const labelTag = snap.label ? ` "${snap.label}"` : '';
|
|
85
|
+
return (`[qulib] Saved baseline ${snap.id}${labelTag}\n` +
|
|
86
|
+
` url: ${snap.url}\n` +
|
|
87
|
+
` releaseConfidence: ${snap.releaseConfidence}/100\n` +
|
|
88
|
+
` gaps: ${snap.gapCount}`);
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Core of `baseline save`, factored out so node:test can drive it without a process.
|
|
92
|
+
* Either --from-report (deterministic, reads a real on-disk report.json) or a live
|
|
93
|
+
* crawl of --url. --from-report is preferred for CI/agents that already ran analyze.
|
|
94
|
+
*/
|
|
95
|
+
export async function runBaselineSave(options, out = (line) => console.log(line)) {
|
|
96
|
+
const url = UrlSchema.parse(options.url);
|
|
97
|
+
let analysis;
|
|
98
|
+
if (options.fromReport && options.fromReport.trim()) {
|
|
99
|
+
analysis = await loadGapAnalysisFromReport(options.fromReport.trim());
|
|
100
|
+
}
|
|
101
|
+
else {
|
|
102
|
+
if (!options.json) {
|
|
103
|
+
out(`[qulib] Crawling ${url} to capture a fresh baseline (analyze_app)…`);
|
|
104
|
+
}
|
|
105
|
+
const result = await analyzeApp({
|
|
106
|
+
url,
|
|
107
|
+
writeArtifacts: false,
|
|
108
|
+
config: liveCrawlConfig(),
|
|
109
|
+
});
|
|
110
|
+
analysis = result.gapAnalysis;
|
|
111
|
+
}
|
|
112
|
+
const snapshot = await saveBaseline(analysis, url, {
|
|
113
|
+
...(options.dir ? { baseDir: resolve(process.cwd(), options.dir) } : {}),
|
|
114
|
+
...(options.label !== undefined ? { label: options.label } : {}),
|
|
115
|
+
});
|
|
116
|
+
if (options.json) {
|
|
117
|
+
out(JSON.stringify(snapshot, null, 2));
|
|
118
|
+
}
|
|
119
|
+
else {
|
|
120
|
+
out(formatSavedSnapshot(snapshot));
|
|
121
|
+
}
|
|
122
|
+
return snapshot;
|
|
123
|
+
}
|
|
124
|
+
/** Render a list of snapshots as a human table. */
|
|
125
|
+
export function formatBaselineList(url, snaps) {
|
|
126
|
+
if (snaps.length === 0) {
|
|
127
|
+
return `[qulib] No baselines saved for ${url} yet. Run: qulib baseline save --url ${url} --from-report output/report.json`;
|
|
128
|
+
}
|
|
129
|
+
const lines = [`[qulib] ${snaps.length} baseline(s) for ${url} (newest first):`];
|
|
130
|
+
for (const s of snaps) {
|
|
131
|
+
const labelTag = s.label ? ` "${s.label}"` : '';
|
|
132
|
+
lines.push(` ${s.id} — confidence ${s.releaseConfidence}/100, ${s.gapCount} gap(s)${labelTag}`);
|
|
133
|
+
}
|
|
134
|
+
return lines.join('\n');
|
|
135
|
+
}
|
|
136
|
+
/** Core of `baseline list`, factored out for direct testing. */
|
|
137
|
+
export async function runBaselineList(options, out = (line) => console.log(line)) {
|
|
138
|
+
const url = UrlSchema.parse(options.url);
|
|
139
|
+
const snaps = await listBaselines(url, {
|
|
140
|
+
...(options.dir ? { baseDir: resolve(process.cwd(), options.dir) } : {}),
|
|
141
|
+
});
|
|
142
|
+
if (options.json) {
|
|
143
|
+
out(JSON.stringify(snaps, null, 2));
|
|
144
|
+
}
|
|
145
|
+
else {
|
|
146
|
+
out(formatBaselineList(url, snaps));
|
|
147
|
+
}
|
|
148
|
+
return snaps;
|
|
149
|
+
}
|
|
150
|
+
/** Render a delta as a human report. */
|
|
151
|
+
export function formatBaselineDelta(delta) {
|
|
152
|
+
const lines = [];
|
|
153
|
+
lines.push(`[qulib] Baseline comparison`);
|
|
154
|
+
lines.push(` from: ${delta.fromId} (${delta.fromReleaseConfidence}/100)`);
|
|
155
|
+
lines.push(` to: ${delta.toId} (${delta.toReleaseConfidence}/100)`);
|
|
156
|
+
lines.push(` ${delta.summary}`);
|
|
157
|
+
const section = (title, items) => {
|
|
158
|
+
if (items.length === 0)
|
|
159
|
+
return;
|
|
160
|
+
lines.push(` ${title}:`);
|
|
161
|
+
for (const g of items) {
|
|
162
|
+
lines.push(` • [${g.severity}] ${g.path} (${g.category}) — ${g.reason}`);
|
|
163
|
+
}
|
|
164
|
+
};
|
|
165
|
+
section('new gaps', delta.newGaps);
|
|
166
|
+
section('resolved gaps', delta.resolvedGaps);
|
|
167
|
+
section('severity changes', delta.severityChanges);
|
|
168
|
+
return lines.join('\n');
|
|
169
|
+
}
|
|
170
|
+
/**
|
|
171
|
+
* Core of `baseline compare`, factored out for direct testing.
|
|
172
|
+
* Resolution order:
|
|
173
|
+
* 1. --from and --to explicit ids → load both, compare.
|
|
174
|
+
* 2. --url → take the two most-recent baselines (prior = older, current = newest).
|
|
175
|
+
* Fails clearly when fewer than two baselines are available — never invents a delta.
|
|
176
|
+
*/
|
|
177
|
+
export async function runBaselineCompare(options, out = (line) => console.log(line)) {
|
|
178
|
+
const baseDirOpt = options.dir ? { baseDir: resolve(process.cwd(), options.dir) } : {};
|
|
179
|
+
let prior;
|
|
180
|
+
let current;
|
|
181
|
+
if (options.from || options.to) {
|
|
182
|
+
if (!options.from || !options.to) {
|
|
183
|
+
throw new Error('baseline compare with explicit ids requires BOTH --from and --to.');
|
|
184
|
+
}
|
|
185
|
+
prior = await loadBaseline(options.from, baseDirOpt);
|
|
186
|
+
current = await loadBaseline(options.to, baseDirOpt);
|
|
187
|
+
}
|
|
188
|
+
else if (options.url) {
|
|
189
|
+
const url = UrlSchema.parse(options.url);
|
|
190
|
+
const snaps = await listBaselines(url, baseDirOpt);
|
|
191
|
+
if (snaps.length < 2) {
|
|
192
|
+
throw new Error(`baseline compare needs at least two saved baselines for ${url}; found ${snaps.length}. ` +
|
|
193
|
+
`Save another with: qulib baseline save --url ${url} --from-report output/report.json`);
|
|
194
|
+
}
|
|
195
|
+
// listBaselines is newest-first: snaps[0] is current, snaps[1] is the prior.
|
|
196
|
+
current = snaps[0];
|
|
197
|
+
prior = snaps[1];
|
|
198
|
+
}
|
|
199
|
+
else {
|
|
200
|
+
throw new Error('baseline compare requires either --from + --to, or --url.');
|
|
201
|
+
}
|
|
202
|
+
const delta = compareBaselines(prior, current);
|
|
203
|
+
if (options.json) {
|
|
204
|
+
out(JSON.stringify(delta, null, 2));
|
|
205
|
+
}
|
|
206
|
+
else {
|
|
207
|
+
out(formatBaselineDelta(delta));
|
|
208
|
+
}
|
|
209
|
+
return delta;
|
|
210
|
+
}
|
|
211
|
+
export function registerBaselineCommand(program) {
|
|
212
|
+
const baseline = program
|
|
213
|
+
.command('baseline')
|
|
214
|
+
.description('Capture and compare quality-gap baselines over time. Snapshot a release\'s gaps, ' +
|
|
215
|
+
'then diff later runs to surface new / resolved / severity-changed gaps and confidence drift.');
|
|
216
|
+
baseline
|
|
217
|
+
.command('save')
|
|
218
|
+
.description('Save a baseline snapshot for a URL (from a report.json or a fresh live crawl)')
|
|
219
|
+
.requiredOption('--url <url>', 'URL the baseline is keyed to')
|
|
220
|
+
.option('--from-report <path>', 'Path to a report.json from `qulib analyze` (deterministic, no network). If omitted, the URL is crawled live.')
|
|
221
|
+
.option('--label <label>', 'Optional human label for this snapshot, e.g. before-refactor')
|
|
222
|
+
.option('--dir <path>', 'Baseline storage root (default: <cwd>/.qulib-baselines)')
|
|
223
|
+
.option('--json', 'Emit the saved BaselineSnapshot as JSON to stdout', false)
|
|
224
|
+
.action(async (options) => {
|
|
225
|
+
await runBaselineSave({
|
|
226
|
+
url: options.url,
|
|
227
|
+
fromReport: options.fromReport,
|
|
228
|
+
label: options.label,
|
|
229
|
+
dir: options.dir,
|
|
230
|
+
json: Boolean(options.json),
|
|
231
|
+
});
|
|
232
|
+
});
|
|
233
|
+
baseline
|
|
234
|
+
.command('list')
|
|
235
|
+
.description('List saved baselines for a URL, newest first')
|
|
236
|
+
.requiredOption('--url <url>', 'URL whose baselines to list')
|
|
237
|
+
.option('--dir <path>', 'Baseline storage root (default: <cwd>/.qulib-baselines)')
|
|
238
|
+
.option('--json', 'Emit the BaselineSnapshot[] as JSON to stdout', false)
|
|
239
|
+
.action(async (options) => {
|
|
240
|
+
await runBaselineList({ url: options.url, dir: options.dir, json: Boolean(options.json) });
|
|
241
|
+
});
|
|
242
|
+
baseline
|
|
243
|
+
.command('compare')
|
|
244
|
+
.description('Compare two baselines — pass --from and --to ids, or --url for the two most-recent')
|
|
245
|
+
.option('--from <id>', 'Prior baseline id')
|
|
246
|
+
.option('--to <id>', 'Current baseline id')
|
|
247
|
+
.option('--url <url>', 'Compare the two most-recent baselines for this URL')
|
|
248
|
+
.option('--dir <path>', 'Baseline storage root (default: <cwd>/.qulib-baselines)')
|
|
249
|
+
.option('--json', 'Emit the full BaselineDelta as JSON to stdout', false)
|
|
250
|
+
.action(async (options) => {
|
|
251
|
+
await runBaselineCompare({
|
|
252
|
+
from: options.from,
|
|
253
|
+
to: options.to,
|
|
254
|
+
url: options.url,
|
|
255
|
+
dir: options.dir,
|
|
256
|
+
json: Boolean(options.json),
|
|
257
|
+
});
|
|
258
|
+
});
|
|
259
|
+
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"confidence-run.d.ts","sourceRoot":"","sources":["../../src/cli/confidence-run.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAQzC,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AAGzE,MAAM,WAAW,iBAAiB;IAChC,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAiBD,uEAAuE;AACvE,wBAAgB,sBAAsB,CAAC,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,CAuCxF;AAED;;;GAGG;AACH,wBAAsB,aAAa,CACjC,OAAO,EAAE,iBAAiB,EAC1B,GAAG,GAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAkC,GACxD,OAAO,CAAC,iBAAiB,CAAC,CAmE5B;AAED,wBAAgB,yBAAyB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,
|
|
1
|
+
{"version":3,"file":"confidence-run.d.ts","sourceRoot":"","sources":["../../src/cli/confidence-run.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAQzC,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AAGzE,MAAM,WAAW,iBAAiB;IAChC,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAiBD,uEAAuE;AACvE,wBAAgB,sBAAsB,CAAC,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,CAuCxF;AAED;;;GAGG;AACH,wBAAsB,aAAa,CACjC,OAAO,EAAE,iBAAiB,EAC1B,GAAG,GAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAkC,GACxD,OAAO,CAAC,iBAAiB,CAAC,CAmE5B;AAED,wBAAgB,yBAAyB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAuBhE"}
|
|
@@ -47,6 +47,11 @@ export function formatConfidenceReport(rc, subjectRef) {
|
|
|
47
47
|
for (const b of rc.blockers)
|
|
48
48
|
lines.push(` • ${b}`);
|
|
49
49
|
}
|
|
50
|
+
if (rc.honestyNotes.length > 0) {
|
|
51
|
+
lines.push(' honesty notes:');
|
|
52
|
+
for (const n of rc.honestyNotes)
|
|
53
|
+
lines.push(` • ${n}`);
|
|
54
|
+
}
|
|
50
55
|
lines.push(' contributions:');
|
|
51
56
|
for (const c of rc.contributions) {
|
|
52
57
|
const scoreLabel = c.score !== null ? `${c.score}/100` : 'n/a';
|
|
@@ -66,11 +71,6 @@ export function formatConfidenceReport(rc, subjectRef) {
|
|
|
66
71
|
for (const r of rc.recommendedNextChecks)
|
|
67
72
|
lines.push(` • ${r}`);
|
|
68
73
|
}
|
|
69
|
-
if (rc.honestyNotes.length > 0) {
|
|
70
|
-
lines.push(' honesty notes:');
|
|
71
|
-
for (const n of rc.honestyNotes)
|
|
72
|
-
lines.push(` • ${n}`);
|
|
73
|
-
}
|
|
74
74
|
return lines.join('\n');
|
|
75
75
|
}
|
|
76
76
|
/**
|
|
@@ -139,12 +139,16 @@ export async function runConfidence(options, out = (line) => console.log(line))
|
|
|
139
139
|
return rc;
|
|
140
140
|
}
|
|
141
141
|
export function registerConfidenceCommand(program) {
|
|
142
|
+
// Canonical flagship command. Alias: release-confidence (for integrations that prefer the
|
|
143
|
+
// full concept name over the short form). Both names are kept through 1.0.
|
|
142
144
|
program
|
|
143
145
|
.command('confidence')
|
|
146
|
+
.alias('release-confidence')
|
|
144
147
|
.description('Compute a fused Release Confidence verdict from qulib evidence collectors. ' +
|
|
145
148
|
'Pass --url to include live-app quality + a11y + coverage evidence. ' +
|
|
146
149
|
'Pass --repo to include test-automation maturity + API coverage. ' +
|
|
147
|
-
'Both may be combined.'
|
|
150
|
+
'Both may be combined. ' +
|
|
151
|
+
'(Alias: release-confidence)')
|
|
148
152
|
.option('--url <url>', 'URL of the deployed app to analyze')
|
|
149
153
|
.option('--repo <path>', 'Path to the local repository to score')
|
|
150
154
|
.option('--json', 'Emit the full ReleaseConfidence object as JSON to stdout', false)
|
package/dist/cli/index.js
CHANGED
|
@@ -41,6 +41,8 @@ import { runAutomatedAuthLogin } from './auth-login-run.js';
|
|
|
41
41
|
import { registerScaffoldCommand } from './scaffold-run.js';
|
|
42
42
|
import { registerScoreAutomationCommand } from './score-automation-run.js';
|
|
43
43
|
import { registerConfidenceCommand } from './confidence-run.js';
|
|
44
|
+
import { registerBaselineCommand } from './baseline-run.js';
|
|
45
|
+
import { registerAnalyzeDiffCommand } from './analyze-diff-run.js';
|
|
44
46
|
const program = new Command();
|
|
45
47
|
const AnalyzeUrlSchema = z.string().url();
|
|
46
48
|
const FormLoginCliSchema = z.object({
|
|
@@ -207,6 +209,8 @@ program
|
|
|
207
209
|
registerScaffoldCommand(program);
|
|
208
210
|
registerScoreAutomationCommand(program);
|
|
209
211
|
registerConfidenceCommand(program);
|
|
212
|
+
registerBaselineCommand(program);
|
|
213
|
+
registerAnalyzeDiffCommand(program);
|
|
210
214
|
program
|
|
211
215
|
.command('clean')
|
|
212
216
|
.description('Remove all generated reports and scan state')
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"score-automation-run.d.ts","sourceRoot":"","sources":["../../src/cli/score-automation-run.ts"],"names":[],"mappings":"AA4BA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,KAAK,EACV,kBAAkB,EAEnB,MAAM,0CAA0C,CAAC;AAIlD,MAAM,WAAW,sBAAsB;IACrC,kDAAkD;IAClD,IAAI,EAAE,MAAM,CAAC;IACb,6FAA6F;IAC7F,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAED;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,EAAE,GAAG,GAAE,MAAsB,GAAG,MAAM,CAYnG;AA+BD,gGAAgG;AAChG,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,kBAAkB,GAAG,MAAM,CAiBtE;AAED;;;;;GAKG;AACH,wBAAsB,kBAAkB,CACtC,OAAO,EAAE,sBAAsB,EAC/B,GAAG,GAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAkC,GACxD,OAAO,CAAC,kBAAkB,CAAC,CAW7B;AAED,wBAAgB,8BAA8B,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,
|
|
1
|
+
{"version":3,"file":"score-automation-run.d.ts","sourceRoot":"","sources":["../../src/cli/score-automation-run.ts"],"names":[],"mappings":"AA4BA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,KAAK,EACV,kBAAkB,EAEnB,MAAM,0CAA0C,CAAC;AAIlD,MAAM,WAAW,sBAAsB;IACrC,kDAAkD;IAClD,IAAI,EAAE,MAAM,CAAC;IACb,6FAA6F;IAC7F,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAED;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,EAAE,GAAG,GAAE,MAAsB,GAAG,MAAM,CAYnG;AA+BD,gGAAgG;AAChG,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,kBAAkB,GAAG,MAAM,CAiBtE;AAED;;;;;GAKG;AACH,wBAAsB,kBAAkB,CACtC,OAAO,EAAE,sBAAsB,EAC/B,GAAG,GAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAkC,GACxD,OAAO,CAAC,kBAAkB,CAAC,CAW7B;AAED,wBAAgB,8BAA8B,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAerE"}
|
|
@@ -112,9 +112,13 @@ export async function runScoreAutomation(options, out = (line) => console.log(li
|
|
|
112
112
|
return maturity;
|
|
113
113
|
}
|
|
114
114
|
export function registerScoreAutomationCommand(program) {
|
|
115
|
+
// Canonical name kept for backwards compatibility. Alias: automation-score
|
|
116
|
+
// (confidence-family naming — shorter and consistent with the qulib_ MCP convention).
|
|
115
117
|
program
|
|
116
118
|
.command('score-automation')
|
|
117
|
-
.
|
|
119
|
+
.alias('automation-score')
|
|
120
|
+
.description("Score a local repo's test-automation maturity (overall + per-dimension, with honest applicability). " +
|
|
121
|
+
'(Alias: automation-score)')
|
|
118
122
|
.requiredOption('--repo <path>', 'Path to the local repository to score')
|
|
119
123
|
.option('--json', 'Emit the full AutomationMaturity object as JSON to stdout', false)
|
|
120
124
|
.action(async (options) => {
|
package/dist/index.d.ts
CHANGED
|
@@ -12,6 +12,7 @@ export { discoverApiSurface, discoverApiSurfaceWithRepo } from './tools/repo/api
|
|
|
12
12
|
export type { ApiSurface, DiscoveredEndpoint, DiscoverApiSurfaceOptions } from './tools/repo/api-surface.js';
|
|
13
13
|
export { computeAutomationMaturity } from './tools/scoring/automation-maturity.js';
|
|
14
14
|
export { computeApiCoverage } from './tools/scoring/api-coverage.js';
|
|
15
|
+
export { detectPromptLeakage } from './tools/scoring/prompt-leakage.js';
|
|
15
16
|
export type { ApiCoverageResult, ApiEndpointCoverage } from './tools/scoring/api-coverage.js';
|
|
16
17
|
export { scaffoldTests } from './scaffold-tests.js';
|
|
17
18
|
export type { ScaffoldOptions, ScaffoldResult, ProjectConfig } from './scaffold-tests.js';
|
|
@@ -29,6 +30,10 @@ export type { CallLlmConfigSlice } from './llm/provider.js';
|
|
|
29
30
|
export type { HarnessConfig, AuthConfig, RouteInventory, GapAnalysis, RepoAnalysis, DetectedAuth, AuthExploration, AuthPath, AuthPathRequirements, CostIntelligence, LlmUsageRecord, RepeatedAiPattern, DeterministicMaturity, PublicSurface, AutomationMaturity, AutomationMaturityDimension, FrameworkDetectionResult, DetectedFrameworkPrimary, RecipeId, RecipeConfig, } from './schemas/index.js';
|
|
30
31
|
export { RecipeIdSchema } from './schemas/index.js';
|
|
31
32
|
export { computeReleaseConfidence } from './tools/scoring/confidence.js';
|
|
33
|
+
export { analyzeRunDiff, formatAnalyzeDiffMarkdown, loadGapAnalysisFile } from './cli/analyze-diff-run.js';
|
|
34
|
+
export { buildPageHeatmap, renderHeatmapSection, HEATMAP_DIMENSIONS, DIMENSION_LABELS } from './reporters/heatmap.js';
|
|
35
|
+
export type { PageHeatmap, HeatmapRow, HeatmapCell, HeatmapDimension } from './reporters/heatmap.js';
|
|
36
|
+
export type { AnalyzeDiffResult } from './cli/analyze-diff-run.js';
|
|
32
37
|
export { ciResultsToEvidence } from './adapters/ci-results-adapter.js';
|
|
33
38
|
export type { CiRunInput } from './adapters/ci-results-adapter.js';
|
|
34
39
|
export { prMetadataToEvidence } from './adapters/pr-metadata-adapter.js';
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EACL,UAAU,EACV,mBAAmB,EACnB,YAAY,EACZ,YAAY,EACZ,aAAa,EACb,cAAc,EACd,gBAAgB,GACjB,MAAM,wBAAwB,CAAC;AAChC,YAAY,EACV,WAAW,EACX,gBAAgB,EAChB,iBAAiB,EACjB,aAAa,GACd,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,YAAY,EACV,YAAY,EACZ,kBAAkB,EAClB,SAAS,EACT,cAAc,EACd,uBAAuB,GACxB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,UAAU,EACV,oBAAoB,EACpB,4BAA4B,EAC5B,yBAAyB,EACzB,qBAAqB,GACtB,MAAM,wBAAwB,CAAC;AAChC,YAAY,EACV,yBAAyB,EACzB,4BAA4B,GAC7B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAC;AAC1G,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,EAAE,kBAAkB,EAAE,0BAA0B,EAAE,MAAM,6BAA6B,CAAC;AAC7F,YAAY,EAAE,UAAU,EAAE,kBAAkB,EAAE,yBAAyB,EAAE,MAAM,6BAA6B,CAAC;AAC7G,OAAO,EAAE,yBAAyB,EAAE,MAAM,wCAAwC,CAAC;AACnF,OAAO,EAAE,kBAAkB,EAAE,MAAM,iCAAiC,CAAC;AACrE,YAAY,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,MAAM,iCAAiC,CAAC;AAC9F,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAC1F,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAClI,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAC5D,OAAO,EAAE,gCAAgC,EAAE,MAAM,4BAA4B,CAAC;AAC9E,OAAO,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAC;AACvF,YAAY,EAAE,cAAc,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AACjF,YAAY,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AACrE,YAAY,EACV,aAAa,EACb,cAAc,EACd,kBAAkB,GACnB,MAAM,oCAAoC,CAAC;AAC5C,OAAO,EAAE,iBAAiB,EAAE,MAAM,oCAAoC,CAAC;AACvE,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAC5D,YAAY,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AAC9E,YAAY,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAC5D,YAAY,EACV,aAAa,EACb,UAAU,EACV,cAAc,EACd,WAAW,EACX,YAAY,EACZ,YAAY,EACZ,eAAe,EACf,QAAQ,EACR,oBAAoB,EACpB,gBAAgB,EAChB,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,aAAa,EACb,kBAAkB,EAClB,2BAA2B,EAC3B,wBAAwB,EACxB,wBAAwB,EACxB,QAAQ,EACR,YAAY,GACb,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAEpD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAC;AACvE,YAAY,EAAE,UAAU,EAAE,MAAM,kCAAkC,CAAC;AACnE,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AACzE,YAAY,EAAE,eAAe,EAAE,WAAW,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,mCAAmC,CAAC;AACtH,OAAO,EAAE,6BAA6B,EAAE,MAAM,0CAA0C,CAAC;AACzF,OAAO,EAAE,cAAc,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,qCAAqC,CAAC;AAC7G,YAAY,EACV,kBAAkB,EAClB,YAAY,EACZ,iBAAiB,EACjB,eAAe,EACf,gBAAgB,EAChB,iBAAiB,EACjB,sBAAsB,EACtB,iBAAiB,EACjB,oBAAoB,EACpB,SAAS,EACT,aAAa,EACb,UAAU,EACV,WAAW,EACX,UAAU,GACX,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,wBAAwB,EACxB,kBAAkB,EAClB,uBAAuB,EACvB,qBAAqB,EACrB,sBAAsB,EACtB,uBAAuB,EACvB,uBAAuB,GACxB,MAAM,oBAAoB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EACL,UAAU,EACV,mBAAmB,EACnB,YAAY,EACZ,YAAY,EACZ,aAAa,EACb,cAAc,EACd,gBAAgB,GACjB,MAAM,wBAAwB,CAAC;AAChC,YAAY,EACV,WAAW,EACX,gBAAgB,EAChB,iBAAiB,EACjB,aAAa,GACd,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,YAAY,EACV,YAAY,EACZ,kBAAkB,EAClB,SAAS,EACT,cAAc,EACd,uBAAuB,GACxB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,UAAU,EACV,oBAAoB,EACpB,4BAA4B,EAC5B,yBAAyB,EACzB,qBAAqB,GACtB,MAAM,wBAAwB,CAAC;AAChC,YAAY,EACV,yBAAyB,EACzB,4BAA4B,GAC7B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAC;AAC1G,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,EAAE,kBAAkB,EAAE,0BAA0B,EAAE,MAAM,6BAA6B,CAAC;AAC7F,YAAY,EAAE,UAAU,EAAE,kBAAkB,EAAE,yBAAyB,EAAE,MAAM,6BAA6B,CAAC;AAC7G,OAAO,EAAE,yBAAyB,EAAE,MAAM,wCAAwC,CAAC;AACnF,OAAO,EAAE,kBAAkB,EAAE,MAAM,iCAAiC,CAAC;AACrE,OAAO,EAAE,mBAAmB,EAAE,MAAM,mCAAmC,CAAC;AACxE,YAAY,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,MAAM,iCAAiC,CAAC;AAC9F,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAC1F,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAClI,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAC5D,OAAO,EAAE,gCAAgC,EAAE,MAAM,4BAA4B,CAAC;AAC9E,OAAO,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAC;AACvF,YAAY,EAAE,cAAc,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AACjF,YAAY,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AACrE,YAAY,EACV,aAAa,EACb,cAAc,EACd,kBAAkB,GACnB,MAAM,oCAAoC,CAAC;AAC5C,OAAO,EAAE,iBAAiB,EAAE,MAAM,oCAAoC,CAAC;AACvE,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAC5D,YAAY,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AAC9E,YAAY,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAC5D,YAAY,EACV,aAAa,EACb,UAAU,EACV,cAAc,EACd,WAAW,EACX,YAAY,EACZ,YAAY,EACZ,eAAe,EACf,QAAQ,EACR,oBAAoB,EACpB,gBAAgB,EAChB,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,aAAa,EACb,kBAAkB,EAClB,2BAA2B,EAC3B,wBAAwB,EACxB,wBAAwB,EACxB,QAAQ,EACR,YAAY,GACb,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAEpD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,OAAO,EAAE,cAAc,EAAE,yBAAyB,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAE3G,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AACtH,YAAY,EAAE,WAAW,EAAE,UAAU,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AACrG,YAAY,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAEnE,OAAO,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAC;AACvE,YAAY,EAAE,UAAU,EAAE,MAAM,kCAAkC,CAAC;AACnE,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AACzE,YAAY,EAAE,eAAe,EAAE,WAAW,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,mCAAmC,CAAC;AACtH,OAAO,EAAE,6BAA6B,EAAE,MAAM,0CAA0C,CAAC;AACzF,OAAO,EAAE,cAAc,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,qCAAqC,CAAC;AAC7G,YAAY,EACV,kBAAkB,EAClB,YAAY,EACZ,iBAAiB,EACjB,eAAe,EACf,gBAAgB,EAChB,iBAAiB,EACjB,sBAAsB,EACtB,iBAAiB,EACjB,oBAAoB,EACpB,SAAS,EACT,aAAa,EACb,UAAU,EACV,WAAW,EACX,UAAU,GACX,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,wBAAwB,EACxB,kBAAkB,EAClB,uBAAuB,EACvB,qBAAqB,EACrB,sBAAsB,EACtB,uBAAuB,EACvB,uBAAuB,GACxB,MAAM,oBAAoB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -8,6 +8,7 @@ export { scanRepo } from './tools/repo/scan.js';
|
|
|
8
8
|
export { discoverApiSurface, discoverApiSurfaceWithRepo } from './tools/repo/api-surface.js';
|
|
9
9
|
export { computeAutomationMaturity } from './tools/scoring/automation-maturity.js';
|
|
10
10
|
export { computeApiCoverage } from './tools/scoring/api-coverage.js';
|
|
11
|
+
export { detectPromptLeakage } from './tools/scoring/prompt-leakage.js';
|
|
11
12
|
export { scaffoldTests } from './scaffold-tests.js';
|
|
12
13
|
export { expandRecipes, buildAuthScenarios, buildA11yScenarios, buildNavScenarios, buildSeedScenarios } from './recipes/index.js';
|
|
13
14
|
export { createProvider } from './llm/provider-registry.js';
|
|
@@ -18,6 +19,10 @@ export { redactUrlForTelemetry } from './telemetry/emit.js';
|
|
|
18
19
|
export { RecipeIdSchema } from './schemas/index.js';
|
|
19
20
|
// P3 — Confidence Layer exports
|
|
20
21
|
export { computeReleaseConfidence } from './tools/scoring/confidence.js';
|
|
22
|
+
// analyze-diff — structured diff between two analyze_app outputs
|
|
23
|
+
export { analyzeRunDiff, formatAnalyzeDiffMarkdown, loadGapAnalysisFile } from './cli/analyze-diff-run.js';
|
|
24
|
+
// per-page coverage heatmap
|
|
25
|
+
export { buildPageHeatmap, renderHeatmapSection, HEATMAP_DIMENSIONS, DIMENSION_LABELS } from './reporters/heatmap.js';
|
|
21
26
|
// P4 — Evidence adapters (CI results + PR metadata)
|
|
22
27
|
export { ciResultsToEvidence } from './adapters/ci-results-adapter.js';
|
|
23
28
|
export { prMetadataToEvidence } from './adapters/pr-metadata-adapter.js';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"think.d.ts","sourceRoot":"","sources":["../../src/phases/think.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACjE,OAAO,EAAqB,KAAK,WAAW,EAAE,MAAM,mCAAmC,CAAC;AACxF,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"think.d.ts","sourceRoot":"","sources":["../../src/phases/think.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACjE,OAAO,EAAqB,KAAK,WAAW,EAAE,MAAM,mCAAmC,CAAC;AACxF,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAIlD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAIrE,wBAAsB,KAAK,CACzB,QAAQ,EAAE,aAAa,EACvB,MAAM,EAAE,aAAa,EACrB,SAAS,GAAE,mBAA8C,GACxD,OAAO,CAAC,WAAW,CAAC,CAyDtB;AAED,OAAO,EAAE,4BAA4B,EAAE,MAAM,qBAAqB,CAAC;AACnE,YAAY,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC"}
|
package/dist/phases/think.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { GapAnalysisSchema } from '../schemas/gap-analysis.schema.js';
|
|
2
2
|
import { analyzeGaps } from '../tools/scoring/gaps.js';
|
|
3
|
+
import { detectPromptLeakage } from '../tools/scoring/prompt-leakage.js';
|
|
3
4
|
import { logDecision } from '../harness/decision-logger.js';
|
|
4
5
|
import { finalizeGapAnalysisFromDraft } from './think-finalize.js';
|
|
5
6
|
import { emitTelemetry } from '../telemetry/emit.js';
|
|
@@ -13,6 +14,8 @@ export async function think(observed, config, artifacts = { writeArtifacts: true
|
|
|
13
14
|
};
|
|
14
15
|
emitTelemetry(artifacts.telemetry, 'phase.think.started', sessionId, { mode });
|
|
15
16
|
const gapBlock = analyzeGaps(observed.routes, observed.repo, mode, config);
|
|
17
|
+
// Merge prompt-leakage gaps from all scanned routes (additive signal).
|
|
18
|
+
const promptLeakageGaps = observed.routes.routes.flatMap((route) => detectPromptLeakage(route));
|
|
16
19
|
const draft = {
|
|
17
20
|
analyzedAt: gapBlock.analyzedAt,
|
|
18
21
|
mode: gapBlock.mode,
|
|
@@ -20,7 +23,7 @@ export async function think(observed, config, artifacts = { writeArtifacts: true
|
|
|
20
23
|
coveragePagesScanned: gapBlock.coveragePagesScanned,
|
|
21
24
|
coverageBudgetExceeded: gapBlock.coverageBudgetExceeded,
|
|
22
25
|
coverageWarning: gapBlock.coverageWarning,
|
|
23
|
-
gaps: gapBlock.gaps,
|
|
26
|
+
gaps: [...gapBlock.gaps, ...promptLeakageGaps],
|
|
24
27
|
};
|
|
25
28
|
const partialAnalysis = GapAnalysisSchema.parse({
|
|
26
29
|
...draft,
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-page coverage heatmap for Markdown reports.
|
|
3
|
+
*
|
|
4
|
+
* buildPageHeatmap() is a pure function — no I/O, no side-effects.
|
|
5
|
+
* It derives a rows × dimensions matrix from GapAnalysis.gaps and sorts
|
|
6
|
+
* rows worst-first (most critical coverage problems bubble to the top).
|
|
7
|
+
*
|
|
8
|
+
* Dimensions map to the GapSchema.category enum values so the heatmap
|
|
9
|
+
* always stays in sync with what the scanner can produce.
|
|
10
|
+
*/
|
|
11
|
+
import type { Gap } from '../schemas/gap-analysis.schema.js';
|
|
12
|
+
/** The ordered set of gap categories that appear as heatmap columns. */
|
|
13
|
+
export declare const HEATMAP_DIMENSIONS: readonly ["untested-route", "a11y", "console-error", "broken-link", "coverage", "untested-api-endpoint", "auth-surface", "prompt-leakage"];
|
|
14
|
+
export type HeatmapDimension = (typeof HEATMAP_DIMENSIONS)[number];
|
|
15
|
+
/** Display labels for each dimension column header. */
|
|
16
|
+
export declare const DIMENSION_LABELS: Record<HeatmapDimension, string>;
|
|
17
|
+
/** One cell in the heatmap: the worst severity for that page × dimension. */
|
|
18
|
+
export type HeatmapCell = {
|
|
19
|
+
/** Worst Gap severity found, or null when no gap exists. */
|
|
20
|
+
worstSeverity: Gap['severity'] | null;
|
|
21
|
+
/** Glyph to render in Markdown. */
|
|
22
|
+
glyph: string;
|
|
23
|
+
/** Count of gaps contributing to this cell. */
|
|
24
|
+
count: number;
|
|
25
|
+
};
|
|
26
|
+
/** One row in the heatmap: a page path and its per-dimension cells. */
|
|
27
|
+
export type HeatmapRow = {
|
|
28
|
+
path: string;
|
|
29
|
+
/** Map from dimension to cell. Guaranteed to contain every HEATMAP_DIMENSION key. */
|
|
30
|
+
cells: Record<HeatmapDimension, HeatmapCell>;
|
|
31
|
+
/** Sum of severity scores across all cells — used for worst-first sort. */
|
|
32
|
+
worstScore: number;
|
|
33
|
+
};
|
|
34
|
+
/** The full heatmap structure returned by buildPageHeatmap(). */
|
|
35
|
+
export type PageHeatmap = {
|
|
36
|
+
/** Rows sorted worst-first (highest total severity score first). */
|
|
37
|
+
rows: HeatmapRow[];
|
|
38
|
+
/** Ordered dimension labels for use as column headers. */
|
|
39
|
+
dimensions: typeof HEATMAP_DIMENSIONS;
|
|
40
|
+
/** Total number of gaps that fed into the heatmap. */
|
|
41
|
+
totalGaps: number;
|
|
42
|
+
};
|
|
43
|
+
/**
|
|
44
|
+
* Build a per-page coverage heatmap from a flat list of gaps.
|
|
45
|
+
*
|
|
46
|
+
* @param gaps The gaps array from GapAnalysis.
|
|
47
|
+
* @returns A PageHeatmap with rows sorted worst-first.
|
|
48
|
+
*/
|
|
49
|
+
export declare function buildPageHeatmap(gaps: Gap[]): PageHeatmap;
|
|
50
|
+
/**
|
|
51
|
+
* Render a PageHeatmap as a Markdown section string.
|
|
52
|
+
* Returns an empty string when there are no rows (nothing scanned).
|
|
53
|
+
*/
|
|
54
|
+
export declare function renderHeatmapSection(heatmap: PageHeatmap): string;
|
|
55
|
+
//# sourceMappingURL=heatmap.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"heatmap.d.ts","sourceRoot":"","sources":["../../src/reporters/heatmap.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,mCAAmC,CAAC;AAM7D,wEAAwE;AACxE,eAAO,MAAM,kBAAkB,4IASoB,CAAC;AAEpD,MAAM,MAAM,gBAAgB,GAAG,CAAC,OAAO,kBAAkB,CAAC,CAAC,MAAM,CAAC,CAAC;AAEnE,uDAAuD;AACvD,eAAO,MAAM,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,EAAE,MAAM,CAS7D,CAAC;AAuBF,6EAA6E;AAC7E,MAAM,MAAM,WAAW,GAAG;IACxB,4DAA4D;IAC5D,aAAa,EAAE,GAAG,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC;IACtC,mCAAmC;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,+CAA+C;IAC/C,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,uEAAuE;AACvE,MAAM,MAAM,UAAU,GAAG;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,qFAAqF;IACrF,KAAK,EAAE,MAAM,CAAC,gBAAgB,EAAE,WAAW,CAAC,CAAC;IAC7C,2EAA2E;IAC3E,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,iEAAiE;AACjE,MAAM,MAAM,WAAW,GAAG;IACxB,oEAAoE;IACpE,IAAI,EAAE,UAAU,EAAE,CAAC;IACnB,0DAA0D;IAC1D,UAAU,EAAE,OAAO,kBAAkB,CAAC;IACtC,sDAAsD;IACtD,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAMF;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,WAAW,CAuDzD;AAMD;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,WAAW,GAAG,MAAM,CAmCjE"}
|