@qulib/core 0.4.2 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +135 -8
- package/dist/__tests__/cli-smoke-fixture.d.ts +2 -0
- package/dist/__tests__/cli-smoke-fixture.d.ts.map +1 -0
- package/dist/__tests__/cli-smoke-fixture.js +58 -0
- package/dist/__tests__/fixture-server.d.ts +6 -0
- package/dist/__tests__/fixture-server.d.ts.map +1 -0
- package/dist/__tests__/fixture-server.js +141 -0
- package/dist/analyze.d.ts.map +1 -1
- package/dist/analyze.js +84 -5
- package/dist/cli/auth-login-run.d.ts.map +1 -1
- package/dist/cli/auth-login-run.js +26 -2
- package/dist/cli/index.js +12 -6
- package/dist/index.d.ts +6 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -5
- package/dist/llm/providers/anthropic.js +1 -1
- package/dist/phases/observe.js +2 -2
- package/dist/phases/think-finalize.d.ts.map +1 -1
- package/dist/phases/think-finalize.js +7 -1
- package/dist/phases/think.js +1 -1
- package/dist/schemas/automation-maturity.schema.d.ts +8 -0
- package/dist/schemas/automation-maturity.schema.d.ts.map +1 -1
- package/dist/schemas/automation-maturity.schema.js +1 -0
- package/dist/schemas/repo-analysis.schema.d.ts +7 -0
- package/dist/schemas/repo-analysis.schema.d.ts.map +1 -1
- package/dist/telemetry/telemetry.interface.d.ts +1 -1
- package/dist/telemetry/telemetry.interface.d.ts.map +1 -1
- package/dist/tools/apply-auth.d.ts +4 -0
- package/dist/tools/apply-auth.d.ts.map +1 -0
- package/dist/tools/apply-auth.js +35 -0
- package/dist/tools/auth/apply.d.ts +4 -0
- package/dist/tools/auth/apply.d.ts.map +1 -0
- package/dist/tools/auth/apply.js +35 -0
- package/dist/tools/auth/block-gap.d.ts +9 -0
- package/dist/tools/auth/block-gap.d.ts.map +1 -0
- package/dist/tools/auth/block-gap.js +52 -0
- package/dist/tools/auth/custom-providers.d.ts +15 -0
- package/dist/tools/auth/custom-providers.d.ts.map +1 -0
- package/dist/tools/auth/custom-providers.js +62 -0
- package/dist/tools/auth/detect.d.ts +23 -0
- package/dist/tools/auth/detect.d.ts.map +1 -0
- package/dist/tools/auth/detect.js +526 -0
- package/dist/tools/auth/detector.d.ts +23 -0
- package/dist/tools/auth/detector.d.ts.map +1 -0
- package/dist/tools/auth/detector.js +526 -0
- package/dist/tools/auth/explore.d.ts +4 -0
- package/dist/tools/auth/explore.d.ts.map +1 -0
- package/dist/tools/auth/explore.js +346 -0
- package/dist/tools/auth/explorer.d.ts +4 -0
- package/dist/tools/auth/explorer.d.ts.map +1 -0
- package/dist/tools/auth/explorer.js +346 -0
- package/dist/tools/auth/gaps.d.ts +9 -0
- package/dist/tools/auth/gaps.d.ts.map +1 -0
- package/dist/tools/auth/gaps.js +52 -0
- package/dist/tools/auth/oauth-providers.d.ts +7 -0
- package/dist/tools/auth/oauth-providers.d.ts.map +1 -0
- package/dist/tools/auth/oauth-providers.js +21 -0
- package/dist/tools/auth/providers.d.ts +7 -0
- package/dist/tools/auth/providers.d.ts.map +1 -0
- package/dist/tools/auth/providers.js +21 -0
- package/dist/tools/auth/surface-analyzer.d.ts +4 -0
- package/dist/tools/auth/surface-analyzer.d.ts.map +1 -0
- package/dist/tools/auth/surface-analyzer.js +170 -0
- package/dist/tools/auth/surface.d.ts +4 -0
- package/dist/tools/auth/surface.d.ts.map +1 -0
- package/dist/tools/auth/surface.js +170 -0
- package/dist/tools/auth/user-providers.d.ts +15 -0
- package/dist/tools/auth/user-providers.d.ts.map +1 -0
- package/dist/tools/auth/user-providers.js +62 -0
- package/dist/tools/auth-block-gap.d.ts +6 -0
- package/dist/tools/auth-block-gap.d.ts.map +1 -1
- package/dist/tools/auth-block-gap.js +42 -9
- package/dist/tools/auth-detector.d.ts +9 -8
- package/dist/tools/auth-detector.d.ts.map +1 -1
- package/dist/tools/auth-detector.js +106 -8
- package/dist/tools/explorers/browser.d.ts +3 -0
- package/dist/tools/explorers/browser.d.ts.map +1 -0
- package/dist/tools/explorers/browser.js +13 -0
- package/dist/tools/explorers/cypress-explorer.d.ts +8 -0
- package/dist/tools/explorers/cypress-explorer.d.ts.map +1 -0
- package/dist/tools/explorers/cypress-explorer.js +5 -0
- package/dist/tools/explorers/cypress.d.ts +8 -0
- package/dist/tools/explorers/cypress.d.ts.map +1 -0
- package/dist/tools/explorers/cypress.js +5 -0
- package/dist/tools/explorers/explorer.interface.d.ts +7 -0
- package/dist/tools/explorers/explorer.interface.d.ts.map +1 -0
- package/dist/tools/explorers/explorer.interface.js +1 -0
- package/dist/tools/explorers/factory.d.ts +4 -0
- package/dist/tools/explorers/factory.d.ts.map +1 -0
- package/dist/tools/explorers/factory.js +12 -0
- package/dist/tools/explorers/playwright-explorer.d.ts +8 -0
- package/dist/tools/explorers/playwright-explorer.d.ts.map +1 -0
- package/dist/tools/explorers/playwright-explorer.js +172 -0
- package/dist/tools/explorers/playwright.d.ts +8 -0
- package/dist/tools/explorers/playwright.d.ts.map +1 -0
- package/dist/tools/explorers/playwright.js +172 -0
- package/dist/tools/explorers/types.d.ts +7 -0
- package/dist/tools/explorers/types.d.ts.map +1 -0
- package/dist/tools/explorers/types.js +1 -0
- package/dist/tools/playwright-explorer.js +1 -1
- package/dist/tools/repo/detect-framework.d.ts +15 -0
- package/dist/tools/repo/detect-framework.d.ts.map +1 -0
- package/dist/tools/repo/detect-framework.js +153 -0
- package/dist/tools/repo/framework-detector.d.ts +15 -0
- package/dist/tools/repo/framework-detector.d.ts.map +1 -0
- package/dist/tools/repo/framework-detector.js +153 -0
- package/dist/tools/repo/scan.d.ts +19 -0
- package/dist/tools/repo/scan.d.ts.map +1 -0
- package/dist/tools/repo/scan.js +181 -0
- package/dist/tools/repo/scanner.d.ts +19 -0
- package/dist/tools/repo/scanner.d.ts.map +1 -0
- package/dist/tools/repo/scanner.js +181 -0
- package/dist/tools/scoring/automation-maturity.d.ts +4 -0
- package/dist/tools/scoring/automation-maturity.d.ts.map +1 -0
- package/dist/tools/scoring/automation-maturity.js +231 -0
- package/dist/tools/scoring/gap-engine.d.ts +8 -0
- package/dist/tools/scoring/gap-engine.d.ts.map +1 -0
- package/dist/tools/scoring/gap-engine.js +138 -0
- package/dist/tools/scoring/gaps.d.ts +8 -0
- package/dist/tools/scoring/gaps.d.ts.map +1 -0
- package/dist/tools/scoring/gaps.js +138 -0
- package/dist/tools/scoring/public-surface.d.ts +5 -0
- package/dist/tools/scoring/public-surface.d.ts.map +1 -0
- package/dist/tools/scoring/public-surface.js +13 -0
- package/package.json +3 -3
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
import { existsSync, readdirSync, statSync } from 'node:fs';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { AutomationMaturitySchema } from '../../schemas/automation-maturity.schema.js';
|
|
4
|
+
/**
|
|
5
|
+
* Dimension weights (sum = 1). Breadth + harness adoption dominate: shipping risk is mostly
|
|
6
|
+
* untested routes and missing Playwright/Cypress-level coverage.
|
|
7
|
+
*/
|
|
8
|
+
const W_TEST_BREADTH = 0.28;
|
|
9
|
+
const W_FRAMEWORK = 0.22;
|
|
10
|
+
const W_TEST_ID = 0.18;
|
|
11
|
+
const W_CI = 0.14;
|
|
12
|
+
const W_AUTH_TESTS = 0.1;
|
|
13
|
+
const W_COMPONENT_RATIO = 0.08;
|
|
14
|
+
function hasCiAtRoot(repoPath) {
|
|
15
|
+
const ev = [];
|
|
16
|
+
const gh = join(repoPath, '.github', 'workflows');
|
|
17
|
+
if (existsSync(gh) && statSync(gh).isDirectory()) {
|
|
18
|
+
try {
|
|
19
|
+
const files = readdirSync(gh).filter((f) => f.endsWith('.yml') || f.endsWith('.yaml'));
|
|
20
|
+
if (files.length > 0) {
|
|
21
|
+
ev.push(`.github/workflows (${files.length} workflow file(s))`);
|
|
22
|
+
return { ok: true, evidence: ev };
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
catch {
|
|
26
|
+
/* ignore */
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
if (existsSync(join(repoPath, '.circleci'))) {
|
|
30
|
+
ev.push('.circleci/ present');
|
|
31
|
+
return { ok: true, evidence: ev };
|
|
32
|
+
}
|
|
33
|
+
for (const f of ['.gitlab-ci.yml', 'Jenkinsfile']) {
|
|
34
|
+
if (existsSync(join(repoPath, f))) {
|
|
35
|
+
ev.push(`${f} present`);
|
|
36
|
+
return { ok: true, evidence: ev };
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
return { ok: false, evidence: ['No GitHub Actions, CircleCI, GitLab CI, or Jenkinsfile detected at repo root'] };
|
|
40
|
+
}
|
|
41
|
+
function scoreLevel(overall) {
|
|
42
|
+
if (overall < 20)
|
|
43
|
+
return { level: 1, label: 'L1 — nascent automation' };
|
|
44
|
+
if (overall < 40)
|
|
45
|
+
return { level: 2, label: 'L2 — emerging coverage' };
|
|
46
|
+
if (overall < 60)
|
|
47
|
+
return { level: 3, label: 'L3 — building maturity' };
|
|
48
|
+
if (overall < 80)
|
|
49
|
+
return { level: 4, label: 'L4 — strong automation' };
|
|
50
|
+
return { level: 5, label: 'L5 — advanced QA automation' };
|
|
51
|
+
}
|
|
52
|
+
export function computeAutomationMaturity(repo) {
|
|
53
|
+
const routePaths = [...new Set(repo.routes.map((r) => r.path))];
|
|
54
|
+
let coveredRoutes = 0;
|
|
55
|
+
for (const p of routePaths) {
|
|
56
|
+
const covered = repo.testFiles.some((tf) => tf.coveredPaths.some((c) => p === c || (c !== '/' && p.startsWith(c))));
|
|
57
|
+
if (covered)
|
|
58
|
+
coveredRoutes++;
|
|
59
|
+
}
|
|
60
|
+
const breadthScore = routePaths.length === 0 ? 100 : Math.round((100 * coveredRoutes) / routePaths.length);
|
|
61
|
+
const breadthDim = {
|
|
62
|
+
dimension: 'test-coverage-breadth',
|
|
63
|
+
score: breadthScore,
|
|
64
|
+
weight: W_TEST_BREADTH,
|
|
65
|
+
evidence: routePaths.length === 0
|
|
66
|
+
? ['No static routes inferred from repo layout']
|
|
67
|
+
: [
|
|
68
|
+
`${coveredRoutes}/${routePaths.length} inferred routes appear in at least one test coveredPaths`,
|
|
69
|
+
],
|
|
70
|
+
recommendations: breadthScore >= 80
|
|
71
|
+
? []
|
|
72
|
+
: ['Add route-level smoke tests that assert critical paths referenced in production URLs.'],
|
|
73
|
+
};
|
|
74
|
+
const types = new Set(repo.testFiles.map((t) => t.type));
|
|
75
|
+
let frameworkScore = 0;
|
|
76
|
+
const fwEvidence = [`Test runners seen: ${[...types].join(', ') || 'none'}`];
|
|
77
|
+
if (types.has('playwright') || types.has('cypress-e2e') || types.has('cypress-component')) {
|
|
78
|
+
frameworkScore = 100;
|
|
79
|
+
fwEvidence.push('Playwright or Cypress present — good browser harness signal.');
|
|
80
|
+
}
|
|
81
|
+
else if (types.has('jest') || types.has('vitest')) {
|
|
82
|
+
frameworkScore = 55;
|
|
83
|
+
fwEvidence.push('Jest/Vitest only — add Playwright or Cypress for deployment-facing checks.');
|
|
84
|
+
}
|
|
85
|
+
else if (repo.testFiles.length > 0) {
|
|
86
|
+
frameworkScore = 30;
|
|
87
|
+
fwEvidence.push('Tests exist but no recognized browser harness in scanned files.');
|
|
88
|
+
}
|
|
89
|
+
else {
|
|
90
|
+
frameworkScore = 0;
|
|
91
|
+
fwEvidence.push('No test files matched qulib scan globs.');
|
|
92
|
+
}
|
|
93
|
+
const frameworkDim = {
|
|
94
|
+
dimension: 'framework-adoption',
|
|
95
|
+
score: frameworkScore,
|
|
96
|
+
weight: W_FRAMEWORK,
|
|
97
|
+
evidence: fwEvidence,
|
|
98
|
+
recommendations: frameworkScore >= 80 ? [] : ['Standardize on Playwright or Cypress for E2E against deployed URLs.'],
|
|
99
|
+
};
|
|
100
|
+
const missingIds = repo.missingTestIds.length;
|
|
101
|
+
const interactiveTsxScanned = repo.interactiveTsxFilesScanned ?? missingIds;
|
|
102
|
+
let hygieneScore = 0;
|
|
103
|
+
let hygieneApplicability = 'applicable';
|
|
104
|
+
let hygieneReason;
|
|
105
|
+
let hygieneGuidance;
|
|
106
|
+
const hygieneEvidence = [];
|
|
107
|
+
if (interactiveTsxScanned === 0) {
|
|
108
|
+
hygieneApplicability = 'unknown';
|
|
109
|
+
hygieneReason = 'No interactive TSX files scanned — cannot compute a missing-id ratio honestly.';
|
|
110
|
+
hygieneGuidance =
|
|
111
|
+
'Qulib could not collect enough signal to score this dimension. Run against a repo with more test files or a larger page scan to improve signal.';
|
|
112
|
+
hygieneEvidence.push(hygieneReason);
|
|
113
|
+
}
|
|
114
|
+
else {
|
|
115
|
+
const missingRatio = missingIds / interactiveTsxScanned;
|
|
116
|
+
hygieneScore = Math.round(Math.max(0, 100 * (1 - missingRatio)));
|
|
117
|
+
hygieneEvidence.push(`${missingIds}/${interactiveTsxScanned} interactive TSX file(s) lacked data-testid (heuristic scan).`);
|
|
118
|
+
}
|
|
119
|
+
const hygieneDim = {
|
|
120
|
+
dimension: 'test-id-hygiene',
|
|
121
|
+
score: hygieneScore,
|
|
122
|
+
weight: W_TEST_ID,
|
|
123
|
+
evidence: hygieneEvidence,
|
|
124
|
+
recommendations: hygieneApplicability === 'applicable' && hygieneScore < 85
|
|
125
|
+
? ['Add stable data-testid (or role-based selectors) on interactive components used in tests.']
|
|
126
|
+
: [],
|
|
127
|
+
applicability: hygieneApplicability,
|
|
128
|
+
...(hygieneReason && { reason: hygieneReason }),
|
|
129
|
+
...(hygieneGuidance && { guidance: hygieneGuidance }),
|
|
130
|
+
};
|
|
131
|
+
const ci = hasCiAtRoot(repo.repoPath);
|
|
132
|
+
const ciDim = {
|
|
133
|
+
dimension: 'ci-integration',
|
|
134
|
+
score: ci.ok ? 100 : 0,
|
|
135
|
+
weight: W_CI,
|
|
136
|
+
evidence: ci.evidence,
|
|
137
|
+
recommendations: ci.ok ? [] : ['Add a CI workflow that runs unit/E2E tests on every PR.'],
|
|
138
|
+
};
|
|
139
|
+
const authRe = /\/(login|auth|signin)(\/|$)/i;
|
|
140
|
+
const authRouteFileRe = /(login|auth|signin)/i;
|
|
141
|
+
const authCovered = repo.testFiles.some((tf) => tf.coveredPaths.some((c) => authRe.test(c)));
|
|
142
|
+
const repoHasAuthRoute = repo.routes.some((r) => authRe.test(r.path));
|
|
143
|
+
const repoHasAuthTestFile = repo.testFiles.some((tf) => authRouteFileRe.test(tf.file));
|
|
144
|
+
const repoHasAnyAuthSignal = repoHasAuthRoute || repoHasAuthTestFile || authCovered;
|
|
145
|
+
let authScore = 0;
|
|
146
|
+
let authApplicability = 'applicable';
|
|
147
|
+
let authReason;
|
|
148
|
+
let authGuidance;
|
|
149
|
+
const authEvidence = [];
|
|
150
|
+
if (!repoHasAnyAuthSignal) {
|
|
151
|
+
authApplicability = 'not_applicable';
|
|
152
|
+
authReason = 'No auth routes, auth-named test files, or auth path coverage detected — repo appears auth-free.';
|
|
153
|
+
authGuidance =
|
|
154
|
+
'No auth signal detected in this app. If authentication exists, run qulib with a storage-state file to enable auth-test-coverage scoring.';
|
|
155
|
+
authEvidence.push(authReason);
|
|
156
|
+
}
|
|
157
|
+
else {
|
|
158
|
+
authScore = authCovered ? 90 : 25;
|
|
159
|
+
authEvidence.push(authCovered
|
|
160
|
+
? 'At least one test references /login, /auth, or /signin in coveredPaths.'
|
|
161
|
+
: 'Repo has auth-shaped routes or test files but no auth-route coverage in extracted test path strings.');
|
|
162
|
+
}
|
|
163
|
+
const authDim = {
|
|
164
|
+
dimension: 'auth-test-coverage',
|
|
165
|
+
score: authScore,
|
|
166
|
+
weight: W_AUTH_TESTS,
|
|
167
|
+
evidence: authEvidence,
|
|
168
|
+
recommendations: authApplicability === 'applicable' && !authCovered
|
|
169
|
+
? ['Add focused tests for sign-in and post-auth landing behavior.']
|
|
170
|
+
: [],
|
|
171
|
+
applicability: authApplicability,
|
|
172
|
+
...(authReason && { reason: authReason }),
|
|
173
|
+
...(authGuidance && { guidance: authGuidance }),
|
|
174
|
+
};
|
|
175
|
+
const cypressE2e = repo.testFiles.filter((t) => t.type === 'cypress-e2e').length;
|
|
176
|
+
const cypressComp = repo.testFiles.filter((t) => t.type === 'cypress-component').length;
|
|
177
|
+
const cypressTotal = cypressE2e + cypressComp;
|
|
178
|
+
let compRatioScore = 0;
|
|
179
|
+
let compApplicability = 'applicable';
|
|
180
|
+
let compReason;
|
|
181
|
+
let compGuidance;
|
|
182
|
+
const compEvidence = [];
|
|
183
|
+
if (cypressTotal === 0) {
|
|
184
|
+
compApplicability = 'not_applicable';
|
|
185
|
+
compReason = 'No Cypress (e2e or component) tests detected — component-test-ratio does not apply.';
|
|
186
|
+
compGuidance =
|
|
187
|
+
'No Cypress component test setup detected. Add cypress/component/ tests and a component config to enable this dimension.';
|
|
188
|
+
compEvidence.push(compReason);
|
|
189
|
+
}
|
|
190
|
+
else {
|
|
191
|
+
compRatioScore = Math.round((100 * cypressComp) / cypressTotal);
|
|
192
|
+
compEvidence.push(`Cypress e2e files (matched): ${cypressE2e}, component: ${cypressComp}.`);
|
|
193
|
+
}
|
|
194
|
+
const compDim = {
|
|
195
|
+
dimension: 'component-test-ratio',
|
|
196
|
+
score: compRatioScore,
|
|
197
|
+
weight: W_COMPONENT_RATIO,
|
|
198
|
+
evidence: compEvidence,
|
|
199
|
+
recommendations: compApplicability === 'applicable' && cypressComp > 0
|
|
200
|
+
? ['Balance component vs E2E Cypress tests so critical flows stay fast in CI.']
|
|
201
|
+
: [],
|
|
202
|
+
applicability: compApplicability,
|
|
203
|
+
...(compReason && { reason: compReason }),
|
|
204
|
+
...(compGuidance && { guidance: compGuidance }),
|
|
205
|
+
};
|
|
206
|
+
const dimensions = [breadthDim, frameworkDim, hygieneDim, ciDim, authDim, compDim];
|
|
207
|
+
// Overall score normalizes over applicable dimensions only.
|
|
208
|
+
// overallScore = round( Σ score_i * weight_i / Σ weight_i ) for i ∈ applicable.
|
|
209
|
+
// If no dimension is applicable (degenerate repo), overall = 0 and level = L1.
|
|
210
|
+
const applicableDims = dimensions.filter((d) => (d.applicability ?? 'applicable') === 'applicable');
|
|
211
|
+
const weightSum = applicableDims.reduce((s, d) => s + d.weight, 0);
|
|
212
|
+
const overallScore = weightSum > 0
|
|
213
|
+
? Math.round(applicableDims.reduce((s, d) => s + d.score * d.weight, 0) / weightSum)
|
|
214
|
+
: 0;
|
|
215
|
+
const { level, label } = scoreLevel(overallScore);
|
|
216
|
+
const topRecommendations = [...applicableDims]
|
|
217
|
+
.sort((a, b) => a.score - b.score)
|
|
218
|
+
.flatMap((d) => d.recommendations)
|
|
219
|
+
.filter(Boolean)
|
|
220
|
+
.slice(0, 8);
|
|
221
|
+
return AutomationMaturitySchema.parse({
|
|
222
|
+
computedAt: new Date().toISOString(),
|
|
223
|
+
repoPath: repo.repoPath,
|
|
224
|
+
overallScore,
|
|
225
|
+
level,
|
|
226
|
+
label,
|
|
227
|
+
dimensions,
|
|
228
|
+
topRecommendations,
|
|
229
|
+
scoreFormula: 'overallScore = round( Σ (score * weight) / Σ weight ) for applicable dimensions only. not_applicable and unknown dimensions are excluded from the denominator.',
|
|
230
|
+
});
|
|
231
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { type GapAnalysis, type Gap } from '../../schemas/gap-analysis.schema.js';
|
|
2
|
+
import type { RouteInventory } from '../../schemas/route-inventory.schema.js';
|
|
3
|
+
import type { RepoAnalysis } from '../../schemas/repo-analysis.schema.js';
|
|
4
|
+
import type { HarnessConfig } from '../../schemas/config.schema.js';
|
|
5
|
+
export declare function computeQualityScoreFromGaps(gaps: Gap[], scoringWeights?: HarnessConfig['scoringWeights']): number;
|
|
6
|
+
export declare function computeCoverageScore(routes: RouteInventory): number | null;
|
|
7
|
+
export declare function analyzeGaps(routes: RouteInventory, repo: RepoAnalysis | null, mode: 'url-only' | 'url-repo', config: HarnessConfig): Omit<GapAnalysis, 'scenarios' | 'generatedTests'>;
|
|
8
|
+
//# sourceMappingURL=gap-engine.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"gap-engine.d.ts","sourceRoot":"","sources":["../../../src/tools/scoring/gap-engine.ts"],"names":[],"mappings":"AACA,OAAO,EAAa,KAAK,WAAW,EAAE,KAAK,GAAG,EAAE,MAAM,sCAAsC,CAAC;AAC7F,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,yCAAyC,CAAC;AAC9E,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,uCAAuC,CAAC;AAC1E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAQpE,wBAAgB,2BAA2B,CACzC,IAAI,EAAE,GAAG,EAAE,EACX,cAAc,CAAC,EAAE,aAAa,CAAC,gBAAgB,CAAC,GAC/C,MAAM,CAkBR;AAED,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,cAAc,GAAG,MAAM,GAAG,IAAI,CAa1E;AAED,wBAAgB,WAAW,CACzB,MAAM,EAAE,cAAc,EACtB,IAAI,EAAE,YAAY,GAAG,IAAI,EACzB,IAAI,EAAE,UAAU,GAAG,UAAU,EAC7B,MAAM,EAAE,aAAa,GACpB,IAAI,CAAC,WAAW,EAAE,WAAW,GAAG,gBAAgB,CAAC,CAqGnD"}
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { GapSchema } from '../../schemas/gap-analysis.schema.js';
|
|
3
|
+
// TODO: Add category-specific weight overrides (e.g., auth-surface gaps should cost more than untested-route gaps by default).
|
|
4
|
+
// Requires a 2D weight matrix: { [severity]: { [category]: number } }.
|
|
5
|
+
// Deferred until there is empirical data from real scan runs to calibrate.
|
|
6
|
+
const DEFAULT_SCORING_WEIGHTS = { critical: 25, high: 20, medium: 8, low: 3 };
|
|
7
|
+
export function computeQualityScoreFromGaps(gaps, scoringWeights) {
|
|
8
|
+
let critical = 0;
|
|
9
|
+
let high = 0;
|
|
10
|
+
let medium = 0;
|
|
11
|
+
let low = 0;
|
|
12
|
+
for (const g of gaps) {
|
|
13
|
+
if (g.severity === 'critical')
|
|
14
|
+
critical++;
|
|
15
|
+
else if (g.severity === 'high')
|
|
16
|
+
high++;
|
|
17
|
+
else if (g.severity === 'medium')
|
|
18
|
+
medium++;
|
|
19
|
+
else
|
|
20
|
+
low++;
|
|
21
|
+
}
|
|
22
|
+
const w = {
|
|
23
|
+
critical: scoringWeights?.critical ?? DEFAULT_SCORING_WEIGHTS.critical,
|
|
24
|
+
high: scoringWeights?.high ?? DEFAULT_SCORING_WEIGHTS.high,
|
|
25
|
+
medium: scoringWeights?.medium ?? DEFAULT_SCORING_WEIGHTS.medium,
|
|
26
|
+
low: scoringWeights?.low ?? DEFAULT_SCORING_WEIGHTS.low,
|
|
27
|
+
};
|
|
28
|
+
return Math.max(0, 100 - critical * w.critical - high * w.high - medium * w.medium - low * w.low);
|
|
29
|
+
}
|
|
30
|
+
export function computeCoverageScore(routes) {
|
|
31
|
+
const scanned = routes.routes.length;
|
|
32
|
+
const skipped = routes.pagesSkipped;
|
|
33
|
+
const denom = scanned + skipped;
|
|
34
|
+
// TODO: return null here once the explorer exposes an explicit "discovered-but-unknown" signal
|
|
35
|
+
// (i.e. routes were found but the full set couldn't be confirmed — a low score is misleading)
|
|
36
|
+
if (denom === 0) {
|
|
37
|
+
if (routes.budgetExceeded) {
|
|
38
|
+
return 0;
|
|
39
|
+
}
|
|
40
|
+
return scanned === 0 ? 0 : 100;
|
|
41
|
+
}
|
|
42
|
+
return Math.round((100 * scanned) / denom);
|
|
43
|
+
}
|
|
44
|
+
export function analyzeGaps(routes, repo, mode, config) {
|
|
45
|
+
const coveredPaths = new Set();
|
|
46
|
+
if (repo) {
|
|
47
|
+
for (const testFile of repo.testFiles) {
|
|
48
|
+
for (const path of testFile.coveredPaths) {
|
|
49
|
+
coveredPaths.add(path);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
const gaps = [];
|
|
54
|
+
const addGap = (gap) => {
|
|
55
|
+
const validated = GapSchema.parse(gap);
|
|
56
|
+
gaps.push(validated);
|
|
57
|
+
};
|
|
58
|
+
let hasNavigationFailures = false;
|
|
59
|
+
for (const route of routes.routes) {
|
|
60
|
+
if (repo && !coveredPaths.has(route.path)) {
|
|
61
|
+
const highRisk = /checkout|payment|auth|login|order/i.test(route.path);
|
|
62
|
+
addGap({
|
|
63
|
+
id: randomUUID(),
|
|
64
|
+
path: route.path,
|
|
65
|
+
severity: highRisk ? 'high' : 'medium',
|
|
66
|
+
reason: `Route is not covered by existing tests: ${route.path}`,
|
|
67
|
+
category: 'untested-route',
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
const navErrors = route.consoleErrors.filter((e) => e.startsWith('Navigation error:'));
|
|
71
|
+
if (navErrors.length > 0) {
|
|
72
|
+
hasNavigationFailures = true;
|
|
73
|
+
addGap({
|
|
74
|
+
id: randomUUID(),
|
|
75
|
+
path: route.path,
|
|
76
|
+
severity: 'high',
|
|
77
|
+
reason: `Navigation failed: ${navErrors.join('; ')}`,
|
|
78
|
+
category: 'console-error',
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
else if (route.consoleErrors.length > 0) {
|
|
82
|
+
addGap({
|
|
83
|
+
id: randomUUID(),
|
|
84
|
+
path: route.path,
|
|
85
|
+
severity: 'high',
|
|
86
|
+
reason: `Console errors detected (${route.consoleErrors.length})`,
|
|
87
|
+
category: 'console-error',
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
if (route.brokenLinks.length > 0) {
|
|
91
|
+
addGap({
|
|
92
|
+
id: randomUUID(),
|
|
93
|
+
path: route.path,
|
|
94
|
+
severity: 'medium',
|
|
95
|
+
reason: `Broken or invalid links detected (${route.brokenLinks.length})`,
|
|
96
|
+
category: 'broken-link',
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
for (const violation of route.a11yViolations) {
|
|
100
|
+
const impact = violation.impact.toLowerCase();
|
|
101
|
+
const severity = impact === 'critical'
|
|
102
|
+
? 'critical'
|
|
103
|
+
: impact === 'serious'
|
|
104
|
+
? 'high'
|
|
105
|
+
: impact === 'moderate'
|
|
106
|
+
? 'medium'
|
|
107
|
+
: 'low';
|
|
108
|
+
addGap({
|
|
109
|
+
id: randomUUID(),
|
|
110
|
+
path: route.path,
|
|
111
|
+
severity,
|
|
112
|
+
reason: `A11y violation ${violation.id} (${violation.impact}): ${violation.helpUrl}`,
|
|
113
|
+
category: 'a11y',
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
const releaseConfidence = computeQualityScoreFromGaps(gaps, config.scoringWeights);
|
|
118
|
+
const pagesScanned = routes.routes.length;
|
|
119
|
+
let coverageWarning;
|
|
120
|
+
if (routes.budgetExceeded) {
|
|
121
|
+
coverageWarning = 'budget-exceeded';
|
|
122
|
+
}
|
|
123
|
+
else if (hasNavigationFailures) {
|
|
124
|
+
coverageWarning = 'navigation-failures';
|
|
125
|
+
}
|
|
126
|
+
else if (pagesScanned < config.minPagesForConfidence) {
|
|
127
|
+
coverageWarning = 'low-coverage';
|
|
128
|
+
}
|
|
129
|
+
return {
|
|
130
|
+
analyzedAt: new Date().toISOString(),
|
|
131
|
+
mode,
|
|
132
|
+
releaseConfidence,
|
|
133
|
+
coveragePagesScanned: pagesScanned,
|
|
134
|
+
coverageBudgetExceeded: routes.budgetExceeded,
|
|
135
|
+
coverageWarning,
|
|
136
|
+
gaps,
|
|
137
|
+
};
|
|
138
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { type GapAnalysis, type Gap } from '../../schemas/gap-analysis.schema.js';
|
|
2
|
+
import type { RouteInventory } from '../../schemas/route-inventory.schema.js';
|
|
3
|
+
import type { RepoAnalysis } from '../../schemas/repo-analysis.schema.js';
|
|
4
|
+
import type { HarnessConfig } from '../../schemas/config.schema.js';
|
|
5
|
+
export declare function computeQualityScoreFromGaps(gaps: Gap[], scoringWeights?: HarnessConfig['scoringWeights']): number;
|
|
6
|
+
export declare function computeCoverageScore(routes: RouteInventory): number | null;
|
|
7
|
+
export declare function analyzeGaps(routes: RouteInventory, repo: RepoAnalysis | null, mode: 'url-only' | 'url-repo', config: HarnessConfig): Omit<GapAnalysis, 'scenarios' | 'generatedTests'>;
|
|
8
|
+
//# sourceMappingURL=gaps.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"gaps.d.ts","sourceRoot":"","sources":["../../../src/tools/scoring/gaps.ts"],"names":[],"mappings":"AACA,OAAO,EAAa,KAAK,WAAW,EAAE,KAAK,GAAG,EAAE,MAAM,sCAAsC,CAAC;AAC7F,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,yCAAyC,CAAC;AAC9E,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,uCAAuC,CAAC;AAC1E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAQpE,wBAAgB,2BAA2B,CACzC,IAAI,EAAE,GAAG,EAAE,EACX,cAAc,CAAC,EAAE,aAAa,CAAC,gBAAgB,CAAC,GAC/C,MAAM,CAkBR;AAED,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,cAAc,GAAG,MAAM,GAAG,IAAI,CAa1E;AAED,wBAAgB,WAAW,CACzB,MAAM,EAAE,cAAc,EACtB,IAAI,EAAE,YAAY,GAAG,IAAI,EACzB,IAAI,EAAE,UAAU,GAAG,UAAU,EAC7B,MAAM,EAAE,aAAa,GACpB,IAAI,CAAC,WAAW,EAAE,WAAW,GAAG,gBAAgB,CAAC,CAqGnD"}
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { GapSchema } from '../../schemas/gap-analysis.schema.js';
|
|
3
|
+
// TODO: Add category-specific weight overrides (e.g., auth-surface gaps should cost more than untested-route gaps by default).
|
|
4
|
+
// Requires a 2D weight matrix: { [severity]: { [category]: number } }.
|
|
5
|
+
// Deferred until there is empirical data from real scan runs to calibrate.
|
|
6
|
+
const DEFAULT_SCORING_WEIGHTS = { critical: 25, high: 20, medium: 8, low: 3 };
|
|
7
|
+
export function computeQualityScoreFromGaps(gaps, scoringWeights) {
|
|
8
|
+
let critical = 0;
|
|
9
|
+
let high = 0;
|
|
10
|
+
let medium = 0;
|
|
11
|
+
let low = 0;
|
|
12
|
+
for (const g of gaps) {
|
|
13
|
+
if (g.severity === 'critical')
|
|
14
|
+
critical++;
|
|
15
|
+
else if (g.severity === 'high')
|
|
16
|
+
high++;
|
|
17
|
+
else if (g.severity === 'medium')
|
|
18
|
+
medium++;
|
|
19
|
+
else
|
|
20
|
+
low++;
|
|
21
|
+
}
|
|
22
|
+
const w = {
|
|
23
|
+
critical: scoringWeights?.critical ?? DEFAULT_SCORING_WEIGHTS.critical,
|
|
24
|
+
high: scoringWeights?.high ?? DEFAULT_SCORING_WEIGHTS.high,
|
|
25
|
+
medium: scoringWeights?.medium ?? DEFAULT_SCORING_WEIGHTS.medium,
|
|
26
|
+
low: scoringWeights?.low ?? DEFAULT_SCORING_WEIGHTS.low,
|
|
27
|
+
};
|
|
28
|
+
return Math.max(0, 100 - critical * w.critical - high * w.high - medium * w.medium - low * w.low);
|
|
29
|
+
}
|
|
30
|
+
export function computeCoverageScore(routes) {
|
|
31
|
+
const scanned = routes.routes.length;
|
|
32
|
+
const skipped = routes.pagesSkipped;
|
|
33
|
+
const denom = scanned + skipped;
|
|
34
|
+
// TODO: return null here once the explorer exposes an explicit "discovered-but-unknown" signal
|
|
35
|
+
// (i.e. routes were found but the full set couldn't be confirmed — a low score is misleading)
|
|
36
|
+
if (denom === 0) {
|
|
37
|
+
if (routes.budgetExceeded) {
|
|
38
|
+
return 0;
|
|
39
|
+
}
|
|
40
|
+
return scanned === 0 ? 0 : 100;
|
|
41
|
+
}
|
|
42
|
+
return Math.round((100 * scanned) / denom);
|
|
43
|
+
}
|
|
44
|
+
export function analyzeGaps(routes, repo, mode, config) {
|
|
45
|
+
const coveredPaths = new Set();
|
|
46
|
+
if (repo) {
|
|
47
|
+
for (const testFile of repo.testFiles) {
|
|
48
|
+
for (const path of testFile.coveredPaths) {
|
|
49
|
+
coveredPaths.add(path);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
const gaps = [];
|
|
54
|
+
const addGap = (gap) => {
|
|
55
|
+
const validated = GapSchema.parse(gap);
|
|
56
|
+
gaps.push(validated);
|
|
57
|
+
};
|
|
58
|
+
let hasNavigationFailures = false;
|
|
59
|
+
for (const route of routes.routes) {
|
|
60
|
+
if (repo && !coveredPaths.has(route.path)) {
|
|
61
|
+
const highRisk = /checkout|payment|auth|login|order/i.test(route.path);
|
|
62
|
+
addGap({
|
|
63
|
+
id: randomUUID(),
|
|
64
|
+
path: route.path,
|
|
65
|
+
severity: highRisk ? 'high' : 'medium',
|
|
66
|
+
reason: `Route is not covered by existing tests: ${route.path}`,
|
|
67
|
+
category: 'untested-route',
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
const navErrors = route.consoleErrors.filter((e) => e.startsWith('Navigation error:'));
|
|
71
|
+
if (navErrors.length > 0) {
|
|
72
|
+
hasNavigationFailures = true;
|
|
73
|
+
addGap({
|
|
74
|
+
id: randomUUID(),
|
|
75
|
+
path: route.path,
|
|
76
|
+
severity: 'high',
|
|
77
|
+
reason: `Navigation failed: ${navErrors.join('; ')}`,
|
|
78
|
+
category: 'console-error',
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
else if (route.consoleErrors.length > 0) {
|
|
82
|
+
addGap({
|
|
83
|
+
id: randomUUID(),
|
|
84
|
+
path: route.path,
|
|
85
|
+
severity: 'high',
|
|
86
|
+
reason: `Console errors detected (${route.consoleErrors.length})`,
|
|
87
|
+
category: 'console-error',
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
if (route.brokenLinks.length > 0) {
|
|
91
|
+
addGap({
|
|
92
|
+
id: randomUUID(),
|
|
93
|
+
path: route.path,
|
|
94
|
+
severity: 'medium',
|
|
95
|
+
reason: `Broken or invalid links detected (${route.brokenLinks.length})`,
|
|
96
|
+
category: 'broken-link',
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
for (const violation of route.a11yViolations) {
|
|
100
|
+
const impact = violation.impact.toLowerCase();
|
|
101
|
+
const severity = impact === 'critical'
|
|
102
|
+
? 'critical'
|
|
103
|
+
: impact === 'serious'
|
|
104
|
+
? 'high'
|
|
105
|
+
: impact === 'moderate'
|
|
106
|
+
? 'medium'
|
|
107
|
+
: 'low';
|
|
108
|
+
addGap({
|
|
109
|
+
id: randomUUID(),
|
|
110
|
+
path: route.path,
|
|
111
|
+
severity,
|
|
112
|
+
reason: `A11y violation ${violation.id} (${violation.impact}): ${violation.helpUrl}`,
|
|
113
|
+
category: 'a11y',
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
const releaseConfidence = computeQualityScoreFromGaps(gaps, config.scoringWeights);
|
|
118
|
+
const pagesScanned = routes.routes.length;
|
|
119
|
+
let coverageWarning;
|
|
120
|
+
if (routes.budgetExceeded) {
|
|
121
|
+
coverageWarning = 'budget-exceeded';
|
|
122
|
+
}
|
|
123
|
+
else if (hasNavigationFailures) {
|
|
124
|
+
coverageWarning = 'navigation-failures';
|
|
125
|
+
}
|
|
126
|
+
else if (pagesScanned < config.minPagesForConfidence) {
|
|
127
|
+
coverageWarning = 'low-coverage';
|
|
128
|
+
}
|
|
129
|
+
return {
|
|
130
|
+
analyzedAt: new Date().toISOString(),
|
|
131
|
+
mode,
|
|
132
|
+
releaseConfidence,
|
|
133
|
+
coveragePagesScanned: pagesScanned,
|
|
134
|
+
coverageBudgetExceeded: routes.budgetExceeded,
|
|
135
|
+
coverageWarning,
|
|
136
|
+
gaps,
|
|
137
|
+
};
|
|
138
|
+
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { RouteInventory } from '../../schemas/route-inventory.schema.js';
|
|
2
|
+
import type { Gap } from '../../schemas/gap-analysis.schema.js';
|
|
3
|
+
import type { PublicSurface } from '../../schemas/public-surface.schema.js';
|
|
4
|
+
export declare function buildPublicSurface(pages: RouteInventory['routes'], gaps: Gap[]): PublicSurface;
|
|
5
|
+
//# sourceMappingURL=public-surface.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"public-surface.d.ts","sourceRoot":"","sources":["../../../src/tools/scoring/public-surface.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,yCAAyC,CAAC;AAC9E,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,sCAAsC,CAAC;AAChE,OAAO,KAAK,EAAE,aAAa,EAAmD,MAAM,wCAAwC,CAAC;AAE7H,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,cAAc,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,aAAa,CAY9F"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export function buildPublicSurface(pages, gaps) {
|
|
2
|
+
const accessibilityViolations = [];
|
|
3
|
+
const brokenLinks = [];
|
|
4
|
+
for (const r of pages) {
|
|
5
|
+
for (const v of r.a11yViolations) {
|
|
6
|
+
accessibilityViolations.push({ ...v, path: r.path });
|
|
7
|
+
}
|
|
8
|
+
for (const b of r.brokenLinks) {
|
|
9
|
+
brokenLinks.push({ ...b, path: r.path });
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
return { pages, gaps, accessibilityViolations, brokenLinks };
|
|
13
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@qulib/core",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.5.0",
|
|
4
4
|
"description": "Qulib — analyze deployed web apps for honest quality gaps (CLI + programmatic API)",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Tapesh Nagarwal",
|
|
@@ -48,8 +48,8 @@
|
|
|
48
48
|
"analyze": "tsx src/cli/index.ts analyze",
|
|
49
49
|
"clean": "tsx src/cli/index.ts clean",
|
|
50
50
|
"build": "tsc",
|
|
51
|
-
"test": "node --import tsx/esm --test src/llm/cost-intelligence.test.ts src/tools/
|
|
52
|
-
"test:integration": "node --import tsx/esm --test src/analyze.integration.test.ts",
|
|
51
|
+
"test": "node --import tsx/esm --test src/llm/__tests__/cost-intelligence.test.ts src/tools/scoring/__tests__/gaps.test.ts src/tools/auth/__tests__/gaps.test.ts src/tools/auth/__tests__/detect.test.ts src/tools/scoring/__tests__/automation-maturity.test.ts src/harness/__tests__/state-manager.test.ts src/telemetry/__tests__/redact-url.test.ts src/cli/__tests__/auth-login.test.ts src/cli/__tests__/cli-version.test.ts src/__tests__/analyze.storage-state-invalid.test.ts src/__tests__/analyze.fixtures.test.ts",
|
|
52
|
+
"test:integration": "node --import tsx/esm --test src/__tests__/analyze.integration.test.ts",
|
|
53
53
|
"smoke": "tsx src/cli/index.ts analyze --url https://example.com --ephemeral",
|
|
54
54
|
"cost-doctor": "tsx src/cli/index.ts cost doctor"
|
|
55
55
|
},
|