@qulib/core 0.8.2 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -5
- package/bin/qulib.js +2 -3
- package/dist/__tests__/playwright-available.d.ts +32 -0
- package/dist/__tests__/playwright-available.d.ts.map +1 -0
- package/dist/__tests__/playwright-available.js +35 -0
- package/dist/adapters/ci-results-adapter.d.ts +67 -0
- package/dist/adapters/ci-results-adapter.d.ts.map +1 -0
- package/dist/adapters/ci-results-adapter.js +143 -0
- package/dist/adapters/cypress-e2e-adapter.d.ts.map +1 -1
- package/dist/adapters/cypress-e2e-adapter.js +25 -2
- package/dist/adapters/playwright-adapter.d.ts.map +1 -1
- package/dist/adapters/playwright-adapter.js +25 -2
- package/dist/adapters/pr-metadata-adapter.d.ts +75 -0
- package/dist/adapters/pr-metadata-adapter.d.ts.map +1 -0
- package/dist/adapters/pr-metadata-adapter.js +146 -0
- package/dist/adapters/validate-specs.d.ts +55 -0
- package/dist/adapters/validate-specs.d.ts.map +1 -0
- package/dist/adapters/validate-specs.js +67 -0
- package/dist/baseline/baseline.d.ts +54 -0
- package/dist/baseline/baseline.d.ts.map +1 -0
- package/dist/baseline/baseline.js +252 -0
- package/dist/baseline/baseline.schema.d.ts +233 -0
- package/dist/baseline/baseline.schema.d.ts.map +1 -0
- package/dist/baseline/baseline.schema.js +59 -0
- package/dist/cli/confidence-run.d.ts +16 -0
- package/dist/cli/confidence-run.d.ts.map +1 -0
- package/dist/cli/confidence-run.js +158 -0
- package/dist/cli/index.d.ts +11 -1
- package/dist/cli/index.d.ts.map +1 -1
- package/dist/cli/index.js +80 -4
- package/dist/cli/scaffold-run.d.ts +86 -0
- package/dist/cli/scaffold-run.d.ts.map +1 -0
- package/dist/cli/scaffold-run.js +232 -0
- package/dist/cli/score-automation-run.d.ts +25 -0
- package/dist/cli/score-automation-run.d.ts.map +1 -0
- package/dist/cli/score-automation-run.js +123 -0
- package/dist/examples/notquality-dogfood/fixture.d.ts +166 -0
- package/dist/examples/notquality-dogfood/fixture.d.ts.map +1 -0
- package/dist/examples/notquality-dogfood/fixture.js +174 -0
- package/dist/examples/notquality-dogfood/run.d.ts +34 -0
- package/dist/examples/notquality-dogfood/run.d.ts.map +1 -0
- package/dist/examples/notquality-dogfood/run.js +139 -0
- package/dist/index.d.ts +14 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +11 -0
- package/dist/recipes/a11y.d.ts +36 -0
- package/dist/recipes/a11y.d.ts.map +1 -0
- package/dist/recipes/a11y.js +118 -0
- package/dist/recipes/auth.d.ts +38 -0
- package/dist/recipes/auth.d.ts.map +1 -0
- package/dist/recipes/auth.js +156 -0
- package/dist/recipes/index.d.ts +26 -0
- package/dist/recipes/index.d.ts.map +1 -0
- package/dist/recipes/index.js +41 -0
- package/dist/recipes/nav.d.ts +34 -0
- package/dist/recipes/nav.d.ts.map +1 -0
- package/dist/recipes/nav.js +128 -0
- package/dist/recipes/seed.d.ts +34 -0
- package/dist/recipes/seed.d.ts.map +1 -0
- package/dist/recipes/seed.js +87 -0
- package/dist/scaffold-tests.d.ts +21 -0
- package/dist/scaffold-tests.d.ts.map +1 -1
- package/dist/scaffold-tests.js +12 -2
- package/dist/schemas/confidence.schema.d.ts +526 -0
- package/dist/schemas/confidence.schema.d.ts.map +1 -0
- package/dist/schemas/confidence.schema.js +161 -0
- package/dist/schemas/index.d.ts +3 -0
- package/dist/schemas/index.d.ts.map +1 -1
- package/dist/schemas/index.js +3 -0
- package/dist/schemas/recipe.schema.d.ts +66 -0
- package/dist/schemas/recipe.schema.d.ts.map +1 -0
- package/dist/schemas/recipe.schema.js +45 -0
- package/dist/schemas/views.schema.d.ts +234 -0
- package/dist/schemas/views.schema.d.ts.map +1 -0
- package/dist/schemas/views.schema.js +82 -0
- package/dist/tools/scoring/confidence-from-qulib.d.ts +34 -0
- package/dist/tools/scoring/confidence-from-qulib.d.ts.map +1 -0
- package/dist/tools/scoring/confidence-from-qulib.js +206 -0
- package/dist/tools/scoring/confidence-views.d.ts +40 -0
- package/dist/tools/scoring/confidence-views.d.ts.map +1 -0
- package/dist/tools/scoring/confidence-views.js +163 -0
- package/dist/tools/scoring/confidence.d.ts +32 -0
- package/dist/tools/scoring/confidence.d.ts.map +1 -0
- package/dist/tools/scoring/confidence.js +180 -0
- package/dist/tools/scoring/levels.d.ts +15 -0
- package/dist/tools/scoring/levels.d.ts.map +1 -0
- package/dist/tools/scoring/levels.js +21 -0
- package/package.json +13 -7
package/README.md
CHANGED
|
@@ -140,16 +140,38 @@ After a normal **`analyze`**, `output/report.json` includes **`gapAnalysis.costI
|
|
|
140
140
|
Re-print that block from disk:
|
|
141
141
|
|
|
142
142
|
```bash
|
|
143
|
-
|
|
144
|
-
# or:
|
|
143
|
+
qulib cost doctor
|
|
144
|
+
# or: qulib cost doctor --report output/report.json
|
|
145
145
|
```
|
|
146
146
|
|
|
147
147
|
## CLI (from npm)
|
|
148
148
|
|
|
149
|
+
**Release confidence — the flagship command:**
|
|
150
|
+
|
|
151
|
+
```bash
|
|
152
|
+
npx @qulib/core confidence --url https://example.com
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
Returns ship / caution / hold / block with a 0–100 score, top risks, and recommended next checks. Add `--repo` to also score test-automation maturity and API coverage.
|
|
156
|
+
|
|
157
|
+
**Analyze (full gap report):**
|
|
158
|
+
|
|
149
159
|
```bash
|
|
150
160
|
npx @qulib/core analyze --url https://example.com
|
|
151
161
|
```
|
|
152
162
|
|
|
163
|
+
**Scaffold a test suite:**
|
|
164
|
+
|
|
165
|
+
```bash
|
|
166
|
+
npx @qulib/core scaffold --url https://example.com --framework cypress-e2e
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
**Score automation maturity (repo-only, no URL needed):**
|
|
170
|
+
|
|
171
|
+
```bash
|
|
172
|
+
npx @qulib/core score-automation --repo /path/to/repo
|
|
173
|
+
```
|
|
174
|
+
|
|
153
175
|
Use `npx playwright install chromium` the first time you scan (Playwright is a dependency).
|
|
154
176
|
|
|
155
177
|
## Programmatic API
|
|
@@ -344,10 +366,13 @@ Use these as conservative reference numbers:
|
|
|
344
366
|
|
|
345
367
|
| Tool | When to use | Key input |
|
|
346
368
|
|---|---|---|
|
|
347
|
-
|
|
|
348
|
-
| `
|
|
369
|
+
| **`qulib_score_confidence`** | **Flagship.** Fused verdict (ship/caution/hold/block) from all collectors | `url` and/or `repoPath`, optional `includeViews.replay` |
|
|
370
|
+
| `analyze_app` | Live-app QA scan: release confidence + gaps + a11y | `url`, optional `auth`, optional LLM knobs |
|
|
371
|
+
| `qulib_score_automation` | Score local repo test-automation maturity | absolute `repoPath`, optional `includeFullDimensions` |
|
|
372
|
+
| `qulib_score_api` | Discover API endpoints and score their test coverage | absolute `repoPath`, optional `enableTier3`, `includeEndpointDetail` |
|
|
373
|
+
| `qulib_scaffold_tests` | Generate Cypress/Playwright scaffold from a live URL | `url`, optional `framework`, `maxPagesToScan`, `recipes` |
|
|
349
374
|
| `explore_auth` | Deeper auth-path discovery on unfamiliar apps | `url`, optional `timeoutMs` |
|
|
350
|
-
| `
|
|
375
|
+
| `detect_auth` | Fast single-pass auth pattern guess | `url`, optional `timeoutMs` |
|
|
351
376
|
|
|
352
377
|
## Output directories
|
|
353
378
|
|
package/bin/qulib.js
CHANGED
|
@@ -5,10 +5,9 @@ import { dirname, resolve } from 'node:path';
|
|
|
5
5
|
|
|
6
6
|
const __filename = fileURLToPath(import.meta.url);
|
|
7
7
|
const __dirname = dirname(__filename);
|
|
8
|
-
const cliPath = resolve(__dirname, '../
|
|
8
|
+
const cliPath = resolve(__dirname, '../dist/cli/index.js');
|
|
9
9
|
|
|
10
|
-
const
|
|
11
|
-
const child = spawn(npxCmd, ['tsx', cliPath, ...process.argv.slice(2)], {
|
|
10
|
+
const child = spawn(process.execPath, [cliPath, ...process.argv.slice(2)], {
|
|
12
11
|
stdio: 'inherit',
|
|
13
12
|
});
|
|
14
13
|
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Lightweight helper used by test files that require a Playwright Chromium
|
|
3
|
+
* installation. Checks for the browser at the filesystem level — no side
|
|
4
|
+
* effects, no subprocess launch.
|
|
5
|
+
*
|
|
6
|
+
* Two ways to skip:
|
|
7
|
+
* 1. Set PLAYWRIGHT_SKIP=1 in the environment — explicit opt-out, useful for
|
|
8
|
+
* fresh-clone CI jobs that intentionally skip browser tests.
|
|
9
|
+
* 2. The Playwright Chromium executable is simply absent from the expected
|
|
10
|
+
* install path (e.g. the developer never ran `npx playwright install`).
|
|
11
|
+
*
|
|
12
|
+
* Usage in test files:
|
|
13
|
+
*
|
|
14
|
+
* import { chromiumAvailable, CHROMIUM_SKIP_REASON } from './playwright-available.js';
|
|
15
|
+
*
|
|
16
|
+
* test('my browser test', { skip: !chromiumAvailable, skipMessage: CHROMIUM_SKIP_REASON }, () => { ... });
|
|
17
|
+
*
|
|
18
|
+
* Or for a subtree of tests (t.before guard):
|
|
19
|
+
*
|
|
20
|
+
* if (!chromiumAvailable) { t.skip(); return; }
|
|
21
|
+
*/
|
|
22
|
+
/**
|
|
23
|
+
* True when a Playwright Chromium browser binary is present on disk AND the
|
|
24
|
+
* caller has not set PLAYWRIGHT_SKIP=1.
|
|
25
|
+
*/
|
|
26
|
+
export declare const chromiumAvailable: boolean;
|
|
27
|
+
/**
|
|
28
|
+
* Human-readable reason string passed to `t.skip()` / `skipMessage` so the
|
|
29
|
+
* skip is self-documenting in test output.
|
|
30
|
+
*/
|
|
31
|
+
export declare const CHROMIUM_SKIP_REASON: string;
|
|
32
|
+
//# sourceMappingURL=playwright-available.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"playwright-available.d.ts","sourceRoot":"","sources":["../../src/__tests__/playwright-available.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAKH;;;GAGG;AACH,eAAO,MAAM,iBAAiB,EAAE,OAC0C,CAAC;AAE3E;;;GAGG;AACH,eAAO,MAAM,oBAAoB,EAAE,MAEqE,CAAC"}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Lightweight helper used by test files that require a Playwright Chromium
|
|
3
|
+
* installation. Checks for the browser at the filesystem level — no side
|
|
4
|
+
* effects, no subprocess launch.
|
|
5
|
+
*
|
|
6
|
+
* Two ways to skip:
|
|
7
|
+
* 1. Set PLAYWRIGHT_SKIP=1 in the environment — explicit opt-out, useful for
|
|
8
|
+
* fresh-clone CI jobs that intentionally skip browser tests.
|
|
9
|
+
* 2. The Playwright Chromium executable is simply absent from the expected
|
|
10
|
+
* install path (e.g. the developer never ran `npx playwright install`).
|
|
11
|
+
*
|
|
12
|
+
* Usage in test files:
|
|
13
|
+
*
|
|
14
|
+
* import { chromiumAvailable, CHROMIUM_SKIP_REASON } from './playwright-available.js';
|
|
15
|
+
*
|
|
16
|
+
* test('my browser test', { skip: !chromiumAvailable, skipMessage: CHROMIUM_SKIP_REASON }, () => { ... });
|
|
17
|
+
*
|
|
18
|
+
* Or for a subtree of tests (t.before guard):
|
|
19
|
+
*
|
|
20
|
+
* if (!chromiumAvailable) { t.skip(); return; }
|
|
21
|
+
*/
|
|
22
|
+
import { chromium } from '@playwright/test';
|
|
23
|
+
import { existsSync } from 'node:fs';
|
|
24
|
+
/**
|
|
25
|
+
* True when a Playwright Chromium browser binary is present on disk AND the
|
|
26
|
+
* caller has not set PLAYWRIGHT_SKIP=1.
|
|
27
|
+
*/
|
|
28
|
+
export const chromiumAvailable = !process.env['PLAYWRIGHT_SKIP'] && existsSync(chromium.executablePath());
|
|
29
|
+
/**
|
|
30
|
+
* Human-readable reason string passed to `t.skip()` / `skipMessage` so the
|
|
31
|
+
* skip is self-documenting in test output.
|
|
32
|
+
*/
|
|
33
|
+
export const CHROMIUM_SKIP_REASON = process.env['PLAYWRIGHT_SKIP']
|
|
34
|
+
? 'PLAYWRIGHT_SKIP=1 is set — browser tests opted out. Unset PLAYWRIGHT_SKIP to enable.'
|
|
35
|
+
: 'Playwright Chromium is not installed. Run `npx playwright install chromium` to enable these tests.';
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CI-results evidence adapter (P4 — evidence collectors).
|
|
3
|
+
*
|
|
4
|
+
* Maps a CI run summary (test pass/fail counts, build status, optional flakiness
|
|
5
|
+
* data) into an `EvidenceItem` for `computeReleaseConfidence`, using the
|
|
6
|
+
* `ci-results` source kind reserved in confidence.schema.ts.
|
|
7
|
+
*
|
|
8
|
+
* Design:
|
|
9
|
+
* - Pure function — no I/O, no side effects. The caller owns fetching the CI data
|
|
10
|
+
* (e.g. from the `gh` API or a CI provider webhook); this adapter scores + maps.
|
|
11
|
+
* - Applicability:
|
|
12
|
+
* `applicable` — a real run with ≥1 test executed
|
|
13
|
+
* `not_applicable` — no CI run on record for this ref (e.g. no CI configured)
|
|
14
|
+
* `unknown` — run exists but results are incomplete/in-progress
|
|
15
|
+
* - Score formula: pass-rate * build-weight * freshness-factor
|
|
16
|
+
* pass-rate = passed / (passed + failed + errored) [0..1]
|
|
17
|
+
* build-weight = 0.85 if build succeeded, 0.0 if build failed (a build failure
|
|
18
|
+
* overrides the test pass-rate — tests that never ran are not "passing")
|
|
19
|
+
* freshness = 1.0 when ageSeconds < FRESH_THRESHOLD, decaying linearly to
|
|
20
|
+
* MIN_FRESHNESS over STALE_THRESHOLD. Beyond STALE_THRESHOLD the
|
|
21
|
+
* applicability is coerced to `unknown` (matches §2.6 rule 3 of the spec).
|
|
22
|
+
* - A build failure OR 0 tests passing marks evidence ['critical'] and forces a
|
|
23
|
+
* blocking recommendation (the aggregator uses recommendations for narrative).
|
|
24
|
+
* - Multi-tenant: tenantId flows through unchanged (caller sets it on the EvidenceItem
|
|
25
|
+
* via the `collector.tool` string; the full tenant stamp is the subject, not the item).
|
|
26
|
+
*/
|
|
27
|
+
import type { EvidenceItem } from '../schemas/confidence.schema.js';
|
|
28
|
+
/**
|
|
29
|
+
* Raw CI run data the caller provides (from `gh run view --json`, provider API, etc.).
|
|
30
|
+
* Only the counts matter for scoring; everything else is for evidence strings + provenance.
|
|
31
|
+
*/
|
|
32
|
+
export interface CiRunInput {
|
|
33
|
+
/** ISO-8601 timestamp when the run completed. */
|
|
34
|
+
completedAt: string;
|
|
35
|
+
/** Whether the CI build step itself succeeded (compilation, lint, etc. — before tests). */
|
|
36
|
+
buildPassed: boolean;
|
|
37
|
+
/** Number of test cases that passed. */
|
|
38
|
+
testsPassed: number;
|
|
39
|
+
/** Number of test cases that failed (hard). */
|
|
40
|
+
testsFailed: number;
|
|
41
|
+
/** Number of test cases that errored (infra/setup failure, not assertion failure). */
|
|
42
|
+
testsErrored: number;
|
|
43
|
+
/**
|
|
44
|
+
* Optional: number of tests that were flaky (passed on retry). Presence lowers score
|
|
45
|
+
* slightly but doesn't fail the run — flaky tests are a warn, not a block.
|
|
46
|
+
*/
|
|
47
|
+
testsFlaky?: number;
|
|
48
|
+
/**
|
|
49
|
+
* Optional: CI provider URL for the run (e.g. `https://github.com/…/actions/runs/…`).
|
|
50
|
+
* Never fabricated — omit rather than invent.
|
|
51
|
+
*/
|
|
52
|
+
runUrl?: string;
|
|
53
|
+
/** Optional: CI workflow/pipeline name for the evidence string. */
|
|
54
|
+
workflowName?: string;
|
|
55
|
+
/**
|
|
56
|
+
* Optional: collector freshness budget override (seconds). Defaults to 24 h for stale.
|
|
57
|
+
*/
|
|
58
|
+
staleAfterSeconds?: number;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Produce a `ci-results` EvidenceItem from a raw CI run summary.
|
|
62
|
+
*
|
|
63
|
+
* Deterministic and pure. Returns `not_applicable` when the run is absent, `unknown`
|
|
64
|
+
* when stale or incomplete, and `applicable` with a real score otherwise.
|
|
65
|
+
*/
|
|
66
|
+
export declare function ciResultsToEvidence(run: CiRunInput, collectedAt?: string): EvidenceItem;
|
|
67
|
+
//# sourceMappingURL=ci-results-adapter.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ci-results-adapter.d.ts","sourceRoot":"","sources":["../../src/adapters/ci-results-adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAsB,MAAM,iCAAiC,CAAC;AASxF;;;GAGG;AACH,MAAM,WAAW,UAAU;IACzB,iDAAiD;IACjD,WAAW,EAAE,MAAM,CAAC;IACpB,2FAA2F;IAC3F,WAAW,EAAE,OAAO,CAAC;IACrB,wCAAwC;IACxC,WAAW,EAAE,MAAM,CAAC;IACpB,+CAA+C;IAC/C,WAAW,EAAE,MAAM,CAAC;IACpB,sFAAsF;IACtF,YAAY,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,mEAAmE;IACnE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;OAEG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,UAAU,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,YAAY,CA+GvF"}
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CI-results evidence adapter (P4 — evidence collectors).
|
|
3
|
+
*
|
|
4
|
+
* Maps a CI run summary (test pass/fail counts, build status, optional flakiness
|
|
5
|
+
* data) into an `EvidenceItem` for `computeReleaseConfidence`, using the
|
|
6
|
+
* `ci-results` source kind reserved in confidence.schema.ts.
|
|
7
|
+
*
|
|
8
|
+
* Design:
|
|
9
|
+
* - Pure function — no I/O, no side effects. The caller owns fetching the CI data
|
|
10
|
+
* (e.g. from the `gh` API or a CI provider webhook); this adapter scores + maps.
|
|
11
|
+
* - Applicability:
|
|
12
|
+
* `applicable` — a real run with ≥1 test executed
|
|
13
|
+
* `not_applicable` — no CI run on record for this ref (e.g. no CI configured)
|
|
14
|
+
* `unknown` — run exists but results are incomplete/in-progress
|
|
15
|
+
* - Score formula: pass-rate * build-weight * freshness-factor
|
|
16
|
+
* pass-rate = passed / (passed + failed + errored) [0..1]
|
|
17
|
+
* build-weight = 0.85 if build succeeded, 0.0 if build failed (a build failure
|
|
18
|
+
* overrides the test pass-rate — tests that never ran are not "passing")
|
|
19
|
+
* freshness = 1.0 when ageSeconds < FRESH_THRESHOLD, decaying linearly to
|
|
20
|
+
* MIN_FRESHNESS over STALE_THRESHOLD. Beyond STALE_THRESHOLD the
|
|
21
|
+
* applicability is coerced to `unknown` (matches §2.6 rule 3 of the spec).
|
|
22
|
+
* - A build failure OR 0 tests passing marks evidence ['critical'] and forces a
|
|
23
|
+
* blocking recommendation (the aggregator uses recommendations for narrative).
|
|
24
|
+
* - Multi-tenant: tenantId flows through unchanged (caller sets it on the EvidenceItem
|
|
25
|
+
* via the `collector.tool` string; the full tenant stamp is the subject, not the item).
|
|
26
|
+
*/
|
|
27
|
+
// Freshness constants (seconds). Configurable at call-site via CiRunInput.
|
|
28
|
+
const DEFAULT_FRESH_THRESHOLD_S = 60 * 60 * 4; // 4 h
|
|
29
|
+
const DEFAULT_STALE_THRESHOLD_S = 60 * 60 * 24; // 24 h
|
|
30
|
+
const MIN_FRESHNESS = 0.5; // floor before we coerce to unknown
|
|
31
|
+
const CI_WEIGHT = 0.10; // matches DEFAULT_WEIGHTS['ci-results'] in confidence.ts
|
|
32
|
+
/**
|
|
33
|
+
* Produce a `ci-results` EvidenceItem from a raw CI run summary.
|
|
34
|
+
*
|
|
35
|
+
* Deterministic and pure. Returns `not_applicable` when the run is absent, `unknown`
|
|
36
|
+
* when stale or incomplete, and `applicable` with a real score otherwise.
|
|
37
|
+
*/
|
|
38
|
+
export function ciResultsToEvidence(run, collectedAt) {
|
|
39
|
+
const now = collectedAt ?? new Date().toISOString();
|
|
40
|
+
const source = 'ci-results';
|
|
41
|
+
const weight = CI_WEIGHT;
|
|
42
|
+
// Freshness computation.
|
|
43
|
+
const ageMs = Date.parse(now) - Date.parse(run.completedAt);
|
|
44
|
+
const ageS = Math.max(0, ageMs / 1000);
|
|
45
|
+
const staleThreshold = run.staleAfterSeconds ?? DEFAULT_STALE_THRESHOLD_S;
|
|
46
|
+
if (ageS > staleThreshold) {
|
|
47
|
+
return {
|
|
48
|
+
source,
|
|
49
|
+
score: 0,
|
|
50
|
+
weight,
|
|
51
|
+
applicability: 'unknown',
|
|
52
|
+
blocking: false,
|
|
53
|
+
evidence: [
|
|
54
|
+
`CI run completed at ${run.completedAt} — stale (${Math.round(ageS / 3600)}h > ${staleThreshold / 3600}h threshold).`,
|
|
55
|
+
],
|
|
56
|
+
recommendations: ['Re-run CI against the current commit before shipping.'],
|
|
57
|
+
reason: `CI run is stale (${Math.round(ageS / 3600)}h old, threshold ${staleThreshold / 3600}h).`,
|
|
58
|
+
collectedAt: now,
|
|
59
|
+
collector: { tool: 'qulib.ci-results-adapter' },
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
const total = run.testsPassed + run.testsFailed + run.testsErrored;
|
|
63
|
+
// Build failure: tests may not even have run — score 0, blocking recommendation.
|
|
64
|
+
if (!run.buildPassed) {
|
|
65
|
+
const evidence = [`Build FAILED (${run.workflowName ?? 'CI workflow'}).`];
|
|
66
|
+
if (total > 0)
|
|
67
|
+
evidence.push(`${run.testsPassed}/${total} tests passed before build failure.`);
|
|
68
|
+
if (run.runUrl)
|
|
69
|
+
evidence.push(`Run: ${run.runUrl}`);
|
|
70
|
+
return {
|
|
71
|
+
source,
|
|
72
|
+
score: 0,
|
|
73
|
+
weight,
|
|
74
|
+
applicability: 'applicable',
|
|
75
|
+
blocking: false, // the aggregator decides blocking; we report honestly
|
|
76
|
+
evidence,
|
|
77
|
+
recommendations: ['Fix the build failure before shipping.'],
|
|
78
|
+
collectedAt: now,
|
|
79
|
+
collector: { tool: 'qulib.ci-results-adapter', inputRef: run.runUrl },
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
// No tests ran at all — cannot score honestly.
|
|
83
|
+
if (total === 0) {
|
|
84
|
+
return {
|
|
85
|
+
source,
|
|
86
|
+
score: 0,
|
|
87
|
+
weight,
|
|
88
|
+
applicability: 'unknown',
|
|
89
|
+
blocking: false,
|
|
90
|
+
evidence: [
|
|
91
|
+
`Build passed (${run.workflowName ?? 'CI workflow'}) but 0 tests executed.`,
|
|
92
|
+
...(run.runUrl ? [`Run: ${run.runUrl}`] : []),
|
|
93
|
+
],
|
|
94
|
+
recommendations: ['Add a test suite to CI for a meaningful confidence signal.'],
|
|
95
|
+
reason: 'Build passed but zero tests were executed — no test signal.',
|
|
96
|
+
collectedAt: now,
|
|
97
|
+
collector: { tool: 'qulib.ci-results-adapter', inputRef: run.runUrl },
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
// Normal case: compute pass-rate.
|
|
101
|
+
const passRate = run.testsPassed / total; // 0..1
|
|
102
|
+
const freshnessRatio = ageS <= DEFAULT_FRESH_THRESHOLD_S
|
|
103
|
+
? 1.0
|
|
104
|
+
: MIN_FRESHNESS +
|
|
105
|
+
(1 - MIN_FRESHNESS) *
|
|
106
|
+
(1 - (ageS - DEFAULT_FRESH_THRESHOLD_S) / (staleThreshold - DEFAULT_FRESH_THRESHOLD_S));
|
|
107
|
+
const rawScore = passRate * freshnessRatio;
|
|
108
|
+
const score = Math.round(Math.max(0, Math.min(100, rawScore * 100)));
|
|
109
|
+
const evidence = [];
|
|
110
|
+
evidence.push(`${run.testsPassed}/${total} tests passed` +
|
|
111
|
+
(run.testsFailed > 0 ? ` (${run.testsFailed} failed)` : '') +
|
|
112
|
+
(run.testsErrored > 0 ? ` (${run.testsErrored} errored)` : '') +
|
|
113
|
+
` — ${Math.round(passRate * 100)}% pass-rate.`);
|
|
114
|
+
if (run.workflowName)
|
|
115
|
+
evidence.push(`Workflow: ${run.workflowName}.`);
|
|
116
|
+
if (run.testsFlaky && run.testsFlaky > 0) {
|
|
117
|
+
evidence.push(`${run.testsFlaky} test(s) flaky (passed on retry).`);
|
|
118
|
+
}
|
|
119
|
+
if (run.runUrl)
|
|
120
|
+
evidence.push(`Run: ${run.runUrl}`);
|
|
121
|
+
if (freshnessRatio < 1.0) {
|
|
122
|
+
evidence.push(`Freshness factor ${freshnessRatio.toFixed(2)} applied (run age ${Math.round(ageS / 3600)}h).`);
|
|
123
|
+
}
|
|
124
|
+
const recommendations = [];
|
|
125
|
+
if (run.testsFailed > 0)
|
|
126
|
+
recommendations.push(`Fix ${run.testsFailed} failing test(s) before shipping.`);
|
|
127
|
+
if (run.testsErrored > 0)
|
|
128
|
+
recommendations.push(`Investigate ${run.testsErrored} errored test(s) (infra/setup).`);
|
|
129
|
+
if (run.testsFlaky && run.testsFlaky > 0) {
|
|
130
|
+
recommendations.push(`Stabilize ${run.testsFlaky} flaky test(s) to improve signal quality.`);
|
|
131
|
+
}
|
|
132
|
+
return {
|
|
133
|
+
source,
|
|
134
|
+
score,
|
|
135
|
+
weight,
|
|
136
|
+
applicability: 'applicable',
|
|
137
|
+
blocking: false,
|
|
138
|
+
evidence,
|
|
139
|
+
recommendations,
|
|
140
|
+
collectedAt: now,
|
|
141
|
+
collector: { tool: 'qulib.ci-results-adapter', inputRef: run.runUrl },
|
|
142
|
+
};
|
|
143
|
+
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cypress-e2e-adapter.d.ts","sourceRoot":"","sources":["../../src/adapters/cypress-e2e-adapter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAY,MAAM,mCAAmC,CAAC;
|
|
1
|
+
{"version":3,"file":"cypress-e2e-adapter.d.ts","sourceRoot":"","sources":["../../src/adapters/cypress-e2e-adapter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAY,MAAM,mCAAmC,CAAC;AA+DlG,qBAAa,iBAAkB,YAAW,WAAW;IACnD,QAAQ,CAAC,WAAW,iBAAiB;IAErC,MAAM,CAAC,QAAQ,EAAE,eAAe,GAAG,aAAa;IAiChD,SAAS,CAAC,SAAS,EAAE,eAAe,EAAE,GAAG,aAAa,EAAE;CAGzD"}
|
|
@@ -38,15 +38,38 @@ function renderStep(step) {
|
|
|
38
38
|
return ` // TODO: ${step.description}`;
|
|
39
39
|
}
|
|
40
40
|
}
|
|
41
|
+
/**
|
|
42
|
+
* Render a recipe-specific step override for Cypress.
|
|
43
|
+
* Returns null when no override applies — falls through to renderStep.
|
|
44
|
+
*/
|
|
45
|
+
function renderRecipeStep(step, scenario) {
|
|
46
|
+
const tags = scenario.tags ?? [];
|
|
47
|
+
// a11y recipe: title assertion — cy.title() instead of cy.get('title')
|
|
48
|
+
if (tags.includes('recipe-a11y') && step.action === 'assert-text' && step.target === 'title') {
|
|
49
|
+
return ` cy.title().should('not.be.empty');`;
|
|
50
|
+
}
|
|
51
|
+
// a11y recipe: nav count using proper Cypress assertion
|
|
52
|
+
if (tags.includes('recipe-a11y') && step.action === 'assert-count') {
|
|
53
|
+
const t = step.target != null ? JSON.stringify(step.target) : null;
|
|
54
|
+
if (t) {
|
|
55
|
+
return ` cy.get(${t}).its('length').should('be.gte', ${parseInt(step.value ?? '1', 10)});`;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
return null;
|
|
59
|
+
}
|
|
41
60
|
export class CypressE2EAdapter {
|
|
42
61
|
adapterType = 'cypress-e2e';
|
|
43
62
|
render(scenario) {
|
|
44
63
|
const slug = slugify(scenario.title);
|
|
45
64
|
const filename = `${slug}.cy.ts`;
|
|
46
|
-
const stepLines = scenario.steps
|
|
65
|
+
const stepLines = scenario.steps
|
|
66
|
+
.map((step) => renderRecipeStep(step, scenario) ?? renderStep(step))
|
|
67
|
+
.join('\n');
|
|
68
|
+
const recipeTag = (scenario.tags ?? []).find((t) => t.startsWith('recipe-'));
|
|
69
|
+
const recipeComment = recipeTag ? `\n// recipe: ${recipeTag.replace('recipe-', '')}` : '';
|
|
47
70
|
const code = [
|
|
48
71
|
`// ${scenario.description}`,
|
|
49
|
-
`// qulib-generated — scenario: ${scenario.id}`,
|
|
72
|
+
`// qulib-generated — scenario: ${scenario.id}${recipeComment}`,
|
|
50
73
|
``,
|
|
51
74
|
`describe(${JSON.stringify(scenario.title)}, () => {`,
|
|
52
75
|
` it(${JSON.stringify(scenario.description)}, () => {`,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"playwright-adapter.d.ts","sourceRoot":"","sources":["../../src/adapters/playwright-adapter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAY,MAAM,mCAAmC,CAAC;
|
|
1
|
+
{"version":3,"file":"playwright-adapter.d.ts","sourceRoot":"","sources":["../../src/adapters/playwright-adapter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAY,MAAM,mCAAmC,CAAC;AAqElG,qBAAa,iBAAkB,YAAW,WAAW;IACnD,QAAQ,CAAC,WAAW,gBAAgB;IAEpC,MAAM,CAAC,QAAQ,EAAE,eAAe,GAAG,aAAa;IAmChD,SAAS,CAAC,SAAS,EAAE,eAAe,EAAE,GAAG,aAAa,EAAE;CAGzD"}
|
|
@@ -44,15 +44,38 @@ function renderStep(step) {
|
|
|
44
44
|
return ` // TODO: ${step.description}`;
|
|
45
45
|
}
|
|
46
46
|
}
|
|
47
|
+
/**
|
|
48
|
+
* Render a recipe-specific step override for Playwright.
|
|
49
|
+
* Returns null when no override applies — falls through to renderStep.
|
|
50
|
+
*/
|
|
51
|
+
function renderRecipeStep(step, scenario) {
|
|
52
|
+
const tags = scenario.tags ?? [];
|
|
53
|
+
// a11y recipe: title assertion — page.title() instead of page.locator('title')
|
|
54
|
+
if (tags.includes('recipe-a11y') && step.action === 'assert-text' && step.target === 'title') {
|
|
55
|
+
return ` expect(await page.title()).not.toBe('');`;
|
|
56
|
+
}
|
|
57
|
+
// a11y recipe: nav count using Playwright count()
|
|
58
|
+
if (tags.includes('recipe-a11y') && step.action === 'assert-count') {
|
|
59
|
+
const t = step.target != null ? JSON.stringify(step.target) : null;
|
|
60
|
+
if (t) {
|
|
61
|
+
return ` expect(await page.locator(${t}).count()).toBeGreaterThanOrEqual(${parseInt(step.value ?? '1', 10)});`;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
return null;
|
|
65
|
+
}
|
|
47
66
|
export class PlaywrightAdapter {
|
|
48
67
|
adapterType = 'playwright';
|
|
49
68
|
render(scenario) {
|
|
50
69
|
const slug = slugify(scenario.title);
|
|
51
70
|
const filename = `${slug}.spec.ts`;
|
|
52
|
-
const stepLines = scenario.steps
|
|
71
|
+
const stepLines = scenario.steps
|
|
72
|
+
.map((step) => renderRecipeStep(step, scenario) ?? renderStep(step))
|
|
73
|
+
.join('\n');
|
|
74
|
+
const recipeTag = (scenario.tags ?? []).find((t) => t.startsWith('recipe-'));
|
|
75
|
+
const recipeComment = recipeTag ? `\n// recipe: ${recipeTag.replace('recipe-', '')}` : '';
|
|
53
76
|
const code = [
|
|
54
77
|
`// ${scenario.description}`,
|
|
55
|
-
`// qulib-generated — scenario: ${scenario.id}`,
|
|
78
|
+
`// qulib-generated — scenario: ${scenario.id}${recipeComment}`,
|
|
56
79
|
``,
|
|
57
80
|
`import { test, expect } from '@playwright/test';`,
|
|
58
81
|
``,
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* PR-metadata evidence adapter (P4 — evidence collectors).
|
|
3
|
+
*
|
|
4
|
+
* Maps a pull-request review/checks payload (as returned by `gh pr view --json
|
|
5
|
+
* reviewDecision,statusCheckRollup,mergeable,number,url,additions,deletions`)
|
|
6
|
+
* into an `EvidenceItem` for `computeReleaseConfidence`, using the
|
|
7
|
+
* `deploy-metadata` source kind — the closest P3-reserved kind for
|
|
8
|
+
* PR-level ship-readiness.
|
|
9
|
+
*
|
|
10
|
+
* Design:
|
|
11
|
+
* - Pure function. The caller fetches the `gh` JSON; this adapter scores it.
|
|
12
|
+
* - Applicability:
|
|
13
|
+
* `applicable` — a PR exists and review/check state is readable
|
|
14
|
+
* `not_applicable` — no PR for this ref (direct push, script, etc.)
|
|
15
|
+
* `unknown` — PR exists but checks are still pending / state ambiguous
|
|
16
|
+
* - Score formula (0..100):
|
|
17
|
+
* Base 60 points: PR is open and merge-ready (no conflicts)
|
|
18
|
+
* +20: reviewDecision === 'APPROVED'
|
|
19
|
+
* +20: all status checks pass (statusCheckRollup every entry state === 'SUCCESS')
|
|
20
|
+
* Deductions:
|
|
21
|
+
* -10: any failing status check (per failing check, capped at 20)
|
|
22
|
+
* -15: reviewDecision === 'CHANGES_REQUESTED'
|
|
23
|
+
* - The adapter NEVER fabricates a PR number or URL; if absent from the payload
|
|
24
|
+
* the evidence strings omit them (WAVE-GUARDRAILS: no fabricated data).
|
|
25
|
+
*/
|
|
26
|
+
import type { EvidenceItem } from '../schemas/confidence.schema.js';
|
|
27
|
+
/**
|
|
28
|
+
* Status-check entry shape from `gh pr view --json statusCheckRollup`.
|
|
29
|
+
* Only the fields we actually use — extra fields are harmlessly ignored.
|
|
30
|
+
*/
|
|
31
|
+
export interface StatusCheck {
|
|
32
|
+
/** e.g. "SUCCESS", "FAILURE", "PENDING", "SKIPPED", "NEUTRAL" */
|
|
33
|
+
state: string;
|
|
34
|
+
/** CI job/check name — optional, used for the evidence string */
|
|
35
|
+
name?: string;
|
|
36
|
+
/** Direct URL to the check — never fabricated; omit rather than invent */
|
|
37
|
+
targetUrl?: string;
|
|
38
|
+
}
|
|
39
|
+
/** GitHub review decision values as returned by the `gh` CLI. */
|
|
40
|
+
export type ReviewDecision = 'APPROVED' | 'CHANGES_REQUESTED' | 'REVIEW_REQUIRED' | null | undefined;
|
|
41
|
+
/** Mergeable state as returned by `gh pr view --json mergeable`. */
|
|
42
|
+
export type MergeableState = 'MERGEABLE' | 'CONFLICTING' | 'UNKNOWN' | null | undefined;
|
|
43
|
+
/**
|
|
44
|
+
* Raw PR payload the caller provides. All fields are optional — the adapter
|
|
45
|
+
* degrades gracefully when partial data is available.
|
|
46
|
+
*/
|
|
47
|
+
export interface PrMetadataInput {
|
|
48
|
+
/** PR number. Never fabricated — omit if absent. */
|
|
49
|
+
number?: number;
|
|
50
|
+
/** PR URL. Never fabricated — omit if absent. */
|
|
51
|
+
url?: string;
|
|
52
|
+
/** Review decision from the `gh` response. null/undefined = no review yet. */
|
|
53
|
+
reviewDecision?: ReviewDecision;
|
|
54
|
+
/** Array of status check entries (the `statusCheckRollup` field). */
|
|
55
|
+
statusCheckRollup?: StatusCheck[];
|
|
56
|
+
/**
|
|
57
|
+
* Whether the PR is currently mergeable (no conflicts).
|
|
58
|
+
* null/UNKNOWN counts as unknown-state.
|
|
59
|
+
*/
|
|
60
|
+
mergeable?: MergeableState;
|
|
61
|
+
/** ISO-8601 when this payload was collected. Defaults to now. */
|
|
62
|
+
collectedAt?: string;
|
|
63
|
+
/**
|
|
64
|
+
* Set to true when there is explicitly NO PR for this ref (e.g. direct push).
|
|
65
|
+
* Produces a `not_applicable` contribution so the aggregator abstains honestly.
|
|
66
|
+
*/
|
|
67
|
+
noPr?: boolean;
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Produce a `deploy-metadata` EvidenceItem from a PR metadata payload.
|
|
71
|
+
* Returns `not_applicable` when `noPr` is true, `unknown` when checks are
|
|
72
|
+
* pending or state is ambiguous, and `applicable` with a real score otherwise.
|
|
73
|
+
*/
|
|
74
|
+
export declare function prMetadataToEvidence(input: PrMetadataInput, collectedAt?: string): EvidenceItem;
|
|
75
|
+
//# sourceMappingURL=pr-metadata-adapter.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pr-metadata-adapter.d.ts","sourceRoot":"","sources":["../../src/adapters/pr-metadata-adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAsB,MAAM,iCAAiC,CAAC;AAIxF;;;GAGG;AACH,MAAM,WAAW,WAAW;IAC1B,iEAAiE;IACjE,KAAK,EAAE,MAAM,CAAC;IACd,iEAAiE;IACjE,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,0EAA0E;IAC1E,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,iEAAiE;AACjE,MAAM,MAAM,cAAc,GAAG,UAAU,GAAG,mBAAmB,GAAG,iBAAiB,GAAG,IAAI,GAAG,SAAS,CAAC;AAErG,oEAAoE;AACpE,MAAM,MAAM,cAAc,GAAG,WAAW,GAAG,aAAa,GAAG,SAAS,GAAG,IAAI,GAAG,SAAS,CAAC;AAExF;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC9B,oDAAoD;IACpD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,iDAAiD;IACjD,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,8EAA8E;IAC9E,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,qEAAqE;IACrE,iBAAiB,CAAC,EAAE,WAAW,EAAE,CAAC;IAClC;;;OAGG;IACH,SAAS,CAAC,EAAE,cAAc,CAAC;IAC3B,iEAAiE;IACjE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAED;;;;GAIG;AACH,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,eAAe,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,YAAY,CA+H/F"}
|