@qulib/core 0.2.2 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -3
- package/dist/analyze.d.ts +16 -4
- package/dist/analyze.d.ts.map +1 -1
- package/dist/analyze.js +98 -38
- package/dist/cli/cost-doctor.d.ts +2 -0
- package/dist/cli/cost-doctor.d.ts.map +1 -0
- package/dist/cli/cost-doctor.js +72 -0
- package/dist/cli/index.js +14 -0
- package/dist/harness/progress-log.d.ts +7 -0
- package/dist/harness/progress-log.d.ts.map +1 -0
- package/dist/harness/progress-log.js +1 -0
- package/dist/harness/run-options.d.ts +2 -0
- package/dist/harness/run-options.d.ts.map +1 -1
- package/dist/index.d.ts +4 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/llm/content-hash.d.ts +2 -0
- package/dist/llm/content-hash.d.ts.map +1 -0
- package/dist/llm/content-hash.js +4 -0
- package/dist/llm/context-builder.js +1 -1
- package/dist/llm/cost-intelligence.d.ts +29 -0
- package/dist/llm/cost-intelligence.d.ts.map +1 -0
- package/dist/llm/cost-intelligence.js +153 -0
- package/dist/llm/provider.d.ts +11 -1
- package/dist/llm/provider.d.ts.map +1 -1
- package/dist/llm/provider.js +43 -4
- package/dist/phases/act.d.ts.map +1 -1
- package/dist/phases/act.js +4 -1
- package/dist/phases/observe.js +1 -1
- package/dist/phases/think-finalize.d.ts +6 -0
- package/dist/phases/think-finalize.d.ts.map +1 -0
- package/dist/phases/think-finalize.js +164 -0
- package/dist/phases/think.d.ts +2 -0
- package/dist/phases/think.d.ts.map +1 -1
- package/dist/phases/think.js +16 -65
- package/dist/reporters/markdown-reporter.d.ts.map +1 -1
- package/dist/reporters/markdown-reporter.js +23 -3
- package/dist/schemas/config.schema.d.ts +7 -0
- package/dist/schemas/config.schema.d.ts.map +1 -1
- package/dist/schemas/config.schema.js +18 -1
- package/dist/schemas/cost-intelligence.schema.d.ts +229 -0
- package/dist/schemas/cost-intelligence.schema.d.ts.map +1 -0
- package/dist/schemas/cost-intelligence.schema.js +41 -0
- package/dist/schemas/decision-log.schema.d.ts +2 -2
- package/dist/schemas/gap-analysis.schema.d.ts +270 -31
- package/dist/schemas/gap-analysis.schema.d.ts.map +1 -1
- package/dist/schemas/gap-analysis.schema.js +7 -3
- package/dist/schemas/index.d.ts +3 -1
- package/dist/schemas/index.d.ts.map +1 -1
- package/dist/schemas/index.js +3 -1
- package/dist/schemas/public-surface.schema.d.ts +268 -0
- package/dist/schemas/public-surface.schema.d.ts.map +1 -0
- package/dist/schemas/public-surface.schema.js +15 -0
- package/dist/tools/auth-block-gap.d.ts +3 -0
- package/dist/tools/auth-block-gap.d.ts.map +1 -0
- package/dist/tools/auth-block-gap.js +19 -0
- package/dist/tools/auth-detector.d.ts +2 -1
- package/dist/tools/auth-detector.d.ts.map +1 -1
- package/dist/tools/auth-detector.js +28 -3
- package/dist/tools/auth-explorer.d.ts +2 -1
- package/dist/tools/auth-explorer.d.ts.map +1 -1
- package/dist/tools/auth-explorer.js +30 -8
- package/dist/tools/auth-surface-analyzer.d.ts +4 -0
- package/dist/tools/auth-surface-analyzer.d.ts.map +1 -0
- package/dist/tools/auth-surface-analyzer.js +154 -0
- package/dist/tools/cypress-explorer.d.ts +2 -1
- package/dist/tools/cypress-explorer.d.ts.map +1 -1
- package/dist/tools/cypress-explorer.js +1 -1
- package/dist/tools/explorer.interface.d.ts +2 -1
- package/dist/tools/explorer.interface.d.ts.map +1 -1
- package/dist/tools/gap-engine.d.ts +3 -1
- package/dist/tools/gap-engine.d.ts.map +1 -1
- package/dist/tools/gap-engine.js +39 -12
- package/dist/tools/playwright-explorer.d.ts +2 -1
- package/dist/tools/playwright-explorer.d.ts.map +1 -1
- package/dist/tools/playwright-explorer.js +21 -3
- package/dist/tools/public-surface.d.ts +5 -0
- package/dist/tools/public-surface.d.ts.map +1 -0
- package/dist/tools/public-surface.js +13 -0
- package/package.json +7 -3
package/README.md
CHANGED
|
@@ -81,6 +81,29 @@ Or via MCP:
|
|
|
81
81
|
|
|
82
82
|
> "Use qulib's detect_auth tool on https://app.example.com — what's the recommended auth setup?"
|
|
83
83
|
|
|
84
|
+
## Release confidence
|
|
85
|
+
|
|
86
|
+
The score (0–100) is derived from **deterministic gaps** (untested routes vs repo, console errors, broken links, axe violations). High-severity items subtract more than low-severity ones. If **`coveragePagesScanned` is below `minPagesForConfidence`**, the score is **capped at 40** and `coverageWarning` is set to **`low-coverage`** so a shallow crawl cannot masquerade as high confidence.
|
|
87
|
+
|
|
88
|
+
When **`mode` is `auth-required`**, the scan never reached real app pages behind login: **release confidence is 0**, gaps are empty, and Cost Intelligence reflects the blocked state (L0 maturity).
|
|
89
|
+
|
|
90
|
+
## LLM scenario budget (naming)
|
|
91
|
+
|
|
92
|
+
- **`llmTokenBudget`** (legacy name, still required in config files): **max output tokens for a single** scenario-generation LLM completion. It maps to the provider’s **per-request completion cap**, not a multi-call or “whole run” token budget.
|
|
93
|
+
- **`llmMaxOutputTokensPerCall`** (optional): when set, **overrides** `llmTokenBudget` for the same purpose—clearer naming.
|
|
94
|
+
- **`enableLlmScenarios`**: when **`false`**, Qulib never calls an LLM for scenarios (templates only).
|
|
95
|
+
|
|
96
|
+
## Cost Intelligence and `qulib cost doctor`
|
|
97
|
+
|
|
98
|
+
After a normal **`analyze`**, `output/report.json` includes **`gapAnalysis.costIntelligence`**: usage records (**`actual`** vs **`estimated`** vs **`none`**), per-completion ceiling, budget warnings, repeated prompt fingerprints (when the same hash appears twice in one run), deterministic maturity (L0–L3 with an explicit ceiling for L4/L5), and conversion recommendations.
|
|
99
|
+
|
|
100
|
+
Re-print that block from disk:
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
npx tsx src/cli/index.ts cost doctor
|
|
104
|
+
# or: npx tsx src/cli/index.ts cost doctor --report output/report.json
|
|
105
|
+
```
|
|
106
|
+
|
|
84
107
|
## CLI (from npm)
|
|
85
108
|
|
|
86
109
|
```bash
|
|
@@ -101,6 +124,8 @@ const config: HarnessConfig = {
|
|
|
101
124
|
timeoutMs: 30000,
|
|
102
125
|
retryCount: 2,
|
|
103
126
|
llmTokenBudget: 4000,
|
|
127
|
+
llmMaxOutputTokensPerCall: undefined,
|
|
128
|
+
enableLlmScenarios: true,
|
|
104
129
|
testGenerationLimit: 10,
|
|
105
130
|
readOnlyMode: true,
|
|
106
131
|
requireHumanReview: true,
|
|
@@ -116,7 +141,7 @@ const result = await analyzeApp({
|
|
|
116
141
|
writeArtifacts: false,
|
|
117
142
|
});
|
|
118
143
|
|
|
119
|
-
console.log(result.releaseConfidence, result.gapAnalysis);
|
|
144
|
+
console.log(result.releaseConfidence, result.gapAnalysis.costIntelligence);
|
|
120
145
|
```
|
|
121
146
|
|
|
122
147
|
## Repository
|
|
@@ -134,7 +159,7 @@ This package is part of **[Qulib](https://github.com/TapeshN/qulib)** ([repo REA
|
|
|
134
159
|
- Optional **authenticated** crawling via `auth` in config (`form-login` or Playwright `storage-state`).
|
|
135
160
|
- Repo scanner: routes, tests, Cypress structure.
|
|
136
161
|
- Gap engine: deterministic gaps, **release confidence** with a low-page coverage floor, coverage warnings.
|
|
137
|
-
- Reports: `output/report.json` and `output/report.md` when not using **`--ephemeral
|
|
162
|
+
- Reports: `output/report.json` and `output/report.md` when not using **`--ephemeral`** (both include **Cost Intelligence** when present on `gapAnalysis`).
|
|
138
163
|
- State under `.scan-state/` unless **`--ephemeral`** (no disk writes; full JSON on stdout).
|
|
139
164
|
- **`npm run clean`** removes generated `output/` and `.scan-state/` and restores `.gitkeep` placeholders.
|
|
140
165
|
|
|
@@ -172,6 +197,9 @@ Use the same **hostname** for `--url` as your app’s canonical host when you ca
|
|
|
172
197
|
- `npm run dev` — CLI via `tsx` (append subcommands, e.g. `npm run dev -- clean`)
|
|
173
198
|
- `npm run analyze -- --url <url> [--repo <path>] [--config <file>] [--ephemeral]`
|
|
174
199
|
- `npm run clean` — reset `output/` and `.scan-state/` here
|
|
200
|
+
- `npm run test` — unit tests (cost intelligence + hashing)
|
|
201
|
+
- `npm run smoke` — ephemeral analyze of `https://example.com` (uses this package’s `qulib.config.ts`)
|
|
202
|
+
- `npm run cost-doctor` — print Cost Intelligence from `output/report.json` (run a non-ephemeral `analyze` first)
|
|
175
203
|
- `npm run build` — compile to `dist/`
|
|
176
204
|
|
|
177
205
|
From the **repository root**:
|
|
@@ -209,7 +237,7 @@ npx playwright install chromium
|
|
|
209
237
|
|
|
210
238
|
## Output and state (cwd = `packages/core` when you `cd` here)
|
|
211
239
|
|
|
212
|
-
**Ephemeral:** stdout prints one JSON object: `gapAnalysis
|
|
240
|
+
**Ephemeral:** stdout prints one JSON object: `gapAnalysis` (including **`costIntelligence`** when populated), `discoveredRoutes`, `repoInventory`, `decisionLog`.
|
|
213
241
|
|
|
214
242
|
**Persistent:**
|
|
215
243
|
|
package/dist/analyze.d.ts
CHANGED
|
@@ -1,22 +1,34 @@
|
|
|
1
|
-
import type
|
|
2
|
-
import {
|
|
3
|
-
import type
|
|
1
|
+
import { type HarnessConfig, type DetectedAuth } from './schemas/config.schema.js';
|
|
2
|
+
import type { Gap, GapAnalysis } from './schemas/gap-analysis.schema.js';
|
|
3
|
+
import { type RouteInventory } from './schemas/route-inventory.schema.js';
|
|
4
4
|
import type { RepoAnalysis } from './schemas/repo-analysis.schema.js';
|
|
5
5
|
import type { DecisionLogEntry } from './schemas/decision-log.schema.js';
|
|
6
|
+
import { type PublicSurface } from './schemas/public-surface.schema.js';
|
|
7
|
+
import type { AnalyzeProgressSink } from './harness/progress-log.js';
|
|
8
|
+
export type AnalyzeStatus = 'complete' | 'blocked' | 'partial';
|
|
6
9
|
export interface AnalyzeOptions {
|
|
7
10
|
url: string;
|
|
8
11
|
repoPath?: string;
|
|
9
12
|
config: HarnessConfig;
|
|
10
13
|
writeArtifacts?: boolean;
|
|
11
14
|
skipAuthDetection?: boolean;
|
|
15
|
+
progressLog?: AnalyzeProgressSink;
|
|
12
16
|
}
|
|
13
17
|
export interface AnalyzeResult {
|
|
14
|
-
|
|
18
|
+
status: AnalyzeStatus;
|
|
19
|
+
coverageScore: number | null;
|
|
20
|
+
/** Quality of evaluated pages only; `null` when no evaluable surface produced a score (fully blocked). */
|
|
21
|
+
releaseConfidence: number | null;
|
|
22
|
+
/** Same entries as `gapAnalysis.gaps` for consumers that read a flat `gaps` field. */
|
|
23
|
+
gaps: Gap[];
|
|
15
24
|
gapAnalysis: GapAnalysis;
|
|
25
|
+
/** Authenticated crawl scope only; empty routes when the scan stopped at an auth wall without credentials. */
|
|
16
26
|
routeInventory: RouteInventory;
|
|
17
27
|
repoInventory: RepoAnalysis | null;
|
|
18
28
|
decisionLog: DecisionLogEntry[];
|
|
19
29
|
detectedAuth?: DetectedAuth;
|
|
30
|
+
/** Crawled public routes and their gap-derived surface (complete scans); OAuth-wall scans use the pre-auth crawl only. */
|
|
31
|
+
publicSurface: PublicSurface | null;
|
|
20
32
|
}
|
|
21
33
|
export declare function analyzeApp(options: AnalyzeOptions): Promise<AnalyzeResult>;
|
|
22
34
|
//# sourceMappingURL=analyze.d.ts.map
|
package/dist/analyze.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"analyze.d.ts","sourceRoot":"","sources":["../src/analyze.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"analyze.d.ts","sourceRoot":"","sources":["../src/analyze.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,aAAa,EAAE,KAAK,YAAY,EAAE,MAAM,4BAA4B,CAAC;AACnF,OAAO,KAAK,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,kCAAkC,CAAC;AACzE,OAAO,EAAwB,KAAK,cAAc,EAAE,MAAM,qCAAqC,CAAC;AAChG,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AACtE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kCAAkC,CAAC;AACzE,OAAO,EAAuB,KAAK,aAAa,EAAE,MAAM,oCAAoC,CAAC;AAU7F,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAErE,MAAM,MAAM,aAAa,GAAG,UAAU,GAAG,SAAS,GAAG,SAAS,CAAC;AAE/D,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,aAAa,CAAC;IACtB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,WAAW,CAAC,EAAE,mBAAmB,CAAC;CACnC;AAED,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,aAAa,CAAC;IACtB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,0GAA0G;IAC1G,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,sFAAsF;IACtF,IAAI,EAAE,GAAG,EAAE,CAAC;IACZ,WAAW,EAAE,WAAW,CAAC;IACzB,8GAA8G;IAC9G,cAAc,EAAE,cAAc,CAAC;IAC/B,aAAa,EAAE,YAAY,GAAG,IAAI,CAAC;IACnC,WAAW,EAAE,gBAAgB,EAAE,CAAC;IAChC,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,0HAA0H;IAC1H,aAAa,EAAE,aAAa,GAAG,IAAI,CAAC;CACrC;AAcD,wBAAsB,UAAU,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC,CA8HhF"}
|
package/dist/analyze.js
CHANGED
|
@@ -1,61 +1,121 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { RouteInventorySchema } from './schemas/route-inventory.schema.js';
|
|
2
|
+
import { PublicSurfaceSchema } from './schemas/public-surface.schema.js';
|
|
2
3
|
import { observe } from './phases/observe.js';
|
|
3
4
|
import { think } from './phases/think.js';
|
|
4
5
|
import { act } from './phases/act.js';
|
|
5
6
|
import { detectAuth } from './tools/auth-detector.js';
|
|
7
|
+
import { analyzeGaps, computeCoverageScore, computeQualityScoreFromGaps } from './tools/gap-engine.js';
|
|
8
|
+
import { analyzeAuthSurfaceGaps } from './tools/auth-surface-analyzer.js';
|
|
9
|
+
import { buildPublicSurface } from './tools/public-surface.js';
|
|
10
|
+
import { buildAuthBlockGap } from './tools/auth-block-gap.js';
|
|
11
|
+
import { finalizeGapAnalysisFromDraft } from './phases/think-finalize.js';
|
|
12
|
+
function logScanEnd(progress, result) {
|
|
13
|
+
const rc = result.releaseConfidence === null ? 'null' : String(result.releaseConfidence);
|
|
14
|
+
const cs = result.coverageScore === null ? 'null' : String(result.coverageScore);
|
|
15
|
+
progress?.info(`status=${result.status} | coverageScore=${cs} | releaseConfidence=${rc} | gaps=${result.gaps.length}`);
|
|
16
|
+
for (const g of result.gaps) {
|
|
17
|
+
progress?.debug(`gap id=${g.id} severity=${g.severity} category=${g.category}`);
|
|
18
|
+
}
|
|
19
|
+
if (process.env.QULIB_DEBUG === '1') {
|
|
20
|
+
progress?.debug(`gaps json=${JSON.stringify(result.gaps)}`);
|
|
21
|
+
}
|
|
22
|
+
}
|
|
6
23
|
export async function analyzeApp(options) {
|
|
7
24
|
const writeArtifacts = options.writeArtifacts ?? false;
|
|
8
25
|
const decisionLog = [];
|
|
26
|
+
const progress = options.progressLog;
|
|
9
27
|
const artifacts = {
|
|
10
28
|
writeArtifacts,
|
|
11
29
|
decisionMemory: decisionLog,
|
|
30
|
+
...(progress !== undefined && { progressLog: progress }),
|
|
12
31
|
};
|
|
32
|
+
progress?.info(`Starting scan → ${options.url} maxPagesToScan=${options.config.maxPagesToScan}`);
|
|
33
|
+
let detectedAuth;
|
|
34
|
+
let authWall = false;
|
|
13
35
|
if (!options.config.auth && !options.skipAuthDetection) {
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
const gapAnalysis = GapAnalysisSchema.parse({
|
|
17
|
-
analyzedAt: new Date().toISOString(),
|
|
18
|
-
mode: 'auth-required',
|
|
19
|
-
releaseConfidence: 0,
|
|
20
|
-
coveragePagesScanned: 0,
|
|
21
|
-
coverageBudgetExceeded: false,
|
|
22
|
-
coverageWarning: 'auth-required',
|
|
23
|
-
gaps: [],
|
|
24
|
-
scenarios: [],
|
|
25
|
-
generatedTests: [],
|
|
26
|
-
});
|
|
27
|
-
return {
|
|
28
|
-
releaseConfidence: 0,
|
|
29
|
-
gapAnalysis,
|
|
30
|
-
routeInventory: {
|
|
31
|
-
scannedAt: new Date().toISOString(),
|
|
32
|
-
baseUrl: options.url,
|
|
33
|
-
routes: [],
|
|
34
|
-
pagesSkipped: 0,
|
|
35
|
-
budgetExceeded: false,
|
|
36
|
-
},
|
|
37
|
-
repoInventory: null,
|
|
38
|
-
decisionLog: [
|
|
39
|
-
{
|
|
40
|
-
timestamp: new Date().toISOString(),
|
|
41
|
-
phase: 'observe',
|
|
42
|
-
decision: 'auth-required',
|
|
43
|
-
reason: detection.recommendation,
|
|
44
|
-
metadata: { detection },
|
|
45
|
-
},
|
|
46
|
-
],
|
|
47
|
-
detectedAuth: detection,
|
|
48
|
-
};
|
|
49
|
-
}
|
|
36
|
+
detectedAuth = await detectAuth(options.url, options.config.timeoutMs, progress);
|
|
37
|
+
authWall = Boolean(detectedAuth.hasAuth);
|
|
50
38
|
}
|
|
51
39
|
const observed = await observe(options.url, options.repoPath, options.config, artifacts);
|
|
40
|
+
if (authWall && !options.config.auth && detectedAuth) {
|
|
41
|
+
decisionLog.push({
|
|
42
|
+
timestamp: new Date().toISOString(),
|
|
43
|
+
phase: 'observe',
|
|
44
|
+
decision: 'auth-required',
|
|
45
|
+
reason: detectedAuth.recommendation,
|
|
46
|
+
metadata: { detection: detectedAuth },
|
|
47
|
+
});
|
|
48
|
+
const status = observed.routes.routes.length === 0 ? 'blocked' : 'partial';
|
|
49
|
+
if (status === 'blocked') {
|
|
50
|
+
progress?.warn('Scan blocked by auth wall');
|
|
51
|
+
}
|
|
52
|
+
else {
|
|
53
|
+
progress?.warn('Auth wall: continuing with public surface only (partial)');
|
|
54
|
+
}
|
|
55
|
+
const mode = observed.repo ? 'url-repo' : 'url-only';
|
|
56
|
+
const publicAnalysis = analyzeGaps(observed.routes, observed.repo, mode, options.config);
|
|
57
|
+
const publicSurface = PublicSurfaceSchema.parse(buildPublicSurface(observed.routes.routes, publicAnalysis.gaps));
|
|
58
|
+
progress?.info(`Public surface crawl: ${publicSurface.pages.length} page(s) reachable pre-auth`);
|
|
59
|
+
const authSurfaceGaps = await analyzeAuthSurfaceGaps(options.url, detectedAuth, options.config.timeoutMs);
|
|
60
|
+
const authBlockGap = buildAuthBlockGap(options.url);
|
|
61
|
+
const qualityInputGaps = [...publicAnalysis.gaps, ...authSurfaceGaps];
|
|
62
|
+
const qualityScore = computeQualityScoreFromGaps(qualityInputGaps);
|
|
63
|
+
const draftRelease = status === 'blocked' ? null : qualityScore;
|
|
64
|
+
const draft = {
|
|
65
|
+
analyzedAt: new Date().toISOString(),
|
|
66
|
+
mode: 'auth-required',
|
|
67
|
+
releaseConfidence: draftRelease,
|
|
68
|
+
coveragePagesScanned: 0,
|
|
69
|
+
coverageBudgetExceeded: false,
|
|
70
|
+
coverageWarning: 'auth-required',
|
|
71
|
+
gaps: [...authSurfaceGaps, authBlockGap],
|
|
72
|
+
};
|
|
73
|
+
const costContext = {
|
|
74
|
+
mode: publicAnalysis.mode,
|
|
75
|
+
coveragePagesScanned: observed.routes.routes.length,
|
|
76
|
+
releaseConfidence: qualityScore,
|
|
77
|
+
gaps: publicAnalysis.gaps,
|
|
78
|
+
};
|
|
79
|
+
const gapAnalysis = await finalizeGapAnalysisFromDraft(draft, options.config, artifacts, costContext);
|
|
80
|
+
const emptyAuthRoutes = RouteInventorySchema.parse({
|
|
81
|
+
scannedAt: new Date().toISOString(),
|
|
82
|
+
baseUrl: options.url,
|
|
83
|
+
routes: [],
|
|
84
|
+
pagesSkipped: 0,
|
|
85
|
+
budgetExceeded: false,
|
|
86
|
+
});
|
|
87
|
+
await act(gapAnalysis, options.config, artifacts);
|
|
88
|
+
const result = {
|
|
89
|
+
status,
|
|
90
|
+
coverageScore: computeCoverageScore(observed.routes),
|
|
91
|
+
releaseConfidence: draftRelease,
|
|
92
|
+
gaps: gapAnalysis.gaps,
|
|
93
|
+
gapAnalysis,
|
|
94
|
+
routeInventory: emptyAuthRoutes,
|
|
95
|
+
repoInventory: observed.repo,
|
|
96
|
+
decisionLog,
|
|
97
|
+
detectedAuth,
|
|
98
|
+
publicSurface,
|
|
99
|
+
};
|
|
100
|
+
logScanEnd(progress, result);
|
|
101
|
+
return result;
|
|
102
|
+
}
|
|
52
103
|
const analysis = await think(observed, options.config, artifacts);
|
|
53
104
|
await act(analysis, options.config, artifacts);
|
|
54
|
-
|
|
105
|
+
const publicSurface = PublicSurfaceSchema.parse(buildPublicSurface(observed.routes.routes, analysis.gaps));
|
|
106
|
+
progress?.info(`Public surface crawl: ${publicSurface.pages.length} page(s) reachable pre-auth`);
|
|
107
|
+
const result = {
|
|
108
|
+
status: 'complete',
|
|
109
|
+
coverageScore: computeCoverageScore(observed.routes),
|
|
55
110
|
releaseConfidence: analysis.releaseConfidence,
|
|
111
|
+
gaps: analysis.gaps,
|
|
56
112
|
gapAnalysis: analysis,
|
|
57
113
|
routeInventory: observed.routes,
|
|
58
114
|
repoInventory: observed.repo,
|
|
59
115
|
decisionLog,
|
|
116
|
+
...(detectedAuth !== undefined && { detectedAuth }),
|
|
117
|
+
publicSurface,
|
|
60
118
|
};
|
|
119
|
+
logScanEnd(progress, result);
|
|
120
|
+
return result;
|
|
61
121
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cost-doctor.d.ts","sourceRoot":"","sources":["../../src/cli/cost-doctor.ts"],"names":[],"mappings":"AAIA,wBAAsB,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CA6ErE"}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import { readFile } from 'node:fs/promises';
|
|
2
|
+
import { resolve } from 'node:path';
|
|
3
|
+
import { GapAnalysisSchema } from '../schemas/gap-analysis.schema.js';
|
|
4
|
+
export async function runCostDoctor(reportPath) {
|
|
5
|
+
const abs = resolve(process.cwd(), reportPath);
|
|
6
|
+
let raw;
|
|
7
|
+
try {
|
|
8
|
+
raw = await readFile(abs, 'utf8');
|
|
9
|
+
}
|
|
10
|
+
catch {
|
|
11
|
+
throw new Error(`Could not read ${abs}. Run \`qulib analyze --url <url>\` (without --ephemeral) from this directory first.`);
|
|
12
|
+
}
|
|
13
|
+
const parsed = GapAnalysisSchema.safeParse(JSON.parse(raw));
|
|
14
|
+
if (!parsed.success) {
|
|
15
|
+
throw new Error('File is not a valid gap analysis report (report.json schema mismatch).');
|
|
16
|
+
}
|
|
17
|
+
const ci = parsed.data.costIntelligence;
|
|
18
|
+
if (!ci) {
|
|
19
|
+
console.log('[qulib] No costIntelligence in this report (older scan). Re-run analyze with the current qulib version to populate Cost Intelligence.');
|
|
20
|
+
return;
|
|
21
|
+
}
|
|
22
|
+
console.log('# Qulib cost doctor\n');
|
|
23
|
+
console.log(`Report: ${abs}`);
|
|
24
|
+
console.log(`Analyzed at: ${parsed.data.analyzedAt}\n`);
|
|
25
|
+
console.log('## Token ceiling (per LLM completion)\n');
|
|
26
|
+
console.log(`- maxOutputTokensPerLlmCall: ${ci.maxOutputTokensPerLlmCall}`);
|
|
27
|
+
console.log(`- budgetRole: ${ci.budgetRole}\n`);
|
|
28
|
+
console.log('## Usage (this scan)\n');
|
|
29
|
+
console.log(`- Input / output tokens: ${ci.usageSummary.totalInputTokens} / ${ci.usageSummary.totalOutputTokens} (${ci.usageSummary.dataQuality})`);
|
|
30
|
+
if (ci.budgetWarnings.length) {
|
|
31
|
+
console.log('\n## Budget warnings\n');
|
|
32
|
+
for (const w of ci.budgetWarnings) {
|
|
33
|
+
console.log(`- ${w}`);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
else {
|
|
37
|
+
console.log('\n## Budget warnings\n\n- (none)\n');
|
|
38
|
+
}
|
|
39
|
+
if (ci.repeatedOperations.length) {
|
|
40
|
+
console.log('\n## Repeated AI patterns\n');
|
|
41
|
+
for (const r of ci.repeatedOperations) {
|
|
42
|
+
console.log(`- ${r.promptHash} ×${r.count}`);
|
|
43
|
+
console.log(` ${r.recommendation}`);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
else {
|
|
47
|
+
console.log('\n## Repeated AI patterns\n\n- (none in this run)\n');
|
|
48
|
+
}
|
|
49
|
+
console.log('\n## Deterministic maturity\n');
|
|
50
|
+
console.log(`- ${ci.deterministicMaturity.label}`);
|
|
51
|
+
console.log(`- ${ci.deterministicMaturity.rationale}`);
|
|
52
|
+
if (ci.deterministicMaturity.ceilingNote) {
|
|
53
|
+
console.log(`- _${ci.deterministicMaturity.ceilingNote}_`);
|
|
54
|
+
}
|
|
55
|
+
console.log('\n## Conversion recommendations\n');
|
|
56
|
+
for (const c of ci.conversionRecommendations) {
|
|
57
|
+
console.log(`- ${c}`);
|
|
58
|
+
}
|
|
59
|
+
const topGap = [...parsed.data.gaps].sort((a, b) => {
|
|
60
|
+
const o = { critical: 0, high: 1, medium: 2, low: 3 };
|
|
61
|
+
return o[a.severity] - o[b.severity];
|
|
62
|
+
})[0];
|
|
63
|
+
console.log('\n## Next best deterministic check\n');
|
|
64
|
+
if (topGap) {
|
|
65
|
+
console.log(`- Prioritize **${topGap.category}** on \`${topGap.path}\` (${topGap.severity}): ${topGap.reason}`);
|
|
66
|
+
}
|
|
67
|
+
else {
|
|
68
|
+
console.log('- No gaps in this report; extend crawl coverage or add auth before chasing new checks.');
|
|
69
|
+
}
|
|
70
|
+
console.log('\n---\n');
|
|
71
|
+
console.log('TODO: correlate multiple historical reports and CI adapters for cross-run “cost doctor” diffing (not implemented yet).');
|
|
72
|
+
}
|
package/dist/cli/index.js
CHANGED
|
@@ -93,8 +93,13 @@ async function runAnalyze(options) {
|
|
|
93
93
|
});
|
|
94
94
|
if (ephemeral) {
|
|
95
95
|
console.log(JSON.stringify({
|
|
96
|
+
status: result.status,
|
|
97
|
+
coverageScore: result.coverageScore,
|
|
98
|
+
releaseConfidence: result.releaseConfidence,
|
|
99
|
+
gaps: result.gaps,
|
|
96
100
|
gapAnalysis: result.gapAnalysis,
|
|
97
101
|
discoveredRoutes: result.routeInventory,
|
|
102
|
+
publicSurface: result.publicSurface,
|
|
98
103
|
repoInventory: result.repoInventory,
|
|
99
104
|
decisionLog: result.decisionLog,
|
|
100
105
|
...(result.detectedAuth !== undefined && { detectedAuth: result.detectedAuth }),
|
|
@@ -126,6 +131,15 @@ program
|
|
|
126
131
|
await fs.writeFile('.scan-state/.gitkeep', '', 'utf8');
|
|
127
132
|
console.log('[qulib] clean complete');
|
|
128
133
|
});
|
|
134
|
+
const costCmd = program.command('cost').description('Cost intelligence helpers');
|
|
135
|
+
costCmd
|
|
136
|
+
.command('doctor')
|
|
137
|
+
.description('Print Cost Intelligence from output/report.json (run analyze without --ephemeral first)')
|
|
138
|
+
.option('--report <file>', 'Path to report.json relative to cwd', 'output/report.json')
|
|
139
|
+
.action(async (opts) => {
|
|
140
|
+
const { runCostDoctor } = await import('./cost-doctor.js');
|
|
141
|
+
await runCostDoctor(opts.report);
|
|
142
|
+
});
|
|
129
143
|
program
|
|
130
144
|
.command('analyze')
|
|
131
145
|
.description('Analyze an app for quality gaps')
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"progress-log.d.ts","sourceRoot":"","sources":["../../src/harness/progress-log.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,mBAAmB;IAClC,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import type { DecisionLogEntry } from '../schemas/decision-log.schema.js';
|
|
2
|
+
import type { AnalyzeProgressSink } from './progress-log.js';
|
|
2
3
|
export type RunArtifactsOptions = {
|
|
3
4
|
writeArtifacts: boolean;
|
|
4
5
|
decisionMemory?: DecisionLogEntry[];
|
|
6
|
+
progressLog?: AnalyzeProgressSink;
|
|
5
7
|
};
|
|
6
8
|
//# sourceMappingURL=run-options.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run-options.d.ts","sourceRoot":"","sources":["../../src/harness/run-options.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mCAAmC,CAAC;
|
|
1
|
+
{"version":3,"file":"run-options.d.ts","sourceRoot":"","sources":["../../src/harness/run-options.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mCAAmC,CAAC;AAC1E,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAE7D,MAAM,MAAM,mBAAmB,GAAG;IAChC,cAAc,EAAE,OAAO,CAAC;IACxB,cAAc,CAAC,EAAE,gBAAgB,EAAE,CAAC;IACpC,WAAW,CAAC,EAAE,mBAAmB,CAAC;CACnC,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -2,6 +2,8 @@ export { analyzeApp } from './analyze.js';
|
|
|
2
2
|
export { detectAuth } from './tools/auth-detector.js';
|
|
3
3
|
export { exploreAuth } from './tools/auth-explorer.js';
|
|
4
4
|
export { addUserProvider, removeUserProvider, listUserProviders } from './tools/user-providers.js';
|
|
5
|
-
export
|
|
6
|
-
export type {
|
|
5
|
+
export { resolveMaxOutputTokensPerLlmCall } from './schemas/config.schema.js';
|
|
6
|
+
export type { AnalyzeOptions, AnalyzeResult, AnalyzeStatus } from './analyze.js';
|
|
7
|
+
export type { AnalyzeProgressSink } from './harness/progress-log.js';
|
|
8
|
+
export type { HarnessConfig, AuthConfig, RouteInventory, GapAnalysis, RepoAnalysis, DetectedAuth, AuthExploration, AuthPath, AuthPathRequirements, CostIntelligence, LlmUsageRecord, RepeatedAiPattern, DeterministicMaturity, PublicSurface, } from './schemas/index.js';
|
|
7
9
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AACnG,YAAY,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AACnG,OAAO,EAAE,gCAAgC,EAAE,MAAM,4BAA4B,CAAC;AAC9E,YAAY,EAAE,cAAc,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AACjF,YAAY,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AACrE,YAAY,EACV,aAAa,EACb,UAAU,EACV,cAAc,EACd,WAAW,EACX,YAAY,EACZ,YAAY,EACZ,eAAe,EACf,QAAQ,EACR,oBAAoB,EACpB,gBAAgB,EAChB,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,aAAa,GACd,MAAM,oBAAoB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -2,3 +2,4 @@ export { analyzeApp } from './analyze.js';
|
|
|
2
2
|
export { detectAuth } from './tools/auth-detector.js';
|
|
3
3
|
export { exploreAuth } from './tools/auth-explorer.js';
|
|
4
4
|
export { addUserProvider, removeUserProvider, listUserProviders } from './tools/user-providers.js';
|
|
5
|
+
export { resolveMaxOutputTokensPerLlmCall } from './schemas/config.schema.js';
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"content-hash.d.ts","sourceRoot":"","sources":["../../src/llm/content-hash.ts"],"names":[],"mappings":"AAEA,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAE/D"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
export function buildGapPrompt(gaps, limit) {
|
|
2
2
|
const topGaps = [...gaps]
|
|
3
3
|
.sort((a, b) => {
|
|
4
|
-
const order = {
|
|
4
|
+
const order = { critical: 0, high: 1, medium: 2, low: 3 };
|
|
5
5
|
return order[a.severity] - order[b.severity];
|
|
6
6
|
})
|
|
7
7
|
.slice(0, limit);
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { GapAnalysis } from '../schemas/gap-analysis.schema.js';
|
|
2
|
+
import type { CostIntelligence, DeterministicMaturity, LlmUsageRecord, RepeatedAiPattern } from '../schemas/cost-intelligence.schema.js';
|
|
3
|
+
export declare function summarizeUsageQuality(records: LlmUsageRecord[]): CostIntelligence['usageSummary'];
|
|
4
|
+
export declare function buildBudgetWarnings(records: LlmUsageRecord[], maxOutputTokensPerLlmCall: number): string[];
|
|
5
|
+
export declare function findRepeatedPromptPatterns(records: LlmUsageRecord[]): RepeatedAiPattern[];
|
|
6
|
+
export declare function buildConversionRecommendations(params: {
|
|
7
|
+
scenarioSource: 'llm' | 'template';
|
|
8
|
+
repeatedOperations: RepeatedAiPattern[];
|
|
9
|
+
budgetWarnings: string[];
|
|
10
|
+
gapCount: number;
|
|
11
|
+
}): string[];
|
|
12
|
+
export declare function computeDeterministicMaturity(params: {
|
|
13
|
+
mode: GapAnalysis['mode'];
|
|
14
|
+
coveragePagesScanned: number;
|
|
15
|
+
gapCount: number;
|
|
16
|
+
scenarioSource: 'llm' | 'template';
|
|
17
|
+
repeatedOperations: RepeatedAiPattern[];
|
|
18
|
+
releaseConfidence: number | null;
|
|
19
|
+
requireHumanReview: boolean;
|
|
20
|
+
}): DeterministicMaturity;
|
|
21
|
+
export declare function assembleCostIntelligence(params: {
|
|
22
|
+
maxOutputTokensPerLlmCall: number;
|
|
23
|
+
records: LlmUsageRecord[];
|
|
24
|
+
partial: Pick<GapAnalysis, 'mode' | 'coveragePagesScanned' | 'releaseConfidence' | 'gaps'>;
|
|
25
|
+
scenarioSource: 'llm' | 'template';
|
|
26
|
+
requireHumanReview: boolean;
|
|
27
|
+
}): CostIntelligence;
|
|
28
|
+
export declare function costIntelligenceForAuthBlocked(maxOutputTokensPerLlmCall: number): CostIntelligence;
|
|
29
|
+
//# sourceMappingURL=cost-intelligence.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cost-intelligence.d.ts","sourceRoot":"","sources":["../../src/llm/cost-intelligence.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mCAAmC,CAAC;AACrE,OAAO,KAAK,EACV,gBAAgB,EAChB,qBAAqB,EACrB,cAAc,EACd,iBAAiB,EAClB,MAAM,wCAAwC,CAAC;AAEhD,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,cAAc,EAAE,GAAG,gBAAgB,CAAC,cAAc,CAAC,CAYjG;AAED,wBAAgB,mBAAmB,CACjC,OAAO,EAAE,cAAc,EAAE,EACzB,yBAAyB,EAAE,MAAM,GAChC,MAAM,EAAE,CAuBV;AAED,wBAAgB,0BAA0B,CAAC,OAAO,EAAE,cAAc,EAAE,GAAG,iBAAiB,EAAE,CAiBzF;AAED,wBAAgB,8BAA8B,CAAC,MAAM,EAAE;IACrD,cAAc,EAAE,KAAK,GAAG,UAAU,CAAC;IACnC,kBAAkB,EAAE,iBAAiB,EAAE,CAAC;IACxC,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;CAClB,GAAG,MAAM,EAAE,CAuBX;AAED,wBAAgB,4BAA4B,CAAC,MAAM,EAAE;IACnD,IAAI,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IAC1B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,KAAK,GAAG,UAAU,CAAC;IACnC,kBAAkB,EAAE,iBAAiB,EAAE,CAAC;IACxC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,kBAAkB,EAAE,OAAO,CAAC;CAC7B,GAAG,qBAAqB,CA+CxB;AAED,wBAAgB,wBAAwB,CAAC,MAAM,EAAE;IAC/C,yBAAyB,EAAE,MAAM,CAAC;IAClC,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,OAAO,EAAE,IAAI,CACX,WAAW,EACX,MAAM,GAAG,sBAAsB,GAAG,mBAAmB,GAAG,MAAM,CAC/D,CAAC;IACF,cAAc,EAAE,KAAK,GAAG,UAAU,CAAC;IACnC,kBAAkB,EAAE,OAAO,CAAC;CAC7B,GAAG,gBAAgB,CA8BnB;AAED,wBAAgB,8BAA8B,CAAC,yBAAyB,EAAE,MAAM,GAAG,gBAAgB,CAoBlG"}
|