@thomasminh1995/depverdict 0.6.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +518 -0
- package/bin/depverdict.js +7 -0
- package/bin/upgradelens.js +7 -0
- package/docs/GR-01-Semantic-Grounding-Failure-Analysis.md +309 -0
- package/docs/GR-02-Versioned-Action-Evaluation-Criteria.md +187 -0
- package/docs/GR-03-Extractive-Contract-Safety-Experiment.md +242 -0
- package/docs/GR-04-Versioned-Production-Extractive-Contract.md +154 -0
- package/docs/IA-01-Repository-Usage-Discovery.md +122 -0
- package/docs/IA-02-Repository-Impact-Analysis.md +118 -0
- package/docs/IA-03-Repository-Impact-Evidence.md +160 -0
- package/docs/IA-04-CLI-Orchestration.md +235 -0
- package/docs/IA-05-Real-Provider-Validation.md +235 -0
- package/docs/IA-05-VinGrade-Validation.md +339 -0
- package/docs/MVP-01.md +48 -0
- package/docs/MVP-02-Architecture.md +536 -0
- package/docs/MVP-02-CLI-HTTP-Runtime.md +50 -0
- package/docs/MVP-02-HTTP-Lifecycle.md +37 -0
- package/docs/MVP-02-Knowledge-Manifest-Generation.md +75 -0
- package/docs/MVP-02-Knowledge-Manifest.md +279 -0
- package/docs/MVP-02-Knowledge-Research-Orchestration.md +62 -0
- package/docs/MVP-02-Knowledge-Store.md +87 -0
- package/docs/MVP-02-PyPI-Registry-Adapter.md +97 -0
- package/docs/MVP-02-Research-Planning.md +196 -0
- package/docs/MVP-02-Source-Provenance.md +81 -0
- package/docs/MVP-02-npm-Registry-Adapter.md +99 -0
- package/docs/RR-01-End-to-End-and-Real-Provider-Validation.md +393 -0
- package/docs/RR-01-RERUN-Extractive-Contract-Validation.md +566 -0
- package/docs/RR-02-Full-Product-Workflow-and-Developer-CLI-UX-Review.md +35 -0
- package/docs/RR02-FIX-01-Persistent-Qualification-Resolution.md +32 -0
- package/docs/RR02-FIX-02-Stage-aware-CLI-Progress-and-Heartbeat.md +28 -0
- package/docs/RR02-FIX-03-npm-Capture-Evidence-Exclusion.md +33 -0
- package/docs/RR02-FIX-03A-Complete-Package-Exclusion-and-Evidence-Commit.md +223 -0
- package/docs/RR02-FIX-04-Event-loop-safe-Heartbeat.md +393 -0
- package/docs/RR02-FIX-05-Materialize-Persisted-Qualification.md +207 -0
- package/docs/RR02-RERUN-CLI-Qualification-Progress-UX-and-Package-Validation.md +342 -0
- package/docs/VinGrade-MVP-02-Validation.md +480 -0
- package/docs/VinGrade-RC02-Live-Validation.md +273 -0
- package/docs/ai-capability-discovery.md +573 -0
- package/docs/ai-engineering-production-readiness.md +390 -0
- package/docs/ai-engineering-review.md +251 -0
- package/docs/ai-runtime-governance-discovery.md +754 -0
- package/docs/architecture-overview.md +78 -0
- package/docs/cli-progress.md +111 -0
- package/docs/decisions/diff-01-brand-distribution-identity.md +450 -0
- package/docs/decisions/diff-02-identity-compatibility-contract.md +274 -0
- package/docs/decisions/diff-03-repository-docs-community-migration.md +165 -0
- package/docs/decisions/diff-04-release-evidence-gap-acceptance.md +110 -0
- package/docs/discovery/mvp-05-ai-migration-planning-discovery.md +350 -0
- package/docs/gateway-runtime-discovery.md +614 -0
- package/docs/live-ai-validation.md +299 -0
- package/docs/migration-planning-qualification-resolution.md +99 -0
- package/docs/migrations/upgradelens-to-depverdict.md +97 -0
- package/docs/mp-r03-deterministic-upgrade-decision-architecture.md +135 -0
- package/docs/mp-r04-evidence-bounded-migration-handoff-architecture.md +116 -0
- package/docs/mp-r05-product-completion-and-decision-first-cli-architecture.md +162 -0
- package/docs/mvp-05-deterministic-context-runtime.md +127 -0
- package/docs/mvp-05-migration-checklist-contract.md +166 -0
- package/docs/mvp-05-migration-checklist-orchestration.md +105 -0
- package/docs/mvp-05-migration-evaluation-and-qualification.md +135 -0
- package/docs/mvp-05-provider-neutral-generator.md +79 -0
- package/docs/ollama-local-smoke-validation.md +156 -0
- package/docs/openai-compatible-runtime-discovery.md +789 -0
- package/docs/openrouter-one-dependency-validation.md +205 -0
- package/docs/oss-02-package-guard-hardening-architecture.md +131 -0
- package/docs/oss-04-public-ci-package-metadata-architecture.md +142 -0
- package/docs/package-content-policy.md +98 -0
- package/docs/releases/v0.5.0-technical-preview.md +114 -0
- package/docs/releases/v0.6.0-alpha.1-depverdict-preview.md +101 -0
- package/docs/reviews/diff-02-identity-contract-compatibility.md +413 -0
- package/docs/reviews/diff-03-repository-docs-community-migration.md +364 -0
- package/docs/reviews/diff-04-depverdict-distribution-identity-readiness-rereview.md +453 -0
- package/docs/reviews/diff-04-fix-post-rename-identity-release-remediation.md +310 -0
- package/docs/reviews/diff-05-final-preview-distribution-qualification.md +561 -0
- package/docs/reviews/mvp-05-final-product-value-workflow-rereview.md +471 -0
- package/docs/reviews/mvp-05-product-workflow-review.md +605 -0
- package/docs/reviews/oss-01-duplicate-artifact-investigation-cleanup.md +411 -0
- package/docs/reviews/oss-02-package-guard-hardening.md +303 -0
- package/docs/reviews/oss-03-community-scaffolding.md +378 -0
- package/docs/reviews/oss-04-public-ci-package-metadata.md +439 -0
- package/docs/reviews/oss-05-technical-preview-qualification.md +482 -0
- package/docs/reviews/upgradelens-vs-upgradedepdetective-source-comparison.md +355 -0
- package/docs/reviews/v0.5.0-pre-release-smoke.md +263 -0
- package/docs/reviews/v0.5.0-version-bump-release-verification.md +306 -0
- package/docs/runtime-contract-discovery.md +521 -0
- package/docs/structured-output-compatibility-report.md +100 -0
- package/docs/ts-fix-01-exact-duplicate-occurrence-target-selection-architecture.md +111 -0
- package/docs/version-analysis-architecture.md +827 -0
- package/eval/README.md +86 -0
- package/eval/datasets/generic/declared-constraint.json +59 -0
- package/eval/datasets/generic/evidence-conflict.json +73 -0
- package/eval/datasets/generic/major-breaking-release.json +66 -0
- package/eval/datasets/generic/missing-evidence.json +46 -0
- package/eval/datasets/generic/patch-release-low.json +59 -0
- package/eval/datasets/node/axios-patch-low.json +59 -0
- package/eval/datasets/node/react-major-breaking.json +66 -0
- package/eval/datasets/node/react-minor-compatibility.json +66 -0
- package/eval/datasets/python/fastapi-deprecation.json +66 -0
- package/eval/datasets/python/pydantic-major-breaking.json +66 -0
- package/eval/migration-planning/golden-dataset-v2.json +306 -0
- package/eval/migration-planning/golden-dataset.json +214 -0
- package/eval/schemas/expected-result.schema.json +88 -0
- package/eval/schemas/golden-case.schema.json +181 -0
- package/package.json +57 -0
- package/schemas/ai-scorecard.schema.json +132 -0
- package/schemas/benchmark-report.schema.json +189 -0
- package/schemas/benchmark.schema.json +55 -0
- package/schemas/capability-profile.schema.json +44 -0
- package/schemas/conformance-report.schema.json +150 -0
- package/schemas/deployment-profile.schema.json +64 -0
- package/schemas/evaluation-report.schema.json +158 -0
- package/schemas/knowledge-evidence-bundle.schema.json +150 -0
- package/schemas/knowledge-manifest.schema.json +548 -0
- package/schemas/metrics.schema.json +178 -0
- package/schemas/migration-checklist-extractive-candidate.schema.json +42 -0
- package/schemas/migration-checklist.schema.json +706 -0
- package/schemas/migration-evaluation-dataset-v2.schema.json +208 -0
- package/schemas/migration-evaluation-dataset.schema.json +204 -0
- package/schemas/migration-planning-qualification-record.schema.json +234 -0
- package/schemas/project-manifest.schema.json +308 -0
- package/schemas/qualification-record.schema.json +56 -0
- package/schemas/repository-impact-evidence.schema.json +232 -0
- package/schemas/repository-impact.schema.json +202 -0
- package/schemas/upgrade-decision.schema.json +273 -0
- package/schemas/usage-index.schema.json +179 -0
- package/schemas/version-analysis.schema.json +449 -0
- package/src/ai-runtime-debug.js +325 -0
- package/src/ai-runtime-error.js +42 -0
- package/src/ai-runtime.js +174 -0
- package/src/ai-scorecard.js +204 -0
- package/src/ai-version-analysis.js +484 -0
- package/src/artifact-root-compatibility.js +91 -0
- package/src/benchmark-report.js +111 -0
- package/src/benchmark-runner.js +191 -0
- package/src/canonical-json.js +69 -0
- package/src/cli.js +1299 -0
- package/src/conformance-report.js +158 -0
- package/src/conformance-runner.js +253 -0
- package/src/constants.js +73 -0
- package/src/cooperative-scheduler.js +79 -0
- package/src/dependencies.js +44 -0
- package/src/dependency-ai-context.js +625 -0
- package/src/detectors.js +253 -0
- package/src/discovery.js +234 -0
- package/src/ecosystem-version-adapter.js +294 -0
- package/src/environment-compatibility.js +77 -0
- package/src/evaluation-comparator.js +158 -0
- package/src/evaluation-report.js +76 -0
- package/src/evaluation-runner.js +248 -0
- package/src/evidence-source-adapter.js +472 -0
- package/src/files.js +89 -0
- package/src/governance-diagnostics.js +64 -0
- package/src/governance-loader.js +63 -0
- package/src/governance-metadata.js +346 -0
- package/src/governance-validator.js +360 -0
- package/src/http/bounded-fetch.js +278 -0
- package/src/http/cli-http-runtime.js +44 -0
- package/src/impact/input-loader.js +157 -0
- package/src/impact/matcher.js +40 -0
- package/src/impact/repository-impact.js +199 -0
- package/src/impact/runtime.js +24 -0
- package/src/impact/status.js +62 -0
- package/src/impact/writer.js +30 -0
- package/src/impact-evidence/input-loader.js +202 -0
- package/src/impact-evidence/repository-impact-evidence.js +234 -0
- package/src/impact-evidence/runtime.js +16 -0
- package/src/impact-evidence/writer.js +30 -0
- package/src/index.js +563 -0
- package/src/installed-version-baseline.js +196 -0
- package/src/knowledge-cache.js +324 -0
- package/src/knowledge-evidence-bundle.js +101 -0
- package/src/knowledge-evidence-producer.js +233 -0
- package/src/knowledge-manifest-builder.js +188 -0
- package/src/knowledge-manifest-writer.js +32 -0
- package/src/knowledge-manifest.js +255 -0
- package/src/knowledge-research.js +615 -0
- package/src/metrics-engine.js +205 -0
- package/src/migration-checklist/ai-candidate.js +320 -0
- package/src/migration-checklist/assembler.js +37 -0
- package/src/migration-checklist/context-runtime.js +828 -0
- package/src/migration-checklist/evaluation/action-criteria.js +244 -0
- package/src/migration-checklist/evaluation/comparator-v2.js +332 -0
- package/src/migration-checklist/evaluation/comparator.js +279 -0
- package/src/migration-checklist/evaluation/dataset-v2.js +227 -0
- package/src/migration-checklist/evaluation/dataset.js +336 -0
- package/src/migration-checklist/evaluation/extractive-fixtures-v2.js +148 -0
- package/src/migration-checklist/evaluation/metrics-v2.js +226 -0
- package/src/migration-checklist/evaluation/metrics.js +158 -0
- package/src/migration-checklist/evaluation/qualification-v2.js +321 -0
- package/src/migration-checklist/evaluation/qualification.js +239 -0
- package/src/migration-checklist/evaluation/runner-v2.js +294 -0
- package/src/migration-checklist/evaluation/runner.js +194 -0
- package/src/migration-checklist/evaluation/scorecard-v2.js +106 -0
- package/src/migration-checklist/evaluation/scorecard.js +86 -0
- package/src/migration-checklist/extractive-candidate.js +166 -0
- package/src/migration-checklist/extractive-prompt.js +62 -0
- package/src/migration-checklist/generator.js +702 -0
- package/src/migration-checklist/grounding-policy.js +117 -0
- package/src/migration-checklist/input-loader.js +613 -0
- package/src/migration-checklist/migration-checklist.js +635 -0
- package/src/migration-checklist/presentation.js +292 -0
- package/src/migration-checklist/progress.js +96 -0
- package/src/migration-checklist/prompt.js +83 -0
- package/src/migration-checklist/qualification-guard.js +462 -0
- package/src/migration-checklist/qualification-resolution.js +122 -0
- package/src/migration-checklist/qualification-store.js +225 -0
- package/src/migration-checklist/runtime.js +205 -0
- package/src/migration-checklist/verification.js +134 -0
- package/src/migration-checklist/writer.js +35 -0
- package/src/openai-compatible-provider.js +451 -0
- package/src/orchestration/failure-log.js +32 -0
- package/src/orchestration/pipeline.js +200 -0
- package/src/orchestration/progress-events.js +337 -0
- package/src/orchestration/progress-reporter.js +131 -0
- package/src/orchestration/text-writer.js +22 -0
- package/src/portable.js +13 -0
- package/src/product-completion.js +249 -0
- package/src/project-manifest-input.js +90 -0
- package/src/project-manifest.js +141 -0
- package/src/python-requirements.js +137 -0
- package/src/registry/npm-packument.js +256 -0
- package/src/registry/npm-registry-adapter.js +262 -0
- package/src/registry/pypi-project.js +300 -0
- package/src/registry/pypi-registry-adapter.js +235 -0
- package/src/registry/sanitize-registry-body.js +51 -0
- package/src/renderers/console.js +160 -0
- package/src/renderers/impact-presentation.js +278 -0
- package/src/renderers/markdown.js +172 -0
- package/src/research-plan.js +455 -0
- package/src/runtime-conformance.js +275 -0
- package/src/source-provenance.js +393 -0
- package/src/source-url.js +62 -0
- package/src/structured-output-schema.js +66 -0
- package/src/target-selector.js +306 -0
- package/src/upgrade-decision/input-loader.js +107 -0
- package/src/upgrade-decision/presentation.js +43 -0
- package/src/upgrade-decision/runtime.js +21 -0
- package/src/upgrade-decision/upgrade-decision.js +626 -0
- package/src/upgrade-decision/writer.js +30 -0
- package/src/usage/analyzer-registry.js +63 -0
- package/src/usage/coverage.js +116 -0
- package/src/usage/input-loader.js +139 -0
- package/src/usage/js/analyzer.js +240 -0
- package/src/usage/js/parser.js +21 -0
- package/src/usage/runtime.js +187 -0
- package/src/usage/scope.js +44 -0
- package/src/usage/source-files.js +50 -0
- package/src/usage/usage-index.js +217 -0
- package/src/usage/writer.js +31 -0
- package/src/version-analysis-loader.js +203 -0
- package/src/version-analysis-manifest.js +314 -0
- package/src/version-analysis-writer.js +30 -0
|
@@ -0,0 +1,294 @@
|
|
|
1
|
+
import { compareText } from './portable.js';
|
|
2
|
+
|
|
3
|
+
export class EcosystemVersionError extends Error {
|
|
4
|
+
constructor(message, code = 'VERSION_UNSUPPORTED') {
|
|
5
|
+
super(message);
|
|
6
|
+
this.name = 'EcosystemVersionError';
|
|
7
|
+
this.code = code;
|
|
8
|
+
}
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
function ok(value) {
|
|
12
|
+
return { ok: true, value };
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function fail(reason) {
|
|
16
|
+
return { ok: false, reason };
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function trim(value) {
|
|
20
|
+
return typeof value === 'string' ? value.trim() : '';
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function compareNumericParts(left, right) {
|
|
24
|
+
const length = Math.max(left.length, right.length);
|
|
25
|
+
for (let index = 0; index < length; index += 1) {
|
|
26
|
+
const leftPart = left[index] ?? 0;
|
|
27
|
+
const rightPart = right[index] ?? 0;
|
|
28
|
+
if (leftPart !== rightPart) return leftPart < rightPart ? -1 : 1;
|
|
29
|
+
}
|
|
30
|
+
return 0;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
const SEMVER_RE = /^v?(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?(?:\+[0-9A-Za-z.-]+)?$/;
|
|
34
|
+
|
|
35
|
+
function parseSemver(value) {
|
|
36
|
+
const match = SEMVER_RE.exec(trim(value));
|
|
37
|
+
if (!match) return null;
|
|
38
|
+
return {
|
|
39
|
+
major: Number(match[1]),
|
|
40
|
+
minor: Number(match[2]),
|
|
41
|
+
patch: Number(match[3]),
|
|
42
|
+
prerelease: match[4] ?? null
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function normalizeSemver(value) {
|
|
47
|
+
const parsed = parseSemver(value);
|
|
48
|
+
if (!parsed) return null;
|
|
49
|
+
const base = `${parsed.major}.${parsed.minor}.${parsed.patch}`;
|
|
50
|
+
return parsed.prerelease ? `${base}-${parsed.prerelease}` : base;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function comparePrerelease(left, right) {
|
|
54
|
+
if (left === right) return 0;
|
|
55
|
+
if (left === null) return 1;
|
|
56
|
+
if (right === null) return -1;
|
|
57
|
+
const leftParts = left.split('.');
|
|
58
|
+
const rightParts = right.split('.');
|
|
59
|
+
const length = Math.max(leftParts.length, rightParts.length);
|
|
60
|
+
for (let index = 0; index < length; index += 1) {
|
|
61
|
+
const leftPart = leftParts[index];
|
|
62
|
+
const rightPart = rightParts[index];
|
|
63
|
+
if (leftPart === undefined) return -1;
|
|
64
|
+
if (rightPart === undefined) return 1;
|
|
65
|
+
const leftNumeric = /^\d+$/.test(leftPart);
|
|
66
|
+
const rightNumeric = /^\d+$/.test(rightPart);
|
|
67
|
+
if (leftNumeric && rightNumeric && Number(leftPart) !== Number(rightPart)) {
|
|
68
|
+
return Number(leftPart) < Number(rightPart) ? -1 : 1;
|
|
69
|
+
}
|
|
70
|
+
if (leftNumeric !== rightNumeric) return leftNumeric ? -1 : 1;
|
|
71
|
+
const text = compareText(leftPart, rightPart);
|
|
72
|
+
if (text !== 0) return text;
|
|
73
|
+
}
|
|
74
|
+
return 0;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function compareSemver(left, right) {
|
|
78
|
+
const parsedLeft = parseSemver(left);
|
|
79
|
+
const parsedRight = parseSemver(right);
|
|
80
|
+
if (!parsedLeft || !parsedRight) return null;
|
|
81
|
+
const numeric = compareNumericParts(
|
|
82
|
+
[parsedLeft.major, parsedLeft.minor, parsedLeft.patch],
|
|
83
|
+
[parsedRight.major, parsedRight.minor, parsedRight.patch]
|
|
84
|
+
);
|
|
85
|
+
if (numeric !== 0) return numeric;
|
|
86
|
+
return comparePrerelease(parsedLeft.prerelease, parsedRight.prerelease);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function semverClassification(current, target) {
|
|
90
|
+
const left = parseSemver(current);
|
|
91
|
+
const right = parseSemver(target);
|
|
92
|
+
if (!left || !right) return 'unknown';
|
|
93
|
+
if (left.major !== right.major) return 'major';
|
|
94
|
+
if (left.minor !== right.minor) return 'minor';
|
|
95
|
+
if (left.patch !== right.patch) return 'patch';
|
|
96
|
+
if (left.prerelease !== right.prerelease) return 'prerelease';
|
|
97
|
+
return 'other';
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function isSemverConstraint(value) {
|
|
101
|
+
const text = trim(value);
|
|
102
|
+
if (!text) return false;
|
|
103
|
+
if (/^(?:workspace:|file:|link:|npm:|git\+|git:|https?:|ssh:|github:|gitlab:|bitbucket:)/i.test(text)) {
|
|
104
|
+
return false;
|
|
105
|
+
}
|
|
106
|
+
return /(?:[\s,|]|[~^<>=*xX])/.test(text) && /\d|[*xX]/.test(text);
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
function compareReleaseVersions(adapter, left, right) {
|
|
110
|
+
const compared = adapter.compareVersionOrder(left.version, right.version);
|
|
111
|
+
return compared === 0 ? compareText(left.version ?? left.tag ?? '', right.version ?? right.tag ?? '') : compared;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function versionedReleases(releases, adapter) {
|
|
115
|
+
return releases
|
|
116
|
+
.filter((release) => typeof release.version === 'string' && adapter.normalizeVersion(release.version).ok)
|
|
117
|
+
.map((release) => ({ ...release, version: adapter.normalizeVersion(release.version).value }));
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
function selectExactReleases(releases, adapter, current, target) {
|
|
121
|
+
const direction = adapter.compareVersions(current, target).direction;
|
|
122
|
+
const normalizedCurrent = adapter.normalizeVersion(current).value;
|
|
123
|
+
const normalizedTarget = adapter.normalizeVersion(target).value;
|
|
124
|
+
return versionedReleases(releases, adapter)
|
|
125
|
+
.filter((release) => {
|
|
126
|
+
const afterCurrent = adapter.compareVersionOrder(release.version, normalizedCurrent);
|
|
127
|
+
const beforeTarget = adapter.compareVersionOrder(release.version, normalizedTarget);
|
|
128
|
+
if (direction === 'upgrade') return afterCurrent > 0 && beforeTarget <= 0;
|
|
129
|
+
if (direction === 'downgrade') return afterCurrent < 0 && beforeTarget >= 0;
|
|
130
|
+
return beforeTarget === 0;
|
|
131
|
+
})
|
|
132
|
+
.sort((left, right) => compareReleaseVersions(adapter, left, right));
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
function selectTargetRelease(releases, adapter, target) {
|
|
136
|
+
const normalizedTarget = adapter.normalizeVersion(target).value;
|
|
137
|
+
return versionedReleases(releases, adapter)
|
|
138
|
+
.filter((release) => adapter.compareVersionOrder(release.version, normalizedTarget) === 0)
|
|
139
|
+
.sort((left, right) => compareReleaseVersions(adapter, left, right));
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
function createNodeSemverAdapter() {
|
|
143
|
+
return {
|
|
144
|
+
ecosystem: 'node',
|
|
145
|
+
registries: ['npm'],
|
|
146
|
+
normalizeVersion(value) {
|
|
147
|
+
const normalized = normalizeSemver(value);
|
|
148
|
+
return normalized ? ok(normalized) : fail('invalid-semver');
|
|
149
|
+
},
|
|
150
|
+
resolveDeclaredBaseline(declaredVersion) {
|
|
151
|
+
const value = trim(declaredVersion);
|
|
152
|
+
if (!value) return { kind: 'unsupported', reason: 'missing-declared-version' };
|
|
153
|
+
const exact = value.startsWith('=') ? value.slice(1).trim() : value;
|
|
154
|
+
const normalized = normalizeSemver(exact);
|
|
155
|
+
if (normalized) return { kind: 'exactVersion', version: normalized };
|
|
156
|
+
if (isSemverConstraint(value)) return { kind: 'declaredConstraint', constraint: value };
|
|
157
|
+
return { kind: 'unsupported', reason: 'unsupported-semver-declaration' };
|
|
158
|
+
},
|
|
159
|
+
targetSatisfiesDeclaration(declaredVersion, target) {
|
|
160
|
+
const baseline = this.resolveDeclaredBaseline(declaredVersion);
|
|
161
|
+
const normalizedTarget = this.normalizeVersion(target);
|
|
162
|
+
if (!normalizedTarget.ok) return 'unknown';
|
|
163
|
+
if (baseline.kind === 'exactVersion') return baseline.version === normalizedTarget.value ? 'yes' : 'no';
|
|
164
|
+
if (baseline.kind !== 'declaredConstraint') return 'unknown';
|
|
165
|
+
const text = baseline.constraint.trim();
|
|
166
|
+
const caret = /^\^(\d+)(?:\.(\d+))?(?:\.(\d+))?/.exec(text);
|
|
167
|
+
if (caret) {
|
|
168
|
+
const parsed = parseSemver(normalizedTarget.value);
|
|
169
|
+
return parsed && parsed.major === Number(caret[1]) ? 'yes' : 'no';
|
|
170
|
+
}
|
|
171
|
+
return 'unknown';
|
|
172
|
+
},
|
|
173
|
+
compareVersionOrder(left, right) {
|
|
174
|
+
const result = compareSemver(left, right);
|
|
175
|
+
if (result === null) throw new EcosystemVersionError('Cannot compare invalid SemVer values.');
|
|
176
|
+
return result;
|
|
177
|
+
},
|
|
178
|
+
compareVersions(current, target) {
|
|
179
|
+
const result = compareSemver(current, target);
|
|
180
|
+
if (result === null) return { direction: 'unknown', classification: 'unknown' };
|
|
181
|
+
return {
|
|
182
|
+
direction: result < 0 ? 'upgrade' : result > 0 ? 'downgrade' : 'same',
|
|
183
|
+
classification: result === 0 ? 'other' : semverClassification(current, target)
|
|
184
|
+
};
|
|
185
|
+
},
|
|
186
|
+
selectRelevantReleases(releases, input) {
|
|
187
|
+
if (input.mode === 'declaredConstraint') return selectTargetRelease(releases, this, input.target);
|
|
188
|
+
return selectExactReleases(releases, this, input.current, input.target);
|
|
189
|
+
}
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
const PEP440_RE = /^v?(\d+(?:\.\d+)*)(?:(a|b|rc)(\d+))?$/i;
|
|
194
|
+
|
|
195
|
+
function parsePep440(value) {
|
|
196
|
+
const match = PEP440_RE.exec(trim(value));
|
|
197
|
+
if (!match) return null;
|
|
198
|
+
return {
|
|
199
|
+
release: match[1].split('.').map((part) => Number(part)),
|
|
200
|
+
prerelease: match[2] ? `${match[2].toLowerCase()}${Number(match[3])}` : null
|
|
201
|
+
};
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
function normalizePep440(value) {
|
|
205
|
+
const parsed = parsePep440(value);
|
|
206
|
+
if (!parsed) return null;
|
|
207
|
+
const release = parsed.release.join('.');
|
|
208
|
+
return parsed.prerelease ? `${release}${parsed.prerelease}` : release;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
function comparePep440(left, right) {
|
|
212
|
+
const parsedLeft = parsePep440(left);
|
|
213
|
+
const parsedRight = parsePep440(right);
|
|
214
|
+
if (!parsedLeft || !parsedRight) return null;
|
|
215
|
+
const release = compareNumericParts(parsedLeft.release, parsedRight.release);
|
|
216
|
+
if (release !== 0) return release;
|
|
217
|
+
return comparePrerelease(parsedLeft.prerelease, parsedRight.prerelease);
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
function pep440Classification(current, target) {
|
|
221
|
+
const left = parsePep440(current);
|
|
222
|
+
const right = parsePep440(target);
|
|
223
|
+
if (!left || !right) return 'unknown';
|
|
224
|
+
const leftParts = [left.release[0] ?? 0, left.release[1] ?? 0, left.release[2] ?? 0];
|
|
225
|
+
const rightParts = [right.release[0] ?? 0, right.release[1] ?? 0, right.release[2] ?? 0];
|
|
226
|
+
if (leftParts[0] !== rightParts[0]) return 'major';
|
|
227
|
+
if (leftParts[1] !== rightParts[1]) return 'minor';
|
|
228
|
+
if (leftParts[2] !== rightParts[2]) return 'patch';
|
|
229
|
+
if (left.prerelease !== right.prerelease) return 'prerelease';
|
|
230
|
+
return 'other';
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
function isPep440Constraint(value) {
|
|
234
|
+
const text = trim(value);
|
|
235
|
+
return /^(?:~=|==|!=|<=|>=|<|>)/.test(text) || text.includes(',');
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
function createPythonPep440Adapter() {
|
|
239
|
+
return {
|
|
240
|
+
ecosystem: 'python',
|
|
241
|
+
registries: ['pypi'],
|
|
242
|
+
normalizeVersion(value) {
|
|
243
|
+
const normalized = normalizePep440(value);
|
|
244
|
+
return normalized ? ok(normalized) : fail('invalid-pep440-version');
|
|
245
|
+
},
|
|
246
|
+
resolveDeclaredBaseline(declaredVersion) {
|
|
247
|
+
const value = trim(declaredVersion);
|
|
248
|
+
if (!value) return { kind: 'unsupported', reason: 'missing-declared-version' };
|
|
249
|
+
const exactMatch = /^==\s*(.+)$/.exec(value);
|
|
250
|
+
const exact = exactMatch ? exactMatch[1].trim() : value;
|
|
251
|
+
const normalized = normalizePep440(exact);
|
|
252
|
+
if (normalized) return { kind: 'exactVersion', version: normalized };
|
|
253
|
+
if (isPep440Constraint(value)) return { kind: 'declaredConstraint', constraint: value };
|
|
254
|
+
return { kind: 'unsupported', reason: 'unsupported-pep440-declaration' };
|
|
255
|
+
},
|
|
256
|
+
targetSatisfiesDeclaration(declaredVersion, target) {
|
|
257
|
+
const baseline = this.resolveDeclaredBaseline(declaredVersion);
|
|
258
|
+
const normalizedTarget = this.normalizeVersion(target);
|
|
259
|
+
if (!normalizedTarget.ok) return 'unknown';
|
|
260
|
+
if (baseline.kind === 'exactVersion') return baseline.version === normalizedTarget.value ? 'yes' : 'no';
|
|
261
|
+
return 'unknown';
|
|
262
|
+
},
|
|
263
|
+
compareVersionOrder(left, right) {
|
|
264
|
+
const result = comparePep440(left, right);
|
|
265
|
+
if (result === null) throw new EcosystemVersionError('Cannot compare invalid PEP 440 values.');
|
|
266
|
+
return result;
|
|
267
|
+
},
|
|
268
|
+
compareVersions(current, target) {
|
|
269
|
+
const result = comparePep440(current, target);
|
|
270
|
+
if (result === null) return { direction: 'unknown', classification: 'unknown' };
|
|
271
|
+
return {
|
|
272
|
+
direction: result < 0 ? 'upgrade' : result > 0 ? 'downgrade' : 'same',
|
|
273
|
+
classification: result === 0 ? 'other' : pep440Classification(current, target)
|
|
274
|
+
};
|
|
275
|
+
},
|
|
276
|
+
selectRelevantReleases(releases, input) {
|
|
277
|
+
if (input.mode === 'declaredConstraint') return selectTargetRelease(releases, this, input.target);
|
|
278
|
+
return selectExactReleases(releases, this, input.current, input.target);
|
|
279
|
+
}
|
|
280
|
+
};
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
export function createDefaultEcosystemVersionAdapterRegistry() {
|
|
284
|
+
return new Map([
|
|
285
|
+
['node', createNodeSemverAdapter()],
|
|
286
|
+
['python', createPythonPep440Adapter()]
|
|
287
|
+
]);
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
export function getEcosystemVersionAdapter(ecosystem, registry = createDefaultEcosystemVersionAdapterRegistry()) {
|
|
291
|
+
const adapter = registry.get(ecosystem);
|
|
292
|
+
if (!adapter) throw new EcosystemVersionError(`Unsupported ecosystem ${ecosystem}.`, 'UNSUPPORTED_ECOSYSTEM');
|
|
293
|
+
return adapter;
|
|
294
|
+
}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
const SUFFIXES = Object.freeze([
|
|
2
|
+
'AI_PROVIDER',
|
|
3
|
+
'AI_ENDPOINT',
|
|
4
|
+
'AI_MODEL',
|
|
5
|
+
'AI_AUTHORIZATION',
|
|
6
|
+
'AI_TIMEOUT_MS',
|
|
7
|
+
'AI_TIMEOUT_SECONDS',
|
|
8
|
+
'AI_MAX_RESPONSE_BYTES',
|
|
9
|
+
'AI_DEBUG'
|
|
10
|
+
]);
|
|
11
|
+
|
|
12
|
+
export const ENVIRONMENT_IDENTITY_MAP = Object.freeze(SUFFIXES.map((suffix) => Object.freeze({
|
|
13
|
+
canonical: `DEPVERDICT_${suffix}`,
|
|
14
|
+
legacy: `UPGRADELENS_${suffix}`,
|
|
15
|
+
secret: suffix === 'AI_AUTHORIZATION'
|
|
16
|
+
})));
|
|
17
|
+
|
|
18
|
+
function configured(value) {
|
|
19
|
+
return value !== undefined && value !== '';
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Resolve the supported identity-prefixed environment without exposing values.
|
|
24
|
+
* The returned object retains unrelated environment keys for callers such as
|
|
25
|
+
* NO_COLOR, and adds only the selected canonical identity keys.
|
|
26
|
+
*/
|
|
27
|
+
export function resolveIdentityEnvironment(env = {}, options = {}) {
|
|
28
|
+
const source = env ?? {};
|
|
29
|
+
const overrides = options.overrides ?? {};
|
|
30
|
+
const diagnostics = options.diagnosticState ?? new Set();
|
|
31
|
+
const resolved = { ...source };
|
|
32
|
+
for (const key of Object.keys(resolved)) {
|
|
33
|
+
if (key.startsWith('DEPVERDICT_') || key.startsWith('UPGRADELENS_')) {
|
|
34
|
+
delete resolved[key];
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
for (const mapping of ENVIRONMENT_IDENTITY_MAP) {
|
|
39
|
+
const explicit = overrides[mapping.canonical];
|
|
40
|
+
const canonical = source[mapping.canonical];
|
|
41
|
+
const legacy = source[mapping.legacy];
|
|
42
|
+
let value;
|
|
43
|
+
|
|
44
|
+
if (configured(explicit)) {
|
|
45
|
+
value = explicit;
|
|
46
|
+
} else if (configured(canonical)) {
|
|
47
|
+
value = canonical;
|
|
48
|
+
if (configured(legacy)) {
|
|
49
|
+
const key = `conflict:${mapping.canonical}`;
|
|
50
|
+
if (!diagnostics.has(key)) {
|
|
51
|
+
diagnostics.add(key);
|
|
52
|
+
options.onDiagnostic?.(
|
|
53
|
+
`ENVIRONMENT_IDENTITY_CONFLICT: ${mapping.canonical} overrides deprecated ${mapping.legacy}.`
|
|
54
|
+
);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
} else if (configured(legacy)) {
|
|
58
|
+
value = legacy;
|
|
59
|
+
const key = `legacy:${mapping.canonical}`;
|
|
60
|
+
if (!diagnostics.has(key)) {
|
|
61
|
+
diagnostics.add(key);
|
|
62
|
+
options.onDiagnostic?.(
|
|
63
|
+
`LEGACY_ENVIRONMENT_USED: ${mapping.legacy} is deprecated; use ${mapping.canonical}.`
|
|
64
|
+
);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
if (configured(value)) resolved[mapping.canonical] = value;
|
|
69
|
+
else delete resolved[mapping.canonical];
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
return Object.freeze(resolved);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export function legacyEnvironmentKey(canonicalKey) {
|
|
76
|
+
return ENVIRONMENT_IDENTITY_MAP.find(({ canonical }) => canonical === canonicalKey)?.legacy;
|
|
77
|
+
}
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
import { AI_VERSION_ANALYSIS_CANDIDATE_SCHEMA } from './ai-version-analysis.js';
|
|
2
|
+
import { compareText } from './portable.js';
|
|
3
|
+
|
|
4
|
+
const candidateRequiredFields = new Set(AI_VERSION_ANALYSIS_CANDIDATE_SCHEMA.required);
|
|
5
|
+
|
|
6
|
+
function sorted(values = []) {
|
|
7
|
+
return [...values].sort(compareText);
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
function uniqueRefs(values = []) {
|
|
11
|
+
return sorted([...new Set(values)]);
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
function equalJson(left, right) {
|
|
15
|
+
return JSON.stringify(left) === JSON.stringify(right);
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function check(passed, expected, actual, message) {
|
|
19
|
+
return {
|
|
20
|
+
passed,
|
|
21
|
+
...(expected !== undefined ? { expected } : {}),
|
|
22
|
+
...(actual !== undefined ? { actual } : {}),
|
|
23
|
+
...(message ? { message } : {})
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function flattenFindingRefs(findings = []) {
|
|
28
|
+
return uniqueRefs(findings.flatMap((finding) => finding.evidenceRefs ?? []));
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function evidenceReferenceActual(result) {
|
|
32
|
+
return {
|
|
33
|
+
summary: uniqueRefs(result.summaryEvidenceRefs),
|
|
34
|
+
risk: uniqueRefs(result.riskEvidenceRefs),
|
|
35
|
+
findings: flattenFindingRefs(result.findings)
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function findingExpectationActual(result) {
|
|
40
|
+
return result.findings.map((finding) => ({
|
|
41
|
+
kind: finding.kind,
|
|
42
|
+
appliesToVersions: sorted(finding.appliesToVersions),
|
|
43
|
+
evidenceRefs: uniqueRefs(finding.evidenceRefs),
|
|
44
|
+
summary: finding.summary
|
|
45
|
+
})).sort((left, right) =>
|
|
46
|
+
compareText(left.kind, right.kind)
|
|
47
|
+
|| compareText(left.appliesToVersions.join('\0'), right.appliesToVersions.join('\0'))
|
|
48
|
+
|| compareText(left.evidenceRefs.join('\0'), right.evidenceRefs.join('\0'))
|
|
49
|
+
);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function findingExpected(expectedResult) {
|
|
53
|
+
return expectedResult.expectedFindings.map((finding) => ({
|
|
54
|
+
kind: finding.kind,
|
|
55
|
+
appliesToVersions: sorted(finding.appliesToVersions),
|
|
56
|
+
evidenceRefs: uniqueRefs(finding.evidenceRefs),
|
|
57
|
+
requiredKeywords: sorted(finding.requiredKeywords ?? [])
|
|
58
|
+
})).sort((left, right) =>
|
|
59
|
+
compareText(left.kind, right.kind)
|
|
60
|
+
|| compareText(left.appliesToVersions.join('\0'), right.appliesToVersions.join('\0'))
|
|
61
|
+
|| compareText(left.evidenceRefs.join('\0'), right.evidenceRefs.join('\0'))
|
|
62
|
+
);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function findingsMatch(expected, actual) {
|
|
66
|
+
if (expected.length !== actual.length) return false;
|
|
67
|
+
for (let index = 0; index < expected.length; index += 1) {
|
|
68
|
+
const left = expected[index];
|
|
69
|
+
const right = actual[index];
|
|
70
|
+
if (left.kind !== right.kind) return false;
|
|
71
|
+
if (!equalJson(left.appliesToVersions, right.appliesToVersions)) return false;
|
|
72
|
+
if (!equalJson(left.evidenceRefs, right.evidenceRefs)) return false;
|
|
73
|
+
for (const keyword of left.requiredKeywords) {
|
|
74
|
+
if (!right.summary.toLowerCase().includes(keyword.toLowerCase())) return false;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
return true;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function resultHasCandidateShape(result) {
|
|
81
|
+
return [...candidateRequiredFields].every((field) => field in result)
|
|
82
|
+
&& typeof result.summary === 'string'
|
|
83
|
+
&& Array.isArray(result.findings)
|
|
84
|
+
&& Array.isArray(result.summaryEvidenceRefs)
|
|
85
|
+
&& Array.isArray(result.riskEvidenceRefs);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
export function compareEvaluationResult(goldenCase, result) {
|
|
89
|
+
const expected = goldenCase.expectedResult;
|
|
90
|
+
const actualEvidenceRefs = evidenceReferenceActual(result);
|
|
91
|
+
const expectedEvidenceRefs = {
|
|
92
|
+
summary: uniqueRefs(expected.expectedEvidenceRefs.summary),
|
|
93
|
+
risk: uniqueRefs(expected.expectedEvidenceRefs.risk),
|
|
94
|
+
findings: uniqueRefs(expected.expectedEvidenceRefs.findings)
|
|
95
|
+
};
|
|
96
|
+
const expectedFindings = findingExpected(expected);
|
|
97
|
+
const actualFindings = findingExpectationActual(result);
|
|
98
|
+
const checks = {
|
|
99
|
+
schema: check(resultHasCandidateShape(result), true, resultHasCandidateShape(result), 'AI result has required structured fields.'),
|
|
100
|
+
summaryPresent: check(
|
|
101
|
+
typeof result.summary === 'string' && result.summary.trim().length > 0,
|
|
102
|
+
true,
|
|
103
|
+
typeof result.summary === 'string' ? result.summary.trim().length > 0 : false
|
|
104
|
+
),
|
|
105
|
+
riskLevel: check(result.riskLevel === expected.riskLevel, expected.riskLevel, result.riskLevel),
|
|
106
|
+
humanReview: check(
|
|
107
|
+
result.requiresHumanReview === expected.requiresHumanReview,
|
|
108
|
+
expected.requiresHumanReview,
|
|
109
|
+
result.requiresHumanReview
|
|
110
|
+
),
|
|
111
|
+
humanReviewReasons: check(
|
|
112
|
+
equalJson(sorted(expected.humanReviewReasons), sorted(result.humanReviewReasons)),
|
|
113
|
+
sorted(expected.humanReviewReasons),
|
|
114
|
+
sorted(result.humanReviewReasons)
|
|
115
|
+
),
|
|
116
|
+
evidenceCoverage: check(
|
|
117
|
+
result.evidenceCoverage === expected.evidenceCoverage,
|
|
118
|
+
expected.evidenceCoverage,
|
|
119
|
+
result.evidenceCoverage
|
|
120
|
+
),
|
|
121
|
+
validationState: check(
|
|
122
|
+
result.validation?.status === expected.validation.status
|
|
123
|
+
&& equalJson(sorted(expected.validation.warningCodes), sorted(result.validation?.warningCodes ?? [])),
|
|
124
|
+
expected.validation,
|
|
125
|
+
result.validation
|
|
126
|
+
),
|
|
127
|
+
evidenceReferences: check(
|
|
128
|
+
equalJson(expectedEvidenceRefs, actualEvidenceRefs),
|
|
129
|
+
expectedEvidenceRefs,
|
|
130
|
+
actualEvidenceRefs
|
|
131
|
+
),
|
|
132
|
+
findings: check(
|
|
133
|
+
findingsMatch(expectedFindings, actualFindings),
|
|
134
|
+
expectedFindings,
|
|
135
|
+
actualFindings.map((finding) => ({
|
|
136
|
+
kind: finding.kind,
|
|
137
|
+
appliesToVersions: finding.appliesToVersions,
|
|
138
|
+
evidenceRefs: finding.evidenceRefs
|
|
139
|
+
}))
|
|
140
|
+
)
|
|
141
|
+
};
|
|
142
|
+
const passed = Object.values(checks).every((item) => item.passed);
|
|
143
|
+
return {
|
|
144
|
+
id: goldenCase.id,
|
|
145
|
+
title: goldenCase.title,
|
|
146
|
+
category: goldenCase.category,
|
|
147
|
+
ecosystem: goldenCase.dependency.ecosystem,
|
|
148
|
+
passed,
|
|
149
|
+
checks,
|
|
150
|
+
actual: {
|
|
151
|
+
status: result.status,
|
|
152
|
+
riskLevel: result.riskLevel,
|
|
153
|
+
requiresHumanReview: result.requiresHumanReview,
|
|
154
|
+
evidenceCoverage: result.evidenceCoverage,
|
|
155
|
+
validationStatus: result.validation?.status ?? 'unknown'
|
|
156
|
+
}
|
|
157
|
+
};
|
|
158
|
+
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { readFile } from 'node:fs/promises';
|
|
2
|
+
|
|
3
|
+
import Ajv2020 from 'ajv/dist/2020.js';
|
|
4
|
+
import addFormats from 'ajv-formats';
|
|
5
|
+
|
|
6
|
+
import { ARTIFACT_GENERATOR_NAME, VERSION } from './constants.js';
|
|
7
|
+
import { VERSION_ANALYSIS_PROMPT_VERSION } from './ai-version-analysis.js';
|
|
8
|
+
|
|
9
|
+
export const EVALUATION_REPORT_SCHEMA_VERSION = '1.0.0';
|
|
10
|
+
|
|
11
|
+
const schema = JSON.parse(await readFile(
|
|
12
|
+
new URL('../schemas/evaluation-report.schema.json', import.meta.url),
|
|
13
|
+
'utf8'
|
|
14
|
+
));
|
|
15
|
+
const ajv = new Ajv2020({ allErrors: true, strict: true, strictRequired: false });
|
|
16
|
+
addFormats(ajv);
|
|
17
|
+
const validateSchema = ajv.compile(schema);
|
|
18
|
+
|
|
19
|
+
function ratio(passed, total) {
|
|
20
|
+
return total === 0 ? 1 : passed / total;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function metric(caseResults, selector) {
|
|
24
|
+
return ratio(caseResults.filter((item) => selector(item).passed).length, caseResults.length);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function buildEvaluationReport({
|
|
28
|
+
datasetPath,
|
|
29
|
+
datasetVersion = '1.0.0',
|
|
30
|
+
promptVersion = VERSION_ANALYSIS_PROMPT_VERSION,
|
|
31
|
+
model = { provider: 'unknown', name: 'unknown' },
|
|
32
|
+
caseResults,
|
|
33
|
+
generatedAt = new Date()
|
|
34
|
+
}) {
|
|
35
|
+
const passed = caseResults.filter((item) => item.passed).length;
|
|
36
|
+
const report = {
|
|
37
|
+
schemaVersion: EVALUATION_REPORT_SCHEMA_VERSION,
|
|
38
|
+
generatedAt: generatedAt instanceof Date ? generatedAt.toISOString() : generatedAt,
|
|
39
|
+
generator: { name: ARTIFACT_GENERATOR_NAME, version: VERSION },
|
|
40
|
+
evaluation: {
|
|
41
|
+
datasetPath,
|
|
42
|
+
datasetVersion,
|
|
43
|
+
promptVersion,
|
|
44
|
+
caseCount: caseResults.length
|
|
45
|
+
},
|
|
46
|
+
model,
|
|
47
|
+
summary: {
|
|
48
|
+
totalCases: caseResults.length,
|
|
49
|
+
passed,
|
|
50
|
+
failed: caseResults.length - passed
|
|
51
|
+
},
|
|
52
|
+
metrics: {
|
|
53
|
+
riskClassificationAccuracy: metric(caseResults, (item) => item.checks.riskLevel),
|
|
54
|
+
humanReviewAccuracy: metric(caseResults, (item) => item.checks.humanReview),
|
|
55
|
+
evidenceReferenceAccuracy: metric(caseResults, (item) => item.checks.evidenceReferences),
|
|
56
|
+
schemaValidationPassRate: metric(caseResults, (item) => item.checks.schema)
|
|
57
|
+
},
|
|
58
|
+
cases: caseResults
|
|
59
|
+
};
|
|
60
|
+
return validateEvaluationReport(report);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export function validateEvaluationReport(report) {
|
|
64
|
+
if (report?.schemaVersion !== EVALUATION_REPORT_SCHEMA_VERSION) {
|
|
65
|
+
throw new Error(`Evaluation Report validation error: unsupported schema version; expected ${EVALUATION_REPORT_SCHEMA_VERSION}.`);
|
|
66
|
+
}
|
|
67
|
+
if (!validateSchema(report)) {
|
|
68
|
+
throw new Error(`Evaluation Report validation error: schema validation failed: ${ajv.errorsText(validateSchema.errors, { separator: '; ' })}`);
|
|
69
|
+
}
|
|
70
|
+
return report;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export function serializeEvaluationReport(report) {
|
|
74
|
+
validateEvaluationReport(report);
|
|
75
|
+
return `${JSON.stringify(report, null, 2)}\n`;
|
|
76
|
+
}
|