@devflow-tools/benchmark 0.17.8 → 0.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/accuracy-command-options.d.ts +3 -0
- package/dist/accuracy-command-options.d.ts.map +1 -0
- package/dist/accuracy-command-options.js +17 -0
- package/dist/accuracy-command-options.js.map +1 -0
- package/dist/context-symbol-quality-command.d.ts +59 -0
- package/dist/context-symbol-quality-command.d.ts.map +1 -0
- package/dist/context-symbol-quality-command.js +482 -0
- package/dist/context-symbol-quality-command.js.map +1 -0
- package/dist/evaluation-bundle-builder.d.ts +48 -0
- package/dist/evaluation-bundle-builder.d.ts.map +1 -0
- package/dist/evaluation-bundle-builder.js +496 -0
- package/dist/evaluation-bundle-builder.js.map +1 -0
- package/dist/evaluation-bundle-command.d.ts +2 -0
- package/dist/evaluation-bundle-command.d.ts.map +1 -0
- package/dist/evaluation-bundle-command.js +15 -0
- package/dist/evaluation-bundle-command.js.map +1 -0
- package/dist/evaluation-bundle.d.ts +55 -0
- package/dist/evaluation-bundle.d.ts.map +1 -0
- package/dist/evaluation-bundle.js +272 -0
- package/dist/evaluation-bundle.js.map +1 -0
- package/dist/evaluation-run-state.d.ts +23 -0
- package/dist/evaluation-run-state.d.ts.map +1 -0
- package/dist/evaluation-run-state.js +115 -0
- package/dist/evaluation-run-state.js.map +1 -0
- package/dist/evidence-quality-metrics.d.ts +15 -0
- package/dist/evidence-quality-metrics.d.ts.map +1 -1
- package/dist/evidence-quality-metrics.js +51 -1
- package/dist/evidence-quality-metrics.js.map +1 -1
- package/dist/graders/deterministic.d.ts.map +1 -1
- package/dist/graders/deterministic.js +5 -3
- package/dist/graders/deterministic.js.map +1 -1
- package/dist/index.d.ts +15 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +12 -1
- package/dist/index.js.map +1 -1
- package/dist/release-quality-bundle-command.d.ts +3 -0
- package/dist/release-quality-bundle-command.d.ts.map +1 -0
- package/dist/release-quality-bundle-command.js +72 -0
- package/dist/release-quality-bundle-command.js.map +1 -0
- package/dist/release-quality-bundle-evaluator.d.ts +61 -0
- package/dist/release-quality-bundle-evaluator.d.ts.map +1 -0
- package/dist/release-quality-bundle-evaluator.js +321 -0
- package/dist/release-quality-bundle-evaluator.js.map +1 -0
- package/dist/release-quality-bundle-materializer.d.ts +79 -0
- package/dist/release-quality-bundle-materializer.d.ts.map +1 -0
- package/dist/release-quality-bundle-materializer.js +395 -0
- package/dist/release-quality-bundle-materializer.js.map +1 -0
- package/dist/release-quality-comparison.d.ts +34 -0
- package/dist/release-quality-comparison.d.ts.map +1 -0
- package/dist/release-quality-comparison.js +60 -0
- package/dist/release-quality-comparison.js.map +1 -0
- package/dist/release-quality-evaluation-command.d.ts +42 -0
- package/dist/release-quality-evaluation-command.d.ts.map +1 -0
- package/dist/release-quality-evaluation-command.js +91 -0
- package/dist/release-quality-evaluation-command.js.map +1 -0
- package/dist/release-quality-host-proof.d.ts +30 -0
- package/dist/release-quality-host-proof.d.ts.map +1 -0
- package/dist/release-quality-host-proof.js +109 -0
- package/dist/release-quality-host-proof.js.map +1 -0
- package/dist/release-quality-output-cleanup.d.ts +2 -0
- package/dist/release-quality-output-cleanup.d.ts.map +1 -0
- package/dist/release-quality-output-cleanup.js +18 -0
- package/dist/release-quality-output-cleanup.js.map +1 -0
- package/dist/release-quality-policy.d.ts +87 -0
- package/dist/release-quality-policy.d.ts.map +1 -0
- package/dist/release-quality-policy.js +179 -0
- package/dist/release-quality-policy.js.map +1 -0
- package/dist/release-quality-query-sampling.d.ts +36 -0
- package/dist/release-quality-query-sampling.d.ts.map +1 -0
- package/dist/release-quality-query-sampling.js +87 -0
- package/dist/release-quality-query-sampling.js.map +1 -0
- package/dist/release-quality-source-snapshot.d.ts +18 -0
- package/dist/release-quality-source-snapshot.d.ts.map +1 -0
- package/dist/release-quality-source-snapshot.js +46 -0
- package/dist/release-quality-source-snapshot.js.map +1 -0
- package/dist/retrieval-quality-corpus.d.ts +0 -1
- package/dist/retrieval-quality-corpus.d.ts.map +1 -1
- package/dist/retrieval-quality-corpus.js +1 -1
- package/dist/retrieval-quality-corpus.js.map +1 -1
- package/dist/retrieval-quality.d.ts.map +1 -1
- package/dist/retrieval-quality.js +29 -26
- package/dist/retrieval-quality.js.map +1 -1
- package/dist/run-accuracy-tests.js +82 -2
- package/dist/run-accuracy-tests.js.map +1 -1
- package/dist/suite-runner.js +1 -1
- package/dist/suite-runner.js.map +1 -1
- package/dist/unified-runtime-corpus.d.ts +64 -0
- package/dist/unified-runtime-corpus.d.ts.map +1 -0
- package/dist/unified-runtime-corpus.js +161 -0
- package/dist/unified-runtime-corpus.js.map +1 -0
- package/dist/unified-runtime-evaluation.d.ts +133 -0
- package/dist/unified-runtime-evaluation.d.ts.map +1 -0
- package/dist/unified-runtime-evaluation.js +388 -0
- package/dist/unified-runtime-evaluation.js.map +1 -0
- package/dist/unified-runtime-observation.d.ts +101 -0
- package/dist/unified-runtime-observation.d.ts.map +1 -0
- package/dist/unified-runtime-observation.js +2 -0
- package/dist/unified-runtime-observation.js.map +1 -0
- package/dist/unified-runtime-runner.d.ts +73 -0
- package/dist/unified-runtime-runner.d.ts.map +1 -0
- package/dist/unified-runtime-runner.js +620 -0
- package/dist/unified-runtime-runner.js.map +1 -0
- package/fixtures/evidence-runtime-corpus.json +0 -100
- package/fixtures/release-quality/bundles.json +1 -0
- package/fixtures/release-quality/calibration.json +602 -0
- package/fixtures/release-quality/ecc/corpus.json +257 -0
- package/fixtures/release-quality/ecc/input.json +62 -0
- package/fixtures/release-quality/llm-wiki/corpus.json +263 -0
- package/fixtures/release-quality/llm-wiki/input.json +72 -0
- package/fixtures/release-quality/mem0/corpus.json +288 -0
- package/fixtures/release-quality/mem0/input.json +68 -0
- package/fixtures/release-quality/projects.json +29 -0
- package/fixtures/release-quality/release-quality-policy.json +232 -0
- package/fixtures/release-quality/rsbuild-react/corpus.json +263 -0
- package/fixtures/release-quality/rsbuild-react/input.json +62 -0
- package/fixtures/retrieval-quality-corpus.json +7 -7
- package/fixtures/semantic-regex-inventory.json +3 -10
- package/fixtures/unified-runtime/baselines/2026-08-08.json +37 -0
- package/fixtures/unified-runtime/context.json +206 -0
- package/fixtures/unified-runtime/knowledge.json +224 -0
- package/fixtures/unified-runtime/memory.json +224 -0
- package/fixtures/unified-runtime/real-projects/rsbuild-react-2026-08-07/context.json +32 -0
- package/fixtures/unified-runtime/real-projects/rsbuild-react-2026-08-07/knowledge.json +32 -0
- package/fixtures/unified-runtime/real-projects/rsbuild-react-2026-08-07/memory.json +32 -0
- package/fixtures/unified-runtime/real-projects/rsbuild-react-2026-08-07/semantic.json +409 -0
- package/fixtures/unified-runtime/runs/rsbuild-react-2026-08-05.json +54 -0
- package/fixtures/unified-runtime/semantic.json +409 -0
- package/package.json +24 -6
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"release-quality-comparison.js","sourceRoot":"","sources":["../src/release-quality-comparison.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AA8BvD,MAAM,UAAU,aAAa,CAC3B,QAA+B,EAC/B,SAAgC;IAEhC,IAAI,QAAQ,CAAC,WAAW,KAAK,CAAC,IAAI,SAAS,CAAC,WAAW,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC;IACnG,IAAI,QAAQ,CAAC,WAAW,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,EAAE,kBAAkB,EAAE,CAAC;IACtE,IAAI,SAAS,CAAC,WAAW,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;IAChE,MAAM,eAAe,GAAG,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IACnF,MAAM,cAAc,GAAG,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAClF,IAAI,eAAe,KAAK,cAAc;QAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;IACvE,OAAO,EAAE,MAAM,EAAE,eAAe,GAAG,cAAc,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;AACjF,CAAC;AAED,MAAM,UAAU,qBAAqB,CACnC,QAA0B,EAC1B,SAA2B;IAM3B,MAAM,WAAW,GAAG,4BAA4B,CAAC,QAAQ,CAAC,QAAQ,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC;IACxF,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,EAAE,MAAM,EAAE,gBAAgB,EAAE,WAAW,EAAE,CAAC;IAC7E,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;IACnD,IAAI,KAAK,CAAC,MAAM,KAAK,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,MAAM;WACrD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,SAAS,CAAC,EAAE,CAAC;QAC/D,OAAO,EAAE,MAAM,EAAE,gBAAgB,EAAE,WAAW,EAAE,CAAC,qBAAqB,CAAC,EAAE,CAAC;IAC5E,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;QACnD,IAAI;QACJ,aAAa,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAE,EAAE,SAAS,CAAC,OAAO,CAAC,IAAI,CAAE,CAAC;KACjE,CAAC,CAAC,CAAC;IACJ,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACrE,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,gBAAgB;QACnE,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW;YAC5C,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,kBAAkB;gBAC5D,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU;oBAC1C,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,gBAAgB,CAAC;IACtF,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;AAC7B,CAAC;AAED,SAAS,4BAA4B,CACnC,QAA4C,EAC5C,SAA6C;IAE7C,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,IAAI,QAAQ,CAAC,aAAa,KAAK,SAAS,CAAC,aAAa;QAAE,UAAU,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;IACtG,IAAI,QAAQ,CAAC,QAAQ,KAAK,SAAS,CAAC,QAAQ;QAAE,UAAU,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IACjF,IAAI,QAAQ,CAAC,UAAU,KAAK,SAAS,CAAC,UAAU;QAAE,UAAU,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IACrF,IAAI,iBAAiB,CAAC,QAAQ,CAAC,WAAW,CAAC,KAAK,iBAAiB,CAAC,SAAS,CAAC,WAAW,CAAC;QAAE,UAAU,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IACjI,IAAI,iBAAiB,CAAC,QAAQ,CAAC,qBAAqB,CAAC,KAAK,iBAAiB,CAAC,SAAS,CAAC,qBAAqB,CAAC;QAAE,UAAU,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;IAC3J,IAAI,QAAQ,CAAC,mBAAmB,KAAK,SAAS,CAAC,mBAAmB;QAAE,UAAU,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;IACjH,IAAI,iBAAiB,CAAC,QAAQ,CAAC,aAAa,CAAC,KAAK,iBAAiB,CAAC,SAAS,CAAC,aAAa,CAAC;QAAE,UAAU,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;IACzI,IAAI,QAAQ,CAAC,iBAAiB,KAAK,SAAS,CAAC,iBAAiB;QAAE,UAAU,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;IAC3G,IAAI,QAAQ,CAAC,UAAU,KAAK,SAAS,CAAC,UAAU;QAAE,UAAU,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IACrF,IAAI,QAAQ,CAAC,aAAa,KAAK,SAAS,CAAC,aAAa,IAAI,QAAQ,CAAC,YAAY,KAAK,SAAS,CAAC,YAAY;QAAE,UAAU,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC9I,OAAO,UAAU,CAAC;AACpB,CAAC"}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import type { EvidenceRerankerPort } from '@devflow-tools/sdk';
|
|
2
|
+
import { type EvidenceQualityMetric } from './evidence-quality-metrics.js';
|
|
3
|
+
import { type ReleaseQualityBundleReport } from './release-quality-bundle-evaluator.js';
|
|
4
|
+
export interface ReleaseQualityEvaluationReport {
|
|
5
|
+
schemaVersion: 'devflow.release-quality-evaluation.v1';
|
|
6
|
+
projects: Array<{
|
|
7
|
+
projectId: string;
|
|
8
|
+
report: ReleaseQualityBundleReport;
|
|
9
|
+
}>;
|
|
10
|
+
macro: Record<'context' | 'memory' | 'knowledge', {
|
|
11
|
+
candidateRecall: number | null;
|
|
12
|
+
recallAt5: number | null;
|
|
13
|
+
precisionAt5: number | null;
|
|
14
|
+
forbiddenContamination: number | null;
|
|
15
|
+
provenanceCompleteness: number | null;
|
|
16
|
+
}>;
|
|
17
|
+
micro: Record<'context' | 'memory' | 'knowledge', {
|
|
18
|
+
candidateRecall: EvidenceQualityMetric;
|
|
19
|
+
recallAt5: EvidenceQualityMetric;
|
|
20
|
+
precisionAt5: EvidenceQualityMetric;
|
|
21
|
+
forbiddenContamination: EvidenceQualityMetric;
|
|
22
|
+
provenanceCompleteness: EvidenceQualityMetric;
|
|
23
|
+
}>;
|
|
24
|
+
memoryDurability: EvidenceQualityMetric;
|
|
25
|
+
memoryTemporalCorrectness: EvidenceQualityMetric;
|
|
26
|
+
knowledgeVersionCorrectness: EvidenceQualityMetric;
|
|
27
|
+
consumption: EvidenceQualityMetric | {
|
|
28
|
+
status: 'unavailable';
|
|
29
|
+
reasonCode: 'host_ledger_required';
|
|
30
|
+
};
|
|
31
|
+
verification: EvidenceQualityMetric | {
|
|
32
|
+
status: 'unavailable';
|
|
33
|
+
reasonCode: 'verification_receipt_required';
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
export declare function evaluateReleaseQualityBundles(input: {
|
|
37
|
+
bundlesRoot: string;
|
|
38
|
+
mappingPath: string;
|
|
39
|
+
reranker: EvidenceRerankerPort;
|
|
40
|
+
hostProofPath?: string;
|
|
41
|
+
}): Promise<ReleaseQualityEvaluationReport>;
|
|
42
|
+
//# sourceMappingURL=release-quality-evaluation-command.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"release-quality-evaluation-command.d.ts","sourceRoot":"","sources":["../src/release-quality-evaluation-command.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAE/D,OAAO,EAEL,KAAK,qBAAqB,EAC3B,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAEL,KAAK,0BAA0B,EAChC,MAAM,uCAAuC,CAAC;AAkB/C,MAAM,WAAW,8BAA8B;IAC7C,aAAa,EAAE,uCAAuC,CAAC;IACvD,QAAQ,EAAE,KAAK,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,0BAA0B,CAAA;KAAE,CAAC,CAAC;IAC3E,KAAK,EAAE,MAAM,CAAC,SAAS,GAAG,QAAQ,GAAG,WAAW,EAAE;QAChD,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;QAC/B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;QACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QAC5B,sBAAsB,EAAE,MAAM,GAAG,IAAI,CAAC;QACtC,sBAAsB,EAAE,MAAM,GAAG,IAAI,CAAC;KACvC,CAAC,CAAC;IACH,KAAK,EAAE,MAAM,CAAC,SAAS,GAAG,QAAQ,GAAG,WAAW,EAAE;QAChD,eAAe,EAAE,qBAAqB,CAAC;QACvC,SAAS,EAAE,qBAAqB,CAAC;QACjC,YAAY,EAAE,qBAAqB,CAAC;QACpC,sBAAsB,EAAE,qBAAqB,CAAC;QAC9C,sBAAsB,EAAE,qBAAqB,CAAC;KAC/C,CAAC,CAAC;IACH,gBAAgB,EAAE,qBAAqB,CAAC;IACxC,yBAAyB,EAAE,qBAAqB,CAAC;IACjD,2BAA2B,EAAE,qBAAqB,CAAC;IACnD,WAAW,EAAE,qBAAqB,GAAG;QAAE,MAAM,EAAE,aAAa,CAAC;QAAC,UAAU,EAAE,sBAAsB,CAAA;KAAE,CAAC;IACnG,YAAY,EAAE,qBAAqB,GAAG;QAAE,MAAM,EAAE,aAAa,CAAC;QAAC,UAAU,EAAE,+BAA+B,CAAA;KAAE,CAAC;CAC9G;AAED,wBAAsB,6BAA6B,CAAC,KAAK,EAAE;IACzD,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,oBAAoB,CAAC;IAC/B,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB,GAAG,OAAO,CAAC,8BAA8B,CAAC,CAwD1C"}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import { readFileSync, realpathSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { join, resolve } from 'node:path';
|
|
3
|
+
import { fileURLToPath } from 'node:url';
|
|
4
|
+
import { createSamplingEvidenceReranker } from '@devflow-tools/mcp-server/retrieval';
|
|
5
|
+
import { stableRuntimeJson } from '@devflow-tools/sdk';
|
|
6
|
+
import { evidenceQualityMetric, } from './evidence-quality-metrics.js';
|
|
7
|
+
import { evaluateReleaseQualityBundle, } from './release-quality-bundle-evaluator.js';
|
|
8
|
+
import { createDeepSeekSamplingAdapter } from './context-symbol-quality-command.js';
|
|
9
|
+
import { validateReleaseQualityHostProofSet, } from './release-quality-host-proof.js';
|
|
10
|
+
export async function evaluateReleaseQualityBundles(input) {
|
|
11
|
+
const mapping = JSON.parse(readFileSync(resolve(input.mappingPath), 'utf8'));
|
|
12
|
+
if (mapping.schemaVersion !== 'devflow.release-quality-bundles.v1' || mapping.bundles.length === 0) {
|
|
13
|
+
throw new Error('RELEASE_QUALITY_BUNDLE_MAPPING_INVALID');
|
|
14
|
+
}
|
|
15
|
+
const projects = [];
|
|
16
|
+
for (const entry of mapping.bundles) {
|
|
17
|
+
const report = await evaluateReleaseQualityBundle({
|
|
18
|
+
bundlePath: join(resolve(input.bundlesRoot), entry.datasetDirectory),
|
|
19
|
+
reranker: input.reranker,
|
|
20
|
+
});
|
|
21
|
+
if (report.bundleId !== entry.bundleId
|
|
22
|
+
|| report.projectSnapshotId !== entry.projectSnapshotId
|
|
23
|
+
|| report.builderIdentity !== entry.builderIdentity) {
|
|
24
|
+
throw new Error(`RELEASE_QUALITY_BUNDLE_MAPPING_MISMATCH:${entry.projectId}`);
|
|
25
|
+
}
|
|
26
|
+
projects.push({ projectId: entry.projectId, report });
|
|
27
|
+
}
|
|
28
|
+
const channels = ['context', 'memory', 'knowledge'];
|
|
29
|
+
const metricNames = [
|
|
30
|
+
'candidateRecall', 'recallAt5', 'precisionAt5', 'forbiddenContamination', 'provenanceCompleteness',
|
|
31
|
+
];
|
|
32
|
+
const macro = Object.fromEntries(channels.map(channel => [channel, Object.fromEntries(metricNames.map(name => [
|
|
33
|
+
name,
|
|
34
|
+
average(projects.map(project => project.report.channels[channel][name].caseMacro)
|
|
35
|
+
.filter((value) => value !== null)),
|
|
36
|
+
]))]));
|
|
37
|
+
const micro = Object.fromEntries(channels.map(channel => [channel, Object.fromEntries(metricNames.map(name => [
|
|
38
|
+
name,
|
|
39
|
+
combine(projects.map(project => project.report.channels[channel][name].refMicro)),
|
|
40
|
+
]))]));
|
|
41
|
+
const hostProofs = input.hostProofPath ? validateReleaseQualityHostProofSet(JSON.parse(readFileSync(resolve(input.hostProofPath), 'utf8')), mapping.bundles.map(entry => entry.bundleId)) : undefined;
|
|
42
|
+
return {
|
|
43
|
+
schemaVersion: 'devflow.release-quality-evaluation.v1',
|
|
44
|
+
projects,
|
|
45
|
+
macro,
|
|
46
|
+
micro,
|
|
47
|
+
memoryDurability: combine(projects.map(project => project.report.memoryDurability)),
|
|
48
|
+
memoryTemporalCorrectness: combine(projects.map(project => project.report.memoryTemporalCorrectness)),
|
|
49
|
+
knowledgeVersionCorrectness: combine(projects.map(project => project.report.knowledgeVersionCorrectness)),
|
|
50
|
+
consumption: hostProofs
|
|
51
|
+
? evidenceQualityMetric(hostProofs.reduce((total, proof) => total + proof.consumedRefs.length, 0), hostProofs.reduce((total, proof) => total + proof.selectedRefs.length, 0))
|
|
52
|
+
: { status: 'unavailable', reasonCode: 'host_ledger_required' },
|
|
53
|
+
verification: hostProofs
|
|
54
|
+
? evidenceQualityMetric(hostProofs.reduce((total, proof) => total + proof.verifiedRefs.length, 0), hostProofs.reduce((total, proof) => total + proof.consumedRefs.length, 0))
|
|
55
|
+
: { status: 'unavailable', reasonCode: 'verification_receipt_required' },
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
async function main(argv) {
|
|
59
|
+
const rerankerAdapter = createDeepSeekSamplingAdapter(requiredOption(argv, '--sampling-model'));
|
|
60
|
+
const report = await evaluateReleaseQualityBundles({
|
|
61
|
+
bundlesRoot: requiredOption(argv, '--bundles'),
|
|
62
|
+
mappingPath: requiredOption(argv, '--mapping'),
|
|
63
|
+
reranker: createSamplingEvidenceReranker(rerankerAdapter),
|
|
64
|
+
...(option(argv, '--host-proof') ? { hostProofPath: option(argv, '--host-proof') } : {}),
|
|
65
|
+
});
|
|
66
|
+
const output = option(argv, '--output');
|
|
67
|
+
const json = `${stableRuntimeJson(report)}\n`;
|
|
68
|
+
if (output)
|
|
69
|
+
writeFileSync(resolve(output), json);
|
|
70
|
+
process.stdout.write(json);
|
|
71
|
+
}
|
|
72
|
+
function combine(metrics) {
|
|
73
|
+
return evidenceQualityMetric(metrics.reduce((total, metric) => total + metric.numerator, 0), metrics.reduce((total, metric) => total + metric.denominator, 0));
|
|
74
|
+
}
|
|
75
|
+
function average(values) {
|
|
76
|
+
return values.length === 0 ? null : values.reduce((total, value) => total + value, 0) / values.length;
|
|
77
|
+
}
|
|
78
|
+
function requiredOption(argv, name) {
|
|
79
|
+
const value = option(argv, name);
|
|
80
|
+
if (!value)
|
|
81
|
+
throw new Error(`${name} requires a path`);
|
|
82
|
+
return value;
|
|
83
|
+
}
|
|
84
|
+
function option(argv, name) {
|
|
85
|
+
const index = argv.indexOf(name);
|
|
86
|
+
return index >= 0 ? argv[index + 1] : undefined;
|
|
87
|
+
}
|
|
88
|
+
if (process.argv[1] && (realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url)) || realpathSync(process.argv[1]).endsWith('/release-quality-evaluation-command.ts'))) {
|
|
89
|
+
await main(process.argv.slice(2));
|
|
90
|
+
}
|
|
91
|
+
//# sourceMappingURL=release-quality-evaluation-command.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"release-quality-evaluation-command.js","sourceRoot":"","sources":["../src/release-quality-evaluation-command.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACpE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,8BAA8B,EAAE,MAAM,qCAAqC,CAAC;AAErF,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EACL,qBAAqB,GAEtB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EACL,4BAA4B,GAE7B,MAAM,uCAAuC,CAAC;AAC/C,OAAO,EAAE,6BAA6B,EAAE,MAAM,qCAAqC,CAAC;AACpF,OAAO,EACL,kCAAkC,GAEnC,MAAM,iCAAiC,CAAC;AAqCzC,MAAM,CAAC,KAAK,UAAU,6BAA6B,CAAC,KAKnD;IACC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC,CAAkB,CAAC;IAC9F,IAAI,OAAO,CAAC,aAAa,KAAK,oCAAoC,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnG,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;IAC5D,CAAC;IACD,MAAM,QAAQ,GAA+C,EAAE,CAAC;IAChE,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,MAAM,4BAA4B,CAAC;YAChD,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,KAAK,CAAC,gBAAgB,CAAC;YACpE,QAAQ,EAAE,KAAK,CAAC,QAAQ;SACzB,CAAC,CAAC;QACH,IAAI,MAAM,CAAC,QAAQ,KAAK,KAAK,CAAC,QAAQ;eACjC,MAAM,CAAC,iBAAiB,KAAK,KAAK,CAAC,iBAAiB;eACpD,MAAM,CAAC,eAAe,KAAK,KAAK,CAAC,eAAe,EAAE,CAAC;YACtD,MAAM,IAAI,KAAK,CAAC,2CAA2C,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;QAChF,CAAC;QACD,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IACxD,CAAC;IACD,MAAM,QAAQ,GAAG,CAAC,SAAS,EAAE,QAAQ,EAAE,WAAW,CAAU,CAAC;IAC7D,MAAM,WAAW,GAAG;QAClB,iBAAiB,EAAE,WAAW,EAAE,cAAc,EAAE,wBAAwB,EAAE,wBAAwB;KAC1F,CAAC;IACX,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5G,IAAI;YACJ,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC;iBAC9E,MAAM,CAAC,CAAC,KAAK,EAAmB,EAAE,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC;SACvD,CAAC,CAAC,CAAC,CAAC,CAA4C,CAAC;IAClD,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5G,IAAI;YACJ,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC;SAClF,CAAC,CAAC,CAAC,CAAC,CAA4C,CAAC;IAClD,MAAM,UAAU,GAAG,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,kCAAkC,CACzE,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC,CAAiC,EAC9F,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAC7C,CAAC,CAAC,CAAC,SAAS,CAAC;IACd,OAAO;QACL,aAAa,EAAE,uCAAuC;QACtD,QAAQ;QACR,KAAK;QACL,KAAK;QACL,gBAAgB,EAAE,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;QACnF,yBAAyB,EAAE,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,yBAAyB,CAAC,CAAC;QACrG,2BAA2B,EAAE,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,2BAA2B,CAAC,CAAC;QACzG,WAAW,EAAE,UAAU;YACrB,CAAC,CAAC,qBAAqB,CACrB,UAAU,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC,EACzE,UAAU,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC,CAC1E;YACD,CAAC,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,UAAU,EAAE,sBAAsB,EAAE;QACjE,YAAY,EAAE,UAAU;YACtB,CAAC,CAAC,qBAAqB,CACrB,UAAU,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC,EACzE,UAAU,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC,CAC1E;YACD,CAAC,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,UAAU,EAAE,+BAA+B,EAAE;KAC3E,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,IAAI,CAAC,IAAc;IAChC,MAAM,eAAe,GAAG,6BAA6B,CAAC,cAAc,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAC,CAAC;IAChG,MAAM,MAAM,GAAG,MAAM,6BAA6B,CAAC;QACjD,WAAW,EAAE,cAAc,CAAC,IAAI,EAAE,WAAW,CAAC;QAC9C,WAAW,EAAE,cAAc,CAAC,IAAI,EAAE,WAAW,CAAC;QAC9C,QAAQ,EAAE,8BAA8B,CAAC,eAAe,CAAC;QACzD,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,MAAM,CAAC,IAAI,EAAE,cAAc,CAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC1F,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IACxC,MAAM,IAAI,GAAG,GAAG,iBAAiB,CAAC,MAAM,CAAC,IAAI,CAAC;IAC9C,IAAI,MAAM;QAAE,aAAa,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,CAAC;IACjD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,OAAO,CAAC,OAAgC;IAC/C,OAAO,qBAAqB,CAC1B,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,EAC9D,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CACjE,CAAC;AACJ,CAAC;AAED,SAAS,OAAO,CAAC,MAAgB;IAC/B,OAAO,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;AACxG,CAAC;AAED,SAAS,cAAc,CAAC,IAAc,EAAE,IAAY;IAClD,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACjC,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,kBAAkB,CAAC,CAAC;IACvD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,MAAM,CAAC,IAAc,EAAE,IAAY;IAC1C,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACjC,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAClD,CAAC;AAED,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,YAAY,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,wCAAwC,CAAC,CAAC,EAAE,CAAC;IAC5L,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACpC,CAAC"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { RetrievalLedgerEventRecord } from '@devflow-tools/database';
|
|
2
|
+
export interface ReleaseQualityHostProofV1 {
|
|
3
|
+
schemaVersion: 'devflow.release-quality-host-proof.v1';
|
|
4
|
+
releaseRunId: string;
|
|
5
|
+
bundleId: string;
|
|
6
|
+
selectedRefs: string[];
|
|
7
|
+
consumedRefs: string[];
|
|
8
|
+
verifiedRefs: string[];
|
|
9
|
+
ledgerReceiptIds: string[];
|
|
10
|
+
}
|
|
11
|
+
export interface ReleaseQualityHostProofSetV1 {
|
|
12
|
+
schemaVersion: 'devflow.release-quality-host-proofs.v1';
|
|
13
|
+
releaseRunId: string;
|
|
14
|
+
proofs: ReleaseQualityHostProofV1[];
|
|
15
|
+
}
|
|
16
|
+
export declare function validateReleaseQualityHostProofSet(value: ReleaseQualityHostProofSetV1, expectedBundleIds: string[]): ReleaseQualityHostProofV1[];
|
|
17
|
+
export declare function deriveReleaseQualityHostProof(input: {
|
|
18
|
+
events: RetrievalLedgerEventRecord[];
|
|
19
|
+
releaseRunId: string;
|
|
20
|
+
bundleId: string;
|
|
21
|
+
projectRoot: string;
|
|
22
|
+
sessionId: string;
|
|
23
|
+
executionId?: string;
|
|
24
|
+
requestId?: string;
|
|
25
|
+
}): ReleaseQualityHostProofV1;
|
|
26
|
+
export declare function validateReleaseQualityHostProof(proof: ReleaseQualityHostProofV1, identity: {
|
|
27
|
+
releaseRunId: string;
|
|
28
|
+
bundleId: string;
|
|
29
|
+
}): void;
|
|
30
|
+
//# sourceMappingURL=release-quality-host-proof.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"release-quality-host-proof.d.ts","sourceRoot":"","sources":["../src/release-quality-host-proof.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,yBAAyB,CAAC;AAE1E,MAAM,WAAW,yBAAyB;IACxC,aAAa,EAAE,uCAAuC,CAAC;IACvD,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,gBAAgB,EAAE,MAAM,EAAE,CAAC;CAC5B;AAED,MAAM,WAAW,4BAA4B;IAC3C,aAAa,EAAE,wCAAwC,CAAC;IACxD,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,yBAAyB,EAAE,CAAC;CACrC;AAED,wBAAgB,kCAAkC,CAChD,KAAK,EAAE,4BAA4B,EACnC,iBAAiB,EAAE,MAAM,EAAE,GAC1B,yBAAyB,EAAE,CAe7B;AAED,wBAAgB,6BAA6B,CAAC,KAAK,EAAE;IACnD,MAAM,EAAE,0BAA0B,EAAE,CAAC;IACrC,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,GAAG,yBAAyB,CA4C5B;AAED,wBAAgB,+BAA+B,CAC7C,KAAK,EAAE,yBAAyB,EAChC,QAAQ,EAAE;IAAE,YAAY,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,GACnD,IAAI,CAkBN"}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
export function validateReleaseQualityHostProofSet(value, expectedBundleIds) {
|
|
2
|
+
if (value.schemaVersion !== 'devflow.release-quality-host-proofs.v1'
|
|
3
|
+
|| !nonEmpty(value.releaseRunId) || !Array.isArray(value.proofs) || value.proofs.length === 0) {
|
|
4
|
+
throw new Error('RELEASE_QUALITY_HOST_PROOF_SET_INVALID');
|
|
5
|
+
}
|
|
6
|
+
const expected = [...expectedBundleIds].sort();
|
|
7
|
+
const actual = value.proofs.map(proof => proof.bundleId).sort();
|
|
8
|
+
if (new Set(actual).size !== actual.length || expected.length !== actual.length
|
|
9
|
+
|| expected.some((bundleId, index) => bundleId !== actual[index])) {
|
|
10
|
+
throw new Error('RELEASE_QUALITY_HOST_PROOF_BUNDLE_SET_MISMATCH');
|
|
11
|
+
}
|
|
12
|
+
for (const proof of value.proofs) {
|
|
13
|
+
validateReleaseQualityHostProof(proof, { releaseRunId: value.releaseRunId, bundleId: proof.bundleId });
|
|
14
|
+
}
|
|
15
|
+
return value.proofs;
|
|
16
|
+
}
|
|
17
|
+
export function deriveReleaseQualityHostProof(input) {
|
|
18
|
+
if (!nonEmpty(input.releaseRunId) || !nonEmpty(input.bundleId)
|
|
19
|
+
|| !nonEmpty(input.projectRoot) || !nonEmpty(input.sessionId)) {
|
|
20
|
+
throw new Error('RELEASE_QUALITY_HOST_IDENTITY_INVALID');
|
|
21
|
+
}
|
|
22
|
+
const scoped = input.events.filter(event => {
|
|
23
|
+
if (event.projectRoot !== input.projectRoot || event.sessionId !== input.sessionId
|
|
24
|
+
|| (input.executionId && event.executionId !== input.executionId)
|
|
25
|
+
|| (input.requestId && event.requestId !== input.requestId))
|
|
26
|
+
return false;
|
|
27
|
+
return event.payload.releaseRunId === input.releaseRunId
|
|
28
|
+
&& event.payload.bundleId === input.bundleId;
|
|
29
|
+
});
|
|
30
|
+
if (scoped.length === 0)
|
|
31
|
+
throw new Error('RELEASE_QUALITY_HOST_LEDGER_EMPTY');
|
|
32
|
+
const selected = uniqueStageRefs(scoped, 'selected');
|
|
33
|
+
const consumed = uniqueStageRefs(scoped, 'consumed');
|
|
34
|
+
const verifiedEvents = scoped.filter(event => event.stage === 'verified');
|
|
35
|
+
const verified = uniqueStageRefs(verifiedEvents, 'verified');
|
|
36
|
+
const selectedSet = new Set(selected);
|
|
37
|
+
const consumedSet = new Set(consumed);
|
|
38
|
+
if (consumed.some(ref => !selectedSet.has(ref))) {
|
|
39
|
+
throw new Error('RELEASE_QUALITY_HOST_CONSUMPTION_WITHOUT_SELECTION');
|
|
40
|
+
}
|
|
41
|
+
if (verified.some(ref => !consumedSet.has(ref))) {
|
|
42
|
+
throw new Error('RELEASE_QUALITY_HOST_VERIFICATION_WITHOUT_CONSUMPTION');
|
|
43
|
+
}
|
|
44
|
+
if (verifiedEvents.some(event => !nonEmpty(event.verificationReceipt))) {
|
|
45
|
+
throw new Error('RELEASE_QUALITY_HOST_VERIFICATION_RECEIPT_MISSING');
|
|
46
|
+
}
|
|
47
|
+
requireChannels(selected, 'selected');
|
|
48
|
+
requireChannels(consumed, 'consumed');
|
|
49
|
+
requireChannels(verified, 'verified');
|
|
50
|
+
const ledgerReceiptIds = unique(verifiedEvents.map(event => event.verificationReceipt));
|
|
51
|
+
if (ledgerReceiptIds.length !== verified.length) {
|
|
52
|
+
throw new Error('RELEASE_QUALITY_HOST_VERIFICATION_RECEIPT_INCOMPLETE');
|
|
53
|
+
}
|
|
54
|
+
return {
|
|
55
|
+
schemaVersion: 'devflow.release-quality-host-proof.v1',
|
|
56
|
+
releaseRunId: input.releaseRunId,
|
|
57
|
+
bundleId: input.bundleId,
|
|
58
|
+
selectedRefs: selected,
|
|
59
|
+
consumedRefs: consumed,
|
|
60
|
+
verifiedRefs: verified,
|
|
61
|
+
ledgerReceiptIds,
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
export function validateReleaseQualityHostProof(proof, identity) {
|
|
65
|
+
if (proof.schemaVersion !== 'devflow.release-quality-host-proof.v1'
|
|
66
|
+
|| proof.releaseRunId !== identity.releaseRunId || proof.bundleId !== identity.bundleId) {
|
|
67
|
+
throw new Error('RELEASE_QUALITY_HOST_PROOF_IDENTITY_INVALID');
|
|
68
|
+
}
|
|
69
|
+
const selected = new Set(requireRefs(proof.selectedRefs, 'selected'));
|
|
70
|
+
const consumed = new Set(requireRefs(proof.consumedRefs, 'consumed'));
|
|
71
|
+
const verified = new Set(requireRefs(proof.verifiedRefs, 'verified'));
|
|
72
|
+
requireRefs(proof.ledgerReceiptIds, 'ledgerReceiptIds');
|
|
73
|
+
if ([...consumed].some(ref => !selected.has(ref))) {
|
|
74
|
+
throw new Error('RELEASE_QUALITY_HOST_CONSUMPTION_WITHOUT_SELECTION');
|
|
75
|
+
}
|
|
76
|
+
if ([...verified].some(ref => !consumed.has(ref))) {
|
|
77
|
+
throw new Error('RELEASE_QUALITY_HOST_VERIFICATION_WITHOUT_CONSUMPTION');
|
|
78
|
+
}
|
|
79
|
+
requireChannels([...selected], 'selected');
|
|
80
|
+
requireChannels([...consumed], 'consumed');
|
|
81
|
+
requireChannels([...verified], 'verified');
|
|
82
|
+
}
|
|
83
|
+
function uniqueStageRefs(events, stage) {
|
|
84
|
+
const refs = events.filter(event => event.stage === stage).map(event => event.sourceId);
|
|
85
|
+
if (new Set(refs).size !== refs.length)
|
|
86
|
+
throw new Error(`RELEASE_QUALITY_HOST_DUPLICATE_${stage.toUpperCase()}_REF`);
|
|
87
|
+
return refs;
|
|
88
|
+
}
|
|
89
|
+
function requireRefs(value, label) {
|
|
90
|
+
if (!Array.isArray(value) || value.length === 0 || value.some(item => !nonEmpty(item))
|
|
91
|
+
|| new Set(value).size !== value.length) {
|
|
92
|
+
throw new Error(`RELEASE_QUALITY_HOST_${label.toUpperCase()}_INVALID`);
|
|
93
|
+
}
|
|
94
|
+
return value;
|
|
95
|
+
}
|
|
96
|
+
function requireChannels(refs, stage) {
|
|
97
|
+
for (const channel of ['code', 'memory', 'knowledge']) {
|
|
98
|
+
if (!refs.some(ref => ref.startsWith(`${channel}:`))) {
|
|
99
|
+
throw new Error(`RELEASE_QUALITY_HOST_${stage.toUpperCase()}_CHANNEL_MISSING:${channel}`);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
function nonEmpty(value) {
|
|
104
|
+
return typeof value === 'string' && value.trim().length > 0;
|
|
105
|
+
}
|
|
106
|
+
function unique(values) {
|
|
107
|
+
return [...new Set(values)];
|
|
108
|
+
}
|
|
109
|
+
//# sourceMappingURL=release-quality-host-proof.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"release-quality-host-proof.js","sourceRoot":"","sources":["../src/release-quality-host-proof.ts"],"names":[],"mappings":"AAkBA,MAAM,UAAU,kCAAkC,CAChD,KAAmC,EACnC,iBAA2B;IAE3B,IAAI,KAAK,CAAC,aAAa,KAAK,wCAAwC;WAC/D,CAAC,QAAQ,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChG,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;IAC5D,CAAC;IACD,MAAM,QAAQ,GAAG,CAAC,GAAG,iBAAiB,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/C,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC;IAChE,IAAI,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM;WAC1E,QAAQ,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,KAAK,EAAE,EAAE,CAAC,QAAQ,KAAK,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;QACpE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;IACpE,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACjC,+BAA+B,CAAC,KAAK,EAAE,EAAE,YAAY,EAAE,KAAK,CAAC,YAAY,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;IACzG,CAAC;IACD,OAAO,KAAK,CAAC,MAAM,CAAC;AACtB,CAAC;AAED,MAAM,UAAU,6BAA6B,CAAC,KAQ7C;IACC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC;WACzD,CAAC,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC;QAChE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAC3D,CAAC;IACD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;QACzC,IAAI,KAAK,CAAC,WAAW,KAAK,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,SAAS,KAAK,KAAK,CAAC,SAAS;eAC7E,CAAC,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,WAAW,KAAK,KAAK,CAAC,WAAW,CAAC;eAC9D,CAAC,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,SAAS,KAAK,KAAK,CAAC,SAAS,CAAC;YAAE,OAAO,KAAK,CAAC;QAC5E,OAAO,KAAK,CAAC,OAAO,CAAC,YAAY,KAAK,KAAK,CAAC,YAAY;eACnD,KAAK,CAAC,OAAO,CAAC,QAAQ,KAAK,KAAK,CAAC,QAAQ,CAAC;IACjD,CAAC,CAAC,CAAC;IACH,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;IAC9E,MAAM,QAAQ,GAAG,eAAe,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IACrD,MAAM,QAAQ,GAAG,eAAe,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IACrD,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,UAAU,CAAC,CAAC;IAC1E,MAAM,QAAQ,GAAG,eAAe,CAAC,cAAc,EAAE,UAAU,CAAC,CAAC;IAC7D,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IACtC,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IACtC,IAAI,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;IACxE,CAAC;IACD,IAAI,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;IAC3E,CAAC;IACD,IAAI,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC,EAAE,CAAC;QACvE,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;IACvE,CAAC;IACD,eAAe,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IACtC,eAAe,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IACtC,eAAe,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IACtC,MAAM,gBAAgB,GAAG,MAAM,CAAC,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,mBAAoB,CAAC,CAAC,CAAC;IACzF,IAAI,gBAAgB,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC;QAChD,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;IAC1E,CAAC;IACD,OAAO;QACL,aAAa,EAAE,uCAAuC;QACtD,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,YAAY,EAAE,QAAQ;QACtB,YAAY,EAAE,QAAQ;QACtB,YAAY,EAAE,QAAQ;QACtB,gBAAgB;KACjB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,+BAA+B,CAC7C,KAAgC,EAChC,QAAoD;IAEpD,IAAI,KAAK,CAAC,aAAa,KAAK,uCAAuC;WAC9D,KAAK,CAAC,YAAY,KAAK,QAAQ,CAAC,YAAY,IAAI,KAAK,CAAC,QAAQ,KAAK,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAC1F,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IACjE,CAAC;IACD,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC,CAAC;IACtE,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC,CAAC;IACtE,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC,CAAC;IACtE,WAAW,CAAC,KAAK,CAAC,gBAAgB,EAAE,kBAAkB,CAAC,CAAC;IACxD,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QAClD,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;IACxE,CAAC;IACD,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QAClD,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;IAC3E,CAAC;IACD,eAAe,CAAC,CAAC,GAAG,QAAQ,CAAC,EAAE,UAAU,CAAC,CAAC;IAC3C,eAAe,CAAC,CAAC,GAAG,QAAQ,CAAC,EAAE,UAAU,CAAC,CAAC;IAC3C,eAAe,CAAC,CAAC,GAAG,QAAQ,CAAC,EAAE,UAAU,CAAC,CAAC;AAC7C,CAAC;AAED,SAAS,eAAe,CAAC,MAAoC,EAAE,KAA0C;IACvG,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACxF,IAAI,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,kCAAkC,KAAK,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;IACrH,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,WAAW,CAAC,KAAc,EAAE,KAAa;IAChD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;WACjF,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;QAC1C,MAAM,IAAI,KAAK,CAAC,wBAAwB,KAAK,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;IACzE,CAAC;IACD,OAAO,KAAiB,CAAC;AAC3B,CAAC;AAED,SAAS,eAAe,CAAC,IAAc,EAAE,KAAa;IACpD,KAAK,MAAM,OAAO,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,WAAW,CAAC,EAAE,CAAC;QACtD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,OAAO,GAAG,CAAC,CAAC,EAAE,CAAC;YACrD,MAAM,IAAI,KAAK,CAAC,wBAAwB,KAAK,CAAC,WAAW,EAAE,oBAAoB,OAAO,EAAE,CAAC,CAAC;QAC5F,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;AAC9D,CAAC;AAED,SAAS,MAAM,CAAC,MAAgB;IAC9B,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;AAC9B,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"release-quality-output-cleanup.d.ts","sourceRoot":"","sources":["../src/release-quality-output-cleanup.ts"],"names":[],"mappings":"AASA,wBAAgB,0BAA0B,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAY7D"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { chmodSync, existsSync, lstatSync, readdirSync, rmSync, } from 'node:fs';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
export function removeReleaseQualityOutput(path) {
|
|
4
|
+
if (!existsSync(path))
|
|
5
|
+
return;
|
|
6
|
+
const stat = lstatSync(path);
|
|
7
|
+
if (!stat.isSymbolicLink() && stat.isDirectory()) {
|
|
8
|
+
chmodSync(path, 0o755);
|
|
9
|
+
for (const name of readdirSync(path)) {
|
|
10
|
+
removeReleaseQualityOutput(join(path, name));
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
else if (!stat.isSymbolicLink()) {
|
|
14
|
+
chmodSync(path, 0o644);
|
|
15
|
+
}
|
|
16
|
+
rmSync(path, { recursive: stat.isDirectory() && !stat.isSymbolicLink(), force: true });
|
|
17
|
+
}
|
|
18
|
+
//# sourceMappingURL=release-quality-output-cleanup.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"release-quality-output-cleanup.js","sourceRoot":"","sources":["../src/release-quality-output-cleanup.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,SAAS,EACT,UAAU,EACV,SAAS,EACT,WAAW,EACX,MAAM,GACP,MAAM,SAAS,CAAC;AACjB,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,MAAM,UAAU,0BAA0B,CAAC,IAAY;IACrD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO;IAC9B,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAC7B,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;QACjD,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QACvB,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;YACrC,0BAA0B,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;SAAM,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,CAAC;QAClC,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACzB,CAAC;IACD,MAAM,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AACzF,CAAC"}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
export type ReleaseQualityThresholdName = 'candidateRecallAt20' | 'eligibilitySurvival' | 'recallAt5' | 'precisionAt5' | 'forbiddenContamination' | 'abstentionAccuracy' | 'coverageGapAccuracy' | 'consumptionRate' | 'verificationRate' | 'temporalCorrectness' | 'explicitSaveDurability' | 'knowledgeVersionCorrectness' | 'provenanceCompleteness';
|
|
2
|
+
export interface ReleaseQualityPolicyV1 {
|
|
3
|
+
schemaVersion: 'devflow.release-quality-policy.v1';
|
|
4
|
+
policyHash: string;
|
|
5
|
+
bundleIds: string[];
|
|
6
|
+
requiredSlices: Array<{
|
|
7
|
+
key: string;
|
|
8
|
+
minimumCases: number;
|
|
9
|
+
thresholds: Partial<Record<ReleaseQualityThresholdName, number>>;
|
|
10
|
+
}>;
|
|
11
|
+
baselineRuntimeIdentity: {
|
|
12
|
+
commit: string;
|
|
13
|
+
lockfileHash: string;
|
|
14
|
+
buildConfigHash: string;
|
|
15
|
+
tarballHashes: Record<string, string>;
|
|
16
|
+
};
|
|
17
|
+
comparisonProfile: {
|
|
18
|
+
providerModel: {
|
|
19
|
+
provider: string;
|
|
20
|
+
model: string;
|
|
21
|
+
};
|
|
22
|
+
readerProfileHash: string;
|
|
23
|
+
budgetHash: string;
|
|
24
|
+
candidateTopK: 20;
|
|
25
|
+
selectedTopK: 5;
|
|
26
|
+
};
|
|
27
|
+
externalHostProfile: {
|
|
28
|
+
host: 'claude-code';
|
|
29
|
+
version: string;
|
|
30
|
+
promptHashes: Record<string, string>;
|
|
31
|
+
permissions: string[];
|
|
32
|
+
maxTurns: number;
|
|
33
|
+
timeoutMs: number;
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
export interface ReleaseQualityPolicyEvaluationInput {
|
|
37
|
+
projects: Array<{
|
|
38
|
+
projectId: string;
|
|
39
|
+
report: {
|
|
40
|
+
cases: Array<{
|
|
41
|
+
channel: string;
|
|
42
|
+
caseClass: string;
|
|
43
|
+
}>;
|
|
44
|
+
channels: Record<string, Record<string, {
|
|
45
|
+
refMicro: {
|
|
46
|
+
numerator: number;
|
|
47
|
+
denominator: number;
|
|
48
|
+
value: number | null;
|
|
49
|
+
};
|
|
50
|
+
}>>;
|
|
51
|
+
memoryDurability: {
|
|
52
|
+
numerator: number;
|
|
53
|
+
denominator: number;
|
|
54
|
+
value: number | null;
|
|
55
|
+
};
|
|
56
|
+
memoryTemporalCorrectness: {
|
|
57
|
+
numerator: number;
|
|
58
|
+
denominator: number;
|
|
59
|
+
value: number | null;
|
|
60
|
+
};
|
|
61
|
+
knowledgeVersionCorrectness: {
|
|
62
|
+
numerator: number;
|
|
63
|
+
denominator: number;
|
|
64
|
+
value: number | null;
|
|
65
|
+
};
|
|
66
|
+
};
|
|
67
|
+
}>;
|
|
68
|
+
host?: {
|
|
69
|
+
consumptionRate: {
|
|
70
|
+
numerator: number;
|
|
71
|
+
denominator: number;
|
|
72
|
+
value: number | null;
|
|
73
|
+
};
|
|
74
|
+
verificationRate: {
|
|
75
|
+
numerator: number;
|
|
76
|
+
denominator: number;
|
|
77
|
+
value: number | null;
|
|
78
|
+
};
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
export interface ReleaseQualityPolicyEvaluation {
|
|
82
|
+
passed: boolean;
|
|
83
|
+
failures: string[];
|
|
84
|
+
}
|
|
85
|
+
export declare function loadReleaseQualityPolicy(path: string): ReleaseQualityPolicyV1;
|
|
86
|
+
export declare function evaluateReleaseQualityPolicy(policy: ReleaseQualityPolicyV1, input: ReleaseQualityPolicyEvaluationInput): ReleaseQualityPolicyEvaluation;
|
|
87
|
+
//# sourceMappingURL=release-quality-policy.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"release-quality-policy.d.ts","sourceRoot":"","sources":["../src/release-quality-policy.ts"],"names":[],"mappings":"AAGA,MAAM,MAAM,2BAA2B,GACnC,qBAAqB,GACrB,qBAAqB,GACrB,WAAW,GACX,cAAc,GACd,wBAAwB,GACxB,oBAAoB,GACpB,qBAAqB,GACrB,iBAAiB,GACjB,kBAAkB,GAClB,qBAAqB,GACrB,wBAAwB,GACxB,6BAA6B,GAC7B,wBAAwB,CAAC;AAE7B,MAAM,WAAW,sBAAsB;IACrC,aAAa,EAAE,mCAAmC,CAAC;IACnD,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,cAAc,EAAE,KAAK,CAAC;QACpB,GAAG,EAAE,MAAM,CAAC;QACZ,YAAY,EAAE,MAAM,CAAC;QACrB,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,2BAA2B,EAAE,MAAM,CAAC,CAAC,CAAC;KAClE,CAAC,CAAC;IACH,uBAAuB,EAAE;QACvB,MAAM,EAAE,MAAM,CAAC;QACf,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,EAAE,MAAM,CAAC;QACxB,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KACvC,CAAC;IACF,iBAAiB,EAAE;QACjB,aAAa,EAAE;YAAE,QAAQ,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAA;SAAE,CAAC;QACnD,iBAAiB,EAAE,MAAM,CAAC;QAC1B,UAAU,EAAE,MAAM,CAAC;QACnB,aAAa,EAAE,EAAE,CAAC;QAClB,YAAY,EAAE,CAAC,CAAC;KACjB,CAAC;IACF,mBAAmB,EAAE;QACnB,IAAI,EAAE,aAAa,CAAC;QACpB,OAAO,EAAE,MAAM,CAAC;QAChB,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACrC,WAAW,EAAE,MAAM,EAAE,CAAC;QACtB,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;CACH;AAED,MAAM,WAAW,mCAAmC;IAClD,QAAQ,EAAE,KAAK,CAAC;QACd,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE;YACN,KAAK,EAAE,KAAK,CAAC;gBAAE,OAAO,EAAE,MAAM,CAAC;gBAAC,SAAS,EAAE,MAAM,CAAA;aAAE,CAAC,CAAC;YACrD,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE;gBAAE,QAAQ,EAAE;oBAAE,SAAS,EAAE,MAAM,CAAC;oBAAC,WAAW,EAAE,MAAM,CAAC;oBAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;iBAAE,CAAA;aAAE,CAAC,CAAC,CAAC;YACzH,gBAAgB,EAAE;gBAAE,SAAS,EAAE,MAAM,CAAC;gBAAC,WAAW,EAAE,MAAM,CAAC;gBAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;aAAE,CAAC;YACnF,yBAAyB,EAAE;gBAAE,SAAS,EAAE,MAAM,CAAC;gBAAC,WAAW,EAAE,MAAM,CAAC;gBAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;aAAE,CAAC;YAC5F,2BAA2B,EAAE;gBAAE,SAAS,EAAE,MAAM,CAAC;gBAAC,WAAW,EAAE,MAAM,CAAC;gBAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;aAAE,CAAC;SAC/F,CAAC;KACH,CAAC,CAAC;IACH,IAAI,CAAC,EAAE;QACL,eAAe,EAAE;YAAE,SAAS,EAAE,MAAM,CAAC;YAAC,WAAW,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;SAAE,CAAC;QAClF,gBAAgB,EAAE;YAAE,SAAS,EAAE,MAAM,CAAC;YAAC,WAAW,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;SAAE,CAAC;KACpF,CAAC;CACH;AAED,MAAM,WAAW,8BAA8B;IAC7C,MAAM,EAAE,OAAO,CAAC;IAChB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AASD,wBAAgB,wBAAwB,CAAC,IAAI,EAAE,MAAM,GAAG,sBAAsB,CAiC7E;AAED,wBAAgB,4BAA4B,CAC1C,MAAM,EAAE,sBAAsB,EAC9B,KAAK,EAAE,mCAAmC,GACzC,8BAA8B,CAoChC"}
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
import { readFileSync } from 'node:fs';
|
|
2
|
+
import { stableRuntimeHash } from '@devflow-tools/sdk';
|
|
3
|
+
const THRESHOLD_NAMES = new Set([
|
|
4
|
+
'candidateRecallAt20', 'eligibilitySurvival', 'recallAt5', 'precisionAt5',
|
|
5
|
+
'forbiddenContamination', 'abstentionAccuracy', 'coverageGapAccuracy',
|
|
6
|
+
'consumptionRate', 'verificationRate', 'temporalCorrectness',
|
|
7
|
+
'explicitSaveDurability', 'knowledgeVersionCorrectness', 'provenanceCompleteness',
|
|
8
|
+
]);
|
|
9
|
+
export function loadReleaseQualityPolicy(path) {
|
|
10
|
+
const raw = JSON.parse(readFileSync(path, 'utf8'));
|
|
11
|
+
if (!record(raw))
|
|
12
|
+
fail('schema');
|
|
13
|
+
assertKeys(raw, [
|
|
14
|
+
'schemaVersion', 'policyHash', 'bundleIds', 'requiredSlices', 'baselineRuntimeIdentity',
|
|
15
|
+
'comparisonProfile', 'externalHostProfile',
|
|
16
|
+
]);
|
|
17
|
+
const policy = raw;
|
|
18
|
+
if (policy.schemaVersion !== 'devflow.release-quality-policy.v1')
|
|
19
|
+
fail('schema');
|
|
20
|
+
requireUniqueNonEmpty(policy.bundleIds, 'bundle_ids');
|
|
21
|
+
if (!Array.isArray(policy.requiredSlices) || policy.requiredSlices.length === 0)
|
|
22
|
+
fail('required_slices');
|
|
23
|
+
if (policy.requiredSlices.some(slice => !record(slice)))
|
|
24
|
+
fail('required_slices');
|
|
25
|
+
requireUniqueNonEmpty(policy.requiredSlices.map(slice => slice.key), 'slice_keys');
|
|
26
|
+
for (const slice of policy.requiredSlices) {
|
|
27
|
+
assertKeys(slice, ['key', 'minimumCases', 'thresholds']);
|
|
28
|
+
if (!Number.isSafeInteger(slice.minimumCases) || slice.minimumCases < 1)
|
|
29
|
+
fail('minimum_cases');
|
|
30
|
+
const thresholds = Object.entries(slice.thresholds ?? {});
|
|
31
|
+
if (thresholds.length === 0)
|
|
32
|
+
fail('thresholds');
|
|
33
|
+
for (const [name, value] of thresholds) {
|
|
34
|
+
if (!THRESHOLD_NAMES.has(name)
|
|
35
|
+
|| typeof value !== 'number' || !Number.isFinite(value) || value < 0 || value > 1)
|
|
36
|
+
fail('thresholds');
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
validateBaseline(policy.baselineRuntimeIdentity);
|
|
40
|
+
validateComparisonProfile(policy.comparisonProfile);
|
|
41
|
+
validateExternalHost(policy.externalHostProfile);
|
|
42
|
+
const promptBundleIds = Object.keys(policy.externalHostProfile.promptHashes).sort();
|
|
43
|
+
const policyBundleIds = [...policy.bundleIds].sort();
|
|
44
|
+
if (promptBundleIds.length !== policyBundleIds.length
|
|
45
|
+
|| promptBundleIds.some((bundleId, index) => bundleId !== policyBundleIds[index]))
|
|
46
|
+
fail('external_host_prompt_hashes');
|
|
47
|
+
const { policyHash, ...body } = policy;
|
|
48
|
+
if (!nonEmpty(policyHash) || stableRuntimeHash(body) !== policyHash)
|
|
49
|
+
fail('policy_hash');
|
|
50
|
+
return policy;
|
|
51
|
+
}
|
|
52
|
+
export function evaluateReleaseQualityPolicy(policy, input) {
|
|
53
|
+
const failures = [];
|
|
54
|
+
for (const slice of policy.requiredSlices) {
|
|
55
|
+
const match = /^(.+)\/(context|memory|knowledge)\/(positive|lifecycle)$/u.exec(slice.key);
|
|
56
|
+
if (!match) {
|
|
57
|
+
if (slice.key === 'host') {
|
|
58
|
+
evaluateHostSlice(slice, input.host, failures);
|
|
59
|
+
continue;
|
|
60
|
+
}
|
|
61
|
+
failures.push(`${slice.key}:slice_key_invalid`);
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
const [, projectId, channel, caseClass] = match;
|
|
65
|
+
const project = input.projects.find(item => item.projectId === projectId);
|
|
66
|
+
if (!project) {
|
|
67
|
+
failures.push(`${slice.key}:project_missing`);
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
const cases = project.report.cases.filter(item => item.channel === channel && item.caseClass === caseClass);
|
|
71
|
+
if (cases.length < slice.minimumCases) {
|
|
72
|
+
failures.push(`${slice.key}:minimum_cases:${cases.length}/${slice.minimumCases}`);
|
|
73
|
+
continue;
|
|
74
|
+
}
|
|
75
|
+
for (const [name, threshold] of Object.entries(slice.thresholds)) {
|
|
76
|
+
const metric = metricFor(project.report, channel, name);
|
|
77
|
+
if (!metric || metric.denominator <= 0 || metric.value === null) {
|
|
78
|
+
failures.push(`${slice.key}:${name}:unmeasurable`);
|
|
79
|
+
continue;
|
|
80
|
+
}
|
|
81
|
+
const passes = name === 'forbiddenContamination'
|
|
82
|
+
? metric.value <= threshold
|
|
83
|
+
: metric.value >= threshold;
|
|
84
|
+
if (!passes)
|
|
85
|
+
failures.push(`${slice.key}:${name}:${metric.value}<${threshold}`);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
return { passed: failures.length === 0, failures };
|
|
89
|
+
}
|
|
90
|
+
function evaluateHostSlice(slice, host, failures) {
|
|
91
|
+
if (!host) {
|
|
92
|
+
failures.push('host:unavailable');
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
if (slice.minimumCases > host.consumptionRate.denominator
|
|
96
|
+
|| slice.minimumCases > host.verificationRate.denominator) {
|
|
97
|
+
failures.push(`host:minimum_cases:${slice.minimumCases}`);
|
|
98
|
+
}
|
|
99
|
+
for (const [name, threshold] of Object.entries(slice.thresholds)) {
|
|
100
|
+
const metric = name === 'consumptionRate' ? host.consumptionRate : host.verificationRate;
|
|
101
|
+
if (metric.denominator <= 0 || metric.value === null || metric.value < threshold) {
|
|
102
|
+
failures.push(`host:${name}:unmeasurable_or_below_threshold`);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
function metricFor(report, channel, name) {
|
|
107
|
+
if (name === 'temporalCorrectness')
|
|
108
|
+
return report.memoryTemporalCorrectness;
|
|
109
|
+
if (name === 'explicitSaveDurability')
|
|
110
|
+
return report.memoryDurability;
|
|
111
|
+
if (name === 'knowledgeVersionCorrectness')
|
|
112
|
+
return report.knowledgeVersionCorrectness;
|
|
113
|
+
return report.channels[channel]?.[metricName(name)]?.refMicro;
|
|
114
|
+
}
|
|
115
|
+
function metricName(name) {
|
|
116
|
+
if (name === 'candidateRecallAt20')
|
|
117
|
+
return 'candidateRecall';
|
|
118
|
+
return name;
|
|
119
|
+
}
|
|
120
|
+
function validateBaseline(identity) {
|
|
121
|
+
if (record(identity))
|
|
122
|
+
assertKeys(identity, ['commit', 'lockfileHash', 'buildConfigHash', 'tarballHashes']);
|
|
123
|
+
if (!identity || !/^[0-9a-f]{40}$/u.test(identity.commit)
|
|
124
|
+
|| !nonEmpty(identity.lockfileHash) || !nonEmpty(identity.buildConfigHash))
|
|
125
|
+
fail('baseline_runtime_identity');
|
|
126
|
+
const tarballs = Object.entries(identity.tarballHashes ?? {});
|
|
127
|
+
if (tarballs.length === 0 || tarballs.some(([name, hash]) => !nonEmpty(name) || !nonEmpty(hash)))
|
|
128
|
+
fail('baseline_runtime_identity');
|
|
129
|
+
}
|
|
130
|
+
function validateComparisonProfile(profile) {
|
|
131
|
+
if (record(profile))
|
|
132
|
+
assertKeys(profile, [
|
|
133
|
+
'providerModel', 'readerProfileHash', 'budgetHash', 'candidateTopK', 'selectedTopK',
|
|
134
|
+
]);
|
|
135
|
+
if (record(profile?.providerModel))
|
|
136
|
+
assertKeys(profile.providerModel, ['provider', 'model']);
|
|
137
|
+
if (!profile || !nonEmpty(profile.providerModel?.provider) || !nonEmpty(profile.providerModel?.model)
|
|
138
|
+
|| !nonEmpty(profile.readerProfileHash) || !nonEmpty(profile.budgetHash)
|
|
139
|
+
|| profile.candidateTopK !== 20 || profile.selectedTopK !== 5)
|
|
140
|
+
fail('comparison_profile');
|
|
141
|
+
}
|
|
142
|
+
function validateExternalHost(profile) {
|
|
143
|
+
if (record(profile))
|
|
144
|
+
assertKeys(profile, [
|
|
145
|
+
'host', 'version', 'promptHashes', 'permissions', 'maxTurns', 'timeoutMs',
|
|
146
|
+
]);
|
|
147
|
+
const promptHashes = Object.entries(profile?.promptHashes ?? {});
|
|
148
|
+
if (!profile || profile.host !== 'claude-code' || !nonEmpty(profile.version)
|
|
149
|
+
|| promptHashes.length === 0 || promptHashes.some(([bundleId, hash]) => !nonEmpty(bundleId) || !nonEmpty(hash))
|
|
150
|
+
|| !Array.isArray(profile.permissions) || profile.permissions.some(permission => !nonEmpty(permission))
|
|
151
|
+
|| !['mcp__devflow__get_project_context', 'mcp__devflow__get_memory', 'mcp__devflow__get_knowledge']
|
|
152
|
+
.every(tool => profile.permissions.includes(tool))
|
|
153
|
+
|| !['mcp__devflow__run_workflow', 'mcp__devflow__devflow_report_step']
|
|
154
|
+
.every(tool => profile.permissions.includes(tool))
|
|
155
|
+
|| !Number.isSafeInteger(profile.maxTurns) || profile.maxTurns < 1
|
|
156
|
+
|| !Number.isSafeInteger(profile.timeoutMs) || profile.timeoutMs < 1)
|
|
157
|
+
fail('external_host_profile');
|
|
158
|
+
}
|
|
159
|
+
function requireUniqueNonEmpty(values, field) {
|
|
160
|
+
if (!Array.isArray(values) || values.length === 0 || new Set(values).size !== values.length
|
|
161
|
+
|| values.some(value => !nonEmpty(value)))
|
|
162
|
+
fail(field);
|
|
163
|
+
}
|
|
164
|
+
function nonEmpty(value) {
|
|
165
|
+
return typeof value === 'string' && value.trim().length > 0;
|
|
166
|
+
}
|
|
167
|
+
function assertKeys(value, expected) {
|
|
168
|
+
const actual = Object.keys(value).sort();
|
|
169
|
+
const allowed = [...expected].sort();
|
|
170
|
+
if (actual.length !== allowed.length || actual.some((key, index) => key !== allowed[index]))
|
|
171
|
+
fail('fields');
|
|
172
|
+
}
|
|
173
|
+
function record(value) {
|
|
174
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
175
|
+
}
|
|
176
|
+
function fail(field) {
|
|
177
|
+
throw new Error(`Invalid release quality policy: ${field}`);
|
|
178
|
+
}
|
|
179
|
+
//# sourceMappingURL=release-quality-policy.js.map
|