crysk 0.0.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +176 -0
- package/README.md +611 -3
- package/bin/pr-hero.js +166 -0
- package/config/models/anthropic.json +5 -0
- package/dist/SKILL-7e3725zn.md +496 -0
- package/dist/SKILL-7q35wcaz.md +66 -0
- package/dist/SKILL-9at22381.md +74 -0
- package/dist/SKILL-ex2hxxh3.md +134 -0
- package/dist/SKILL-y8rv8msh.md +134 -0
- package/dist/SKILL-ygka1bs0.md +515 -0
- package/dist/adjudicator-dtg2axnh.md +86 -0
- package/dist/admission-config.example-a9vqmmad.json +9 -0
- package/dist/ci-admission-camfbx2b.md +122 -0
- package/dist/ci-admission-qk9r9taq.md +124 -0
- package/dist/cli.js +42501 -0
- package/dist/consolidate-ac0mnfyj.md +24 -0
- package/dist/deep-review-lifecycle-m698ybjp.md +362 -0
- package/dist/deep-review-logic-a6b1gczf.md +156 -0
- package/dist/deep-review-parity-0ec904db.md +120 -0
- package/dist/deep-review-reliability-qyakbjb9.md +125 -0
- package/dist/deep-review-resilience-3tz8y26e.md +118 -0
- package/dist/index.js +20190 -0
- package/dist/opencode-ci-7b0be64p.md +141 -0
- package/dist/review-refuter-j4e9gezy.md +71 -0
- package/dist/scout-yvp153q7.md +140 -0
- package/dist/summarizer-zca44nxa.md +18 -0
- package/dist/workflow-w1g9szna.yml +151 -0
- package/package.json +81 -4
- package/prompts/consolidate.md +24 -0
- package/prompts/default/PROVENANCE.md +18 -0
- package/prompts/default/deep-review-lifecycle.md +362 -0
- package/prompts/default/deep-review-logic.md +156 -0
- package/prompts/default/deep-review-parity.md +120 -0
- package/prompts/default/deep-review-reliability.md +125 -0
- package/prompts/default/deep-review-resilience.md +118 -0
- package/prompts/default/review-refuter.md +71 -0
- package/prompts/scout.md +140 -0
- package/prompts/summarizer.md +18 -0
- package/skills/crysk/SKILL.md +74 -0
- package/skills/crysk-ci-setup/SKILL.md +134 -0
- package/skills/crysk-ci-setup/assets/admission-config.example.json +9 -0
- package/skills/crysk-ci-setup/assets/workflow.yml +151 -0
- package/skills/crysk-ci-setup/references/ci-admission.md +124 -0
- package/skills/crysk-ci-setup/references/opencode-ci.md +141 -0
- package/skills/crysk-loop/SKILL.md +515 -0
- package/skills/crysk-loop/adjudicator.md +86 -0
- package/skills/crysk-pr/SKILL.md +66 -0
- package/src/agent-env.ts +1084 -0
- package/src/asset-manifest.ts +83 -0
- package/src/assets.d.ts +6 -0
- package/src/assets.ts +183 -0
- package/src/brand-paths.ts +155 -0
- package/src/brand.ts +255 -0
- package/src/ci/admission-ledger.ts +350 -0
- package/src/ci/classify-error.ts +73 -0
- package/src/ci/cli-record.ts +271 -0
- package/src/ci/force-comment.ts +39 -0
- package/src/ci/gates.ts +945 -0
- package/src/ci/outcome.ts +2181 -0
- package/src/ci/override.ts +187 -0
- package/src/ci/publish.ts +1639 -0
- package/src/ci/reporter.ts +626 -0
- package/src/ci/review-admission.ts +910 -0
- package/src/ci/review-levels.ts +310 -0
- package/src/ci/review-policy.ts +18 -0
- package/src/ci/review-risk.ts +184 -0
- package/src/ci/setup.ts +844 -0
- package/src/ci/trigger.ts +1353 -0
- package/src/cli.ts +348 -0
- package/src/commands/activity.ts +67 -0
- package/src/commands/ci-setup-questions.ts +397 -0
- package/src/commands/ci-setup.ts +381 -0
- package/src/commands/config.ts +99 -0
- package/src/commands/doctor.ts +108 -0
- package/src/commands/init.ts +139 -0
- package/src/commands/mcp.ts +10 -0
- package/src/commands/menu.ts +129 -0
- package/src/commands/post.ts +302 -0
- package/src/commands/skills.ts +83 -0
- package/src/commands/status.ts +173 -0
- package/src/commands/triage.ts +342 -0
- package/src/commands/uninstall.ts +37 -0
- package/src/commands/upgrade.ts +214 -0
- package/src/commands/usage.ts +98 -0
- package/src/compare/compare.ts +28 -0
- package/src/compare/floor-test.ts +471 -0
- package/src/compare/martian-adapter.ts +80 -0
- package/src/config/config.ts +267 -0
- package/src/corpus/corpus.ts +1094 -0
- package/src/corpus/preflight.ts +1789 -0
- package/src/diversity/accounting.ts +90 -0
- package/src/diversity/adjudication.ts +145 -0
- package/src/diversity/admission.ts +60 -0
- package/src/diversity/benchmark.ts +195 -0
- package/src/diversity/clustering.ts +285 -0
- package/src/diversity/errors.ts +7 -0
- package/src/diversity/identity.ts +285 -0
- package/src/diversity/index.ts +11 -0
- package/src/diversity/pipeline-integration.ts +448 -0
- package/src/diversity/projection.ts +57 -0
- package/src/diversity/self-contained.ts +68 -0
- package/src/diversity/target-validation.ts +179 -0
- package/src/doctor-migration.ts +209 -0
- package/src/doctor.ts +723 -0
- package/src/errors.ts +83 -0
- package/src/execution/admission.ts +75 -0
- package/src/execution/atomic-write.ts +35 -0
- package/src/execution/attempt-evidence.ts +415 -0
- package/src/execution/bucket-id.ts +136 -0
- package/src/execution/concurrency-limiter.ts +175 -0
- package/src/execution/contracts.ts +443 -0
- package/src/execution/failure-classification.ts +42 -0
- package/src/execution/failure-policy.ts +240 -0
- package/src/execution/harness.ts +2548 -0
- package/src/execution/settlement.ts +348 -0
- package/src/execution/spawned-process.ts +66 -0
- package/src/execution/spend-limiter.ts +424 -0
- package/src/execution/step-artifacts.ts +55 -0
- package/src/execution/usage-normalized.ts +469 -0
- package/src/git/git.ts +396 -0
- package/src/git/identity.ts +64 -0
- package/src/git/refs.ts +140 -0
- package/src/home-preflight.ts +443 -0
- package/src/home.ts +243 -0
- package/src/ignore-file.ts +336 -0
- package/src/ignore-read.ts +118 -0
- package/src/index.ts +34 -0
- package/src/legacy-launchd.ts +29 -0
- package/src/mcp/mcp.ts +150 -0
- package/src/mcp/preflight.ts +407 -0
- package/src/menu-context.ts +192 -0
- package/src/model/catalog.ts +251 -0
- package/src/model/exec-snapshots.ts +390 -0
- package/src/model/free-discovery.ts +302 -0
- package/src/model/provider-capabilities.ts +814 -0
- package/src/model/routing.ts +600 -0
- package/src/pr/admission.ts +984 -0
- package/src/pr/answered-findings.ts +169 -0
- package/src/pr/ci-admission-gate.ts +743 -0
- package/src/pr/ci-publish.ts +76 -0
- package/src/pr/discovery.ts +432 -0
- package/src/pr/inline.ts +507 -0
- package/src/pr/plan.ts +366 -0
- package/src/pr/posting.ts +115 -0
- package/src/pr/pr-status.ts +845 -0
- package/src/pr/pr.ts +3642 -0
- package/src/pr/preflight.ts +1031 -0
- package/src/pr/publish-outcome.ts +361 -0
- package/src/pr/range.ts +119 -0
- package/src/pr/reverts-preflight.ts +606 -0
- package/src/pr/reverts.ts +377 -0
- package/src/pr/review-pr.ts +927 -0
- package/src/pr/status.ts +154 -0
- package/src/pr/target.ts +388 -0
- package/src/pr/teardown.ts +437 -0
- package/src/pr/worktree-setup.ts +60 -0
- package/src/production-runtime.ts +1877 -0
- package/src/progress.ts +487 -0
- package/src/rereview/adjudicate.ts +89 -0
- package/src/rereview/classify.ts +362 -0
- package/src/rereview/identity.ts +211 -0
- package/src/rereview/plan.ts +308 -0
- package/src/rereview/prepare.ts +1025 -0
- package/src/rereview/state.ts +767 -0
- package/src/rereview/verify.ts +241 -0
- package/src/review/boundary.ts +105 -0
- package/src/review/consolidate.ts +246 -0
- package/src/review/dedupe.ts +368 -0
- package/src/review/drafts.ts +448 -0
- package/src/review/findings-conformance.ts +120 -0
- package/src/review/findings.ts +714 -0
- package/src/review/pipeline.ts +3977 -0
- package/src/review/preflight.ts +3575 -0
- package/src/review/prompt-set.ts +177 -0
- package/src/review/proof-refs.ts +187 -0
- package/src/review/report.ts +1821 -0
- package/src/review/review.ts +490 -0
- package/src/review/root-cause.ts +120 -0
- package/src/review/route-preflight.ts +289 -0
- package/src/review/run.ts +601 -0
- package/src/review/scout.ts +323 -0
- package/src/review/size-gate.ts +595 -0
- package/src/review/spec.ts +235 -0
- package/src/review/step-runner.ts +233 -0
- package/src/runner-authority.ts +596 -0
- package/src/security/binding-policy.ts +115 -0
- package/src/security/credential-broker.ts +773 -0
- package/src/security/evidence-redaction.ts +62 -0
- package/src/security/execution-authority.ts +27 -0
- package/src/security/redact.ts +71 -0
- package/src/security/workspace-read-broker.ts +466 -0
- package/src/server/client.ts +190 -0
- package/src/server/preflight.ts +217 -0
- package/src/server/server.ts +327 -0
- package/src/signal-handler.ts +50 -0
- package/src/store/activity.ts +413 -0
- package/src/store/backfill-preflight.ts +49 -0
- package/src/store/backfill.ts +141 -0
- package/src/store/gc-preflight.ts +206 -0
- package/src/store/gc.ts +424 -0
- package/src/store/metrics-preflight.ts +229 -0
- package/src/store/metrics.ts +236 -0
- package/src/store/preflight.ts +748 -0
- package/src/store/store.ts +693 -0
- package/src/system-tools.ts +355 -0
- package/src/transport-registry.ts +1124 -0
- package/src/transports/claude-code-cli.ts +1348 -0
- package/src/transports/opencode-admission.ts +515 -0
- package/src/transports/opencode-client.ts +3328 -0
- package/src/transports/opencode-evidence.ts +557 -0
- package/src/transports/opencode-mcp.ts +248 -0
- package/src/transports/opencode-sdk.ts +2294 -0
- package/src/transports/opencode-server.ts +535 -0
- package/src/triage/adjudication.ts +113 -0
- package/src/triage/reply.ts +125 -0
- package/src/triage/triage.ts +255 -0
- package/src/ui/activity.ts +173 -0
- package/src/ui/config-edit.ts +816 -0
- package/src/ui/config.ts +462 -0
- package/src/ui/menu.ts +1325 -0
- package/src/ui/plan.ts +1039 -0
- package/src/ui/primitives.ts +300 -0
- package/src/ui/progress.ts +298 -0
- package/src/ui/result.ts +552 -0
- package/src/ui/review-menu.ts +312 -0
- package/src/ui/select.ts +528 -0
- package/src/ui/tree.ts +178 -0
- package/src/uninstaller.ts +479 -0
- package/src/updater.ts +836 -0
- package/src/usage.ts +77 -0
- package/src/watch/preflight.ts +1389 -0
- package/src/watch/watch.ts +1235 -0
- package/src/wizard.ts +877 -0
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import type { NormalizedUsage } from "../execution/usage-normalized";
|
|
2
|
+
import { sumNormalizedUsage } from "../execution/usage-normalized";
|
|
3
|
+
import type { FindingObservation } from "./clustering";
|
|
4
|
+
|
|
5
|
+
export type DiversityAttemptStatus =
|
|
6
|
+
| "completed"
|
|
7
|
+
| "failed"
|
|
8
|
+
| "retry"
|
|
9
|
+
| "duplicate"
|
|
10
|
+
| "refuted";
|
|
11
|
+
|
|
12
|
+
export interface DiversityAttemptRecord {
|
|
13
|
+
readonly attemptId: string;
|
|
14
|
+
readonly legId: string;
|
|
15
|
+
readonly armId: string;
|
|
16
|
+
readonly specialty: string;
|
|
17
|
+
readonly replicate: number;
|
|
18
|
+
readonly attempt: number;
|
|
19
|
+
readonly status: DiversityAttemptStatus;
|
|
20
|
+
readonly usage: NormalizedUsage;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export interface DiversityObservationRecord {
|
|
24
|
+
readonly observation: FindingObservation;
|
|
25
|
+
readonly attemptId: string;
|
|
26
|
+
readonly legId: string;
|
|
27
|
+
readonly armId: string;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export interface DiversityLedger {
|
|
31
|
+
readonly attempts: readonly DiversityAttemptRecord[];
|
|
32
|
+
readonly observations: readonly DiversityObservationRecord[];
|
|
33
|
+
readonly failures: readonly DiversityAttemptRecord[];
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export interface DiversityAccountingTotals {
|
|
37
|
+
readonly cashCostUsd: number;
|
|
38
|
+
readonly notionalCostUsd: number;
|
|
39
|
+
readonly attemptCount: number;
|
|
40
|
+
readonly observationCount: number;
|
|
41
|
+
readonly failureCount: number;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export function appendAttempt(
|
|
45
|
+
ledger: DiversityLedger,
|
|
46
|
+
attempt: DiversityAttemptRecord,
|
|
47
|
+
): DiversityLedger {
|
|
48
|
+
const attempts = [...ledger.attempts, attempt];
|
|
49
|
+
const failures =
|
|
50
|
+
attempt.status === "failed"
|
|
51
|
+
? [...ledger.failures, attempt]
|
|
52
|
+
: ledger.failures;
|
|
53
|
+
return { ...ledger, attempts, failures };
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export function appendObservation(
|
|
57
|
+
ledger: DiversityLedger,
|
|
58
|
+
record: DiversityObservationRecord,
|
|
59
|
+
): DiversityLedger {
|
|
60
|
+
return {
|
|
61
|
+
...ledger,
|
|
62
|
+
observations: [...ledger.observations, record],
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export function emptyDiversityLedger(): DiversityLedger {
|
|
67
|
+
return { attempts: [], observations: [], failures: [] };
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export function summarizeDiversityAccounting(
|
|
71
|
+
ledger: DiversityLedger,
|
|
72
|
+
): DiversityAccountingTotals {
|
|
73
|
+
const usage = ledger.attempts.map((attempt) => attempt.usage);
|
|
74
|
+
const summed =
|
|
75
|
+
usage.length === 0
|
|
76
|
+
? undefined
|
|
77
|
+
: usage.reduce((acc, current) => sumNormalizedUsage(acc, current));
|
|
78
|
+
return {
|
|
79
|
+
cashCostUsd: summed?.cashCostUsd ?? 0,
|
|
80
|
+
notionalCostUsd: summed?.notionalCostUsd ?? 0,
|
|
81
|
+
attemptCount: ledger.attempts.length,
|
|
82
|
+
observationCount: ledger.observations.length,
|
|
83
|
+
failureCount: ledger.failures.length,
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
export function retainPartialFailureEvidence(ledger: DiversityLedger): boolean {
|
|
88
|
+
if (ledger.failures.length === 0) return true;
|
|
89
|
+
return ledger.observations.length > 0 || ledger.failures.length > 0;
|
|
90
|
+
}
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import type {
|
|
3
|
+
CausalDisposition,
|
|
4
|
+
EvidenceClass,
|
|
5
|
+
HopTrail,
|
|
6
|
+
Severity,
|
|
7
|
+
} from "#review/findings";
|
|
8
|
+
import type { AdjudicationGroup } from "./clustering";
|
|
9
|
+
|
|
10
|
+
export interface CodeEvidenceReport {
|
|
11
|
+
readonly inspectedLocations: readonly {
|
|
12
|
+
path: string;
|
|
13
|
+
line?: number;
|
|
14
|
+
symbol?: string;
|
|
15
|
+
}[];
|
|
16
|
+
readonly reachableBehavior: readonly string[];
|
|
17
|
+
readonly proofRefs: readonly string[];
|
|
18
|
+
readonly limitations: readonly string[];
|
|
19
|
+
readonly sha256: string;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export interface ClusterAdjudication {
|
|
23
|
+
readonly evidenceReportSha256: string;
|
|
24
|
+
readonly relation:
|
|
25
|
+
| "same_defect"
|
|
26
|
+
| "distinct_defects"
|
|
27
|
+
| "no_defect"
|
|
28
|
+
| "inconclusive";
|
|
29
|
+
readonly hypotheses: readonly {
|
|
30
|
+
id: string;
|
|
31
|
+
outcome: "supported" | "refuted" | "latent" | "inconclusive";
|
|
32
|
+
proofRefs: readonly string[];
|
|
33
|
+
}[];
|
|
34
|
+
readonly canonicalFindings: readonly {
|
|
35
|
+
path: string;
|
|
36
|
+
line?: number;
|
|
37
|
+
symbol?: string;
|
|
38
|
+
severity: Severity;
|
|
39
|
+
category: number;
|
|
40
|
+
evidenceClass: EvidenceClass;
|
|
41
|
+
causalDisposition: CausalDisposition;
|
|
42
|
+
claim: string;
|
|
43
|
+
proofRefs: readonly string[];
|
|
44
|
+
hopsUsed: number;
|
|
45
|
+
hopTrail: HopTrail;
|
|
46
|
+
}[];
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export function hashCodeEvidenceReport(
|
|
50
|
+
report: Omit<CodeEvidenceReport, "sha256">,
|
|
51
|
+
): string {
|
|
52
|
+
return createHash("sha256").update(JSON.stringify(report)).digest("hex");
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export function bindCodeEvidenceReport(
|
|
56
|
+
report: Omit<CodeEvidenceReport, "sha256">,
|
|
57
|
+
): CodeEvidenceReport {
|
|
58
|
+
const sha256 = hashCodeEvidenceReport(report);
|
|
59
|
+
return { ...report, sha256 };
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export function validateHashBinding(
|
|
63
|
+
adjudication: ClusterAdjudication,
|
|
64
|
+
report: CodeEvidenceReport,
|
|
65
|
+
): boolean {
|
|
66
|
+
return adjudication.evidenceReportSha256 === report.sha256;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export function synthesizeDeterministicAdjudication(
|
|
70
|
+
group: AdjudicationGroup,
|
|
71
|
+
): ClusterAdjudication | undefined {
|
|
72
|
+
if (group.ambiguous) return undefined;
|
|
73
|
+
const observations = group.clusters.flatMap(
|
|
74
|
+
(cluster) => cluster.observations,
|
|
75
|
+
);
|
|
76
|
+
if (observations.length === 0) return undefined;
|
|
77
|
+
const inspectedLocations = observations.map((observation) => ({
|
|
78
|
+
path: observation.path,
|
|
79
|
+
line: observation.line,
|
|
80
|
+
symbol: observation.symbol,
|
|
81
|
+
}));
|
|
82
|
+
const proofRefs = [
|
|
83
|
+
...new Set(observations.flatMap((observation) => observation.proofRefs)),
|
|
84
|
+
];
|
|
85
|
+
if (proofRefs.length === 0) return undefined;
|
|
86
|
+
const report = bindCodeEvidenceReport({
|
|
87
|
+
inspectedLocations,
|
|
88
|
+
reachableBehavior: observations.map((observation) => observation.claim),
|
|
89
|
+
proofRefs,
|
|
90
|
+
limitations: [],
|
|
91
|
+
});
|
|
92
|
+
const canonicalFindings = observations.map((observation) => ({
|
|
93
|
+
path: observation.path,
|
|
94
|
+
line: observation.line,
|
|
95
|
+
symbol: observation.symbol,
|
|
96
|
+
severity: observation.severity,
|
|
97
|
+
category: observation.category,
|
|
98
|
+
evidenceClass: "deterministic" as const,
|
|
99
|
+
causalDisposition: "introduced" as const,
|
|
100
|
+
claim: observation.claim,
|
|
101
|
+
proofRefs: [...observation.proofRefs],
|
|
102
|
+
hopsUsed: 0,
|
|
103
|
+
hopTrail: [] as HopTrail,
|
|
104
|
+
}));
|
|
105
|
+
const adjudication: ClusterAdjudication = {
|
|
106
|
+
evidenceReportSha256: report.sha256,
|
|
107
|
+
relation: "same_defect",
|
|
108
|
+
hypotheses: observations.map((observation, index) => ({
|
|
109
|
+
id: anonymousHypothesisId(index),
|
|
110
|
+
outcome: "supported" as const,
|
|
111
|
+
proofRefs: [...observation.proofRefs],
|
|
112
|
+
})),
|
|
113
|
+
canonicalFindings,
|
|
114
|
+
};
|
|
115
|
+
return adjudicateGroupConservatively(group, report, adjudication);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
export function adjudicateGroupConservatively(
|
|
119
|
+
group: AdjudicationGroup,
|
|
120
|
+
report?: CodeEvidenceReport,
|
|
121
|
+
adjudication?: ClusterAdjudication,
|
|
122
|
+
): ClusterAdjudication {
|
|
123
|
+
if (!report || !adjudication) {
|
|
124
|
+
return {
|
|
125
|
+
evidenceReportSha256: report?.sha256 ?? "",
|
|
126
|
+
relation: "inconclusive",
|
|
127
|
+
hypotheses: group.clusters.flatMap((cluster, index) =>
|
|
128
|
+
cluster.observations.map((_, hypothesisIndex) => ({
|
|
129
|
+
id: `H${index + 1}-${hypothesisIndex + 1}`,
|
|
130
|
+
outcome: "inconclusive" as const,
|
|
131
|
+
proofRefs: [],
|
|
132
|
+
})),
|
|
133
|
+
),
|
|
134
|
+
canonicalFindings: [],
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
if (!validateHashBinding(adjudication, report)) {
|
|
138
|
+
return adjudicateGroupConservatively(group);
|
|
139
|
+
}
|
|
140
|
+
return adjudication;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
export function anonymousHypothesisId(index: number): string {
|
|
144
|
+
return `H${index + 1}`;
|
|
145
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import {
|
|
2
|
+
SCHEMA_VERSION_V1_1,
|
|
3
|
+
validateFindingsDocument,
|
|
4
|
+
} from "#review/findings";
|
|
5
|
+
import { FINDINGS_CONFORMANCE_CASES } from "#review/findings-conformance";
|
|
6
|
+
import { DiversityCapabilityError } from "./errors";
|
|
7
|
+
|
|
8
|
+
export interface InternalCapabilityReport {
|
|
9
|
+
readonly ok: boolean;
|
|
10
|
+
readonly c2SchemaVersion: string;
|
|
11
|
+
readonly reason?: string;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export function checkInternalFindingsCapability(): InternalCapabilityReport {
|
|
15
|
+
try {
|
|
16
|
+
for (const conformanceCase of FINDINGS_CONFORMANCE_CASES) {
|
|
17
|
+
const parsed = JSON.parse(conformanceCase.raw);
|
|
18
|
+
const shouldAccept = conformanceCase.expect === "accept";
|
|
19
|
+
try {
|
|
20
|
+
validateFindingsDocument(parsed);
|
|
21
|
+
if (!shouldAccept) {
|
|
22
|
+
return {
|
|
23
|
+
ok: false,
|
|
24
|
+
c2SchemaVersion: SCHEMA_VERSION_V1_1,
|
|
25
|
+
reason: `conformance case ${conformanceCase.id} expected reject`,
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
} catch (error) {
|
|
29
|
+
if (shouldAccept) {
|
|
30
|
+
return {
|
|
31
|
+
ok: false,
|
|
32
|
+
c2SchemaVersion: SCHEMA_VERSION_V1_1,
|
|
33
|
+
reason: `conformance case ${conformanceCase.id} expected accept: ${(error as Error).message}`,
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
return { ok: true, c2SchemaVersion: SCHEMA_VERSION_V1_1 };
|
|
39
|
+
} catch (error) {
|
|
40
|
+
return {
|
|
41
|
+
ok: false,
|
|
42
|
+
c2SchemaVersion: SCHEMA_VERSION_V1_1,
|
|
43
|
+
reason: (error as Error).message,
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export function requireInternalFindingsCapability(): void {
|
|
49
|
+
assertDiversityCapabilityOrThrow();
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export function assertDiversityCapabilityOrThrow(
|
|
53
|
+
report: InternalCapabilityReport = checkInternalFindingsCapability(),
|
|
54
|
+
): void {
|
|
55
|
+
if (!report.ok) {
|
|
56
|
+
throw new DiversityCapabilityError(
|
|
57
|
+
report.reason ?? "internal findings v1.1 capability unavailable",
|
|
58
|
+
);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
import type { BenchmarkTarget } from "./identity";
|
|
2
|
+
|
|
3
|
+
export interface DiversityBenchmarkArm {
|
|
4
|
+
readonly armId: string;
|
|
5
|
+
readonly enabled: boolean;
|
|
6
|
+
readonly replicates: number;
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
export interface DiversityBenchmarkPlan {
|
|
10
|
+
readonly planId: string;
|
|
11
|
+
readonly target: BenchmarkTarget;
|
|
12
|
+
readonly controlArm: DiversityBenchmarkArm;
|
|
13
|
+
readonly treatmentArm: DiversityBenchmarkArm;
|
|
14
|
+
readonly buildFingerprint: string;
|
|
15
|
+
readonly promptFingerprint: string;
|
|
16
|
+
readonly interleaved: true;
|
|
17
|
+
readonly maxCashUsd: number;
|
|
18
|
+
readonly stopOnInvalidRun: true;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export class DiversityBenchmarkError extends Error {}
|
|
22
|
+
|
|
23
|
+
function must(condition: boolean, message: string): void {
|
|
24
|
+
if (!condition) throw new DiversityBenchmarkError(message);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function validateBenchmarkPlan(
|
|
28
|
+
candidate: unknown,
|
|
29
|
+
): DiversityBenchmarkPlan {
|
|
30
|
+
must(
|
|
31
|
+
typeof candidate === "object" && candidate !== null,
|
|
32
|
+
"benchmark plan must be an object",
|
|
33
|
+
);
|
|
34
|
+
const plan = candidate as Record<string, unknown>;
|
|
35
|
+
must(
|
|
36
|
+
typeof plan.planId === "string" && plan.planId.length > 0,
|
|
37
|
+
"planId required",
|
|
38
|
+
);
|
|
39
|
+
must(
|
|
40
|
+
typeof plan.target === "object" && plan.target !== null,
|
|
41
|
+
"target required",
|
|
42
|
+
);
|
|
43
|
+
const target = plan.target as Record<string, unknown>;
|
|
44
|
+
must(typeof target.repoId === "string", "target.repoId required");
|
|
45
|
+
must(typeof target.pr === "number" && target.pr > 0, "target.pr required");
|
|
46
|
+
must(
|
|
47
|
+
typeof target.baseSha === "string" && target.baseSha.length === 40,
|
|
48
|
+
"target.baseSha must be 40 hex chars",
|
|
49
|
+
);
|
|
50
|
+
must(
|
|
51
|
+
typeof target.headSha === "string" && target.headSha.length === 40,
|
|
52
|
+
"target.headSha must be 40 hex chars",
|
|
53
|
+
);
|
|
54
|
+
const control = plan.controlArm as Record<string, unknown>;
|
|
55
|
+
const treatment = plan.treatmentArm as Record<string, unknown>;
|
|
56
|
+
must(control?.enabled === false, "control arm must be disabled diversity");
|
|
57
|
+
must(treatment?.enabled === true, "treatment arm must enable diversity");
|
|
58
|
+
must(
|
|
59
|
+
typeof treatment.replicates === "number" && treatment.replicates >= 3,
|
|
60
|
+
"treatment replicates must be >= 3",
|
|
61
|
+
);
|
|
62
|
+
must(plan.interleaved === true, "plan must be interleaved");
|
|
63
|
+
must(
|
|
64
|
+
typeof plan.buildFingerprint === "string" &&
|
|
65
|
+
plan.buildFingerprint.length > 0,
|
|
66
|
+
"buildFingerprint required",
|
|
67
|
+
);
|
|
68
|
+
must(
|
|
69
|
+
typeof plan.promptFingerprint === "string" &&
|
|
70
|
+
plan.promptFingerprint.length > 0,
|
|
71
|
+
"promptFingerprint required",
|
|
72
|
+
);
|
|
73
|
+
must(plan.stopOnInvalidRun === true, "stopOnInvalidRun must be true");
|
|
74
|
+
must(
|
|
75
|
+
typeof plan.maxCashUsd === "number" && plan.maxCashUsd > 0,
|
|
76
|
+
"maxCashUsd required",
|
|
77
|
+
);
|
|
78
|
+
return plan as unknown as DiversityBenchmarkPlan;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export interface BenchmarkRunEvidence {
|
|
82
|
+
readonly runId: string;
|
|
83
|
+
readonly armId: string;
|
|
84
|
+
readonly replicate: number;
|
|
85
|
+
readonly observedBuildFingerprint: string;
|
|
86
|
+
readonly observedPromptFingerprint: string;
|
|
87
|
+
readonly controlCompleted: boolean;
|
|
88
|
+
readonly treatmentCompleted: boolean;
|
|
89
|
+
readonly blindingIntact: boolean;
|
|
90
|
+
readonly interleavingIntact: boolean;
|
|
91
|
+
readonly cashCostUsd: number;
|
|
92
|
+
readonly notionalCostUsd: number;
|
|
93
|
+
readonly uniqueTruePositives: number;
|
|
94
|
+
readonly recall: number;
|
|
95
|
+
readonly cleanPrRestraint: number;
|
|
96
|
+
readonly blindSpots: number;
|
|
97
|
+
readonly capExceeded?: boolean;
|
|
98
|
+
readonly replicateOmitted?: boolean;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
export interface ScoredBenchmarkRun extends BenchmarkRunEvidence {
|
|
102
|
+
readonly valid: boolean;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
export function deriveRunValidity(
|
|
106
|
+
run: BenchmarkRunEvidence,
|
|
107
|
+
plan: DiversityBenchmarkPlan,
|
|
108
|
+
): boolean {
|
|
109
|
+
if (run.capExceeded === true) return false;
|
|
110
|
+
if (run.replicateOmitted === true) return false;
|
|
111
|
+
if (!run.blindingIntact || !run.interleavingIntact) return false;
|
|
112
|
+
if (!run.controlCompleted || !run.treatmentCompleted) return false;
|
|
113
|
+
if (run.observedBuildFingerprint !== plan.buildFingerprint) return false;
|
|
114
|
+
if (run.observedPromptFingerprint !== plan.promptFingerprint) return false;
|
|
115
|
+
if (run.cashCostUsd > plan.maxCashUsd) return false;
|
|
116
|
+
if (run.replicate < 1 || run.replicate > plan.treatmentArm.replicates) {
|
|
117
|
+
return false;
|
|
118
|
+
}
|
|
119
|
+
return true;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
export interface BenchmarkScoreReport {
|
|
123
|
+
readonly cashCostPerUniqueTp: number | null;
|
|
124
|
+
readonly notionalCostPerUniqueTp: number | null;
|
|
125
|
+
readonly recall: number;
|
|
126
|
+
readonly cleanPrRestraint: number;
|
|
127
|
+
readonly blindSpots: number;
|
|
128
|
+
readonly validRuns: number;
|
|
129
|
+
readonly invalidRuns: number;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
export function scoreBenchmarkRuns(
|
|
133
|
+
runs: readonly BenchmarkRunEvidence[],
|
|
134
|
+
plan?: DiversityBenchmarkPlan,
|
|
135
|
+
): BenchmarkScoreReport {
|
|
136
|
+
const scored: ScoredBenchmarkRun[] = plan
|
|
137
|
+
? runs.map((run) => ({ ...run, valid: deriveRunValidity(run, plan) }))
|
|
138
|
+
: runs.map((run) => ({
|
|
139
|
+
...run,
|
|
140
|
+
valid: (run as ScoredBenchmarkRun).valid ?? false,
|
|
141
|
+
}));
|
|
142
|
+
const validRuns = scored.filter((run) => run.valid);
|
|
143
|
+
const invalidRuns = scored.length - validRuns.length;
|
|
144
|
+
const uniqueTp = validRuns.reduce(
|
|
145
|
+
(sum, run) => sum + run.uniqueTruePositives,
|
|
146
|
+
0,
|
|
147
|
+
);
|
|
148
|
+
const cash = validRuns.reduce((sum, run) => sum + run.cashCostUsd, 0);
|
|
149
|
+
const notional = validRuns.reduce((sum, run) => sum + run.notionalCostUsd, 0);
|
|
150
|
+
const recall =
|
|
151
|
+
validRuns.length === 0
|
|
152
|
+
? 0
|
|
153
|
+
: validRuns.reduce((sum, run) => sum + run.recall, 0) / validRuns.length;
|
|
154
|
+
const cleanPrRestraint =
|
|
155
|
+
validRuns.length === 0
|
|
156
|
+
? 0
|
|
157
|
+
: validRuns.reduce((sum, run) => sum + run.cleanPrRestraint, 0) /
|
|
158
|
+
validRuns.length;
|
|
159
|
+
const blindSpots = validRuns.reduce((sum, run) => sum + run.blindSpots, 0);
|
|
160
|
+
return {
|
|
161
|
+
cashCostPerUniqueTp: uniqueTp > 0 ? cash / uniqueTp : null,
|
|
162
|
+
notionalCostPerUniqueTp: uniqueTp > 0 ? notional / uniqueTp : null,
|
|
163
|
+
recall,
|
|
164
|
+
cleanPrRestraint,
|
|
165
|
+
blindSpots,
|
|
166
|
+
validRuns: validRuns.length,
|
|
167
|
+
invalidRuns,
|
|
168
|
+
};
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
export function requiresExplicitLiveAuthorization(): boolean {
|
|
172
|
+
return true;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
export interface PromotionDecision {
|
|
176
|
+
readonly approved: boolean;
|
|
177
|
+
readonly scope: string;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
export function recordPromotion(
|
|
181
|
+
decision: PromotionDecision,
|
|
182
|
+
): PromotionDecision {
|
|
183
|
+
if (!decision.approved) {
|
|
184
|
+
throw new DiversityBenchmarkError(
|
|
185
|
+
"promotion requires explicit human approval",
|
|
186
|
+
);
|
|
187
|
+
}
|
|
188
|
+
return decision;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
export function refuseLiveRunWithoutAuthorization(): never {
|
|
192
|
+
throw new DiversityBenchmarkError(
|
|
193
|
+
"live benchmark run requires separate authorization",
|
|
194
|
+
);
|
|
195
|
+
}
|