audit-tools 0.32.41 → 0.32.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/audit/cli/dispatch/packetPrompt.d.ts.map +1 -1
- package/dist/audit/cli/dispatch/packetPrompt.js +13 -1
- package/dist/audit/cli/dispatch/packetPrompt.js.map +1 -1
- package/dist/audit/cli/rollingAuditDispatch.d.ts.map +1 -1
- package/dist/audit/cli/rollingAuditDispatch.js +21 -1
- package/dist/audit/cli/rollingAuditDispatch.js.map +1 -1
- package/dist/audit/cli/scoreTokensCommand.d.ts +18 -0
- package/dist/audit/cli/scoreTokensCommand.d.ts.map +1 -0
- package/dist/audit/cli/scoreTokensCommand.js +130 -0
- package/dist/audit/cli/scoreTokensCommand.js.map +1 -0
- package/dist/audit/cli.d.ts.map +1 -1
- package/dist/audit/cli.js +5 -1
- package/dist/audit/cli.js.map +1 -1
- package/dist/audit/extractors/graphPathUtils.d.ts +2 -1
- package/dist/audit/extractors/graphPathUtils.d.ts.map +1 -1
- package/dist/audit/extractors/graphPathUtils.js +5 -5
- package/dist/audit/extractors/graphPathUtils.js.map +1 -1
- package/dist/audit/io/tokenUsageLedger.d.ts +57 -0
- package/dist/audit/io/tokenUsageLedger.d.ts.map +1 -0
- package/dist/audit/io/tokenUsageLedger.js +52 -0
- package/dist/audit/io/tokenUsageLedger.js.map +1 -0
- package/dist/audit/orchestrator/continuityScore.d.ts +1 -2
- package/dist/audit/orchestrator/continuityScore.d.ts.map +1 -1
- package/dist/audit/orchestrator/continuityScore.js +8 -130
- package/dist/audit/orchestrator/continuityScore.js.map +1 -1
- package/dist/audit/orchestrator/fileAnchors.d.ts +11 -0
- package/dist/audit/orchestrator/fileAnchors.d.ts.map +1 -1
- package/dist/audit/orchestrator/fileAnchors.js +42 -0
- package/dist/audit/orchestrator/fileAnchors.js.map +1 -1
- package/dist/audit/orchestrator/reviewPacketGraphEdges.d.ts +3 -3
- package/dist/audit/orchestrator/reviewPacketGraphEdges.d.ts.map +1 -1
- package/dist/audit/orchestrator/reviewPacketGraphEdges.js +5 -38
- package/dist/audit/orchestrator/reviewPacketGraphEdges.js.map +1 -1
- package/dist/audit/orchestrator/reviewPackets.d.ts.map +1 -1
- package/dist/audit/orchestrator/reviewPackets.js +6 -7
- package/dist/audit/orchestrator/reviewPackets.js.map +1 -1
- package/dist/audit/reporting/scoreTokens.d.ts +112 -0
- package/dist/audit/reporting/scoreTokens.d.ts.map +1 -0
- package/dist/audit/reporting/scoreTokens.js +195 -0
- package/dist/audit/reporting/scoreTokens.js.map +1 -0
- package/dist/audit/types/externalAnalyzer.d.ts +10 -10
- package/dist/remediate/contractPipeline/counterexampleFingerprint.d.ts +27 -0
- package/dist/remediate/contractPipeline/counterexampleFingerprint.d.ts.map +1 -0
- package/dist/remediate/contractPipeline/counterexampleFingerprint.js +37 -0
- package/dist/remediate/contractPipeline/counterexampleFingerprint.js.map +1 -0
- package/dist/remediate/index.d.ts +34 -0
- package/dist/remediate/index.d.ts.map +1 -1
- package/dist/remediate/index.js +92 -26
- package/dist/remediate/index.js.map +1 -1
- package/dist/remediate/state/accessMemory.d.ts +26 -0
- package/dist/remediate/state/accessMemory.d.ts.map +1 -1
- package/dist/remediate/state/accessMemory.js +52 -1
- package/dist/remediate/state/accessMemory.js.map +1 -1
- package/dist/remediate/steps/contractPipeline.d.ts.map +1 -1
- package/dist/remediate/steps/contractPipeline.js +35 -4
- package/dist/remediate/steps/contractPipeline.js.map +1 -1
- package/dist/remediate/steps/contractPipelinePrompts.d.ts.map +1 -1
- package/dist/remediate/steps/contractPipelinePrompts.js +1 -1
- package/dist/remediate/steps/contractPipelinePrompts.js.map +1 -1
- package/dist/remediate/steps/dispatch/acceptNode.d.ts +18 -0
- package/dist/remediate/steps/dispatch/acceptNode.d.ts.map +1 -1
- package/dist/remediate/steps/dispatch/acceptNode.js +26 -0
- package/dist/remediate/steps/dispatch/acceptNode.js.map +1 -1
- package/dist/remediate/steps/nextStep.d.ts +9 -0
- package/dist/remediate/steps/nextStep.d.ts.map +1 -1
- package/dist/remediate/steps/nextStep.js +19 -7
- package/dist/remediate/steps/nextStep.js.map +1 -1
- package/dist/remediate/steps/rollingSession.d.ts.map +1 -1
- package/dist/remediate/steps/rollingSession.js +21 -4
- package/dist/remediate/steps/rollingSession.js.map +1 -1
- package/dist/remediate/validation/artifacts.d.ts.map +1 -1
- package/dist/remediate/validation/artifacts.js +15 -26
- package/dist/remediate/validation/artifacts.js.map +1 -1
- package/dist/remediate/validation/contractPipeline.d.ts +1 -1
- package/dist/remediate/validation/contractPipeline.d.ts.map +1 -1
- package/dist/remediate/validation/contractPipeline.js +1 -1
- package/dist/remediate/validation/contractPipeline.js.map +1 -1
- package/dist/remediate/validation/contractPipelineGates.d.ts +52 -0
- package/dist/remediate/validation/contractPipelineGates.d.ts.map +1 -1
- package/dist/remediate/validation/contractPipelineGates.js +61 -0
- package/dist/remediate/validation/contractPipelineGates.js.map +1 -1
- package/dist/shared/continuityScore.d.ts +13 -0
- package/dist/shared/continuityScore.d.ts.map +1 -0
- package/dist/shared/continuityScore.js +156 -0
- package/dist/shared/continuityScore.js.map +1 -0
- package/dist/shared/dispatch/ownershipScheduler.d.ts +21 -0
- package/dist/shared/dispatch/ownershipScheduler.d.ts.map +1 -1
- package/dist/shared/dispatch/ownershipScheduler.js +10 -1
- package/dist/shared/dispatch/ownershipScheduler.js.map +1 -1
- package/dist/shared/dispatch/rollingDispatch.d.ts +15 -0
- package/dist/shared/dispatch/rollingDispatch.d.ts.map +1 -1
- package/dist/shared/dispatch/rollingDispatch.js.map +1 -1
- package/dist/shared/graph/graphPaths.d.ts +21 -0
- package/dist/shared/graph/graphPaths.d.ts.map +1 -0
- package/dist/shared/graph/graphPaths.js +61 -0
- package/dist/shared/graph/graphPaths.js.map +1 -0
- package/dist/shared/index.d.ts +2 -0
- package/dist/shared/index.d.ts.map +1 -1
- package/dist/shared/index.js +2 -0
- package/dist/shared/index.js.map +1 -1
- package/dist/shared/providers/openAiCompatibleProvider.d.ts +29 -0
- package/dist/shared/providers/openAiCompatibleProvider.d.ts.map +1 -1
- package/dist/shared/providers/openAiCompatibleProvider.js +60 -0
- package/dist/shared/providers/openAiCompatibleProvider.js.map +1 -1
- package/dist/shared/providers/types.d.ts +19 -0
- package/dist/shared/providers/types.d.ts.map +1 -1
- package/package.json +1 -1
- package/wrapper/audit-code-wrapper-lib.mjs +15 -39
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* score-tokens — the COST counterpart to the A-2 quality oracle (`score-audit`).
|
|
3
|
+
*
|
|
4
|
+
* `scoreTokens` is a PURE function of a run's recorded token-usage ledger
|
|
5
|
+
* (`token-usage.jsonl`, appended post-hoc at packet-completion time by
|
|
6
|
+
* `appendTokenUsageLine` — see `src/audit/io/tokenUsageLedger.ts`). It reports
|
|
7
|
+
* real per-step input/output tokens and prefix cache-hit ratio, exactly what the
|
|
8
|
+
* provider reported on a COMPLETED response — never a metered/token-counting API
|
|
9
|
+
* call and never `estimateTokensFromBytes` (that estimator is for PLANNING only).
|
|
10
|
+
*
|
|
11
|
+
* No-silent-scoring (mirrors score-audit's `unmatched[]` discipline): a step
|
|
12
|
+
* whose usage is entirely unreported (the agentic-CLI providers — claude-code /
|
|
13
|
+
* codex / opencode — spawn an external process with no structured usage) is
|
|
14
|
+
* counted in `totals.unmeasured_steps` and excluded from the token/cache-hit
|
|
15
|
+
* aggregates — NEVER folded in as a silent 0.
|
|
16
|
+
*
|
|
17
|
+
* Pure module: no IO, no clock (the ledger's wall-clock timestamp is read but
|
|
18
|
+
* never used to influence the verdict), no model identity — the same (entries,
|
|
19
|
+
* prefixHashes) always yields a byte-identical scorecard, order-independent.
|
|
20
|
+
*/
|
|
21
|
+
/** One packet's recorded usage, read from a `token-usage.jsonl` line. */
|
|
22
|
+
export interface TokenUsageEntry {
|
|
23
|
+
packet_id: string;
|
|
24
|
+
pool_id?: string | null;
|
|
25
|
+
lens?: string | null;
|
|
26
|
+
input_tokens?: number | null;
|
|
27
|
+
output_tokens?: number | null;
|
|
28
|
+
cache_read_tokens?: number | null;
|
|
29
|
+
cache_creation_tokens?: number | null;
|
|
30
|
+
}
|
|
31
|
+
/** Per-packet prefix-cache-eligibility verdict (structural, provider-independent). */
|
|
32
|
+
export interface PrefixStability {
|
|
33
|
+
/** True iff every packet in the corpus hashed to the SAME fixed-prefix. */
|
|
34
|
+
stable: boolean;
|
|
35
|
+
/** Packet ids whose fixed-prefix hash diverged from the majority, sorted. */
|
|
36
|
+
diverging_packet_ids: string[];
|
|
37
|
+
}
|
|
38
|
+
/** The deterministic scorecard `scoreTokens` emits. */
|
|
39
|
+
export interface TokenScorecard {
|
|
40
|
+
schema_version: "score-tokens-scorecard/v1";
|
|
41
|
+
run_id: string;
|
|
42
|
+
steps: Array<{
|
|
43
|
+
packet_id: string;
|
|
44
|
+
pool_id: string | null;
|
|
45
|
+
lens: string | null;
|
|
46
|
+
input_tokens: number | null;
|
|
47
|
+
output_tokens: number | null;
|
|
48
|
+
cache_read_tokens: number | null;
|
|
49
|
+
/** cacheRead / (cacheRead + input) when BOTH legs are measured, else null. */
|
|
50
|
+
cache_hit_ratio: number | null;
|
|
51
|
+
}>;
|
|
52
|
+
totals: {
|
|
53
|
+
/** Sum of `input_tokens` over MEASURED steps only. */
|
|
54
|
+
input_tokens: number;
|
|
55
|
+
/** Sum of `output_tokens` over MEASURED steps only. */
|
|
56
|
+
output_tokens: number;
|
|
57
|
+
/** Steps with at least one reported usage field. */
|
|
58
|
+
measured_steps: number;
|
|
59
|
+
/** Steps whose usage was entirely unreported (never folded into totals as 0). */
|
|
60
|
+
unmeasured_steps: number;
|
|
61
|
+
};
|
|
62
|
+
/** Aggregate cache_hit_ratio over measured steps only; null when none qualify. */
|
|
63
|
+
cache_hit_ratio_overall: number | null;
|
|
64
|
+
/** Structural (provider-independent) prompt-prefix cache-eligibility signal. */
|
|
65
|
+
prefix_stability: PrefixStability;
|
|
66
|
+
/** Per-pool (== per-provider identity, `pool_id` is `provider#account/model`) coverage. */
|
|
67
|
+
provider_coverage: Record<string, "measured" | "unmeasured">;
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Score a run's recorded token-usage ledger — the pure score-tokens oracle.
|
|
71
|
+
*
|
|
72
|
+
* `prefixHashes` is an OPTIONAL caller-supplied map of `packet_id ->
|
|
73
|
+
* fixed-prefix hash` (see {@link packetPromptPrefixHash}), computed by the
|
|
74
|
+
* caller from the run's recorded packet prompts. Omit it when wiring real
|
|
75
|
+
* prompt hashes end-to-end is not available — `prefix_stability` then defaults
|
|
76
|
+
* to `{ stable: true, diverging_packet_ids: [] }` (no signal, not a false
|
|
77
|
+
* failure).
|
|
78
|
+
*/
|
|
79
|
+
export declare function scoreTokens(runId: string, entries: TokenUsageEntry[], prefixHashes?: Record<string, string>): TokenScorecard;
|
|
80
|
+
/**
|
|
81
|
+
* The structural, provider-independent prefix-stability boundary: hash of the
|
|
82
|
+
* FIXED prefix of a packet's rendered prompt — everything BEFORE the volatile
|
|
83
|
+
* `## Packet` section (the same boundary
|
|
84
|
+
* `tests/audit/dispatch-helpers.test.mjs` pins for prompt-caching:
|
|
85
|
+
* `prompt.slice(0, prompt.indexOf("## Packet"))`). Two packets in the same run
|
|
86
|
+
* whose fixed prefix hashes to the SAME value are byte-identical up to that
|
|
87
|
+
* point — the structural precondition for a provider's prefix cache to hit,
|
|
88
|
+
* independent of whether the provider actually reports a cache-hit ratio.
|
|
89
|
+
*
|
|
90
|
+
* A prompt with no `## Packet` marker hashes its entirety (defensive — never
|
|
91
|
+
* throws on an unexpected prompt shape).
|
|
92
|
+
*/
|
|
93
|
+
export declare function packetPromptPrefixHash(prompt: string): string;
|
|
94
|
+
/**
|
|
95
|
+
* The SOLE gate predicate: did the overall cache-hit ratio REGRESS (drop)
|
|
96
|
+
* against a baseline? Mirrors score-audit's `hallucinationRegressed` mechanism
|
|
97
|
+
* exactly, direction-flipped (higher cache-hit ratio is BETTER, so a regression
|
|
98
|
+
* is a DECREASE, not an increase).
|
|
99
|
+
*
|
|
100
|
+
* - With no baseline, nothing has regressed.
|
|
101
|
+
* - A `null` current ratio (no measured cache-eligible steps this run) cannot
|
|
102
|
+
* regress.
|
|
103
|
+
* - A `null` baseline ratio is treated as 0 (the floor a ratio can't drop
|
|
104
|
+
* below), so a null baseline never triggers a regression.
|
|
105
|
+
*
|
|
106
|
+
* Token totals are tracked (`totals`) but never gate — track-don't-gate, same
|
|
107
|
+
* policy as score-audit's precision/recall.
|
|
108
|
+
*/
|
|
109
|
+
export declare function cacheHitRatioRegressed(current: TokenScorecard, baseline: TokenScorecard | null | undefined, epsilon?: number): boolean;
|
|
110
|
+
/** A compact, deterministic human summary of a {@link TokenScorecard}. */
|
|
111
|
+
export declare function renderTokenScorecardMarkdown(scorecard: TokenScorecard): string;
|
|
112
|
+
//# sourceMappingURL=scoreTokens.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scoreTokens.d.ts","sourceRoot":"","sources":["../../../src/audit/reporting/scoreTokens.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAIH,yEAAyE;AACzE,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,qBAAqB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACvC;AAED,sFAAsF;AACtF,MAAM,WAAW,eAAe;IAC9B,2EAA2E;IAC3E,MAAM,EAAE,OAAO,CAAC;IAChB,6EAA6E;IAC7E,oBAAoB,EAAE,MAAM,EAAE,CAAC;CAChC;AAED,uDAAuD;AACvD,MAAM,WAAW,cAAc;IAC7B,cAAc,EAAE,2BAA2B,CAAC;IAC5C,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,KAAK,CAAC;QACX,SAAS,EAAE,MAAM,CAAC;QAClB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;QACvB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;QACpB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QAC5B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;QAC7B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;QACjC,8EAA8E;QAC9E,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;KAChC,CAAC,CAAC;IACH,MAAM,EAAE;QACN,sDAAsD;QACtD,YAAY,EAAE,MAAM,CAAC;QACrB,uDAAuD;QACvD,aAAa,EAAE,MAAM,CAAC;QACtB,oDAAoD;QACpD,cAAc,EAAE,MAAM,CAAC;QACvB,iFAAiF;QACjF,gBAAgB,EAAE,MAAM,CAAC;KAC1B,CAAC;IACF,kFAAkF;IAClF,uBAAuB,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC,gFAAgF;IAChF,gBAAgB,EAAE,eAAe,CAAC;IAClC,2FAA2F;IAC3F,iBAAiB,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,GAAG,YAAY,CAAC,CAAC;CAC9D;AAgBD;;;;;;;;;GASG;AACH,wBAAgB,WAAW,CACzB,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,eAAe,EAAE,EAC1B,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GACpC,cAAc,CA2EhB;AAuBD;;;;;;;;;;;;GAYG;AACH,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAI7D;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,cAAc,EACvB,QAAQ,EAAE,cAAc,GAAG,IAAI,GAAG,SAAS,EAC3C,OAAO,SAAO,GACb,OAAO,CAMT;AAED,0EAA0E;AAC1E,wBAAgB,4BAA4B,CAAC,SAAS,EAAE,cAAc,GAAG,MAAM,CAwB9E"}
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* score-tokens — the COST counterpart to the A-2 quality oracle (`score-audit`).
|
|
3
|
+
*
|
|
4
|
+
* `scoreTokens` is a PURE function of a run's recorded token-usage ledger
|
|
5
|
+
* (`token-usage.jsonl`, appended post-hoc at packet-completion time by
|
|
6
|
+
* `appendTokenUsageLine` — see `src/audit/io/tokenUsageLedger.ts`). It reports
|
|
7
|
+
* real per-step input/output tokens and prefix cache-hit ratio, exactly what the
|
|
8
|
+
* provider reported on a COMPLETED response — never a metered/token-counting API
|
|
9
|
+
* call and never `estimateTokensFromBytes` (that estimator is for PLANNING only).
|
|
10
|
+
*
|
|
11
|
+
* No-silent-scoring (mirrors score-audit's `unmatched[]` discipline): a step
|
|
12
|
+
* whose usage is entirely unreported (the agentic-CLI providers — claude-code /
|
|
13
|
+
* codex / opencode — spawn an external process with no structured usage) is
|
|
14
|
+
* counted in `totals.unmeasured_steps` and excluded from the token/cache-hit
|
|
15
|
+
* aggregates — NEVER folded in as a silent 0.
|
|
16
|
+
*
|
|
17
|
+
* Pure module: no IO, no clock (the ledger's wall-clock timestamp is read but
|
|
18
|
+
* never used to influence the verdict), no model identity — the same (entries,
|
|
19
|
+
* prefixHashes) always yields a byte-identical scorecard, order-independent.
|
|
20
|
+
*/
|
|
21
|
+
import { createHash } from "node:crypto";
|
|
22
|
+
function ratio(numerator, denominator) {
|
|
23
|
+
return denominator === 0 ? null : numerator / denominator;
|
|
24
|
+
}
|
|
25
|
+
/** True iff at least one of the four usage legs was reported (not null/undefined). */
|
|
26
|
+
function isMeasured(entry) {
|
|
27
|
+
return (entry.input_tokens != null ||
|
|
28
|
+
entry.output_tokens != null ||
|
|
29
|
+
entry.cache_read_tokens != null ||
|
|
30
|
+
entry.cache_creation_tokens != null);
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Score a run's recorded token-usage ledger — the pure score-tokens oracle.
|
|
34
|
+
*
|
|
35
|
+
* `prefixHashes` is an OPTIONAL caller-supplied map of `packet_id ->
|
|
36
|
+
* fixed-prefix hash` (see {@link packetPromptPrefixHash}), computed by the
|
|
37
|
+
* caller from the run's recorded packet prompts. Omit it when wiring real
|
|
38
|
+
* prompt hashes end-to-end is not available — `prefix_stability` then defaults
|
|
39
|
+
* to `{ stable: true, diverging_packet_ids: [] }` (no signal, not a false
|
|
40
|
+
* failure).
|
|
41
|
+
*/
|
|
42
|
+
export function scoreTokens(runId, entries, prefixHashes) {
|
|
43
|
+
// Deterministic (sorted by packet_id) iteration so the scorecard is
|
|
44
|
+
// byte-identical regardless of input order.
|
|
45
|
+
const sorted = [...entries].sort((a, b) => a.packet_id.localeCompare(b.packet_id));
|
|
46
|
+
const steps = [];
|
|
47
|
+
let totalInput = 0;
|
|
48
|
+
let totalOutput = 0;
|
|
49
|
+
let measuredSteps = 0;
|
|
50
|
+
let unmeasuredSteps = 0;
|
|
51
|
+
let cacheHitReadSum = 0;
|
|
52
|
+
let cacheHitDenomSum = 0;
|
|
53
|
+
let cacheHitQualifyingSteps = 0;
|
|
54
|
+
const providerCoverage = new Map();
|
|
55
|
+
for (const entry of sorted) {
|
|
56
|
+
const poolKey = entry.pool_id ?? "unknown";
|
|
57
|
+
const measured = isMeasured(entry);
|
|
58
|
+
if (measured) {
|
|
59
|
+
measuredSteps += 1;
|
|
60
|
+
totalInput += entry.input_tokens ?? 0;
|
|
61
|
+
totalOutput += entry.output_tokens ?? 0;
|
|
62
|
+
// A pool is "measured" the moment ANY of its steps reported usage.
|
|
63
|
+
providerCoverage.set(poolKey, "measured");
|
|
64
|
+
}
|
|
65
|
+
else {
|
|
66
|
+
unmeasuredSteps += 1;
|
|
67
|
+
// Never downgrade a pool already proven "measured" by an earlier step.
|
|
68
|
+
if (providerCoverage.get(poolKey) !== "measured") {
|
|
69
|
+
providerCoverage.set(poolKey, "unmeasured");
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
// cache_hit_ratio requires BOTH legs measured (a lone cache_read with no
|
|
73
|
+
// input denominator, or vice versa, is not a ratio — surfaced as null).
|
|
74
|
+
const cacheRead = entry.cache_read_tokens;
|
|
75
|
+
const input = entry.input_tokens;
|
|
76
|
+
let cacheHitRatio = null;
|
|
77
|
+
if (cacheRead != null && input != null) {
|
|
78
|
+
cacheHitRatio = ratio(cacheRead, cacheRead + input);
|
|
79
|
+
if (cacheHitRatio !== null) {
|
|
80
|
+
cacheHitReadSum += cacheRead;
|
|
81
|
+
cacheHitDenomSum += cacheRead + input;
|
|
82
|
+
cacheHitQualifyingSteps += 1;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
steps.push({
|
|
86
|
+
packet_id: entry.packet_id,
|
|
87
|
+
pool_id: entry.pool_id ?? null,
|
|
88
|
+
lens: entry.lens ?? null,
|
|
89
|
+
input_tokens: entry.input_tokens ?? null,
|
|
90
|
+
output_tokens: entry.output_tokens ?? null,
|
|
91
|
+
cache_read_tokens: entry.cache_read_tokens ?? null,
|
|
92
|
+
cache_hit_ratio: cacheHitRatio,
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
return {
|
|
96
|
+
schema_version: "score-tokens-scorecard/v1",
|
|
97
|
+
run_id: runId,
|
|
98
|
+
steps,
|
|
99
|
+
totals: {
|
|
100
|
+
input_tokens: totalInput,
|
|
101
|
+
output_tokens: totalOutput,
|
|
102
|
+
measured_steps: measuredSteps,
|
|
103
|
+
unmeasured_steps: unmeasuredSteps,
|
|
104
|
+
},
|
|
105
|
+
cache_hit_ratio_overall: cacheHitQualifyingSteps > 0 ? ratio(cacheHitReadSum, cacheHitDenomSum) : null,
|
|
106
|
+
prefix_stability: computePrefixStability(prefixHashes),
|
|
107
|
+
provider_coverage: Object.fromEntries([...providerCoverage.entries()].sort(([a], [b]) => a.localeCompare(b))),
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Derive `prefix_stability` from an optional `packet_id -> fixed-prefix hash`
|
|
112
|
+
* map. Absent/empty input defaults to "stable" (no structural signal available,
|
|
113
|
+
* not a false failure). Otherwise every packet must share the FIRST (by sorted
|
|
114
|
+
* packet_id) hash; any divergence is surfaced, never silently averaged away.
|
|
115
|
+
*/
|
|
116
|
+
function computePrefixStability(prefixHashes) {
|
|
117
|
+
const entries = prefixHashes ? Object.entries(prefixHashes) : [];
|
|
118
|
+
if (entries.length === 0)
|
|
119
|
+
return { stable: true, diverging_packet_ids: [] };
|
|
120
|
+
const sortedEntries = [...entries].sort(([a], [b]) => a.localeCompare(b));
|
|
121
|
+
const referenceHash = sortedEntries[0][1];
|
|
122
|
+
const diverging = sortedEntries
|
|
123
|
+
.filter(([, hash]) => hash !== referenceHash)
|
|
124
|
+
.map(([packetId]) => packetId)
|
|
125
|
+
.sort();
|
|
126
|
+
return { stable: diverging.length === 0, diverging_packet_ids: diverging };
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* The structural, provider-independent prefix-stability boundary: hash of the
|
|
130
|
+
* FIXED prefix of a packet's rendered prompt — everything BEFORE the volatile
|
|
131
|
+
* `## Packet` section (the same boundary
|
|
132
|
+
* `tests/audit/dispatch-helpers.test.mjs` pins for prompt-caching:
|
|
133
|
+
* `prompt.slice(0, prompt.indexOf("## Packet"))`). Two packets in the same run
|
|
134
|
+
* whose fixed prefix hashes to the SAME value are byte-identical up to that
|
|
135
|
+
* point — the structural precondition for a provider's prefix cache to hit,
|
|
136
|
+
* independent of whether the provider actually reports a cache-hit ratio.
|
|
137
|
+
*
|
|
138
|
+
* A prompt with no `## Packet` marker hashes its entirety (defensive — never
|
|
139
|
+
* throws on an unexpected prompt shape).
|
|
140
|
+
*/
|
|
141
|
+
export function packetPromptPrefixHash(prompt) {
|
|
142
|
+
const markerIdx = prompt.indexOf("## Packet");
|
|
143
|
+
const prefix = markerIdx >= 0 ? prompt.slice(0, markerIdx) : prompt;
|
|
144
|
+
return createHash("sha256").update(prefix, "utf8").digest("hex");
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* The SOLE gate predicate: did the overall cache-hit ratio REGRESS (drop)
|
|
148
|
+
* against a baseline? Mirrors score-audit's `hallucinationRegressed` mechanism
|
|
149
|
+
* exactly, direction-flipped (higher cache-hit ratio is BETTER, so a regression
|
|
150
|
+
* is a DECREASE, not an increase).
|
|
151
|
+
*
|
|
152
|
+
* - With no baseline, nothing has regressed.
|
|
153
|
+
* - A `null` current ratio (no measured cache-eligible steps this run) cannot
|
|
154
|
+
* regress.
|
|
155
|
+
* - A `null` baseline ratio is treated as 0 (the floor a ratio can't drop
|
|
156
|
+
* below), so a null baseline never triggers a regression.
|
|
157
|
+
*
|
|
158
|
+
* Token totals are tracked (`totals`) but never gate — track-don't-gate, same
|
|
159
|
+
* policy as score-audit's precision/recall.
|
|
160
|
+
*/
|
|
161
|
+
export function cacheHitRatioRegressed(current, baseline, epsilon = 1e-9) {
|
|
162
|
+
if (!baseline)
|
|
163
|
+
return false;
|
|
164
|
+
const currentRatio = current.cache_hit_ratio_overall;
|
|
165
|
+
if (currentRatio === null)
|
|
166
|
+
return false;
|
|
167
|
+
const baselineRatio = baseline.cache_hit_ratio_overall ?? 0;
|
|
168
|
+
return currentRatio < baselineRatio - epsilon;
|
|
169
|
+
}
|
|
170
|
+
/** A compact, deterministic human summary of a {@link TokenScorecard}. */
|
|
171
|
+
export function renderTokenScorecardMarkdown(scorecard) {
|
|
172
|
+
const pct = (value) => value === null ? "n/a" : `${(value * 100).toFixed(1)}%`;
|
|
173
|
+
const t = scorecard.totals;
|
|
174
|
+
const lines = [
|
|
175
|
+
`# Token scorecard — ${scorecard.run_id}`,
|
|
176
|
+
"",
|
|
177
|
+
`- Steps: ${t.measured_steps + t.unmeasured_steps} (measured ${t.measured_steps}, unmeasured ${t.unmeasured_steps})`,
|
|
178
|
+
`- Input tokens (measured only): ${t.input_tokens}`,
|
|
179
|
+
`- Output tokens (measured only): ${t.output_tokens}`,
|
|
180
|
+
`- Cache-hit ratio (gated): ${pct(scorecard.cache_hit_ratio_overall)}`,
|
|
181
|
+
`- Prefix stability: ${scorecard.prefix_stability.stable ? "stable" : "DIVERGING"}` +
|
|
182
|
+
(scorecard.prefix_stability.diverging_packet_ids.length > 0
|
|
183
|
+
? ` (${scorecard.prefix_stability.diverging_packet_ids.join(", ")})`
|
|
184
|
+
: ""),
|
|
185
|
+
];
|
|
186
|
+
const coverageEntries = Object.entries(scorecard.provider_coverage);
|
|
187
|
+
if (coverageEntries.length > 0) {
|
|
188
|
+
lines.push("", "## Provider coverage");
|
|
189
|
+
for (const [poolId, status] of coverageEntries) {
|
|
190
|
+
lines.push(`- \`${poolId}\`: ${status}`);
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
return lines.join("\n") + "\n";
|
|
194
|
+
}
|
|
195
|
+
//# sourceMappingURL=scoreTokens.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scoreTokens.js","sourceRoot":"","sources":["../../../src/audit/reporting/scoreTokens.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAqDzC,SAAS,KAAK,CAAC,SAAiB,EAAE,WAAmB;IACnD,OAAO,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,GAAG,WAAW,CAAC;AAC5D,CAAC;AAED,sFAAsF;AACtF,SAAS,UAAU,CAAC,KAAsB;IACxC,OAAO,CACL,KAAK,CAAC,YAAY,IAAI,IAAI;QAC1B,KAAK,CAAC,aAAa,IAAI,IAAI;QAC3B,KAAK,CAAC,iBAAiB,IAAI,IAAI;QAC/B,KAAK,CAAC,qBAAqB,IAAI,IAAI,CACpC,CAAC;AACJ,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,WAAW,CACzB,KAAa,EACb,OAA0B,EAC1B,YAAqC;IAErC,oEAAoE;IACpE,4CAA4C;IAC5C,MAAM,MAAM,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;IAEnF,MAAM,KAAK,GAA4B,EAAE,CAAC;IAC1C,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,IAAI,uBAAuB,GAAG,CAAC,CAAC;IAChC,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAqC,CAAC;IAEtE,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,IAAI,SAAS,CAAC;QAC3C,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QAEnC,IAAI,QAAQ,EAAE,CAAC;YACb,aAAa,IAAI,CAAC,CAAC;YACnB,UAAU,IAAI,KAAK,CAAC,YAAY,IAAI,CAAC,CAAC;YACtC,WAAW,IAAI,KAAK,CAAC,aAAa,IAAI,CAAC,CAAC;YACxC,mEAAmE;YACnE,gBAAgB,CAAC,GAAG,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;QAC5C,CAAC;aAAM,CAAC;YACN,eAAe,IAAI,CAAC,CAAC;YACrB,uEAAuE;YACvE,IAAI,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,UAAU,EAAE,CAAC;gBACjD,gBAAgB,CAAC,GAAG,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;YAC9C,CAAC;QACH,CAAC;QAED,yEAAyE;QACzE,wEAAwE;QACxE,MAAM,SAAS,GAAG,KAAK,CAAC,iBAAiB,CAAC;QAC1C,MAAM,KAAK,GAAG,KAAK,CAAC,YAAY,CAAC;QACjC,IAAI,aAAa,GAAkB,IAAI,CAAC;QACxC,IAAI,SAAS,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;YACvC,aAAa,GAAG,KAAK,CAAC,SAAS,EAAE,SAAS,GAAG,KAAK,CAAC,CAAC;YACpD,IAAI,aAAa,KAAK,IAAI,EAAE,CAAC;gBAC3B,eAAe,IAAI,SAAS,CAAC;gBAC7B,gBAAgB,IAAI,SAAS,GAAG,KAAK,CAAC;gBACtC,uBAAuB,IAAI,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;QAED,KAAK,CAAC,IAAI,CAAC;YACT,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,OAAO,EAAE,KAAK,CAAC,OAAO,IAAI,IAAI;YAC9B,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,IAAI;YACxB,YAAY,EAAE,KAAK,CAAC,YAAY,IAAI,IAAI;YACxC,aAAa,EAAE,KAAK,CAAC,aAAa,IAAI,IAAI;YAC1C,iBAAiB,EAAE,KAAK,CAAC,iBAAiB,IAAI,IAAI;YAClD,eAAe,EAAE,aAAa;SAC/B,CAAC,CAAC;IACL,CAAC;IAED,OAAO;QACL,cAAc,EAAE,2BAA2B;QAC3C,MAAM,EAAE,KAAK;QACb,KAAK;QACL,MAAM,EAAE;YACN,YAAY,EAAE,UAAU;YACxB,aAAa,EAAE,WAAW;YAC1B,cAAc,EAAE,aAAa;YAC7B,gBAAgB,EAAE,eAAe;SAClC;QACD,uBAAuB,EACrB,uBAAuB,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,eAAe,EAAE,gBAAgB,CAAC,CAAC,CAAC,CAAC,IAAI;QAC/E,gBAAgB,EAAE,sBAAsB,CAAC,YAAY,CAAC;QACtD,iBAAiB,EAAE,MAAM,CAAC,WAAW,CACnC,CAAC,GAAG,gBAAgB,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CACvE;KACF,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,SAAS,sBAAsB,CAC7B,YAAgD;IAEhD,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACjE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,oBAAoB,EAAE,EAAE,EAAE,CAAC;IAE5E,MAAM,aAAa,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1E,MAAM,aAAa,GAAG,aAAa,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAC;IAC3C,MAAM,SAAS,GAAG,aAAa;SAC5B,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,KAAK,aAAa,CAAC;SAC5C,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC;SAC7B,IAAI,EAAE,CAAC;IACV,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,oBAAoB,EAAE,SAAS,EAAE,CAAC;AAC7E,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,sBAAsB,CAAC,MAAc;IACnD,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IAC9C,MAAM,MAAM,GAAG,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IACpE,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACnE,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,sBAAsB,CACpC,OAAuB,EACvB,QAA2C,EAC3C,OAAO,GAAG,IAAI;IAEd,IAAI,CAAC,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC5B,MAAM,YAAY,GAAG,OAAO,CAAC,uBAAuB,CAAC;IACrD,IAAI,YAAY,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IACxC,MAAM,aAAa,GAAG,QAAQ,CAAC,uBAAuB,IAAI,CAAC,CAAC;IAC5D,OAAO,YAAY,GAAG,aAAa,GAAG,OAAO,CAAC;AAChD,CAAC;AAED,0EAA0E;AAC1E,MAAM,UAAU,4BAA4B,CAAC,SAAyB;IACpE,MAAM,GAAG,GAAG,CAAC,KAAoB,EAAU,EAAE,CAC3C,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;IAC1D,MAAM,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC;IAC3B,MAAM,KAAK,GAAG;QACZ,uBAAuB,SAAS,CAAC,MAAM,EAAE;QACzC,EAAE;QACF,YAAY,CAAC,CAAC,cAAc,GAAG,CAAC,CAAC,gBAAgB,cAAc,CAAC,CAAC,cAAc,gBAAgB,CAAC,CAAC,gBAAgB,GAAG;QACpH,mCAAmC,CAAC,CAAC,YAAY,EAAE;QACnD,oCAAoC,CAAC,CAAC,aAAa,EAAE;QACrD,8BAA8B,GAAG,CAAC,SAAS,CAAC,uBAAuB,CAAC,EAAE;QACtE,uBAAuB,SAAS,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,EAAE;YACjF,CAAC,SAAS,CAAC,gBAAgB,CAAC,oBAAoB,CAAC,MAAM,GAAG,CAAC;gBACzD,CAAC,CAAC,KAAK,SAAS,CAAC,gBAAgB,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;gBACpE,CAAC,CAAC,EAAE,CAAC;KACV,CAAC;IACF,MAAM,eAAe,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;IACpE,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/B,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,sBAAsB,CAAC,CAAC;QACvC,KAAK,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,eAAe,EAAE,CAAC;YAC/C,KAAK,CAAC,IAAI,CAAC,OAAO,MAAM,OAAO,MAAM,EAAE,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;AACjC,CAAC"}
|
|
@@ -94,16 +94,16 @@ export declare const ExternalAnalyzerToolStatusSchema: z.ZodObject<{
|
|
|
94
94
|
status: "failed" | "findings" | "spawn_error" | "success" | "skipped" | "not_resolved" | "parse_error";
|
|
95
95
|
resolved: boolean;
|
|
96
96
|
tool: string;
|
|
97
|
-
command?: string | undefined;
|
|
98
97
|
error?: string | undefined;
|
|
98
|
+
command?: string | undefined;
|
|
99
99
|
exit_code?: number | null | undefined;
|
|
100
100
|
output_snippet?: string | undefined;
|
|
101
101
|
}, {
|
|
102
102
|
status: "failed" | "findings" | "spawn_error" | "success" | "skipped" | "not_resolved" | "parse_error";
|
|
103
103
|
resolved: boolean;
|
|
104
104
|
tool: string;
|
|
105
|
-
command?: string | undefined;
|
|
106
105
|
error?: string | undefined;
|
|
106
|
+
command?: string | undefined;
|
|
107
107
|
exit_code?: number | null | undefined;
|
|
108
108
|
output_snippet?: string | undefined;
|
|
109
109
|
}>;
|
|
@@ -166,16 +166,16 @@ export declare const ExternalAnalyzerResultsSchema: z.ZodObject<{
|
|
|
166
166
|
status: "failed" | "findings" | "spawn_error" | "success" | "skipped" | "not_resolved" | "parse_error";
|
|
167
167
|
resolved: boolean;
|
|
168
168
|
tool: string;
|
|
169
|
-
command?: string | undefined;
|
|
170
169
|
error?: string | undefined;
|
|
170
|
+
command?: string | undefined;
|
|
171
171
|
exit_code?: number | null | undefined;
|
|
172
172
|
output_snippet?: string | undefined;
|
|
173
173
|
}, {
|
|
174
174
|
status: "failed" | "findings" | "spawn_error" | "success" | "skipped" | "not_resolved" | "parse_error";
|
|
175
175
|
resolved: boolean;
|
|
176
176
|
tool: string;
|
|
177
|
-
command?: string | undefined;
|
|
178
177
|
error?: string | undefined;
|
|
178
|
+
command?: string | undefined;
|
|
179
179
|
exit_code?: number | null | undefined;
|
|
180
180
|
output_snippet?: string | undefined;
|
|
181
181
|
}>, "many">>;
|
|
@@ -243,8 +243,8 @@ export declare const ExternalAnalyzerResultsSchema: z.ZodObject<{
|
|
|
243
243
|
status: "failed" | "findings" | "spawn_error" | "success" | "skipped" | "not_resolved" | "parse_error";
|
|
244
244
|
resolved: boolean;
|
|
245
245
|
tool: string;
|
|
246
|
-
command?: string | undefined;
|
|
247
246
|
error?: string | undefined;
|
|
247
|
+
command?: string | undefined;
|
|
248
248
|
exit_code?: number | null | undefined;
|
|
249
249
|
output_snippet?: string | undefined;
|
|
250
250
|
}[] | undefined;
|
|
@@ -280,8 +280,8 @@ export declare const ExternalAnalyzerResultsSchema: z.ZodObject<{
|
|
|
280
280
|
status: "failed" | "findings" | "spawn_error" | "success" | "skipped" | "not_resolved" | "parse_error";
|
|
281
281
|
resolved: boolean;
|
|
282
282
|
tool: string;
|
|
283
|
-
command?: string | undefined;
|
|
284
283
|
error?: string | undefined;
|
|
284
|
+
command?: string | undefined;
|
|
285
285
|
exit_code?: number | null | undefined;
|
|
286
286
|
output_snippet?: string | undefined;
|
|
287
287
|
}[] | undefined;
|
|
@@ -314,16 +314,16 @@ export declare const ExternalAnalyzerAcquisitionMarkerSchema: z.ZodObject<{
|
|
|
314
314
|
status: "failed" | "findings" | "spawn_error" | "success" | "skipped" | "not_resolved" | "parse_error";
|
|
315
315
|
resolved: boolean;
|
|
316
316
|
tool: string;
|
|
317
|
-
command?: string | undefined;
|
|
318
317
|
error?: string | undefined;
|
|
318
|
+
command?: string | undefined;
|
|
319
319
|
exit_code?: number | null | undefined;
|
|
320
320
|
output_snippet?: string | undefined;
|
|
321
321
|
}, {
|
|
322
322
|
status: "failed" | "findings" | "spawn_error" | "success" | "skipped" | "not_resolved" | "parse_error";
|
|
323
323
|
resolved: boolean;
|
|
324
324
|
tool: string;
|
|
325
|
-
command?: string | undefined;
|
|
326
325
|
error?: string | undefined;
|
|
326
|
+
command?: string | undefined;
|
|
327
327
|
exit_code?: number | null | undefined;
|
|
328
328
|
output_snippet?: string | undefined;
|
|
329
329
|
}>, "many">;
|
|
@@ -333,8 +333,8 @@ export declare const ExternalAnalyzerAcquisitionMarkerSchema: z.ZodObject<{
|
|
|
333
333
|
status: "failed" | "findings" | "spawn_error" | "success" | "skipped" | "not_resolved" | "parse_error";
|
|
334
334
|
resolved: boolean;
|
|
335
335
|
tool: string;
|
|
336
|
-
command?: string | undefined;
|
|
337
336
|
error?: string | undefined;
|
|
337
|
+
command?: string | undefined;
|
|
338
338
|
exit_code?: number | null | undefined;
|
|
339
339
|
output_snippet?: string | undefined;
|
|
340
340
|
}[];
|
|
@@ -345,8 +345,8 @@ export declare const ExternalAnalyzerAcquisitionMarkerSchema: z.ZodObject<{
|
|
|
345
345
|
status: "failed" | "findings" | "spawn_error" | "success" | "skipped" | "not_resolved" | "parse_error";
|
|
346
346
|
resolved: boolean;
|
|
347
347
|
tool: string;
|
|
348
|
-
command?: string | undefined;
|
|
349
348
|
error?: string | undefined;
|
|
349
|
+
command?: string | undefined;
|
|
350
350
|
exit_code?: number | null | undefined;
|
|
351
351
|
output_snippet?: string | undefined;
|
|
352
352
|
}[];
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Content fingerprint for a counterexample.
|
|
3
|
+
*
|
|
4
|
+
* The judge↔repair convergence gate ({@link evaluateJudgeGate} in
|
|
5
|
+
* `../steps/contractPipeline.js`) must detect genuine re-occurrence of the
|
|
6
|
+
* same unrepaired defect across independent adversarial rounds. The
|
|
7
|
+
* reviewer-supplied counterexample `id` is free text (see
|
|
8
|
+
* `Counterexample.id` in `audit-tools/shared`'s obligations types) and is NOT
|
|
9
|
+
* a stable cross-round identity: two independent critic rounds each commonly
|
|
10
|
+
* label their genuinely-distinct top counterexample "CE-001" (the prompt
|
|
11
|
+
* schema's own example value). Keying convergence on the raw id string then
|
|
12
|
+
* misreads "two different CEs that happen to share a label" as "the same CE
|
|
13
|
+
* re-accepted after a repair" and falsely escalates a stall while a real new
|
|
14
|
+
* defect is being correctly repaired.
|
|
15
|
+
*
|
|
16
|
+
* This fingerprint keys on CONTENT instead: the tool-constrained
|
|
17
|
+
* `violated_obligation_ids` vocabulary (sorted, normalized) paired with the
|
|
18
|
+
* normalized `claim` prose, falling back to the normalized claim alone when
|
|
19
|
+
* no obligations are cited. Mirrors the derivation style of
|
|
20
|
+
* `src/shared/findingIdentitySignature.ts` (tiered stable fields, aggressive
|
|
21
|
+
* title normalization, truncated content hash) rather than introducing a
|
|
22
|
+
* second identity convention.
|
|
23
|
+
*/
|
|
24
|
+
import { type Counterexample } from "audit-tools/shared";
|
|
25
|
+
/** Deterministic, content-derived identity for a counterexample. */
|
|
26
|
+
export declare function counterexampleFingerprint(ce: Counterexample): string;
|
|
27
|
+
//# sourceMappingURL=counterexampleFingerprint.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"counterexampleFingerprint.d.ts","sourceRoot":"","sources":["../../../src/remediate/contractPipeline/counterexampleFingerprint.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,OAAO,EAA+B,KAAK,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAEtF,oEAAoE;AACpE,wBAAgB,yBAAyB,CAAC,EAAE,EAAE,cAAc,GAAG,MAAM,CAWpE"}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Content fingerprint for a counterexample.
|
|
3
|
+
*
|
|
4
|
+
* The judge↔repair convergence gate ({@link evaluateJudgeGate} in
|
|
5
|
+
* `../steps/contractPipeline.js`) must detect genuine re-occurrence of the
|
|
6
|
+
* same unrepaired defect across independent adversarial rounds. The
|
|
7
|
+
* reviewer-supplied counterexample `id` is free text (see
|
|
8
|
+
* `Counterexample.id` in `audit-tools/shared`'s obligations types) and is NOT
|
|
9
|
+
* a stable cross-round identity: two independent critic rounds each commonly
|
|
10
|
+
* label their genuinely-distinct top counterexample "CE-001" (the prompt
|
|
11
|
+
* schema's own example value). Keying convergence on the raw id string then
|
|
12
|
+
* misreads "two different CEs that happen to share a label" as "the same CE
|
|
13
|
+
* re-accepted after a repair" and falsely escalates a stall while a real new
|
|
14
|
+
* defect is being correctly repaired.
|
|
15
|
+
*
|
|
16
|
+
* This fingerprint keys on CONTENT instead: the tool-constrained
|
|
17
|
+
* `violated_obligation_ids` vocabulary (sorted, normalized) paired with the
|
|
18
|
+
* normalized `claim` prose, falling back to the normalized claim alone when
|
|
19
|
+
* no obligations are cited. Mirrors the derivation style of
|
|
20
|
+
* `src/shared/findingIdentitySignature.ts` (tiered stable fields, aggressive
|
|
21
|
+
* title normalization, truncated content hash) rather than introducing a
|
|
22
|
+
* second identity convention.
|
|
23
|
+
*/
|
|
24
|
+
import { hashContent, normalizeTitle } from "audit-tools/shared";
|
|
25
|
+
/** Deterministic, content-derived identity for a counterexample. */
|
|
26
|
+
export function counterexampleFingerprint(ce) {
|
|
27
|
+
const obligations = (ce.violated_obligation_ids ?? [])
|
|
28
|
+
.map((id) => id.trim().toLowerCase())
|
|
29
|
+
.filter((id) => id.length > 0)
|
|
30
|
+
.sort();
|
|
31
|
+
const claim = normalizeTitle(ce.claim);
|
|
32
|
+
const key = obligations.length > 0
|
|
33
|
+
? `obligations|${obligations.join(",")}|${claim}`
|
|
34
|
+
: `claim|${claim}`;
|
|
35
|
+
return hashContent(key, { length: 8 });
|
|
36
|
+
}
|
|
37
|
+
//# sourceMappingURL=counterexampleFingerprint.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"counterexampleFingerprint.js","sourceRoot":"","sources":["../../../src/remediate/contractPipeline/counterexampleFingerprint.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,OAAO,EAAE,WAAW,EAAE,cAAc,EAAuB,MAAM,oBAAoB,CAAC;AAEtF,oEAAoE;AACpE,MAAM,UAAU,yBAAyB,CAAC,EAAkB;IAC1D,MAAM,WAAW,GAAG,CAAC,EAAE,CAAC,uBAAuB,IAAI,EAAE,CAAC;SACnD,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;SACpC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;SAC7B,IAAI,EAAE,CAAC;IACV,MAAM,KAAK,GAAG,cAAc,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC;IACvC,MAAM,GAAG,GACP,WAAW,CAAC,MAAM,GAAG,CAAC;QACpB,CAAC,CAAC,eAAe,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,EAAE;QACjD,CAAC,CAAC,SAAS,KAAK,EAAE,CAAC;IACvB,OAAO,WAAW,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC;AACzC,CAAC"}
|
|
@@ -1,6 +1,40 @@
|
|
|
1
1
|
import { Command } from "commander";
|
|
2
2
|
import { runCommand } from "./utils/commands.js";
|
|
3
|
+
import { type ContractPipelineArtifactName } from "./contractPipeline/artifactStore.js";
|
|
4
|
+
import type { ValidationIssue } from "audit-tools/shared";
|
|
3
5
|
declare const program: Command;
|
|
6
|
+
export interface ValidateArtifactActionResult {
|
|
7
|
+
status: "ok" | "error";
|
|
8
|
+
name?: ContractPipelineArtifactName;
|
|
9
|
+
issue_count?: number;
|
|
10
|
+
issues?: ValidationIssue[];
|
|
11
|
+
message?: string;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* The `validate-artifact --name X` self-check's full logic, exported so tests
|
|
15
|
+
* can call it directly (no dist build / subprocess race). Returns the JSON body
|
|
16
|
+
* + exit code the CLI action prints/exits with, without doing either itself.
|
|
17
|
+
*
|
|
18
|
+
* Beyond the per-artifact structural validator, this ALSO loads the on-disk
|
|
19
|
+
* sibling contract-pipeline artifacts (under `<artifactsDir>/intake/contract/`)
|
|
20
|
+
* and runs the SAME cross-artifact gates the plural `validate-artifacts` sweep
|
|
21
|
+
* and `next-step` enforce (evaluateContractPipelineCrossGates — single-sourced
|
|
22
|
+
* in validation/contractPipelineGates.ts), substituting the in-flight `name`
|
|
23
|
+
* payload for its on-disk version so the in-flight edit always wins over a
|
|
24
|
+
* stale/absent sibling. Without this, a shape-valid artifact missing its
|
|
25
|
+
* cross-artifact obligations (e.g. a test_validator_plan missing its CE-006
|
|
26
|
+
* scoped negative) could self-validate "ok" here and only fail later at
|
|
27
|
+
* next-step — the exact authoring round-trip this closes.
|
|
28
|
+
*/
|
|
29
|
+
export declare function runValidateArtifactAction(options: {
|
|
30
|
+
name: string;
|
|
31
|
+
file?: string;
|
|
32
|
+
root: string;
|
|
33
|
+
artifactsDir: string;
|
|
34
|
+
}): Promise<{
|
|
35
|
+
result: ValidateArtifactActionResult;
|
|
36
|
+
exitCode: number;
|
|
37
|
+
}>;
|
|
4
38
|
export { program };
|
|
5
39
|
/**
|
|
6
40
|
* Parse argv through the program after normalizing the `=value` form of the
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/remediate/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAKpC,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/remediate/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAKpC,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAajD,OAAO,EAML,KAAK,4BAA4B,EAClC,MAAM,qCAAqC,CAAC;AAE7C,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAyD1D,QAAA,MAAM,OAAO,SAAgB,CAAC;AAoP9B,MAAM,WAAW,4BAA4B;IAC3C,MAAM,EAAE,IAAI,GAAG,OAAO,CAAC;IACvB,IAAI,CAAC,EAAE,4BAA4B,CAAC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,eAAe,EAAE,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,yBAAyB,CAAC,OAAO,EAAE;IACvD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;CACtB,GAAG,OAAO,CAAC;IAAE,MAAM,EAAE,4BAA4B,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC,CAuFtE;AAiCD,OAAO,EAAE,OAAO,EAAE,CAAC;AAEnB;;;;;;;GAOG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,IAAI,CAEjD;AAUD;;;;GAIG;AACH,wBAAgB,wBAAwB,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAU/E;AAED;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAC/B,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,MAAM,GACvB,MAAM,CAuBR;AAYD;;;;;;;;GAQG;AACH,wBAAgB,yBAAyB,CACvC,IAAI,EAAE,MAAM,EACZ,YAAY,EAAE,MAAM,GACnB,MAAM,CAIR;AAED,wBAAgB,kBAAkB,CAChC,IAAI,GAAE;IACJ,GAAG,CAAC,EAAE,OAAO,UAAU,CAAC;IACxB,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IAChC,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CAC9B,GACL,MAAM,CAcR;AA4FD,wBAAgB,kBAAkB,CAChC,KAAK,EAAE,OAAO,EACd,GAAG,GAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAkB,EACxC,OAAO,SAAY,EACnB,IAAI,SAAgB,GACnB,IAAI,CA4EN"}
|