@compaction/cli 0.1.2 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/README.md +11 -1
  2. package/dist/cli/commands/compact.js +15 -0
  3. package/dist/cli/commands/context.d.ts +2 -0
  4. package/dist/cli/commands/context.js +130 -0
  5. package/dist/cli/commands/import.js +3 -1
  6. package/dist/cli/commands/init.js +184 -157
  7. package/dist/cli/commands/run.js +13 -1
  8. package/dist/cli/index.js +4 -1
  9. package/dist/cli/onboarding/InitTui.d.ts +52 -0
  10. package/dist/cli/onboarding/InitTui.js +153 -0
  11. package/dist/cli/onboarding/model.d.ts +39 -0
  12. package/dist/cli/onboarding/model.js +77 -0
  13. package/dist/cli/onboarding/should-use-tui.d.ts +25 -0
  14. package/dist/cli/onboarding/should-use-tui.js +35 -0
  15. package/dist/cli/onboarding/wordmark.d.ts +28 -0
  16. package/dist/cli/onboarding/wordmark.js +72 -0
  17. package/dist/core/aggregate-format.d.ts +3 -1
  18. package/dist/core/aggregate-format.js +18 -0
  19. package/dist/core/context-store-eval-cases.d.ts +6 -0
  20. package/dist/core/context-store-eval-cases.js +331 -0
  21. package/dist/core/context-store-eval.d.ts +140 -0
  22. package/dist/core/context-store-eval.js +138 -0
  23. package/dist/core/context-store-fs.d.ts +73 -0
  24. package/dist/core/context-store-fs.js +157 -0
  25. package/dist/core/context-store-sufficiency.d.ts +98 -0
  26. package/dist/core/context-store-sufficiency.js +135 -0
  27. package/dist/core/context-store.d.ts +155 -0
  28. package/dist/core/context-store.js +228 -0
  29. package/dist/core/report-generator.d.ts +32 -1
  30. package/dist/core/report-generator.js +67 -0
  31. package/dist/core/run-aggregator.d.ts +64 -0
  32. package/dist/core/run-aggregator.js +132 -3
  33. package/dist/core/session-aggregate.d.ts +43 -0
  34. package/dist/core/session-aggregate.js +89 -7
  35. package/dist/core/trace-adapters.d.ts +8 -0
  36. package/dist/core/trace-adapters.js +4 -0
  37. package/dist/core/types.d.ts +55 -0
  38. package/package.json +5 -1
@@ -0,0 +1,228 @@
1
+ import { contentHash } from "./content-hash.js";
2
+ import { estimateTextTokens } from "./token-estimator.js";
3
+ import { computeTraceFingerprint } from "./trace-fingerprint.js";
4
+ /**
5
+ * V0.4 context store — rung-1 (local, deterministic, source-grounded).
6
+ *
7
+ * SCOPE (accepted design doc `docs/design/v0.4-context-store-retrieval.md`,
8
+ * 2026-06-24): a durable, local, file-based store of source-pointed context items
9
+ * accumulated across runs, plus DETERMINISTIC local retrieval and assembly under a
10
+ * token budget. This module is the rung-1 core.
11
+ *
12
+ * HARD NON-GOALS (load-bearing — do not add here):
13
+ * - NO semantic / embedding retrieval. Retrieval ranks by deterministic signals only
14
+ * (source-pointer match, lexical overlap, recency). Semantic retrieval needs a model
15
+ * → not free-CLI local-first → deferred to the gated private engine/API.
16
+ * - NO network call, credential, upload, or model invocation. This module is pure +
17
+ * local-file I/O only (see `loadContextStore` / `appendContextItems`).
18
+ * - NO hosted DB / vector store / auth.
19
+ * - NO large-memory / 300M-token / retrieval-correctness claim. Those are gated on the
20
+ * four-axis eval harness (a follow-up increment) and rung-6 evidence.
21
+ *
22
+ * HONESTY: token figures here are `local_estimate` (chars/4), never provider-reported.
23
+ * Every stored item retains its `source_pointer` + recoverability so the assembled
24
+ * context stays recoverable to its sources (the substrate invariant).
25
+ */
26
+ /** Algorithm + schema version, so a persisted store is self-describing. */
27
+ export const CONTEXT_STORE_ITEM_VERSION = "context-store-item-v1";
28
+ function normalizeContent(content) {
29
+ return content.trim().replace(/\s+/g, " ");
30
+ }
31
+ /**
32
+ * Build a single context item, deriving the dedup hash + local token estimate.
33
+ * Pure — no I/O, no clock (the caller supplies `created_at` so this stays
34
+ * deterministic and testable).
35
+ */
36
+ export function makeContextItem(input) {
37
+ const content = normalizeContent(input.content);
38
+ return {
39
+ version: CONTEXT_STORE_ITEM_VERSION,
40
+ content_sha256: contentHash(content),
41
+ content,
42
+ estimated_tokens: estimateTextTokens(content),
43
+ source_pointer: input.source_pointer,
44
+ recoverability: input.recoverability ?? "unknown",
45
+ trace_fingerprint: input.trace_fingerprint,
46
+ source_trace_id: input.source_trace_id,
47
+ provenance: input.provenance,
48
+ labels: input.labels ? [...input.labels] : [],
49
+ created_at: input.created_at
50
+ };
51
+ }
52
+ /**
53
+ * Derive context items from a `StateCapsule`. Each retained fact / open question /
54
+ * safety note becomes a source-pointed item, inheriting the capsule's source pointer +
55
+ * recoverability + the originating trace id (and the supplied per-run fingerprint).
56
+ * Pure — `created_at` is supplied by the caller.
57
+ */
58
+ export function contextItemsFromCapsule(capsule, opts) {
59
+ const primaryProvenance = capsule.provenance_entries?.[0];
60
+ const base = {
61
+ source_pointer: capsule.source_pointer ?? primaryProvenance?.source_pointer,
62
+ recoverability: capsule.source_recoverable ?? primaryProvenance?.source_recoverable ?? "unknown",
63
+ source_trace_id: capsule.source_trace_id ?? capsule.traceId,
64
+ provenance: primaryProvenance,
65
+ trace_fingerprint: opts.trace_fingerprint,
66
+ created_at: opts.created_at
67
+ };
68
+ const items = [];
69
+ for (const fact of capsule.retainedFacts) {
70
+ if (fact.trim().length === 0)
71
+ continue;
72
+ items.push(makeContextItem({ ...base, content: fact, labels: ["retained_fact"] }));
73
+ }
74
+ for (const question of capsule.openQuestions) {
75
+ if (question.trim().length === 0)
76
+ continue;
77
+ items.push(makeContextItem({ ...base, content: question, labels: ["open_question"] }));
78
+ }
79
+ for (const note of capsule.safetyNotes) {
80
+ if (note.trim().length === 0)
81
+ continue;
82
+ items.push(makeContextItem({ ...base, content: note, labels: ["safety_note"] }));
83
+ }
84
+ return dedupeContextItems(items);
85
+ }
86
+ /**
87
+ * Derive context items from a normalized `AgentTrace` (the local artifact `compaction capture`
88
+ * / `import` produces). Each non-empty message becomes a source-pointed item recoverable to its
89
+ * `trace=<id> message=<id>` origin, tagged with the run's distinctness fingerprint. This is the
90
+ * free-CLI populate path (state-capsules require the gated engine; trace messages do not). Pure —
91
+ * `created_at` is supplied by the caller.
92
+ */
93
+ export function contextItemsFromTrace(trace, opts) {
94
+ const trace_fingerprint = computeTraceFingerprint(trace).content_sha256;
95
+ const items = [];
96
+ for (const message of trace.messages) {
97
+ if (message.content.trim().length === 0)
98
+ continue;
99
+ const labels = [message.role];
100
+ if (message.toolName)
101
+ labels.push(message.toolName);
102
+ items.push(makeContextItem({
103
+ content: message.content,
104
+ source_pointer: `trace=${trace.id} message=${message.id}`,
105
+ source_trace_id: trace.id,
106
+ recoverability: true,
107
+ trace_fingerprint,
108
+ labels,
109
+ created_at: opts.created_at
110
+ }));
111
+ }
112
+ return dedupeContextItems(items);
113
+ }
114
+ /**
115
+ * Dedup by `content_sha256`, keeping the EARLIEST item deterministically (ties broken
116
+ * by `created_at`, then by the existing order). Reuses the fingerprint discipline:
117
+ * byte-identical content is one item, not many. Output order follows the
118
+ * first-occurrence order of each distinct hash in the input array.
119
+ */
120
+ export function dedupeContextItems(items) {
121
+ const byHash = new Map();
122
+ for (const item of items) {
123
+ const existing = byHash.get(item.content_sha256);
124
+ if (!existing || item.created_at < existing.created_at) {
125
+ // Merge labels so a later duplicate's labels are not lost.
126
+ if (existing) {
127
+ const labels = Array.from(new Set([...item.labels, ...existing.labels])).sort();
128
+ byHash.set(item.content_sha256, { ...item, labels });
129
+ }
130
+ else {
131
+ byHash.set(item.content_sha256, item);
132
+ }
133
+ }
134
+ else {
135
+ const labels = Array.from(new Set([...existing.labels, ...item.labels])).sort();
136
+ byHash.set(item.content_sha256, { ...existing, labels });
137
+ }
138
+ }
139
+ return Array.from(byHash.values());
140
+ }
141
+ const WORD_RE = /[a-z0-9]+/g;
142
+ function tokenSet(text) {
143
+ const set = new Set();
144
+ const matches = text.toLowerCase().match(WORD_RE);
145
+ if (matches) {
146
+ for (const word of matches)
147
+ set.add(word);
148
+ }
149
+ return set;
150
+ }
151
+ /**
152
+ * Deterministic, local retrieval. Ranks items by:
153
+ * score = sourceMatchWeight * (exact source/trace match ? 1 : 0)
154
+ * + lexicalWeight * lexical overlap coefficient in [0,1]
155
+ *
156
+ * Ties are broken deterministically by recency (newer first) then `content_sha256`
157
+ * ascending, so the same store + query always yields the same order. No model, no
158
+ * embeddings, no network — overlap is a pure word-set intersection.
159
+ */
160
+ export function retrieveContextItems(items, query, options = {}) {
161
+ const sourceWeight = options.sourceMatchWeight ?? 1;
162
+ const lexicalWeight = options.lexicalWeight ?? 1;
163
+ const queryWords = tokenSet(query.text);
164
+ const wantedPointers = new Set(query.source_pointers ?? []);
165
+ const wantedTraces = new Set(query.trace_ids ?? []);
166
+ const ranked = items.map((item) => {
167
+ const sourceMatch = (item.source_pointer !== undefined && wantedPointers.has(item.source_pointer)) ||
168
+ (item.source_trace_id !== undefined && wantedTraces.has(item.source_trace_id));
169
+ let overlap = 0;
170
+ if (queryWords.size > 0) {
171
+ const itemWords = tokenSet(item.content);
172
+ if (itemWords.size > 0) {
173
+ let intersection = 0;
174
+ const [small, large] = queryWords.size <= itemWords.size ? [queryWords, itemWords] : [itemWords, queryWords];
175
+ for (const word of small) {
176
+ if (large.has(word))
177
+ intersection += 1;
178
+ }
179
+ // Overlap coefficient: intersection / min(|q|, |item|) — in [0, 1].
180
+ overlap = intersection / Math.min(queryWords.size, itemWords.size);
181
+ }
182
+ }
183
+ const score = sourceWeight * (sourceMatch ? 1 : 0) + lexicalWeight * overlap;
184
+ return { item, score, source_match: sourceMatch, lexical_overlap: overlap };
185
+ });
186
+ ranked.sort((a, b) => {
187
+ if (b.score !== a.score)
188
+ return b.score - a.score;
189
+ if (a.item.created_at !== b.item.created_at)
190
+ return a.item.created_at < b.item.created_at ? 1 : -1;
191
+ return a.item.content_sha256 < b.item.content_sha256 ? -1 : 1;
192
+ });
193
+ const withSignal = ranked.filter((entry) => entry.score > 0);
194
+ return options.limit !== undefined ? withSignal.slice(0, options.limit) : withSignal;
195
+ }
196
+ /**
197
+ * Assemble ranked items into the sufficient active context under a token budget.
198
+ * Greedy in ranked order; dedups by `content_sha256`; PRESERVES the source pointer +
199
+ * provenance on every included item (so the assembled context stays recoverable).
200
+ * Token figures are local estimates.
201
+ */
202
+ export function assembleContext(ranked, options) {
203
+ const items = ranked.map((entry) => ("item" in entry ? entry.item : entry));
204
+ const included = [];
205
+ const dropped = [];
206
+ const seen = new Set();
207
+ let total = 0;
208
+ for (const item of items) {
209
+ if (seen.has(item.content_sha256))
210
+ continue;
211
+ seen.add(item.content_sha256);
212
+ if (total + item.estimated_tokens <= options.tokenBudget) {
213
+ included.push(item);
214
+ total += item.estimated_tokens;
215
+ }
216
+ else {
217
+ dropped.push(item);
218
+ }
219
+ }
220
+ return {
221
+ text: included.map((item) => item.content).join("\n"),
222
+ included,
223
+ dropped,
224
+ estimated_tokens: total,
225
+ within_budget: dropped.length === 0
226
+ };
227
+ }
228
+ //# sourceMappingURL=context-store.js.map
@@ -1,6 +1,37 @@
1
1
  import type { SkillInjectionAdvisory } from "./skill-injection-detector.js";
2
- import type { AgentTrace, CompactionPolicy, CompactionReport, CostEstimate, TokenEstimate, WasteFinding } from "./types.js";
2
+ import type { AgentTrace, CompactionPolicy, CompactionReport, CostEstimate, SavingsEvidence, TokenEstimate, WasteFinding } from "./types.js";
3
3
  import type { UsageMetadata } from "./usage-metadata.js";
4
+ /**
5
+ * Attach the content-addressed per-run trace fingerprint to a compaction report (PURE,
6
+ * additive, backward-compatible). This is the report-generation attachment point that
7
+ * threads trace identity into the savings-evidence pipeline so each per-run saving is
8
+ * attributable to a verifiable run identity and a re-captured run can be de-duplicated in
9
+ * the aggregate.
10
+ *
11
+ * It changes NO existing report field/label — it only sets the OPTIONAL `trace_fingerprint`
12
+ * from `computeTraceFingerprint(trace)` (a one-way SHA-256 digest + structural counts; no
13
+ * message content). Returns a new object; the input report is not mutated. This is an
14
+ * attribution/integrity signal, NOT a billing, provider, or semantic-preservation claim.
15
+ */
16
+ export declare function withTraceFingerprint(report: CompactionReport, trace: AgentTrace): CompactionReport;
17
+ /**
18
+ * Attach the composed per-run SAVINGS EVIDENCE record to a compaction report (PURE,
19
+ * additive, backward-compatible). V0.2 "evidence labels per result": it composes the
20
+ * report's EXISTING numbers into ONE explicitly-labeled record and asserts the weakest
21
+ * honest rung — a single run is rung 1 (local estimate). It makes NO new/stronger claim:
22
+ * `billing_confirmed` is always false, `semantic_preservation` always `not_evaluated`, and
23
+ * the measured rung 1.5 (`measured_caveated_estimate_delta`) is NEVER produced here (only by
24
+ * the aggregate increment across N≥3 distinct runs).
25
+ *
26
+ * `costSource` defaults to the weakest honest label (`local_estimate`); pass a stronger
27
+ * source ONLY when the run genuinely carries it. `recoverability` defaults to
28
+ * `not_evaluated` (the eval is the separate `compact --eval` path); pass the eval result
29
+ * when one ran. Returns a new object; the input report is not mutated.
30
+ */
31
+ export declare function withSavingsEvidence(report: CompactionReport, opts?: {
32
+ costSource?: SavingsEvidence["cost_source"];
33
+ recoverability?: SavingsEvidence["recoverability"];
34
+ }): CompactionReport;
4
35
  export declare function formatAnalyzeReport(trace: AgentTrace, tokens: TokenEstimate, cost: CostEstimate, findings?: WasteFinding[], usage?: UsageMetadata, skillInjectionAdvisory?: SkillInjectionAdvisory): string;
5
36
  export declare function formatCompactionReport(report: CompactionReport): string;
6
37
  export declare function formatCompactionMarkdownReport(report: CompactionReport, policy: CompactionPolicy): string;
@@ -1,5 +1,71 @@
1
1
  import { calculateCost } from "./cost-calculator.js";
2
2
  import { isKnownModel } from "./pricing.js";
3
+ import { computeTraceFingerprint } from "./trace-fingerprint.js";
4
+ /**
5
+ * Attach the content-addressed per-run trace fingerprint to a compaction report (PURE,
6
+ * additive, backward-compatible). This is the report-generation attachment point that
7
+ * threads trace identity into the savings-evidence pipeline so each per-run saving is
8
+ * attributable to a verifiable run identity and a re-captured run can be de-duplicated in
9
+ * the aggregate.
10
+ *
11
+ * It changes NO existing report field/label — it only sets the OPTIONAL `trace_fingerprint`
12
+ * from `computeTraceFingerprint(trace)` (a one-way SHA-256 digest + structural counts; no
13
+ * message content). Returns a new object; the input report is not mutated. This is an
14
+ * attribution/integrity signal, NOT a billing, provider, or semantic-preservation claim.
15
+ */
16
+ export function withTraceFingerprint(report, trace) {
17
+ return { ...report, trace_fingerprint: computeTraceFingerprint(trace) };
18
+ }
19
+ /**
20
+ * Attach the composed per-run SAVINGS EVIDENCE record to a compaction report (PURE,
21
+ * additive, backward-compatible). V0.2 "evidence labels per result": it composes the
22
+ * report's EXISTING numbers into ONE explicitly-labeled record and asserts the weakest
23
+ * honest rung — a single run is rung 1 (local estimate). It makes NO new/stronger claim:
24
+ * `billing_confirmed` is always false, `semantic_preservation` always `not_evaluated`, and
25
+ * the measured rung 1.5 (`measured_caveated_estimate_delta`) is NEVER produced here (only by
26
+ * the aggregate increment across N≥3 distinct runs).
27
+ *
28
+ * `costSource` defaults to the weakest honest label (`local_estimate`); pass a stronger
29
+ * source ONLY when the run genuinely carries it. `recoverability` defaults to
30
+ * `not_evaluated` (the eval is the separate `compact --eval` path); pass the eval result
31
+ * when one ran. Returns a new object; the input report is not mutated.
32
+ */
33
+ export function withSavingsEvidence(report, opts = {}) {
34
+ const cost_source = opts.costSource ?? "local_estimate";
35
+ const recoverability = opts.recoverability ?? "not_evaluated";
36
+ const savings_evidence = {
37
+ trace_identity: report.trace_fingerprint ? "fingerprinted" : "unidentified",
38
+ cost_source,
39
+ recoverability,
40
+ semantic_preservation: "not_evaluated",
41
+ evidence_rung: "local_estimate_single_run",
42
+ billing_confirmed: false,
43
+ label: `Per-run ${cost_source.replace(/_/g, " ")} delta (single run → local-estimate rung). ` +
44
+ `Recoverability: ${recoverability}` +
45
+ (recoverability === "not_evaluated" ? " (run `compact --eval`)" : "") +
46
+ ". Semantic preservation: not_evaluated. NOT billing-confirmed, NOT realized savings, NOT extrapolated."
47
+ };
48
+ return { ...report, savings_evidence };
49
+ }
50
+ /** Render the composed per-run savings-evidence record for the markdown report. */
51
+ function formatSavingsEvidenceLines(report) {
52
+ const se = report.savings_evidence;
53
+ if (!se) {
54
+ return [];
55
+ }
56
+ return [
57
+ "",
58
+ "## Savings Evidence (per-run, labeled)",
59
+ "",
60
+ `- Trace identity: ${se.trace_identity}`,
61
+ `- Cost source: ${se.cost_source}`,
62
+ `- Recoverability: ${se.recoverability}`,
63
+ `- Semantic preservation: ${se.semantic_preservation}`,
64
+ `- Evidence rung: ${se.evidence_rung}`,
65
+ `- Billing-confirmed: ${se.billing_confirmed}`,
66
+ `- ${se.label}`
67
+ ];
68
+ }
3
69
  function formatTokenBreakdown(tokens) {
4
70
  return `${tokens.totalTokens} total (${tokens.inputTokens} input, ${tokens.outputTokens} output)`;
5
71
  }
@@ -201,6 +267,7 @@ export function formatCompactionMarkdownReport(report, policy) {
201
267
  `- Saving per run: $${report.saving_per_run.toFixed(6)} (estimated)`,
202
268
  ...formatValueProofLines(report),
203
269
  ...formatSpendBySourceLines(report),
270
+ ...formatSavingsEvidenceLines(report),
204
271
  "",
205
272
  "## Waste Pattern",
206
273
  "",
@@ -1,4 +1,17 @@
1
1
  import type { CompactionReport, WasteFinding } from "./types.js";
2
+ /**
3
+ * Duplicate-safety summary for the `summary` rollup, by content fingerprint. A count of DISTINCT
4
+ * captured runs — NOT a billing/provider/semantic claim. A run captured/aggregated twice with the
5
+ * SAME fingerprint is counted once in the headline so summed savings are never inflated; a run
6
+ * WITHOUT a fingerprint cannot be de-duplicated and is counted as distinct.
7
+ */
8
+ export interface RunDistinctnessSummary {
9
+ total_runs: number;
10
+ distinct_runs: number;
11
+ duplicate_runs: number;
12
+ runs_with_fingerprint: number;
13
+ runs_without_fingerprint: number;
14
+ }
2
15
  export interface RunSummaryBucket {
3
16
  total_runs: number;
4
17
  total_original_input_tokens: number;
@@ -22,12 +35,54 @@ export interface SkippedReportFile {
22
35
  path: string;
23
36
  reason: string;
24
37
  }
38
+ /**
39
+ * V0.2 Increment 2 — the MEASURED-across-distinct-runs estimate delta. Aggregates the per-run
40
+ * estimate reductions over N DISTINCT runs (dedup by content fingerprint) into a mean + dispersion,
41
+ * and applies the accepted criterion (`docs/design/billing-delta-loop.md`): **N ≥ 3 distinct runs
42
+ * AND the mean ±2·SE interval excludes zero**. When BOTH hold, the figure earns the
43
+ * `measured_caveated_estimate_delta` rung (claims-and-evidence-ladder rung 1.5) — a measured
44
+ * estimate, NOT a single anecdote. Below the bar it stays rung 1 (`local_estimate_single_run`).
45
+ *
46
+ * It is STILL an estimate (chars/4 + price-table): explicitly NOT `provider-reported`, NOT
47
+ * eval-backed (rung 3), and NEVER `billing-confirmed` (rung 5). No time-period extrapolation.
48
+ */
49
+ export interface MeasuredEstimateDelta {
50
+ /** N distinct runs used (≥3 required to qualify). */
51
+ distinct_run_count: number;
52
+ /** Mean per-run percent reduction across the distinct runs (estimate). */
53
+ mean_percent_reduction: number;
54
+ /** Standard error of the mean percent reduction (0 when N<2 — no dispersion). */
55
+ std_error: number;
56
+ /** Lower / upper bound of the mean ±2·SE interval. */
57
+ ci_low: number;
58
+ ci_high: number;
59
+ /** Does the ±2·SE interval exclude zero (lower bound > 0)? */
60
+ excludes_zero: boolean;
61
+ /** Criterion met: N≥3 AND the ±2·SE interval excludes zero. */
62
+ qualifies: boolean;
63
+ /** The claims-ladder rung this aggregate stands on. */
64
+ evidence_rung: "measured_caveated_estimate_delta" | "local_estimate_single_run";
65
+ /** One-line honest label — estimate-class, never billing-confirmed, never eval-backed. */
66
+ label: string;
67
+ }
25
68
  export interface RunSummary extends RunSummaryBucket {
26
69
  average_percent_reduction: number;
70
+ /**
71
+ * V0.2 measured-across-distinct-runs estimate delta (the "measured" rung). Mean ±2·SE over the
72
+ * distinct runs' estimate reductions; `qualifies` only when N≥3 AND the interval excludes zero.
73
+ * Estimate-class — never billing-confirmed, never eval-backed.
74
+ */
75
+ measured_estimate_delta: MeasuredEstimateDelta;
27
76
  savings_by_policy: Record<string, RunSummaryBucket>;
28
77
  savings_by_waste_pattern: Record<string, RunSummaryBucket>;
29
78
  top_savings_runs: TopSavingsRun[];
30
79
  skipped_files: SkippedReportFile[];
80
+ /**
81
+ * Duplicate-safety summary: the headline totals above are summed over DISTINCT runs only (a
82
+ * run captured/aggregated twice with the same content fingerprint is counted once), so savings
83
+ * are never inflated. Runs without a fingerprint are each counted as distinct.
84
+ */
85
+ distinctness: RunDistinctnessSummary;
31
86
  generated_at: string;
32
87
  source_glob: string;
33
88
  }
@@ -35,6 +90,15 @@ interface LoadedReport {
35
90
  path: string;
36
91
  report: CompactionReport;
37
92
  }
93
+ /** Minimum DISTINCT runs for a measured (not single-anecdote) estimate delta. */
94
+ export declare const MIN_RUNS_FOR_MEASURED_DELTA = 3;
95
+ /**
96
+ * Compute the measured-across-distinct-runs estimate delta over a set of per-run percent
97
+ * reductions (PURE). Applies the accepted criterion: N ≥ 3 AND the mean ±2·SE interval excludes
98
+ * zero. Estimate-class only — the result is NEVER billing-confirmed or eval-backed. Uses the
99
+ * sample standard deviation (n−1); with n<2 there is no dispersion (SE 0) so it cannot qualify.
100
+ */
101
+ export declare function computeMeasuredEstimateDelta(percentReductions: number[]): MeasuredEstimateDelta;
38
102
  export declare function validateCompactionReport(value: unknown): {
39
103
  report?: CompactionReport;
40
104
  reason?: string;
@@ -1,5 +1,6 @@
1
1
  import { readdir, readFile } from "node:fs/promises";
2
2
  import { join } from "node:path";
3
+ import { classifyDistinctness } from "./trace-fingerprint.js";
3
4
  const REQUIRED_STRING_FIELDS = ["run_id", "trace_title", "policy_name"];
4
5
  const REQUIRED_NUMBER_FIELDS = [
5
6
  "original_input_tokens",
@@ -25,6 +26,103 @@ function roundTo(value, digits) {
25
26
  const factor = 10 ** digits;
26
27
  return Math.round((value + Number.EPSILON) * factor) / factor;
27
28
  }
29
+ /** Minimum DISTINCT runs for a measured (not single-anecdote) estimate delta. */
30
+ export const MIN_RUNS_FOR_MEASURED_DELTA = 3;
31
+ /**
32
+ * Compute the measured-across-distinct-runs estimate delta over a set of per-run percent
33
+ * reductions (PURE). Applies the accepted criterion: N ≥ 3 AND the mean ±2·SE interval excludes
34
+ * zero. Estimate-class only — the result is NEVER billing-confirmed or eval-backed. Uses the
35
+ * sample standard deviation (n−1); with n<2 there is no dispersion (SE 0) so it cannot qualify.
36
+ */
37
+ export function computeMeasuredEstimateDelta(percentReductions) {
38
+ const n = percentReductions.length;
39
+ const mean = n === 0 ? 0 : percentReductions.reduce((sum, x) => sum + x, 0) / n;
40
+ let se = 0;
41
+ if (n >= 2) {
42
+ const variance = percentReductions.reduce((sum, x) => sum + (x - mean) ** 2, 0) / (n - 1);
43
+ se = Math.sqrt(variance) / Math.sqrt(n);
44
+ }
45
+ const ciLow = mean - 2 * se;
46
+ const ciHigh = mean + 2 * se;
47
+ const excludesZero = ciLow > 0;
48
+ const qualifies = n >= MIN_RUNS_FOR_MEASURED_DELTA && excludesZero;
49
+ const meanR = roundTo(mean, 2);
50
+ const lowR = roundTo(ciLow, 2);
51
+ const highR = roundTo(ciHigh, 2);
52
+ let label;
53
+ if (qualifies) {
54
+ label =
55
+ `Measured-caveated ESTIMATE delta across ${n} distinct runs: mean ${meanR}% reduction ` +
56
+ `(±2·SE [${lowR}%, ${highR}%], excludes zero). Estimate (chars/4 + price-table) — ` +
57
+ "NOT billing-confirmed, NOT eval-backed, NOT realized savings, NOT extrapolated.";
58
+ }
59
+ else if (n < MIN_RUNS_FOR_MEASURED_DELTA) {
60
+ label =
61
+ `Single-run/anecdote estimate (${n} distinct run(s); need ≥${MIN_RUNS_FOR_MEASURED_DELTA} ` +
62
+ "for a measured delta). Local estimate — NOT billing-confirmed.";
63
+ }
64
+ else {
65
+ label =
66
+ `Estimate delta across ${n} distinct runs does NOT meet the measured bar ` +
67
+ `(±2·SE [${lowR}%, ${highR}%] includes zero — dispersion too wide). Local estimate — NOT billing-confirmed.`;
68
+ }
69
+ return {
70
+ distinct_run_count: n,
71
+ mean_percent_reduction: meanR,
72
+ std_error: roundTo(se, 4),
73
+ ci_low: lowR,
74
+ ci_high: highR,
75
+ excludes_zero: excludesZero,
76
+ qualifies,
77
+ evidence_rung: qualifies ? "measured_caveated_estimate_delta" : "local_estimate_single_run",
78
+ label
79
+ };
80
+ }
81
+ /** Read the OPTIONAL content fingerprint digest from a report (older reports have none). */
82
+ function fingerprintDigestOf(report) {
83
+ const fp = report.trace_fingerprint;
84
+ return fp && typeof fp.content_sha256 === "string" && fp.content_sha256.length > 0 ? fp.content_sha256 : null;
85
+ }
86
+ /**
87
+ * De-duplicate loaded reports by content fingerprint for a duplicate-safe headline.
88
+ *
89
+ * Keeps the FIRST report per distinct fingerprint; later same-fingerprint reports are duplicates
90
+ * and are dropped from the totals so a re-captured run never double-counts its savings. Reports
91
+ * without a fingerprint are each kept (cannot dedup what cannot be identified). PURE; never
92
+ * inflates savings.
93
+ */
94
+ function dedupeReportsByFingerprint(reports) {
95
+ const seen = new Set();
96
+ const distinctReports = [];
97
+ let withFingerprint = 0;
98
+ let withoutFingerprint = 0;
99
+ for (const loaded of reports) {
100
+ const fp = fingerprintDigestOf(loaded.report);
101
+ if (fp !== null) {
102
+ withFingerprint += 1;
103
+ if (seen.has(fp))
104
+ continue;
105
+ seen.add(fp);
106
+ distinctReports.push(loaded);
107
+ }
108
+ else {
109
+ withoutFingerprint += 1;
110
+ distinctReports.push(loaded);
111
+ }
112
+ }
113
+ const classified = classifyDistinctness(reports.map((loaded) => ({ runId: loaded.report.run_id, content_sha256: fingerprintDigestOf(loaded.report) })));
114
+ const distinctRunCount = classified.distinct_verified_count + withoutFingerprint;
115
+ return {
116
+ distinctReports,
117
+ distinctness: {
118
+ total_runs: reports.length,
119
+ distinct_runs: distinctRunCount,
120
+ duplicate_runs: reports.length - distinctRunCount,
121
+ runs_with_fingerprint: withFingerprint,
122
+ runs_without_fingerprint: withoutFingerprint
123
+ }
124
+ };
125
+ }
28
126
  function addReportToBucket(bucket, report) {
29
127
  bucket.total_runs += 1;
30
128
  bucket.total_original_input_tokens += report.original_input_tokens;
@@ -101,18 +199,23 @@ export async function readCompactionRunReports(runsDirectory = ".compaction/runs
101
199
  return { reports, skippedFiles };
102
200
  }
103
201
  export function aggregateCompactionReports(reports, skippedFiles = [], generatedAt = new Date().toISOString(), sourceGlob = ".compaction/runs/*/report.json") {
202
+ // Duplicate-safe headline: fold totals over DISTINCT reports only (a run captured/aggregated
203
+ // twice with the same content fingerprint is counted once). Never inflates summed savings.
204
+ const { distinctReports, distinctness } = dedupeReportsByFingerprint(reports);
104
205
  const summary = {
105
206
  ...emptyBucket(),
106
207
  average_percent_reduction: 0,
208
+ measured_estimate_delta: computeMeasuredEstimateDelta([]),
107
209
  savings_by_policy: {},
108
210
  savings_by_waste_pattern: {},
109
211
  top_savings_runs: [],
110
212
  skipped_files: skippedFiles,
213
+ distinctness,
111
214
  generated_at: generatedAt,
112
215
  source_glob: sourceGlob
113
216
  };
114
217
  let percentReductionTotal = 0;
115
- for (const { path, report } of reports) {
218
+ for (const { path, report } of distinctReports) {
116
219
  addReportToBucket(summary, report);
117
220
  percentReductionTotal += report.percent_reduction;
118
221
  const policyBucket = (summary.savings_by_policy[report.policy_name] ??= emptyBucket());
@@ -131,7 +234,11 @@ export function aggregateCompactionReports(reports, skippedFiles = [], generated
131
234
  waste_pattern: report.waste_pattern
132
235
  });
133
236
  }
134
- summary.average_percent_reduction = reports.length === 0 ? 0 : roundTo(percentReductionTotal / reports.length, 2);
237
+ summary.average_percent_reduction =
238
+ distinctReports.length === 0 ? 0 : roundTo(percentReductionTotal / distinctReports.length, 2);
239
+ // V0.2 Increment 2: the measured-across-distinct-runs estimate delta over the DISTINCT runs'
240
+ // per-run reductions. Qualifies for rung 1.5 only when N≥3 AND the mean ±2·SE excludes zero.
241
+ summary.measured_estimate_delta = computeMeasuredEstimateDelta(summary.top_savings_runs.map((run) => run.percent_reduction));
135
242
  normalizeBucket(summary);
136
243
  for (const bucket of Object.values(summary.savings_by_policy)) {
137
244
  normalizeBucket(bucket);
@@ -156,6 +263,18 @@ export async function createRunSummary(runsDirectory = ".compaction/runs") {
156
263
  function formatCurrency(value) {
157
264
  return `$${value.toFixed(6)}`;
158
265
  }
266
+ /**
267
+ * "distinct runs N of M (K duplicates detected)" — a count of distinct captured runs by content
268
+ * fingerprint. The totals above are summed over the distinct set, so a re-captured run never
269
+ * double-counts. NOT a billing/provider/semantic claim.
270
+ */
271
+ function distinctnessLine(d) {
272
+ const noFp = d.runs_without_fingerprint > 0
273
+ ? `; ${d.runs_without_fingerprint} run(s) without a fingerprint counted as distinct (cannot de-duplicate)`
274
+ : "";
275
+ return (`distinct runs ${d.distinct_runs} of ${d.total_runs} (${d.duplicate_runs} duplicate(s) detected by content fingerprint, ` +
276
+ `excluded from the headline so savings are not double-counted)${noFp}`);
277
+ }
159
278
  function formatBucketLines(bucket) {
160
279
  return [
161
280
  `runs: ${bucket.total_runs}`,
@@ -184,8 +303,10 @@ export function formatRunSummary(summary) {
184
303
  `Generated at: ${summary.generated_at}`,
185
304
  "",
186
305
  "Totals",
306
+ `- ${distinctnessLine(summary.distinctness)}`,
187
307
  ...formatBucketLines(summary).map((line) => `- ${line}`),
188
308
  `- average percent reduction: ${summary.average_percent_reduction.toFixed(2)}%`,
309
+ `- measured estimate delta [${summary.measured_estimate_delta.evidence_rung}]: ${summary.measured_estimate_delta.label}`,
189
310
  "",
190
311
  "Savings by policy",
191
312
  ...(policyLines.length === 0 ? ["- none"] : policyLines),
@@ -213,6 +334,7 @@ export function formatRunSummaryMarkdown(summary) {
213
334
  "",
214
335
  "## Totals",
215
336
  "",
337
+ `- Distinct runs: ${distinctnessLine(summary.distinctness)}`,
216
338
  `- Total runs: ${summary.total_runs}`,
217
339
  `- Total original input tokens: ${summary.total_original_input_tokens}`,
218
340
  `- Total compacted input tokens: ${summary.total_compacted_input_tokens}`,
@@ -221,7 +343,14 @@ export function formatRunSummaryMarkdown(summary) {
221
343
  `- Total cost before per run: ${formatCurrency(summary.total_cost_before_per_run)}`,
222
344
  `- Total cost after per run: ${formatCurrency(summary.total_cost_after_per_run)}`,
223
345
  `- Total saving per run: ${formatCurrency(summary.total_saving_per_run)}`,
224
- `- Saving figures are estimated and summed over the ${summary.total_runs} recorded run${summary.total_runs === 1 ? "" : "s"}.`,
346
+ `- Saving figures are estimated and summed over the ${summary.distinctness.distinct_runs} distinct recorded run${summary.distinctness.distinct_runs === 1 ? "" : "s"} (${summary.distinctness.duplicate_runs} duplicate(s) detected by content fingerprint and excluded).`,
347
+ "",
348
+ "## Measured Estimate Delta (across distinct runs)",
349
+ "",
350
+ `- Rung: ${summary.measured_estimate_delta.evidence_rung}`,
351
+ `- Distinct runs: ${summary.measured_estimate_delta.distinct_run_count}`,
352
+ `- Mean percent reduction: ${summary.measured_estimate_delta.mean_percent_reduction.toFixed(2)}% (±2·SE [${summary.measured_estimate_delta.ci_low.toFixed(2)}%, ${summary.measured_estimate_delta.ci_high.toFixed(2)}%])`,
353
+ `- ${summary.measured_estimate_delta.label}`,
225
354
  "",
226
355
  "## Savings by policy",
227
356
  "",