@compaction/cli 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -1
- package/dist/cli/commands/compact.js +15 -0
- package/dist/cli/commands/context.d.ts +2 -0
- package/dist/cli/commands/context.js +130 -0
- package/dist/cli/commands/import.js +3 -1
- package/dist/cli/commands/init.js +184 -157
- package/dist/cli/commands/run.js +13 -1
- package/dist/cli/index.js +4 -1
- package/dist/cli/onboarding/InitTui.d.ts +52 -0
- package/dist/cli/onboarding/InitTui.js +153 -0
- package/dist/cli/onboarding/model.d.ts +39 -0
- package/dist/cli/onboarding/model.js +77 -0
- package/dist/cli/onboarding/should-use-tui.d.ts +25 -0
- package/dist/cli/onboarding/should-use-tui.js +35 -0
- package/dist/cli/onboarding/wordmark.d.ts +28 -0
- package/dist/cli/onboarding/wordmark.js +72 -0
- package/dist/core/aggregate-format.d.ts +3 -1
- package/dist/core/aggregate-format.js +18 -0
- package/dist/core/context-store-eval-cases.d.ts +6 -0
- package/dist/core/context-store-eval-cases.js +331 -0
- package/dist/core/context-store-eval.d.ts +140 -0
- package/dist/core/context-store-eval.js +138 -0
- package/dist/core/context-store-fs.d.ts +73 -0
- package/dist/core/context-store-fs.js +157 -0
- package/dist/core/context-store-sufficiency.d.ts +98 -0
- package/dist/core/context-store-sufficiency.js +135 -0
- package/dist/core/context-store.d.ts +155 -0
- package/dist/core/context-store.js +228 -0
- package/dist/core/report-generator.d.ts +32 -1
- package/dist/core/report-generator.js +67 -0
- package/dist/core/run-aggregator.d.ts +64 -0
- package/dist/core/run-aggregator.js +132 -3
- package/dist/core/session-aggregate.d.ts +43 -0
- package/dist/core/session-aggregate.js +89 -7
- package/dist/core/trace-adapters.d.ts +8 -0
- package/dist/core/trace-adapters.js +4 -0
- package/dist/core/types.d.ts +55 -0
- package/package.json +5 -1
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
import { contentHash } from "./content-hash.js";
|
|
2
|
+
import { estimateTextTokens } from "./token-estimator.js";
|
|
3
|
+
import { computeTraceFingerprint } from "./trace-fingerprint.js";
|
|
4
|
+
/**
|
|
5
|
+
* V0.4 context store — rung-1 (local, deterministic, source-grounded).
|
|
6
|
+
*
|
|
7
|
+
* SCOPE (accepted design doc `docs/design/v0.4-context-store-retrieval.md`,
|
|
8
|
+
* 2026-06-24): a durable, local, file-based store of source-pointed context items
|
|
9
|
+
* accumulated across runs, plus DETERMINISTIC local retrieval and assembly under a
|
|
10
|
+
* token budget. This module is the rung-1 core.
|
|
11
|
+
*
|
|
12
|
+
* HARD NON-GOALS (load-bearing — do not add here):
|
|
13
|
+
* - NO semantic / embedding retrieval. Retrieval ranks by deterministic signals only
|
|
14
|
+
* (source-pointer match, lexical overlap, recency). Semantic retrieval needs a model
|
|
15
|
+
* → not free-CLI local-first → deferred to the gated private engine/API.
|
|
16
|
+
* - NO network call, credential, upload, or model invocation. This module is pure +
|
|
17
|
+
* local-file I/O only (see `loadContextStore` / `appendContextItems`).
|
|
18
|
+
* - NO hosted DB / vector store / auth.
|
|
19
|
+
* - NO large-memory / 300M-token / retrieval-correctness claim. Those are gated on the
|
|
20
|
+
* four-axis eval harness (a follow-up increment) and rung-6 evidence.
|
|
21
|
+
*
|
|
22
|
+
* HONESTY: token figures here are `local_estimate` (chars/4), never provider-reported.
|
|
23
|
+
* Every stored item retains its `source_pointer` + recoverability so the assembled
|
|
24
|
+
* context stays recoverable to its sources (the substrate invariant).
|
|
25
|
+
*/
|
|
26
|
+
/** Algorithm + schema version, so a persisted store is self-describing. */
|
|
27
|
+
export const CONTEXT_STORE_ITEM_VERSION = "context-store-item-v1";
|
|
28
|
+
function normalizeContent(content) {
|
|
29
|
+
return content.trim().replace(/\s+/g, " ");
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Build a single context item, deriving the dedup hash + local token estimate.
|
|
33
|
+
* Pure — no I/O, no clock (the caller supplies `created_at` so this stays
|
|
34
|
+
* deterministic and testable).
|
|
35
|
+
*/
|
|
36
|
+
export function makeContextItem(input) {
|
|
37
|
+
const content = normalizeContent(input.content);
|
|
38
|
+
return {
|
|
39
|
+
version: CONTEXT_STORE_ITEM_VERSION,
|
|
40
|
+
content_sha256: contentHash(content),
|
|
41
|
+
content,
|
|
42
|
+
estimated_tokens: estimateTextTokens(content),
|
|
43
|
+
source_pointer: input.source_pointer,
|
|
44
|
+
recoverability: input.recoverability ?? "unknown",
|
|
45
|
+
trace_fingerprint: input.trace_fingerprint,
|
|
46
|
+
source_trace_id: input.source_trace_id,
|
|
47
|
+
provenance: input.provenance,
|
|
48
|
+
labels: input.labels ? [...input.labels] : [],
|
|
49
|
+
created_at: input.created_at
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Derive context items from a `StateCapsule`. Each retained fact / open question /
|
|
54
|
+
* safety note becomes a source-pointed item, inheriting the capsule's source pointer +
|
|
55
|
+
* recoverability + the originating trace id (and the supplied per-run fingerprint).
|
|
56
|
+
* Pure — `created_at` is supplied by the caller.
|
|
57
|
+
*/
|
|
58
|
+
export function contextItemsFromCapsule(capsule, opts) {
|
|
59
|
+
const primaryProvenance = capsule.provenance_entries?.[0];
|
|
60
|
+
const base = {
|
|
61
|
+
source_pointer: capsule.source_pointer ?? primaryProvenance?.source_pointer,
|
|
62
|
+
recoverability: capsule.source_recoverable ?? primaryProvenance?.source_recoverable ?? "unknown",
|
|
63
|
+
source_trace_id: capsule.source_trace_id ?? capsule.traceId,
|
|
64
|
+
provenance: primaryProvenance,
|
|
65
|
+
trace_fingerprint: opts.trace_fingerprint,
|
|
66
|
+
created_at: opts.created_at
|
|
67
|
+
};
|
|
68
|
+
const items = [];
|
|
69
|
+
for (const fact of capsule.retainedFacts) {
|
|
70
|
+
if (fact.trim().length === 0)
|
|
71
|
+
continue;
|
|
72
|
+
items.push(makeContextItem({ ...base, content: fact, labels: ["retained_fact"] }));
|
|
73
|
+
}
|
|
74
|
+
for (const question of capsule.openQuestions) {
|
|
75
|
+
if (question.trim().length === 0)
|
|
76
|
+
continue;
|
|
77
|
+
items.push(makeContextItem({ ...base, content: question, labels: ["open_question"] }));
|
|
78
|
+
}
|
|
79
|
+
for (const note of capsule.safetyNotes) {
|
|
80
|
+
if (note.trim().length === 0)
|
|
81
|
+
continue;
|
|
82
|
+
items.push(makeContextItem({ ...base, content: note, labels: ["safety_note"] }));
|
|
83
|
+
}
|
|
84
|
+
return dedupeContextItems(items);
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Derive context items from a normalized `AgentTrace` (the local artifact `compaction capture`
|
|
88
|
+
* / `import` produces). Each non-empty message becomes a source-pointed item recoverable to its
|
|
89
|
+
* `trace=<id> message=<id>` origin, tagged with the run's distinctness fingerprint. This is the
|
|
90
|
+
* free-CLI populate path (state-capsules require the gated engine; trace messages do not). Pure —
|
|
91
|
+
* `created_at` is supplied by the caller.
|
|
92
|
+
*/
|
|
93
|
+
export function contextItemsFromTrace(trace, opts) {
|
|
94
|
+
const trace_fingerprint = computeTraceFingerprint(trace).content_sha256;
|
|
95
|
+
const items = [];
|
|
96
|
+
for (const message of trace.messages) {
|
|
97
|
+
if (message.content.trim().length === 0)
|
|
98
|
+
continue;
|
|
99
|
+
const labels = [message.role];
|
|
100
|
+
if (message.toolName)
|
|
101
|
+
labels.push(message.toolName);
|
|
102
|
+
items.push(makeContextItem({
|
|
103
|
+
content: message.content,
|
|
104
|
+
source_pointer: `trace=${trace.id} message=${message.id}`,
|
|
105
|
+
source_trace_id: trace.id,
|
|
106
|
+
recoverability: true,
|
|
107
|
+
trace_fingerprint,
|
|
108
|
+
labels,
|
|
109
|
+
created_at: opts.created_at
|
|
110
|
+
}));
|
|
111
|
+
}
|
|
112
|
+
return dedupeContextItems(items);
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Dedup by `content_sha256`, keeping the EARLIEST item deterministically (ties broken
|
|
116
|
+
* by `created_at`, then by the existing order). Reuses the fingerprint discipline:
|
|
117
|
+
* byte-identical content is one item, not many. Output order follows the
|
|
118
|
+
* first-occurrence order of each distinct hash in the input array.
|
|
119
|
+
*/
|
|
120
|
+
export function dedupeContextItems(items) {
|
|
121
|
+
const byHash = new Map();
|
|
122
|
+
for (const item of items) {
|
|
123
|
+
const existing = byHash.get(item.content_sha256);
|
|
124
|
+
if (!existing || item.created_at < existing.created_at) {
|
|
125
|
+
// Merge labels so a later duplicate's labels are not lost.
|
|
126
|
+
if (existing) {
|
|
127
|
+
const labels = Array.from(new Set([...item.labels, ...existing.labels])).sort();
|
|
128
|
+
byHash.set(item.content_sha256, { ...item, labels });
|
|
129
|
+
}
|
|
130
|
+
else {
|
|
131
|
+
byHash.set(item.content_sha256, item);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
else {
|
|
135
|
+
const labels = Array.from(new Set([...existing.labels, ...item.labels])).sort();
|
|
136
|
+
byHash.set(item.content_sha256, { ...existing, labels });
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
return Array.from(byHash.values());
|
|
140
|
+
}
|
|
141
|
+
const WORD_RE = /[a-z0-9]+/g;
|
|
142
|
+
function tokenSet(text) {
|
|
143
|
+
const set = new Set();
|
|
144
|
+
const matches = text.toLowerCase().match(WORD_RE);
|
|
145
|
+
if (matches) {
|
|
146
|
+
for (const word of matches)
|
|
147
|
+
set.add(word);
|
|
148
|
+
}
|
|
149
|
+
return set;
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* Deterministic, local retrieval. Ranks items by:
|
|
153
|
+
* score = sourceMatchWeight * (exact source/trace match ? 1 : 0)
|
|
154
|
+
* + lexicalWeight * lexical overlap coefficient in [0,1]
|
|
155
|
+
*
|
|
156
|
+
* Ties are broken deterministically by recency (newer first) then `content_sha256`
|
|
157
|
+
* ascending, so the same store + query always yields the same order. No model, no
|
|
158
|
+
* embeddings, no network — overlap is a pure word-set intersection.
|
|
159
|
+
*/
|
|
160
|
+
export function retrieveContextItems(items, query, options = {}) {
|
|
161
|
+
const sourceWeight = options.sourceMatchWeight ?? 1;
|
|
162
|
+
const lexicalWeight = options.lexicalWeight ?? 1;
|
|
163
|
+
const queryWords = tokenSet(query.text);
|
|
164
|
+
const wantedPointers = new Set(query.source_pointers ?? []);
|
|
165
|
+
const wantedTraces = new Set(query.trace_ids ?? []);
|
|
166
|
+
const ranked = items.map((item) => {
|
|
167
|
+
const sourceMatch = (item.source_pointer !== undefined && wantedPointers.has(item.source_pointer)) ||
|
|
168
|
+
(item.source_trace_id !== undefined && wantedTraces.has(item.source_trace_id));
|
|
169
|
+
let overlap = 0;
|
|
170
|
+
if (queryWords.size > 0) {
|
|
171
|
+
const itemWords = tokenSet(item.content);
|
|
172
|
+
if (itemWords.size > 0) {
|
|
173
|
+
let intersection = 0;
|
|
174
|
+
const [small, large] = queryWords.size <= itemWords.size ? [queryWords, itemWords] : [itemWords, queryWords];
|
|
175
|
+
for (const word of small) {
|
|
176
|
+
if (large.has(word))
|
|
177
|
+
intersection += 1;
|
|
178
|
+
}
|
|
179
|
+
// Overlap coefficient: intersection / min(|q|, |item|) — in [0, 1].
|
|
180
|
+
overlap = intersection / Math.min(queryWords.size, itemWords.size);
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
const score = sourceWeight * (sourceMatch ? 1 : 0) + lexicalWeight * overlap;
|
|
184
|
+
return { item, score, source_match: sourceMatch, lexical_overlap: overlap };
|
|
185
|
+
});
|
|
186
|
+
ranked.sort((a, b) => {
|
|
187
|
+
if (b.score !== a.score)
|
|
188
|
+
return b.score - a.score;
|
|
189
|
+
if (a.item.created_at !== b.item.created_at)
|
|
190
|
+
return a.item.created_at < b.item.created_at ? 1 : -1;
|
|
191
|
+
return a.item.content_sha256 < b.item.content_sha256 ? -1 : 1;
|
|
192
|
+
});
|
|
193
|
+
const withSignal = ranked.filter((entry) => entry.score > 0);
|
|
194
|
+
return options.limit !== undefined ? withSignal.slice(0, options.limit) : withSignal;
|
|
195
|
+
}
|
|
196
|
+
/**
|
|
197
|
+
* Assemble ranked items into the sufficient active context under a token budget.
|
|
198
|
+
* Greedy in ranked order; dedups by `content_sha256`; PRESERVES the source pointer +
|
|
199
|
+
* provenance on every included item (so the assembled context stays recoverable).
|
|
200
|
+
* Token figures are local estimates.
|
|
201
|
+
*/
|
|
202
|
+
export function assembleContext(ranked, options) {
|
|
203
|
+
const items = ranked.map((entry) => ("item" in entry ? entry.item : entry));
|
|
204
|
+
const included = [];
|
|
205
|
+
const dropped = [];
|
|
206
|
+
const seen = new Set();
|
|
207
|
+
let total = 0;
|
|
208
|
+
for (const item of items) {
|
|
209
|
+
if (seen.has(item.content_sha256))
|
|
210
|
+
continue;
|
|
211
|
+
seen.add(item.content_sha256);
|
|
212
|
+
if (total + item.estimated_tokens <= options.tokenBudget) {
|
|
213
|
+
included.push(item);
|
|
214
|
+
total += item.estimated_tokens;
|
|
215
|
+
}
|
|
216
|
+
else {
|
|
217
|
+
dropped.push(item);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
return {
|
|
221
|
+
text: included.map((item) => item.content).join("\n"),
|
|
222
|
+
included,
|
|
223
|
+
dropped,
|
|
224
|
+
estimated_tokens: total,
|
|
225
|
+
within_budget: dropped.length === 0
|
|
226
|
+
};
|
|
227
|
+
}
|
|
228
|
+
//# sourceMappingURL=context-store.js.map
|
|
@@ -1,6 +1,37 @@
|
|
|
1
1
|
import type { SkillInjectionAdvisory } from "./skill-injection-detector.js";
|
|
2
|
-
import type { AgentTrace, CompactionPolicy, CompactionReport, CostEstimate, TokenEstimate, WasteFinding } from "./types.js";
|
|
2
|
+
import type { AgentTrace, CompactionPolicy, CompactionReport, CostEstimate, SavingsEvidence, TokenEstimate, WasteFinding } from "./types.js";
|
|
3
3
|
import type { UsageMetadata } from "./usage-metadata.js";
|
|
4
|
+
/**
|
|
5
|
+
* Attach the content-addressed per-run trace fingerprint to a compaction report (PURE,
|
|
6
|
+
* additive, backward-compatible). This is the report-generation attachment point that
|
|
7
|
+
* threads trace identity into the savings-evidence pipeline so each per-run saving is
|
|
8
|
+
* attributable to a verifiable run identity and a re-captured run can be de-duplicated in
|
|
9
|
+
* the aggregate.
|
|
10
|
+
*
|
|
11
|
+
* It changes NO existing report field/label — it only sets the OPTIONAL `trace_fingerprint`
|
|
12
|
+
* from `computeTraceFingerprint(trace)` (a one-way SHA-256 digest + structural counts; no
|
|
13
|
+
* message content). Returns a new object; the input report is not mutated. This is an
|
|
14
|
+
* attribution/integrity signal, NOT a billing, provider, or semantic-preservation claim.
|
|
15
|
+
*/
|
|
16
|
+
export declare function withTraceFingerprint(report: CompactionReport, trace: AgentTrace): CompactionReport;
|
|
17
|
+
/**
|
|
18
|
+
* Attach the composed per-run SAVINGS EVIDENCE record to a compaction report (PURE,
|
|
19
|
+
* additive, backward-compatible). V0.2 "evidence labels per result": it composes the
|
|
20
|
+
* report's EXISTING numbers into ONE explicitly-labeled record and asserts the weakest
|
|
21
|
+
* honest rung — a single run is rung 1 (local estimate). It makes NO new/stronger claim:
|
|
22
|
+
* `billing_confirmed` is always false, `semantic_preservation` always `not_evaluated`, and
|
|
23
|
+
* the measured rung 1.5 (`measured_caveated_estimate_delta`) is NEVER produced here (only by
|
|
24
|
+
* the aggregate increment across N≥3 distinct runs).
|
|
25
|
+
*
|
|
26
|
+
* `costSource` defaults to the weakest honest label (`local_estimate`); pass a stronger
|
|
27
|
+
* source ONLY when the run genuinely carries it. `recoverability` defaults to
|
|
28
|
+
* `not_evaluated` (the eval is the separate `compact --eval` path); pass the eval result
|
|
29
|
+
* when one ran. Returns a new object; the input report is not mutated.
|
|
30
|
+
*/
|
|
31
|
+
export declare function withSavingsEvidence(report: CompactionReport, opts?: {
|
|
32
|
+
costSource?: SavingsEvidence["cost_source"];
|
|
33
|
+
recoverability?: SavingsEvidence["recoverability"];
|
|
34
|
+
}): CompactionReport;
|
|
4
35
|
export declare function formatAnalyzeReport(trace: AgentTrace, tokens: TokenEstimate, cost: CostEstimate, findings?: WasteFinding[], usage?: UsageMetadata, skillInjectionAdvisory?: SkillInjectionAdvisory): string;
|
|
5
36
|
export declare function formatCompactionReport(report: CompactionReport): string;
|
|
6
37
|
export declare function formatCompactionMarkdownReport(report: CompactionReport, policy: CompactionPolicy): string;
|
|
@@ -1,5 +1,71 @@
|
|
|
1
1
|
import { calculateCost } from "./cost-calculator.js";
|
|
2
2
|
import { isKnownModel } from "./pricing.js";
|
|
3
|
+
import { computeTraceFingerprint } from "./trace-fingerprint.js";
|
|
4
|
+
/**
|
|
5
|
+
* Attach the content-addressed per-run trace fingerprint to a compaction report (PURE,
|
|
6
|
+
* additive, backward-compatible). This is the report-generation attachment point that
|
|
7
|
+
* threads trace identity into the savings-evidence pipeline so each per-run saving is
|
|
8
|
+
* attributable to a verifiable run identity and a re-captured run can be de-duplicated in
|
|
9
|
+
* the aggregate.
|
|
10
|
+
*
|
|
11
|
+
* It changes NO existing report field/label — it only sets the OPTIONAL `trace_fingerprint`
|
|
12
|
+
* from `computeTraceFingerprint(trace)` (a one-way SHA-256 digest + structural counts; no
|
|
13
|
+
* message content). Returns a new object; the input report is not mutated. This is an
|
|
14
|
+
* attribution/integrity signal, NOT a billing, provider, or semantic-preservation claim.
|
|
15
|
+
*/
|
|
16
|
+
export function withTraceFingerprint(report, trace) {
|
|
17
|
+
return { ...report, trace_fingerprint: computeTraceFingerprint(trace) };
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Attach the composed per-run SAVINGS EVIDENCE record to a compaction report (PURE,
|
|
21
|
+
* additive, backward-compatible). V0.2 "evidence labels per result": it composes the
|
|
22
|
+
* report's EXISTING numbers into ONE explicitly-labeled record and asserts the weakest
|
|
23
|
+
* honest rung — a single run is rung 1 (local estimate). It makes NO new/stronger claim:
|
|
24
|
+
* `billing_confirmed` is always false, `semantic_preservation` always `not_evaluated`, and
|
|
25
|
+
* the measured rung 1.5 (`measured_caveated_estimate_delta`) is NEVER produced here (only by
|
|
26
|
+
* the aggregate increment across N≥3 distinct runs).
|
|
27
|
+
*
|
|
28
|
+
* `costSource` defaults to the weakest honest label (`local_estimate`); pass a stronger
|
|
29
|
+
* source ONLY when the run genuinely carries it. `recoverability` defaults to
|
|
30
|
+
* `not_evaluated` (the eval is the separate `compact --eval` path); pass the eval result
|
|
31
|
+
* when one ran. Returns a new object; the input report is not mutated.
|
|
32
|
+
*/
|
|
33
|
+
export function withSavingsEvidence(report, opts = {}) {
|
|
34
|
+
const cost_source = opts.costSource ?? "local_estimate";
|
|
35
|
+
const recoverability = opts.recoverability ?? "not_evaluated";
|
|
36
|
+
const savings_evidence = {
|
|
37
|
+
trace_identity: report.trace_fingerprint ? "fingerprinted" : "unidentified",
|
|
38
|
+
cost_source,
|
|
39
|
+
recoverability,
|
|
40
|
+
semantic_preservation: "not_evaluated",
|
|
41
|
+
evidence_rung: "local_estimate_single_run",
|
|
42
|
+
billing_confirmed: false,
|
|
43
|
+
label: `Per-run ${cost_source.replace(/_/g, " ")} delta (single run → local-estimate rung). ` +
|
|
44
|
+
`Recoverability: ${recoverability}` +
|
|
45
|
+
(recoverability === "not_evaluated" ? " (run `compact --eval`)" : "") +
|
|
46
|
+
". Semantic preservation: not_evaluated. NOT billing-confirmed, NOT realized savings, NOT extrapolated."
|
|
47
|
+
};
|
|
48
|
+
return { ...report, savings_evidence };
|
|
49
|
+
}
|
|
50
|
+
/** Render the composed per-run savings-evidence record for the markdown report. */
|
|
51
|
+
function formatSavingsEvidenceLines(report) {
|
|
52
|
+
const se = report.savings_evidence;
|
|
53
|
+
if (!se) {
|
|
54
|
+
return [];
|
|
55
|
+
}
|
|
56
|
+
return [
|
|
57
|
+
"",
|
|
58
|
+
"## Savings Evidence (per-run, labeled)",
|
|
59
|
+
"",
|
|
60
|
+
`- Trace identity: ${se.trace_identity}`,
|
|
61
|
+
`- Cost source: ${se.cost_source}`,
|
|
62
|
+
`- Recoverability: ${se.recoverability}`,
|
|
63
|
+
`- Semantic preservation: ${se.semantic_preservation}`,
|
|
64
|
+
`- Evidence rung: ${se.evidence_rung}`,
|
|
65
|
+
`- Billing-confirmed: ${se.billing_confirmed}`,
|
|
66
|
+
`- ${se.label}`
|
|
67
|
+
];
|
|
68
|
+
}
|
|
3
69
|
function formatTokenBreakdown(tokens) {
|
|
4
70
|
return `${tokens.totalTokens} total (${tokens.inputTokens} input, ${tokens.outputTokens} output)`;
|
|
5
71
|
}
|
|
@@ -201,6 +267,7 @@ export function formatCompactionMarkdownReport(report, policy) {
|
|
|
201
267
|
`- Saving per run: $${report.saving_per_run.toFixed(6)} (estimated)`,
|
|
202
268
|
...formatValueProofLines(report),
|
|
203
269
|
...formatSpendBySourceLines(report),
|
|
270
|
+
...formatSavingsEvidenceLines(report),
|
|
204
271
|
"",
|
|
205
272
|
"## Waste Pattern",
|
|
206
273
|
"",
|
|
@@ -1,4 +1,17 @@
|
|
|
1
1
|
import type { CompactionReport, WasteFinding } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Duplicate-safety summary for the `summary` rollup, by content fingerprint. A count of DISTINCT
|
|
4
|
+
* captured runs — NOT a billing/provider/semantic claim. A run captured/aggregated twice with the
|
|
5
|
+
* SAME fingerprint is counted once in the headline so summed savings are never inflated; a run
|
|
6
|
+
* WITHOUT a fingerprint cannot be de-duplicated and is counted as distinct.
|
|
7
|
+
*/
|
|
8
|
+
export interface RunDistinctnessSummary {
|
|
9
|
+
total_runs: number;
|
|
10
|
+
distinct_runs: number;
|
|
11
|
+
duplicate_runs: number;
|
|
12
|
+
runs_with_fingerprint: number;
|
|
13
|
+
runs_without_fingerprint: number;
|
|
14
|
+
}
|
|
2
15
|
export interface RunSummaryBucket {
|
|
3
16
|
total_runs: number;
|
|
4
17
|
total_original_input_tokens: number;
|
|
@@ -22,12 +35,54 @@ export interface SkippedReportFile {
|
|
|
22
35
|
path: string;
|
|
23
36
|
reason: string;
|
|
24
37
|
}
|
|
38
|
+
/**
|
|
39
|
+
* V0.2 Increment 2 — the MEASURED-across-distinct-runs estimate delta. Aggregates the per-run
|
|
40
|
+
* estimate reductions over N DISTINCT runs (dedup by content fingerprint) into a mean + dispersion,
|
|
41
|
+
* and applies the accepted criterion (`docs/design/billing-delta-loop.md`): **N ≥ 3 distinct runs
|
|
42
|
+
* AND the mean ±2·SE interval excludes zero**. When BOTH hold, the figure earns the
|
|
43
|
+
* `measured_caveated_estimate_delta` rung (claims-and-evidence-ladder rung 1.5) — a measured
|
|
44
|
+
* estimate, NOT a single anecdote. Below the bar it stays rung 1 (`local_estimate_single_run`).
|
|
45
|
+
*
|
|
46
|
+
* It is STILL an estimate (chars/4 + price-table): explicitly NOT `provider-reported`, NOT
|
|
47
|
+
* eval-backed (rung 3), and NEVER `billing-confirmed` (rung 5). No time-period extrapolation.
|
|
48
|
+
*/
|
|
49
|
+
export interface MeasuredEstimateDelta {
|
|
50
|
+
/** N distinct runs used (≥3 required to qualify). */
|
|
51
|
+
distinct_run_count: number;
|
|
52
|
+
/** Mean per-run percent reduction across the distinct runs (estimate). */
|
|
53
|
+
mean_percent_reduction: number;
|
|
54
|
+
/** Standard error of the mean percent reduction (0 when N<2 — no dispersion). */
|
|
55
|
+
std_error: number;
|
|
56
|
+
/** Lower / upper bound of the mean ±2·SE interval. */
|
|
57
|
+
ci_low: number;
|
|
58
|
+
ci_high: number;
|
|
59
|
+
/** Does the ±2·SE interval exclude zero (lower bound > 0)? */
|
|
60
|
+
excludes_zero: boolean;
|
|
61
|
+
/** Criterion met: N≥3 AND the ±2·SE interval excludes zero. */
|
|
62
|
+
qualifies: boolean;
|
|
63
|
+
/** The claims-ladder rung this aggregate stands on. */
|
|
64
|
+
evidence_rung: "measured_caveated_estimate_delta" | "local_estimate_single_run";
|
|
65
|
+
/** One-line honest label — estimate-class, never billing-confirmed, never eval-backed. */
|
|
66
|
+
label: string;
|
|
67
|
+
}
|
|
25
68
|
export interface RunSummary extends RunSummaryBucket {
|
|
26
69
|
average_percent_reduction: number;
|
|
70
|
+
/**
|
|
71
|
+
* V0.2 measured-across-distinct-runs estimate delta (the "measured" rung). Mean ±2·SE over the
|
|
72
|
+
* distinct runs' estimate reductions; `qualifies` only when N≥3 AND the interval excludes zero.
|
|
73
|
+
* Estimate-class — never billing-confirmed, never eval-backed.
|
|
74
|
+
*/
|
|
75
|
+
measured_estimate_delta: MeasuredEstimateDelta;
|
|
27
76
|
savings_by_policy: Record<string, RunSummaryBucket>;
|
|
28
77
|
savings_by_waste_pattern: Record<string, RunSummaryBucket>;
|
|
29
78
|
top_savings_runs: TopSavingsRun[];
|
|
30
79
|
skipped_files: SkippedReportFile[];
|
|
80
|
+
/**
|
|
81
|
+
* Duplicate-safety summary: the headline totals above are summed over DISTINCT runs only (a
|
|
82
|
+
* run captured/aggregated twice with the same content fingerprint is counted once), so savings
|
|
83
|
+
* are never inflated. Runs without a fingerprint are each counted as distinct.
|
|
84
|
+
*/
|
|
85
|
+
distinctness: RunDistinctnessSummary;
|
|
31
86
|
generated_at: string;
|
|
32
87
|
source_glob: string;
|
|
33
88
|
}
|
|
@@ -35,6 +90,15 @@ interface LoadedReport {
|
|
|
35
90
|
path: string;
|
|
36
91
|
report: CompactionReport;
|
|
37
92
|
}
|
|
93
|
+
/** Minimum DISTINCT runs for a measured (not single-anecdote) estimate delta. */
|
|
94
|
+
export declare const MIN_RUNS_FOR_MEASURED_DELTA = 3;
|
|
95
|
+
/**
|
|
96
|
+
* Compute the measured-across-distinct-runs estimate delta over a set of per-run percent
|
|
97
|
+
* reductions (PURE). Applies the accepted criterion: N ≥ 3 AND the mean ±2·SE interval excludes
|
|
98
|
+
* zero. Estimate-class only — the result is NEVER billing-confirmed or eval-backed. Uses the
|
|
99
|
+
* sample standard deviation (n−1); with n<2 there is no dispersion (SE 0) so it cannot qualify.
|
|
100
|
+
*/
|
|
101
|
+
export declare function computeMeasuredEstimateDelta(percentReductions: number[]): MeasuredEstimateDelta;
|
|
38
102
|
export declare function validateCompactionReport(value: unknown): {
|
|
39
103
|
report?: CompactionReport;
|
|
40
104
|
reason?: string;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { readdir, readFile } from "node:fs/promises";
|
|
2
2
|
import { join } from "node:path";
|
|
3
|
+
import { classifyDistinctness } from "./trace-fingerprint.js";
|
|
3
4
|
const REQUIRED_STRING_FIELDS = ["run_id", "trace_title", "policy_name"];
|
|
4
5
|
const REQUIRED_NUMBER_FIELDS = [
|
|
5
6
|
"original_input_tokens",
|
|
@@ -25,6 +26,103 @@ function roundTo(value, digits) {
|
|
|
25
26
|
const factor = 10 ** digits;
|
|
26
27
|
return Math.round((value + Number.EPSILON) * factor) / factor;
|
|
27
28
|
}
|
|
29
|
+
/** Minimum DISTINCT runs for a measured (not single-anecdote) estimate delta. */
|
|
30
|
+
export const MIN_RUNS_FOR_MEASURED_DELTA = 3;
|
|
31
|
+
/**
|
|
32
|
+
* Compute the measured-across-distinct-runs estimate delta over a set of per-run percent
|
|
33
|
+
* reductions (PURE). Applies the accepted criterion: N ≥ 3 AND the mean ±2·SE interval excludes
|
|
34
|
+
* zero. Estimate-class only — the result is NEVER billing-confirmed or eval-backed. Uses the
|
|
35
|
+
* sample standard deviation (n−1); with n<2 there is no dispersion (SE 0) so it cannot qualify.
|
|
36
|
+
*/
|
|
37
|
+
export function computeMeasuredEstimateDelta(percentReductions) {
|
|
38
|
+
const n = percentReductions.length;
|
|
39
|
+
const mean = n === 0 ? 0 : percentReductions.reduce((sum, x) => sum + x, 0) / n;
|
|
40
|
+
let se = 0;
|
|
41
|
+
if (n >= 2) {
|
|
42
|
+
const variance = percentReductions.reduce((sum, x) => sum + (x - mean) ** 2, 0) / (n - 1);
|
|
43
|
+
se = Math.sqrt(variance) / Math.sqrt(n);
|
|
44
|
+
}
|
|
45
|
+
const ciLow = mean - 2 * se;
|
|
46
|
+
const ciHigh = mean + 2 * se;
|
|
47
|
+
const excludesZero = ciLow > 0;
|
|
48
|
+
const qualifies = n >= MIN_RUNS_FOR_MEASURED_DELTA && excludesZero;
|
|
49
|
+
const meanR = roundTo(mean, 2);
|
|
50
|
+
const lowR = roundTo(ciLow, 2);
|
|
51
|
+
const highR = roundTo(ciHigh, 2);
|
|
52
|
+
let label;
|
|
53
|
+
if (qualifies) {
|
|
54
|
+
label =
|
|
55
|
+
`Measured-caveated ESTIMATE delta across ${n} distinct runs: mean ${meanR}% reduction ` +
|
|
56
|
+
`(±2·SE [${lowR}%, ${highR}%], excludes zero). Estimate (chars/4 + price-table) — ` +
|
|
57
|
+
"NOT billing-confirmed, NOT eval-backed, NOT realized savings, NOT extrapolated.";
|
|
58
|
+
}
|
|
59
|
+
else if (n < MIN_RUNS_FOR_MEASURED_DELTA) {
|
|
60
|
+
label =
|
|
61
|
+
`Single-run/anecdote estimate (${n} distinct run(s); need ≥${MIN_RUNS_FOR_MEASURED_DELTA} ` +
|
|
62
|
+
"for a measured delta). Local estimate — NOT billing-confirmed.";
|
|
63
|
+
}
|
|
64
|
+
else {
|
|
65
|
+
label =
|
|
66
|
+
`Estimate delta across ${n} distinct runs does NOT meet the measured bar ` +
|
|
67
|
+
`(±2·SE [${lowR}%, ${highR}%] includes zero — dispersion too wide). Local estimate — NOT billing-confirmed.`;
|
|
68
|
+
}
|
|
69
|
+
return {
|
|
70
|
+
distinct_run_count: n,
|
|
71
|
+
mean_percent_reduction: meanR,
|
|
72
|
+
std_error: roundTo(se, 4),
|
|
73
|
+
ci_low: lowR,
|
|
74
|
+
ci_high: highR,
|
|
75
|
+
excludes_zero: excludesZero,
|
|
76
|
+
qualifies,
|
|
77
|
+
evidence_rung: qualifies ? "measured_caveated_estimate_delta" : "local_estimate_single_run",
|
|
78
|
+
label
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
/** Read the OPTIONAL content fingerprint digest from a report (older reports have none). */
|
|
82
|
+
function fingerprintDigestOf(report) {
|
|
83
|
+
const fp = report.trace_fingerprint;
|
|
84
|
+
return fp && typeof fp.content_sha256 === "string" && fp.content_sha256.length > 0 ? fp.content_sha256 : null;
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* De-duplicate loaded reports by content fingerprint for a duplicate-safe headline.
|
|
88
|
+
*
|
|
89
|
+
* Keeps the FIRST report per distinct fingerprint; later same-fingerprint reports are duplicates
|
|
90
|
+
* and are dropped from the totals so a re-captured run never double-counts its savings. Reports
|
|
91
|
+
* without a fingerprint are each kept (cannot dedup what cannot be identified). PURE; never
|
|
92
|
+
* inflates savings.
|
|
93
|
+
*/
|
|
94
|
+
function dedupeReportsByFingerprint(reports) {
|
|
95
|
+
const seen = new Set();
|
|
96
|
+
const distinctReports = [];
|
|
97
|
+
let withFingerprint = 0;
|
|
98
|
+
let withoutFingerprint = 0;
|
|
99
|
+
for (const loaded of reports) {
|
|
100
|
+
const fp = fingerprintDigestOf(loaded.report);
|
|
101
|
+
if (fp !== null) {
|
|
102
|
+
withFingerprint += 1;
|
|
103
|
+
if (seen.has(fp))
|
|
104
|
+
continue;
|
|
105
|
+
seen.add(fp);
|
|
106
|
+
distinctReports.push(loaded);
|
|
107
|
+
}
|
|
108
|
+
else {
|
|
109
|
+
withoutFingerprint += 1;
|
|
110
|
+
distinctReports.push(loaded);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
const classified = classifyDistinctness(reports.map((loaded) => ({ runId: loaded.report.run_id, content_sha256: fingerprintDigestOf(loaded.report) })));
|
|
114
|
+
const distinctRunCount = classified.distinct_verified_count + withoutFingerprint;
|
|
115
|
+
return {
|
|
116
|
+
distinctReports,
|
|
117
|
+
distinctness: {
|
|
118
|
+
total_runs: reports.length,
|
|
119
|
+
distinct_runs: distinctRunCount,
|
|
120
|
+
duplicate_runs: reports.length - distinctRunCount,
|
|
121
|
+
runs_with_fingerprint: withFingerprint,
|
|
122
|
+
runs_without_fingerprint: withoutFingerprint
|
|
123
|
+
}
|
|
124
|
+
};
|
|
125
|
+
}
|
|
28
126
|
function addReportToBucket(bucket, report) {
|
|
29
127
|
bucket.total_runs += 1;
|
|
30
128
|
bucket.total_original_input_tokens += report.original_input_tokens;
|
|
@@ -101,18 +199,23 @@ export async function readCompactionRunReports(runsDirectory = ".compaction/runs
|
|
|
101
199
|
return { reports, skippedFiles };
|
|
102
200
|
}
|
|
103
201
|
export function aggregateCompactionReports(reports, skippedFiles = [], generatedAt = new Date().toISOString(), sourceGlob = ".compaction/runs/*/report.json") {
|
|
202
|
+
// Duplicate-safe headline: fold totals over DISTINCT reports only (a run captured/aggregated
|
|
203
|
+
// twice with the same content fingerprint is counted once). Never inflates summed savings.
|
|
204
|
+
const { distinctReports, distinctness } = dedupeReportsByFingerprint(reports);
|
|
104
205
|
const summary = {
|
|
105
206
|
...emptyBucket(),
|
|
106
207
|
average_percent_reduction: 0,
|
|
208
|
+
measured_estimate_delta: computeMeasuredEstimateDelta([]),
|
|
107
209
|
savings_by_policy: {},
|
|
108
210
|
savings_by_waste_pattern: {},
|
|
109
211
|
top_savings_runs: [],
|
|
110
212
|
skipped_files: skippedFiles,
|
|
213
|
+
distinctness,
|
|
111
214
|
generated_at: generatedAt,
|
|
112
215
|
source_glob: sourceGlob
|
|
113
216
|
};
|
|
114
217
|
let percentReductionTotal = 0;
|
|
115
|
-
for (const { path, report } of
|
|
218
|
+
for (const { path, report } of distinctReports) {
|
|
116
219
|
addReportToBucket(summary, report);
|
|
117
220
|
percentReductionTotal += report.percent_reduction;
|
|
118
221
|
const policyBucket = (summary.savings_by_policy[report.policy_name] ??= emptyBucket());
|
|
@@ -131,7 +234,11 @@ export function aggregateCompactionReports(reports, skippedFiles = [], generated
|
|
|
131
234
|
waste_pattern: report.waste_pattern
|
|
132
235
|
});
|
|
133
236
|
}
|
|
134
|
-
summary.average_percent_reduction =
|
|
237
|
+
summary.average_percent_reduction =
|
|
238
|
+
distinctReports.length === 0 ? 0 : roundTo(percentReductionTotal / distinctReports.length, 2);
|
|
239
|
+
// V0.2 Increment 2: the measured-across-distinct-runs estimate delta over the DISTINCT runs'
|
|
240
|
+
// per-run reductions. Qualifies for rung 1.5 only when N≥3 AND the mean ±2·SE excludes zero.
|
|
241
|
+
summary.measured_estimate_delta = computeMeasuredEstimateDelta(summary.top_savings_runs.map((run) => run.percent_reduction));
|
|
135
242
|
normalizeBucket(summary);
|
|
136
243
|
for (const bucket of Object.values(summary.savings_by_policy)) {
|
|
137
244
|
normalizeBucket(bucket);
|
|
@@ -156,6 +263,18 @@ export async function createRunSummary(runsDirectory = ".compaction/runs") {
|
|
|
156
263
|
function formatCurrency(value) {
|
|
157
264
|
return `$${value.toFixed(6)}`;
|
|
158
265
|
}
|
|
266
|
+
/**
|
|
267
|
+
* "distinct runs N of M (K duplicates detected)" — a count of distinct captured runs by content
|
|
268
|
+
* fingerprint. The totals above are summed over the distinct set, so a re-captured run never
|
|
269
|
+
* double-counts. NOT a billing/provider/semantic claim.
|
|
270
|
+
*/
|
|
271
|
+
function distinctnessLine(d) {
|
|
272
|
+
const noFp = d.runs_without_fingerprint > 0
|
|
273
|
+
? `; ${d.runs_without_fingerprint} run(s) without a fingerprint counted as distinct (cannot de-duplicate)`
|
|
274
|
+
: "";
|
|
275
|
+
return (`distinct runs ${d.distinct_runs} of ${d.total_runs} (${d.duplicate_runs} duplicate(s) detected by content fingerprint, ` +
|
|
276
|
+
`excluded from the headline so savings are not double-counted)${noFp}`);
|
|
277
|
+
}
|
|
159
278
|
function formatBucketLines(bucket) {
|
|
160
279
|
return [
|
|
161
280
|
`runs: ${bucket.total_runs}`,
|
|
@@ -184,8 +303,10 @@ export function formatRunSummary(summary) {
|
|
|
184
303
|
`Generated at: ${summary.generated_at}`,
|
|
185
304
|
"",
|
|
186
305
|
"Totals",
|
|
306
|
+
`- ${distinctnessLine(summary.distinctness)}`,
|
|
187
307
|
...formatBucketLines(summary).map((line) => `- ${line}`),
|
|
188
308
|
`- average percent reduction: ${summary.average_percent_reduction.toFixed(2)}%`,
|
|
309
|
+
`- measured estimate delta [${summary.measured_estimate_delta.evidence_rung}]: ${summary.measured_estimate_delta.label}`,
|
|
189
310
|
"",
|
|
190
311
|
"Savings by policy",
|
|
191
312
|
...(policyLines.length === 0 ? ["- none"] : policyLines),
|
|
@@ -213,6 +334,7 @@ export function formatRunSummaryMarkdown(summary) {
|
|
|
213
334
|
"",
|
|
214
335
|
"## Totals",
|
|
215
336
|
"",
|
|
337
|
+
`- Distinct runs: ${distinctnessLine(summary.distinctness)}`,
|
|
216
338
|
`- Total runs: ${summary.total_runs}`,
|
|
217
339
|
`- Total original input tokens: ${summary.total_original_input_tokens}`,
|
|
218
340
|
`- Total compacted input tokens: ${summary.total_compacted_input_tokens}`,
|
|
@@ -221,7 +343,14 @@ export function formatRunSummaryMarkdown(summary) {
|
|
|
221
343
|
`- Total cost before per run: ${formatCurrency(summary.total_cost_before_per_run)}`,
|
|
222
344
|
`- Total cost after per run: ${formatCurrency(summary.total_cost_after_per_run)}`,
|
|
223
345
|
`- Total saving per run: ${formatCurrency(summary.total_saving_per_run)}`,
|
|
224
|
-
`- Saving figures are estimated and summed over the ${summary.
|
|
346
|
+
`- Saving figures are estimated and summed over the ${summary.distinctness.distinct_runs} distinct recorded run${summary.distinctness.distinct_runs === 1 ? "" : "s"} (${summary.distinctness.duplicate_runs} duplicate(s) detected by content fingerprint and excluded).`,
|
|
347
|
+
"",
|
|
348
|
+
"## Measured Estimate Delta (across distinct runs)",
|
|
349
|
+
"",
|
|
350
|
+
`- Rung: ${summary.measured_estimate_delta.evidence_rung}`,
|
|
351
|
+
`- Distinct runs: ${summary.measured_estimate_delta.distinct_run_count}`,
|
|
352
|
+
`- Mean percent reduction: ${summary.measured_estimate_delta.mean_percent_reduction.toFixed(2)}% (±2·SE [${summary.measured_estimate_delta.ci_low.toFixed(2)}%, ${summary.measured_estimate_delta.ci_high.toFixed(2)}%])`,
|
|
353
|
+
`- ${summary.measured_estimate_delta.label}`,
|
|
225
354
|
"",
|
|
226
355
|
"## Savings by policy",
|
|
227
356
|
"",
|