@compaction/cli 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -12
- package/dist/cli/commands/activity.d.ts +8 -0
- package/dist/cli/commands/activity.js +35 -0
- package/dist/cli/commands/apply-context.js +68 -0
- package/dist/cli/commands/billing-delta.js +12 -1
- package/dist/cli/commands/capture-claude-code.d.ts +44 -0
- package/dist/cli/commands/capture-claude-code.js +206 -0
- package/dist/cli/commands/capture.js +283 -1
- package/dist/cli/commands/compact.js +9 -5
- package/dist/cli/commands/dev.d.ts +17 -0
- package/dist/cli/commands/dev.js +131 -0
- package/dist/cli/commands/gateway.d.ts +31 -0
- package/dist/cli/commands/gateway.js +312 -0
- package/dist/cli/commands/hooks.d.ts +5 -0
- package/dist/cli/commands/hooks.js +181 -0
- package/dist/cli/commands/init.js +422 -17
- package/dist/cli/commands/input-compaction-ab.d.ts +2 -0
- package/dist/cli/commands/input-compaction-ab.js +125 -0
- package/dist/cli/commands/optimize-hosted.d.ts +54 -0
- package/dist/cli/commands/optimize-hosted.js +123 -0
- package/dist/cli/commands/optimize.js +8 -0
- package/dist/cli/commands/output-shaping-ab.d.ts +2 -0
- package/dist/cli/commands/output-shaping-ab.js +132 -0
- package/dist/cli/commands/output-shaping.d.ts +7 -0
- package/dist/cli/commands/output-shaping.js +43 -0
- package/dist/cli/commands/policies.d.ts +2 -0
- package/dist/cli/commands/policies.js +80 -0
- package/dist/cli/commands/precall.d.ts +19 -0
- package/dist/cli/commands/precall.js +436 -0
- package/dist/cli/commands/recommend.js +1 -1
- package/dist/cli/commands/run.js +297 -12
- package/dist/cli/commands/upgrade-status.d.ts +42 -0
- package/dist/cli/commands/upgrade-status.js +152 -0
- package/dist/cli/cursor-export-read.d.ts +13 -0
- package/dist/cli/cursor-export-read.js +53 -0
- package/dist/cli/cursor-live-preflight.d.ts +19 -0
- package/dist/cli/cursor-live-preflight.js +46 -0
- package/dist/cli/index.js +36 -2
- package/dist/cli/onboarding/GatewayTui.d.ts +33 -0
- package/dist/cli/onboarding/GatewayTui.js +113 -0
- package/dist/cli/onboarding/model.d.ts +32 -1
- package/dist/cli/onboarding/model.js +27 -2
- package/dist/cli/terminal-logo.d.ts +27 -0
- package/dist/cli/terminal-logo.js +49 -0
- package/dist/core/activity-event.d.ts +107 -0
- package/dist/core/activity-event.js +182 -0
- package/dist/core/activity-store.d.ts +63 -0
- package/dist/core/activity-store.js +254 -0
- package/dist/core/activity-view.d.ts +76 -0
- package/dist/core/activity-view.js +120 -0
- package/dist/core/api-client/index.d.ts +3 -1
- package/dist/core/api-client/index.js +4 -0
- package/dist/core/api-client/payload.d.ts +5 -1
- package/dist/core/api-client/payload.js +8 -6
- package/dist/core/api-client/persisted-config.d.ts +57 -0
- package/dist/core/api-client/persisted-config.js +156 -0
- package/dist/core/api-client/tool.d.ts +29 -0
- package/dist/core/api-client/tool.js +50 -0
- package/dist/core/api-client/types.d.ts +15 -4
- package/dist/core/auto-apply-ask.d.ts +58 -0
- package/dist/core/auto-apply-ask.js +105 -0
- package/dist/core/auto-apply-gates.d.ts +76 -0
- package/dist/core/auto-apply-gates.js +113 -0
- package/dist/core/before-call-activity.d.ts +35 -0
- package/dist/core/before-call-activity.js +103 -0
- package/dist/core/before-call-recovery.d.ts +19 -0
- package/dist/core/before-call-recovery.js +46 -0
- package/dist/core/before-call-stdin.d.ts +59 -0
- package/dist/core/before-call-stdin.js +78 -0
- package/dist/core/before-call.d.ts +149 -0
- package/dist/core/before-call.js +358 -0
- package/dist/core/billing-delta/billing-delta-record.d.ts +28 -0
- package/dist/core/billing-delta/billing-delta-record.js +26 -0
- package/dist/core/capture-record.d.ts +40 -0
- package/dist/core/capture-record.js +55 -0
- package/dist/core/claude-code-before-call.d.ts +37 -0
- package/dist/core/claude-code-before-call.js +120 -0
- package/dist/core/claude-code-connect.d.ts +31 -0
- package/dist/core/claude-code-connect.js +87 -0
- package/dist/core/claude-code-hook-record.d.ts +71 -0
- package/dist/core/claude-code-hook-record.js +0 -0
- package/dist/core/claude-code-hooks.d.ts +77 -0
- package/dist/core/claude-code-hooks.js +141 -0
- package/dist/core/codex-capture.d.ts +45 -0
- package/dist/core/codex-capture.js +204 -0
- package/dist/core/command-runner.js +5 -1
- package/dist/core/compaction-artifacts.d.ts +51 -0
- package/dist/core/compaction-artifacts.js +242 -0
- package/dist/core/compactor.d.ts +9 -0
- package/dist/core/compactor.js +95 -0
- package/dist/core/cross-surface-event.d.ts +306 -0
- package/dist/core/cross-surface-event.js +330 -0
- package/dist/core/cursor-capture.d.ts +54 -0
- package/dist/core/cursor-capture.js +215 -0
- package/dist/core/cursor-preflight-probe.d.ts +8 -0
- package/dist/core/cursor-preflight-probe.js +88 -0
- package/dist/core/cursor-preflight.d.ts +86 -0
- package/dist/core/cursor-preflight.js +126 -0
- package/dist/core/gateway/apply-activation.d.ts +39 -0
- package/dist/core/gateway/apply-activation.js +84 -0
- package/dist/core/gateway/apply-policy.d.ts +64 -0
- package/dist/core/gateway/apply-policy.js +221 -0
- package/dist/core/gateway/apply-receipt.d.ts +36 -0
- package/dist/core/gateway/apply-receipt.js +75 -0
- package/dist/core/gateway/cache-proof.d.ts +45 -0
- package/dist/core/gateway/cache-proof.js +65 -0
- package/dist/core/gateway/configure.d.ts +50 -0
- package/dist/core/gateway/configure.js +169 -0
- package/dist/core/gateway/openai-usage.d.ts +56 -0
- package/dist/core/gateway/openai-usage.js +128 -0
- package/dist/core/gateway/receipt.d.ts +138 -0
- package/dist/core/gateway/receipt.js +120 -0
- package/dist/core/gateway/recovery.d.ts +23 -0
- package/dist/core/gateway/recovery.js +68 -0
- package/dist/core/gateway/server.d.ts +51 -0
- package/dist/core/gateway/server.js +276 -0
- package/dist/core/gateway/status.d.ts +45 -0
- package/dist/core/gateway/status.js +109 -0
- package/dist/core/hook-usage-aggregate.d.ts +47 -0
- package/dist/core/hook-usage-aggregate.js +161 -0
- package/dist/core/input-compaction-ab.d.ts +111 -0
- package/dist/core/input-compaction-ab.js +158 -0
- package/dist/core/local-run-record.d.ts +109 -0
- package/dist/core/local-run-record.js +223 -0
- package/dist/core/output-shaping-ab.d.ts +140 -0
- package/dist/core/output-shaping-ab.js +146 -0
- package/dist/core/output-shaping-attach.d.ts +31 -0
- package/dist/core/output-shaping-attach.js +57 -0
- package/dist/core/output-shaping.d.ts +56 -0
- package/dist/core/output-shaping.js +89 -0
- package/dist/core/policy-middleware.d.ts +121 -0
- package/dist/core/policy-middleware.js +919 -0
- package/dist/core/policy-preferences.d.ts +99 -0
- package/dist/core/policy-preferences.js +232 -0
- package/dist/core/run-aggregator.d.ts +11 -1
- package/dist/core/run-aggregator.js +29 -2
- package/dist/core/run-flow-report.d.ts +82 -0
- package/dist/core/run-flow-report.js +71 -0
- package/dist/core/safety-report.js +8 -1
- package/dist/core/shim-capture-bridge.d.ts +32 -0
- package/dist/core/shim-capture-bridge.js +88 -0
- package/dist/core/skill-injection-policy.d.ts +72 -0
- package/dist/core/skill-injection-policy.js +183 -0
- package/dist/core/spend-attribution.js +1 -1
- package/dist/core/token-accounting.d.ts +1 -1
- package/dist/core/tool-shim.d.ts +129 -0
- package/dist/core/tool-shim.js +447 -0
- package/dist/core/trace-parser.d.ts +13 -13
- package/dist/core/trace-parser.js +6 -6
- package/dist/core/types.d.ts +12 -2
- package/dist/core/waste-detector.d.ts +20 -0
- package/dist/core/waste-detector.js +160 -6
- package/package.json +1 -1
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Aggregate the content-free Claude Code hook usage records (PUBLIC CLI/SDK code, engine-free).
|
|
3
|
+
* Rolls up the records written by `capture claude-code --from-hook`
|
|
4
|
+
* (`.compaction/hooks/**/records/*.json`) into a CONTENT-FREE source-status view — by tool, by session,
|
|
5
|
+
* and over a time window — mirroring the `/app` source-status model (events / input / output / token
|
|
6
|
+
* sources / providers / models / provider-reported-vs-unavailable). Local-first; no network.
|
|
7
|
+
*
|
|
8
|
+
* Honesty rails (binding):
|
|
9
|
+
* - Content-free in / content-free out: counts, honest source, ids, providers, models, timestamps only.
|
|
10
|
+
* No prompt/completion/message content, no transcript content, no `last_assistant_message`.
|
|
11
|
+
* - Missing usage stays **null / unavailable**, never 0. A tool with no recorded input tokens reports
|
|
12
|
+
* `inputTokens: null`, not 0.
|
|
13
|
+
* - Token source labels are preserved exactly (provider-reported / local-estimate / unknown).
|
|
14
|
+
* - Idempotent: records are deduped by `dedupKey` so the same Stop/session state is counted once.
|
|
15
|
+
* - NO savings claim — this surfaces usage only.
|
|
16
|
+
*/
|
|
17
|
+
import { readdir, readFile, stat } from "node:fs/promises";
|
|
18
|
+
import path from "node:path";
|
|
19
|
+
import { CLAUDE_CODE_HOOK_RECORD_SCHEMA } from "./claude-code-hook-record.js";
|
|
20
|
+
export const HOOK_USAGE_AGGREGATE_SCHEMA = "compaction.hook-usage-aggregate.v1";
|
|
21
|
+
/** A loaded record plus nothing else — we only ever read the content-free fields. */
|
|
22
|
+
function isHookRecord(value) {
|
|
23
|
+
if (typeof value !== "object" || value === null)
|
|
24
|
+
return false;
|
|
25
|
+
const v = value;
|
|
26
|
+
return v.schema === CLAUDE_CODE_HOOK_RECORD_SCHEMA && typeof v.dedupKey === "string" && typeof v.tool === "string";
|
|
27
|
+
}
|
|
28
|
+
/** Recursively find `*.json` files under any `records/` directory below baseDir. */
|
|
29
|
+
async function findRecordFiles(baseDir) {
|
|
30
|
+
const out = [];
|
|
31
|
+
async function walk(dir) {
|
|
32
|
+
let names;
|
|
33
|
+
try {
|
|
34
|
+
names = await readdir(dir);
|
|
35
|
+
}
|
|
36
|
+
catch {
|
|
37
|
+
return; // missing dir → no records (not an error)
|
|
38
|
+
}
|
|
39
|
+
for (const name of names) {
|
|
40
|
+
const full = path.join(dir, name);
|
|
41
|
+
let isDir = false;
|
|
42
|
+
try {
|
|
43
|
+
isDir = (await stat(full)).isDirectory();
|
|
44
|
+
}
|
|
45
|
+
catch {
|
|
46
|
+
continue;
|
|
47
|
+
}
|
|
48
|
+
if (isDir) {
|
|
49
|
+
await walk(full);
|
|
50
|
+
}
|
|
51
|
+
else if (name.endsWith(".json") && path.basename(dir) === "records") {
|
|
52
|
+
out.push(full);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
await walk(baseDir);
|
|
57
|
+
return out;
|
|
58
|
+
}
|
|
59
|
+
/** Load + validate all hook usage records under baseDir (default `.compaction/hooks`). Invalid files skipped. */
|
|
60
|
+
export async function loadHookUsageRecords(baseDir = ".compaction/hooks") {
|
|
61
|
+
const files = await findRecordFiles(baseDir);
|
|
62
|
+
const records = [];
|
|
63
|
+
for (const file of files) {
|
|
64
|
+
try {
|
|
65
|
+
const parsed = JSON.parse(await readFile(file, "utf8"));
|
|
66
|
+
if (isHookRecord(parsed))
|
|
67
|
+
records.push(parsed);
|
|
68
|
+
}
|
|
69
|
+
catch {
|
|
70
|
+
// skip unreadable/invalid record file (never throw — local-first, best-effort)
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
return records;
|
|
74
|
+
}
|
|
75
|
+
function addToken(acc, value) {
|
|
76
|
+
if (value === null)
|
|
77
|
+
return acc; // missing contributes nothing and never coerces to 0
|
|
78
|
+
return (acc ?? 0) + value;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Aggregate hook records into a content-free source-status view. Dedups by `dedupKey`. Missing token
|
|
82
|
+
* counts stay null. Token source labels preserved exactly. No savings, no content.
|
|
83
|
+
*/
|
|
84
|
+
export function aggregateHookUsageRecords(records, options = {}) {
|
|
85
|
+
const since = options.since ?? null;
|
|
86
|
+
// Apply the time-window filter first, then dedup by dedupKey (records are already deduped at write time;
|
|
87
|
+
// this is defensive). `totalRecords` counts records IN the window so "deduped from N" reads accurately
|
|
88
|
+
// under --since. When `since` is set, a record with a missing/non-string recordedAt is EXCLUDED (a record
|
|
89
|
+
// with no timestamp cannot be proven to fall inside the window).
|
|
90
|
+
let totalRecords = 0;
|
|
91
|
+
const deduped = new Map();
|
|
92
|
+
for (const r of records) {
|
|
93
|
+
if (since !== null && (typeof r.recordedAt !== "string" || r.recordedAt < since))
|
|
94
|
+
continue;
|
|
95
|
+
totalRecords += 1;
|
|
96
|
+
if (!deduped.has(r.dedupKey))
|
|
97
|
+
deduped.set(r.dedupKey, r);
|
|
98
|
+
}
|
|
99
|
+
const byTool = new Map();
|
|
100
|
+
const sessionsByTool = new Map();
|
|
101
|
+
for (const r of deduped.values()) {
|
|
102
|
+
let agg = byTool.get(r.tool);
|
|
103
|
+
if (!agg) {
|
|
104
|
+
agg = {
|
|
105
|
+
tool: r.tool,
|
|
106
|
+
events: 0,
|
|
107
|
+
inputTokens: null,
|
|
108
|
+
outputTokens: null,
|
|
109
|
+
reasoningTokens: null,
|
|
110
|
+
tokenSources: [],
|
|
111
|
+
providers: [],
|
|
112
|
+
models: [],
|
|
113
|
+
outputRecorded: 0,
|
|
114
|
+
providerReportedEvents: 0,
|
|
115
|
+
unavailableEvents: 0,
|
|
116
|
+
sessions: 0,
|
|
117
|
+
firstRecordedAt: null,
|
|
118
|
+
lastRecordedAt: null
|
|
119
|
+
};
|
|
120
|
+
byTool.set(r.tool, agg);
|
|
121
|
+
sessionsByTool.set(r.tool, new Set());
|
|
122
|
+
}
|
|
123
|
+
agg.events += 1;
|
|
124
|
+
agg.inputTokens = addToken(agg.inputTokens, r.inputTokens ?? null);
|
|
125
|
+
agg.outputTokens = addToken(agg.outputTokens, r.outputTokens ?? null);
|
|
126
|
+
if (typeof r.outputTokens === "number")
|
|
127
|
+
agg.outputRecorded += 1;
|
|
128
|
+
if (r.providerReported === true)
|
|
129
|
+
agg.providerReportedEvents += 1;
|
|
130
|
+
if ((r.inputTokens ?? null) === null && (r.outputTokens ?? null) === null)
|
|
131
|
+
agg.unavailableEvents += 1;
|
|
132
|
+
if (r.tokenSource && !agg.tokenSources.includes(r.tokenSource))
|
|
133
|
+
agg.tokenSources.push(r.tokenSource);
|
|
134
|
+
if (r.provider && !agg.providers.includes(r.provider))
|
|
135
|
+
agg.providers.push(r.provider);
|
|
136
|
+
if (r.model && !agg.models.includes(r.model))
|
|
137
|
+
agg.models.push(r.model);
|
|
138
|
+
if (r.sessionId)
|
|
139
|
+
sessionsByTool.get(r.tool).add(r.sessionId);
|
|
140
|
+
if (typeof r.recordedAt === "string") {
|
|
141
|
+
if (agg.firstRecordedAt === null || r.recordedAt < agg.firstRecordedAt)
|
|
142
|
+
agg.firstRecordedAt = r.recordedAt;
|
|
143
|
+
if (agg.lastRecordedAt === null || r.recordedAt > agg.lastRecordedAt)
|
|
144
|
+
agg.lastRecordedAt = r.recordedAt;
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
for (const [tool, agg] of byTool) {
|
|
148
|
+
agg.sessions = sessionsByTool.get(tool).size;
|
|
149
|
+
agg.tokenSources.sort();
|
|
150
|
+
agg.providers.sort();
|
|
151
|
+
agg.models.sort();
|
|
152
|
+
}
|
|
153
|
+
return {
|
|
154
|
+
schema: HOOK_USAGE_AGGREGATE_SCHEMA,
|
|
155
|
+
totalRecords,
|
|
156
|
+
dedupedRecords: deduped.size,
|
|
157
|
+
since,
|
|
158
|
+
tools: Array.from(byTool.values()).sort((a, b) => a.tool.localeCompare(b.tool))
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
//# sourceMappingURL=hook-usage-aggregate.js.map
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Input-compaction A/B experiment record + summary (PUBLIC CLI/SDK code — engine-free, ships in the npm CLI).
|
|
3
|
+
* Design: `docs/design/input-compaction-ab.md` (accepted 2026-06-30). The INPUT-side twin of
|
|
4
|
+
* `output-shaping-ab.ts`: links a control arm (full input context) and a treatment arm (Compaction-compacted
|
|
5
|
+
* input context) of Codex runs and reports the **observed provider-reported INPUT-token delta**, gated for a
|
|
6
|
+
* confirmed claim on a **context-preservation** eval — NOT the output short-but-sufficient eval.
|
|
7
|
+
*
|
|
8
|
+
* Boundary / honesty (binding):
|
|
9
|
+
* - This PUBLIC module computes the **observed** input-token measurement for operator visibility. It NEVER
|
|
10
|
+
* produces a CONFIRMED input-savings number — `confirmedSavingsEligible` is the literal `false`. A confirmed
|
|
11
|
+
* number is produced solely by the PRIVATE engine gate (`src/engine/input-compaction-verification.ts`).
|
|
12
|
+
* - **Input eval ≠ output eval.** Input requires context-preservation: task-critical context preserved,
|
|
13
|
+
* source recoverability, instruction/commitment preservation, no material loss, and the treatment output
|
|
14
|
+
* still solves the same task. Reusing the output short-but-sufficient eval here would be a bug.
|
|
15
|
+
* - Input tokens that are not provider-reported (e.g. Cursor) → `unavailable`; can never be eligible.
|
|
16
|
+
* - Missing usage stays null, never 0. Token source preserved exactly. Content-free (counts + flags + ids).
|
|
17
|
+
* - Output-token delta is kept **observable but secondary** — never the primary claim for this input gate.
|
|
18
|
+
*/
|
|
19
|
+
import type { TokenSource } from "./api-client/index.js";
|
|
20
|
+
import type { CaptureUsageSidecar } from "./output-shaping-ab.js";
|
|
21
|
+
export declare const INPUT_COMPACTION_AB_SCHEMA: "input-compaction.ab-experiment.v1";
|
|
22
|
+
export type InputCompactionAbArm = "control" | "treatment";
|
|
23
|
+
/** Tri-state per-treatment outcome: true = passed, false = failed, null/undefined = not evaluated. */
|
|
24
|
+
export type EvalOutcome = boolean | null;
|
|
25
|
+
export interface InputCompactionAbRun {
|
|
26
|
+
arm: InputCompactionAbArm;
|
|
27
|
+
/** Provider-reported INPUT tokens; null when unavailable (never invented as 0). */
|
|
28
|
+
inputTokens: number | null;
|
|
29
|
+
/** Provider-reported OUTPUT tokens (observed, secondary — not the input claim). */
|
|
30
|
+
outputTokens: number | null;
|
|
31
|
+
providerReported: boolean;
|
|
32
|
+
tokenSource: TokenSource;
|
|
33
|
+
/** Task-critical context preserved (overall operator/eval judgment). */
|
|
34
|
+
contextPreserved?: EvalOutcome;
|
|
35
|
+
/** The treatment output still solves the same task. */
|
|
36
|
+
taskSolved?: EvalOutcome;
|
|
37
|
+
/** Source recoverability preserved (from `compact --eval` recoverability). */
|
|
38
|
+
sourceRecoverability?: EvalOutcome;
|
|
39
|
+
/** Instruction/commitment preservation (from `compact --eval` commitment-preservation). */
|
|
40
|
+
commitmentPreservation?: EvalOutcome;
|
|
41
|
+
/** No material context loss. */
|
|
42
|
+
noMaterialLoss?: EvalOutcome;
|
|
43
|
+
/** Semantic/meaning preservation is NOT scored in v1 (D4) — always `not_evaluated`. */
|
|
44
|
+
semanticEval: "not_evaluated";
|
|
45
|
+
reference?: string;
|
|
46
|
+
}
|
|
47
|
+
export interface InputCompactionAbExperiment {
|
|
48
|
+
schema: typeof INPUT_COMPACTION_AB_SCHEMA;
|
|
49
|
+
experimentId: string;
|
|
50
|
+
/** Short generic label of the shared task/context SHAPE (not prompt content). */
|
|
51
|
+
taskShape: string;
|
|
52
|
+
createdAt: string;
|
|
53
|
+
runs: InputCompactionAbRun[];
|
|
54
|
+
}
|
|
55
|
+
export interface InitInputCompactionAbInput {
|
|
56
|
+
experimentId: string;
|
|
57
|
+
taskShape: string;
|
|
58
|
+
createdAt?: string;
|
|
59
|
+
}
|
|
60
|
+
export declare function initInputCompactionAbExperiment(input: InitInputCompactionAbInput): InputCompactionAbExperiment;
|
|
61
|
+
export declare function addInputCompactionAbRun(experiment: InputCompactionAbExperiment, run: InputCompactionAbRun): InputCompactionAbExperiment;
|
|
62
|
+
/** Build a treatment run from a capture-usage sidecar + the recorded context-preservation outcome. */
|
|
63
|
+
export interface TreatmentOutcomeInput {
|
|
64
|
+
contextPreserved?: EvalOutcome;
|
|
65
|
+
taskSolved?: EvalOutcome;
|
|
66
|
+
sourceRecoverability?: EvalOutcome;
|
|
67
|
+
commitmentPreservation?: EvalOutcome;
|
|
68
|
+
noMaterialLoss?: EvalOutcome;
|
|
69
|
+
}
|
|
70
|
+
export declare function runFromSidecar(arm: InputCompactionAbArm, sidecar: CaptureUsageSidecar, outcome?: TreatmentOutcomeInput, reference?: string): InputCompactionAbRun;
|
|
71
|
+
export type InputCompactionAbConfidence = "unavailable" | "review_required" | "observed_not_confirmed" | "eligible_for_engine_confirmation";
|
|
72
|
+
/** Context-preservation rollup for the treatment arm. `pass` only if ALL required checks passed. */
|
|
73
|
+
export type ContextPreservationStatus = "pass" | "fail" | "not_evaluated" | "mixed";
|
|
74
|
+
export interface InputCompactionAbSummary {
|
|
75
|
+
experimentId: string;
|
|
76
|
+
nControl: number;
|
|
77
|
+
nTreatment: number;
|
|
78
|
+
/** Mean control (full-context) INPUT tokens — provider-reported; null when unavailable. */
|
|
79
|
+
inputTokensBefore: number | null;
|
|
80
|
+
/** Mean treatment (compacted-context) INPUT tokens — provider-reported; null when unavailable. */
|
|
81
|
+
inputTokensAfter: number | null;
|
|
82
|
+
/** before − after: positive ⇒ treatment used FEWER input tokens. OBSERVED, not confirmed. */
|
|
83
|
+
inputTokenDelta: number | null;
|
|
84
|
+
inputTokenReductionPct: number | null;
|
|
85
|
+
/** OBSERVED output-token delta (secondary; never the input claim). */
|
|
86
|
+
outputTokensBefore: number | null;
|
|
87
|
+
outputTokensAfter: number | null;
|
|
88
|
+
outputTokenDelta: number | null;
|
|
89
|
+
tokenSource: "provider-reported" | "mixed" | "local-estimate" | "unavailable";
|
|
90
|
+
/** Context-preservation rollup across treatment runs (the INPUT eval — NOT short-but-sufficient). */
|
|
91
|
+
contextPreservation: ContextPreservationStatus;
|
|
92
|
+
semanticEval: "not_evaluated";
|
|
93
|
+
confidence: InputCompactionAbConfidence;
|
|
94
|
+
/** ALWAYS false in the public CLI — a confirmed input-savings number is produced only by the engine gate. */
|
|
95
|
+
confirmedSavingsEligible: false;
|
|
96
|
+
reasons: string[];
|
|
97
|
+
}
|
|
98
|
+
/** Roll up the treatment arm's context-preservation checks. `pass` only if every required check is true. */
|
|
99
|
+
export declare function treatmentContextPreservation(treatment: InputCompactionAbRun[]): ContextPreservationStatus;
|
|
100
|
+
/**
|
|
101
|
+
* Observed input-compaction A/B + a conservative confidence. NEVER confirms a saving (engine gate does).
|
|
102
|
+
* Decision order is fail-safe: unavailable → review_required → observed_not_confirmed →
|
|
103
|
+
* eligible_for_engine_confirmation. Input means/delta come ONLY from provider-reported input tokens.
|
|
104
|
+
*/
|
|
105
|
+
export declare function summarizeInputCompactionAb(experiment: InputCompactionAbExperiment): InputCompactionAbSummary;
|
|
106
|
+
/** Provider-reported INPUT-token arrays per arm — the only honest input to the engine confirmation gate. */
|
|
107
|
+
export declare function providerReportedInputArms(experiment: InputCompactionAbExperiment): {
|
|
108
|
+
controlInputTokens: number[];
|
|
109
|
+
treatmentInputTokens: number[];
|
|
110
|
+
bothProviderReported: boolean;
|
|
111
|
+
};
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
export const INPUT_COMPACTION_AB_SCHEMA = "input-compaction.ab-experiment.v1";
|
|
2
|
+
export function initInputCompactionAbExperiment(input) {
|
|
3
|
+
return {
|
|
4
|
+
schema: INPUT_COMPACTION_AB_SCHEMA,
|
|
5
|
+
experimentId: input.experimentId,
|
|
6
|
+
taskShape: input.taskShape,
|
|
7
|
+
createdAt: input.createdAt ?? new Date().toISOString(),
|
|
8
|
+
runs: []
|
|
9
|
+
};
|
|
10
|
+
}
|
|
11
|
+
export function addInputCompactionAbRun(experiment, run) {
|
|
12
|
+
return { ...experiment, runs: [...experiment.runs, run] };
|
|
13
|
+
}
|
|
14
|
+
export function runFromSidecar(arm, sidecar, outcome = {}, reference) {
|
|
15
|
+
const base = {
|
|
16
|
+
arm,
|
|
17
|
+
inputTokens: sidecar.inputTokens,
|
|
18
|
+
outputTokens: sidecar.outputTokens,
|
|
19
|
+
providerReported: sidecar.providerReported === true,
|
|
20
|
+
tokenSource: sidecar.tokenSource,
|
|
21
|
+
semanticEval: "not_evaluated",
|
|
22
|
+
...(reference ? { reference } : {})
|
|
23
|
+
};
|
|
24
|
+
if (arm !== "treatment")
|
|
25
|
+
return base;
|
|
26
|
+
return {
|
|
27
|
+
...base,
|
|
28
|
+
contextPreserved: outcome.contextPreserved ?? null,
|
|
29
|
+
taskSolved: outcome.taskSolved ?? null,
|
|
30
|
+
sourceRecoverability: outcome.sourceRecoverability ?? null,
|
|
31
|
+
commitmentPreservation: outcome.commitmentPreservation ?? null,
|
|
32
|
+
noMaterialLoss: outcome.noMaterialLoss ?? null
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
const MIN_RUNS_PER_ARM = 3;
|
|
36
|
+
const REQUIRED_CHECKS = [
|
|
37
|
+
"contextPreserved",
|
|
38
|
+
"taskSolved",
|
|
39
|
+
"sourceRecoverability",
|
|
40
|
+
"commitmentPreservation",
|
|
41
|
+
"noMaterialLoss"
|
|
42
|
+
];
|
|
43
|
+
function mean(xs) {
|
|
44
|
+
return xs.reduce((s, x) => s + x, 0) / xs.length;
|
|
45
|
+
}
|
|
46
|
+
function meanOrNull(xs) {
|
|
47
|
+
return xs.length > 0 ? mean(xs) : null;
|
|
48
|
+
}
|
|
49
|
+
function armSource(runs) {
|
|
50
|
+
if (runs.length === 0)
|
|
51
|
+
return "unavailable";
|
|
52
|
+
if (runs.every((r) => r.providerReported && typeof r.inputTokens === "number"))
|
|
53
|
+
return "provider-reported";
|
|
54
|
+
if (runs.some((r) => r.providerReported && typeof r.inputTokens === "number"))
|
|
55
|
+
return "mixed";
|
|
56
|
+
return runs.some((r) => r.tokenSource === "local-estimate") ? "local-estimate" : "unavailable";
|
|
57
|
+
}
|
|
58
|
+
/** Roll up the treatment arm's context-preservation checks. `pass` only if every required check is true. */
|
|
59
|
+
export function treatmentContextPreservation(treatment) {
|
|
60
|
+
if (treatment.length === 0)
|
|
61
|
+
return "not_evaluated";
|
|
62
|
+
const values = [];
|
|
63
|
+
for (const r of treatment)
|
|
64
|
+
for (const k of REQUIRED_CHECKS)
|
|
65
|
+
values.push(r[k] ?? null);
|
|
66
|
+
if (values.some((v) => v === undefined || v === null)) {
|
|
67
|
+
return values.some((v) => v === true || v === false) ? "mixed" : "not_evaluated";
|
|
68
|
+
}
|
|
69
|
+
if (values.every((v) => v === true))
|
|
70
|
+
return "pass";
|
|
71
|
+
if (values.every((v) => v === false))
|
|
72
|
+
return "fail";
|
|
73
|
+
return "mixed";
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Observed input-compaction A/B + a conservative confidence. NEVER confirms a saving (engine gate does).
|
|
77
|
+
* Decision order is fail-safe: unavailable → review_required → observed_not_confirmed →
|
|
78
|
+
* eligible_for_engine_confirmation. Input means/delta come ONLY from provider-reported input tokens.
|
|
79
|
+
*/
|
|
80
|
+
export function summarizeInputCompactionAb(experiment) {
|
|
81
|
+
const control = experiment.runs.filter((r) => r.arm === "control");
|
|
82
|
+
const treatment = experiment.runs.filter((r) => r.arm === "treatment");
|
|
83
|
+
const reasons = [];
|
|
84
|
+
const controlIn = control.filter((r) => r.providerReported && typeof r.inputTokens === "number").map((r) => r.inputTokens);
|
|
85
|
+
const treatmentIn = treatment.filter((r) => r.providerReported && typeof r.inputTokens === "number").map((r) => r.inputTokens);
|
|
86
|
+
const controlOut = control.filter((r) => typeof r.outputTokens === "number").map((r) => r.outputTokens);
|
|
87
|
+
const treatmentOut = treatment.filter((r) => typeof r.outputTokens === "number").map((r) => r.outputTokens);
|
|
88
|
+
const bothProviderReported = control.length > 0 &&
|
|
89
|
+
treatment.length > 0 &&
|
|
90
|
+
control.every((r) => r.providerReported && typeof r.inputTokens === "number") &&
|
|
91
|
+
treatment.every((r) => r.providerReported && typeof r.inputTokens === "number");
|
|
92
|
+
const inputTokensBefore = meanOrNull(controlIn);
|
|
93
|
+
const inputTokensAfter = meanOrNull(treatmentIn);
|
|
94
|
+
const inputTokenDelta = inputTokensBefore !== null && inputTokensAfter !== null ? inputTokensBefore - inputTokensAfter : null;
|
|
95
|
+
const inputTokenReductionPct = inputTokenDelta !== null && inputTokensBefore !== null && inputTokensBefore > 0 ? (inputTokenDelta / inputTokensBefore) * 100 : null;
|
|
96
|
+
const outputTokensBefore = meanOrNull(controlOut);
|
|
97
|
+
const outputTokensAfter = meanOrNull(treatmentOut);
|
|
98
|
+
const outputTokenDelta = outputTokensBefore !== null && outputTokensAfter !== null ? outputTokensBefore - outputTokensAfter : null;
|
|
99
|
+
const cSrc = armSource(control);
|
|
100
|
+
const tSrc = armSource(treatment);
|
|
101
|
+
const tokenSource = bothProviderReported
|
|
102
|
+
? "provider-reported"
|
|
103
|
+
: cSrc === "local-estimate" || tSrc === "local-estimate"
|
|
104
|
+
? "local-estimate"
|
|
105
|
+
: cSrc === "mixed" || tSrc === "mixed"
|
|
106
|
+
? "mixed"
|
|
107
|
+
: "unavailable";
|
|
108
|
+
const contextPreservation = treatmentContextPreservation(treatment);
|
|
109
|
+
let confidence;
|
|
110
|
+
if (!bothProviderReported) {
|
|
111
|
+
confidence = "unavailable";
|
|
112
|
+
reasons.push("input savings unavailable: both arms must have provider-reported INPUT tokens (local-estimate / missing input can never produce an input-savings number).");
|
|
113
|
+
}
|
|
114
|
+
else if (contextPreservation !== "pass") {
|
|
115
|
+
confidence = "review_required";
|
|
116
|
+
reasons.push(`review required: treatment context-preservation status is "${contextPreservation}" — confirmation requires task-critical context preserved + source recoverability + commitment preservation + no material loss + treatment-solves-task (record each per treatment run).`);
|
|
117
|
+
}
|
|
118
|
+
else if (control.length < MIN_RUNS_PER_ARM || treatment.length < MIN_RUNS_PER_ARM) {
|
|
119
|
+
confidence = "observed_not_confirmed";
|
|
120
|
+
reasons.push(`observed, not confirmed: N ≥ ${MIN_RUNS_PER_ARM} per arm is required (have control=${control.length}, treatment=${treatment.length}).`);
|
|
121
|
+
}
|
|
122
|
+
else {
|
|
123
|
+
confidence = "eligible_for_engine_confirmation";
|
|
124
|
+
reasons.push("eligible for engine confirmation: provider-reported both arms, N ≥ 3 per arm, context-preservation passed (incl. task-solved). The CONFIRMED verdict is the engine gate (measured ±2·SE criterion + reduction).");
|
|
125
|
+
}
|
|
126
|
+
reasons.push("note: semantic/meaning preservation is not_evaluated (v1); output-token delta is observed-only, not the input claim.");
|
|
127
|
+
return {
|
|
128
|
+
experimentId: experiment.experimentId,
|
|
129
|
+
nControl: control.length,
|
|
130
|
+
nTreatment: treatment.length,
|
|
131
|
+
inputTokensBefore,
|
|
132
|
+
inputTokensAfter,
|
|
133
|
+
inputTokenDelta,
|
|
134
|
+
inputTokenReductionPct,
|
|
135
|
+
outputTokensBefore,
|
|
136
|
+
outputTokensAfter,
|
|
137
|
+
outputTokenDelta,
|
|
138
|
+
tokenSource,
|
|
139
|
+
contextPreservation,
|
|
140
|
+
semanticEval: "not_evaluated",
|
|
141
|
+
confidence,
|
|
142
|
+
confirmedSavingsEligible: false,
|
|
143
|
+
reasons
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
/** Provider-reported INPUT-token arrays per arm — the only honest input to the engine confirmation gate. */
|
|
147
|
+
export function providerReportedInputArms(experiment) {
|
|
148
|
+
const control = experiment.runs.filter((r) => r.arm === "control");
|
|
149
|
+
const treatment = experiment.runs.filter((r) => r.arm === "treatment");
|
|
150
|
+
const controlInputTokens = control.filter((r) => r.providerReported && typeof r.inputTokens === "number").map((r) => r.inputTokens);
|
|
151
|
+
const treatmentInputTokens = treatment.filter((r) => r.providerReported && typeof r.inputTokens === "number").map((r) => r.inputTokens);
|
|
152
|
+
const bothProviderReported = control.length > 0 &&
|
|
153
|
+
treatment.length > 0 &&
|
|
154
|
+
controlInputTokens.length === control.length &&
|
|
155
|
+
treatmentInputTokens.length === treatment.length;
|
|
156
|
+
return { controlInputTokens, treatmentInputTokens, bothProviderReported };
|
|
157
|
+
}
|
|
158
|
+
//# sourceMappingURL=input-compaction-ab.js.map
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import type { CrossSurfaceEvent } from "./cross-surface-event.js";
|
|
2
|
+
import type { RunFlowTokenReport } from "./run-flow-report.js";
|
|
3
|
+
/** Default local accumulation directory (sibling of `.compaction/runs`; flat, one file per run). */
|
|
4
|
+
export declare const DEFAULT_RUN_RECORDS_DIRECTORY = ".compaction/run-records";
|
|
5
|
+
/** File name of the per-run copy written next to the run's other artifacts. */
|
|
6
|
+
export declare const RUN_TOKEN_RECORD_FILENAME = "run-token-record.json";
|
|
7
|
+
/** Contract copy (docs/ops/app-source-status-contract.md) — tier labels, verbatim. */
|
|
8
|
+
export declare const TIER_LIVE_LABEL = "live (provider-reported)";
|
|
9
|
+
export declare const TIER_ESTIMATED_LABEL = "estimated (local-estimate)";
|
|
10
|
+
/** Contract copy — the literal output-savings line every tool renders in v1 (no tool has passed the gate). */
|
|
11
|
+
export declare const OUTPUT_SAVINGS_UNAVAILABLE_LINE = "output-token savings: unavailable \u2014 needs a provider-reported A/B that passes the measured criterion + the short-but-sufficient eval.";
|
|
12
|
+
/**
|
|
13
|
+
* One persisted local run record. `token_report` is the EXISTING `RunFlowTokenReport` shape, embedded
|
|
14
|
+
* unchanged (no schema migration — additive file, reused record shape).
|
|
15
|
+
*/
|
|
16
|
+
export interface LocalRunTokenRecord {
|
|
17
|
+
record_version: 1;
|
|
18
|
+
run_id: string;
|
|
19
|
+
recorded_at: string;
|
|
20
|
+
token_report: RunFlowTokenReport;
|
|
21
|
+
/**
|
|
22
|
+
* ADDITIVE, OPTIONAL cross-surface event per `docs/ops/cross-surface-evidence-contract.md`
|
|
23
|
+
* (typed in `src/core/cross-surface-event.ts`). `run cursor` populates it (the first writer —
|
|
24
|
+
* D3 alignment via `buildCursorRunCrossSurfaceEvent`); other tools remain contract §7 named
|
|
25
|
+
* follow-ups. When present it rides through write/read/rollup untouched. Validation of this
|
|
26
|
+
* field is the REPORT-ONLY `validateCrossSurfaceEvent` — an imperfect event here never causes
|
|
27
|
+
* the record to be skipped (record-level validation is unchanged).
|
|
28
|
+
*/
|
|
29
|
+
cross_surface_event?: CrossSurfaceEvent;
|
|
30
|
+
}
|
|
31
|
+
export declare function buildLocalRunTokenRecord(params: {
|
|
32
|
+
runId: string;
|
|
33
|
+
tokenReport: RunFlowTokenReport;
|
|
34
|
+
recordedAt?: string;
|
|
35
|
+
/** Optional additive cross-surface event (see `LocalRunTokenRecord.cross_surface_event`). */
|
|
36
|
+
crossSurfaceEvent?: CrossSurfaceEvent;
|
|
37
|
+
}): LocalRunTokenRecord;
|
|
38
|
+
/** Validate a parsed record. Invalid records are SKIPPED with a reason — never guessed at. */
|
|
39
|
+
export declare function validateLocalRunTokenRecord(value: unknown): {
|
|
40
|
+
record?: LocalRunTokenRecord;
|
|
41
|
+
reason?: string;
|
|
42
|
+
};
|
|
43
|
+
export interface WrittenRunRecordPaths {
|
|
44
|
+
/** The copy next to the run's other artifacts (self-contained artifact set). */
|
|
45
|
+
artifactPath: string;
|
|
46
|
+
/** The accumulated copy `compaction summary` rolls up. */
|
|
47
|
+
accumulatedPath: string;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Persist the record LOCALLY, twice: once next to the run's artifacts (`run-token-record.json` in the
|
|
51
|
+
* run's output directory) and once in the flat accumulation directory so `summary` can roll runs up
|
|
52
|
+
* without scanning arbitrary --out locations. Local file writes only — never a network call.
|
|
53
|
+
*/
|
|
54
|
+
export declare function writeLocalRunTokenRecord(record: LocalRunTokenRecord, artifactDirectory: string, recordsDirectory?: string): Promise<WrittenRunRecordPaths>;
|
|
55
|
+
export interface SkippedRunRecordFile {
|
|
56
|
+
path: string;
|
|
57
|
+
reason: string;
|
|
58
|
+
}
|
|
59
|
+
export interface LoadedRunRecord {
|
|
60
|
+
path: string;
|
|
61
|
+
record: LocalRunTokenRecord;
|
|
62
|
+
}
|
|
63
|
+
/** Read all accumulated records. A missing directory means "no records yet" (empty, not an error). */
|
|
64
|
+
export declare function readLocalRunTokenRecords(recordsDirectory?: string): Promise<{
|
|
65
|
+
records: LoadedRunRecord[];
|
|
66
|
+
skipped: SkippedRunRecordFile[];
|
|
67
|
+
}>;
|
|
68
|
+
/** Per-axis tier buckets (mirrors the contract tiers). Tokens are summed ONLY within their own tier. */
|
|
69
|
+
export interface PerToolAxisRollup {
|
|
70
|
+
provider_reported: {
|
|
71
|
+
runs: number;
|
|
72
|
+
tokens: number;
|
|
73
|
+
};
|
|
74
|
+
local_estimate: {
|
|
75
|
+
runs: number;
|
|
76
|
+
tokens: number;
|
|
77
|
+
};
|
|
78
|
+
/** Runs where this axis could not be counted honestly — with the honest reasons (never a 0). */
|
|
79
|
+
unavailable: {
|
|
80
|
+
runs: number;
|
|
81
|
+
reasons: string[];
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
export interface PerToolRunRecordRollup {
|
|
85
|
+
tool: string;
|
|
86
|
+
runs: number;
|
|
87
|
+
input: PerToolAxisRollup;
|
|
88
|
+
output: PerToolAxisRollup;
|
|
89
|
+
/** LITERAL contract value: no tool has passed the measured + eval-confirmed output-shaping gate. */
|
|
90
|
+
output_savings: "unavailable";
|
|
91
|
+
/** Distinct honest notes carried from the runs' capture limitations (content-free, capped). */
|
|
92
|
+
notes: string[];
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Roll accumulated records up PER TOOL (PURE). Counts + sources only — no cost, no savings math, no
|
|
96
|
+
* content. Tools are sorted for deterministic output. `output_savings` is always the literal
|
|
97
|
+
* "unavailable" (contract v1: no tool has measured + eval-confirmed output-shaping evidence).
|
|
98
|
+
*/
|
|
99
|
+
export declare function summarizePerToolRunRecords(records: LocalRunTokenRecord[]): PerToolRunRecordRollup[];
|
|
100
|
+
/** Header stating exactly what this section is (and is not) — printed above the per-tool lines. */
|
|
101
|
+
export declare const PER_TOOL_RECORDS_HEADER: string;
|
|
102
|
+
/** The honest empty state — never fabricated rows for tools that have no records. */
|
|
103
|
+
export declare const PER_TOOL_RECORDS_EMPTY_LINE = "- none yet \u2014 `compaction run \u2026` / `run codex` / `run cursor` accumulate per-run token_source records locally";
|
|
104
|
+
/**
|
|
105
|
+
* Render the per-tool rollup as indented list lines (shared by the console summary and summary.md).
|
|
106
|
+
* Copy mirrors the `/app` source-status contract: live (provider-reported) / estimated (local-estimate) /
|
|
107
|
+
* unavailable — <reason>; plus the literal output-savings unavailable line for EVERY tool.
|
|
108
|
+
*/
|
|
109
|
+
export declare function formatPerToolRunRecordLines(rollups: PerToolRunRecordRollup[]): string[];
|