@compaction/cli 0.1.4 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -1
- package/dist/cli/commands/compact.js +15 -6
- package/dist/cli/commands/context.d.ts +2 -0
- package/dist/cli/commands/context.js +130 -0
- package/dist/cli/commands/import.js +3 -1
- package/dist/cli/commands/run.js +13 -6
- package/dist/cli/index.js +4 -1
- package/dist/core/context-store-eval-cases.d.ts +6 -0
- package/dist/core/context-store-eval-cases.js +331 -0
- package/dist/core/context-store-eval.d.ts +140 -0
- package/dist/core/context-store-eval.js +138 -0
- package/dist/core/context-store-fs.d.ts +73 -0
- package/dist/core/context-store-fs.js +157 -0
- package/dist/core/context-store-sufficiency.d.ts +98 -0
- package/dist/core/context-store-sufficiency.js +135 -0
- package/dist/core/context-store.d.ts +155 -0
- package/dist/core/context-store.js +228 -0
- package/dist/core/report-generator.d.ts +19 -1
- package/dist/core/report-generator.js +51 -0
- package/dist/core/run-aggregator.d.ts +45 -0
- package/dist/core/run-aggregator.js +64 -0
- package/dist/core/trace-adapters.d.ts +8 -0
- package/dist/core/trace-adapters.js +4 -0
- package/dist/core/types.d.ts +33 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -24,6 +24,14 @@ Everything below runs on your machine and writes local files only:
|
|
|
24
24
|
billing-confirmed**.
|
|
25
25
|
- **Inspect spend and roll up runs.** `spend`, `summary`, and `aggregate` summarize where context
|
|
26
26
|
spend came from and roll up local runs — local files only, no hosted dashboard.
|
|
27
|
+
- **Build a local context store and retrieve from it.** `context add` accumulates source-pointed
|
|
28
|
+
items from a captured/imported trace into a local, size-capped store; `context get`
|
|
29
|
+
deterministically retrieves and assembles the active context for a query under a token budget,
|
|
30
|
+
printing it **with source pointers** plus local retrieval **diagnostics** (what was
|
|
31
|
+
considered / included / dropped, the recoverable fraction, latency). Local-only and
|
|
32
|
+
deterministic — **no model, no embeddings, no network**. It reports **diagnostics only** (no
|
|
33
|
+
quality or sufficiency verdict on your data) and makes **no savings claim** (see `spend` /
|
|
34
|
+
`summary` for cost).
|
|
27
35
|
- **Write local reports.** `analyze` writes local JSON + Markdown reports under an
|
|
28
36
|
operator-chosen output directory. Local files only.
|
|
29
37
|
- **Create redacted, content-free share bundles.** `feedback --redact` writes a redacted,
|
|
@@ -66,7 +74,7 @@ npx @compaction/cli --help # no install at all
|
|
|
66
74
|
```
|
|
67
75
|
|
|
68
76
|
The installed binary is `compaction`. After a global install, sanity-check with
|
|
69
|
-
`compaction --version` (→ `0.
|
|
77
|
+
`compaction --version` (→ `0.2.0`) and `compaction --help`.
|
|
70
78
|
|
|
71
79
|
### curl one-liner
|
|
72
80
|
|
|
@@ -133,6 +141,8 @@ compaction capture provider-usage --endpoint <url> [--credential-env-var <NAME>]
|
|
|
133
141
|
compaction import <trace-file> --source <source> --out <dir>
|
|
134
142
|
compaction import --list-sources
|
|
135
143
|
compaction analyze <trace-file>
|
|
144
|
+
compaction context add <trace-file> # build a local context store from a captured/imported trace
|
|
145
|
+
compaction context get "<query>" # retrieve + assemble the active context (with source pointers)
|
|
136
146
|
compaction spend <trace-or-optimization-path> # context-spend attribution
|
|
137
147
|
compaction summary # roll up local run reports
|
|
138
148
|
compaction aggregate # per-session / cross-session local rollups
|
|
@@ -2,7 +2,7 @@ import path from "node:path";
|
|
|
2
2
|
import chalk from "chalk";
|
|
3
3
|
import { Option } from "commander";
|
|
4
4
|
import { writeJsonArtifact, writeTextArtifact } from "../../core/artifact-writer.js";
|
|
5
|
-
import { withTraceFingerprint } from "../../core/report-generator.js";
|
|
5
|
+
import { formatCompactionMarkdownReport, withSavingsEvidence, withTraceFingerprint } from "../../core/report-generator.js";
|
|
6
6
|
import { describeTokenAccounting } from "../../core/token-accounting.js";
|
|
7
7
|
import { parseTraceFile } from "../../core/trace-parser.js";
|
|
8
8
|
import { buildRunLabelsFile, hasAnyLabel } from "../../core/run-labels.js";
|
|
@@ -124,11 +124,20 @@ export function registerCompactCommand(program) {
|
|
|
124
124
|
// hand-assembled bundle — the eval consumes the identical PolicyMiddlewareResult.
|
|
125
125
|
const policyResult = applyCompactionPolicy({ trace, compactSkillInjections: approveSkillInjectionPolicy });
|
|
126
126
|
const artifacts = await writeCompactionArtifacts(trace, options.out, runId, policyResult, undefined, { reviewerType, reviewSummaryPresent, approveSkillInjectionPolicy });
|
|
127
|
-
// Attribution/integrity (V0.2 savings-evidence): re-persist
|
|
128
|
-
// content-addressed trace fingerprint so `aggregate`/`summary` can
|
|
129
|
-
// verifiable run and de-duplicate
|
|
130
|
-
//
|
|
131
|
-
|
|
127
|
+
// Attribution/integrity + per-run evidence label (V0.2 savings-evidence): re-persist
|
|
128
|
+
// report.json with (1) the content-addressed trace fingerprint so `aggregate`/`summary` can
|
|
129
|
+
// attribute each saving to a verifiable run and de-duplicate a run rolled up twice, and
|
|
130
|
+
// (2) the composed `savings_evidence` record stating the per-run label explicitly. The
|
|
131
|
+
// compact path's figures are local estimates (chars/4) — `local_estimate` rung 1; never
|
|
132
|
+
// billing-confirmed, never semantic. Additive, backward-compatible; changes no existing field.
|
|
133
|
+
const evidenceReport = withSavingsEvidence(withTraceFingerprint(artifacts.report, trace), {
|
|
134
|
+
costSource: "local_estimate"
|
|
135
|
+
});
|
|
136
|
+
await writeJsonArtifact(options.out, "report.json", evidenceReport);
|
|
137
|
+
// Re-render report.md from the ENRICHED report so the per-run savings_evidence record
|
|
138
|
+
// appears in the human-readable artifact too (the engine rendered report.md from the base
|
|
139
|
+
// report before the CLI added the public-layer fingerprint/evidence enrichment).
|
|
140
|
+
await writeTextArtifact(options.out, "report.md", formatCompactionMarkdownReport(evidenceReport, artifacts.policy));
|
|
132
141
|
console.log(chalk.cyan("compaction compact"));
|
|
133
142
|
console.log(artifacts.consoleReport);
|
|
134
143
|
// Operator-visible only when the approval gesture was given; default output is unchanged.
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
2
|
+
import { resolve } from "node:path";
|
|
3
|
+
import chalk from "chalk";
|
|
4
|
+
import { assembleContext, contextItemsFromTrace, retrieveContextItems } from "../../core/context-store.js";
|
|
5
|
+
import { appendContextItems, loadContextStore } from "../../core/context-store-fs.js";
|
|
6
|
+
/**
|
|
7
|
+
* V0.4 `compaction context` — a LOCAL memory + retrieval surface over the user's own captured
|
|
8
|
+
* traces (design: docs/design/v0.4-context-command.md, accepted 2026-06-24).
|
|
9
|
+
*
|
|
10
|
+
* HARD RAILS (load-bearing): local-only — no network, no model, no embeddings, no engine. It
|
|
11
|
+
* makes NO savings/cost claim (cost lives in `spend`/`summary`) and NO quality/recall/correctness
|
|
12
|
+
* verdict on user data (the four-axis sufficiency harness stays internal — there is no ground
|
|
13
|
+
* truth for an arbitrary user query). `get` prints local retrieval DIAGNOSTICS only.
|
|
14
|
+
*/
|
|
15
|
+
const DEFAULT_STORE_DIR = ".compaction/context-store";
|
|
16
|
+
function readTrace(path) {
|
|
17
|
+
if (!existsSync(path)) {
|
|
18
|
+
throw new Error(`File not found: ${path}`);
|
|
19
|
+
}
|
|
20
|
+
let parsed;
|
|
21
|
+
try {
|
|
22
|
+
parsed = JSON.parse(readFileSync(path, "utf8"));
|
|
23
|
+
}
|
|
24
|
+
catch {
|
|
25
|
+
throw new Error(`Not valid JSON: ${path}`);
|
|
26
|
+
}
|
|
27
|
+
const trace = parsed;
|
|
28
|
+
if (!trace || typeof trace.id !== "string" || !Array.isArray(trace.messages)) {
|
|
29
|
+
throw new Error("Not a normalized AgentTrace (expected { id, messages: [...] }). " +
|
|
30
|
+
"Produce one with `compaction capture` or `compaction import`.");
|
|
31
|
+
}
|
|
32
|
+
for (const message of trace.messages) {
|
|
33
|
+
const m = message;
|
|
34
|
+
if (!m || typeof m.id !== "string" || typeof m.content !== "string") {
|
|
35
|
+
throw new Error("Malformed trace: every message needs a string `id` and `content`. " +
|
|
36
|
+
"Produce a trace with `compaction capture` or `compaction import`.");
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
return parsed;
|
|
40
|
+
}
|
|
41
|
+
function collect(value, previous) {
|
|
42
|
+
return [...previous, value];
|
|
43
|
+
}
|
|
44
|
+
export function registerContextCommand(program) {
|
|
45
|
+
const context = program
|
|
46
|
+
.command("context")
|
|
47
|
+
.description("Local memory: build a durable context store from your own captured/imported traces and " +
|
|
48
|
+
"retrieve the active context for a next step (local-only; no model, no network, no savings claim).");
|
|
49
|
+
context
|
|
50
|
+
.command("add")
|
|
51
|
+
.argument("<artifact>", "Path to a local normalized AgentTrace JSON (from `compaction capture`/`import`)")
|
|
52
|
+
.option("--store-dir <dir>", "Context store directory (local, gitignored)", DEFAULT_STORE_DIR)
|
|
53
|
+
.description("Append context items derived from a local trace into the local context store (size-cap enforced).")
|
|
54
|
+
.action(async (artifact, options) => {
|
|
55
|
+
const storeDir = resolve(options.storeDir ?? DEFAULT_STORE_DIR);
|
|
56
|
+
const trace = readTrace(resolve(artifact));
|
|
57
|
+
const items = contextItemsFromTrace(trace, { created_at: new Date().toISOString() });
|
|
58
|
+
const result = await appendContextItems(storeDir, items);
|
|
59
|
+
console.log(chalk.cyan("compaction context add"));
|
|
60
|
+
console.log(`Source: ${artifact}`);
|
|
61
|
+
console.log(`Store: ${storeDir}`);
|
|
62
|
+
console.log(`Items added: ${result.appended.length} skipped (duplicate): ${result.skipped.length} ` +
|
|
63
|
+
`evicted (size cap): ${result.evicted.length}`);
|
|
64
|
+
console.log(`Store now holds: ${result.total} items.`);
|
|
65
|
+
console.log("Local-only: nothing was uploaded. This makes no savings claim (see `compaction spend`/`summary` for cost).");
|
|
66
|
+
console.log(chalk.bold('Next: compaction context get "<your query>"'));
|
|
67
|
+
});
|
|
68
|
+
context
|
|
69
|
+
.command("get")
|
|
70
|
+
.argument("<query>", "Free-text description of the current step")
|
|
71
|
+
.option("--store-dir <dir>", "Context store directory (local, gitignored)", DEFAULT_STORE_DIR)
|
|
72
|
+
.option("--budget <tokens>", "Token budget for the assembled context (local estimate)", "4000")
|
|
73
|
+
.option("--source <pointer>", "Required source pointer for coverage diagnostics (repeatable)", collect, [])
|
|
74
|
+
.option("--limit <n>", "Max candidate items to consider after ranking")
|
|
75
|
+
.description("Retrieve + assemble the active context for a query from the local store (deterministic, local). " +
|
|
76
|
+
"Prints the assembled context WITH source pointers + local retrieval diagnostics — not a quality score.")
|
|
77
|
+
.action(async (query, options) => {
|
|
78
|
+
const storeDir = resolve(options.storeDir ?? DEFAULT_STORE_DIR);
|
|
79
|
+
const budget = Number.parseInt(options.budget ?? "4000", 10);
|
|
80
|
+
if (Number.isNaN(budget) || budget < 0) {
|
|
81
|
+
console.error("--budget must be a non-negative integer number of tokens.");
|
|
82
|
+
process.exitCode = 1;
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
85
|
+
let limit;
|
|
86
|
+
if (options.limit !== undefined) {
|
|
87
|
+
limit = Number.parseInt(options.limit, 10);
|
|
88
|
+
if (Number.isNaN(limit) || limit < 0) {
|
|
89
|
+
console.error("--limit must be a non-negative integer.");
|
|
90
|
+
process.exitCode = 1;
|
|
91
|
+
return;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
const items = await loadContextStore(storeDir);
|
|
95
|
+
const retrievalOptions = limit !== undefined ? { limit } : {};
|
|
96
|
+
const start = performance.now();
|
|
97
|
+
const ranked = retrieveContextItems(items, { text: query, source_pointers: options.source && options.source.length > 0 ? options.source : undefined }, retrievalOptions);
|
|
98
|
+
const assembled = assembleContext(ranked, { tokenBudget: budget });
|
|
99
|
+
const latencyMs = performance.now() - start;
|
|
100
|
+
console.log(chalk.cyan("compaction context get"));
|
|
101
|
+
console.log(`Query: ${JSON.stringify(query)}`);
|
|
102
|
+
console.log(`Store: ${storeDir} (${items.length} items)`);
|
|
103
|
+
if (assembled.included.length === 0) {
|
|
104
|
+
console.log("Assembled context: (no matching context in the store for this query)");
|
|
105
|
+
}
|
|
106
|
+
else {
|
|
107
|
+
console.log(`Assembled context (~${assembled.estimated_tokens} tokens, local estimate; budget ${budget}):`);
|
|
108
|
+
for (const item of assembled.included) {
|
|
109
|
+
const pointer = item.source_pointer ?? "(no source pointer)";
|
|
110
|
+
const oneLine = item.content.replace(/\s+/g, " ");
|
|
111
|
+
console.log(` [${pointer}] ${oneLine}`);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
const recoverable = assembled.included.filter((i) => i.recoverability === true && i.source_pointer).length;
|
|
115
|
+
console.log("Diagnostics (local retrieval, not a quality score):");
|
|
116
|
+
console.log(` considered: ${ranked.length} included: ${assembled.included.length} dropped: ${assembled.dropped.length}`);
|
|
117
|
+
console.log(` recoverable (has source pointer): ${recoverable}/${assembled.included.length}`);
|
|
118
|
+
if (options.source && options.source.length > 0) {
|
|
119
|
+
const includedPointers = new Set(assembled.included.map((i) => i.source_pointer).filter((p) => Boolean(p)));
|
|
120
|
+
const covered = options.source.filter((s) => includedPointers.has(s)).length;
|
|
121
|
+
console.log(` source coverage: ${covered}/${options.source.length}`);
|
|
122
|
+
}
|
|
123
|
+
else {
|
|
124
|
+
console.log(" source coverage: n/a (no --source given)");
|
|
125
|
+
}
|
|
126
|
+
console.log(` latency: ${latencyMs.toFixed(1)} ms`);
|
|
127
|
+
console.log("No model, no network. Local memory/retrieval view — it makes no savings claim and no quality/correctness claim.");
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
//# sourceMappingURL=context.js.map
|
|
@@ -5,8 +5,10 @@ function printSupportedSources() {
|
|
|
5
5
|
console.log(chalk.cyan("compaction import sources"));
|
|
6
6
|
for (const adapter of listTraceAdapters()) {
|
|
7
7
|
console.log(`- ${adapter.id}: ${adapter.displayName}`);
|
|
8
|
+
console.log(` readiness: ${adapter.readiness}`);
|
|
8
9
|
}
|
|
9
|
-
console.log("
|
|
10
|
+
console.log("All sources are local file import (no live integration). Readiness above is an honest integration-readiness " +
|
|
11
|
+
"label (AGENTS.md Level 0–6) + validation status — not a stronger integration claim than the evidence supports.");
|
|
10
12
|
}
|
|
11
13
|
export function registerImportCommand(program) {
|
|
12
14
|
program
|
package/dist/cli/commands/run.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import chalk from "chalk";
|
|
2
2
|
import { parseRunCommand, executeLocalCommand, persistLocalCommandRun } from "../../core/command-runner.js";
|
|
3
|
-
import { writeJsonArtifact } from "../../core/artifact-writer.js";
|
|
4
|
-
import { withTraceFingerprint } from "../../core/report-generator.js";
|
|
3
|
+
import { writeJsonArtifact, writeTextArtifact } from "../../core/artifact-writer.js";
|
|
4
|
+
import { formatCompactionMarkdownReport, withSavingsEvidence, withTraceFingerprint } from "../../core/report-generator.js";
|
|
5
5
|
import { localCommandRunToAgentTrace } from "../../core/run-trace-converter.js";
|
|
6
6
|
import { runEngineCommand } from "../engine-degrade.js";
|
|
7
7
|
export function registerRunCommand(program) {
|
|
@@ -24,10 +24,17 @@ export function registerRunCommand(program) {
|
|
|
24
24
|
const trace = localCommandRunToAgentTrace(run);
|
|
25
25
|
const tracePath = await writeJsonArtifact(persistedRun.outputDirectory, "trace.json", trace);
|
|
26
26
|
const artifacts = await writeCompactionArtifacts(trace, persistedRun.outputDirectory, run.runId);
|
|
27
|
-
// Attribution/integrity (V0.2 savings-evidence): re-persist
|
|
28
|
-
//
|
|
29
|
-
//
|
|
30
|
-
|
|
27
|
+
// Attribution/integrity + per-run evidence label (V0.2 savings-evidence): re-persist
|
|
28
|
+
// report.json with the trace fingerprint (attribution/dedup) and the composed
|
|
29
|
+
// `savings_evidence` record (per-run label). `run` figures are local estimates (chars/4) —
|
|
30
|
+
// `local_estimate` rung 1. Additive; no existing report field/label changes.
|
|
31
|
+
const evidenceReport = withSavingsEvidence(withTraceFingerprint(artifacts.report, trace), {
|
|
32
|
+
costSource: "local_estimate"
|
|
33
|
+
});
|
|
34
|
+
await writeJsonArtifact(persistedRun.outputDirectory, "report.json", evidenceReport);
|
|
35
|
+
// Re-render report.md from the ENRICHED report so the per-run savings_evidence record
|
|
36
|
+
// appears in the human-readable artifact too (engine rendered it from the base report).
|
|
37
|
+
await writeTextArtifact(persistedRun.outputDirectory, "report.md", formatCompactionMarkdownReport(evidenceReport, artifacts.policy));
|
|
31
38
|
console.log(`Exit code: ${run.exitCode ?? "none"}`);
|
|
32
39
|
console.log(`Signal: ${run.signal ?? "none"}`);
|
|
33
40
|
console.log(`Duration: ${run.durationMs} ms`);
|
package/dist/cli/index.js
CHANGED
|
@@ -11,6 +11,7 @@ import { registerBillingDeltaCommand } from "./commands/billing-delta.js";
|
|
|
11
11
|
import { registerInitCommand } from "./commands/init.js";
|
|
12
12
|
import { registerCaptureCommand } from "./commands/capture.js";
|
|
13
13
|
import { registerCompactCommand } from "./commands/compact.js";
|
|
14
|
+
import { registerContextCommand } from "./commands/context.js";
|
|
14
15
|
import { registerEvalCommand } from "./commands/eval.js";
|
|
15
16
|
import { registerExportSessionSeedCommand } from "./commands/export-session-seed.js";
|
|
16
17
|
import { registerFeedbackCommand } from "./commands/feedback.js";
|
|
@@ -27,7 +28,7 @@ const program = new Command();
|
|
|
27
28
|
program
|
|
28
29
|
.name("compaction")
|
|
29
30
|
.description("CLI-first trace compaction tools for AI agent workflows.")
|
|
30
|
-
.version("0.
|
|
31
|
+
.version("0.2.0");
|
|
31
32
|
registerAnalyzeCommand(program);
|
|
32
33
|
registerAdapterCommand(program);
|
|
33
34
|
registerAggregateCommand(program);
|
|
@@ -39,6 +40,7 @@ registerBillingDeltaCommand(program);
|
|
|
39
40
|
registerInitCommand(program);
|
|
40
41
|
registerCaptureCommand(program);
|
|
41
42
|
registerCompactCommand(program);
|
|
43
|
+
registerContextCommand(program);
|
|
42
44
|
registerEvalCommand(program);
|
|
43
45
|
registerExportSessionSeedCommand(program);
|
|
44
46
|
registerFeedbackCommand(program);
|
|
@@ -62,6 +64,7 @@ const PUBLIC_COMMANDS = new Set([
|
|
|
62
64
|
"capture",
|
|
63
65
|
"import",
|
|
64
66
|
"analyze",
|
|
67
|
+
"context",
|
|
65
68
|
"spend",
|
|
66
69
|
"summary",
|
|
67
70
|
"aggregate",
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { ContextStoreEvalFixture } from "./context-store-eval.js";
|
|
2
|
+
export declare const CONTEXT_STORE_EVAL_FIXTURES: ContextStoreEvalFixture[];
|
|
3
|
+
export declare const CONTEXT_STORE_EVAL_RICHER_FIXTURES: ContextStoreEvalFixture[];
|
|
4
|
+
export declare const CONTEXT_STORE_EVAL_EDGE_FIXTURES: ContextStoreEvalFixture[];
|
|
5
|
+
/** Every fixture (healthy + richer + edge), for the internal sufficiency report. */
|
|
6
|
+
export declare const CONTEXT_STORE_ALL_EVAL_FIXTURES: ContextStoreEvalFixture[];
|
|
@@ -0,0 +1,331 @@
|
|
|
1
|
+
import { makeContextItem } from "./context-store.js";
|
|
2
|
+
/** Label a fixture `sufficiency` — gated against the sufficiency thresholds. */
|
|
3
|
+
function sufficiency(fixture) {
|
|
4
|
+
return { ...fixture, role: "sufficiency" };
|
|
5
|
+
}
|
|
6
|
+
/** Label a fixture `adversarial` — NOT gated; a regression anchor with an expected outcome. */
|
|
7
|
+
function adversarial(fixture, expected) {
|
|
8
|
+
return { ...fixture, role: "adversarial", expected };
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Canonical fixtures for the V0.4 context-store eval harness (rung-1).
|
|
12
|
+
*
|
|
13
|
+
* These are SYNTHETIC fixtures — hand-authored stores + queries with known task-critical
|
|
14
|
+
* items — used to MEASURE deterministic retrieval/assembly behavior. They are not captured
|
|
15
|
+
* sessions and carry no real-world claim. `created_at` values are fixed so the suite is
|
|
16
|
+
* fully deterministic (apart from observed latency).
|
|
17
|
+
*/
|
|
18
|
+
const T0 = "2026-06-20T09:00:00.000Z";
|
|
19
|
+
const T1 = "2026-06-21T09:00:00.000Z";
|
|
20
|
+
const T2 = "2026-06-22T09:00:00.000Z";
|
|
21
|
+
function fact(content, source_pointer, created_at) {
|
|
22
|
+
return makeContextItem({
|
|
23
|
+
content,
|
|
24
|
+
source_pointer,
|
|
25
|
+
source_trace_id: source_pointer.split(" ")[0]?.replace("trace=", ""),
|
|
26
|
+
recoverability: true,
|
|
27
|
+
created_at
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
/** Fixture A: the task-critical item is lexically obvious and fits the budget. */
|
|
31
|
+
const lexicalRecall = {
|
|
32
|
+
name: "lexical-recall",
|
|
33
|
+
description: "Task-critical item is lexically related to the query and fits the budget.",
|
|
34
|
+
store: [
|
|
35
|
+
fact("the auth token must be refreshed before every batch upload", "trace=A message=m1", T0),
|
|
36
|
+
fact("the user prefers a restrained blue palette in the TUI", "trace=A message=m2", T1),
|
|
37
|
+
fact("retry the upload with exponential backoff on 503", "trace=A message=m3", T2)
|
|
38
|
+
],
|
|
39
|
+
query: { text: "how do we handle the auth token before an upload batch" },
|
|
40
|
+
tokenBudget: 1000,
|
|
41
|
+
goldItems: ["the auth token must be refreshed before every batch upload"],
|
|
42
|
+
requiredSourcePointers: ["trace=A message=m1"],
|
|
43
|
+
continuationRequirements: ["auth token", "batch upload"]
|
|
44
|
+
};
|
|
45
|
+
/**
|
|
46
|
+
* Fixture B: the source-pointer boost is the DECIDING factor. The task-critical item
|
|
47
|
+
* (trace B) has weak lexical overlap with the query, while a non-gold distractor (trace C)
|
|
48
|
+
* has high lexical overlap. The budget fits only one item — so without the trace-id boost
|
|
49
|
+
* the distractor would win the slot and the task-critical item would be lost. The boost is
|
|
50
|
+
* what makes the right item survive.
|
|
51
|
+
*/
|
|
52
|
+
const goldDecision = "decision: keep the public surface at eight commands for rung-1";
|
|
53
|
+
const sourcePointerRecall = {
|
|
54
|
+
name: "source-pointer-recall",
|
|
55
|
+
description: "Under budget pressure, the exact trace-id hint is the deciding factor that keeps a lexically-weak task-critical item.",
|
|
56
|
+
store: [
|
|
57
|
+
fact(goldDecision, "trace=B message=m1", T0),
|
|
58
|
+
// Distractor: high lexical overlap with the query, but NOT task-critical (trace C, unboosted).
|
|
59
|
+
fact("what did we decide about the command surface here today", "trace=C message=m1", T1)
|
|
60
|
+
],
|
|
61
|
+
query: { text: "what did we decide about the command surface", trace_ids: ["B"] },
|
|
62
|
+
// Budget fits exactly one item — forces the boost to decide which survives.
|
|
63
|
+
tokenBudget: makeContextItem({ content: goldDecision, created_at: T0 }).estimated_tokens,
|
|
64
|
+
goldItems: [goldDecision],
|
|
65
|
+
requiredSourcePointers: ["trace=B message=m1"],
|
|
66
|
+
continuationRequirements: ["eight commands"]
|
|
67
|
+
};
|
|
68
|
+
/** Fixture C: the budget forces a drop — the higher-ranked task-critical item must win. */
|
|
69
|
+
const budgetPressure = {
|
|
70
|
+
name: "budget-pressure",
|
|
71
|
+
description: "Under a tight budget the task-critical (higher-ranked) item must survive.",
|
|
72
|
+
store: [
|
|
73
|
+
fact("critical: the migration must run before the API deploy", "trace=D message=m1", T0),
|
|
74
|
+
fact("aside: someone mentioned the office coffee machine is broken", "trace=D message=m2", T1)
|
|
75
|
+
],
|
|
76
|
+
query: { text: "what must happen before the api deploy migration" },
|
|
77
|
+
// Budget large enough for one short item only.
|
|
78
|
+
tokenBudget: makeContextItem({ content: "critical: the migration must run before the API deploy", created_at: T0 })
|
|
79
|
+
.estimated_tokens,
|
|
80
|
+
goldItems: ["critical: the migration must run before the API deploy"],
|
|
81
|
+
requiredSourcePointers: ["trace=D message=m1"],
|
|
82
|
+
continuationRequirements: ["migration", "api deploy"]
|
|
83
|
+
};
|
|
84
|
+
export const CONTEXT_STORE_EVAL_FIXTURES = [
|
|
85
|
+
sufficiency(lexicalRecall),
|
|
86
|
+
sufficiency(sourcePointerRecall),
|
|
87
|
+
sufficiency(budgetPressure)
|
|
88
|
+
];
|
|
89
|
+
// ---------------------------------------------------------------------------
|
|
90
|
+
// Richer / adversarial fixtures (V0.4 rung-1, 2026-06-24).
|
|
91
|
+
//
|
|
92
|
+
// PURPOSE: make the eval harness HARDER TO GAME before any threshold is set. Unlike the
|
|
93
|
+
// healthy baseline above (every axis passes), several of these fixtures are designed so a
|
|
94
|
+
// SPECIFIC axis legitimately FAILS — proving the harness reports honest sub-1.0 metrics and
|
|
95
|
+
// cannot be satisfied by a degenerate retriever. Expected per-axis outcomes live next to the
|
|
96
|
+
// assertions in `tests/core/context-store-eval-richer.test.ts`. No thresholds, no claim.
|
|
97
|
+
// ---------------------------------------------------------------------------
|
|
98
|
+
/** Flexible item builder — source pointer / recoverability are optional (for the negative cases). */
|
|
99
|
+
function mk(content, opts = {}) {
|
|
100
|
+
return makeContextItem({
|
|
101
|
+
content,
|
|
102
|
+
source_pointer: opts.pointer,
|
|
103
|
+
source_trace_id: opts.trace ?? (opts.pointer ? opts.pointer.split(" ")[0]?.replace("trace=", "") : undefined),
|
|
104
|
+
recoverability: opts.recoverable ?? true,
|
|
105
|
+
created_at: opts.created_at ?? T0
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
function tokensOf(content) {
|
|
109
|
+
return makeContextItem({ content, created_at: T0 }).estimated_tokens;
|
|
110
|
+
}
|
|
111
|
+
/** R1 — task-critical item survives budget pressure against several genuinely-retrieved competitors. */
|
|
112
|
+
const budgetPressureMulti = {
|
|
113
|
+
name: "budget-pressure-multi",
|
|
114
|
+
description: "Gold outranks several genuinely-retrieved competitors and wins the only budget slot.",
|
|
115
|
+
store: [
|
|
116
|
+
mk("critical: rotate the signing key before the release cutover", { pointer: "trace=E message=m1" }),
|
|
117
|
+
mk("rotate the release plan before cutover today", { pointer: "trace=E message=m2", created_at: T1 }),
|
|
118
|
+
mk("the signing key was discussed", { pointer: "trace=E message=m3", created_at: T2 })
|
|
119
|
+
],
|
|
120
|
+
query: { text: "rotate the signing key before release cutover" },
|
|
121
|
+
tokenBudget: tokensOf("critical: rotate the signing key before the release cutover"),
|
|
122
|
+
goldItems: ["critical: rotate the signing key before the release cutover"],
|
|
123
|
+
requiredSourcePointers: ["trace=E message=m1"],
|
|
124
|
+
continuationRequirements: ["signing key", "release cutover"]
|
|
125
|
+
};
|
|
126
|
+
/** R2 — source coverage FAILS: a second required source is dropped under budget, gold still survives. */
|
|
127
|
+
const sourceCoverageMiss = {
|
|
128
|
+
name: "source-coverage-miss",
|
|
129
|
+
description: "Two sources are required but the budget fits only the gold item; the other required source is missing.",
|
|
130
|
+
store: [
|
|
131
|
+
mk("the api key rotation runbook lives in the wiki", { pointer: "trace=A message=m1" }),
|
|
132
|
+
mk("the staging database snapshot policy", { pointer: "trace=B message=m1", created_at: T1 })
|
|
133
|
+
],
|
|
134
|
+
query: { text: "where is the api key rotation runbook" },
|
|
135
|
+
tokenBudget: tokensOf("the api key rotation runbook lives in the wiki"),
|
|
136
|
+
goldItems: ["the api key rotation runbook lives in the wiki"],
|
|
137
|
+
requiredSourcePointers: ["trace=A message=m1", "trace=B message=m1"],
|
|
138
|
+
continuationRequirements: ["rotation runbook"]
|
|
139
|
+
};
|
|
140
|
+
/** R3 — continuation success FAILS: one required marker is absent from the assembled text. */
|
|
141
|
+
const continuationMarkerAbsent = {
|
|
142
|
+
name: "continuation-marker-absent",
|
|
143
|
+
description: "The gold item is assembled but a required continuation marker is not present in it.",
|
|
144
|
+
store: [mk("deploy step: run the smoke test then promote the build", { pointer: "trace=F message=m1" })],
|
|
145
|
+
query: { text: "what is the deploy step for promote" },
|
|
146
|
+
tokenBudget: 1000,
|
|
147
|
+
goldItems: ["deploy step: run the smoke test then promote the build"],
|
|
148
|
+
requiredSourcePointers: ["trace=F message=m1"],
|
|
149
|
+
// "smoke test" is present; "rollback plan" is absent -> continuation_success = 0.5.
|
|
150
|
+
continuationRequirements: ["smoke test", "rollback plan"]
|
|
151
|
+
};
|
|
152
|
+
// R4/R5 share a store: a high-lexical-overlap distractor (trace C) vs a lexically-weak gold (trace B).
|
|
153
|
+
const trapGold = "rollback procedure: restore the retained original artifact";
|
|
154
|
+
const trapDistractor = "which note covers the deploy step we discussed";
|
|
155
|
+
const lexicalTrapStore = [
|
|
156
|
+
mk(trapGold, { pointer: "trace=B message=m1" }),
|
|
157
|
+
mk(trapDistractor, { pointer: "trace=C message=m1", created_at: T1 })
|
|
158
|
+
];
|
|
159
|
+
/** R4 — lexical trap, source pointer SAVES it: the trace-id boost lifts the gold over the distractor. */
|
|
160
|
+
const lexicalTrapPointerSaves = {
|
|
161
|
+
name: "lexical-trap-pointer-saves",
|
|
162
|
+
description: "An irrelevant high-overlap distractor would win, but the trace-id hint lifts the gold into the slot.",
|
|
163
|
+
store: lexicalTrapStore,
|
|
164
|
+
query: { text: "which note covers the deploy rollback step", trace_ids: ["B"] },
|
|
165
|
+
tokenBudget: tokensOf(trapGold),
|
|
166
|
+
goldItems: [trapGold],
|
|
167
|
+
requiredSourcePointers: ["trace=B message=m1"],
|
|
168
|
+
continuationRequirements: ["retained original"]
|
|
169
|
+
};
|
|
170
|
+
/** R5 — lexical trap, NO pointer: the irrelevant distractor wins the slot, gold is lost (recall 0). */
|
|
171
|
+
const lexicalTrapNoPointer = {
|
|
172
|
+
name: "lexical-trap-no-pointer",
|
|
173
|
+
description: "Without a source-pointer hint the high-overlap distractor wins the slot and the gold is legitimately lost.",
|
|
174
|
+
store: lexicalTrapStore,
|
|
175
|
+
query: { text: "which note covers the deploy step" },
|
|
176
|
+
tokenBudget: tokensOf(trapDistractor),
|
|
177
|
+
goldItems: [trapGold],
|
|
178
|
+
requiredSourcePointers: ["trace=B message=m1"],
|
|
179
|
+
continuationRequirements: ["retained original"]
|
|
180
|
+
};
|
|
181
|
+
/** R6 — duplicate/redundant context must NOT inflate success: 3 byte-identical golds collapse to one. */
|
|
182
|
+
const goldCanary = "the canary must pass before full rollout";
|
|
183
|
+
const duplicateNoInflation = {
|
|
184
|
+
name: "duplicate-no-inflation",
|
|
185
|
+
description: "Three byte-identical copies of the gold plus one filler assemble to two distinct items, not four.",
|
|
186
|
+
store: [
|
|
187
|
+
mk(goldCanary, { pointer: "trace=G message=m1" }),
|
|
188
|
+
mk(goldCanary, { pointer: "trace=G message=m1", created_at: T1 }),
|
|
189
|
+
mk(goldCanary, { pointer: "trace=G message=m1", created_at: T2 }),
|
|
190
|
+
mk("full rollout schedule for the canary release", { pointer: "trace=H message=m1", created_at: T2 })
|
|
191
|
+
],
|
|
192
|
+
query: { text: "when does the canary pass before rollout" },
|
|
193
|
+
tokenBudget: 1000,
|
|
194
|
+
goldItems: [goldCanary],
|
|
195
|
+
requiredSourcePointers: ["trace=G message=m1"],
|
|
196
|
+
continuationRequirements: ["canary"]
|
|
197
|
+
};
|
|
198
|
+
/** R7 — recoverability DROPS when an assembled item lacks a source pointer. */
|
|
199
|
+
const recoverabilityMissingPointers = {
|
|
200
|
+
name: "recoverability-missing-pointers",
|
|
201
|
+
description: "Two task-critical items are assembled; one carries no source pointer, so recoverability is 0.5.",
|
|
202
|
+
store: [
|
|
203
|
+
mk("retain the original artifact for rollback", { pointer: "trace=A message=m1" }),
|
|
204
|
+
// No source pointer: flagged recoverable but cannot actually be recovered (no pointer to recover to).
|
|
205
|
+
mk("the diff summary should be reviewed before apply", { recoverable: true, created_at: T1 })
|
|
206
|
+
],
|
|
207
|
+
query: { text: "what to retain and review before apply rollback" },
|
|
208
|
+
tokenBudget: 1000,
|
|
209
|
+
goldItems: ["retain the original artifact for rollback", "the diff summary should be reviewed before apply"],
|
|
210
|
+
requiredSourcePointers: ["trace=A message=m1"],
|
|
211
|
+
continuationRequirements: ["retain", "review"]
|
|
212
|
+
};
|
|
213
|
+
export const CONTEXT_STORE_EVAL_RICHER_FIXTURES = [
|
|
214
|
+
sufficiency(budgetPressureMulti),
|
|
215
|
+
adversarial(sourceCoverageMiss, { recall: 1, source_coverage: 0.5, continuation_success: 1, recoverability: 1 }),
|
|
216
|
+
adversarial(continuationMarkerAbsent, { recall: 1, source_coverage: 1, continuation_success: 0.5, recoverability: 1 }),
|
|
217
|
+
sufficiency(lexicalTrapPointerSaves),
|
|
218
|
+
adversarial(lexicalTrapNoPointer, { recall: 0, source_coverage: 0, continuation_success: 0, recoverability: 1 }),
|
|
219
|
+
sufficiency(duplicateNoInflation),
|
|
220
|
+
adversarial(recoverabilityMissingPointers, { recall: 1, source_coverage: 1, continuation_success: 1, recoverability: 0.5 })
|
|
221
|
+
];
|
|
222
|
+
// ---------------------------------------------------------------------------
|
|
223
|
+
// Edge-class fixtures (V0.4 rung-1, 2026-06-24) — the missing classes the accepted
|
|
224
|
+
// threshold note (docs/design/v0.4-context-store-thresholds.md) asked for BEFORE any
|
|
225
|
+
// threshold is implemented:
|
|
226
|
+
// - multi-gold -> makes recall/coverage FRACTIONAL (today they are binary), so a
|
|
227
|
+
// future bar can be calibrated against real partials.
|
|
228
|
+
// - empty store -> makes zero-context behavior explicit and honest.
|
|
229
|
+
// - budget / limit -> recall + coverage degrade for the RIGHT, distinguishable reason
|
|
230
|
+
// (token budget vs retrieval `limit`).
|
|
231
|
+
// - adversarial anchor -> a low score is the CORRECT behavior (the info is not in the store).
|
|
232
|
+
// Fixtures/tests only. No thresholds, no claim, no command.
|
|
233
|
+
// ---------------------------------------------------------------------------
|
|
234
|
+
// Three equally-relevant task-critical items (each fully contains the query terms, so they
|
|
235
|
+
// tie on lexical overlap and order deterministically by recency). Used by the multi-gold and
|
|
236
|
+
// budget/limit fixtures so exactly which items survive is deterministic.
|
|
237
|
+
const mgQuery = "rollout checklist: migration, signing, oncall";
|
|
238
|
+
const mgA = "rollout checklist migration signing oncall — run the database migration first";
|
|
239
|
+
const mgB = "rollout checklist migration signing oncall — rotate the signing key next";
|
|
240
|
+
const mgC = "rollout checklist migration signing oncall — page the oncall engineer last";
|
|
241
|
+
/** F1 — multi-gold, PARTIAL recall: budget fits two of three task-critical items. */
|
|
242
|
+
const multiGoldPartialRecall = {
|
|
243
|
+
name: "multi-gold-partial-recall",
|
|
244
|
+
description: "Three equally-relevant gold items, budget fits two -> recall is a real fraction (not 0/1).",
|
|
245
|
+
store: [
|
|
246
|
+
mk(mgA, { pointer: "trace=MG message=a", created_at: T0 }),
|
|
247
|
+
mk(mgB, { pointer: "trace=MG message=b", created_at: T1 }),
|
|
248
|
+
mk(mgC, { pointer: "trace=MG message=c", created_at: T2 })
|
|
249
|
+
],
|
|
250
|
+
query: { text: mgQuery },
|
|
251
|
+
// Room for the two newest (mgC, mgB by recency tie-break); mgA is dropped.
|
|
252
|
+
tokenBudget: tokensOf(mgC) + tokensOf(mgB),
|
|
253
|
+
goldItems: [mgA, mgB, mgC],
|
|
254
|
+
// No required sources / markers here: isolate PARTIAL RECALL as the signal.
|
|
255
|
+
requiredSourcePointers: [],
|
|
256
|
+
continuationRequirements: []
|
|
257
|
+
};
|
|
258
|
+
/** F2 — empty store: zero-context behavior is explicit and honest (no crash, no inflation). */
|
|
259
|
+
const emptyStore = {
|
|
260
|
+
name: "empty-store",
|
|
261
|
+
description: "Nothing has been accumulated yet; retrieval/assembly must report honest zeros.",
|
|
262
|
+
store: [],
|
|
263
|
+
query: { text: "what did we decide about the migration", trace_ids: ["Z"] },
|
|
264
|
+
tokenBudget: 1000,
|
|
265
|
+
goldItems: ["a task-critical fact that is not in the store"],
|
|
266
|
+
requiredSourcePointers: ["trace=Z message=m1"],
|
|
267
|
+
continuationRequirements: ["migration decision"]
|
|
268
|
+
};
|
|
269
|
+
/** F3 — budget pressure: recall AND coverage degrade together, for the budget reason. */
|
|
270
|
+
const budgetDegradesRecallAndCoverage = {
|
|
271
|
+
name: "budget-degrades-recall-and-coverage",
|
|
272
|
+
description: "All three sources are required but the budget fits two -> recall and coverage both drop.",
|
|
273
|
+
store: [
|
|
274
|
+
mk(mgA, { pointer: "trace=MG message=a", created_at: T0 }),
|
|
275
|
+
mk(mgB, { pointer: "trace=MG message=b", created_at: T1 }),
|
|
276
|
+
mk(mgC, { pointer: "trace=MG message=c", created_at: T2 })
|
|
277
|
+
],
|
|
278
|
+
query: { text: mgQuery },
|
|
279
|
+
tokenBudget: tokensOf(mgC) + tokensOf(mgB),
|
|
280
|
+
goldItems: [mgA, mgB, mgC],
|
|
281
|
+
// All three required -> the dropped item's source is uncovered (coverage degrades WITH recall).
|
|
282
|
+
requiredSourcePointers: ["trace=MG message=a", "trace=MG message=b", "trace=MG message=c"],
|
|
283
|
+
continuationRequirements: []
|
|
284
|
+
};
|
|
285
|
+
/** F4 — retrieval `limit` omits one gold: recall degrades because of the LIMIT, not the budget. */
|
|
286
|
+
const limitOmitsGold = {
|
|
287
|
+
name: "limit-omits-gold",
|
|
288
|
+
description: "Budget is generous but retrieval limit=2 caps the candidate set -> one gold never reaches assembly.",
|
|
289
|
+
store: [
|
|
290
|
+
mk(mgA, { pointer: "trace=MG message=a", created_at: T0 }),
|
|
291
|
+
mk(mgB, { pointer: "trace=MG message=b", created_at: T1 }),
|
|
292
|
+
mk(mgC, { pointer: "trace=MG message=c", created_at: T2 })
|
|
293
|
+
],
|
|
294
|
+
query: { text: mgQuery },
|
|
295
|
+
tokenBudget: 1000, // generous: nothing is dropped BY BUDGET
|
|
296
|
+
retrievalOptions: { limit: 2 },
|
|
297
|
+
goldItems: [mgA, mgB, mgC],
|
|
298
|
+
requiredSourcePointers: [],
|
|
299
|
+
continuationRequirements: []
|
|
300
|
+
};
|
|
301
|
+
/** F5 — adversarial anchor: a low score is the CORRECT behavior because the info isn't stored. */
|
|
302
|
+
const adversarialCorrectLowScore = {
|
|
303
|
+
name: "adversarial-correct-low-score",
|
|
304
|
+
description: "The store holds only unrelated items; the task-critical fact is absent, so recall 0 is correct (not a retriever failure).",
|
|
305
|
+
store: [
|
|
306
|
+
mk("the office wifi password was rotated on tuesday", { pointer: "trace=N message=m1", created_at: T0 }),
|
|
307
|
+
mk("lunch is catered on fridays this quarter", { pointer: "trace=N message=m2", created_at: T1 })
|
|
308
|
+
],
|
|
309
|
+
query: { text: "what is the database migration rollback procedure" },
|
|
310
|
+
tokenBudget: 1000,
|
|
311
|
+
// The required fact was never captured into the store -> a faithful retriever cannot surface it.
|
|
312
|
+
goldItems: ["the database migration rollback procedure is to restore the retained original"],
|
|
313
|
+
requiredSourcePointers: ["trace=DB message=m1"],
|
|
314
|
+
continuationRequirements: ["rollback procedure"]
|
|
315
|
+
};
|
|
316
|
+
export const CONTEXT_STORE_EVAL_EDGE_FIXTURES = [
|
|
317
|
+
// All edge fixtures are adversarial anchors: their partial/zero scores are the CORRECT
|
|
318
|
+
// behavior (budget / limit / empty / absent-info), not a sufficiency failure.
|
|
319
|
+
adversarial(multiGoldPartialRecall, { recall: 2 / 3, source_coverage: 1, continuation_success: 1, recoverability: 1 }),
|
|
320
|
+
adversarial(emptyStore, { recall: 0, source_coverage: 0, continuation_success: 0, recoverability: 1 }),
|
|
321
|
+
adversarial(budgetDegradesRecallAndCoverage, { recall: 2 / 3, source_coverage: 2 / 3, continuation_success: 1, recoverability: 1 }),
|
|
322
|
+
adversarial(limitOmitsGold, { recall: 2 / 3, source_coverage: 1, continuation_success: 1, recoverability: 1 }),
|
|
323
|
+
adversarial(adversarialCorrectLowScore, { recall: 0, source_coverage: 0, continuation_success: 0, recoverability: 1 })
|
|
324
|
+
];
|
|
325
|
+
/** Every fixture (healthy + richer + edge), for the internal sufficiency report. */
|
|
326
|
+
export const CONTEXT_STORE_ALL_EVAL_FIXTURES = [
|
|
327
|
+
...CONTEXT_STORE_EVAL_FIXTURES,
|
|
328
|
+
...CONTEXT_STORE_EVAL_RICHER_FIXTURES,
|
|
329
|
+
...CONTEXT_STORE_EVAL_EDGE_FIXTURES
|
|
330
|
+
];
|
|
331
|
+
//# sourceMappingURL=context-store-eval-cases.js.map
|