@bcmyguest/openmem 0.8.0 → 0.9.0

package/README.md

@@ -23,16 +23,16 @@
 <p align="center">
 <picture>
 <source media="(prefers-color-scheme: dark)" srcset="https://bcmyguest.github.io/openmem/media/overview-context-wide-dark.png" />
-<img src="https://bcmyguest.github.io/openmem/media/overview-context-wide.png" alt="B, context management: coding agent, context proxy trimming at 120k tokens with no LLM call, model API, SQLite session store read back with smart_compact_search and smart_compact_sql; OpenCode 2 uses a native plugin" />
+<img src="https://bcmyguest.github.io/openmem/media/overview-context-wide.png" alt="B, context management: coding agent, context proxy trimming at 125k tokens (Haiku 95k) with no LLM call, model API, SQLite session store read back with smart_compact_search and smart_compact_sql; OpenCode 2 uses a native plugin" />
 </picture>
 </p>
 
-**Benchmarks:** [smart compaction against Claude Code's native compaction and keep-everything](https://bcmyguest.github.io/openmem/docs/context-management/#benchmarks) on [ContextBench](https://github.com/RulinShao/ContextBench) synthetic tasks (Claude Haiku 5.5). Compact-Bench real-coding results are in progress.
+**Benchmarks:** In seed-matched [ContextBench](https://github.com/RulinShao/ContextBench) log tasks on Haiku 5.5, smart compaction scored 0.469 higher than keep-everything (95% bootstrap interval 0.339–0.578; eight pairs). All arms scored 1.0 on kv R4000. Costs fell at R4000, L6400, and L12800; the L1600 saving remains uncertain. Claude Code's default native arm made no compaction in these tasks. [Methods and results](docs/context-management.md#benchmarks) · [paired intervals and pending work](docs/evidence-plan.md#completed-seed-paired-comparisons). Compact-Bench real-coding results are pending.
 
 <p align="center">
 <picture>
 <source media="(prefers-color-scheme: dark)" srcset="https://bcmyguest.github.io/openmem/media/contextbench-quality-compact-dark.png" />
-<img src="https://bcmyguest.github.io/openmem/media/contextbench-quality-compact.png" alt="Line charts of ContextBench score by task size, kv-store and log-triage: OpenMem smart compaction stays at 0.90 to 1.00; native compaction and keep everything fall as tasks grow" width="480" />
+<img src="https://bcmyguest.github.io/openmem/media/contextbench-quality-compact.png" alt="ContextBench scores: kv R4000 scores 1.00 in all arms; on log tasks, OpenMem scores 1.00 at L1600, 0.88 at L6400 and 1.00 at L12800, above both baselines" width="480" />
 </picture>
 </p>
 
@@ -91,7 +91,7 @@ flowchart LR
 
 ```mermaid
 flowchart LR
-  A["<b>Coding agent</b><br/>Claude Code · Codex · OpenCode 1"] -->|requests| X["<b>Context proxy</b><br/>LiteLLM pre-call hook<br/>trims at 120k tokens · no LLM call"]
+  A["<b>Coding agent</b><br/>Claude Code · Codex · OpenCode 1"] -->|requests| X["<b>Context proxy</b><br/>LiteLLM pre-call hook<br/>trims at 125k tokens (Haiku 95k) · no LLM call"]
   X -->|trimmed request| API["<b>Model API</b><br/>Anthropic · ChatGPT (Codex)"]
   X -->|every turn| S[("<b>Session store</b><br/>SQLite · every turn, verbatim")]
   S -->|"MCP: smart_compact_search · smart_compact_sql"| A
@@ -125,6 +125,7 @@ OpenMem is not affiliated with these projects; they shape the problem or the pat
 | **[LiteLLM](https://github.com/BerriAI/litellm)** | Model proxy used on Hindsight stacks; its pre-call hook hosts the context proxy. |
 | **[Hindsight coding agents](https://www.npmjs.com/package/@vectorize-io/hindsight-coding-agents)** | Reference implementation for knowledge pages, initiative capture and ingest status; OpenMem's hooks replace it in `openmem` mode. |
 | **[Mainplane](https://github.com/mainplane-ai/mainplane)** | Inspiration for smart compaction: continue in a fresh context with a pointer to the saved transcript and retrieve as you go, instead of an LLM summary. |
+| **[Letta](https://www.letta.com/) / [MemGPT](https://arxiv.org/abs/2310.08560)** | Inspiration for smart compaction: evicted turns kept verbatim in searchable recall storage the agent pages back with tools, and [file-style search over the transcript](https://www.letta.com/blog/benchmarking-ai-agent-memory/) (74.0% on LoCoMo). OpenMem does this in a proxy and an MCP server, so no Letta server is needed. |
 | **[rtk](https://github.com/rtk-ai/rtk)** | Compresses shell command output before the agent sees it; `install-coding-agent` runs `rtk init` when it is installed. Inspired the context proxy's compression of large tool results (reimplemented, not bundled). |
 | **[opencode-clm](https://github.com/bcmyguest/opencode-clm)** | OpenCode plugin for [Context Language Models](https://arxiv.org/abs/2609.37725): the model edits a file mirror of its own context. Same goal as smart compaction, with the model doing the trimming ([comparison](https://bcmyguest.github.io/openmem/docs/context-research/)). |
 | **[Meridian](https://github.com/rynfar/meridian)** | Bridges a Claude subscription to an Anthropic-compatible API; optional Hindsight model backend ([guide](https://bcmyguest.github.io/openmem/docs/model-configuration/)). |

package/cli/coding-agent/context-trim.mjs

@@ -16,7 +16,7 @@ import { blobRef, chain, dropSession, openSession, scheduleEmbedding, storeSetti
 
 export { RECALL_CLOSE, RECALL_OPEN, RECALL_PREFACE, recallBlock } from "./recall-frame.mjs";
 
-export const DEFAULT_TRIM_AT_TOKENS = 120000;
+export const DEFAULT_TRIM_AT_TOKENS = 125000;
 /**
  * Trim thresholds by model family (a substring of the lowercased model id), used while CONTEXT_TRIM_AT_TOKENS is
  * unset; same table as MODEL_TRIM_AT_TOKENS in proxy/openmem_context/settings.py.
@@ -547,8 +547,10 @@ export function pointerText(sid, dropped) {
     + "The tools smart_compact_search and smart_compact_sql are available to you in this request. "
     + `You MUST call smart_compact_search(session_id="${sid}", query=... or pattern=...) before answering `
     + 'anything that depends on them, and never answer "not found", "never received" or "no search tool" '
-    + "without calling it; "
-    + "every message of this session is stored and queryable with "
+    + "without calling it. "
+    + "To recover a file you read earlier, search for the text you need instead of reading it again, "
+    + "and re-read only the lines you still lack (offset/limit). "
+    + "Every message of this session is stored and queryable with "
     + `smart_compact_search and smart_compact_sql(session_id="${sid}", sql=...).]`;
 }
 

package/docs/context-management.md

@@ -39,72 +39,61 @@ The cut is sticky: later requests with the same history get the same rewrite byt
 
 ## Benchmarks
 
-ContextBench synthetic tasks, run through Claude Code in headless strict mode (no Bash or file tools, so the agent cannot park data on disk) with one model, Claude Haiku 5.5. Three settings: keep everything (no compaction), Claude Code's native auto-compaction, and OpenMem smart compaction (SQLite store, lexical search and `smart_compact_sql`). `kv-store` streams key-value batches and then asks for values; `log-triage` streams log batches and then asks lookup and count questions. Sizes grow left to right. Seed counts differ per cell (n in the table). [Context research](context-research.md#benchmark-method) has the method, every run and the caveats. The numbers and charts below are generated by `site/scripts/bench/compute-contextbench.py` and `site/scripts/bench/render-contextbench.mjs` from the result files.
+Three settings on the same held-out [ContextBench](https://github.com/RulinShao/ContextBench) seeds through Claude Code in headless strict mode (no Bash or file tools): keep everything, Claude Code's default native compaction, and OpenMem smart compaction. The model is Claude Haiku 5.5. `kv-store` asks for earlier values; `log-triage` asks lookup and count questions. Each cell has two or three seeds. The default native arm recorded zero compactions in these tasks. [Method and limits](context-research.md#run-e-held-out-seed-matched-comparison); [paired 95% bootstrap intervals](evidence-plan.md#completed-seed-paired-comparisons).
 
 <!-- contextbench:start (generated by site/scripts/bench/render-contextbench.mjs; do not edit) -->
 <p align="center">
 <picture>
 <source media="(prefers-color-scheme: dark)" srcset="https://bcmyguest.github.io/openmem/media/contextbench-quality-dark.png" />
-<img src="https://bcmyguest.github.io/openmem/media/contextbench-quality.png" alt="Line charts of answer quality by task size (kv-store score, log-triage score, lookup and count questions) for OpenMem smart compaction, native compaction and keep everything. Largest sizes. kv-store R16000: OpenMem 1.00, native 0.00, keep everything 0.00. log-triage L12800: OpenMem 1.00, no native trial, keep everything 0.38." width="480" />
+<img src="https://bcmyguest.github.io/openmem/media/contextbench-quality.png" alt="Line charts of answer quality by task size (kv-store score, log-triage score, lookup and count questions) for OpenMem smart compaction, native compaction and keep everything. Largest sizes. kv-store R4000: OpenMem 1.00, native 1.00, keep everything 1.00. log-triage L12800: OpenMem 1.00, native 0.35, keep everything 0.35." width="480" />
 </picture>
 </p>
 
 Source: [ContextBench](https://github.com/RulinShao/ContextBench) synthetic tasks, Claude Haiku 5.5. Score is reward_lenient (tag-insensitive), the mean over the completed seeds of each cell. † a seed dropped the `#tag` from its kv answers and scored 0 under the strict reward; the values were right. ‡ the harness step limit stopped the trial before any question, so the 0 is not a recall score.
 
-| Task, size | OpenMem smart compaction | Native compaction | Keep everything |
+| Task, size | OpenMem smart compaction | Native compaction (default) | Keep everything |
 | --- | --- | --- | --- |
-| kv-store R1000 | 1.00, n = 3 | 1.00, n = 1 | 1.00, n = 1 |
-| kv-store R4000 | 1.00 †, n = 3 | 0.04, n = 1 | 1.00, n = 2 |
-| kv-store R16000 | 1.00 †, n = 3 | 0.00, n = 1 | 0.00 ‡, n = 2 |
-| log-triage L1600 | 0.90 (lookup 1.00, count 0.81), n = 3 | 0.50 (lookup 1.00, count 0.00), n = 1 | 0.71 (lookup 1.00, count 0.42), n = 1 |
-| log-triage L6400 | 1.00 (lookup 1.00, count 1.00), n = 3 | 0.08 (lookup 0.17, count 0.00), n = 2 | 0.46 (lookup 0.92, count 0.00), n = 2 |
-| log-triage L12800 | 1.00 (lookup 1.00, count 1.00), n = 3 | no completed trial | 0.38 (lookup 0.75, count 0.00), n = 1 |
+| kv-store R4000 | 1.00 †, n = 3 | 1.00 †, n = 3 | 1.00, n = 3 |
+| log-triage L1600 | 1.00 (lookup 1.00, count 1.00), n = 3 | 0.58 (lookup 1.00, count 0.17), n = 3 | 0.57 (lookup 0.97, count 0.17), n = 3 |
+| log-triage L6400 | 0.88 (lookup 1.00, count 0.75), n = 3 | 0.46 (lookup 0.92, count 0.00), n = 3 | 0.49 (lookup 0.97, count 0.00), n = 3 |
+| log-triage L12800 | 1.00 (lookup 1.00, count 1.00), n = 3 | 0.35 (lookup 0.69, count 0.00), n = 3 | 0.35 (lookup 0.71, count 0.00), n = 2 |
 
 ### Time
 
-<p align="center">
-<picture>
-<source media="(prefers-color-scheme: dark)" srcset="https://bcmyguest.github.io/openmem/media/contextbench-compaction-dark.png" />
-<img src="https://bcmyguest.github.io/openmem/media/contextbench-compaction.png" alt="Bar chart of seconds per compaction event, median bars with 10th to 90th percentile whiskers. Median native summary call 10.7 s; median OpenMem request that carried a trim 4.0 s, other OpenMem requests 1.7 s." width="480" />
-</picture>
-</p>
-
-Native compaction is a model call. Claude Code's summary call took median 10.7 s (p10–p90 7.6–16.1 s, n = 22); Claude Code's own compaction timer read median 12.4 s (p10–p90 9.1–17.7 s, n = 22). The summary calls are the requests after which the next prompt drops below half its size; that finds 22 of the 22 compactions Claude Code reported.
-
-Smart compaction makes no model call: the proxy cuts the history in its pre-call hook and records no separate hook time. The request that carried a trim took median 4.0 s (p10–p90 3.1–5.4 s, n = 81), against median 1.7 s (p10–p90 1.4–2.5 s, n = 2723) for the other requests of the same run. That request also rewrites the prompt cache for the shortened history, so the difference is an upper bound on the trim's own time, assuming LiteLLM's request timer covers the pre-call hook (not verified).
+This comparison recorded no native comped. Its default native arm therefore behaved as a keep-everything baseline at these sizes. The saved usage log cannot identify which request paid for a new OpenMem trim, so it does not support a trim-latency estimate.
 
 <p align="center">
 <picture>
 <source media="(prefers-color-scheme: dark)" srcset="https://bcmyguest.github.io/openmem/media/contextbench-time-dark.png" />
-<img src="https://bcmyguest.github.io/openmem/media/contextbench-time.png" alt="Line charts of agent wall time per trial by task size for kv-store and log-triage. At log-triage L12800 an OpenMem trial took 24.4 min and a keep-everything trial 13.5 min. The runs had different API load, so treat the gap as rough." width="480" />
+<img src="https://bcmyguest.github.io/openmem/media/contextbench-time.png" alt="Line charts of agent wall time per trial by task size for kv-store and log-triage. At log-triage L12800 an OpenMem trial took 13.2 min and a keep-everything trial 16.1 min." width="480" />
 </picture>
 </p>
 
 | Setting | Trials | Agent wall time per trial, mean | Requests | Request latency, mean | Prompt-cache hit rate |
 | --- | --- | --- | --- | --- | --- |
-| OpenMem smart compaction | 18 | 12.3 min | 2804 | 1.99 s | 95% |
-| Native compaction | 6 | 8.1 min | 729 | 0.86 s | 93% |
-| Keep everything | 9 | 9.7 min | 1077 | 0.59 s | 99% |
+| OpenMem smart compaction | 12 | 8.8 min | 2103 | 1.83 s | 90% |
+| Native compaction (default) | 12 | 8.0 min | 1831 | 2.12 s | 99% |
+| Keep everything | 11 | 8.2 min | 1567 | 2.47 s | 99% |
 
-Wall time per trial is longer with OpenMem (593 s vs 472 s for native compaction); OpenMem also makes more requests per trial (130 vs 117), which include its store searches.
+Wall time per trial is longer with OpenMem (527 s vs 479 s for native compaction); OpenMem also makes more requests per trial (175 vs 153), which include its store searches.
 
 Source: [ContextBench](https://github.com/RulinShao/ContextBench) synthetic tasks, Claude Haiku 5.5.
 
-Wall time and request latency come from different runs: the baselines (run B) and smart compaction (run D) used different LiteLLM builds, ran at different times under different API load, and only run D had the OpenMem hook. Treat the comparison as rough.
+Wall times still include variable model service latency; the seed-paired intervals are in [the evidence plan](evidence-plan.md#completed-seed-paired-comparisons).
 
 ### Cost and prompt cache
 
 <p align="center">
 <picture>
 <source media="(prefers-color-scheme: dark)" srcset="https://bcmyguest.github.io/openmem/media/contextbench-cost-dark.png" />
-<img src="https://bcmyguest.github.io/openmem/media/contextbench-cost.png" alt="Line charts of list-price cost per trial by task size for kv-store and log-triage. At log-triage L12800 keep everything cost $6.12 per trial and OpenMem $0.74. Where both ran, OpenMem cost within 15% of native compaction." width="480" />
+<img src="https://bcmyguest.github.io/openmem/media/contextbench-cost.png" alt="Line charts of list-price cost per trial by task size for kv-store and log-triage. At log-triage L12800, keep everything cost $6.07 per trial and OpenMem $0.38." width="480" />
 </picture>
 </p>
 
 <p align="center">
 <picture>
 <source media="(prefers-color-scheme: dark)" srcset="https://bcmyguest.github.io/openmem/media/contextbench-cache-dark.png" />
-<img src="https://bcmyguest.github.io/openmem/media/contextbench-cache.png" alt="Line charts of prompt-cache hit rate by task size for kv-store and log-triage. Prompt-cache hit rate, cells where native ran: OpenMem 95%, native 93%. A trim rewrites the cache once; the cut is then stable." width="480" />
+<img src="https://bcmyguest.github.io/openmem/media/contextbench-cache.png" alt="Line charts of prompt-cache hit rate by task size for kv-store and log-triage. Prompt-cache hit rate, cells where native ran: OpenMem 90%, native 99%. A trim rewrites the cache once; the cut is then stable." width="480" />
 </picture>
 </p>
 
@@ -112,45 +101,31 @@ Source: [ContextBench](https://github.com/RulinShao/ContextBench) synthetic task
 
 | Task, size | List-price cost per trial (smart / native / keep) | Prompt-cache hit rate (smart / native / keep) | Tokens per trial, cache reads in parentheses (smart / native / keep) | Peak prompt (smart / native / keep) |
 | --- | --- | --- | --- | --- |
-| kv-store R1000 | $0.05 / $0.05 / $0.05 | 97% / 97% / 97% | 2.9M (2.8M) / 2.8M (2.7M) / 2.8M (2.7M) | 101k / 97k / 96k |
-| kv-store R4000 | $0.22 / $0.24 / $1.05 | 93% / 88% / 98% | 5.0M (4.6M) / 3.4M (3.0M) / 15.8M (15.4M) | 119k / 120k / 358k |
-| kv-store R16000 | $0.85 / $0.88 / $3.64 | 90% / 88% / 98% | 13.8M (12.4M) / 12.8M (11.2M) / 56.0M (55.0M) | 119k / 119k / 979k |
-| log-triage L1600 | $0.27 / $0.31 / $0.57 | 98% / 99% / 98% | 9.0M (8.9M) / 8.6M (8.4M) / 10.2M (9.9M) | 120k / 120k / 178k |
-| log-triage L6400 | $0.32 / $0.36 / $1.88 | 96% / 95% / 99% | 10.6M (10.2M) / 8.7M (8.2M) / 32.6M (32.2M) | 120k / 123k / 386k |
-| log-triage L12800 | $0.74 / — / $6.12 | 96% / — / 99% | 18.6M (17.7M) / — / 109.4M (108.6M) | 120k / — / 765k |
+| kv-store R4000 | $0.14 / $1.07 / $1.05 | 90% / 98% / 98% | 4.6M (4.1M) / 16.1M (15.7M) / 15.7M (15.4M) | 93k / 359k / 359k |
+| log-triage L1600 | $0.36 / $0.54 / $0.68 | 84% / 98% / 99% | 7.9M (6.5M) / 10.2M (10.0M) / 12.5M (12.3M) | 94k / 218k / 268k |
+| log-triage L6400 | $0.20 / $1.90 / $1.87 | 94% / 99% / 99% | 8.9M (8.3M) / 32.6M (32.2M) / 32.3M (31.9M) | 96k / 397k / 388k |
+| log-triage L12800 | $0.38 / $6.11 / $6.07 | 92% / 99% / 99% | 14.4M (13.2M) / 109.3M (108.6M) / 108.8M (108.0M) | 95k / 764k / 762k |
 <!-- contextbench:end -->
 
-### Latest version: search-tools note and cache-stable trims
-
-The charts and tables above are run D. Three changes since, measured on one task (log-triage L1600 seed 44):
-
-1. **Haiku trims at 95k** instead of about 120k, so every request stays under the 100k line where list prices rise 5× per token.
-2. **The trim note names the search tools.** Without it the model never searched its trimmed history on this task and answered 16 of 24 questions UNKNOWN. With it, every trial searched and answered every question.
-3. **Trims keep the prompt cache intact.** A trim now drops the model's thinking from before the cut and keeps the thinking after it. The start of the prompt stays identical from one request to the next, so the cache keeps hitting.
-
-| Version | Trials | Score | Cost per trial | Cache hit rate | Tokens written to cache | Largest prompt | Wall time |
-| --- | --- | --- | --- | --- | --- | --- | --- |
-| 95k trim | 1 | 0.17 | $0.15 | 93% | 404k | 94k | 7.9 min |
-| + search-tools note | 5 | 0.94 | $0.22 | 92% | 598k | 95k | 9.8 min |
-| + cache-stable trims | 5 | 1.00 | $0.11 | 98% | 163k | 95k | 7.3 min |
+### What this shows
 
-The cache fix halves mean cost, and all 5 trials scored 1.00. Most of the gap is one search-tools-only trial that trimmed 3 times, wrote 2.2M tokens to the cache and cost $0.59; without it that version averages $0.13.
+Across eight seed-matched log trials, OpenMem's mean score exceeded keep-everything by 0.469 (95% bootstrap interval 0.339–0.578). Across nine pairs it exceeded default native by 0.495 (0.384–0.593). All three arms scored 1.0 on kv R4000. OpenMem cost less on every paired R4000, L6400, and L12800 trial. The L1600 cost interval includes no saving, and its wall-time interval includes a slowdown over 20%. These small synthetic samples do not establish the same effect for coding sessions.
 
-For scale, the same size on seed 42 (one trial each, a different task instance): keep everything scored 0.71 at $0.57 with a 178k largest prompt; native compaction scored 0.50 at $0.31. The latest version costs about a fifth of keep everything and a third of native compaction. Most of that saving is the price tier, not token count: total tokens differ by less than 1.5× (6.9M against 10.2M). Treat the ratios as rough until the baselines run on seed 44. [Context research](context-research.md) has every trial.
+On Sonnet 5 log L6400, two seed-matched pairs favored OpenMem by 0.0625 score (0.0417–0.0833) and cost 0.401 times keep-everything (0.361–0.441). [Exclusions, price basis, and pending work](evidence-plan.md#completed-seed-paired-comparisons). Scratchpad decision consistency remains untested on this published comparison.
 
 ### Compact-Bench (real coding sessions)
 
 [Compact-Bench](https://github.com/cai-jianfeng/compact-bench)'s Task Preservation track resumes real SWE-bench repair sessions after compaction and checks whether the task is preserved, damaged, rescued or left unsolved, per method: full history, Claude Code summary, Codex summary and OpenMem.
 
 <!-- compact-bench:start (generated by site/scripts/bench/render-contextbench.mjs; do not edit) -->
-Compact-Bench results pending. The run is in progress; partial numbers are not published. Methods: Full history, Claude Code summary, Codex summary, OpenMem.
+Compact-Bench results pending. No run has completed yet. Methods: Full history, Claude Code summary, Codex summary, OpenMem.
 <!-- compact-bench:end -->
 
 ## When to turn it on
 
 | Turn it on when | Leave it off when |
 | --- | --- |
-| Sessions grow past `trim_at_tokens` (default 120,000 estimated tokens, 95,000 for Haiku) | Sessions stay under that limit |
+| Sessions grow past `trim_at_tokens` (default 125,000 estimated tokens, 95,000 for Haiku) | Sessions stay under that limit |
 | You want trimmed turns searchable instead of summarized away | You cannot run the `context-proxy` container (Claude Code, OpenCode 1 and Codex send every request to it and fail while it is down) |
 | You use Claude Code, Codex or OpenCode (OpenCode 2 needs no container) | You use only Cursor CLI (not routed through the proxy) |
 
@@ -232,7 +207,7 @@ The installer does not wire the unified MCP for OpenCode 2 ([compatibility](comp
 | `[context_proxy] enabled` | unset | none | Master switch: the value of every agent key that is not set. Unset: `opencode2` on, the others off |
 | `[context_proxy.agents] claude-code`, `opencode1`, `codex`, `opencode2` | follow `enabled`; with it unset, `opencode2` on and the others off | `CONTEXT_PROXY_ENABLED` for `opencode2` (in OpenCode's environment) | Per-agent on/off; see [Turn it on or off](#turn-it-on-or-off) |
 | `[context_proxy] port` | `4100` | none (copied to the stack `.env` as `CONTEXT_PROXY_PORT`) | Host port, bound to 127.0.0.1 |
-| `[context_proxy] trim_at_tokens` | `95000` for Haiku, `120000` otherwise | `CONTEXT_TRIM_AT_TOKENS` | Estimated prompt tokens that trigger a trim; unset, the threshold follows the request's model ([settings](context-proxy.md)), and a set value applies to every model; the estimate is calibrated from the provider's reported usage ([token estimate](context-proxy.md#token-estimate)) |
+| `[context_proxy] trim_at_tokens` | `95000` for Haiku, `125000` otherwise | `CONTEXT_TRIM_AT_TOKENS` | Estimated prompt tokens that trigger a trim; unset, the threshold follows the request's model ([settings](context-proxy.md)), and a set value applies to every model; the estimate is calibrated from the provider's reported usage ([token estimate](context-proxy.md#token-estimate)) |
 | `[context_proxy] trim_to_fraction` | `0.3` | `CONTEXT_TRIM_TO_FRACTION` | How deep a trim cuts a long in-flight turn: older tool exchanges stay only while the request is under this fraction of `trim_at_tokens`. Between 0 and 1; another value falls back to 0.3 |
 | `[context_proxy] chars_per_token` | `2.5` | `CONTEXT_CHARS_PER_TOKEN` | Characters per token before a session reports usage |
 | `[context_proxy] compress_over_tokens` | `4000` | `CONTEXT_COMPRESS_OVER_TOKENS` | Tool results above this are compressed; `0` disables compression |

package/docs/context-proxy.md

@@ -25,7 +25,7 @@ Set these in the stack `.env` or in the shell that runs compose.
 | Variable | Default | Effect |
 | --- | --- | --- |
 | `CONTEXT_PROXY_ENABLED` | `1` | `0` forwards every request unmodified |
-| `CONTEXT_TRIM_AT_TOKENS` | unset: `95000` for Haiku, `120000` otherwise | Prompt tokens (estimated as below) that trigger a trim. Unset or not an integer, the threshold follows the request's model (`MODEL_TRIM_AT_TOKENS` in `proxy/openmem_context/settings.py`): a model id containing `haiku` trims at 95000, under the higher price that Haiku charges for prompts over 100k tokens. A set value applies to every model |
+| `CONTEXT_TRIM_AT_TOKENS` | unset: `95000` for Haiku, `125000` otherwise | Prompt tokens (estimated as below) that trigger a trim. Unset or not an integer, the threshold follows the request's model (`MODEL_TRIM_AT_TOKENS` in `proxy/openmem_context/settings.py`): a model id containing `haiku` trims at 95000, under the higher price that Haiku charges for prompts over 100k tokens. A set value applies to every model |
 | `CONTEXT_TRIM_TO_FRACTION` | `0.3` | A trim keeps older in-turn tool exchanges only while the request stays under this fraction of `CONTEXT_TRIM_AT_TOKENS`; must be between 0 and 1, else the default applies (config.toml `[context_proxy] trim_to_fraction`) |
 | `CONTEXT_CHARS_PER_TOKEN` | `2.5` | Characters per token for the estimate until the session reports usage (config.toml `[context_proxy] chars_per_token`) |
 | `CONTEXT_COMPRESS_OVER_TOKENS` | `4000` | Tool results above this are compressed; `0` disables compression |

package/docs/context-research.md

@@ -31,6 +31,7 @@ flowchart LR
 | [MemGPT](https://arxiv.org/abs/2310.08560) (Packer et al., 2023) | OS-style virtual context: a small main context plus searchable recall storage, paged in by function calls. | Verbatim recall storage behind search tools instead of a summary. | No self-editing memory; the proxy trims and the agent only searches. |
 | [MemGPT DMR](https://arxiv.org/abs/2310.08560) (Table 2) and [MSC-Self-Instruct](https://huggingface.co/datasets/MemGPT/MSC-Self-Instruct) | Deep memory retrieval questions over earlier Multi-Session Chat sessions. GPT-4 scored 32.1% fixed-context and 92.5% with MemGPT; GPT-4 Turbo scored 35.3% and 93.4%. | Evidence that search over stored turns beats a fixed window. | Its conversations fit in today's windows, so it doesn't force trimming. Not run here. |
 | MemGPT nested key-value retrieval (same paper) | Multi-hop lookups across key-value pairs. Fixed-context GPT-4 and GPT-4 Turbo reach 0% by 3 nesting levels; MemGPT with GPT-4 is unaffected by nesting depth. | Closest published analogue to our kv-store task. | Not run here. |
+| [The Complexity Trap](https://arxiv.org/html/2508.21433v3) (Lindenbauer et al., JetBrains Research, 2025) | SWE-agent on SWE-bench Verified: observation masking (replace old tool outputs with a placeholder, keep the last 10) against LLM summarization. Masking roughly halves cost against the raw agent and matches summarization's solve rate; Qwen3-Coder 480B scored 54.8% at $0.61 masked, 53.8% at $0.64 summarized, 53.4% at $1.29 raw. | Evidence that removal without an LLM summary is a sound default. | Masking drops only observations and keeps the agent's reasoning and actions; we drop whole turns but keep them searchable. |
 | [Letta: Is a Filesystem All You Need?](https://www.letta.com/blog/benchmarking-ai-agent-memory/) (2025-08-12) | Conversation history stored as files; agent uses `grep`, `search_files`, `open`, `close`. 74.0% on LoCoMo with GPT-4o mini, against 68.5% for Mem0's best graph variant. | File-like tools (search, open) work well; models already use them iteratively. | Letta's files are server-side and reached through tools. We copy that: the store is reached through MCP tools, not a shell. |
 | [Mainplane](https://github.com/mainplane-ai/mainplane) | Each session is one append-only state file. `/continue` starts a new session with "You are continuing session X. Retrieve key context from it as you go." A standing "Context Management" system-prompt section says the previous session's log "is searchable with rg". | The pointer-and-retrieve continuation, and a standing system instruction telling the agent the content exists and how to fetch it. | We trim in place, keeping the same session and prompt-cache prefix, instead of starting a new session. Access is through tools, not `rg`. |
 | [rtk](https://github.com/rtk-ai/rtk) | "CLI proxy that reduces LLM token consumption by 60-90% on common dev commands"; an optional `rtk-proxy` compresses provider-API payloads. | Compress large tool output before the model reads it; `install-coding-agent` runs `rtk init` when rtk is installed. | Our compression keeps the full text as a blob behind `smart_compact_get`, so nothing is lost. |
@@ -196,14 +197,24 @@ Same task and setup, run 2026-10-30. This version (commit 47d17fa) builds on the
 
 The cache fix halves the mean cost, from $0.218 to $0.108, and every trial scored 1.00. Most of the cost gap comes from one search-tools-only trial (trial 5). It trimmed 3 times, wrote 2.2M tokens to the cache at a 0.693 hit rate, and cost $0.589. Without that trial the search-tools-only mean is $0.126. No cache-fix trial behaved that way, but five trials cannot rule it out. Every trial in both runs kept its largest prompt under 95k tokens.
 
+## Run E: held-out seed-matched comparison
+
+Run E used Haiku 5.5, Claude Code strict mode, the same proxy build, and seeds 45–47 for each arm. The primary arms were `e-none` (no compaction), `e-native` (Claude Code default), and `e-smart` (OpenMem SQLite search, 95k Haiku threshold, search-tools pointer and cache-stable trim). Tasks were kv R4000 and log L1600/L6400/L12800. Keep-everything has only seeds 45–46 at L12800. `e-native-153k` is a sensitivity arm and is excluded from the primary chart. The default native arm recorded no compactions, so these trials compare OpenMem chiefly with two keep-everything variants. Raw rows are in `bench/results-E/summary.json`; the [published chart](context-management.md#benchmarks) reads the trial artifacts through `site/scripts/bench/compute-contextbench.py`.
+
+The paired log-score advantage was +0.469 against keep-everything (95% bootstrap interval 0.339–0.578, eight pairs) and +0.495 against default native (0.384–0.593, nine pairs). Kv R4000 scored 1.0 in all arms. Smart/none cost ratios were 0.130 [0.128, 0.132] at R4000, 0.811 [0.102, 1.528] at L1600, 0.109 [0.107, 0.111] at L6400, and 0.062 [0.062, 0.063] at L12800. The L1600 interval crosses 1. The [evidence plan](evidence-plan.md#completed-seed-paired-comparisons) gives each size's score and wall-time interval and explains the bootstrap method.
+
+Sonnet 5 has two paired log L6400 seeds in `bench/results-S/summary.json`. Smart minus none score was +0.0625 [0.0417, 0.0833]; its cost ratio was 0.401 [0.361, 0.441]. The two smart trials cost $7.86 at list price against $19.85 for none. This supports the direction on that one task, with too few seeds for broad model claims.
+
+The coding-task replay used Haiku 4.5. Its `step2c` brief required changing strings pinned by existing exact-string tests while forbidding assertion edits; all four `step2c` trials therefore report `tests_rc=1`. Two of four smart-arm replays searched OpenMem once. This does not diagnose the earlier hour-long subagents, whose transcripts were not scored for scratchpad behavior. The decision-consistency trial set remains partial. Scratchpad read-path work is outside `main`, so no scratchpad benefit is claimed here.
+
 ## Limitations
 
-- One model (Haiku 5.5) and three seeds per task; run B's baselines have one or two seeds per cell.
+- The primary run E uses Haiku 5.5 and two or three seeds per cell. Sonnet has two paired seeds on one task. Earlier run B baselines have one or two seeds per cell.
 - Synthetic tasks with exact keys, which favour lexical search. No paraphrase or semantic task yet, so run D never exercised vectors.
 - Haiku drops the `#tag` in kv answers in every setting; compare `reward_lenient`.
 - Runs before D trimmed late (203–236k real tokens) because of the chars/4 estimate; run D's calibrated estimate fired at 116.5–120.1k.
 - kv R16000 under s1 hit the harness step limit at 979k tokens before any question, so it is not a recall score.
-- Counts in run D overcount when the agent's own tool output and answers echo log lines, and drop when the agent skips SQL. Runs A and earlier were limited by the JSONL log's `line_matches`.
+- Counts in run D overcount when the agent's own tool output and answers echo log lines, and drop when the agent skips SQL. Runs A and earlier were limited by the JSONL log's `line_matches`. Run E did not repair the scoring issue.
 - The d-hybrid embedder runs single-threaded and fell behind under 4 concurrent sessions.
 - Compression never fired, so s3a and s3b test the same thing.
 
@@ -212,4 +223,4 @@ The cache fix halves the mean cost, from $0.218 to $0.108, and every trial score
 1. Fix counts: exclude tool output from counted lines, add a role filter, and add a count clause to the standing instruction. Deferred.
 2. Add a paraphrase or semantic task (Chroma's semantic needle variant or LongMemEval), so hybrid search is used and measured.
 3. Run Compact-Bench real coding sessions.
-4. Repeat with a second model.
+4. Finish decision-consistency and scratchpad tests with a validated Codex context limit before drawing a coding-agent conclusion.

package/docs/context-store.md

@@ -238,7 +238,7 @@ It is a constant (`STANDING_INSTRUCTION` in `proxy/openmem_context/core.py` and
 Pointer block placed at the start of the cut turn after a trim:
 
 ```
-[openmem: continuing session <sid>; <n> earlier messages are NOT in your context. They were delivered to you and later removed to fit your context window, so a gap in what you see (missing batches, lines, keys) is in them. The tools smart_compact_search and smart_compact_sql are available to you in this request. You MUST call smart_compact_search(session_id="<sid>", query=... or pattern=...) before answering anything that depends on them, and never answer "not found", "never received" or "no search tool" without calling it; every message of this session is stored and queryable with smart_compact_search and smart_compact_sql(session_id="<sid>", sql=...).]
+[openmem: continuing session <sid>; <n> earlier messages are NOT in your context. They were delivered to you and later removed to fit your context window, so a gap in what you see (missing batches, lines, keys) is in them. The tools smart_compact_search and smart_compact_sql are available to you in this request. You MUST call smart_compact_search(session_id="<sid>", query=... or pattern=...) before answering anything that depends on them, and never answer "not found", "never received" or "no search tool" without calling it. To recover a file you read earlier, search for the text you need instead of reading it again, and re-read only the lines you still lack (offset/limit). Every message of this session is stored and queryable with smart_compact_search and smart_compact_sql(session_id="<sid>", sql=...).]
 ```
 
 It names no file path, so the proxy (in its container) and the OpenCode 2 plugin (on the host) produce the same bytes.

package/docs/evidence-plan.md

@@ -0,0 +1,216 @@
+# Evidence plan: smart compaction, run E
+
+Original experiment plan for the claim in [context-management.md](context-management.md#benchmarks): *OpenMem smart compaction keeps ContextBench scores at or above keep-everything while costing less than native compaction and keep-everything.* The budget below is the original estimate, **$58 at list price** (phases 1–5), with a $75 hard stop. Results and deviations are recorded here as trials finish.
+
+## Current run status (2026-10-31)
+
+| Work | Evidence now available | State |
+| --- | --- | --- |
+| Phase 1, run E | `bench/results-E/summary.json` contains 51 rows across e-none (13), e-native (12), e-native-153k (12), and e-smart (14), including sensitivity and rerun rows. | Seed-paired H1–H3 and H5 comparisons below; H6 remains unmeasured. |
+| Phase 2, scratchpad | `scratchpad-phase2` has commits `8631472` (pins table) and `0809dc3` (MCP write tool). Separate trial worktrees contain step2c read-path commits, including `2e55743`. | Read path is not integrated into `main`; the brief/test conflict below needs resolution before treating it as complete. |
+| Phase 4, Sonnet | `bench/results-S/summary.json` contains two seed-matched pairs on log L6400. Mean lenient reward: e-none 0.479, e-smart 0.542. Total list-price cost: $19.85 versus $7.86 for the two trials per arm. | H4 comparison below. The first pair exceeded the original $10 abort threshold; the user approved the second pair. |
+| Coding-task replay | Eight trials completed: two seeds each for `decision-fixes` and `step2c` under e-native and e-smart. `bench/replay/results/report.md` holds raw metrics. | Scoring review pending; see below. |
+| Phase 3, decision consistency | Four completed sudoku t150 rows (seeds 45 and 46, e-none and e-smart), all with lenient reward 1.0. One decision-ledger D40 trial has a `result.json`; two more trial directories are partial. | Incomplete. No active runner was visible at this check; remaining trials need a restart and a fresh summary. |
+| Phases 5 and 6 | Phase 5 threshold ablation was optional; phase 6 Compact-Bench was deferred in the original plan. | No completed results found in this check. |
+
+The three result sets above total $101.72 at list price ($65.54 run E, $27.71 Sonnet, $8.47 replay), excluding earlier runs. This exceeds the original $75 stop by $26.72. These are artifact totals, not a reconciled billing statement. Further API trials are paused pending a new budget decision.
+
+The replay exposed two harness fixes. `bench/ctx_metrics.py` now skips non-object message entries; commit `09c8ae1` contains that fix. The replay script uses a short `/tmp/rp-XXXXXX` `TMPDIR` and links it into each trial directory to keep sandbox socket paths short. The previous agent reported that a three-turn smoke run showed no sandbox error but made no Bash call; sandboxed Bash remains unverified. The replay tree was previously ignored by `.git/info/exclude`; its source and packaged task inputs are now tracked. The previous agent reported removing eight invalid pre-fix trials; the results directory now contains eight completed trials.
+
+The replay report's `tests_pass` column is unsuitable for `step2c`: the brief requires changing the standing instruction and pointer text while forbidding changes to existing assertions, yet the existing exact-string tests pin both values. The four completed `step2c` trials all have `tests_rc=1`. The observed failures differ across trials, and one trial also has Python test failures, so a passing score cannot be inferred from `tests_rc` alone. Score the brief's requested behavior, Python tests, the exact failing checks, and the agent's report separately before drawing an arm comparison. The original two-trial inspection found that agents claimed passing npm tests without running the full suite; inspect all four final reports before treating that finding as a full-run result.
+
+The replay uses Claude Haiku 4.5 and host Claude Code with bubblewrap; run E uses the ContextBench harness. Its raw means (e-native versus e-smart: 74.0 versus 132.5 tool calls, 295 versus 463 seconds, $0.85 versus $1.27 per trial) describe these eight trials only. Only two of four e-smart replay trials called `smart_compact_search`, once each; the Sonnet ContextBench smart trials called it 11 and 12 times. These results do not establish an effect of trimming or a model-specific cause for stalled coding agents because task success and the `step2c` scorer remain unresolved.
+
+### Completed seed-paired comparisons
+
+Source: `bench/results-E/summary.json` and `bench/results-S/summary.json`. Each trial row is one observation. Within each task size, pair arms by seed; resample the paired seeds with replacement 20,000 times for two-sided 95% percentile-bootstrap intervals. The planned e-native-153k sensitivity arm and phase 3 sudoku rows are outside the primary comparison. No primary row has a recorded exception or rate-limit error. With only two or three seeds per cell, these intervals describe the observed seeds and give weak coverage for new tasks or seeds.
+
+| Run E task | Pairs smart/none | Reward difference smart − none, 95% CI | Reward difference smart − native, 95% CI | Smart/none cost ratio, 95% CI |
+| --- | ---: | ---: | ---: | ---: |
+| kv R4000 | 3 | 0.000 [0.000, 0.000] | 0.000 [0.000, 0.000] | 0.130 [0.128, 0.132] |
+| log L1600 | 3 | +0.431 [0.250, 0.542] | +0.417 [0.333, 0.500] | 0.811 [0.102, 1.528] |
+| log L6400 | 3 | +0.389 [0.125, 0.542] | +0.417 [0.167, 0.583] | 0.109 [0.107, 0.111] |
+| log L12800 | 2 | +0.646 [0.625, 0.667] | +0.653 [0.583, 0.708] (3 pairs) | 0.062 [0.062, 0.063] |
+
+Pooling the available log pairs gives smart − none reward +0.469 [0.339, 0.578] across 8 pairs and smart − native +0.495 [0.384, 0.593] across 9 pairs. This pooled interval resamples individual task/seed pairs; it describes the observed mix of log sizes. R4000 is a ceiling case: all three arms scored 1.0, so it supports non-inferiority on these seeds but not strict superiority over native. H1 was not disproved. H2 is supported on log tasks and fails its strict-superiority condition on kv R4000. Smart cost was lower on every R4000, L6400, and L12800 pair. At L1600, the cost-ratio CI crosses 1 and one smart trial cost more than its baseline, so H3's pre-registered all-family cost claim is not established.
+
+For H5, the paired smart/none agent-wall-time ratios were 0.996 [0.794, 1.140] at R4000, 1.532 [0.640, 2.021] at L1600, 1.004 [0.979, 1.052] at L6400, and 0.876 [0.582, 1.169] at L12800. The wide L1600 interval includes the 1.2 slowdown threshold; these trials neither establish nor rule out a slowdown there.
+
+On Sonnet log L6400 (seeds 45–46), smart − none reward was +0.0625 [0.0417, 0.0833]. The mean paired smart/none cost ratio was 0.401 [0.361, 0.441]; the two-arm totals were $7.86 and $19.85. Total reported input, cache, and output tokens were 22.73M smart versus 69.26M none (ratio 0.328). Both observed pairs meet H4's score and cost rule, with only two seeds and one task size.
+
+H6 cannot be scored from these artifacts. In `usage.jsonl`, `trimmed=true` marks both a newly cut request and later requests carrying the same cut; the flag is not a new-trim indicator. The LiteLLM `latency_s` logger receives model-call start/end timestamps, and the saved trim events do not record hook duration. Comparing all `trimmed=true` requests with untrimmed requests would answer a different question.
+
+## 0. What the published numbers rest on
+
+| Run | What | n | Where |
+| --- | --- | --- | --- |
+| Strict | 4 arms, kv R4000 + log L6400, seeds 42–44 | 24 | context-research.md §Strict run |
+| B | s1-none, s2-native, s3a-smart by size | 1–2 seeds/cell | context-research.md §Run B |
+| D | d-sqlite + d-hybrid at 120k trim, 6 sizes × seeds 42–44 | 36 | `bench/results-D/`, chart source |
+| Dp | search-tools pointer, log L1600 seed 44 only | 5 | `bench/results-Dp/` |
+| Df | cache-stable trims, log L1600 seed 44 only | 5 | `bench/results-Df/` |
+
+Weaknesses the plan must remove or confirm:
+
+1. Baselines (run B) and smart (run D) come from different runs, LiteLLM builds and days; `docs/context-management.md:93` says so. The latency claim (trim request 4.0 s vs 1.7 s) is cross-run.
+2. Df (the config quoted as current, `docs/context-management.md:125-139`) has n = 5 on **one task instance**, seed 44, the seed it was tuned on.
+3. The README chart (`README.md:30-36`) and `site/src/data/contextbench.json` are run D (`site/scripts/bench/compute-contextbench.py:27-29` reads `results-B` s1/s2 and `results-D` d-sqlite); the text under it quotes Df. Chart and text disagree on config (120k vs 95k trim, pointer, cache fix).
+4. The 5× cost ratio depends on Haiku 5.5's >100k price tier (`bench/summarize.py:31`, `PRICE[True]`); `context-management.md:139` already concedes total tokens differ <1.5×.
+5. Native arm fired at a forced window (`bench/run_strict.sh:31` `NATIVE_WINDOW=153000`), not Claude Code's default.
+6. No task scores the agent's own earlier decisions, the thing whole-turn eviction drops.
+
+## 1. Hypotheses and decision rules (pre-registered)
+
+Primary metric: `reward_lenient` per trial (kv tag-insensitive; `bench/summarize.py:120-135` `answer_check`). Secondary: list-price USD per trial (`summarize.py:37-43`), agent wall time (result.json via `bench/ctx_metrics.py`), request latency (`usage.jsonl` `latency_s`), peak prompt tokens, trims, native compactions.
+
+| Id | Hypothesis | Disproved if |
+| --- | --- | --- |
+| H1 quality | Smart ≥ keep-everything on held-out seeds at R4000/L6400/L12800 | paired (same seed) mean difference smart − none < −0.05 with 95% bootstrap CI excluding 0 |
+| H2 quality vs native | Smart > native (default) | paired difference ≤ 0 or CI includes 0 at n = 9 pairs per task family |
+| H3 cost | Smart < native and < none per trial on Haiku | smart/none cost ratio CI includes 1.0, or smart > native on any family |
+| H4 price-tier | On Sonnet 5, at Sonnet's per-model trim threshold (125k), smart passes if it costs no more per trial than e-none (no compaction) and scores the same or better on `reward_lenient` | smart costs more than e-none per trial, or scores worse on `reward_lenient` (then the cost claim is published as Haiku-only) |
+| H5 speed | Within one run, smart's agent wall time per trial is not slower than none by more than 20% | wall-time ratio smart/none > 1.2 with CI excluding 1.2 |
+| H6 trim latency | Requests carrying a trim cost ≤ 1 s extra over the same arm's untrimmed requests | median difference > 1 s (then document the hook cost, do not hide it) |
+| H7 decisions | Whole-turn eviction drops the agent's own decisions: smart without scratchpad scores < none on the decision task | smart ≥ none (then risk (c) is not observable here; accept as-is) |
+| H8 scratchpad | Scratchpad restores H7: smart+scratchpad ≥ none and > smart | smart+scratchpad − smart ≤ 0 (then drop the feature) |
+
+Statistics for every table: mean, n, 95% percentile-bootstrap CI (10,000 resamples), paired by seed where arms share seeds; report per family and pooled; list every trial (no exclusions beyond the pre-registered harness failures below). A trial is excluded only when `summarize.py` classes it rate-limited (`RATE_LIMIT_RE`, line 29), harness-failed (`cc_stream.py` no result event), or the proxy fail-opened (`events.kind = 'fail_open'`, `core.py:355`); excluded trials are listed and rerun once (`bench/rerun_strict.sh` pattern; `ctx/queue-E.txt` accepts appended lines, `ctx_sched.py:5`).
+
+Null outcome is publishable: if H1–H3 fail, `context-management.md` Benchmarks is rewritten to say so and the README chart is removed.
+
+## 2. Phases
+
+Trial-cost and time anchors (list price; `results-D/d-sqlite/usage.jsonl`, `docs/context-research.md:152,184-195`, `context-management.md:83-88`):
+
+| Cell | $ per trial | min per trial |
+| --- | --- | --- |
+| smart log L1600 (Df, 95k) | 0.11 | 7.3 |
+| smart kv R4000 / log L6400 (run D, 120k) | 0.27 / 0.30 | 10–12 |
+| smart log L12800 (run D) | 0.58 | 24 |
+| none L1600 / R4000 / L6400 / L12800 (run B) | 0.57 / ~1.5 / ~1.5 / ~3 | 8–15 |
+| native (forced 153k) L1600 / R4000 / L6400 | 0.31 / ~0.5 / ~0.5 | 8 |
+
+Four trials run concurrently (`ctx_sched.py` MAX = 4). Wall time ≈ trials × 11 min / 4 + 20% for reruns.
+
+### Phase 0: harness prep (no API spend, ~2 h)
+
+Files:
+
+- `bench/run_ctx.sh`: add run `E` with arms `e-none`, `e-native`, `e-smart`; ports in the table at lines 51-55 (next free: 4257–4259); `ARMS` case at 40-41; `TRIM_AT=` empty for E (line 45) so `MODEL_TRIM_AT_TOKENS` gives Haiku 95k (`proxy/openmem_context/settings.py:14`); `e-smart` at the current `main` SHA exported like `DF_SHA`/`R_DF` (lines 126-127; `./run_ctx.sh prep E`). `e-none`: pass-through proxy + `DISABLE_AUTO_COMPACT=1` (as `run_strict.sh:113`). `e-native`: pass-through proxy, **no** `CLAUDE_CODE_AUTO_COMPACT_WINDOW`, **no** `DISABLE_AUTO_COMPACT` — Claude Code's default. Justification: the comparison a user sees is default Claude Code; the forced 153k window was a pilot artefact. If Claude Code's default never compacts under 1M on R4000 (then e-native ≡ e-none up to noise) the result still answers "what does the default do"; add a `e-native-95k` sensitivity arm (`CLAUDE_CODE_AUTO_COMPACT_WINDOW` sized to fire near 95k, confirmed in the first trial's `compactions` field, `cc_stream.py:171`) on log L6400 only, 3 seeds, +$1.5.
+- Record every arm's exact env in the trial `config.json` (already Harbor's) and the proxy image tag (`D_IMAGE`, line 130) in `docs/context-research.md` run E header.
+- `bench/ci.py` (new, ~60 lines): reads `summarize.py` trial rows, prints per-cell mean / n / bootstrap CI and paired differences by seed. `bench/stats.py` is a token-count script, not statistics; do not extend it.
+- Seeds: 45, 46, 47 (held out; tasks already packaged: `bench/cb/tasks/*-seed45..49` exist for all sizes). Seed 44 is never used for scoring.
+- Queue `bench/ctx/queue-E.txt`; run with `python bench/ctx_sched.py 4 ctx/queue-E.txt`; summarise with `python bench/summarize.py results-E state-E` and `python bench/ci.py results-E`.
+- Smoke: one `e-smart` log L1600 seed 45 trial ($0.11) to confirm the 95k trim fires (`state-E/context/sessions/<sid>.log.jsonl`, `summarize.py:96-100`).
+
+### Phase 1: three arms, same day, held-out seeds (H1–H3, H5, H6)
+
+Cells: {e-none, e-native, e-smart} × {kv R4000, log L1600, log L6400, log L12800} × seeds {45, 46, 47}, except e-none at L12800 limited to 2 seeds (765k prompts, ~$3 each) and kv R16000 excluded (e-none hits the 979k step limit, not a recall score; `context-research.md` Limitations).
+
+| Arm | Trials | $ |
+| --- | --- | --- |
+| e-smart | 12 | 3 × (0.11 + 0.27 + 0.30 + 0.58) = 3.8 |
+| e-native (default) | 12 | 3 × (0.31 + 0.5 + 0.5 + 1.0) = 6.9 (if it never compacts, cost rises toward e-none: +5) |
+| e-none | 11 | 3 × (0.57 + 1.5 + 1.5) + 2 × 3 = 16.7 |
+| e-native-95k (optional) | 3 | 1.5 |
+| reruns (20%) | ~7 | 5 |
+
+Phase 1 total ≈ **$34**, 38 trials, ≈ 2.5–3 h wall. All arms interleaved in one queue so API load is shared.
+
+Analysis: H1–H3 paired by seed; H5 from result.json agent wall time, same run; H6 **within e-smart**: `usage.jsonl` rows with `trimmed = 1` vs untrimmed rows of the same session, median and bootstrap CI of the difference, plus `events` table trim rows (`core.py:438`) for the hook's own time if recorded — check `record_usage` in `proxy/openmem_context/tokens.py`; if latency_s is measured outside the hook, say so. Also report requests per trial and searches per trial (sc_search, sc_sql columns, `summarize.py:232`).
+
+Stop rule: if after the first 12 trials (one seed per cell) e-smart mean `reward_lenient` < e-none − 0.15, pause and write it up before spending more.
+
+### Phase 2: scratchpad feature (no API spend, ~1 day)
+
+Design (minimal; cache-safe):
+
+- Storage: new table `pins (id INTEGER PRIMARY KEY, at TEXT, text TEXT)` as schema version 3 in `proxy/openmem_context/sessiondb.py` (migrations at lines 21-40; JS twin in `cli/mcp/context-store.mjs` / `cli/coding-agent/context-trim.mjs:370-381`).
+- Write path: MCP tool `smart_compact_pin(session_id, text, replace=false)` in `cli/mcp/context-store.mjs` next to `smart_compact_search` (line 60); ≤ 2,000 chars total, newest-wins on `replace`; returns the current pinned block so it also sits in recent context.
+- Read path: `_build_note` (`proxy/openmem_context/core.py:270-290`) appends `[openmem: pinned notes]\n<pins>` after `pointer_text`. The note is frozen per trim (`threads[root]["note"]`, line 416), so the pinned block changes only at trim time and the cached prefix stays byte-identical between trims. Pins written after a trim appear at the next trim; the tool result carries them meanwhile.
+- Prompting: one sentence in `STANDING_INSTRUCTION` (`core.py:40`) and the pointer: "Decisions you must keep: smart_compact_pin(...)". Add the same to `cli/coding-agent/context-trim.mjs` for parity.
+- Settings: `CONTEXT_PIN_MAX_CHARS` (default 2000) in `settings.py` dataclass (line 70+), 0 disables.
+
+Tests:
+
+- `proxy/tests/test_context_proxy.py` (87 tests; run `cd proxy && python3 -m pytest -q`, pytest from `bench/.venv/bin/pytest` if the host lacks it): trim note contains pins; pins survive a second trim; `replace` semantics; byte-identical note between two requests after one trim with a pin added in between (cache invariant).
+- `proxy/tests/test_session_db.py`: schema v3 migration from v2 db.
+- `scripts/tests/test_context_store.mjs` + new `scripts/tests/test_context_store_pin.mjs` (`npm test` runs `scripts/tests/test_*.mjs`, `package.json:88`): tool schema, cap, replace, read-only tools untouched.
+- `scripts/tests/test_context_parity.mjs`: JS port renders the same note.
+- Docs: `docs/context-management.md` tool table, `proxy/README.md` if it lists env vars, CLAUDE.md context section.
+
+### Phase 3: decision-consistency task (H7, H8)
+
+Research summary (what exists, why it does or does not fit):
+
+| Benchmark | What it scores | Fit for (c) | Harness cost |
+| --- | --- | --- | --- |
+| ContextBench `sudoku_sketchpad` (vendored, `bench/cb/contextbench/tasks/sudoku_sketchpad`, tasks t4–t150 seeds 42–49) | agent tracks board state across turns; the grader folds actions | **Good**: the agent's own state, not user facts; already packaged for strict mode | zero |
+| ContextBench `needle_retention` (vendored) | gold lines verbatim in the *final context window* | **Unusable**: scores retained context, so any trimmer scores 0 by construction | — |
+| Compact-Bench Task Preservation ([arXiv 2610.31710](https://arxiv.org/abs/2610.31710)); adapter `bench/compact-bench-openmem/` exists, clone `bench/compact-bench` absent (needs commit 9dd9a3f) | continuation of SWE repairs after compaction of the agent's own progress | **Best external match** for (c) | high: repo venvs, 33 tasks, 2 at a time, several hours; defer to phase 6 |
+| [LongMemEval](https://arxiv.org/abs/2410.10813), [LoCoMo](https://arxiv.org/abs/2402.17753), [MemoryAgentBench](https://arxiv.org/abs/2507.05257) | QA over user-supplied history (MemoryAgentBench adds conflict resolution) | tests retrieval of *given* facts, which kv/log already cover; none scores the agent's own decisions | medium: new stream adapter |
+
+Decision: run **sudoku-sketchpad** (existing) and add one small family, `decision_ledger`, to `bench/cb/contextbench/tasks/` (register in `FAMILIES`, `bench/cb/contextbench/tasks/scaffold.py:25`; same `generate(seed, ...)` shape as `log_triage/generate.py:74`):
+
+- Stream: N rounds. Round k gives the agent two or three valid options for item k ("pick the codename for service k from {A,B,C}") and asks it to reply `DECIDE k=<choice>`. Filler batches (log lines reused from `log_triage`) push the context past 95k. Then Q questions: "which codename did you pick for k?" → `ANSWER k=<choice>`, plus M "apply your choice" questions whose correct answer is a deterministic function of the earlier choice (e.g. "the port for k is 8000 + index of your choice").
+- Grader (`bench/cb/contextbench/graders/decision.py`, mirrors `graders/log.py:64`): parse the agent's `DECIDE` lines from early turns (not from gold), compare each `ANSWER` to the agent's own decision; reward = fraction consistent. Deterministic; no LLM judge.
+- Sizes: `D40-F2600` (40 decisions, 2,600 filler lines, ~118k streamed tokens, one trim) and `D80-F6400` (3–4 trims). Seeds 45–47.
+
+Cells: {e-none, e-smart, e-smart-pin} × {sudoku t150, ledger D40-F2600, ledger D80-F6400} × 3 seeds = 27 trials; e-native omitted (H7/H8 are about eviction, not summaries). Cost ≈ 27 × 0.35 = **$9.5**, ≈ 1.5 h.
+
+Decision rules: H7, H8 as in §1. If `e-smart` scores ≥ `e-none` on both families, record "eviction loss not observed at these sizes" and ship the scratchpad only if H8's lift is ≥ 0.05 with CI excluding 0.
+
+### Phase 4: Sonnet, one task, end of programme (H4)
+
+Model `claude-sonnet-5` (confirm id against the LiteLLM route `bench/litellm/bench-config.yaml:7-12`; flat 1M pricing, so `PRICE` in `summarize.py:31` needs a per-model table — add `PRICES[model]`, and `site/scripts/bench/compute-contextbench.py` inherits it). Trim threshold: leave `CONTEXT_TRIM_AT_TOKENS` unset so Sonnet gets its per-model default of 125k (`settings.py:11`, `DEFAULT_TRIM_AT_TOKENS`); Haiku's 95k override (`MODEL_TRIM_AT_TOKENS`, `settings.py:14`) does not apply to Sonnet.
+
+Cells: {e-none, e-smart} × log L6400 × seeds {45, 46} = 4 trials. Cost per trial at Sonnet rates (~20× Haiku base, no cliff): none ≈ 10M tokens mostly cache reads ≈ $4–6; smart ≈ 7M ≈ $2–3. Total ≈ **$16**; abort after the first pair if it exceeds $10. The first pair cost $12.71, over the abort threshold; the user waived the abort rule and approved a second pair, and Sonnet stops there. Report cost ratio with CI and the token ratio; H4 decides how the cost claim is worded.
+
+### Phase 5: threshold ablation (optional, cheap)
+
+Only if phase 1 shows trims drive cost (cost correlates with trim count across trials, Spearman ρ > 0.5). e-smart only, log L6400 seed 45, grid `CONTEXT_TRIM_AT_TOKENS ∈ {60k, 95k, 120k}` × `CONTEXT_TRIM_TO_FRACTION ∈ {0.3, 0.6}` (`settings.py:16,80,111`), 1 trial each = 6 trials ≈ **$2**, 40 min. Report score, cost, trims, cache hit rate; pick nothing from it without a confirmation seed.
+
+### Phase 6 (deferred, not budgeted): Compact-Bench Task Preservation
+
+Re-clone `cai-jianfeng/compact-bench` at 9dd9a3f into `bench/compact-bench`, run `bench/compact-bench-openmem/run.sh --instance pylint-dev__pylint-6903 --methods claude,openmem --rounds 1` as smoke, then `drive.py`. Needs its own budget line (33 tasks × 4 methods on Haiku; estimate after the smoke).
+
+### Cheap aside for risk (a)
+
+In phase 1 e-smart trials Claude Code's auto-compaction is disabled. One extra e-smart trial on log L6400 with `DISABLE_AUTO_COMPACT` unset ($0.30) shows whether the two managers ever collide at 95k; record `compactions` and trims. Not prioritised.
+
+## 3. Budget and schedule
+
+| Phase | Trials | $ | Wall |
+| --- | --- | --- | --- |
+| 0 prep + smoke | 1 | 0.1 | 2 h build |
+| 1 three arms | 38 | 34 | 3 h |
+| 2 scratchpad | 0 | 0 | 1 day build |
+| 3 decision task | 27 | 9.5 | 1.5 h + 0.5 day build |
+| 4 Sonnet | 4 | 16 | 1 h |
+| 5 ablation | 6 | 2 | 40 min |
+| a aside | 1 | 0.3 | — |
+| **Total** | **77** | **≈ 62 (cap 75)** | 2.5 days build, ~6 h run |
+
+Phase 4 is the overrun risk; its abort rule keeps the programme under $60 if Sonnet costs come in high.
+
+## 4. Harness pitfalls (from the code)
+
+- Retries must `--resume`; a retried turn with `--session-id` collides (`bench/cc_stream.py:345`). Retry events are counted in `cc_retry_events` (`summarize.py:203`).
+- Rate limits: `RATE_LIMIT_RE` (`summarize.py:29`) classes 429/529/overloaded; run B lost trials to this. Keep MAX = 4 concurrent; stagger is 5 s (`ctx_sched.py:28`).
+- Ports are hard-coded per run:arm (`run_ctx.sh:51-55`); a new run needs new entries or proxies collide.
+- `TRIM_AT` semantics: unset → 120000; set-empty → proxy per-model threshold (`run_strict.sh:29`, `run_ctx.sh:45`).
+- kv `#tag` drops make strict `reward` 0 while values are right; compare `reward_lenient`, rerun format-invalid trials with `bench/rerun_strict.sh`.
+- `ctx_sched.py` skips tasks whose dir is missing (`ctx/skipped.txt`) and only exits when `ctx/STOP` exists.
+- Results dirs contain root-owned files from Docker; write summaries elsewhere (`bench/ctx/`), never edit `results-*`.
+- `needle_retention` grades the final context window; do not run it against any trimming arm.
+- Counts in log-triage overcount when the agent's own output echoes log lines (`context-research.md` "Echo overcount"); unchanged in run E, reported separately as `count` sub-score.
+- The proxy's `fail_open` event means an untrimmed request went out silently (`core.py:355`); check `events` per session before scoring an e-smart trial.
+- `latency_s` in `usage.jsonl` may or may not include the pre-call hook; verify in `tokens.py:record_usage` before claiming H6.
+
+## 5. Docs to update after the runs
+
+- `docs/context-research.md`: new "Run E" section (method, every trial, CIs, exclusions), "Run E-S" (Sonnet), "Run E-D" (decision task), Limitations rewritten, Next steps pruned.
+- `docs/context-management.md`: Benchmarks tables and Time/Cost sections regenerated from run E (same-run baselines; drop the cross-run caveat at line 93); the "three changes since" block (lines 125-139) folds into the main table; scratchpad tool documented.
+- `site/scripts/bench/compute-contextbench.py:27-29`: point at `results-E/state-E` for all three arms; add per-model prices; `cd site && pnpm media` regenerates `contextbench-*.png`; `site/src/data/contextbench.json` committed.
+- `README.md:30-36`: chart alt text and link; today the chart is run D (120k trim, no pointer) while the text quotes Df, fix by regenerating from run E. If H1–H3 fail, replace the chart with the null result sentence.
+- `CLAUDE.md`: bench section mentions run E layout and `bench/ci.py`.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bcmyguest/openmem",
-  "version": "0.8.0",
+  "version": "0.9.0",
   "description": "OpenMem — memory interface for coding agents: pluggable providers (Hindsight, mem0, Cognee, …), shared wizard/indexer, multi-format import/export (PMM and more).",
   "type": "module",
   "license": "Apache-2.0",

package/proxy/openmem_context/core.py

@@ -100,8 +100,10 @@ def pointer_text(sid, dropped):
         f"The tools smart_compact_search and smart_compact_sql are available to you in this request. "
         f'You MUST call smart_compact_search(session_id="{sid}", query=... or pattern=...) before answering '
         f'anything that depends on them, and never answer "not found", "never received" or "no search tool" '
-        f"without calling it; "
-        f'every message of this session is stored and queryable with '
+        f"without calling it. "
+        f"To recover a file you read earlier, search for the text you need instead of reading it again, "
+        f"and re-read only the lines you still lack (offset/limit). "
+        f"Every message of this session is stored and queryable with "
         f'smart_compact_search and smart_compact_sql(session_id="{sid}", sql=...).]'
     )
 

package/proxy/openmem_context/settings.py

@@ -8,7 +8,7 @@ from pathlib import Path
 from .store import ASCII_WHITESPACE
 from .tokens import DEFAULT_CHARS_PER_TOKEN
 
-DEFAULT_TRIM_AT_TOKENS = 120000
+DEFAULT_TRIM_AT_TOKENS = 125000
 # Trim thresholds by model family (a substring of the lowercased model id), used while CONTEXT_TRIM_AT_TOKENS is
 # unset. Haiku's prompts over 100k tokens bill at a higher tier, so its sessions trim below it.
 MODEL_TRIM_AT_TOKENS = (("haiku", 95000),)

package/scripts/run-setup.sh

@@ -1,18 +1,27 @@
 #!/usr/bin/env bash
 # Non-interactive checkout bootstrap (CI, remote shells without a TTY).
-# Prefer: npx openmem  or  cd checkout && npx . setup
+# Prefer: ./setup  or  npm run setup
+# Do not use `npx . setup` here: a checkout does not link its own package bins
+# into node_modules/.bin, so npx fails with "openmem: not found" (exit 127).
 set -euo pipefail
 REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
 cd "${REPO}"
 
 npm install --no-fund --no-audit
 
+run_setup() {
+  if [ -e "${REPO}/setup" ]; then
+    exec "${REPO}/setup" "$@"
+  fi
+  exec npm run setup -- "$@"
+}
+
 if [ -f "${HOME}/.config/hindsight/wizard.conf" ]; then
-  exec npx . setup --yes --no-smoke "$@"
+  run_setup --yes --no-smoke "$@"
 fi
 
 if [ -t 0 ] && [ -t 1 ]; then
-  exec npx . setup --no-smoke "$@"
+  run_setup --no-smoke "$@"
 fi
 
 cat <<EOF
@@ -23,8 +32,8 @@ Run the wizard on a terminal (Docker, provider, indexing):
   cd ${REPO}
   mkdir -p ~/.config/personal-memory
   cp config/openmem.toml.example ~/.config/personal-memory/config.toml
-  npx . setup
+  ./setup
 
-Or from npm (no clone): npx openmem
+Or from npm (no clone): npx @bcmyguest/openmem
 EOF
 exit 0

package/wizard/lib/pm-config.mjs

@@ -12,7 +12,7 @@ const DEFAULT_CONTEXT_PROXY_PORT = 4100;
 export const CONTEXT_AGENT_KEYS = ["claude-code", "opencode1", "codex", "opencode2"];
 export const CONTEXT_PROXY_AGENT_KEYS = ["claude-code", "opencode1", "codex"];
 // Same defaults as proxy/openmem_context/settings.py and cli/coding-agent/context-trim.mjs.
-const DEFAULT_TRIM_AT_TOKENS = 120000;
+const DEFAULT_TRIM_AT_TOKENS = 125000;
 const DEFAULT_COMPRESS_OVER_TOKENS = 4000;
 const DEFAULT_COMPACTION_MAX_CHARS = 12000;
 /** Config keys that reach the proxy container (stack .env) and the OpenCode 2 plugin as env. */
@@ -285,7 +285,7 @@ export function contextAgentsFromPmConfig(cfg) {
 }
 
 /**
- * Trim settings with defaults: `[context_proxy] trim_at_tokens` (120000; unset, the proxy and plugin trim Haiku at 95000), `compress_over_tokens`
+ * Trim settings with defaults: `[context_proxy] trim_at_tokens` (125000; unset, the proxy and plugin trim Haiku at 95000), `compress_over_tokens`
  * (4000), `standing_instruction` (true), `[compaction] max_chars` (12000), `native` (true). Env overrides are applied by
  * `contextTrimEnvFromPmConfig` + `trimSettings()`, not here.
  * @param {Record<string, unknown>} cfg