context-compress 2026.3.21 → 2026.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +258 -44
- package/dist/cli/doctor.d.ts.map +1 -1
- package/dist/cli/doctor.js +2 -10
- package/dist/cli/doctor.js.map +1 -1
- package/dist/cli/filter.d.ts +52 -0
- package/dist/cli/filter.d.ts.map +1 -0
- package/dist/cli/filter.js +200 -0
- package/dist/cli/filter.js.map +1 -0
- package/dist/cli/index.d.ts +8 -4
- package/dist/cli/index.d.ts.map +1 -1
- package/dist/cli/index.js +19 -6
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/lite.d.ts +15 -0
- package/dist/cli/lite.d.ts.map +1 -0
- package/dist/cli/lite.js +37 -0
- package/dist/cli/lite.js.map +1 -0
- package/dist/cli/setup.d.ts +23 -1
- package/dist/cli/setup.d.ts.map +1 -1
- package/dist/cli/setup.js +122 -21
- package/dist/cli/setup.js.map +1 -1
- package/dist/executor.d.ts +7 -1
- package/dist/executor.d.ts.map +1 -1
- package/dist/executor.js +51 -4
- package/dist/executor.js.map +1 -1
- package/dist/filters.d.ts +52 -0
- package/dist/filters.d.ts.map +1 -0
- package/dist/filters.js +719 -0
- package/dist/filters.js.map +1 -0
- package/dist/hooks/pretooluse.js +57 -0
- package/dist/hooks/pretooluse.js.map +1 -1
- package/dist/network.d.ts.map +1 -1
- package/dist/network.js +11 -0
- package/dist/network.js.map +1 -1
- package/dist/server.bundle.mjs +1333 -619
- package/dist/server.bundle.mjs.map +4 -4
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +44 -610
- package/dist/server.js.map +1 -1
- package/dist/stats.d.ts +7 -1
- package/dist/stats.d.ts.map +1 -1
- package/dist/stats.js +65 -0
- package/dist/stats.js.map +1 -1
- package/dist/store.d.ts +1 -0
- package/dist/store.d.ts.map +1 -1
- package/dist/store.js +15 -2
- package/dist/store.js.map +1 -1
- package/dist/tools/batch-execute.d.ts +4 -0
- package/dist/tools/batch-execute.d.ts.map +1 -0
- package/dist/tools/batch-execute.js +75 -0
- package/dist/tools/batch-execute.js.map +1 -0
- package/dist/tools/context.d.ts +17 -0
- package/dist/tools/context.d.ts.map +1 -0
- package/dist/tools/context.js +2 -0
- package/dist/tools/context.js.map +1 -0
- package/dist/tools/discover.d.ts +4 -0
- package/dist/tools/discover.d.ts.map +1 -0
- package/dist/tools/discover.js +65 -0
- package/dist/tools/discover.js.map +1 -0
- package/dist/tools/execute-file.d.ts +4 -0
- package/dist/tools/execute-file.d.ts.map +1 -0
- package/dist/tools/execute-file.js +66 -0
- package/dist/tools/execute-file.js.map +1 -0
- package/dist/tools/execute.d.ts +4 -0
- package/dist/tools/execute.d.ts.map +1 -0
- package/dist/tools/execute.js +54 -0
- package/dist/tools/execute.js.map +1 -0
- package/dist/tools/fetch-and-index.d.ts +4 -0
- package/dist/tools/fetch-and-index.d.ts.map +1 -0
- package/dist/tools/fetch-and-index.js +91 -0
- package/dist/tools/fetch-and-index.js.map +1 -0
- package/dist/tools/index-content.d.ts +4 -0
- package/dist/tools/index-content.d.ts.map +1 -0
- package/dist/tools/index-content.js +85 -0
- package/dist/tools/index-content.js.map +1 -0
- package/dist/tools/search.d.ts +4 -0
- package/dist/tools/search.d.ts.map +1 -0
- package/dist/tools/search.js +57 -0
- package/dist/tools/search.js.map +1 -0
- package/dist/tools/stats.d.ts +4 -0
- package/dist/tools/stats.d.ts.map +1 -0
- package/dist/tools/stats.js +10 -0
- package/dist/tools/stats.js.map +1 -0
- package/dist/types.d.ts +11 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/util/auto-mode.d.ts +40 -0
- package/dist/util/auto-mode.d.ts.map +1 -0
- package/dist/util/auto-mode.js +181 -0
- package/dist/util/auto-mode.js.map +1 -0
- package/dist/util/fetch-code.d.ts +10 -0
- package/dist/util/fetch-code.d.ts.map +1 -0
- package/dist/util/fetch-code.js +87 -0
- package/dist/util/fetch-code.js.map +1 -0
- package/dist/util/intent-filter.d.ts +17 -0
- package/dist/util/intent-filter.d.ts.map +1 -0
- package/dist/util/intent-filter.js +28 -0
- package/dist/util/intent-filter.js.map +1 -0
- package/dist/util/label.d.ts +4 -0
- package/dist/util/label.d.ts.map +1 -0
- package/dist/util/label.js +14 -0
- package/dist/util/label.js.map +1 -0
- package/dist/util/path.d.ts +8 -0
- package/dist/util/path.d.ts.map +1 -0
- package/dist/util/path.js +21 -0
- package/dist/util/path.js.map +1 -0
- package/dist/util/stream-compress.d.ts +36 -0
- package/dist/util/stream-compress.d.ts.map +1 -0
- package/dist/util/stream-compress.js +104 -0
- package/dist/util/stream-compress.js.map +1 -0
- package/dist/util/version.d.ts +2 -0
- package/dist/util/version.d.ts.map +1 -0
- package/dist/util/version.js +15 -0
- package/dist/util/version.js.map +1 -0
- package/docs/token-reduction-report.md +164 -88
- package/hooks/pretooluse.mjs +38 -0
- package/package.json +5 -4
|
@@ -6,6 +6,8 @@
|
|
|
6
6
|
> provides a detailed before/after comparison for 12 common operations,
|
|
7
7
|
> and addresses the natural question: "doesn't less tokens mean losing context?"
|
|
8
8
|
|
|
9
|
+
**Version**: 2026.5.0 | **Last updated**: 2026-05-10
|
|
10
|
+
|
|
9
11
|
---
|
|
10
12
|
|
|
11
13
|
## Table of Contents
|
|
@@ -17,6 +19,7 @@
|
|
|
17
19
|
- [Context Window Impact](#context-window-impact)
|
|
18
20
|
- [Cost Impact](#cost-impact)
|
|
19
21
|
- [Deep Dive: How Playwright Snapshot Goes from 56KB to 299B](#deep-dive-how-playwright-snapshot-goes-from-56kb-to-299b)
|
|
22
|
+
- [Security and Reliability](#security-and-reliability)
|
|
20
23
|
- [FAQ: Doesn't Less Tokens Mean Losing Context?](#faq-doesnt-less-tokens-mean-losing-context)
|
|
21
24
|
|
|
22
25
|
---
|
|
@@ -26,28 +29,48 @@
|
|
|
26
29
|
Every byte of tool output that enters Claude Code's context window **consumes tokens permanently**. In a typical coding session:
|
|
27
30
|
|
|
28
31
|
```
|
|
29
|
-
Read a bundled file → 776KB →
|
|
30
|
-
Playwright browser snapshot → 56KB →
|
|
31
|
-
npm test (42 tests) → 4KB →
|
|
32
|
-
git diff (3 commits) → 8KB →
|
|
32
|
+
Read a bundled file → 776KB → 155K-259K tokens
|
|
33
|
+
Playwright browser snapshot → 56KB → 11K-19K tokens
|
|
34
|
+
npm test (42 tests) → 4KB → 748-1,246 tokens
|
|
35
|
+
git diff (3 commits) → 8KB → 1,600-2,667 tokens
|
|
33
36
|
─────────────────
|
|
34
|
-
Total:
|
|
35
|
-
←
|
|
37
|
+
Total: 169K-282K tokens
|
|
38
|
+
← can overflow 200K window
|
|
36
39
|
```
|
|
37
40
|
|
|
38
|
-
|
|
41
|
+
> **Token estimation**: 1 token ≈ 3-5 bytes depending on content. We use a range (bytes/5 to bytes/3) because Anthropic does not publish a local tokenizer for Claude 3+ models.
|
|
42
|
+
|
|
43
|
+
With just 4 operations, you risk **overflowing the entire context window**. Earlier conversation messages get compressed or lost. The agent forgets what you asked. Quality degrades.
|
|
39
44
|
|
|
40
45
|
The worst part: **99% of that tool output is noise** — import statements, boilerplate, minified code, irrelevant test output. The agent doesn't benefit from seeing it. It just crowds out the conversation.
|
|
41
46
|
|
|
42
47
|
---
|
|
43
48
|
|
|
44
|
-
## The Solution:
|
|
49
|
+
## The Solution: 4-Layer Architecture
|
|
45
50
|
|
|
46
51
|
context-compress doesn't delete data — it **defers** it. All data is preserved and searchable. Only the relevant parts enter context.
|
|
47
52
|
|
|
53
|
+
### Layer 0: Command-Specific Output Filters (v2026.3.22)
|
|
54
|
+
|
|
55
|
+
Before generic compression, output passes through command-aware filters that strip noise specific to each tool:
|
|
56
|
+
|
|
57
|
+
| Command | What's Stripped | Typical Savings |
|
|
58
|
+
|:--|:--|:--|
|
|
59
|
+
| `git push/pull/fetch/clone` | Remote progress lines, object counting, delta resolution | 40-60% |
|
|
60
|
+
| `git status` | Hint lines (`use "git add"...`), blank lines | 20-30% |
|
|
61
|
+
| `npm/yarn/pnpm install` | Deprecation warnings, funding prompts, tree-drawing chars | 30-50% |
|
|
62
|
+
| `npm test / jest / vitest / pytest` | Passing test details, keeps only failures + summary | 70-95% |
|
|
63
|
+
| `cargo build / make / gradle` | Download progress, "Compiling X/Y", lock waits | 50-70% |
|
|
64
|
+
| `docker build` | Layer hash lines (` ---> abc123`), build context transfer | 30-50% |
|
|
65
|
+
| `ls -R / find / tree` | Directory grouping for large listings (50+ files) | 60-80% |
|
|
66
|
+
|
|
67
|
+
Additionally, all output passes through:
|
|
68
|
+
- **ANSI stripping**: Terminal escape codes (colors, cursor movement) are always removed — pure noise for LLMs
|
|
69
|
+
- **Progress line removal**: Spinner characters, percentage bars, download/ETA lines are filtered
|
|
70
|
+
|
|
48
71
|
### Layer 1: Sandbox Execution
|
|
49
72
|
|
|
50
|
-
The agent writes code to process data. Only `console.log()` output enters context.
|
|
73
|
+
The agent writes code to process data. Only `console.log()` output enters context. 11 languages supported: JavaScript, TypeScript, Python, Shell, Ruby, Go, Rust, PHP, Perl, R, Elixir.
|
|
51
74
|
|
|
52
75
|
```
|
|
53
76
|
execute_file("server.bundle.mjs", code: `
|
|
@@ -61,9 +84,11 @@ Context: 420 bytes (the extracted schema)
|
|
|
61
84
|
|
|
62
85
|
The agent isn't blindly losing context — it's **choosing** what matters via code.
|
|
63
86
|
|
|
87
|
+
**Safeguards**: Code input limited to 1MB. Subprocess timeout (default 30s). Output hard cap (100MB). Process group kill on timeout. Concurrent executions limited to 8 globally.
|
|
88
|
+
|
|
64
89
|
### Layer 2: FTS5 Knowledge Base
|
|
65
90
|
|
|
66
|
-
Full data is stored in a searchable SQLite FTS5 database with BM25 ranking, Porter stemming,
|
|
91
|
+
Full data is stored in a searchable SQLite FTS5 database with BM25 ranking, Porter stemming, trigram matching, and Levenshtein fuzzy correction (with early-exit optimization).
|
|
67
92
|
|
|
68
93
|
```
|
|
69
94
|
index(path: "snapshot.md") → 56KB stored, 42 chunks created
|
|
@@ -74,6 +99,8 @@ search("order table row headers") → 180B match returned
|
|
|
74
99
|
|
|
75
100
|
Data is **not lost**. It's **indexed and searchable on demand**.
|
|
76
101
|
|
|
102
|
+
**Persistence option**: Set `persistDb: true` in config to survive MCP server restarts.
|
|
103
|
+
|
|
77
104
|
### Layer 3: Intent-Based Auto-Filter
|
|
78
105
|
|
|
79
106
|
When the agent provides an `intent` parameter, large outputs are automatically filtered:
|
|
@@ -93,116 +120,130 @@ Small outputs are **never compressed**. Large outputs are filtered by what was a
|
|
|
93
120
|
|
|
94
121
|
The following comparison uses realistic output sizes measured from the context-compress project itself.
|
|
95
122
|
|
|
96
|
-
> **Token calculation**: 1 token ≈
|
|
123
|
+
> **Token calculation**: 1 token ≈ 3-5 bytes. The "Tokens" column shows the midpoint estimate (bytes/4). See [Cost Impact](#cost-impact) for range-based calculations.
|
|
97
124
|
|
|
98
125
|
### 1. Read large source file (server.ts ~21KB)
|
|
99
126
|
|
|
100
|
-
| | Bytes | Tokens | Method |
|
|
127
|
+
| | Bytes | Tokens (est.) | Method |
|
|
101
128
|
|:--|--:|--:|:--|
|
|
102
|
-
| **Before** | 21,000 | 5,250 | `Read` tool → full file dumped into context |
|
|
103
|
-
| **After** | 350 | 88 | `execute_file` → agent prints only what it needs |
|
|
104
|
-
| **Saved** | |
|
|
129
|
+
| **Before** | 21,000 | ~5,250 | `Read` tool → full file dumped into context |
|
|
130
|
+
| **After** | 350 | ~88 | `execute_file` → agent prints only what it needs |
|
|
131
|
+
| **Saved** | | **~5,162** | **98.3% reduction** |
|
|
105
132
|
|
|
106
133
|
### 2. Read bundled file (server.bundle.mjs ~776KB)
|
|
107
134
|
|
|
108
|
-
| | Bytes | Tokens | Method |
|
|
135
|
+
| | Bytes | Tokens (est.) | Method |
|
|
109
136
|
|:--|--:|--:|:--|
|
|
110
|
-
| **Before** | 776,304 | 194,076 | `Read` tool → full file in context (truncated at 2000 lines) |
|
|
111
|
-
| **After** | 420 | 105 | `execute_file` → extract specific function/pattern |
|
|
112
|
-
| **Saved** | |
|
|
137
|
+
| **Before** | 776,304 | ~194,076 | `Read` tool → full file in context (truncated at 2000 lines) |
|
|
138
|
+
| **After** | 420 | ~105 | `execute_file` → extract specific function/pattern |
|
|
139
|
+
| **Saved** | | **~193,971** | **99.9% reduction** |
|
|
113
140
|
|
|
114
141
|
### 3. npm test output (42 tests, ~3.7KB)
|
|
115
142
|
|
|
116
|
-
| | Bytes | Tokens | Method |
|
|
143
|
+
| | Bytes | Tokens (est.) | Method |
|
|
117
144
|
|:--|--:|--:|:--|
|
|
118
|
-
| **Before** | 3,739 | 935 | `Bash` → full stdout in context |
|
|
119
|
-
| **After** | 180 | 45 | `execute` with `intent: "failing tests"` → summary only |
|
|
120
|
-
| **Saved** | |
|
|
145
|
+
| **Before** | 3,739 | ~935 | `Bash` → full stdout in context |
|
|
146
|
+
| **After** | 180 | ~45 | `execute` with `intent: "failing tests"` → summary only |
|
|
147
|
+
| **Saved** | | **~890** | **95.2% reduction** |
|
|
121
148
|
|
|
122
149
|
### 4. git log (full history, ~5KB)
|
|
123
150
|
|
|
124
|
-
| | Bytes | Tokens | Method |
|
|
151
|
+
| | Bytes | Tokens (est.) | Method |
|
|
125
152
|
|:--|--:|--:|:--|
|
|
126
|
-
| **Before** | 5,000 | 1,250 | `Bash git log` → all commits in context |
|
|
127
|
-
| **After** | 250 | 63 | `execute` + `search` for specific commits |
|
|
128
|
-
| **Saved** | |
|
|
153
|
+
| **Before** | 5,000 | ~1,250 | `Bash git log` → all commits in context |
|
|
154
|
+
| **After** | 250 | ~63 | `execute` + `search` for specific commits |
|
|
155
|
+
| **Saved** | | **~1,187** | **95.0% reduction** |
|
|
129
156
|
|
|
130
157
|
### 5. git diff (3 commits, ~8KB)
|
|
131
158
|
|
|
132
|
-
| | Bytes | Tokens | Method |
|
|
159
|
+
| | Bytes | Tokens (est.) | Method |
|
|
133
160
|
|:--|--:|--:|:--|
|
|
134
|
-
| **Before** | 8,000 | 2,000 | `Bash git diff` → full patch in context |
|
|
135
|
-
| **After** | 400 | 100 | `execute` + `search` for changed functions |
|
|
136
|
-
| **Saved** | |
|
|
161
|
+
| **Before** | 8,000 | ~2,000 | `Bash git diff` → full patch in context |
|
|
162
|
+
| **After** | 400 | ~100 | `execute` + `search` for changed functions |
|
|
163
|
+
| **Saved** | | **~1,900** | **95.0% reduction** |
|
|
137
164
|
|
|
138
165
|
### 6. grep across codebase (~1.4KB)
|
|
139
166
|
|
|
140
|
-
| | Bytes | Tokens | Method |
|
|
167
|
+
| | Bytes | Tokens (est.) | Method |
|
|
141
168
|
|:--|--:|--:|:--|
|
|
142
|
-
| **Before** | 1,442 | 361 | `Grep` → all matching lines in context |
|
|
143
|
-
| **After** | 1,442 | 361 | Same — small output passes through as-is |
|
|
169
|
+
| **Before** | 1,442 | ~361 | `Grep` → all matching lines in context |
|
|
170
|
+
| **After** | 1,442 | ~361 | Same — small output passes through as-is |
|
|
144
171
|
| **Saved** | | **0** | **0% — no overhead for small outputs** |
|
|
145
172
|
|
|
146
173
|
### 7. Playwright browser_snapshot (~56KB)
|
|
147
174
|
|
|
148
|
-
| | Bytes | Tokens | Method |
|
|
175
|
+
| | Bytes | Tokens (est.) | Method |
|
|
149
176
|
|:--|--:|--:|:--|
|
|
150
|
-
| **Before** | 56,000 | 14,000 | `browser_snapshot` → full accessibility tree in context |
|
|
151
|
-
| **After** | 299 | 75 | save → `index` → `search` for specific elements |
|
|
152
|
-
| **Saved** | |
|
|
177
|
+
| **Before** | 56,000 | ~14,000 | `browser_snapshot` → full accessibility tree in context |
|
|
178
|
+
| **After** | 299 | ~75 | save → `index` → `search` for specific elements |
|
|
179
|
+
| **Saved** | | **~13,925** | **99.5% reduction** |
|
|
153
180
|
|
|
154
181
|
### 8. curl API response (JSON ~12KB)
|
|
155
182
|
|
|
156
|
-
| | Bytes | Tokens | Method |
|
|
183
|
+
| | Bytes | Tokens (est.) | Method |
|
|
157
184
|
|:--|--:|--:|:--|
|
|
158
|
-
| **Before** | 12,000 | 3,000 | `Bash curl` → full JSON response in context |
|
|
159
|
-
| **After** | 350 | 88 | `execute` → extract specific fields with code |
|
|
160
|
-
| **Saved** | |
|
|
185
|
+
| **Before** | 12,000 | ~3,000 | `Bash curl` → full JSON response in context |
|
|
186
|
+
| **After** | 350 | ~88 | `execute` → extract specific fields with code |
|
|
187
|
+
| **Saved** | | **~2,912** | **97.1% reduction** |
|
|
161
188
|
|
|
162
189
|
### 9. fetch_and_index (web docs ~45KB)
|
|
163
190
|
|
|
164
|
-
| | Bytes | Tokens | Method |
|
|
191
|
+
| | Bytes | Tokens (est.) | Method |
|
|
165
192
|
|:--|--:|--:|:--|
|
|
166
|
-
| **Before** | 45,000 | 11,250 | `WebFetch` → full page markdown in context |
|
|
167
|
-
| **After** | 3,000 | 750 | `fetch_and_index` → 3KB preview + rest searchable |
|
|
168
|
-
| **Saved** | |
|
|
193
|
+
| **Before** | 45,000 | ~11,250 | `WebFetch` → full page markdown in context |
|
|
194
|
+
| **After** | 3,000 | ~750 | `fetch_and_index` → 3KB preview + rest searchable |
|
|
195
|
+
| **Saved** | | **~10,500** | **93.3% reduction** |
|
|
196
|
+
|
|
197
|
+
**Security**: SSRF protection with DNS rebinding prevention, IP pinning, redirect blocking, and 10MB response size limit. Prompt injection detection on fetched content.
|
|
169
198
|
|
|
170
199
|
### 10. batch_execute (5 commands, ~25KB total)
|
|
171
200
|
|
|
172
|
-
| | Bytes | Tokens | Method |
|
|
201
|
+
| | Bytes | Tokens (est.) | Method |
|
|
173
202
|
|:--|--:|--:|:--|
|
|
174
|
-
| **Before** | 25,000 | 6,250 | 5x `Bash` → all output in context |
|
|
175
|
-
| **After** | 1,500 | 375 | `batch_execute` + search across all in 1 call |
|
|
176
|
-
| **Saved** | |
|
|
203
|
+
| **Before** | 25,000 | ~6,250 | 5x `Bash` → all output in context |
|
|
204
|
+
| **After** | 1,500 | ~375 | `batch_execute` + search across all in 1 call |
|
|
205
|
+
| **Saved** | | **~5,875** | **94.0% reduction** |
|
|
206
|
+
|
|
207
|
+
**Performance**: Commands run with bounded concurrency (max 4 parallel). Global execution limit of 8 prevents resource exhaustion.
|
|
177
208
|
|
|
178
209
|
### 11. Read CSV/JSON data file (~100KB)
|
|
179
210
|
|
|
180
|
-
| | Bytes | Tokens | Method |
|
|
211
|
+
| | Bytes | Tokens (est.) | Method |
|
|
181
212
|
|:--|--:|--:|:--|
|
|
182
|
-
| **Before** | 100,000 | 25,000 | `Read` → file contents in context |
|
|
183
|
-
| **After** | 500 | 125 | `execute_file` → extract/aggregate specific data |
|
|
184
|
-
| **Saved** | |
|
|
213
|
+
| **Before** | 100,000 | ~25,000 | `Read` → file contents in context |
|
|
214
|
+
| **After** | 500 | ~125 | `execute_file` → extract/aggregate specific data |
|
|
215
|
+
| **Saved** | | **~24,875** | **99.5% reduction** |
|
|
185
216
|
|
|
186
217
|
### 12. npm install log (~15KB)
|
|
187
218
|
|
|
188
|
-
| | Bytes | Tokens | Method |
|
|
219
|
+
| | Bytes | Tokens (est.) | Method |
|
|
220
|
+
|:--|--:|--:|:--|
|
|
221
|
+
| **Before** | 15,000 | ~3,750 | `Bash npm install` → full install log in context |
|
|
222
|
+
| **After** | 200 | ~50 | `execute` with `intent: "errors"` → only issues shown |
|
|
223
|
+
| **Saved** | | **~3,700** | **98.7% reduction** |
|
|
224
|
+
|
|
225
|
+
### 13. npm test with ANSI + verbose output (~8KB, v2026.3.22)
|
|
226
|
+
|
|
227
|
+
| | Bytes | Tokens (est.) | Method |
|
|
189
228
|
|:--|--:|--:|:--|
|
|
190
|
-
| **Before** |
|
|
191
|
-
| **After** |
|
|
192
|
-
| **Saved** | |
|
|
229
|
+
| **Before** | 8,000 | ~2,000 | `Bash npm test` → full ANSI-colored verbose output in context |
|
|
230
|
+
| **After** | 350 | ~88 | Command filter strips ANSI + passing tests → only failures + summary |
|
|
231
|
+
| **Saved** | | **~1,912** | **95.6% reduction** |
|
|
232
|
+
|
|
233
|
+
**Pipeline**: ANSI stripping → command filter (test runner detection) → progress line removal → deduplication → smart truncation. All 5 layers applied automatically.
|
|
193
234
|
|
|
194
235
|
---
|
|
195
236
|
|
|
196
237
|
## Session Totals
|
|
197
238
|
|
|
198
|
-
Combining all
|
|
239
|
+
Combining all 13 operations from a single coding session:
|
|
199
240
|
|
|
200
241
|
```
|
|
201
|
-
BEFORE: 1,
|
|
202
|
-
AFTER: 9 KB → 2
|
|
242
|
+
BEFORE: 1,051 KB → ~263K tokens consumed (bytes/4 midpoint)
|
|
243
|
+
AFTER: 9 KB → ~2.3K tokens consumed
|
|
203
244
|
────────────────────────
|
|
204
|
-
SAVED: 1,
|
|
205
|
-
REDUCTION: 99.
|
|
245
|
+
SAVED: 1,042 KB → ~260K tokens
|
|
246
|
+
REDUCTION: 99.1%
|
|
206
247
|
```
|
|
207
248
|
|
|
208
249
|
---
|
|
@@ -216,42 +257,43 @@ Claude Code uses a 200K token context window.
|
|
|
216
257
|
│ 200,000 token context window │
|
|
217
258
|
│ │
|
|
218
259
|
│ WITHOUT context-compress: │
|
|
219
|
-
│ ████████████████████████████████████████████████████
|
|
220
|
-
│ ←
|
|
260
|
+
│ ████████████████████████████████████████████████████ ~132% │
|
|
261
|
+
│ ← 13 operations OVERFLOW the window. Conversation lost. │
|
|
221
262
|
│ │
|
|
222
263
|
│ WITH context-compress: │
|
|
223
|
-
│ █ 1.
|
|
224
|
-
│ ←
|
|
264
|
+
│ █ ~1.2% │
|
|
265
|
+
│ ← 13 operations use ~1.2%. ~98.8% free for conversation. │
|
|
225
266
|
└─────────────────────────────────────────────────────────────┘
|
|
226
267
|
```
|
|
227
268
|
|
|
228
269
|
| Metric | Before | After |
|
|
229
270
|
|:--|--:|--:|
|
|
230
|
-
| Tokens consumed |
|
|
231
|
-
| % of context window |
|
|
232
|
-
| Operations before compaction | ~9 | **~1,
|
|
233
|
-
| Conversation longevity | Short | **~
|
|
271
|
+
| Tokens consumed (est.) | ~263,000 | ~2,300 |
|
|
272
|
+
| % of context window | ~132% | ~1.2% |
|
|
273
|
+
| Operations before compaction | ~9 | **~1,080** |
|
|
274
|
+
| Conversation longevity | Short | **~117x longer** |
|
|
234
275
|
|
|
235
276
|
---
|
|
236
277
|
|
|
237
278
|
## Cost Impact
|
|
238
279
|
|
|
239
|
-
Input token pricing (per session, 12 operations):
|
|
280
|
+
Input token pricing (per session, 12 operations). Using midpoint estimate (bytes/4):
|
|
240
281
|
|
|
241
282
|
| Model | Before | After | Saved per Session |
|
|
242
283
|
|:--|--:|--:|--:|
|
|
243
|
-
|
|
|
244
|
-
|
|
|
284
|
+
| Haiku 4.5 ($0.80/MTok) | $0.21 | $0.002 | **$0.21** |
|
|
285
|
+
| Sonnet 4.6 ($3/MTok) | $0.78 | $0.007 | **$0.78** |
|
|
286
|
+
| Opus 4.6 ($15/MTok) | $3.92 | $0.033 | **$3.89** |
|
|
245
287
|
|
|
246
|
-
### Extrapolated Savings
|
|
288
|
+
### Extrapolated Monthly Savings
|
|
247
289
|
|
|
248
|
-
| Usage | Sonnet
|
|
249
|
-
|
|
250
|
-
| 5 sessions/day | $
|
|
251
|
-
| 10 sessions/day | $
|
|
252
|
-
| 20 sessions/day | $
|
|
290
|
+
| Usage | Haiku | Sonnet | Opus |
|
|
291
|
+
|:--|--:|--:|--:|
|
|
292
|
+
| 5 sessions/day | $31.05 | $116.44 | **$582.19** |
|
|
293
|
+
| 10 sessions/day | $62.10 | $232.88 | **$1,164.38** |
|
|
294
|
+
| 20 sessions/day | $124.20 | $465.75 | **$2,328.75** |
|
|
253
295
|
|
|
254
|
-
> Note: These are input token savings only. Actual savings vary based on session complexity. Output tokens are unaffected.
|
|
296
|
+
> Note: These are input token savings only. Actual savings vary based on session complexity. Output tokens are unaffected. Token estimates use bytes/4 midpoint; actual counts may vary 20-30%.
|
|
255
297
|
|
|
256
298
|
---
|
|
257
299
|
|
|
@@ -317,7 +359,7 @@ The `browser_snapshot()` tool returns a full accessibility tree:
|
|
|
317
359
|
... (thousands more lines for a real application)
|
|
318
360
|
```
|
|
319
361
|
|
|
320
|
-
**All 56,000 bytes (14,000 tokens) dumped into context. Gone.**
|
|
362
|
+
**All 56,000 bytes (~14,000 tokens) dumped into context. Gone.**
|
|
321
363
|
|
|
322
364
|
The agent probably only needed the login form. But it paid for the entire page.
|
|
323
365
|
|
|
@@ -363,6 +405,37 @@ The other 55,701 bytes are still in FTS5 — fully searchable. Need the order ta
|
|
|
363
405
|
|
|
364
406
|
---
|
|
365
407
|
|
|
408
|
+
## Security and Reliability
|
|
409
|
+
|
|
410
|
+
context-compress v2026.5.0 includes comprehensive security and reliability features:
|
|
411
|
+
|
|
412
|
+
### Security
|
|
413
|
+
|
|
414
|
+
| Feature | Description |
|
|
415
|
+
|:--|:--|
|
|
416
|
+
| Environment isolation | Opt-in credential passthrough (`passthroughEnvVars` defaults to empty) |
|
|
417
|
+
| SSRF protection | 4-layer defense: hostname validation, DNS rebinding prevention, IP pinning, redirect blocking |
|
|
418
|
+
| Input limits | Code: 1MB max. Fetch response: 10MB max. Index content: 50MB max |
|
|
419
|
+
| Concurrency control | Global limit of 8 concurrent executions. batch_execute: max 4 parallel |
|
|
420
|
+
| Prompt injection detection | Regex-based advisory warnings on fetched content (7 patterns) |
|
|
421
|
+
| Path traversal protection | `realpathSync` with symlink resolution + project boundary enforcement |
|
|
422
|
+
| Process isolation | Timeout, output caps (100MB), process group kill, safe environment |
|
|
423
|
+
|
|
424
|
+
### Reliability
|
|
425
|
+
|
|
426
|
+
| Feature | Description |
|
|
427
|
+
|:--|:--|
|
|
428
|
+
| Graceful shutdown | Active subprocess tracking, SIGTERM/SIGINT cleanup, uncaughtException handling |
|
|
429
|
+
| DB resilience | In-memory fallback on disk-full. WAL mode for crash recovery. Stale DB cleanup |
|
|
430
|
+
| Output processing | ANSI stripping, progress line removal, command-specific filters, line deduplication, error grouping, smart 60/40 head/tail truncation |
|
|
431
|
+
| Cumulative stats | Cross-session token savings persisted to `stats.json` when `persistDb` is enabled |
|
|
432
|
+
| Search fallback | 3-layer: Porter stemming → trigram (lazy) → Levenshtein fuzzy correction |
|
|
433
|
+
| Configuration | ENV > file > defaults with Zod validation and sanity clamping |
|
|
434
|
+
|
|
435
|
+
For the full security model, see [SECURITY.md](../SECURITY.md).
|
|
436
|
+
|
|
437
|
+
---
|
|
438
|
+
|
|
366
439
|
## FAQ: Doesn't Less Tokens Mean Losing Context?
|
|
367
440
|
|
|
368
441
|
**This is the right question to ask.** If we're feeding the agent fewer tokens, doesn't it see less?
|
|
@@ -374,7 +447,7 @@ The other 55,701 bytes are still in FTS5 — fully searchable. Need the order ta
|
|
|
374
447
|
```
|
|
375
448
|
WITHOUT context-compress (passive exposure):
|
|
376
449
|
┌──────────────────────────────────────────────────────┐
|
|
377
|
-
│ 194,
|
|
450
|
+
│ ~194,000 tokens loaded into context │
|
|
378
451
|
│ │
|
|
379
452
|
│ 99% = imports, boilerplate, minified code, │
|
|
380
453
|
│ source maps, irrelevant functions... │
|
|
@@ -390,7 +463,7 @@ WITHOUT context-compress (passive exposure):
|
|
|
390
463
|
|
|
391
464
|
WITH context-compress (active retrieval):
|
|
392
465
|
┌──────────────────────────────────────────────────────┐
|
|
393
|
-
│ 105 tokens loaded into context │
|
|
466
|
+
│ ~105 tokens loaded into context │
|
|
394
467
|
│ │
|
|
395
468
|
│ 100% = exactly the function you care about │
|
|
396
469
|
│ │
|
|
@@ -443,11 +516,13 @@ context-compress trades **passive exposure to noise** for **active retrieval of
|
|
|
443
516
|
|
|
444
517
|
| Tool | Mechanism | Best For |
|
|
445
518
|
|:--|:--|:--|
|
|
446
|
-
| `execute` | Runs code in sandbox. Only `console.log` enters context | CLI commands, API calls, test runners |
|
|
519
|
+
| `execute` | Runs code in sandbox (11 languages). Only `console.log` enters context | CLI commands, API calls, test runners |
|
|
447
520
|
| `execute_file` | Reads file into sandbox. Only printed summary enters context | Large source files, CSVs, logs, data files |
|
|
448
521
|
| `index` + `search` | FTS5 stores all data. BM25 returns only matching chunks | Documentation, snapshots, large datasets |
|
|
449
522
|
| `fetch_and_index` | HTML → markdown → FTS5. Returns 3KB preview + searchable index | Web pages, API docs, reference material |
|
|
450
|
-
| `batch_execute` | Runs N commands, indexes all output, searches across all in 1 call | Multi-step workflows, exploration |
|
|
523
|
+
| `batch_execute` | Runs N commands (max 4 parallel), indexes all output, searches across all in 1 call | Multi-step workflows, exploration |
|
|
524
|
+
| `discover` | Shows knowledge base inventory and optimization suggestions | Understanding available indexed data |
|
|
525
|
+
| `stats` | Real-time session statistics with token range estimates and cost | Monitoring compression effectiveness |
|
|
451
526
|
|
|
452
527
|
The core principle:
|
|
453
528
|
|
|
@@ -455,5 +530,6 @@ The core principle:
|
|
|
455
530
|
|
|
456
531
|
---
|
|
457
532
|
|
|
458
|
-
*Generated from real benchmarks on the context-compress
|
|
459
|
-
*Token
|
|
533
|
+
*Generated from real benchmarks on the context-compress v2026.5.0 codebase.*
|
|
534
|
+
*Token estimates use bytes/4 midpoint. Actual token counts may vary by 20-30% depending on content type.*
|
|
535
|
+
*See SECURITY.md for the full trust model and security architecture.*
|
package/hooks/pretooluse.mjs
CHANGED
|
@@ -6,6 +6,36 @@ var blockCurl = process.env.CONTEXT_COMPRESS_BLOCK_CURL !== "0";
|
|
|
6
6
|
var blockWebFetch = process.env.CONTEXT_COMPRESS_BLOCK_WEBFETCH !== "0";
|
|
7
7
|
var nudgeOnRead = process.env.CONTEXT_COMPRESS_NUDGE_READ !== "0";
|
|
8
8
|
var nudgeOnGrep = process.env.CONTEXT_COMPRESS_NUDGE_GREP !== "0";
|
|
9
|
+
var filterBash = process.env.CONTEXT_COMPRESS_FILTER_BASH === "1";
|
|
10
|
+
var ccBin = process.env.CONTEXT_COMPRESS_BIN ?? "context-compress";
|
|
11
|
+
var ccMode = process.env.CONTEXT_COMPRESS_MODE;
|
|
12
|
+
var WRAP_TARGETS = [
|
|
13
|
+
/^git\s+(status|log|diff|show|blame|branch|stash\s+list|grep|ls-files)/,
|
|
14
|
+
/^(npm|yarn|pnpm|bun)\s+(install|i|add|test|run\s|update|outdated|audit|list|ls|view|info)/,
|
|
15
|
+
/^cargo\s+(build|test|check|run|clippy|tree|search|metadata)/,
|
|
16
|
+
/^(pytest|jest|mocha|vitest|tap|bats)\b/,
|
|
17
|
+
/^(find|grep|rg|fd|ag|ripgrep)\b/,
|
|
18
|
+
/^ls\s+(-R|-la|-al)/,
|
|
19
|
+
/^docker\s+(build|ps|logs|images|inspect|stats)/,
|
|
20
|
+
/^kubectl\s+(get|describe|logs|top|api-resources)/,
|
|
21
|
+
/^terraform\s+(plan|show|state\s+list|state\s+show|validate)/,
|
|
22
|
+
/^helm\s+(list|status|history|get)/,
|
|
23
|
+
/^(make|gradle|bazel|nx|turbo)\b/,
|
|
24
|
+
/^ps\s+(aux|-ef)/,
|
|
25
|
+
/^(top|htop)\b/,
|
|
26
|
+
/^(df|du)\b/,
|
|
27
|
+
/^(go|rustc)\s+(test|build|vet|run)/
|
|
28
|
+
];
|
|
29
|
+
function shouldWrap(cmd) {
|
|
30
|
+
const trimmed = cmd.trim();
|
|
31
|
+
if (/(?:^|\s)(?:>|>>|\d?>&)\s*\S/.test(trimmed)) return false;
|
|
32
|
+
if (/\|/.test(trimmed)) return false;
|
|
33
|
+
if (/&&|\|\||;/.test(trimmed)) return false;
|
|
34
|
+
return WRAP_TARGETS.some((re) => re.test(trimmed));
|
|
35
|
+
}
|
|
36
|
+
function shellQuote(s) {
|
|
37
|
+
return `'${s.replace(/'/g, "'\\''")}'`;
|
|
38
|
+
}
|
|
9
39
|
var raw = "";
|
|
10
40
|
process.stdin.setEncoding("utf-8");
|
|
11
41
|
for await (const chunk of process.stdin) raw += chunk;
|
|
@@ -37,6 +67,14 @@ if (tool === "Bash") {
|
|
|
37
67
|
}
|
|
38
68
|
});
|
|
39
69
|
}
|
|
70
|
+
if (filterBash && shouldWrap(command)) {
|
|
71
|
+
const modeFlag = ccMode ? ` --mode ${ccMode}` : "";
|
|
72
|
+
respond({
|
|
73
|
+
updatedInput: {
|
|
74
|
+
command: `${ccBin} wrap${modeFlag} ${shellQuote(command)}`
|
|
75
|
+
}
|
|
76
|
+
});
|
|
77
|
+
}
|
|
40
78
|
process.exit(0);
|
|
41
79
|
}
|
|
42
80
|
if (tool === "Read" && nudgeOnRead) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "context-compress",
|
|
3
|
-
"version": "2026.
|
|
3
|
+
"version": "2026.5.0",
|
|
4
4
|
"description": "Context-aware MCP server that compresses tool output for Claude Code",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/server.js",
|
|
@@ -12,12 +12,13 @@
|
|
|
12
12
|
"build:hooks": "esbuild src/hooks/pretooluse.ts --bundle --platform=node --format=esm --outfile=hooks/pretooluse.mjs --target=node18 && node -e \"const{createHash}=require('crypto'),{readFileSync,writeFileSync}=require('fs');const h=createHash('sha256').update(readFileSync('hooks/pretooluse.mjs')).digest('hex');writeFileSync('hooks/pretooluse.sha256',h+'\\n')\"",
|
|
13
13
|
"dev": "tsx src/index.ts",
|
|
14
14
|
"typecheck": "tsc --noEmit",
|
|
15
|
-
"lint": "biome check src/",
|
|
16
|
-
"lint:fix": "biome check --write src/",
|
|
15
|
+
"lint": "biome check --max-diagnostics=200 src/",
|
|
16
|
+
"lint:fix": "biome check --write --max-diagnostics=200 src/",
|
|
17
17
|
"test": "node --import tsx --test tests/**/*.test.ts",
|
|
18
18
|
"test:unit": "node --import tsx --test tests/unit/*.test.ts",
|
|
19
19
|
"test:integration": "node --import tsx --test tests/integration/*.test.ts",
|
|
20
|
-
"clean": "rm -rf dist",
|
|
20
|
+
"clean": "rm -rf dist dist-bin",
|
|
21
|
+
"build:bin": "bun build --compile --target=bun-darwin-arm64 ./src/cli/lite.ts --outfile=./dist-bin/cc-lite-darwin-arm64 && bun build --compile --target=bun-darwin-x64 ./src/cli/lite.ts --outfile=./dist-bin/cc-lite-darwin-x64 && bun build --compile --target=bun-linux-x64 ./src/cli/lite.ts --outfile=./dist-bin/cc-lite-linux-x64 && bun build --compile --target=bun-linux-arm64 ./src/cli/lite.ts --outfile=./dist-bin/cc-lite-linux-arm64",
|
|
21
22
|
"prepublishOnly": "npm run lint && npm run test && npm run build"
|
|
22
23
|
},
|
|
23
24
|
"engines": {
|