@astudioplus/compressor 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +52 -0
- package/LICENSE +20 -0
- package/README.md +167 -0
- package/dist/adapters/agents-md.d.ts +2 -0
- package/dist/adapters/agents-md.js +91 -0
- package/dist/adapters/apply.d.ts +3 -0
- package/dist/adapters/apply.js +83 -0
- package/dist/adapters/claude-code.d.ts +2 -0
- package/dist/adapters/claude-code.js +403 -0
- package/dist/adapters/copilot.d.ts +2 -0
- package/dist/adapters/copilot.js +418 -0
- package/dist/adapters/cursor.d.ts +2 -0
- package/dist/adapters/cursor.js +149 -0
- package/dist/adapters/index.d.ts +11 -0
- package/dist/adapters/index.js +19 -0
- package/dist/adapters/markers.d.ts +7 -0
- package/dist/adapters/markers.js +129 -0
- package/dist/adapters/types.d.ts +44 -0
- package/dist/adapters/types.js +1 -0
- package/dist/bench/ablate.d.ts +35 -0
- package/dist/bench/ablate.js +163 -0
- package/dist/bench/cell.d.ts +33 -0
- package/dist/bench/cell.js +437 -0
- package/dist/bench/results.d.ts +37 -0
- package/dist/bench/results.js +157 -0
- package/dist/bench/runner.d.ts +24 -0
- package/dist/bench/runner.js +121 -0
- package/dist/bench/tasks.d.ts +4 -0
- package/dist/bench/tasks.js +147 -0
- package/dist/bench/types.d.ts +109 -0
- package/dist/bench/types.js +1 -0
- package/dist/claude/transcripts.d.ts +30 -0
- package/dist/claude/transcripts.js +154 -0
- package/dist/cli/commands/benchmark.d.ts +33 -0
- package/dist/cli/commands/benchmark.js +203 -0
- package/dist/cli/commands/compress.d.ts +8 -0
- package/dist/cli/commands/compress.js +45 -0
- package/dist/cli/commands/count.d.ts +5 -0
- package/dist/cli/commands/count.js +25 -0
- package/dist/cli/commands/hook.d.ts +6 -0
- package/dist/cli/commands/hook.js +30 -0
- package/dist/cli/commands/init.d.ts +16 -0
- package/dist/cli/commands/init.js +76 -0
- package/dist/cli/commands/report.d.ts +90 -0
- package/dist/cli/commands/report.js +464 -0
- package/dist/cli/commands/savings.d.ts +38 -0
- package/dist/cli/commands/savings.js +196 -0
- package/dist/cli/commands/set-mode.d.ts +5 -0
- package/dist/cli/commands/set-mode.js +13 -0
- package/dist/cli/commands/stats.d.ts +5 -0
- package/dist/cli/commands/stats.js +51 -0
- package/dist/cli/commands/status.d.ts +1 -0
- package/dist/cli/commands/status.js +11 -0
- package/dist/cli/commands/uninstall.d.ts +7 -0
- package/dist/cli/commands/uninstall.js +22 -0
- package/dist/cli/index.d.ts +2 -0
- package/dist/cli/index.js +146 -0
- package/dist/copilot-hook-entry.d.ts +1 -0
- package/dist/copilot-hook-entry.js +36 -0
- package/dist/copilot-hook.js +1000 -0
- package/dist/engine/detect.d.ts +2 -0
- package/dist/engine/detect.js +47 -0
- package/dist/engine/index.d.ts +4 -0
- package/dist/engine/index.js +90 -0
- package/dist/engine/policy.d.ts +2 -0
- package/dist/engine/policy.js +48 -0
- package/dist/engine/tiers/code.d.ts +7 -0
- package/dist/engine/tiers/code.js +206 -0
- package/dist/engine/tiers/logs.d.ts +4 -0
- package/dist/engine/tiers/logs.js +139 -0
- package/dist/engine/tiers/structural.d.ts +28 -0
- package/dist/engine/tiers/structural.js +199 -0
- package/dist/engine/types.d.ts +71 -0
- package/dist/engine/types.js +5 -0
- package/dist/hook/copilot.d.ts +5 -0
- package/dist/hook/copilot.js +136 -0
- package/dist/hook/core.d.ts +36 -0
- package/dist/hook/core.js +138 -0
- package/dist/hook/exit.d.ts +22 -0
- package/dist/hook/exit.js +56 -0
- package/dist/hook/post-tool-use.d.ts +5 -0
- package/dist/hook/post-tool-use.js +57 -0
- package/dist/hook-entry.d.ts +1 -0
- package/dist/hook-entry.js +35 -0
- package/dist/hook.js +946 -0
- package/dist/index.d.ts +15 -0
- package/dist/index.js +16 -0
- package/dist/ledger/read.d.ts +9 -0
- package/dist/ledger/read.js +91 -0
- package/dist/ledger/write.d.ts +29 -0
- package/dist/ledger/write.js +61 -0
- package/dist/packs/atoms.d.ts +3 -0
- package/dist/packs/atoms.js +108 -0
- package/dist/packs/modes.d.ts +3 -0
- package/dist/packs/modes.js +34 -0
- package/dist/packs/render.d.ts +24 -0
- package/dist/packs/render.js +115 -0
- package/dist/packs/types.d.ts +32 -0
- package/dist/packs/types.js +1 -0
- package/dist/paths.d.ts +29 -0
- package/dist/paths.js +87 -0
- package/dist/tokens/estimate.d.ts +12 -0
- package/dist/tokens/estimate.js +23 -0
- package/dist/tokens/exact.d.ts +5 -0
- package/dist/tokens/exact.js +16 -0
- package/dist/tokens/index.d.ts +2 -0
- package/dist/tokens/index.js +2 -0
- package/package.json +77 -0
package/CHANGELOG.md
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project will be documented in this file.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
|
+
|
|
8
|
+
## [0.1.0] - 2026-06-10
|
|
9
|
+
|
|
10
|
+
Initial release.
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
|
|
14
|
+
#### Engine
|
|
15
|
+
- Pure, dependency-free compression engine: `compress(content, meta, policy, estimator)` with injected token estimator.
|
|
16
|
+
- Tier 1 (structural): ANSI stripping, blank-run collapse, repeated-line dedupe, head/tail truncation with recoverable markers stating exact `Read offset/limit`.
|
|
17
|
+
- Tier 2 (code-aware): comment/blank-line stripping with original line numbers preserved; skeleton view (imports + signatures) above threshold.
|
|
18
|
+
- Tier 3 (log-aware, slim only): test logs reduced to failures + summary, build logs to errors/warnings + status.
|
|
19
|
+
- Per-mode policy thresholds; targeted reads (explicit offset/limit) always pass through; idempotency guard on the omission marker; fail-open on any error.
|
|
20
|
+
- Content-kind detection (code / test log / build log / generic) and configurable marker styles (plain / deterrent / informative).
|
|
21
|
+
|
|
22
|
+
#### Packs
|
|
23
|
+
- Instruction atoms (`{id, category, text, modes}`): 13 active output- and behavior-discipline atoms composing the `optimized` and `slim` modes; `full` removes all artifacts for a true baseline.
|
|
24
|
+
- Rejected atoms retained in the catalog with empirical rationale (e.g. `tokens.drop-articles`), reproducible via benchmark `--ablate-add`.
|
|
25
|
+
- Deterministic renderers (no timestamps, byte-stable for prompt caching) with embedded atom-ID manifests: Claude Code output style, Copilot/AGENTS.md/legacy-`.cursorrules` marked sections, Cursor `.mdc`.
|
|
26
|
+
|
|
27
|
+
#### Adapters
|
|
28
|
+
- `claude-code`: output style + `outputStyle` settings entry + surgical PostToolUse hook merge; project hook entry written to `.claude/settings.local.json`; foreign `outputStyle` stashed and restored on uninstall; project and global scope.
|
|
29
|
+
- `copilot`: marked section in `.github/copilot-instructions.md` + `postToolUse` hook config in `.github/hooks/compressor.json`; `--global` installs a machine-wide hook to `~/.copilot/hooks/` (`$COPILOT_HOME` honored).
|
|
30
|
+
- `cursor`: `.cursor/rules/compressor.mdc` (`alwaysApply`); legacy `.cursorrules` updated only when pre-existing (instructions only).
|
|
31
|
+
- `agents-md`: marked section in `AGENTS.md` (instructions only).
|
|
32
|
+
- Shared conventions: marker-based idempotent upserts, ownership predicates that never touch foreign entries, `--dry-run` diffs, `status` derived from files with honest per-agent capability notes.
|
|
33
|
+
|
|
34
|
+
#### Hooks
|
|
35
|
+
- Claude Code PostToolUse hook (`dist/hook.js`, bundled, no CLI dependency): shape-preserving replacement of `tool_response` via `updatedToolOutput`; matcher `Read|Bash|Grep|Glob`; fail-open.
|
|
36
|
+
- Copilot CLI `postToolUse` hook (`dist/copilot-hook.js`): `toolResult.textResultForLlm` in, `modifiedResult` out; self-filters by tool name (no matcher support); fail-open.
|
|
37
|
+
- Shared protocol-independent hook core (leaf picking, worthwhileness floor of 200 chars / 10%, shape-preserving rebuild).
|
|
38
|
+
|
|
39
|
+
#### Benchmark harness
|
|
40
|
+
- Runner driving headless `claude --bare -p` per cell (task × variant × trial) with per-cell `CLAUDE_CONFIG_DIR` isolation, pinned and verified served model, cost ceiling with no-cost circuit breaker, and data-quality flags.
|
|
41
|
+
- Token accounting from result JSON and session transcripts (deduped, sidechains included) — never from estimators.
|
|
42
|
+
- Binary success checks per task; suites (`basic`, `main`, `hookab`, `ablate`, `interactive`) with zero-dependency fixtures.
|
|
43
|
+
- Ablation variants: per-atom (`--ablate`), rejected-atom demonstration (`--ablate-add`), category groups (`--ablate-group`), `--no-hook` arms, and marker-style fan-out (`--marker-styles`).
|
|
44
|
+
- `report`: per-variant medians + IQR, deltas vs full and vs ablation baselines, run comparison, table/md/json output.
|
|
45
|
+
|
|
46
|
+
#### Ledger and savings
|
|
47
|
+
- Append-only local savings ledger written by the hooks: monthly JSONL under `~/.compressor/ledger/`, sizes and transform ids only (no paths, no content), fail-open, `COMPRESSOR_NO_LEDGER=1` kill switch.
|
|
48
|
+
- `savings` command: totals and day/tool/mode breakdowns with explicit "estimated" labelling and window labels; optional self-contained HTML report (inline SVG, no JS).
|
|
49
|
+
- `stats` command: actual usage aggregated from local Claude Code transcripts.
|
|
50
|
+
- `count` command: estimated token counts per file, `--exact` via the Anthropic `count_tokens` endpoint.
|
|
51
|
+
|
|
52
|
+
[0.1.0]: https://github.com/anvanster/compressor/releases/tag/v0.1.0
|
package/LICENSE
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 anvanster
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE DEALINGS IN THE SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
# compressor
|
|
2
|
+
|
|
3
|
+
Token optimization for AI coding agents. Three parts: **mode-switchable instruction packs** (full / optimized / slim) installed into Claude Code, GitHub Copilot, Cursor, and AGENTS.md-reading agents; **tool-output compression hooks** that shrink file reads and command output before they enter the model's context; and a **measurement harness** that runs real agent sessions and reports token deltas with success rates. The project's one rule: every optimization is measured or it doesn't ship — including the ideas that didn't survive measurement, which are documented below alongside the ones that did.
|
|
4
|
+
|
|
5
|
+
## Measured results
|
|
6
|
+
|
|
7
|
+
All numbers come from benchmark runs against live Claude models (real API usage from result JSON and session transcripts, not estimates). Task success is checked by shell command, not judgment. Run ids refer to result files under `bench/results/`.
|
|
8
|
+
|
|
9
|
+
| What | Measured effect | Run |
|
|
10
|
+
|---|---|---|
|
|
11
|
+
| Compression hook, big-file work | Median context **−24%** (optimized) / **−75%** (slim) vs no hook on the wide-refactor task; tail cells roughly **−90%** (−90.1% optimized / −89.5% slim, worst cell vs worst cell; worst no-hook cells reached ~3.0M context tokens) | bench-20260610-114234, bench-20260610-123102 |
|
|
12
|
+
| Instruction packs, multi-turn conversations | Output **−11% to −24%** (optimized) / −5% to −25% (slim) vs full on three of four conversation types; the fourth (a short add-function exchange) was +3%/+4%, within noise | bench-20260610-183001 |
|
|
13
|
+
| Instruction packs, single-shot tasks | Output **−5.6%** (optimized) / **−2.3%** (slim) — consistent with −6.1%/−2.5% in an earlier run, but IQRs overlap at 3 trials; directional only | bench-20260610-114234 (earlier: bench-20260609-232940) |
|
|
14
|
+
| Behavior atoms (ablation) | Removing the behavior-atom group costs +6.6% output; on the wide-refactor task the median output nearly doubled (6,003 → 11,697 tokens, n=2 — the two no-behavior trials were 7,022 and 16,372) — they carry the instruction-pack effect | bench-20260610-124626 |
|
|
15
|
+
| Task success | **100% in every measured arm** — all modes, all ablations, hook on and off | all runs above |
|
|
16
|
+
|
|
17
|
+
Most cells ran 2–4 trials. Medians are reported; treat single-task deltas as directional unless stated otherwise.
|
|
18
|
+
|
|
19
|
+
### Negative results
|
|
20
|
+
|
|
21
|
+
A project that only publishes its wins is advertising. These were measured and rejected:
|
|
22
|
+
|
|
23
|
+
| Hypothesis | Result | Run |
|
|
24
|
+
|---|---|---|
|
|
25
|
+
| "Drop articles and filler words to save tokens" (the viral trick) | Refuted: −2.2% output vs an already-concise baseline — noise, with grammar damage as the only reliable effect. The atom stays in `src/packs/atoms.ts` as rejected, with the run id in its rationale | bench-20260610-124626 |
|
|
26
|
+
| Rewording truncation markers (deterrent / informative phrasing) discourages wasteful pagination | Negative: marker phrasing did not move pagination behavior (if anything the trend ran the other way, within binomial noise at n=3). Plain markers kept | bench-20260610-181302 |
|
|
27
|
+
| Output-discipline atoms help single-shot agentic tasks | No measurable marginal effect in ablation (−2.6%, noise). Their measured value is prose-heavy tasks (slim −20% on an architecture-summary task) and multi-turn conversations | bench-20260610-124626, bench-20260610-114234 |
|
|
28
|
+
|
|
29
|
+
## Install
|
|
30
|
+
|
|
31
|
+
Requires Node **>= 20**.
|
|
32
|
+
|
|
33
|
+
Once published, install the CLI globally:
|
|
34
|
+
|
|
35
|
+
```sh
|
|
36
|
+
npm install -g @astudioplus/compressor
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
Or install from source:
|
|
40
|
+
|
|
41
|
+
```sh
|
|
42
|
+
git clone https://github.com/anvanster/compressor.git
|
|
43
|
+
cd compressor
|
|
44
|
+
npm install
|
|
45
|
+
npm run build # compiles the CLI and bundles the hook entries (dist/hook.js, dist/copilot-hook.js)
|
|
46
|
+
npm link # optional: puts `compressor` on your PATH
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Without `npm link`, substitute `node /path/to/compressor/dist/cli/index.js` for `compressor` below. Note that the installed hook entries embed the absolute path of this clone — don't move the directory after running `init` (re-run `init` if you do).
|
|
50
|
+
|
|
51
|
+
## Quickstart
|
|
52
|
+
|
|
53
|
+
### Claude Code
|
|
54
|
+
|
|
55
|
+
```sh
|
|
56
|
+
cd your-project
|
|
57
|
+
compressor init # claude-code, mode optimized, project scope
|
|
58
|
+
compressor status
|
|
59
|
+
compressor set-mode slim # switch modes
|
|
60
|
+
compressor set-mode full # removes everything — a true baseline
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
What `init` installs at project scope:
|
|
64
|
+
|
|
65
|
+
- `.claude/output-styles/compressor-<mode>.md` — the instruction pack as an output style (system prompt, cache-friendly)
|
|
66
|
+
- `.claude/settings.json` — `outputStyle` set to `compressor-<mode>` (a pre-existing foreign `outputStyle` is stashed and restored on uninstall)
|
|
67
|
+
- `.claude/settings.local.json` — the PostToolUse hook entry (`node "<clone>/dist/hook.js" --mode <mode>`, matcher `Read|Bash|Grep|Glob`). It carries a machine-specific absolute path, so it goes in the conventionally gitignored local file, never the shared `settings.json`
|
|
68
|
+
|
|
69
|
+
With `--global` everything goes to `~/.claude/` instead, hook included in `~/.claude/settings.json`. Changes take effect on the next session (`/clear` or a new session).
|
|
70
|
+
|
|
71
|
+
### GitHub Copilot
|
|
72
|
+
|
|
73
|
+
```sh
|
|
74
|
+
compressor init --agent copilot # project: instructions + hook
|
|
75
|
+
compressor init --agent copilot --global # machine-wide hook only (Copilot CLI)
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
Project scope installs a marked section in `.github/copilot-instructions.md` plus a `postToolUse` hook config at `.github/hooks/compressor.json`. Global scope installs the hook to `~/.copilot/hooks/compressor.json` (`$COPILOT_HOME` honored) — instructions have no user-global mechanism in Copilot, so global is hook-only.
|
|
79
|
+
|
|
80
|
+
The honest caveat (also shown by `compressor status`): **compression is effective in Copilot CLI on this machine only.** The hook command is an absolute local path, so the Copilot cloud agent and teammates who pull a committed `.github/hooks/compressor.json` get a fail-open no-op, and the VS Code IDE surface does not execute hook files at all (instructions still apply there). Copilot `postToolUse` is fail-open by platform design, so a dead command never blocks the agent.
|
|
81
|
+
|
|
82
|
+
### Cursor
|
|
83
|
+
|
|
84
|
+
```sh
|
|
85
|
+
compressor init --agent cursor
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
Installs `.cursor/rules/compressor.mdc` (`alwaysApply: true`). A legacy `.cursorrules` file gets a marked section only if it already exists — it is never created. **Instructions only**: Cursor's hooks can replace MCP tool output but not built-in Read/Shell output, so the compression half of compressor does not apply.
|
|
89
|
+
|
|
90
|
+
### AGENTS.md
|
|
91
|
+
|
|
92
|
+
```sh
|
|
93
|
+
compressor init --agent agents-md
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
Upserts a marked section in `AGENTS.md`, read natively by Cursor, Copilot, Codex, Windsurf and others. Claude Code does **not** read AGENTS.md (use the claude-code adapter). Instructions only — no hook mechanism exists through this file.
|
|
97
|
+
|
|
98
|
+
Multiple agents in one go: `compressor init --agent claude-code copilot agents-md`.
|
|
99
|
+
|
|
100
|
+
### Support matrix
|
|
101
|
+
|
|
102
|
+
| Agent | Instruction pack | Compression hook | Global scope |
|
|
103
|
+
|---|---|---|---|
|
|
104
|
+
| claude-code | yes (output style) | yes (PostToolUse) [^1] | yes (style + hook) |
|
|
105
|
+
| copilot | yes (project only) | yes — Copilot CLI on this machine only [^2] | hook only [^3] |
|
|
106
|
+
| cursor | yes (`.mdc` rules) | no [^4] | no (per-project standard) |
|
|
107
|
+
| agents-md | yes (marked section) | no | no (per-project standard) |
|
|
108
|
+
|
|
109
|
+
[^1]: At project scope the hook entry lives in `.claude/settings.local.json` because it embeds a machine-specific absolute path.
|
|
110
|
+
[^2]: The installed command is an absolute local path: cloud agent and teammates get a fail-open no-op; the VS Code IDE runs no hook files.
|
|
111
|
+
[^3]: `~/.copilot/hooks/compressor.json`, loaded by Copilot CLI only; Copilot has no user-global instructions mechanism.
|
|
112
|
+
[^4]: Cursor `postToolUse` can replace MCP tool output only; built-in Read/Shell output cannot be rewritten.
|
|
113
|
+
|
|
114
|
+
## Modes
|
|
115
|
+
|
|
116
|
+
| Mode | Instructions | Tool-output compression |
|
|
117
|
+
|---|---|---|
|
|
118
|
+
| `full` | none — `set-mode full` **removes** every compressor artifact and the hook, giving a true baseline rather than "empty instructions present" | none (passthrough) |
|
|
119
|
+
| `optimized` | answer-first output discipline plus context discipline (targeted reads, no re-reads, no tool-output echo) — 11 atoms | tier 1 structural (ANSI strip, blank-run collapse, repeated-line dedupe, recoverable truncation at ~5,000 est. tokens) + comment-strip above ~2,000 est. tokens; output below ~600 est. tokens is never touched |
|
|
120
|
+
| `slim` | optimized plus code-first responses under a hard ~10% explanation budget — 12 atoms | tiers 1–2 (skeleton view above ~6,000 est. tokens) plus log-aware filtering (test failures + summary, build errors) above ~800 est. tokens; touch floor ~300 |
|
|
121
|
+
|
|
122
|
+
Every omission is marked, sized, and recoverable — truncation markers state the exact `Read offset/limit` to retrieve the omitted lines, line numbers are never renumbered, and targeted reads (explicit offset/limit) pass through untouched. The hook is fail-open: any error means the original output passes through unchanged. Thresholds are estimated tokens (cheap estimator — used for thresholds only, never for reported savings).
|
|
123
|
+
|
|
124
|
+
## CLI reference
|
|
125
|
+
|
|
126
|
+
| Command | What it does | Key flags (defaults) |
|
|
127
|
+
|---|---|---|
|
|
128
|
+
| `init` | install the instruction pack + hook for the given agents | `--agent <name...>` (`claude-code`), `--mode optimized\|slim` (`optimized`), `--global`, `--dry-run` |
|
|
129
|
+
| `set-mode <full\|optimized\|slim>` | switch mode; `full` removes all compressor artifacts (true baseline) | `--agent <name...>` (`claude-code`), `--global`, `--dry-run` |
|
|
130
|
+
| `status` | show installation state per agent — derived from files and markers, no state file to drift | `--global` |
|
|
131
|
+
| `uninstall` | remove all compressor-owned artifacts | `--agent <name...>` (`claude-code`), `--global`, `--dry-run` |
|
|
132
|
+
| `compress` | compress stdin to stdout via the engine; stats on stderr | `--mode` (`optimized`), `--kind read\|bash\|search\|other` (`other`), `--file-path <path>`, `--marker-style plain\|deterrent\|informative` |
|
|
133
|
+
| `count <file...>` | token counts per file — estimated by default | `--exact` (Anthropic `count_tokens`, needs `ANTHROPIC_API_KEY`), `--model` (`claude-sonnet-4-6`) |
|
|
134
|
+
| `stats` | aggregate actual token usage from Claude Code transcripts | `--project <path>` (cwd), `--since` (`30d`) |
|
|
135
|
+
| `savings` | show what the compression hook saved (live ledger, estimated tokens) | `--since` (`30d`, or `all`), `--by day\|tool\|mode` (`day`), `--html <path>`, `--ledger-dir <dir>` |
|
|
136
|
+
| `benchmark` | run the benchmark suite: cells = task × variant × trial, results as JSONL | `--suite` (`bench/suites/basic.json`), `--modes` (`full,optimized,slim`), `--trials` (`5`), `--model` (`claude-sonnet-4-6`), `--ablate <ids>`, `--ablate-add <ids>`, `--ablate-group <output\|behavior>`, `--no-hook`, `--hook-args <args>`, `--marker-styles <styles>`, `--concurrency` (`2`), `--max-budget-usd` (`5`), `--out` (`bench/results`) |
|
|
137
|
+
| `report` | aggregate a run: per-variant medians + IQR, deltas vs full and vs ablation baselines | `--run <id>` (latest), `--compare <runs...>`, `--format table\|md\|json` (`table`), `--out` (`bench/results`) |
|
|
138
|
+
| `hook post-tool-use` | Claude Code PostToolUse protocol entry: payload on stdin, updated output on stdout | `--mode` (`optimized`), `--marker-style` |
|
|
139
|
+
| `hook copilot-post-tool-use` | Copilot postToolUse protocol entry: payload on stdin, `modifiedResult` JSON on stdout | `--mode` (`optimized`), `--marker-style` |
|
|
140
|
+
|
|
141
|
+
## Seeing your savings
|
|
142
|
+
|
|
143
|
+
Three views, deliberately not conflated:
|
|
144
|
+
|
|
145
|
+
- **`compressor savings`** — the live ledger. Every time the hook makes a worthwhile compression (at least 200 chars and 10% saved) during a real session, it appends an event to `~/.compressor/ledger/<YYYY-MM>.jsonl`. Events carry **sizes and transform ids only — no file paths, no content**. Token numbers here are estimates from a cheap chars/3.5 estimator and are labelled as such (chars are exact; the roughly 15–20% undercount of Claude tokenization belongs to the `js-tiktoken` estimator used by `count` and `compress`, not to this one). `--html` writes a self-contained report (inline SVG, no JS, no network). Kill switch: `COMPRESSOR_NO_LEDGER=1` disables all recording before any IO. The ledger is fail-open — a broken ledger never breaks your agent.
|
|
146
|
+
- **`compressor stats`** — raw token usage aggregated from your local Claude Code transcripts (the authoritative usage fields). Currently shows usage totals, not a before/after comparison.
|
|
147
|
+
- **`compressor benchmark`** + **`compressor report`** — the measured numbers. Real `claude` headless sessions in isolated per-cell config dirs, real API usage, binary success checks, budget ceiling, medians + IQR. Everything in the tables above came from here.
|
|
148
|
+
|
|
149
|
+
## Limitations
|
|
150
|
+
|
|
151
|
+
Stated plainly, because the alternative is users discovering them:
|
|
152
|
+
|
|
153
|
+
- **Copilot compression is CLI-on-this-machine only.** The hook command embeds an absolute local path; cloud agents and teammates get a fail-open no-op, and the Copilot IDE surface runs no hook files. A relocatable invocation needs the package on npm first.
|
|
154
|
+
- **Cursor and AGENTS.md get instructions only.** No mechanism exists to rewrite built-in tool output there.
|
|
155
|
+
- **Pagination bimodality is unsolved.** On the huge-log task the agent sometimes paginates (~457k context) and sometimes slurps the file (~248k) regardless of hook arm, and a marker-phrasing experiment failed to move it (run bench-20260610-181302). A structural fix (recovery-read budgets) is future work.
|
|
156
|
+
- **Output atoms are unproven in single-shot agentic use.** Ablation showed no marginal effect there; their measured value is prose tasks and multi-turn conversations. No harm measured either.
|
|
157
|
+
- **Sample sizes are small.** Most results are 2–4 trials per cell; headline numbers are medians and the per-mode aggregate deltas have overlapping IQRs. Negative and directional results are labelled as such.
|
|
158
|
+
- **Not yet on npm.** Install from source; the hook path is anchored to your clone.
|
|
159
|
+
|
|
160
|
+
## Further reading
|
|
161
|
+
|
|
162
|
+
- **Benchmarking and methodology**: [docs/MEASUREMENTS.md](docs/MEASUREMENTS.md) — harness design, isolation, run records, and how to reproduce the numbers above.
|
|
163
|
+
- **Architecture**: [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) — engine tiers, adapter contracts, hook protocols, and the design decisions behind them.
|
|
164
|
+
|
|
165
|
+
## License
|
|
166
|
+
|
|
167
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import { readFile, stat } from 'node:fs/promises';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { parseAtomManifest, renderMarkedSection } from "../packs/render.js";
|
|
4
|
+
import { readMarkedSection, removeMarkedSection, upsertMarkedSection, } from "./markers.js";
|
|
5
|
+
// AGENTS.md is plain Markdown read verbatim by 25+ tools (agents.md standard);
|
|
6
|
+
// our marked section is just text to those models. No tool offers a hook
|
|
7
|
+
// mechanism through it, so only the instruction half of compressor applies.
|
|
8
|
+
const READERS_NOTE = 'instructions only — read natively by Cursor, Copilot, Codex, Windsurf; Claude Code does NOT read AGENTS.md';
|
|
9
|
+
function agentsMdPath(ctx) {
|
|
10
|
+
return path.join(ctx.projectDir, 'AGENTS.md');
|
|
11
|
+
}
|
|
12
|
+
function isErrnoException(error) {
|
|
13
|
+
return error instanceof Error && 'code' in error;
|
|
14
|
+
}
|
|
15
|
+
async function readFileOrNull(filePath) {
|
|
16
|
+
try {
|
|
17
|
+
return await readFile(filePath, 'utf8');
|
|
18
|
+
}
|
|
19
|
+
catch (error) {
|
|
20
|
+
if (isErrnoException(error) && error.code === 'ENOENT') {
|
|
21
|
+
return null;
|
|
22
|
+
}
|
|
23
|
+
throw error;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
async function dirExists(dirPath) {
|
|
27
|
+
try {
|
|
28
|
+
return (await stat(dirPath)).isDirectory();
|
|
29
|
+
}
|
|
30
|
+
catch {
|
|
31
|
+
return false;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
async function fileExists(filePath) {
|
|
35
|
+
try {
|
|
36
|
+
return (await stat(filePath)).isFile();
|
|
37
|
+
}
|
|
38
|
+
catch {
|
|
39
|
+
return false;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
export const agentsMdAdapter = {
|
|
43
|
+
name: 'agents-md',
|
|
44
|
+
async detect(ctx) {
|
|
45
|
+
// AGENTS.md already present, or an agent that reads it is plausibly in
|
|
46
|
+
// use (.cursor → Cursor, .github → Copilot) — the standard is useful
|
|
47
|
+
// exactly when such agents are around.
|
|
48
|
+
return ((await fileExists(agentsMdPath(ctx))) ||
|
|
49
|
+
(await dirExists(path.join(ctx.projectDir, '.cursor'))) ||
|
|
50
|
+
(await dirExists(path.join(ctx.projectDir, '.github'))));
|
|
51
|
+
},
|
|
52
|
+
async install(mode, ctx) {
|
|
53
|
+
if (ctx.global) {
|
|
54
|
+
throw new Error('agents-md: AGENTS.md is a per-project standard; use project scope');
|
|
55
|
+
}
|
|
56
|
+
const file = agentsMdPath(ctx);
|
|
57
|
+
const before = await readFileOrNull(file);
|
|
58
|
+
const after = upsertMarkedSection(before, renderMarkedSection(mode, 'agents-md').body);
|
|
59
|
+
return after === before ? [] : [{ path: file, before, after }];
|
|
60
|
+
},
|
|
61
|
+
async uninstall(ctx) {
|
|
62
|
+
if (ctx.global) {
|
|
63
|
+
// install refuses global scope, so nothing of ours can exist there
|
|
64
|
+
return [];
|
|
65
|
+
}
|
|
66
|
+
const file = agentsMdPath(ctx);
|
|
67
|
+
const before = await readFileOrNull(file);
|
|
68
|
+
if (before === null || readMarkedSection(before) === null) {
|
|
69
|
+
return [];
|
|
70
|
+
}
|
|
71
|
+
// Never delete the file: whether WE created it is not derivable from
|
|
72
|
+
// disk (a user-created empty file that received our section is
|
|
73
|
+
// byte-identical to one we created), so err KEEP — worst case an empty
|
|
74
|
+
// file remains. Matches the cursor .cursorrules precedent.
|
|
75
|
+
return [{ path: file, before, after: removeMarkedSection(before) }];
|
|
76
|
+
},
|
|
77
|
+
async status(ctx) {
|
|
78
|
+
const body = await readFileOrNull(agentsMdPath(ctx));
|
|
79
|
+
const section = body === null ? null : readMarkedSection(body);
|
|
80
|
+
if (section === null) {
|
|
81
|
+
return { agent: 'agents-md', installed: false, detail: 'not installed' };
|
|
82
|
+
}
|
|
83
|
+
const mode = parseAtomManifest(section)?.mode;
|
|
84
|
+
return {
|
|
85
|
+
agent: 'agents-md',
|
|
86
|
+
installed: true,
|
|
87
|
+
...(mode !== undefined ? { mode } : {}),
|
|
88
|
+
detail: `AGENTS.md section (project) — ${READERS_NOTE}`,
|
|
89
|
+
};
|
|
90
|
+
},
|
|
91
|
+
};
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { mkdir, rm, rmdir, writeFile } from 'node:fs/promises';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
/** Remove now-empty dirs left after a delete, climbing no higher than `.claude`. */
|
|
4
|
+
async function pruneEmptyClaudeDirs(filePath) {
|
|
5
|
+
let dir = path.dirname(filePath);
|
|
6
|
+
while (dir.split(path.sep).includes('.claude')) {
|
|
7
|
+
try {
|
|
8
|
+
await rmdir(dir);
|
|
9
|
+
}
|
|
10
|
+
catch {
|
|
11
|
+
return;
|
|
12
|
+
}
|
|
13
|
+
if (path.basename(dir) === '.claude') {
|
|
14
|
+
return;
|
|
15
|
+
}
|
|
16
|
+
dir = path.dirname(dir);
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
export async function applyChanges(changes) {
|
|
20
|
+
for (const change of changes) {
|
|
21
|
+
if (change.after === null) {
|
|
22
|
+
await rm(change.path, { force: true });
|
|
23
|
+
await pruneEmptyClaudeDirs(change.path);
|
|
24
|
+
}
|
|
25
|
+
else {
|
|
26
|
+
await mkdir(path.dirname(change.path), { recursive: true });
|
|
27
|
+
await writeFile(change.path, change.after, 'utf8');
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
function splitLines(text) {
|
|
32
|
+
const lines = text.split('\n');
|
|
33
|
+
if (lines.length > 0 && lines[lines.length - 1] === '') {
|
|
34
|
+
lines.pop();
|
|
35
|
+
}
|
|
36
|
+
return lines;
|
|
37
|
+
}
|
|
38
|
+
function diffParts(before, after) {
|
|
39
|
+
const max = Math.min(before.length, after.length);
|
|
40
|
+
let prefixLen = 0;
|
|
41
|
+
while (prefixLen < max && before[prefixLen] === after[prefixLen]) {
|
|
42
|
+
prefixLen += 1;
|
|
43
|
+
}
|
|
44
|
+
let suffixLen = 0;
|
|
45
|
+
while (suffixLen < max - prefixLen &&
|
|
46
|
+
before[before.length - 1 - suffixLen] === after[after.length - 1 - suffixLen]) {
|
|
47
|
+
suffixLen += 1;
|
|
48
|
+
}
|
|
49
|
+
return {
|
|
50
|
+
prefixLen,
|
|
51
|
+
removed: before.slice(prefixLen, before.length - suffixLen),
|
|
52
|
+
added: after.slice(prefixLen, after.length - suffixLen),
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
const DIFF_CONTEXT = 2;
|
|
56
|
+
const DIFF_BODY_MAX_LINES = 200;
|
|
57
|
+
export function renderChanges(changes) {
|
|
58
|
+
const out = [];
|
|
59
|
+
for (const change of changes) {
|
|
60
|
+
const beforeLines = change.before === null ? [] : splitLines(change.before);
|
|
61
|
+
const afterLines = change.after === null ? [] : splitLines(change.after);
|
|
62
|
+
if (change.after === null) {
|
|
63
|
+
out.push(`delete ${change.path} (+0/-${beforeLines.length})`);
|
|
64
|
+
continue;
|
|
65
|
+
}
|
|
66
|
+
if (change.before === null) {
|
|
67
|
+
out.push(`create ${change.path} (+${afterLines.length}/-0)`);
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
const { prefixLen, removed, added } = diffParts(beforeLines, afterLines);
|
|
71
|
+
out.push(`update ${change.path} (+${added.length}/-${removed.length})`);
|
|
72
|
+
if (beforeLines.length < DIFF_BODY_MAX_LINES &&
|
|
73
|
+
afterLines.length < DIFF_BODY_MAX_LINES) {
|
|
74
|
+
const suffixStart = prefixLen + removed.length;
|
|
75
|
+
out.push(...beforeLines
|
|
76
|
+
.slice(Math.max(0, prefixLen - DIFF_CONTEXT), prefixLen)
|
|
77
|
+
.map((line) => ` ${line}`), ...removed.map((line) => `- ${line}`), ...added.map((line) => `+ ${line}`), ...beforeLines
|
|
78
|
+
.slice(suffixStart, suffixStart + DIFF_CONTEXT)
|
|
79
|
+
.map((line) => ` ${line}`));
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
return out.join('\n');
|
|
83
|
+
}
|