@sliday/tamp 0.3.1 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/tamp.js +4 -2
- package/compress.js +85 -2
- package/config.js +4 -0
- package/package.json +1 -1
package/bin/tamp.js
CHANGED
|
@@ -34,6 +34,8 @@ const STAGE_INFO = {
|
|
|
34
34
|
dedup: 'Replace duplicate tool_results with references',
|
|
35
35
|
diff: 'Replace similar re-reads with unified diffs',
|
|
36
36
|
prune: 'Strip lockfile hashes, registry URLs, npm metadata',
|
|
37
|
+
'strip-comments': 'Remove code comments (lossy, opt-in)',
|
|
38
|
+
textpress: 'LLM semantic compression via Ollama/OpenRouter (opt-in)',
|
|
37
39
|
}
|
|
38
40
|
|
|
39
41
|
// --- Determine stages ---
|
|
@@ -43,7 +45,7 @@ let selectedStages
|
|
|
43
45
|
if (process.env.TAMP_STAGES) {
|
|
44
46
|
selectedStages = process.env.TAMP_STAGES.split(',').map(s => s.trim()).filter(Boolean)
|
|
45
47
|
} else if (skipPrompt) {
|
|
46
|
-
selectedStages = Object.keys(STAGE_INFO)
|
|
48
|
+
selectedStages = Object.keys(STAGE_INFO).filter(s => s !== 'strip-comments' && s !== 'textpress')
|
|
47
49
|
} else {
|
|
48
50
|
log('')
|
|
49
51
|
log(` ${c.bold}${c.cyan}Tamp${c.reset} ${c.dim}v${pkg.version}${c.reset} — Token compression proxy`)
|
|
@@ -53,7 +55,7 @@ if (process.env.TAMP_STAGES) {
|
|
|
53
55
|
choices: Object.entries(STAGE_INFO).map(([value, desc]) => ({
|
|
54
56
|
name: `${value.padEnd(12)} ${c.dim}— ${desc}${c.reset}`,
|
|
55
57
|
value,
|
|
56
|
-
checked:
|
|
58
|
+
checked: value !== 'strip-comments' && value !== 'textpress',
|
|
57
59
|
})),
|
|
58
60
|
})
|
|
59
61
|
if (selectedStages.length === 0) {
|
package/compress.js
CHANGED
|
@@ -102,6 +102,78 @@ function pruneJSON(text) {
|
|
|
102
102
|
return result
|
|
103
103
|
}
|
|
104
104
|
|
|
105
|
+
// --- Stage: strip-comments (opt-in, not in defaults) ---
|
|
106
|
+
function stripComments(text) {
|
|
107
|
+
return text
|
|
108
|
+
.replace(/\/\*\*[\s\S]*?\*\//g, '') // JSDoc /** ... */
|
|
109
|
+
.replace(/\/\*[\s\S]*?\*\//g, '') // block /* ... */
|
|
110
|
+
.replace(/\/\/.*$/gm, '') // line // ...
|
|
111
|
+
.replace(/^\s*#(?!!).*$/gm, '') // Python/shell # (not #!)
|
|
112
|
+
.replace(/<!--[\s\S]*?-->/g, '') // HTML <!-- ... -->
|
|
113
|
+
.replace(/\n\s*\n\s*\n/g, '\n\n') // collapse resulting blank lines
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
// --- Stage: textpress (opt-in, uses Ollama or OpenRouter free model) ---
|
|
117
|
+
const TEXTPRESS_PROMPT = 'You are an expert at making text more concise without changing its meaning. Don\'t reword, don\'t improve. Find ways to combine and shorten. Keep ALL names, versions, paths, function signatures, numbers. Return ONLY the shortened text. No explanations.'
|
|
118
|
+
|
|
119
|
+
async function textpressCompress(text, config) {
|
|
120
|
+
if (text.length < 200) return null
|
|
121
|
+
|
|
122
|
+
// Try Ollama first (local, free, no rate limits)
|
|
123
|
+
const ollamaUrl = config.textpressOllamaUrl || 'http://localhost:11434'
|
|
124
|
+
const ollamaModel = config.textpressOllamaModel || 'qwen3.5:0.8b'
|
|
125
|
+
try {
|
|
126
|
+
const controller = new AbortController()
|
|
127
|
+
const timeout = setTimeout(() => controller.abort(), 15000)
|
|
128
|
+
const res = await fetch(`${ollamaUrl}/api/chat`, {
|
|
129
|
+
method: 'POST',
|
|
130
|
+
headers: { 'Content-Type': 'application/json' },
|
|
131
|
+
body: JSON.stringify({
|
|
132
|
+
model: ollamaModel,
|
|
133
|
+
messages: [{ role: 'system', content: TEXTPRESS_PROMPT }, { role: 'user', content: text }],
|
|
134
|
+
stream: false,
|
|
135
|
+
options: { temperature: 0, num_predict: Math.ceil(text.length * 0.7) },
|
|
136
|
+
think: false,
|
|
137
|
+
}),
|
|
138
|
+
signal: controller.signal,
|
|
139
|
+
})
|
|
140
|
+
clearTimeout(timeout)
|
|
141
|
+
if (res.ok) {
|
|
142
|
+
const data = await res.json()
|
|
143
|
+
const output = (data.message?.content || '').trim()
|
|
144
|
+
if (output.length > 0 && output.length < text.length * 0.9) return output
|
|
145
|
+
}
|
|
146
|
+
} catch { /* Ollama not available, try OpenRouter */ }
|
|
147
|
+
|
|
148
|
+
// Fallback: OpenRouter free model
|
|
149
|
+
const orKey = config.textpressApiKey || process.env.OPENROUTER_API_KEY
|
|
150
|
+
if (!orKey) return null
|
|
151
|
+
const orModel = config.textpressModel || 'google/gemini-3.1-flash-lite-preview'
|
|
152
|
+
try {
|
|
153
|
+
const controller = new AbortController()
|
|
154
|
+
const timeout = setTimeout(() => controller.abort(), 10000)
|
|
155
|
+
const res = await fetch('https://openrouter.ai/api/v1/chat/completions', {
|
|
156
|
+
method: 'POST',
|
|
157
|
+
headers: { 'Content-Type': 'application/json', 'Authorization': `Bearer ${orKey}` },
|
|
158
|
+
body: JSON.stringify({
|
|
159
|
+
model: orModel,
|
|
160
|
+
messages: [{ role: 'system', content: TEXTPRESS_PROMPT }, { role: 'user', content: text }],
|
|
161
|
+
max_tokens: Math.ceil(text.length * 0.5),
|
|
162
|
+
temperature: 0,
|
|
163
|
+
}),
|
|
164
|
+
signal: controller.signal,
|
|
165
|
+
})
|
|
166
|
+
clearTimeout(timeout)
|
|
167
|
+
if (res.ok) {
|
|
168
|
+
const data = await res.json()
|
|
169
|
+
const output = (data.choices?.[0]?.message?.content || '').replace(/<think>[\s\S]*?<\/think>/g, '').trim()
|
|
170
|
+
if (output.length > 0 && output.length < text.length * 0.9) return output
|
|
171
|
+
}
|
|
172
|
+
} catch { /* OpenRouter not available */ }
|
|
173
|
+
|
|
174
|
+
return null
|
|
175
|
+
}
|
|
176
|
+
|
|
105
177
|
// --- Core compression ---
|
|
106
178
|
export function compressText(text, config) {
|
|
107
179
|
if (text.length < config.minSize) return null
|
|
@@ -117,11 +189,17 @@ export function compressText(text, config) {
|
|
|
117
189
|
if (config.stages.includes('whitespace')) {
|
|
118
190
|
processed = normalizeWhitespace(processed)
|
|
119
191
|
}
|
|
192
|
+
if (config.stages.includes('strip-comments')) {
|
|
193
|
+
processed = stripComments(processed)
|
|
194
|
+
}
|
|
120
195
|
if (processed.length < text.length * 0.9) {
|
|
121
196
|
return { text: processed, method: 'normalize', originalLen: text.length, compressedLen: processed.length, originalTokens: countTokens(text), compressedTokens: countTokens(processed) }
|
|
122
197
|
}
|
|
123
198
|
if (config.stages.includes('llmlingua') && config.llmLinguaUrl) {
|
|
124
|
-
return { async: true, text: processed, cls }
|
|
199
|
+
return { async: true, asyncMethod: 'llmlingua', text: processed, cls }
|
|
200
|
+
}
|
|
201
|
+
if (config.stages.includes('textpress')) {
|
|
202
|
+
return { async: true, asyncMethod: 'textpress', text: processed, cls }
|
|
125
203
|
}
|
|
126
204
|
return null
|
|
127
205
|
}
|
|
@@ -182,7 +260,12 @@ async function compressBlock(text, config) {
|
|
|
182
260
|
const sync = compressText(text, config)
|
|
183
261
|
let result
|
|
184
262
|
if (sync && sync.async) {
|
|
185
|
-
|
|
263
|
+
if (sync.asyncMethod === 'textpress') {
|
|
264
|
+
const compressed = await textpressCompress(sync.text, config)
|
|
265
|
+
result = compressed ? { text: compressed, method: 'textpress', originalLen: text.length, compressedLen: compressed.length, originalTokens: countTokens(text), compressedTokens: countTokens(compressed) } : null
|
|
266
|
+
} else {
|
|
267
|
+
result = await compressWithLLMLingua(sync.text, config)
|
|
268
|
+
}
|
|
186
269
|
} else {
|
|
187
270
|
result = sync
|
|
188
271
|
}
|
package/config.js
CHANGED
|
@@ -15,6 +15,10 @@ export function loadConfig(env = process.env) {
|
|
|
15
15
|
maxBody: parseInt(env.TAMP_MAX_BODY, 10) || 10_485_760,
|
|
16
16
|
cacheSafe: true,
|
|
17
17
|
llmLinguaUrl: env.TAMP_LLMLINGUA_URL || null,
|
|
18
|
+
textpressOllamaUrl: env.TAMP_TEXTPRESS_OLLAMA_URL || 'http://localhost:11434',
|
|
19
|
+
textpressOllamaModel: env.TAMP_TEXTPRESS_OLLAMA_MODEL || 'qwen3.5:0.8b',
|
|
20
|
+
textpressModel: env.TAMP_TEXTPRESS_MODEL || 'google/gemini-3.1-flash-lite-preview',
|
|
21
|
+
textpressApiKey: env.TAMP_TEXTPRESS_API_KEY || env.OPENROUTER_API_KEY || null,
|
|
18
22
|
tokenCost: parseFloat(env.TAMP_TOKEN_COST) || 3,
|
|
19
23
|
})
|
|
20
24
|
}
|
package/package.json
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
"providers.js",
|
|
10
10
|
"stats.js"
|
|
11
11
|
],
|
|
12
|
-
"version": "0.3.
|
|
12
|
+
"version": "0.3.3",
|
|
13
13
|
"description": "Token compression proxy for coding agents. Works with Claude Code, Aider, Cursor, Cline, Windsurf. 50.5% fewer input tokens.",
|
|
14
14
|
"type": "module",
|
|
15
15
|
"main": "index.js",
|