@ajdev0/token-shrink 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +345 -0
- package/dist/chunk-7H6PGILN.js +86 -0
- package/dist/chunk-7H6PGILN.js.map +1 -0
- package/dist/chunk-7SQ6HMWM.js +145 -0
- package/dist/chunk-7SQ6HMWM.js.map +1 -0
- package/dist/chunk-HRF3BIOV.js +518 -0
- package/dist/chunk-HRF3BIOV.js.map +1 -0
- package/dist/chunk-LPJMNP4N.js +179 -0
- package/dist/chunk-LPJMNP4N.js.map +1 -0
- package/dist/cli-EpVqinpB.d.cts +96 -0
- package/dist/cli-EpVqinpB.d.ts +96 -0
- package/dist/cli.cjs +771 -0
- package/dist/cli.cjs.map +1 -0
- package/dist/cli.d.cts +4 -0
- package/dist/cli.d.ts +4 -0
- package/dist/cli.js +10 -0
- package/dist/cli.js.map +1 -0
- package/dist/index.cjs +988 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +169 -0
- package/dist/index.d.ts +169 -0
- package/dist/index.js +68 -0
- package/dist/index.js.map +1 -0
- package/dist/mcp.cjs +856 -0
- package/dist/mcp.cjs.map +1 -0
- package/dist/mcp.d.cts +58 -0
- package/dist/mcp.d.ts +58 -0
- package/dist/mcp.js +26 -0
- package/dist/mcp.js.map +1 -0
- package/dist/registry-JLP6X4QB.js +14 -0
- package/dist/registry-JLP6X4QB.js.map +1 -0
- package/dist/tree-sitter-typescript.wasm +0 -0
- package/package.json +59 -0
- package/wasm/tree-sitter-typescript.wasm +0 -0
package/dist/cli.cjs
ADDED
|
@@ -0,0 +1,771 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
"use strict";
|
|
3
|
+
var __create = Object.create;
|
|
4
|
+
var __defProp = Object.defineProperty;
|
|
5
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
6
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
7
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
8
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
9
|
+
var __esm = (fn, res) => function __init() {
|
|
10
|
+
return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
|
|
11
|
+
};
|
|
12
|
+
var __export = (target, all) => {
|
|
13
|
+
for (var name in all)
|
|
14
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
15
|
+
};
|
|
16
|
+
var __copyProps = (to, from, except, desc) => {
|
|
17
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
18
|
+
for (let key of __getOwnPropNames(from))
|
|
19
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
20
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
21
|
+
}
|
|
22
|
+
return to;
|
|
23
|
+
};
|
|
24
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
25
|
+
// If the importer is in node compatibility mode or this is not an ESM
|
|
26
|
+
// file that has been converted to a CommonJS file using a Babel-
|
|
27
|
+
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
28
|
+
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
29
|
+
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
30
|
+
mod
|
|
31
|
+
));
|
|
32
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
33
|
+
|
|
34
|
+
// src/parser/registry.ts
|
|
35
|
+
var registry_exports = {};
|
|
36
|
+
__export(registry_exports, {
|
|
37
|
+
allExtensions: () => allExtensions,
|
|
38
|
+
languageForFile: () => languageForFile,
|
|
39
|
+
registry: () => registry,
|
|
40
|
+
supportedLanguageIds: () => supportedLanguageIds
|
|
41
|
+
});
|
|
42
|
+
function languageForFile(filePath) {
|
|
43
|
+
const langId = extensionIndex.get(import_node_path2.default.extname(filePath).toLowerCase());
|
|
44
|
+
if (!langId) return null;
|
|
45
|
+
return registry[langId];
|
|
46
|
+
}
|
|
47
|
+
var import_node_path2, BLOCK, SWIFT_BLOCK, GO_BLOCK, RUST_BLOCK, PY_BLOCK, C_BLOCK, JAVA_BLOCK, PHP_BLOCK, KOTLIN_BLOCK, DART_BLOCK, DIRECTIVE_KEEP, registry, extensionIndex, supportedLanguageIds, allExtensions;
|
|
48
|
+
var init_registry = __esm({
|
|
49
|
+
"src/parser/registry.ts"() {
|
|
50
|
+
"use strict";
|
|
51
|
+
import_node_path2 = __toESM(require("path"), 1);
|
|
52
|
+
BLOCK = { query: "(statement_block) @block", replacement: { token: "/* ... */" } };
|
|
53
|
+
SWIFT_BLOCK = { query: "(statements) @block", replacement: { token: "/* ... */" } };
|
|
54
|
+
GO_BLOCK = { query: "(block) @block", replacement: { token: "/* ... */" } };
|
|
55
|
+
RUST_BLOCK = { query: "(block) @block", replacement: { token: "/* ... */" } };
|
|
56
|
+
PY_BLOCK = { query: "(block) @block", replacement: { token: "pass" } };
|
|
57
|
+
C_BLOCK = { query: "(compound_statement) @block", replacement: { token: "/* ... */" } };
|
|
58
|
+
JAVA_BLOCK = {
|
|
59
|
+
query: "(block) @block",
|
|
60
|
+
replacement: { token: "/* ... */" }
|
|
61
|
+
};
|
|
62
|
+
PHP_BLOCK = {
|
|
63
|
+
query: "(compound_statement) @block",
|
|
64
|
+
replacement: { token: "/* ... */" }
|
|
65
|
+
};
|
|
66
|
+
KOTLIN_BLOCK = { query: "(block) @block", replacement: { token: "/* ... */" } };
|
|
67
|
+
DART_BLOCK = { query: "(block) @block", replacement: { token: "/* ... */" } };
|
|
68
|
+
DIRECTIVE_KEEP = {
|
|
69
|
+
token: "/* ... */",
|
|
70
|
+
keepIf: /^\s*'use (client|server)'/
|
|
71
|
+
};
|
|
72
|
+
registry = {
|
|
73
|
+
typescript: {
|
|
74
|
+
name: "TypeScript",
|
|
75
|
+
wasm: "tree-sitter-typescript.wasm",
|
|
76
|
+
url: "https://github.com/tree-sitter/tree-sitter-typescript/releases/latest/download/tree-sitter-typescript.wasm",
|
|
77
|
+
extensions: [".ts", ".cts", ".mts"],
|
|
78
|
+
rules: [BLOCK]
|
|
79
|
+
},
|
|
80
|
+
javascript: {
|
|
81
|
+
name: "JavaScript",
|
|
82
|
+
wasm: "tree-sitter-javascript.wasm",
|
|
83
|
+
url: "https://github.com/tree-sitter/tree-sitter-javascript/releases/latest/download/tree-sitter-javascript.wasm",
|
|
84
|
+
extensions: [".js", ".cjs", ".mjs"],
|
|
85
|
+
rules: [BLOCK]
|
|
86
|
+
},
|
|
87
|
+
tsx: {
|
|
88
|
+
name: "React / Next.js (TSX)",
|
|
89
|
+
wasm: "tree-sitter-tsx.wasm",
|
|
90
|
+
url: "https://github.com/tree-sitter/tree-sitter-typescript/releases/latest/download/tree-sitter-tsx.wasm",
|
|
91
|
+
extensions: [".tsx"],
|
|
92
|
+
rules: [{ query: "(statement_block) @block", replacement: DIRECTIVE_KEEP }]
|
|
93
|
+
},
|
|
94
|
+
jsx: {
|
|
95
|
+
name: "React (JSX)",
|
|
96
|
+
wasm: "tree-sitter-javascript.wasm",
|
|
97
|
+
url: "https://github.com/tree-sitter/tree-sitter-javascript/releases/latest/download/tree-sitter-javascript.wasm",
|
|
98
|
+
extensions: [".jsx"],
|
|
99
|
+
rules: [{ query: "(statement_block) @block", replacement: DIRECTIVE_KEEP }]
|
|
100
|
+
},
|
|
101
|
+
python: {
|
|
102
|
+
name: "Python",
|
|
103
|
+
wasm: "tree-sitter-python.wasm",
|
|
104
|
+
url: "https://github.com/tree-sitter/tree-sitter-python/releases/latest/download/tree-sitter-python.wasm",
|
|
105
|
+
extensions: [".py", ".pyi"],
|
|
106
|
+
rules: [PY_BLOCK]
|
|
107
|
+
},
|
|
108
|
+
dart: {
|
|
109
|
+
name: "Dart / Flutter",
|
|
110
|
+
wasm: "tree-sitter-dart.wasm",
|
|
111
|
+
url: "https://github.com/UserNobody14/tree-sitter-dart.wasm/releases/latest/download/tree-sitter-dart.wasm",
|
|
112
|
+
extensions: [".dart"],
|
|
113
|
+
rules: [DART_BLOCK]
|
|
114
|
+
},
|
|
115
|
+
swift: {
|
|
116
|
+
name: "Swift / SwiftUI",
|
|
117
|
+
wasm: "tree-sitter-swift.wasm",
|
|
118
|
+
url: "https://github.com/alex-pinkus/tree-sitter-swift/releases/latest/download/tree-sitter-swift.wasm",
|
|
119
|
+
extensions: [".swift"],
|
|
120
|
+
rules: [SWIFT_BLOCK]
|
|
121
|
+
},
|
|
122
|
+
go: {
|
|
123
|
+
name: "Go",
|
|
124
|
+
wasm: "tree-sitter-go.wasm",
|
|
125
|
+
url: "https://github.com/tree-sitter/tree-sitter-go/releases/latest/download/tree-sitter-go.wasm",
|
|
126
|
+
extensions: [".go"],
|
|
127
|
+
rules: [GO_BLOCK]
|
|
128
|
+
},
|
|
129
|
+
rust: {
|
|
130
|
+
name: "Rust",
|
|
131
|
+
wasm: "tree-sitter-rust.wasm",
|
|
132
|
+
url: "https://github.com/tree-sitter/tree-sitter-rust/releases/latest/download/tree-sitter-rust.wasm",
|
|
133
|
+
extensions: [".rs"],
|
|
134
|
+
rules: [RUST_BLOCK]
|
|
135
|
+
},
|
|
136
|
+
java: {
|
|
137
|
+
name: "Java",
|
|
138
|
+
wasm: "tree-sitter-java.wasm",
|
|
139
|
+
url: "https://github.com/tree-sitter/tree-sitter-java/releases/latest/download/tree-sitter-java.wasm",
|
|
140
|
+
extensions: [".java"],
|
|
141
|
+
rules: [JAVA_BLOCK]
|
|
142
|
+
},
|
|
143
|
+
kotlin: {
|
|
144
|
+
name: "Kotlin",
|
|
145
|
+
wasm: "tree-sitter-kotlin.wasm",
|
|
146
|
+
url: "https://github.com/fwcd/tree-sitter-kotlin/releases/latest/download/tree-sitter-kotlin.wasm",
|
|
147
|
+
extensions: [".kt", ".kts"],
|
|
148
|
+
rules: [KOTLIN_BLOCK]
|
|
149
|
+
},
|
|
150
|
+
c: {
|
|
151
|
+
name: "C",
|
|
152
|
+
wasm: "tree-sitter-c.wasm",
|
|
153
|
+
url: "https://github.com/tree-sitter/tree-sitter-c/releases/latest/download/tree-sitter-c.wasm",
|
|
154
|
+
extensions: [".c", ".h"],
|
|
155
|
+
rules: [C_BLOCK]
|
|
156
|
+
},
|
|
157
|
+
cpp: {
|
|
158
|
+
name: "C++",
|
|
159
|
+
wasm: "tree-sitter-cpp.wasm",
|
|
160
|
+
url: "https://github.com/tree-sitter/tree-sitter-cpp/releases/latest/download/tree-sitter-cpp.wasm",
|
|
161
|
+
extensions: [".cc", ".cpp", ".cxx", ".hpp", ".hh", ".hxx"],
|
|
162
|
+
rules: [C_BLOCK]
|
|
163
|
+
},
|
|
164
|
+
php: {
|
|
165
|
+
name: "PHP",
|
|
166
|
+
wasm: "tree-sitter-php.wasm",
|
|
167
|
+
url: "https://github.com/tree-sitter/tree-sitter-php/releases/latest/download/tree-sitter-php.wasm",
|
|
168
|
+
extensions: [".php"],
|
|
169
|
+
rules: [PHP_BLOCK]
|
|
170
|
+
}
|
|
171
|
+
};
|
|
172
|
+
extensionIndex = /* @__PURE__ */ new Map();
|
|
173
|
+
for (const [id, spec] of Object.entries(registry)) {
|
|
174
|
+
for (const ext2 of spec.extensions) {
|
|
175
|
+
extensionIndex.set(ext2, id);
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
supportedLanguageIds = Object.keys(registry);
|
|
179
|
+
allExtensions = [...extensionIndex.keys()];
|
|
180
|
+
}
|
|
181
|
+
});
|
|
182
|
+
|
|
183
|
+
// src/cli.ts
|
|
184
|
+
var cli_exports = {};
|
|
185
|
+
__export(cli_exports, {
|
|
186
|
+
startServer: () => startServer
|
|
187
|
+
});
|
|
188
|
+
module.exports = __toCommonJS(cli_exports);
|
|
189
|
+
var import_fastify = __toESM(require("fastify"), 1);
|
|
190
|
+
var import_picocolors = __toESM(require("picocolors"), 1);
|
|
191
|
+
var import_node_path6 = __toESM(require("path"), 1);
|
|
192
|
+
|
|
193
|
+
// src/server/assembler.ts
|
|
194
|
+
var import_node_fs = __toESM(require("fs"), 1);
|
|
195
|
+
var import_node_path = __toESM(require("path"), 1);
|
|
196
|
+
function assemble(activeFilePath, cache, opts = {}) {
|
|
197
|
+
const { includeStats = false, maxSkeletons = 50, pruneActiveFile = false } = opts;
|
|
198
|
+
const abs = import_node_path.default.resolve(activeFilePath);
|
|
199
|
+
const activeSource = import_node_fs.default.existsSync(abs) ? import_node_fs.default.readFileSync(abs, "utf8") : "";
|
|
200
|
+
const activeEntry = cache.get(abs);
|
|
201
|
+
const included = [];
|
|
202
|
+
const unresolved = [];
|
|
203
|
+
const seen = /* @__PURE__ */ new Set();
|
|
204
|
+
if (activeEntry) {
|
|
205
|
+
for (const imp of activeEntry.imports) {
|
|
206
|
+
const entry = cache.get(imp);
|
|
207
|
+
if (!entry) {
|
|
208
|
+
unresolved.push(imp);
|
|
209
|
+
continue;
|
|
210
|
+
}
|
|
211
|
+
if (seen.has(imp)) continue;
|
|
212
|
+
seen.add(imp);
|
|
213
|
+
included.push({ filePath: imp, language: entry.language });
|
|
214
|
+
if (included.length >= maxSkeletons) break;
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
if (!activeEntry && activeSource) {
|
|
218
|
+
const specifiers = extractSpecifiers(activeSource);
|
|
219
|
+
for (const spec of specifiers) {
|
|
220
|
+
const resolved = resolveLocal(abs, spec);
|
|
221
|
+
if (!resolved) {
|
|
222
|
+
unresolved.push(spec);
|
|
223
|
+
continue;
|
|
224
|
+
}
|
|
225
|
+
if (seen.has(resolved)) continue;
|
|
226
|
+
seen.add(resolved);
|
|
227
|
+
const cached = cache.get(resolved);
|
|
228
|
+
included.push({ filePath: resolved, language: cached?.language ?? null });
|
|
229
|
+
if (included.length >= maxSkeletons) break;
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
const lines = [];
|
|
233
|
+
lines.push("# Compressed Code Context", "");
|
|
234
|
+
lines.push(`Active file: \`${rel(abs)}\``, "");
|
|
235
|
+
lines.push("## Ring 0 \u2014 Active file (full text)", "");
|
|
236
|
+
const ring0 = pruneActiveFile && activeEntry ? activeEntry.skeleton : activeSource;
|
|
237
|
+
lines.push(`\`\`\`${ext(activeFilePath)}`);
|
|
238
|
+
lines.push(ring0.trim() || "(empty or unreadable file)");
|
|
239
|
+
lines.push("```", "");
|
|
240
|
+
lines.push(
|
|
241
|
+
`## Ring 1 \u2014 Pruned dependencies (${included.length})`,
|
|
242
|
+
"",
|
|
243
|
+
"Implementation bodies removed; type signatures, interfaces and exports retained.",
|
|
244
|
+
""
|
|
245
|
+
);
|
|
246
|
+
if (included.length === 0) {
|
|
247
|
+
lines.push("_No local dependency skeletons available._", "");
|
|
248
|
+
}
|
|
249
|
+
for (const inc of included) {
|
|
250
|
+
const entry = cache.get(inc.filePath);
|
|
251
|
+
const label = rel(inc.filePath);
|
|
252
|
+
lines.push(`### \`${label}\``, "");
|
|
253
|
+
if (entry) {
|
|
254
|
+
lines.push(`\`\`\`${ext(inc.filePath)}`);
|
|
255
|
+
lines.push(entry.skeleton.trim());
|
|
256
|
+
lines.push("```", "");
|
|
257
|
+
} else {
|
|
258
|
+
lines.push("_Unindexed file._", "");
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
if (unresolved.length > 0) {
|
|
262
|
+
lines.push("## Unresolved imports", "");
|
|
263
|
+
for (const u of unresolved) lines.push(`- \`${u}\``);
|
|
264
|
+
lines.push("");
|
|
265
|
+
}
|
|
266
|
+
if (includeStats) {
|
|
267
|
+
const depTokens = included.reduce(
|
|
268
|
+
(acc, inc) => {
|
|
269
|
+
const e = cache.get(inc.filePath);
|
|
270
|
+
return e ? acc + approximateTokens(e.skeleton) : acc;
|
|
271
|
+
},
|
|
272
|
+
0
|
|
273
|
+
);
|
|
274
|
+
lines.push("---", "");
|
|
275
|
+
lines.push(
|
|
276
|
+
`_Token estimate \u2014 active: ${approximateTokens(activeSource)} \xB7 pruned deps: ${depTokens}._`,
|
|
277
|
+
""
|
|
278
|
+
);
|
|
279
|
+
}
|
|
280
|
+
return {
|
|
281
|
+
markdown: lines.join("\n"),
|
|
282
|
+
activeFilePath: abs,
|
|
283
|
+
activeSource,
|
|
284
|
+
included,
|
|
285
|
+
unresolved
|
|
286
|
+
};
|
|
287
|
+
}
|
|
288
|
+
function extractSpecifiers(source) {
|
|
289
|
+
const found = [];
|
|
290
|
+
const re = /(?:from\s+['"`]|import\s+['"`]|require\s*\(\s*['"`])([^'"`]+)['"`]/g;
|
|
291
|
+
let m;
|
|
292
|
+
while ((m = re.exec(source)) !== null) {
|
|
293
|
+
if (m[1]) found.push(m[1]);
|
|
294
|
+
}
|
|
295
|
+
return [...new Set(found)];
|
|
296
|
+
}
|
|
297
|
+
function resolveLocal(importer, specifier) {
|
|
298
|
+
if (!/^[.~@]/.test(specifier)) return null;
|
|
299
|
+
if (specifier.startsWith("@/")) specifier = specifier.slice(2);
|
|
300
|
+
else if (specifier.startsWith("~")) specifier = specifier.slice(1);
|
|
301
|
+
const base = import_node_path.default.dirname(import_node_path.default.resolve(importer));
|
|
302
|
+
const p = import_node_path.default.resolve(base, specifier);
|
|
303
|
+
const candidates = [
|
|
304
|
+
p,
|
|
305
|
+
`${p}.ts`,
|
|
306
|
+
`${p}.tsx`,
|
|
307
|
+
`${p}.js`,
|
|
308
|
+
`${p}.jsx`,
|
|
309
|
+
`${p}.py`,
|
|
310
|
+
`${p}.go`,
|
|
311
|
+
`${p}.rs`,
|
|
312
|
+
`${p}.dart`,
|
|
313
|
+
`${p}.swift`,
|
|
314
|
+
`${p}.java`,
|
|
315
|
+
`${p}.kt`,
|
|
316
|
+
`${p}.c`,
|
|
317
|
+
`${p}.cpp`,
|
|
318
|
+
import_node_path.default.join(p, "index.ts"),
|
|
319
|
+
import_node_path.default.join(p, "index.js")
|
|
320
|
+
];
|
|
321
|
+
for (const c of candidates) {
|
|
322
|
+
if (import_node_fs.default.existsSync(c) && import_node_fs.default.statSync(c).isFile()) return import_node_path.default.resolve(c);
|
|
323
|
+
}
|
|
324
|
+
return null;
|
|
325
|
+
}
|
|
326
|
+
function rel(p) {
|
|
327
|
+
try {
|
|
328
|
+
const cwd = process.cwd();
|
|
329
|
+
if (p.startsWith(cwd)) return "." + p.slice(cwd.length);
|
|
330
|
+
} catch {
|
|
331
|
+
}
|
|
332
|
+
return p;
|
|
333
|
+
}
|
|
334
|
+
function ext(p) {
|
|
335
|
+
return import_node_path.default.extname(p).replace(/^\./, "") || "text";
|
|
336
|
+
}
|
|
337
|
+
function approximateTokens(text) {
|
|
338
|
+
if (!text) return 0;
|
|
339
|
+
const tokens = text.match(/[A-Za-z0-9_$]+|[^A-Za-z0-9_\s$]/g);
|
|
340
|
+
return tokens ? tokens.length : 0;
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
// src/watcher/sync.ts
|
|
344
|
+
var import_node_fs3 = __toESM(require("fs"), 1);
|
|
345
|
+
var import_node_path5 = __toESM(require("path"), 1);
|
|
346
|
+
var import_node_crypto = require("crypto");
|
|
347
|
+
var import_chokidar = __toESM(require("chokidar"), 1);
|
|
348
|
+
|
|
349
|
+
// src/parser/pruner.ts
|
|
350
|
+
var import_web_tree_sitter = __toESM(require("web-tree-sitter"), 1);
|
|
351
|
+
var import_node_module = require("module");
|
|
352
|
+
var import_node_path4 = __toESM(require("path"), 1);
|
|
353
|
+
init_registry();
|
|
354
|
+
|
|
355
|
+
// src/parser/wasm.ts
|
|
356
|
+
var import_node_fs2 = __toESM(require("fs"), 1);
|
|
357
|
+
var import_node_path3 = __toESM(require("path"), 1);
|
|
358
|
+
function resolveWasmDir() {
|
|
359
|
+
const candidates = [
|
|
360
|
+
import_node_path3.default.resolve(__dirname, "../../wasm"),
|
|
361
|
+
import_node_path3.default.resolve(__dirname, "../wasm"),
|
|
362
|
+
import_node_path3.default.resolve(process.cwd(), "wasm")
|
|
363
|
+
];
|
|
364
|
+
for (const dir2 of candidates) {
|
|
365
|
+
try {
|
|
366
|
+
import_node_fs2.default.mkdirSync(dir2, { recursive: true });
|
|
367
|
+
return dir2;
|
|
368
|
+
} catch {
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
const dir = import_node_path3.default.resolve(process.cwd(), "wasm");
|
|
372
|
+
import_node_fs2.default.mkdirSync(dir, { recursive: true });
|
|
373
|
+
return dir;
|
|
374
|
+
}
|
|
375
|
+
var wasmDir = null;
|
|
376
|
+
function cacheDir() {
|
|
377
|
+
if (!wasmDir) wasmDir = resolveWasmDir();
|
|
378
|
+
return wasmDir;
|
|
379
|
+
}
|
|
380
|
+
function cachePath(spec) {
|
|
381
|
+
return import_node_path3.default.join(cacheDir(), spec.wasm);
|
|
382
|
+
}
|
|
383
|
+
function looksLikeWasm(bytes) {
|
|
384
|
+
return bytes.length >= 4 && bytes[0] === 0 && bytes[1] === 97 && bytes[2] === 115 && bytes[3] === 109;
|
|
385
|
+
}
|
|
386
|
+
var inflight = /* @__PURE__ */ new Map();
|
|
387
|
+
function download(spec) {
|
|
388
|
+
const pending = inflight.get(spec.wasm);
|
|
389
|
+
if (pending) return pending;
|
|
390
|
+
const job = (async () => {
|
|
391
|
+
const res = await fetch(spec.url, { redirect: "follow" });
|
|
392
|
+
if (!res.ok) {
|
|
393
|
+
throw new Error(
|
|
394
|
+
`Failed to download grammar "${spec.wasm}" from ${spec.url} (HTTP ${res.status})`
|
|
395
|
+
);
|
|
396
|
+
}
|
|
397
|
+
const bytes = new Uint8Array(await res.arrayBuffer());
|
|
398
|
+
if (!looksLikeWasm(bytes)) {
|
|
399
|
+
throw new Error(
|
|
400
|
+
`Downloaded grammar "${spec.wasm}" is not valid WebAssembly (got HTML/redirect).`
|
|
401
|
+
);
|
|
402
|
+
}
|
|
403
|
+
const target = cachePath(spec);
|
|
404
|
+
const tmp = `${target}.${process.pid}.tmp`;
|
|
405
|
+
import_node_fs2.default.writeFileSync(tmp, bytes);
|
|
406
|
+
import_node_fs2.default.renameSync(tmp, target);
|
|
407
|
+
inflight.delete(spec.wasm);
|
|
408
|
+
return bytes;
|
|
409
|
+
})().catch((err) => {
|
|
410
|
+
inflight.delete(spec.wasm);
|
|
411
|
+
throw err;
|
|
412
|
+
});
|
|
413
|
+
inflight.set(spec.wasm, job);
|
|
414
|
+
return job;
|
|
415
|
+
}
|
|
416
|
+
async function getGrammar(spec, force = false) {
|
|
417
|
+
if (!force && import_node_fs2.default.existsSync(cachePath(spec))) {
|
|
418
|
+
const cached = import_node_fs2.default.readFileSync(cachePath(spec));
|
|
419
|
+
if (looksLikeWasm(cached)) return cached;
|
|
420
|
+
}
|
|
421
|
+
return download(spec);
|
|
422
|
+
}
|
|
423
|
+
async function warmGrammars(specs = []) {
|
|
424
|
+
const registryModule = await Promise.resolve().then(() => (init_registry(), registry_exports));
|
|
425
|
+
const list = specs.length > 0 ? specs : Object.values(registryModule.registry);
|
|
426
|
+
const unique = /* @__PURE__ */ new Map();
|
|
427
|
+
for (const spec of list) unique.set(spec.wasm, spec);
|
|
428
|
+
const results = await Promise.allSettled(
|
|
429
|
+
[...unique.values()].map((spec) => getGrammar(spec))
|
|
430
|
+
);
|
|
431
|
+
return results.filter((r) => r.status === "fulfilled").length;
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
// src/parser/pruner.ts
|
|
435
|
+
var languageCache = /* @__PURE__ */ new Map();
|
|
436
|
+
var initPromise = null;
|
|
437
|
+
function ensureParserInit() {
|
|
438
|
+
if (!initPromise) {
|
|
439
|
+
initPromise = import_web_tree_sitter.default.init({
|
|
440
|
+
locateFile: (file) => {
|
|
441
|
+
const require2 = (0, import_node_module.createRequire)(__filename);
|
|
442
|
+
try {
|
|
443
|
+
const pkgRoot = import_node_path4.default.dirname(require2.resolve("web-tree-sitter/package.json"));
|
|
444
|
+
return import_node_path4.default.join(pkgRoot, file);
|
|
445
|
+
} catch {
|
|
446
|
+
return file;
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
});
|
|
450
|
+
}
|
|
451
|
+
return initPromise;
|
|
452
|
+
}
|
|
453
|
+
function loadLanguage(spec, force = false) {
|
|
454
|
+
if (!force) {
|
|
455
|
+
const cached = languageCache.get(spec.wasm);
|
|
456
|
+
if (cached) return Promise.resolve(cached);
|
|
457
|
+
}
|
|
458
|
+
if (force) languageCache.delete(spec.wasm);
|
|
459
|
+
return (async () => {
|
|
460
|
+
await ensureParserInit();
|
|
461
|
+
const grammar = await getGrammar(spec, force);
|
|
462
|
+
const language = await import_web_tree_sitter.default.Language.load(grammar);
|
|
463
|
+
languageCache.set(spec.wasm, language);
|
|
464
|
+
return language;
|
|
465
|
+
})();
|
|
466
|
+
}
|
|
467
|
+
async function prune(filePath, source, opts = {}) {
|
|
468
|
+
const spec = languageForFile(filePath);
|
|
469
|
+
if (!spec || spec.rules.length === 0) {
|
|
470
|
+
return { code: source, language: null, removed: 0 };
|
|
471
|
+
}
|
|
472
|
+
const language = await loadLanguage(spec, opts.forceDownload);
|
|
473
|
+
const parser = new import_web_tree_sitter.default();
|
|
474
|
+
parser.setLanguage(language);
|
|
475
|
+
const tree = parser.parse(source);
|
|
476
|
+
const ranges = [];
|
|
477
|
+
for (const rule of spec.rules) {
|
|
478
|
+
const query = language.query(rule.query);
|
|
479
|
+
const captures = query.captures(tree.rootNode);
|
|
480
|
+
for (const cap of captures) {
|
|
481
|
+
if (cap.node.startIndex === cap.node.endIndex) continue;
|
|
482
|
+
if (rule.replacement.keepIf?.test(cap.node.text)) continue;
|
|
483
|
+
ranges.push({
|
|
484
|
+
start: cap.node.startIndex,
|
|
485
|
+
end: cap.node.endIndex,
|
|
486
|
+
token: rule.replacement.token
|
|
487
|
+
});
|
|
488
|
+
}
|
|
489
|
+
query.delete();
|
|
490
|
+
}
|
|
491
|
+
tree.delete();
|
|
492
|
+
parser.delete();
|
|
493
|
+
const { code, removed } = spliceRanges(source, ranges);
|
|
494
|
+
return { code, language: spec.name, removed };
|
|
495
|
+
}
|
|
496
|
+
function spliceRanges(source, ranges) {
|
|
497
|
+
let removed = 0;
|
|
498
|
+
const sorted = [...ranges].sort((a, b) => b.start - a.start);
|
|
499
|
+
let out = source;
|
|
500
|
+
for (const r of sorted) {
|
|
501
|
+
if (r.start < 0 || r.end > out.length || r.end < r.start) continue;
|
|
502
|
+
removed += r.end - r.start;
|
|
503
|
+
out = out.slice(0, r.start) + r.token + out.slice(r.end);
|
|
504
|
+
}
|
|
505
|
+
return { code: out, removed };
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
// src/watcher/sync.ts
|
|
509
|
+
var DEFAULT_IGNORED = [
|
|
510
|
+
/(^|[/\\])\.[^/\\]+/,
|
|
511
|
+
// dotfiles / dot-dirs (.git, .cursor, .env, ...)
|
|
512
|
+
/node_modules/,
|
|
513
|
+
/[/\\](build|dist|out|coverage|\.next|\.turbo|\.cache)[/\\]/
|
|
514
|
+
];
|
|
515
|
+
var ContextCache = class {
|
|
516
|
+
entries = /* @__PURE__ */ new Map();
|
|
517
|
+
initPromise = null;
|
|
518
|
+
ensureInit() {
|
|
519
|
+
if (!this.initPromise) {
|
|
520
|
+
this.initPromise = ensureParserInit();
|
|
521
|
+
}
|
|
522
|
+
return this.initPromise;
|
|
523
|
+
}
|
|
524
|
+
/** Returns the cached skeleton for a file, if present. */
|
|
525
|
+
getSkeleton(filePath) {
|
|
526
|
+
return this.entries.get(import_node_path5.default.resolve(filePath)) ?? null;
|
|
527
|
+
}
|
|
528
|
+
get imports() {
|
|
529
|
+
return this.entries;
|
|
530
|
+
}
|
|
531
|
+
};
|
|
532
|
+
function hashOf(text) {
|
|
533
|
+
return (0, import_node_crypto.createHash)("sha1").update(text).digest("hex");
|
|
534
|
+
}
|
|
535
|
+
function extractImports(_filePath, source) {
|
|
536
|
+
const out = [];
|
|
537
|
+
const re = /(?:from\s+['"`]([^'"`]+)['"`]|import\s+['"`]([^'"`]+)['"`]|require\s*\(\s*['"]([^'"]+)['"]\s*\))/g;
|
|
538
|
+
let m;
|
|
539
|
+
while ((m = re.exec(source)) !== null) {
|
|
540
|
+
const spec = m[1] ?? m[2] ?? m[3];
|
|
541
|
+
if (spec) out.push(spec);
|
|
542
|
+
}
|
|
543
|
+
return [...new Set(out)].filter(Boolean);
|
|
544
|
+
}
|
|
545
|
+
function resolveImport(importer, specifier, root) {
|
|
546
|
+
if (!/^[.~@]/.test(specifier)) {
|
|
547
|
+
if (specifier.startsWith("@/")) {
|
|
548
|
+
return resolveCandidate(import_node_path5.default.join(root, specifier.slice(2)));
|
|
549
|
+
}
|
|
550
|
+
if (specifier.startsWith("@")) {
|
|
551
|
+
return null;
|
|
552
|
+
}
|
|
553
|
+
if (specifier.startsWith("~")) {
|
|
554
|
+
return resolveCandidate(import_node_path5.default.join(root, specifier.slice(1)));
|
|
555
|
+
}
|
|
556
|
+
return null;
|
|
557
|
+
}
|
|
558
|
+
const base = import_node_path5.default.dirname(import_node_path5.default.resolve(importer));
|
|
559
|
+
return resolveCandidate(import_node_path5.default.resolve(base, specifier));
|
|
560
|
+
}
|
|
561
|
+
function resolveCandidate(p) {
|
|
562
|
+
const candidates = [
|
|
563
|
+
p,
|
|
564
|
+
`${p}.ts`,
|
|
565
|
+
`${p}.tsx`,
|
|
566
|
+
`${p}.js`,
|
|
567
|
+
`${p}.jsx`,
|
|
568
|
+
`${p}.mjs`,
|
|
569
|
+
`${p}.cjs`,
|
|
570
|
+
`${p}.py`,
|
|
571
|
+
`${p}.go`,
|
|
572
|
+
`${p}.rs`,
|
|
573
|
+
`${p}.dart`,
|
|
574
|
+
`${p}.swift`,
|
|
575
|
+
`${p}.java`,
|
|
576
|
+
`${p}.kt`,
|
|
577
|
+
`${p}.c`,
|
|
578
|
+
`${p}.cpp`,
|
|
579
|
+
`${p}.h`,
|
|
580
|
+
`${p}.hpp`,
|
|
581
|
+
`${p}.php`,
|
|
582
|
+
import_node_path5.default.join(p, "index.ts"),
|
|
583
|
+
import_node_path5.default.join(p, "index.js"),
|
|
584
|
+
import_node_path5.default.join(p, "index.tsx"),
|
|
585
|
+
import_node_path5.default.join(p, "index.jsx"),
|
|
586
|
+
import_node_path5.default.join(p, "index.py")
|
|
587
|
+
];
|
|
588
|
+
for (const c of candidates) {
|
|
589
|
+
if (import_node_fs3.default.existsSync(c) && import_node_fs3.default.statSync(c).isFile()) return import_node_path5.default.resolve(c);
|
|
590
|
+
}
|
|
591
|
+
return null;
|
|
592
|
+
}
|
|
593
|
+
function createWatcher(opts) {
|
|
594
|
+
const cache = new ContextCache();
|
|
595
|
+
const { root, ignored = DEFAULT_IGNORED, onIndexed } = opts;
|
|
596
|
+
const pending = /* @__PURE__ */ new Map();
|
|
597
|
+
const DEBOUNCE_MS = 100;
|
|
598
|
+
async function handle(filePath) {
|
|
599
|
+
const abs = import_node_path5.default.resolve(filePath);
|
|
600
|
+
let source;
|
|
601
|
+
try {
|
|
602
|
+
const st = import_node_fs3.default.statSync(abs);
|
|
603
|
+
if (!st.isFile()) return;
|
|
604
|
+
source = import_node_fs3.default.readFileSync(abs, "utf8");
|
|
605
|
+
} catch {
|
|
606
|
+
return;
|
|
607
|
+
}
|
|
608
|
+
const hash = hashOf(source);
|
|
609
|
+
const cached = cache.entries.get(abs);
|
|
610
|
+
if (cached && cached.hash === hash) return;
|
|
611
|
+
await cache.ensureInit();
|
|
612
|
+
const { code, language } = await prune(abs, source);
|
|
613
|
+
const imports = await extractImports(abs, source);
|
|
614
|
+
const entry = {
|
|
615
|
+
hash,
|
|
616
|
+
skeleton: code,
|
|
617
|
+
language,
|
|
618
|
+
imports: imports.map((spec) => resolveImport(abs, spec, root)).filter((p) => p !== null)
|
|
619
|
+
};
|
|
620
|
+
cache.entries.set(abs, entry);
|
|
621
|
+
onIndexed?.(abs, entry);
|
|
622
|
+
}
|
|
623
|
+
const watcher = import_chokidar.default.watch(root, {
|
|
624
|
+
// Combine path matchers with a stat check so non-regular files (e.g. unix
|
|
625
|
+
// sockets) are never opened with fs.watch (which raises UVException).
|
|
626
|
+
ignored(_p, stats) {
|
|
627
|
+
if (stats && !stats.isFile() && !stats.isDirectory()) return true;
|
|
628
|
+
return isIgnored(import_node_path5.default.resolve(String(_p)), ignored);
|
|
629
|
+
},
|
|
630
|
+
alwaysStat: true,
|
|
631
|
+
ignoreInitial: true,
|
|
632
|
+
persistent: true,
|
|
633
|
+
awaitWriteFinish: { stabilityThreshold: 50, pollInterval: 10 }
|
|
634
|
+
});
|
|
635
|
+
function debounce(filePath) {
|
|
636
|
+
const abs = import_node_path5.default.resolve(filePath);
|
|
637
|
+
const existing = pending.get(abs);
|
|
638
|
+
if (existing) clearTimeout(existing);
|
|
639
|
+
if (!existing) void handle(abs);
|
|
640
|
+
pending.set(
|
|
641
|
+
abs,
|
|
642
|
+
setTimeout(() => pending.delete(abs), DEBOUNCE_MS)
|
|
643
|
+
);
|
|
644
|
+
}
|
|
645
|
+
watcher.on("add", debounce);
|
|
646
|
+
watcher.on("change", debounce);
|
|
647
|
+
watcher.on("unlink", (filePath) => {
|
|
648
|
+
const abs = import_node_path5.default.resolve(filePath);
|
|
649
|
+
cache.entries.delete(abs);
|
|
650
|
+
});
|
|
651
|
+
return {
|
|
652
|
+
cache,
|
|
653
|
+
watcher,
|
|
654
|
+
/** Index an existing file right now (bypasses the watcher). */
|
|
655
|
+
index: handle,
|
|
656
|
+
/**
|
|
657
|
+
* Index the whole tree once (cold start). Returns the number of files
|
|
658
|
+
* successfully indexed.
|
|
659
|
+
*/
|
|
660
|
+
async indexAll() {
|
|
661
|
+
const files = [];
|
|
662
|
+
await walk(root, (f) => files.push(f), ignored);
|
|
663
|
+
let ok = 0;
|
|
664
|
+
for (const f of files) {
|
|
665
|
+
try {
|
|
666
|
+
await handle(f);
|
|
667
|
+
ok++;
|
|
668
|
+
} catch {
|
|
669
|
+
}
|
|
670
|
+
}
|
|
671
|
+
return ok;
|
|
672
|
+
},
|
|
673
|
+
close: () => watcher.close()
|
|
674
|
+
};
|
|
675
|
+
}
|
|
676
|
+
async function walk(root, push, ignored) {
|
|
677
|
+
const entries = await import_node_fs3.default.promises.readdir(root, { withFileTypes: true });
|
|
678
|
+
for (const e of entries) {
|
|
679
|
+
const abs = import_node_path5.default.join(root, e.name);
|
|
680
|
+
if (isIgnored(abs, ignored)) continue;
|
|
681
|
+
if (e.isDirectory()) {
|
|
682
|
+
await walk(abs, push, ignored);
|
|
683
|
+
} else if (e.isFile()) {
|
|
684
|
+
push(abs);
|
|
685
|
+
}
|
|
686
|
+
}
|
|
687
|
+
}
|
|
688
|
+
function isIgnored(abs, ignored) {
|
|
689
|
+
for (const m of ignored) {
|
|
690
|
+
if (typeof m === "string" && abs.includes(m)) return true;
|
|
691
|
+
if (m instanceof RegExp && m.test(abs)) return true;
|
|
692
|
+
}
|
|
693
|
+
return false;
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
// src/cli.ts
|
|
697
|
+
function parseArgv(argv = process.argv) {
|
|
698
|
+
const out = {};
|
|
699
|
+
for (let i = 0; i < argv.length; i++) {
|
|
700
|
+
const cur = argv[i];
|
|
701
|
+
if (cur?.startsWith("--")) {
|
|
702
|
+
const key = cur.slice(2);
|
|
703
|
+
const next = argv[i + 1];
|
|
704
|
+
if (next && !next.startsWith("--")) {
|
|
705
|
+
out[key] = next;
|
|
706
|
+
i++;
|
|
707
|
+
} else {
|
|
708
|
+
out[key] = "true";
|
|
709
|
+
}
|
|
710
|
+
}
|
|
711
|
+
}
|
|
712
|
+
return out;
|
|
713
|
+
}
|
|
714
|
+
async function startServer(opts = {}) {
|
|
715
|
+
const args = parseArgv();
|
|
716
|
+
const port = opts.port ?? Number(args.port ?? process.env.PORT ?? 3e3);
|
|
717
|
+
const host = opts.host ?? args.host ?? "0.0.0.0";
|
|
718
|
+
const root = import_node_path6.default.resolve(opts.root ?? args.root ?? process.cwd());
|
|
719
|
+
const silent = opts.silent ?? args.silent === "true";
|
|
720
|
+
const log = (msg) => {
|
|
721
|
+
if (!silent) console.log(import_picocolors.default.dim(msg));
|
|
722
|
+
};
|
|
723
|
+
void warmGrammars().catch(() => {
|
|
724
|
+
});
|
|
725
|
+
const watcher = createWatcher({ root, ignored: opts.ignored });
|
|
726
|
+
log(`Indexing ${root} in the background\u2026`);
|
|
727
|
+
void watcher.indexAll().then((n) => log(`Indexed ${n} files.`));
|
|
728
|
+
const app = (0, import_fastify.default)({ logger: !silent });
|
|
729
|
+
app.get("/health", async () => ({
|
|
730
|
+
status: "ok",
|
|
731
|
+
service: "token-shrink",
|
|
732
|
+
version: "2.0.0",
|
|
733
|
+
root,
|
|
734
|
+
indexed: watcher.cache.entries.size
|
|
735
|
+
}));
|
|
736
|
+
app.post("/v1/context", async (req, reply) => {
|
|
737
|
+
const body = req.body ?? {};
|
|
738
|
+
if (!body.activeFilePath) {
|
|
739
|
+
return reply.status(400).send({ error: "`activeFilePath` is required" });
|
|
740
|
+
}
|
|
741
|
+
const result = assemble(body.activeFilePath, watcher.cache.entries, {
|
|
742
|
+
maxSkeletons: body.maxSkeletons,
|
|
743
|
+
includeStats: body.includeStats
|
|
744
|
+
});
|
|
745
|
+
return {
|
|
746
|
+
markdown: result.markdown,
|
|
747
|
+
activeFilePath: result.activeFilePath,
|
|
748
|
+
dependencies: result.included.map((i) => i.filePath),
|
|
749
|
+
unresolved: result.unresolved
|
|
750
|
+
};
|
|
751
|
+
});
|
|
752
|
+
app.setNotFoundHandler(async (req, reply) => {
|
|
753
|
+
void req;
|
|
754
|
+
return reply.status(404).send({ error: "Not found" });
|
|
755
|
+
});
|
|
756
|
+
await app.listen({ port, host });
|
|
757
|
+
log(`Listening on http://${host}:${port}`);
|
|
758
|
+
return { app, watcher };
|
|
759
|
+
}
|
|
760
|
+
var argv1 = process.argv[1] ? import_node_path6.default.basename(process.argv[1]) : "";
|
|
761
|
+
if (argv1 === "cli.js" || argv1 === "cli.mjs" || argv1 === "cli.cjs" || argv1 === "cli.ts") {
|
|
762
|
+
startServer().catch((err) => {
|
|
763
|
+
console.error(import_picocolors.default.red(`[token-shrink] ${err.message}`));
|
|
764
|
+
process.exitCode = 1;
|
|
765
|
+
});
|
|
766
|
+
}
|
|
767
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
768
|
+
0 && (module.exports = {
|
|
769
|
+
startServer
|
|
770
|
+
});
|
|
771
|
+
//# sourceMappingURL=cli.cjs.map
|