@ajdev0/token-shrink 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs ADDED
@@ -0,0 +1,988 @@
1
+ #!/usr/bin/env node
2
+ "use strict";
3
+ var __create = Object.create;
4
+ var __defProp = Object.defineProperty;
5
+ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
6
+ var __getOwnPropNames = Object.getOwnPropertyNames;
7
+ var __getProtoOf = Object.getPrototypeOf;
8
+ var __hasOwnProp = Object.prototype.hasOwnProperty;
9
+ var __esm = (fn, res) => function __init() {
10
+ return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
11
+ };
12
+ var __export = (target, all) => {
13
+ for (var name in all)
14
+ __defProp(target, name, { get: all[name], enumerable: true });
15
+ };
16
+ var __copyProps = (to, from, except, desc) => {
17
+ if (from && typeof from === "object" || typeof from === "function") {
18
+ for (let key of __getOwnPropNames(from))
19
+ if (!__hasOwnProp.call(to, key) && key !== except)
20
+ __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
21
+ }
22
+ return to;
23
+ };
24
+ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
25
+ // If the importer is in node compatibility mode or this is not an ESM
26
+ // file that has been converted to a CommonJS file using a Babel-
27
+ // compatible transform (i.e. "__esModule" has not been set), then set
28
+ // "default" to the CommonJS "module.exports" for node compatibility.
29
+ isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
30
+ mod
31
+ ));
32
+ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
33
+
34
+ // src/parser/registry.ts
35
+ var registry_exports = {};
36
+ __export(registry_exports, {
37
+ allExtensions: () => allExtensions,
38
+ languageForFile: () => languageForFile,
39
+ registry: () => registry,
40
+ supportedLanguageIds: () => supportedLanguageIds
41
+ });
42
+ function languageForFile(filePath) {
43
+ const langId = extensionIndex.get(import_node_path.default.extname(filePath).toLowerCase());
44
+ if (!langId) return null;
45
+ return registry[langId];
46
+ }
47
+ var import_node_path, BLOCK, SWIFT_BLOCK, GO_BLOCK, RUST_BLOCK, PY_BLOCK, C_BLOCK, JAVA_BLOCK, PHP_BLOCK, KOTLIN_BLOCK, DART_BLOCK, DIRECTIVE_KEEP, registry, extensionIndex, supportedLanguageIds, allExtensions;
48
+ var init_registry = __esm({
49
+ "src/parser/registry.ts"() {
50
+ "use strict";
51
+ import_node_path = __toESM(require("path"), 1);
52
+ BLOCK = { query: "(statement_block) @block", replacement: { token: "/* ... */" } };
53
+ SWIFT_BLOCK = { query: "(statements) @block", replacement: { token: "/* ... */" } };
54
+ GO_BLOCK = { query: "(block) @block", replacement: { token: "/* ... */" } };
55
+ RUST_BLOCK = { query: "(block) @block", replacement: { token: "/* ... */" } };
56
+ PY_BLOCK = { query: "(block) @block", replacement: { token: "pass" } };
57
+ C_BLOCK = { query: "(compound_statement) @block", replacement: { token: "/* ... */" } };
58
+ JAVA_BLOCK = {
59
+ query: "(block) @block",
60
+ replacement: { token: "/* ... */" }
61
+ };
62
+ PHP_BLOCK = {
63
+ query: "(compound_statement) @block",
64
+ replacement: { token: "/* ... */" }
65
+ };
66
+ KOTLIN_BLOCK = { query: "(block) @block", replacement: { token: "/* ... */" } };
67
+ DART_BLOCK = { query: "(block) @block", replacement: { token: "/* ... */" } };
68
+ DIRECTIVE_KEEP = {
69
+ token: "/* ... */",
70
+ keepIf: /^\s*'use (client|server)'/
71
+ };
72
+ registry = {
73
+ typescript: {
74
+ name: "TypeScript",
75
+ wasm: "tree-sitter-typescript.wasm",
76
+ url: "https://github.com/tree-sitter/tree-sitter-typescript/releases/latest/download/tree-sitter-typescript.wasm",
77
+ extensions: [".ts", ".cts", ".mts"],
78
+ rules: [BLOCK]
79
+ },
80
+ javascript: {
81
+ name: "JavaScript",
82
+ wasm: "tree-sitter-javascript.wasm",
83
+ url: "https://github.com/tree-sitter/tree-sitter-javascript/releases/latest/download/tree-sitter-javascript.wasm",
84
+ extensions: [".js", ".cjs", ".mjs"],
85
+ rules: [BLOCK]
86
+ },
87
+ tsx: {
88
+ name: "React / Next.js (TSX)",
89
+ wasm: "tree-sitter-tsx.wasm",
90
+ url: "https://github.com/tree-sitter/tree-sitter-typescript/releases/latest/download/tree-sitter-tsx.wasm",
91
+ extensions: [".tsx"],
92
+ rules: [{ query: "(statement_block) @block", replacement: DIRECTIVE_KEEP }]
93
+ },
94
+ jsx: {
95
+ name: "React (JSX)",
96
+ wasm: "tree-sitter-javascript.wasm",
97
+ url: "https://github.com/tree-sitter/tree-sitter-javascript/releases/latest/download/tree-sitter-javascript.wasm",
98
+ extensions: [".jsx"],
99
+ rules: [{ query: "(statement_block) @block", replacement: DIRECTIVE_KEEP }]
100
+ },
101
+ python: {
102
+ name: "Python",
103
+ wasm: "tree-sitter-python.wasm",
104
+ url: "https://github.com/tree-sitter/tree-sitter-python/releases/latest/download/tree-sitter-python.wasm",
105
+ extensions: [".py", ".pyi"],
106
+ rules: [PY_BLOCK]
107
+ },
108
+ dart: {
109
+ name: "Dart / Flutter",
110
+ wasm: "tree-sitter-dart.wasm",
111
+ url: "https://github.com/UserNobody14/tree-sitter-dart.wasm/releases/latest/download/tree-sitter-dart.wasm",
112
+ extensions: [".dart"],
113
+ rules: [DART_BLOCK]
114
+ },
115
+ swift: {
116
+ name: "Swift / SwiftUI",
117
+ wasm: "tree-sitter-swift.wasm",
118
+ url: "https://github.com/alex-pinkus/tree-sitter-swift/releases/latest/download/tree-sitter-swift.wasm",
119
+ extensions: [".swift"],
120
+ rules: [SWIFT_BLOCK]
121
+ },
122
+ go: {
123
+ name: "Go",
124
+ wasm: "tree-sitter-go.wasm",
125
+ url: "https://github.com/tree-sitter/tree-sitter-go/releases/latest/download/tree-sitter-go.wasm",
126
+ extensions: [".go"],
127
+ rules: [GO_BLOCK]
128
+ },
129
+ rust: {
130
+ name: "Rust",
131
+ wasm: "tree-sitter-rust.wasm",
132
+ url: "https://github.com/tree-sitter/tree-sitter-rust/releases/latest/download/tree-sitter-rust.wasm",
133
+ extensions: [".rs"],
134
+ rules: [RUST_BLOCK]
135
+ },
136
+ java: {
137
+ name: "Java",
138
+ wasm: "tree-sitter-java.wasm",
139
+ url: "https://github.com/tree-sitter/tree-sitter-java/releases/latest/download/tree-sitter-java.wasm",
140
+ extensions: [".java"],
141
+ rules: [JAVA_BLOCK]
142
+ },
143
+ kotlin: {
144
+ name: "Kotlin",
145
+ wasm: "tree-sitter-kotlin.wasm",
146
+ url: "https://github.com/fwcd/tree-sitter-kotlin/releases/latest/download/tree-sitter-kotlin.wasm",
147
+ extensions: [".kt", ".kts"],
148
+ rules: [KOTLIN_BLOCK]
149
+ },
150
+ c: {
151
+ name: "C",
152
+ wasm: "tree-sitter-c.wasm",
153
+ url: "https://github.com/tree-sitter/tree-sitter-c/releases/latest/download/tree-sitter-c.wasm",
154
+ extensions: [".c", ".h"],
155
+ rules: [C_BLOCK]
156
+ },
157
+ cpp: {
158
+ name: "C++",
159
+ wasm: "tree-sitter-cpp.wasm",
160
+ url: "https://github.com/tree-sitter/tree-sitter-cpp/releases/latest/download/tree-sitter-cpp.wasm",
161
+ extensions: [".cc", ".cpp", ".cxx", ".hpp", ".hh", ".hxx"],
162
+ rules: [C_BLOCK]
163
+ },
164
+ php: {
165
+ name: "PHP",
166
+ wasm: "tree-sitter-php.wasm",
167
+ url: "https://github.com/tree-sitter/tree-sitter-php/releases/latest/download/tree-sitter-php.wasm",
168
+ extensions: [".php"],
169
+ rules: [PHP_BLOCK]
170
+ }
171
+ };
172
+ extensionIndex = /* @__PURE__ */ new Map();
173
+ for (const [id, spec] of Object.entries(registry)) {
174
+ for (const ext2 of spec.extensions) {
175
+ extensionIndex.set(ext2, id);
176
+ }
177
+ }
178
+ supportedLanguageIds = Object.keys(registry);
179
+ allExtensions = [...extensionIndex.keys()];
180
+ }
181
+ });
182
+
183
+ // src/index.ts
184
+ var index_exports = {};
185
+ __export(index_exports, {
186
+ AUTO_RULE_PATH: () => AUTO_RULE_PATH,
187
+ AUTO_RULE_SENTINEL: () => AUTO_RULE_SENTINEL,
188
+ CLAUDE_RULE_PATH: () => CLAUDE_RULE_PATH,
189
+ CLAUDE_RULE_SENTINEL: () => CLAUDE_RULE_SENTINEL,
190
+ CURSOR_RULE_PATH: () => CURSOR_RULE_PATH,
191
+ DEFAULT_IGNORED: () => DEFAULT_IGNORED,
192
+ RULE_TARGET_PATH: () => RULE_TARGET_PATH,
193
+ allExtensions: () => allExtensions,
194
+ approximateTokens: () => approximateTokens,
195
+ assemble: () => assemble,
196
+ createAutoRule: () => createAutoRule,
197
+ createCursorRule: () => createCursorRule,
198
+ createWatcher: () => createWatcher,
199
+ extractImports: () => extractImports,
200
+ extractSpecifiers: () => extractSpecifiers,
201
+ getGrammar: () => getGrammar,
202
+ hashOf: () => hashOf,
203
+ languageForFile: () => languageForFile,
204
+ prune: () => prune,
205
+ registry: () => registry,
206
+ resolveImport: () => resolveImport,
207
+ spliceRanges: () => spliceRanges,
208
+ startMcpServer: () => startMcpServer,
209
+ startServer: () => startServer,
210
+ supportedLanguageIds: () => supportedLanguageIds,
211
+ version: () => version,
212
+ warmGrammars: () => warmGrammars
213
+ });
214
+ module.exports = __toCommonJS(index_exports);
215
+
216
+ // src/parser/pruner.ts
217
+ var import_web_tree_sitter = __toESM(require("web-tree-sitter"), 1);
218
+ var import_node_module = require("module");
219
+ var import_node_path3 = __toESM(require("path"), 1);
220
+ init_registry();
221
+
222
+ // src/parser/wasm.ts
223
+ var import_node_fs = __toESM(require("fs"), 1);
224
+ var import_node_path2 = __toESM(require("path"), 1);
225
+ function resolveWasmDir() {
226
+ const candidates = [
227
+ import_node_path2.default.resolve(__dirname, "../../wasm"),
228
+ import_node_path2.default.resolve(__dirname, "../wasm"),
229
+ import_node_path2.default.resolve(process.cwd(), "wasm")
230
+ ];
231
+ for (const dir2 of candidates) {
232
+ try {
233
+ import_node_fs.default.mkdirSync(dir2, { recursive: true });
234
+ return dir2;
235
+ } catch {
236
+ }
237
+ }
238
+ const dir = import_node_path2.default.resolve(process.cwd(), "wasm");
239
+ import_node_fs.default.mkdirSync(dir, { recursive: true });
240
+ return dir;
241
+ }
242
+ var wasmDir = null;
243
+ function cacheDir() {
244
+ if (!wasmDir) wasmDir = resolveWasmDir();
245
+ return wasmDir;
246
+ }
247
+ function cachePath(spec) {
248
+ return import_node_path2.default.join(cacheDir(), spec.wasm);
249
+ }
250
+ function looksLikeWasm(bytes) {
251
+ return bytes.length >= 4 && bytes[0] === 0 && bytes[1] === 97 && bytes[2] === 115 && bytes[3] === 109;
252
+ }
253
+ var inflight = /* @__PURE__ */ new Map();
254
+ function download(spec) {
255
+ const pending = inflight.get(spec.wasm);
256
+ if (pending) return pending;
257
+ const job = (async () => {
258
+ const res = await fetch(spec.url, { redirect: "follow" });
259
+ if (!res.ok) {
260
+ throw new Error(
261
+ `Failed to download grammar "${spec.wasm}" from ${spec.url} (HTTP ${res.status})`
262
+ );
263
+ }
264
+ const bytes = new Uint8Array(await res.arrayBuffer());
265
+ if (!looksLikeWasm(bytes)) {
266
+ throw new Error(
267
+ `Downloaded grammar "${spec.wasm}" is not valid WebAssembly (got HTML/redirect).`
268
+ );
269
+ }
270
+ const target = cachePath(spec);
271
+ const tmp = `${target}.${process.pid}.tmp`;
272
+ import_node_fs.default.writeFileSync(tmp, bytes);
273
+ import_node_fs.default.renameSync(tmp, target);
274
+ inflight.delete(spec.wasm);
275
+ return bytes;
276
+ })().catch((err) => {
277
+ inflight.delete(spec.wasm);
278
+ throw err;
279
+ });
280
+ inflight.set(spec.wasm, job);
281
+ return job;
282
+ }
283
+ async function getGrammar(spec, force = false) {
284
+ if (!force && import_node_fs.default.existsSync(cachePath(spec))) {
285
+ const cached = import_node_fs.default.readFileSync(cachePath(spec));
286
+ if (looksLikeWasm(cached)) return cached;
287
+ }
288
+ return download(spec);
289
+ }
290
+ async function warmGrammars(specs = []) {
291
+ const registryModule = await Promise.resolve().then(() => (init_registry(), registry_exports));
292
+ const list = specs.length > 0 ? specs : Object.values(registryModule.registry);
293
+ const unique = /* @__PURE__ */ new Map();
294
+ for (const spec of list) unique.set(spec.wasm, spec);
295
+ const results = await Promise.allSettled(
296
+ [...unique.values()].map((spec) => getGrammar(spec))
297
+ );
298
+ return results.filter((r) => r.status === "fulfilled").length;
299
+ }
300
+
301
+ // src/parser/pruner.ts
302
+ var languageCache = /* @__PURE__ */ new Map();
303
+ var initPromise = null;
304
+ function ensureParserInit() {
305
+ if (!initPromise) {
306
+ initPromise = import_web_tree_sitter.default.init({
307
+ locateFile: (file) => {
308
+ const require2 = (0, import_node_module.createRequire)(__filename);
309
+ try {
310
+ const pkgRoot = import_node_path3.default.dirname(require2.resolve("web-tree-sitter/package.json"));
311
+ return import_node_path3.default.join(pkgRoot, file);
312
+ } catch {
313
+ return file;
314
+ }
315
+ }
316
+ });
317
+ }
318
+ return initPromise;
319
+ }
320
+ function loadLanguage(spec, force = false) {
321
+ if (!force) {
322
+ const cached = languageCache.get(spec.wasm);
323
+ if (cached) return Promise.resolve(cached);
324
+ }
325
+ if (force) languageCache.delete(spec.wasm);
326
+ return (async () => {
327
+ await ensureParserInit();
328
+ const grammar = await getGrammar(spec, force);
329
+ const language = await import_web_tree_sitter.default.Language.load(grammar);
330
+ languageCache.set(spec.wasm, language);
331
+ return language;
332
+ })();
333
+ }
334
+ async function prune(filePath, source, opts = {}) {
335
+ const spec = languageForFile(filePath);
336
+ if (!spec || spec.rules.length === 0) {
337
+ return { code: source, language: null, removed: 0 };
338
+ }
339
+ const language = await loadLanguage(spec, opts.forceDownload);
340
+ const parser = new import_web_tree_sitter.default();
341
+ parser.setLanguage(language);
342
+ const tree = parser.parse(source);
343
+ const ranges = [];
344
+ for (const rule of spec.rules) {
345
+ const query = language.query(rule.query);
346
+ const captures = query.captures(tree.rootNode);
347
+ for (const cap of captures) {
348
+ if (cap.node.startIndex === cap.node.endIndex) continue;
349
+ if (rule.replacement.keepIf?.test(cap.node.text)) continue;
350
+ ranges.push({
351
+ start: cap.node.startIndex,
352
+ end: cap.node.endIndex,
353
+ token: rule.replacement.token
354
+ });
355
+ }
356
+ query.delete();
357
+ }
358
+ tree.delete();
359
+ parser.delete();
360
+ const { code, removed } = spliceRanges(source, ranges);
361
+ return { code, language: spec.name, removed };
362
+ }
363
+ function spliceRanges(source, ranges) {
364
+ let removed = 0;
365
+ const sorted = [...ranges].sort((a, b) => b.start - a.start);
366
+ let out = source;
367
+ for (const r of sorted) {
368
+ if (r.start < 0 || r.end > out.length || r.end < r.start) continue;
369
+ removed += r.end - r.start;
370
+ out = out.slice(0, r.start) + r.token + out.slice(r.end);
371
+ }
372
+ return { code: out, removed };
373
+ }
374
+
375
+ // src/index.ts
376
+ init_registry();
377
+
378
+ // src/watcher/sync.ts
379
+ var import_node_fs2 = __toESM(require("fs"), 1);
380
+ var import_node_path4 = __toESM(require("path"), 1);
381
+ var import_node_crypto = require("crypto");
382
+ var import_chokidar = __toESM(require("chokidar"), 1);
383
+ var DEFAULT_IGNORED = [
384
+ /(^|[/\\])\.[^/\\]+/,
385
+ // dotfiles / dot-dirs (.git, .cursor, .env, ...)
386
+ /node_modules/,
387
+ /[/\\](build|dist|out|coverage|\.next|\.turbo|\.cache)[/\\]/
388
+ ];
389
+ var ContextCache = class {
390
+ entries = /* @__PURE__ */ new Map();
391
+ initPromise = null;
392
+ ensureInit() {
393
+ if (!this.initPromise) {
394
+ this.initPromise = ensureParserInit();
395
+ }
396
+ return this.initPromise;
397
+ }
398
+ /** Returns the cached skeleton for a file, if present. */
399
+ getSkeleton(filePath) {
400
+ return this.entries.get(import_node_path4.default.resolve(filePath)) ?? null;
401
+ }
402
+ get imports() {
403
+ return this.entries;
404
+ }
405
+ };
406
+ function hashOf(text) {
407
+ return (0, import_node_crypto.createHash)("sha1").update(text).digest("hex");
408
+ }
409
+ function extractImports(_filePath, source) {
410
+ const out = [];
411
+ const re = /(?:from\s+['"`]([^'"`]+)['"`]|import\s+['"`]([^'"`]+)['"`]|require\s*\(\s*['"]([^'"]+)['"]\s*\))/g;
412
+ let m;
413
+ while ((m = re.exec(source)) !== null) {
414
+ const spec = m[1] ?? m[2] ?? m[3];
415
+ if (spec) out.push(spec);
416
+ }
417
+ return [...new Set(out)].filter(Boolean);
418
+ }
419
+ function resolveImport(importer, specifier, root) {
420
+ if (!/^[.~@]/.test(specifier)) {
421
+ if (specifier.startsWith("@/")) {
422
+ return resolveCandidate(import_node_path4.default.join(root, specifier.slice(2)));
423
+ }
424
+ if (specifier.startsWith("@")) {
425
+ return null;
426
+ }
427
+ if (specifier.startsWith("~")) {
428
+ return resolveCandidate(import_node_path4.default.join(root, specifier.slice(1)));
429
+ }
430
+ return null;
431
+ }
432
+ const base = import_node_path4.default.dirname(import_node_path4.default.resolve(importer));
433
+ return resolveCandidate(import_node_path4.default.resolve(base, specifier));
434
+ }
435
+ function resolveCandidate(p) {
436
+ const candidates = [
437
+ p,
438
+ `${p}.ts`,
439
+ `${p}.tsx`,
440
+ `${p}.js`,
441
+ `${p}.jsx`,
442
+ `${p}.mjs`,
443
+ `${p}.cjs`,
444
+ `${p}.py`,
445
+ `${p}.go`,
446
+ `${p}.rs`,
447
+ `${p}.dart`,
448
+ `${p}.swift`,
449
+ `${p}.java`,
450
+ `${p}.kt`,
451
+ `${p}.c`,
452
+ `${p}.cpp`,
453
+ `${p}.h`,
454
+ `${p}.hpp`,
455
+ `${p}.php`,
456
+ import_node_path4.default.join(p, "index.ts"),
457
+ import_node_path4.default.join(p, "index.js"),
458
+ import_node_path4.default.join(p, "index.tsx"),
459
+ import_node_path4.default.join(p, "index.jsx"),
460
+ import_node_path4.default.join(p, "index.py")
461
+ ];
462
+ for (const c of candidates) {
463
+ if (import_node_fs2.default.existsSync(c) && import_node_fs2.default.statSync(c).isFile()) return import_node_path4.default.resolve(c);
464
+ }
465
+ return null;
466
+ }
467
+ function createWatcher(opts) {
468
+ const cache = new ContextCache();
469
+ const { root, ignored = DEFAULT_IGNORED, onIndexed } = opts;
470
+ const pending = /* @__PURE__ */ new Map();
471
+ const DEBOUNCE_MS = 100;
472
+ async function handle(filePath) {
473
+ const abs = import_node_path4.default.resolve(filePath);
474
+ let source;
475
+ try {
476
+ const st = import_node_fs2.default.statSync(abs);
477
+ if (!st.isFile()) return;
478
+ source = import_node_fs2.default.readFileSync(abs, "utf8");
479
+ } catch {
480
+ return;
481
+ }
482
+ const hash = hashOf(source);
483
+ const cached = cache.entries.get(abs);
484
+ if (cached && cached.hash === hash) return;
485
+ await cache.ensureInit();
486
+ const { code, language } = await prune(abs, source);
487
+ const imports = await extractImports(abs, source);
488
+ const entry = {
489
+ hash,
490
+ skeleton: code,
491
+ language,
492
+ imports: imports.map((spec) => resolveImport(abs, spec, root)).filter((p) => p !== null)
493
+ };
494
+ cache.entries.set(abs, entry);
495
+ onIndexed?.(abs, entry);
496
+ }
497
+ const watcher = import_chokidar.default.watch(root, {
498
+ // Combine path matchers with a stat check so non-regular files (e.g. unix
499
+ // sockets) are never opened with fs.watch (which raises UVException).
500
+ ignored(_p, stats) {
501
+ if (stats && !stats.isFile() && !stats.isDirectory()) return true;
502
+ return isIgnored(import_node_path4.default.resolve(String(_p)), ignored);
503
+ },
504
+ alwaysStat: true,
505
+ ignoreInitial: true,
506
+ persistent: true,
507
+ awaitWriteFinish: { stabilityThreshold: 50, pollInterval: 10 }
508
+ });
509
+ function debounce(filePath) {
510
+ const abs = import_node_path4.default.resolve(filePath);
511
+ const existing = pending.get(abs);
512
+ if (existing) clearTimeout(existing);
513
+ if (!existing) void handle(abs);
514
+ pending.set(
515
+ abs,
516
+ setTimeout(() => pending.delete(abs), DEBOUNCE_MS)
517
+ );
518
+ }
519
+ watcher.on("add", debounce);
520
+ watcher.on("change", debounce);
521
+ watcher.on("unlink", (filePath) => {
522
+ const abs = import_node_path4.default.resolve(filePath);
523
+ cache.entries.delete(abs);
524
+ });
525
+ return {
526
+ cache,
527
+ watcher,
528
+ /** Index an existing file right now (bypasses the watcher). */
529
+ index: handle,
530
+ /**
531
+ * Index the whole tree once (cold start). Returns the number of files
532
+ * successfully indexed.
533
+ */
534
+ async indexAll() {
535
+ const files = [];
536
+ await walk(root, (f) => files.push(f), ignored);
537
+ let ok = 0;
538
+ for (const f of files) {
539
+ try {
540
+ await handle(f);
541
+ ok++;
542
+ } catch {
543
+ }
544
+ }
545
+ return ok;
546
+ },
547
+ close: () => watcher.close()
548
+ };
549
+ }
550
+ async function walk(root, push, ignored) {
551
+ const entries = await import_node_fs2.default.promises.readdir(root, { withFileTypes: true });
552
+ for (const e of entries) {
553
+ const abs = import_node_path4.default.join(root, e.name);
554
+ if (isIgnored(abs, ignored)) continue;
555
+ if (e.isDirectory()) {
556
+ await walk(abs, push, ignored);
557
+ } else if (e.isFile()) {
558
+ push(abs);
559
+ }
560
+ }
561
+ }
562
+ function isIgnored(abs, ignored) {
563
+ for (const m of ignored) {
564
+ if (typeof m === "string" && abs.includes(m)) return true;
565
+ if (m instanceof RegExp && m.test(abs)) return true;
566
+ }
567
+ return false;
568
+ }
569
+
570
+ // src/server/assembler.ts
571
+ var import_node_fs3 = __toESM(require("fs"), 1);
572
+ var import_node_path5 = __toESM(require("path"), 1);
573
+ function assemble(activeFilePath, cache, opts = {}) {
574
+ const { includeStats = false, maxSkeletons = 50, pruneActiveFile = false } = opts;
575
+ const abs = import_node_path5.default.resolve(activeFilePath);
576
+ const activeSource = import_node_fs3.default.existsSync(abs) ? import_node_fs3.default.readFileSync(abs, "utf8") : "";
577
+ const activeEntry = cache.get(abs);
578
+ const included = [];
579
+ const unresolved = [];
580
+ const seen = /* @__PURE__ */ new Set();
581
+ if (activeEntry) {
582
+ for (const imp of activeEntry.imports) {
583
+ const entry = cache.get(imp);
584
+ if (!entry) {
585
+ unresolved.push(imp);
586
+ continue;
587
+ }
588
+ if (seen.has(imp)) continue;
589
+ seen.add(imp);
590
+ included.push({ filePath: imp, language: entry.language });
591
+ if (included.length >= maxSkeletons) break;
592
+ }
593
+ }
594
+ if (!activeEntry && activeSource) {
595
+ const specifiers = extractSpecifiers(activeSource);
596
+ for (const spec of specifiers) {
597
+ const resolved = resolveLocal(abs, spec);
598
+ if (!resolved) {
599
+ unresolved.push(spec);
600
+ continue;
601
+ }
602
+ if (seen.has(resolved)) continue;
603
+ seen.add(resolved);
604
+ const cached = cache.get(resolved);
605
+ included.push({ filePath: resolved, language: cached?.language ?? null });
606
+ if (included.length >= maxSkeletons) break;
607
+ }
608
+ }
609
+ const lines = [];
610
+ lines.push("# Compressed Code Context", "");
611
+ lines.push(`Active file: \`${rel(abs)}\``, "");
612
+ lines.push("## Ring 0 \u2014 Active file (full text)", "");
613
+ const ring0 = pruneActiveFile && activeEntry ? activeEntry.skeleton : activeSource;
614
+ lines.push(`\`\`\`${ext(activeFilePath)}`);
615
+ lines.push(ring0.trim() || "(empty or unreadable file)");
616
+ lines.push("```", "");
617
+ lines.push(
618
+ `## Ring 1 \u2014 Pruned dependencies (${included.length})`,
619
+ "",
620
+ "Implementation bodies removed; type signatures, interfaces and exports retained.",
621
+ ""
622
+ );
623
+ if (included.length === 0) {
624
+ lines.push("_No local dependency skeletons available._", "");
625
+ }
626
+ for (const inc of included) {
627
+ const entry = cache.get(inc.filePath);
628
+ const label = rel(inc.filePath);
629
+ lines.push(`### \`${label}\``, "");
630
+ if (entry) {
631
+ lines.push(`\`\`\`${ext(inc.filePath)}`);
632
+ lines.push(entry.skeleton.trim());
633
+ lines.push("```", "");
634
+ } else {
635
+ lines.push("_Unindexed file._", "");
636
+ }
637
+ }
638
+ if (unresolved.length > 0) {
639
+ lines.push("## Unresolved imports", "");
640
+ for (const u of unresolved) lines.push(`- \`${u}\``);
641
+ lines.push("");
642
+ }
643
+ if (includeStats) {
644
+ const depTokens = included.reduce(
645
+ (acc, inc) => {
646
+ const e = cache.get(inc.filePath);
647
+ return e ? acc + approximateTokens(e.skeleton) : acc;
648
+ },
649
+ 0
650
+ );
651
+ lines.push("---", "");
652
+ lines.push(
653
+ `_Token estimate \u2014 active: ${approximateTokens(activeSource)} \xB7 pruned deps: ${depTokens}._`,
654
+ ""
655
+ );
656
+ }
657
+ return {
658
+ markdown: lines.join("\n"),
659
+ activeFilePath: abs,
660
+ activeSource,
661
+ included,
662
+ unresolved
663
+ };
664
+ }
665
+ function extractSpecifiers(source) {
666
+ const found = [];
667
+ const re = /(?:from\s+['"`]|import\s+['"`]|require\s*\(\s*['"`])([^'"`]+)['"`]/g;
668
+ let m;
669
+ while ((m = re.exec(source)) !== null) {
670
+ if (m[1]) found.push(m[1]);
671
+ }
672
+ return [...new Set(found)];
673
+ }
674
+ function resolveLocal(importer, specifier) {
675
+ if (!/^[.~@]/.test(specifier)) return null;
676
+ if (specifier.startsWith("@/")) specifier = specifier.slice(2);
677
+ else if (specifier.startsWith("~")) specifier = specifier.slice(1);
678
+ const base = import_node_path5.default.dirname(import_node_path5.default.resolve(importer));
679
+ const p = import_node_path5.default.resolve(base, specifier);
680
+ const candidates = [
681
+ p,
682
+ `${p}.ts`,
683
+ `${p}.tsx`,
684
+ `${p}.js`,
685
+ `${p}.jsx`,
686
+ `${p}.py`,
687
+ `${p}.go`,
688
+ `${p}.rs`,
689
+ `${p}.dart`,
690
+ `${p}.swift`,
691
+ `${p}.java`,
692
+ `${p}.kt`,
693
+ `${p}.c`,
694
+ `${p}.cpp`,
695
+ import_node_path5.default.join(p, "index.ts"),
696
+ import_node_path5.default.join(p, "index.js")
697
+ ];
698
+ for (const c of candidates) {
699
+ if (import_node_fs3.default.existsSync(c) && import_node_fs3.default.statSync(c).isFile()) return import_node_path5.default.resolve(c);
700
+ }
701
+ return null;
702
+ }
703
+ function rel(p) {
704
+ try {
705
+ const cwd = process.cwd();
706
+ if (p.startsWith(cwd)) return "." + p.slice(cwd.length);
707
+ } catch {
708
+ }
709
+ return p;
710
+ }
711
+ function ext(p) {
712
+ return import_node_path5.default.extname(p).replace(/^\./, "") || "text";
713
+ }
714
+ function approximateTokens(text) {
715
+ if (!text) return 0;
716
+ const tokens = text.match(/[A-Za-z0-9_$]+|[^A-Za-z0-9_\s$]/g);
717
+ return tokens ? tokens.length : 0;
718
+ }
719
+
720
+ // src/cli.ts
721
+ var import_fastify = __toESM(require("fastify"), 1);
722
+ var import_picocolors = __toESM(require("picocolors"), 1);
723
+ var import_node_path6 = __toESM(require("path"), 1);
724
+ function parseArgv(argv = process.argv) {
725
+ const out = {};
726
+ for (let i = 0; i < argv.length; i++) {
727
+ const cur = argv[i];
728
+ if (cur?.startsWith("--")) {
729
+ const key = cur.slice(2);
730
+ const next = argv[i + 1];
731
+ if (next && !next.startsWith("--")) {
732
+ out[key] = next;
733
+ i++;
734
+ } else {
735
+ out[key] = "true";
736
+ }
737
+ }
738
+ }
739
+ return out;
740
+ }
741
+ async function startServer(opts = {}) {
742
+ const args = parseArgv();
743
+ const port = opts.port ?? Number(args.port ?? process.env.PORT ?? 3e3);
744
+ const host = opts.host ?? args.host ?? "0.0.0.0";
745
+ const root = import_node_path6.default.resolve(opts.root ?? args.root ?? process.cwd());
746
+ const silent = opts.silent ?? args.silent === "true";
747
+ const log = (msg) => {
748
+ if (!silent) console.log(import_picocolors.default.dim(msg));
749
+ };
750
+ void warmGrammars().catch(() => {
751
+ });
752
+ const watcher = createWatcher({ root, ignored: opts.ignored });
753
+ log(`Indexing ${root} in the background\u2026`);
754
+ void watcher.indexAll().then((n) => log(`Indexed ${n} files.`));
755
+ const app = (0, import_fastify.default)({ logger: !silent });
756
+ app.get("/health", async () => ({
757
+ status: "ok",
758
+ service: "token-shrink",
759
+ version: "2.0.0",
760
+ root,
761
+ indexed: watcher.cache.entries.size
762
+ }));
763
+ app.post("/v1/context", async (req, reply) => {
764
+ const body = req.body ?? {};
765
+ if (!body.activeFilePath) {
766
+ return reply.status(400).send({ error: "`activeFilePath` is required" });
767
+ }
768
+ const result = assemble(body.activeFilePath, watcher.cache.entries, {
769
+ maxSkeletons: body.maxSkeletons,
770
+ includeStats: body.includeStats
771
+ });
772
+ return {
773
+ markdown: result.markdown,
774
+ activeFilePath: result.activeFilePath,
775
+ dependencies: result.included.map((i) => i.filePath),
776
+ unresolved: result.unresolved
777
+ };
778
+ });
779
+ app.setNotFoundHandler(async (req, reply) => {
780
+ void req;
781
+ return reply.status(404).send({ error: "Not found" });
782
+ });
783
+ await app.listen({ port, host });
784
+ log(`Listening on http://${host}:${port}`);
785
+ return { app, watcher };
786
+ }
787
+ var argv1 = process.argv[1] ? import_node_path6.default.basename(process.argv[1]) : "";
788
+ if (argv1 === "cli.js" || argv1 === "cli.mjs" || argv1 === "cli.cjs" || argv1 === "cli.ts") {
789
+ startServer().catch((err) => {
790
+ console.error(import_picocolors.default.red(`[token-shrink] ${err.message}`));
791
+ process.exitCode = 1;
792
+ });
793
+ }
794
+
795
+ // src/mcp.ts
796
+ var import_mcp = require("@modelcontextprotocol/sdk/server/mcp.js");
797
+ var import_stdio = require("@modelcontextprotocol/sdk/server/stdio.js");
798
+ var import_zod = require("zod");
799
+ var import_node_fs4 = __toESM(require("fs"), 1);
800
+ var import_node_path7 = __toESM(require("path"), 1);
801
+ var RULE_TARGET_PATH = {
802
+ cursor: ".cursor/rules/token-shrink.mdc",
803
+ claude: ".claude/rules/token-shrink.md"
804
+ };
805
+ var AUTO_RULE_SENTINEL = "# auto-generated by token-shrink";
806
+ var CURSOR_RULE_PATH = RULE_TARGET_PATH.cursor;
807
+ var AUTO_RULE_PATH = CURSOR_RULE_PATH;
808
+ var CLAUDE_RULE_PATH = RULE_TARGET_PATH.claude;
809
+ var CLAUDE_RULE_SENTINEL = "# auto-generated by token-shrink (claude)";
810
+ var ruleTargets = {
811
+ cursor: {
812
+ relPath: CURSOR_RULE_PATH,
813
+ sentinel: AUTO_RULE_SENTINEL,
814
+ body: `---
815
+ description: Compress dependency context with token-shrink on every task
816
+ globs: **/*.{ts,tsx,js,jsx,py,go,rs,dart,swift,java,kt,c,cpp,h,hpp,php}
817
+ alwaysApply: true
818
+ ---
819
+ Before working on a file in this repo, call the \`get_compressed_code_context\` MCP tool with
820
+ that file's path, and use the returned Ring 0 + Ring 1 payload as the context for that file
821
+ and its direct imports.
822
+
823
+ ${AUTO_RULE_SENTINEL}
824
+ `
825
+ },
826
+ claude: {
827
+ relPath: CLAUDE_RULE_PATH,
828
+ sentinel: CLAUDE_RULE_SENTINEL,
829
+ body: `---
830
+ description: Compress dependency context with token-shrink on every task
831
+ paths: ["**/*.{ts,tsx,js,jsx,py,go,rs,dart,swift,java,kt,c,cpp,h,hpp,php}"]
832
+ ---
833
+ Before working on a file in this repo, call the \`get_compressed_code_context\` MCP tool with
834
+ that file's path, and use the returned Ring 0 + Ring 1 payload as the context for that file
835
+ and its direct imports.
836
+
837
+ ${CLAUDE_RULE_SENTINEL}
838
+ `
839
+ }
840
+ };
841
+ function createAutoRule(root, target) {
842
+ const spec = ruleTargets[target];
843
+ const rulePath = import_node_path7.default.join(root, spec.relPath);
844
+ try {
845
+ if (import_node_fs4.default.existsSync(rulePath)) {
846
+ const existing = import_node_fs4.default.readFileSync(rulePath, "utf8");
847
+ if (existing.includes(spec.sentinel)) {
848
+ return { created: false, skipped: "exists", filePath: rulePath };
849
+ }
850
+ return { created: false, skipped: "user", filePath: rulePath };
851
+ }
852
+ import_node_fs4.default.mkdirSync(import_node_path7.default.dirname(rulePath), { recursive: true });
853
+ import_node_fs4.default.writeFileSync(rulePath, spec.body, "utf8");
854
+ return { created: true, skipped: "none", filePath: rulePath };
855
+ } catch (err) {
856
+ process.stderr.write(
857
+ `[token-shrink] Failed to write rule for "${target}": ${err.message}
858
+ `
859
+ );
860
+ return { created: false, skipped: "none", filePath: rulePath };
861
+ }
862
+ }
863
+ function createCursorRule(root) {
864
+ return createAutoRule(root, "cursor");
865
+ }
866
+ function resolveTargets(ruleTarget) {
867
+ if (!ruleTarget) return ["cursor", "claude"];
868
+ const list = Array.isArray(ruleTarget) ? ruleTarget : [ruleTarget];
869
+ if (list.includes("all")) return ["cursor", "claude"];
870
+ return list;
871
+ }
872
+ async function startMcpServer(opts = {}) {
873
+ const root = import_node_path7.default.resolve(opts.root ?? process.env.ROOT ?? process.cwd());
874
+ const log = (msg) => {
875
+ if (!opts.silent) process.stderr.write(`[token-shrink] ${msg}
876
+ `);
877
+ };
878
+ const watcher = createWatcher({ root, ignored: opts.ignored });
879
+ if (!opts.silent) {
880
+ log(`Indexing ${root} in the background\u2026`);
881
+ }
882
+ if (opts.createRule !== false) {
883
+ for (const target of resolveTargets(opts.ruleTarget)) {
884
+ const res = createAutoRule(root, target);
885
+ if (res.created) {
886
+ log(`Wrote ${target} rule to ${res.filePath}`);
887
+ } else if (res.skipped === "user") {
888
+ log(`${target} rule exists (user-authored); leaving it untouched.`);
889
+ }
890
+ }
891
+ }
892
+ void watcher.indexAll().then((n) => log(`Indexed ${n} files.`));
893
+ const server = new import_mcp.McpServer(
894
+ { name: "token-shrink", version: "2.0.0" },
895
+ { capabilities: { tools: {} } }
896
+ );
897
+ server.registerTool(
898
+ "get_compressed_code_context",
899
+ {
900
+ title: "Get Compressed Code Context",
901
+ description: "Returns a compressed, framework-aware AST context payload for a file: the active file\u2019s full source (Ring 0) plus pruned skeletons of its direct imports (Ring 1). Implementation bodies are removed but type signatures, interfaces, and module exports are preserved for ~80-90% token reduction.",
902
+ inputSchema: {
903
+ activeFilePath: import_zod.z.string().describe("Path to the file the agent is working on"),
904
+ maxSkeletons: import_zod.z.number().int().min(1).max(200).optional().describe("Cap on number of dependency skeletons to include"),
905
+ includeStats: import_zod.z.boolean().optional().describe("Append approximate token-count stats")
906
+ }
907
+ },
908
+ async ({ activeFilePath, maxSkeletons, includeStats }) => {
909
+ const result = assemble(activeFilePath, watcher.cache.entries, {
910
+ maxSkeletons,
911
+ includeStats
912
+ });
913
+ return {
914
+ content: [
915
+ {
916
+ type: "text",
917
+ text: result.markdown
918
+ },
919
+ {
920
+ type: "text",
921
+ text: `[stats] active=${result.activeFilePath} dependencies=${result.included.length} unresolved=${result.unresolved.length}`
922
+ }
923
+ ]
924
+ };
925
+ }
926
+ );
927
+ const transport = new import_stdio.StdioServerTransport();
928
+ await server.connect(transport);
929
+ log("MCP server connected.");
930
+ return server;
931
+ }
932
+ var argv12 = process.argv[1] ? import_node_path7.default.basename(process.argv[1]) : "";
933
+ if (argv12 === "mcp.js" || argv12 === "mcp.mjs" || argv12 === "mcp.cjs" || argv12 === "mcp.ts") {
934
+ const rootArg = (() => {
935
+ const i = process.argv.indexOf("--root");
936
+ if (i !== -1 && process.argv[i + 1]) return process.argv[i + 1];
937
+ const eq = process.argv.find((a) => a.startsWith("--root="));
938
+ if (eq) return eq.slice("--root=".length);
939
+ return void 0;
940
+ })();
941
+ const createRule = !(process.env.TOKEN_SHRINK_CREATE_RULE === "0" || process.env.TOKEN_SHRINK_CREATE_RULE === "false" || process.env.CONTEXT_SHRINK_CREATE_RULE === "0" || process.env.CONTEXT_SHRINK_CREATE_RULE === "false" || process.argv.includes("--no-create-rule") || (() => {
942
+ const f = process.argv.find((a) => a.startsWith("--create-rule="));
943
+ return f ? f.slice("--create-rule=".length) === "false" : false;
944
+ })());
945
+ const rawTargets = process.argv.filter((a) => a.startsWith("--rule-target=")).flatMap((a) => a.slice("--rule-target=".length).split(","));
946
+ const ruleTarget = rawTargets.length > 0 && !rawTargets.includes("all") ? [...new Set(rawTargets)].filter(
947
+ (v) => v === "cursor" || v === "claude"
948
+ ) : void 0;
949
+ void startMcpServer({ root: rootArg, createRule, ruleTarget }).catch((err) => {
950
+ process.stderr.write(`[token-shrink] MCP server error: ${err.message}
951
+ `);
952
+ process.exitCode = 1;
953
+ });
954
+ }
955
+
956
+ // src/index.ts
957
+ var version = "2.0.0";
958
+ // Annotate the CommonJS export names for ESM import in node:
959
+ 0 && (module.exports = {
960
+ AUTO_RULE_PATH,
961
+ AUTO_RULE_SENTINEL,
962
+ CLAUDE_RULE_PATH,
963
+ CLAUDE_RULE_SENTINEL,
964
+ CURSOR_RULE_PATH,
965
+ DEFAULT_IGNORED,
966
+ RULE_TARGET_PATH,
967
+ allExtensions,
968
+ approximateTokens,
969
+ assemble,
970
+ createAutoRule,
971
+ createCursorRule,
972
+ createWatcher,
973
+ extractImports,
974
+ extractSpecifiers,
975
+ getGrammar,
976
+ hashOf,
977
+ languageForFile,
978
+ prune,
979
+ registry,
980
+ resolveImport,
981
+ spliceRanges,
982
+ startMcpServer,
983
+ startServer,
984
+ supportedLanguageIds,
985
+ version,
986
+ warmGrammars
987
+ });
988
+ //# sourceMappingURL=index.cjs.map