@spikedpunch/mast 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (257) hide show
  1. package/LICENSE +21 -0
  2. package/MAST_SPEC.md +2926 -0
  3. package/README.md +842 -0
  4. package/assets/skill.md +45 -0
  5. package/dist/ast/extract.d.ts +65 -0
  6. package/dist/ast/extract.d.ts.map +1 -0
  7. package/dist/ast/extract.js +147 -0
  8. package/dist/ast/extract.js.map +1 -0
  9. package/dist/ast/extractor.d.ts +85 -0
  10. package/dist/ast/extractor.d.ts.map +1 -0
  11. package/dist/ast/extractor.js +2 -0
  12. package/dist/ast/extractor.js.map +1 -0
  13. package/dist/ast/extractors/markdown.d.ts +40 -0
  14. package/dist/ast/extractors/markdown.d.ts.map +1 -0
  15. package/dist/ast/extractors/markdown.js +175 -0
  16. package/dist/ast/extractors/markdown.js.map +1 -0
  17. package/dist/ast/extractors/typescript.d.ts +179 -0
  18. package/dist/ast/extractors/typescript.d.ts.map +1 -0
  19. package/dist/ast/extractors/typescript.js +1439 -0
  20. package/dist/ast/extractors/typescript.js.map +1 -0
  21. package/dist/ast/parser.d.ts +5 -0
  22. package/dist/ast/parser.d.ts.map +1 -0
  23. package/dist/ast/parser.js +21 -0
  24. package/dist/ast/parser.js.map +1 -0
  25. package/dist/ast/types.d.ts +586 -0
  26. package/dist/ast/types.d.ts.map +1 -0
  27. package/dist/ast/types.js +3 -0
  28. package/dist/ast/types.js.map +1 -0
  29. package/dist/cli/docs-cmd.d.ts +22 -0
  30. package/dist/cli/docs-cmd.d.ts.map +1 -0
  31. package/dist/cli/docs-cmd.js +100 -0
  32. package/dist/cli/docs-cmd.js.map +1 -0
  33. package/dist/cli/index-cmd.d.ts +72 -0
  34. package/dist/cli/index-cmd.d.ts.map +1 -0
  35. package/dist/cli/index-cmd.js +263 -0
  36. package/dist/cli/index-cmd.js.map +1 -0
  37. package/dist/cli/index.d.ts +3 -0
  38. package/dist/cli/index.d.ts.map +1 -0
  39. package/dist/cli/index.js +4 -0
  40. package/dist/cli/index.js.map +1 -0
  41. package/dist/cli/init.d.ts +14 -0
  42. package/dist/cli/init.d.ts.map +1 -0
  43. package/dist/cli/init.js +64 -0
  44. package/dist/cli/init.js.map +1 -0
  45. package/dist/cli/install-hooks.d.ts +3 -0
  46. package/dist/cli/install-hooks.d.ts.map +1 -0
  47. package/dist/cli/install-hooks.js +34 -0
  48. package/dist/cli/install-hooks.js.map +1 -0
  49. package/dist/cli/metrics-cmd.d.ts +3 -0
  50. package/dist/cli/metrics-cmd.d.ts.map +1 -0
  51. package/dist/cli/metrics-cmd.js +164 -0
  52. package/dist/cli/metrics-cmd.js.map +1 -0
  53. package/dist/cli/program.d.ts +14 -0
  54. package/dist/cli/program.d.ts.map +1 -0
  55. package/dist/cli/program.js +44 -0
  56. package/dist/cli/program.js.map +1 -0
  57. package/dist/cli/query.d.ts +46 -0
  58. package/dist/cli/query.d.ts.map +1 -0
  59. package/dist/cli/query.js +153 -0
  60. package/dist/cli/query.js.map +1 -0
  61. package/dist/cli/search-cmd.d.ts +33 -0
  62. package/dist/cli/search-cmd.d.ts.map +1 -0
  63. package/dist/cli/search-cmd.js +114 -0
  64. package/dist/cli/search-cmd.js.map +1 -0
  65. package/dist/cli/serve.d.ts +3 -0
  66. package/dist/cli/serve.d.ts.map +1 -0
  67. package/dist/cli/serve.js +19 -0
  68. package/dist/cli/serve.js.map +1 -0
  69. package/dist/cli/skill-install.d.ts +39 -0
  70. package/dist/cli/skill-install.d.ts.map +1 -0
  71. package/dist/cli/skill-install.js +57 -0
  72. package/dist/cli/skill-install.js.map +1 -0
  73. package/dist/cli/status.d.ts +35 -0
  74. package/dist/cli/status.d.ts.map +1 -0
  75. package/dist/cli/status.js +117 -0
  76. package/dist/cli/status.js.map +1 -0
  77. package/dist/cli/upgrade-cmd.d.ts +48 -0
  78. package/dist/cli/upgrade-cmd.d.ts.map +1 -0
  79. package/dist/cli/upgrade-cmd.js +129 -0
  80. package/dist/cli/upgrade-cmd.js.map +1 -0
  81. package/dist/cli/version.d.ts +3 -0
  82. package/dist/cli/version.d.ts.map +1 -0
  83. package/dist/cli/version.js +19 -0
  84. package/dist/cli/version.js.map +1 -0
  85. package/dist/env.d.ts +17 -0
  86. package/dist/env.d.ts.map +1 -0
  87. package/dist/env.js +17 -0
  88. package/dist/env.js.map +1 -0
  89. package/dist/graph/checker-resolver.d.ts +156 -0
  90. package/dist/graph/checker-resolver.d.ts.map +1 -0
  91. package/dist/graph/checker-resolver.js +458 -0
  92. package/dist/graph/checker-resolver.js.map +1 -0
  93. package/dist/graph/db.d.ts +318 -0
  94. package/dist/graph/db.d.ts.map +1 -0
  95. package/dist/graph/db.js +266 -0
  96. package/dist/graph/db.js.map +1 -0
  97. package/dist/graph/index.d.ts +4 -0
  98. package/dist/graph/index.d.ts.map +1 -0
  99. package/dist/graph/index.js +5 -0
  100. package/dist/graph/index.js.map +1 -0
  101. package/dist/graph/local-type-env.d.ts +45 -0
  102. package/dist/graph/local-type-env.d.ts.map +1 -0
  103. package/dist/graph/local-type-env.js +59 -0
  104. package/dist/graph/local-type-env.js.map +1 -0
  105. package/dist/graph/path-range.d.ts +40 -0
  106. package/dist/graph/path-range.d.ts.map +1 -0
  107. package/dist/graph/path-range.js +42 -0
  108. package/dist/graph/path-range.js.map +1 -0
  109. package/dist/graph/populate.d.ts +263 -0
  110. package/dist/graph/populate.d.ts.map +1 -0
  111. package/dist/graph/populate.js +984 -0
  112. package/dist/graph/populate.js.map +1 -0
  113. package/dist/graph/queries.d.ts +114 -0
  114. package/dist/graph/queries.d.ts.map +1 -0
  115. package/dist/graph/queries.js +415 -0
  116. package/dist/graph/queries.js.map +1 -0
  117. package/dist/graph/sqliteBatch.d.ts +48 -0
  118. package/dist/graph/sqliteBatch.d.ts.map +1 -0
  119. package/dist/graph/sqliteBatch.js +93 -0
  120. package/dist/graph/sqliteBatch.js.map +1 -0
  121. package/dist/indexer/freshness.d.ts +48 -0
  122. package/dist/indexer/freshness.d.ts.map +1 -0
  123. package/dist/indexer/freshness.js +72 -0
  124. package/dist/indexer/freshness.js.map +1 -0
  125. package/dist/indexer/import-resolver.d.ts +50 -0
  126. package/dist/indexer/import-resolver.d.ts.map +1 -0
  127. package/dist/indexer/import-resolver.js +318 -0
  128. package/dist/indexer/import-resolver.js.map +1 -0
  129. package/dist/indexer/index.d.ts +249 -0
  130. package/dist/indexer/index.d.ts.map +1 -0
  131. package/dist/indexer/index.js +560 -0
  132. package/dist/indexer/index.js.map +1 -0
  133. package/dist/indexer/walker.d.ts +60 -0
  134. package/dist/indexer/walker.d.ts.map +1 -0
  135. package/dist/indexer/walker.js +128 -0
  136. package/dist/indexer/walker.js.map +1 -0
  137. package/dist/indexer/watcher.d.ts +80 -0
  138. package/dist/indexer/watcher.d.ts.map +1 -0
  139. package/dist/indexer/watcher.js +151 -0
  140. package/dist/indexer/watcher.js.map +1 -0
  141. package/dist/mcp/context.d.ts +23 -0
  142. package/dist/mcp/context.d.ts.map +1 -0
  143. package/dist/mcp/context.js +2 -0
  144. package/dist/mcp/context.js.map +1 -0
  145. package/dist/mcp/register-tools.d.ts +21 -0
  146. package/dist/mcp/register-tools.d.ts.map +1 -0
  147. package/dist/mcp/register-tools.js +42 -0
  148. package/dist/mcp/register-tools.js.map +1 -0
  149. package/dist/mcp/server.d.ts +58 -0
  150. package/dist/mcp/server.d.ts.map +1 -0
  151. package/dist/mcp/server.js +177 -0
  152. package/dist/mcp/server.js.map +1 -0
  153. package/dist/mcp/staleness.d.ts +72 -0
  154. package/dist/mcp/staleness.d.ts.map +1 -0
  155. package/dist/mcp/staleness.js +213 -0
  156. package/dist/mcp/staleness.js.map +1 -0
  157. package/dist/mcp/startup.d.ts +40 -0
  158. package/dist/mcp/startup.d.ts.map +1 -0
  159. package/dist/mcp/startup.js +114 -0
  160. package/dist/mcp/startup.js.map +1 -0
  161. package/dist/mcp/tools/_helpers.d.ts +26 -0
  162. package/dist/mcp/tools/_helpers.d.ts.map +1 -0
  163. package/dist/mcp/tools/_helpers.js +54 -0
  164. package/dist/mcp/tools/_helpers.js.map +1 -0
  165. package/dist/mcp/tools/callers.d.ts +4 -0
  166. package/dist/mcp/tools/callers.d.ts.map +1 -0
  167. package/dist/mcp/tools/callers.js +112 -0
  168. package/dist/mcp/tools/callers.js.map +1 -0
  169. package/dist/mcp/tools/dependencies.d.ts +4 -0
  170. package/dist/mcp/tools/dependencies.d.ts.map +1 -0
  171. package/dist/mcp/tools/dependencies.js +36 -0
  172. package/dist/mcp/tools/dependencies.js.map +1 -0
  173. package/dist/mcp/tools/efficiency.d.ts +4 -0
  174. package/dist/mcp/tools/efficiency.d.ts.map +1 -0
  175. package/dist/mcp/tools/efficiency.js +39 -0
  176. package/dist/mcp/tools/efficiency.js.map +1 -0
  177. package/dist/mcp/tools/exports.d.ts +4 -0
  178. package/dist/mcp/tools/exports.d.ts.map +1 -0
  179. package/dist/mcp/tools/exports.js +56 -0
  180. package/dist/mcp/tools/exports.js.map +1 -0
  181. package/dist/mcp/tools/implementors.d.ts +4 -0
  182. package/dist/mcp/tools/implementors.d.ts.map +1 -0
  183. package/dist/mcp/tools/implementors.js +47 -0
  184. package/dist/mcp/tools/implementors.js.map +1 -0
  185. package/dist/mcp/tools/project-skeleton.d.ts +4 -0
  186. package/dist/mcp/tools/project-skeleton.d.ts.map +1 -0
  187. package/dist/mcp/tools/project-skeleton.js +61 -0
  188. package/dist/mcp/tools/project-skeleton.js.map +1 -0
  189. package/dist/mcp/tools/reindex.d.ts +14 -0
  190. package/dist/mcp/tools/reindex.d.ts.map +1 -0
  191. package/dist/mcp/tools/reindex.js +32 -0
  192. package/dist/mcp/tools/reindex.js.map +1 -0
  193. package/dist/mcp/tools/rename-impact.d.ts +4 -0
  194. package/dist/mcp/tools/rename-impact.d.ts.map +1 -0
  195. package/dist/mcp/tools/rename-impact.js +124 -0
  196. package/dist/mcp/tools/rename-impact.js.map +1 -0
  197. package/dist/mcp/tools/search.d.ts +4 -0
  198. package/dist/mcp/tools/search.d.ts.map +1 -0
  199. package/dist/mcp/tools/search.js +69 -0
  200. package/dist/mcp/tools/search.js.map +1 -0
  201. package/dist/mcp/tools/signature.d.ts +4 -0
  202. package/dist/mcp/tools/signature.d.ts.map +1 -0
  203. package/dist/mcp/tools/signature.js +127 -0
  204. package/dist/mcp/tools/signature.js.map +1 -0
  205. package/dist/mcp/tools/status.d.ts +4 -0
  206. package/dist/mcp/tools/status.d.ts.map +1 -0
  207. package/dist/mcp/tools/status.js +30 -0
  208. package/dist/mcp/tools/status.js.map +1 -0
  209. package/dist/search/declex.d.ts +109 -0
  210. package/dist/search/declex.d.ts.map +1 -0
  211. package/dist/search/declex.js +242 -0
  212. package/dist/search/declex.js.map +1 -0
  213. package/dist/search/fts.d.ts +77 -0
  214. package/dist/search/fts.d.ts.map +1 -0
  215. package/dist/search/fts.js +223 -0
  216. package/dist/search/fts.js.map +1 -0
  217. package/dist/search/fused.d.ts +110 -0
  218. package/dist/search/fused.d.ts.map +1 -0
  219. package/dist/search/fused.js +323 -0
  220. package/dist/search/fused.js.map +1 -0
  221. package/dist/search/potential-matches.d.ts +88 -0
  222. package/dist/search/potential-matches.d.ts.map +1 -0
  223. package/dist/search/potential-matches.js +83 -0
  224. package/dist/search/potential-matches.js.map +1 -0
  225. package/dist/search/scope.d.ts +66 -0
  226. package/dist/search/scope.d.ts.map +1 -0
  227. package/dist/search/scope.js +71 -0
  228. package/dist/search/scope.js.map +1 -0
  229. package/dist/store/config.d.ts +57 -0
  230. package/dist/store/config.d.ts.map +1 -0
  231. package/dist/store/config.js +208 -0
  232. package/dist/store/config.js.map +1 -0
  233. package/dist/store/lock.d.ts +101 -0
  234. package/dist/store/lock.d.ts.map +1 -0
  235. package/dist/store/lock.js +182 -0
  236. package/dist/store/lock.js.map +1 -0
  237. package/dist/store/lockMetrics.d.ts +52 -0
  238. package/dist/store/lockMetrics.d.ts.map +1 -0
  239. package/dist/store/lockMetrics.js +29 -0
  240. package/dist/store/lockMetrics.js.map +1 -0
  241. package/dist/store/sqliteChunkStore.d.ts +44 -0
  242. package/dist/store/sqliteChunkStore.d.ts.map +1 -0
  243. package/dist/store/sqliteChunkStore.js +135 -0
  244. package/dist/store/sqliteChunkStore.js.map +1 -0
  245. package/dist/telemetry/lockMetricsSummary.d.ts +42 -0
  246. package/dist/telemetry/lockMetricsSummary.d.ts.map +1 -0
  247. package/dist/telemetry/lockMetricsSummary.js +139 -0
  248. package/dist/telemetry/lockMetricsSummary.js.map +1 -0
  249. package/dist/telemetry/metrics.d.ts +140 -0
  250. package/dist/telemetry/metrics.d.ts.map +1 -0
  251. package/dist/telemetry/metrics.js +312 -0
  252. package/dist/telemetry/metrics.js.map +1 -0
  253. package/dist/telemetry/tokenizer.d.ts +125 -0
  254. package/dist/telemetry/tokenizer.d.ts.map +1 -0
  255. package/dist/telemetry/tokenizer.js +174 -0
  256. package/dist/telemetry/tokenizer.js.map +1 -0
  257. package/package.json +81 -0
@@ -0,0 +1,125 @@
1
+ /**
2
+ * The honest tokenizer label, reported verbatim by `mast_efficiency`
3
+ * (`tokenizer` field) and the `mast metrics` footer.
4
+ *
5
+ * Single source of truth — every consumer reads this constant so the wording
6
+ * cannot drift. `@anthropic-ai/tokenizer@0.0.4` implements the Claude 2-era
7
+ * vocabulary and Anthropic never published Claude 3+ vocabularies, so absolute
8
+ * counts drift for every model that consumes MAST output today. The savings
9
+ * *ratio* mostly cancels the per-count error, which is why the mechanism stays
10
+ * and only the labeling is corrected (§14.5). An exact mode via the API's
11
+ * `count_tokens` endpoint is a documented future seam, not implemented here.
12
+ */
13
+ export declare const TOKENIZER_LABEL = "@anthropic-ai/tokenizer (claude-2 era, approximate for current models)";
14
+ /**
15
+ * Count tokens in `text` using `@anthropic-ai/tokenizer`.
16
+ *
17
+ * Used for `ToolStats.tokens_returned` and `tokens_full_file_upper_bound`.
18
+ * Counts are approximate for current models — see {@link TOKENIZER_LABEL}.
19
+ * The tokenizer loads its vocab table on first call and caches it in memory;
20
+ * subsequent calls are fast (microseconds).
21
+ */
22
+ export declare function countTokens(text: string): number;
23
+ /**
24
+ * Filesystem access needed by {@link estimateFullFileBound}, injected so
25
+ * tests can observe cache-hit / cache-miss behaviour deterministically
26
+ * (§4.4 — depend on an interface, not a `node:fs` module mock).
27
+ *
28
+ * `mtimeSeconds` and `sizeBytes` are returned from one `stat` method, not two
29
+ * — `fs.statSync` already yields both from a single syscall, and the budget
30
+ * path (F8) needs the size for every cache miss, so splitting them into
31
+ * `statMtime` + a hypothetical `statSize` would double the stat cost for no
32
+ * benefit.
33
+ */
34
+ export interface FullFileReader {
35
+ /** Reads a file's full UTF-8 contents. Callers catch failures. */
36
+ readonly readFile: (absolutePath: string) => string;
37
+ /**
38
+ * Returns the file's mtime (unix epoch seconds, matching the `files` table
39
+ * convention — see `staleness.ts`) and size in bytes. Callers catch
40
+ * failures.
41
+ */
42
+ readonly stat: (absolutePath: string) => {
43
+ readonly mtimeSeconds: number;
44
+ readonly sizeBytes: number;
45
+ };
46
+ }
47
+ /**
48
+ * Upper bound on the full-file token cache. Tokenizing whole files is the
49
+ * expensive part of every read-tool call (§14.2), so repeated hits on the
50
+ * same unchanged file must not re-tokenize — but an unbounded cache would
51
+ * grow without limit across a long-running `mast serve` process.
52
+ *
53
+ * F8 (eval/GITNEXUS_COMPARISON.md §13.7): 200 was tuned for "one working
54
+ * session's file set", before `mast_project_skeleton` — which references
55
+ * EVERY project file — existed as a caller. Entries are a path string plus
56
+ * two numbers, a few hundred bytes each, so 8192 entries is ~2-3 MB worst
57
+ * case: comfortably covers the ~10k-file corpora at the 150k-chunk scale
58
+ * target (plan Stage 4.5), so repeated calls against a monorepo-sized
59
+ * project CONVERGE to all-exact counts (via the per-call tokenization
60
+ * budget below) instead of thrashing forever.
61
+ */
62
+ export declare const FULL_FILE_BOUND_CACHE_LIMIT = 8192;
63
+ /**
64
+ * Maximum number of cache-miss paths {@link estimateFullFileBound} will
65
+ * exactly read and tokenize in one call. Paths beyond this budget are
66
+ * size-estimated instead (see {@link BYTES_PER_TOKEN_ESTIMATE}).
67
+ *
68
+ * F8: measured ~24ms/file to read+tokenize a full file, so 32 files is
69
+ * ~0.8s worst-case telemetry overhead per call — versus the ~28s/call this
70
+ * function cost when it tokenized all 1,334 files in a monorepo-sized
71
+ * project on every `mast_project_skeleton` call. §14.1 sets telemetry's
72
+ * "negligible overhead" goal; whatever the exact bound, instrumentation must
73
+ * never cost more than the query it instruments, and this budget converts an
74
+ * unbounded, thrashing cost into a bounded, converging one.
75
+ */
76
+ export declare const FULL_FILE_TOKENIZE_BUDGET_PER_CALL = 32;
77
+ /**
78
+ * Bytes-per-token heuristic used to estimate the token count of a cache-miss
79
+ * path once {@link FULL_FILE_TOKENIZE_BUDGET_PER_CALL} is exhausted for a
80
+ * call. This is the standard ~4-bytes/token rule of thumb for source text —
81
+ * defensible here because `tokens_full_file_upper_bound` is already an
82
+ * explicitly approximate, upper-bound methodology number (the tokenizer
83
+ * itself is a claude-2-era vocabulary, §14.5 / {@link TOKENIZER_LABEL}), not
84
+ * a billing figure.
85
+ */
86
+ export declare const BYTES_PER_TOKEN_ESTIMATE = 4;
87
+ /**
88
+ * Test-only seam: seeds the full-file cache via the real `cacheTouch` insert
89
+ * + evict path, without paying for real tokenization.
90
+ *
91
+ * F8 raised {@link FULL_FILE_BOUND_CACHE_LIMIT} to 8192. Profiling showed
92
+ * `countTokens` costs ~22ms/call *regardless of content length* (a fixed
93
+ * per-invocation floor, not an I/O or content-size cost), so proving
94
+ * eviction still fires once the raised bound is exceeded — by driving 8192+
95
+ * real reads through {@link estimateFullFileBound} — would cost minutes and
96
+ * dwarf the very call this change speeds up. Seeding through `cacheTouch`
97
+ * exercises the exact same insert-and-evict-oldest branch production code
98
+ * uses; only the (irrelevant, for this purpose) cost of real tokenization is
99
+ * skipped. Not used by any non-test code.
100
+ */
101
+ export declare function __seedFullFileCacheForTests(absolutePath: string, mtime: number, tokens: number): void;
102
+ /**
103
+ * Estimate the full-file token upper bound for a set of file paths (§14.2).
104
+ *
105
+ * For each unique path, reads the full file under `projectRoot` and sums
106
+ * `countTokens` over its contents — the "what would a naive Read of every
107
+ * result file have cost?" counterfactual used to compute `efficiency_ratio`.
108
+ *
109
+ * Missing or unreadable files contribute 0 and never throw: the metrics path
110
+ * must never break a tool response over a stale or deleted file reference.
111
+ * Repeated calls referencing the same file at the same mtime reuse the
112
+ * cached token count instead of re-reading and re-tokenizing.
113
+ *
114
+ * F8 — "cap the work, not the cache" (eval/GITNEXUS_COMPARISON.md §13.7):
115
+ * a cache miss is only exactly tokenized while
116
+ * {@link FULL_FILE_TOKENIZE_BUDGET_PER_CALL} allows it, in the caller's path
117
+ * order (after dedup) so results are deterministic. Cache hits are free and
118
+ * never consume budget. Once the budget is exhausted for a call, further
119
+ * cache misses are size-estimated (`sizeBytes / BYTES_PER_TOKEN_ESTIMATE`)
120
+ * rather than read — and NOT cached, since an estimate must never masquerade
121
+ * as an exact cached count. Successive calls over the same path set
122
+ * progressively convert estimates to exact, cached counts.
123
+ */
124
+ export declare function estimateFullFileBound(filePaths: readonly string[], projectRoot: string, reader?: FullFileReader): number;
125
+ //# sourceMappingURL=tokenizer.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../../src/telemetry/tokenizer.ts"],"names":[],"mappings":"AAQA;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,eAAe,2EAC8C,CAAC;AAE3E;;;;;;;GAOG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEhD;AAED;;;;;;;;;;GAUG;AACH,MAAM,WAAW,cAAc;IAC7B,kEAAkE;IAClE,QAAQ,CAAC,QAAQ,EAAE,CAAC,YAAY,EAAE,MAAM,KAAK,MAAM,CAAC;IACpD;;;;OAIG;IACH,QAAQ,CAAC,IAAI,EAAE,CAAC,YAAY,EAAE,MAAM,KAAK;QAAE,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;CACxG;AAeD;;;;;;;;;;;;;;GAcG;AACH,eAAO,MAAM,2BAA2B,OAAO,CAAC;AAEhD;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,kCAAkC,KAAK,CAAC;AAErD;;;;;;;;GAQG;AACH,eAAO,MAAM,wBAAwB,IAAI,CAAC;AAiB1C;;;;;;;;;;;;;GAaG;AACH,wBAAgB,2BAA2B,CAAC,YAAY,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAErG;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,qBAAqB,CACnC,SAAS,EAAE,SAAS,MAAM,EAAE,EAC5B,WAAW,EAAE,MAAM,EACnB,MAAM,GAAE,cAAsC,GAC7C,MAAM,CA8CR"}
@@ -0,0 +1,174 @@
1
+ import { readFileSync, statSync } from 'node:fs';
2
+ import { join } from 'node:path';
3
+ import { countTokens as anthropicCountTokens } from '@anthropic-ai/tokenizer';
4
+ // ---------------------------------------------------------------------------
5
+ // Token counting (@anthropic-ai/tokenizer — claude-2 era, approximate)
6
+ // ---------------------------------------------------------------------------
7
+ /**
8
+ * The honest tokenizer label, reported verbatim by `mast_efficiency`
9
+ * (`tokenizer` field) and the `mast metrics` footer.
10
+ *
11
+ * Single source of truth — every consumer reads this constant so the wording
12
+ * cannot drift. `@anthropic-ai/tokenizer@0.0.4` implements the Claude 2-era
13
+ * vocabulary and Anthropic never published Claude 3+ vocabularies, so absolute
14
+ * counts drift for every model that consumes MAST output today. The savings
15
+ * *ratio* mostly cancels the per-count error, which is why the mechanism stays
16
+ * and only the labeling is corrected (§14.5). An exact mode via the API's
17
+ * `count_tokens` endpoint is a documented future seam, not implemented here.
18
+ */
19
+ export const TOKENIZER_LABEL = '@anthropic-ai/tokenizer (claude-2 era, approximate for current models)';
20
+ /**
21
+ * Count tokens in `text` using `@anthropic-ai/tokenizer`.
22
+ *
23
+ * Used for `ToolStats.tokens_returned` and `tokens_full_file_upper_bound`.
24
+ * Counts are approximate for current models — see {@link TOKENIZER_LABEL}.
25
+ * The tokenizer loads its vocab table on first call and caches it in memory;
26
+ * subsequent calls are fast (microseconds).
27
+ */
28
+ export function countTokens(text) {
29
+ return anthropicCountTokens(text);
30
+ }
31
+ const defaultFullFileReader = {
32
+ readFile: (absolutePath) => readFileSync(absolutePath, 'utf8'),
33
+ stat: (absolutePath) => {
34
+ const s = statSync(absolutePath);
35
+ return { mtimeSeconds: s.mtimeMs / 1_000, sizeBytes: s.size };
36
+ },
37
+ };
38
+ /**
39
+ * Upper bound on the full-file token cache. Tokenizing whole files is the
40
+ * expensive part of every read-tool call (§14.2), so repeated hits on the
41
+ * same unchanged file must not re-tokenize — but an unbounded cache would
42
+ * grow without limit across a long-running `mast serve` process.
43
+ *
44
+ * F8 (eval/GITNEXUS_COMPARISON.md §13.7): 200 was tuned for "one working
45
+ * session's file set", before `mast_project_skeleton` — which references
46
+ * EVERY project file — existed as a caller. Entries are a path string plus
47
+ * two numbers, a few hundred bytes each, so 8192 entries is ~2-3 MB worst
48
+ * case: comfortably covers the ~10k-file corpora at the 150k-chunk scale
49
+ * target (plan Stage 4.5), so repeated calls against a monorepo-sized
50
+ * project CONVERGE to all-exact counts (via the per-call tokenization
51
+ * budget below) instead of thrashing forever.
52
+ */
53
+ export const FULL_FILE_BOUND_CACHE_LIMIT = 8192;
54
+ /**
55
+ * Maximum number of cache-miss paths {@link estimateFullFileBound} will
56
+ * exactly read and tokenize in one call. Paths beyond this budget are
57
+ * size-estimated instead (see {@link BYTES_PER_TOKEN_ESTIMATE}).
58
+ *
59
+ * F8: measured ~24ms/file to read+tokenize a full file, so 32 files is
60
+ * ~0.8s worst-case telemetry overhead per call — versus the ~28s/call this
61
+ * function cost when it tokenized all 1,334 files in a monorepo-sized
62
+ * project on every `mast_project_skeleton` call. §14.1 sets telemetry's
63
+ * "negligible overhead" goal; whatever the exact bound, instrumentation must
64
+ * never cost more than the query it instruments, and this budget converts an
65
+ * unbounded, thrashing cost into a bounded, converging one.
66
+ */
67
+ export const FULL_FILE_TOKENIZE_BUDGET_PER_CALL = 32;
68
+ /**
69
+ * Bytes-per-token heuristic used to estimate the token count of a cache-miss
70
+ * path once {@link FULL_FILE_TOKENIZE_BUDGET_PER_CALL} is exhausted for a
71
+ * call. This is the standard ~4-bytes/token rule of thumb for source text —
72
+ * defensible here because `tokens_full_file_upper_bound` is already an
73
+ * explicitly approximate, upper-bound methodology number (the tokenizer
74
+ * itself is a claude-2-era vocabulary, §14.5 / {@link TOKENIZER_LABEL}), not
75
+ * a billing figure.
76
+ */
77
+ export const BYTES_PER_TOKEN_ESTIMATE = 4;
78
+ // Module-level so the cache survives across tool calls within one `mast
79
+ // serve` process. Map iteration order is insertion order, so the first key
80
+ // is always the least-recently-used one — cheap LRU-ish eviction with no
81
+ // extra dependency (re-inserting a key on cache hit moves it to the tail).
82
+ const fullFileCache = new Map();
83
+ function cacheTouch(absolutePath, entry) {
84
+ fullFileCache.delete(absolutePath);
85
+ fullFileCache.set(absolutePath, entry);
86
+ if (fullFileCache.size > FULL_FILE_BOUND_CACHE_LIMIT) {
87
+ const oldestKey = fullFileCache.keys().next().value;
88
+ if (oldestKey !== undefined)
89
+ fullFileCache.delete(oldestKey);
90
+ }
91
+ }
92
+ /**
93
+ * Test-only seam: seeds the full-file cache via the real `cacheTouch` insert
94
+ * + evict path, without paying for real tokenization.
95
+ *
96
+ * F8 raised {@link FULL_FILE_BOUND_CACHE_LIMIT} to 8192. Profiling showed
97
+ * `countTokens` costs ~22ms/call *regardless of content length* (a fixed
98
+ * per-invocation floor, not an I/O or content-size cost), so proving
99
+ * eviction still fires once the raised bound is exceeded — by driving 8192+
100
+ * real reads through {@link estimateFullFileBound} — would cost minutes and
101
+ * dwarf the very call this change speeds up. Seeding through `cacheTouch`
102
+ * exercises the exact same insert-and-evict-oldest branch production code
103
+ * uses; only the (irrelevant, for this purpose) cost of real tokenization is
104
+ * skipped. Not used by any non-test code.
105
+ */
106
+ export function __seedFullFileCacheForTests(absolutePath, mtime, tokens) {
107
+ cacheTouch(absolutePath, { mtime, tokens });
108
+ }
109
+ /**
110
+ * Estimate the full-file token upper bound for a set of file paths (§14.2).
111
+ *
112
+ * For each unique path, reads the full file under `projectRoot` and sums
113
+ * `countTokens` over its contents — the "what would a naive Read of every
114
+ * result file have cost?" counterfactual used to compute `efficiency_ratio`.
115
+ *
116
+ * Missing or unreadable files contribute 0 and never throw: the metrics path
117
+ * must never break a tool response over a stale or deleted file reference.
118
+ * Repeated calls referencing the same file at the same mtime reuse the
119
+ * cached token count instead of re-reading and re-tokenizing.
120
+ *
121
+ * F8 — "cap the work, not the cache" (eval/GITNEXUS_COMPARISON.md §13.7):
122
+ * a cache miss is only exactly tokenized while
123
+ * {@link FULL_FILE_TOKENIZE_BUDGET_PER_CALL} allows it, in the caller's path
124
+ * order (after dedup) so results are deterministic. Cache hits are free and
125
+ * never consume budget. Once the budget is exhausted for a call, further
126
+ * cache misses are size-estimated (`sizeBytes / BYTES_PER_TOKEN_ESTIMATE`)
127
+ * rather than read — and NOT cached, since an estimate must never masquerade
128
+ * as an exact cached count. Successive calls over the same path set
129
+ * progressively convert estimates to exact, cached counts.
130
+ */
131
+ export function estimateFullFileBound(filePaths, projectRoot, reader = defaultFullFileReader) {
132
+ const uniquePaths = new Set(filePaths);
133
+ let total = 0;
134
+ let tokenizeBudgetRemaining = FULL_FILE_TOKENIZE_BUDGET_PER_CALL;
135
+ for (const relPath of uniquePaths) {
136
+ const absPath = join(projectRoot, relPath);
137
+ let mtimeSeconds;
138
+ let sizeBytes;
139
+ try {
140
+ ({ mtimeSeconds, sizeBytes } = reader.stat(absPath));
141
+ }
142
+ catch {
143
+ continue; // file missing/unreadable — contributes 0, consumes no budget
144
+ }
145
+ const cached = fullFileCache.get(absPath);
146
+ if (cached !== undefined && cached.mtime === mtimeSeconds) {
147
+ total += cached.tokens;
148
+ // Refresh recency even on a hit so a hot file survives eviction.
149
+ cacheTouch(absPath, cached);
150
+ continue;
151
+ }
152
+ if (tokenizeBudgetRemaining <= 0) {
153
+ // Beyond the per-call budget: estimate from size instead of paying for
154
+ // an exact read+tokenize. Deliberately not cached (see doc comment).
155
+ total += Math.ceil(sizeBytes / BYTES_PER_TOKEN_ESTIMATE);
156
+ continue;
157
+ }
158
+ // A failed read below still spends this slot rather than freeing it for
159
+ // the next path — the simplest honest rule, and consistent with the
160
+ // fact that the stat() that granted the slot already succeeded.
161
+ tokenizeBudgetRemaining--;
162
+ try {
163
+ const content = reader.readFile(absPath);
164
+ const tokens = countTokens(content);
165
+ cacheTouch(absPath, { mtime: mtimeSeconds, tokens });
166
+ total += tokens;
167
+ }
168
+ catch {
169
+ continue; // unreadable despite a successful stat — contributes 0
170
+ }
171
+ }
172
+ return total;
173
+ }
174
+ //# sourceMappingURL=tokenizer.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../../src/telemetry/tokenizer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACjD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,WAAW,IAAI,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAE9E,8EAA8E;AAC9E,uEAAuE;AACvE,8EAA8E;AAE9E;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,MAAM,eAAe,GAC1B,wEAAwE,CAAC;AAE3E;;;;;;;GAOG;AACH,MAAM,UAAU,WAAW,CAAC,IAAY;IACtC,OAAO,oBAAoB,CAAC,IAAI,CAAC,CAAC;AACpC,CAAC;AAwBD,MAAM,qBAAqB,GAAmB;IAC5C,QAAQ,EAAE,CAAC,YAAY,EAAE,EAAE,CAAC,YAAY,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9D,IAAI,EAAE,CAAC,YAAY,EAAE,EAAE;QACrB,MAAM,CAAC,GAAG,QAAQ,CAAC,YAAY,CAAC,CAAC;QACjC,OAAO,EAAE,YAAY,EAAE,CAAC,CAAC,OAAO,GAAG,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAChE,CAAC;CACF,CAAC;AAOF;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG,IAAI,CAAC;AAEhD;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,MAAM,kCAAkC,GAAG,EAAE,CAAC;AAErD;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,CAAC;AAE1C,wEAAwE;AACxE,2EAA2E;AAC3E,yEAAyE;AACzE,2EAA2E;AAC3E,MAAM,aAAa,GAAG,IAAI,GAAG,EAA8B,CAAC;AAE5D,SAAS,UAAU,CAAC,YAAoB,EAAE,KAAyB;IACjE,aAAa,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;IACnC,aAAa,CAAC,GAAG,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;IACvC,IAAI,aAAa,CAAC,IAAI,GAAG,2BAA2B,EAAE,CAAC;QACrD,MAAM,SAAS,GAAG,aAAa,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;QACpD,IAAI,SAAS,KAAK,SAAS;YAAE,aAAa,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC/D,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,2BAA2B,CAAC,YAAoB,EAAE,KAAa,EAAE,MAAc;IAC7F,UAAU,CAAC,YAAY,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;AAC9C,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,UAAU,qBAAqB,CACnC,SAA4B,EAC5B,WAAmB,EACnB,SAAyB,qBAAqB;IAE9C,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IACvC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,uBAAuB,GAAG,kCAAkC,CAAC;IAEjE,KAAK,MAAM,OAAO,IAAI,WAAW,EAAE,CAAC;QAClC,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QAE3C,IAAI,YAAoB,CAAC;QACzB,IAAI,SAAiB,CAAC;QACtB,IAAI,CAAC;YACH,CAAC,EAAE,YAAY,EAAE,SAAS,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;QACvD,CAAC;QAAC,MAAM,CAAC;YACP,SAAS,CAAC,8DAA8D;QAC1E,CAAC;QAED,MAAM,MAAM,GAAG,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAC1C,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,KAAK,KAAK,YAAY,EAAE,CAAC;YAC1D,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC;YACvB,iEAAiE;YACjE,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAC5B,SAAS;QACX,CAAC;QAED,IAAI,uBAAuB,IAAI,CAAC,EAAE,CAAC;YACjC,uEAAuE;YACvE,qEAAqE;YACrE,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,SAAS,GAAG,wBAAwB,CAAC,CAAC;YACzD,SAAS;QACX,CAAC;QAED,wEAAwE;QACxE,oEAAoE;QACpE,gEAAgE;QAChE,uBAAuB,EAAE,CAAC;QAC1B,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YACzC,MAAM,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;YACpC,UAAU,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,CAAC,CAAC;YACrD,KAAK,IAAI,MAAM,CAAC;QAClB,CAAC;QAAC,MAAM,CAAC;YACP,SAAS,CAAC,uDAAuD;QACnE,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC"}
package/package.json ADDED
@@ -0,0 +1,81 @@
1
+ {
2
+ "name": "@spikedpunch/mast",
3
+ "version": "0.1.0",
4
+ "publishConfig": {
5
+ "access": "public"
6
+ },
7
+ "repository": {
8
+ "type": "git",
9
+ "url": "git+https://github.com/SpikedPunchVictim/mast.git"
10
+ },
11
+ "homepage": "https://github.com/SpikedPunchVictim/mast#readme",
12
+ "bugs": {
13
+ "url": "https://github.com/SpikedPunchVictim/mast/issues"
14
+ },
15
+ "keywords": [
16
+ "mcp",
17
+ "code-search",
18
+ "ast",
19
+ "tree-sitter",
20
+ "monorepo",
21
+ "typescript",
22
+ "claude"
23
+ ],
24
+ "description": "MAST — Monorepo AST Search Tool: semantic code search MCP server and CLI",
25
+ "type": "module",
26
+ "bin": {
27
+ "mast": "./dist/cli/index.js"
28
+ },
29
+ "exports": {
30
+ ".": {
31
+ "import": "./dist/cli/index.js",
32
+ "types": "./dist/cli/index.d.ts"
33
+ },
34
+ "./graph": {
35
+ "import": "./dist/graph/index.js",
36
+ "types": "./dist/graph/index.d.ts"
37
+ }
38
+ },
39
+ "files": [
40
+ "dist",
41
+ "assets",
42
+ "MAST_SPEC.md"
43
+ ],
44
+ "dependencies": {
45
+ "@anthropic-ai/tokenizer": "^0.0.4",
46
+ "@modelcontextprotocol/sdk": "^1.29.0",
47
+ "better-sqlite3": "^12.10.0",
48
+ "chokidar": "^5.0.0",
49
+ "commander": "^14.0.3",
50
+ "fast-glob": "^3.3.3",
51
+ "kysely": "^0.29.0",
52
+ "proper-lockfile": "^4.1.2",
53
+ "tree-sitter": "0.22.4",
54
+ "tree-sitter-typescript": "0.23.2",
55
+ "tsconfig-paths": "^4.2.0",
56
+ "typescript": "^5.3.0",
57
+ "uuid": "^14.0.0",
58
+ "zod": "^4.4.3"
59
+ },
60
+ "devDependencies": {
61
+ "@types/better-sqlite3": "^7.6.13",
62
+ "@types/node": "^24.0.0",
63
+ "@types/proper-lockfile": "^4.1.4",
64
+ "@typescript-eslint/eslint-plugin": "^8.60.1",
65
+ "@typescript-eslint/parser": "^8.60.1",
66
+ "eslint": "^10.4.1",
67
+ "vitest": "^1.0.0"
68
+ },
69
+ "engines": {
70
+ "node": ">=22.0.0"
71
+ },
72
+ "license": "MIT",
73
+ "scripts": {
74
+ "build": "tsc",
75
+ "typecheck": "tsc --noEmit && tsc -p tsconfig.test.json",
76
+ "test": "vitest run",
77
+ "test:watch": "vitest",
78
+ "lint": "eslint src eval",
79
+ "format": "prettier --write src"
80
+ }
81
+ }