@spikedpunch/mast 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (257) hide show
  1. package/LICENSE +21 -0
  2. package/MAST_SPEC.md +2926 -0
  3. package/README.md +842 -0
  4. package/assets/skill.md +45 -0
  5. package/dist/ast/extract.d.ts +65 -0
  6. package/dist/ast/extract.d.ts.map +1 -0
  7. package/dist/ast/extract.js +147 -0
  8. package/dist/ast/extract.js.map +1 -0
  9. package/dist/ast/extractor.d.ts +85 -0
  10. package/dist/ast/extractor.d.ts.map +1 -0
  11. package/dist/ast/extractor.js +2 -0
  12. package/dist/ast/extractor.js.map +1 -0
  13. package/dist/ast/extractors/markdown.d.ts +40 -0
  14. package/dist/ast/extractors/markdown.d.ts.map +1 -0
  15. package/dist/ast/extractors/markdown.js +175 -0
  16. package/dist/ast/extractors/markdown.js.map +1 -0
  17. package/dist/ast/extractors/typescript.d.ts +179 -0
  18. package/dist/ast/extractors/typescript.d.ts.map +1 -0
  19. package/dist/ast/extractors/typescript.js +1439 -0
  20. package/dist/ast/extractors/typescript.js.map +1 -0
  21. package/dist/ast/parser.d.ts +5 -0
  22. package/dist/ast/parser.d.ts.map +1 -0
  23. package/dist/ast/parser.js +21 -0
  24. package/dist/ast/parser.js.map +1 -0
  25. package/dist/ast/types.d.ts +586 -0
  26. package/dist/ast/types.d.ts.map +1 -0
  27. package/dist/ast/types.js +3 -0
  28. package/dist/ast/types.js.map +1 -0
  29. package/dist/cli/docs-cmd.d.ts +22 -0
  30. package/dist/cli/docs-cmd.d.ts.map +1 -0
  31. package/dist/cli/docs-cmd.js +100 -0
  32. package/dist/cli/docs-cmd.js.map +1 -0
  33. package/dist/cli/index-cmd.d.ts +72 -0
  34. package/dist/cli/index-cmd.d.ts.map +1 -0
  35. package/dist/cli/index-cmd.js +263 -0
  36. package/dist/cli/index-cmd.js.map +1 -0
  37. package/dist/cli/index.d.ts +3 -0
  38. package/dist/cli/index.d.ts.map +1 -0
  39. package/dist/cli/index.js +4 -0
  40. package/dist/cli/index.js.map +1 -0
  41. package/dist/cli/init.d.ts +14 -0
  42. package/dist/cli/init.d.ts.map +1 -0
  43. package/dist/cli/init.js +64 -0
  44. package/dist/cli/init.js.map +1 -0
  45. package/dist/cli/install-hooks.d.ts +3 -0
  46. package/dist/cli/install-hooks.d.ts.map +1 -0
  47. package/dist/cli/install-hooks.js +34 -0
  48. package/dist/cli/install-hooks.js.map +1 -0
  49. package/dist/cli/metrics-cmd.d.ts +3 -0
  50. package/dist/cli/metrics-cmd.d.ts.map +1 -0
  51. package/dist/cli/metrics-cmd.js +164 -0
  52. package/dist/cli/metrics-cmd.js.map +1 -0
  53. package/dist/cli/program.d.ts +14 -0
  54. package/dist/cli/program.d.ts.map +1 -0
  55. package/dist/cli/program.js +44 -0
  56. package/dist/cli/program.js.map +1 -0
  57. package/dist/cli/query.d.ts +46 -0
  58. package/dist/cli/query.d.ts.map +1 -0
  59. package/dist/cli/query.js +153 -0
  60. package/dist/cli/query.js.map +1 -0
  61. package/dist/cli/search-cmd.d.ts +33 -0
  62. package/dist/cli/search-cmd.d.ts.map +1 -0
  63. package/dist/cli/search-cmd.js +114 -0
  64. package/dist/cli/search-cmd.js.map +1 -0
  65. package/dist/cli/serve.d.ts +3 -0
  66. package/dist/cli/serve.d.ts.map +1 -0
  67. package/dist/cli/serve.js +19 -0
  68. package/dist/cli/serve.js.map +1 -0
  69. package/dist/cli/skill-install.d.ts +39 -0
  70. package/dist/cli/skill-install.d.ts.map +1 -0
  71. package/dist/cli/skill-install.js +57 -0
  72. package/dist/cli/skill-install.js.map +1 -0
  73. package/dist/cli/status.d.ts +35 -0
  74. package/dist/cli/status.d.ts.map +1 -0
  75. package/dist/cli/status.js +117 -0
  76. package/dist/cli/status.js.map +1 -0
  77. package/dist/cli/upgrade-cmd.d.ts +48 -0
  78. package/dist/cli/upgrade-cmd.d.ts.map +1 -0
  79. package/dist/cli/upgrade-cmd.js +129 -0
  80. package/dist/cli/upgrade-cmd.js.map +1 -0
  81. package/dist/cli/version.d.ts +3 -0
  82. package/dist/cli/version.d.ts.map +1 -0
  83. package/dist/cli/version.js +19 -0
  84. package/dist/cli/version.js.map +1 -0
  85. package/dist/env.d.ts +17 -0
  86. package/dist/env.d.ts.map +1 -0
  87. package/dist/env.js +17 -0
  88. package/dist/env.js.map +1 -0
  89. package/dist/graph/checker-resolver.d.ts +156 -0
  90. package/dist/graph/checker-resolver.d.ts.map +1 -0
  91. package/dist/graph/checker-resolver.js +458 -0
  92. package/dist/graph/checker-resolver.js.map +1 -0
  93. package/dist/graph/db.d.ts +318 -0
  94. package/dist/graph/db.d.ts.map +1 -0
  95. package/dist/graph/db.js +266 -0
  96. package/dist/graph/db.js.map +1 -0
  97. package/dist/graph/index.d.ts +4 -0
  98. package/dist/graph/index.d.ts.map +1 -0
  99. package/dist/graph/index.js +5 -0
  100. package/dist/graph/index.js.map +1 -0
  101. package/dist/graph/local-type-env.d.ts +45 -0
  102. package/dist/graph/local-type-env.d.ts.map +1 -0
  103. package/dist/graph/local-type-env.js +59 -0
  104. package/dist/graph/local-type-env.js.map +1 -0
  105. package/dist/graph/path-range.d.ts +40 -0
  106. package/dist/graph/path-range.d.ts.map +1 -0
  107. package/dist/graph/path-range.js +42 -0
  108. package/dist/graph/path-range.js.map +1 -0
  109. package/dist/graph/populate.d.ts +263 -0
  110. package/dist/graph/populate.d.ts.map +1 -0
  111. package/dist/graph/populate.js +984 -0
  112. package/dist/graph/populate.js.map +1 -0
  113. package/dist/graph/queries.d.ts +114 -0
  114. package/dist/graph/queries.d.ts.map +1 -0
  115. package/dist/graph/queries.js +415 -0
  116. package/dist/graph/queries.js.map +1 -0
  117. package/dist/graph/sqliteBatch.d.ts +48 -0
  118. package/dist/graph/sqliteBatch.d.ts.map +1 -0
  119. package/dist/graph/sqliteBatch.js +93 -0
  120. package/dist/graph/sqliteBatch.js.map +1 -0
  121. package/dist/indexer/freshness.d.ts +48 -0
  122. package/dist/indexer/freshness.d.ts.map +1 -0
  123. package/dist/indexer/freshness.js +72 -0
  124. package/dist/indexer/freshness.js.map +1 -0
  125. package/dist/indexer/import-resolver.d.ts +50 -0
  126. package/dist/indexer/import-resolver.d.ts.map +1 -0
  127. package/dist/indexer/import-resolver.js +318 -0
  128. package/dist/indexer/import-resolver.js.map +1 -0
  129. package/dist/indexer/index.d.ts +249 -0
  130. package/dist/indexer/index.d.ts.map +1 -0
  131. package/dist/indexer/index.js +560 -0
  132. package/dist/indexer/index.js.map +1 -0
  133. package/dist/indexer/walker.d.ts +60 -0
  134. package/dist/indexer/walker.d.ts.map +1 -0
  135. package/dist/indexer/walker.js +128 -0
  136. package/dist/indexer/walker.js.map +1 -0
  137. package/dist/indexer/watcher.d.ts +80 -0
  138. package/dist/indexer/watcher.d.ts.map +1 -0
  139. package/dist/indexer/watcher.js +151 -0
  140. package/dist/indexer/watcher.js.map +1 -0
  141. package/dist/mcp/context.d.ts +23 -0
  142. package/dist/mcp/context.d.ts.map +1 -0
  143. package/dist/mcp/context.js +2 -0
  144. package/dist/mcp/context.js.map +1 -0
  145. package/dist/mcp/register-tools.d.ts +21 -0
  146. package/dist/mcp/register-tools.d.ts.map +1 -0
  147. package/dist/mcp/register-tools.js +42 -0
  148. package/dist/mcp/register-tools.js.map +1 -0
  149. package/dist/mcp/server.d.ts +58 -0
  150. package/dist/mcp/server.d.ts.map +1 -0
  151. package/dist/mcp/server.js +177 -0
  152. package/dist/mcp/server.js.map +1 -0
  153. package/dist/mcp/staleness.d.ts +72 -0
  154. package/dist/mcp/staleness.d.ts.map +1 -0
  155. package/dist/mcp/staleness.js +213 -0
  156. package/dist/mcp/staleness.js.map +1 -0
  157. package/dist/mcp/startup.d.ts +40 -0
  158. package/dist/mcp/startup.d.ts.map +1 -0
  159. package/dist/mcp/startup.js +114 -0
  160. package/dist/mcp/startup.js.map +1 -0
  161. package/dist/mcp/tools/_helpers.d.ts +26 -0
  162. package/dist/mcp/tools/_helpers.d.ts.map +1 -0
  163. package/dist/mcp/tools/_helpers.js +54 -0
  164. package/dist/mcp/tools/_helpers.js.map +1 -0
  165. package/dist/mcp/tools/callers.d.ts +4 -0
  166. package/dist/mcp/tools/callers.d.ts.map +1 -0
  167. package/dist/mcp/tools/callers.js +112 -0
  168. package/dist/mcp/tools/callers.js.map +1 -0
  169. package/dist/mcp/tools/dependencies.d.ts +4 -0
  170. package/dist/mcp/tools/dependencies.d.ts.map +1 -0
  171. package/dist/mcp/tools/dependencies.js +36 -0
  172. package/dist/mcp/tools/dependencies.js.map +1 -0
  173. package/dist/mcp/tools/efficiency.d.ts +4 -0
  174. package/dist/mcp/tools/efficiency.d.ts.map +1 -0
  175. package/dist/mcp/tools/efficiency.js +39 -0
  176. package/dist/mcp/tools/efficiency.js.map +1 -0
  177. package/dist/mcp/tools/exports.d.ts +4 -0
  178. package/dist/mcp/tools/exports.d.ts.map +1 -0
  179. package/dist/mcp/tools/exports.js +56 -0
  180. package/dist/mcp/tools/exports.js.map +1 -0
  181. package/dist/mcp/tools/implementors.d.ts +4 -0
  182. package/dist/mcp/tools/implementors.d.ts.map +1 -0
  183. package/dist/mcp/tools/implementors.js +47 -0
  184. package/dist/mcp/tools/implementors.js.map +1 -0
  185. package/dist/mcp/tools/project-skeleton.d.ts +4 -0
  186. package/dist/mcp/tools/project-skeleton.d.ts.map +1 -0
  187. package/dist/mcp/tools/project-skeleton.js +61 -0
  188. package/dist/mcp/tools/project-skeleton.js.map +1 -0
  189. package/dist/mcp/tools/reindex.d.ts +14 -0
  190. package/dist/mcp/tools/reindex.d.ts.map +1 -0
  191. package/dist/mcp/tools/reindex.js +32 -0
  192. package/dist/mcp/tools/reindex.js.map +1 -0
  193. package/dist/mcp/tools/rename-impact.d.ts +4 -0
  194. package/dist/mcp/tools/rename-impact.d.ts.map +1 -0
  195. package/dist/mcp/tools/rename-impact.js +124 -0
  196. package/dist/mcp/tools/rename-impact.js.map +1 -0
  197. package/dist/mcp/tools/search.d.ts +4 -0
  198. package/dist/mcp/tools/search.d.ts.map +1 -0
  199. package/dist/mcp/tools/search.js +69 -0
  200. package/dist/mcp/tools/search.js.map +1 -0
  201. package/dist/mcp/tools/signature.d.ts +4 -0
  202. package/dist/mcp/tools/signature.d.ts.map +1 -0
  203. package/dist/mcp/tools/signature.js +127 -0
  204. package/dist/mcp/tools/signature.js.map +1 -0
  205. package/dist/mcp/tools/status.d.ts +4 -0
  206. package/dist/mcp/tools/status.d.ts.map +1 -0
  207. package/dist/mcp/tools/status.js +30 -0
  208. package/dist/mcp/tools/status.js.map +1 -0
  209. package/dist/search/declex.d.ts +109 -0
  210. package/dist/search/declex.d.ts.map +1 -0
  211. package/dist/search/declex.js +242 -0
  212. package/dist/search/declex.js.map +1 -0
  213. package/dist/search/fts.d.ts +77 -0
  214. package/dist/search/fts.d.ts.map +1 -0
  215. package/dist/search/fts.js +223 -0
  216. package/dist/search/fts.js.map +1 -0
  217. package/dist/search/fused.d.ts +110 -0
  218. package/dist/search/fused.d.ts.map +1 -0
  219. package/dist/search/fused.js +323 -0
  220. package/dist/search/fused.js.map +1 -0
  221. package/dist/search/potential-matches.d.ts +88 -0
  222. package/dist/search/potential-matches.d.ts.map +1 -0
  223. package/dist/search/potential-matches.js +83 -0
  224. package/dist/search/potential-matches.js.map +1 -0
  225. package/dist/search/scope.d.ts +66 -0
  226. package/dist/search/scope.d.ts.map +1 -0
  227. package/dist/search/scope.js +71 -0
  228. package/dist/search/scope.js.map +1 -0
  229. package/dist/store/config.d.ts +57 -0
  230. package/dist/store/config.d.ts.map +1 -0
  231. package/dist/store/config.js +208 -0
  232. package/dist/store/config.js.map +1 -0
  233. package/dist/store/lock.d.ts +101 -0
  234. package/dist/store/lock.d.ts.map +1 -0
  235. package/dist/store/lock.js +182 -0
  236. package/dist/store/lock.js.map +1 -0
  237. package/dist/store/lockMetrics.d.ts +52 -0
  238. package/dist/store/lockMetrics.d.ts.map +1 -0
  239. package/dist/store/lockMetrics.js +29 -0
  240. package/dist/store/lockMetrics.js.map +1 -0
  241. package/dist/store/sqliteChunkStore.d.ts +44 -0
  242. package/dist/store/sqliteChunkStore.d.ts.map +1 -0
  243. package/dist/store/sqliteChunkStore.js +135 -0
  244. package/dist/store/sqliteChunkStore.js.map +1 -0
  245. package/dist/telemetry/lockMetricsSummary.d.ts +42 -0
  246. package/dist/telemetry/lockMetricsSummary.d.ts.map +1 -0
  247. package/dist/telemetry/lockMetricsSummary.js +139 -0
  248. package/dist/telemetry/lockMetricsSummary.js.map +1 -0
  249. package/dist/telemetry/metrics.d.ts +140 -0
  250. package/dist/telemetry/metrics.d.ts.map +1 -0
  251. package/dist/telemetry/metrics.js +312 -0
  252. package/dist/telemetry/metrics.js.map +1 -0
  253. package/dist/telemetry/tokenizer.d.ts +125 -0
  254. package/dist/telemetry/tokenizer.d.ts.map +1 -0
  255. package/dist/telemetry/tokenizer.js +174 -0
  256. package/dist/telemetry/tokenizer.js.map +1 -0
  257. package/package.json +81 -0
@@ -0,0 +1,45 @@
1
+ # Using MAST
2
+
3
+ You have MAST tools for navigating this codebase. MAST parses **TypeScript, JavaScript,
4
+ and Markdown** into an AST-level index and answers structural questions from a symbol
5
+ graph, so prefer it over reading files or grepping.
6
+
7
+ ## Rules
8
+
9
+ - Call `mast_status` at the start of a session to confirm the index is fresh.
10
+ - **Search before opening any file.** No file path without a MAST result behind it.
11
+ - **Use code tokens in queries** — function names, type names, column names.
12
+ `createTable uuid primaryKey` beats `migration pattern`. An exact symbol name in the
13
+ query anchors its declaration to the top.
14
+ - When a query returns nothing, **change vocabulary — do not repeat it**.
15
+ - Call `mast_reindex` after writing files, to keep the index current.
16
+
17
+ ## Picking the right tool
18
+
19
+ | tool | use it for |
20
+ |---|---|
21
+ | `mast_search` | lexical BM25 + declaration-exact discovery |
22
+ | `mast_signature` | a symbol's declaration and resolved parameter types |
23
+ | `mast_callers` | who calls a function — run before any refactor |
24
+ | `mast_implementors` | which classes implement an interface |
25
+ | `mast_exports` | a module's public API, without reading the file |
26
+ | `mast_dependencies` | what a file imports, and what it re-exports |
27
+ | `mast_project_skeleton` | a directory map of all exported symbols |
28
+ | `mast_rename_impact` | rename checklist: verified callers, review sites, barrel exports |
29
+ | `mast_efficiency` | token accounting for the session |
30
+ | `mast_status` | index freshness and health |
31
+ | `mast_reindex` | refresh the index after edits |
32
+
33
+ ## Reading the answers honestly
34
+
35
+ MAST reports what it does not know, and those signals are load-bearing:
36
+
37
+ - A result carrying **`file_busy_returning_stale_cache`** or a staleness flag means the
38
+ answer may predate the file on disk. Do not treat it as current.
39
+ - A **`truncated`** or **`potential_truncated`** flag means the set was capped. The real
40
+ count is larger than what you were shown.
41
+ - An **empty result is not proof of absence.** MAST indexes TypeScript, JavaScript, and
42
+ Markdown only — a symbol defined in Python, Go, Java, or any other language is absent
43
+ from the index, not absent from the repository. Check the flags before concluding
44
+ "it isn't there", and never delete or rewrite code on the strength of an empty result
45
+ alone.
@@ -0,0 +1,65 @@
1
+ import type { FileExtraction, IdentifierRow } from './extractor.js';
2
+ import type { Chunk } from './types.js';
3
+ import { type ExtractedSignature } from './extractors/typescript.js';
4
+ export type { ExtractedSignature } from './extractors/typescript.js';
5
+ export declare function supportsExtension(extension: string): boolean;
6
+ export type ExtractResult = FileExtraction;
7
+ /**
8
+ * Extract chunks, symbols, imports, edges, and identifier rows from a single
9
+ * file, dispatching to the registered extractor for its extension.
10
+ *
11
+ * Throws if the file cannot be read. On parse errors the caller should catch
12
+ * and log at `warn` level per §7.1 (never abort the full run).
13
+ */
14
+ export declare function extractFile(filePath: string, projectRoot: string, contextLines: number, chunkSplitThreshold: number, markdownHeadingDepth?: number): ExtractResult;
15
+ /**
16
+ * Disambiguate `chunk_id` collisions within one file's chunk list. The first
17
+ * chunk seen for a given id keeps it; the 2nd+ occurrence gets a namespaced
18
+ * ordinal suffix, assigned in stable source order so two runs over identical
19
+ * input produce identical ids (§7.1 determinism).
20
+ *
21
+ * The preimage is `${file_path}:${start_line}#dup:${n}` — NOT the raw
22
+ * `file:start:n` — because that would collide with the existing sub-chunk id
23
+ * scheme `sha256(file:startLine:subIndex)` (typescript.ts, `pushChunks`).
24
+ * `#dup` namespaces the two preimage families apart. Content-hash ids were
25
+ * rejected: they would break the documented "a content edit keeps the same
26
+ * chunk_id" contract that `vectorKey` (embedder.ts) and `isFileUnchanged`
27
+ * (indexer/index.ts) depend on. A `(file, start, end, type)` composite was
28
+ * also rejected — it still collides when multiple same-type siblings share
29
+ * one line (see lab3.ts in GITNEXUS_COMPARISON.md §15.3).
30
+ */
31
+ export declare function dedupeChunkIds(chunks: readonly Chunk[]): readonly Chunk[];
32
+ /**
33
+ * Re-key `identifierRows` (produced by the extractor against pre-dedup
34
+ * `chunk_id`s) to the post-dedup ids from `dedupeChunkIds`, so
35
+ * `identifier_fts` rows stay attributable to the right chunk.
36
+ *
37
+ * **Positional**, not keyed by `chunk_id` — which is exactly the point. Before
38
+ * dedup a `chunk_id` does NOT identify a chunk uniquely (that is what dedup is
39
+ * for), so a row carrying a colliding id names a *group*, not a member. The
40
+ * previous implementation resolved that ambiguity with a per-id FIFO and was
41
+ * exact only while every chunk in a group also produced a row — an invariant
42
+ * stated in this comment and enforced by nothing. It held in practice (measured:
43
+ * zero violations over n8n at its pinned SHA, 19,056 files / 4 collision groups,
44
+ * and mast's own src, 130 files), but it rested on an argument about identifier
45
+ * emission inside a 1,637-line extractor rather than on anything a compiler or a
46
+ * test could see. `FileExtraction.identifierRows` now carries one row per chunk,
47
+ * so the correspondence is total and this function is a zip (D040).
48
+ *
49
+ * @throws Error if the list is neither empty nor one-per-chunk. That is a broken
50
+ * extractor contract, not a data condition: it is caught per-file by `runIndex`,
51
+ * counted as a parse error, and retried — loudly, rather than writing identifier
52
+ * rows onto the wrong chunk where nothing downstream could detect it.
53
+ */
54
+ export declare function remapIdentifierRows(originalChunks: readonly Chunk[], dedupedChunks: readonly Chunk[], identifierRows: readonly IdentifierRow[]): readonly IdentifierRow[];
55
+ /**
56
+ * Extract body-free signatures (with params + return type) for every symbol in
57
+ * a file. Used by `mast_signature` and `mast_exports` at query time so they
58
+ * report declarations, not function bodies (§10.2). Returns `[]` for
59
+ * unsupported extensions or on parse failure.
60
+ *
61
+ * Deliberately TypeScript-only: signatures are a code concept, and doc chunks
62
+ * have none — a non-TS extension takes the same `[]` path as an unsupported one.
63
+ */
64
+ export declare function extractFileSignatures(absPath: string): readonly ExtractedSignature[];
65
+ //# sourceMappingURL=extract.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"extract.d.ts","sourceRoot":"","sources":["../../src/ast/extract.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAqB,cAAc,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AACvF,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,EAAkD,KAAK,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAGrH,YAAY,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAkBrE,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAE5D;AAMD,MAAM,MAAM,aAAa,GAAG,cAAc,CAAC;AAE3C;;;;;;GAMG;AACH,wBAAgB,WAAW,CACzB,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,MAAM,EACnB,YAAY,EAAE,MAAM,EACpB,mBAAmB,EAAE,MAAM,EAC3B,oBAAoB,SAAI,GACvB,aAAa,CAsCf;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,GAAG,SAAS,KAAK,EAAE,CAQzE;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,mBAAmB,CACjC,cAAc,EAAE,SAAS,KAAK,EAAE,EAChC,aAAa,EAAE,SAAS,KAAK,EAAE,EAC/B,cAAc,EAAE,SAAS,aAAa,EAAE,GACvC,SAAS,aAAa,EAAE,CAe1B;AAED;;;;;;;;GAQG;AACH,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,kBAAkB,EAAE,CAUpF"}
@@ -0,0 +1,147 @@
1
+ import { extname } from 'node:path';
2
+ import { readFileSync, statSync } from 'node:fs';
3
+ import { parseSource } from './parser.js';
4
+ import { TypeScriptExtractor, extractSignatures, sha256 } from './extractors/typescript.js';
5
+ import { MarkdownExtractor } from './extractors/markdown.js';
6
+ // ---------------------------------------------------------------------------
7
+ // Extractor registry
8
+ // ---------------------------------------------------------------------------
9
+ // Adding a language = adding one entry here. Each extractor owns its full
10
+ // extraction story (parsing included) behind the LanguageExtractor contract,
11
+ // so the pipeline never branches on language.
12
+ const EXTRACTORS = [new TypeScriptExtractor(), new MarkdownExtractor()];
13
+ const EXT_TO_EXTRACTOR = new Map();
14
+ for (const ext of EXTRACTORS) {
15
+ for (const extension of ext.extensions) {
16
+ EXT_TO_EXTRACTOR.set(extension, ext);
17
+ }
18
+ }
19
+ export function supportsExtension(extension) {
20
+ return EXT_TO_EXTRACTOR.has(extension);
21
+ }
22
+ /**
23
+ * Extract chunks, symbols, imports, edges, and identifier rows from a single
24
+ * file, dispatching to the registered extractor for its extension.
25
+ *
26
+ * Throws if the file cannot be read. On parse errors the caller should catch
27
+ * and log at `warn` level per §7.1 (never abort the full run).
28
+ */
29
+ export function extractFile(filePath, projectRoot, contextLines, chunkSplitThreshold, markdownHeadingDepth = 2) {
30
+ const extension = extname(filePath);
31
+ const extractor = EXT_TO_EXTRACTOR.get(extension);
32
+ if (extractor === undefined) {
33
+ throw new Error(`No extractor registered for extension "${extension}"`);
34
+ }
35
+ const src = readFileSync(filePath, 'utf-8');
36
+ const mtime = statSync(filePath).mtimeMs / 1_000;
37
+ // Relative path for chunk IDs and stored file_path field.
38
+ const relativePath = filePath.startsWith(projectRoot)
39
+ ? filePath.slice(projectRoot.length).replace(/^\//, '')
40
+ : filePath;
41
+ const extraction = extractor.extract(src, relativePath, mtime, {
42
+ projectRoot,
43
+ contextLines,
44
+ chunkSplitThreshold,
45
+ markdownHeadingDepth,
46
+ });
47
+ // Collision guard (GITNEXUS_COMPARISON.md §15.3): genuine same-line
48
+ // siblings — multiple real declarations sharing one physical line (e.g.
49
+ // several methods on one line), or generated/minified files where every
50
+ // top-level statement shares start_line — produce identical chunk_ids
51
+ // under the position-based `sha256(file:start_line)` scheme. Applied here,
52
+ // the one dispatch chokepoint, so every extractor (typescript, markdown) is
53
+ // covered without scattering disambiguation through individual chunkId
54
+ // call sites. The first chunk at a given id is left unchanged, so the
55
+ // ~99.97% of files with no collision see zero id churn.
56
+ const chunks = dedupeChunkIds(extraction.chunks);
57
+ // Re-key BEFORE dropping empties: the re-keying is positional, and filtering
58
+ // first is what destroys the position (D040).
59
+ const identifierRows = remapIdentifierRows(extraction.chunks, chunks, extraction.identifierRows)
60
+ .filter((row) => row.identifiers.length > 0);
61
+ return { ...extraction, chunks, identifierRows };
62
+ }
63
+ /**
64
+ * Disambiguate `chunk_id` collisions within one file's chunk list. The first
65
+ * chunk seen for a given id keeps it; the 2nd+ occurrence gets a namespaced
66
+ * ordinal suffix, assigned in stable source order so two runs over identical
67
+ * input produce identical ids (§7.1 determinism).
68
+ *
69
+ * The preimage is `${file_path}:${start_line}#dup:${n}` — NOT the raw
70
+ * `file:start:n` — because that would collide with the existing sub-chunk id
71
+ * scheme `sha256(file:startLine:subIndex)` (typescript.ts, `pushChunks`).
72
+ * `#dup` namespaces the two preimage families apart. Content-hash ids were
73
+ * rejected: they would break the documented "a content edit keeps the same
74
+ * chunk_id" contract that `vectorKey` (embedder.ts) and `isFileUnchanged`
75
+ * (indexer/index.ts) depend on. A `(file, start, end, type)` composite was
76
+ * also rejected — it still collides when multiple same-type siblings share
77
+ * one line (see lab3.ts in GITNEXUS_COMPARISON.md §15.3).
78
+ */
79
+ export function dedupeChunkIds(chunks) {
80
+ const occurrences = new Map();
81
+ return chunks.map((chunk) => {
82
+ const occurrence = occurrences.get(chunk.chunk_id) ?? 0;
83
+ occurrences.set(chunk.chunk_id, occurrence + 1);
84
+ if (occurrence === 0)
85
+ return chunk;
86
+ return { ...chunk, chunk_id: sha256(`${chunk.file_path}:${chunk.start_line}#dup:${occurrence}`) };
87
+ });
88
+ }
89
+ /**
90
+ * Re-key `identifierRows` (produced by the extractor against pre-dedup
91
+ * `chunk_id`s) to the post-dedup ids from `dedupeChunkIds`, so
92
+ * `identifier_fts` rows stay attributable to the right chunk.
93
+ *
94
+ * **Positional**, not keyed by `chunk_id` — which is exactly the point. Before
95
+ * dedup a `chunk_id` does NOT identify a chunk uniquely (that is what dedup is
96
+ * for), so a row carrying a colliding id names a *group*, not a member. The
97
+ * previous implementation resolved that ambiguity with a per-id FIFO and was
98
+ * exact only while every chunk in a group also produced a row — an invariant
99
+ * stated in this comment and enforced by nothing. It held in practice (measured:
100
+ * zero violations over n8n at its pinned SHA, 19,056 files / 4 collision groups,
101
+ * and mast's own src, 130 files), but it rested on an argument about identifier
102
+ * emission inside a 1,637-line extractor rather than on anything a compiler or a
103
+ * test could see. `FileExtraction.identifierRows` now carries one row per chunk,
104
+ * so the correspondence is total and this function is a zip (D040).
105
+ *
106
+ * @throws Error if the list is neither empty nor one-per-chunk. That is a broken
107
+ * extractor contract, not a data condition: it is caught per-file by `runIndex`,
108
+ * counted as a parse error, and retried — loudly, rather than writing identifier
109
+ * rows onto the wrong chunk where nothing downstream could detect it.
110
+ */
111
+ export function remapIdentifierRows(originalChunks, dedupedChunks, identifierRows) {
112
+ if (identifierRows.length === 0)
113
+ return identifierRows;
114
+ if (identifierRows.length !== originalChunks.length || dedupedChunks.length !== originalChunks.length) {
115
+ throw new Error(`identifier row remap requires one identifier row per chunk: got ${identifierRows.length} rows, ` +
116
+ `${originalChunks.length} chunks, ${dedupedChunks.length} deduped chunks`);
117
+ }
118
+ return identifierRows.map((row, i) => {
119
+ const deduped = dedupedChunks[i];
120
+ if (deduped === undefined || deduped.chunk_id === row.chunk_id)
121
+ return row;
122
+ return { ...row, chunk_id: deduped.chunk_id };
123
+ });
124
+ }
125
+ /**
126
+ * Extract body-free signatures (with params + return type) for every symbol in
127
+ * a file. Used by `mast_signature` and `mast_exports` at query time so they
128
+ * report declarations, not function bodies (§10.2). Returns `[]` for
129
+ * unsupported extensions or on parse failure.
130
+ *
131
+ * Deliberately TypeScript-only: signatures are a code concept, and doc chunks
132
+ * have none — a non-TS extension takes the same `[]` path as an unsupported one.
133
+ */
134
+ export function extractFileSignatures(absPath) {
135
+ const extension = extname(absPath);
136
+ if (!(EXT_TO_EXTRACTOR.get(extension) instanceof TypeScriptExtractor))
137
+ return [];
138
+ try {
139
+ const src = readFileSync(absPath, 'utf-8');
140
+ const tree = parseSource(src, extension);
141
+ return extractSignatures(tree, src);
142
+ }
143
+ catch {
144
+ return [];
145
+ }
146
+ }
147
+ //# sourceMappingURL=extract.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"extract.js","sourceRoot":"","sources":["../../src/ast/extract.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAG1C,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,EAA2B,MAAM,4BAA4B,CAAC;AACrH,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAI7D,8EAA8E;AAC9E,qBAAqB;AACrB,8EAA8E;AAE9E,0EAA0E;AAC1E,6EAA6E;AAC7E,8CAA8C;AAC9C,MAAM,UAAU,GAAiC,CAAC,IAAI,mBAAmB,EAAE,EAAE,IAAI,iBAAiB,EAAE,CAAC,CAAC;AAEtG,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAA6B,CAAC;AAC9D,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;IAC7B,KAAK,MAAM,SAAS,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC;QACvC,gBAAgB,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;IACvC,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,SAAiB;IACjD,OAAO,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;AACzC,CAAC;AAQD;;;;;;GAMG;AACH,MAAM,UAAU,WAAW,CACzB,QAAgB,EAChB,WAAmB,EACnB,YAAoB,EACpB,mBAA2B,EAC3B,oBAAoB,GAAG,CAAC;IAExB,MAAM,SAAS,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IACpC,MAAM,SAAS,GAAG,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAClD,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CAAC,0CAA0C,SAAS,GAAG,CAAC,CAAC;IAC1E,CAAC;IAED,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC5C,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,OAAO,GAAG,KAAK,CAAC;IAEjD,0DAA0D;IAC1D,MAAM,YAAY,GAAG,QAAQ,CAAC,UAAU,CAAC,WAAW,CAAC;QACnD,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;QACvD,CAAC,CAAC,QAAQ,CAAC;IAEb,MAAM,UAAU,GAAG,SAAS,CAAC,OAAO,CAAC,GAAG,EAAE,YAAY,EAAE,KAAK,EAAE;QAC7D,WAAW;QACX,YAAY;QACZ,mBAAmB;QACnB,oBAAoB;KACrB,CAAC,CAAC;IAEH,oEAAoE;IACpE,wEAAwE;IACxE,wEAAwE;IACxE,sEAAsE;IACtE,2EAA2E;IAC3E,4EAA4E;IAC5E,uEAAuE;IACvE,sEAAsE;IACtE,wDAAwD;IACxD,MAAM,MAAM,GAAG,cAAc,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;IACjD,6EAA6E;IAC7E,8CAA8C;IAC9C,MAAM,cAAc,GAAG,mBAAmB,CAAC,UAAU,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,cAAc,CAAC;SAC7F,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAE/C,OAAO,EAAE,GAAG,UAAU,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC;AACnD,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,cAAc,CAAC,MAAwB;IACrD,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC9C,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QAC1B,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACxD,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,EAAE,UAAU,GAAG,CAAC,CAAC,CAAC;QAChD,IAAI,UAAU,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QACnC,OAAO,EAAE,GAAG,KAAK,EAAE,QAAQ,EAAE,MAAM,CAAC,GAAG,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,UAAU,QAAQ,UAAU,EAAE,CAAC,EAAE,CAAC;IACpG,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,UAAU,mBAAmB,CACjC,cAAgC,EAChC,aAA+B,EAC/B,cAAwC;IAExC,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,cAAc,CAAC;IAEvD,IAAI,cAAc,CAAC,MAAM,KAAK,cAAc,CAAC,MAAM,IAAI,aAAa,CAAC,MAAM,KAAK,cAAc,CAAC,MAAM,EAAE,CAAC;QACtG,MAAM,IAAI,KAAK,CACb,mEAAmE,cAAc,CAAC,MAAM,SAAS;YACjG,GAAG,cAAc,CAAC,MAAM,YAAY,aAAa,CAAC,MAAM,iBAAiB,CAC1E,CAAC;IACJ,CAAC;IAED,OAAO,cAAc,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE;QACnC,MAAM,OAAO,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;QACjC,IAAI,OAAO,KAAK,SAAS,IAAI,OAAO,CAAC,QAAQ,KAAK,GAAG,CAAC,QAAQ;YAAE,OAAO,GAAG,CAAC;QAC3E,OAAO,EAAE,GAAG,GAAG,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC;IAChD,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,qBAAqB,CAAC,OAAe;IACnD,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACnC,IAAI,CAAC,CAAC,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAC,YAAY,mBAAmB,CAAC;QAAE,OAAO,EAAE,CAAC;IACjF,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAC3C,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QACzC,OAAO,iBAAiB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACtC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC"}
@@ -0,0 +1,85 @@
1
+ import type { Chunk, Language, SymbolRecord, ImportRecord, EdgeRecord } from './types.js';
2
+ /**
3
+ * One `identifier_fts` row: the identifier tokens extracted from a chunk.
4
+ * Produced by the extractor (not the graph layer) because what counts as an
5
+ * "identifier" is a language-level judgment — e.g. markdown prose contributes
6
+ * none, since a doc that merely mentions a symbol name is not a call site.
7
+ */
8
+ export interface IdentifierRow {
9
+ readonly chunk_id: string;
10
+ /** Space-separated identifier tokens for the FTS index. */
11
+ readonly identifiers: string;
12
+ }
13
+ /**
14
+ * File-level `export * from './x'` (star barrel re-export). Named re-exports
15
+ * need no dedicated record — they ride the existing symbols + edges pipeline
16
+ * (a marker symbol of kind `export` plus a RE_EXPORTS edge, §10.1). Stars
17
+ * have no per-symbol identity, so they persist as `re_export_files` rows.
18
+ */
19
+ export interface StarReExportRecord {
20
+ readonly module: string;
21
+ /** Resolved relative path for intra-monorepo targets; null for external. */
22
+ readonly resolvedPath: string | null;
23
+ /** 1-indexed source line of the `export *` statement. */
24
+ readonly line: number;
25
+ }
26
+ /**
27
+ * Everything the indexing pipeline needs from one source file. Each extractor
28
+ * owns the full story for its language: languages without a symbol graph
29
+ * (markdown) return empty `symbols` / `edges` / … and the pipeline needs no
30
+ * per-language branches.
31
+ */
32
+ export interface FileExtraction {
33
+ /** Concrete language of THIS file — may differ from the extractor's primary
34
+ * language (the TypeScript extractor also handles `.js`/`.jsx`). */
35
+ readonly language: Language;
36
+ readonly chunks: readonly Chunk[];
37
+ readonly symbols: readonly SymbolRecord[];
38
+ readonly imports: readonly ImportRecord[];
39
+ readonly edges: readonly EdgeRecord[];
40
+ /**
41
+ * Either empty, or **exactly one row per chunk, in chunk order** — including
42
+ * chunks whose identifier bag is empty. `extractFile` relies on that positional
43
+ * correspondence to re-key rows after `dedupeChunkIds`, and drops the empty ones
44
+ * afterwards; a partial list cannot be re-keyed, because `chunk_id` does not
45
+ * identify a chunk uniquely before dedup (D040).
46
+ */
47
+ readonly identifierRows: readonly IdentifierRow[];
48
+ readonly starReExports: readonly StarReExportRecord[];
49
+ }
50
+ /** Per-run knobs passed to every extractor; each reads what applies to it. */
51
+ export interface ExtractorOptions {
52
+ /** Absolute project root — used for import specifier resolution (§13.7). */
53
+ readonly projectRoot: string;
54
+ /** Source lines expanded around AST boundaries in stored content. */
55
+ readonly contextLines: number;
56
+ /** Lines above which a declaration/section splits into sub-chunks (§10.1). */
57
+ readonly chunkSplitThreshold: number;
58
+ /** Max ATX heading level that starts a markdown doc chunk (§10.1). */
59
+ readonly markdownHeadingDepth: number;
60
+ }
61
+ /**
62
+ * Contract every language extractor must satisfy.
63
+ *
64
+ * Parsing strategy is the implementer's concern — the TypeScript extractor
65
+ * parses with tree-sitter internally, the markdown extractor line-scans. The
66
+ * pipeline (extract.ts) only dispatches by extension and consumes the
67
+ * `FileExtraction`; it never special-cases a language.
68
+ */
69
+ export interface LanguageExtractor {
70
+ /** Primary language identifier for this extractor. */
71
+ readonly language: Language;
72
+ /** File extensions this extractor handles (e.g. `['.ts', '.tsx']`). */
73
+ readonly extensions: readonly string[];
74
+ /**
75
+ * Extract everything the index needs from one file. Called once per file
76
+ * per index run. May throw on unparseable input — the indexer catches and
77
+ * logs per §7.1 (never aborts the full run).
78
+ *
79
+ * @param src - Raw UTF-8 source text.
80
+ * @param filePath - Relative to project root; used in chunk IDs.
81
+ * @param fileMtime - File modification time at index time (unix seconds).
82
+ */
83
+ extract(src: string, filePath: string, fileMtime: number, options: ExtractorOptions): FileExtraction;
84
+ }
85
+ //# sourceMappingURL=extractor.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"extractor.d.ts","sourceRoot":"","sources":["../../src/ast/extractor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,YAAY,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAM1F;;;;;GAKG;AACH,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,2DAA2D;IAC3D,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAED;;;;;GAKG;AACH,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,4EAA4E;IAC5E,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,yDAAyD;IACzD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAED;;;;;GAKG;AACH,MAAM,WAAW,cAAc;IAC7B;yEACqE;IACrE,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;IAClC,QAAQ,CAAC,OAAO,EAAE,SAAS,YAAY,EAAE,CAAC;IAC1C,QAAQ,CAAC,OAAO,EAAE,SAAS,YAAY,EAAE,CAAC;IAC1C,QAAQ,CAAC,KAAK,EAAE,SAAS,UAAU,EAAE,CAAC;IACtC;;;;;;OAMG;IACH,QAAQ,CAAC,cAAc,EAAE,SAAS,aAAa,EAAE,CAAC;IAClD,QAAQ,CAAC,aAAa,EAAE,SAAS,kBAAkB,EAAE,CAAC;CACvD;AAED,8EAA8E;AAC9E,MAAM,WAAW,gBAAgB;IAC/B,4EAA4E;IAC5E,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,qEAAqE;IACrE,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,8EAA8E;IAC9E,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,sEAAsE;IACtE,QAAQ,CAAC,oBAAoB,EAAE,MAAM,CAAC;CACvC;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,iBAAiB;IAChC,sDAAsD;IACtD,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC;IAC5B,uEAAuE;IACvE,QAAQ,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,CAAC;IACvC;;;;;;;;OAQG;IACH,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,GAAG,cAAc,CAAC;CACtG"}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=extractor.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"extractor.js","sourceRoot":"","sources":["../../src/ast/extractor.ts"],"names":[],"mappings":""}
@@ -0,0 +1,40 @@
1
+ import type { Chunk } from '../types.js';
2
+ import type { LanguageExtractor, FileExtraction, ExtractorOptions } from '../extractor.js';
3
+ /**
4
+ * Heading-based chunker for markdown documents.
5
+ *
6
+ * No parser: a line scan with fence tracking is sufficient and avoids
7
+ * shipping a markdown grammar — parsing strategy is internal to each
8
+ * `LanguageExtractor`. Doc chunks carry no symbol graph: `extract` returns
9
+ * empty `symbols` / `imports` / `edges`, and empty `identifierRows` because
10
+ * `identifier_fts` feeds `mast_callers` potential_matches, where prose that
11
+ * merely mentions a symbol name is noise, not a call site — docs participate
12
+ * in search via chunk_fts + vectors only.
13
+ *
14
+ * Chunking rules (documented in MAST_SPEC.md §10.1):
15
+ * - A new chunk starts at every ATX heading of level ≤ `maxHeadingDepth`;
16
+ * deeper headings fold into the enclosing section's content.
17
+ * - `symbol_name` is the heading path: file name + every ancestor heading +
18
+ * the section's own heading, joined with " > ".
19
+ * - Content before the first boundary heading becomes a preamble chunk whose
20
+ * `symbol_name` is the file name alone.
21
+ * - `#` lines inside fenced code blocks (``` or ~~~) are not headings.
22
+ * - Setext headings (`===` / `---` underlines) are not recognised — this
23
+ * repo's docs use ATX exclusively, and supporting both complicates the
24
+ * fence-aware scan for no observed payoff.
25
+ * - Sections longer than `chunkSplitThreshold` lines split into overlapping
26
+ * sub-chunks using the same window/overlap/ID scheme as code declarations,
27
+ * so no single doc chunk exceeds the token budget the threshold encodes.
28
+ * - No `context_lines` expansion: sections are self-delimiting, and expansion
29
+ * would duplicate neighbouring sections' text into every chunk.
30
+ * - Doc chunks carry no `declaration_hash`/`body_hash` — there is no symbol
31
+ * row to compare against, so the incremental "unchanged file" fast path
32
+ * conservatively rewrites markdown files instead (see indexer/index.ts).
33
+ */
34
+ export declare class MarkdownExtractor implements LanguageExtractor {
35
+ readonly language = "markdown";
36
+ readonly extensions: readonly string[];
37
+ extract(src: string, filePath: string, fileMtime: number, options: ExtractorOptions): FileExtraction;
38
+ extractChunks(src: string, filePath: string, fileMtime: number, chunkSplitThreshold: number, maxHeadingDepth: number): Chunk[];
39
+ }
40
+ //# sourceMappingURL=markdown.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"markdown.d.ts","sourceRoot":"","sources":["../../../src/ast/extractors/markdown.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,KAAK,EAAE,iBAAiB,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAoB3F;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,qBAAa,iBAAkB,YAAW,iBAAiB;IACzD,QAAQ,CAAC,QAAQ,cAAc;IAC/B,QAAQ,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,CAAW;IAEjD,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,GAAG,cAAc;IAKpG,aAAa,CACX,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,mBAAmB,EAAE,MAAM,EAC3B,eAAe,EAAE,MAAM,GACtB,KAAK,EAAE;CA6EX"}
@@ -0,0 +1,175 @@
1
+ import { chunkId, sha256 } from './typescript.js';
2
+ // ---------------------------------------------------------------------------
3
+ // Markdown extractor — heading-based doc chunking (§10.1)
4
+ // ---------------------------------------------------------------------------
5
+ /** ATX heading: 1–6 `#` followed by whitespace and the heading text. */
6
+ const HEADING_RE = /^(#{1,6})\s+(.*?)\s*#*\s*$/;
7
+ /** Fence opener/closer: ``` or ~~~ optionally followed by an info string. */
8
+ const FENCE_RE = /^\s{0,3}(```|~~~)/;
9
+ /**
10
+ * Heading-based chunker for markdown documents.
11
+ *
12
+ * No parser: a line scan with fence tracking is sufficient and avoids
13
+ * shipping a markdown grammar — parsing strategy is internal to each
14
+ * `LanguageExtractor`. Doc chunks carry no symbol graph: `extract` returns
15
+ * empty `symbols` / `imports` / `edges`, and empty `identifierRows` because
16
+ * `identifier_fts` feeds `mast_callers` potential_matches, where prose that
17
+ * merely mentions a symbol name is noise, not a call site — docs participate
18
+ * in search via chunk_fts + vectors only.
19
+ *
20
+ * Chunking rules (documented in MAST_SPEC.md §10.1):
21
+ * - A new chunk starts at every ATX heading of level ≤ `maxHeadingDepth`;
22
+ * deeper headings fold into the enclosing section's content.
23
+ * - `symbol_name` is the heading path: file name + every ancestor heading +
24
+ * the section's own heading, joined with " > ".
25
+ * - Content before the first boundary heading becomes a preamble chunk whose
26
+ * `symbol_name` is the file name alone.
27
+ * - `#` lines inside fenced code blocks (``` or ~~~) are not headings.
28
+ * - Setext headings (`===` / `---` underlines) are not recognised — this
29
+ * repo's docs use ATX exclusively, and supporting both complicates the
30
+ * fence-aware scan for no observed payoff.
31
+ * - Sections longer than `chunkSplitThreshold` lines split into overlapping
32
+ * sub-chunks using the same window/overlap/ID scheme as code declarations,
33
+ * so no single doc chunk exceeds the token budget the threshold encodes.
34
+ * - No `context_lines` expansion: sections are self-delimiting, and expansion
35
+ * would duplicate neighbouring sections' text into every chunk.
36
+ * - Doc chunks carry no `declaration_hash`/`body_hash` — there is no symbol
37
+ * row to compare against, so the incremental "unchanged file" fast path
38
+ * conservatively rewrites markdown files instead (see indexer/index.ts).
39
+ */
40
+ export class MarkdownExtractor {
41
+ language = 'markdown';
42
+ extensions = ['.md'];
43
+ extract(src, filePath, fileMtime, options) {
44
+ const chunks = this.extractChunks(src, filePath, fileMtime, options.chunkSplitThreshold, options.markdownHeadingDepth);
45
+ return { language: 'markdown', chunks, symbols: [], imports: [], edges: [], identifierRows: [], starReExports: [] };
46
+ }
47
+ extractChunks(src, filePath, fileMtime, chunkSplitThreshold, maxHeadingDepth) {
48
+ if (src.trim() === '')
49
+ return [];
50
+ const lines = src.split('\n');
51
+ // Drop a trailing empty element produced by a terminating newline so the
52
+ // last section's end_line matches the real last line of the file.
53
+ if (lines[lines.length - 1] === '')
54
+ lines.pop();
55
+ const fileName = fileNameOf(filePath);
56
+ const sections = [];
57
+ // Last-seen heading text per level (1-indexed); shallower ancestors form
58
+ // the heading path of each new section. Sparse when a document skips
59
+ // levels (e.g. `#` straight to `###`) — holes are filtered from paths.
60
+ const headingStack = [];
61
+ let inFence = false;
62
+ let fenceMarker = '';
63
+ for (let i = 0; i < lines.length; i++) {
64
+ const line = lines[i] ?? '';
65
+ const fence = FENCE_RE.exec(line);
66
+ if (fence !== null) {
67
+ const marker = fence[1] ?? '';
68
+ if (!inFence) {
69
+ inFence = true;
70
+ fenceMarker = marker;
71
+ }
72
+ else if (marker === fenceMarker) {
73
+ inFence = false;
74
+ }
75
+ continue;
76
+ }
77
+ if (inFence)
78
+ continue;
79
+ const heading = HEADING_RE.exec(line);
80
+ if (heading === null)
81
+ continue;
82
+ const level = (heading[1] ?? '').length;
83
+ const text = heading[2] ?? '';
84
+ // Track ancestors at every level so folded deep headings still
85
+ // contribute to the path of later, shallower-or-equal siblings' children.
86
+ headingStack.splice(level - 1);
87
+ headingStack[level - 1] = text;
88
+ if (level > maxHeadingDepth)
89
+ continue; // folds into the current section
90
+ // First boundary heading below line 1 implies a preamble chunk.
91
+ if (sections.length === 0 && i > 0) {
92
+ sections.push({ startLine: 1, symbolName: fileName });
93
+ }
94
+ const ancestors = headingStack.slice(0, level).filter((t) => t !== undefined);
95
+ const path = [fileName, ...ancestors].join(' > ');
96
+ sections.push({ startLine: i + 1, symbolName: path });
97
+ }
98
+ // No headings at all — the whole file is one preamble chunk.
99
+ if (sections.length === 0) {
100
+ sections.push({ startLine: 1, symbolName: fileName });
101
+ }
102
+ const chunks = [];
103
+ for (let s = 0; s < sections.length; s++) {
104
+ const section = sections[s];
105
+ if (section === undefined)
106
+ continue;
107
+ const next = sections[s + 1];
108
+ const endLine = next !== undefined ? next.startLine - 1 : lines.length;
109
+ pushSectionChunks(chunks, {
110
+ lines,
111
+ filePath,
112
+ fileMtime,
113
+ chunkSplitThreshold,
114
+ symbolName: section.symbolName,
115
+ startLine: section.startLine,
116
+ endLine,
117
+ });
118
+ }
119
+ return chunks;
120
+ }
121
+ }
122
+ // ---------------------------------------------------------------------------
123
+ // Helpers
124
+ // ---------------------------------------------------------------------------
125
+ /** Basename of a relative path — the heading-path root. */
126
+ function fileNameOf(filePath) {
127
+ const idx = filePath.lastIndexOf('/');
128
+ return idx === -1 ? filePath : filePath.slice(idx + 1);
129
+ }
130
+ /**
131
+ * Push one doc chunk — or N overlapping sub-chunks when the section exceeds
132
+ * `chunkSplitThreshold` lines. Mirrors the code-side split rule (window =
133
+ * threshold, 10-line overlap, sub-index-qualified chunk IDs) so downstream
134
+ * consumers see one consistent scheme.
135
+ */
136
+ function pushSectionChunks(chunks, opts) {
137
+ const { lines, filePath, fileMtime, symbolName, startLine, endLine } = opts;
138
+ const base = {
139
+ file_path: filePath,
140
+ chunk_type: 'doc',
141
+ symbol_name: symbolName,
142
+ parent_symbol: null,
143
+ is_exported: false,
144
+ language: 'markdown',
145
+ file_mtime: fileMtime,
146
+ };
147
+ if (endLine - startLine + 1 <= opts.chunkSplitThreshold) {
148
+ chunks.push({
149
+ ...base,
150
+ chunk_id: chunkId(filePath, startLine),
151
+ start_line: startLine,
152
+ end_line: endLine,
153
+ content: lines.slice(startLine - 1, endLine).join('\n'),
154
+ });
155
+ return;
156
+ }
157
+ const OVERLAP_LINES = 10;
158
+ let subStart = startLine;
159
+ let subIndex = 0;
160
+ while (subStart <= endLine) {
161
+ const subEnd = Math.min(subStart + opts.chunkSplitThreshold - 1, endLine);
162
+ chunks.push({
163
+ ...base,
164
+ chunk_id: sha256(`${filePath}:${startLine}:${subIndex}`),
165
+ start_line: subStart,
166
+ end_line: subEnd,
167
+ content: lines.slice(subStart - 1, subEnd).join('\n'),
168
+ });
169
+ if (subEnd >= endLine)
170
+ break;
171
+ subStart = subEnd - OVERLAP_LINES + 1;
172
+ subIndex++;
173
+ }
174
+ }
175
+ //# sourceMappingURL=markdown.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"markdown.js","sourceRoot":"","sources":["../../../src/ast/extractors/markdown.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAElD,8EAA8E;AAC9E,0DAA0D;AAC1D,8EAA8E;AAE9E,wEAAwE;AACxE,MAAM,UAAU,GAAG,4BAA4B,CAAC;AAEhD,6EAA6E;AAC7E,MAAM,QAAQ,GAAG,mBAAmB,CAAC;AASrC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,MAAM,OAAO,iBAAiB;IACnB,QAAQ,GAAG,UAAU,CAAC;IACtB,UAAU,GAAsB,CAAC,KAAK,CAAC,CAAC;IAEjD,OAAO,CAAC,GAAW,EAAE,QAAgB,EAAE,SAAiB,EAAE,OAAyB;QACjF,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,mBAAmB,EAAE,OAAO,CAAC,oBAAoB,CAAC,CAAC;QACvH,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,cAAc,EAAE,EAAE,EAAE,aAAa,EAAE,EAAE,EAAE,CAAC;IACtH,CAAC;IAED,aAAa,CACX,GAAW,EACX,QAAgB,EAChB,SAAiB,EACjB,mBAA2B,EAC3B,eAAuB;QAEvB,IAAI,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE;YAAE,OAAO,EAAE,CAAC;QAEjC,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC9B,yEAAyE;QACzE,kEAAkE;QAClE,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,EAAE;YAAE,KAAK,CAAC,GAAG,EAAE,CAAC;QAEhD,MAAM,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;QACtC,MAAM,QAAQ,GAAc,EAAE,CAAC;QAC/B,yEAAyE;QACzE,qEAAqE;QACrE,uEAAuE;QACvE,MAAM,YAAY,GAA2B,EAAE,CAAC;QAChD,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,IAAI,WAAW,GAAG,EAAE,CAAC;QAErB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAE5B,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClC,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBACnB,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC9B,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,OAAO,GAAG,IAAI,CAAC;oBACf,WAAW,GAAG,MAAM,CAAC;gBACvB,CAAC;qBAAM,IAAI,MAAM,KAAK,WAAW,EAAE,CAAC;oBAClC,OAAO,GAAG,KAAK,CAAC;gBAClB,CAAC;gBACD,SAAS;YACX,CAAC;YACD,IAAI,OAAO;gBAAE,SAAS;YAEtB,MAAM,OAAO,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACtC,IAAI,OAAO,KAAK,IAAI;gBAAE,SAAS;YAC/B,MAAM,KAAK,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;YACxC,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAE9B,+DAA+D;YAC/D,0EAA0E;YAC1E,YAAY,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;YAC/B,YAAY,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC;YAE/B,IAAI,KAAK,GAAG,eAAe;gBAAE,SAAS,CAAC,iCAAiC;YAExE,gEAAgE;YAChE,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACnC,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC,CAAC;YACxD,CAAC;YACD,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC;YAC3F,MAAM,IAAI,GAAG,CAAC,QAAQ,EAAE,GAAG,SAAS,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAClD,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,CAAC,GAAG,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;QACxD,CAAC;QAED,6DAA6D;QAC7D,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC,CAAC;QACxD,CAAC;QAED,MAAM,MAAM,GAAY,EAAE,CAAC;QAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACzC,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;YAC5B,IAAI,OAAO,KAAK,SAAS;gBAAE,SAAS;YACpC,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAC7B,MAAM,OAAO,GAAG,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC;YACvE,iBAAiB,CAAC,MAAM,EAAE;gBACxB,KAAK;gBACL,QAAQ;gBACR,SAAS;gBACT,mBAAmB;gBACnB,UAAU,EAAE,OAAO,CAAC,UAAU;gBAC9B,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,OAAO;aACR,CAAC,CAAC;QACL,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAED,8EAA8E;AAC9E,UAAU;AACV,8EAA8E;AAE9E,2DAA2D;AAC3D,SAAS,UAAU,CAAC,QAAgB;IAClC,MAAM,GAAG,GAAG,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IACtC,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;AACzD,CAAC;AAYD;;;;;GAKG;AACH,SAAS,iBAAiB,CAAC,MAAe,EAAE,IAAqB;IAC/D,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC;IAC5E,MAAM,IAAI,GAAG;QACX,SAAS,EAAE,QAAQ;QACnB,UAAU,EAAE,KAAc;QAC1B,WAAW,EAAE,UAAU;QACvB,aAAa,EAAE,IAAI;QACnB,WAAW,EAAE,KAAK;QAClB,QAAQ,EAAE,UAAmB;QAC7B,UAAU,EAAE,SAAS;KACtB,CAAC;IAEF,IAAI,OAAO,GAAG,SAAS,GAAG,CAAC,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACxD,MAAM,CAAC,IAAI,CAAC;YACV,GAAG,IAAI;YACP,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,SAAS,CAAC;YACtC,UAAU,EAAE,SAAS;YACrB,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;SACxD,CAAC,CAAC;QACH,OAAO;IACT,CAAC;IAED,MAAM,aAAa,GAAG,EAAE,CAAC;IACzB,IAAI,QAAQ,GAAG,SAAS,CAAC;IACzB,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,OAAO,QAAQ,IAAI,OAAO,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,mBAAmB,GAAG,CAAC,EAAE,OAAO,CAAC,CAAC;QAC1E,MAAM,CAAC,IAAI,CAAC;YACV,GAAG,IAAI;YACP,QAAQ,EAAE,MAAM,CAAC,GAAG,QAAQ,IAAI,SAAS,IAAI,QAAQ,EAAE,CAAC;YACxD,UAAU,EAAE,QAAQ;YACpB,QAAQ,EAAE,MAAM;YAChB,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,QAAQ,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;SACtD,CAAC,CAAC;QACH,IAAI,MAAM,IAAI,OAAO;YAAE,MAAM;QAC7B,QAAQ,GAAG,MAAM,GAAG,aAAa,GAAG,CAAC,CAAC;QACtC,QAAQ,EAAE,CAAC;IACb,CAAC;AACH,CAAC"}