mdcontext 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (251) hide show
  1. package/.changeset/config.json +9 -9
  2. package/.claude/settings.local.json +25 -0
  3. package/.github/workflows/claude-code-review.yml +44 -0
  4. package/.github/workflows/claude.yml +85 -0
  5. package/CONTRIBUTING.md +186 -0
  6. package/NOTES/NOTES +44 -0
  7. package/README.md +206 -3
  8. package/biome.json +1 -1
  9. package/dist/chunk-23UPXDNL.js +3044 -0
  10. package/dist/chunk-2W7MO2DL.js +1366 -0
  11. package/dist/chunk-3NUAZGMA.js +1689 -0
  12. package/dist/chunk-7TOWB2XB.js +366 -0
  13. package/dist/chunk-7XOTOADQ.js +3065 -0
  14. package/dist/chunk-AH2PDM2K.js +3042 -0
  15. package/dist/chunk-BNXWSZ63.js +3742 -0
  16. package/dist/chunk-BTL5DJVU.js +3222 -0
  17. package/dist/chunk-HDHYG7E4.js +104 -0
  18. package/dist/chunk-HLR4KZBP.js +3234 -0
  19. package/dist/chunk-IP3FRFEB.js +1045 -0
  20. package/dist/chunk-KHU56VDO.js +3042 -0
  21. package/dist/chunk-KRYIFLQR.js +85 -89
  22. package/dist/chunk-LBSDNLEM.js +287 -0
  23. package/dist/chunk-MNTQ7HCP.js +2643 -0
  24. package/dist/chunk-MUJELQQ6.js +1387 -0
  25. package/dist/chunk-MXJGMSLV.js +2199 -0
  26. package/dist/chunk-N6QJGC3Z.js +2636 -0
  27. package/dist/chunk-OBELGBPM.js +1713 -0
  28. package/dist/chunk-OT7R5XTA.js +3192 -0
  29. package/dist/chunk-P7X4RA2T.js +106 -0
  30. package/dist/chunk-PIDUQNC2.js +3185 -0
  31. package/dist/chunk-POGCDIH4.js +3187 -0
  32. package/dist/chunk-PSIEOQGZ.js +3043 -0
  33. package/dist/chunk-PVRT3IHA.js +3238 -0
  34. package/dist/chunk-QNN4TT23.js +1430 -0
  35. package/dist/chunk-RE3R45RJ.js +3042 -0
  36. package/dist/chunk-S7E6TFX6.js +718 -657
  37. package/dist/chunk-SG6GLU4U.js +1378 -0
  38. package/dist/chunk-SJCDV2ST.js +274 -0
  39. package/dist/chunk-SYE5XLF3.js +104 -0
  40. package/dist/chunk-T5VLYBZD.js +103 -0
  41. package/dist/chunk-TOQB7VWU.js +3238 -0
  42. package/dist/chunk-VFNMZ4ZQ.js +3228 -0
  43. package/dist/chunk-VVTGZNBT.js +1533 -1423
  44. package/dist/chunk-W7Q4RFEV.js +104 -0
  45. package/dist/chunk-XTYYVRLO.js +3190 -0
  46. package/dist/chunk-Y6MDYVJD.js +3063 -0
  47. package/dist/cli/main.js +4072 -629
  48. package/dist/index.d.ts +420 -33
  49. package/dist/index.js +8 -15
  50. package/dist/mcp/server.js +103 -7
  51. package/dist/schema-BAWSG7KY.js +22 -0
  52. package/dist/schema-E3QUPL26.js +20 -0
  53. package/dist/schema-EHL7WUT6.js +20 -0
  54. package/docs/019-USAGE.md +44 -5
  55. package/docs/020-current-implementation.md +8 -8
  56. package/docs/021-DOGFOODING-FINDINGS.md +1 -1
  57. package/docs/CONFIG.md +1123 -0
  58. package/docs/ERRORS.md +383 -0
  59. package/docs/summarization.md +320 -0
  60. package/justfile +40 -0
  61. package/package.json +39 -33
  62. package/research/INDEX.md +315 -0
  63. package/research/code-review/README.md +90 -0
  64. package/research/code-review/cli-error-handling-review.md +979 -0
  65. package/research/code-review/code-review-validation-report.md +464 -0
  66. package/research/code-review/main-ts-review.md +1128 -0
  67. package/research/config-docs/SUMMARY.md +357 -0
  68. package/research/config-docs/TEST-RESULTS.md +776 -0
  69. package/research/config-docs/TODO.md +542 -0
  70. package/research/config-docs/analysis.md +744 -0
  71. package/research/config-docs/fix-validation.md +502 -0
  72. package/research/config-docs/help-audit.md +264 -0
  73. package/research/config-docs/help-system-analysis.md +890 -0
  74. package/research/frontmatter/COMMENTS-ARE-SKIPPED.md +149 -0
  75. package/research/frontmatter/LLM-CODE-NAVIGATION.md +276 -0
  76. package/research/issue-review.md +603 -0
  77. package/research/llm-summarization/agent-cli-tools-2026.md +1082 -0
  78. package/research/llm-summarization/alternative-providers-2026.md +1428 -0
  79. package/research/llm-summarization/anthropic-2026.md +367 -0
  80. package/research/llm-summarization/claude-cli-integration.md +1706 -0
  81. package/research/llm-summarization/cli-integration-patterns.md +3155 -0
  82. package/research/llm-summarization/openai-2026.md +473 -0
  83. package/research/llm-summarization/openai-compatible-providers-2026.md +1022 -0
  84. package/research/llm-summarization/opencode-cli-integration.md +1552 -0
  85. package/research/llm-summarization/prompt-engineering-2026.md +1426 -0
  86. package/research/llm-summarization/prototype-results.md +56 -0
  87. package/research/llm-summarization/provider-switching-patterns-2026.md +2153 -0
  88. package/research/llm-summarization/typescript-llm-libraries-2026.md +2436 -0
  89. package/research/mdcontext-pudding/00-EXECUTIVE-SUMMARY.md +282 -0
  90. package/research/mdcontext-pudding/01-index-embed.md +956 -0
  91. package/research/mdcontext-pudding/02-search-COMMANDS.md +142 -0
  92. package/research/mdcontext-pudding/02-search-SUMMARY.md +146 -0
  93. package/research/mdcontext-pudding/02-search.md +970 -0
  94. package/research/mdcontext-pudding/03-context.md +779 -0
  95. package/research/mdcontext-pudding/04-navigation-and-analytics.md +803 -0
  96. package/research/mdcontext-pudding/04-tree.md +704 -0
  97. package/research/mdcontext-pudding/05-config.md +1038 -0
  98. package/research/mdcontext-pudding/06-links-summary.txt +87 -0
  99. package/research/mdcontext-pudding/06-links.md +679 -0
  100. package/research/mdcontext-pudding/07-stats.md +693 -0
  101. package/research/mdcontext-pudding/BUG-FIX-PLAN.md +388 -0
  102. package/research/mdcontext-pudding/P0-BUG-VALIDATION.md +167 -0
  103. package/research/mdcontext-pudding/README.md +168 -0
  104. package/research/mdcontext-pudding/TESTING-SUMMARY.md +128 -0
  105. package/research/research-quality-review.md +834 -0
  106. package/research/semantic-search/embedding-text-analysis.md +156 -0
  107. package/research/semantic-search/multi-word-failure-reproduction.md +171 -0
  108. package/research/semantic-search/query-processing-analysis.md +207 -0
  109. package/research/semantic-search/root-cause-and-solution.md +114 -0
  110. package/research/semantic-search/threshold-validation-report.md +69 -0
  111. package/research/semantic-search/vector-search-analysis.md +63 -0
  112. package/research/test-path-issues.md +276 -0
  113. package/review/ALP-76/1-error-type-design.md +962 -0
  114. package/review/ALP-76/2-error-handling-patterns.md +906 -0
  115. package/review/ALP-76/3-error-presentation.md +624 -0
  116. package/review/ALP-76/4-test-coverage.md +625 -0
  117. package/review/ALP-76/5-migration-completeness.md +440 -0
  118. package/review/ALP-76/6-effect-best-practices.md +755 -0
  119. package/scripts/apply-branch-protection.sh +47 -0
  120. package/scripts/branch-protection-templates.json +79 -0
  121. package/scripts/prototype-summarization.ts +346 -0
  122. package/scripts/rebuild-hnswlib.js +32 -37
  123. package/scripts/setup-branch-protection.sh +64 -0
  124. package/src/__tests__/fixtures/semantic-search/multi-word-corpus/.mdcontext/active-provider.json +7 -0
  125. package/src/__tests__/fixtures/semantic-search/multi-word-corpus/.mdcontext/bm25.json +541 -0
  126. package/src/__tests__/fixtures/semantic-search/multi-word-corpus/.mdcontext/bm25.meta.json +5 -0
  127. package/src/__tests__/fixtures/semantic-search/multi-word-corpus/.mdcontext/config.json +8 -0
  128. package/src/__tests__/fixtures/semantic-search/multi-word-corpus/.mdcontext/embeddings/openai_text-embedding-3-small_512/vectors.bin +0 -0
  129. package/src/__tests__/fixtures/semantic-search/multi-word-corpus/.mdcontext/embeddings/openai_text-embedding-3-small_512/vectors.meta.bin +0 -0
  130. package/src/__tests__/fixtures/semantic-search/multi-word-corpus/.mdcontext/indexes/documents.json +60 -0
  131. package/src/__tests__/fixtures/semantic-search/multi-word-corpus/.mdcontext/indexes/links.json +13 -0
  132. package/src/__tests__/fixtures/semantic-search/multi-word-corpus/.mdcontext/indexes/sections.json +1197 -0
  133. package/src/__tests__/fixtures/semantic-search/multi-word-corpus/configuration-management.md +99 -0
  134. package/src/__tests__/fixtures/semantic-search/multi-word-corpus/distributed-systems.md +92 -0
  135. package/src/__tests__/fixtures/semantic-search/multi-word-corpus/error-handling.md +78 -0
  136. package/src/__tests__/fixtures/semantic-search/multi-word-corpus/failure-automation.md +55 -0
  137. package/src/__tests__/fixtures/semantic-search/multi-word-corpus/job-context.md +69 -0
  138. package/src/__tests__/fixtures/semantic-search/multi-word-corpus/process-orchestration.md +99 -0
  139. package/src/cli/argv-preprocessor.test.ts +2 -2
  140. package/src/cli/cli.test.ts +230 -33
  141. package/src/cli/commands/config-cmd.ts +642 -0
  142. package/src/cli/commands/context.ts +97 -9
  143. package/src/cli/commands/duplicates.ts +122 -0
  144. package/src/cli/commands/embeddings.ts +529 -0
  145. package/src/cli/commands/index-cmd.ts +210 -30
  146. package/src/cli/commands/index.ts +3 -0
  147. package/src/cli/commands/search.ts +894 -64
  148. package/src/cli/commands/stats.ts +3 -0
  149. package/src/cli/commands/tree.ts +26 -5
  150. package/src/cli/config-layer.ts +176 -0
  151. package/src/cli/error-handler.test.ts +235 -0
  152. package/src/cli/error-handler.ts +655 -0
  153. package/src/cli/flag-schemas.ts +66 -0
  154. package/src/cli/help.ts +209 -7
  155. package/src/cli/main.ts +348 -58
  156. package/src/cli/options.ts +10 -0
  157. package/src/cli/shared-error-handling.ts +199 -0
  158. package/src/cli/utils.ts +150 -17
  159. package/src/config/file-provider.test.ts +320 -0
  160. package/src/config/file-provider.ts +273 -0
  161. package/src/config/index.ts +72 -0
  162. package/src/config/integration.test.ts +667 -0
  163. package/src/config/precedence.test.ts +277 -0
  164. package/src/config/precedence.ts +451 -0
  165. package/src/config/schema.test.ts +414 -0
  166. package/src/config/schema.ts +603 -0
  167. package/src/config/service.test.ts +320 -0
  168. package/src/config/service.ts +243 -0
  169. package/src/config/testing.test.ts +264 -0
  170. package/src/config/testing.ts +110 -0
  171. package/src/core/types.ts +6 -33
  172. package/src/duplicates/detector.test.ts +183 -0
  173. package/src/duplicates/detector.ts +414 -0
  174. package/src/duplicates/index.ts +18 -0
  175. package/src/embeddings/embedding-namespace.test.ts +300 -0
  176. package/src/embeddings/embedding-namespace.ts +947 -0
  177. package/src/embeddings/heading-boost.test.ts +222 -0
  178. package/src/embeddings/hnsw-build-options.test.ts +198 -0
  179. package/src/embeddings/hyde.test.ts +272 -0
  180. package/src/embeddings/hyde.ts +264 -0
  181. package/src/embeddings/index.ts +2 -0
  182. package/src/embeddings/openai-provider.ts +332 -83
  183. package/src/embeddings/pricing.json +22 -0
  184. package/src/embeddings/provider-constants.ts +204 -0
  185. package/src/embeddings/provider-errors.test.ts +967 -0
  186. package/src/embeddings/provider-errors.ts +565 -0
  187. package/src/embeddings/provider-factory.test.ts +240 -0
  188. package/src/embeddings/provider-factory.ts +225 -0
  189. package/src/embeddings/provider-integration.test.ts +788 -0
  190. package/src/embeddings/query-preprocessing.test.ts +187 -0
  191. package/src/embeddings/semantic-search-threshold.test.ts +508 -0
  192. package/src/embeddings/semantic-search.ts +780 -93
  193. package/src/embeddings/types.ts +293 -16
  194. package/src/embeddings/vector-store.ts +486 -77
  195. package/src/embeddings/voyage-provider.ts +313 -0
  196. package/src/errors/errors.test.ts +845 -0
  197. package/src/errors/index.ts +533 -0
  198. package/src/index/ignore-patterns.test.ts +354 -0
  199. package/src/index/ignore-patterns.ts +305 -0
  200. package/src/index/indexer.ts +286 -48
  201. package/src/index/storage.ts +94 -30
  202. package/src/index/types.ts +40 -2
  203. package/src/index/watcher.ts +67 -9
  204. package/src/index.ts +22 -0
  205. package/src/integration/search-keyword.test.ts +678 -0
  206. package/src/mcp/server.ts +135 -6
  207. package/src/parser/parser.ts +18 -19
  208. package/src/parser/section-filter.test.ts +277 -0
  209. package/src/parser/section-filter.ts +125 -3
  210. package/src/search/__tests__/hybrid-search.test.ts +650 -0
  211. package/src/search/bm25-store.ts +366 -0
  212. package/src/search/cross-encoder.test.ts +253 -0
  213. package/src/search/cross-encoder.ts +406 -0
  214. package/src/search/fuzzy-search.test.ts +419 -0
  215. package/src/search/fuzzy-search.ts +273 -0
  216. package/src/search/hybrid-search.ts +448 -0
  217. package/src/search/path-matcher.test.ts +276 -0
  218. package/src/search/path-matcher.ts +33 -0
  219. package/src/search/searcher.test.ts +99 -1
  220. package/src/search/searcher.ts +189 -67
  221. package/src/search/wink-bm25.d.ts +30 -0
  222. package/src/summarization/cli-providers/claude.ts +202 -0
  223. package/src/summarization/cli-providers/detection.test.ts +273 -0
  224. package/src/summarization/cli-providers/detection.ts +118 -0
  225. package/src/summarization/cli-providers/index.ts +8 -0
  226. package/src/summarization/cost.test.ts +139 -0
  227. package/src/summarization/cost.ts +102 -0
  228. package/src/summarization/error-handler.test.ts +127 -0
  229. package/src/summarization/error-handler.ts +111 -0
  230. package/src/summarization/index.ts +102 -0
  231. package/src/summarization/pipeline.test.ts +498 -0
  232. package/src/summarization/pipeline.ts +231 -0
  233. package/src/summarization/prompts.test.ts +269 -0
  234. package/src/summarization/prompts.ts +133 -0
  235. package/src/summarization/provider-factory.test.ts +396 -0
  236. package/src/summarization/provider-factory.ts +178 -0
  237. package/src/summarization/types.ts +184 -0
  238. package/src/summarize/summarizer.ts +104 -35
  239. package/src/types/huggingface-transformers.d.ts +66 -0
  240. package/tests/fixtures/cli/.mdcontext/active-provider.json +7 -0
  241. package/tests/fixtures/cli/.mdcontext/embeddings/openai_text-embedding-3-small_512/vectors.bin +0 -0
  242. package/tests/fixtures/cli/.mdcontext/embeddings/openai_text-embedding-3-small_512/vectors.meta.bin +0 -0
  243. package/tests/fixtures/cli/.mdcontext/indexes/documents.json +4 -4
  244. package/tests/fixtures/cli/.mdcontext/indexes/sections.json +14 -0
  245. package/tests/integration/embed-index.test.ts +712 -0
  246. package/tests/integration/search-context.test.ts +469 -0
  247. package/tests/integration/search-semantic.test.ts +522 -0
  248. package/vitest.config.ts +1 -6
  249. package/AGENTS.md +0 -46
  250. package/tests/fixtures/cli/.mdcontext/vectors.bin +0 -0
  251. package/tests/fixtures/cli/.mdcontext/vectors.meta.json +0 -1264
@@ -1,85 +1,104 @@
1
1
  // src/utils/tokens.ts
2
2
  import { Effect } from "effect";
3
+
3
4
  var encoder = null;
4
5
  var getEncoder = Effect.gen(function* () {
5
- if (encoder === null) {
6
- const { get_encoding } = yield* Effect.promise(() => import("tiktoken"));
7
- encoder = get_encoding("cl100k_base");
8
- }
9
- return encoder;
10
- });
11
- var countTokens = (text) => Effect.gen(function* () {
12
- const enc = yield* getEncoder;
13
- const tokens = enc.encode(text);
14
- return tokens.length;
6
+ if (encoder === null) {
7
+ const { get_encoding } = yield* Effect.promise(() => import("tiktoken"));
8
+ encoder = get_encoding("cl100k_base");
9
+ }
10
+ return encoder;
15
11
  });
12
+ var countTokens = (text) =>
13
+ Effect.gen(function* () {
14
+ const enc = yield* getEncoder;
15
+ const tokens = enc.encode(text);
16
+ return tokens.length;
17
+ });
16
18
  var countTokensApprox = (text) => {
17
- if (text.length === 0) return 0;
18
- const cjkPattern = /[\u4e00-\u9fff\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af\u3400-\u4dbf]/g;
19
- const cjkMatches = text.match(cjkPattern) || [];
20
- const cjkCount = cjkMatches.length;
21
- const emojiPattern = /[\u{1F300}-\u{1F9FF}\u{2600}-\u{26FF}\u{2700}-\u{27BF}\u{1F600}-\u{1F64F}\u{1F680}-\u{1F6FF}\u{2300}-\u{23FF}\u{2190}-\u{21FF}\u{25A0}-\u{25FF}\u{2B00}-\u{2BFF}]/gu;
22
- const emojiMatches = text.match(emojiPattern) || [];
23
- const emojiCount = emojiMatches.length;
24
- const variationSelectorPattern = /[\uFE0E\uFE0F]/g;
25
- const variationMatches = text.match(variationSelectorPattern) || [];
26
- const variationCount = variationMatches.length;
27
- let workingText = text;
28
- const codeBlockMatches = text.match(/```[\s\S]*?```/g) || [];
29
- let codeBlockTokens = 0;
30
- for (const block of codeBlockMatches) {
31
- const hasLang = /^```\w+/.test(block);
32
- const overhead = hasLang ? 6 : 4;
33
- const content = block.replace(/^```\w*\n?/, "").replace(/\n?```$/, "");
34
- const contentNewlines = (content.match(/\n/g) || []).length;
35
- const contentTokens = content.length > 0 ? content.length / 2.5 : 0;
36
- codeBlockTokens += Math.max(
37
- overhead,
38
- overhead + contentNewlines + contentTokens
39
- );
40
- workingText = workingText.replace(block, "");
41
- }
42
- const inlineCodeMatches = workingText.match(/`[^`]+`/g) || [];
43
- let inlineCodeTokens = 0;
44
- for (const match of inlineCodeMatches) {
45
- const content = match.slice(1, -1);
46
- inlineCodeTokens += 2 + content.length / 2.5;
47
- workingText = workingText.replace(match, "");
48
- }
49
- const pathMatches = workingText.match(/(?:\/[\w.-]+)+/g) || [];
50
- let pathTokens = 0;
51
- for (const match of pathMatches) {
52
- const slashCount = (match.match(/\//g) || []).length;
53
- const contentLength = match.length - slashCount;
54
- pathTokens += slashCount + contentLength / 3.5;
55
- workingText = workingText.replace(match, "");
56
- }
57
- const punctuationMatches = workingText.match(/[!?,.:;'"()[\]{}@#$%^&*+=|\\<>~\-/]/g) || [];
58
- const punctuationCount = punctuationMatches.length;
59
- const proseNewlines = (workingText.match(/\n/g) || []).length;
60
- const proseLength = Math.max(
61
- 0,
62
- workingText.length - proseNewlines - cjkCount - emojiCount - variationCount - punctuationCount
63
- );
64
- const proseTokens = proseLength / 3.5;
65
- const proseNewlineTokens = proseNewlines * 1;
66
- const punctuationBonus = punctuationCount * 0.8;
67
- const cjkTokens = cjkCount * 1.2;
68
- const emojiTokens = emojiCount * 2.5;
69
- const variationTokens = variationCount * 1;
70
- const estimate = proseTokens + proseNewlineTokens + codeBlockTokens + inlineCodeTokens + pathTokens + punctuationBonus + cjkTokens + emojiTokens + variationTokens;
71
- return Math.ceil(estimate * 1.1);
19
+ if (text.length === 0) return 0;
20
+ const cjkPattern =
21
+ /[\u4e00-\u9fff\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af\u3400-\u4dbf]/g;
22
+ const cjkMatches = text.match(cjkPattern) || [];
23
+ const cjkCount = cjkMatches.length;
24
+ const emojiPattern =
25
+ /[\u{1F300}-\u{1F9FF}\u{2600}-\u{26FF}\u{2700}-\u{27BF}\u{1F600}-\u{1F64F}\u{1F680}-\u{1F6FF}\u{2300}-\u{23FF}\u{2190}-\u{21FF}\u{25A0}-\u{25FF}\u{2B00}-\u{2BFF}]/gu;
26
+ const emojiMatches = text.match(emojiPattern) || [];
27
+ const emojiCount = emojiMatches.length;
28
+ const variationSelectorPattern = /[\uFE0E\uFE0F]/g;
29
+ const variationMatches = text.match(variationSelectorPattern) || [];
30
+ const variationCount = variationMatches.length;
31
+ let workingText = text;
32
+ const codeBlockMatches = text.match(/```[\s\S]*?```/g) || [];
33
+ let codeBlockTokens = 0;
34
+ for (const block of codeBlockMatches) {
35
+ const hasLang = /^```\w+/.test(block);
36
+ const overhead = hasLang ? 6 : 4;
37
+ const content = block.replace(/^```\w*\n?/, "").replace(/\n?```$/, "");
38
+ const contentNewlines = (content.match(/\n/g) || []).length;
39
+ const contentTokens = content.length > 0 ? content.length / 2.5 : 0;
40
+ codeBlockTokens += Math.max(
41
+ overhead,
42
+ overhead + contentNewlines + contentTokens,
43
+ );
44
+ workingText = workingText.replace(block, "");
45
+ }
46
+ const inlineCodeMatches = workingText.match(/`[^`]+`/g) || [];
47
+ let inlineCodeTokens = 0;
48
+ for (const match of inlineCodeMatches) {
49
+ const content = match.slice(1, -1);
50
+ inlineCodeTokens += 2 + content.length / 2.5;
51
+ workingText = workingText.replace(match, "");
52
+ }
53
+ const pathMatches = workingText.match(/(?:\/[\w.-]+)+/g) || [];
54
+ let pathTokens = 0;
55
+ for (const match of pathMatches) {
56
+ const slashCount = (match.match(/\//g) || []).length;
57
+ const contentLength = match.length - slashCount;
58
+ pathTokens += slashCount + contentLength / 3.5;
59
+ workingText = workingText.replace(match, "");
60
+ }
61
+ const punctuationMatches =
62
+ workingText.match(/[!?,.:;'"()[\]{}@#$%^&*+=|\\<>~\-/]/g) || [];
63
+ const punctuationCount = punctuationMatches.length;
64
+ const proseNewlines = (workingText.match(/\n/g) || []).length;
65
+ const proseLength = Math.max(
66
+ 0,
67
+ workingText.length -
68
+ proseNewlines -
69
+ cjkCount -
70
+ emojiCount -
71
+ variationCount -
72
+ punctuationCount,
73
+ );
74
+ const proseTokens = proseLength / 3.5;
75
+ const proseNewlineTokens = proseNewlines * 1;
76
+ const punctuationBonus = punctuationCount * 0.8;
77
+ const cjkTokens = cjkCount * 1.2;
78
+ const emojiTokens = emojiCount * 2.5;
79
+ const variationTokens = variationCount * 1;
80
+ const estimate =
81
+ proseTokens +
82
+ proseNewlineTokens +
83
+ codeBlockTokens +
84
+ inlineCodeTokens +
85
+ pathTokens +
86
+ punctuationBonus +
87
+ cjkTokens +
88
+ emojiTokens +
89
+ variationTokens;
90
+ return Math.ceil(estimate * 1.1);
72
91
  };
73
92
  var countWords = (text) => {
74
- const trimmed = text.trim();
75
- if (trimmed.length === 0) return 0;
76
- return trimmed.split(/\s+/).length;
93
+ const trimmed = text.trim();
94
+ if (trimmed.length === 0) return 0;
95
+ return trimmed.split(/\s+/).length;
77
96
  };
78
97
  var freeEncoder = () => {
79
- if (encoder !== null) {
80
- encoder.free();
81
- encoder = null;
82
- }
98
+ if (encoder !== null) {
99
+ encoder.free();
100
+ encoder = null;
101
+ }
83
102
  };
84
103
 
85
104
  // src/parser/parser.ts
@@ -90,653 +109,695 @@ import remarkGfm from "remark-gfm";
90
109
  import remarkParse from "remark-parse";
91
110
  import { unified } from "unified";
92
111
  import { visit } from "unist-util-visit";
112
+
93
113
  var processor = unified().use(remarkParse).use(remarkGfm);
94
114
  var generateId = (input) => {
95
- return crypto.createHash("md5").update(input).digest("hex").slice(0, 12);
115
+ return crypto.createHash("md5").update(input).digest("hex").slice(0, 12);
96
116
  };
97
117
  var slugify = (text) => {
98
- return text.toLowerCase().replace(/[^\w\s-]/g, "").replace(/\s+/g, "-").replace(/-+/g, "-").trim();
118
+ return text
119
+ .toLowerCase()
120
+ .replace(/[^\w\s-]/g, "")
121
+ .replace(/\s+/g, "-")
122
+ .replace(/-+/g, "-")
123
+ .trim();
99
124
  };
100
125
  var isInternalLink = (href) => {
101
- if (href.startsWith("http://") || href.startsWith("https://")) return false;
102
- if (href.startsWith("mailto:")) return false;
103
- if (href.startsWith("#")) return true;
104
- if (href.endsWith(".md") || href.includes(".md#")) return true;
105
- return !href.includes("://");
126
+ if (href.startsWith("http://") || href.startsWith("https://")) return false;
127
+ if (href.startsWith("mailto:")) return false;
128
+ if (href.startsWith("#")) return true;
129
+ if (href.endsWith(".md") || href.includes(".md#")) return true;
130
+ return !href.includes("://");
106
131
  };
107
132
  var extractPlainText = (node) => {
108
- const texts = [];
109
- visit(node, "text", (textNode) => {
110
- texts.push(textNode.value);
111
- });
112
- return texts.join(" ");
133
+ const texts = [];
134
+ visit(node, "text", (textNode) => {
135
+ texts.push(textNode.value);
136
+ });
137
+ return texts.join(" ");
113
138
  };
114
139
  var getNodeEndLine = (node) => {
115
- return node?.position?.end?.line ?? 0;
140
+ return node?.position?.end?.line ?? 0;
116
141
  };
117
142
  var getNodeStartLine = (node) => {
118
- return node?.position?.start?.line ?? 0;
143
+ return node?.position?.start?.line ?? 0;
119
144
  };
120
145
  var extractRawSections = (tree) => {
121
- const sections = [];
122
- const headings = [];
123
- tree.children.forEach((node, index) => {
124
- if (node.type === "heading") {
125
- const heading = node;
126
- headings.push({
127
- heading: extractPlainText(heading),
128
- level: heading.depth,
129
- line: getNodeStartLine(node),
130
- index
131
- });
132
- }
133
- });
134
- headings.forEach((h, i) => {
135
- const nextHeading = headings[i + 1];
136
- const endIndex = nextHeading ? nextHeading.index : tree.children.length;
137
- const contentNodes = tree.children.slice(h.index + 1, endIndex);
138
- const lastContentNode = contentNodes[contentNodes.length - 1];
139
- const endLine = lastContentNode ? getNodeEndLine(lastContentNode) : h.line;
140
- sections.push({
141
- heading: h.heading,
142
- level: h.level,
143
- startLine: h.line,
144
- endLine,
145
- contentStartLine: h.line + 1,
146
- contentNodes
147
- });
148
- });
149
- return sections;
146
+ const sections = [];
147
+ const headings = [];
148
+ tree.children.forEach((node, index) => {
149
+ if (node.type === "heading") {
150
+ const heading = node;
151
+ headings.push({
152
+ heading: extractPlainText(heading),
153
+ level: heading.depth,
154
+ line: getNodeStartLine(node),
155
+ index,
156
+ });
157
+ }
158
+ });
159
+ headings.forEach((h, i) => {
160
+ const nextHeading = headings[i + 1];
161
+ const endIndex = nextHeading ? nextHeading.index : tree.children.length;
162
+ const contentNodes = tree.children.slice(h.index + 1, endIndex);
163
+ const lastContentNode = contentNodes[contentNodes.length - 1];
164
+ const endLine = lastContentNode ? getNodeEndLine(lastContentNode) : h.line;
165
+ sections.push({
166
+ heading: h.heading,
167
+ level: h.level,
168
+ startLine: h.line,
169
+ endLine,
170
+ contentStartLine: h.line + 1,
171
+ contentNodes,
172
+ });
173
+ });
174
+ return sections;
150
175
  };
151
176
  var buildSectionHierarchy = (rawSections, docId, lines) => {
152
- const result = [];
153
- const stack = [];
154
- for (const raw of rawSections) {
155
- const contentLines = lines.slice(raw.startLine - 1, raw.endLine);
156
- const content = contentLines.join("\n");
157
- const plainText = extractSectionPlainText(raw.contentNodes);
158
- const hasCode = raw.contentNodes.some(
159
- (n) => n.type === "code"
160
- );
161
- const hasList = raw.contentNodes.some(
162
- (n) => n.type === "list"
163
- );
164
- const hasTable = raw.contentNodes.some(
165
- (n) => n.type === "table"
166
- );
167
- const section = {
168
- id: `${docId}-${slugify(raw.heading)}`,
169
- heading: raw.heading,
170
- level: raw.level,
171
- content,
172
- plainText,
173
- startLine: raw.startLine,
174
- endLine: raw.endLine,
175
- children: [],
176
- metadata: {
177
- wordCount: countWords(plainText),
178
- tokenCount: countTokensApprox(content),
179
- hasCode,
180
- hasList,
181
- hasTable
182
- }
183
- };
184
- while (stack.length > 0 && stack[stack.length - 1].level >= raw.level) {
185
- stack.pop();
186
- }
187
- if (stack.length === 0) {
188
- result.push(section);
189
- } else {
190
- const parent = stack[stack.length - 1];
191
- parent.section.children.push(section);
192
- }
193
- stack.push({ section, level: raw.level });
194
- }
195
- return result;
177
+ const result = [];
178
+ const stack = [];
179
+ for (const raw of rawSections) {
180
+ const contentLines = lines.slice(raw.startLine - 1, raw.endLine);
181
+ const content = contentLines.join("\n");
182
+ const plainText = extractSectionPlainText(raw.contentNodes);
183
+ const hasCode = raw.contentNodes.some((n) => n.type === "code");
184
+ const hasList = raw.contentNodes.some((n) => n.type === "list");
185
+ const hasTable = raw.contentNodes.some((n) => n.type === "table");
186
+ const section = {
187
+ id: `${docId}-${slugify(raw.heading)}`,
188
+ heading: raw.heading,
189
+ level: raw.level,
190
+ content,
191
+ plainText,
192
+ startLine: raw.startLine,
193
+ endLine: raw.endLine,
194
+ children: [],
195
+ metadata: {
196
+ wordCount: countWords(plainText),
197
+ tokenCount: countTokensApprox(content),
198
+ hasCode,
199
+ hasList,
200
+ hasTable,
201
+ },
202
+ };
203
+ while (stack.length > 0 && stack[stack.length - 1].level >= raw.level) {
204
+ stack.pop();
205
+ }
206
+ if (stack.length === 0) {
207
+ result.push(section);
208
+ } else {
209
+ const parent = stack[stack.length - 1];
210
+ parent.section.children.push(section);
211
+ }
212
+ stack.push({ section, level: raw.level });
213
+ }
214
+ return result;
196
215
  };
197
216
  var extractSectionPlainText = (nodes) => {
198
- const texts = [];
199
- for (const node of nodes) {
200
- if ("value" in node && typeof node.value === "string") {
201
- texts.push(node.value);
202
- } else if ("children" in node) {
203
- texts.push(extractPlainText(node));
204
- }
205
- }
206
- return texts.join(" ");
217
+ const texts = [];
218
+ for (const node of nodes) {
219
+ if ("value" in node && typeof node.value === "string") {
220
+ texts.push(node.value);
221
+ } else if ("children" in node) {
222
+ texts.push(extractPlainText(node));
223
+ }
224
+ }
225
+ return texts.join(" ");
207
226
  };
208
227
  var countAllSections = (sections) => {
209
- let count = 0;
210
- for (const section of sections) {
211
- count += 1;
212
- count += countAllSections(section.children);
213
- }
214
- return count;
228
+ let count = 0;
229
+ for (const section of sections) {
230
+ count += 1;
231
+ count += countAllSections(section.children);
232
+ }
233
+ return count;
215
234
  };
216
235
  var extractLinks = (tree, docId) => {
217
- const links = [];
218
- let currentSectionId = docId;
219
- visit(tree, (node) => {
220
- if (node.type === "heading") {
221
- currentSectionId = `${docId}-${slugify(extractPlainText(node))}`;
222
- }
223
- if (node.type === "link") {
224
- const link = node;
225
- const internal = isInternalLink(link.url);
226
- links.push({
227
- type: internal ? "internal" : "external",
228
- href: link.url,
229
- text: extractPlainText(link),
230
- sectionId: currentSectionId,
231
- line: getNodeStartLine(node)
232
- });
233
- }
234
- if (node.type === "image") {
235
- const img = node;
236
- links.push({
237
- type: "image",
238
- href: img.url,
239
- text: img.alt ?? "",
240
- sectionId: currentSectionId,
241
- line: getNodeStartLine(node)
242
- });
243
- }
244
- });
245
- return links;
236
+ const links = [];
237
+ let currentSectionId = docId;
238
+ visit(tree, (node) => {
239
+ if (node.type === "heading") {
240
+ currentSectionId = `${docId}-${slugify(extractPlainText(node))}`;
241
+ }
242
+ if (node.type === "link") {
243
+ const link = node;
244
+ const internal = isInternalLink(link.url);
245
+ links.push({
246
+ type: internal ? "internal" : "external",
247
+ href: link.url,
248
+ text: extractPlainText(link),
249
+ sectionId: currentSectionId,
250
+ line: getNodeStartLine(node),
251
+ });
252
+ }
253
+ if (node.type === "image") {
254
+ const img = node;
255
+ links.push({
256
+ type: "image",
257
+ href: img.url,
258
+ text: img.alt ?? "",
259
+ sectionId: currentSectionId,
260
+ line: getNodeStartLine(node),
261
+ });
262
+ }
263
+ });
264
+ return links;
246
265
  };
247
266
  var extractCodeBlocks = (tree, docId) => {
248
- const codeBlocks = [];
249
- let currentSectionId = docId;
250
- visit(tree, (node) => {
251
- if (node.type === "heading") {
252
- currentSectionId = `${docId}-${slugify(extractPlainText(node))}`;
253
- }
254
- if (node.type === "code") {
255
- const code = node;
256
- codeBlocks.push({
257
- language: code.lang ?? null,
258
- content: code.value,
259
- sectionId: currentSectionId,
260
- startLine: getNodeStartLine(node),
261
- endLine: getNodeEndLine(node)
262
- });
263
- }
264
- });
265
- return codeBlocks;
267
+ const codeBlocks = [];
268
+ let currentSectionId = docId;
269
+ visit(tree, (node) => {
270
+ if (node.type === "heading") {
271
+ currentSectionId = `${docId}-${slugify(extractPlainText(node))}`;
272
+ }
273
+ if (node.type === "code") {
274
+ const code = node;
275
+ codeBlocks.push({
276
+ language: code.lang ?? null,
277
+ content: code.value,
278
+ sectionId: currentSectionId,
279
+ startLine: getNodeStartLine(node),
280
+ endLine: getNodeEndLine(node),
281
+ });
282
+ }
283
+ });
284
+ return codeBlocks;
266
285
  };
267
- var parse = (content, options = {}) => Effect2.gen(function* () {
268
- const path3 = options.path ?? "unknown";
269
- const docId = generateId(path3);
270
- const now = /* @__PURE__ */ new Date();
271
- let frontmatter = {};
272
- let markdownContent = content;
273
- try {
274
- const parsed = matter(content);
275
- frontmatter = parsed.data;
276
- markdownContent = parsed.content;
277
- } catch (error) {
278
- const msg = error instanceof Error ? error.message : String(error);
279
- console.warn(
280
- `Warning: Malformed frontmatter in ${path3}, skipping: ${msg.split("\n")[0]}`
281
- );
282
- }
283
- const tree = processor.parse(markdownContent);
284
- const lines = markdownContent.split("\n");
285
- const rawSections = extractRawSections(tree);
286
- const sections = buildSectionHierarchy(rawSections, docId, lines);
287
- const links = extractLinks(tree, docId);
288
- const codeBlocks = extractCodeBlocks(tree, docId);
289
- const firstH1 = sections.find((s) => s.level === 1);
290
- const title = firstH1?.heading ?? (typeof frontmatter.title === "string" ? frontmatter.title : null) ?? path3.split("/").pop()?.replace(/\.md$/, "") ?? "Untitled";
291
- const totalContent = sections.map((s) => s.content).join("\n");
292
- const metadata = {
293
- wordCount: countWords(totalContent),
294
- tokenCount: countTokensApprox(content),
295
- headingCount: countAllSections(sections),
296
- linkCount: links.length,
297
- codeBlockCount: codeBlocks.length,
298
- lastModified: options.lastModified ?? now,
299
- indexedAt: now
300
- };
301
- const document = {
302
- id: docId,
303
- path: path3,
304
- title,
305
- frontmatter,
306
- sections,
307
- links,
308
- codeBlocks,
309
- metadata
310
- };
311
- return document;
312
- });
313
- var parseFile = (filePath) => Effect2.gen(function* () {
314
- const fs3 = yield* Effect2.promise(() => import("fs/promises"));
315
- let content;
316
- let stats;
317
- try {
318
- ;
319
- [content, stats] = yield* Effect2.all([
320
- Effect2.promise(() => fs3.readFile(filePath, "utf-8")),
321
- Effect2.promise(() => fs3.stat(filePath))
322
- ]);
323
- } catch (error) {
324
- return yield* Effect2.fail({
325
- _tag: "IoError",
326
- message: error instanceof Error ? error.message : "Unknown error",
327
- path: filePath
328
- });
329
- }
330
- return yield* parse(content, {
331
- path: filePath,
332
- lastModified: stats.mtime
333
- });
334
- });
286
+ var parse = (content, options = {}) =>
287
+ Effect2.gen(function* () {
288
+ const path3 = options.path ?? "unknown";
289
+ const docId = generateId(path3);
290
+ const now = /* @__PURE__ */ new Date();
291
+ let frontmatter = {};
292
+ let markdownContent = content;
293
+ try {
294
+ const parsed = matter(content);
295
+ frontmatter = parsed.data;
296
+ markdownContent = parsed.content;
297
+ } catch (error) {
298
+ const msg = error instanceof Error ? error.message : String(error);
299
+ console.warn(
300
+ `Warning: Malformed frontmatter in ${path3}, skipping: ${msg.split("\n")[0]}`,
301
+ );
302
+ }
303
+ const tree = processor.parse(markdownContent);
304
+ const lines = markdownContent.split("\n");
305
+ const rawSections = extractRawSections(tree);
306
+ const sections = buildSectionHierarchy(rawSections, docId, lines);
307
+ const links = extractLinks(tree, docId);
308
+ const codeBlocks = extractCodeBlocks(tree, docId);
309
+ const firstH1 = sections.find((s) => s.level === 1);
310
+ const title =
311
+ firstH1?.heading ??
312
+ (typeof frontmatter.title === "string" ? frontmatter.title : null) ??
313
+ path3.split("/").pop()?.replace(/\.md$/, "") ??
314
+ "Untitled";
315
+ const totalContent = sections.map((s) => s.content).join("\n");
316
+ const metadata = {
317
+ wordCount: countWords(totalContent),
318
+ tokenCount: countTokensApprox(content),
319
+ headingCount: countAllSections(sections),
320
+ linkCount: links.length,
321
+ codeBlockCount: codeBlocks.length,
322
+ lastModified: options.lastModified ?? now,
323
+ indexedAt: now,
324
+ };
325
+ const document = {
326
+ id: docId,
327
+ path: path3,
328
+ title,
329
+ frontmatter,
330
+ sections,
331
+ links,
332
+ codeBlocks,
333
+ metadata,
334
+ };
335
+ return document;
336
+ });
337
+ var parseFile = (filePath) =>
338
+ Effect2.gen(function* () {
339
+ const fs3 = yield* Effect2.promise(() => import("fs/promises"));
340
+ let content;
341
+ let stats;
342
+ try {
343
+ [content, stats] = yield* Effect2.all([
344
+ Effect2.promise(() => fs3.readFile(filePath, "utf-8")),
345
+ Effect2.promise(() => fs3.stat(filePath)),
346
+ ]);
347
+ } catch (error) {
348
+ return yield* Effect2.fail({
349
+ _tag: "IoError",
350
+ message: error instanceof Error ? error.message : "Unknown error",
351
+ path: filePath,
352
+ });
353
+ }
354
+ return yield* parse(content, {
355
+ path: filePath,
356
+ lastModified: stats.mtime,
357
+ });
358
+ });
335
359
 
336
360
  // src/index/types.ts
337
361
  var INDEX_DIR = ".mdcontext";
338
362
  var INDEX_VERSION = 1;
339
363
  var getIndexPaths = (rootPath) => ({
340
- root: `${rootPath}/${INDEX_DIR}`,
341
- config: `${rootPath}/${INDEX_DIR}/config.json`,
342
- documents: `${rootPath}/${INDEX_DIR}/indexes/documents.json`,
343
- sections: `${rootPath}/${INDEX_DIR}/indexes/sections.json`,
344
- links: `${rootPath}/${INDEX_DIR}/indexes/links.json`,
345
- cache: `${rootPath}/${INDEX_DIR}/cache`,
346
- parsed: `${rootPath}/${INDEX_DIR}/cache/parsed`
364
+ root: `${rootPath}/${INDEX_DIR}`,
365
+ config: `${rootPath}/${INDEX_DIR}/config.json`,
366
+ documents: `${rootPath}/${INDEX_DIR}/indexes/documents.json`,
367
+ sections: `${rootPath}/${INDEX_DIR}/indexes/sections.json`,
368
+ links: `${rootPath}/${INDEX_DIR}/indexes/links.json`,
369
+ cache: `${rootPath}/${INDEX_DIR}/cache`,
370
+ parsed: `${rootPath}/${INDEX_DIR}/cache/parsed`,
347
371
  });
348
372
 
349
373
  // src/index/storage.ts
350
374
  import * as crypto2 from "crypto";
375
+ import { Effect as Effect3 } from "effect";
351
376
  import * as fs from "fs/promises";
352
377
  import * as path from "path";
353
- import { Effect as Effect3 } from "effect";
354
- var ensureDir = (dirPath) => Effect3.tryPromise({
355
- try: () => fs.mkdir(dirPath, { recursive: true }),
356
- catch: (e) => new Error(`Failed to create directory ${dirPath}: ${e}`)
357
- }).pipe(Effect3.map(() => void 0));
358
- var readJsonFile = (filePath) => Effect3.tryPromise({
359
- try: async () => {
360
- try {
361
- const content = await fs.readFile(filePath, "utf-8");
362
- return JSON.parse(content);
363
- } catch {
364
- return null;
365
- }
366
- },
367
- catch: (e) => new Error(`Failed to read ${filePath}: ${e}`)
368
- });
369
- var writeJsonFile = (filePath, data) => Effect3.gen(function* () {
370
- const dir = path.dirname(filePath);
371
- yield* ensureDir(dir);
372
- yield* Effect3.tryPromise({
373
- try: () => fs.writeFile(filePath, JSON.stringify(data, null, 2)),
374
- catch: (e) => new Error(`Failed to write ${filePath}: ${e}`)
375
- });
376
- });
378
+
379
+ var ensureDir = (dirPath) =>
380
+ Effect3.tryPromise({
381
+ try: () => fs.mkdir(dirPath, { recursive: true }),
382
+ catch: (e) => new Error(`Failed to create directory ${dirPath}: ${e}`),
383
+ }).pipe(Effect3.map(() => void 0));
384
+ var readJsonFile = (filePath) =>
385
+ Effect3.tryPromise({
386
+ try: async () => {
387
+ try {
388
+ const content = await fs.readFile(filePath, "utf-8");
389
+ return JSON.parse(content);
390
+ } catch {
391
+ return null;
392
+ }
393
+ },
394
+ catch: (e) => new Error(`Failed to read ${filePath}: ${e}`),
395
+ });
396
+ var writeJsonFile = (filePath, data) =>
397
+ Effect3.gen(function* () {
398
+ const dir = path.dirname(filePath);
399
+ yield* ensureDir(dir);
400
+ yield* Effect3.tryPromise({
401
+ try: () => fs.writeFile(filePath, JSON.stringify(data, null, 2)),
402
+ catch: (e) => new Error(`Failed to write ${filePath}: ${e}`),
403
+ });
404
+ });
377
405
  var computeHash = (content) => {
378
- return crypto2.createHash("sha256").update(content).digest("hex").slice(0, 16);
406
+ return crypto2
407
+ .createHash("sha256")
408
+ .update(content)
409
+ .digest("hex")
410
+ .slice(0, 16);
379
411
  };
380
412
  var createStorage = (rootPath) => ({
381
- rootPath: path.resolve(rootPath),
382
- paths: getIndexPaths(path.resolve(rootPath))
383
- });
384
- var initializeIndex = (storage) => Effect3.gen(function* () {
385
- yield* ensureDir(storage.paths.root);
386
- yield* ensureDir(storage.paths.parsed);
387
- yield* ensureDir(path.dirname(storage.paths.documents));
388
- const existingConfig = yield* loadConfig(storage);
389
- if (!existingConfig) {
390
- const config = {
391
- version: INDEX_VERSION,
392
- rootPath: storage.rootPath,
393
- include: ["**/*.md", "**/*.mdx"],
394
- exclude: ["**/node_modules/**", "**/.*/**"],
395
- createdAt: (/* @__PURE__ */ new Date()).toISOString(),
396
- updatedAt: (/* @__PURE__ */ new Date()).toISOString()
397
- };
398
- yield* saveConfig(storage, config);
399
- }
413
+ rootPath: path.resolve(rootPath),
414
+ paths: getIndexPaths(path.resolve(rootPath)),
400
415
  });
416
+ var initializeIndex = (storage) =>
417
+ Effect3.gen(function* () {
418
+ yield* ensureDir(storage.paths.root);
419
+ yield* ensureDir(storage.paths.parsed);
420
+ yield* ensureDir(path.dirname(storage.paths.documents));
421
+ const existingConfig = yield* loadConfig(storage);
422
+ if (!existingConfig) {
423
+ const config = {
424
+ version: INDEX_VERSION,
425
+ rootPath: storage.rootPath,
426
+ include: ["**/*.md", "**/*.mdx"],
427
+ exclude: ["**/node_modules/**", "**/.*/**"],
428
+ createdAt: /* @__PURE__ */ new Date().toISOString(),
429
+ updatedAt: /* @__PURE__ */ new Date().toISOString(),
430
+ };
431
+ yield* saveConfig(storage, config);
432
+ }
433
+ });
401
434
  var loadConfig = (storage) => readJsonFile(storage.paths.config);
402
- var saveConfig = (storage, config) => writeJsonFile(storage.paths.config, {
403
- ...config,
404
- updatedAt: (/* @__PURE__ */ new Date()).toISOString()
405
- });
435
+ var saveConfig = (storage, config) =>
436
+ writeJsonFile(storage.paths.config, {
437
+ ...config,
438
+ updatedAt: /* @__PURE__ */ new Date().toISOString(),
439
+ });
406
440
  var loadDocumentIndex = (storage) => readJsonFile(storage.paths.documents);
407
- var saveDocumentIndex = (storage, index) => writeJsonFile(storage.paths.documents, index);
441
+ var saveDocumentIndex = (storage, index) =>
442
+ writeJsonFile(storage.paths.documents, index);
408
443
  var createEmptyDocumentIndex = (rootPath) => ({
409
- version: INDEX_VERSION,
410
- rootPath,
411
- documents: {}
444
+ version: INDEX_VERSION,
445
+ rootPath,
446
+ documents: {},
412
447
  });
413
448
  var loadSectionIndex = (storage) => readJsonFile(storage.paths.sections);
414
- var saveSectionIndex = (storage, index) => writeJsonFile(storage.paths.sections, index);
449
+ var saveSectionIndex = (storage, index) =>
450
+ writeJsonFile(storage.paths.sections, index);
415
451
  var createEmptySectionIndex = () => ({
416
- version: INDEX_VERSION,
417
- sections: {},
418
- byHeading: {},
419
- byDocument: {}
452
+ version: INDEX_VERSION,
453
+ sections: {},
454
+ byHeading: {},
455
+ byDocument: {},
420
456
  });
421
457
  var loadLinkIndex = (storage) => readJsonFile(storage.paths.links);
422
- var saveLinkIndex = (storage, index) => writeJsonFile(storage.paths.links, index);
458
+ var saveLinkIndex = (storage, index) =>
459
+ writeJsonFile(storage.paths.links, index);
423
460
  var createEmptyLinkIndex = () => ({
424
- version: INDEX_VERSION,
425
- forward: {},
426
- backward: {},
427
- broken: []
428
- });
429
- var indexExists = (storage) => Effect3.tryPromise({
430
- try: async () => {
431
- try {
432
- await fs.access(storage.paths.config);
433
- return true;
434
- } catch {
435
- return false;
436
- }
437
- },
438
- catch: (e) => new Error(`Failed to check index existence: ${e}`)
461
+ version: INDEX_VERSION,
462
+ forward: {},
463
+ backward: {},
464
+ broken: [],
439
465
  });
466
+ var indexExists = (storage) =>
467
+ Effect3.tryPromise({
468
+ try: async () => {
469
+ try {
470
+ await fs.access(storage.paths.config);
471
+ return true;
472
+ } catch {
473
+ return false;
474
+ }
475
+ },
476
+ catch: (e) => new Error(`Failed to check index existence: ${e}`),
477
+ });
440
478
 
479
+ import { Effect as Effect4 } from "effect";
441
480
  // src/index/indexer.ts
442
481
  import * as fs2 from "fs/promises";
443
482
  import * as path2 from "path";
444
- import { Effect as Effect4 } from "effect";
445
- var isMarkdownFile = (filename) => filename.endsWith(".md") || filename.endsWith(".mdx");
483
+
484
+ var isMarkdownFile = (filename) =>
485
+ filename.endsWith(".md") || filename.endsWith(".mdx");
446
486
  var shouldExclude = (filePath, exclude) => {
447
- const normalized = filePath.toLowerCase();
448
- for (const pattern of exclude) {
449
- if (pattern.includes("node_modules") && normalized.includes("node_modules")) {
450
- return true;
451
- }
452
- if (pattern.startsWith("**/.*") && normalized.includes("/.")) {
453
- return true;
454
- }
455
- }
456
- return false;
487
+ const normalized = filePath.toLowerCase();
488
+ for (const pattern of exclude) {
489
+ if (
490
+ pattern.includes("node_modules") &&
491
+ normalized.includes("node_modules")
492
+ ) {
493
+ return true;
494
+ }
495
+ if (pattern.startsWith("**/.*") && normalized.includes("/.")) {
496
+ return true;
497
+ }
498
+ }
499
+ return false;
457
500
  };
458
501
  var walkDirectory = async (dir, exclude) => {
459
- const files = [];
460
- const entries = await fs2.readdir(dir, { withFileTypes: true });
461
- for (const entry of entries) {
462
- const fullPath = path2.join(dir, entry.name);
463
- if (entry.name.startsWith(".") || entry.name === "node_modules") {
464
- continue;
465
- }
466
- if (shouldExclude(fullPath, exclude)) {
467
- continue;
468
- }
469
- if (entry.isDirectory()) {
470
- const subFiles = await walkDirectory(fullPath, exclude);
471
- files.push(...subFiles);
472
- } else if (entry.isFile() && isMarkdownFile(entry.name)) {
473
- files.push(fullPath);
474
- }
475
- }
476
- return files;
502
+ const files = [];
503
+ const entries = await fs2.readdir(dir, { withFileTypes: true });
504
+ for (const entry of entries) {
505
+ const fullPath = path2.join(dir, entry.name);
506
+ if (entry.name.startsWith(".") || entry.name === "node_modules") {
507
+ continue;
508
+ }
509
+ if (shouldExclude(fullPath, exclude)) {
510
+ continue;
511
+ }
512
+ if (entry.isDirectory()) {
513
+ const subFiles = await walkDirectory(fullPath, exclude);
514
+ files.push(...subFiles);
515
+ } else if (entry.isFile() && isMarkdownFile(entry.name)) {
516
+ files.push(fullPath);
517
+ }
518
+ }
519
+ return files;
477
520
  };
478
521
  var flattenSections = (sections, docId, docPath) => {
479
- const result = [];
480
- const traverse = (section) => {
481
- result.push({
482
- id: section.id,
483
- documentId: docId,
484
- documentPath: docPath,
485
- heading: section.heading,
486
- level: section.level,
487
- startLine: section.startLine,
488
- endLine: section.endLine,
489
- tokenCount: section.metadata.tokenCount,
490
- hasCode: section.metadata.hasCode,
491
- hasList: section.metadata.hasList,
492
- hasTable: section.metadata.hasTable
493
- });
494
- for (const child of section.children) {
495
- traverse(child);
496
- }
497
- };
498
- for (const section of sections) {
499
- traverse(section);
500
- }
501
- return result;
522
+ const result = [];
523
+ const traverse = (section) => {
524
+ result.push({
525
+ id: section.id,
526
+ documentId: docId,
527
+ documentPath: docPath,
528
+ heading: section.heading,
529
+ level: section.level,
530
+ startLine: section.startLine,
531
+ endLine: section.endLine,
532
+ tokenCount: section.metadata.tokenCount,
533
+ hasCode: section.metadata.hasCode,
534
+ hasList: section.metadata.hasList,
535
+ hasTable: section.metadata.hasTable,
536
+ });
537
+ for (const child of section.children) {
538
+ traverse(child);
539
+ }
540
+ };
541
+ for (const section of sections) {
542
+ traverse(section);
543
+ }
544
+ return result;
502
545
  };
503
546
  var resolveInternalLink = (href, fromPath, rootPath) => {
504
- if (href.startsWith("#")) {
505
- return fromPath;
506
- }
507
- if (href.startsWith("http://") || href.startsWith("https://")) {
508
- return null;
509
- }
510
- const linkPath = href.split("#")[0] ?? "";
511
- if (!linkPath) return null;
512
- const fromDir = path2.dirname(fromPath);
513
- const resolved = path2.resolve(fromDir, linkPath);
514
- if (!resolved.startsWith(rootPath)) {
515
- return null;
516
- }
517
- return path2.relative(rootPath, resolved);
547
+ if (href.startsWith("#")) {
548
+ return fromPath;
549
+ }
550
+ if (href.startsWith("http://") || href.startsWith("https://")) {
551
+ return null;
552
+ }
553
+ const linkPath = href.split("#")[0] ?? "";
554
+ if (!linkPath) return null;
555
+ const fromDir = path2.dirname(fromPath);
556
+ const resolved = path2.resolve(fromDir, linkPath);
557
+ if (!resolved.startsWith(rootPath)) {
558
+ return null;
559
+ }
560
+ return path2.relative(rootPath, resolved);
518
561
  };
519
- var buildIndex = (rootPath, options = {}) => Effect4.gen(function* () {
520
- const startTime = Date.now();
521
- const storage = createStorage(rootPath);
522
- const errors = [];
523
- yield* initializeIndex(storage);
524
- const existingDocIndex = yield* loadDocumentIndex(storage);
525
- const docIndex = options.force || !existingDocIndex ? createEmptyDocumentIndex(storage.rootPath) : existingDocIndex;
526
- const existingSectionIndex = yield* loadSectionIndex(storage);
527
- const existingLinkIndex = yield* loadLinkIndex(storage);
528
- const sectionIndex = existingSectionIndex ?? createEmptySectionIndex();
529
- const linkIndex = existingLinkIndex ?? createEmptyLinkIndex();
530
- const exclude = options.exclude ?? ["**/node_modules/**", "**/.*/**"];
531
- const files = yield* Effect4.tryPromise({
532
- try: () => walkDirectory(storage.rootPath, exclude),
533
- catch: (e) => new Error(`Failed to walk directory: ${e}`)
534
- });
535
- let documentsIndexed = 0;
536
- let sectionsIndexed = 0;
537
- let linksIndexed = 0;
538
- const mutableDocuments = {
539
- ...docIndex.documents
540
- };
541
- const mutableSections = {
542
- ...sectionIndex.sections
543
- };
544
- const mutableByHeading = Object.fromEntries(
545
- Object.entries(sectionIndex.byHeading).map(([k, v]) => [k, [...v]])
546
- );
547
- const mutableByDocument = Object.fromEntries(
548
- Object.entries(sectionIndex.byDocument).map(([k, v]) => [k, [...v]])
549
- );
550
- const mutableForward = Object.fromEntries(
551
- Object.entries(linkIndex.forward).map(([k, v]) => [k, [...v]])
552
- );
553
- const mutableBackward = Object.fromEntries(
554
- Object.entries(linkIndex.backward).map(([k, v]) => [k, [...v]])
555
- );
556
- const brokenLinks = [...linkIndex.broken];
557
- for (const filePath of files) {
558
- const relativePath = path2.relative(storage.rootPath, filePath);
559
- const processFile = Effect4.gen(function* () {
560
- const [content, stats] = yield* Effect4.promise(
561
- () => Promise.all([fs2.readFile(filePath, "utf-8"), fs2.stat(filePath)])
562
- );
563
- const hash = computeHash(content);
564
- const existingEntry = mutableDocuments[relativePath];
565
- if (!options.force && existingEntry && existingEntry.hash === hash && existingEntry.mtime === stats.mtime.getTime()) {
566
- return;
567
- }
568
- const doc = yield* parse(content, {
569
- path: relativePath,
570
- lastModified: stats.mtime
571
- }).pipe(
572
- Effect4.mapError(
573
- (e) => new Error(`Parse error in ${relativePath}: ${e.message}`)
574
- )
575
- );
576
- if (existingEntry) {
577
- const oldSectionIds = mutableByDocument[existingEntry.id] ?? [];
578
- for (const sectionId of oldSectionIds) {
579
- const oldSection = mutableSections[sectionId];
580
- if (oldSection) {
581
- const headingKey = oldSection.heading.toLowerCase();
582
- const headingList = mutableByHeading[headingKey];
583
- if (headingList) {
584
- const idx = headingList.indexOf(sectionId);
585
- if (idx !== -1) headingList.splice(idx, 1);
586
- }
587
- }
588
- delete mutableSections[sectionId];
589
- }
590
- delete mutableByDocument[existingEntry.id];
591
- delete mutableForward[relativePath];
592
- }
593
- mutableDocuments[relativePath] = {
594
- id: doc.id,
595
- path: relativePath,
596
- title: doc.title,
597
- mtime: stats.mtime.getTime(),
598
- hash,
599
- tokenCount: doc.metadata.tokenCount,
600
- sectionCount: doc.metadata.headingCount
601
- };
602
- documentsIndexed++;
603
- const sections = flattenSections(doc.sections, doc.id, relativePath);
604
- mutableByDocument[doc.id] = [];
605
- for (const section of sections) {
606
- mutableSections[section.id] = section;
607
- mutableByDocument[doc.id]?.push(section.id);
608
- const headingKey = section.heading.toLowerCase();
609
- if (!mutableByHeading[headingKey]) {
610
- mutableByHeading[headingKey] = [];
611
- }
612
- mutableByHeading[headingKey]?.push(section.id);
613
- sectionsIndexed++;
614
- }
615
- const internalLinks = doc.links.filter((l) => l.type === "internal");
616
- const outgoingLinks = [];
617
- for (const link of internalLinks) {
618
- const target = resolveInternalLink(
619
- link.href,
620
- filePath,
621
- storage.rootPath
622
- );
623
- if (target) {
624
- outgoingLinks.push(target);
625
- if (!mutableBackward[target]) {
626
- mutableBackward[target] = [];
627
- }
628
- if (!mutableBackward[target]?.includes(relativePath)) {
629
- mutableBackward[target]?.push(relativePath);
630
- }
631
- linksIndexed++;
632
- }
633
- }
634
- mutableForward[relativePath] = outgoingLinks;
635
- }).pipe(
636
- Effect4.catchAll((error) => {
637
- errors.push({
638
- path: relativePath,
639
- message: error instanceof Error ? error.message : String(error)
640
- });
641
- return Effect4.void;
642
- })
643
- );
644
- yield* processFile;
645
- }
646
- for (const [_from, targets] of Object.entries(mutableForward)) {
647
- for (const target of targets) {
648
- if (!mutableDocuments[target] && !brokenLinks.includes(target)) {
649
- brokenLinks.push(target);
650
- }
651
- }
652
- }
653
- yield* saveDocumentIndex(storage, {
654
- version: docIndex.version,
655
- rootPath: storage.rootPath,
656
- documents: mutableDocuments
657
- });
658
- yield* saveSectionIndex(storage, {
659
- version: sectionIndex.version,
660
- sections: mutableSections,
661
- byHeading: mutableByHeading,
662
- byDocument: mutableByDocument
663
- });
664
- yield* saveLinkIndex(storage, {
665
- version: linkIndex.version,
666
- forward: mutableForward,
667
- backward: mutableBackward,
668
- broken: brokenLinks
669
- });
670
- const duration = Date.now() - startTime;
671
- const totalLinks = Object.values(mutableForward).reduce(
672
- (sum, links) => sum + links.length,
673
- 0
674
- );
675
- return {
676
- documentsIndexed,
677
- sectionsIndexed,
678
- linksIndexed,
679
- totalDocuments: Object.keys(mutableDocuments).length,
680
- totalSections: Object.keys(mutableSections).length,
681
- totalLinks,
682
- duration,
683
- errors
684
- };
685
- });
686
- var getOutgoingLinks = (rootPath, filePath) => Effect4.gen(function* () {
687
- const storage = createStorage(rootPath);
688
- const linkIndex = yield* loadLinkIndex(storage);
689
- if (!linkIndex) {
690
- return [];
691
- }
692
- const relativePath = path2.relative(storage.rootPath, path2.resolve(filePath));
693
- return linkIndex.forward[relativePath] ?? [];
694
- });
695
- var getIncomingLinks = (rootPath, filePath) => Effect4.gen(function* () {
696
- const storage = createStorage(rootPath);
697
- const linkIndex = yield* loadLinkIndex(storage);
698
- if (!linkIndex) {
699
- return [];
700
- }
701
- const relativePath = path2.relative(storage.rootPath, path2.resolve(filePath));
702
- return linkIndex.backward[relativePath] ?? [];
703
- });
704
- var getBrokenLinks = (rootPath) => Effect4.gen(function* () {
705
- const storage = createStorage(rootPath);
706
- const linkIndex = yield* loadLinkIndex(storage);
707
- if (!linkIndex) {
708
- return [];
709
- }
710
- return linkIndex.broken;
711
- });
562
+ var buildIndex = (rootPath, options = {}) =>
563
+ Effect4.gen(function* () {
564
+ const startTime = Date.now();
565
+ const storage = createStorage(rootPath);
566
+ const errors = [];
567
+ yield* initializeIndex(storage);
568
+ const existingDocIndex = yield* loadDocumentIndex(storage);
569
+ const docIndex =
570
+ options.force || !existingDocIndex
571
+ ? createEmptyDocumentIndex(storage.rootPath)
572
+ : existingDocIndex;
573
+ const existingSectionIndex = yield* loadSectionIndex(storage);
574
+ const existingLinkIndex = yield* loadLinkIndex(storage);
575
+ const sectionIndex = existingSectionIndex ?? createEmptySectionIndex();
576
+ const linkIndex = existingLinkIndex ?? createEmptyLinkIndex();
577
+ const exclude = options.exclude ?? ["**/node_modules/**", "**/.*/**"];
578
+ const files = yield* Effect4.tryPromise({
579
+ try: () => walkDirectory(storage.rootPath, exclude),
580
+ catch: (e) => new Error(`Failed to walk directory: ${e}`),
581
+ });
582
+ let documentsIndexed = 0;
583
+ let sectionsIndexed = 0;
584
+ let linksIndexed = 0;
585
+ const mutableDocuments = {
586
+ ...docIndex.documents,
587
+ };
588
+ const mutableSections = {
589
+ ...sectionIndex.sections,
590
+ };
591
+ const mutableByHeading = Object.fromEntries(
592
+ Object.entries(sectionIndex.byHeading).map(([k, v]) => [k, [...v]]),
593
+ );
594
+ const mutableByDocument = Object.fromEntries(
595
+ Object.entries(sectionIndex.byDocument).map(([k, v]) => [k, [...v]]),
596
+ );
597
+ const mutableForward = Object.fromEntries(
598
+ Object.entries(linkIndex.forward).map(([k, v]) => [k, [...v]]),
599
+ );
600
+ const mutableBackward = Object.fromEntries(
601
+ Object.entries(linkIndex.backward).map(([k, v]) => [k, [...v]]),
602
+ );
603
+ const brokenLinks = [...linkIndex.broken];
604
+ for (const filePath of files) {
605
+ const relativePath = path2.relative(storage.rootPath, filePath);
606
+ const processFile = Effect4.gen(function* () {
607
+ const [content, stats] = yield* Effect4.promise(() =>
608
+ Promise.all([fs2.readFile(filePath, "utf-8"), fs2.stat(filePath)]),
609
+ );
610
+ const hash = computeHash(content);
611
+ const existingEntry = mutableDocuments[relativePath];
612
+ if (
613
+ !options.force &&
614
+ existingEntry &&
615
+ existingEntry.hash === hash &&
616
+ existingEntry.mtime === stats.mtime.getTime()
617
+ ) {
618
+ return;
619
+ }
620
+ const doc = yield* parse(content, {
621
+ path: relativePath,
622
+ lastModified: stats.mtime,
623
+ }).pipe(
624
+ Effect4.mapError(
625
+ (e) => new Error(`Parse error in ${relativePath}: ${e.message}`),
626
+ ),
627
+ );
628
+ if (existingEntry) {
629
+ const oldSectionIds = mutableByDocument[existingEntry.id] ?? [];
630
+ for (const sectionId of oldSectionIds) {
631
+ const oldSection = mutableSections[sectionId];
632
+ if (oldSection) {
633
+ const headingKey = oldSection.heading.toLowerCase();
634
+ const headingList = mutableByHeading[headingKey];
635
+ if (headingList) {
636
+ const idx = headingList.indexOf(sectionId);
637
+ if (idx !== -1) headingList.splice(idx, 1);
638
+ }
639
+ }
640
+ delete mutableSections[sectionId];
641
+ }
642
+ delete mutableByDocument[existingEntry.id];
643
+ delete mutableForward[relativePath];
644
+ }
645
+ mutableDocuments[relativePath] = {
646
+ id: doc.id,
647
+ path: relativePath,
648
+ title: doc.title,
649
+ mtime: stats.mtime.getTime(),
650
+ hash,
651
+ tokenCount: doc.metadata.tokenCount,
652
+ sectionCount: doc.metadata.headingCount,
653
+ };
654
+ documentsIndexed++;
655
+ const sections = flattenSections(doc.sections, doc.id, relativePath);
656
+ mutableByDocument[doc.id] = [];
657
+ for (const section of sections) {
658
+ mutableSections[section.id] = section;
659
+ mutableByDocument[doc.id]?.push(section.id);
660
+ const headingKey = section.heading.toLowerCase();
661
+ if (!mutableByHeading[headingKey]) {
662
+ mutableByHeading[headingKey] = [];
663
+ }
664
+ mutableByHeading[headingKey]?.push(section.id);
665
+ sectionsIndexed++;
666
+ }
667
+ const internalLinks = doc.links.filter((l) => l.type === "internal");
668
+ const outgoingLinks = [];
669
+ for (const link of internalLinks) {
670
+ const target = resolveInternalLink(
671
+ link.href,
672
+ filePath,
673
+ storage.rootPath,
674
+ );
675
+ if (target) {
676
+ outgoingLinks.push(target);
677
+ if (!mutableBackward[target]) {
678
+ mutableBackward[target] = [];
679
+ }
680
+ if (!mutableBackward[target]?.includes(relativePath)) {
681
+ mutableBackward[target]?.push(relativePath);
682
+ }
683
+ linksIndexed++;
684
+ }
685
+ }
686
+ mutableForward[relativePath] = outgoingLinks;
687
+ }).pipe(
688
+ Effect4.catchAll((error) => {
689
+ errors.push({
690
+ path: relativePath,
691
+ message: error instanceof Error ? error.message : String(error),
692
+ });
693
+ return Effect4.void;
694
+ }),
695
+ );
696
+ yield* processFile;
697
+ }
698
+ for (const [_from, targets] of Object.entries(mutableForward)) {
699
+ for (const target of targets) {
700
+ if (!mutableDocuments[target] && !brokenLinks.includes(target)) {
701
+ brokenLinks.push(target);
702
+ }
703
+ }
704
+ }
705
+ yield* saveDocumentIndex(storage, {
706
+ version: docIndex.version,
707
+ rootPath: storage.rootPath,
708
+ documents: mutableDocuments,
709
+ });
710
+ yield* saveSectionIndex(storage, {
711
+ version: sectionIndex.version,
712
+ sections: mutableSections,
713
+ byHeading: mutableByHeading,
714
+ byDocument: mutableByDocument,
715
+ });
716
+ yield* saveLinkIndex(storage, {
717
+ version: linkIndex.version,
718
+ forward: mutableForward,
719
+ backward: mutableBackward,
720
+ broken: brokenLinks,
721
+ });
722
+ const duration = Date.now() - startTime;
723
+ const totalLinks = Object.values(mutableForward).reduce(
724
+ (sum, links) => sum + links.length,
725
+ 0,
726
+ );
727
+ return {
728
+ documentsIndexed,
729
+ sectionsIndexed,
730
+ linksIndexed,
731
+ totalDocuments: Object.keys(mutableDocuments).length,
732
+ totalSections: Object.keys(mutableSections).length,
733
+ totalLinks,
734
+ duration,
735
+ errors,
736
+ };
737
+ });
738
+ var getOutgoingLinks = (rootPath, filePath) =>
739
+ Effect4.gen(function* () {
740
+ const storage = createStorage(rootPath);
741
+ const linkIndex = yield* loadLinkIndex(storage);
742
+ if (!linkIndex) {
743
+ return [];
744
+ }
745
+ const relativePath = path2.relative(
746
+ storage.rootPath,
747
+ path2.resolve(filePath),
748
+ );
749
+ return linkIndex.forward[relativePath] ?? [];
750
+ });
751
+ var getIncomingLinks = (rootPath, filePath) =>
752
+ Effect4.gen(function* () {
753
+ const storage = createStorage(rootPath);
754
+ const linkIndex = yield* loadLinkIndex(storage);
755
+ if (!linkIndex) {
756
+ return [];
757
+ }
758
+ const relativePath = path2.relative(
759
+ storage.rootPath,
760
+ path2.resolve(filePath),
761
+ );
762
+ return linkIndex.backward[relativePath] ?? [];
763
+ });
764
+ var getBrokenLinks = (rootPath) =>
765
+ Effect4.gen(function* () {
766
+ const storage = createStorage(rootPath);
767
+ const linkIndex = yield* loadLinkIndex(storage);
768
+ if (!linkIndex) {
769
+ return [];
770
+ }
771
+ return linkIndex.broken;
772
+ });
712
773
 
713
774
  export {
714
- countTokens,
715
- countTokensApprox,
716
- countWords,
717
- freeEncoder,
718
- parse,
719
- parseFile,
720
- INDEX_DIR,
721
- INDEX_VERSION,
722
- getIndexPaths,
723
- computeHash,
724
- createStorage,
725
- initializeIndex,
726
- loadConfig,
727
- saveConfig,
728
- loadDocumentIndex,
729
- saveDocumentIndex,
730
- createEmptyDocumentIndex,
731
- loadSectionIndex,
732
- saveSectionIndex,
733
- createEmptySectionIndex,
734
- loadLinkIndex,
735
- saveLinkIndex,
736
- createEmptyLinkIndex,
737
- indexExists,
738
- buildIndex,
739
- getOutgoingLinks,
740
- getIncomingLinks,
741
- getBrokenLinks
775
+ countTokens,
776
+ countTokensApprox,
777
+ countWords,
778
+ freeEncoder,
779
+ parse,
780
+ parseFile,
781
+ INDEX_DIR,
782
+ INDEX_VERSION,
783
+ getIndexPaths,
784
+ computeHash,
785
+ createStorage,
786
+ initializeIndex,
787
+ loadConfig,
788
+ saveConfig,
789
+ loadDocumentIndex,
790
+ saveDocumentIndex,
791
+ createEmptyDocumentIndex,
792
+ loadSectionIndex,
793
+ saveSectionIndex,
794
+ createEmptySectionIndex,
795
+ loadLinkIndex,
796
+ saveLinkIndex,
797
+ createEmptyLinkIndex,
798
+ indexExists,
799
+ buildIndex,
800
+ getOutgoingLinks,
801
+ getIncomingLinks,
802
+ getBrokenLinks,
742
803
  };