mdcontext 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (251) hide show
  1. package/.changeset/config.json +9 -9
  2. package/.claude/settings.local.json +25 -0
  3. package/.github/workflows/claude-code-review.yml +44 -0
  4. package/.github/workflows/claude.yml +85 -0
  5. package/CONTRIBUTING.md +186 -0
  6. package/NOTES/NOTES +44 -0
  7. package/README.md +206 -3
  8. package/biome.json +1 -1
  9. package/dist/chunk-23UPXDNL.js +3044 -0
  10. package/dist/chunk-2W7MO2DL.js +1366 -0
  11. package/dist/chunk-3NUAZGMA.js +1689 -0
  12. package/dist/chunk-7TOWB2XB.js +366 -0
  13. package/dist/chunk-7XOTOADQ.js +3065 -0
  14. package/dist/chunk-AH2PDM2K.js +3042 -0
  15. package/dist/chunk-BNXWSZ63.js +3742 -0
  16. package/dist/chunk-BTL5DJVU.js +3222 -0
  17. package/dist/chunk-HDHYG7E4.js +104 -0
  18. package/dist/chunk-HLR4KZBP.js +3234 -0
  19. package/dist/chunk-IP3FRFEB.js +1045 -0
  20. package/dist/chunk-KHU56VDO.js +3042 -0
  21. package/dist/chunk-KRYIFLQR.js +85 -89
  22. package/dist/chunk-LBSDNLEM.js +287 -0
  23. package/dist/chunk-MNTQ7HCP.js +2643 -0
  24. package/dist/chunk-MUJELQQ6.js +1387 -0
  25. package/dist/chunk-MXJGMSLV.js +2199 -0
  26. package/dist/chunk-N6QJGC3Z.js +2636 -0
  27. package/dist/chunk-OBELGBPM.js +1713 -0
  28. package/dist/chunk-OT7R5XTA.js +3192 -0
  29. package/dist/chunk-P7X4RA2T.js +106 -0
  30. package/dist/chunk-PIDUQNC2.js +3185 -0
  31. package/dist/chunk-POGCDIH4.js +3187 -0
  32. package/dist/chunk-PSIEOQGZ.js +3043 -0
  33. package/dist/chunk-PVRT3IHA.js +3238 -0
  34. package/dist/chunk-QNN4TT23.js +1430 -0
  35. package/dist/chunk-RE3R45RJ.js +3042 -0
  36. package/dist/chunk-S7E6TFX6.js +718 -657
  37. package/dist/chunk-SG6GLU4U.js +1378 -0
  38. package/dist/chunk-SJCDV2ST.js +274 -0
  39. package/dist/chunk-SYE5XLF3.js +104 -0
  40. package/dist/chunk-T5VLYBZD.js +103 -0
  41. package/dist/chunk-TOQB7VWU.js +3238 -0
  42. package/dist/chunk-VFNMZ4ZQ.js +3228 -0
  43. package/dist/chunk-VVTGZNBT.js +1533 -1423
  44. package/dist/chunk-W7Q4RFEV.js +104 -0
  45. package/dist/chunk-XTYYVRLO.js +3190 -0
  46. package/dist/chunk-Y6MDYVJD.js +3063 -0
  47. package/dist/cli/main.js +4072 -629
  48. package/dist/index.d.ts +420 -33
  49. package/dist/index.js +8 -15
  50. package/dist/mcp/server.js +103 -7
  51. package/dist/schema-BAWSG7KY.js +22 -0
  52. package/dist/schema-E3QUPL26.js +20 -0
  53. package/dist/schema-EHL7WUT6.js +20 -0
  54. package/docs/019-USAGE.md +44 -5
  55. package/docs/020-current-implementation.md +8 -8
  56. package/docs/021-DOGFOODING-FINDINGS.md +1 -1
  57. package/docs/CONFIG.md +1123 -0
  58. package/docs/ERRORS.md +383 -0
  59. package/docs/summarization.md +320 -0
  60. package/justfile +40 -0
  61. package/package.json +39 -33
  62. package/research/INDEX.md +315 -0
  63. package/research/code-review/README.md +90 -0
  64. package/research/code-review/cli-error-handling-review.md +979 -0
  65. package/research/code-review/code-review-validation-report.md +464 -0
  66. package/research/code-review/main-ts-review.md +1128 -0
  67. package/research/config-docs/SUMMARY.md +357 -0
  68. package/research/config-docs/TEST-RESULTS.md +776 -0
  69. package/research/config-docs/TODO.md +542 -0
  70. package/research/config-docs/analysis.md +744 -0
  71. package/research/config-docs/fix-validation.md +502 -0
  72. package/research/config-docs/help-audit.md +264 -0
  73. package/research/config-docs/help-system-analysis.md +890 -0
  74. package/research/frontmatter/COMMENTS-ARE-SKIPPED.md +149 -0
  75. package/research/frontmatter/LLM-CODE-NAVIGATION.md +276 -0
  76. package/research/issue-review.md +603 -0
  77. package/research/llm-summarization/agent-cli-tools-2026.md +1082 -0
  78. package/research/llm-summarization/alternative-providers-2026.md +1428 -0
  79. package/research/llm-summarization/anthropic-2026.md +367 -0
  80. package/research/llm-summarization/claude-cli-integration.md +1706 -0
  81. package/research/llm-summarization/cli-integration-patterns.md +3155 -0
  82. package/research/llm-summarization/openai-2026.md +473 -0
  83. package/research/llm-summarization/openai-compatible-providers-2026.md +1022 -0
  84. package/research/llm-summarization/opencode-cli-integration.md +1552 -0
  85. package/research/llm-summarization/prompt-engineering-2026.md +1426 -0
  86. package/research/llm-summarization/prototype-results.md +56 -0
  87. package/research/llm-summarization/provider-switching-patterns-2026.md +2153 -0
  88. package/research/llm-summarization/typescript-llm-libraries-2026.md +2436 -0
  89. package/research/mdcontext-pudding/00-EXECUTIVE-SUMMARY.md +282 -0
  90. package/research/mdcontext-pudding/01-index-embed.md +956 -0
  91. package/research/mdcontext-pudding/02-search-COMMANDS.md +142 -0
  92. package/research/mdcontext-pudding/02-search-SUMMARY.md +146 -0
  93. package/research/mdcontext-pudding/02-search.md +970 -0
  94. package/research/mdcontext-pudding/03-context.md +779 -0
  95. package/research/mdcontext-pudding/04-navigation-and-analytics.md +803 -0
  96. package/research/mdcontext-pudding/04-tree.md +704 -0
  97. package/research/mdcontext-pudding/05-config.md +1038 -0
  98. package/research/mdcontext-pudding/06-links-summary.txt +87 -0
  99. package/research/mdcontext-pudding/06-links.md +679 -0
  100. package/research/mdcontext-pudding/07-stats.md +693 -0
  101. package/research/mdcontext-pudding/BUG-FIX-PLAN.md +388 -0
  102. package/research/mdcontext-pudding/P0-BUG-VALIDATION.md +167 -0
  103. package/research/mdcontext-pudding/README.md +168 -0
  104. package/research/mdcontext-pudding/TESTING-SUMMARY.md +128 -0
  105. package/research/research-quality-review.md +834 -0
  106. package/research/semantic-search/embedding-text-analysis.md +156 -0
  107. package/research/semantic-search/multi-word-failure-reproduction.md +171 -0
  108. package/research/semantic-search/query-processing-analysis.md +207 -0
  109. package/research/semantic-search/root-cause-and-solution.md +114 -0
  110. package/research/semantic-search/threshold-validation-report.md +69 -0
  111. package/research/semantic-search/vector-search-analysis.md +63 -0
  112. package/research/test-path-issues.md +276 -0
  113. package/review/ALP-76/1-error-type-design.md +962 -0
  114. package/review/ALP-76/2-error-handling-patterns.md +906 -0
  115. package/review/ALP-76/3-error-presentation.md +624 -0
  116. package/review/ALP-76/4-test-coverage.md +625 -0
  117. package/review/ALP-76/5-migration-completeness.md +440 -0
  118. package/review/ALP-76/6-effect-best-practices.md +755 -0
  119. package/scripts/apply-branch-protection.sh +47 -0
  120. package/scripts/branch-protection-templates.json +79 -0
  121. package/scripts/prototype-summarization.ts +346 -0
  122. package/scripts/rebuild-hnswlib.js +32 -37
  123. package/scripts/setup-branch-protection.sh +64 -0
  124. package/src/__tests__/fixtures/semantic-search/multi-word-corpus/.mdcontext/active-provider.json +7 -0
  125. package/src/__tests__/fixtures/semantic-search/multi-word-corpus/.mdcontext/bm25.json +541 -0
  126. package/src/__tests__/fixtures/semantic-search/multi-word-corpus/.mdcontext/bm25.meta.json +5 -0
  127. package/src/__tests__/fixtures/semantic-search/multi-word-corpus/.mdcontext/config.json +8 -0
  128. package/src/__tests__/fixtures/semantic-search/multi-word-corpus/.mdcontext/embeddings/openai_text-embedding-3-small_512/vectors.bin +0 -0
  129. package/src/__tests__/fixtures/semantic-search/multi-word-corpus/.mdcontext/embeddings/openai_text-embedding-3-small_512/vectors.meta.bin +0 -0
  130. package/src/__tests__/fixtures/semantic-search/multi-word-corpus/.mdcontext/indexes/documents.json +60 -0
  131. package/src/__tests__/fixtures/semantic-search/multi-word-corpus/.mdcontext/indexes/links.json +13 -0
  132. package/src/__tests__/fixtures/semantic-search/multi-word-corpus/.mdcontext/indexes/sections.json +1197 -0
  133. package/src/__tests__/fixtures/semantic-search/multi-word-corpus/configuration-management.md +99 -0
  134. package/src/__tests__/fixtures/semantic-search/multi-word-corpus/distributed-systems.md +92 -0
  135. package/src/__tests__/fixtures/semantic-search/multi-word-corpus/error-handling.md +78 -0
  136. package/src/__tests__/fixtures/semantic-search/multi-word-corpus/failure-automation.md +55 -0
  137. package/src/__tests__/fixtures/semantic-search/multi-word-corpus/job-context.md +69 -0
  138. package/src/__tests__/fixtures/semantic-search/multi-word-corpus/process-orchestration.md +99 -0
  139. package/src/cli/argv-preprocessor.test.ts +2 -2
  140. package/src/cli/cli.test.ts +230 -33
  141. package/src/cli/commands/config-cmd.ts +642 -0
  142. package/src/cli/commands/context.ts +97 -9
  143. package/src/cli/commands/duplicates.ts +122 -0
  144. package/src/cli/commands/embeddings.ts +529 -0
  145. package/src/cli/commands/index-cmd.ts +210 -30
  146. package/src/cli/commands/index.ts +3 -0
  147. package/src/cli/commands/search.ts +894 -64
  148. package/src/cli/commands/stats.ts +3 -0
  149. package/src/cli/commands/tree.ts +26 -5
  150. package/src/cli/config-layer.ts +176 -0
  151. package/src/cli/error-handler.test.ts +235 -0
  152. package/src/cli/error-handler.ts +655 -0
  153. package/src/cli/flag-schemas.ts +66 -0
  154. package/src/cli/help.ts +209 -7
  155. package/src/cli/main.ts +348 -58
  156. package/src/cli/options.ts +10 -0
  157. package/src/cli/shared-error-handling.ts +199 -0
  158. package/src/cli/utils.ts +150 -17
  159. package/src/config/file-provider.test.ts +320 -0
  160. package/src/config/file-provider.ts +273 -0
  161. package/src/config/index.ts +72 -0
  162. package/src/config/integration.test.ts +667 -0
  163. package/src/config/precedence.test.ts +277 -0
  164. package/src/config/precedence.ts +451 -0
  165. package/src/config/schema.test.ts +414 -0
  166. package/src/config/schema.ts +603 -0
  167. package/src/config/service.test.ts +320 -0
  168. package/src/config/service.ts +243 -0
  169. package/src/config/testing.test.ts +264 -0
  170. package/src/config/testing.ts +110 -0
  171. package/src/core/types.ts +6 -33
  172. package/src/duplicates/detector.test.ts +183 -0
  173. package/src/duplicates/detector.ts +414 -0
  174. package/src/duplicates/index.ts +18 -0
  175. package/src/embeddings/embedding-namespace.test.ts +300 -0
  176. package/src/embeddings/embedding-namespace.ts +947 -0
  177. package/src/embeddings/heading-boost.test.ts +222 -0
  178. package/src/embeddings/hnsw-build-options.test.ts +198 -0
  179. package/src/embeddings/hyde.test.ts +272 -0
  180. package/src/embeddings/hyde.ts +264 -0
  181. package/src/embeddings/index.ts +2 -0
  182. package/src/embeddings/openai-provider.ts +332 -83
  183. package/src/embeddings/pricing.json +22 -0
  184. package/src/embeddings/provider-constants.ts +204 -0
  185. package/src/embeddings/provider-errors.test.ts +967 -0
  186. package/src/embeddings/provider-errors.ts +565 -0
  187. package/src/embeddings/provider-factory.test.ts +240 -0
  188. package/src/embeddings/provider-factory.ts +225 -0
  189. package/src/embeddings/provider-integration.test.ts +788 -0
  190. package/src/embeddings/query-preprocessing.test.ts +187 -0
  191. package/src/embeddings/semantic-search-threshold.test.ts +508 -0
  192. package/src/embeddings/semantic-search.ts +780 -93
  193. package/src/embeddings/types.ts +293 -16
  194. package/src/embeddings/vector-store.ts +486 -77
  195. package/src/embeddings/voyage-provider.ts +313 -0
  196. package/src/errors/errors.test.ts +845 -0
  197. package/src/errors/index.ts +533 -0
  198. package/src/index/ignore-patterns.test.ts +354 -0
  199. package/src/index/ignore-patterns.ts +305 -0
  200. package/src/index/indexer.ts +286 -48
  201. package/src/index/storage.ts +94 -30
  202. package/src/index/types.ts +40 -2
  203. package/src/index/watcher.ts +67 -9
  204. package/src/index.ts +22 -0
  205. package/src/integration/search-keyword.test.ts +678 -0
  206. package/src/mcp/server.ts +135 -6
  207. package/src/parser/parser.ts +18 -19
  208. package/src/parser/section-filter.test.ts +277 -0
  209. package/src/parser/section-filter.ts +125 -3
  210. package/src/search/__tests__/hybrid-search.test.ts +650 -0
  211. package/src/search/bm25-store.ts +366 -0
  212. package/src/search/cross-encoder.test.ts +253 -0
  213. package/src/search/cross-encoder.ts +406 -0
  214. package/src/search/fuzzy-search.test.ts +419 -0
  215. package/src/search/fuzzy-search.ts +273 -0
  216. package/src/search/hybrid-search.ts +448 -0
  217. package/src/search/path-matcher.test.ts +276 -0
  218. package/src/search/path-matcher.ts +33 -0
  219. package/src/search/searcher.test.ts +99 -1
  220. package/src/search/searcher.ts +189 -67
  221. package/src/search/wink-bm25.d.ts +30 -0
  222. package/src/summarization/cli-providers/claude.ts +202 -0
  223. package/src/summarization/cli-providers/detection.test.ts +273 -0
  224. package/src/summarization/cli-providers/detection.ts +118 -0
  225. package/src/summarization/cli-providers/index.ts +8 -0
  226. package/src/summarization/cost.test.ts +139 -0
  227. package/src/summarization/cost.ts +102 -0
  228. package/src/summarization/error-handler.test.ts +127 -0
  229. package/src/summarization/error-handler.ts +111 -0
  230. package/src/summarization/index.ts +102 -0
  231. package/src/summarization/pipeline.test.ts +498 -0
  232. package/src/summarization/pipeline.ts +231 -0
  233. package/src/summarization/prompts.test.ts +269 -0
  234. package/src/summarization/prompts.ts +133 -0
  235. package/src/summarization/provider-factory.test.ts +396 -0
  236. package/src/summarization/provider-factory.ts +178 -0
  237. package/src/summarization/types.ts +184 -0
  238. package/src/summarize/summarizer.ts +104 -35
  239. package/src/types/huggingface-transformers.d.ts +66 -0
  240. package/tests/fixtures/cli/.mdcontext/active-provider.json +7 -0
  241. package/tests/fixtures/cli/.mdcontext/embeddings/openai_text-embedding-3-small_512/vectors.bin +0 -0
  242. package/tests/fixtures/cli/.mdcontext/embeddings/openai_text-embedding-3-small_512/vectors.meta.bin +0 -0
  243. package/tests/fixtures/cli/.mdcontext/indexes/documents.json +4 -4
  244. package/tests/fixtures/cli/.mdcontext/indexes/sections.json +14 -0
  245. package/tests/integration/embed-index.test.ts +712 -0
  246. package/tests/integration/search-context.test.ts +469 -0
  247. package/tests/integration/search-semantic.test.ts +522 -0
  248. package/vitest.config.ts +1 -6
  249. package/AGENTS.md +0 -46
  250. package/tests/fixtures/cli/.mdcontext/vectors.bin +0 -0
  251. package/tests/fixtures/cli/.mdcontext/vectors.meta.json +0 -1264
@@ -1,271 +1,305 @@
1
1
  import {
2
- INDEX_DIR,
3
- countTokensApprox,
4
- createStorage,
5
- loadDocumentIndex,
6
- loadSectionIndex,
7
- parseFile
2
+ countTokensApprox,
3
+ createStorage,
4
+ INDEX_DIR,
5
+ loadDocumentIndex,
6
+ loadSectionIndex,
7
+ parseFile,
8
8
  } from "./chunk-S7E6TFX6.js";
9
9
 
10
10
  // src/summarize/formatters.ts
11
11
  var formatSummary = (summary, options = {}) => {
12
- const maxTokens = options.maxTokens;
13
- const flatSections = [];
14
- const collectSections = (section, depth = 0, parentNumber = "", index = 0) => {
15
- const number = parentNumber ? `${parentNumber}.${index + 1}` : `${index + 1}`;
16
- flatSections.push({ section, depth, number });
17
- section.children.forEach((child, i) => {
18
- collectSections(child, depth + 1, number, i);
19
- });
20
- };
21
- summary.sections.forEach((section, i) => {
22
- collectSections(section, 0, "", i);
23
- });
24
- const buildOutput = (includedSectionIndices, truncationInfo, includeTopics2) => {
25
- const lines = [];
26
- if (truncationInfo.showWarning && truncationInfo.truncatedCount > 0 && truncationInfo.tokensTotal > 0) {
27
- const pct = Math.round(
28
- truncationInfo.tokensShown / truncationInfo.tokensTotal * 100
29
- );
30
- lines.push(
31
- `\u26A0\uFE0F Truncated: Showing ~${truncationInfo.tokensShown}/${truncationInfo.tokensTotal} tokens (${pct}%)`
32
- );
33
- if (truncationInfo.includedNumbers.length > 0) {
34
- const includedDisplay = truncationInfo.includedNumbers.length <= 6 ? truncationInfo.includedNumbers.join(", ") : truncationInfo.includedNumbers.slice(0, 5).join(", ") + `, ... (+${truncationInfo.includedNumbers.length - 5} more)`;
35
- lines.push(`Sections included: ${includedDisplay}`);
36
- }
37
- if (truncationInfo.excludedNumbers.length > 0) {
38
- const excludedDisplay = truncationInfo.excludedNumbers.length <= 6 ? truncationInfo.excludedNumbers.join(", ") : truncationInfo.excludedNumbers.slice(0, 5).join(", ") + `, ... (+${truncationInfo.excludedNumbers.length - 5} more)`;
39
- lines.push(`Sections excluded: ${excludedDisplay}`);
40
- }
41
- lines.push(
42
- "Use --full for complete content or --section to target specific sections."
43
- );
44
- lines.push("");
45
- }
46
- lines.push(`# ${summary.title}`);
47
- lines.push(`Path: ${summary.path}`);
48
- const tokenLineIndex = lines.length;
49
- lines.push("PLACEHOLDER");
50
- lines.push("");
51
- const fullTopicsLine2 = summary.keyTopics.length > 0 ? `**Topics:** ${summary.keyTopics.join(", ")}` : "";
52
- if (includeTopics2 && fullTopicsLine2) {
53
- lines.push(fullTopicsLine2);
54
- lines.push("");
55
- }
56
- const sectionLines = [];
57
- for (let i = 0; i < flatSections.length; i++) {
58
- if (!includedSectionIndices.has(i)) continue;
59
- const { section, depth } = flatSections[i];
60
- const indent = " ".repeat(depth);
61
- const prefix = "#".repeat(section.level);
62
- sectionLines.push(`${indent}${prefix} ${section.heading}`);
63
- if (section.summary) {
64
- sectionLines.push(`${indent}${section.summary}`);
65
- }
66
- }
67
- lines.push(sectionLines.join("\n"));
68
- const tempOutput = lines.join("\n");
69
- const tokensWithoutLine = countTokensApprox(
70
- tempOutput.replace("PLACEHOLDER", "")
71
- );
72
- let estimatedTotal = tokensWithoutLine + 8;
73
- for (let iter = 0; iter < 3; iter++) {
74
- const testTokenLine = `Tokens: ${estimatedTotal} (${(summary.compressionRatio * 100).toFixed(0)}% reduction from ${summary.originalTokens})`;
75
- const testOutput = tempOutput.replace("PLACEHOLDER", testTokenLine);
76
- const actualTotal = countTokensApprox(testOutput);
77
- if (actualTotal === estimatedTotal) break;
78
- estimatedTotal = actualTotal;
79
- }
80
- const finalTokenLine = `Tokens: ${estimatedTotal} (${(summary.compressionRatio * 100).toFixed(0)}% reduction from ${summary.originalTokens})`;
81
- lines[tokenLineIndex] = finalTokenLine;
82
- return lines.join("\n");
83
- };
84
- if (maxTokens === void 0) {
85
- const allIndices = new Set(flatSections.map((_, i) => i));
86
- const hasPriorTruncation = summary.truncated && summary.truncatedCount;
87
- return buildOutput(
88
- allIndices,
89
- {
90
- showWarning: !!hasPriorTruncation,
91
- truncatedCount: summary.truncatedCount ?? 0,
92
- includedNumbers: flatSections.map((s) => s.number),
93
- excludedNumbers: [],
94
- tokensShown: summary.summaryTokens,
95
- tokensTotal: summary.originalTokens
96
- },
97
- true
98
- );
99
- }
100
- const includedIndices = /* @__PURE__ */ new Set();
101
- let truncatedCount = 0;
102
- let includeTopics = true;
103
- const SAFETY_MARGIN = 1.15;
104
- const minHeaderTemplate = [
105
- `# ${summary.title}`,
106
- `Path: ${summary.path}`,
107
- `Tokens: 9999 (${(summary.compressionRatio * 100).toFixed(0)}% reduction from ${summary.originalTokens})`,
108
- "",
109
- ""
110
- ].join("\n");
111
- const minHeaderTokens = Math.ceil(
112
- countTokensApprox(minHeaderTemplate) * SAFETY_MARGIN
113
- );
114
- const fullTopicsLine = summary.keyTopics.length > 0 ? `**Topics:** ${summary.keyTopics.join(", ")}
115
- ` : "";
116
- const topicsTokens = fullTopicsLine ? Math.ceil(countTokensApprox(fullTopicsLine) * SAFETY_MARGIN) : 0;
117
- const truncationWarningTokens = Math.ceil(
118
- countTokensApprox(
119
- `\u26A0\uFE0F Truncated: Showing ~9999/9999 tokens (99%)
12
+ const maxTokens = options.maxTokens;
13
+ const flatSections = [];
14
+ const collectSections = (
15
+ section,
16
+ depth = 0,
17
+ parentNumber = "",
18
+ index = 0,
19
+ ) => {
20
+ const number = parentNumber
21
+ ? `${parentNumber}.${index + 1}`
22
+ : `${index + 1}`;
23
+ flatSections.push({ section, depth, number });
24
+ section.children.forEach((child, i) => {
25
+ collectSections(child, depth + 1, number, i);
26
+ });
27
+ };
28
+ summary.sections.forEach((section, i) => {
29
+ collectSections(section, 0, "", i);
30
+ });
31
+ const buildOutput = (
32
+ includedSectionIndices,
33
+ truncationInfo,
34
+ includeTopics2,
35
+ ) => {
36
+ const lines = [];
37
+ if (
38
+ truncationInfo.showWarning &&
39
+ truncationInfo.truncatedCount > 0 &&
40
+ truncationInfo.tokensTotal > 0
41
+ ) {
42
+ const pct = Math.round(
43
+ (truncationInfo.tokensShown / truncationInfo.tokensTotal) * 100,
44
+ );
45
+ lines.push(
46
+ `\u26A0\uFE0F Truncated: Showing ~${truncationInfo.tokensShown}/${truncationInfo.tokensTotal} tokens (${pct}%)`,
47
+ );
48
+ if (truncationInfo.includedNumbers.length > 0) {
49
+ const includedDisplay =
50
+ truncationInfo.includedNumbers.length <= 6
51
+ ? truncationInfo.includedNumbers.join(", ")
52
+ : truncationInfo.includedNumbers.slice(0, 5).join(", ") +
53
+ `, ... (+${truncationInfo.includedNumbers.length - 5} more)`;
54
+ lines.push(`Sections included: ${includedDisplay}`);
55
+ }
56
+ if (truncationInfo.excludedNumbers.length > 0) {
57
+ const excludedDisplay =
58
+ truncationInfo.excludedNumbers.length <= 6
59
+ ? truncationInfo.excludedNumbers.join(", ")
60
+ : truncationInfo.excludedNumbers.slice(0, 5).join(", ") +
61
+ `, ... (+${truncationInfo.excludedNumbers.length - 5} more)`;
62
+ lines.push(`Sections excluded: ${excludedDisplay}`);
63
+ }
64
+ lines.push(
65
+ "Use --full for complete content or --section to target specific sections.",
66
+ );
67
+ lines.push("");
68
+ }
69
+ lines.push(`# ${summary.title}`);
70
+ lines.push(`Path: ${summary.path}`);
71
+ const tokenLineIndex = lines.length;
72
+ lines.push("PLACEHOLDER");
73
+ lines.push("");
74
+ const fullTopicsLine2 =
75
+ summary.keyTopics.length > 0
76
+ ? `**Topics:** ${summary.keyTopics.join(", ")}`
77
+ : "";
78
+ if (includeTopics2 && fullTopicsLine2) {
79
+ lines.push(fullTopicsLine2);
80
+ lines.push("");
81
+ }
82
+ const sectionLines = [];
83
+ for (let i = 0; i < flatSections.length; i++) {
84
+ if (!includedSectionIndices.has(i)) continue;
85
+ const { section, depth } = flatSections[i];
86
+ const indent = " ".repeat(depth);
87
+ const prefix = "#".repeat(section.level);
88
+ sectionLines.push(`${indent}${prefix} ${section.heading}`);
89
+ if (section.summary) {
90
+ sectionLines.push(`${indent}${section.summary}`);
91
+ }
92
+ }
93
+ lines.push(sectionLines.join("\n"));
94
+ const tempOutput = lines.join("\n");
95
+ const tokensWithoutLine = countTokensApprox(
96
+ tempOutput.replace("PLACEHOLDER", ""),
97
+ );
98
+ let estimatedTotal = tokensWithoutLine + 8;
99
+ for (let iter = 0; iter < 3; iter++) {
100
+ const testTokenLine = `Tokens: ${estimatedTotal} (${(summary.compressionRatio * 100).toFixed(0)}% reduction from ${summary.originalTokens})`;
101
+ const testOutput = tempOutput.replace("PLACEHOLDER", testTokenLine);
102
+ const actualTotal = countTokensApprox(testOutput);
103
+ if (actualTotal === estimatedTotal) break;
104
+ estimatedTotal = actualTotal;
105
+ }
106
+ const finalTokenLine = `Tokens: ${estimatedTotal} (${(summary.compressionRatio * 100).toFixed(0)}% reduction from ${summary.originalTokens})`;
107
+ lines[tokenLineIndex] = finalTokenLine;
108
+ return lines.join("\n");
109
+ };
110
+ if (maxTokens === void 0) {
111
+ const allIndices = new Set(flatSections.map((_, i) => i));
112
+ const hasPriorTruncation = summary.truncated && summary.truncatedCount;
113
+ return buildOutput(
114
+ allIndices,
115
+ {
116
+ showWarning: !!hasPriorTruncation,
117
+ truncatedCount: summary.truncatedCount ?? 0,
118
+ includedNumbers: flatSections.map((s) => s.number),
119
+ excludedNumbers: [],
120
+ tokensShown: summary.summaryTokens,
121
+ tokensTotal: summary.originalTokens,
122
+ },
123
+ true,
124
+ );
125
+ }
126
+ const includedIndices = /* @__PURE__ */ new Set();
127
+ let truncatedCount = 0;
128
+ let includeTopics = true;
129
+ const SAFETY_MARGIN = 1.15;
130
+ const minHeaderTemplate = [
131
+ `# ${summary.title}`,
132
+ `Path: ${summary.path}`,
133
+ `Tokens: 9999 (${(summary.compressionRatio * 100).toFixed(0)}% reduction from ${summary.originalTokens})`,
134
+ "",
135
+ "",
136
+ ].join("\n");
137
+ const minHeaderTokens = Math.ceil(
138
+ countTokensApprox(minHeaderTemplate) * SAFETY_MARGIN,
139
+ );
140
+ const fullTopicsLine =
141
+ summary.keyTopics.length > 0
142
+ ? `**Topics:** ${summary.keyTopics.join(", ")}
143
+ `
144
+ : "";
145
+ const topicsTokens = fullTopicsLine
146
+ ? Math.ceil(countTokensApprox(fullTopicsLine) * SAFETY_MARGIN)
147
+ : 0;
148
+ const truncationWarningTokens = Math.ceil(
149
+ countTokensApprox(
150
+ `\u26A0\uFE0F Truncated: Showing ~9999/9999 tokens (99%)
120
151
  Sections included: 1, 2, 3, 4, 5, ... (+99 more)
121
152
  Sections excluded: 6, 7, 8, 9, 10, ... (+99 more)
122
153
  Use --full for complete content or --section to target specific sections.
123
- `
124
- ) * SAFETY_MARGIN
125
- );
126
- let headerTokens = minHeaderTokens + topicsTokens;
127
- if (headerTokens >= maxTokens) {
128
- includeTopics = false;
129
- headerTokens = minHeaderTokens;
130
- }
131
- let contentBudget = maxTokens - headerTokens - truncationWarningTokens;
132
- let tokensUsed = 0;
133
- for (let i = 0; i < flatSections.length; i++) {
134
- const { section, depth } = flatSections[i];
135
- const indent = " ".repeat(depth);
136
- const prefix = "#".repeat(section.level);
137
- const sectionContent = section.summary ? `${indent}${prefix} ${section.heading}
138
- ${indent}${section.summary}` : `${indent}${prefix} ${section.heading}`;
139
- const sectionTokens = Math.ceil(
140
- countTokensApprox(sectionContent) * SAFETY_MARGIN
141
- );
142
- if (tokensUsed + sectionTokens <= contentBudget) {
143
- includedIndices.add(i);
144
- tokensUsed += sectionTokens;
145
- } else {
146
- truncatedCount++;
147
- }
148
- }
149
- if (truncatedCount === 0) {
150
- contentBudget += truncationWarningTokens;
151
- }
152
- const includedNumbers = [];
153
- const excludedNumbers = [];
154
- for (let i = 0; i < flatSections.length; i++) {
155
- if (includedIndices.has(i)) {
156
- includedNumbers.push(flatSections[i].number);
157
- } else {
158
- excludedNumbers.push(flatSections[i].number);
159
- }
160
- }
161
- let tokensShown = 0;
162
- for (const idx of includedIndices) {
163
- tokensShown += flatSections[idx].section.summaryTokens;
164
- }
165
- let output = buildOutput(
166
- includedIndices,
167
- {
168
- showWarning: truncatedCount > 0,
169
- truncatedCount,
170
- includedNumbers,
171
- excludedNumbers,
172
- tokensShown,
173
- tokensTotal: summary.originalTokens
174
- },
175
- includeTopics
176
- );
177
- let actualTokens = countTokensApprox(output);
178
- const sortedIndices = Array.from(includedIndices).sort((a, b) => b - a);
179
- let removalIndex = 0;
180
- while (actualTokens > maxTokens && removalIndex < sortedIndices.length) {
181
- const indexToRemove = sortedIndices[removalIndex];
182
- includedIndices.delete(indexToRemove);
183
- truncatedCount++;
184
- removalIndex++;
185
- const removedNumber = flatSections[indexToRemove].number;
186
- const includedIdx = includedNumbers.indexOf(removedNumber);
187
- if (includedIdx !== -1) {
188
- includedNumbers.splice(includedIdx, 1);
189
- excludedNumbers.push(removedNumber);
190
- }
191
- tokensShown -= flatSections[indexToRemove].section.summaryTokens;
192
- output = buildOutput(
193
- includedIndices,
194
- {
195
- showWarning: true,
196
- truncatedCount,
197
- includedNumbers,
198
- excludedNumbers,
199
- tokensShown,
200
- tokensTotal: summary.originalTokens
201
- },
202
- includeTopics
203
- );
204
- actualTokens = countTokensApprox(output);
205
- }
206
- if (actualTokens > maxTokens && includeTopics) {
207
- includeTopics = false;
208
- output = buildOutput(
209
- includedIndices,
210
- {
211
- showWarning: truncatedCount > 0,
212
- truncatedCount,
213
- includedNumbers,
214
- excludedNumbers,
215
- tokensShown,
216
- tokensTotal: summary.originalTokens
217
- },
218
- includeTopics
219
- );
220
- actualTokens = countTokensApprox(output);
221
- }
222
- if (actualTokens > maxTokens && truncatedCount > 0) {
223
- output = buildOutput(
224
- includedIndices,
225
- {
226
- showWarning: false,
227
- truncatedCount,
228
- includedNumbers,
229
- excludedNumbers,
230
- tokensShown,
231
- tokensTotal: summary.originalTokens
232
- },
233
- includeTopics
234
- );
235
- actualTokens = countTokensApprox(output);
236
- }
237
- return output;
154
+ `,
155
+ ) * SAFETY_MARGIN,
156
+ );
157
+ let headerTokens = minHeaderTokens + topicsTokens;
158
+ if (headerTokens >= maxTokens) {
159
+ includeTopics = false;
160
+ headerTokens = minHeaderTokens;
161
+ }
162
+ let contentBudget = maxTokens - headerTokens - truncationWarningTokens;
163
+ let tokensUsed = 0;
164
+ for (let i = 0; i < flatSections.length; i++) {
165
+ const { section, depth } = flatSections[i];
166
+ const indent = " ".repeat(depth);
167
+ const prefix = "#".repeat(section.level);
168
+ const sectionContent = section.summary
169
+ ? `${indent}${prefix} ${section.heading}
170
+ ${indent}${section.summary}`
171
+ : `${indent}${prefix} ${section.heading}`;
172
+ const sectionTokens = Math.ceil(
173
+ countTokensApprox(sectionContent) * SAFETY_MARGIN,
174
+ );
175
+ if (tokensUsed + sectionTokens <= contentBudget) {
176
+ includedIndices.add(i);
177
+ tokensUsed += sectionTokens;
178
+ } else {
179
+ truncatedCount++;
180
+ }
181
+ }
182
+ if (truncatedCount === 0) {
183
+ contentBudget += truncationWarningTokens;
184
+ }
185
+ const includedNumbers = [];
186
+ const excludedNumbers = [];
187
+ for (let i = 0; i < flatSections.length; i++) {
188
+ if (includedIndices.has(i)) {
189
+ includedNumbers.push(flatSections[i].number);
190
+ } else {
191
+ excludedNumbers.push(flatSections[i].number);
192
+ }
193
+ }
194
+ let tokensShown = 0;
195
+ for (const idx of includedIndices) {
196
+ tokensShown += flatSections[idx].section.summaryTokens;
197
+ }
198
+ let output = buildOutput(
199
+ includedIndices,
200
+ {
201
+ showWarning: truncatedCount > 0,
202
+ truncatedCount,
203
+ includedNumbers,
204
+ excludedNumbers,
205
+ tokensShown,
206
+ tokensTotal: summary.originalTokens,
207
+ },
208
+ includeTopics,
209
+ );
210
+ let actualTokens = countTokensApprox(output);
211
+ const sortedIndices = Array.from(includedIndices).sort((a, b) => b - a);
212
+ let removalIndex = 0;
213
+ while (actualTokens > maxTokens && removalIndex < sortedIndices.length) {
214
+ const indexToRemove = sortedIndices[removalIndex];
215
+ includedIndices.delete(indexToRemove);
216
+ truncatedCount++;
217
+ removalIndex++;
218
+ const removedNumber = flatSections[indexToRemove].number;
219
+ const includedIdx = includedNumbers.indexOf(removedNumber);
220
+ if (includedIdx !== -1) {
221
+ includedNumbers.splice(includedIdx, 1);
222
+ excludedNumbers.push(removedNumber);
223
+ }
224
+ tokensShown -= flatSections[indexToRemove].section.summaryTokens;
225
+ output = buildOutput(
226
+ includedIndices,
227
+ {
228
+ showWarning: true,
229
+ truncatedCount,
230
+ includedNumbers,
231
+ excludedNumbers,
232
+ tokensShown,
233
+ tokensTotal: summary.originalTokens,
234
+ },
235
+ includeTopics,
236
+ );
237
+ actualTokens = countTokensApprox(output);
238
+ }
239
+ if (actualTokens > maxTokens && includeTopics) {
240
+ includeTopics = false;
241
+ output = buildOutput(
242
+ includedIndices,
243
+ {
244
+ showWarning: truncatedCount > 0,
245
+ truncatedCount,
246
+ includedNumbers,
247
+ excludedNumbers,
248
+ tokensShown,
249
+ tokensTotal: summary.originalTokens,
250
+ },
251
+ includeTopics,
252
+ );
253
+ actualTokens = countTokensApprox(output);
254
+ }
255
+ if (actualTokens > maxTokens && truncatedCount > 0) {
256
+ output = buildOutput(
257
+ includedIndices,
258
+ {
259
+ showWarning: false,
260
+ truncatedCount,
261
+ includedNumbers,
262
+ excludedNumbers,
263
+ tokensShown,
264
+ tokensTotal: summary.originalTokens,
265
+ },
266
+ includeTopics,
267
+ );
268
+ actualTokens = countTokensApprox(output);
269
+ }
270
+ return output;
238
271
  };
239
272
  var formatAssembledContext = (context) => {
240
- const lines = [];
241
- lines.push("# Context Assembly");
242
- lines.push(`Total tokens: ${context.totalTokens}/${context.budget}`);
243
- lines.push(`Sources: ${context.sources.length}`);
244
- lines.push("");
245
- for (const source of context.sources) {
246
- lines.push("---");
247
- lines.push("");
248
- lines.push(source.content);
249
- }
250
- if (context.overflow.length > 0) {
251
- lines.push("---");
252
- lines.push("");
253
- lines.push("## Overflow (not included due to budget)");
254
- for (const overflowPath of context.overflow) {
255
- lines.push(`- ${overflowPath}`);
256
- }
257
- }
258
- return lines.join("\n");
273
+ const lines = [];
274
+ lines.push("# Context Assembly");
275
+ lines.push(`Total tokens: ${context.totalTokens}/${context.budget}`);
276
+ lines.push(`Sources: ${context.sources.length}`);
277
+ lines.push("");
278
+ for (const source of context.sources) {
279
+ lines.push("---");
280
+ lines.push("");
281
+ lines.push(source.content);
282
+ }
283
+ if (context.overflow.length > 0) {
284
+ lines.push("---");
285
+ lines.push("");
286
+ lines.push("## Overflow (not included due to budget)");
287
+ for (const overflowPath of context.overflow) {
288
+ lines.push(`- ${overflowPath}`);
289
+ }
290
+ }
291
+ return lines.join("\n");
259
292
  };
260
293
 
294
+ import { Effect } from "effect";
261
295
  // src/summarize/summarizer.ts
262
296
  import * as fs from "fs/promises";
263
297
  import * as path from "path";
264
- import { Effect } from "effect";
298
+
265
299
  var TOKEN_BUDGETS = {
266
- brief: 100,
267
- summary: 500,
268
- full: Infinity
300
+ brief: 100,
301
+ summary: 500,
302
+ full: Infinity,
269
303
  };
270
304
  var MIN_SENTENCE_LENGTH = 10;
271
305
  var SENTENCE_SCORE_DEFINITION = 2;
@@ -283,1237 +317,1313 @@ var MAX_TOPIC_LENGTH = 50;
283
317
  var MAX_TOPICS = 10;
284
318
  var MIN_PARTIAL_BUDGET = 50;
285
319
  var extractKeyPoints = (content, maxSentences) => {
286
- const sentences = content.replace(/\n+/g, " ").split(/(?<=[.!?])\s+/).filter((s) => s.trim().length > MIN_SENTENCE_LENGTH);
287
- if (sentences.length <= maxSentences) {
288
- return sentences;
289
- }
290
- const scored = sentences.map((s) => {
291
- let score = 0;
292
- if (s.includes(":")) score += SENTENCE_SCORE_DEFINITION;
293
- if (/^[A-Z]/.test(s)) score += SENTENCE_SCORE_PROPER_START;
294
- if (s.length > SENTENCE_LENGTH_MIN && s.length < SENTENCE_LENGTH_MAX)
295
- score += SENTENCE_SCORE_MEDIUM_LENGTH;
296
- if (/\*\*|`/.test(s)) score += SENTENCE_SCORE_EMPHASIS;
297
- return { sentence: s, score };
298
- });
299
- scored.sort((a, b) => b.score - a.score);
300
- return scored.slice(0, maxSentences).map((s) => s.sentence);
320
+ const sentences = content
321
+ .replace(/\n+/g, " ")
322
+ .split(/(?<=[.!?])\s+/)
323
+ .filter((s) => s.trim().length > MIN_SENTENCE_LENGTH);
324
+ if (sentences.length <= maxSentences) {
325
+ return sentences;
326
+ }
327
+ const scored = sentences.map((s) => {
328
+ let score = 0;
329
+ if (s.includes(":")) score += SENTENCE_SCORE_DEFINITION;
330
+ if (/^[A-Z]/.test(s)) score += SENTENCE_SCORE_PROPER_START;
331
+ if (s.length > SENTENCE_LENGTH_MIN && s.length < SENTENCE_LENGTH_MAX)
332
+ score += SENTENCE_SCORE_MEDIUM_LENGTH;
333
+ if (/\*\*|`/.test(s)) score += SENTENCE_SCORE_EMPHASIS;
334
+ return { sentence: s, score };
335
+ });
336
+ scored.sort((a, b) => b.score - a.score);
337
+ return scored.slice(0, maxSentences).map((s) => s.sentence);
301
338
  };
302
339
  var summarizeSection = (section, level) => {
303
- const originalTokens = section.metadata.tokenCount;
304
- const children = section.children.map(
305
- (child) => summarizeSection(child, level)
306
- );
307
- const targetTokens = Math.min(
308
- TOKEN_BUDGETS[level],
309
- Math.max(originalTokens * SUMMARY_COMPRESSION_RATIO, MIN_SECTION_TOKENS)
310
- );
311
- let summary;
312
- if (level === "full" || originalTokens <= targetTokens) {
313
- summary = section.plainText;
314
- } else if (level === "brief") {
315
- const meta = [];
316
- if (section.metadata.hasCode) meta.push("code");
317
- if (section.metadata.hasList) meta.push("list");
318
- if (section.metadata.hasTable) meta.push("table");
319
- summary = meta.length > 0 ? `[${meta.join(", ")}]` : "";
320
- } else {
321
- const maxSentences = Math.max(
322
- MIN_SUMMARY_SENTENCES,
323
- Math.floor(targetTokens / TOKENS_PER_SENTENCE_ESTIMATE)
324
- );
325
- const keyPoints = extractKeyPoints(section.plainText, maxSentences);
326
- if (keyPoints.length > 0) {
327
- summary = keyPoints.join(" ");
328
- } else {
329
- const words = section.plainText.split(/\s+/).slice(0, targetTokens);
330
- summary = words.join(" ") + (words.length < section.plainText.split(/\s+/).length ? "..." : "");
331
- }
332
- }
333
- const summaryTokens = countTokensApprox(summary);
334
- return {
335
- heading: section.heading,
336
- level: section.level,
337
- originalTokens,
338
- summaryTokens,
339
- summary,
340
- children,
341
- hasCode: section.metadata.hasCode,
342
- hasList: section.metadata.hasList,
343
- hasTable: section.metadata.hasTable
344
- };
340
+ const originalTokens = section.metadata.tokenCount;
341
+ const children = section.children.map((child) =>
342
+ summarizeSection(child, level),
343
+ );
344
+ const targetTokens = Math.min(
345
+ TOKEN_BUDGETS[level],
346
+ Math.max(originalTokens * SUMMARY_COMPRESSION_RATIO, MIN_SECTION_TOKENS),
347
+ );
348
+ let summary;
349
+ if (level === "full" || originalTokens <= targetTokens) {
350
+ summary = section.plainText;
351
+ } else if (level === "brief") {
352
+ const meta = [];
353
+ if (section.metadata.hasCode) meta.push("code");
354
+ if (section.metadata.hasList) meta.push("list");
355
+ if (section.metadata.hasTable) meta.push("table");
356
+ summary = meta.length > 0 ? `[${meta.join(", ")}]` : "";
357
+ } else {
358
+ const maxSentences = Math.max(
359
+ MIN_SUMMARY_SENTENCES,
360
+ Math.floor(targetTokens / TOKENS_PER_SENTENCE_ESTIMATE),
361
+ );
362
+ const keyPoints = extractKeyPoints(section.plainText, maxSentences);
363
+ if (keyPoints.length > 0) {
364
+ summary = keyPoints.join(" ");
365
+ } else {
366
+ const words = section.plainText.split(/\s+/).slice(0, targetTokens);
367
+ summary =
368
+ words.join(" ") +
369
+ (words.length < section.plainText.split(/\s+/).length ? "..." : "");
370
+ }
371
+ }
372
+ const summaryTokens = countTokensApprox(summary);
373
+ return {
374
+ heading: section.heading,
375
+ level: section.level,
376
+ originalTokens,
377
+ summaryTokens,
378
+ summary,
379
+ children,
380
+ hasCode: section.metadata.hasCode,
381
+ hasList: section.metadata.hasList,
382
+ hasTable: section.metadata.hasTable,
383
+ };
345
384
  };
346
385
  var extractTopics = (document) => {
347
- const topics = /* @__PURE__ */ new Set();
348
- const processSection = (section) => {
349
- const cleanHeading = section.heading.replace(/[:#\-_]/g, " ").trim().toLowerCase();
350
- if (cleanHeading.length > MIN_TOPIC_LENGTH && cleanHeading.length < MAX_TOPIC_LENGTH) {
351
- topics.add(cleanHeading);
352
- }
353
- for (const child of section.children) {
354
- processSection(child);
355
- }
356
- };
357
- for (const section of document.sections) {
358
- processSection(section);
359
- }
360
- const frontmatter = document.frontmatter;
361
- if (frontmatter.tags && Array.isArray(frontmatter.tags)) {
362
- for (const tag of frontmatter.tags) {
363
- if (typeof tag === "string") {
364
- topics.add(tag.toLowerCase());
365
- }
366
- }
367
- }
368
- return Array.from(topics).slice(0, MAX_TOPICS);
386
+ const topics = /* @__PURE__ */ new Set();
387
+ const processSection = (section) => {
388
+ const cleanHeading = section.heading
389
+ .replace(/[:#\-_]/g, " ")
390
+ .trim()
391
+ .toLowerCase();
392
+ if (
393
+ cleanHeading.length > MIN_TOPIC_LENGTH &&
394
+ cleanHeading.length < MAX_TOPIC_LENGTH
395
+ ) {
396
+ topics.add(cleanHeading);
397
+ }
398
+ for (const child of section.children) {
399
+ processSection(child);
400
+ }
401
+ };
402
+ for (const section of document.sections) {
403
+ processSection(section);
404
+ }
405
+ const frontmatter = document.frontmatter;
406
+ if (frontmatter.tags && Array.isArray(frontmatter.tags)) {
407
+ for (const tag of frontmatter.tags) {
408
+ if (typeof tag === "string") {
409
+ topics.add(tag.toLowerCase());
410
+ }
411
+ }
412
+ }
413
+ return Array.from(topics).slice(0, MAX_TOPICS);
369
414
  };
370
415
  var summarizeDocument = (document, options = {}) => {
371
- const level = options.level ?? "summary";
372
- const maxTokens = options.maxTokens ?? TOKEN_BUDGETS[level];
373
- const allSections = document.sections.map((s) => summarizeSection(s, level));
374
- const originalTokens = document.metadata.tokenCount;
375
- let totalSummaryTokens = 0;
376
- const flatSections = [];
377
- const flattenWithTokens = (section) => {
378
- flatSections.push(section);
379
- totalSummaryTokens += section.summaryTokens;
380
- for (const child of section.children) {
381
- flattenWithTokens(child);
382
- }
383
- };
384
- for (const section of allSections) {
385
- flattenWithTokens(section);
386
- }
387
- const topics = extractTopics(document);
388
- const headerTemplate = `# ${document.title}
416
+ const level = options.level ?? "summary";
417
+ const maxTokens = options.maxTokens ?? TOKEN_BUDGETS[level];
418
+ const allSections = document.sections.map((s) => summarizeSection(s, level));
419
+ const originalTokens = document.metadata.tokenCount;
420
+ let totalSummaryTokens = 0;
421
+ const flatSections = [];
422
+ const flattenWithTokens = (section) => {
423
+ flatSections.push(section);
424
+ totalSummaryTokens += section.summaryTokens;
425
+ for (const child of section.children) {
426
+ flattenWithTokens(child);
427
+ }
428
+ };
429
+ for (const section of allSections) {
430
+ flattenWithTokens(section);
431
+ }
432
+ const topics = extractTopics(document);
433
+ const headerTemplate = `# ${document.title}
389
434
  Path: ${document.path}
390
435
  Tokens: 9999 (99% reduction from ${document.metadata.tokenCount})
391
436
  `;
392
- const topicsLine = topics.length > 0 ? `
437
+ const topicsLine =
438
+ topics.length > 0
439
+ ? `
393
440
  **Topics:** ${topics.join(", ")}
394
- ` : "";
395
- const truncationWarning = "\n\u26A0\uFE0F TRUNCATED: 999 sections omitted to fit token budget";
396
- const baseOverhead = countTokensApprox(
397
- headerTemplate + topicsLine + truncationWarning
398
- );
399
- const formattingOverhead = Math.ceil(baseOverhead * 1.2) + 20;
400
- const contentBudget = maxTokens - formattingOverhead;
401
- let truncated = false;
402
- let truncatedCount = 0;
403
- let sections;
404
- let summaryTokens;
405
- if (totalSummaryTokens > contentBudget && contentBudget > 0) {
406
- let tokensUsed = 0;
407
- const truncateSections = (sectionList) => {
408
- const result2 = [];
409
- for (const section of sectionList) {
410
- const sectionOwnTokens = section.summaryTokens;
411
- const fitsInBudget = tokensUsed + sectionOwnTokens <= contentBudget;
412
- if (fitsInBudget) {
413
- tokensUsed += sectionOwnTokens;
414
- const truncatedChildren = truncateSections(section.children);
415
- result2.push({
416
- ...section,
417
- children: truncatedChildren
418
- });
419
- } else {
420
- truncatedCount++;
421
- const rescuedChildren = truncateSections(section.children);
422
- result2.push(...rescuedChildren);
423
- }
424
- }
425
- return result2;
426
- };
427
- sections = truncateSections(allSections);
428
- summaryTokens = tokensUsed;
429
- truncated = truncatedCount > 0;
430
- } else {
431
- sections = allSections;
432
- summaryTokens = totalSummaryTokens;
433
- }
434
- const compressionRatio = originalTokens > 0 ? 1 - summaryTokens / originalTokens : 0;
435
- const result = {
436
- path: document.path,
437
- title: document.title,
438
- originalTokens,
439
- summaryTokens,
440
- compressionRatio,
441
- sections,
442
- keyTopics: topics
443
- };
444
- if (truncated) {
445
- return {
446
- ...result,
447
- truncated: true,
448
- truncatedCount
449
- };
450
- }
451
- return result;
441
+ `
442
+ : "";
443
+ const truncationWarning =
444
+ "\n\u26A0\uFE0F TRUNCATED: 999 sections omitted to fit token budget";
445
+ const baseOverhead = countTokensApprox(
446
+ headerTemplate + topicsLine + truncationWarning,
447
+ );
448
+ const formattingOverhead = Math.ceil(baseOverhead * 1.2) + 20;
449
+ const contentBudget = maxTokens - formattingOverhead;
450
+ let truncated = false;
451
+ let truncatedCount = 0;
452
+ let sections;
453
+ let summaryTokens;
454
+ if (totalSummaryTokens > contentBudget && contentBudget > 0) {
455
+ let tokensUsed = 0;
456
+ const truncateSections = (sectionList) => {
457
+ const result2 = [];
458
+ for (const section of sectionList) {
459
+ const sectionOwnTokens = section.summaryTokens;
460
+ const fitsInBudget = tokensUsed + sectionOwnTokens <= contentBudget;
461
+ if (fitsInBudget) {
462
+ tokensUsed += sectionOwnTokens;
463
+ const truncatedChildren = truncateSections(section.children);
464
+ result2.push({
465
+ ...section,
466
+ children: truncatedChildren,
467
+ });
468
+ } else {
469
+ truncatedCount++;
470
+ const rescuedChildren = truncateSections(section.children);
471
+ result2.push(...rescuedChildren);
472
+ }
473
+ }
474
+ return result2;
475
+ };
476
+ sections = truncateSections(allSections);
477
+ summaryTokens = tokensUsed;
478
+ truncated = truncatedCount > 0;
479
+ } else {
480
+ sections = allSections;
481
+ summaryTokens = totalSummaryTokens;
482
+ }
483
+ const compressionRatio =
484
+ originalTokens > 0 ? 1 - summaryTokens / originalTokens : 0;
485
+ const result = {
486
+ path: document.path,
487
+ title: document.title,
488
+ originalTokens,
489
+ summaryTokens,
490
+ compressionRatio,
491
+ sections,
492
+ keyTopics: topics,
493
+ };
494
+ if (truncated) {
495
+ return {
496
+ ...result,
497
+ truncated: true,
498
+ truncatedCount,
499
+ };
500
+ }
501
+ return result;
452
502
  };
453
- var summarizeFile = (filePath, options = {}) => Effect.gen(function* () {
454
- const document = yield* parseFile(filePath).pipe(
455
- Effect.mapError((e) => new Error(`${e._tag}: ${e.message}`))
456
- );
457
- return summarizeDocument(document, options);
458
- });
459
- var assembleContext = (rootPath, sourcePaths, options) => Effect.gen(function* () {
460
- const budget = options.budget;
461
- const level = options.level ?? "summary";
462
- const sources = [];
463
- const overflow = [];
464
- let totalTokens = 0;
465
- const perSourceBudget = Math.floor(budget / sourcePaths.length);
466
- for (const sourcePath of sourcePaths) {
467
- const resolvedPath = path.isAbsolute(sourcePath) ? sourcePath : path.join(rootPath, sourcePath);
468
- try {
469
- const summary = yield* summarizeFile(resolvedPath, {
470
- level,
471
- maxTokens: perSourceBudget
472
- });
473
- const content = formatSummary(summary);
474
- const tokens = countTokensApprox(content);
475
- if (totalTokens + tokens <= budget) {
476
- sources.push({
477
- path: path.relative(rootPath, resolvedPath),
478
- title: summary.title,
479
- tokens,
480
- content
481
- });
482
- totalTokens += tokens;
483
- } else {
484
- const remaining = budget - totalTokens;
485
- if (remaining > MIN_PARTIAL_BUDGET) {
486
- const briefSummary = yield* summarizeFile(resolvedPath, {
487
- level: "brief",
488
- maxTokens: remaining
489
- });
490
- const briefContent = formatSummary(briefSummary);
491
- const briefTokens = countTokensApprox(briefContent);
492
- sources.push({
493
- path: path.relative(rootPath, resolvedPath),
494
- title: briefSummary.title,
495
- tokens: briefTokens,
496
- content: briefContent
497
- });
498
- totalTokens += briefTokens;
499
- } else {
500
- overflow.push(path.relative(rootPath, resolvedPath));
501
- }
502
- }
503
- } catch (_e) {
504
- overflow.push(sourcePath);
505
- }
506
- }
507
- return {
508
- sources,
509
- totalTokens,
510
- budget,
511
- overflow
512
- };
513
- });
503
+ var summarizeFile = (filePath, options = {}) =>
504
+ Effect.gen(function* () {
505
+ const document = yield* parseFile(filePath).pipe(
506
+ Effect.mapError((e) => new Error(`${e._tag}: ${e.message}`)),
507
+ );
508
+ return summarizeDocument(document, options);
509
+ });
510
+ var assembleContext = (rootPath, sourcePaths, options) =>
511
+ Effect.gen(function* () {
512
+ const budget = options.budget;
513
+ const level = options.level ?? "summary";
514
+ const sources = [];
515
+ const overflow = [];
516
+ let totalTokens = 0;
517
+ const perSourceBudget = Math.floor(budget / sourcePaths.length);
518
+ for (const sourcePath of sourcePaths) {
519
+ const resolvedPath = path.isAbsolute(sourcePath)
520
+ ? sourcePath
521
+ : path.join(rootPath, sourcePath);
522
+ try {
523
+ const summary = yield* summarizeFile(resolvedPath, {
524
+ level,
525
+ maxTokens: perSourceBudget,
526
+ });
527
+ const content = formatSummary(summary);
528
+ const tokens = countTokensApprox(content);
529
+ if (totalTokens + tokens <= budget) {
530
+ sources.push({
531
+ path: path.relative(rootPath, resolvedPath),
532
+ title: summary.title,
533
+ tokens,
534
+ content,
535
+ });
536
+ totalTokens += tokens;
537
+ } else {
538
+ const remaining = budget - totalTokens;
539
+ if (remaining > MIN_PARTIAL_BUDGET) {
540
+ const briefSummary = yield* summarizeFile(resolvedPath, {
541
+ level: "brief",
542
+ maxTokens: remaining,
543
+ });
544
+ const briefContent = formatSummary(briefSummary);
545
+ const briefTokens = countTokensApprox(briefContent);
546
+ sources.push({
547
+ path: path.relative(rootPath, resolvedPath),
548
+ title: briefSummary.title,
549
+ tokens: briefTokens,
550
+ content: briefContent,
551
+ });
552
+ totalTokens += briefTokens;
553
+ } else {
554
+ overflow.push(path.relative(rootPath, resolvedPath));
555
+ }
556
+ }
557
+ } catch (_e) {
558
+ overflow.push(sourcePath);
559
+ }
560
+ }
561
+ return {
562
+ sources,
563
+ totalTokens,
564
+ budget,
565
+ overflow,
566
+ };
567
+ });
514
568
 
569
+ // src/embeddings/openai-provider.ts
570
+ import { Console, Effect as Effect2, Effect as Effect4 } from "effect";
515
571
  // src/embeddings/semantic-search.ts
516
572
  import * as fs3 from "fs/promises";
573
+ import OpenAI from "openai";
517
574
  import * as path3 from "path";
518
- import { Effect as Effect4 } from "effect";
519
575
 
520
- // src/embeddings/openai-provider.ts
521
- import { Console, Effect as Effect2 } from "effect";
522
- import OpenAI from "openai";
523
576
  var PRICING = {
524
- "text-embedding-3-small": 0.02,
525
- "text-embedding-3-large": 0.13,
526
- "text-embedding-ada-002": 0.1
577
+ "text-embedding-3-small": 0.02,
578
+ "text-embedding-3-large": 0.13,
579
+ "text-embedding-ada-002": 0.1,
527
580
  };
528
581
  var MissingApiKeyError = class extends Error {
529
- constructor() {
530
- super("OPENAI_API_KEY not set");
531
- this.name = "MissingApiKeyError";
532
- }
582
+ constructor() {
583
+ super("OPENAI_API_KEY not set");
584
+ this.name = "MissingApiKeyError";
585
+ }
533
586
  };
534
587
  var InvalidApiKeyError = class extends Error {
535
- constructor(message) {
536
- super(message ?? "Invalid OPENAI_API_KEY");
537
- this.name = "InvalidApiKeyError";
538
- }
588
+ constructor(message) {
589
+ super(message ?? "Invalid OPENAI_API_KEY");
590
+ this.name = "InvalidApiKeyError";
591
+ }
539
592
  };
540
593
  var OpenAIProvider = class {
541
- name;
542
- dimensions;
543
- client;
544
- model;
545
- batchSize;
546
- constructor(options = {}) {
547
- const apiKey = options.apiKey ?? process.env.OPENAI_API_KEY;
548
- if (!apiKey) {
549
- throw new MissingApiKeyError();
550
- }
551
- this.client = new OpenAI({ apiKey });
552
- this.model = options.model ?? "text-embedding-3-small";
553
- this.batchSize = options.batchSize ?? 100;
554
- this.name = `openai:${this.model}`;
555
- this.dimensions = 512;
556
- }
557
- async embed(texts) {
558
- if (texts.length === 0) {
559
- return { embeddings: [], tokensUsed: 0, cost: 0 };
560
- }
561
- const allEmbeddings = [];
562
- let totalTokens = 0;
563
- try {
564
- for (let i = 0; i < texts.length; i += this.batchSize) {
565
- const batch = texts.slice(i, i + this.batchSize);
566
- const response = await this.client.embeddings.create({
567
- model: this.model,
568
- input: batch,
569
- dimensions: 512
570
- // Ensure consistent dimensions
571
- });
572
- for (const item of response.data) {
573
- allEmbeddings.push(item.embedding);
574
- }
575
- totalTokens += response.usage?.total_tokens ?? 0;
576
- }
577
- } catch (error) {
578
- if (error instanceof OpenAI.AuthenticationError) {
579
- throw new InvalidApiKeyError(error.message);
580
- }
581
- throw error;
582
- }
583
- const pricePerMillion = PRICING[this.model] ?? 0.02;
584
- const cost = totalTokens / 1e6 * pricePerMillion;
585
- return {
586
- embeddings: allEmbeddings,
587
- tokensUsed: totalTokens,
588
- cost
589
- };
590
- }
594
+ name;
595
+ dimensions;
596
+ client;
597
+ model;
598
+ batchSize;
599
+ constructor(options = {}) {
600
+ const apiKey = options.apiKey ?? process.env.OPENAI_API_KEY;
601
+ if (!apiKey) {
602
+ throw new MissingApiKeyError();
603
+ }
604
+ this.client = new OpenAI({ apiKey });
605
+ this.model = options.model ?? "text-embedding-3-small";
606
+ this.batchSize = options.batchSize ?? 100;
607
+ this.name = `openai:${this.model}`;
608
+ this.dimensions = 512;
609
+ }
610
+ async embed(texts) {
611
+ if (texts.length === 0) {
612
+ return { embeddings: [], tokensUsed: 0, cost: 0 };
613
+ }
614
+ const allEmbeddings = [];
615
+ let totalTokens = 0;
616
+ try {
617
+ for (let i = 0; i < texts.length; i += this.batchSize) {
618
+ const batch = texts.slice(i, i + this.batchSize);
619
+ const response = await this.client.embeddings.create({
620
+ model: this.model,
621
+ input: batch,
622
+ dimensions: 512,
623
+ // Ensure consistent dimensions
624
+ });
625
+ for (const item of response.data) {
626
+ allEmbeddings.push(item.embedding);
627
+ }
628
+ totalTokens += response.usage?.total_tokens ?? 0;
629
+ }
630
+ } catch (error) {
631
+ if (error instanceof OpenAI.AuthenticationError) {
632
+ throw new InvalidApiKeyError(error.message);
633
+ }
634
+ throw error;
635
+ }
636
+ const pricePerMillion = PRICING[this.model] ?? 0.02;
637
+ const cost = (totalTokens / 1e6) * pricePerMillion;
638
+ return {
639
+ embeddings: allEmbeddings,
640
+ tokensUsed: totalTokens,
641
+ cost,
642
+ };
643
+ }
591
644
  };
592
645
  var createOpenAIProvider = (options) => new OpenAIProvider(options);
593
- var handleApiKeyError = (effect) => effect.pipe(
594
- Effect2.catchIf(
595
- (e) => e instanceof MissingApiKeyError,
596
- () => Effect2.gen(function* () {
597
- yield* Console.error("");
598
- yield* Console.error("Error: OPENAI_API_KEY not set");
599
- yield* Console.error("");
600
- yield* Console.error(
601
- "To use semantic search, set your OpenAI API key:"
602
- );
603
- yield* Console.error(" export OPENAI_API_KEY=sk-...");
604
- yield* Console.error("");
605
- yield* Console.error("Or add to .env file in project root.");
606
- return yield* Effect2.fail(new Error("Missing API key"));
607
- })
608
- ),
609
- Effect2.catchIf(
610
- (e) => e instanceof InvalidApiKeyError,
611
- (e) => Effect2.gen(function* () {
612
- yield* Console.error("");
613
- yield* Console.error("Error: Invalid OPENAI_API_KEY");
614
- yield* Console.error("");
615
- yield* Console.error("The provided API key was rejected by OpenAI.");
616
- yield* Console.error("Please check your API key is correct:");
617
- yield* Console.error(" export OPENAI_API_KEY=sk-...");
618
- yield* Console.error("");
619
- yield* Console.error(`Details: ${e.message}`);
620
- return yield* Effect2.fail(new Error("Invalid API key"));
621
- })
622
- )
623
- );
646
+ var handleApiKeyError = (effect) =>
647
+ effect.pipe(
648
+ Effect2.catchIf(
649
+ (e) => e instanceof MissingApiKeyError,
650
+ () =>
651
+ Effect2.gen(function* () {
652
+ yield* Console.error("");
653
+ yield* Console.error("Error: OPENAI_API_KEY not set");
654
+ yield* Console.error("");
655
+ yield* Console.error(
656
+ "To use semantic search, set your OpenAI API key:",
657
+ );
658
+ yield* Console.error(" export OPENAI_API_KEY=sk-...");
659
+ yield* Console.error("");
660
+ yield* Console.error("Or add to .env file in project root.");
661
+ return yield* Effect2.fail(new Error("Missing API key"));
662
+ }),
663
+ ),
664
+ Effect2.catchIf(
665
+ (e) => e instanceof InvalidApiKeyError,
666
+ (e) =>
667
+ Effect2.gen(function* () {
668
+ yield* Console.error("");
669
+ yield* Console.error("Error: Invalid OPENAI_API_KEY");
670
+ yield* Console.error("");
671
+ yield* Console.error("The provided API key was rejected by OpenAI.");
672
+ yield* Console.error("Please check your API key is correct:");
673
+ yield* Console.error(" export OPENAI_API_KEY=sk-...");
674
+ yield* Console.error("");
675
+ yield* Console.error(`Details: ${e.message}`);
676
+ return yield* Effect2.fail(new Error("Invalid API key"));
677
+ }),
678
+ ),
679
+ );
624
680
 
681
+ import { Effect as Effect3 } from "effect";
625
682
  // src/embeddings/vector-store.ts
626
683
  import * as fs2 from "fs/promises";
627
- import * as path2 from "path";
628
- import { Effect as Effect3 } from "effect";
629
684
  import HierarchicalNSW from "hnswlib-node";
685
+ import * as path2 from "path";
686
+
630
687
  var VECTOR_INDEX_FILE = "vectors.bin";
631
688
  var VECTOR_META_FILE = "vectors.meta.json";
632
689
  var INDEX_VERSION = 1;
633
690
  var HnswVectorStore = class {
634
- rootPath;
635
- dimensions;
636
- index = null;
637
- entries = /* @__PURE__ */ new Map();
638
- idToIndex = /* @__PURE__ */ new Map();
639
- nextIndex = 0;
640
- provider = "unknown";
641
- totalCost = 0;
642
- totalTokens = 0;
643
- constructor(rootPath, dimensions) {
644
- this.rootPath = path2.resolve(rootPath);
645
- this.dimensions = dimensions;
646
- }
647
- getIndexDir() {
648
- return path2.join(this.rootPath, INDEX_DIR);
649
- }
650
- getVectorPath() {
651
- return path2.join(this.getIndexDir(), VECTOR_INDEX_FILE);
652
- }
653
- getMetaPath() {
654
- return path2.join(this.getIndexDir(), VECTOR_META_FILE);
655
- }
656
- ensureIndex() {
657
- if (!this.index) {
658
- this.index = new HierarchicalNSW.HierarchicalNSW(
659
- "cosine",
660
- this.dimensions
661
- );
662
- this.index.initIndex(1e4, 16, 200, 100);
663
- }
664
- return this.index;
665
- }
666
- add(entries) {
667
- return Effect3.sync(() => {
668
- const index = this.ensureIndex();
669
- for (const entry of entries) {
670
- if (this.idToIndex.has(entry.id)) {
671
- continue;
672
- }
673
- const idx = this.nextIndex++;
674
- if (idx >= index.getMaxElements()) {
675
- index.resizeIndex(index.getMaxElements() * 2);
676
- }
677
- index.addPoint(entry.embedding, idx);
678
- this.entries.set(idx, entry);
679
- this.idToIndex.set(entry.id, idx);
680
- }
681
- });
682
- }
683
- search(vector, limit, threshold = 0) {
684
- return Effect3.sync(() => {
685
- if (!this.index || this.entries.size === 0) {
686
- return [];
687
- }
688
- const result = this.index.searchKnn(
689
- vector,
690
- Math.min(limit, this.entries.size)
691
- );
692
- const results = [];
693
- for (let i = 0; i < result.neighbors.length; i++) {
694
- const idx = result.neighbors[i];
695
- const distance = result.distances[i];
696
- if (idx === void 0 || distance === void 0) {
697
- continue;
698
- }
699
- const similarity = 1 - distance;
700
- if (similarity < threshold) {
701
- continue;
702
- }
703
- const entry = this.entries.get(idx);
704
- if (entry) {
705
- results.push({
706
- id: entry.id,
707
- sectionId: entry.sectionId,
708
- documentPath: entry.documentPath,
709
- heading: entry.heading,
710
- similarity
711
- });
712
- }
713
- }
714
- return results;
715
- });
716
- }
717
- save() {
718
- return Effect3.gen(
719
- function* () {
720
- if (!this.index) {
721
- return;
722
- }
723
- const indexDir = this.getIndexDir();
724
- yield* Effect3.promise(() => fs2.mkdir(indexDir, { recursive: true }));
725
- yield* Effect3.promise(
726
- () => this.index.writeIndex(this.getVectorPath())
727
- );
728
- const meta = {
729
- version: INDEX_VERSION,
730
- provider: this.provider,
731
- dimensions: this.dimensions,
732
- entries: Object.fromEntries(
733
- Array.from(this.entries.entries()).map(([idx, entry]) => [
734
- idx.toString(),
735
- entry
736
- ])
737
- ),
738
- totalCost: this.totalCost,
739
- totalTokens: this.totalTokens,
740
- createdAt: (/* @__PURE__ */ new Date()).toISOString(),
741
- updatedAt: (/* @__PURE__ */ new Date()).toISOString()
742
- };
743
- yield* Effect3.promise(
744
- () => fs2.writeFile(this.getMetaPath(), JSON.stringify(meta, null, 2))
745
- );
746
- }.bind(this)
747
- );
748
- }
749
- load() {
750
- return Effect3.gen(
751
- function* () {
752
- const vectorPath = this.getVectorPath();
753
- const metaPath = this.getMetaPath();
754
- const filesExist = yield* Effect3.tryPromise({
755
- try: async () => {
756
- await fs2.access(vectorPath);
757
- await fs2.access(metaPath);
758
- return true;
759
- },
760
- catch: () => false
761
- }).pipe(Effect3.catchAll(() => Effect3.succeed(false)));
762
- if (!filesExist) {
763
- return false;
764
- }
765
- const metaContent = yield* Effect3.promise(
766
- () => fs2.readFile(metaPath, "utf-8")
767
- );
768
- const meta = JSON.parse(metaContent);
769
- if (meta.dimensions !== this.dimensions) {
770
- return false;
771
- }
772
- this.index = new HierarchicalNSW.HierarchicalNSW(
773
- "cosine",
774
- this.dimensions
775
- );
776
- yield* Effect3.promise(() => this.index.readIndex(vectorPath));
777
- this.entries.clear();
778
- this.idToIndex.clear();
779
- this.nextIndex = 0;
780
- for (const [idxStr, entry] of Object.entries(meta.entries)) {
781
- const idx = parseInt(idxStr, 10);
782
- this.entries.set(idx, entry);
783
- this.idToIndex.set(entry.id, idx);
784
- this.nextIndex = Math.max(this.nextIndex, idx + 1);
785
- }
786
- this.provider = meta.provider;
787
- this.totalCost = meta.totalCost;
788
- this.totalTokens = meta.totalTokens;
789
- return true;
790
- }.bind(this)
791
- );
792
- }
793
- getStats() {
794
- return {
795
- count: this.entries.size,
796
- dimensions: this.dimensions,
797
- provider: this.provider,
798
- totalCost: this.totalCost,
799
- totalTokens: this.totalTokens
800
- };
801
- }
802
- setProvider(name) {
803
- this.provider = name;
804
- }
805
- addCost(cost, tokens) {
806
- this.totalCost += cost;
807
- this.totalTokens += tokens;
808
- }
691
+ rootPath;
692
+ dimensions;
693
+ index = null;
694
+ entries = /* @__PURE__ */ new Map();
695
+ idToIndex = /* @__PURE__ */ new Map();
696
+ nextIndex = 0;
697
+ provider = "unknown";
698
+ totalCost = 0;
699
+ totalTokens = 0;
700
+ constructor(rootPath, dimensions) {
701
+ this.rootPath = path2.resolve(rootPath);
702
+ this.dimensions = dimensions;
703
+ }
704
+ getIndexDir() {
705
+ return path2.join(this.rootPath, INDEX_DIR);
706
+ }
707
+ getVectorPath() {
708
+ return path2.join(this.getIndexDir(), VECTOR_INDEX_FILE);
709
+ }
710
+ getMetaPath() {
711
+ return path2.join(this.getIndexDir(), VECTOR_META_FILE);
712
+ }
713
+ ensureIndex() {
714
+ if (!this.index) {
715
+ this.index = new HierarchicalNSW.HierarchicalNSW(
716
+ "cosine",
717
+ this.dimensions,
718
+ );
719
+ this.index.initIndex(1e4, 16, 200, 100);
720
+ }
721
+ return this.index;
722
+ }
723
+ add(entries) {
724
+ return Effect3.sync(() => {
725
+ const index = this.ensureIndex();
726
+ for (const entry of entries) {
727
+ if (this.idToIndex.has(entry.id)) {
728
+ continue;
729
+ }
730
+ const idx = this.nextIndex++;
731
+ if (idx >= index.getMaxElements()) {
732
+ index.resizeIndex(index.getMaxElements() * 2);
733
+ }
734
+ index.addPoint(entry.embedding, idx);
735
+ this.entries.set(idx, entry);
736
+ this.idToIndex.set(entry.id, idx);
737
+ }
738
+ });
739
+ }
740
+ search(vector, limit, threshold = 0) {
741
+ return Effect3.sync(() => {
742
+ if (!this.index || this.entries.size === 0) {
743
+ return [];
744
+ }
745
+ const result = this.index.searchKnn(
746
+ vector,
747
+ Math.min(limit, this.entries.size),
748
+ );
749
+ const results = [];
750
+ for (let i = 0; i < result.neighbors.length; i++) {
751
+ const idx = result.neighbors[i];
752
+ const distance = result.distances[i];
753
+ if (idx === void 0 || distance === void 0) {
754
+ continue;
755
+ }
756
+ const similarity = 1 - distance;
757
+ if (similarity < threshold) {
758
+ continue;
759
+ }
760
+ const entry = this.entries.get(idx);
761
+ if (entry) {
762
+ results.push({
763
+ id: entry.id,
764
+ sectionId: entry.sectionId,
765
+ documentPath: entry.documentPath,
766
+ heading: entry.heading,
767
+ similarity,
768
+ });
769
+ }
770
+ }
771
+ return results;
772
+ });
773
+ }
774
+ save() {
775
+ return Effect3.gen(
776
+ function* () {
777
+ if (!this.index) {
778
+ return;
779
+ }
780
+ const indexDir = this.getIndexDir();
781
+ yield* Effect3.promise(() => fs2.mkdir(indexDir, { recursive: true }));
782
+ yield* Effect3.promise(() =>
783
+ this.index.writeIndex(this.getVectorPath()),
784
+ );
785
+ const meta = {
786
+ version: INDEX_VERSION,
787
+ provider: this.provider,
788
+ dimensions: this.dimensions,
789
+ entries: Object.fromEntries(
790
+ Array.from(this.entries.entries()).map(([idx, entry]) => [
791
+ idx.toString(),
792
+ entry,
793
+ ]),
794
+ ),
795
+ totalCost: this.totalCost,
796
+ totalTokens: this.totalTokens,
797
+ createdAt: /* @__PURE__ */ new Date().toISOString(),
798
+ updatedAt: /* @__PURE__ */ new Date().toISOString(),
799
+ };
800
+ yield* Effect3.promise(() =>
801
+ fs2.writeFile(this.getMetaPath(), JSON.stringify(meta, null, 2)),
802
+ );
803
+ }.bind(this),
804
+ );
805
+ }
806
+ load() {
807
+ return Effect3.gen(
808
+ function* () {
809
+ const vectorPath = this.getVectorPath();
810
+ const metaPath = this.getMetaPath();
811
+ const filesExist = yield* Effect3.tryPromise({
812
+ try: async () => {
813
+ await fs2.access(vectorPath);
814
+ await fs2.access(metaPath);
815
+ return true;
816
+ },
817
+ catch: () => false,
818
+ }).pipe(Effect3.catchAll(() => Effect3.succeed(false)));
819
+ if (!filesExist) {
820
+ return false;
821
+ }
822
+ const metaContent = yield* Effect3.promise(() =>
823
+ fs2.readFile(metaPath, "utf-8"),
824
+ );
825
+ const meta = JSON.parse(metaContent);
826
+ if (meta.dimensions !== this.dimensions) {
827
+ return false;
828
+ }
829
+ this.index = new HierarchicalNSW.HierarchicalNSW(
830
+ "cosine",
831
+ this.dimensions,
832
+ );
833
+ yield* Effect3.promise(() => this.index.readIndex(vectorPath));
834
+ this.entries.clear();
835
+ this.idToIndex.clear();
836
+ this.nextIndex = 0;
837
+ for (const [idxStr, entry] of Object.entries(meta.entries)) {
838
+ const idx = parseInt(idxStr, 10);
839
+ this.entries.set(idx, entry);
840
+ this.idToIndex.set(entry.id, idx);
841
+ this.nextIndex = Math.max(this.nextIndex, idx + 1);
842
+ }
843
+ this.provider = meta.provider;
844
+ this.totalCost = meta.totalCost;
845
+ this.totalTokens = meta.totalTokens;
846
+ return true;
847
+ }.bind(this),
848
+ );
849
+ }
850
+ getStats() {
851
+ return {
852
+ count: this.entries.size,
853
+ dimensions: this.dimensions,
854
+ provider: this.provider,
855
+ totalCost: this.totalCost,
856
+ totalTokens: this.totalTokens,
857
+ };
858
+ }
859
+ setProvider(name) {
860
+ this.provider = name;
861
+ }
862
+ addCost(cost, tokens) {
863
+ this.totalCost += cost;
864
+ this.totalTokens += tokens;
865
+ }
809
866
  };
810
- var createVectorStore = (rootPath, dimensions) => new HnswVectorStore(rootPath, dimensions);
867
+ var createVectorStore = (rootPath, dimensions) =>
868
+ new HnswVectorStore(rootPath, dimensions);
811
869
 
812
870
  // src/embeddings/semantic-search.ts
813
- var generateEmbeddingText = (section, content, documentTitle, parentHeading) => {
814
- const parts = [];
815
- parts.push(`# ${section.heading}`);
816
- if (parentHeading) {
817
- parts.push(`Parent section: ${parentHeading}`);
818
- }
819
- parts.push(`Document: ${documentTitle}`);
820
- parts.push("");
821
- parts.push(content);
822
- return parts.join("\n");
871
+ var generateEmbeddingText = (
872
+ section,
873
+ content,
874
+ documentTitle,
875
+ parentHeading,
876
+ ) => {
877
+ const parts = [];
878
+ parts.push(`# ${section.heading}`);
879
+ if (parentHeading) {
880
+ parts.push(`Parent section: ${parentHeading}`);
881
+ }
882
+ parts.push(`Document: ${documentTitle}`);
883
+ parts.push("");
884
+ parts.push(content);
885
+ return parts.join("\n");
823
886
  };
824
887
  var EMBEDDING_PRICE_PER_MILLION = 0.02;
825
- var estimateEmbeddingCost = (rootPath, options = {}) => Effect4.gen(function* () {
826
- const resolvedRoot = path3.resolve(rootPath);
827
- const storage = createStorage(resolvedRoot);
828
- const docIndex = yield* loadDocumentIndex(storage);
829
- const sectionIndex = yield* loadSectionIndex(storage);
830
- if (!docIndex || !sectionIndex) {
831
- return yield* Effect4.fail(
832
- new Error("Index not found. Run 'mdcontext index' first.")
833
- );
834
- }
835
- const byDir = /* @__PURE__ */ new Map();
836
- for (const section of Object.values(sectionIndex.sections)) {
837
- if (section.tokenCount < 10) continue;
838
- if (options.excludePatterns?.length) {
839
- const excluded = options.excludePatterns.some((pattern) => {
840
- const regex = new RegExp(
841
- `^${pattern.replace(/\*/g, ".*").replace(/\?/g, ".")}$`
842
- );
843
- return regex.test(section.documentPath);
844
- });
845
- if (excluded) continue;
846
- }
847
- const dir = path3.dirname(section.documentPath) || ".";
848
- if (!byDir.has(dir)) {
849
- byDir.set(dir, { files: /* @__PURE__ */ new Set(), sections: 0, tokens: 0 });
850
- }
851
- const entry = byDir.get(dir);
852
- entry.files.add(section.documentPath);
853
- entry.sections++;
854
- entry.tokens += section.tokenCount;
855
- }
856
- const directoryEstimates = [];
857
- let totalFiles = 0;
858
- let totalSections = 0;
859
- let totalTokens = 0;
860
- for (const [dir, data] of byDir) {
861
- directoryEstimates.push({
862
- directory: dir,
863
- fileCount: data.files.size,
864
- sectionCount: data.sections,
865
- estimatedTokens: data.tokens,
866
- estimatedCost: data.tokens / 1e6 * EMBEDDING_PRICE_PER_MILLION
867
- });
868
- totalFiles += data.files.size;
869
- totalSections += data.sections;
870
- totalTokens += data.tokens;
871
- }
872
- directoryEstimates.sort((a, b) => a.directory.localeCompare(b.directory));
873
- const estimatedTimeSeconds = Math.ceil(totalSections / 100) * 1.5;
874
- return {
875
- totalFiles,
876
- totalSections,
877
- totalTokens,
878
- totalCost: totalTokens / 1e6 * EMBEDDING_PRICE_PER_MILLION,
879
- estimatedTimeSeconds,
880
- byDirectory: directoryEstimates
881
- };
882
- });
883
- var buildEmbeddings = (rootPath, options = {}) => Effect4.gen(function* () {
884
- const startTime = Date.now();
885
- const resolvedRoot = path3.resolve(rootPath);
886
- const storage = createStorage(resolvedRoot);
887
- const docIndex = yield* loadDocumentIndex(storage);
888
- const sectionIndex = yield* loadSectionIndex(storage);
889
- if (!docIndex || !sectionIndex) {
890
- return yield* Effect4.fail(
891
- new Error("Index not found. Run 'mdcontext index' first.")
892
- );
893
- }
894
- const provider = options.provider ?? (yield* Effect4.try({
895
- try: () => createOpenAIProvider(),
896
- catch: (e) => e
897
- }));
898
- const dimensions = provider.dimensions;
899
- const vectorStore = createVectorStore(
900
- resolvedRoot,
901
- dimensions
902
- );
903
- vectorStore.setProvider(provider.name);
904
- if (!options.force) {
905
- const loaded = yield* vectorStore.load();
906
- if (loaded) {
907
- const stats = vectorStore.getStats();
908
- if (stats.count > 0) {
909
- const duration2 = Date.now() - startTime;
910
- const estimatedSavings = stats.totalTokens / 1e6 * EMBEDDING_PRICE_PER_MILLION;
911
- return {
912
- sectionsEmbedded: 0,
913
- tokensUsed: 0,
914
- cost: 0,
915
- duration: duration2,
916
- filesProcessed: 0,
917
- cacheHit: true,
918
- existingVectors: stats.count,
919
- estimatedSavings
920
- };
921
- }
922
- }
923
- }
924
- const isExcluded = (docPath) => {
925
- if (!options.excludePatterns?.length) return false;
926
- return options.excludePatterns.some((pattern) => {
927
- const regex = new RegExp(
928
- `^${pattern.replace(/\*/g, ".*").replace(/\?/g, ".")}$`
929
- );
930
- return regex.test(docPath);
931
- });
932
- };
933
- const sectionsByDoc = /* @__PURE__ */ new Map();
934
- for (const section of Object.values(sectionIndex.sections)) {
935
- const document = docIndex.documents[section.documentPath];
936
- if (!document) continue;
937
- if (section.tokenCount < 10) continue;
938
- if (isExcluded(section.documentPath)) continue;
939
- let parentHeading;
940
- if (section.level > 1) {
941
- const docSections = sectionIndex.byDocument[document.id] ?? [];
942
- for (const sibId of docSections) {
943
- const sib = sectionIndex.sections[sibId];
944
- if (sib && sib.level === section.level - 1 && sib.startLine < section.startLine) {
945
- parentHeading = sib.heading;
946
- }
947
- }
948
- }
949
- const docPath = section.documentPath;
950
- if (!sectionsByDoc.has(docPath)) {
951
- sectionsByDoc.set(docPath, []);
952
- }
953
- sectionsByDoc.get(docPath).push({ section, parentHeading });
954
- }
955
- if (sectionsByDoc.size === 0) {
956
- const duration2 = Date.now() - startTime;
957
- return {
958
- sectionsEmbedded: 0,
959
- tokensUsed: 0,
960
- cost: 0,
961
- duration: duration2,
962
- filesProcessed: 0
963
- };
964
- }
965
- const sectionsToEmbed = [];
966
- const docPaths = Array.from(sectionsByDoc.keys());
967
- let filesProcessed = 0;
968
- for (let fileIndex = 0; fileIndex < docPaths.length; fileIndex++) {
969
- const docPath = docPaths[fileIndex];
970
- const sections = sectionsByDoc.get(docPath);
971
- const document = docIndex.documents[docPath];
972
- if (!document) continue;
973
- if (options.onFileProgress) {
974
- options.onFileProgress({
975
- fileIndex: fileIndex + 1,
976
- totalFiles: docPaths.length,
977
- filePath: docPath,
978
- sectionCount: sections.length
979
- });
980
- }
981
- const filePath = path3.join(resolvedRoot, docPath);
982
- let fileContent;
983
- try {
984
- fileContent = yield* Effect4.promise(
985
- () => fs3.readFile(filePath, "utf-8")
986
- );
987
- } catch {
988
- continue;
989
- }
990
- filesProcessed++;
991
- const lines = fileContent.split("\n");
992
- for (const { section, parentHeading } of sections) {
993
- const content = lines.slice(section.startLine - 1, section.endLine).join("\n");
994
- const text = generateEmbeddingText(
995
- section,
996
- content,
997
- document.title,
998
- parentHeading
999
- );
1000
- sectionsToEmbed.push({ section, text });
1001
- }
1002
- }
1003
- if (sectionsToEmbed.length === 0) {
1004
- const duration2 = Date.now() - startTime;
1005
- return {
1006
- sectionsEmbedded: 0,
1007
- tokensUsed: 0,
1008
- cost: 0,
1009
- duration: duration2,
1010
- filesProcessed
1011
- };
1012
- }
1013
- const texts = sectionsToEmbed.map((s) => s.text);
1014
- const result = yield* Effect4.tryPromise({
1015
- try: () => provider.embed(texts),
1016
- catch: (e) => {
1017
- if (e instanceof InvalidApiKeyError) return e;
1018
- return new Error(
1019
- `Embedding failed: ${e instanceof Error ? e.message : String(e)}`
1020
- );
1021
- }
1022
- });
1023
- const entries = [];
1024
- for (let i = 0; i < sectionsToEmbed.length; i++) {
1025
- const { section } = sectionsToEmbed[i] ?? { section: null };
1026
- const embedding = result.embeddings[i];
1027
- if (!section || !embedding) continue;
1028
- entries.push({
1029
- id: section.id,
1030
- sectionId: section.id,
1031
- documentPath: section.documentPath,
1032
- heading: section.heading,
1033
- embedding
1034
- });
1035
- }
1036
- yield* vectorStore.add(entries);
1037
- vectorStore.addCost(result.cost, result.tokensUsed);
1038
- yield* vectorStore.save();
1039
- const duration = Date.now() - startTime;
1040
- return {
1041
- sectionsEmbedded: entries.length,
1042
- tokensUsed: result.tokensUsed,
1043
- cost: result.cost,
1044
- duration,
1045
- filesProcessed
1046
- };
1047
- });
1048
- var semanticSearch = (rootPath, query, options = {}) => Effect4.gen(function* () {
1049
- const resolvedRoot = path3.resolve(rootPath);
1050
- const provider = yield* Effect4.try({
1051
- try: () => createOpenAIProvider(),
1052
- catch: (e) => e
1053
- });
1054
- const dimensions = provider.dimensions;
1055
- const vectorStore = createVectorStore(resolvedRoot, dimensions);
1056
- const loaded = yield* vectorStore.load();
1057
- if (!loaded) {
1058
- return yield* Effect4.fail(
1059
- new Error("Embeddings not found. Run 'mdcontext embed' first.")
1060
- );
1061
- }
1062
- const queryResult = yield* Effect4.tryPromise({
1063
- try: () => provider.embed([query]),
1064
- catch: (e) => new Error(
1065
- `Query embedding failed: ${e instanceof Error ? e.message : String(e)}`
1066
- )
1067
- });
1068
- const queryVector = queryResult.embeddings[0];
1069
- if (!queryVector) {
1070
- return yield* Effect4.fail(new Error("Failed to generate query embedding"));
1071
- }
1072
- const limit = options.limit ?? 10;
1073
- const threshold = options.threshold ?? 0;
1074
- const searchResults = yield* vectorStore.search(
1075
- queryVector,
1076
- limit * 2,
1077
- threshold
1078
- );
1079
- let filteredResults = searchResults;
1080
- if (options.pathPattern) {
1081
- const pattern = options.pathPattern.replace(/\./g, "\\.").replace(/\*/g, ".*");
1082
- const regex = new RegExp(`^${pattern}$`, "i");
1083
- filteredResults = searchResults.filter((r) => regex.test(r.documentPath));
1084
- }
1085
- const results = filteredResults.slice(0, limit).map((r) => ({
1086
- sectionId: r.sectionId,
1087
- documentPath: r.documentPath,
1088
- heading: r.heading,
1089
- similarity: r.similarity
1090
- }));
1091
- return results;
1092
- });
1093
- var getEmbeddingStats = (rootPath) => Effect4.gen(function* () {
1094
- const resolvedRoot = path3.resolve(rootPath);
1095
- const vectorStore = createVectorStore(resolvedRoot, 1536);
1096
- const loaded = yield* vectorStore.load();
1097
- if (!loaded) {
1098
- return {
1099
- hasEmbeddings: false,
1100
- count: 0,
1101
- provider: "none",
1102
- dimensions: 0,
1103
- totalCost: 0,
1104
- totalTokens: 0
1105
- };
1106
- }
1107
- const stats = vectorStore.getStats();
1108
- return {
1109
- hasEmbeddings: true,
1110
- count: stats.count,
1111
- provider: stats.provider,
1112
- dimensions: stats.dimensions,
1113
- totalCost: stats.totalCost,
1114
- totalTokens: stats.totalTokens
1115
- };
1116
- });
888
+ var estimateEmbeddingCost = (rootPath, options = {}) =>
889
+ Effect4.gen(function* () {
890
+ const resolvedRoot = path3.resolve(rootPath);
891
+ const storage = createStorage(resolvedRoot);
892
+ const docIndex = yield* loadDocumentIndex(storage);
893
+ const sectionIndex = yield* loadSectionIndex(storage);
894
+ if (!docIndex || !sectionIndex) {
895
+ return yield* Effect4.fail(
896
+ new Error("Index not found. Run 'mdcontext index' first."),
897
+ );
898
+ }
899
+ const byDir = /* @__PURE__ */ new Map();
900
+ for (const section of Object.values(sectionIndex.sections)) {
901
+ if (section.tokenCount < 10) continue;
902
+ if (options.excludePatterns?.length) {
903
+ const excluded = options.excludePatterns.some((pattern) => {
904
+ const regex = new RegExp(
905
+ `^${pattern.replace(/\*/g, ".*").replace(/\?/g, ".")}$`,
906
+ );
907
+ return regex.test(section.documentPath);
908
+ });
909
+ if (excluded) continue;
910
+ }
911
+ const dir = path3.dirname(section.documentPath) || ".";
912
+ if (!byDir.has(dir)) {
913
+ byDir.set(dir, {
914
+ files: /* @__PURE__ */ new Set(),
915
+ sections: 0,
916
+ tokens: 0,
917
+ });
918
+ }
919
+ const entry = byDir.get(dir);
920
+ entry.files.add(section.documentPath);
921
+ entry.sections++;
922
+ entry.tokens += section.tokenCount;
923
+ }
924
+ const directoryEstimates = [];
925
+ let totalFiles = 0;
926
+ let totalSections = 0;
927
+ let totalTokens = 0;
928
+ for (const [dir, data] of byDir) {
929
+ directoryEstimates.push({
930
+ directory: dir,
931
+ fileCount: data.files.size,
932
+ sectionCount: data.sections,
933
+ estimatedTokens: data.tokens,
934
+ estimatedCost: (data.tokens / 1e6) * EMBEDDING_PRICE_PER_MILLION,
935
+ });
936
+ totalFiles += data.files.size;
937
+ totalSections += data.sections;
938
+ totalTokens += data.tokens;
939
+ }
940
+ directoryEstimates.sort((a, b) => a.directory.localeCompare(b.directory));
941
+ const estimatedTimeSeconds = Math.ceil(totalSections / 100) * 1.5;
942
+ return {
943
+ totalFiles,
944
+ totalSections,
945
+ totalTokens,
946
+ totalCost: (totalTokens / 1e6) * EMBEDDING_PRICE_PER_MILLION,
947
+ estimatedTimeSeconds,
948
+ byDirectory: directoryEstimates,
949
+ };
950
+ });
951
+ var buildEmbeddings = (rootPath, options = {}) =>
952
+ Effect4.gen(function* () {
953
+ const startTime = Date.now();
954
+ const resolvedRoot = path3.resolve(rootPath);
955
+ const storage = createStorage(resolvedRoot);
956
+ const docIndex = yield* loadDocumentIndex(storage);
957
+ const sectionIndex = yield* loadSectionIndex(storage);
958
+ if (!docIndex || !sectionIndex) {
959
+ return yield* Effect4.fail(
960
+ new Error("Index not found. Run 'mdcontext index' first."),
961
+ );
962
+ }
963
+ const provider =
964
+ options.provider ??
965
+ (yield* Effect4.try({
966
+ try: () => createOpenAIProvider(),
967
+ catch: (e) => e,
968
+ }));
969
+ const dimensions = provider.dimensions;
970
+ const vectorStore = createVectorStore(resolvedRoot, dimensions);
971
+ vectorStore.setProvider(provider.name);
972
+ if (!options.force) {
973
+ const loaded = yield* vectorStore.load();
974
+ if (loaded) {
975
+ const stats = vectorStore.getStats();
976
+ if (stats.count > 0) {
977
+ const duration2 = Date.now() - startTime;
978
+ const estimatedSavings =
979
+ (stats.totalTokens / 1e6) * EMBEDDING_PRICE_PER_MILLION;
980
+ return {
981
+ sectionsEmbedded: 0,
982
+ tokensUsed: 0,
983
+ cost: 0,
984
+ duration: duration2,
985
+ filesProcessed: 0,
986
+ cacheHit: true,
987
+ existingVectors: stats.count,
988
+ estimatedSavings,
989
+ };
990
+ }
991
+ }
992
+ }
993
+ const isExcluded = (docPath) => {
994
+ if (!options.excludePatterns?.length) return false;
995
+ return options.excludePatterns.some((pattern) => {
996
+ const regex = new RegExp(
997
+ `^${pattern.replace(/\*/g, ".*").replace(/\?/g, ".")}$`,
998
+ );
999
+ return regex.test(docPath);
1000
+ });
1001
+ };
1002
+ const sectionsByDoc = /* @__PURE__ */ new Map();
1003
+ for (const section of Object.values(sectionIndex.sections)) {
1004
+ const document = docIndex.documents[section.documentPath];
1005
+ if (!document) continue;
1006
+ if (section.tokenCount < 10) continue;
1007
+ if (isExcluded(section.documentPath)) continue;
1008
+ let parentHeading;
1009
+ if (section.level > 1) {
1010
+ const docSections = sectionIndex.byDocument[document.id] ?? [];
1011
+ for (const sibId of docSections) {
1012
+ const sib = sectionIndex.sections[sibId];
1013
+ if (
1014
+ sib &&
1015
+ sib.level === section.level - 1 &&
1016
+ sib.startLine < section.startLine
1017
+ ) {
1018
+ parentHeading = sib.heading;
1019
+ }
1020
+ }
1021
+ }
1022
+ const docPath = section.documentPath;
1023
+ if (!sectionsByDoc.has(docPath)) {
1024
+ sectionsByDoc.set(docPath, []);
1025
+ }
1026
+ sectionsByDoc.get(docPath).push({ section, parentHeading });
1027
+ }
1028
+ if (sectionsByDoc.size === 0) {
1029
+ const duration2 = Date.now() - startTime;
1030
+ return {
1031
+ sectionsEmbedded: 0,
1032
+ tokensUsed: 0,
1033
+ cost: 0,
1034
+ duration: duration2,
1035
+ filesProcessed: 0,
1036
+ };
1037
+ }
1038
+ const sectionsToEmbed = [];
1039
+ const docPaths = Array.from(sectionsByDoc.keys());
1040
+ let filesProcessed = 0;
1041
+ for (let fileIndex = 0; fileIndex < docPaths.length; fileIndex++) {
1042
+ const docPath = docPaths[fileIndex];
1043
+ const sections = sectionsByDoc.get(docPath);
1044
+ const document = docIndex.documents[docPath];
1045
+ if (!document) continue;
1046
+ if (options.onFileProgress) {
1047
+ options.onFileProgress({
1048
+ fileIndex: fileIndex + 1,
1049
+ totalFiles: docPaths.length,
1050
+ filePath: docPath,
1051
+ sectionCount: sections.length,
1052
+ });
1053
+ }
1054
+ const filePath = path3.join(resolvedRoot, docPath);
1055
+ let fileContent;
1056
+ try {
1057
+ fileContent = yield* Effect4.promise(() =>
1058
+ fs3.readFile(filePath, "utf-8"),
1059
+ );
1060
+ } catch {
1061
+ continue;
1062
+ }
1063
+ filesProcessed++;
1064
+ const lines = fileContent.split("\n");
1065
+ for (const { section, parentHeading } of sections) {
1066
+ const content = lines
1067
+ .slice(section.startLine - 1, section.endLine)
1068
+ .join("\n");
1069
+ const text = generateEmbeddingText(
1070
+ section,
1071
+ content,
1072
+ document.title,
1073
+ parentHeading,
1074
+ );
1075
+ sectionsToEmbed.push({ section, text });
1076
+ }
1077
+ }
1078
+ if (sectionsToEmbed.length === 0) {
1079
+ const duration2 = Date.now() - startTime;
1080
+ return {
1081
+ sectionsEmbedded: 0,
1082
+ tokensUsed: 0,
1083
+ cost: 0,
1084
+ duration: duration2,
1085
+ filesProcessed,
1086
+ };
1087
+ }
1088
+ const texts = sectionsToEmbed.map((s) => s.text);
1089
+ const result = yield* Effect4.tryPromise({
1090
+ try: () => provider.embed(texts),
1091
+ catch: (e) => {
1092
+ if (e instanceof InvalidApiKeyError) return e;
1093
+ return new Error(
1094
+ `Embedding failed: ${e instanceof Error ? e.message : String(e)}`,
1095
+ );
1096
+ },
1097
+ });
1098
+ const entries = [];
1099
+ for (let i = 0; i < sectionsToEmbed.length; i++) {
1100
+ const { section } = sectionsToEmbed[i] ?? { section: null };
1101
+ const embedding = result.embeddings[i];
1102
+ if (!section || !embedding) continue;
1103
+ entries.push({
1104
+ id: section.id,
1105
+ sectionId: section.id,
1106
+ documentPath: section.documentPath,
1107
+ heading: section.heading,
1108
+ embedding,
1109
+ });
1110
+ }
1111
+ yield* vectorStore.add(entries);
1112
+ vectorStore.addCost(result.cost, result.tokensUsed);
1113
+ yield* vectorStore.save();
1114
+ const duration = Date.now() - startTime;
1115
+ return {
1116
+ sectionsEmbedded: entries.length,
1117
+ tokensUsed: result.tokensUsed,
1118
+ cost: result.cost,
1119
+ duration,
1120
+ filesProcessed,
1121
+ };
1122
+ });
1123
+ var semanticSearch = (rootPath, query, options = {}) =>
1124
+ Effect4.gen(function* () {
1125
+ const resolvedRoot = path3.resolve(rootPath);
1126
+ const provider = yield* Effect4.try({
1127
+ try: () => createOpenAIProvider(),
1128
+ catch: (e) => e,
1129
+ });
1130
+ const dimensions = provider.dimensions;
1131
+ const vectorStore = createVectorStore(resolvedRoot, dimensions);
1132
+ const loaded = yield* vectorStore.load();
1133
+ if (!loaded) {
1134
+ return yield* Effect4.fail(
1135
+ new Error("Embeddings not found. Run 'mdcontext embed' first."),
1136
+ );
1137
+ }
1138
+ const queryResult = yield* Effect4.tryPromise({
1139
+ try: () => provider.embed([query]),
1140
+ catch: (e) =>
1141
+ new Error(
1142
+ `Query embedding failed: ${e instanceof Error ? e.message : String(e)}`,
1143
+ ),
1144
+ });
1145
+ const queryVector = queryResult.embeddings[0];
1146
+ if (!queryVector) {
1147
+ return yield* Effect4.fail(
1148
+ new Error("Failed to generate query embedding"),
1149
+ );
1150
+ }
1151
+ const limit = options.limit ?? 10;
1152
+ const threshold = options.threshold ?? 0;
1153
+ const searchResults = yield* vectorStore.search(
1154
+ queryVector,
1155
+ limit * 2,
1156
+ threshold,
1157
+ );
1158
+ let filteredResults = searchResults;
1159
+ if (options.pathPattern) {
1160
+ const pattern = options.pathPattern
1161
+ .replace(/\./g, "\\.")
1162
+ .replace(/\*/g, ".*");
1163
+ const regex = new RegExp(`^${pattern}$`, "i");
1164
+ filteredResults = searchResults.filter((r) => regex.test(r.documentPath));
1165
+ }
1166
+ const results = filteredResults.slice(0, limit).map((r) => ({
1167
+ sectionId: r.sectionId,
1168
+ documentPath: r.documentPath,
1169
+ heading: r.heading,
1170
+ similarity: r.similarity,
1171
+ }));
1172
+ return results;
1173
+ });
1174
+ var getEmbeddingStats = (rootPath) =>
1175
+ Effect4.gen(function* () {
1176
+ const resolvedRoot = path3.resolve(rootPath);
1177
+ const vectorStore = createVectorStore(resolvedRoot, 1536);
1178
+ const loaded = yield* vectorStore.load();
1179
+ if (!loaded) {
1180
+ return {
1181
+ hasEmbeddings: false,
1182
+ count: 0,
1183
+ provider: "none",
1184
+ dimensions: 0,
1185
+ totalCost: 0,
1186
+ totalTokens: 0,
1187
+ };
1188
+ }
1189
+ const stats = vectorStore.getStats();
1190
+ return {
1191
+ hasEmbeddings: true,
1192
+ count: stats.count,
1193
+ provider: stats.provider,
1194
+ dimensions: stats.dimensions,
1195
+ totalCost: stats.totalCost,
1196
+ totalTokens: stats.totalTokens,
1197
+ };
1198
+ });
1117
1199
 
1200
+ import { Effect as Effect5 } from "effect";
1118
1201
  // src/search/searcher.ts
1119
1202
  import * as fs4 from "fs/promises";
1120
1203
  import * as path4 from "path";
1121
- import { Effect as Effect5 } from "effect";
1122
1204
 
1123
1205
  // src/search/query-parser.ts
1124
1206
  var tokenize = (query) => {
1125
- const tokens = [];
1126
- let i = 0;
1127
- while (i < query.length) {
1128
- if (/\s/.test(query[i])) {
1129
- i++;
1130
- continue;
1131
- }
1132
- if (query[i] === '"') {
1133
- const start = i + 1;
1134
- i++;
1135
- while (i < query.length && query[i] !== '"') {
1136
- i++;
1137
- }
1138
- const value = query.slice(start, i);
1139
- tokens.push({ type: "PHRASE", value });
1140
- i++;
1141
- continue;
1142
- }
1143
- if (query[i] === "(") {
1144
- tokens.push({ type: "LPAREN", value: "(" });
1145
- i++;
1146
- continue;
1147
- }
1148
- if (query[i] === ")") {
1149
- tokens.push({ type: "RPAREN", value: ")" });
1150
- i++;
1151
- continue;
1152
- }
1153
- const wordMatch = query.slice(i).match(/^[^\s()"]+/);
1154
- if (wordMatch) {
1155
- const word = wordMatch[0];
1156
- const upperWord = word.toUpperCase();
1157
- if (upperWord === "AND") {
1158
- tokens.push({ type: "AND", value: "AND" });
1159
- } else if (upperWord === "OR") {
1160
- tokens.push({ type: "OR", value: "OR" });
1161
- } else if (upperWord === "NOT") {
1162
- tokens.push({ type: "NOT", value: "NOT" });
1163
- } else {
1164
- tokens.push({ type: "TERM", value: word });
1165
- }
1166
- i += word.length;
1167
- continue;
1168
- }
1169
- i++;
1170
- }
1171
- return tokens;
1207
+ const tokens = [];
1208
+ let i = 0;
1209
+ while (i < query.length) {
1210
+ if (/\s/.test(query[i])) {
1211
+ i++;
1212
+ continue;
1213
+ }
1214
+ if (query[i] === '"') {
1215
+ const start = i + 1;
1216
+ i++;
1217
+ while (i < query.length && query[i] !== '"') {
1218
+ i++;
1219
+ }
1220
+ const value = query.slice(start, i);
1221
+ tokens.push({ type: "PHRASE", value });
1222
+ i++;
1223
+ continue;
1224
+ }
1225
+ if (query[i] === "(") {
1226
+ tokens.push({ type: "LPAREN", value: "(" });
1227
+ i++;
1228
+ continue;
1229
+ }
1230
+ if (query[i] === ")") {
1231
+ tokens.push({ type: "RPAREN", value: ")" });
1232
+ i++;
1233
+ continue;
1234
+ }
1235
+ const wordMatch = query.slice(i).match(/^[^\s()"]+/);
1236
+ if (wordMatch) {
1237
+ const word = wordMatch[0];
1238
+ const upperWord = word.toUpperCase();
1239
+ if (upperWord === "AND") {
1240
+ tokens.push({ type: "AND", value: "AND" });
1241
+ } else if (upperWord === "OR") {
1242
+ tokens.push({ type: "OR", value: "OR" });
1243
+ } else if (upperWord === "NOT") {
1244
+ tokens.push({ type: "NOT", value: "NOT" });
1245
+ } else {
1246
+ tokens.push({ type: "TERM", value: word });
1247
+ }
1248
+ i += word.length;
1249
+ continue;
1250
+ }
1251
+ i++;
1252
+ }
1253
+ return tokens;
1172
1254
  };
1173
1255
  var Parser = class {
1174
- tokens;
1175
- pos = 0;
1176
- terms = [];
1177
- phrases = [];
1178
- constructor(tokens) {
1179
- this.tokens = tokens;
1180
- }
1181
- current() {
1182
- return this.tokens[this.pos];
1183
- }
1184
- advance() {
1185
- return this.tokens[this.pos++];
1186
- }
1187
- match(type) {
1188
- if (this.current()?.type === type) {
1189
- this.advance();
1190
- return true;
1191
- }
1192
- return false;
1193
- }
1194
- parse() {
1195
- if (this.tokens.length === 0) {
1196
- return null;
1197
- }
1198
- return this.parseExpr();
1199
- }
1200
- parseExpr() {
1201
- let left = this.parseAndExpr();
1202
- while (this.match("OR")) {
1203
- const right = this.parseAndExpr();
1204
- left = { type: "or", left, right };
1205
- }
1206
- return left;
1207
- }
1208
- parseAndExpr() {
1209
- let left = this.parseNotExpr();
1210
- while (this.match("AND") || this.isImplicitAnd()) {
1211
- const right = this.parseNotExpr();
1212
- left = { type: "and", left, right };
1213
- }
1214
- return left;
1215
- }
1216
- isImplicitAnd() {
1217
- const tok = this.current();
1218
- return tok?.type === "TERM" || tok?.type === "PHRASE" || tok?.type === "NOT" || tok?.type === "LPAREN";
1219
- }
1220
- parseNotExpr() {
1221
- if (this.match("NOT")) {
1222
- const operand = this.parseNotExpr();
1223
- return { type: "not", operand };
1224
- }
1225
- return this.parsePrimary();
1226
- }
1227
- parsePrimary() {
1228
- const tok = this.current();
1229
- if (this.match("LPAREN")) {
1230
- const expr = this.parseExpr();
1231
- this.match("RPAREN");
1232
- return expr;
1233
- }
1234
- if (tok?.type === "PHRASE") {
1235
- this.advance();
1236
- this.phrases.push(tok.value);
1237
- return { type: "phrase", value: tok.value };
1238
- }
1239
- if (tok?.type === "TERM") {
1240
- this.advance();
1241
- this.terms.push(tok.value);
1242
- return { type: "term", value: tok.value };
1243
- }
1244
- return { type: "term", value: "" };
1245
- }
1256
+ tokens;
1257
+ pos = 0;
1258
+ terms = [];
1259
+ phrases = [];
1260
+ constructor(tokens) {
1261
+ this.tokens = tokens;
1262
+ }
1263
+ current() {
1264
+ return this.tokens[this.pos];
1265
+ }
1266
+ advance() {
1267
+ return this.tokens[this.pos++];
1268
+ }
1269
+ match(type) {
1270
+ if (this.current()?.type === type) {
1271
+ this.advance();
1272
+ return true;
1273
+ }
1274
+ return false;
1275
+ }
1276
+ parse() {
1277
+ if (this.tokens.length === 0) {
1278
+ return null;
1279
+ }
1280
+ return this.parseExpr();
1281
+ }
1282
+ parseExpr() {
1283
+ let left = this.parseAndExpr();
1284
+ while (this.match("OR")) {
1285
+ const right = this.parseAndExpr();
1286
+ left = { type: "or", left, right };
1287
+ }
1288
+ return left;
1289
+ }
1290
+ parseAndExpr() {
1291
+ let left = this.parseNotExpr();
1292
+ while (this.match("AND") || this.isImplicitAnd()) {
1293
+ const right = this.parseNotExpr();
1294
+ left = { type: "and", left, right };
1295
+ }
1296
+ return left;
1297
+ }
1298
+ isImplicitAnd() {
1299
+ const tok = this.current();
1300
+ return (
1301
+ tok?.type === "TERM" ||
1302
+ tok?.type === "PHRASE" ||
1303
+ tok?.type === "NOT" ||
1304
+ tok?.type === "LPAREN"
1305
+ );
1306
+ }
1307
+ parseNotExpr() {
1308
+ if (this.match("NOT")) {
1309
+ const operand = this.parseNotExpr();
1310
+ return { type: "not", operand };
1311
+ }
1312
+ return this.parsePrimary();
1313
+ }
1314
+ parsePrimary() {
1315
+ const tok = this.current();
1316
+ if (this.match("LPAREN")) {
1317
+ const expr = this.parseExpr();
1318
+ this.match("RPAREN");
1319
+ return expr;
1320
+ }
1321
+ if (tok?.type === "PHRASE") {
1322
+ this.advance();
1323
+ this.phrases.push(tok.value);
1324
+ return { type: "phrase", value: tok.value };
1325
+ }
1326
+ if (tok?.type === "TERM") {
1327
+ this.advance();
1328
+ this.terms.push(tok.value);
1329
+ return { type: "term", value: tok.value };
1330
+ }
1331
+ return { type: "term", value: "" };
1332
+ }
1246
1333
  };
1247
1334
  var parseQuery = (query) => {
1248
- const tokens = tokenize(query);
1249
- if (tokens.length === 0) {
1250
- return null;
1251
- }
1252
- const parser = new Parser(tokens);
1253
- const ast = parser.parse();
1254
- if (!ast) {
1255
- return null;
1256
- }
1257
- return {
1258
- ast,
1259
- terms: parser.terms,
1260
- phrases: parser.phrases
1261
- };
1335
+ const tokens = tokenize(query);
1336
+ if (tokens.length === 0) {
1337
+ return null;
1338
+ }
1339
+ const parser = new Parser(tokens);
1340
+ const ast = parser.parse();
1341
+ if (!ast) {
1342
+ return null;
1343
+ }
1344
+ return {
1345
+ ast,
1346
+ terms: parser.terms,
1347
+ phrases: parser.phrases,
1348
+ };
1262
1349
  };
1263
1350
  var isAdvancedQuery = (query) => {
1264
- const tokens = tokenize(query);
1265
- return tokens.some(
1266
- (t) => t.type === "AND" || t.type === "OR" || t.type === "NOT" || t.type === "PHRASE" || t.type === "LPAREN"
1267
- );
1351
+ const tokens = tokenize(query);
1352
+ return tokens.some(
1353
+ (t) =>
1354
+ t.type === "AND" ||
1355
+ t.type === "OR" ||
1356
+ t.type === "NOT" ||
1357
+ t.type === "PHRASE" ||
1358
+ t.type === "LPAREN",
1359
+ );
1268
1360
  };
1269
1361
  var evaluateQuery = (ast, text) => {
1270
- const lowerText = text.toLowerCase();
1271
- const evaluate = (node) => {
1272
- switch (node.type) {
1273
- case "term": {
1274
- if (!node.value) return true;
1275
- return lowerText.includes(node.value.toLowerCase());
1276
- }
1277
- case "phrase": {
1278
- return lowerText.includes(node.value.toLowerCase());
1279
- }
1280
- case "and": {
1281
- return evaluate(node.left) && evaluate(node.right);
1282
- }
1283
- case "or": {
1284
- return evaluate(node.left) || evaluate(node.right);
1285
- }
1286
- case "not": {
1287
- return !evaluate(node.operand);
1288
- }
1289
- }
1290
- };
1291
- return evaluate(ast);
1362
+ const lowerText = text.toLowerCase();
1363
+ const evaluate = (node) => {
1364
+ switch (node.type) {
1365
+ case "term": {
1366
+ if (!node.value) return true;
1367
+ return lowerText.includes(node.value.toLowerCase());
1368
+ }
1369
+ case "phrase": {
1370
+ return lowerText.includes(node.value.toLowerCase());
1371
+ }
1372
+ case "and": {
1373
+ return evaluate(node.left) && evaluate(node.right);
1374
+ }
1375
+ case "or": {
1376
+ return evaluate(node.left) || evaluate(node.right);
1377
+ }
1378
+ case "not": {
1379
+ return !evaluate(node.operand);
1380
+ }
1381
+ }
1382
+ };
1383
+ return evaluate(ast);
1292
1384
  };
1293
1385
  var buildHighlightPattern = (parsed) => {
1294
- const patterns = [];
1295
- const escapeChars = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
1296
- for (const term of parsed.terms) {
1297
- if (term) {
1298
- patterns.push(`\\b${escapeChars(term)}\\b`);
1299
- }
1300
- }
1301
- for (const phrase of parsed.phrases) {
1302
- if (phrase) {
1303
- patterns.push(escapeChars(phrase));
1304
- }
1305
- }
1306
- if (patterns.length === 0) {
1307
- return /.^/;
1308
- }
1309
- return new RegExp(patterns.join("|"), "gi");
1386
+ const patterns = [];
1387
+ const escapeChars = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
1388
+ for (const term of parsed.terms) {
1389
+ if (term) {
1390
+ patterns.push(`\\b${escapeChars(term)}\\b`);
1391
+ }
1392
+ }
1393
+ for (const phrase of parsed.phrases) {
1394
+ if (phrase) {
1395
+ patterns.push(escapeChars(phrase));
1396
+ }
1397
+ }
1398
+ if (patterns.length === 0) {
1399
+ return /.^/;
1400
+ }
1401
+ return new RegExp(patterns.join("|"), "gi");
1310
1402
  };
1311
1403
 
1312
1404
  // src/search/searcher.ts
1313
1405
  var matchPath = (filePath, pattern) => {
1314
- const regexPattern = pattern.replace(/\./g, "\\.").replace(/\*/g, ".*").replace(/\?/g, ".");
1315
- const regex = new RegExp(`^${regexPattern}$`, "i");
1316
- return regex.test(filePath);
1406
+ const regexPattern = pattern
1407
+ .replace(/\./g, "\\.")
1408
+ .replace(/\*/g, ".*")
1409
+ .replace(/\?/g, ".");
1410
+ const regex = new RegExp(`^${regexPattern}$`, "i");
1411
+ return regex.test(filePath);
1317
1412
  };
1318
- var search = (rootPath, options = {}) => Effect5.gen(function* () {
1319
- const storage = createStorage(rootPath);
1320
- const docIndex = yield* loadDocumentIndex(storage);
1321
- const sectionIndex = yield* loadSectionIndex(storage);
1322
- if (!docIndex || !sectionIndex) {
1323
- return [];
1324
- }
1325
- const results = [];
1326
- const headingRegex = options.heading ? new RegExp(options.heading, "i") : null;
1327
- for (const section of Object.values(sectionIndex.sections)) {
1328
- if (headingRegex && !headingRegex.test(section.heading)) {
1329
- continue;
1330
- }
1331
- if (options.pathPattern && !matchPath(section.documentPath, options.pathPattern)) {
1332
- continue;
1333
- }
1334
- if (options.hasCode !== void 0 && section.hasCode !== options.hasCode) {
1335
- continue;
1336
- }
1337
- if (options.hasList !== void 0 && section.hasList !== options.hasList) {
1338
- continue;
1339
- }
1340
- if (options.hasTable !== void 0 && section.hasTable !== options.hasTable) {
1341
- continue;
1342
- }
1343
- if (options.minLevel !== void 0 && section.level < options.minLevel) {
1344
- continue;
1345
- }
1346
- if (options.maxLevel !== void 0 && section.level > options.maxLevel) {
1347
- continue;
1348
- }
1349
- const document = docIndex.documents[section.documentPath];
1350
- if (document) {
1351
- results.push({ section, document });
1352
- }
1353
- if (options.limit !== void 0 && results.length >= options.limit) {
1354
- break;
1355
- }
1356
- }
1357
- return results;
1358
- });
1359
- var searchContent = (rootPath, options = {}) => Effect5.gen(function* () {
1360
- const storage = createStorage(rootPath);
1361
- const docIndex = yield* loadDocumentIndex(storage);
1362
- const sectionIndex = yield* loadSectionIndex(storage);
1363
- if (!docIndex || !sectionIndex) {
1364
- return [];
1365
- }
1366
- let parsedQuery = null;
1367
- let contentRegex = null;
1368
- let highlightRegex = null;
1369
- if (options.content) {
1370
- if (isAdvancedQuery(options.content)) {
1371
- parsedQuery = parseQuery(options.content);
1372
- if (parsedQuery) {
1373
- highlightRegex = buildHighlightPattern(parsedQuery);
1374
- }
1375
- } else {
1376
- contentRegex = new RegExp(options.content, "gi");
1377
- highlightRegex = contentRegex;
1378
- }
1379
- }
1380
- const headingRegex = options.heading ? new RegExp(options.heading, "i") : null;
1381
- const results = [];
1382
- const sectionsByDoc = {};
1383
- for (const section of Object.values(sectionIndex.sections)) {
1384
- const docSections = sectionsByDoc[section.documentPath];
1385
- if (docSections) {
1386
- docSections.push(section);
1387
- } else {
1388
- sectionsByDoc[section.documentPath] = [section];
1389
- }
1390
- }
1391
- for (const [docPath, sections] of Object.entries(sectionsByDoc)) {
1392
- if (options.pathPattern && !matchPath(docPath, options.pathPattern)) {
1393
- continue;
1394
- }
1395
- const document = docIndex.documents[docPath];
1396
- if (!document) continue;
1397
- let fileContent = null;
1398
- let fileLines = [];
1399
- if (parsedQuery || contentRegex) {
1400
- const filePath = path4.join(storage.rootPath, docPath);
1401
- try {
1402
- fileContent = yield* Effect5.promise(
1403
- () => fs4.readFile(filePath, "utf-8")
1404
- );
1405
- fileLines = fileContent.split("\n");
1406
- } catch {
1407
- continue;
1408
- }
1409
- }
1410
- for (const section of sections) {
1411
- if (headingRegex && !headingRegex.test(section.heading)) {
1412
- continue;
1413
- }
1414
- if (options.hasCode !== void 0 && section.hasCode !== options.hasCode) {
1415
- continue;
1416
- }
1417
- if (options.hasList !== void 0 && section.hasList !== options.hasList) {
1418
- continue;
1419
- }
1420
- if (options.hasTable !== void 0 && section.hasTable !== options.hasTable) {
1421
- continue;
1422
- }
1423
- if (options.minLevel !== void 0 && section.level < options.minLevel) {
1424
- continue;
1425
- }
1426
- if (options.maxLevel !== void 0 && section.level > options.maxLevel) {
1427
- continue;
1428
- }
1429
- if ((parsedQuery || contentRegex) && fileContent) {
1430
- const sectionLines = fileLines.slice(
1431
- section.startLine - 1,
1432
- section.endLine
1433
- );
1434
- const sectionContent = sectionLines.join("\n");
1435
- if (parsedQuery) {
1436
- if (!evaluateQuery(parsedQuery.ast, sectionContent)) {
1437
- continue;
1438
- }
1439
- }
1440
- const matches = [];
1441
- const searchRegex = contentRegex || highlightRegex;
1442
- const contextBefore = options.contextBefore ?? 1;
1443
- const contextAfter = options.contextAfter ?? 1;
1444
- if (searchRegex) {
1445
- for (let i = 0; i < sectionLines.length; i++) {
1446
- const line = sectionLines[i];
1447
- if (line && searchRegex.test(line)) {
1448
- searchRegex.lastIndex = 0;
1449
- const absoluteLineNum = section.startLine + i;
1450
- const snippetStart = Math.max(0, i - contextBefore);
1451
- const snippetEnd = Math.min(
1452
- sectionLines.length,
1453
- i + contextAfter + 1
1454
- );
1455
- const snippetLines = sectionLines.slice(
1456
- snippetStart,
1457
- snippetEnd
1458
- );
1459
- const snippet = snippetLines.join("\n");
1460
- const contextLines = [];
1461
- for (let j = snippetStart; j < snippetEnd; j++) {
1462
- const ctxLine = sectionLines[j];
1463
- if (ctxLine !== void 0) {
1464
- contextLines.push({
1465
- lineNumber: section.startLine + j,
1466
- line: ctxLine,
1467
- isMatch: j === i
1468
- });
1469
- }
1470
- }
1471
- matches.push({
1472
- lineNumber: absoluteLineNum,
1473
- line,
1474
- snippet,
1475
- contextLines
1476
- });
1477
- }
1478
- }
1479
- }
1480
- if (parsedQuery || matches.length > 0) {
1481
- const result = {
1482
- section,
1483
- document,
1484
- sectionContent
1485
- };
1486
- if (matches.length > 0) {
1487
- results.push({ ...result, matches });
1488
- } else {
1489
- results.push(result);
1490
- }
1491
- if (options.limit !== void 0 && results.length >= options.limit) {
1492
- return results;
1493
- }
1494
- }
1495
- } else if (!parsedQuery && !contentRegex) {
1496
- results.push({ section, document });
1497
- if (options.limit !== void 0 && results.length >= options.limit) {
1498
- return results;
1499
- }
1500
- }
1501
- }
1502
- }
1503
- return results;
1504
- });
1413
+ var search = (rootPath, options = {}) =>
1414
+ Effect5.gen(function* () {
1415
+ const storage = createStorage(rootPath);
1416
+ const docIndex = yield* loadDocumentIndex(storage);
1417
+ const sectionIndex = yield* loadSectionIndex(storage);
1418
+ if (!docIndex || !sectionIndex) {
1419
+ return [];
1420
+ }
1421
+ const results = [];
1422
+ const headingRegex = options.heading
1423
+ ? new RegExp(options.heading, "i")
1424
+ : null;
1425
+ for (const section of Object.values(sectionIndex.sections)) {
1426
+ if (headingRegex && !headingRegex.test(section.heading)) {
1427
+ continue;
1428
+ }
1429
+ if (
1430
+ options.pathPattern &&
1431
+ !matchPath(section.documentPath, options.pathPattern)
1432
+ ) {
1433
+ continue;
1434
+ }
1435
+ if (options.hasCode !== void 0 && section.hasCode !== options.hasCode) {
1436
+ continue;
1437
+ }
1438
+ if (options.hasList !== void 0 && section.hasList !== options.hasList) {
1439
+ continue;
1440
+ }
1441
+ if (
1442
+ options.hasTable !== void 0 &&
1443
+ section.hasTable !== options.hasTable
1444
+ ) {
1445
+ continue;
1446
+ }
1447
+ if (options.minLevel !== void 0 && section.level < options.minLevel) {
1448
+ continue;
1449
+ }
1450
+ if (options.maxLevel !== void 0 && section.level > options.maxLevel) {
1451
+ continue;
1452
+ }
1453
+ const document = docIndex.documents[section.documentPath];
1454
+ if (document) {
1455
+ results.push({ section, document });
1456
+ }
1457
+ if (options.limit !== void 0 && results.length >= options.limit) {
1458
+ break;
1459
+ }
1460
+ }
1461
+ return results;
1462
+ });
1463
+ var searchContent = (rootPath, options = {}) =>
1464
+ Effect5.gen(function* () {
1465
+ const storage = createStorage(rootPath);
1466
+ const docIndex = yield* loadDocumentIndex(storage);
1467
+ const sectionIndex = yield* loadSectionIndex(storage);
1468
+ if (!docIndex || !sectionIndex) {
1469
+ return [];
1470
+ }
1471
+ let parsedQuery = null;
1472
+ let contentRegex = null;
1473
+ let highlightRegex = null;
1474
+ if (options.content) {
1475
+ if (isAdvancedQuery(options.content)) {
1476
+ parsedQuery = parseQuery(options.content);
1477
+ if (parsedQuery) {
1478
+ highlightRegex = buildHighlightPattern(parsedQuery);
1479
+ }
1480
+ } else {
1481
+ contentRegex = new RegExp(options.content, "gi");
1482
+ highlightRegex = contentRegex;
1483
+ }
1484
+ }
1485
+ const headingRegex = options.heading
1486
+ ? new RegExp(options.heading, "i")
1487
+ : null;
1488
+ const results = [];
1489
+ const sectionsByDoc = {};
1490
+ for (const section of Object.values(sectionIndex.sections)) {
1491
+ const docSections = sectionsByDoc[section.documentPath];
1492
+ if (docSections) {
1493
+ docSections.push(section);
1494
+ } else {
1495
+ sectionsByDoc[section.documentPath] = [section];
1496
+ }
1497
+ }
1498
+ for (const [docPath, sections] of Object.entries(sectionsByDoc)) {
1499
+ if (options.pathPattern && !matchPath(docPath, options.pathPattern)) {
1500
+ continue;
1501
+ }
1502
+ const document = docIndex.documents[docPath];
1503
+ if (!document) continue;
1504
+ let fileContent = null;
1505
+ let fileLines = [];
1506
+ if (parsedQuery || contentRegex) {
1507
+ const filePath = path4.join(storage.rootPath, docPath);
1508
+ try {
1509
+ fileContent = yield* Effect5.promise(() =>
1510
+ fs4.readFile(filePath, "utf-8"),
1511
+ );
1512
+ fileLines = fileContent.split("\n");
1513
+ } catch {
1514
+ continue;
1515
+ }
1516
+ }
1517
+ for (const section of sections) {
1518
+ if (headingRegex && !headingRegex.test(section.heading)) {
1519
+ continue;
1520
+ }
1521
+ if (options.hasCode !== void 0 && section.hasCode !== options.hasCode) {
1522
+ continue;
1523
+ }
1524
+ if (options.hasList !== void 0 && section.hasList !== options.hasList) {
1525
+ continue;
1526
+ }
1527
+ if (
1528
+ options.hasTable !== void 0 &&
1529
+ section.hasTable !== options.hasTable
1530
+ ) {
1531
+ continue;
1532
+ }
1533
+ if (options.minLevel !== void 0 && section.level < options.minLevel) {
1534
+ continue;
1535
+ }
1536
+ if (options.maxLevel !== void 0 && section.level > options.maxLevel) {
1537
+ continue;
1538
+ }
1539
+ if ((parsedQuery || contentRegex) && fileContent) {
1540
+ const sectionLines = fileLines.slice(
1541
+ section.startLine - 1,
1542
+ section.endLine,
1543
+ );
1544
+ const sectionContent = sectionLines.join("\n");
1545
+ if (parsedQuery) {
1546
+ if (!evaluateQuery(parsedQuery.ast, sectionContent)) {
1547
+ continue;
1548
+ }
1549
+ }
1550
+ const matches = [];
1551
+ const searchRegex = contentRegex || highlightRegex;
1552
+ const contextBefore = options.contextBefore ?? 1;
1553
+ const contextAfter = options.contextAfter ?? 1;
1554
+ if (searchRegex) {
1555
+ for (let i = 0; i < sectionLines.length; i++) {
1556
+ const line = sectionLines[i];
1557
+ if (line && searchRegex.test(line)) {
1558
+ searchRegex.lastIndex = 0;
1559
+ const absoluteLineNum = section.startLine + i;
1560
+ const snippetStart = Math.max(0, i - contextBefore);
1561
+ const snippetEnd = Math.min(
1562
+ sectionLines.length,
1563
+ i + contextAfter + 1,
1564
+ );
1565
+ const snippetLines = sectionLines.slice(
1566
+ snippetStart,
1567
+ snippetEnd,
1568
+ );
1569
+ const snippet = snippetLines.join("\n");
1570
+ const contextLines = [];
1571
+ for (let j = snippetStart; j < snippetEnd; j++) {
1572
+ const ctxLine = sectionLines[j];
1573
+ if (ctxLine !== void 0) {
1574
+ contextLines.push({
1575
+ lineNumber: section.startLine + j,
1576
+ line: ctxLine,
1577
+ isMatch: j === i,
1578
+ });
1579
+ }
1580
+ }
1581
+ matches.push({
1582
+ lineNumber: absoluteLineNum,
1583
+ line,
1584
+ snippet,
1585
+ contextLines,
1586
+ });
1587
+ }
1588
+ }
1589
+ }
1590
+ if (parsedQuery || matches.length > 0) {
1591
+ const result = {
1592
+ section,
1593
+ document,
1594
+ sectionContent,
1595
+ };
1596
+ if (matches.length > 0) {
1597
+ results.push({ ...result, matches });
1598
+ } else {
1599
+ results.push(result);
1600
+ }
1601
+ if (options.limit !== void 0 && results.length >= options.limit) {
1602
+ return results;
1603
+ }
1604
+ }
1605
+ } else if (!parsedQuery && !contentRegex) {
1606
+ results.push({ section, document });
1607
+ if (options.limit !== void 0 && results.length >= options.limit) {
1608
+ return results;
1609
+ }
1610
+ }
1611
+ }
1612
+ }
1613
+ return results;
1614
+ });
1505
1615
 
1506
1616
  export {
1507
- formatSummary,
1508
- formatAssembledContext,
1509
- summarizeFile,
1510
- assembleContext,
1511
- handleApiKeyError,
1512
- estimateEmbeddingCost,
1513
- buildEmbeddings,
1514
- semanticSearch,
1515
- getEmbeddingStats,
1516
- isAdvancedQuery,
1517
- search,
1518
- searchContent
1617
+ formatSummary,
1618
+ formatAssembledContext,
1619
+ summarizeFile,
1620
+ assembleContext,
1621
+ handleApiKeyError,
1622
+ estimateEmbeddingCost,
1623
+ buildEmbeddings,
1624
+ semanticSearch,
1625
+ getEmbeddingStats,
1626
+ isAdvancedQuery,
1627
+ search,
1628
+ searchContent,
1519
1629
  };