@cosmocoder/mcp-web-docs 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (240) hide show
  1. package/LICENSE +22 -0
  2. package/README.md +368 -0
  3. package/build/__mocks__/embeddings.d.ts +17 -0
  4. package/build/__mocks__/embeddings.js +66 -0
  5. package/build/__mocks__/embeddings.js.map +1 -0
  6. package/build/config.d.ts +44 -0
  7. package/build/config.js +158 -0
  8. package/build/config.js.map +1 -0
  9. package/build/config.test.d.ts +1 -0
  10. package/build/config.test.js +165 -0
  11. package/build/config.test.js.map +1 -0
  12. package/build/crawler/auth.d.ts +128 -0
  13. package/build/crawler/auth.js +546 -0
  14. package/build/crawler/auth.js.map +1 -0
  15. package/build/crawler/auth.test.d.ts +1 -0
  16. package/build/crawler/auth.test.js +174 -0
  17. package/build/crawler/auth.test.js.map +1 -0
  18. package/build/crawler/base.d.ts +24 -0
  19. package/build/crawler/base.js +149 -0
  20. package/build/crawler/base.js.map +1 -0
  21. package/build/crawler/base.test.d.ts +1 -0
  22. package/build/crawler/base.test.js +234 -0
  23. package/build/crawler/base.test.js.map +1 -0
  24. package/build/crawler/browser-config.d.ts +2 -0
  25. package/build/crawler/browser-config.js +29 -0
  26. package/build/crawler/browser-config.js.map +1 -0
  27. package/build/crawler/browser-config.test.d.ts +1 -0
  28. package/build/crawler/browser-config.test.js +56 -0
  29. package/build/crawler/browser-config.test.js.map +1 -0
  30. package/build/crawler/cheerio.d.ts +11 -0
  31. package/build/crawler/cheerio.js +134 -0
  32. package/build/crawler/cheerio.js.map +1 -0
  33. package/build/crawler/chromium.d.ts +21 -0
  34. package/build/crawler/chromium.js +596 -0
  35. package/build/crawler/chromium.js.map +1 -0
  36. package/build/crawler/content-extractor-types.d.ts +25 -0
  37. package/build/crawler/content-extractor-types.js +2 -0
  38. package/build/crawler/content-extractor-types.js.map +1 -0
  39. package/build/crawler/content-extractors.d.ts +9 -0
  40. package/build/crawler/content-extractors.js +9 -0
  41. package/build/crawler/content-extractors.js.map +1 -0
  42. package/build/crawler/content-utils.d.ts +2 -0
  43. package/build/crawler/content-utils.js +22 -0
  44. package/build/crawler/content-utils.js.map +1 -0
  45. package/build/crawler/content-utils.test.d.ts +1 -0
  46. package/build/crawler/content-utils.test.js +99 -0
  47. package/build/crawler/content-utils.test.js.map +1 -0
  48. package/build/crawler/crawlee-crawler.d.ts +63 -0
  49. package/build/crawler/crawlee-crawler.js +342 -0
  50. package/build/crawler/crawlee-crawler.js.map +1 -0
  51. package/build/crawler/crawlee-crawler.test.d.ts +1 -0
  52. package/build/crawler/crawlee-crawler.test.js +280 -0
  53. package/build/crawler/crawlee-crawler.test.js.map +1 -0
  54. package/build/crawler/default-extractor.d.ts +4 -0
  55. package/build/crawler/default-extractor.js +26 -0
  56. package/build/crawler/default-extractor.js.map +1 -0
  57. package/build/crawler/default-extractor.test.d.ts +1 -0
  58. package/build/crawler/default-extractor.test.js +200 -0
  59. package/build/crawler/default-extractor.test.js.map +1 -0
  60. package/build/crawler/default.d.ts +11 -0
  61. package/build/crawler/default.js +138 -0
  62. package/build/crawler/default.js.map +1 -0
  63. package/build/crawler/docs-crawler.d.ts +26 -0
  64. package/build/crawler/docs-crawler.js +97 -0
  65. package/build/crawler/docs-crawler.js.map +1 -0
  66. package/build/crawler/docs-crawler.test.d.ts +1 -0
  67. package/build/crawler/docs-crawler.test.js +185 -0
  68. package/build/crawler/docs-crawler.test.js.map +1 -0
  69. package/build/crawler/factory.d.ts +6 -0
  70. package/build/crawler/factory.js +83 -0
  71. package/build/crawler/factory.js.map +1 -0
  72. package/build/crawler/github-pages-extractor.d.ts +4 -0
  73. package/build/crawler/github-pages-extractor.js +33 -0
  74. package/build/crawler/github-pages-extractor.js.map +1 -0
  75. package/build/crawler/github-pages-extractor.test.d.ts +1 -0
  76. package/build/crawler/github-pages-extractor.test.js +184 -0
  77. package/build/crawler/github-pages-extractor.test.js.map +1 -0
  78. package/build/crawler/github.d.ts +20 -0
  79. package/build/crawler/github.js +181 -0
  80. package/build/crawler/github.js.map +1 -0
  81. package/build/crawler/github.test.d.ts +1 -0
  82. package/build/crawler/github.test.js +326 -0
  83. package/build/crawler/github.test.js.map +1 -0
  84. package/build/crawler/puppeteer.d.ts +16 -0
  85. package/build/crawler/puppeteer.js +191 -0
  86. package/build/crawler/puppeteer.js.map +1 -0
  87. package/build/crawler/queue-manager.d.ts +43 -0
  88. package/build/crawler/queue-manager.js +169 -0
  89. package/build/crawler/queue-manager.js.map +1 -0
  90. package/build/crawler/queue-manager.test.d.ts +1 -0
  91. package/build/crawler/queue-manager.test.js +509 -0
  92. package/build/crawler/queue-manager.test.js.map +1 -0
  93. package/build/crawler/site-rules.d.ts +11 -0
  94. package/build/crawler/site-rules.js +104 -0
  95. package/build/crawler/site-rules.js.map +1 -0
  96. package/build/crawler/site-rules.test.d.ts +1 -0
  97. package/build/crawler/site-rules.test.js +139 -0
  98. package/build/crawler/site-rules.test.js.map +1 -0
  99. package/build/crawler/storybook-extractor.d.ts +34 -0
  100. package/build/crawler/storybook-extractor.js +767 -0
  101. package/build/crawler/storybook-extractor.js.map +1 -0
  102. package/build/crawler/storybook-extractor.test.d.ts +1 -0
  103. package/build/crawler/storybook-extractor.test.js +491 -0
  104. package/build/crawler/storybook-extractor.test.js.map +1 -0
  105. package/build/embeddings/fastembed.d.ts +25 -0
  106. package/build/embeddings/fastembed.js +188 -0
  107. package/build/embeddings/fastembed.js.map +1 -0
  108. package/build/embeddings/fastembed.test.d.ts +1 -0
  109. package/build/embeddings/fastembed.test.js +307 -0
  110. package/build/embeddings/fastembed.test.js.map +1 -0
  111. package/build/embeddings/openai.d.ts +8 -0
  112. package/build/embeddings/openai.js +56 -0
  113. package/build/embeddings/openai.js.map +1 -0
  114. package/build/embeddings/types.d.ts +4 -0
  115. package/build/embeddings/types.js +2 -0
  116. package/build/embeddings/types.js.map +1 -0
  117. package/build/index.d.ts +2 -0
  118. package/build/index.js +1007 -0
  119. package/build/index.js.map +1 -0
  120. package/build/index.test.d.ts +1 -0
  121. package/build/index.test.js +364 -0
  122. package/build/index.test.js.map +1 -0
  123. package/build/indexing/queue-manager.d.ts +36 -0
  124. package/build/indexing/queue-manager.js +86 -0
  125. package/build/indexing/queue-manager.js.map +1 -0
  126. package/build/indexing/queue-manager.test.d.ts +1 -0
  127. package/build/indexing/queue-manager.test.js +257 -0
  128. package/build/indexing/queue-manager.test.js.map +1 -0
  129. package/build/indexing/status.d.ts +39 -0
  130. package/build/indexing/status.js +207 -0
  131. package/build/indexing/status.js.map +1 -0
  132. package/build/indexing/status.test.d.ts +1 -0
  133. package/build/indexing/status.test.js +246 -0
  134. package/build/indexing/status.test.js.map +1 -0
  135. package/build/processor/content.d.ts +16 -0
  136. package/build/processor/content.js +286 -0
  137. package/build/processor/content.js.map +1 -0
  138. package/build/processor/content.test.d.ts +1 -0
  139. package/build/processor/content.test.js +369 -0
  140. package/build/processor/content.test.js.map +1 -0
  141. package/build/processor/markdown.d.ts +11 -0
  142. package/build/processor/markdown.js +256 -0
  143. package/build/processor/markdown.js.map +1 -0
  144. package/build/processor/markdown.test.d.ts +1 -0
  145. package/build/processor/markdown.test.js +312 -0
  146. package/build/processor/markdown.test.js.map +1 -0
  147. package/build/processor/metadata-parser.d.ts +37 -0
  148. package/build/processor/metadata-parser.js +245 -0
  149. package/build/processor/metadata-parser.js.map +1 -0
  150. package/build/processor/metadata-parser.test.d.ts +1 -0
  151. package/build/processor/metadata-parser.test.js +357 -0
  152. package/build/processor/metadata-parser.test.js.map +1 -0
  153. package/build/processor/processor.d.ts +8 -0
  154. package/build/processor/processor.js +190 -0
  155. package/build/processor/processor.js.map +1 -0
  156. package/build/processor/processor.test.d.ts +1 -0
  157. package/build/processor/processor.test.js +357 -0
  158. package/build/processor/processor.test.js.map +1 -0
  159. package/build/rag/cache.d.ts +10 -0
  160. package/build/rag/cache.js +10 -0
  161. package/build/rag/cache.js.map +1 -0
  162. package/build/rag/code-generator.d.ts +11 -0
  163. package/build/rag/code-generator.js +30 -0
  164. package/build/rag/code-generator.js.map +1 -0
  165. package/build/rag/context-assembler.d.ts +23 -0
  166. package/build/rag/context-assembler.js +113 -0
  167. package/build/rag/context-assembler.js.map +1 -0
  168. package/build/rag/docs-search.d.ts +55 -0
  169. package/build/rag/docs-search.js +380 -0
  170. package/build/rag/docs-search.js.map +1 -0
  171. package/build/rag/pipeline.d.ts +26 -0
  172. package/build/rag/pipeline.js +91 -0
  173. package/build/rag/pipeline.js.map +1 -0
  174. package/build/rag/query-processor.d.ts +14 -0
  175. package/build/rag/query-processor.js +57 -0
  176. package/build/rag/query-processor.js.map +1 -0
  177. package/build/rag/reranker.d.ts +55 -0
  178. package/build/rag/reranker.js +210 -0
  179. package/build/rag/reranker.js.map +1 -0
  180. package/build/rag/response-generator.d.ts +20 -0
  181. package/build/rag/response-generator.js +101 -0
  182. package/build/rag/response-generator.js.map +1 -0
  183. package/build/rag/retriever.d.ts +19 -0
  184. package/build/rag/retriever.js +111 -0
  185. package/build/rag/retriever.js.map +1 -0
  186. package/build/rag/validator.d.ts +22 -0
  187. package/build/rag/validator.js +128 -0
  188. package/build/rag/validator.js.map +1 -0
  189. package/build/rag/version-manager.d.ts +23 -0
  190. package/build/rag/version-manager.js +98 -0
  191. package/build/rag/version-manager.js.map +1 -0
  192. package/build/setupTests.d.ts +4 -0
  193. package/build/setupTests.js +50 -0
  194. package/build/setupTests.js.map +1 -0
  195. package/build/storage/storage.d.ts +38 -0
  196. package/build/storage/storage.js +700 -0
  197. package/build/storage/storage.js.map +1 -0
  198. package/build/storage/storage.test.d.ts +1 -0
  199. package/build/storage/storage.test.js +338 -0
  200. package/build/storage/storage.test.js.map +1 -0
  201. package/build/types/rag.d.ts +27 -0
  202. package/build/types/rag.js +2 -0
  203. package/build/types/rag.js.map +1 -0
  204. package/build/types.d.ts +120 -0
  205. package/build/types.js +2 -0
  206. package/build/types.js.map +1 -0
  207. package/build/util/content-utils.d.ts +31 -0
  208. package/build/util/content-utils.js +120 -0
  209. package/build/util/content-utils.js.map +1 -0
  210. package/build/util/content.d.ts +1 -0
  211. package/build/util/content.js +16 -0
  212. package/build/util/content.js.map +1 -0
  213. package/build/util/docs.d.ts +1 -0
  214. package/build/util/docs.js +26 -0
  215. package/build/util/docs.js.map +1 -0
  216. package/build/util/docs.test.d.ts +1 -0
  217. package/build/util/docs.test.js +49 -0
  218. package/build/util/docs.test.js.map +1 -0
  219. package/build/util/favicon.d.ts +6 -0
  220. package/build/util/favicon.js +88 -0
  221. package/build/util/favicon.js.map +1 -0
  222. package/build/util/favicon.test.d.ts +1 -0
  223. package/build/util/favicon.test.js +140 -0
  224. package/build/util/favicon.test.js.map +1 -0
  225. package/build/util/logger.d.ts +17 -0
  226. package/build/util/logger.js +72 -0
  227. package/build/util/logger.js.map +1 -0
  228. package/build/util/logger.test.d.ts +1 -0
  229. package/build/util/logger.test.js +46 -0
  230. package/build/util/logger.test.js.map +1 -0
  231. package/build/util/security.d.ts +312 -0
  232. package/build/util/security.js +719 -0
  233. package/build/util/security.js.map +1 -0
  234. package/build/util/security.test.d.ts +1 -0
  235. package/build/util/security.test.js +524 -0
  236. package/build/util/security.test.js.map +1 -0
  237. package/build/util/site-detector.d.ts +22 -0
  238. package/build/util/site-detector.js +42 -0
  239. package/build/util/site-detector.js.map +1 -0
  240. package/package.json +112 -0
@@ -0,0 +1,312 @@
1
+ import { processMarkdownContent, processExtractedContent } from './markdown.js';
2
+ describe('Markdown Processor', () => {
3
+ describe('processMarkdownContent', () => {
4
+ it('should process simple markdown content', async () => {
5
+ const page = {
6
+ url: 'https://example.com/docs/guide.md',
7
+ path: '/docs/guide.md',
8
+ title: 'Guide',
9
+ content: `# Introduction
10
+
11
+ This is the introduction to our guide.
12
+
13
+ ## Getting Started
14
+
15
+ Here's how to get started with our library.
16
+
17
+ ## Installation
18
+
19
+ Run the following command:
20
+
21
+ \`\`\`bash
22
+ npm install our-library
23
+ \`\`\`
24
+ `,
25
+ };
26
+ const result = await processMarkdownContent(page);
27
+ expect(result).toBeDefined();
28
+ // Title comes from page title or front matter, not H1 necessarily
29
+ expect(result?.article.title).toBeTruthy();
30
+ expect(result?.article.url).toBe(page.url);
31
+ expect(result?.article.path).toBe(page.path);
32
+ expect(result?.article.components.length).toBeGreaterThan(0);
33
+ });
34
+ it('should extract front matter', async () => {
35
+ const page = {
36
+ url: 'https://example.com/docs/page.md',
37
+ path: '/docs/page.md',
38
+ title: 'Page',
39
+ content: `---
40
+ title: Custom Title
41
+ description: A custom description
42
+ ---
43
+
44
+ # Heading
45
+
46
+ Content here.
47
+ `,
48
+ };
49
+ const result = await processMarkdownContent(page);
50
+ expect(result).toBeDefined();
51
+ expect(result?.article.title).toBe('Custom Title');
52
+ });
53
+ it('should preserve code blocks', async () => {
54
+ const page = {
55
+ url: 'https://example.com/docs/code.md',
56
+ path: '/docs/code.md',
57
+ title: 'Code Examples',
58
+ content: `# Code Examples
59
+
60
+ Here is some JavaScript code:
61
+
62
+ \`\`\`javascript
63
+ function hello() {
64
+ console.log('Hello, world!');
65
+ }
66
+ \`\`\`
67
+
68
+ And here is Python:
69
+
70
+ \`\`\`python
71
+ def hello():
72
+ print("Hello, world!")
73
+ \`\`\`
74
+ `,
75
+ };
76
+ const result = await processMarkdownContent(page);
77
+ expect(result).toBeDefined();
78
+ expect(result?.content).toContain('```javascript');
79
+ expect(result?.content).toContain('console.log');
80
+ expect(result?.content).toContain('```python');
81
+ });
82
+ it('should handle markdown without headers', async () => {
83
+ const page = {
84
+ url: 'https://example.com/docs/simple.md',
85
+ path: '/docs/simple.md',
86
+ title: 'Simple Page',
87
+ content: `This is just a simple paragraph.
88
+
89
+ Another paragraph here.
90
+
91
+ And a final one.
92
+ `,
93
+ };
94
+ const result = await processMarkdownContent(page);
95
+ expect(result).toBeDefined();
96
+ expect(result?.article.components.length).toBeGreaterThan(0);
97
+ });
98
+ it('should return undefined for empty content', async () => {
99
+ const page = {
100
+ url: 'https://example.com/docs/empty.md',
101
+ path: '/docs/empty.md',
102
+ title: 'Empty',
103
+ content: '',
104
+ };
105
+ const result = await processMarkdownContent(page);
106
+ expect(result).toBeUndefined();
107
+ });
108
+ it('should handle whitespace-only content', async () => {
109
+ const page = {
110
+ url: 'https://example.com/docs/whitespace.md',
111
+ path: '/docs/whitespace.md',
112
+ title: 'Whitespace',
113
+ content: ' \n\n \n ',
114
+ };
115
+ const result = await processMarkdownContent(page);
116
+ expect(result).toBeUndefined();
117
+ });
118
+ it('should handle nested headers', async () => {
119
+ const page = {
120
+ url: 'https://example.com/docs/nested.md',
121
+ path: '/docs/nested.md',
122
+ title: 'Nested',
123
+ content: `# Top Level
124
+
125
+ Introduction text.
126
+
127
+ ## Second Level
128
+
129
+ Section content.
130
+
131
+ ### Third Level
132
+
133
+ Subsection content.
134
+
135
+ #### Fourth Level
136
+
137
+ Deep content.
138
+ `,
139
+ };
140
+ const result = await processMarkdownContent(page);
141
+ expect(result).toBeDefined();
142
+ expect(result?.article.components.length).toBeGreaterThanOrEqual(4);
143
+ });
144
+ it('should detect Docusaurus-style headers with unicode markers', async () => {
145
+ const page = {
146
+ url: 'https://example.com/docs/docusaurus.md',
147
+ path: '/docs/docusaurus.md',
148
+ title: 'Docusaurus Page',
149
+ content: `# Main Title
150
+
151
+ Introduction paragraph.
152
+
153
+ Hooks\u200B
154
+
155
+ This section covers hooks.
156
+
157
+ Example\u200B
158
+
159
+ Here is an example.
160
+ `,
161
+ };
162
+ const result = await processMarkdownContent(page);
163
+ expect(result).toBeDefined();
164
+ expect(result?.article.components.length).toBeGreaterThan(1);
165
+ });
166
+ it('should handle special characters in content', async () => {
167
+ const page = {
168
+ url: 'https://example.com/docs/special.md',
169
+ path: '/docs/special.md',
170
+ title: 'Special Characters',
171
+ content: `# Special Characters
172
+
173
+ Here are some special characters: < > & " ' \` * _ [] () {}
174
+
175
+ ## Math Symbols
176
+
177
+ α β γ δ ε × ÷ ± ≠ ≤ ≥
178
+
179
+ ## Emojis
180
+
181
+ 🚀 📚 💡 ✅ ❌
182
+ `,
183
+ };
184
+ const result = await processMarkdownContent(page);
185
+ expect(result).toBeDefined();
186
+ expect(result?.content).toContain('<');
187
+ expect(result?.content).toContain('α');
188
+ expect(result?.content).toContain('🚀');
189
+ });
190
+ });
191
+ describe('processExtractedContent', () => {
192
+ it('should process pre-extracted content', async () => {
193
+ const page = {
194
+ url: 'https://example.com/component',
195
+ path: '/component',
196
+ title: 'Button Component',
197
+ content: `# Button
198
+
199
+ A versatile button component.
200
+
201
+ ## Props
202
+
203
+ | Prop | Type | Default |
204
+ |------|------|---------|
205
+ | variant | string | 'primary' |
206
+ | disabled | boolean | false |
207
+
208
+ ## Example
209
+
210
+ \`\`\`jsx
211
+ <Button variant="secondary">Click me</Button>
212
+ \`\`\`
213
+ `,
214
+ extractorUsed: 'StorybookExtractor',
215
+ };
216
+ const result = await processExtractedContent(page);
217
+ expect(result).toBeDefined();
218
+ expect(result?.article.title).toBe('Button');
219
+ expect(result?.article.components.length).toBeGreaterThan(0);
220
+ });
221
+ it('should handle content without sections', async () => {
222
+ const page = {
223
+ url: 'https://example.com/simple',
224
+ path: '/simple',
225
+ title: 'Simple Page',
226
+ content: `Just some plain text content without any headers or structure.
227
+
228
+ This is another paragraph.`,
229
+ extractorUsed: 'DefaultExtractor',
230
+ };
231
+ const result = await processExtractedContent(page);
232
+ expect(result).toBeDefined();
233
+ expect(result?.article.components.length).toBe(1);
234
+ expect(result?.content).toContain('Just some plain text');
235
+ });
236
+ it('should return undefined for empty extracted content', async () => {
237
+ const page = {
238
+ url: 'https://example.com/empty',
239
+ path: '/empty',
240
+ title: 'Empty',
241
+ content: '',
242
+ extractorUsed: 'StorybookExtractor',
243
+ };
244
+ const result = await processExtractedContent(page);
245
+ expect(result).toBeUndefined();
246
+ });
247
+ it('should preserve markdown formatting in extracted content', async () => {
248
+ const page = {
249
+ url: 'https://example.com/formatted',
250
+ path: '/formatted',
251
+ title: 'Formatted',
252
+ content: `# Formatted Content
253
+
254
+ Here is **bold** and *italic* text.
255
+
256
+ - List item 1
257
+ - List item 2
258
+ - List item 3
259
+
260
+ > A blockquote here
261
+
262
+ And some \`inline code\` too.
263
+ `,
264
+ extractorUsed: 'GithubPagesExtractor',
265
+ };
266
+ const result = await processExtractedContent(page);
267
+ expect(result).toBeDefined();
268
+ expect(result?.content).toContain('**bold**');
269
+ expect(result?.content).toContain('*italic*');
270
+ expect(result?.content).toContain('- List item');
271
+ });
272
+ it('should use page title when no H1 found', async () => {
273
+ const page = {
274
+ url: 'https://example.com/notitle',
275
+ path: '/notitle',
276
+ title: 'Page Title from Crawl',
277
+ content: `## Only H2 Headers
278
+
279
+ Some content here.
280
+
281
+ ## Another Section
282
+
283
+ More content.
284
+ `,
285
+ extractorUsed: 'StorybookExtractor',
286
+ };
287
+ const result = await processExtractedContent(page);
288
+ expect(result).toBeDefined();
289
+ expect(result?.article.title).toBe('Page Title from Crawl');
290
+ });
291
+ it('should handle very long content', async () => {
292
+ const longContent = `# Long Document
293
+
294
+ ${Array(100)
295
+ .fill(null)
296
+ .map((_, i) => `## Section ${i + 1}\n\nThis is the content for section ${i + 1}. Lorem ipsum dolor sit amet.`)
297
+ .join('\n\n')}
298
+ `;
299
+ const page = {
300
+ url: 'https://example.com/long',
301
+ path: '/long',
302
+ title: 'Long Document',
303
+ content: longContent,
304
+ extractorUsed: 'DefaultExtractor',
305
+ };
306
+ const result = await processExtractedContent(page);
307
+ expect(result).toBeDefined();
308
+ expect(result?.article.components.length).toBeGreaterThan(50);
309
+ });
310
+ });
311
+ });
312
+ //# sourceMappingURL=markdown.test.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"markdown.test.js","sourceRoot":"","sources":["../../src/processor/markdown.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAC;AAGhF,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;IAClC,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;QACtC,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;YACtD,MAAM,IAAI,GAAgB;gBACxB,GAAG,EAAE,mCAAmC;gBACxC,IAAI,EAAE,gBAAgB;gBACtB,KAAK,EAAE,OAAO;gBACd,OAAO,EAAE;;;;;;;;;;;;;;;CAehB;aACM,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,sBAAsB,CAAC,IAAI,CAAC,CAAC;YAElD,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC7B,kEAAkE;YAClE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,UAAU,EAAE,CAAC;YAC3C,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC3C,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7C,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAC/D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;YAC3C,MAAM,IAAI,GAAgB;gBACxB,GAAG,EAAE,kCAAkC;gBACvC,IAAI,EAAE,eAAe;gBACrB,KAAK,EAAE,MAAM;gBACb,OAAO,EAAE;;;;;;;;CAQhB;aACM,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,sBAAsB,CAAC,IAAI,CAAC,CAAC;YAElD,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC7B,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACrD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;YAC3C,MAAM,IAAI,GAAgB;gBACxB,GAAG,EAAE,kCAAkC;gBACvC,IAAI,EAAE,eAAe;gBACrB,KAAK,EAAE,eAAe;gBACtB,OAAO,EAAE;;;;;;;;;;;;;;;;CAgBhB;aACM,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,sBAAsB,CAAC,IAAI,CAAC,CAAC;YAElD,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC7B,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;YACnD,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;YACjD,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;YACtD,MAAM,IAAI,GAAgB;gBACxB,GAAG,EAAE,oCAAoC;gBACzC,IAAI,EAAE,iBAAiB;gBACvB,KAAK,EAAE,aAAa;gBACpB,OAAO,EAAE;;;;;CAKhB;aACM,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,sBAAsB,CAAC,IAAI,CAAC,CAAC;YAElD,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC7B,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAC/D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,MAAM,IAAI,GAAgB;gBACxB,GAAG,EAAE,mCAAmC;gBACxC,IAAI,EAAE,gBAAgB;gBACtB,KAAK,EAAE,OAAO;gBACd,OAAO,EAAE,EAAE;aACZ,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,sBAAsB,CAAC,IAAI,CAAC,CAAC;YAClD,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,EAAE,CAAC;QACjC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACrD,MAAM,IAAI,GAAgB;gBACxB,GAAG,EAAE,wCAAwC;gBAC7C,IAAI,EAAE,qBAAqB;gBAC3B,KAAK,EAAE,YAAY;gBACnB,OAAO,EAAE,iBAAiB;aAC3B,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,sBAAsB,CAAC,IAAI,CAAC,CAAC;YAClD,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,EAAE,CAAC;QACjC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8BAA8B,EAAE,KAAK,IAAI,EAAE;YAC5C,MAAM,IAAI,GAAgB;gBACxB,GAAG,EAAE,oCAAoC;gBACzC,IAAI,EAAE,iBAAiB;gBACvB,KAAK,EAAE,QAAQ;gBACf,OAAO,EAAE;;;;;;;;;;;;;;;CAehB;aACM,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,sBAAsB,CAAC,IAAI,CAAC,CAAC;YAElD,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC7B,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;QACtE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6DAA6D,EAAE,KAAK,IAAI,EAAE;YAC3E,MAAM,IAAI,GAAgB;gBACxB,GAAG,EAAE,wCAAwC;gBAC7C,IAAI,EAAE,qBAAqB;gBAC3B,KAAK,EAAE,iBAAiB;gBACxB,OAAO,EAAE;;;;;;;;;;;CAWhB;aACM,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,sBAAsB,CAAC,IAAI,CAAC,CAAC;YAElD,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC7B,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAC/D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;YAC3D,MAAM,IAAI,GAAgB;gBACxB,GAAG,EAAE,qCAAqC;gBAC1C,IAAI,EAAE,kBAAkB;gBACxB,KAAK,EAAE,oBAAoB;gBAC3B,OAAO,EAAE;;;;;;;;;;;CAWhB;aACM,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,sBAAsB,CAAC,IAAI,CAAC,CAAC;YAElD,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC7B,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YACvC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YACvC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;QACvC,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;YACpD,MAAM,IAAI,GAAgB;gBACxB,GAAG,EAAE,+BAA+B;gBACpC,IAAI,EAAE,YAAY;gBAClB,KAAK,EAAE,kBAAkB;gBACzB,OAAO,EAAE;;;;;;;;;;;;;;;;CAgBhB;gBACO,aAAa,EAAE,oBAAoB;aACpC,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,uBAAuB,CAAC,IAAI,CAAC,CAAC;YAEnD,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC7B,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC7C,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAC/D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;YACtD,MAAM,IAAI,GAAgB;gBACxB,GAAG,EAAE,4BAA4B;gBACjC,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,aAAa;gBACpB,OAAO,EAAE;;2BAEU;gBACnB,aAAa,EAAE,kBAAkB;aAClC,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,uBAAuB,CAAC,IAAI,CAAC,CAAC;YAEnD,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC7B,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClD,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,SAAS,CAAC,sBAAsB,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qDAAqD,EAAE,KAAK,IAAI,EAAE;YACnE,MAAM,IAAI,GAAgB;gBACxB,GAAG,EAAE,2BAA2B;gBAChC,IAAI,EAAE,QAAQ;gBACd,KAAK,EAAE,OAAO;gBACd,OAAO,EAAE,EAAE;gBACX,aAAa,EAAE,oBAAoB;aACpC,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,uBAAuB,CAAC,IAAI,CAAC,CAAC;YACnD,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,EAAE,CAAC;QACjC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0DAA0D,EAAE,KAAK,IAAI,EAAE;YACxE,MAAM,IAAI,GAAgB;gBACxB,GAAG,EAAE,+BAA+B;gBACpC,IAAI,EAAE,YAAY;gBAClB,KAAK,EAAE,WAAW;gBAClB,OAAO,EAAE;;;;;;;;;;;CAWhB;gBACO,aAAa,EAAE,sBAAsB;aACtC,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,uBAAuB,CAAC,IAAI,CAAC,CAAC;YAEnD,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC7B,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;YAC9C,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;YAC9C,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;YACtD,MAAM,IAAI,GAAgB;gBACxB,GAAG,EAAE,6BAA6B;gBAClC,IAAI,EAAE,UAAU;gBAChB,KAAK,EAAE,uBAAuB;gBAC9B,OAAO,EAAE;;;;;;;CAOhB;gBACO,aAAa,EAAE,oBAAoB;aACpC,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,uBAAuB,CAAC,IAAI,CAAC,CAAC;YAEnD,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC7B,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QAC9D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iCAAiC,EAAE,KAAK,IAAI,EAAE;YAC/C,MAAM,WAAW,GAAG;;EAExB,KAAK,CAAC,GAAG,CAAC;iBACT,IAAI,CAAC,IAAI,CAAC;iBACV,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,uCAAuC,CAAC,GAAG,CAAC,+BAA+B,CAAC;iBAC7G,IAAI,CAAC,MAAM,CAAC;CACd,CAAC;YAEI,MAAM,IAAI,GAAgB;gBACxB,GAAG,EAAE,0BAA0B;gBAC/B,IAAI,EAAE,OAAO;gBACb,KAAK,EAAE,eAAe;gBACtB,OAAO,EAAE,WAAW;gBACpB,aAAa,EAAE,kBAAkB;aAClC,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,uBAAuB,CAAC,IAAI,CAAC,CAAC;YAEnD,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC7B,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC;QAChE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
@@ -0,0 +1,37 @@
1
+ /**
2
+ * Parses structured metadata (props, code blocks) from markdown content.
3
+ * Used to populate chunk metadata fields for better search results.
4
+ */
5
+ export interface ParsedProp {
6
+ name: string;
7
+ type: string;
8
+ required: boolean;
9
+ defaultValue?: string;
10
+ description: string;
11
+ }
12
+ export interface ParsedCodeBlock {
13
+ code: string;
14
+ language: string;
15
+ context: string;
16
+ }
17
+ export interface ParsedMetadata {
18
+ props: ParsedProp[];
19
+ codeBlocks: ParsedCodeBlock[];
20
+ contentType: 'overview' | 'api' | 'example' | 'usage';
21
+ }
22
+ /**
23
+ * Extract props from markdown content containing a props table
24
+ */
25
+ export declare function extractProps(content: string): ParsedProp[];
26
+ /**
27
+ * Extract code blocks from markdown content
28
+ */
29
+ export declare function extractCodeBlocks(content: string): ParsedCodeBlock[];
30
+ /**
31
+ * Determine the content type based on content analysis
32
+ */
33
+ export declare function determineContentType(content: string): 'overview' | 'api' | 'example' | 'usage';
34
+ /**
35
+ * Parse all metadata from markdown content
36
+ */
37
+ export declare function parseMetadata(content: string): ParsedMetadata;
@@ -0,0 +1,245 @@
1
+ /**
2
+ * Parses structured metadata (props, code blocks) from markdown content.
3
+ * Used to populate chunk metadata fields for better search results.
4
+ */
5
+ import { logger } from '../util/logger.js';
6
+ /**
7
+ * Split a table row by pipe character, handling escaped pipes (\ |)
8
+ */
9
+ function splitTableRow(line) {
10
+ // Replace escaped pipes with a placeholder
11
+ const placeholder = '\x00PIPE\x00';
12
+ const escaped = line.replace(/\\ \||\\\|/g, placeholder);
13
+ // Split by unescaped pipe
14
+ const cells = escaped.split('|').map((cell) => cell.trim().replace(new RegExp(placeholder, 'g'), '|'));
15
+ // Remove empty first/last cells from | at start/end
16
+ if (cells[0] === '')
17
+ cells.shift();
18
+ if (cells[cells.length - 1] === '')
19
+ cells.pop();
20
+ return cells;
21
+ }
22
+ /**
23
+ * Check if a line is a table separator (|---|---|...)
24
+ */
25
+ function isTableSeparator(line) {
26
+ return /^\|?[\s-:|]+\|?$/.test(line.trim());
27
+ }
28
+ /**
29
+ * Parse a markdown table into rows of key-value pairs
30
+ * Handles tables with blank lines between rows and escaped pipes
31
+ */
32
+ function parseMarkdownTable(tableLines) {
33
+ if (tableLines.length < 2)
34
+ return [];
35
+ // Find header row (first non-empty row with pipes)
36
+ const headerIndex = tableLines.findIndex((l) => l.includes('|') && !isTableSeparator(l));
37
+ if (headerIndex === -1)
38
+ return [];
39
+ const headers = splitTableRow(tableLines[headerIndex]).map((h) => h.toLowerCase().trim());
40
+ if (headers.length === 0)
41
+ return [];
42
+ // Find separator (skip it)
43
+ const separatorIndex = tableLines.findIndex((l, i) => i > headerIndex && isTableSeparator(l));
44
+ // Get data rows (everything after separator that contains pipes)
45
+ const dataLines = tableLines.slice(separatorIndex + 1).filter((l) => l.includes('|') && !isTableSeparator(l));
46
+ const results = [];
47
+ for (const line of dataLines) {
48
+ const cells = splitTableRow(line);
49
+ if (cells.length >= 1) {
50
+ const row = {};
51
+ headers.forEach((header, i) => {
52
+ if (header) {
53
+ row[header] = cells[i] || '';
54
+ }
55
+ });
56
+ // Only add if we got at least the name column
57
+ if (row['name'] || row['prop'] || row['property']) {
58
+ results.push(row);
59
+ }
60
+ }
61
+ }
62
+ return results;
63
+ }
64
+ /**
65
+ * Extract props from markdown content containing a props table
66
+ */
67
+ export function extractProps(content) {
68
+ const props = [];
69
+ const seenNames = new Set();
70
+ // Split content into lines for table detection
71
+ const lines = content.split('\n');
72
+ // Find Props section and extract table
73
+ let inPropsSection = false;
74
+ let tableLines = [];
75
+ let foundTable = false;
76
+ for (let i = 0; i < lines.length; i++) {
77
+ const line = lines[i];
78
+ const trimmedLine = line.trim();
79
+ // Check for Props heading
80
+ if (/^#{1,3}\s*Props\s*$/i.test(trimmedLine)) {
81
+ inPropsSection = true;
82
+ tableLines = [];
83
+ continue;
84
+ }
85
+ // Check for any table with Name column (might not have Props heading)
86
+ if (trimmedLine.startsWith('|') && /\|\s*Name\s*\|/i.test(trimmedLine) && !foundTable) {
87
+ inPropsSection = true;
88
+ tableLines = [trimmedLine];
89
+ continue;
90
+ }
91
+ // If in props section, collect table lines
92
+ if (inPropsSection) {
93
+ // End of props section on new heading
94
+ if (trimmedLine.startsWith('#') && !trimmedLine.toLowerCase().includes('prop')) {
95
+ // Process collected table
96
+ if (tableLines.length >= 2) {
97
+ foundTable = true;
98
+ const rows = parseMarkdownTable(tableLines);
99
+ for (const row of rows) {
100
+ const name = (row['name'] || row['prop'] || row['property'] || '').replace(/[*`"]/g, '').trim();
101
+ const type = (row['type'] || row['types'] || row['description'] || '').replace(/[`"]/g, '').trim();
102
+ const description = row['description'] || row['desc'] || '';
103
+ const defaultVal = row['default'] || row['defaultvalue'] || '';
104
+ if (name && name !== '-' && !seenNames.has(name)) {
105
+ seenNames.add(name);
106
+ props.push({
107
+ name,
108
+ type,
109
+ required: name.includes('*') || type.includes('required'),
110
+ defaultValue: defaultVal && defaultVal !== '-' ? defaultVal.replace(/[`"]/g, '') : undefined,
111
+ description: description.trim(),
112
+ });
113
+ }
114
+ }
115
+ }
116
+ inPropsSection = false;
117
+ tableLines = [];
118
+ continue;
119
+ }
120
+ // Collect table lines (including empty lines within table)
121
+ if (trimmedLine.includes('|') || trimmedLine === '') {
122
+ tableLines.push(trimmedLine);
123
+ }
124
+ }
125
+ }
126
+ // Process any remaining table
127
+ if (tableLines.length >= 2) {
128
+ const rows = parseMarkdownTable(tableLines);
129
+ for (const row of rows) {
130
+ const name = (row['name'] || row['prop'] || row['property'] || '').replace(/[*`"]/g, '').trim();
131
+ const type = (row['type'] || row['types'] || row['description'] || '').replace(/[`"]/g, '').trim();
132
+ const description = row['description'] || row['desc'] || '';
133
+ const defaultVal = row['default'] || row['defaultvalue'] || '';
134
+ if (name && name !== '-' && !seenNames.has(name)) {
135
+ seenNames.add(name);
136
+ props.push({
137
+ name,
138
+ type,
139
+ required: name.includes('*') || type.includes('required'),
140
+ defaultValue: defaultVal && defaultVal !== '-' ? defaultVal.replace(/[`"]/g, '') : undefined,
141
+ description: description.trim(),
142
+ });
143
+ }
144
+ }
145
+ }
146
+ // Fallback: try to extract from inline patterns if no table found
147
+ if (props.length === 0) {
148
+ const inlinePropsRegex = /[`*](\w+)[`*]\s*[-–:]\s*([^(\n]+?)(?:\s*\((?:type[:\s]*)?([^)]+)\))?(?:\n|$)/gi;
149
+ let inlineMatch;
150
+ while ((inlineMatch = inlinePropsRegex.exec(content)) !== null) {
151
+ const name = inlineMatch[1];
152
+ const description = inlineMatch[2].trim();
153
+ const type = inlineMatch[3]?.trim() || '';
154
+ if (name && !seenNames.has(name)) {
155
+ seenNames.add(name);
156
+ props.push({
157
+ name,
158
+ type,
159
+ required: false,
160
+ description,
161
+ });
162
+ }
163
+ }
164
+ }
165
+ logger.debug(`[MetadataParser] Extracted ${props.length} props from content`);
166
+ return props;
167
+ }
168
+ /**
169
+ * Extract code blocks from markdown content
170
+ */
171
+ export function extractCodeBlocks(content) {
172
+ const codeBlocks = [];
173
+ // Match fenced code blocks with optional language
174
+ const codeBlockRegex = /```(\w*)\n([\s\S]*?)```/g;
175
+ let match;
176
+ let prevIndex = 0;
177
+ while ((match = codeBlockRegex.exec(content)) !== null) {
178
+ const language = match[1] || 'plaintext';
179
+ const code = match[2].trim();
180
+ // Get context: text before the code block (last heading or paragraph)
181
+ const textBefore = content.slice(Math.max(0, prevIndex), match.index);
182
+ const lines = textBefore
183
+ .trim()
184
+ .split('\n')
185
+ .filter((l) => l.trim());
186
+ let context = '';
187
+ // Find the closest heading or meaningful text
188
+ for (let i = lines.length - 1; i >= 0; i--) {
189
+ const line = lines[i].trim();
190
+ if (line.startsWith('#')) {
191
+ context = line.replace(/^#+\s*/, '');
192
+ break;
193
+ }
194
+ else if (line.length > 10 && !line.startsWith('|')) {
195
+ context = line.slice(0, 100);
196
+ break;
197
+ }
198
+ }
199
+ if (code.length > 0) {
200
+ codeBlocks.push({
201
+ code,
202
+ language,
203
+ context,
204
+ });
205
+ }
206
+ prevIndex = match.index + match[0].length;
207
+ }
208
+ logger.debug(`[MetadataParser] Extracted ${codeBlocks.length} code blocks from content`);
209
+ return codeBlocks;
210
+ }
211
+ /**
212
+ * Determine the content type based on content analysis
213
+ */
214
+ export function determineContentType(content) {
215
+ const lowerContent = content.toLowerCase();
216
+ // Check for API documentation patterns
217
+ if (lowerContent.includes('props') ||
218
+ lowerContent.includes('parameters') ||
219
+ lowerContent.includes('arguments') ||
220
+ lowerContent.includes('returns') ||
221
+ lowerContent.includes('type:') ||
222
+ /\|\s*name\s*\|.*\|\s*type\s*\|/i.test(content)) {
223
+ return 'api';
224
+ }
225
+ // Check for example patterns
226
+ if (lowerContent.includes('example') || lowerContent.includes('usage example') || (content.match(/```/g)?.length || 0) > 2) {
227
+ return 'example';
228
+ }
229
+ // Check for usage patterns
230
+ if (lowerContent.includes('how to use') || lowerContent.includes('getting started') || lowerContent.includes('installation')) {
231
+ return 'usage';
232
+ }
233
+ return 'overview';
234
+ }
235
+ /**
236
+ * Parse all metadata from markdown content
237
+ */
238
+ export function parseMetadata(content) {
239
+ return {
240
+ props: extractProps(content),
241
+ codeBlocks: extractCodeBlocks(content),
242
+ contentType: determineContentType(content),
243
+ };
244
+ }
245
+ //# sourceMappingURL=metadata-parser.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"metadata-parser.js","sourceRoot":"","sources":["../../src/processor/metadata-parser.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAsB3C;;GAEG;AACH,SAAS,aAAa,CAAC,IAAY;IACjC,2CAA2C;IAC3C,MAAM,WAAW,GAAG,cAAc,CAAC;IACnC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;IAEzD,0BAA0B;IAC1B,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,WAAW,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;IAEvG,oDAAoD;IACpD,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,EAAE;QAAE,KAAK,CAAC,KAAK,EAAE,CAAC;IACnC,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,EAAE;QAAE,KAAK,CAAC,GAAG,EAAE,CAAC;IAEhD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAC,IAAY;IACpC,OAAO,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;AAC9C,CAAC;AAED;;;GAGG;AACH,SAAS,kBAAkB,CAAC,UAAoB;IAC9C,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,EAAE,CAAC;IAErC,mDAAmD;IACnD,MAAM,WAAW,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC;IACzF,IAAI,WAAW,KAAK,CAAC,CAAC;QAAE,OAAO,EAAE,CAAC;IAElC,MAAM,OAAO,GAAG,aAAa,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;IAC1F,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEpC,2BAA2B;IAC3B,MAAM,cAAc,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,WAAW,IAAI,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC;IAE9F,iEAAiE;IACjE,MAAM,SAAS,GAAG,UAAU,CAAC,KAAK,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC;IAE9G,MAAM,OAAO,GAA6B,EAAE,CAAC;IAE7C,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACtB,MAAM,GAAG,GAA2B,EAAE,CAAC;YACvC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBAC5B,IAAI,MAAM,EAAE,CAAC;oBACX,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC/B,CAAC;YACH,CAAC,CAAC,CAAC;YACH,8CAA8C;YAC9C,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;gBAClD,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACpB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,OAAe;IAC1C,MAAM,KAAK,GAAiB,EAAE,CAAC;IAC/B,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;IAEpC,+CAA+C;IAC/C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAElC,uCAAuC;IACvC,IAAI,cAAc,GAAG,KAAK,CAAC;IAC3B,IAAI,UAAU,GAAa,EAAE,CAAC;IAC9B,IAAI,UAAU,GAAG,KAAK,CAAC;IAEvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACtB,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAEhC,0BAA0B;QAC1B,IAAI,sBAAsB,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;YAC7C,cAAc,GAAG,IAAI,CAAC;YACtB,UAAU,GAAG,EAAE,CAAC;YAChB,SAAS;QACX,CAAC;QAED,sEAAsE;QACtE,IAAI,WAAW,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,iBAAiB,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YACtF,cAAc,GAAG,IAAI,CAAC;YACtB,UAAU,GAAG,CAAC,WAAW,CAAC,CAAC;YAC3B,SAAS;QACX,CAAC;QAED,2CAA2C;QAC3C,IAAI,cAAc,EAAE,CAAC;YACnB,sCAAsC;YACtC,IAAI,WAAW,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC/E,0BAA0B;gBAC1B,IAAI,UAAU,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;oBAC3B,UAAU,GAAG,IAAI,CAAC;oBAClB,MAAM,IAAI,GAAG,kBAAkB,CAAC,UAAU,CAAC,CAAC;oBAC5C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;wBACvB,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;wBAChG,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;wBACnG,MAAM,WAAW,GAAG,GAAG,CAAC,aAAa,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;wBAC5D,MAAM,UAAU,GAAG,GAAG,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;wBAE/D,IAAI,IAAI,IAAI,IAAI,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;4BACjD,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;4BACpB,KAAK,CAAC,IAAI,CAAC;gCACT,IAAI;gCACJ,IAAI;gCACJ,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;gCACzD,YAAY,EAAE,UAAU,IAAI,UAAU,KAAK,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS;gCAC5F,WAAW,EAAE,WAAW,CAAC,IAAI,EAAE;6BAChC,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;gBACH,CAAC;gBACD,cAAc,GAAG,KAAK,CAAC;gBACvB,UAAU,GAAG,EAAE,CAAC;gBAChB,SAAS;YACX,CAAC;YAED,2DAA2D;YAC3D,IAAI,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,WAAW,KAAK,EAAE,EAAE,CAAC;gBACpD,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;IACH,CAAC;IAED,8BAA8B;IAC9B,IAAI,UAAU,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,kBAAkB,CAAC,UAAU,CAAC,CAAC;QAC5C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YAChG,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YACnG,MAAM,WAAW,GAAG,GAAG,CAAC,aAAa,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YAC5D,MAAM,UAAU,GAAG,GAAG,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YAE/D,IAAI,IAAI,IAAI,IAAI,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACjD,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACpB,KAAK,CAAC,IAAI,CAAC;oBACT,IAAI;oBACJ,IAAI;oBACJ,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;oBACzD,YAAY,EAAE,UAAU,IAAI,UAAU,KAAK,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS;oBAC5F,WAAW,EAAE,WAAW,CAAC,IAAI,EAAE;iBAChC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,kEAAkE;IAClE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,gBAAgB,GAAG,gFAAgF,CAAC;QAC1G,IAAI,WAAW,CAAC;QAChB,OAAO,CAAC,WAAW,GAAG,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC/D,MAAM,IAAI,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;YAC5B,MAAM,WAAW,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC1C,MAAM,IAAI,GAAG,WAAW,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YAE1C,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACjC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACpB,KAAK,CAAC,IAAI,CAAC;oBACT,IAAI;oBACJ,IAAI;oBACJ,QAAQ,EAAE,KAAK;oBACf,WAAW;iBACZ,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,8BAA8B,KAAK,CAAC,MAAM,qBAAqB,CAAC,CAAC;IAC9E,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,OAAe;IAC/C,MAAM,UAAU,GAAsB,EAAE,CAAC;IAEzC,kDAAkD;IAClD,MAAM,cAAc,GAAG,0BAA0B,CAAC;IAElD,IAAI,KAAK,CAAC;IACV,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACvD,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,WAAW,CAAC;QACzC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAE7B,sEAAsE;QACtE,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;QACtE,MAAM,KAAK,GAAG,UAAU;aACrB,IAAI,EAAE;aACN,KAAK,CAAC,IAAI,CAAC;aACX,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3B,IAAI,OAAO,GAAG,EAAE,CAAC;QAEjB,8CAA8C;QAC9C,KAAK,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC7B,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACzB,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;gBACrC,MAAM;YACR,CAAC;iBAAM,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACrD,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;gBAC7B,MAAM;YACR,CAAC;QACH,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpB,UAAU,CAAC,IAAI,CAAC;gBACd,IAAI;gBACJ,QAAQ;gBACR,OAAO;aACR,CAAC,CAAC;QACL,CAAC;QAED,SAAS,GAAG,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IAC5C,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,8BAA8B,UAAU,CAAC,MAAM,2BAA2B,CAAC,CAAC;IACzF,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAAC,OAAe;IAClD,MAAM,YAAY,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAE3C,uCAAuC;IACvC,IACE,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC;QAC9B,YAAY,CAAC,QAAQ,CAAC,YAAY,CAAC;QACnC,YAAY,CAAC,QAAQ,CAAC,WAAW,CAAC;QAClC,YAAY,CAAC,QAAQ,CAAC,SAAS,CAAC;QAChC,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC;QAC9B,iCAAiC,CAAC,IAAI,CAAC,OAAO,CAAC,EAC/C,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,6BAA6B;IAC7B,IAAI,YAAY,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,YAAY,CAAC,QAAQ,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;QAC3H,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,2BAA2B;IAC3B,IAAI,YAAY,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,YAAY,CAAC,QAAQ,CAAC,iBAAiB,CAAC,IAAI,YAAY,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;QAC7H,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,OAAe;IAC3C,OAAO;QACL,KAAK,EAAE,YAAY,CAAC,OAAO,CAAC;QAC5B,UAAU,EAAE,iBAAiB,CAAC,OAAO,CAAC;QACtC,WAAW,EAAE,oBAAoB,CAAC,OAAO,CAAC;KAC3C,CAAC;AACJ,CAAC"}
@@ -0,0 +1 @@
1
+ export {};