@cosmocoder/mcp-web-docs 2.0.13 → 2.0.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/config.d.ts +0 -1
- package/build/config.js +0 -5
- package/build/config.js.map +1 -1
- package/build/config.test.js +1 -25
- package/build/config.test.js.map +1 -1
- package/build/crawler/base.test.js +1 -0
- package/build/crawler/base.test.js.map +1 -1
- package/build/crawler/content-extractor-types.d.ts +3 -0
- package/build/crawler/crawlee-crawler.js +14 -9
- package/build/crawler/crawlee-crawler.js.map +1 -1
- package/build/crawler/crawlee-crawler.test.js +58 -7
- package/build/crawler/crawlee-crawler.test.js.map +1 -1
- package/build/crawler/default-extractor.js +1 -0
- package/build/crawler/default-extractor.js.map +1 -1
- package/build/crawler/default-extractor.test.js +1 -0
- package/build/crawler/default-extractor.test.js.map +1 -1
- package/build/crawler/docs-crawler.test.js +18 -10
- package/build/crawler/docs-crawler.test.js.map +1 -1
- package/build/crawler/github-pages-extractor.js +2 -0
- package/build/crawler/github-pages-extractor.js.map +1 -1
- package/build/crawler/github-pages-extractor.test.js +2 -0
- package/build/crawler/github-pages-extractor.test.js.map +1 -1
- package/build/crawler/github.js +1 -0
- package/build/crawler/github.js.map +1 -1
- package/build/crawler/github.test.js +1 -0
- package/build/crawler/github.test.js.map +1 -1
- package/build/crawler/queue-manager.test.js +7 -0
- package/build/crawler/queue-manager.test.js.map +1 -1
- package/build/crawler/storybook-extractor.js +3 -0
- package/build/crawler/storybook-extractor.js.map +1 -1
- package/build/crawler/storybook-extractor.test.js +2 -0
- package/build/crawler/storybook-extractor.test.js.map +1 -1
- package/build/index.test.js +2 -2
- package/build/index.test.js.map +1 -1
- package/build/processor/markdown.d.ts +15 -6
- package/build/processor/markdown.js +7 -30
- package/build/processor/markdown.js.map +1 -1
- package/build/processor/markdown.test.js +39 -6
- package/build/processor/markdown.test.js.map +1 -1
- package/build/processor/processor.js +8 -26
- package/build/processor/processor.js.map +1 -1
- package/build/processor/processor.test.js +118 -172
- package/build/processor/processor.test.js.map +1 -1
- package/build/types.d.ts +2 -0
- package/package.json +2 -3
- package/build/processor/content.d.ts +0 -16
- package/build/processor/content.js +0 -287
- package/build/processor/content.js.map +0 -1
- package/build/processor/content.test.d.ts +0 -1
- package/build/processor/content.test.js +0 -369
- package/build/processor/content.test.js.map +0 -1
|
@@ -1,5 +1,7 @@
|
|
|
1
|
+
import { JSDOM } from 'jsdom';
|
|
1
2
|
import { WebDocumentProcessor } from './processor.js';
|
|
2
3
|
import { createMockEmbeddings, createFailingEmbeddings } from '../__mocks__/embeddings.js';
|
|
4
|
+
import { contentExtractors } from '../crawler/content-extractors.js';
|
|
3
5
|
describe('WebDocumentProcessor', () => {
|
|
4
6
|
let processor;
|
|
5
7
|
let mockEmbeddings;
|
|
@@ -8,36 +10,11 @@ describe('WebDocumentProcessor', () => {
|
|
|
8
10
|
processor = new WebDocumentProcessor(mockEmbeddings, 500);
|
|
9
11
|
});
|
|
10
12
|
describe('process', () => {
|
|
11
|
-
it('should process HTML content', async () => {
|
|
12
|
-
const crawlResult = {
|
|
13
|
-
url: 'https://example.com/docs/page',
|
|
14
|
-
path: '/docs/page',
|
|
15
|
-
title: 'Test Page',
|
|
16
|
-
content: `
|
|
17
|
-
<html>
|
|
18
|
-
<body>
|
|
19
|
-
<main>
|
|
20
|
-
<h1>Test Documentation</h1>
|
|
21
|
-
<p>This is some test content for the documentation page.</p>
|
|
22
|
-
<h2>Features</h2>
|
|
23
|
-
<p>Here are some features of our product.</p>
|
|
24
|
-
</main>
|
|
25
|
-
</body>
|
|
26
|
-
</html>
|
|
27
|
-
`,
|
|
28
|
-
};
|
|
29
|
-
const result = await processor.process(crawlResult);
|
|
30
|
-
expect(result).toBeDefined();
|
|
31
|
-
expect(result.metadata.url).toBe(crawlResult.url);
|
|
32
|
-
// Title may come from crawl result or H1, depending on processor logic
|
|
33
|
-
expect(result.metadata.title).toBeTruthy();
|
|
34
|
-
expect(result.chunks.length).toBeGreaterThan(0);
|
|
35
|
-
expect(result.chunks[0].vector.length).toBe(mockEmbeddings.dimensions);
|
|
36
|
-
});
|
|
37
13
|
it('should process markdown content', async () => {
|
|
38
14
|
const crawlResult = {
|
|
39
15
|
url: 'https://example.com/docs/readme.md',
|
|
40
|
-
path: '/docs/readme.
|
|
16
|
+
path: '/docs/readme.html',
|
|
17
|
+
contentFormat: 'markdown',
|
|
41
18
|
title: 'README',
|
|
42
19
|
content: `# Project README
|
|
43
20
|
|
|
@@ -62,75 +39,81 @@ Here's how to use the package.
|
|
|
62
39
|
expect(result.metadata.title).toBeTruthy();
|
|
63
40
|
expect(result.chunks.length).toBeGreaterThan(0);
|
|
64
41
|
});
|
|
65
|
-
it('should
|
|
66
|
-
const
|
|
67
|
-
url: 'https://
|
|
68
|
-
path: '/
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
## Props
|
|
75
|
-
|
|
76
|
-
| Prop | Type |
|
|
77
|
-
|------|------|
|
|
78
|
-
| variant | string |
|
|
79
|
-
|
|
80
|
-
## Example
|
|
81
|
-
|
|
82
|
-
\`\`\`jsx
|
|
83
|
-
<Button variant="primary">Click</Button>
|
|
84
|
-
\`\`\`
|
|
85
|
-
`,
|
|
86
|
-
extractorUsed: 'StorybookExtractor',
|
|
42
|
+
it('should treat extractorUsed as diagnostic-only', async () => {
|
|
43
|
+
const input = {
|
|
44
|
+
url: 'https://example.com/guide',
|
|
45
|
+
path: '/guide.html',
|
|
46
|
+
contentFormat: 'markdown',
|
|
47
|
+
title: 'Guide',
|
|
48
|
+
content: '# Explicit Markdown\n\n## Section\n\nBody',
|
|
49
|
+
extractorUsed: 'FirstExtractor',
|
|
87
50
|
};
|
|
88
|
-
const
|
|
89
|
-
|
|
90
|
-
expect(
|
|
91
|
-
expect(
|
|
51
|
+
const first = await processor.process(input);
|
|
52
|
+
const second = await processor.process({ ...input, extractorUsed: 'RenamedExtractor' });
|
|
53
|
+
expect(second.metadata.title).toBe(first.metadata.title);
|
|
54
|
+
expect(second.chunks.map(({ content, title }) => ({ content, title }))).toEqual(first.chunks.map(({ content, title }) => ({ content, title })));
|
|
92
55
|
});
|
|
93
|
-
it('should process
|
|
94
|
-
const
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
56
|
+
it('should process actual Storybook extractor output', async () => {
|
|
57
|
+
const originalWindow = globalThis.window;
|
|
58
|
+
const storybookWindow = new JSDOM(`<main class="sbdocs-content">
|
|
59
|
+
<h1>Button Component</h1>
|
|
60
|
+
<table class="docblock-argstable">
|
|
61
|
+
<tr><th>Name</th><th>Type</th><th>Default</th></tr>
|
|
62
|
+
<tr><td>variant</td><td>string</td><td>primary</td></tr>
|
|
63
|
+
</table>
|
|
64
|
+
</main>`).window;
|
|
65
|
+
const storybookDocument = storybookWindow.document;
|
|
66
|
+
globalThis.window = storybookWindow;
|
|
67
|
+
storybookWindow.getComputedStyle = vi.fn().mockReturnValue({
|
|
68
|
+
display: 'block',
|
|
69
|
+
visibility: 'visible',
|
|
70
|
+
opacity: '1',
|
|
71
|
+
});
|
|
72
|
+
const storybook = contentExtractors.storybook;
|
|
73
|
+
const expandSpy = vi.spyOn(storybook, 'expandAllTypeValues').mockResolvedValue(undefined);
|
|
74
|
+
const apiSpy = vi.spyOn(storybook, 'waitForStorybookAPI').mockResolvedValue(undefined);
|
|
75
|
+
const contentSpy = vi.spyOn(storybook, 'waitForStorybookContent').mockResolvedValue(storybookDocument.querySelector('main'));
|
|
76
|
+
try {
|
|
77
|
+
const storybookContent = await storybook.extractContent(storybookDocument);
|
|
78
|
+
const storybookResult = await processor.process({
|
|
79
|
+
url: 'https://storybook.example.com/button',
|
|
80
|
+
path: '/button',
|
|
81
|
+
extractorUsed: contentExtractors.storybook.constructor.name,
|
|
82
|
+
...storybookContent,
|
|
83
|
+
title: storybookContent.title || 'Button',
|
|
84
|
+
});
|
|
85
|
+
expect(storybookResult.metadata.title).toBe('Button Component');
|
|
86
|
+
expect(storybookResult.chunks[0].content).toContain('Button Component');
|
|
87
|
+
const propsChunk = storybookResult.chunks.find((chunk) => chunk.metadata.props?.length);
|
|
88
|
+
expect(propsChunk?.content).toContain('| variant | string | primary |');
|
|
89
|
+
expect(propsChunk?.metadata.props).toContainEqual(expect.objectContaining({ name: 'variant', type: 'string', defaultValue: 'primary' }));
|
|
90
|
+
}
|
|
91
|
+
finally {
|
|
92
|
+
contentSpy.mockRestore();
|
|
93
|
+
apiSpy.mockRestore();
|
|
94
|
+
expandSpy.mockRestore();
|
|
95
|
+
globalThis.window = originalWindow;
|
|
96
|
+
}
|
|
111
97
|
});
|
|
112
98
|
it('should create chunks with proper metadata', async () => {
|
|
113
99
|
const crawlResult = {
|
|
114
100
|
url: 'https://example.com/api',
|
|
115
101
|
path: '/api',
|
|
102
|
+
contentFormat: 'markdown',
|
|
116
103
|
title: 'API',
|
|
117
|
-
content:
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
104
|
+
content: `# API Reference
|
|
105
|
+
|
|
106
|
+
This document describes the API endpoints.
|
|
107
|
+
|
|
108
|
+
## GET /users
|
|
109
|
+
|
|
110
|
+
Returns a list of users.
|
|
111
|
+
|
|
112
|
+
\`\`\`json
|
|
126
113
|
{
|
|
127
114
|
"users": [...]
|
|
128
115
|
}
|
|
129
|
-
|
|
130
|
-
</main>
|
|
131
|
-
</body>
|
|
132
|
-
</html>
|
|
133
|
-
`,
|
|
116
|
+
\`\`\``,
|
|
134
117
|
};
|
|
135
118
|
const result = await processor.process(crawlResult);
|
|
136
119
|
expect(result).toBeDefined();
|
|
@@ -145,45 +128,29 @@ Follow these steps to get started.
|
|
|
145
128
|
it('should handle large content by creating multiple chunks', async () => {
|
|
146
129
|
const longContent = Array(50)
|
|
147
130
|
.fill(null)
|
|
148
|
-
.map((_, i) =>
|
|
149
|
-
|
|
150
|
-
<p>This is the content for section ${i + 1}. It contains some text that will need to be chunked appropriately for the embedding model. Lorem ipsum dolor sit amet, consectetur adipiscing elit.</p>
|
|
151
|
-
`)
|
|
152
|
-
.join('\n');
|
|
131
|
+
.map((_, i) => `## Section ${i + 1}\n\nThis is the content for section ${i + 1}. It contains some text that will need to be chunked appropriately for the embedding model. Lorem ipsum dolor sit amet, consectetur adipiscing elit.`)
|
|
132
|
+
.join('\n\n');
|
|
153
133
|
const crawlResult = {
|
|
154
134
|
url: 'https://example.com/long',
|
|
155
135
|
path: '/long',
|
|
136
|
+
contentFormat: 'markdown',
|
|
156
137
|
title: 'Long Document',
|
|
157
|
-
content:
|
|
158
|
-
<html>
|
|
159
|
-
<body>
|
|
160
|
-
<main>
|
|
161
|
-
<h1>Long Document</h1>
|
|
162
|
-
${longContent}
|
|
163
|
-
</main>
|
|
164
|
-
</body>
|
|
165
|
-
</html>
|
|
166
|
-
`,
|
|
138
|
+
content: `# Long Document\n\n${longContent}`,
|
|
167
139
|
};
|
|
168
140
|
const result = await processor.process(crawlResult);
|
|
169
141
|
expect(result).toBeDefined();
|
|
170
142
|
expect(result.chunks.length).toBeGreaterThan(1);
|
|
171
143
|
});
|
|
172
|
-
it(
|
|
144
|
+
it.each([
|
|
145
|
+
['empty', ''],
|
|
146
|
+
['whitespace-only', ' \n\n '],
|
|
147
|
+
])('should throw error for %s content', async (_, content) => {
|
|
173
148
|
const crawlResult = {
|
|
174
149
|
url: 'https://example.com/empty',
|
|
175
150
|
path: '/empty',
|
|
151
|
+
contentFormat: 'markdown',
|
|
176
152
|
title: 'Empty',
|
|
177
|
-
content
|
|
178
|
-
};
|
|
179
|
-
await expect(processor.process(crawlResult)).rejects.toThrow();
|
|
180
|
-
});
|
|
181
|
-
it('should throw error for whitespace-only content', async () => {
|
|
182
|
-
const crawlResult = {
|
|
183
|
-
url: 'https://example.com/whitespace',
|
|
184
|
-
path: '/whitespace',
|
|
185
|
-
title: 'Whitespace',
|
|
186
|
-
content: ' \n\n ',
|
|
153
|
+
content,
|
|
187
154
|
};
|
|
188
155
|
await expect(processor.process(crawlResult)).rejects.toThrow();
|
|
189
156
|
});
|
|
@@ -193,17 +160,9 @@ Follow these steps to get started.
|
|
|
193
160
|
const crawlResult = {
|
|
194
161
|
url: 'https://example.com/test',
|
|
195
162
|
path: '/test',
|
|
163
|
+
contentFormat: 'text',
|
|
196
164
|
title: 'Test',
|
|
197
|
-
content:
|
|
198
|
-
<html>
|
|
199
|
-
<body>
|
|
200
|
-
<main>
|
|
201
|
-
<h1>Test</h1>
|
|
202
|
-
<p>Some content here.</p>
|
|
203
|
-
</main>
|
|
204
|
-
</body>
|
|
205
|
-
</html>
|
|
206
|
-
`,
|
|
165
|
+
content: 'Test\n\nSome content here.',
|
|
207
166
|
};
|
|
208
167
|
await expect(failingProcessor.process(crawlResult)).rejects.toThrow('Embeddings service unavailable');
|
|
209
168
|
});
|
|
@@ -211,17 +170,9 @@ Follow these steps to get started.
|
|
|
211
170
|
const crawlResult = {
|
|
212
171
|
url: 'https://example.com/dated',
|
|
213
172
|
path: '/dated',
|
|
173
|
+
contentFormat: 'text',
|
|
214
174
|
title: 'Dated',
|
|
215
|
-
content:
|
|
216
|
-
<html>
|
|
217
|
-
<body>
|
|
218
|
-
<main>
|
|
219
|
-
<h1>Document</h1>
|
|
220
|
-
<p>Content with date.</p>
|
|
221
|
-
</main>
|
|
222
|
-
</body>
|
|
223
|
-
</html>
|
|
224
|
-
`,
|
|
175
|
+
content: 'Document\n\nContent with date.',
|
|
225
176
|
};
|
|
226
177
|
const before = new Date();
|
|
227
178
|
const result = await processor.process(crawlResult);
|
|
@@ -234,18 +185,13 @@ Follow these steps to get started.
|
|
|
234
185
|
const crawlResult = {
|
|
235
186
|
url: 'https://example.com/chunks',
|
|
236
187
|
path: '/chunks',
|
|
188
|
+
contentFormat: 'text',
|
|
237
189
|
title: 'Chunks',
|
|
238
|
-
content: `
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
<p>This is a longer paragraph that should be split into multiple chunks when using a small chunk size. The semantic chunker should create appropriate boundaries.</p>
|
|
244
|
-
<p>Another paragraph with additional content that needs to be processed and chunked appropriately.</p>
|
|
245
|
-
</main>
|
|
246
|
-
</body>
|
|
247
|
-
</html>
|
|
248
|
-
`,
|
|
190
|
+
content: `Document
|
|
191
|
+
|
|
192
|
+
This is a longer paragraph that should be split into multiple chunks when using a small chunk size. The semantic chunker should create appropriate boundaries.
|
|
193
|
+
|
|
194
|
+
Another paragraph with additional content that needs to be processed and chunked appropriately.`,
|
|
249
195
|
};
|
|
250
196
|
const result = await smallChunkProcessor.process(crawlResult);
|
|
251
197
|
// With smaller chunk size, should create more chunks
|
|
@@ -255,6 +201,7 @@ Follow these steps to get started.
|
|
|
255
201
|
const crawlResult = {
|
|
256
202
|
url: 'https://example.com/docs/component.mdx',
|
|
257
203
|
path: '/docs/component.mdx',
|
|
204
|
+
contentFormat: 'markdown',
|
|
258
205
|
title: 'MDX Component',
|
|
259
206
|
content: `# MDX Component
|
|
260
207
|
|
|
@@ -279,6 +226,7 @@ import { MyComponent } from 'library';
|
|
|
279
226
|
const crawlResult = {
|
|
280
227
|
url: 'https://example.com/default',
|
|
281
228
|
path: '/default',
|
|
229
|
+
contentFormat: 'text',
|
|
282
230
|
title: 'Default Extracted',
|
|
283
231
|
content: `Page Title
|
|
284
232
|
|
|
@@ -298,22 +246,23 @@ Another section of content here.`,
|
|
|
298
246
|
const crawlResult = {
|
|
299
247
|
url: 'https://example.com/api-ref',
|
|
300
248
|
path: '/api-ref',
|
|
249
|
+
contentFormat: 'markdown',
|
|
301
250
|
title: 'API Reference',
|
|
302
|
-
content:
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
251
|
+
content: `# API Reference
|
|
252
|
+
|
|
253
|
+
## GET /api/users
|
|
254
|
+
|
|
255
|
+
Returns array of users
|
|
256
|
+
|
|
257
|
+
### Parameters
|
|
258
|
+
|
|
259
|
+
limit: number - Maximum results
|
|
260
|
+
|
|
261
|
+
### Response
|
|
262
|
+
|
|
263
|
+
\`\`\`json
|
|
264
|
+
{"users": []}
|
|
265
|
+
\`\`\``,
|
|
317
266
|
};
|
|
318
267
|
const result = await processor.process(crawlResult);
|
|
319
268
|
expect(result).toBeDefined();
|
|
@@ -325,26 +274,23 @@ Another section of content here.`,
|
|
|
325
274
|
const crawlResult = {
|
|
326
275
|
url: 'https://example.com/examples',
|
|
327
276
|
path: '/examples',
|
|
277
|
+
contentFormat: 'markdown',
|
|
328
278
|
title: 'Examples',
|
|
329
|
-
content:
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
<h2>Basic Example</h2>
|
|
335
|
-
<pre><code>
|
|
279
|
+
content: `# Code Examples
|
|
280
|
+
|
|
281
|
+
## Basic Example
|
|
282
|
+
|
|
283
|
+
\`\`\`js
|
|
336
284
|
const result = doSomething();
|
|
337
285
|
console.log(result);
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
286
|
+
\`\`\`
|
|
287
|
+
|
|
288
|
+
## Advanced Example
|
|
289
|
+
|
|
290
|
+
\`\`\`js
|
|
341
291
|
const config = { advanced: true };
|
|
342
292
|
const result = doSomething(config);
|
|
343
|
-
|
|
344
|
-
</main>
|
|
345
|
-
</body>
|
|
346
|
-
</html>
|
|
347
|
-
`,
|
|
293
|
+
\`\`\``,
|
|
348
294
|
};
|
|
349
295
|
const result = await processor.process(crawlResult);
|
|
350
296
|
expect(result).toBeDefined();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"processor.test.js","sourceRoot":"","sources":["../../src/processor/processor.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AACtD,OAAO,EAAE,oBAAoB,EAAE,uBAAuB,EAAE,MAAM,4BAA4B,CAAC;
|
|
1
|
+
{"version":3,"file":"processor.test.js","sourceRoot":"","sources":["../../src/processor/processor.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAC9B,OAAO,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AACtD,OAAO,EAAE,oBAAoB,EAAE,uBAAuB,EAAE,MAAM,4BAA4B,CAAC;AAG3F,OAAO,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAC;AAErE,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;IACpC,IAAI,SAA+B,CAAC;IACpC,IAAI,cAAkC,CAAC;IAEvC,UAAU,CAAC,GAAG,EAAE;QACd,cAAc,GAAG,oBAAoB,EAAE,CAAC;QACxC,SAAS,GAAG,IAAI,oBAAoB,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;IAC5D,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,SAAS,EAAE,GAAG,EAAE;QACvB,EAAE,CAAC,iCAAiC,EAAE,KAAK,IAAI,EAAE;YAC/C,MAAM,WAAW,GAAgB;gBAC/B,GAAG,EAAE,oCAAoC;gBACzC,IAAI,EAAE,mBAAmB;gBACzB,aAAa,EAAE,UAAU;gBACzB,KAAK,EAAE,QAAQ;gBACf,OAAO,EAAE;;;;;;;;;;;;;;;CAehB;aACM,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;YAEpD,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC7B,yCAAyC;YACzC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,UAAU,EAAE,CAAC;YAC3C,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;YAC7D,MAAM,KAAK,GAAgB;gBACzB,GAAG,EAAE,2BAA2B;gBAChC,IAAI,EAAE,aAAa;gBACnB,aAAa,EAAE,UAAU;gBACzB,KAAK,EAAE,OAAO;gBACd,OAAO,EAAE,2CAA2C;gBACpD,aAAa,EAAE,gBAAgB;aAChC,CAAC;YAEF,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAC7C,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,EAAE,GAAG,KAAK,EAAE,aAAa,EAAE,kBAAkB,EAAE,CAAC,CAAC;YAExF,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YACzD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAC7E,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,CAC/D,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YAChE,MAAM,cAAc,GAAG,UAAU,CAAC,MAAM,CAAC;YACzC,MAAM,eAAe,GAAG,IAAI,KAAK,CAAC;;;;;;cAM1B,CAAC,CAAC,MAAM,CAAC;YACjB,MAAM,iBAAiB,GAAG,eAAe,CAAC,QAAQ,CAAC;YACnD,UAAU,CAAC,MAAM,GAAG,eAAsD,CAAC;YAC3E,eAAe,CAAC,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC;gBACzD,OAAO,EAAE,OAAO;gBAChB,UAAU,EAAE,SAAS;gBACrB,OAAO,EAAE,GAAG;aACb,CAAuD,CAAC;YAEzD,MAAM,SAAS,GAAG,iBAAiB,CAAC,SAKnC,CAAC;YACF,MAAM,SAAS,GAAG,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,qBAAqB,CAAC,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YAC1F,MAAM,MAAM,GAAG,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,qBAAqB,CAAC,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YACvF,MAAM,UAAU,GAAG,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,yBAAyB,CAAC,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC;YAE7H,IAAI,CAAC;gBACH,MAAM,gBAAgB,GAAG,MAAM,SAAS,CAAC,cAAc,CAAC,iBAAiB,CAAC,CAAC;gBAC3E,MAAM,eAAe,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC;oBAC9C,GAAG,EAAE,sCAAsC;oBAC3C,IAAI,EAAE,SAAS;oBACf,aAAa,EAAE,iBAAiB,CAAC,SAAS,CAAC,WAAW,CAAC,IAAI;oBAC3D,GAAG,gBAAgB;oBACnB,KAAK,EAAE,gBAAgB,CAAC,KAAK,IAAI,QAAQ;iBAC1C,CAAC,CAAC;gBACH,MAAM,CAAC,eAAe,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;gBAChE,MAAM,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;gBACxE,MAAM,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;gBACxF,MAAM,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,SAAS,CAAC,gCAAgC,CAAC,CAAC;gBACxE,MAAM,CAAC,UAAU,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,cAAc,CAC/C,MAAM,CAAC,gBAAgB,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,CAAC,CACtF,CAAC;YACJ,CAAC;oBACO,CAAC;gBACP,UAAU,CAAC,WAAW,EAAE,CAAC;gBACzB,MAAM,CAAC,WAAW,EAAE,CAAC;gBACrB,SAAS,CAAC,WAAW,EAAE,CAAC;gBACxB,UAAU,CAAC,MAAM,GAAG,cAAc,CAAC;YACrC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,MAAM,WAAW,GAAgB;gBAC/B,GAAG,EAAE,yBAAyB;gBAC9B,IAAI,EAAE,MAAM;gBACZ,aAAa,EAAE,UAAU;gBACzB,KAAK,EAAE,KAAK;gBACZ,OAAO,EAAE;;;;;;;;;;;;OAYV;aACA,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;YAEpD,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC7B,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;gBAC9B,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;gBACxC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;gBAC1C,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC;gBAC7D,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;gBACrC,MAAM,CAAC,CAAC,UAAU,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YACjF,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yDAAyD,EAAE,KAAK,IAAI,EAAE;YACvE,MAAM,WAAW,GAAG,KAAK,CAAC,EAAE,CAAC;iBAC1B,IAAI,CAAC,IAAI,CAAC;iBACV,GAAG,CACF,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,cAAc,CAAC,GAAG,CAAC,uCAAuC,CAAC,GAAG,CAAC,sJAAsJ,CACxN;iBACA,IAAI,CAAC,MAAM,CAAC,CAAC;YAEhB,MAAM,WAAW,GAAgB;gBAC/B,GAAG,EAAE,0BAA0B;gBAC/B,IAAI,EAAE,OAAO;gBACb,aAAa,EAAE,UAAU;gBACzB,KAAK,EAAE,eAAe;gBACtB,OAAO,EAAE,sBAAsB,WAAW,EAAE;aAC7C,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;YAEpD,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC7B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,IAAI,CAAC;YACN,CAAC,OAAO,EAAE,EAAE,CAAC;YACb,CAAC,iBAAiB,EAAE,YAAY,CAAC;SAClC,CAAC,CAAC,mCAAmC,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE;YAC3D,MAAM,WAAW,GAAgB;gBAC/B,GAAG,EAAE,2BAA2B;gBAChC,IAAI,EAAE,QAAQ;gBACd,aAAa,EAAE,UAAU;gBACzB,KAAK,EAAE,OAAO;gBACd,OAAO;aACR,CAAC;YAEF,MAAM,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QACjE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;YAC3D,MAAM,iBAAiB,GAAG,uBAAuB,EAAE,CAAC;YACpD,MAAM,gBAAgB,GAAG,IAAI,oBAAoB,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC;YAE1E,MAAM,WAAW,GAAgB;gBAC/B,GAAG,EAAE,0BAA0B;gBAC/B,IAAI,EAAE,OAAO;gBACb,aAAa,EAAE,MAAM;gBACrB,KAAK,EAAE,MAAM;gBACb,OAAO,EAAE,4BAA4B;aACtC,CAAC;YAEF,MAAM,MAAM,CAAC,gBAAgB,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,gCAAgC,CAAC,CAAC;QACxG,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;YAC3C,MAAM,WAAW,GAAgB;gBAC/B,GAAG,EAAE,2BAA2B;gBAChC,IAAI,EAAE,QAAQ;gBACd,aAAa,EAAE,MAAM;gBACrB,KAAK,EAAE,OAAO;gBACd,OAAO,EAAE,gCAAgC;aAC1C,CAAC;YAEF,MAAM,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YAC1B,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;YACpD,MAAM,KAAK,GAAG,IAAI,IAAI,EAAE,CAAC;YAEzB,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC,sBAAsB,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;YACvF,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC,mBAAmB,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACrF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACrD,MAAM,mBAAmB,GAAG,IAAI,oBAAoB,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;YAE1E,MAAM,WAAW,GAAgB;gBAC/B,GAAG,EAAE,4BAA4B;gBACjC,IAAI,EAAE,SAAS;gBACf,aAAa,EAAE,MAAM;gBACrB,KAAK,EAAE,QAAQ;gBACf,OAAO,EAAE;;;;gGAI+E;aACzF,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;YAE9D,qDAAqD;YACrD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yBAAyB,EAAE,KAAK,IAAI,EAAE;YACvC,MAAM,WAAW,GAAgB;gBAC/B,GAAG,EAAE,wCAAwC;gBAC7C,IAAI,EAAE,qBAAqB;gBAC3B,aAAa,EAAE,UAAU;gBACzB,KAAK,EAAE,eAAe;gBACtB,OAAO,EAAE;;;;;;;;;;;;;CAahB;aACM,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;YAEpD,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC7B,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,WAAW,GAAgB;gBAC/B,GAAG,EAAE,6BAA6B;gBAClC,IAAI,EAAE,UAAU;gBAChB,aAAa,EAAE,MAAM;gBACrB,KAAK,EAAE,mBAAmB;gBAC1B,OAAO,EAAE;;;;;iCAKgB;gBACzB,aAAa,EAAE,kBAAkB;aAClC,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;YAEpD,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC7B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,0BAA0B,EAAE,GAAG,EAAE;QACxC,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC9C,MAAM,WAAW,GAAgB;gBAC/B,GAAG,EAAE,6BAA6B;gBAClC,IAAI,EAAE,UAAU;gBAChB,aAAa,EAAE,UAAU;gBACzB,KAAK,EAAE,eAAe;gBACtB,OAAO,EAAE;;;;;;;;;;;;;;OAcV;aACA,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;YAEpD,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC7B,oCAAoC;YACpC,oDAAoD;YACpD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;YAClD,MAAM,WAAW,GAAgB;gBAC/B,GAAG,EAAE,8BAA8B;gBACnC,IAAI,EAAE,WAAW;gBACjB,aAAa,EAAE,UAAU;gBACzB,KAAK,EAAE,UAAU;gBACjB,OAAO,EAAE;;;;;;;;;;;;;;OAcV;aACA,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;YAEpD,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC7B,oCAAoC;YACpC,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC;YACpF,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
package/build/types.d.ts
CHANGED
|
@@ -54,10 +54,12 @@ export interface DocumentChunk {
|
|
|
54
54
|
}[];
|
|
55
55
|
};
|
|
56
56
|
}
|
|
57
|
+
export type ContentFormat = 'markdown' | 'text';
|
|
57
58
|
export interface CrawlResult {
|
|
58
59
|
url: string;
|
|
59
60
|
path: string;
|
|
60
61
|
content: string;
|
|
62
|
+
contentFormat: ContentFormat;
|
|
61
63
|
title: string;
|
|
62
64
|
extractorUsed?: string;
|
|
63
65
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cosmocoder/mcp-web-docs",
|
|
3
|
-
"version": "2.0.
|
|
3
|
+
"version": "2.0.14",
|
|
4
4
|
"description": "MCP server for crawling and indexing web documentation - works with any website",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"mcp",
|
|
@@ -60,13 +60,11 @@
|
|
|
60
60
|
"@andersmyrmel/vard": "1.2.0",
|
|
61
61
|
"@lancedb/lancedb": "0.31.0",
|
|
62
62
|
"@modelcontextprotocol/sdk": "1.29.0",
|
|
63
|
-
"@mozilla/readability": "0.6.0",
|
|
64
63
|
"cheerio": "1.2.0",
|
|
65
64
|
"cli-progress": "3.12.0",
|
|
66
65
|
"crawlee": "3.17.0",
|
|
67
66
|
"default-browser": "5.5.0",
|
|
68
67
|
"fastembed": "2.1.0",
|
|
69
|
-
"jsdom": "28.1.0",
|
|
70
68
|
"node-html-markdown": "2.0.0",
|
|
71
69
|
"pdfjs-dist": "5.7.284",
|
|
72
70
|
"playwright": "1.61.1",
|
|
@@ -97,6 +95,7 @@
|
|
|
97
95
|
"globals": "17.7.0",
|
|
98
96
|
"husky": "9.1.7",
|
|
99
97
|
"jiti": "2.7.0",
|
|
98
|
+
"jsdom": "28.1.0",
|
|
100
99
|
"lint-staged": "16.4.0",
|
|
101
100
|
"prettier": "3.8.5",
|
|
102
101
|
"prettier-plugin-brace-style": "0.10.3",
|
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
import { CrawlResult } from '../types.js';
|
|
2
|
-
export interface ArticleComponent {
|
|
3
|
-
title: string;
|
|
4
|
-
body: string;
|
|
5
|
-
}
|
|
6
|
-
export interface Article {
|
|
7
|
-
url: string;
|
|
8
|
-
path: string;
|
|
9
|
-
title: string;
|
|
10
|
-
components: ArticleComponent[];
|
|
11
|
-
}
|
|
12
|
-
export interface ProcessedContent {
|
|
13
|
-
article: Article;
|
|
14
|
-
content: string;
|
|
15
|
-
}
|
|
16
|
-
export declare function processHtmlContent(page: CrawlResult): Promise<ProcessedContent | undefined>;
|