@cosmocoder/mcp-web-docs 2.0.12 → 2.0.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/config.d.ts +0 -1
- package/build/config.js +0 -5
- package/build/config.js.map +1 -1
- package/build/config.test.js +1 -25
- package/build/config.test.js.map +1 -1
- package/build/crawler/base.test.js +1 -0
- package/build/crawler/base.test.js.map +1 -1
- package/build/crawler/content-extractor-types.d.ts +3 -0
- package/build/crawler/crawlee-crawler.js +14 -9
- package/build/crawler/crawlee-crawler.js.map +1 -1
- package/build/crawler/crawlee-crawler.test.js +58 -7
- package/build/crawler/crawlee-crawler.test.js.map +1 -1
- package/build/crawler/default-extractor.js +1 -0
- package/build/crawler/default-extractor.js.map +1 -1
- package/build/crawler/default-extractor.test.js +1 -0
- package/build/crawler/default-extractor.test.js.map +1 -1
- package/build/crawler/docs-crawler.test.js +18 -10
- package/build/crawler/docs-crawler.test.js.map +1 -1
- package/build/crawler/github-pages-extractor.js +2 -0
- package/build/crawler/github-pages-extractor.js.map +1 -1
- package/build/crawler/github-pages-extractor.test.js +2 -0
- package/build/crawler/github-pages-extractor.test.js.map +1 -1
- package/build/crawler/github.js +1 -0
- package/build/crawler/github.js.map +1 -1
- package/build/crawler/github.test.js +1 -0
- package/build/crawler/github.test.js.map +1 -1
- package/build/crawler/queue-manager.test.js +7 -0
- package/build/crawler/queue-manager.test.js.map +1 -1
- package/build/crawler/storybook-extractor.js +3 -0
- package/build/crawler/storybook-extractor.js.map +1 -1
- package/build/crawler/storybook-extractor.test.js +2 -0
- package/build/crawler/storybook-extractor.test.js.map +1 -1
- package/build/index.js +3 -2
- package/build/index.js.map +1 -1
- package/build/index.test.js +10 -3
- package/build/index.test.js.map +1 -1
- package/build/processor/markdown.d.ts +15 -6
- package/build/processor/markdown.js +7 -30
- package/build/processor/markdown.js.map +1 -1
- package/build/processor/markdown.test.js +39 -6
- package/build/processor/markdown.test.js.map +1 -1
- package/build/processor/processor.js +8 -26
- package/build/processor/processor.js.map +1 -1
- package/build/processor/processor.test.js +118 -172
- package/build/processor/processor.test.js.map +1 -1
- package/build/storage/storage.js +18 -6
- package/build/storage/storage.js.map +1 -1
- package/build/storage/storage.test.js +39 -2
- package/build/storage/storage.test.js.map +1 -1
- package/build/types.d.ts +4 -0
- package/package.json +2 -3
- package/build/processor/content.d.ts +0 -16
- package/build/processor/content.js +0 -287
- package/build/processor/content.js.map +0 -1
- package/build/processor/content.test.d.ts +0 -1
- package/build/processor/content.test.js +0 -369
- package/build/processor/content.test.js.map +0 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"processor.js","sourceRoot":"","sources":["../../src/processor/processor.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"processor.js","sourceRoot":"","sources":["../../src/processor/processor.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAC;AAChF,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAErD;;GAEG;AACH,SAAS,uBAAuB,CAC9B,OAAe,EACf,YAA0D,EAC1D,SAAiB,EACjB,OAAe,EACf,MAAgB;IAEhB,MAAM,MAAM,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;IAEtC,OAAO;QACL,OAAO;QACP,SAAS;QACT,OAAO;QACP,MAAM;QACN,GAAG,EAAE,YAAY,CAAC,GAAG;QACrB,KAAK,EAAE,YAAY,CAAC,KAAK;QACzB,IAAI,EAAE,YAAY,CAAC,IAAI;QACvB,QAAQ,EAAE;YACR,IAAI,EAAE,MAAM,CAAC,WAAW;YACxB,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;YACzD,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;SACzE;KACF,CAAC;AACJ,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,eAAe,CAC7B,OAAe,EACf,YAAoB,EACpB,UAA8B,EAC9B,QAIC;IAED,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO;IACT,CAAC;IAED,wEAAwE;IACxE,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;IAChD,IAAI,YAAY,GAAG,EAAE,CAAC;IACtB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;QAEnC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7B,WAAW,IAAI,YAAY,CAAC,MAAM,CAAC;YACnC,SAAS;QACX,CAAC;QAED,gEAAgE;QAChE,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAExD,kDAAkD;QAClD,IAAI,aAAa,GAAG,YAAY,EAAE,CAAC;YACjC,yCAAyC;YACzC,IAAI,YAAY,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACnC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;gBACpD,MAAM,uBAAuB,CAAC,YAAY,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,WAAW,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC;gBACjG,YAAY,GAAG,EAAE,CAAC;YACpB,CAAC;YAED,mCAAmC;YACnC,MAAM,SAAS,GAAG,WAAW,CAAC,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACvE,IAAI,aAAa,GAAG,EAAE,CAAC;YACvB,IAAI,cAAc,GAAG,CAAC,CAAC;YAEvB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;gBACjC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAElD,IAAI,cAAc,GAAG,UAAU,GAAG,YAAY,GAAG,CAAC,EAAE,CAAC;oBACnD,IAAI,aAAa,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBACpC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;wBACrD,MAAM,uBAAuB,CAC3B,aAAa,CAAC,IAAI,EAAE,EACpB,QAAQ,EACR,WAAW,EACX,WAAW,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,EAClD,MAAM,CACP,CAAC;oBACJ,CAAC;oBACD,aAAa,GAAG,QAAQ,CAAC;oBACzB,cAAc,GAAG,UAAU,CAAC;gBAC9B,CAAC;qBACI,CAAC;oBACJ,aAAa,IAAI,GAAG,GAAG,QAAQ,CAAC;oBAChC,cAAc,IAAI,UAAU,CAAC;gBAC/B,CAAC;YACH,CAAC;YAED,iCAAiC;YACjC,IAAI,aAAa,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACpC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;gBACrD,MAAM,uBAAuB,CAC3B,aAAa,CAAC,IAAI,EAAE,EACpB,QAAQ,EACR,WAAW,EACX,WAAW,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,EAClD,MAAM,CACP,CAAC;YACJ,CAAC;QACH,CAAC;QACD,8EAA8E;aACzE,IAAI,UAAU,GAAG,aAAa,GAAG,YAAY,GAAG,CAAC,EAAE,CAAC;YACvD,IAAI,YAAY,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACnC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;gBACpD,MAAM,uBAAuB,CAAC,YAAY,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,WAAW,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC;YACnG,CAAC;YACD,YAAY,GAAG,WAAW,CAAC;YAC3B,UAAU,GAAG,aAAa,CAAC;YAC3B,SAAS,GAAG,WAAW,CAAC;QAC1B,CAAC;QACD,yCAAyC;aACpC,CAAC;YACJ,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,YAAY,IAAI,MAAM,CAAC;YACzB,CAAC;YACD,YAAY,IAAI,WAAW,CAAC;YAC5B,UAAU,IAAI,aAAa,CAAC;QAC9B,CAAC;QAED,WAAW,IAAI,YAAY,CAAC,MAAM,CAAC;IACrC,CAAC;IAED,iCAAiC;IACjC,IAAI,YAAY,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QACpD,MAAM,uBAAuB,CAAC,YAAY,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,WAAW,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC;IACnG,CAAC;AACH,CAAC;AAED,MAAM,OAAO,oBAAoB;IAEZ;IACA;IAFnB,YACmB,UAA8B,EAC9B,eAAuB,IAAI;QAD3B,eAAU,GAAV,UAAU,CAAoB;QAC9B,iBAAY,GAAZ,YAAY,CAAe;IAC3C,CAAC;IAEJ,KAAK,CAAC,OAAO,CAAC,WAAwB;QACpC,MAAM,CAAC,KAAK,CAAC,qCAAqC,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC;QACrE,MAAM,CAAC,KAAK,CAAC,0CAA0C,WAAW,CAAC,OAAO,CAAC,MAAM,QAAQ,CAAC,CAAC;QAC3F,MAAM,CAAC,KAAK,CAAC,0CAA0C,WAAW,CAAC,aAAa,IAAI,SAAS,EAAE,CAAC,CAAC;QAEjG,IAAI,CAAC;YACH,MAAM,CAAC,KAAK,CAAC,gCAAgC,WAAW,CAAC,aAAa,kBAAkB,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC;YAC5G,IAAI,gBAAgB,CAAC;YACrB,QAAQ,WAAW,CAAC,aAAa,EAAE,CAAC;gBAClC,KAAK,UAAU;oBACb,gBAAgB,GAAG,MAAM,sBAAsB,CAAC,WAAW,CAAC,CAAC;oBAC7D,MAAM;gBACR,KAAK,MAAM;oBACT,gBAAgB,GAAG,MAAM,uBAAuB,CAAC,WAAW,CAAC,CAAC;oBAC9D,MAAM;YACV,CAAC;YAED,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACtB,MAAM,CAAC,KAAK,CAAC,+DAA+D,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC;gBAC/F,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;YACtD,CAAC;YAED,MAAM,CAAC,KAAK,CAAC,6DAA6D,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC;YAC7F,MAAM,CAAC,KAAK,CAAC,gCAAgC,gBAAgB,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,aAAa,CAAC,CAAC;YACtG,MAAM,CAAC,KAAK,CAAC,8CAA8C,gBAAgB,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;YAE7F,MAAM,MAAM,GAAoB,EAAE,CAAC;YACnC,IAAI,WAAW,GAAG,CAAC,CAAC;YAEpB,MAAM,QAAQ,GAAG;gBACf,GAAG,EAAE,gBAAgB,CAAC,OAAO,CAAC,GAAG;gBACjC,KAAK,EAAE,gBAAgB,CAAC,OAAO,CAAC,KAAK;gBACrC,IAAI,EAAE,gBAAgB,CAAC,OAAO,CAAC,IAAI;aACpC,CAAC;YAEF,oCAAoC;YACpC,KAAK,MAAM,SAAS,IAAI,gBAAgB,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;gBAC5D,MAAM,CAAC,KAAK,CAAC,gDAAgD,SAAS,CAAC,KAAK,EAAE,CAAC,CAAC;gBAChF,MAAM,CAAC,KAAK,CAAC,iDAAiD,SAAS,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,CAAC;gBAE7F,MAAM,gBAAgB,GAAG,GAAG,SAAS,CAAC,KAAK,OAAO,SAAS,CAAC,IAAI,EAAE,CAAC;gBACnE,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,eAAe,CAAC,gBAAgB,EAAE,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,EAAE,CAAC;oBAC1G,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACnB,WAAW,EAAE,CAAC;gBAChB,CAAC;YACH,CAAC;YAED,MAAM,CAAC,KAAK,CAAC,kCAAkC,WAAW,SAAS,CAAC,CAAC;YAErE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACxB,MAAM,CAAC,IAAI,CAAC,2DAA2D,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC;gBAC1F,MAAM,CAAC,IAAI,CAAC,mDAAmD,WAAW,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;gBAC7F,MAAM,CAAC,IAAI,CAAC,oDAAoD,gBAAgB,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;gBACnG,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;YAClD,CAAC;YAED,MAAM,CAAC,KAAK,CAAC,iDAAiD,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC;YACjF,OAAO;gBACL,QAAQ,EAAE;oBACR,GAAG,EAAE,WAAW,CAAC,GAAG;oBACpB,KAAK,EAAE,gBAAgB,CAAC,OAAO,CAAC,KAAK;oBACrC,WAAW,EAAE,IAAI,IAAI,EAAE;iBACxB;gBACD,MAAM;aACP,CAAC;QACJ,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,2CAA2C,WAAW,CAAC,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;YACnF,MAAM,CAAC,KAAK,CAAC,uCAAuC,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACpG,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;CACF"}
|
|
@@ -1,5 +1,7 @@
|
|
|
1
|
+
import { JSDOM } from 'jsdom';
|
|
1
2
|
import { WebDocumentProcessor } from './processor.js';
|
|
2
3
|
import { createMockEmbeddings, createFailingEmbeddings } from '../__mocks__/embeddings.js';
|
|
4
|
+
import { contentExtractors } from '../crawler/content-extractors.js';
|
|
3
5
|
describe('WebDocumentProcessor', () => {
|
|
4
6
|
let processor;
|
|
5
7
|
let mockEmbeddings;
|
|
@@ -8,36 +10,11 @@ describe('WebDocumentProcessor', () => {
|
|
|
8
10
|
processor = new WebDocumentProcessor(mockEmbeddings, 500);
|
|
9
11
|
});
|
|
10
12
|
describe('process', () => {
|
|
11
|
-
it('should process HTML content', async () => {
|
|
12
|
-
const crawlResult = {
|
|
13
|
-
url: 'https://example.com/docs/page',
|
|
14
|
-
path: '/docs/page',
|
|
15
|
-
title: 'Test Page',
|
|
16
|
-
content: `
|
|
17
|
-
<html>
|
|
18
|
-
<body>
|
|
19
|
-
<main>
|
|
20
|
-
<h1>Test Documentation</h1>
|
|
21
|
-
<p>This is some test content for the documentation page.</p>
|
|
22
|
-
<h2>Features</h2>
|
|
23
|
-
<p>Here are some features of our product.</p>
|
|
24
|
-
</main>
|
|
25
|
-
</body>
|
|
26
|
-
</html>
|
|
27
|
-
`,
|
|
28
|
-
};
|
|
29
|
-
const result = await processor.process(crawlResult);
|
|
30
|
-
expect(result).toBeDefined();
|
|
31
|
-
expect(result.metadata.url).toBe(crawlResult.url);
|
|
32
|
-
// Title may come from crawl result or H1, depending on processor logic
|
|
33
|
-
expect(result.metadata.title).toBeTruthy();
|
|
34
|
-
expect(result.chunks.length).toBeGreaterThan(0);
|
|
35
|
-
expect(result.chunks[0].vector.length).toBe(mockEmbeddings.dimensions);
|
|
36
|
-
});
|
|
37
13
|
it('should process markdown content', async () => {
|
|
38
14
|
const crawlResult = {
|
|
39
15
|
url: 'https://example.com/docs/readme.md',
|
|
40
|
-
path: '/docs/readme.
|
|
16
|
+
path: '/docs/readme.html',
|
|
17
|
+
contentFormat: 'markdown',
|
|
41
18
|
title: 'README',
|
|
42
19
|
content: `# Project README
|
|
43
20
|
|
|
@@ -62,75 +39,81 @@ Here's how to use the package.
|
|
|
62
39
|
expect(result.metadata.title).toBeTruthy();
|
|
63
40
|
expect(result.chunks.length).toBeGreaterThan(0);
|
|
64
41
|
});
|
|
65
|
-
it('should
|
|
66
|
-
const
|
|
67
|
-
url: 'https://
|
|
68
|
-
path: '/
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
## Props
|
|
75
|
-
|
|
76
|
-
| Prop | Type |
|
|
77
|
-
|------|------|
|
|
78
|
-
| variant | string |
|
|
79
|
-
|
|
80
|
-
## Example
|
|
81
|
-
|
|
82
|
-
\`\`\`jsx
|
|
83
|
-
<Button variant="primary">Click</Button>
|
|
84
|
-
\`\`\`
|
|
85
|
-
`,
|
|
86
|
-
extractorUsed: 'StorybookExtractor',
|
|
42
|
+
it('should treat extractorUsed as diagnostic-only', async () => {
|
|
43
|
+
const input = {
|
|
44
|
+
url: 'https://example.com/guide',
|
|
45
|
+
path: '/guide.html',
|
|
46
|
+
contentFormat: 'markdown',
|
|
47
|
+
title: 'Guide',
|
|
48
|
+
content: '# Explicit Markdown\n\n## Section\n\nBody',
|
|
49
|
+
extractorUsed: 'FirstExtractor',
|
|
87
50
|
};
|
|
88
|
-
const
|
|
89
|
-
|
|
90
|
-
expect(
|
|
91
|
-
expect(
|
|
51
|
+
const first = await processor.process(input);
|
|
52
|
+
const second = await processor.process({ ...input, extractorUsed: 'RenamedExtractor' });
|
|
53
|
+
expect(second.metadata.title).toBe(first.metadata.title);
|
|
54
|
+
expect(second.chunks.map(({ content, title }) => ({ content, title }))).toEqual(first.chunks.map(({ content, title }) => ({ content, title })));
|
|
92
55
|
});
|
|
93
|
-
it('should process
|
|
94
|
-
const
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
56
|
+
it('should process actual Storybook extractor output', async () => {
|
|
57
|
+
const originalWindow = globalThis.window;
|
|
58
|
+
const storybookWindow = new JSDOM(`<main class="sbdocs-content">
|
|
59
|
+
<h1>Button Component</h1>
|
|
60
|
+
<table class="docblock-argstable">
|
|
61
|
+
<tr><th>Name</th><th>Type</th><th>Default</th></tr>
|
|
62
|
+
<tr><td>variant</td><td>string</td><td>primary</td></tr>
|
|
63
|
+
</table>
|
|
64
|
+
</main>`).window;
|
|
65
|
+
const storybookDocument = storybookWindow.document;
|
|
66
|
+
globalThis.window = storybookWindow;
|
|
67
|
+
storybookWindow.getComputedStyle = vi.fn().mockReturnValue({
|
|
68
|
+
display: 'block',
|
|
69
|
+
visibility: 'visible',
|
|
70
|
+
opacity: '1',
|
|
71
|
+
});
|
|
72
|
+
const storybook = contentExtractors.storybook;
|
|
73
|
+
const expandSpy = vi.spyOn(storybook, 'expandAllTypeValues').mockResolvedValue(undefined);
|
|
74
|
+
const apiSpy = vi.spyOn(storybook, 'waitForStorybookAPI').mockResolvedValue(undefined);
|
|
75
|
+
const contentSpy = vi.spyOn(storybook, 'waitForStorybookContent').mockResolvedValue(storybookDocument.querySelector('main'));
|
|
76
|
+
try {
|
|
77
|
+
const storybookContent = await storybook.extractContent(storybookDocument);
|
|
78
|
+
const storybookResult = await processor.process({
|
|
79
|
+
url: 'https://storybook.example.com/button',
|
|
80
|
+
path: '/button',
|
|
81
|
+
extractorUsed: contentExtractors.storybook.constructor.name,
|
|
82
|
+
...storybookContent,
|
|
83
|
+
title: storybookContent.title || 'Button',
|
|
84
|
+
});
|
|
85
|
+
expect(storybookResult.metadata.title).toBe('Button Component');
|
|
86
|
+
expect(storybookResult.chunks[0].content).toContain('Button Component');
|
|
87
|
+
const propsChunk = storybookResult.chunks.find((chunk) => chunk.metadata.props?.length);
|
|
88
|
+
expect(propsChunk?.content).toContain('| variant | string | primary |');
|
|
89
|
+
expect(propsChunk?.metadata.props).toContainEqual(expect.objectContaining({ name: 'variant', type: 'string', defaultValue: 'primary' }));
|
|
90
|
+
}
|
|
91
|
+
finally {
|
|
92
|
+
contentSpy.mockRestore();
|
|
93
|
+
apiSpy.mockRestore();
|
|
94
|
+
expandSpy.mockRestore();
|
|
95
|
+
globalThis.window = originalWindow;
|
|
96
|
+
}
|
|
111
97
|
});
|
|
112
98
|
it('should create chunks with proper metadata', async () => {
|
|
113
99
|
const crawlResult = {
|
|
114
100
|
url: 'https://example.com/api',
|
|
115
101
|
path: '/api',
|
|
102
|
+
contentFormat: 'markdown',
|
|
116
103
|
title: 'API',
|
|
117
|
-
content:
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
104
|
+
content: `# API Reference
|
|
105
|
+
|
|
106
|
+
This document describes the API endpoints.
|
|
107
|
+
|
|
108
|
+
## GET /users
|
|
109
|
+
|
|
110
|
+
Returns a list of users.
|
|
111
|
+
|
|
112
|
+
\`\`\`json
|
|
126
113
|
{
|
|
127
114
|
"users": [...]
|
|
128
115
|
}
|
|
129
|
-
|
|
130
|
-
</main>
|
|
131
|
-
</body>
|
|
132
|
-
</html>
|
|
133
|
-
`,
|
|
116
|
+
\`\`\``,
|
|
134
117
|
};
|
|
135
118
|
const result = await processor.process(crawlResult);
|
|
136
119
|
expect(result).toBeDefined();
|
|
@@ -145,45 +128,29 @@ Follow these steps to get started.
|
|
|
145
128
|
it('should handle large content by creating multiple chunks', async () => {
|
|
146
129
|
const longContent = Array(50)
|
|
147
130
|
.fill(null)
|
|
148
|
-
.map((_, i) =>
|
|
149
|
-
|
|
150
|
-
<p>This is the content for section ${i + 1}. It contains some text that will need to be chunked appropriately for the embedding model. Lorem ipsum dolor sit amet, consectetur adipiscing elit.</p>
|
|
151
|
-
`)
|
|
152
|
-
.join('\n');
|
|
131
|
+
.map((_, i) => `## Section ${i + 1}\n\nThis is the content for section ${i + 1}. It contains some text that will need to be chunked appropriately for the embedding model. Lorem ipsum dolor sit amet, consectetur adipiscing elit.`)
|
|
132
|
+
.join('\n\n');
|
|
153
133
|
const crawlResult = {
|
|
154
134
|
url: 'https://example.com/long',
|
|
155
135
|
path: '/long',
|
|
136
|
+
contentFormat: 'markdown',
|
|
156
137
|
title: 'Long Document',
|
|
157
|
-
content:
|
|
158
|
-
<html>
|
|
159
|
-
<body>
|
|
160
|
-
<main>
|
|
161
|
-
<h1>Long Document</h1>
|
|
162
|
-
${longContent}
|
|
163
|
-
</main>
|
|
164
|
-
</body>
|
|
165
|
-
</html>
|
|
166
|
-
`,
|
|
138
|
+
content: `# Long Document\n\n${longContent}`,
|
|
167
139
|
};
|
|
168
140
|
const result = await processor.process(crawlResult);
|
|
169
141
|
expect(result).toBeDefined();
|
|
170
142
|
expect(result.chunks.length).toBeGreaterThan(1);
|
|
171
143
|
});
|
|
172
|
-
it(
|
|
144
|
+
it.each([
|
|
145
|
+
['empty', ''],
|
|
146
|
+
['whitespace-only', ' \n\n '],
|
|
147
|
+
])('should throw error for %s content', async (_, content) => {
|
|
173
148
|
const crawlResult = {
|
|
174
149
|
url: 'https://example.com/empty',
|
|
175
150
|
path: '/empty',
|
|
151
|
+
contentFormat: 'markdown',
|
|
176
152
|
title: 'Empty',
|
|
177
|
-
content
|
|
178
|
-
};
|
|
179
|
-
await expect(processor.process(crawlResult)).rejects.toThrow();
|
|
180
|
-
});
|
|
181
|
-
it('should throw error for whitespace-only content', async () => {
|
|
182
|
-
const crawlResult = {
|
|
183
|
-
url: 'https://example.com/whitespace',
|
|
184
|
-
path: '/whitespace',
|
|
185
|
-
title: 'Whitespace',
|
|
186
|
-
content: ' \n\n ',
|
|
153
|
+
content,
|
|
187
154
|
};
|
|
188
155
|
await expect(processor.process(crawlResult)).rejects.toThrow();
|
|
189
156
|
});
|
|
@@ -193,17 +160,9 @@ Follow these steps to get started.
|
|
|
193
160
|
const crawlResult = {
|
|
194
161
|
url: 'https://example.com/test',
|
|
195
162
|
path: '/test',
|
|
163
|
+
contentFormat: 'text',
|
|
196
164
|
title: 'Test',
|
|
197
|
-
content:
|
|
198
|
-
<html>
|
|
199
|
-
<body>
|
|
200
|
-
<main>
|
|
201
|
-
<h1>Test</h1>
|
|
202
|
-
<p>Some content here.</p>
|
|
203
|
-
</main>
|
|
204
|
-
</body>
|
|
205
|
-
</html>
|
|
206
|
-
`,
|
|
165
|
+
content: 'Test\n\nSome content here.',
|
|
207
166
|
};
|
|
208
167
|
await expect(failingProcessor.process(crawlResult)).rejects.toThrow('Embeddings service unavailable');
|
|
209
168
|
});
|
|
@@ -211,17 +170,9 @@ Follow these steps to get started.
|
|
|
211
170
|
const crawlResult = {
|
|
212
171
|
url: 'https://example.com/dated',
|
|
213
172
|
path: '/dated',
|
|
173
|
+
contentFormat: 'text',
|
|
214
174
|
title: 'Dated',
|
|
215
|
-
content:
|
|
216
|
-
<html>
|
|
217
|
-
<body>
|
|
218
|
-
<main>
|
|
219
|
-
<h1>Document</h1>
|
|
220
|
-
<p>Content with date.</p>
|
|
221
|
-
</main>
|
|
222
|
-
</body>
|
|
223
|
-
</html>
|
|
224
|
-
`,
|
|
175
|
+
content: 'Document\n\nContent with date.',
|
|
225
176
|
};
|
|
226
177
|
const before = new Date();
|
|
227
178
|
const result = await processor.process(crawlResult);
|
|
@@ -234,18 +185,13 @@ Follow these steps to get started.
|
|
|
234
185
|
const crawlResult = {
|
|
235
186
|
url: 'https://example.com/chunks',
|
|
236
187
|
path: '/chunks',
|
|
188
|
+
contentFormat: 'text',
|
|
237
189
|
title: 'Chunks',
|
|
238
|
-
content: `
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
<p>This is a longer paragraph that should be split into multiple chunks when using a small chunk size. The semantic chunker should create appropriate boundaries.</p>
|
|
244
|
-
<p>Another paragraph with additional content that needs to be processed and chunked appropriately.</p>
|
|
245
|
-
</main>
|
|
246
|
-
</body>
|
|
247
|
-
</html>
|
|
248
|
-
`,
|
|
190
|
+
content: `Document
|
|
191
|
+
|
|
192
|
+
This is a longer paragraph that should be split into multiple chunks when using a small chunk size. The semantic chunker should create appropriate boundaries.
|
|
193
|
+
|
|
194
|
+
Another paragraph with additional content that needs to be processed and chunked appropriately.`,
|
|
249
195
|
};
|
|
250
196
|
const result = await smallChunkProcessor.process(crawlResult);
|
|
251
197
|
// With smaller chunk size, should create more chunks
|
|
@@ -255,6 +201,7 @@ Follow these steps to get started.
|
|
|
255
201
|
const crawlResult = {
|
|
256
202
|
url: 'https://example.com/docs/component.mdx',
|
|
257
203
|
path: '/docs/component.mdx',
|
|
204
|
+
contentFormat: 'markdown',
|
|
258
205
|
title: 'MDX Component',
|
|
259
206
|
content: `# MDX Component
|
|
260
207
|
|
|
@@ -279,6 +226,7 @@ import { MyComponent } from 'library';
|
|
|
279
226
|
const crawlResult = {
|
|
280
227
|
url: 'https://example.com/default',
|
|
281
228
|
path: '/default',
|
|
229
|
+
contentFormat: 'text',
|
|
282
230
|
title: 'Default Extracted',
|
|
283
231
|
content: `Page Title
|
|
284
232
|
|
|
@@ -298,22 +246,23 @@ Another section of content here.`,
|
|
|
298
246
|
const crawlResult = {
|
|
299
247
|
url: 'https://example.com/api-ref',
|
|
300
248
|
path: '/api-ref',
|
|
249
|
+
contentFormat: 'markdown',
|
|
301
250
|
title: 'API Reference',
|
|
302
|
-
content:
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
251
|
+
content: `# API Reference
|
|
252
|
+
|
|
253
|
+
## GET /api/users
|
|
254
|
+
|
|
255
|
+
Returns array of users
|
|
256
|
+
|
|
257
|
+
### Parameters
|
|
258
|
+
|
|
259
|
+
limit: number - Maximum results
|
|
260
|
+
|
|
261
|
+
### Response
|
|
262
|
+
|
|
263
|
+
\`\`\`json
|
|
264
|
+
{"users": []}
|
|
265
|
+
\`\`\``,
|
|
317
266
|
};
|
|
318
267
|
const result = await processor.process(crawlResult);
|
|
319
268
|
expect(result).toBeDefined();
|
|
@@ -325,26 +274,23 @@ Another section of content here.`,
|
|
|
325
274
|
const crawlResult = {
|
|
326
275
|
url: 'https://example.com/examples',
|
|
327
276
|
path: '/examples',
|
|
277
|
+
contentFormat: 'markdown',
|
|
328
278
|
title: 'Examples',
|
|
329
|
-
content:
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
<h2>Basic Example</h2>
|
|
335
|
-
<pre><code>
|
|
279
|
+
content: `# Code Examples
|
|
280
|
+
|
|
281
|
+
## Basic Example
|
|
282
|
+
|
|
283
|
+
\`\`\`js
|
|
336
284
|
const result = doSomething();
|
|
337
285
|
console.log(result);
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
286
|
+
\`\`\`
|
|
287
|
+
|
|
288
|
+
## Advanced Example
|
|
289
|
+
|
|
290
|
+
\`\`\`js
|
|
341
291
|
const config = { advanced: true };
|
|
342
292
|
const result = doSomething(config);
|
|
343
|
-
|
|
344
|
-
</main>
|
|
345
|
-
</body>
|
|
346
|
-
</html>
|
|
347
|
-
`,
|
|
293
|
+
\`\`\``,
|
|
348
294
|
};
|
|
349
295
|
const result = await processor.process(crawlResult);
|
|
350
296
|
expect(result).toBeDefined();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"processor.test.js","sourceRoot":"","sources":["../../src/processor/processor.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AACtD,OAAO,EAAE,oBAAoB,EAAE,uBAAuB,EAAE,MAAM,4BAA4B,CAAC;
|
|
1
|
+
{"version":3,"file":"processor.test.js","sourceRoot":"","sources":["../../src/processor/processor.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAC9B,OAAO,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AACtD,OAAO,EAAE,oBAAoB,EAAE,uBAAuB,EAAE,MAAM,4BAA4B,CAAC;AAG3F,OAAO,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAC;AAErE,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;IACpC,IAAI,SAA+B,CAAC;IACpC,IAAI,cAAkC,CAAC;IAEvC,UAAU,CAAC,GAAG,EAAE;QACd,cAAc,GAAG,oBAAoB,EAAE,CAAC;QACxC,SAAS,GAAG,IAAI,oBAAoB,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;IAC5D,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,SAAS,EAAE,GAAG,EAAE;QACvB,EAAE,CAAC,iCAAiC,EAAE,KAAK,IAAI,EAAE;YAC/C,MAAM,WAAW,GAAgB;gBAC/B,GAAG,EAAE,oCAAoC;gBACzC,IAAI,EAAE,mBAAmB;gBACzB,aAAa,EAAE,UAAU;gBACzB,KAAK,EAAE,QAAQ;gBACf,OAAO,EAAE;;;;;;;;;;;;;;;CAehB;aACM,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;YAEpD,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC7B,yCAAyC;YACzC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,UAAU,EAAE,CAAC;YAC3C,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;YAC7D,MAAM,KAAK,GAAgB;gBACzB,GAAG,EAAE,2BAA2B;gBAChC,IAAI,EAAE,aAAa;gBACnB,aAAa,EAAE,UAAU;gBACzB,KAAK,EAAE,OAAO;gBACd,OAAO,EAAE,2CAA2C;gBACpD,aAAa,EAAE,gBAAgB;aAChC,CAAC;YAEF,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAC7C,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,EAAE,GAAG,KAAK,EAAE,aAAa,EAAE,kBAAkB,EAAE,CAAC,CAAC;YAExF,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YACzD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAC7E,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,CAC/D,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YAChE,MAAM,cAAc,GAAG,UAAU,CAAC,MAAM,CAAC;YACzC,MAAM,eAAe,GAAG,IAAI,KAAK,CAAC;;;;;;cAM1B,CAAC,CAAC,MAAM,CAAC;YACjB,MAAM,iBAAiB,GAAG,eAAe,CAAC,QAAQ,CAAC;YACnD,UAAU,CAAC,MAAM,GAAG,eAAsD,CAAC;YAC3E,eAAe,CAAC,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC;gBACzD,OAAO,EAAE,OAAO;gBAChB,UAAU,EAAE,SAAS;gBACrB,OAAO,EAAE,GAAG;aACb,CAAuD,CAAC;YAEzD,MAAM,SAAS,GAAG,iBAAiB,CAAC,SAKnC,CAAC;YACF,MAAM,SAAS,GAAG,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,qBAAqB,CAAC,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YAC1F,MAAM,MAAM,GAAG,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,qBAAqB,CAAC,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YACvF,MAAM,UAAU,GAAG,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,yBAAyB,CAAC,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC;YAE7H,IAAI,CAAC;gBACH,MAAM,gBAAgB,GAAG,MAAM,SAAS,CAAC,cAAc,CAAC,iBAAiB,CAAC,CAAC;gBAC3E,MAAM,eAAe,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC;oBAC9C,GAAG,EAAE,sCAAsC;oBAC3C,IAAI,EAAE,SAAS;oBACf,aAAa,EAAE,iBAAiB,CAAC,SAAS,CAAC,WAAW,CAAC,IAAI;oBAC3D,GAAG,gBAAgB;oBACnB,KAAK,EAAE,gBAAgB,CAAC,KAAK,IAAI,QAAQ;iBAC1C,CAAC,CAAC;gBACH,MAAM,CAAC,eAAe,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;gBAChE,MAAM,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;gBACxE,MAAM,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;gBACxF,MAAM,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,SAAS,CAAC,gCAAgC,CAAC,CAAC;gBACxE,MAAM,CAAC,UAAU,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,cAAc,CAC/C,MAAM,CAAC,gBAAgB,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,CAAC,CACtF,CAAC;YACJ,CAAC;oBACO,CAAC;gBACP,UAAU,CAAC,WAAW,EAAE,CAAC;gBACzB,MAAM,CAAC,WAAW,EAAE,CAAC;gBACrB,SAAS,CAAC,WAAW,EAAE,CAAC;gBACxB,UAAU,CAAC,MAAM,GAAG,cAAc,CAAC;YACrC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,MAAM,WAAW,GAAgB;gBAC/B,GAAG,EAAE,yBAAyB;gBAC9B,IAAI,EAAE,MAAM;gBACZ,aAAa,EAAE,UAAU;gBACzB,KAAK,EAAE,KAAK;gBACZ,OAAO,EAAE;;;;;;;;;;;;OAYV;aACA,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;YAEpD,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC7B,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;gBAC9B,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;gBACxC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;gBAC1C,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC;gBAC7D,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;gBACrC,MAAM,CAAC,CAAC,UAAU,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YACjF,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yDAAyD,EAAE,KAAK,IAAI,EAAE;YACvE,MAAM,WAAW,GAAG,KAAK,CAAC,EAAE,CAAC;iBAC1B,IAAI,CAAC,IAAI,CAAC;iBACV,GAAG,CACF,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,cAAc,CAAC,GAAG,CAAC,uCAAuC,CAAC,GAAG,CAAC,sJAAsJ,CACxN;iBACA,IAAI,CAAC,MAAM,CAAC,CAAC;YAEhB,MAAM,WAAW,GAAgB;gBAC/B,GAAG,EAAE,0BAA0B;gBAC/B,IAAI,EAAE,OAAO;gBACb,aAAa,EAAE,UAAU;gBACzB,KAAK,EAAE,eAAe;gBACtB,OAAO,EAAE,sBAAsB,WAAW,EAAE;aAC7C,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;YAEpD,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC7B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,IAAI,CAAC;YACN,CAAC,OAAO,EAAE,EAAE,CAAC;YACb,CAAC,iBAAiB,EAAE,YAAY,CAAC;SAClC,CAAC,CAAC,mCAAmC,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE;YAC3D,MAAM,WAAW,GAAgB;gBAC/B,GAAG,EAAE,2BAA2B;gBAChC,IAAI,EAAE,QAAQ;gBACd,aAAa,EAAE,UAAU;gBACzB,KAAK,EAAE,OAAO;gBACd,OAAO;aACR,CAAC;YAEF,MAAM,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QACjE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;YAC3D,MAAM,iBAAiB,GAAG,uBAAuB,EAAE,CAAC;YACpD,MAAM,gBAAgB,GAAG,IAAI,oBAAoB,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC;YAE1E,MAAM,WAAW,GAAgB;gBAC/B,GAAG,EAAE,0BAA0B;gBAC/B,IAAI,EAAE,OAAO;gBACb,aAAa,EAAE,MAAM;gBACrB,KAAK,EAAE,MAAM;gBACb,OAAO,EAAE,4BAA4B;aACtC,CAAC;YAEF,MAAM,MAAM,CAAC,gBAAgB,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,gCAAgC,CAAC,CAAC;QACxG,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;YAC3C,MAAM,WAAW,GAAgB;gBAC/B,GAAG,EAAE,2BAA2B;gBAChC,IAAI,EAAE,QAAQ;gBACd,aAAa,EAAE,MAAM;gBACrB,KAAK,EAAE,OAAO;gBACd,OAAO,EAAE,gCAAgC;aAC1C,CAAC;YAEF,MAAM,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YAC1B,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;YACpD,MAAM,KAAK,GAAG,IAAI,IAAI,EAAE,CAAC;YAEzB,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC,sBAAsB,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;YACvF,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC,mBAAmB,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACrF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACrD,MAAM,mBAAmB,GAAG,IAAI,oBAAoB,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;YAE1E,MAAM,WAAW,GAAgB;gBAC/B,GAAG,EAAE,4BAA4B;gBACjC,IAAI,EAAE,SAAS;gBACf,aAAa,EAAE,MAAM;gBACrB,KAAK,EAAE,QAAQ;gBACf,OAAO,EAAE;;;;gGAI+E;aACzF,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;YAE9D,qDAAqD;YACrD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yBAAyB,EAAE,KAAK,IAAI,EAAE;YACvC,MAAM,WAAW,GAAgB;gBAC/B,GAAG,EAAE,wCAAwC;gBAC7C,IAAI,EAAE,qBAAqB;gBAC3B,aAAa,EAAE,UAAU;gBACzB,KAAK,EAAE,eAAe;gBACtB,OAAO,EAAE;;;;;;;;;;;;;CAahB;aACM,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;YAEpD,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC7B,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,WAAW,GAAgB;gBAC/B,GAAG,EAAE,6BAA6B;gBAClC,IAAI,EAAE,UAAU;gBAChB,aAAa,EAAE,MAAM;gBACrB,KAAK,EAAE,mBAAmB;gBAC1B,OAAO,EAAE;;;;;iCAKgB;gBACzB,aAAa,EAAE,kBAAkB;aAClC,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;YAEpD,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC7B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,0BAA0B,EAAE,GAAG,EAAE;QACxC,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC9C,MAAM,WAAW,GAAgB;gBAC/B,GAAG,EAAE,6BAA6B;gBAClC,IAAI,EAAE,UAAU;gBAChB,aAAa,EAAE,UAAU;gBACzB,KAAK,EAAE,eAAe;gBACtB,OAAO,EAAE;;;;;;;;;;;;;;OAcV;aACA,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;YAEpD,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC7B,oCAAoC;YACpC,oDAAoD;YACpD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;YAClD,MAAM,WAAW,GAAgB;gBAC/B,GAAG,EAAE,8BAA8B;gBACnC,IAAI,EAAE,WAAW;gBACjB,aAAa,EAAE,UAAU;gBACzB,KAAK,EAAE,UAAU;gBACjB,OAAO,EAAE;;;;;;;;;;;;;;OAcV;aACA,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;YAEpD,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC7B,oCAAoC;YACpC,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC;YACpF,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
package/build/storage/storage.js
CHANGED
|
@@ -328,6 +328,13 @@ export class DocumentStore {
|
|
|
328
328
|
);
|
|
329
329
|
`,
|
|
330
330
|
},
|
|
331
|
+
{
|
|
332
|
+
version: 6,
|
|
333
|
+
description: 'Persist crawl path prefix',
|
|
334
|
+
sql: `
|
|
335
|
+
ALTER TABLE documents ADD COLUMN path_prefix TEXT;
|
|
336
|
+
`,
|
|
337
|
+
},
|
|
331
338
|
];
|
|
332
339
|
/**
|
|
333
340
|
* Run pending database migrations.
|
|
@@ -609,15 +616,16 @@ export class DocumentStore {
|
|
|
609
616
|
if (!this.sqliteDb) {
|
|
610
617
|
throw new Error('Storage not initialized');
|
|
611
618
|
}
|
|
612
|
-
await this.sqliteDb.run(`INSERT INTO documents (url, title, favicon, last_indexed, requires_auth, auth_domain, version)
|
|
613
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
619
|
+
await this.sqliteDb.run(`INSERT INTO documents (url, title, favicon, last_indexed, requires_auth, auth_domain, version, path_prefix)
|
|
620
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
614
621
|
ON CONFLICT(url) DO UPDATE SET
|
|
615
622
|
title = excluded.title,
|
|
616
623
|
favicon = excluded.favicon,
|
|
617
624
|
last_indexed = excluded.last_indexed,
|
|
618
625
|
requires_auth = excluded.requires_auth,
|
|
619
626
|
auth_domain = excluded.auth_domain,
|
|
620
|
-
version = excluded.version
|
|
627
|
+
version = excluded.version,
|
|
628
|
+
path_prefix = excluded.path_prefix`, [
|
|
621
629
|
metadata.url,
|
|
622
630
|
metadata.title,
|
|
623
631
|
metadata.favicon ?? null,
|
|
@@ -625,6 +633,7 @@ export class DocumentStore {
|
|
|
625
633
|
metadata.requiresAuth ? 1 : 0,
|
|
626
634
|
metadata.authDomain ?? null,
|
|
627
635
|
metadata.version ?? null,
|
|
636
|
+
metadata.pathPrefix ?? null,
|
|
628
637
|
]);
|
|
629
638
|
}
|
|
630
639
|
async recoverDocumentReplacements() {
|
|
@@ -1141,7 +1150,7 @@ export class DocumentStore {
|
|
|
1141
1150
|
}
|
|
1142
1151
|
logger.debug(`[DocumentStore] Listing documents`);
|
|
1143
1152
|
try {
|
|
1144
|
-
const rows = await this.sqliteReadDb.all(`SELECT d.url, d.title, d.favicon, d.last_indexed, d.requires_auth, d.auth_domain, d.version,
|
|
1153
|
+
const rows = await this.sqliteReadDb.all(`SELECT d.url, d.title, d.favicon, d.last_indexed, d.requires_auth, d.auth_domain, d.version, d.path_prefix,
|
|
1145
1154
|
COALESCE((SELECT json_group_array(dt.tag) FROM document_tags dt WHERE dt.url = d.url), '[]') AS tags_json
|
|
1146
1155
|
FROM documents d
|
|
1147
1156
|
ORDER BY d.last_indexed DESC`);
|
|
@@ -1155,6 +1164,7 @@ export class DocumentStore {
|
|
|
1155
1164
|
authDomain: row.auth_domain ?? undefined,
|
|
1156
1165
|
tags: this.parseDocumentTags(row.tags_json),
|
|
1157
1166
|
version: row.version ?? undefined,
|
|
1167
|
+
pathPrefix: row.path_prefix ?? undefined,
|
|
1158
1168
|
}));
|
|
1159
1169
|
}
|
|
1160
1170
|
catch (error) {
|
|
@@ -1230,7 +1240,7 @@ export class DocumentStore {
|
|
|
1230
1240
|
try {
|
|
1231
1241
|
// Log the query being executed
|
|
1232
1242
|
logger.debug(`[DocumentStore] Executing SQLite query for URL: ${url}`);
|
|
1233
|
-
const row = await this.sqliteReadDb.get(`SELECT d.url, d.title, d.favicon, d.last_indexed, d.requires_auth, d.auth_domain, d.version,
|
|
1243
|
+
const row = await this.sqliteReadDb.get(`SELECT d.url, d.title, d.favicon, d.last_indexed, d.requires_auth, d.auth_domain, d.version, d.path_prefix,
|
|
1234
1244
|
COALESCE((SELECT json_group_array(dt.tag) FROM document_tags dt WHERE dt.url = d.url), '[]') AS tags_json
|
|
1235
1245
|
FROM documents d
|
|
1236
1246
|
WHERE d.url = ?`, [url]);
|
|
@@ -1253,6 +1263,7 @@ export class DocumentStore {
|
|
|
1253
1263
|
authDomain: row.auth_domain ?? undefined,
|
|
1254
1264
|
tags: this.parseDocumentTags(row.tags_json),
|
|
1255
1265
|
version: row.version ?? undefined,
|
|
1266
|
+
pathPrefix: row.path_prefix ?? undefined,
|
|
1256
1267
|
};
|
|
1257
1268
|
}
|
|
1258
1269
|
catch (error) {
|
|
@@ -1554,7 +1565,7 @@ export class DocumentStore {
|
|
|
1554
1565
|
}
|
|
1555
1566
|
// Get documents in the collection
|
|
1556
1567
|
const docRows = await this.sqliteReadDb.all(`
|
|
1557
|
-
SELECT d.url, d.title, d.favicon, d.last_indexed, d.requires_auth, d.auth_domain, d.version,
|
|
1568
|
+
SELECT d.url, d.title, d.favicon, d.last_indexed, d.requires_auth, d.auth_domain, d.version, d.path_prefix,
|
|
1558
1569
|
COALESCE((SELECT json_group_array(dt.tag) FROM document_tags dt WHERE dt.url = d.url), '[]') AS tags_json
|
|
1559
1570
|
FROM documents d
|
|
1560
1571
|
INNER JOIN collection_documents cd ON d.url = cd.url
|
|
@@ -1570,6 +1581,7 @@ export class DocumentStore {
|
|
|
1570
1581
|
authDomain: doc.auth_domain ?? undefined,
|
|
1571
1582
|
tags: this.parseDocumentTags(doc.tags_json),
|
|
1572
1583
|
version: doc.version ?? undefined,
|
|
1584
|
+
pathPrefix: doc.path_prefix ?? undefined,
|
|
1573
1585
|
}));
|
|
1574
1586
|
logger.debug(`[DocumentStore] Collection "${normalizedName}" has ${documents.length} documents`);
|
|
1575
1587
|
return {
|