@cosmocoder/mcp-web-docs 2.0.13 → 2.0.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/config.d.ts +0 -1
- package/build/config.js +0 -5
- package/build/config.js.map +1 -1
- package/build/config.test.js +1 -25
- package/build/config.test.js.map +1 -1
- package/build/crawler/base.test.js +1 -0
- package/build/crawler/base.test.js.map +1 -1
- package/build/crawler/content-extractor-types.d.ts +3 -0
- package/build/crawler/crawlee-crawler.js +14 -9
- package/build/crawler/crawlee-crawler.js.map +1 -1
- package/build/crawler/crawlee-crawler.test.js +58 -7
- package/build/crawler/crawlee-crawler.test.js.map +1 -1
- package/build/crawler/default-extractor.js +1 -0
- package/build/crawler/default-extractor.js.map +1 -1
- package/build/crawler/default-extractor.test.js +1 -0
- package/build/crawler/default-extractor.test.js.map +1 -1
- package/build/crawler/docs-crawler.test.js +18 -10
- package/build/crawler/docs-crawler.test.js.map +1 -1
- package/build/crawler/github-pages-extractor.js +2 -0
- package/build/crawler/github-pages-extractor.js.map +1 -1
- package/build/crawler/github-pages-extractor.test.js +2 -0
- package/build/crawler/github-pages-extractor.test.js.map +1 -1
- package/build/crawler/github.js +1 -0
- package/build/crawler/github.js.map +1 -1
- package/build/crawler/github.test.js +1 -0
- package/build/crawler/github.test.js.map +1 -1
- package/build/crawler/queue-manager.test.js +7 -0
- package/build/crawler/queue-manager.test.js.map +1 -1
- package/build/crawler/storybook-extractor.js +3 -0
- package/build/crawler/storybook-extractor.js.map +1 -1
- package/build/crawler/storybook-extractor.test.js +2 -0
- package/build/crawler/storybook-extractor.test.js.map +1 -1
- package/build/index.js +2 -10
- package/build/index.js.map +1 -1
- package/build/index.test.js +19 -5
- package/build/index.test.js.map +1 -1
- package/build/processor/markdown.d.ts +15 -6
- package/build/processor/markdown.js +7 -30
- package/build/processor/markdown.js.map +1 -1
- package/build/processor/markdown.test.js +39 -6
- package/build/processor/markdown.test.js.map +1 -1
- package/build/processor/processor.js +8 -26
- package/build/processor/processor.js.map +1 -1
- package/build/processor/processor.test.js +118 -172
- package/build/processor/processor.test.js.map +1 -1
- package/build/storage/storage.js +34 -34
- package/build/storage/storage.js.map +1 -1
- package/build/storage/storage.test.js +90 -3
- package/build/storage/storage.test.js.map +1 -1
- package/build/types.d.ts +4 -0
- package/package.json +2 -3
- package/build/processor/content.d.ts +0 -16
- package/build/processor/content.js +0 -287
- package/build/processor/content.js.map +0 -1
- package/build/processor/content.test.d.ts +0 -1
- package/build/processor/content.test.js +0 -369
- package/build/processor/content.test.js.map +0 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"processor.js","sourceRoot":"","sources":["../../src/processor/processor.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"processor.js","sourceRoot":"","sources":["../../src/processor/processor.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAC;AAChF,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAErD;;GAEG;AACH,SAAS,uBAAuB,CAC9B,OAAe,EACf,YAA0D,EAC1D,SAAiB,EACjB,OAAe,EACf,MAAgB;IAEhB,MAAM,MAAM,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;IAEtC,OAAO;QACL,OAAO;QACP,SAAS;QACT,OAAO;QACP,MAAM;QACN,GAAG,EAAE,YAAY,CAAC,GAAG;QACrB,KAAK,EAAE,YAAY,CAAC,KAAK;QACzB,IAAI,EAAE,YAAY,CAAC,IAAI;QACvB,QAAQ,EAAE;YACR,IAAI,EAAE,MAAM,CAAC,WAAW;YACxB,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;YACzD,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;SACzE;KACF,CAAC;AACJ,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,eAAe,CAC7B,OAAe,EACf,YAAoB,EACpB,UAA8B,EAC9B,QAIC;IAED,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO;IACT,CAAC;IAED,wEAAwE;IACxE,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;IAChD,IAAI,YAAY,GAAG,EAAE,CAAC;IACtB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;QAEnC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7B,WAAW,IAAI,YAAY,CAAC,MAAM,CAAC;YACnC,SAAS;QACX,CAAC;QAED,gEAAgE;QAChE,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAExD,kDAAkD;QAClD,IAAI,aAAa,GAAG,YAAY,EAAE,CAAC;YACjC,yCAAyC;YACzC,IAAI,YAAY,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACnC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;gBACpD,MAAM,uBAAuB,CAAC,YAAY,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,WAAW,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC;gBACjG,YAAY,GAAG,EAAE,CAAC;YACpB,CAAC;YAED,mCAAmC;YACnC,MAAM,SAAS,GAAG,WAAW,CAAC,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACvE,IAAI,aAAa,GAAG,EAAE,CAAC;YACvB,IAAI,cAAc,GAAG,CAAC,CAAC;YAEvB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;gBACjC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAElD,IAAI,cAAc,GAAG,UAAU,GAAG,YAAY,GAAG,CAAC,EAAE,CAAC;oBACnD,IAAI,aAAa,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBACpC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;wBACrD,MAAM,uBAAuB,CAC3B,aAAa,CAAC,IAAI,EAAE,EACpB,QAAQ,EACR,WAAW,EACX,WAAW,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,EAClD,MAAM,CACP,CAAC;oBACJ,CAAC;oBACD,aAAa,GAAG,QAAQ,CAAC;oBACzB,cAAc,GAAG,UAAU,CAAC;gBAC9B,CAAC;qBACI,CAAC;oBACJ,aAAa,IAAI,GAAG,GAAG,QAAQ,CAAC;oBAChC,cAAc,IAAI,UAAU,CAAC;gBAC/B,CAAC;YACH,CAAC;YAED,iCAAiC;YACjC,IAAI,aAAa,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACpC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;gBACrD,MAAM,uBAAuB,CAC3B,aAAa,CAAC,IAAI,EAAE,EACpB,QAAQ,EACR,WAAW,EACX,WAAW,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,EAClD,MAAM,CACP,CAAC;YACJ,CAAC;QACH,CAAC;QACD,8EAA8E;aACzE,IAAI,UAAU,GAAG,aAAa,GAAG,YAAY,GAAG,CAAC,EAAE,CAAC;YACvD,IAAI,YAAY,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACnC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;gBACpD,MAAM,uBAAuB,CAAC,YAAY,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,WAAW,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC;YACnG,CAAC;YACD,YAAY,GAAG,WAAW,CAAC;YAC3B,UAAU,GAAG,aAAa,CAAC;YAC3B,SAAS,GAAG,WAAW,CAAC;QAC1B,CAAC;QACD,yCAAyC;aACpC,CAAC;YACJ,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,YAAY,IAAI,MAAM,CAAC;YACzB,CAAC;YACD,YAAY,IAAI,WAAW,CAAC;YAC5B,UAAU,IAAI,aAAa,CAAC;QAC9B,CAAC;QAED,WAAW,IAAI,YAAY,CAAC,MAAM,CAAC;IACrC,CAAC;IAED,iCAAiC;IACjC,IAAI,YAAY,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QACpD,MAAM,uBAAuB,CAAC,YAAY,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,WAAW,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC;IACnG,CAAC;AACH,CAAC;AAED,MAAM,OAAO,oBAAoB;IAEZ;IACA;IAFnB,YACmB,UAA8B,EAC9B,eAAuB,IAAI;QAD3B,eAAU,GAAV,UAAU,CAAoB;QAC9B,iBAAY,GAAZ,YAAY,CAAe;IAC3C,CAAC;IAEJ,KAAK,CAAC,OAAO,CAAC,WAAwB;QACpC,MAAM,CAAC,KAAK,CAAC,qCAAqC,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC;QACrE,MAAM,CAAC,KAAK,CAAC,0CAA0C,WAAW,CAAC,OAAO,CAAC,MAAM,QAAQ,CAAC,CAAC;QAC3F,MAAM,CAAC,KAAK,CAAC,0CAA0C,WAAW,CAAC,aAAa,IAAI,SAAS,EAAE,CAAC,CAAC;QAEjG,IAAI,CAAC;YACH,MAAM,CAAC,KAAK,CAAC,gCAAgC,WAAW,CAAC,aAAa,kBAAkB,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC;YAC5G,IAAI,gBAAgB,CAAC;YACrB,QAAQ,WAAW,CAAC,aAAa,EAAE,CAAC;gBAClC,KAAK,UAAU;oBACb,gBAAgB,GAAG,MAAM,sBAAsB,CAAC,WAAW,CAAC,CAAC;oBAC7D,MAAM;gBACR,KAAK,MAAM;oBACT,gBAAgB,GAAG,MAAM,uBAAuB,CAAC,WAAW,CAAC,CAAC;oBAC9D,MAAM;YACV,CAAC;YAED,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACtB,MAAM,CAAC,KAAK,CAAC,+DAA+D,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC;gBAC/F,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;YACtD,CAAC;YAED,MAAM,CAAC,KAAK,CAAC,6DAA6D,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC;YAC7F,MAAM,CAAC,KAAK,CAAC,gCAAgC,gBAAgB,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,aAAa,CAAC,CAAC;YACtG,MAAM,CAAC,KAAK,CAAC,8CAA8C,gBAAgB,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;YAE7F,MAAM,MAAM,GAAoB,EAAE,CAAC;YACnC,IAAI,WAAW,GAAG,CAAC,CAAC;YAEpB,MAAM,QAAQ,GAAG;gBACf,GAAG,EAAE,gBAAgB,CAAC,OAAO,CAAC,GAAG;gBACjC,KAAK,EAAE,gBAAgB,CAAC,OAAO,CAAC,KAAK;gBACrC,IAAI,EAAE,gBAAgB,CAAC,OAAO,CAAC,IAAI;aACpC,CAAC;YAEF,oCAAoC;YACpC,KAAK,MAAM,SAAS,IAAI,gBAAgB,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;gBAC5D,MAAM,CAAC,KAAK,CAAC,gDAAgD,SAAS,CAAC,KAAK,EAAE,CAAC,CAAC;gBAChF,MAAM,CAAC,KAAK,CAAC,iDAAiD,SAAS,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,CAAC;gBAE7F,MAAM,gBAAgB,GAAG,GAAG,SAAS,CAAC,KAAK,OAAO,SAAS,CAAC,IAAI,EAAE,CAAC;gBACnE,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,eAAe,CAAC,gBAAgB,EAAE,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,EAAE,CAAC;oBAC1G,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACnB,WAAW,EAAE,CAAC;gBAChB,CAAC;YACH,CAAC;YAED,MAAM,CAAC,KAAK,CAAC,kCAAkC,WAAW,SAAS,CAAC,CAAC;YAErE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACxB,MAAM,CAAC,IAAI,CAAC,2DAA2D,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC;gBAC1F,MAAM,CAAC,IAAI,CAAC,mDAAmD,WAAW,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;gBAC7F,MAAM,CAAC,IAAI,CAAC,oDAAoD,gBAAgB,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;gBACnG,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;YAClD,CAAC;YAED,MAAM,CAAC,KAAK,CAAC,iDAAiD,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC;YACjF,OAAO;gBACL,QAAQ,EAAE;oBACR,GAAG,EAAE,WAAW,CAAC,GAAG;oBACpB,KAAK,EAAE,gBAAgB,CAAC,OAAO,CAAC,KAAK;oBACrC,WAAW,EAAE,IAAI,IAAI,EAAE;iBACxB;gBACD,MAAM;aACP,CAAC;QACJ,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,2CAA2C,WAAW,CAAC,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;YACnF,MAAM,CAAC,KAAK,CAAC,uCAAuC,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACpG,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;CACF"}
|
|
@@ -1,5 +1,7 @@
|
|
|
1
|
+
import { JSDOM } from 'jsdom';
|
|
1
2
|
import { WebDocumentProcessor } from './processor.js';
|
|
2
3
|
import { createMockEmbeddings, createFailingEmbeddings } from '../__mocks__/embeddings.js';
|
|
4
|
+
import { contentExtractors } from '../crawler/content-extractors.js';
|
|
3
5
|
describe('WebDocumentProcessor', () => {
|
|
4
6
|
let processor;
|
|
5
7
|
let mockEmbeddings;
|
|
@@ -8,36 +10,11 @@ describe('WebDocumentProcessor', () => {
|
|
|
8
10
|
processor = new WebDocumentProcessor(mockEmbeddings, 500);
|
|
9
11
|
});
|
|
10
12
|
describe('process', () => {
|
|
11
|
-
it('should process HTML content', async () => {
|
|
12
|
-
const crawlResult = {
|
|
13
|
-
url: 'https://example.com/docs/page',
|
|
14
|
-
path: '/docs/page',
|
|
15
|
-
title: 'Test Page',
|
|
16
|
-
content: `
|
|
17
|
-
<html>
|
|
18
|
-
<body>
|
|
19
|
-
<main>
|
|
20
|
-
<h1>Test Documentation</h1>
|
|
21
|
-
<p>This is some test content for the documentation page.</p>
|
|
22
|
-
<h2>Features</h2>
|
|
23
|
-
<p>Here are some features of our product.</p>
|
|
24
|
-
</main>
|
|
25
|
-
</body>
|
|
26
|
-
</html>
|
|
27
|
-
`,
|
|
28
|
-
};
|
|
29
|
-
const result = await processor.process(crawlResult);
|
|
30
|
-
expect(result).toBeDefined();
|
|
31
|
-
expect(result.metadata.url).toBe(crawlResult.url);
|
|
32
|
-
// Title may come from crawl result or H1, depending on processor logic
|
|
33
|
-
expect(result.metadata.title).toBeTruthy();
|
|
34
|
-
expect(result.chunks.length).toBeGreaterThan(0);
|
|
35
|
-
expect(result.chunks[0].vector.length).toBe(mockEmbeddings.dimensions);
|
|
36
|
-
});
|
|
37
13
|
it('should process markdown content', async () => {
|
|
38
14
|
const crawlResult = {
|
|
39
15
|
url: 'https://example.com/docs/readme.md',
|
|
40
|
-
path: '/docs/readme.
|
|
16
|
+
path: '/docs/readme.html',
|
|
17
|
+
contentFormat: 'markdown',
|
|
41
18
|
title: 'README',
|
|
42
19
|
content: `# Project README
|
|
43
20
|
|
|
@@ -62,75 +39,81 @@ Here's how to use the package.
|
|
|
62
39
|
expect(result.metadata.title).toBeTruthy();
|
|
63
40
|
expect(result.chunks.length).toBeGreaterThan(0);
|
|
64
41
|
});
|
|
65
|
-
it('should
|
|
66
|
-
const
|
|
67
|
-
url: 'https://
|
|
68
|
-
path: '/
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
## Props
|
|
75
|
-
|
|
76
|
-
| Prop | Type |
|
|
77
|
-
|------|------|
|
|
78
|
-
| variant | string |
|
|
79
|
-
|
|
80
|
-
## Example
|
|
81
|
-
|
|
82
|
-
\`\`\`jsx
|
|
83
|
-
<Button variant="primary">Click</Button>
|
|
84
|
-
\`\`\`
|
|
85
|
-
`,
|
|
86
|
-
extractorUsed: 'StorybookExtractor',
|
|
42
|
+
it('should treat extractorUsed as diagnostic-only', async () => {
|
|
43
|
+
const input = {
|
|
44
|
+
url: 'https://example.com/guide',
|
|
45
|
+
path: '/guide.html',
|
|
46
|
+
contentFormat: 'markdown',
|
|
47
|
+
title: 'Guide',
|
|
48
|
+
content: '# Explicit Markdown\n\n## Section\n\nBody',
|
|
49
|
+
extractorUsed: 'FirstExtractor',
|
|
87
50
|
};
|
|
88
|
-
const
|
|
89
|
-
|
|
90
|
-
expect(
|
|
91
|
-
expect(
|
|
51
|
+
const first = await processor.process(input);
|
|
52
|
+
const second = await processor.process({ ...input, extractorUsed: 'RenamedExtractor' });
|
|
53
|
+
expect(second.metadata.title).toBe(first.metadata.title);
|
|
54
|
+
expect(second.chunks.map(({ content, title }) => ({ content, title }))).toEqual(first.chunks.map(({ content, title }) => ({ content, title })));
|
|
92
55
|
});
|
|
93
|
-
it('should process
|
|
94
|
-
const
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
56
|
+
it('should process actual Storybook extractor output', async () => {
|
|
57
|
+
const originalWindow = globalThis.window;
|
|
58
|
+
const storybookWindow = new JSDOM(`<main class="sbdocs-content">
|
|
59
|
+
<h1>Button Component</h1>
|
|
60
|
+
<table class="docblock-argstable">
|
|
61
|
+
<tr><th>Name</th><th>Type</th><th>Default</th></tr>
|
|
62
|
+
<tr><td>variant</td><td>string</td><td>primary</td></tr>
|
|
63
|
+
</table>
|
|
64
|
+
</main>`).window;
|
|
65
|
+
const storybookDocument = storybookWindow.document;
|
|
66
|
+
globalThis.window = storybookWindow;
|
|
67
|
+
storybookWindow.getComputedStyle = vi.fn().mockReturnValue({
|
|
68
|
+
display: 'block',
|
|
69
|
+
visibility: 'visible',
|
|
70
|
+
opacity: '1',
|
|
71
|
+
});
|
|
72
|
+
const storybook = contentExtractors.storybook;
|
|
73
|
+
const expandSpy = vi.spyOn(storybook, 'expandAllTypeValues').mockResolvedValue(undefined);
|
|
74
|
+
const apiSpy = vi.spyOn(storybook, 'waitForStorybookAPI').mockResolvedValue(undefined);
|
|
75
|
+
const contentSpy = vi.spyOn(storybook, 'waitForStorybookContent').mockResolvedValue(storybookDocument.querySelector('main'));
|
|
76
|
+
try {
|
|
77
|
+
const storybookContent = await storybook.extractContent(storybookDocument);
|
|
78
|
+
const storybookResult = await processor.process({
|
|
79
|
+
url: 'https://storybook.example.com/button',
|
|
80
|
+
path: '/button',
|
|
81
|
+
extractorUsed: contentExtractors.storybook.constructor.name,
|
|
82
|
+
...storybookContent,
|
|
83
|
+
title: storybookContent.title || 'Button',
|
|
84
|
+
});
|
|
85
|
+
expect(storybookResult.metadata.title).toBe('Button Component');
|
|
86
|
+
expect(storybookResult.chunks[0].content).toContain('Button Component');
|
|
87
|
+
const propsChunk = storybookResult.chunks.find((chunk) => chunk.metadata.props?.length);
|
|
88
|
+
expect(propsChunk?.content).toContain('| variant | string | primary |');
|
|
89
|
+
expect(propsChunk?.metadata.props).toContainEqual(expect.objectContaining({ name: 'variant', type: 'string', defaultValue: 'primary' }));
|
|
90
|
+
}
|
|
91
|
+
finally {
|
|
92
|
+
contentSpy.mockRestore();
|
|
93
|
+
apiSpy.mockRestore();
|
|
94
|
+
expandSpy.mockRestore();
|
|
95
|
+
globalThis.window = originalWindow;
|
|
96
|
+
}
|
|
111
97
|
});
|
|
112
98
|
it('should create chunks with proper metadata', async () => {
|
|
113
99
|
const crawlResult = {
|
|
114
100
|
url: 'https://example.com/api',
|
|
115
101
|
path: '/api',
|
|
102
|
+
contentFormat: 'markdown',
|
|
116
103
|
title: 'API',
|
|
117
|
-
content:
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
104
|
+
content: `# API Reference
|
|
105
|
+
|
|
106
|
+
This document describes the API endpoints.
|
|
107
|
+
|
|
108
|
+
## GET /users
|
|
109
|
+
|
|
110
|
+
Returns a list of users.
|
|
111
|
+
|
|
112
|
+
\`\`\`json
|
|
126
113
|
{
|
|
127
114
|
"users": [...]
|
|
128
115
|
}
|
|
129
|
-
|
|
130
|
-
</main>
|
|
131
|
-
</body>
|
|
132
|
-
</html>
|
|
133
|
-
`,
|
|
116
|
+
\`\`\``,
|
|
134
117
|
};
|
|
135
118
|
const result = await processor.process(crawlResult);
|
|
136
119
|
expect(result).toBeDefined();
|
|
@@ -145,45 +128,29 @@ Follow these steps to get started.
|
|
|
145
128
|
it('should handle large content by creating multiple chunks', async () => {
|
|
146
129
|
const longContent = Array(50)
|
|
147
130
|
.fill(null)
|
|
148
|
-
.map((_, i) =>
|
|
149
|
-
|
|
150
|
-
<p>This is the content for section ${i + 1}. It contains some text that will need to be chunked appropriately for the embedding model. Lorem ipsum dolor sit amet, consectetur adipiscing elit.</p>
|
|
151
|
-
`)
|
|
152
|
-
.join('\n');
|
|
131
|
+
.map((_, i) => `## Section ${i + 1}\n\nThis is the content for section ${i + 1}. It contains some text that will need to be chunked appropriately for the embedding model. Lorem ipsum dolor sit amet, consectetur adipiscing elit.`)
|
|
132
|
+
.join('\n\n');
|
|
153
133
|
const crawlResult = {
|
|
154
134
|
url: 'https://example.com/long',
|
|
155
135
|
path: '/long',
|
|
136
|
+
contentFormat: 'markdown',
|
|
156
137
|
title: 'Long Document',
|
|
157
|
-
content:
|
|
158
|
-
<html>
|
|
159
|
-
<body>
|
|
160
|
-
<main>
|
|
161
|
-
<h1>Long Document</h1>
|
|
162
|
-
${longContent}
|
|
163
|
-
</main>
|
|
164
|
-
</body>
|
|
165
|
-
</html>
|
|
166
|
-
`,
|
|
138
|
+
content: `# Long Document\n\n${longContent}`,
|
|
167
139
|
};
|
|
168
140
|
const result = await processor.process(crawlResult);
|
|
169
141
|
expect(result).toBeDefined();
|
|
170
142
|
expect(result.chunks.length).toBeGreaterThan(1);
|
|
171
143
|
});
|
|
172
|
-
it(
|
|
144
|
+
it.each([
|
|
145
|
+
['empty', ''],
|
|
146
|
+
['whitespace-only', ' \n\n '],
|
|
147
|
+
])('should throw error for %s content', async (_, content) => {
|
|
173
148
|
const crawlResult = {
|
|
174
149
|
url: 'https://example.com/empty',
|
|
175
150
|
path: '/empty',
|
|
151
|
+
contentFormat: 'markdown',
|
|
176
152
|
title: 'Empty',
|
|
177
|
-
content
|
|
178
|
-
};
|
|
179
|
-
await expect(processor.process(crawlResult)).rejects.toThrow();
|
|
180
|
-
});
|
|
181
|
-
it('should throw error for whitespace-only content', async () => {
|
|
182
|
-
const crawlResult = {
|
|
183
|
-
url: 'https://example.com/whitespace',
|
|
184
|
-
path: '/whitespace',
|
|
185
|
-
title: 'Whitespace',
|
|
186
|
-
content: ' \n\n ',
|
|
153
|
+
content,
|
|
187
154
|
};
|
|
188
155
|
await expect(processor.process(crawlResult)).rejects.toThrow();
|
|
189
156
|
});
|
|
@@ -193,17 +160,9 @@ Follow these steps to get started.
|
|
|
193
160
|
const crawlResult = {
|
|
194
161
|
url: 'https://example.com/test',
|
|
195
162
|
path: '/test',
|
|
163
|
+
contentFormat: 'text',
|
|
196
164
|
title: 'Test',
|
|
197
|
-
content:
|
|
198
|
-
<html>
|
|
199
|
-
<body>
|
|
200
|
-
<main>
|
|
201
|
-
<h1>Test</h1>
|
|
202
|
-
<p>Some content here.</p>
|
|
203
|
-
</main>
|
|
204
|
-
</body>
|
|
205
|
-
</html>
|
|
206
|
-
`,
|
|
165
|
+
content: 'Test\n\nSome content here.',
|
|
207
166
|
};
|
|
208
167
|
await expect(failingProcessor.process(crawlResult)).rejects.toThrow('Embeddings service unavailable');
|
|
209
168
|
});
|
|
@@ -211,17 +170,9 @@ Follow these steps to get started.
|
|
|
211
170
|
const crawlResult = {
|
|
212
171
|
url: 'https://example.com/dated',
|
|
213
172
|
path: '/dated',
|
|
173
|
+
contentFormat: 'text',
|
|
214
174
|
title: 'Dated',
|
|
215
|
-
content:
|
|
216
|
-
<html>
|
|
217
|
-
<body>
|
|
218
|
-
<main>
|
|
219
|
-
<h1>Document</h1>
|
|
220
|
-
<p>Content with date.</p>
|
|
221
|
-
</main>
|
|
222
|
-
</body>
|
|
223
|
-
</html>
|
|
224
|
-
`,
|
|
175
|
+
content: 'Document\n\nContent with date.',
|
|
225
176
|
};
|
|
226
177
|
const before = new Date();
|
|
227
178
|
const result = await processor.process(crawlResult);
|
|
@@ -234,18 +185,13 @@ Follow these steps to get started.
|
|
|
234
185
|
const crawlResult = {
|
|
235
186
|
url: 'https://example.com/chunks',
|
|
236
187
|
path: '/chunks',
|
|
188
|
+
contentFormat: 'text',
|
|
237
189
|
title: 'Chunks',
|
|
238
|
-
content: `
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
<p>This is a longer paragraph that should be split into multiple chunks when using a small chunk size. The semantic chunker should create appropriate boundaries.</p>
|
|
244
|
-
<p>Another paragraph with additional content that needs to be processed and chunked appropriately.</p>
|
|
245
|
-
</main>
|
|
246
|
-
</body>
|
|
247
|
-
</html>
|
|
248
|
-
`,
|
|
190
|
+
content: `Document
|
|
191
|
+
|
|
192
|
+
This is a longer paragraph that should be split into multiple chunks when using a small chunk size. The semantic chunker should create appropriate boundaries.
|
|
193
|
+
|
|
194
|
+
Another paragraph with additional content that needs to be processed and chunked appropriately.`,
|
|
249
195
|
};
|
|
250
196
|
const result = await smallChunkProcessor.process(crawlResult);
|
|
251
197
|
// With smaller chunk size, should create more chunks
|
|
@@ -255,6 +201,7 @@ Follow these steps to get started.
|
|
|
255
201
|
const crawlResult = {
|
|
256
202
|
url: 'https://example.com/docs/component.mdx',
|
|
257
203
|
path: '/docs/component.mdx',
|
|
204
|
+
contentFormat: 'markdown',
|
|
258
205
|
title: 'MDX Component',
|
|
259
206
|
content: `# MDX Component
|
|
260
207
|
|
|
@@ -279,6 +226,7 @@ import { MyComponent } from 'library';
|
|
|
279
226
|
const crawlResult = {
|
|
280
227
|
url: 'https://example.com/default',
|
|
281
228
|
path: '/default',
|
|
229
|
+
contentFormat: 'text',
|
|
282
230
|
title: 'Default Extracted',
|
|
283
231
|
content: `Page Title
|
|
284
232
|
|
|
@@ -298,22 +246,23 @@ Another section of content here.`,
|
|
|
298
246
|
const crawlResult = {
|
|
299
247
|
url: 'https://example.com/api-ref',
|
|
300
248
|
path: '/api-ref',
|
|
249
|
+
contentFormat: 'markdown',
|
|
301
250
|
title: 'API Reference',
|
|
302
|
-
content:
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
251
|
+
content: `# API Reference
|
|
252
|
+
|
|
253
|
+
## GET /api/users
|
|
254
|
+
|
|
255
|
+
Returns array of users
|
|
256
|
+
|
|
257
|
+
### Parameters
|
|
258
|
+
|
|
259
|
+
limit: number - Maximum results
|
|
260
|
+
|
|
261
|
+
### Response
|
|
262
|
+
|
|
263
|
+
\`\`\`json
|
|
264
|
+
{"users": []}
|
|
265
|
+
\`\`\``,
|
|
317
266
|
};
|
|
318
267
|
const result = await processor.process(crawlResult);
|
|
319
268
|
expect(result).toBeDefined();
|
|
@@ -325,26 +274,23 @@ Another section of content here.`,
|
|
|
325
274
|
const crawlResult = {
|
|
326
275
|
url: 'https://example.com/examples',
|
|
327
276
|
path: '/examples',
|
|
277
|
+
contentFormat: 'markdown',
|
|
328
278
|
title: 'Examples',
|
|
329
|
-
content:
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
<h2>Basic Example</h2>
|
|
335
|
-
<pre><code>
|
|
279
|
+
content: `# Code Examples
|
|
280
|
+
|
|
281
|
+
## Basic Example
|
|
282
|
+
|
|
283
|
+
\`\`\`js
|
|
336
284
|
const result = doSomething();
|
|
337
285
|
console.log(result);
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
286
|
+
\`\`\`
|
|
287
|
+
|
|
288
|
+
## Advanced Example
|
|
289
|
+
|
|
290
|
+
\`\`\`js
|
|
341
291
|
const config = { advanced: true };
|
|
342
292
|
const result = doSomething(config);
|
|
343
|
-
|
|
344
|
-
</main>
|
|
345
|
-
</body>
|
|
346
|
-
</html>
|
|
347
|
-
`,
|
|
293
|
+
\`\`\``,
|
|
348
294
|
};
|
|
349
295
|
const result = await processor.process(crawlResult);
|
|
350
296
|
expect(result).toBeDefined();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"processor.test.js","sourceRoot":"","sources":["../../src/processor/processor.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AACtD,OAAO,EAAE,oBAAoB,EAAE,uBAAuB,EAAE,MAAM,4BAA4B,CAAC;
|
|
1
|
+
{"version":3,"file":"processor.test.js","sourceRoot":"","sources":["../../src/processor/processor.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAC9B,OAAO,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AACtD,OAAO,EAAE,oBAAoB,EAAE,uBAAuB,EAAE,MAAM,4BAA4B,CAAC;AAG3F,OAAO,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAC;AAErE,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;IACpC,IAAI,SAA+B,CAAC;IACpC,IAAI,cAAkC,CAAC;IAEvC,UAAU,CAAC,GAAG,EAAE;QACd,cAAc,GAAG,oBAAoB,EAAE,CAAC;QACxC,SAAS,GAAG,IAAI,oBAAoB,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;IAC5D,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,SAAS,EAAE,GAAG,EAAE;QACvB,EAAE,CAAC,iCAAiC,EAAE,KAAK,IAAI,EAAE;YAC/C,MAAM,WAAW,GAAgB;gBAC/B,GAAG,EAAE,oCAAoC;gBACzC,IAAI,EAAE,mBAAmB;gBACzB,aAAa,EAAE,UAAU;gBACzB,KAAK,EAAE,QAAQ;gBACf,OAAO,EAAE;;;;;;;;;;;;;;;CAehB;aACM,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;YAEpD,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC7B,yCAAyC;YACzC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,UAAU,EAAE,CAAC;YAC3C,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;YAC7D,MAAM,KAAK,GAAgB;gBACzB,GAAG,EAAE,2BAA2B;gBAChC,IAAI,EAAE,aAAa;gBACnB,aAAa,EAAE,UAAU;gBACzB,KAAK,EAAE,OAAO;gBACd,OAAO,EAAE,2CAA2C;gBACpD,aAAa,EAAE,gBAAgB;aAChC,CAAC;YAEF,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAC7C,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,EAAE,GAAG,KAAK,EAAE,aAAa,EAAE,kBAAkB,EAAE,CAAC,CAAC;YAExF,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YACzD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAC7E,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,CAC/D,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YAChE,MAAM,cAAc,GAAG,UAAU,CAAC,MAAM,CAAC;YACzC,MAAM,eAAe,GAAG,IAAI,KAAK,CAAC;;;;;;cAM1B,CAAC,CAAC,MAAM,CAAC;YACjB,MAAM,iBAAiB,GAAG,eAAe,CAAC,QAAQ,CAAC;YACnD,UAAU,CAAC,MAAM,GAAG,eAAsD,CAAC;YAC3E,eAAe,CAAC,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC;gBACzD,OAAO,EAAE,OAAO;gBAChB,UAAU,EAAE,SAAS;gBACrB,OAAO,EAAE,GAAG;aACb,CAAuD,CAAC;YAEzD,MAAM,SAAS,GAAG,iBAAiB,CAAC,SAKnC,CAAC;YACF,MAAM,SAAS,GAAG,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,qBAAqB,CAAC,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YAC1F,MAAM,MAAM,GAAG,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,qBAAqB,CAAC,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YACvF,MAAM,UAAU,GAAG,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,yBAAyB,CAAC,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC;YAE7H,IAAI,CAAC;gBACH,MAAM,gBAAgB,GAAG,MAAM,SAAS,CAAC,cAAc,CAAC,iBAAiB,CAAC,CAAC;gBAC3E,MAAM,eAAe,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC;oBAC9C,GAAG,EAAE,sCAAsC;oBAC3C,IAAI,EAAE,SAAS;oBACf,aAAa,EAAE,iBAAiB,CAAC,SAAS,CAAC,WAAW,CAAC,IAAI;oBAC3D,GAAG,gBAAgB;oBACnB,KAAK,EAAE,gBAAgB,CAAC,KAAK,IAAI,QAAQ;iBAC1C,CAAC,CAAC;gBACH,MAAM,CAAC,eAAe,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;gBAChE,MAAM,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;gBACxE,MAAM,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;gBACxF,MAAM,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,SAAS,CAAC,gCAAgC,CAAC,CAAC;gBACxE,MAAM,CAAC,UAAU,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,cAAc,CAC/C,MAAM,CAAC,gBAAgB,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,CAAC,CACtF,CAAC;YACJ,CAAC;oBACO,CAAC;gBACP,UAAU,CAAC,WAAW,EAAE,CAAC;gBACzB,MAAM,CAAC,WAAW,EAAE,CAAC;gBACrB,SAAS,CAAC,WAAW,EAAE,CAAC;gBACxB,UAAU,CAAC,MAAM,GAAG,cAAc,CAAC;YACrC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,MAAM,WAAW,GAAgB;gBAC/B,GAAG,EAAE,yBAAyB;gBAC9B,IAAI,EAAE,MAAM;gBACZ,aAAa,EAAE,UAAU;gBACzB,KAAK,EAAE,KAAK;gBACZ,OAAO,EAAE;;;;;;;;;;;;OAYV;aACA,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;YAEpD,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC7B,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;gBAC9B,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;gBACxC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;gBAC1C,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC;gBAC7D,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;gBACrC,MAAM,CAAC,CAAC,UAAU,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YACjF,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yDAAyD,EAAE,KAAK,IAAI,EAAE;YACvE,MAAM,WAAW,GAAG,KAAK,CAAC,EAAE,CAAC;iBAC1B,IAAI,CAAC,IAAI,CAAC;iBACV,GAAG,CACF,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,cAAc,CAAC,GAAG,CAAC,uCAAuC,CAAC,GAAG,CAAC,sJAAsJ,CACxN;iBACA,IAAI,CAAC,MAAM,CAAC,CAAC;YAEhB,MAAM,WAAW,GAAgB;gBAC/B,GAAG,EAAE,0BAA0B;gBAC/B,IAAI,EAAE,OAAO;gBACb,aAAa,EAAE,UAAU;gBACzB,KAAK,EAAE,eAAe;gBACtB,OAAO,EAAE,sBAAsB,WAAW,EAAE;aAC7C,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;YAEpD,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC7B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,IAAI,CAAC;YACN,CAAC,OAAO,EAAE,EAAE,CAAC;YACb,CAAC,iBAAiB,EAAE,YAAY,CAAC;SAClC,CAAC,CAAC,mCAAmC,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE;YAC3D,MAAM,WAAW,GAAgB;gBAC/B,GAAG,EAAE,2BAA2B;gBAChC,IAAI,EAAE,QAAQ;gBACd,aAAa,EAAE,UAAU;gBACzB,KAAK,EAAE,OAAO;gBACd,OAAO;aACR,CAAC;YAEF,MAAM,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QACjE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;YAC3D,MAAM,iBAAiB,GAAG,uBAAuB,EAAE,CAAC;YACpD,MAAM,gBAAgB,GAAG,IAAI,oBAAoB,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC;YAE1E,MAAM,WAAW,GAAgB;gBAC/B,GAAG,EAAE,0BAA0B;gBAC/B,IAAI,EAAE,OAAO;gBACb,aAAa,EAAE,MAAM;gBACrB,KAAK,EAAE,MAAM;gBACb,OAAO,EAAE,4BAA4B;aACtC,CAAC;YAEF,MAAM,MAAM,CAAC,gBAAgB,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,gCAAgC,CAAC,CAAC;QACxG,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;YAC3C,MAAM,WAAW,GAAgB;gBAC/B,GAAG,EAAE,2BAA2B;gBAChC,IAAI,EAAE,QAAQ;gBACd,aAAa,EAAE,MAAM;gBACrB,KAAK,EAAE,OAAO;gBACd,OAAO,EAAE,gCAAgC;aAC1C,CAAC;YAEF,MAAM,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YAC1B,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;YACpD,MAAM,KAAK,GAAG,IAAI,IAAI,EAAE,CAAC;YAEzB,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC,sBAAsB,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;YACvF,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC,mBAAmB,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACrF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACrD,MAAM,mBAAmB,GAAG,IAAI,oBAAoB,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;YAE1E,MAAM,WAAW,GAAgB;gBAC/B,GAAG,EAAE,4BAA4B;gBACjC,IAAI,EAAE,SAAS;gBACf,aAAa,EAAE,MAAM;gBACrB,KAAK,EAAE,QAAQ;gBACf,OAAO,EAAE;;;;gGAI+E;aACzF,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;YAE9D,qDAAqD;YACrD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yBAAyB,EAAE,KAAK,IAAI,EAAE;YACvC,MAAM,WAAW,GAAgB;gBAC/B,GAAG,EAAE,wCAAwC;gBAC7C,IAAI,EAAE,qBAAqB;gBAC3B,aAAa,EAAE,UAAU;gBACzB,KAAK,EAAE,eAAe;gBACtB,OAAO,EAAE;;;;;;;;;;;;;CAahB;aACM,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;YAEpD,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC7B,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,WAAW,GAAgB;gBAC/B,GAAG,EAAE,6BAA6B;gBAClC,IAAI,EAAE,UAAU;gBAChB,aAAa,EAAE,MAAM;gBACrB,KAAK,EAAE,mBAAmB;gBAC1B,OAAO,EAAE;;;;;iCAKgB;gBACzB,aAAa,EAAE,kBAAkB;aAClC,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;YAEpD,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC7B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,0BAA0B,EAAE,GAAG,EAAE;QACxC,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC9C,MAAM,WAAW,GAAgB;gBAC/B,GAAG,EAAE,6BAA6B;gBAClC,IAAI,EAAE,UAAU;gBAChB,aAAa,EAAE,UAAU;gBACzB,KAAK,EAAE,eAAe;gBACtB,OAAO,EAAE;;;;;;;;;;;;;;OAcV;aACA,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;YAEpD,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC7B,oCAAoC;YACpC,oDAAoD;YACpD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;YAClD,MAAM,WAAW,GAAgB;gBAC/B,GAAG,EAAE,8BAA8B;gBACnC,IAAI,EAAE,WAAW;gBACjB,aAAa,EAAE,UAAU;gBACzB,KAAK,EAAE,UAAU;gBACjB,OAAO,EAAE;;;;;;;;;;;;;;OAcV;aACA,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;YAEpD,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC7B,oCAAoC;YACpC,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC;YACpF,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
package/build/storage/storage.js
CHANGED
|
@@ -38,6 +38,26 @@ function preprocessQuery(query) {
|
|
|
38
38
|
logger.debug('[QueryPreprocess] Processed query:', result);
|
|
39
39
|
return result;
|
|
40
40
|
}
|
|
41
|
+
function buildExactUrlCondition(urls) {
|
|
42
|
+
return `url IN (${urls.map((url) => `'${escapeFilterValue(url)}'`).join(', ')})`;
|
|
43
|
+
}
|
|
44
|
+
function buildSearchWhereClause(visibilityFilter, options, tagFilteredUrls) {
|
|
45
|
+
const conditions = [`(${visibilityFilter})`];
|
|
46
|
+
if (options.filterByType) {
|
|
47
|
+
conditions.push(`type = '${escapeFilterValue(options.filterByType)}'`);
|
|
48
|
+
}
|
|
49
|
+
if (options.filterUrl) {
|
|
50
|
+
const escapedUrl = escapeFilterValue(options.filterUrl).replace(/\\/g, '\\\\').replace(/%/g, '\\%').replace(/_/g, '\\_');
|
|
51
|
+
conditions.push(`url LIKE '${escapedUrl}%'`);
|
|
52
|
+
}
|
|
53
|
+
if (options.filterUrls) {
|
|
54
|
+
conditions.push(buildExactUrlCondition(options.filterUrls));
|
|
55
|
+
}
|
|
56
|
+
if (tagFilteredUrls) {
|
|
57
|
+
conditions.push(buildExactUrlCondition(tagFilteredUrls));
|
|
58
|
+
}
|
|
59
|
+
return conditions.join(' AND ');
|
|
60
|
+
}
|
|
41
61
|
export class DocumentStore {
|
|
42
62
|
dbPath;
|
|
43
63
|
vectorDbPath;
|
|
@@ -778,15 +798,19 @@ export class DocumentStore {
|
|
|
778
798
|
if (!this.lanceTable) {
|
|
779
799
|
throw new Error('Storage not initialized');
|
|
780
800
|
}
|
|
781
|
-
const { limit = 10, includeVectors = false, filterByType, filterByTags, textQuery } = options;
|
|
801
|
+
const { limit = 10, includeVectors = false, filterByType, filterUrls, filterByTags, textQuery } = options;
|
|
782
802
|
logger.debug(`[DocumentStore] Searching documents with vector:`, {
|
|
783
803
|
dimensions: queryVector.length,
|
|
784
804
|
limit,
|
|
785
805
|
includeVectors,
|
|
786
806
|
filterByType,
|
|
807
|
+
filterUrls,
|
|
787
808
|
filterByTags,
|
|
788
809
|
hasTextQuery: !!textQuery,
|
|
789
810
|
});
|
|
811
|
+
if (filterUrls?.length === 0) {
|
|
812
|
+
return [];
|
|
813
|
+
}
|
|
790
814
|
// Add validation for query vector
|
|
791
815
|
if (queryVector.length === 0 && !textQuery) {
|
|
792
816
|
logger.debug('[DocumentStore] Empty query vector and no text query provided');
|
|
@@ -835,17 +859,9 @@ export class DocumentStore {
|
|
|
835
859
|
}
|
|
836
860
|
logger.debug(`[DocumentStore] Tag filter matched ${tagFilteredUrls.length} documents for vector search`);
|
|
837
861
|
}
|
|
838
|
-
// Build WHERE conditions
|
|
839
862
|
const visibilityFilter = await this.getJournalVisibilityFilter();
|
|
840
|
-
const
|
|
841
|
-
|
|
842
|
-
conditions.push(`type = '${escapeFilterValue(filterByType)}'`);
|
|
843
|
-
}
|
|
844
|
-
if (tagFilteredUrls && tagFilteredUrls.length > 0) {
|
|
845
|
-
const urlConditions = tagFilteredUrls.map((u) => `url = '${escapeFilterValue(u)}'`).join(' OR ');
|
|
846
|
-
conditions.push(`(${urlConditions})`);
|
|
847
|
-
}
|
|
848
|
-
const query = this.lanceTable.search(queryVector).where(conditions.join(' AND ')).limit(limit);
|
|
863
|
+
const whereClause = buildSearchWhereClause(visibilityFilter, options, tagFilteredUrls);
|
|
864
|
+
const query = this.lanceTable.search(queryVector).where(whereClause).limit(limit);
|
|
849
865
|
const results = await query.toArray();
|
|
850
866
|
logger.debug(`[DocumentStore] Found ${results.length} results`);
|
|
851
867
|
// Log the first result for debugging if available
|
|
@@ -919,7 +935,7 @@ export class DocumentStore {
|
|
|
919
935
|
try {
|
|
920
936
|
logger.debug('[DocumentStore] Creating FTS index on content field...');
|
|
921
937
|
await this.lanceTable.createIndex('content', {
|
|
922
|
-
config: lancedb.Index.fts(),
|
|
938
|
+
config: lancedb.Index.fts({ withPosition: true }),
|
|
923
939
|
replace: true, // Replace existing index to prevent accumulation
|
|
924
940
|
});
|
|
925
941
|
this.ftsIndexCreated = true;
|
|
@@ -953,8 +969,10 @@ export class DocumentStore {
|
|
|
953
969
|
logger.debug(`[DocumentStore] Returning cached results`);
|
|
954
970
|
return cached;
|
|
955
971
|
}
|
|
956
|
-
const { limit = 10,
|
|
957
|
-
|
|
972
|
+
const { limit = 10, filterUrls, filterByTags } = options;
|
|
973
|
+
if (filterUrls?.length === 0) {
|
|
974
|
+
return [];
|
|
975
|
+
}
|
|
958
976
|
// If filtering by tags, get the list of URLs that have all those tags
|
|
959
977
|
let tagFilteredUrls;
|
|
960
978
|
if (filterByTags && filterByTags.length > 0) {
|
|
@@ -967,26 +985,8 @@ export class DocumentStore {
|
|
|
967
985
|
}
|
|
968
986
|
logger.debug(`[DocumentStore] Tag filter matched ${tagFilteredUrls.length} documents`);
|
|
969
987
|
}
|
|
970
|
-
|
|
971
|
-
const
|
|
972
|
-
const conditions = [`(${visibilityFilter})`];
|
|
973
|
-
if (filterByType) {
|
|
974
|
-
conditions.push(`type = '${escapeFilterValue(filterByType)}'`);
|
|
975
|
-
}
|
|
976
|
-
if (filterUrl) {
|
|
977
|
-
// Filter by base URL - use LIKE to match URLs that start with the base URL
|
|
978
|
-
// Escape the filterUrl and also escape LIKE wildcards within the value
|
|
979
|
-
const escapedUrl = escapeFilterValue(filterUrl).replace(/\\/g, '\\\\').replace(/%/g, '\\%').replace(/_/g, '\\_');
|
|
980
|
-
conditions.push(`url LIKE '${escapedUrl}%'`);
|
|
981
|
-
}
|
|
982
|
-
if (tagFilteredUrls && tagFilteredUrls.length > 0) {
|
|
983
|
-
// Filter to only include URLs that match the tag filter
|
|
984
|
-
const urlConditions = tagFilteredUrls.map((u) => `url = '${escapeFilterValue(u)}'`).join(' OR ');
|
|
985
|
-
conditions.push(`(${urlConditions})`);
|
|
986
|
-
}
|
|
987
|
-
return conditions.join(' AND ');
|
|
988
|
-
};
|
|
989
|
-
const whereClause = buildWhereClause();
|
|
988
|
+
const visibilityFilter = await this.getJournalVisibilityFilter();
|
|
989
|
+
const whereClause = buildSearchWhereClause(visibilityFilter, options, tagFilteredUrls);
|
|
990
990
|
try {
|
|
991
991
|
if (!this.lanceTable) {
|
|
992
992
|
throw new Error('Storage not initialized');
|