@cosmocoder/mcp-web-docs 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (240) hide show
  1. package/LICENSE +22 -0
  2. package/README.md +368 -0
  3. package/build/__mocks__/embeddings.d.ts +17 -0
  4. package/build/__mocks__/embeddings.js +66 -0
  5. package/build/__mocks__/embeddings.js.map +1 -0
  6. package/build/config.d.ts +44 -0
  7. package/build/config.js +158 -0
  8. package/build/config.js.map +1 -0
  9. package/build/config.test.d.ts +1 -0
  10. package/build/config.test.js +165 -0
  11. package/build/config.test.js.map +1 -0
  12. package/build/crawler/auth.d.ts +128 -0
  13. package/build/crawler/auth.js +546 -0
  14. package/build/crawler/auth.js.map +1 -0
  15. package/build/crawler/auth.test.d.ts +1 -0
  16. package/build/crawler/auth.test.js +174 -0
  17. package/build/crawler/auth.test.js.map +1 -0
  18. package/build/crawler/base.d.ts +24 -0
  19. package/build/crawler/base.js +149 -0
  20. package/build/crawler/base.js.map +1 -0
  21. package/build/crawler/base.test.d.ts +1 -0
  22. package/build/crawler/base.test.js +234 -0
  23. package/build/crawler/base.test.js.map +1 -0
  24. package/build/crawler/browser-config.d.ts +2 -0
  25. package/build/crawler/browser-config.js +29 -0
  26. package/build/crawler/browser-config.js.map +1 -0
  27. package/build/crawler/browser-config.test.d.ts +1 -0
  28. package/build/crawler/browser-config.test.js +56 -0
  29. package/build/crawler/browser-config.test.js.map +1 -0
  30. package/build/crawler/cheerio.d.ts +11 -0
  31. package/build/crawler/cheerio.js +134 -0
  32. package/build/crawler/cheerio.js.map +1 -0
  33. package/build/crawler/chromium.d.ts +21 -0
  34. package/build/crawler/chromium.js +596 -0
  35. package/build/crawler/chromium.js.map +1 -0
  36. package/build/crawler/content-extractor-types.d.ts +25 -0
  37. package/build/crawler/content-extractor-types.js +2 -0
  38. package/build/crawler/content-extractor-types.js.map +1 -0
  39. package/build/crawler/content-extractors.d.ts +9 -0
  40. package/build/crawler/content-extractors.js +9 -0
  41. package/build/crawler/content-extractors.js.map +1 -0
  42. package/build/crawler/content-utils.d.ts +2 -0
  43. package/build/crawler/content-utils.js +22 -0
  44. package/build/crawler/content-utils.js.map +1 -0
  45. package/build/crawler/content-utils.test.d.ts +1 -0
  46. package/build/crawler/content-utils.test.js +99 -0
  47. package/build/crawler/content-utils.test.js.map +1 -0
  48. package/build/crawler/crawlee-crawler.d.ts +63 -0
  49. package/build/crawler/crawlee-crawler.js +342 -0
  50. package/build/crawler/crawlee-crawler.js.map +1 -0
  51. package/build/crawler/crawlee-crawler.test.d.ts +1 -0
  52. package/build/crawler/crawlee-crawler.test.js +280 -0
  53. package/build/crawler/crawlee-crawler.test.js.map +1 -0
  54. package/build/crawler/default-extractor.d.ts +4 -0
  55. package/build/crawler/default-extractor.js +26 -0
  56. package/build/crawler/default-extractor.js.map +1 -0
  57. package/build/crawler/default-extractor.test.d.ts +1 -0
  58. package/build/crawler/default-extractor.test.js +200 -0
  59. package/build/crawler/default-extractor.test.js.map +1 -0
  60. package/build/crawler/default.d.ts +11 -0
  61. package/build/crawler/default.js +138 -0
  62. package/build/crawler/default.js.map +1 -0
  63. package/build/crawler/docs-crawler.d.ts +26 -0
  64. package/build/crawler/docs-crawler.js +97 -0
  65. package/build/crawler/docs-crawler.js.map +1 -0
  66. package/build/crawler/docs-crawler.test.d.ts +1 -0
  67. package/build/crawler/docs-crawler.test.js +185 -0
  68. package/build/crawler/docs-crawler.test.js.map +1 -0
  69. package/build/crawler/factory.d.ts +6 -0
  70. package/build/crawler/factory.js +83 -0
  71. package/build/crawler/factory.js.map +1 -0
  72. package/build/crawler/github-pages-extractor.d.ts +4 -0
  73. package/build/crawler/github-pages-extractor.js +33 -0
  74. package/build/crawler/github-pages-extractor.js.map +1 -0
  75. package/build/crawler/github-pages-extractor.test.d.ts +1 -0
  76. package/build/crawler/github-pages-extractor.test.js +184 -0
  77. package/build/crawler/github-pages-extractor.test.js.map +1 -0
  78. package/build/crawler/github.d.ts +20 -0
  79. package/build/crawler/github.js +181 -0
  80. package/build/crawler/github.js.map +1 -0
  81. package/build/crawler/github.test.d.ts +1 -0
  82. package/build/crawler/github.test.js +326 -0
  83. package/build/crawler/github.test.js.map +1 -0
  84. package/build/crawler/puppeteer.d.ts +16 -0
  85. package/build/crawler/puppeteer.js +191 -0
  86. package/build/crawler/puppeteer.js.map +1 -0
  87. package/build/crawler/queue-manager.d.ts +43 -0
  88. package/build/crawler/queue-manager.js +169 -0
  89. package/build/crawler/queue-manager.js.map +1 -0
  90. package/build/crawler/queue-manager.test.d.ts +1 -0
  91. package/build/crawler/queue-manager.test.js +509 -0
  92. package/build/crawler/queue-manager.test.js.map +1 -0
  93. package/build/crawler/site-rules.d.ts +11 -0
  94. package/build/crawler/site-rules.js +104 -0
  95. package/build/crawler/site-rules.js.map +1 -0
  96. package/build/crawler/site-rules.test.d.ts +1 -0
  97. package/build/crawler/site-rules.test.js +139 -0
  98. package/build/crawler/site-rules.test.js.map +1 -0
  99. package/build/crawler/storybook-extractor.d.ts +34 -0
  100. package/build/crawler/storybook-extractor.js +767 -0
  101. package/build/crawler/storybook-extractor.js.map +1 -0
  102. package/build/crawler/storybook-extractor.test.d.ts +1 -0
  103. package/build/crawler/storybook-extractor.test.js +491 -0
  104. package/build/crawler/storybook-extractor.test.js.map +1 -0
  105. package/build/embeddings/fastembed.d.ts +25 -0
  106. package/build/embeddings/fastembed.js +188 -0
  107. package/build/embeddings/fastembed.js.map +1 -0
  108. package/build/embeddings/fastembed.test.d.ts +1 -0
  109. package/build/embeddings/fastembed.test.js +307 -0
  110. package/build/embeddings/fastembed.test.js.map +1 -0
  111. package/build/embeddings/openai.d.ts +8 -0
  112. package/build/embeddings/openai.js +56 -0
  113. package/build/embeddings/openai.js.map +1 -0
  114. package/build/embeddings/types.d.ts +4 -0
  115. package/build/embeddings/types.js +2 -0
  116. package/build/embeddings/types.js.map +1 -0
  117. package/build/index.d.ts +2 -0
  118. package/build/index.js +1007 -0
  119. package/build/index.js.map +1 -0
  120. package/build/index.test.d.ts +1 -0
  121. package/build/index.test.js +364 -0
  122. package/build/index.test.js.map +1 -0
  123. package/build/indexing/queue-manager.d.ts +36 -0
  124. package/build/indexing/queue-manager.js +86 -0
  125. package/build/indexing/queue-manager.js.map +1 -0
  126. package/build/indexing/queue-manager.test.d.ts +1 -0
  127. package/build/indexing/queue-manager.test.js +257 -0
  128. package/build/indexing/queue-manager.test.js.map +1 -0
  129. package/build/indexing/status.d.ts +39 -0
  130. package/build/indexing/status.js +207 -0
  131. package/build/indexing/status.js.map +1 -0
  132. package/build/indexing/status.test.d.ts +1 -0
  133. package/build/indexing/status.test.js +246 -0
  134. package/build/indexing/status.test.js.map +1 -0
  135. package/build/processor/content.d.ts +16 -0
  136. package/build/processor/content.js +286 -0
  137. package/build/processor/content.js.map +1 -0
  138. package/build/processor/content.test.d.ts +1 -0
  139. package/build/processor/content.test.js +369 -0
  140. package/build/processor/content.test.js.map +1 -0
  141. package/build/processor/markdown.d.ts +11 -0
  142. package/build/processor/markdown.js +256 -0
  143. package/build/processor/markdown.js.map +1 -0
  144. package/build/processor/markdown.test.d.ts +1 -0
  145. package/build/processor/markdown.test.js +312 -0
  146. package/build/processor/markdown.test.js.map +1 -0
  147. package/build/processor/metadata-parser.d.ts +37 -0
  148. package/build/processor/metadata-parser.js +245 -0
  149. package/build/processor/metadata-parser.js.map +1 -0
  150. package/build/processor/metadata-parser.test.d.ts +1 -0
  151. package/build/processor/metadata-parser.test.js +357 -0
  152. package/build/processor/metadata-parser.test.js.map +1 -0
  153. package/build/processor/processor.d.ts +8 -0
  154. package/build/processor/processor.js +190 -0
  155. package/build/processor/processor.js.map +1 -0
  156. package/build/processor/processor.test.d.ts +1 -0
  157. package/build/processor/processor.test.js +357 -0
  158. package/build/processor/processor.test.js.map +1 -0
  159. package/build/rag/cache.d.ts +10 -0
  160. package/build/rag/cache.js +10 -0
  161. package/build/rag/cache.js.map +1 -0
  162. package/build/rag/code-generator.d.ts +11 -0
  163. package/build/rag/code-generator.js +30 -0
  164. package/build/rag/code-generator.js.map +1 -0
  165. package/build/rag/context-assembler.d.ts +23 -0
  166. package/build/rag/context-assembler.js +113 -0
  167. package/build/rag/context-assembler.js.map +1 -0
  168. package/build/rag/docs-search.d.ts +55 -0
  169. package/build/rag/docs-search.js +380 -0
  170. package/build/rag/docs-search.js.map +1 -0
  171. package/build/rag/pipeline.d.ts +26 -0
  172. package/build/rag/pipeline.js +91 -0
  173. package/build/rag/pipeline.js.map +1 -0
  174. package/build/rag/query-processor.d.ts +14 -0
  175. package/build/rag/query-processor.js +57 -0
  176. package/build/rag/query-processor.js.map +1 -0
  177. package/build/rag/reranker.d.ts +55 -0
  178. package/build/rag/reranker.js +210 -0
  179. package/build/rag/reranker.js.map +1 -0
  180. package/build/rag/response-generator.d.ts +20 -0
  181. package/build/rag/response-generator.js +101 -0
  182. package/build/rag/response-generator.js.map +1 -0
  183. package/build/rag/retriever.d.ts +19 -0
  184. package/build/rag/retriever.js +111 -0
  185. package/build/rag/retriever.js.map +1 -0
  186. package/build/rag/validator.d.ts +22 -0
  187. package/build/rag/validator.js +128 -0
  188. package/build/rag/validator.js.map +1 -0
  189. package/build/rag/version-manager.d.ts +23 -0
  190. package/build/rag/version-manager.js +98 -0
  191. package/build/rag/version-manager.js.map +1 -0
  192. package/build/setupTests.d.ts +4 -0
  193. package/build/setupTests.js +50 -0
  194. package/build/setupTests.js.map +1 -0
  195. package/build/storage/storage.d.ts +38 -0
  196. package/build/storage/storage.js +700 -0
  197. package/build/storage/storage.js.map +1 -0
  198. package/build/storage/storage.test.d.ts +1 -0
  199. package/build/storage/storage.test.js +338 -0
  200. package/build/storage/storage.test.js.map +1 -0
  201. package/build/types/rag.d.ts +27 -0
  202. package/build/types/rag.js +2 -0
  203. package/build/types/rag.js.map +1 -0
  204. package/build/types.d.ts +120 -0
  205. package/build/types.js +2 -0
  206. package/build/types.js.map +1 -0
  207. package/build/util/content-utils.d.ts +31 -0
  208. package/build/util/content-utils.js +120 -0
  209. package/build/util/content-utils.js.map +1 -0
  210. package/build/util/content.d.ts +1 -0
  211. package/build/util/content.js +16 -0
  212. package/build/util/content.js.map +1 -0
  213. package/build/util/docs.d.ts +1 -0
  214. package/build/util/docs.js +26 -0
  215. package/build/util/docs.js.map +1 -0
  216. package/build/util/docs.test.d.ts +1 -0
  217. package/build/util/docs.test.js +49 -0
  218. package/build/util/docs.test.js.map +1 -0
  219. package/build/util/favicon.d.ts +6 -0
  220. package/build/util/favicon.js +88 -0
  221. package/build/util/favicon.js.map +1 -0
  222. package/build/util/favicon.test.d.ts +1 -0
  223. package/build/util/favicon.test.js +140 -0
  224. package/build/util/favicon.test.js.map +1 -0
  225. package/build/util/logger.d.ts +17 -0
  226. package/build/util/logger.js +72 -0
  227. package/build/util/logger.js.map +1 -0
  228. package/build/util/logger.test.d.ts +1 -0
  229. package/build/util/logger.test.js +46 -0
  230. package/build/util/logger.test.js.map +1 -0
  231. package/build/util/security.d.ts +312 -0
  232. package/build/util/security.js +719 -0
  233. package/build/util/security.js.map +1 -0
  234. package/build/util/security.test.d.ts +1 -0
  235. package/build/util/security.test.js +524 -0
  236. package/build/util/security.test.js.map +1 -0
  237. package/build/util/site-detector.d.ts +22 -0
  238. package/build/util/site-detector.js +42 -0
  239. package/build/util/site-detector.js.map +1 -0
  240. package/package.json +112 -0
package/build/crawler/github-pages-extractor.test.js ADDED
@@ -0,0 +1,184 @@
1
+ import { JSDOM } from 'jsdom';
2
+ import { GitHubPagesExtractor } from './github-pages-extractor.js';
3
+ describe('GitHubPagesExtractor', () => {
4
+ let extractor;
5
+ beforeEach(() => {
6
+ extractor = new GitHubPagesExtractor();
7
+ });
8
+ const createDocument = (html) => {
9
+ const dom = new JSDOM(html);
10
+ return dom.window.document;
11
+ };
12
+ describe('extractContent', () => {
13
+ it('should extract content from main element', async () => {
14
+ const doc = createDocument(`
15
+ <html>
16
+ <body>
17
+ <nav>Navigation</nav>
18
+ <header>Header</header>
19
+ <main>
20
+ <h1>Main Title</h1>
21
+ <p>First paragraph description</p>
22
+ <p>More content here</p>
23
+ </main>
24
+ <footer>Footer</footer>
25
+ </body>
26
+ </html>
27
+ `);
28
+ const result = await extractor.extractContent(doc);
29
+ expect(result.content).toContain('Main Title');
30
+ expect(result.content).toContain('First paragraph description');
31
+ expect(result.content).not.toContain('Navigation');
32
+ expect(result.content).not.toContain('Footer');
33
+ expect(result.metadata.type).toBe('overview');
34
+ expect(result.metadata.pattern?.name).toBe('Main Title');
35
+ expect(result.metadata.pattern?.description).toBe('First paragraph description');
36
+ });
37
+ it('should extract content from article element', async () => {
38
+ const doc = createDocument(`
39
+ <html>
40
+ <body>
41
+ <nav>Navigation</nav>
42
+ <article>
43
+ <h1>Article Title</h1>
44
+ <p>Article description</p>
45
+ </article>
46
+ </body>
47
+ </html>
48
+ `);
49
+ const result = await extractor.extractContent(doc);
50
+ expect(result.content).toContain('Article Title');
51
+ expect(result.content).toContain('Article description');
52
+ expect(result.metadata.pattern?.name).toBe('Article Title');
53
+ });
54
+ it('should extract content from markdown-body class', async () => {
55
+ const doc = createDocument(`
56
+ <html>
57
+ <body>
58
+ <div class="markdown-body">
59
+ <h1>Markdown Content</h1>
60
+ <p>Markdown description</p>
61
+ </div>
62
+ </body>
63
+ </html>
64
+ `);
65
+ const result = await extractor.extractContent(doc);
66
+ expect(result.content).toContain('Markdown Content');
67
+ expect(result.metadata.pattern?.name).toBe('Markdown Content');
68
+ });
69
+ it('should return empty content when no main element found', async () => {
70
+ const doc = createDocument(`
71
+ <html>
72
+ <body>
73
+ <nav>Navigation only</nav>
74
+ <div>Some random content</div>
75
+ </body>
76
+ </html>
77
+ `);
78
+ const result = await extractor.extractContent(doc);
79
+ expect(result.content).toBe('');
80
+ expect(result.metadata.type).toBe('overview');
81
+ });
82
+ it('should remove script and style elements', async () => {
83
+ const doc = createDocument(`
84
+ <html>
85
+ <body>
86
+ <main>
87
+ <h1>Title</h1>
88
+ <script>console.log('malicious');</script>
89
+ <style>.hidden { display: none; }</style>
90
+ <p>Safe content</p>
91
+ </main>
92
+ </body>
93
+ </html>
94
+ `);
95
+ const result = await extractor.extractContent(doc);
96
+ expect(result.content).toContain('Title');
97
+ expect(result.content).toContain('Safe content');
98
+ expect(result.content).not.toContain('malicious');
99
+ expect(result.content).not.toContain('display: none');
100
+ });
101
+ it('should handle missing title gracefully', async () => {
102
+ const doc = createDocument(`
103
+ <html>
104
+ <body>
105
+ <main>
106
+ <p>Just a paragraph without title</p>
107
+ </main>
108
+ </body>
109
+ </html>
110
+ `);
111
+ const result = await extractor.extractContent(doc);
112
+ expect(result.content).toContain('Just a paragraph');
113
+ expect(result.metadata.pattern?.name).toBe('');
114
+ });
115
+ it('should handle missing description gracefully', async () => {
116
+ const doc = createDocument(`
117
+ <html>
118
+ <body>
119
+ <main>
120
+ <h1>Title Only</h1>
121
+ </main>
122
+ </body>
123
+ </html>
124
+ `);
125
+ const result = await extractor.extractContent(doc);
126
+ expect(result.content).toContain('Title Only');
127
+ expect(result.metadata.pattern?.description).toBe('');
128
+ });
129
+ it('should remove nav, header, and footer before extraction', async () => {
130
+ const doc = createDocument(`
131
+ <html>
132
+ <body>
133
+ <nav>Site Navigation</nav>
134
+ <header>Site Header</header>
135
+ <main>
136
+ <h1>Main Content</h1>
137
+ <p>This is the main content</p>
138
+ </main>
139
+ <footer>Site Footer</footer>
140
+ </body>
141
+ </html>
142
+ `);
143
+ // Check that nav/header/footer are removed from the document
144
+ const result = await extractor.extractContent(doc);
145
+ // The content should not include nav/header/footer text
146
+ expect(result.content).not.toContain('Site Navigation');
147
+ expect(result.content).not.toContain('Site Header');
148
+ expect(result.content).not.toContain('Site Footer');
149
+ expect(result.content).toContain('Main Content');
150
+ });
151
+ it('should set component type in pattern metadata', async () => {
152
+ const doc = createDocument(`
153
+ <html>
154
+ <body>
155
+ <main>
156
+ <h1>Button Component</h1>
157
+ <p>A clickable button element</p>
158
+ </main>
159
+ </body>
160
+ </html>
161
+ `);
162
+ const result = await extractor.extractContent(doc);
163
+ expect(result.metadata.pattern?.type).toBe('component');
164
+ expect(result.metadata.pattern?.usageContexts).toEqual([]);
165
+ expect(result.metadata.pattern?.relatedPatterns).toEqual([]);
166
+ });
167
+ it('should get first paragraph after h1 as description', async () => {
168
+ const doc = createDocument(`
169
+ <html>
170
+ <body>
171
+ <main>
172
+ <h1>Component Name</h1>
173
+ <p>This is the description after h1</p>
174
+ <p>This is not the description</p>
175
+ </main>
176
+ </body>
177
+ </html>
178
+ `);
179
+ const result = await extractor.extractContent(doc);
180
+ expect(result.metadata.pattern?.description).toBe('This is the description after h1');
181
+ });
182
+ });
183
+ });
184
+ //# sourceMappingURL=github-pages-extractor.test.js.map
package/build/crawler/github-pages-extractor.test.js.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"github-pages-extractor.test.js","sourceRoot":"","sources":["../../src/crawler/github-pages-extractor.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAC9B,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AAEnE,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;IACpC,IAAI,SAA+B,CAAC;IAEpC,UAAU,CAAC,GAAG,EAAE;QACd,SAAS,GAAG,IAAI,oBAAoB,EAAE,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,MAAM,cAAc,GAAG,CAAC,IAAY,EAAY,EAAE;QAChD,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC;QAC5B,OAAO,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC;IAC7B,CAAC,CAAC;IAEF,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,MAAM,GAAG,GAAG,cAAc,CAAC;;;;;;;;;;;;;OAa1B,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YAEnD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;YAC/C,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,6BAA6B,CAAC,CAAC;YAChE,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;YACnD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;YAC/C,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC9C,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YACzD,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;QACnF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;YAC3D,MAAM,GAAG,GAAG,cAAc,CAAC;;;;;;;;;;OAU1B,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YAEnD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;YAClD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,qBAAqB,CAAC,CAAC;YACxD,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAC9D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;YAC/D,MAAM,GAAG,GAAG,cAAc,CAAC;;;;;;;;;OAS1B,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YAEnD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;YACrD,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QACjE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wDAAwD,EAAE,KAAK,IAAI,EAAE;YACtE,MAAM,GAAG,GAAG,cAAc,CAAC;;;;;;;OAO1B,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YAEnD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAChC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,GAAG,GAAG,cAAc,CAAC;;;;;;;;;;;OAW1B,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YAEnD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;YAC1C,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;YACjD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;YAClD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;QACxD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;YACtD,MAAM,GAAG,GAAG,cAAc,CAAC;;;;;;;;OAQ1B,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YAEnD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;YACrD,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;YAC5D,MAAM,GAAG,GAAG,cAAc,CAAC;;;;;;;;OAQ1B,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YAEnD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;YAC/C,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACxD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yDAAyD,EAAE,KAAK,IAAI,EAAE;YACvE,MAAM,GAAG,GAAG,cAAc,CAAC;;;;;;;;;;;;OAY1B,CAAC,CAAC;YAEH,6DAA6D;YAC7D,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YAEnD,wDAAwD;YACxD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;YACxD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;YACpD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;YACpD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;YAC7D,MAAM,GAAG,GAAG,cAAc,CAAC;;;;;;;;;OAS1B,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YAEnD,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACxD,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAC3D,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC/D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;YAClE,MAAM,GAAG,GAAG,cAAc,CAAC;;;;;;;;;;OAU1B,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YAEnD,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;QACxF,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
package/build/crawler/github.d.ts ADDED
@@ -0,0 +1,20 @@
1
+ import { CrawlResult } from '../types.js';
2
+ import { BaseCrawler } from './base.js';
3
+ export declare class GitHubCrawler extends BaseCrawler {
4
+ private readonly githubToken?;
5
+ private readonly API_BASE;
6
+ private readonly RAW_CONTENT_BASE;
7
+ private readonly MARKDOWN_EXTENSIONS;
8
+ private readonly DOCUMENTATION_PATHS;
9
+ constructor(maxDepth?: number, maxRequestsPerCrawl?: number, githubToken?: string | undefined, onProgress?: (progress: number, description: string) => void);
10
+ crawl(url: string): AsyncGenerator<CrawlResult, void, unknown>;
11
+ private parseGitHubUrl;
12
+ private findDocumentationDirs;
13
+ private processDirectory;
14
+ private fetchRepoContents;
15
+ private fetchFileContent;
16
+ private isMarkdownFile;
17
+ private shouldProcessDirectory;
18
+ private constructGitHubUrl;
19
+ private extractTitleFromPath;
20
+ }
package/build/crawler/github.js ADDED
@@ -0,0 +1,181 @@
1
+ import { URL } from 'url';
2
+ import { BaseCrawler } from './base.js';
3
+ import { logger } from '../util/logger.js';
4
+ import { GitHubFilesArraySchema } from '../util/security.js';
5
+ export class GitHubCrawler extends BaseCrawler {
6
+ githubToken;
7
+ API_BASE = 'https://api.github.com';
8
+ RAW_CONTENT_BASE = 'https://raw.githubusercontent.com';
9
+ MARKDOWN_EXTENSIONS = ['.md', '.mdx', '.markdown'];
10
+ DOCUMENTATION_PATHS = ['docs', 'doc', 'documentation', 'wiki', 'guide', 'guides', 'tutorial', 'tutorials'];
11
+ constructor(maxDepth = 4, maxRequestsPerCrawl = 1000, githubToken, onProgress) {
12
+ super(maxDepth, maxRequestsPerCrawl, onProgress);
13
+ this.githubToken = githubToken;
14
+ }
15
+ async *crawl(url) {
16
+ logger.debug(`[${this.constructor.name}] Crawling GitHub repository: ${url}`);
17
+ if (this.isAborting) {
18
+ logger.debug('[GitHubCrawler] Crawl aborted');
19
+ return;
20
+ }
21
+ try {
22
+ const repoInfo = this.parseGitHubUrl(url);
23
+ if (!repoInfo) {
24
+ throw new Error(`Invalid GitHub URL: ${url}`);
25
+ }
26
+ // First try to find documentation directory
27
+ const docDirs = await this.findDocumentationDirs(repoInfo);
28
+ if (docDirs.length > 0) {
29
+ // Process documentation directories
30
+ for (const docDir of docDirs) {
31
+ if (this.isAborting)
32
+ break;
33
+ yield* this.processDirectory(repoInfo, docDir);
34
+ }
35
+ }
36
+ else {
37
+ // Fall back to processing markdown files in root
38
+ logger.debug('[GitHubCrawler] No documentation directory found, processing root markdown files');
39
+ yield* this.processDirectory(repoInfo, '');
40
+ }
41
+ }
42
+ catch (error) {
43
+ logger.debug('[GitHubCrawler] Error crawling repository:', error);
44
+ }
45
+ }
46
+ parseGitHubUrl(url) {
47
+ try {
48
+ const urlObj = new URL(url);
49
+ if (urlObj.hostname !== 'github.com')
50
+ return null;
51
+ const [, owner, repo] = urlObj.pathname.split('/');
52
+ if (!owner || !repo)
53
+ return null;
54
+ // Remove .git extension if present
55
+ const cleanRepo = repo.replace(/\.git$/, '');
56
+ return {
57
+ owner,
58
+ repo: cleanRepo,
59
+ branch: 'main', // Default to main, could be determined dynamically
60
+ };
61
+ }
62
+ catch {
63
+ return null;
64
+ }
65
+ }
66
+ async findDocumentationDirs(repoInfo) {
67
+ const dirs = [];
68
+ try {
69
+ const contents = await this.fetchRepoContents(repoInfo, '');
70
+ for (const item of contents) {
71
+ if (item.type === 'dir' && this.DOCUMENTATION_PATHS.some((path) => item.path.toLowerCase() === path.toLowerCase())) {
72
+ dirs.push(item.path);
73
+ }
74
+ }
75
+ }
76
+ catch (error) {
77
+ logger.debug('[GitHubCrawler] Error finding documentation directories:', error);
78
+ }
79
+ return dirs;
80
+ }
81
+ async *processDirectory(repoInfo, path) {
82
+ try {
83
+ const contents = await this.fetchRepoContents(repoInfo, path);
84
+ for (const item of contents) {
85
+ if (this.isAborting)
86
+ break;
87
+ if (item.type === 'file' && this.isMarkdownFile(item.path)) {
88
+ const content = await this.fetchFileContent(repoInfo, item.path);
89
+ if (content) {
90
+ yield {
91
+ url: this.constructGitHubUrl(repoInfo, item.path),
92
+ path: item.path,
93
+ content,
94
+ title: this.extractTitleFromPath(item.path),
95
+ };
96
+ }
97
+ }
98
+ else if (item.type === 'dir' && this.shouldProcessDirectory(item.path)) {
99
+ yield* this.processDirectory(repoInfo, item.path);
100
+ }
101
+ }
102
+ }
103
+ catch (error) {
104
+ logger.debug(`[GitHubCrawler] Error processing directory ${path}:`, error);
105
+ }
106
+ }
107
+ async fetchRepoContents(repoInfo, path) {
108
+ await this.rateLimit();
109
+ const url = `${this.API_BASE}/repos/${repoInfo.owner}/${repoInfo.repo}/contents/${path}`;
110
+ const headers = {
111
+ Accept: 'application/vnd.github.v3+json',
112
+ };
113
+ if (this.githubToken) {
114
+ headers['Authorization'] = `token ${this.githubToken}`;
115
+ }
116
+ try {
117
+ const response = await fetch(url, { headers });
118
+ if (!response.ok) {
119
+ if (response.status === 403) {
120
+ throw new Error('GitHub API rate limit exceeded');
121
+ }
122
+ throw new Error(`GitHub API error: ${response.status}`);
123
+ }
124
+ const rawData = await response.json();
125
+ // Validate the response structure to prevent prototype pollution and ensure expected format
126
+ const validationResult = GitHubFilesArraySchema.safeParse(rawData);
127
+ if (!validationResult.success) {
128
+ logger.warn(`[GitHubCrawler] Invalid GitHub API response structure: ${validationResult.error.message}`);
129
+ return [];
130
+ }
131
+ return validationResult.data;
132
+ }
133
+ catch (error) {
134
+ logger.debug(`[GitHubCrawler] Error fetching repo contents for ${path}:`, error);
135
+ return [];
136
+ }
137
+ }
138
+ async fetchFileContent(repoInfo, path) {
139
+ await this.rateLimit();
140
+ const url = `${this.RAW_CONTENT_BASE}/${repoInfo.owner}/${repoInfo.repo}/${repoInfo.branch}/${path}`;
141
+ const headers = this.githubToken ? { Authorization: `token ${this.githubToken}` } : {};
142
+ try {
143
+ const response = await fetch(url, { headers });
144
+ if (!response.ok) {
145
+ throw new Error(`Failed to fetch file content: ${response.status}`);
146
+ }
147
+ return await response.text();
148
+ }
149
+ catch (error) {
150
+ logger.debug(`[GitHubCrawler] Error fetching content for ${path}:`, error);
151
+ return null;
152
+ }
153
+ }
154
+ isMarkdownFile(path) {
155
+ const lowercasePath = path.toLowerCase();
156
+ return this.MARKDOWN_EXTENSIONS.some((ext) => lowercasePath.endsWith(ext));
157
+ }
158
+ shouldProcessDirectory(path) {
159
+ const lowercasePath = path.toLowerCase();
160
+ // Skip common non-documentation directories
161
+ return (!lowercasePath.includes('node_modules') &&
162
+ !lowercasePath.includes('vendor') &&
163
+ !lowercasePath.includes('test') &&
164
+ !lowercasePath.includes('example') &&
165
+ !lowercasePath.includes('build') &&
166
+ !lowercasePath.includes('dist'));
167
+ }
168
+ constructGitHubUrl(repoInfo, path) {
169
+ return `https://github.com/${repoInfo.owner}/${repoInfo.repo}/blob/${repoInfo.branch}/${path}`;
170
+ }
171
+ extractTitleFromPath(path) {
172
+ // Remove extension and convert to title case
173
+ const basename = path.split('/').pop() || '';
174
+ const nameWithoutExt = basename.replace(/\.[^/.]+$/, '');
175
+ return nameWithoutExt
176
+ .split(/[-_]/)
177
+ .map((word) => word.charAt(0).toUpperCase() + word.slice(1))
178
+ .join(' ');
179
+ }
180
+ }
181
+ //# sourceMappingURL=github.js.map
package/build/crawler/github.js.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"github.js","sourceRoot":"","sources":["../../src/crawler/github.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAE1B,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,sBAAsB,EAA4B,MAAM,qBAAqB,CAAC;AAEvF,MAAM,OAAO,aAAc,SAAQ,WAAW;IASzB;IARF,QAAQ,GAAG,wBAAwB,CAAC;IACpC,gBAAgB,GAAG,mCAAmC,CAAC;IACvD,mBAAmB,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;IACnD,mBAAmB,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,eAAe,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC;IAE5H,YACE,WAAmB,CAAC,EACpB,sBAA8B,IAAI,EACjB,WAAoB,EACrC,UAA4D;QAE5D,KAAK,CAAC,QAAQ,EAAE,mBAAmB,EAAE,UAAU,CAAC,CAAC;QAHhC,gBAAW,GAAX,WAAW,CAAS;IAIvC,CAAC;IAED,KAAK,CAAC,CAAC,KAAK,CAAC,GAAW;QACtB,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,iCAAiC,GAAG,EAAE,CAAC,CAAC;QAE9E,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,MAAM,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;YAC9C,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YAC1C,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,MAAM,IAAI,KAAK,CAAC,uBAAuB,GAAG,EAAE,CAAC,CAAC;YAChD,CAAC;YAED,4CAA4C;YAC5C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAC;YAE3D,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,oCAAoC;gBACpC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;oBAC7B,IAAI,IAAI,CAAC,UAAU;wBAAE,MAAM;oBAC3B,KAAK,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;gBACjD,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,iDAAiD;gBACjD,MAAM,CAAC,KAAK,CAAC,kFAAkF,CAAC,CAAC;gBACjG,KAAK,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAC7C,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,4CAA4C,EAAE,KAAK,CAAC,CAAC;QACpE,CAAC;IACH,CAAC;IAEO,cAAc,CAAC,GAAW;QAChC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,IAAI,MAAM,CAAC,QAAQ,KAAK,YAAY;gBAAE,OAAO,IAAI,CAAC;YAElD,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACnD,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI;gBAAE,OAAO,IAAI,CAAC;YAEjC,mCAAmC;YACnC,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAE7C,OAAO;gBACL,KAAK;gBACL,IAAI,EAAE,SAAS;gBACf,MAAM,EAAE,MAAM,EAAE,mDAAmD;aACpE,CAAC;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,qBAAqB,CAAC,QAAyD;QAC3F,MAAM,IAAI,GAAa,EAAE,CAAC;QAE1B,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAE5D,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;gBAC5B,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,IAAI,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;oBACnH,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACvB,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,0DAA0D,EAAE,KAAK,CAAC,CAAC;QAClF,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,KAAK,CAAC,CAAC,gBAAgB,CAAC,QAAyD,EAAE,IAAY;QACrG,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAE9D,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;gBAC5B,IAAI,IAAI,CAAC,UAAU;oBAAE,MAAM;gBAE3B,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,IAAI,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC3D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;oBACjE,IAAI,OAAO,EAAE,CAAC;wBACZ,MAAM;4BACJ,GAAG,EAAE,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC;4BACjD,IAAI,EAAE,IAAI,CAAC,IAAI;4BACf,OAAO;4BACP,KAAK,EAAE,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC;yBAC5C,CAAC;oBACJ,CAAC;gBACH,CAAC;qBAAM,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;oBACzE,KAAK,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;gBACpD,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,8CAA8C,IAAI,GAAG,EAAE,KAAK,CAAC,CAAC;QAC7E,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,QAAyD,EAAE,IAAY;QACrG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QAEvB,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,QAAQ,UAAU,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,IAAI,aAAa,IAAI,EAAE,CAAC;QACzF,MAAM,OAAO,GAAgB;YAC3B,MAAM,EAAE,gCAAgC;SACzC,CAAC;QAEF,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,OAAO,CAAC,eAAe,CAAC,GAAG,SAAS,IAAI,CAAC,WAAW,EAAE,CAAC;QACzD,CAAC;QAED,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YAE/C,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBAC5B,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;gBACpD,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,qBAAqB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;YAC1D,CAAC;YAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAEtC,4FAA4F;YAC5F,MAAM,gBAAgB,GAAG,sBAAsB,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;YACnE,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAAC;gBAC9B,MAAM,CAAC,IAAI,CAAC,0DAA0D,gBAAgB,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;gBACxG,OAAO,EAAE,CAAC;YACZ,CAAC;YAED,OAAO,gBAAgB,CAAC,IAAI,CAAC;QAC/B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,oDAAoD,IAAI,GAAG,EAAE,KAAK,CAAC,CAAC;YACjF,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,QAAyD,EAAE,IAAY;QACpG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QAEvB,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,gBAAgB,IAAI,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC;QACrG,MAAM,OAAO,GAAgB,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,SAAS,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAEpG,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YAE/C,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,iCAAiC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;YACtE,CAAC;YAED,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC/B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,8CAA8C,IAAI,GAAG,EAAE,KAAK,CAAC,CAAC;YAC3E,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAEO,cAAc,CAAC,IAAY;QACjC,MAAM,aAAa,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QACzC,OAAO,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,aAAa,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7E,CAAC;IAEO,sBAAsB,CAAC,IAAY;QACzC,MAAM,aAAa,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QACzC,4CAA4C;QAC5C,OAAO,CACL,CAAC,aAAa,CAAC,QAAQ,CAAC,cAAc,CAAC;YACvC,CAAC,aAAa,CAAC,QAAQ,CAAC,QAAQ,CAAC;YACjC,CAAC,aAAa,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC/B,CAAC,aAAa,CAAC,QAAQ,CAAC,SAAS,CAAC;YAClC,CAAC,aAAa,CAAC,QAAQ,CAAC,OAAO,CAAC;YAChC,CAAC,aAAa,CAAC,QAAQ,CAAC,MAAM,CAAC,CAChC,CAAC;IACJ,CAAC;IAEO,kBAAkB,CAAC,QAAyD,EAAE,IAAY;QAChG,OAAO,sBAAsB,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,IAAI,SAAS,QAAQ,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC;IACjG,CAAC;IAEO,oBAAoB,CAAC,IAAY;QACvC,6CAA6C;QAC7C,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;QAC7C,MAAM,cAAc,GAAG,QAAQ,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;QACzD,OAAO,cAAc;aAClB,KAAK,CAAC,MAAM,CAAC;aACb,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;aAC3D,IAAI,CAAC,GAAG,CAAC,CAAC;IACf,CAAC;CACF"}
package/build/crawler/github.test.d.ts ADDED
@@ -0,0 +1 @@
1
+ export {};