@cosmocoder/mcp-web-docs 2.0.17 → 2.0.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. package/README.md +2 -0
  2. package/build/crawler/github.d.ts +0 -2
  3. package/build/crawler/github.js +22 -17
  4. package/build/crawler/github.js.map +1 -1
  5. package/build/crawler/github.test.js +91 -84
  6. package/build/crawler/github.test.js.map +1 -1
  7. package/build/crawler/queue-manager.d.ts +1 -1
  8. package/build/crawler/queue-manager.js +8 -8
  9. package/build/crawler/queue-manager.js.map +1 -1
  10. package/build/crawler/queue-manager.test.js +26 -12
  11. package/build/crawler/queue-manager.test.js.map +1 -1
  12. package/build/index.js +50 -47
  13. package/build/index.js.map +1 -1
  14. package/build/index.test.js +56 -57
  15. package/build/index.test.js.map +1 -1
  16. package/build/indexing/status.d.ts +8 -8
  17. package/build/indexing/status.js +40 -38
  18. package/build/indexing/status.js.map +1 -1
  19. package/build/indexing/status.test.js +62 -32
  20. package/build/indexing/status.test.js.map +1 -1
  21. package/build/types.d.ts +3 -0
  22. package/build/util/docs.d.ts +1 -0
  23. package/build/util/docs.js +8 -0
  24. package/build/util/docs.js.map +1 -1
  25. package/build/util/docs.test.js +18 -1
  26. package/build/util/docs.test.js.map +1 -1
  27. package/build/util/security.d.ts +4 -3
  28. package/build/util/security.js +20 -0
  29. package/build/util/security.js.map +1 -1
  30. package/package.json +1 -1
package/README.md CHANGED
@@ -346,6 +346,8 @@ add_documentation({
346
346
  });
347
347
  ```
348
348
 
349
+ For GitHub tree URLs, the first segment after `/tree/` is the branch and any remaining segments are the crawl root. Encode `/` as `%2F` in branch names (for example, `/tree/feature%2Fdocs`).
350
+
349
351
  ### `search_documentation`
350
352
 
351
353
  Search through indexed documentation using hybrid search (FTS + semantic).
package/build/crawler/github.d.ts CHANGED
@@ -3,7 +3,6 @@ import { BaseCrawler } from './base.js';
3
3
  export declare class GitHubCrawler extends BaseCrawler {
4
4
  private readonly githubToken?;
5
5
  private readonly API_BASE;
6
- private readonly RAW_CONTENT_BASE;
7
6
  private readonly MARKDOWN_EXTENSIONS;
8
7
  private readonly DOCUMENTATION_PATHS;
9
8
  constructor(maxDepth?: number, maxRequestsPerCrawl?: number, githubToken?: string | undefined, onProgress?: (progress: number, description: string) => void);
@@ -15,6 +14,5 @@ export declare class GitHubCrawler extends BaseCrawler {
15
14
  private fetchFileContent;
16
15
  private isMarkdownFile;
17
16
  private shouldProcessDirectory;
18
- private constructGitHubUrl;
19
17
  private extractTitleFromPath;
20
18
  }
package/build/crawler/github.js CHANGED
@@ -6,7 +6,6 @@ import { GitHubFilesArraySchema } from '../util/security.js';
6
6
  export class GitHubCrawler extends BaseCrawler {
7
7
  githubToken;
8
8
  API_BASE = 'https://api.github.com';
9
- RAW_CONTENT_BASE = 'https://raw.githubusercontent.com';
10
9
  MARKDOWN_EXTENSIONS = ['.md', '.mdx', '.markdown'];
11
10
  DOCUMENTATION_PATHS = ['docs', 'doc', 'documentation', 'wiki', 'guide', 'guides', 'tutorial', 'tutorials'];
12
11
  constructor(maxDepth = 4, maxRequestsPerCrawl = 1000, githubToken, onProgress) {
@@ -25,7 +24,7 @@ export class GitHubCrawler extends BaseCrawler {
25
24
  throw new Error(`Invalid GitHub URL: ${url}`);
26
25
  }
27
26
  // First try to find documentation directory
28
- const docDirs = await this.findDocumentationDirs(repoInfo);
27
+ const docDirs = repoInfo.startPath ? [repoInfo.startPath] : await this.findDocumentationDirs(repoInfo);
29
28
  if (docDirs.length > 0) {
30
29
  // Process documentation directories
31
30
  for (const docDir of docDirs) {
@@ -51,17 +50,16 @@ export class GitHubCrawler extends BaseCrawler {
51
50
  if (urlObj.hostname !== 'github.com') {
52
51
  return null;
53
52
  }
54
- const [, owner, repo] = urlObj.pathname.split('/');
53
+ const [, owner, repo, view, encodedBranch, ...remainingPath] = urlObj.pathname.split('/');
55
54
  if (!owner || !repo) {
56
55
  return null;
57
56
  }
58
57
  // Remove .git extension if present
59
58
  const cleanRepo = repo.replace(/\.git$/, '');
60
- return {
61
- owner,
62
- repo: cleanRepo,
63
- branch: 'main', // Default to main, could be determined dynamically
64
- };
59
+ // ponytail: literal suffixes are paths; encode "/" as %2F when it belongs to the branch. Probe refs if ambiguity matters.
60
+ const branch = view === 'tree' && encodedBranch ? decodeURIComponent(encodedBranch) : undefined;
61
+ const startPath = branch ? decodeURIComponent(remainingPath.filter(Boolean).join('/')) || undefined : undefined;
62
+ return { owner, repo: cleanRepo, branch, startPath };
65
63
  }
66
64
  catch {
67
65
  return null;
@@ -90,10 +88,14 @@ export class GitHubCrawler extends BaseCrawler {
90
88
  break;
91
89
  }
92
90
  if (item.type === 'file' && this.isMarkdownFile(item.path)) {
93
- const content = await this.fetchFileContent(repoInfo, item.path);
91
+ if (!item.download_url || !item.html_url) {
92
+ logger.debug(`[GitHubCrawler] Skipping ${item.path}: GitHub did not return canonical file URLs`);
93
+ continue;
94
+ }
95
+ const content = await this.fetchFileContent(item.download_url, item.path);
94
96
  if (content) {
95
97
  yield {
96
- url: this.constructGitHubUrl(repoInfo, item.path),
98
+ url: item.html_url,
97
99
  path: item.path,
98
100
  content,
99
101
  contentFormat: 'markdown',
@@ -112,7 +114,14 @@ export class GitHubCrawler extends BaseCrawler {
112
114
  }
113
115
  async fetchRepoContents(repoInfo, path) {
114
116
  await this.rateLimit();
115
- const url = `${this.API_BASE}/repos/${repoInfo.owner}/${repoInfo.repo}/contents/${path}`;
117
+ const encodedPath = path
118
+ .split('/')
119
+ .map((segment) => encodeURIComponent(segment))
120
+ .join('/');
121
+ const url = new URL(`/repos/${encodeURIComponent(repoInfo.owner)}/${encodeURIComponent(repoInfo.repo)}/contents/${encodedPath}`, this.API_BASE);
122
+ if (repoInfo.branch) {
123
+ url.searchParams.set('ref', repoInfo.branch);
124
+ }
116
125
  const headers = {
117
126
  Accept: 'application/vnd.github.v3+json',
118
127
  };
@@ -120,7 +129,7 @@ export class GitHubCrawler extends BaseCrawler {
120
129
  headers['Authorization'] = `token ${this.githubToken}`;
121
130
  }
122
131
  try {
123
- const response = await fetchPublicUrl(url, { headers, signal: this.abortSignal });
132
+ const response = await fetchPublicUrl(url.toString(), { headers, signal: this.abortSignal });
124
133
  if (!response.ok) {
125
134
  if (response.status === 403) {
126
135
  throw new Error('GitHub API rate limit exceeded');
@@ -141,9 +150,8 @@ export class GitHubCrawler extends BaseCrawler {
141
150
  return [];
142
151
  }
143
152
  }
144
- async fetchFileContent(repoInfo, path) {
153
+ async fetchFileContent(url, path) {
145
154
  await this.rateLimit();
146
- const url = `${this.RAW_CONTENT_BASE}/${repoInfo.owner}/${repoInfo.repo}/${repoInfo.branch}/${path}`;
147
155
  const headers = this.githubToken ? { Authorization: `token ${this.githubToken}` } : {};
148
156
  try {
149
157
  const response = await fetchPublicUrl(url, { headers, signal: this.abortSignal });
@@ -171,9 +179,6 @@ export class GitHubCrawler extends BaseCrawler {
171
179
  !lowercasePath.includes('build') &&
172
180
  !lowercasePath.includes('dist'));
173
181
  }
174
- constructGitHubUrl(repoInfo, path) {
175
- return `https://github.com/${repoInfo.owner}/${repoInfo.repo}/blob/${repoInfo.branch}/${path}`;
176
- }
177
182
  extractTitleFromPath(path) {
178
183
  // Remove extension and convert to title case
179
184
  const basename = path.split('/').pop() || '';
package/build/crawler/github.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"github.js","sourceRoot":"","sources":["../../src/crawler/github.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAE1B,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,sBAAsB,EAA4B,MAAM,qBAAqB,CAAC;AAEvF,MAAM,OAAO,aAAc,SAAQ,WAAW;IASzB;IARF,QAAQ,GAAG,wBAAwB,CAAC;IACpC,gBAAgB,GAAG,mCAAmC,CAAC;IACvD,mBAAmB,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;IACnD,mBAAmB,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,eAAe,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC;IAE5H,YACE,WAAmB,CAAC,EACpB,sBAA8B,IAAI,EACjB,WAAoB,EACrC,UAA4D;QAE5D,KAAK,CAAC,QAAQ,EAAE,mBAAmB,EAAE,UAAU,CAAC,CAAC;QAHhC,gBAAW,GAAX,WAAW,CAAS;IAIvC,CAAC;IAED,KAAK,CAAC,CAAC,KAAK,CAAC,GAAW;QACtB,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,iCAAiC,GAAG,EAAE,CAAC,CAAC;QAE9E,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,MAAM,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;YAC9C,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YAC1C,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,MAAM,IAAI,KAAK,CAAC,uBAAuB,GAAG,EAAE,CAAC,CAAC;YAChD,CAAC;YAED,4CAA4C;YAC5C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAC;YAE3D,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,oCAAoC;gBACpC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;oBAC7B,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;wBACpB,MAAM;oBACR,CAAC;oBACD,KAAK,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;gBACjD,CAAC;YACH,CAAC;iBACI,CAAC;gBACJ,iDAAiD;gBACjD,MAAM,CAAC,KAAK,CAAC,kFAAkF,CAAC,CAAC;gBACjG,KAAK,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAC7C,CAAC;QACH,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,4CAA4C,EAAE,KAAK,CAAC,CAAC;QACpE,CAAC;IACH,CAAC;IAEO,cAAc,CAAC,GAAW;QAChC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,IAAI,MAAM,CAAC,QAAQ,KAAK,YAAY,EAAE,CAAC;gBACrC,OAAO,IAAI,CAAC;YACd,CAAC;YAED,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACnD,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC;gBACpB,OAAO,IAAI,CAAC;YACd,CAAC;YAED,mCAAmC;YACnC,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAE7C,OAAO;gBACL,KAAK;gBACL,IAAI,EAAE,SAAS;gBACf,MAAM,EAAE,MAAM,EAAE,mDAAmD;aACpE,CAAC;QACJ,CAAC;QACD,MAAM,CAAC;YACL,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,qBAAqB,CAAC,QAAyD;QAC3F,MAAM,IAAI,GAAa,EAAE,CAAC;QAE1B,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAE5D,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;gBAC5B,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,IAAI,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;oBACnH,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACvB,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,0DAA0D,EAAE,KAAK,CAAC,CAAC;QAClF,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,KAAK,CAAC,CAAC,gBAAgB,CAAC,QAAyD,EAAE,IAAY;QACrG,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAE9D,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;gBAC5B,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;oBACpB,MAAM;gBACR,CAAC;gBAED,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,IAAI,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC3D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;oBACjE,IAAI,OAAO,EAAE,CAAC;wBACZ,MAAM;4BACJ,GAAG,EAAE,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC;4BACjD,IAAI,EAAE,IAAI,CAAC,IAAI;4BACf,OAAO;4BACP,aAAa,EAAE,UAAU;4BACzB,KAAK,EAAE,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC;yBAC5C,CAAC;oBACJ,CAAC;gBACH,CAAC;qBACI,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;oBACvE,KAAK,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;gBACpD,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,8CAA8C,IAAI,GAAG,EAAE,KAAK,CAAC,CAAC;QAC7E,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,QAAyD,EAAE,IAAY;QACrG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QAEvB,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,QAAQ,UAAU,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,IAAI,aAAa,IAAI,EAAE,CAAC;QACzF,MAAM,OAAO,GAAgB;YAC3B,MAAM,EAAE,gCAAgC;SACzC,CAAC;QAEF,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,OAAO,CAAC,eAAe,CAAC,GAAG,SAAS,IAAI,CAAC,WAAW,EAAE,CAAC;QACzD,CAAC;QAED,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;YAElF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBAC5B,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;gBACpD,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,qBAAqB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;YAC1D,CAAC;YAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAEtC,4FAA4F;YAC5F,MAAM,gBAAgB,GAAG,sBAAsB,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;YACnE,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAAC;gBAC9B,MAAM,CAAC,IAAI,CAAC,0DAA0D,gBAAgB,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;gBACxG,OAAO,EAAE,CAAC;YACZ,CAAC;YAED,OAAO,gBAAgB,CAAC,IAAI,CAAC;QAC/B,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,oDAAoD,IAAI,GAAG,EAAE,KAAK,CAAC,CAAC;YACjF,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,QAAyD,EAAE,IAAY;QACpG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QAEvB,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,gBAAgB,IAAI,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC;QACrG,MAAM,OAAO,GAAgB,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,SAAS,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAEpG,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;YAElF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,iCAAiC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;YACtE,CAAC;YAED,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC/B,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,8CAA8C,IAAI,GAAG,EAAE,KAAK,CAAC,CAAC;YAC3E,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAEO,cAAc,CAAC,IAAY;QACjC,MAAM,aAAa,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QACzC,OAAO,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,aAAa,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7E,CAAC;IAEO,sBAAsB,CAAC,IAAY;QACzC,MAAM,aAAa,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QACzC,4CAA4C;QAC5C,OAAO,CACL,CAAC,aAAa,CAAC,QAAQ,CAAC,cAAc,CAAC;YACvC,CAAC,aAAa,CAAC,QAAQ,CAAC,QAAQ,CAAC;YACjC,CAAC,aAAa,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC/B,CAAC,aAAa,CAAC,QAAQ,CAAC,SAAS,CAAC;YAClC,CAAC,aAAa,CAAC,QAAQ,CAAC,OAAO,CAAC;YAChC,CAAC,aAAa,CAAC,QAAQ,CAAC,MAAM,CAAC,CAChC,CAAC;IACJ,CAAC;IAEO,kBAAkB,CAAC,QAAyD,EAAE,IAAY;QAChG,OAAO,sBAAsB,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,IAAI,SAAS,QAAQ,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC;IACjG,CAAC;IAEO,oBAAoB,CAAC,IAAY;QACvC,6CAA6C;QAC7C,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;QAC7C,MAAM,cAAc,GAAG,QAAQ,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;QACzD,OAAO,cAAc;aAClB,KAAK,CAAC,MAAM,CAAC;aACb,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;aAC3D,IAAI,CAAC,GAAG,CAAC,CAAC;IACf,CAAC;CACF"}
1
+ {"version":3,"file":"github.js","sourceRoot":"","sources":["../../src/crawler/github.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAE1B,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,sBAAsB,EAA4B,MAAM,qBAAqB,CAAC;AASvF,MAAM,OAAO,aAAc,SAAQ,WAAW;IAQzB;IAPF,QAAQ,GAAG,wBAAwB,CAAC;IACpC,mBAAmB,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;IACnD,mBAAmB,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,eAAe,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC;IAE5H,YACE,WAAmB,CAAC,EACpB,sBAA8B,IAAI,EACjB,WAAoB,EACrC,UAA4D;QAE5D,KAAK,CAAC,QAAQ,EAAE,mBAAmB,EAAE,UAAU,CAAC,CAAC;QAHhC,gBAAW,GAAX,WAAW,CAAS;IAIvC,CAAC;IAED,KAAK,CAAC,CAAC,KAAK,CAAC,GAAW;QACtB,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,iCAAiC,GAAG,EAAE,CAAC,CAAC;QAE9E,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,MAAM,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;YAC9C,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YAC1C,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,MAAM,IAAI,KAAK,CAAC,uBAAuB,GAAG,EAAE,CAAC,CAAC;YAChD,CAAC;YAED,4CAA4C;YAC5C,MAAM,OAAO,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAC;YACvG,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,oCAAoC;gBACpC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;oBAC7B,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;wBACpB,MAAM;oBACR,CAAC;oBACD,KAAK,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;gBACjD,CAAC;YACH,CAAC;iBACI,CAAC;gBACJ,iDAAiD;gBACjD,MAAM,CAAC,KAAK,CAAC,kFAAkF,CAAC,CAAC;gBACjG,KAAK,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAC7C,CAAC;QACH,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,4CAA4C,EAAE,KAAK,CAAC,CAAC;QACpE,CAAC;IACH,CAAC;IAEO,cAAc,CAAC,GAAW;QAChC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,IAAI,MAAM,CAAC,QAAQ,KAAK,YAAY,EAAE,CAAC;gBACrC,OAAO,IAAI,CAAC;YACd,CAAC;YAED,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,aAAa,EAAE,GAAG,aAAa,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAC1F,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC;gBACpB,OAAO,IAAI,CAAC;YACd,CAAC;YAED,mCAAmC;YACnC,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAE7C,0HAA0H;YAC1H,MAAM,MAAM,GAAG,IAAI,KAAK,MAAM,IAAI,aAAa,CAAC,CAAC,CAAC,kBAAkB,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YAChG,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,kBAAkB,CAAC,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;YAEhH,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;QACvD,CAAC;QACD,MAAM,CAAC;YACL,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,qBAAqB,CAAC,QAAkB;QACpD,MAAM,IAAI,GAAa,EAAE,CAAC;QAE1B,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAE5D,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;gBAC5B,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,IAAI,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;oBACnH,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACvB,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,0DAA0D,EAAE,KAAK,CAAC,CAAC;QAClF,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,KAAK,CAAC,CAAC,gBAAgB,CAAC,QAAkB,EAAE,IAAY;QAC9D,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAE9D,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;gBAC5B,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;oBACpB,MAAM;gBACR,CAAC;gBAED,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,IAAI,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC3D,IAAI,CAAC,IAAI,CAAC,YAAY,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;wBACzC,MAAM,CAAC,KAAK,CAAC,4BAA4B,IAAI,CAAC,IAAI,6CAA6C,CAAC,CAAC;wBACjG,SAAS;oBACX,CAAC;oBAED,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;oBAC1E,IAAI,OAAO,EAAE,CAAC;wBACZ,MAAM;4BACJ,GAAG,EAAE,IAAI,CAAC,QAAQ;4BAClB,IAAI,EAAE,IAAI,CAAC,IAAI;4BACf,OAAO;4BACP,aAAa,EAAE,UAAU;4BACzB,KAAK,EAAE,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC;yBAC5C,CAAC;oBACJ,CAAC;gBACH,CAAC;qBACI,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;oBACvE,KAAK,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;gBACpD,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,8CAA8C,IAAI,GAAG,EAAE,KAAK,CAAC,CAAC;QAC7E,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,QAAkB,EAAE,IAAY;QAC9D,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QAEvB,MAAM,WAAW,GAAG,IAAI;aACrB,KAAK,CAAC,GAAG,CAAC;aACV,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;aAC7C,IAAI,CAAC,GAAG,CAAC,CAAC;QACb,MAAM,GAAG,GAAG,IAAI,GAAG,CACjB,UAAU,kBAAkB,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,kBAAkB,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,WAAW,EAAE,EAC3G,IAAI,CAAC,QAAQ,CACd,CAAC;QACF,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;YACpB,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC/C,CAAC;QACD,MAAM,OAAO,GAAgB;YAC3B,MAAM,EAAE,gCAAgC;SACzC,CAAC;QAEF,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,OAAO,CAAC,eAAe,CAAC,GAAG,SAAS,IAAI,CAAC,WAAW,EAAE,CAAC;QACzD,CAAC;QAED,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;YAE7F,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBAC5B,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;gBACpD,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,qBAAqB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;YAC1D,CAAC;YAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAEtC,4FAA4F;YAC5F,MAAM,gBAAgB,GAAG,sBAAsB,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;YACnE,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAAC;gBAC9B,MAAM,CAAC,IAAI,CAAC,0DAA0D,gBAAgB,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;gBACxG,OAAO,EAAE,CAAC;YACZ,CAAC;YAED,OAAO,gBAAgB,CAAC,IAAI,CAAC;QAC/B,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,oDAAoD,IAAI,GAAG,EAAE,KAAK,CAAC,CAAC;YACjF,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,GAAW,EAAE,IAAY;QACtD,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QAEvB,MAAM,OAAO,GAAgB,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,SAAS,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAEpG,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;YAElF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,iCAAiC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;YACtE,CAAC;YAED,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC/B,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,8CAA8C,IAAI,GAAG,EAAE,KAAK,CAAC,CAAC;YAC3E,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAEO,cAAc,CAAC,IAAY;QACjC,MAAM,aAAa,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QACzC,OAAO,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,aAAa,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7E,CAAC;IAEO,sBAAsB,CAAC,IAAY;QACzC,MAAM,aAAa,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QACzC,4CAA4C;QAC5C,OAAO,CACL,CAAC,aAAa,CAAC,QAAQ,CAAC,cAAc,CAAC;YACvC,CAAC,aAAa,CAAC,QAAQ,CAAC,QAAQ,CAAC;YACjC,CAAC,aAAa,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC/B,CAAC,aAAa,CAAC,QAAQ,CAAC,SAAS,CAAC;YAClC,CAAC,aAAa,CAAC,QAAQ,CAAC,OAAO,CAAC;YAChC,CAAC,aAAa,CAAC,QAAQ,CAAC,MAAM,CAAC,CAChC,CAAC;IACJ,CAAC;IAEO,oBAAoB,CAAC,IAAY;QACvC,6CAA6C;QAC7C,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;QAC7C,MAAM,cAAc,GAAG,QAAQ,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;QACzD,OAAO,cAAc;aAClB,KAAK,CAAC,MAAM,CAAC;aACb,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;aAC3D,IAAI,CAAC,GAAG,CAAC,CAAC;IACf,CAAC;CACF"}
package/build/crawler/github.test.js CHANGED
@@ -1,4 +1,13 @@
1
1
  import { GitHubCrawler } from './github.js';
2
+ function githubFile(path, branch = 'main') {
3
+ return {
4
+ path,
5
+ type: 'file',
6
+ url: `https://api.github.com/repos/owner/repo/contents/${path}`,
7
+ html_url: `https://github.com/owner/repo/blob/${branch}/${path}`,
8
+ download_url: `https://raw.githubusercontent.com/owner/repo/${branch}/${path}`,
9
+ };
10
+ }
2
11
  describe('GitHubCrawler', () => {
3
12
  let crawler;
4
13
  beforeEach(() => {
@@ -50,15 +59,7 @@ describe('GitHubCrawler', () => {
50
59
  { path: 'src', type: 'dir', name: 'src', url: 'https://api.github.com/repos/owner/repo/contents/src' },
51
60
  ]));
52
61
  // Second call: list docs directory
53
- fetchMock.mockResponseOnce(JSON.stringify([
54
- {
55
- path: 'docs/guide.md',
56
- type: 'file',
57
- name: 'guide.md',
58
- url: 'https://api.github.com/repos/owner/repo/contents/docs/guide.md',
59
- },
60
- { path: 'docs/api.md', type: 'file', name: 'api.md', url: 'https://api.github.com/repos/owner/repo/contents/docs/api.md' },
61
- ]));
62
+ fetchMock.mockResponseOnce(JSON.stringify([githubFile('docs/guide.md'), githubFile('docs/api.md')]));
62
63
  // Third call: fetch guide.md content
63
64
  fetchMock.mockResponseOnce('# Guide\n\nThis is the guide content.');
64
65
  // Fourth call: fetch api.md content
@@ -73,10 +74,8 @@ describe('GitHubCrawler', () => {
73
74
  expect(results[0].contentFormat).toBe('markdown');
74
75
  expect(results[1].path).toBe('docs/api.md');
75
76
  });
76
- it('should handle .git extension in repo URL', async () => {
77
- const rootFiles = [
78
- { path: 'README.md', type: 'file', name: 'README.md', url: 'https://api.github.com/repos/owner/repo/contents/README.md' },
79
- ];
77
+ it('should use canonical URLs from a repository whose default branch is master', async () => {
78
+ const rootFiles = [githubFile('README.md', 'master')];
80
79
  // First call: findDocumentationDirs checks root
81
80
  fetchMock.mockResponseOnce(JSON.stringify(rootFiles));
82
81
  // Second call: processDirectory fetches root again (no doc dirs found)
@@ -89,13 +88,13 @@ describe('GitHubCrawler', () => {
89
88
  }
90
89
  expect(results).toHaveLength(1);
91
90
  expect(results[0].title).toBe('README');
91
+ expect(results[0].url).toBe('https://github.com/owner/repo/blob/master/README.md');
92
+ expect(fetchMock.mock.calls[0][0]).toBe('https://api.github.com/repos/owner/repo/contents/');
93
+ expect(fetchMock.mock.calls[1][0]).toBe('https://api.github.com/repos/owner/repo/contents/');
94
+ expect(fetchMock.mock.calls[2][0]).toBe('https://raw.githubusercontent.com/owner/repo/master/README.md');
92
95
  });
93
96
  it('should skip non-markdown files', async () => {
94
- const rootFiles = [
95
- { path: 'index.js', type: 'file', name: 'index.js', url: 'https://api.github.com/repos/owner/repo/contents/index.js' },
96
- { path: 'style.css', type: 'file', name: 'style.css', url: 'https://api.github.com/repos/owner/repo/contents/style.css' },
97
- { path: 'README.md', type: 'file', name: 'README.md', url: 'https://api.github.com/repos/owner/repo/contents/README.md' },
98
- ];
97
+ const rootFiles = [githubFile('index.js'), githubFile('style.css'), githubFile('README.md')];
99
98
  // First call: findDocumentationDirs checks root
100
99
  fetchMock.mockResponseOnce(JSON.stringify(rootFiles));
101
100
  // Second call: processDirectory fetches root again (no doc dirs found)
@@ -110,16 +109,7 @@ describe('GitHubCrawler', () => {
110
109
  expect(results[0].path).toBe('README.md');
111
110
  });
112
111
  it('should handle various markdown extensions', async () => {
113
- const rootFiles = [
114
- { path: 'doc.md', type: 'file', name: 'doc.md', url: 'https://api.github.com/repos/owner/repo/contents/doc.md' },
115
- { path: 'page.mdx', type: 'file', name: 'page.mdx', url: 'https://api.github.com/repos/owner/repo/contents/page.mdx' },
116
- {
117
- path: 'guide.markdown',
118
- type: 'file',
119
- name: 'guide.markdown',
120
- url: 'https://api.github.com/repos/owner/repo/contents/guide.markdown',
121
- },
122
- ];
112
+ const rootFiles = [githubFile('doc.md'), githubFile('page.mdx'), githubFile('guide.markdown')];
123
113
  // First call: findDocumentationDirs checks root
124
114
  fetchMock.mockResponseOnce(JSON.stringify(rootFiles));
125
115
  // Second call: processDirectory fetches root again (no doc dirs found)
@@ -146,14 +136,7 @@ describe('GitHubCrawler', () => {
146
136
  { path: 'test', type: 'dir', name: 'test', url: 'https://api.github.com/repos/owner/repo/contents/test' },
147
137
  { path: 'docs', type: 'dir', name: 'docs', url: 'https://api.github.com/repos/owner/repo/contents/docs' },
148
138
  ]));
149
- fetchMock.mockResponseOnce(JSON.stringify([
150
- {
151
- path: 'docs/guide.md',
152
- type: 'file',
153
- name: 'guide.md',
154
- url: 'https://api.github.com/repos/owner/repo/contents/docs/guide.md',
155
- },
156
- ]));
139
+ fetchMock.mockResponseOnce(JSON.stringify([githubFile('docs/guide.md')]));
157
140
  fetchMock.mockResponseOnce('# Guide');
158
141
  const results = [];
159
142
  for await (const result of crawler.crawl('https://github.com/owner/repo')) {
@@ -185,32 +168,22 @@ describe('GitHubCrawler', () => {
185
168
  // Mock rateLimit for new crawler instance
186
169
  // eslint-disable-next-line @typescript-eslint/no-explicit-any
187
170
  vi.spyOn(tokenCrawler, 'rateLimit').mockResolvedValue(undefined);
188
- fetchMock.mockResponseOnce(JSON.stringify([]));
171
+ const rootFiles = [githubFile('README.md')];
172
+ fetchMock.mockResponseOnce(JSON.stringify(rootFiles));
173
+ fetchMock.mockResponseOnce(JSON.stringify(rootFiles));
174
+ fetchMock.mockResponseOnce('# README');
189
175
  const results = [];
190
176
  for await (const result of tokenCrawler.crawl('https://github.com/owner/repo')) {
191
177
  results.push(result);
192
178
  }
193
- expect(fetchMock).toHaveBeenCalledWith(expect.any(String), expect.objectContaining({
179
+ expect(fetchMock).toHaveBeenLastCalledWith('https://raw.githubusercontent.com/owner/repo/main/README.md', expect.objectContaining({
194
180
  headers: expect.objectContaining({
195
181
  Authorization: 'token test_token_123',
196
182
  }),
197
183
  }));
198
184
  });
199
185
  it('should extract title from file path', async () => {
200
- const rootFiles = [
201
- {
202
- path: 'getting-started.md',
203
- type: 'file',
204
- name: 'getting-started.md',
205
- url: 'https://api.github.com/repos/owner/repo/contents/getting-started.md',
206
- },
207
- {
208
- path: 'api_reference.md',
209
- type: 'file',
210
- name: 'api_reference.md',
211
- url: 'https://api.github.com/repos/owner/repo/contents/api_reference.md',
212
- },
213
- ];
186
+ const rootFiles = [githubFile('getting-started.md'), githubFile('api_reference.md')];
214
187
  // First call: findDocumentationDirs checks root
215
188
  fetchMock.mockResponseOnce(JSON.stringify(rootFiles));
216
189
  // Second call: processDirectory fetches root again (no doc dirs found)
@@ -225,30 +198,75 @@ describe('GitHubCrawler', () => {
225
198
  expect(results[0].title).toBe('Getting Started');
226
199
  expect(results[1].title).toBe('Api Reference');
227
200
  });
228
- it('should construct correct GitHub blob URLs', async () => {
229
- // First call: findDocumentationDirs checks root - find docs directory
230
- fetchMock.mockResponseOnce(JSON.stringify([{ path: 'docs', type: 'dir', name: 'docs', url: 'https://api.github.com/repos/owner/repo/contents/docs' }]));
231
- // Second call: processDirectory fetches docs directory contents
232
- fetchMock.mockResponseOnce(JSON.stringify([
233
- {
234
- path: 'docs/guide.md',
235
- type: 'file',
236
- name: 'guide.md',
237
- url: 'https://api.github.com/repos/owner/repo/contents/docs/guide.md',
238
- },
239
- ]));
240
- // Third call: fetch file content
201
+ it('should use an explicit branch for root, recursive, and file requests', async () => {
202
+ fetchMock.mockResponseOnce(JSON.stringify([{ path: 'docs', type: 'dir', url: 'https://api.github.com/repos/owner/repo/contents/docs' }]));
203
+ fetchMock.mockResponseOnce(JSON.stringify([githubFile('docs/guide.md', 'develop')]));
204
+ fetchMock.mockResponseOnce('# Guide');
205
+ const results = [];
206
+ for await (const result of crawler.crawl('https://github.com/owner/repo/tree/develop')) {
207
+ results.push(result);
208
+ }
209
+ expect(fetchMock.mock.calls[0][0]).toBe('https://api.github.com/repos/owner/repo/contents/?ref=develop');
210
+ expect(fetchMock.mock.calls[1][0]).toBe('https://api.github.com/repos/owner/repo/contents/docs?ref=develop');
211
+ expect(fetchMock.mock.calls[2][0]).toBe('https://raw.githubusercontent.com/owner/repo/develop/docs/guide.md');
212
+ expect(results[0].url).toBe('https://github.com/owner/repo/blob/develop/docs/guide.md');
213
+ });
214
+ it('should treat a literal suffix as a subdirectory on a single-segment branch', async () => {
215
+ fetchMock.mockResponseOnce(JSON.stringify([githubFile('docs/guide.md', 'feature')]));
241
216
  fetchMock.mockResponseOnce('# Guide');
242
217
  const results = [];
218
+ for await (const result of crawler.crawl('https://github.com/owner/repo/tree/feature/docs')) {
219
+ results.push(result);
220
+ }
221
+ expect(fetchMock.mock.calls[0][0]).toBe('https://api.github.com/repos/owner/repo/contents/docs?ref=feature');
222
+ expect(results.map(({ path }) => path)).toEqual(['docs/guide.md']);
223
+ });
224
+ it('should use an encoded slash as part of the branch name', async () => {
225
+ const rootFiles = [githubFile('README.md', 'feature/docs')];
226
+ fetchMock.mockResponseOnce(JSON.stringify(rootFiles));
227
+ fetchMock.mockResponseOnce(JSON.stringify(rootFiles));
228
+ fetchMock.mockResponseOnce('# README');
229
+ const results = [];
230
+ for await (const result of crawler.crawl('https://github.com/owner/repo/tree/feature%2Fdocs')) {
231
+ results.push(result);
232
+ }
233
+ expect(fetchMock.mock.calls[0][0]).toBe('https://api.github.com/repos/owner/repo/contents/?ref=feature%2Fdocs');
234
+ expect(results.map(({ url }) => url)).toEqual(['https://github.com/owner/repo/blob/feature/docs/README.md']);
235
+ });
236
+ it('should skip markdown files without canonical GitHub URLs', async () => {
237
+ const rootFiles = [{ path: 'README.md', type: 'file', url: 'https://api.github.com/repos/owner/repo/contents/README.md' }];
238
+ fetchMock.mockResponseOnce(JSON.stringify(rootFiles));
239
+ fetchMock.mockResponseOnce(JSON.stringify(rootFiles));
240
+ const results = [];
243
241
  for await (const result of crawler.crawl('https://github.com/owner/repo')) {
244
242
  results.push(result);
245
243
  }
246
- expect(results[0].url).toBe('https://github.com/owner/repo/blob/main/docs/guide.md');
244
+ expect(results).toHaveLength(0);
245
+ expect(fetchMock).toHaveBeenCalledTimes(2);
246
+ });
247
+ it.each([
248
+ ['invalid download URL', { download_url: 'not a URL' }],
249
+ ['insecure download URL', { download_url: 'http://raw.githubusercontent.com/owner/repo/main/README.md' }],
250
+ ['internal download URL', { download_url: 'https://127.0.0.1/README.md' }],
251
+ ['attacker download URL', { download_url: 'https://attacker.example/README.md' }],
252
+ ['insecure HTML URL', { html_url: 'http://github.com/owner/repo/blob/main/README.md' }],
253
+ ['non-GitHub HTML URL', { html_url: 'https://attacker.example/owner/repo/blob/main/README.md' }],
254
+ ])('should reject %s', async (_name, override) => {
255
+ const rootFiles = [{ ...githubFile('README.md'), ...override }];
256
+ fetchMock.mockResponseOnce(JSON.stringify(rootFiles));
257
+ fetchMock.mockResponseOnce(JSON.stringify(rootFiles));
258
+ const results = [];
259
+ for await (const result of crawler.crawl('https://github.com/owner/repo')) {
260
+ results.push(result);
261
+ }
262
+ expect(results).toHaveLength(0);
263
+ expect(fetchMock).toHaveBeenCalledTimes(2);
264
+ expect(fetchMock.mock.calls.every(([request]) => String(request).startsWith('https://api.github.com/'))).toBe(true);
247
265
  });
248
266
  it('should handle fetch errors for file content', async () => {
249
- fetchMock.mockResponseOnce(JSON.stringify([
250
- { path: 'guide.md', type: 'file', name: 'guide.md', url: 'https://api.github.com/repos/owner/repo/contents/guide.md' },
251
- ]));
267
+ const rootFiles = [githubFile('guide.md')];
268
+ fetchMock.mockResponseOnce(JSON.stringify(rootFiles));
269
+ fetchMock.mockResponseOnce(JSON.stringify(rootFiles));
252
270
  fetchMock.mockRejectOnce(new Error('Network error'));
253
271
  const results = [];
254
272
  for await (const result of crawler.crawl('https://github.com/owner/repo')) {
@@ -256,6 +274,7 @@ describe('GitHubCrawler', () => {
256
274
  }
257
275
  // Should skip files that fail to fetch
258
276
  expect(results).toHaveLength(0);
277
+ expect(fetchMock.mock.calls[2][0]).toBe('https://raw.githubusercontent.com/owner/repo/main/guide.md');
259
278
  });
260
279
  it('should validate GitHub API response structure', async () => {
261
280
  // Mock invalid response structure
@@ -271,18 +290,9 @@ describe('GitHubCrawler', () => {
271
290
  { path: 'docs', type: 'dir', name: 'docs', url: 'https://api.github.com/repos/owner/repo/contents/docs' },
272
291
  { path: 'guide', type: 'dir', name: 'guide', url: 'https://api.github.com/repos/owner/repo/contents/guide' },
273
292
  ]));
274
- fetchMock.mockResponseOnce(JSON.stringify([
275
- { path: 'docs/api.md', type: 'file', name: 'api.md', url: 'https://api.github.com/repos/owner/repo/contents/docs/api.md' },
276
- ]));
293
+ fetchMock.mockResponseOnce(JSON.stringify([githubFile('docs/api.md')]));
277
294
  fetchMock.mockResponseOnce('# API');
278
- fetchMock.mockResponseOnce(JSON.stringify([
279
- {
280
- path: 'guide/intro.md',
281
- type: 'file',
282
- name: 'intro.md',
283
- url: 'https://api.github.com/repos/owner/repo/contents/guide/intro.md',
284
- },
285
- ]));
295
+ fetchMock.mockResponseOnce(JSON.stringify([githubFile('guide/intro.md')]));
286
296
  fetchMock.mockResponseOnce('# Intro');
287
297
  const results = [];
288
298
  for await (const result of crawler.crawl('https://github.com/owner/repo')) {
@@ -291,10 +301,7 @@ describe('GitHubCrawler', () => {
291
301
  expect(results).toHaveLength(2);
292
302
  });
293
303
  it('should stop crawling when aborted', async () => {
294
- const rootFiles = [
295
- { path: 'doc1.md', type: 'file', name: 'doc1.md', url: 'https://api.github.com/repos/owner/repo/contents/doc1.md' },
296
- { path: 'doc2.md', type: 'file', name: 'doc2.md', url: 'https://api.github.com/repos/owner/repo/contents/doc2.md' },
297
- ];
304
+ const rootFiles = [githubFile('doc1.md'), githubFile('doc2.md')];
298
305
  // First call: findDocumentationDirs checks root
299
306
  fetchMock.mockResponseOnce(JSON.stringify(rootFiles));
300
307
  // Second call: processDirectory fetches root again (no doc dirs found)
package/build/crawler/github.test.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"github.test.js","sourceRoot":"","sources":["../../src/crawler/github.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAG5C,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC7B,IAAI,OAAsB,CAAC;IAE3B,UAAU,CAAC,GAAG,EAAE;QACd,SAAS,CAAC,UAAU,EAAE,CAAC;QACvB,OAAO,GAAG,IAAI,aAAa,EAAE,CAAC;QAC9B,+DAA+D;QAC/D,8DAA8D;QAC9D,EAAE,CAAC,KAAK,CAAC,OAAc,EAAE,WAAW,CAAC,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;IACrE,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,cAAc,GAAG,IAAI,aAAa,EAAE,CAAC;YAC3C,MAAM,CAAC,cAAc,CAAC,CAAC,WAAW,EAAE,CAAC;QACvC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;YAC/D,MAAM,aAAa,GAAG,IAAI,aAAa,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;YACjD,MAAM,CAAC,aAAa,CAAC,CAAC,WAAW,EAAE,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;YACpC,MAAM,YAAY,GAAG,IAAI,aAAa,CAAC,CAAC,EAAE,IAAI,EAAE,kBAAkB,CAAC,CAAC;YACpE,MAAM,CAAC,YAAY,CAAC,CAAC,WAAW,EAAE,CAAC;QACrC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;YACzC,MAAM,UAAU,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,eAAe,GAAG,IAAI,aAAa,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;YAC1E,MAAM,CAAC,eAAe,CAAC,CAAC,WAAW,EAAE,CAAC;QACxC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;QACrB,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;YACjD,MAAM,OAAO,GAAkB,EAAE,CAAC;YAElC,iBAAiB;YACjB,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,gCAAgC,CAAC,EAAE,CAAC;gBAC3E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACrD,MAAM,OAAO,GAAkB,EAAE,CAAC;YAElC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,oBAAoB,CAAC,EAAE,CAAC;gBAC/D,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;YAC/D,kCAAkC;YAClC,SAAS,CAAC,gBAAgB,CACxB,IAAI,CAAC,SAAS,CAAC;gBACb,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,uDAAuD,EAAE;gBACzG,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,sDAAsD,EAAE;aACvG,CAAC,CACH,CAAC;YACF,mCAAmC;YACnC,SAAS,CAAC,gBAAgB,CACxB,IAAI,CAAC,SAAS,CAAC;gBACb;oBACE,IAAI,EAAE,eAAe;oBACrB,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,UAAU;oBAChB,GAAG,EAAE,gEAAgE;iBACtE;gBACD,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,8DAA8D,EAAE;aAC3H,CAAC,CACH,CAAC;YACF,qCAAqC;YACrC,SAAS,CAAC,gBAAgB,CAAC,uCAAuC,CAAC,CAAC;YACpE,oCAAoC;YACpC,SAAS,CAAC,gBAAgB,CAAC,uCAAuC,CAAC,CAAC;YAEpE,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,EAAE,CAAC;gBAC1E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAChC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YAC9C,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;YAC9C,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAClD,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,MAAM,SAAS,GAAG;gBAChB,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,GAAG,EAAE,4DAA4D,EAAE;aAC1H,CAAC;YACF,gDAAgD;YAChD,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YACtD,uEAAuE;YACvE,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YACtD,sCAAsC;YACtC,SAAS,CAAC,gBAAgB,CAAC,6BAA6B,CAAC,CAAC;YAE1D,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,mCAAmC,CAAC,EAAE,CAAC;gBAC9E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAChC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC9C,MAAM,SAAS,GAAG;gBAChB,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,2DAA2D,EAAE;gBACtH,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,GAAG,EAAE,4DAA4D,EAAE;gBACzH,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,GAAG,EAAE,4DAA4D,EAAE;aAC1H,CAAC;YACF,gDAAgD;YAChD,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YACtD,uEAAuE;YACvE,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YACtD,sCAAsC;YACtC,SAAS,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC;YAEvC,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,EAAE,CAAC;gBAC1E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAChC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,MAAM,SAAS,GAAG;gBAChB,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,yDAAyD,EAAE;gBAChH,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,2DAA2D,EAAE;gBACtH;oBACE,IAAI,EAAE,gBAAgB;oBACtB,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,gBAAgB;oBACtB,GAAG,EAAE,iEAAiE;iBACvE;aACF,CAAC;YACF,gDAAgD;YAChD,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YACtD,uEAAuE;YACvE,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YACtD,uBAAuB;YACvB,SAAS,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;YACpC,SAAS,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;YACrC,SAAS,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAEtC,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,EAAE,CAAC;gBAC1E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+DAA+D,EAAE,KAAK,IAAI,EAAE;YAC7E,SAAS,CAAC,gBAAgB,CACxB,IAAI,CAAC,SAAS,CAAC;gBACb;oBACE,IAAI,EAAE,cAAc;oBACpB,IAAI,EAAE,KAAK;oBACX,IAAI,EAAE,cAAc;oBACpB,GAAG,EAAE,+DAA+D;iBACrE;gBACD,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,yDAAyD,EAAE;gBAC/G,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,uDAAuD,EAAE;gBACzG,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,uDAAuD,EAAE;aAC1G,CAAC,CACH,CAAC;YACF,SAAS,CAAC,gBAAgB,CACxB,IAAI,CAAC,SAAS,CAAC;gBACb;oBACE,IAAI,EAAE,eAAe;oBACrB,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,UAAU;oBAChB,GAAG,EAAE,gEAAgE;iBACtE;aACF,CAAC,CACH,CAAC;YACF,SAAS,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAEtC,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,EAAE,CAAC;gBAC1E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,wEAAwE;YACxE,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAChC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,SAAS,CAAC,gBAAgB,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;YAEhD,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,EAAE,CAAC;gBAC1E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,wCAAwC;YACxC,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qCAAqC,EAAE,KAAK,IAAI,EAAE;YACnD,SAAS,CAAC,gBAAgB,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;YAEhD,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,EAAE,CAAC;gBAC1E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YAChE,MAAM,YAAY,GAAG,IAAI,aAAa,CAAC,CAAC,EAAE,IAAI,EAAE,gBAAgB,CAAC,CAAC;YAClE,0CAA0C;YAC1C,8DAA8D;YAC9D,EAAE,CAAC,KAAK,CAAC,YAAmB,EAAE,WAAW,CAAC,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YAExE,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC;YAE/C,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,YAAY,CAAC,KAAK,CAAC,+BAA+B,CAAC,EAAE,CAAC;gBAC/E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,SAAS,CAAC,CAAC,oBAAoB,CACpC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAClB,MAAM,CAAC,gBAAgB,CAAC;gBACtB,OAAO,EAAE,MAAM,CAAC,gBAAgB,CAAC;oBAC/B,aAAa,EAAE,sBAAsB;iBACtC,CAAC;aACH,CAAC,CACH,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qCAAqC,EAAE,KAAK,IAAI,EAAE;YACnD,MAAM,SAAS,GAAG;gBAChB;oBACE,IAAI,EAAE,oBAAoB;oBAC1B,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,oBAAoB;oBAC1B,GAAG,EAAE,qEAAqE;iBAC3E;gBACD;oBACE,IAAI,EAAE,kBAAkB;oBACxB,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,kBAAkB;oBACxB,GAAG,EAAE,mEAAmE;iBACzE;aACF,CAAC;YACF,gDAAgD;YAChD,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YACtD,uEAAuE;YACvE,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YACtD,uBAAuB;YACvB,SAAS,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC;YACxC,SAAS,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC;YAExC,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,EAAE,CAAC;gBAC1E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;YACjD,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,sEAAsE;YACtE,SAAS,CAAC,gBAAgB,CACxB,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,uDAAuD,EAAE,CAAC,CAAC,CAC5H,CAAC;YACF,gEAAgE;YAChE,SAAS,CAAC,gBAAgB,CACxB,IAAI,CAAC,SAAS,CAAC;gBACb;oBACE,IAAI,EAAE,eAAe;oBACrB,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,UAAU;oBAChB,GAAG,EAAE,gEAAgE;iBACtE;aACF,CAAC,CACH,CAAC;YACF,iCAAiC;YACjC,SAAS,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAEtC,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,EAAE,CAAC;gBAC1E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;QACvF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;YAC3D,SAAS,CAAC,gBAAgB,CACxB,IAAI,CAAC,SAAS,CAAC;gBACb,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,2DAA2D,EAAE;aACvH,CAAC,CACH,CAAC;YACF,SAAS,CAAC,cAAc,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC;YAErD,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,EAAE,CAAC;gBAC1E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,uCAAuC;YACvC,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;YAC7D,kCAAkC;YAClC,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC;YAErE,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,EAAE,CAAC;gBAC1E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;YAC9D,SAAS,CAAC,gBAAgB,CACxB,IAAI,CAAC,SAAS,CAAC;gBACb,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,uDAAuD,EAAE;gBACzG,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,wDAAwD,EAAE;aAC7G,CAAC,CACH,CAAC;YACF,SAAS,CAAC,gBAAgB,CACxB,IAAI,CAAC,SAAS,CAAC;gBACb,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,8DAA8D,EAAE;aAC3H,CAAC,CACH,CAAC;YACF,SAAS,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;YACpC,SAAS,CAAC,gBAAgB,CACxB,IAAI,CAAC,SAAS,CAAC;gBACb;oBACE,IAAI,EAAE,gBAAgB;oBACtB,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,UAAU;oBAChB,GAAG,EAAE,iEAAiE;iBACvE;aACF,CAAC,CACH,CAAC;YACF,SAAS,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAEtC,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,EAAE,CAAC;gBAC1E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;YACjD,MAAM,SAAS,GAAG;gBAChB,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,0DAA0D,EAAE;gBACnH,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,0DAA0D,EAAE;aACpH,CAAC;YACF,gDAAgD;YAChD,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YACtD,uEAAuE;YACvE,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YACtD,oCAAoC;YACpC,SAAS,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAEtC,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,MAAM,gBAAgB,GAAG,IAAI,aAAa,EAAE,CAAC;YAC7C,0CAA0C;YAC1C,8DAA8D;YAC9D,EAAE,CAAC,KAAK,CAAC,gBAAuB,EAAE,WAAW,CAAC,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YAE5E,oBAAoB;YACpB,MAAM,SAAS,GAAG,gBAAgB,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;YAE1E,mBAAmB;YACnB,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;YACrC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;gBAChB,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC5B,CAAC;YAED,qCAAqC;YACpC,gBAAuD,CAAC,UAAU,GAAG,IAAI,CAAC;YAE3E,0BAA0B;YAC1B,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,SAAS,EAAE,CAAC;gBACrC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,oCAAoC;YACpC,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YAChE,MAAM,cAAc,GAAG,OAAO,CAAC,aAAa,EAAe,CAAC;YAC5D,SAAS,CAAC,sBAAsB,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE;gBAChD,MAAM,MAAM,GAAG,IAAI,EAAE,MAAqB,CAAC;gBAC3C,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBAC/B,OAAO,IAAI,OAAO,CAAC,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE;oBACtC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;gBAChF,CAAC,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC,IAAI,EAAE,CAAC;YACrE,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC;YAC5C,OAAO,CAAC,KAAK,EAAE,CAAC;YAEhB,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClC,MAAM,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9D,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
1
+ {"version":3,"file":"github.test.js","sourceRoot":"","sources":["../../src/crawler/github.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAG5C,SAAS,UAAU,CAAC,IAAY,EAAE,MAAM,GAAG,MAAM;IAC/C,OAAO;QACL,IAAI;QACJ,IAAI,EAAE,MAAM;QACZ,GAAG,EAAE,oDAAoD,IAAI,EAAE;QAC/D,QAAQ,EAAE,sCAAsC,MAAM,IAAI,IAAI,EAAE;QAChE,YAAY,EAAE,gDAAgD,MAAM,IAAI,IAAI,EAAE;KAC/E,CAAC;AACJ,CAAC;AAED,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC7B,IAAI,OAAsB,CAAC;IAE3B,UAAU,CAAC,GAAG,EAAE;QACd,SAAS,CAAC,UAAU,EAAE,CAAC;QACvB,OAAO,GAAG,IAAI,aAAa,EAAE,CAAC;QAC9B,+DAA+D;QAC/D,8DAA8D;QAC9D,EAAE,CAAC,KAAK,CAAC,OAAc,EAAE,WAAW,CAAC,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;IACrE,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,cAAc,GAAG,IAAI,aAAa,EAAE,CAAC;YAC3C,MAAM,CAAC,cAAc,CAAC,CAAC,WAAW,EAAE,CAAC;QACvC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;YAC/D,MAAM,aAAa,GAAG,IAAI,aAAa,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;YACjD,MAAM,CAAC,aAAa,CAAC,CAAC,WAAW,EAAE,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;YACpC,MAAM,YAAY,GAAG,IAAI,aAAa,CAAC,CAAC,EAAE,IAAI,EAAE,kBAAkB,CAAC,CAAC;YACpE,MAAM,CAAC,YAAY,CAAC,CAAC,WAAW,EAAE,CAAC;QACrC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;YACzC,MAAM,UAAU,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,eAAe,GAAG,IAAI,aAAa,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;YAC1E,MAAM,CAAC,eAAe,CAAC,CAAC,WAAW,EAAE,CAAC;QACxC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;QACrB,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;YACjD,MAAM,OAAO,GAAkB,EAAE,CAAC;YAElC,iBAAiB;YACjB,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,gCAAgC,CAAC,EAAE,CAAC;gBAC3E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACrD,MAAM,OAAO,GAAkB,EAAE,CAAC;YAElC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,oBAAoB,CAAC,EAAE,CAAC;gBAC/D,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;YAC/D,kCAAkC;YAClC,SAAS,CAAC,gBAAgB,CACxB,IAAI,CAAC,SAAS,CAAC;gBACb,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,uDAAuD,EAAE;gBACzG,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,sDAAsD,EAAE;aACvG,CAAC,CACH,CAAC;YACF,mCAAmC;YACnC,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;YACrG,qCAAqC;YACrC,SAAS,CAAC,gBAAgB,CAAC,uCAAuC,CAAC,CAAC;YACpE,oCAAoC;YACpC,SAAS,CAAC,gBAAgB,CAAC,uCAAuC,CAAC,CAAC;YAEpE,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,EAAE,CAAC;gBAC1E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAChC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YAC9C,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;YAC9C,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAClD,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4EAA4E,EAAE,KAAK,IAAI,EAAE;YAC1F,MAAM,SAAS,GAAG,CAAC,UAAU,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAC;YACtD,gDAAgD;YAChD,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YACtD,uEAAuE;YACvE,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YACtD,sCAAsC;YACtC,SAAS,CAAC,gBAAgB,CAAC,6BAA6B,CAAC,CAAC;YAE1D,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,mCAAmC,CAAC,EAAE,CAAC;gBAC9E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAChC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACxC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,qDAAqD,CAAC,CAAC;YACnF,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,mDAAmD,CAAC,CAAC;YAC7F,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,mDAAmD,CAAC,CAAC;YAC7F,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,+DAA+D,CAAC,CAAC;QAC3G,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC9C,MAAM,SAAS,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC,WAAW,CAAC,EAAE,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC;YAC7F,gDAAgD;YAChD,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YACtD,uEAAuE;YACvE,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YACtD,sCAAsC;YACtC,SAAS,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC;YAEvC,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,EAAE,CAAC;gBAC1E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAChC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,MAAM,SAAS,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,UAAU,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC;YAC/F,gDAAgD;YAChD,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YACtD,uEAAuE;YACvE,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YACtD,uBAAuB;YACvB,SAAS,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;YACpC,SAAS,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;YACrC,SAAS,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAEtC,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,EAAE,CAAC;gBAC1E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+DAA+D,EAAE,KAAK,IAAI,EAAE;YAC7E,SAAS,CAAC,gBAAgB,CACxB,IAAI,CAAC,SAAS,CAAC;gBACb;oBACE,IAAI,EAAE,cAAc;oBACpB,IAAI,EAAE,KAAK;oBACX,IAAI,EAAE,cAAc;oBACpB,GAAG,EAAE,+DAA+D;iBACrE;gBACD,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,yDAAyD,EAAE;gBAC/G,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,uDAAuD,EAAE;gBACzG,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,uDAAuD,EAAE;aAC1G,CAAC,CACH,CAAC;YACF,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC;YAC1E,SAAS,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAEtC,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,EAAE,CAAC;gBAC1E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,wEAAwE;YACxE,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAChC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,SAAS,CAAC,gBAAgB,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;YAEhD,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,EAAE,CAAC;gBAC1E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,wCAAwC;YACxC,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qCAAqC,EAAE,KAAK,IAAI,EAAE;YACnD,SAAS,CAAC,gBAAgB,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;YAEhD,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,EAAE,CAAC;gBAC1E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YAChE,MAAM,YAAY,GAAG,IAAI,aAAa,CAAC,CAAC,EAAE,IAAI,EAAE,gBAAgB,CAAC,CAAC;YAClE,0CAA0C;YAC1C,8DAA8D;YAC9D,EAAE,CAAC,KAAK,CAAC,YAAmB,EAAE,WAAW,CAAC,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YAExE,MAAM,SAAS,GAAG,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC;YAC5C,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YACtD,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YACtD,SAAS,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC;YAEvC,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,YAAY,CAAC,KAAK,CAAC,+BAA+B,CAAC,EAAE,CAAC;gBAC/E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,SAAS,CAAC,CAAC,wBAAwB,CACxC,6DAA6D,EAC7D,MAAM,CAAC,gBAAgB,CAAC;gBACtB,OAAO,EAAE,MAAM,CAAC,gBAAgB,CAAC;oBAC/B,aAAa,EAAE,sBAAsB;iBACtC,CAAC;aACH,CAAC,CACH,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qCAAqC,EAAE,KAAK,IAAI,EAAE;YACnD,MAAM,SAAS,GAAG,CAAC,UAAU,CAAC,oBAAoB,CAAC,EAAE,UAAU,CAAC,kBAAkB,CAAC,CAAC,CAAC;YACrF,gDAAgD;YAChD,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YACtD,uEAAuE;YACvE,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YACtD,uBAAuB;YACvB,SAAS,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC;YACxC,SAAS,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC;YAExC,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,EAAE,CAAC;gBAC1E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;YACjD,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sEAAsE,EAAE,KAAK,IAAI,EAAE;YACpF,SAAS,CAAC,gBAAgB,CACxB,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,uDAAuD,EAAE,CAAC,CAAC,CAC9G,CAAC;YACF,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,UAAU,CAAC,eAAe,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;YACrF,SAAS,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAEtC,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,4CAA4C,CAAC,EAAE,CAAC;gBACvF,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,+DAA+D,CAAC,CAAC;YACzG,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,mEAAmE,CAAC,CAAC;YAC7G,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,oEAAoE,CAAC,CAAC;YAC9G,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,0DAA0D,CAAC,CAAC;QAC1F,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4EAA4E,EAAE,KAAK,IAAI,EAAE;YAC1F,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,UAAU,CAAC,eAAe,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;YACrF,SAAS,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAEtC,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,iDAAiD,CAAC,EAAE,CAAC;gBAC5F,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,mEAAmE,CAAC,CAAC;YAC7G,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC;QACrE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wDAAwD,EAAE,KAAK,IAAI,EAAE;YACtE,MAAM,SAAS,GAAG,CAAC,UAAU,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC,CAAC;YAC5D,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YACtD,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YACtD,SAAS,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC;YAEvC,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,mDAAmD,CAAC,EAAE,CAAC;gBAC9F,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,sEAAsE,CAAC,CAAC;YAChH,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,2DAA2D,CAAC,CAAC,CAAC;QAC/G,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0DAA0D,EAAE,KAAK,IAAI,EAAE;YACxE,MAAM,SAAS,GAAG,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,4DAA4D,EAAE,CAAC,CAAC;YAC3H,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YACtD,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YAEtD,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,EAAE,CAAC;gBAC1E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAChC,MAAM,CAAC,SAAS,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,IAAI,CAAC;YACN,CAAC,sBAAsB,EAAE,EAAE,YAAY,EAAE,WAAW,EAAE,CAAC;YACvD,CAAC,uBAAuB,EAAE,EAAE,YAAY,EAAE,4DAA4D,EAAE,CAAC;YACzG,CAAC,uBAAuB,EAAE,EAAE,YAAY,EAAE,6BAA6B,EAAE,CAAC;YAC1E,CAAC,uBAAuB,EAAE,EAAE,YAAY,EAAE,oCAAoC,EAAE,CAAC;YACjF,CAAC,mBAAmB,EAAE,EAAE,QAAQ,EAAE,kDAAkD,EAAE,CAAC;YACvF,CAAC,qBAAqB,EAAE,EAAE,QAAQ,EAAE,yDAAyD,EAAE,CAAC;SACjG,CAAC,CAAC,kBAAkB,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE;YAC/C,MAAM,SAAS,GAAG,CAAC,EAAE,GAAG,UAAU,CAAC,WAAW,CAAC,EAAE,GAAG,QAAQ,EAAE,CAAC,CAAC;YAChE,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YACtD,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YAEtD,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,EAAE,CAAC;gBAC1E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAChC,MAAM,CAAC,SAAS,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;YAC3C,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,UAAU,CAAC,yBAAyB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;YAC3D,MAAM,SAAS,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC;YAC3C,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YACtD,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YACtD,SAAS,CAAC,cAAc,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC;YAErD,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,EAAE,CAAC;gBAC1E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,uCAAuC;YACvC,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAChC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,4DAA4D,CAAC,CAAC;QACxG,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;YAC7D,kCAAkC;YAClC,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC;YAErE,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,EAAE,CAAC;gBAC1E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;YAC9D,SAAS,CAAC,gBAAgB,CACxB,IAAI,CAAC,SAAS,CAAC;gBACb,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,uDAAuD,EAAE;gBACzG,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,wDAAwD,EAAE;aAC7G,CAAC,CACH,CAAC;YACF,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;YACxE,SAAS,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;YACpC,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC;YAC3E,SAAS,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAEtC,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,EAAE,CAAC;gBAC1E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;YACjD,MAAM,SAAS,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC;YACjE,gDAAgD;YAChD,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YACtD,uEAAuE;YACvE,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YACtD,oCAAoC;YACpC,SAAS,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAEtC,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,MAAM,gBAAgB,GAAG,IAAI,aAAa,EAAE,CAAC;YAC7C,0CAA0C;YAC1C,8DAA8D;YAC9D,EAAE,CAAC,KAAK,CAAC,gBAAuB,EAAE,WAAW,CAAC,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YAE5E,oBAAoB;YACpB,MAAM,SAAS,GAAG,gBAAgB,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;YAE1E,mBAAmB;YACnB,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;YACrC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;gBAChB,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC5B,CAAC;YAED,qCAAqC;YACpC,gBAAuD,CAAC,UAAU,GAAG,IAAI,CAAC;YAE3E,0BAA0B;YAC1B,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,SAAS,EAAE,CAAC;gBACrC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,oCAAoC;YACpC,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YAChE,MAAM,cAAc,GAAG,OAAO,CAAC,aAAa,EAAe,CAAC;YAC5D,SAAS,CAAC,sBAAsB,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE;gBAChD,MAAM,MAAM,GAAG,IAAI,EAAE,MAAqB,CAAC;gBAC3C,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBAC/B,OAAO,IAAI,OAAO,CAAC,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE;oBACtC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;gBAChF,CAAC,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC,IAAI,EAAE,CAAC;YACrE,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC;YAC5C,OAAO,CAAC,KAAK,EAAE,CAAC;YAEhB,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClC,MAAM,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9D,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
package/build/crawler/queue-manager.d.ts CHANGED
@@ -3,7 +3,7 @@ import { CrawlResult } from '../types.js';
3
3
  import { SiteDetectionRule } from './site-rules.js';
4
4
  export declare class QueueManager {
5
5
  private requestQueue;
6
- private websiteId;
6
+ private storageId;
7
7
  private results;
8
8
  private static readonly BATCH_SIZE;
9
9
  /** Optional path prefix to restrict crawling to URLs under this path */
package/build/crawler/queue-manager.js CHANGED
@@ -1,10 +1,10 @@
1
1
  import { RequestQueue, Dataset } from 'crawlee';
2
- import { generateDocId, isPathAllowed } from '../util/docs.js';
2
+ import { generateCrawlStorageId, isPathAllowed } from '../util/docs.js';
3
3
  import { logger } from '../util/logger.js';
4
4
  import { discoverUrlsFromLlmsTxt } from './llms-txt.js';
5
5
  export class QueueManager {
6
6
  requestQueue = null;
7
- websiteId = '';
7
+ storageId = '';
8
8
  results = [];
9
9
  static BATCH_SIZE = 20;
10
10
  /** Optional path prefix to restrict crawling to URLs under this path */
@@ -17,7 +17,7 @@ export class QueueManager {
17
17
  filteredByHostnameCount = 0;
18
18
  async initialize(url, pathPrefix) {
19
19
  const parsedUrl = new URL(url);
20
- this.websiteId = generateDocId(url, parsedUrl.hostname);
20
+ this.storageId = generateCrawlStorageId(url);
21
21
  this.pathPrefix = pathPrefix || '';
22
22
  this.allowedHostname = parsedUrl.hostname.toLowerCase();
23
23
  this.filteredByPathCount = 0;
@@ -26,11 +26,11 @@ export class QueueManager {
26
26
  if (this.pathPrefix) {
27
27
  logger.info(`[QueueManager] Path restriction enabled: only crawling URLs under ${this.pathPrefix}`);
28
28
  }
29
- logger.debug(`[QueueManager] Using website ID: ${this.websiteId}`);
29
+ logger.debug(`[QueueManager] Using storage ID: ${this.storageId}`);
30
30
  // Initialize queue
31
- this.requestQueue = await RequestQueue.open(this.websiteId);
31
+ this.requestQueue = await RequestQueue.open(this.storageId);
32
32
  await this.requestQueue.drop();
33
- this.requestQueue = await RequestQueue.open(this.websiteId);
33
+ this.requestQueue = await RequestQueue.open(this.storageId);
34
34
  // Add initial request (strip any hash from URL)
35
35
  const cleanUrl = parsedUrl.origin + parsedUrl.pathname + parsedUrl.search;
36
36
  await this.requestQueue.addRequest({
@@ -38,7 +38,7 @@ export class QueueManager {
38
38
  uniqueKey: parsedUrl.pathname + parsedUrl.search,
39
39
  });
40
40
  // Clear existing dataset
41
- const dataset = await Dataset.open(this.websiteId);
41
+ const dataset = await Dataset.open(this.storageId);
42
42
  await dataset.drop();
43
43
  }
44
44
  /**
@@ -181,7 +181,7 @@ export class QueueManager {
181
181
  if (this.results.length === 0) {
182
182
  return [];
183
183
  }
184
- const dataset = await Dataset.open(this.websiteId);
184
+ const dataset = await Dataset.open(this.storageId);
185
185
  const resultsToProcess = [...this.results];
186
186
  this.results = [];
187
187
  // Process in chunks for better memory management
package/build/crawler/queue-manager.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"queue-manager.js","sourceRoot":"","sources":["../../src/crawler/queue-manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,OAAO,EAA6C,MAAM,SAAS,CAAC;AAC3F,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAG/D,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAC;AAExD,MAAM,OAAO,YAAY;IACf,YAAY,GAAwB,IAAI,CAAC;IACzC,SAAS,GAAW,EAAE,CAAC;IACvB,OAAO,GAAkB,EAAE,CAAC;IAC5B,MAAM,CAAU,UAAU,GAAG,EAAE,CAAC;IACxC,wEAAwE;IAChE,UAAU,GAAW,EAAE,CAAC;IAChC,gGAAgG;IACxF,eAAe,GAAW,EAAE,CAAC;IACrC,yDAAyD;IACjD,mBAAmB,GAAW,CAAC,CAAC;IACxC,sDAAsD;IAC9C,uBAAuB,GAAW,CAAC,CAAC;IAE5C,KAAK,CAAC,UAAU,CAAC,GAAW,EAAE,UAAmB;QAC/C,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,CAAC,SAAS,GAAG,aAAa,CAAC,GAAG,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC;QACxD,IAAI,CAAC,UAAU,GAAG,UAAU,IAAI,EAAE,CAAC;QACnC,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QACxD,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,uBAAuB,GAAG,CAAC,CAAC;QAEjC,MAAM,CAAC,IAAI,CAAC,8DAA8D,IAAI,CAAC,eAAe,uBAAuB,CAAC,CAAC;QACvH,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,MAAM,CAAC,IAAI,CAAC,qEAAqE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QACtG,CAAC;QACD,MAAM,CAAC,KAAK,CAAC,oCAAoC,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QAEnE,mBAAmB;QACnB,IAAI,CAAC,YAAY,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC5D,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;QAC/B,IAAI,CAAC,YAAY,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAE5D,gDAAgD;QAChD,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,GAAG,SAAS,CAAC,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC;QAC1E,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC;YACjC,GAAG,EAAE,QAAQ;YACb,SAAS,EAAE,SAAS,CAAC,QAAQ,GAAG,SAAS,CAAC,MAAM;SACjD,CAAC,CAAC;QAEH,yBAAyB;QACzB,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACnD,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;IACvB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,eAAe,CAAC,OAAe;QACnC,MAAM,IAAI,GAAG,MAAM,uBAAuB,CAAC,OAAO,CAAC,CAAC;QACpD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,CAAC;QACX,CAAC;QAED,OAAO,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,WAAW,CAAC,IAAc;QAC9B,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvB,MAAM,CAAC,IAAI,CAAC,4DAA4D,CAAC,CAAC;YAC1E,OAAO,CAAC,CAAC;QACX,CAAC;QAED,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,KAAK,MAAM,MAAM,IAAI,IAAI,EAAE,CAAC;YAC1B,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;gBAE/B,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAC7C,SAAS;gBACX,CAAC;gBAED,IAAI,IAAI,CAAC,UAAU,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;oBACxE,SAAS;gBACX,CAAC;gBAED,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC;gBACtE,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC;gBAElD,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,aAAa,EAAE,SAAS,EAAE,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC5F,KAAK,EAAE,CAAC;YACV,CAAC;YACD,MAAM,CAAC;gBACL,MAAM,CAAC,KAAK,CAAC,6CAA6C,MAAM,EAAE,CAAC,CAAC;YACtE,CAAC;QACH,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,yBAAyB,KAAK,wBAAwB,IAAI,CAAC,MAAM,GAAG,KAAK,YAAY,CAAC,CAAC;QACnG,OAAO,KAAK,CAAC;IACf,CAAC;IAED,sBAAsB;QACpB,OAAO,IAAI,CAAC,mBAAmB,CAAC;IAClC,CAAC;IAED,0BAA0B;QACxB,OAAO,IAAI,CAAC,uBAAuB,CAAC;IACtC,CAAC;IAED;;;;;;;;;;;OAWG;IACK,iBAAiB,CAAC,QAAgB;QACxC,MAAM,CAAC,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;QACjC,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC;QAErC,cAAc;QACd,IAAI,CAAC,KAAK,OAAO,EAAE,CAAC;YAClB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,8FAA8F;QAC9F,IAAI,CAAC,CAAC,QAAQ,CAAC,GAAG,GAAG,OAAO,CAAC,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED,KAAK,CAAC,mBAAmB,CACvB,YAAuG,EACvG,GAAQ,EACR,IAAuB;QAEvB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,YAAa,CAAC,OAAO,EAAE,CAAC;QACrD,IAAI,SAAS,EAAE,CAAC;YACd,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE;gBACxB,YAAY,EAAE,SAAS,CAAC,mBAAmB,IAAI,CAAC;gBAChD,YAAY,EAAE,SAAS,CAAC,mBAAmB,IAAI,CAAC;gBAChD,UAAU,EAAE,SAAS,CAAC,iBAAiB,IAAI,CAAC;aAC7C,CAAC,CAAC;QACL,CAAC;QAED,wDAAwD;QACxD,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QACnC,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC;QAC7C,MAAM,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5D,MAAM,uBAAuB,GAAG,GAAG,EAAE;YACnC,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC7B,CAAC,CAAC;QACF,MAAM,2BAA2B,GAAG,GAAG,EAAE;YACvC,IAAI,CAAC,uBAAuB,EAAE,CAAC;QACjC,CAAC,CAAC;QAEF,MAAM,cAAc,GAAwB;YAC1C,QAAQ,EAAE,aAAgC;YAC1C,wBAAwB,CAAC,GAAG;gBAC1B,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAE7B,oDAAoD;gBACpD,4DAA4D;gBAC5D,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;oBACb,MAAM,CAAC,KAAK,CAAC,wCAAwC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;oBAChE,OAAO,KAAK,CAAC;gBACf,CAAC;gBAED,yEAAyE;gBACzE,+GAA+G;gBAC/G,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;oBACrC,MAAM,CAAC,KAAK,CAAC,wDAAwD,GAAG,CAAC,QAAQ,cAAc,eAAe,GAAG,CAAC,CAAC;oBACnH,2BAA2B,EAAE,CAAC;oBAC9B,OAAO,KAAK,CAAC;gBACf,CAAC;gBAED,oDAAoD;gBACpD,iFAAiF;gBACjF,IAAI,UAAU,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,EAAE,CAAC;oBAC3D,MAAM,CAAC,KAAK,CAAC,oDAAoD,GAAG,CAAC,QAAQ,aAAa,UAAU,GAAG,CAAC,CAAC;oBACzG,uBAAuB,EAAE,CAAC;oBAC1B,OAAO,KAAK,CAAC;gBACf,CAAC;gBAED,yEAAyE;gBACzE,OAAO;oBACL,GAAG,GAAG;oBACN,GAAG,EAAE,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,MAAM,EAAE,sBAAsB;oBACnE,SAAS,EAAE,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,MAAM;iBACrC,CAAC;YACJ,CAAC;SACF,CAAC;QAEF,+CAA+C;QAC/C,IAAI,IAAI,CAAC,aAAa,EAAE,MAAM,EAAE,CAAC;YAC/B,cAAc,CAAC,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1D,CAAC;QAED,MAAM,aAAa,GAAG,MAAM,YAAY,CAAC,cAAc,CAAC,CAAC;QAEzD,GAAG,CAAC,IAAI,CAAC,iBAAiB,EAAE;YAC1B,cAAc,EAAE,aAAa,CAAC,iBAAiB,CAAC,MAAM;YACtD,IAAI,EAAE,aAAa,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAwB,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;YACpF,GAAG,CAAC,IAAI,CAAC,uBAAuB,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,IAAI,CAAC,uBAAuB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjG,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,cAAc,EAAE,IAAI,CAAC,mBAAmB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACtG,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,YAAY;QAChB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9B,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACnD,MAAM,gBAAgB,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC;QAC3C,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAElB,iDAAiD;QACjD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;YACpD,MAAM,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC3D,CAAC;QAED,OAAO,gBAAgB,CAAC;IAC1B,CAAC;IAED,SAAS,CAAC,MAAmB;QAC3B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC5B,CAAC;IAED,gBAAgB;QACd,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,YAAY,CAAC,UAAU,CAAC;IACxD,CAAC;IAED,eAAe;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAClB,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,GAAG,CAAC,CAAC,CAAC;YAClG,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QAC3B,CAAC;IACH,CAAC"}
1
+ {"version":3,"file":"queue-manager.js","sourceRoot":"","sources":["../../src/crawler/queue-manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,OAAO,EAA6C,MAAM,SAAS,CAAC;AAC3F,OAAO,EAAE,sBAAsB,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAGxE,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAC;AAExD,MAAM,OAAO,YAAY;IACf,YAAY,GAAwB,IAAI,CAAC;IACzC,SAAS,GAAW,EAAE,CAAC;IACvB,OAAO,GAAkB,EAAE,CAAC;IAC5B,MAAM,CAAU,UAAU,GAAG,EAAE,CAAC;IACxC,wEAAwE;IAChE,UAAU,GAAW,EAAE,CAAC;IAChC,gGAAgG;IACxF,eAAe,GAAW,EAAE,CAAC;IACrC,yDAAyD;IACjD,mBAAmB,GAAW,CAAC,CAAC;IACxC,sDAAsD;IAC9C,uBAAuB,GAAW,CAAC,CAAC;IAE5C,KAAK,CAAC,UAAU,CAAC,GAAW,EAAE,UAAmB;QAC/C,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,CAAC,SAAS,GAAG,sBAAsB,CAAC,GAAG,CAAC,CAAC;QAC7C,IAAI,CAAC,UAAU,GAAG,UAAU,IAAI,EAAE,CAAC;QACnC,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QACxD,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,uBAAuB,GAAG,CAAC,CAAC;QAEjC,MAAM,CAAC,IAAI,CAAC,8DAA8D,IAAI,CAAC,eAAe,uBAAuB,CAAC,CAAC;QACvH,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,MAAM,CAAC,IAAI,CAAC,qEAAqE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QACtG,CAAC;QACD,MAAM,CAAC,KAAK,CAAC,oCAAoC,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QAEnE,mBAAmB;QACnB,IAAI,CAAC,YAAY,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC5D,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;QAC/B,IAAI,CAAC,YAAY,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAE5D,gDAAgD;QAChD,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,GAAG,SAAS,CAAC,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC;QAC1E,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC;YACjC,GAAG,EAAE,QAAQ;YACb,SAAS,EAAE,SAAS,CAAC,QAAQ,GAAG,SAAS,CAAC,MAAM;SACjD,CAAC,CAAC;QAEH,yBAAyB;QACzB,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACnD,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;IACvB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,eAAe,CAAC,OAAe;QACnC,MAAM,IAAI,GAAG,MAAM,uBAAuB,CAAC,OAAO,CAAC,CAAC;QACpD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,CAAC;QACX,CAAC;QAED,OAAO,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,WAAW,CAAC,IAAc;QAC9B,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvB,MAAM,CAAC,IAAI,CAAC,4DAA4D,CAAC,CAAC;YAC1E,OAAO,CAAC,CAAC;QACX,CAAC;QAED,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,KAAK,MAAM,MAAM,IAAI,IAAI,EAAE,CAAC;YAC1B,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;gBAE/B,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAC7C,SAAS;gBACX,CAAC;gBAED,IAAI,IAAI,CAAC,UAAU,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;oBACxE,SAAS;gBACX,CAAC;gBAED,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC;gBACtE,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC;gBAElD,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,aAAa,EAAE,SAAS,EAAE,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC5F,KAAK,EAAE,CAAC;YACV,CAAC;YACD,MAAM,CAAC;gBACL,MAAM,CAAC,KAAK,CAAC,6CAA6C,MAAM,EAAE,CAAC,CAAC;YACtE,CAAC;QACH,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,yBAAyB,KAAK,wBAAwB,IAAI,CAAC,MAAM,GAAG,KAAK,YAAY,CAAC,CAAC;QACnG,OAAO,KAAK,CAAC;IACf,CAAC;IAED,sBAAsB;QACpB,OAAO,IAAI,CAAC,mBAAmB,CAAC;IAClC,CAAC;IAED,0BAA0B;QACxB,OAAO,IAAI,CAAC,uBAAuB,CAAC;IACtC,CAAC;IAED;;;;;;;;;;;OAWG;IACK,iBAAiB,CAAC,QAAgB;QACxC,MAAM,CAAC,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;QACjC,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC;QAErC,cAAc;QACd,IAAI,CAAC,KAAK,OAAO,EAAE,CAAC;YAClB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,8FAA8F;QAC9F,IAAI,CAAC,CAAC,QAAQ,CAAC,GAAG,GAAG,OAAO,CAAC,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED,KAAK,CAAC,mBAAmB,CACvB,YAAuG,EACvG,GAAQ,EACR,IAAuB;QAEvB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,YAAa,CAAC,OAAO,EAAE,CAAC;QACrD,IAAI,SAAS,EAAE,CAAC;YACd,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE;gBACxB,YAAY,EAAE,SAAS,CAAC,mBAAmB,IAAI,CAAC;gBAChD,YAAY,EAAE,SAAS,CAAC,mBAAmB,IAAI,CAAC;gBAChD,UAAU,EAAE,SAAS,CAAC,iBAAiB,IAAI,CAAC;aAC7C,CAAC,CAAC;QACL,CAAC;QAED,wDAAwD;QACxD,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QACnC,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC;QAC7C,MAAM,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5D,MAAM,uBAAuB,GAAG,GAAG,EAAE;YACnC,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC7B,CAAC,CAAC;QACF,MAAM,2BAA2B,GAAG,GAAG,EAAE;YACvC,IAAI,CAAC,uBAAuB,EAAE,CAAC;QACjC,CAAC,CAAC;QAEF,MAAM,cAAc,GAAwB;YAC1C,QAAQ,EAAE,aAAgC;YAC1C,wBAAwB,CAAC,GAAG;gBAC1B,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAE7B,oDAAoD;gBACpD,4DAA4D;gBAC5D,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;oBACb,MAAM,CAAC,KAAK,CAAC,wCAAwC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;oBAChE,OAAO,KAAK,CAAC;gBACf,CAAC;gBAED,yEAAyE;gBACzE,+GAA+G;gBAC/G,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;oBACrC,MAAM,CAAC,KAAK,CAAC,wDAAwD,GAAG,CAAC,QAAQ,cAAc,eAAe,GAAG,CAAC,CAAC;oBACnH,2BAA2B,EAAE,CAAC;oBAC9B,OAAO,KAAK,CAAC;gBACf,CAAC;gBAED,oDAAoD;gBACpD,iFAAiF;gBACjF,IAAI,UAAU,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,EAAE,CAAC;oBAC3D,MAAM,CAAC,KAAK,CAAC,oDAAoD,GAAG,CAAC,QAAQ,aAAa,UAAU,GAAG,CAAC,CAAC;oBACzG,uBAAuB,EAAE,CAAC;oBAC1B,OAAO,KAAK,CAAC;gBACf,CAAC;gBAED,yEAAyE;gBACzE,OAAO;oBACL,GAAG,GAAG;oBACN,GAAG,EAAE,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,MAAM,EAAE,sBAAsB;oBACnE,SAAS,EAAE,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,MAAM;iBACrC,CAAC;YACJ,CAAC;SACF,CAAC;QAEF,+CAA+C;QAC/C,IAAI,IAAI,CAAC,aAAa,EAAE,MAAM,EAAE,CAAC;YAC/B,cAAc,CAAC,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1D,CAAC;QAED,MAAM,aAAa,GAAG,MAAM,YAAY,CAAC,cAAc,CAAC,CAAC;QAEzD,GAAG,CAAC,IAAI,CAAC,iBAAiB,EAAE;YAC1B,cAAc,EAAE,aAAa,CAAC,iBAAiB,CAAC,MAAM;YACtD,IAAI,EAAE,aAAa,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAwB,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;YACpF,GAAG,CAAC,IAAI,CAAC,uBAAuB,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,IAAI,CAAC,uBAAuB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjG,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,cAAc,EAAE,IAAI,CAAC,mBAAmB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACtG,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,YAAY;QAChB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9B,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACnD,MAAM,gBAAgB,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC;QAC3C,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAElB,iDAAiD;QACjD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;YACpD,MAAM,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC3D,CAAC;QAED,OAAO,gBAAgB,CAAC;IAC1B,CAAC;IAED,SAAS,CAAC,MAAmB;QAC3B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC5B,CAAC;IAED,gBAAgB;QACd,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,YAAY,CAAC,UAAU,CAAC;IACxD,CAAC;IAED,eAAe;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAClB,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,GAAG,CAAC,CAAC,CAAC;YAClG,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QAC3B,CAAC;IACH,CAAC"}