@cosmocoder/mcp-web-docs 2.0.20 → 2.0.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/config.d.ts +0 -18
- package/build/config.js +0 -20
- package/build/config.js.map +1 -1
- package/build/config.test.js +1 -17
- package/build/config.test.js.map +1 -1
- package/build/crawler/base.d.ts +2 -4
- package/build/crawler/base.js +1 -5
- package/build/crawler/base.js.map +1 -1
- package/build/crawler/base.test.js +4 -16
- package/build/crawler/base.test.js.map +1 -1
- package/build/crawler/docs-crawler.d.ts +4 -6
- package/build/crawler/docs-crawler.js +5 -10
- package/build/crawler/docs-crawler.js.map +1 -1
- package/build/crawler/docs-crawler.test.js +18 -50
- package/build/crawler/docs-crawler.test.js.map +1 -1
- package/build/crawler/github.d.ts +1 -1
- package/build/crawler/github.js +2 -2
- package/build/crawler/github.js.map +1 -1
- package/build/crawler/github.test.js +1 -20
- package/build/crawler/github.test.js.map +1 -1
- package/build/index.js +104 -62
- package/build/index.js.map +1 -1
- package/build/index.test.js +147 -15
- package/build/index.test.js.map +1 -1
- package/build/indexing/queue-manager.d.ts +2 -0
- package/build/indexing/queue-manager.js +12 -18
- package/build/indexing/queue-manager.js.map +1 -1
- package/build/indexing/queue-manager.test.js +5 -12
- package/build/indexing/queue-manager.test.js.map +1 -1
- package/build/types.d.ts +0 -15
- package/package.json +1 -1
package/build/config.d.ts
CHANGED
|
@@ -1,9 +1,6 @@
|
|
|
1
1
|
export interface DocsConfig {
|
|
2
2
|
githubToken?: string;
|
|
3
|
-
maxDepth: number;
|
|
4
|
-
maxRequestsPerCrawl: number;
|
|
5
3
|
maxChunkSize: number;
|
|
6
|
-
maxConcurrentRequests: number;
|
|
7
4
|
cacheSize: number;
|
|
8
5
|
dataDir: string;
|
|
9
6
|
dbPath: string;
|
|
@@ -16,21 +13,6 @@ export declare const RATE_LIMIT: {
|
|
|
16
13
|
timeWindow: number;
|
|
17
14
|
minDelay: number;
|
|
18
15
|
};
|
|
19
|
-
export declare const QUEUE_OPTIONS: {
|
|
20
|
-
maxRequestRetries: number;
|
|
21
|
-
retryDelay: number;
|
|
22
|
-
maxRequestsPerCrawl: number;
|
|
23
|
-
};
|
|
24
|
-
export declare const GITHUB_RATE_LIMIT: {
|
|
25
|
-
unauthenticated: {
|
|
26
|
-
maxRequests: number;
|
|
27
|
-
timeWindow: number;
|
|
28
|
-
};
|
|
29
|
-
authenticated: {
|
|
30
|
-
maxRequests: number;
|
|
31
|
-
timeWindow: number;
|
|
32
|
-
};
|
|
33
|
-
};
|
|
34
16
|
export declare function isValidUrl(urlString: string): boolean;
|
|
35
17
|
/**
|
|
36
18
|
* Validate URL and check for SSRF attacks (blocks private/internal networks)
|
package/build/config.js
CHANGED
|
@@ -5,10 +5,7 @@ import { logger } from './util/logger.js';
|
|
|
5
5
|
import { validatePublicUrl } from './util/security.js';
|
|
6
6
|
const DATA_DIR = join(homedir(), '.mcp-web-docs');
|
|
7
7
|
const DEFAULT_CONFIG = {
|
|
8
|
-
maxDepth: 4,
|
|
9
|
-
maxRequestsPerCrawl: 1000, // Match DocsCrawler default for better coverage
|
|
10
8
|
maxChunkSize: 1000,
|
|
11
|
-
maxConcurrentRequests: 3, // Allow concurrent requests for better performance while maintaining stability
|
|
12
9
|
cacheSize: 1000,
|
|
13
10
|
dataDir: DATA_DIR,
|
|
14
11
|
dbPath: join(DATA_DIR, 'docs.db'),
|
|
@@ -80,23 +77,6 @@ export const RATE_LIMIT = {
|
|
|
80
77
|
timeWindow: 60 * 1000, // 1 minute
|
|
81
78
|
minDelay: 250, // Reduced delay between requests
|
|
82
79
|
};
|
|
83
|
-
// Queue configuration
|
|
84
|
-
export const QUEUE_OPTIONS = {
|
|
85
|
-
maxRequestRetries: 2,
|
|
86
|
-
retryDelay: 1000,
|
|
87
|
-
maxRequestsPerCrawl: 2000, // Increased from default 1000
|
|
88
|
-
};
|
|
89
|
-
// GitHub API rate limits
|
|
90
|
-
export const GITHUB_RATE_LIMIT = {
|
|
91
|
-
unauthenticated: {
|
|
92
|
-
maxRequests: 60,
|
|
93
|
-
timeWindow: 60 * 60 * 1000, // 1 hour
|
|
94
|
-
},
|
|
95
|
-
authenticated: {
|
|
96
|
-
maxRequests: 5000,
|
|
97
|
-
timeWindow: 60 * 60 * 1000, // 1 hour
|
|
98
|
-
},
|
|
99
|
-
};
|
|
100
80
|
// Utility function to validate URLs
|
|
101
81
|
export function isValidUrl(urlString) {
|
|
102
82
|
try {
|
package/build/config.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC;AAC7B,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC;AAC7B,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAWvD,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,eAAe,CAAC,CAAC;AAElD,MAAM,cAAc,GAAe;IACjC,YAAY,EAAE,IAAI;IAClB,SAAS,EAAE,IAAI;IACf,OAAO,EAAE,QAAQ;IACjB,MAAM,EAAE,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC;IACjC,YAAY,EAAE,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC;CACxC,CAAC;AAEF,MAAM,CAAC,KAAK,UAAU,UAAU;IAC9B,MAAM,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;IAE/C,+CAA+C;IAC/C,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;IAC7C,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;IAC9C,CAAC;IAED,0DAA0D;IAC1D,IAAI,CAAC;QACH,MAAM,CAAC,KAAK,CAAC,qCAAqC,QAAQ,EAAE,CAAC,CAAC;QAC9D,MAAM,KAAK,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAE3C,gCAAgC;QAChC,MAAM,iBAAiB,GAAG,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QACpD,OAAO,CAAC,GAAG,CAAC,mBAAmB,GAAG,iBAAiB,CAAC;QACpD,MAAM,KAAK,CAAC,iBAAiB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtD,CAAC;IACD,OAAO,KAAK,EAAE,CAAC;QACb,MAAM,CAAC,KAAK,CAAC,yCAAyC,EAAE,KAAK,CAAC,CAAC;QAC/D,MAAM,KAAK,CAAC;IACd,CAAC;IAED,MAAM,MAAM,GAAe;QACzB,GAAG,cAAc;QACjB,WAAW;KACZ,CAAC;IAEF,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE;QAC7C,GAAG,MAAM;QACT,WAAW,EAAE,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;KAC7C,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,yBAAyB;AACzB,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,aAAa;IACb,YAAY;IACZ,UAAU;IACV,UAAU;IACV,aAAa;IACb,aAAa;IACb,QAAQ;IACR,eAAe;IACf,WAAW;IACX,gBAAgB;IAChB,SAAS;IACT,SAAS;IACT,SAAS;IACT,MAAM;IACN,MAAM;IACN,KAAK;IACL,QAAQ;IACR,oCAAoC;IACpC,eAAe;IACf,SAAS;IACT,OAAO;IACP,QAAQ;IACR,UAAU;IACV,WAAW;IACX,QAAQ;IACR,OAAO;IACP,OAAO;CACR,CAAC;AAEF,0BAA0B;AAC1B,MAAM,CAAC,MAAM,UAAU,GAAG;IACxB,WAAW,EAAE,EAAE,EAAE,kCAAkC;IACnD,UAAU,EAAE,EAAE,GAAG,IAAI,EAAE,WAAW;IAClC,QAAQ,EAAE,GAAG,EAAE,iCAAiC;CACjD,CAAC;AAEF,oCAAoC;AACpC,MAAM,UAAU,UAAU,CAAC,SAAiB;IAC1C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;QAC/B,OAAO,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,CAAC;IAC/D,CAAC;IACD,MAAM,CAAC;QACL,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAAC,SAAiB,EAAE,YAAY,GAAG,KAAK;IACtE,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC3B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,YAAY,EAAE,CAAC;QACjB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CAAC;QACH,iBAAiB,CAAC,SAAS,CAAC,CAAC;QAC7B,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,KAAK,EAAE,CAAC;QACb,MAAM,CAAC,KAAK,CAAC,4CAA4C,SAAS,EAAE,EAAE,KAAK,CAAC,CAAC;QAC7E,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,qCAAqC;AACrC,MAAM,UAAU,YAAY,CAAC,SAAiB;IAC5C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;QAC/B,wBAAwB;QACxB,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAC3C,CAAC;IACD,MAAM,CAAC;QACL,MAAM,IAAI,KAAK,CAAC,gBAAgB,SAAS,EAAE,CAAC,CAAC;IAC/C,CAAC;AACH,CAAC;AAED,4DAA4D;AAC5D,MAAM,UAAU,WAAW,CAAC,SAAiB;IAC3C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;QAC/B,OAAO,GAAG,CAAC,QAAQ,KAAK,YAAY,CAAC;IACvC,CAAC;IACD,MAAM,CAAC;QACL,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}
|
package/build/config.test.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { isValidUrl, isValidPublicUrl, normalizeUrl, isGitHubUrl, IGNORED_PATHS, RATE_LIMIT
|
|
1
|
+
import { isValidUrl, isValidPublicUrl, normalizeUrl, isGitHubUrl, IGNORED_PATHS, RATE_LIMIT } from './config.js';
|
|
2
2
|
describe('Configuration Utilities', () => {
|
|
3
3
|
describe('isValidUrl', () => {
|
|
4
4
|
it('should accept valid HTTP URLs', () => {
|
|
@@ -120,22 +120,6 @@ describe('Configuration Utilities', () => {
|
|
|
120
120
|
expect(RATE_LIMIT.minDelay).toBeGreaterThanOrEqual(0);
|
|
121
121
|
});
|
|
122
122
|
});
|
|
123
|
-
describe('QUEUE_OPTIONS', () => {
|
|
124
|
-
it('should have retry configuration', () => {
|
|
125
|
-
expect(QUEUE_OPTIONS.maxRequestRetries).toBeGreaterThanOrEqual(0);
|
|
126
|
-
expect(QUEUE_OPTIONS.retryDelay).toBeGreaterThan(0);
|
|
127
|
-
expect(QUEUE_OPTIONS.maxRequestsPerCrawl).toBeGreaterThan(0);
|
|
128
|
-
});
|
|
129
|
-
});
|
|
130
|
-
describe('GITHUB_RATE_LIMIT', () => {
|
|
131
|
-
it('should have different limits for authenticated vs unauthenticated', () => {
|
|
132
|
-
expect(GITHUB_RATE_LIMIT.authenticated.maxRequests).toBeGreaterThan(GITHUB_RATE_LIMIT.unauthenticated.maxRequests);
|
|
133
|
-
});
|
|
134
|
-
it('should have time windows defined', () => {
|
|
135
|
-
expect(GITHUB_RATE_LIMIT.authenticated.timeWindow).toBeGreaterThan(0);
|
|
136
|
-
expect(GITHUB_RATE_LIMIT.unauthenticated.timeWindow).toBeGreaterThan(0);
|
|
137
|
-
});
|
|
138
|
-
});
|
|
139
123
|
});
|
|
140
124
|
});
|
|
141
125
|
//# sourceMappingURL=config.test.js.map
|
package/build/config.test.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.test.js","sourceRoot":"","sources":["../src/config.test.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"config.test.js","sourceRoot":"","sources":["../src/config.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,YAAY,EAAE,WAAW,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEjH,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;IACvC,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;QAC1B,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;YACvC,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACpD,MAAM,CAAC,UAAU,CAAC,yBAAyB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzD,MAAM,CAAC,UAAU,CAAC,iCAAiC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjE,MAAM,CAAC,UAAU,CAAC,oCAAoC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;YACxC,MAAM,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrD,MAAM,CAAC,UAAU,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC/D,MAAM,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;YAC1C,MAAM,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACpD,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrD,MAAM,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACtD,MAAM,CAAC,UAAU,CAAC,8BAA8B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;YACpC,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC5C,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC9C,MAAM,CAAC,UAAU,CAAC,yBAAyB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;YAClC,MAAM,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClD,MAAM,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClD,MAAM,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAChC,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;YACzC,MAAM,CAAC,gBAAgB,CAAC,qBAAqB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC3D,MAAM,CAAC,gBAAgB,CAAC,4BAA4B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClE,MAAM,CAAC,gBAAgB,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;YACvD,MAAM,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACzD,MAAM,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACzD,MAAM,CAAC,gBAAgB,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACxD,MAAM,CAAC,gBAAgB,CAAC,oBAAoB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC3D,MAAM,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qDAAqD,EAAE,GAAG,EAAE;YAC7D,MAAM,CAAC,gBAAgB,CAAC,kBAAkB,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC9D,MAAM,CAAC,gBAAgB,CAAC,oBAAoB,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;YAC/D,MAAM,CAAC,gBAAgB,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACxD,MAAM,CAAC,gBAAgB,CAAC,mBAAmB,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,CAAC,gBAAgB,CAAC,wBAAwB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC/D,MAAM,CAAC,gBAAgB,CAAC,iCAAiC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC1E,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;QAC5B,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;YACxC,MAAM,CAAC,YAAY,CAAC,sBAAsB,CAAC,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;YACzE,MAAM,CAAC,YAAY,CAAC,2BAA2B,CAAC,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACrF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;YACxC,MAAM,CAAC,YAAY,CAAC,kCAAkC,CAAC,CAAC,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;YAClG,MAAM,CAAC,YAAY,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;QAC9F,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;YACnD,MAAM,CAAC,YAAY,CAAC,qBAAqB,CAAC,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;YACxE,MAAM,CAAC,YAAY,CAAC,0BAA0B,CAAC,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACpF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;YACvC,MAAM,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;YAC/D,MAAM,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;QACxD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;YAC3C,MAAM,CAAC,YAAY,CAAC,kCAAkC,CAAC,CAAC,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;QACpG,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;YACrC,MAAM,CAAC,WAAW,CAAC,8BAA8B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC/D,MAAM,CAAC,WAAW,CAAC,kDAAkD,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnF,MAAM,CAAC,WAAW,CAAC,oBAAoB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,CAAC,WAAW,CAAC,8BAA8B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChE,MAAM,CAAC,WAAW,CAAC,iCAAiC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnE,MAAM,CAAC,WAAW,CAAC,qBAAqB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;YAC1D,MAAM,CAAC,WAAW,CAAC,mBAAmB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrD,MAAM,CAAC,WAAW,CAAC,wBAAwB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1D,MAAM,CAAC,WAAW,CAAC,uDAAuD,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC3F,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;YAC9C,MAAM,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC7C,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,WAAW,EAAE,GAAG,EAAE;QACzB,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;YAC7B,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;gBACjD,MAAM,CAAC,aAAa,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;gBAC/C,MAAM,CAAC,aAAa,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;gBAC9C,MAAM,CAAC,aAAa,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;gBAC3C,MAAM,CAAC,aAAa,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;YACnD,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;gBACvC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAChD,aAAa,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;oBAC7B,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACrC,CAAC,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;YAC1B,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;gBAC/C,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;gBAClD,MAAM,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;gBACjD,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;YACxD,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
package/build/crawler/base.d.ts
CHANGED
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
import { CrawlResult } from '../types.js';
|
|
2
2
|
export declare abstract class BaseCrawler {
|
|
3
|
-
protected readonly maxDepth: number;
|
|
4
|
-
protected readonly maxRequestsPerCrawl: number;
|
|
5
3
|
protected seenUrls: Set<string>;
|
|
6
4
|
protected isAborting: boolean;
|
|
7
5
|
private readonly abortController;
|
|
@@ -11,11 +9,11 @@ export declare abstract class BaseCrawler {
|
|
|
11
9
|
protected totalUrls: number;
|
|
12
10
|
protected processedUrls: number;
|
|
13
11
|
protected onProgress?: (progress: number, description: string) => void;
|
|
14
|
-
constructor(
|
|
12
|
+
constructor(onProgress?: (progress: number, description: string) => void);
|
|
15
13
|
protected updateProgress(description: string): void;
|
|
16
14
|
protected addDiscoveredUrls(count: number): void;
|
|
17
15
|
protected markUrlProcessed(url: string): void;
|
|
18
|
-
abstract crawl(url: string
|
|
16
|
+
abstract crawl(url: string): AsyncGenerator<CrawlResult, void, unknown>;
|
|
19
17
|
protected shouldCrawl(urlString: string): boolean;
|
|
20
18
|
protected markUrlAsSeen(url: string): void;
|
|
21
19
|
protected getPathFromUrl(urlString: string): string;
|
package/build/crawler/base.js
CHANGED
|
@@ -3,8 +3,6 @@ import { setTimeout as delay } from 'node:timers/promises';
|
|
|
3
3
|
import { IGNORED_PATHS, RATE_LIMIT } from '../config.js';
|
|
4
4
|
import { logger } from '../util/logger.js';
|
|
5
5
|
export class BaseCrawler {
|
|
6
|
-
maxDepth;
|
|
7
|
-
maxRequestsPerCrawl;
|
|
8
6
|
seenUrls;
|
|
9
7
|
isAborting;
|
|
10
8
|
abortController = new AbortController();
|
|
@@ -14,9 +12,7 @@ export class BaseCrawler {
|
|
|
14
12
|
totalUrls;
|
|
15
13
|
processedUrls;
|
|
16
14
|
onProgress;
|
|
17
|
-
constructor(
|
|
18
|
-
this.maxDepth = maxDepth;
|
|
19
|
-
this.maxRequestsPerCrawl = maxRequestsPerCrawl;
|
|
15
|
+
constructor(onProgress) {
|
|
20
16
|
this.seenUrls = new Set();
|
|
21
17
|
this.isAborting = false;
|
|
22
18
|
this.requestCount = 0;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"base.js","sourceRoot":"","sources":["../../src/crawler/base.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAC1B,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAEzD,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,MAAM,OAAgB,WAAW;
|
|
1
|
+
{"version":3,"file":"base.js","sourceRoot":"","sources":["../../src/crawler/base.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAC1B,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAEzD,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,MAAM,OAAgB,WAAW;IACrB,QAAQ,CAAc;IACtB,UAAU,CAAU;IACb,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;IACtC,WAAW,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC;IACrD,YAAY,CAAS;IACrB,eAAe,CAAS;IACtB,SAAS,CAAS;IAClB,aAAa,CAAS;IACtB,UAAU,CAAmD;IAEvE,YAAY,UAA4D;QACtE,IAAI,CAAC,QAAQ,GAAG,IAAI,GAAG,EAAE,CAAC;QAC1B,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QACxB,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;QACtB,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC;QACzB,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,mCAAmC;QACvD,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC;QACvB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;IAES,cAAc,CAAC,WAAmB;QAC1C,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,2CAA2C;YAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC;YACxF,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;QACzC,CAAC;IACH,CAAC;IAES,iBAAiB,CAAC,KAAa;QACvC,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC;QACxB,IAAI,CAAC,cAAc,CAAC,sBAAsB,CAAC,CAAC;IAC9C,CAAC;IAES,gBAAgB,CAAC,GAAW;QACpC,IAAI,CAAC,aAAa,EAAE,CAAC;QACrB,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;QACxB,IAAI,CAAC,cAAc,CAAC,mBAAmB,IAAI,CAAC,aAAa,OAAO,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;IACpF,CAAC;IAIS,WAAW,CAAC,SAAiB;QACrC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;YAE/B,+DAA+D;YAC/D,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;gBACjC,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,gCAAgC,SAAS,EAAE,CAAC,CAAC;gBACnF,OAAO,KAAK,CAAC;YACf,CAAC;YAED,sBAAsB;YACtB,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;gBACb,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,6BAA6B,SAAS,EAAE,CAAC,CAAC;gBAChF,OAAO,KAAK,CAAC;YACf,CAAC;YAED,yDAAyD;YACzD,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,CAAC;YACzD,IAAI,GAAG,IAAI,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;gBAC3C,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,6BAA6B,SAAS,EAAE,CAAC,CAAC;gBAChF,OAAO,KAAK,CAAC;YACf,CAAC;YAED,gDAAgD;YAChD,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;YACxC,MAAM,SAAS,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE;gBAC/C,uDAAuD;gBACvD,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC1B,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;gBAClC,CAAC;gBACD,0BAA0B;gBAC1B,OAAO,IAAI,KAAK,IAAI,OAAO,EAAE,CAAC;YAChC,CAAC,CAAC,CAAC;YAEH,IAAI,SAAS,EAAE,CAAC;gBACd,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,4BAA4B,SAAS,EAAE,CAAC,CAAC;gBAC/E,OAAO,KAAK,CAAC;YACf,CAAC;YAED,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,yBAAyB,SAAS,EAAE,EAAE,KAAK,CAAC,CAAC;YACnF,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAES,aAAa,CAAC,GAAW;QACjC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IAES,cAAc,CAAC,SAAiB;QACxC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;YAC/B,OAAO,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,+BAA+B;QACnE,CAAC;QACD,MAAM,CAAC;YACL,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;IAES,YAAY,CAAC,SAAiB;QACtC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;YAC/B,6CAA6C;YAC7C,GAAG,CAAC,IAAI,GAAG,EAAE,CAAC;YACd,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC3C,CAAC;QACD,MAAM,CAAC;YACL,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;IAES,KAAK,CAAC,SAAS;QACvB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,CAAC,YAAY,EAAE,CAAC;QAEpB,0CAA0C;QAC1C,IAAI,GAAG,GAAG,IAAI,CAAC,eAAe,GAAG,UAAU,CAAC,UAAU,EAAE,CAAC;YACvD,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;YACtB,IAAI,CAAC,eAAe,GAAG,GAAG,CAAC;YAC3B,OAAO;QACT,CAAC;QAED,0DAA0D;QAC1D,IAAI,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC,WAAW,EAAE,CAAC;YAC/C,MAAM,QAAQ,GAAG,UAAU,CAAC,UAAU,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,eAAe,CAAC,CAAC;YACtE,MAAM,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;YAC/D,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;YACtB,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAClC,OAAO;QACT,CAAC;QAED,wCAAwC;QACxC,MAAM,oBAAoB,GAAG,GAAG,GAAG,IAAI,CAAC,eAAe,CAAC;QACxD,IAAI,oBAAoB,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC;YAC/C,MAAM,KAAK,CAAC,UAAU,CAAC,QAAQ,GAAG,oBAAoB,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;QACnG,CAAC;QAED,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACpC,CAAC;IAES,KAAK,CAAC,gBAAgB,CAAI,SAA2B,EAAE,aAAqB,CAAC,EAAE,YAAoB,IAAI;QAC/G,IAAI,SAA4B,CAAC;QAEjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,IAAI,CAAC;gBACH,OAAO,MAAM,SAAS,EAAE,CAAC;YAC3B,CAAC;YACD,OAAO,KAAK,EAAE,CAAC;gBACb,SAAS,GAAG,KAAc,CAAC;gBAC3B,IAAI,CAAC,KAAK,UAAU,GAAG,CAAC,EAAE,CAAC;oBACzB,MAAM;gBACR,CAAC;gBAED,sBAAsB;gBACtB,MAAM,KAAK,GAAG,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;gBACzC,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;QAED,MAAM,SAAS,CAAC;IAClB,CAAC;IAED,KAAK;QACH,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;IAC/B,CAAC;CACF"}
|
|
@@ -62,21 +62,9 @@ describe('BaseCrawler', () => {
|
|
|
62
62
|
crawler = new TestCrawler();
|
|
63
63
|
});
|
|
64
64
|
describe('constructor', () => {
|
|
65
|
-
it('should initialize with default values', () => {
|
|
66
|
-
const defaultCrawler = new TestCrawler();
|
|
67
|
-
expect(defaultCrawler).toBeDefined();
|
|
68
|
-
});
|
|
69
|
-
it('should accept custom maxDepth', () => {
|
|
70
|
-
const customCrawler = new TestCrawler(10);
|
|
71
|
-
expect(customCrawler).toBeDefined();
|
|
72
|
-
});
|
|
73
|
-
it('should accept custom maxRequestsPerCrawl', () => {
|
|
74
|
-
const customCrawler = new TestCrawler(4, 500);
|
|
75
|
-
expect(customCrawler).toBeDefined();
|
|
76
|
-
});
|
|
77
65
|
it('should accept progress callback', () => {
|
|
78
66
|
const progressFn = vi.fn();
|
|
79
|
-
const customCrawler = new TestCrawler(
|
|
67
|
+
const customCrawler = new TestCrawler(progressFn);
|
|
80
68
|
customCrawler.testUpdateProgress('Testing');
|
|
81
69
|
expect(progressFn).toHaveBeenCalled();
|
|
82
70
|
});
|
|
@@ -197,19 +185,19 @@ describe('BaseCrawler', () => {
|
|
|
197
185
|
describe('progress tracking', () => {
|
|
198
186
|
it('should call progress callback with updateProgress', () => {
|
|
199
187
|
const progressFn = vi.fn();
|
|
200
|
-
const progressCrawler = new TestCrawler(
|
|
188
|
+
const progressCrawler = new TestCrawler(progressFn);
|
|
201
189
|
progressCrawler.testUpdateProgress('Processing...');
|
|
202
190
|
expect(progressFn).toHaveBeenCalledWith(expect.any(Number), 'Processing...');
|
|
203
191
|
});
|
|
204
192
|
it('should track discovered URLs', () => {
|
|
205
193
|
const progressFn = vi.fn();
|
|
206
|
-
const progressCrawler = new TestCrawler(
|
|
194
|
+
const progressCrawler = new TestCrawler(progressFn);
|
|
207
195
|
progressCrawler.testAddDiscoveredUrls(5);
|
|
208
196
|
expect(progressFn).toHaveBeenCalled();
|
|
209
197
|
});
|
|
210
198
|
it('should track processed URLs', () => {
|
|
211
199
|
const progressFn = vi.fn();
|
|
212
|
-
const progressCrawler = new TestCrawler(
|
|
200
|
+
const progressCrawler = new TestCrawler(progressFn);
|
|
213
201
|
progressCrawler.testMarkUrlProcessed('https://example.com/page1');
|
|
214
202
|
expect(progressFn).toHaveBeenCalled();
|
|
215
203
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"base.test.js","sourceRoot":"","sources":["../../src/crawler/base.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAGxC;;;;;;;;;;GAUG;AACH,MAAM,WAAY,SAAQ,WAAW;IACnC,6DAA6D;IAC7D,KAAK,CAAC,CAAC,KAAK,CAAC,IAAY;QACvB,MAAM;YACJ,GAAG,EAAE,qBAAqB;YAC1B,IAAI,EAAE,GAAG;YACT,OAAO,EAAE,eAAe;YACxB,aAAa,EAAE,MAAM;YACrB,KAAK,EAAE,MAAM;SACd,CAAC;IACJ,CAAC;IAED,yFAAyF;IAClF,eAAe,CAAC,GAAW;QAChC,OAAO,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAC/B,CAAC;IAEM,iBAAiB,CAAC,GAAW;QAClC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAEM,kBAAkB,CAAC,GAAW;QACnC,OAAO,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;IAClC,CAAC;IAEM,gBAAgB,CAAC,GAAW;QACjC,OAAO,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;IAChC,CAAC;IAEM,KAAK,CAAC,aAAa;QACxB,OAAO,IAAI,CAAC,SAAS,EAAE,CAAC;IAC1B,CAAC;IAEM,KAAK,CAAC,oBAAoB,CAAI,SAA2B,EAAE,UAAmB;QACnF,OAAO,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;IACtD,CAAC;IAEM,SAAS;QACd,IAAI,CAAC,KAAK,EAAE,CAAC;IACf,CAAC;IAED,IAAW,cAAc;QACvB,OAAO,IAAI,CAAC,UAAU,CAAC;IACzB,CAAC;IAEM,kBAAkB,CAAC,WAAmB;QAC3C,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;IACnC,CAAC;IAEM,qBAAqB,CAAC,KAAa;QACxC,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAEM,oBAAoB,CAAC,GAAW;QACrC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;CACF;AAED,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,IAAI,OAAoB,CAAC;IAEzB,UAAU,CAAC,GAAG,EAAE;QACd,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,EAAE,CAAC,
|
|
1
|
+
{"version":3,"file":"base.test.js","sourceRoot":"","sources":["../../src/crawler/base.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAGxC;;;;;;;;;;GAUG;AACH,MAAM,WAAY,SAAQ,WAAW;IACnC,6DAA6D;IAC7D,KAAK,CAAC,CAAC,KAAK,CAAC,IAAY;QACvB,MAAM;YACJ,GAAG,EAAE,qBAAqB;YAC1B,IAAI,EAAE,GAAG;YACT,OAAO,EAAE,eAAe;YACxB,aAAa,EAAE,MAAM;YACrB,KAAK,EAAE,MAAM;SACd,CAAC;IACJ,CAAC;IAED,yFAAyF;IAClF,eAAe,CAAC,GAAW;QAChC,OAAO,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAC/B,CAAC;IAEM,iBAAiB,CAAC,GAAW;QAClC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAEM,kBAAkB,CAAC,GAAW;QACnC,OAAO,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;IAClC,CAAC;IAEM,gBAAgB,CAAC,GAAW;QACjC,OAAO,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;IAChC,CAAC;IAEM,KAAK,CAAC,aAAa;QACxB,OAAO,IAAI,CAAC,SAAS,EAAE,CAAC;IAC1B,CAAC;IAEM,KAAK,CAAC,oBAAoB,CAAI,SAA2B,EAAE,UAAmB;QACnF,OAAO,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;IACtD,CAAC;IAEM,SAAS;QACd,IAAI,CAAC,KAAK,EAAE,CAAC;IACf,CAAC;IAED,IAAW,cAAc;QACvB,OAAO,IAAI,CAAC,UAAU,CAAC;IACzB,CAAC;IAEM,kBAAkB,CAAC,WAAmB;QAC3C,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;IACnC,CAAC;IAEM,qBAAqB,CAAC,KAAa;QACxC,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAEM,oBAAoB,CAAC,GAAW;QACrC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;CACF;AAED,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,IAAI,OAAoB,CAAC;IAEzB,UAAU,CAAC,GAAG,EAAE;QACd,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;YACzC,MAAM,UAAU,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,aAAa,GAAG,IAAI,WAAW,CAAC,UAAU,CAAC,CAAC;YAClD,aAAa,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;YAC5C,MAAM,CAAC,UAAU,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACxC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;YAC3C,0DAA0D;YAC1D,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC5E,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,8BAA8B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC3E,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,8BAA8B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;YACnD,OAAO,CAAC,iBAAiB,CAAC,0BAA0B,CAAC,CAAC;YACtD,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,0BAA0B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC1E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;YAC1D,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,kCAAkC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;YAChD,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC7E,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,6BAA6B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC3E,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC7E,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC/E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;YAC3C,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC5E,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,8BAA8B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6CAA6C,EAAE,GAAG,EAAE;YACrD,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC5E,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,8BAA8B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC3E,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,gCAAgC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;YAC9D,qEAAqE;YACrE,yCAAyC;YACzC,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,0BAA0B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACxE,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,sBAAsB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACtE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;YAC9C,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACzD,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;QAC7B,EAAE,CAAC,yBAAyB,EAAE,GAAG,EAAE;YACjC,MAAM,GAAG,GAAG,+BAA+B,CAAC;YAC5C,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAChD,OAAO,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC;YAC/B,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;YAC1C,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACzF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,mCAAmC,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QACjG,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;YACvD,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACpE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yBAAyB,EAAE,GAAG,EAAE;YACjC,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,qBAAqB,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACpE,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,sBAAsB,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACvE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;QAC5B,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;YACtC,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,kCAAkC,CAAC,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACxG,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;YACtC,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,2BAA2B,CAAC,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACjG,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;YACtC,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,mCAAmC,CAAC,CAAC,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;QAClH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;YACvD,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAClE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;QACrB,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;YAC5C,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC3C,OAAO,CAAC,SAAS,EAAE,CAAC;YACpB,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAChC,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;YAC5D,MAAM,SAAS,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YACvD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC;YAE7D,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC/B,MAAM,CAAC,SAAS,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yBAAyB,EAAE,KAAK,IAAI,EAAE;YACvC,MAAM,SAAS,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,qBAAqB,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YAEhG,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,kBAAkB,CAAC,CAAC,QAAoB,EAAE,EAAE;gBACzE,QAAQ,EAAE,CAAC;gBACX,OAAO,CAA8B,CAAC;YACxC,CAAC,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC;YAE7D,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC/B,MAAM,CAAC,SAAS,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;YAE3C,EAAE,CAAC,eAAe,EAAE,CAAC;QACvB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC9C,MAAM,SAAS,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC;YAE3E,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,kBAAkB,CAAC,CAAC,QAAoB,EAAE,EAAE;gBACzE,QAAQ,EAAE,CAAC;gBACX,OAAO,CAA8B,CAAC;YACxC,CAAC,CAAC,CAAC;YAEH,MAAM,MAAM,CAAC,OAAO,CAAC,oBAAoB,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;YAC7F,MAAM,CAAC,SAAS,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;YAE3C,EAAE,CAAC,eAAe,EAAE,CAAC;QACvB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;QACjC,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;YAC3D,MAAM,UAAU,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,eAAe,GAAG,IAAI,WAAW,CAAC,UAAU,CAAC,CAAC;YAEpD,eAAe,CAAC,kBAAkB,CAAC,eAAe,CAAC,CAAC;YAEpD,MAAM,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,eAAe,CAAC,CAAC;QAC/E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;YACtC,MAAM,UAAU,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,eAAe,GAAG,IAAI,WAAW,CAAC,UAAU,CAAC,CAAC;YAEpD,eAAe,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;YAEzC,MAAM,CAAC,UAAU,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACxC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;YACrC,MAAM,UAAU,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,eAAe,GAAG,IAAI,WAAW,CAAC,UAAU,CAAC,CAAC;YAEpD,eAAe,CAAC,oBAAoB,CAAC,2BAA2B,CAAC,CAAC;YAElE,MAAM,CAAC,UAAU,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACxC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;YAC9D,MAAM,SAAS,GAAG,IAAI,WAAW,EAAE,CAAC;YAEpC,MAAM,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;YACjE,MAAM,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;YAC/D,MAAM,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,oBAAoB,CAAC,qBAAqB,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;QACpF,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;QACrB,EAAE,CAAC,0BAA0B,EAAE,KAAK,IAAI,EAAE;YACxC,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBAChE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC/B,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;YACnD,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -1,8 +1,6 @@
|
|
|
1
|
-
import { CrawlResult
|
|
1
|
+
import { CrawlResult } from '../types.js';
|
|
2
2
|
import { StorageState } from './crawlee-crawler.js';
|
|
3
|
-
export declare class DocsCrawler
|
|
4
|
-
private readonly maxDepth;
|
|
5
|
-
private readonly maxRequestsPerCrawl;
|
|
3
|
+
export declare class DocsCrawler {
|
|
6
4
|
private readonly githubToken?;
|
|
7
5
|
private readonly onProgress?;
|
|
8
6
|
private readonly GITHUB_HOST;
|
|
@@ -10,7 +8,7 @@ export declare class DocsCrawler implements WebCrawler {
|
|
|
10
8
|
private storageState?;
|
|
11
9
|
private pathPrefix?;
|
|
12
10
|
private activeCrawler?;
|
|
13
|
-
constructor(
|
|
11
|
+
constructor(githubToken?: string | undefined, onProgress?: ((progress: number, description: string) => void) | undefined);
|
|
14
12
|
/**
|
|
15
13
|
* Set an optional path prefix to restrict crawling to URLs under this path.
|
|
16
14
|
* Only pages whose path starts with this prefix will be crawled.
|
|
@@ -21,6 +19,6 @@ export declare class DocsCrawler implements WebCrawler {
|
|
|
21
19
|
* Set authentication storage state (cookies) to use when crawling
|
|
22
20
|
*/
|
|
23
21
|
setStorageState(state: StorageState): void;
|
|
24
|
-
crawl(url: string): AsyncGenerator<CrawlResult,
|
|
22
|
+
crawl(url: string): AsyncGenerator<CrawlResult, void, unknown>;
|
|
25
23
|
abort(): void;
|
|
26
24
|
}
|
|
@@ -3,8 +3,6 @@ import { CrawleeCrawler } from './crawlee-crawler.js';
|
|
|
3
3
|
import { GitHubCrawler } from './github.js';
|
|
4
4
|
import { logger } from '../util/logger.js';
|
|
5
5
|
export class DocsCrawler {
|
|
6
|
-
maxDepth;
|
|
7
|
-
maxRequestsPerCrawl;
|
|
8
6
|
githubToken;
|
|
9
7
|
onProgress;
|
|
10
8
|
GITHUB_HOST = 'github.com';
|
|
@@ -12,9 +10,7 @@ export class DocsCrawler {
|
|
|
12
10
|
storageState;
|
|
13
11
|
pathPrefix;
|
|
14
12
|
activeCrawler;
|
|
15
|
-
constructor(
|
|
16
|
-
this.maxDepth = maxDepth;
|
|
17
|
-
this.maxRequestsPerCrawl = maxRequestsPerCrawl;
|
|
13
|
+
constructor(githubToken, onProgress) {
|
|
18
14
|
this.githubToken = githubToken;
|
|
19
15
|
this.onProgress = onProgress;
|
|
20
16
|
}
|
|
@@ -39,12 +35,12 @@ export class DocsCrawler {
|
|
|
39
35
|
logger.debug(`[DocsCrawler] Starting crawl of ${startUrl}`);
|
|
40
36
|
if (this.isAborting) {
|
|
41
37
|
logger.debug('[DocsCrawler] Crawl aborted');
|
|
42
|
-
return
|
|
38
|
+
return;
|
|
43
39
|
}
|
|
44
40
|
// Handle GitHub repositories
|
|
45
41
|
if (startUrl.host === this.GITHUB_HOST) {
|
|
46
42
|
logger.debug('[DocsCrawler] Detected GitHub repository');
|
|
47
|
-
const githubCrawler = new GitHubCrawler(this.
|
|
43
|
+
const githubCrawler = new GitHubCrawler(this.githubToken, this.onProgress);
|
|
48
44
|
this.activeCrawler = githubCrawler;
|
|
49
45
|
try {
|
|
50
46
|
for await (const page of githubCrawler.crawl(url)) {
|
|
@@ -53,7 +49,7 @@ export class DocsCrawler {
|
|
|
53
49
|
}
|
|
54
50
|
yield page;
|
|
55
51
|
}
|
|
56
|
-
return
|
|
52
|
+
return;
|
|
57
53
|
}
|
|
58
54
|
catch (e) {
|
|
59
55
|
logger.debug('[DocsCrawler] GitHub crawler failed:', e);
|
|
@@ -68,7 +64,7 @@ export class DocsCrawler {
|
|
|
68
64
|
}
|
|
69
65
|
// Use Crawlee for all other sites
|
|
70
66
|
logger.debug('[DocsCrawler] Using Crawlee crawler');
|
|
71
|
-
const crawleeCrawler = new CrawleeCrawler(this.
|
|
67
|
+
const crawleeCrawler = new CrawleeCrawler(this.onProgress);
|
|
72
68
|
this.activeCrawler = crawleeCrawler;
|
|
73
69
|
// Pass authentication if available
|
|
74
70
|
if (this.storageState) {
|
|
@@ -93,7 +89,6 @@ export class DocsCrawler {
|
|
|
93
89
|
else {
|
|
94
90
|
logger.debug(`[DocsCrawler] Crawlee crawler completed (${pageCount} pages)`);
|
|
95
91
|
}
|
|
96
|
-
return 'crawlee';
|
|
97
92
|
}
|
|
98
93
|
catch (e) {
|
|
99
94
|
logger.debug('[DocsCrawler] Crawlee crawler failed:', e);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"docs-crawler.js","sourceRoot":"","sources":["../../src/crawler/docs-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAE1B,OAAO,EAAE,cAAc,EAAgB,MAAM,sBAAsB,CAAC;AACpE,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,MAAM,OAAO,WAAW;IAQH;IACA;
|
|
1
|
+
{"version":3,"file":"docs-crawler.js","sourceRoot":"","sources":["../../src/crawler/docs-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAE1B,OAAO,EAAE,cAAc,EAAgB,MAAM,sBAAsB,CAAC;AACpE,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,MAAM,OAAO,WAAW;IAQH;IACA;IARF,WAAW,GAAG,YAAY,CAAC;IACpC,UAAU,GAAG,KAAK,CAAC;IACnB,YAAY,CAAgB;IAC5B,UAAU,CAAU;IACpB,aAAa,CAAqB;IAE1C,YACmB,WAAoB,EACpB,UAA4D;QAD5D,gBAAW,GAAX,WAAW,CAAS;QACpB,eAAU,GAAV,UAAU,CAAkD;IAC5E,CAAC;IAEJ;;;;OAIG;IACH,aAAa,CAAC,MAAc;QAC1B,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC;QACzB,MAAM,CAAC,IAAI,CAAC,8CAA8C,MAAM,EAAE,CAAC,CAAC;IACtE,CAAC;IAED;;OAEG;IACH,eAAe,CAAC,KAAmB;QACjC,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,MAAM,CAAC,IAAI,CAAC,wCAAwC,KAAK,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC;IAC5F,CAAC;IAED,KAAK,CAAC,CAAC,KAAK,CAAC,GAAW;QACtB,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC9B,MAAM,CAAC,KAAK,CAAC,mCAAmC,QAAQ,EAAE,CAAC,CAAC;QAE5D,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,MAAM,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;YAC5C,OAAO;QACT,CAAC;QAED,6BAA6B;QAC7B,IAAI,QAAQ,CAAC,IAAI,KAAK,IAAI,CAAC,WAAW,EAAE,CAAC;YACvC,MAAM,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAC;YACzD,MAAM,aAAa,GAAG,IAAI,aAAa,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YAC3E,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;YAEnC,IAAI,CAAC;gBACH,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;oBAClD,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;wBACpB,MAAM;oBACR,CAAC;oBACD,MAAM,IAAI,CAAC;gBACb,CAAC;gBACD,OAAO;YACT,CAAC;YACD,OAAO,CAAC,EAAE,CAAC;gBACT,MAAM,CAAC,KAAK,CAAC,sCAAsC,EAAE,CAAC,CAAC,CAAC;gBACxD,uDAAuD;gBACvD,MAAM,CAAC,CAAC;YACV,CAAC;oBACO,CAAC;gBACP,IAAI,IAAI,CAAC,aAAa,KAAK,aAAa,EAAE,CAAC;oBACzC,IAAI,CAAC,aAAa,GAAG,SAAS,CAAC;gBACjC,CAAC;YACH,CAAC;QACH,CAAC;QAED,kCAAkC;QAClC,MAAM,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACpD,MAAM,cAAc,GAAG,IAAI,cAAc,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC3D,IAAI,CAAC,aAAa,GAAG,cAAc,CAAC;QAEpC,mCAAmC;QACnC,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,cAAc,CAAC,eAAe,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACpD,CAAC;QAED,6CAA6C;QAC7C,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,cAAc,CAAC,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAChD,CAAC;QAED,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,IAAI,CAAC;YACH,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;gBACnD,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;oBACpB,MAAM;gBACR,CAAC;gBACD,SAAS,EAAE,CAAC;gBACZ,MAAM,IAAI,CAAC;YACb,CAAC;YAED,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;gBACpB,MAAM,CAAC,IAAI,CAAC,oDAAoD,GAAG,EAAE,CAAC,CAAC;YACzE,CAAC;iBACI,CAAC;gBACJ,MAAM,CAAC,KAAK,CAAC,4CAA4C,SAAS,SAAS,CAAC,CAAC;YAC/E,CAAC;QACH,CAAC;QACD,OAAO,CAAC,EAAE,CAAC;YACT,MAAM,CAAC,KAAK,CAAC,uCAAuC,EAAE,CAAC,CAAC,CAAC;YACzD,MAAM,CAAC,CAAC;QACV,CAAC;gBACO,CAAC;YACP,IAAI,IAAI,CAAC,aAAa,KAAK,cAAc,EAAE,CAAC;gBAC1C,IAAI,CAAC,aAAa,GAAG,SAAS,CAAC;YACjC,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK;QACH,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,IAAI,CAAC,aAAa,EAAE,KAAK,EAAE,CAAC;IAC9B,CAAC;CACF"}
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import { DocsCrawler } from './docs-crawler.js';
|
|
2
2
|
const mockGitHubCrawl = vi.fn();
|
|
3
3
|
const mockGitHubAbort = vi.fn();
|
|
4
|
+
const mockGitHubConstructor = vi.fn();
|
|
4
5
|
vi.mock('./github.js', () => ({
|
|
5
|
-
GitHubCrawler: function () {
|
|
6
|
+
GitHubCrawler: function (...args) {
|
|
7
|
+
mockGitHubConstructor(...args);
|
|
6
8
|
return {
|
|
7
9
|
crawl: mockGitHubCrawl,
|
|
8
10
|
abort: mockGitHubAbort,
|
|
@@ -11,10 +13,12 @@ vi.mock('./github.js', () => ({
|
|
|
11
13
|
}));
|
|
12
14
|
const mockCrawleeCrawl = vi.fn();
|
|
13
15
|
const mockCrawleeAbort = vi.fn();
|
|
16
|
+
const mockCrawleeConstructor = vi.fn();
|
|
14
17
|
const mockSetStorageState = vi.fn();
|
|
15
18
|
const mockSetPathPrefix = vi.fn();
|
|
16
19
|
vi.mock('./crawlee-crawler.js', () => ({
|
|
17
|
-
CrawleeCrawler: function () {
|
|
20
|
+
CrawleeCrawler: function (...args) {
|
|
21
|
+
mockCrawleeConstructor(...args);
|
|
18
22
|
return {
|
|
19
23
|
crawl: mockCrawleeCrawl,
|
|
20
24
|
abort: mockCrawleeAbort,
|
|
@@ -29,14 +33,19 @@ describe('DocsCrawler', () => {
|
|
|
29
33
|
vi.clearAllMocks();
|
|
30
34
|
crawler = new DocsCrawler();
|
|
31
35
|
});
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
36
|
+
it('passes the GitHub token and progress callback to the selected crawler', async () => {
|
|
37
|
+
const progressFn = vi.fn();
|
|
38
|
+
const configuredCrawler = new DocsCrawler('github_token', progressFn);
|
|
39
|
+
mockGitHubCrawl.mockImplementation(async function* () {
|
|
40
|
+
yield* [];
|
|
35
41
|
});
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
expect(customCrawler).toBeDefined();
|
|
42
|
+
mockCrawleeCrawl.mockImplementation(async function* () {
|
|
43
|
+
yield* [];
|
|
39
44
|
});
|
|
45
|
+
await configuredCrawler.crawl('https://github.com/owner/repo').next();
|
|
46
|
+
await configuredCrawler.crawl('https://example.com').next();
|
|
47
|
+
expect(mockGitHubConstructor).toHaveBeenCalledWith('github_token', progressFn);
|
|
48
|
+
expect(mockCrawleeConstructor).toHaveBeenCalledWith(progressFn);
|
|
40
49
|
});
|
|
41
50
|
describe('crawl', () => {
|
|
42
51
|
describe('GitHub URLs', () => {
|
|
@@ -63,19 +72,6 @@ describe('DocsCrawler', () => {
|
|
|
63
72
|
expect(results).toHaveLength(1);
|
|
64
73
|
expect(results[0].url).toContain('github.com');
|
|
65
74
|
});
|
|
66
|
-
it('should return github type for GitHub URLs', async () => {
|
|
67
|
-
mockGitHubCrawl.mockImplementation(async function* () {
|
|
68
|
-
yield { url: 'https://github.com/owner/repo', path: '/', content: 'test', contentFormat: 'markdown', title: 'Test' };
|
|
69
|
-
});
|
|
70
|
-
const generator = crawler.crawl('https://github.com/owner/repo');
|
|
71
|
-
// Manually iterate to capture the return value
|
|
72
|
-
let result = await generator.next();
|
|
73
|
-
while (!result.done) {
|
|
74
|
-
result = await generator.next();
|
|
75
|
-
}
|
|
76
|
-
const crawlerType = result.value;
|
|
77
|
-
expect(crawlerType).toBe('github');
|
|
78
|
-
});
|
|
79
75
|
it('should propagate errors from GitHubCrawler', async () => {
|
|
80
76
|
// eslint-disable-next-line require-yield
|
|
81
77
|
mockGitHubCrawl.mockImplementation(async function* () {
|
|
@@ -107,33 +103,6 @@ describe('DocsCrawler', () => {
|
|
|
107
103
|
}
|
|
108
104
|
expect(results).toHaveLength(2);
|
|
109
105
|
});
|
|
110
|
-
it('should return crawlee type regardless of page count', async () => {
|
|
111
|
-
mockCrawleeCrawl.mockImplementation(async function* () {
|
|
112
|
-
yield { url: 'https://example.com/page1', path: '/page1', content: 'Page 1', contentFormat: 'text', title: 'Page 1' };
|
|
113
|
-
});
|
|
114
|
-
const results = [];
|
|
115
|
-
const generator = crawler.crawl('https://example.com');
|
|
116
|
-
let result = await generator.next();
|
|
117
|
-
while (!result.done) {
|
|
118
|
-
results.push(result.value);
|
|
119
|
-
result = await generator.next();
|
|
120
|
-
}
|
|
121
|
-
const crawlerType = result.value;
|
|
122
|
-
expect(results).toHaveLength(1);
|
|
123
|
-
expect(crawlerType).toBe('crawlee');
|
|
124
|
-
});
|
|
125
|
-
it('should return crawlee type even with 0 pages found', async () => {
|
|
126
|
-
mockCrawleeCrawl.mockImplementation(() => ({
|
|
127
|
-
[Symbol.asyncIterator]: () => ({ next: () => Promise.resolve({ done: true, value: undefined }) }),
|
|
128
|
-
}));
|
|
129
|
-
const generator = crawler.crawl('https://example.com');
|
|
130
|
-
let result = await generator.next();
|
|
131
|
-
while (!result.done) {
|
|
132
|
-
result = await generator.next();
|
|
133
|
-
}
|
|
134
|
-
const crawlerType = result.value;
|
|
135
|
-
expect(crawlerType).toBe('crawlee');
|
|
136
|
-
});
|
|
137
106
|
});
|
|
138
107
|
describe('abort', () => {
|
|
139
108
|
it.each([
|
|
@@ -173,9 +142,8 @@ describe('DocsCrawler', () => {
|
|
|
173
142
|
});
|
|
174
143
|
const generator = crawler.crawl('https://example.com');
|
|
175
144
|
const result = await generator.next();
|
|
176
|
-
// Should return immediately with crawlee type when aborted
|
|
177
145
|
expect(result.done).toBe(true);
|
|
178
|
-
expect(
|
|
146
|
+
expect(mockCrawleeCrawl).not.toHaveBeenCalled();
|
|
179
147
|
});
|
|
180
148
|
});
|
|
181
149
|
describe('setStorageState', () => {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"docs-crawler.test.js","sourceRoot":"","sources":["../../src/crawler/docs-crawler.test.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,eAAe,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;AAChC,MAAM,eAAe,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;AAChC,EAAE,CAAC,IAAI,CAAC,aAAa,EAAE,GAAG,EAAE,CAAC,CAAC;IAC5B,aAAa,EAAE;
|
|
1
|
+
{"version":3,"file":"docs-crawler.test.js","sourceRoot":"","sources":["../../src/crawler/docs-crawler.test.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,eAAe,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;AAChC,MAAM,eAAe,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;AAChC,MAAM,qBAAqB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;AACtC,EAAE,CAAC,IAAI,CAAC,aAAa,EAAE,GAAG,EAAE,CAAC,CAAC;IAC5B,aAAa,EAAE,UAAU,GAAG,IAAe;QACzC,qBAAqB,CAAC,GAAG,IAAI,CAAC,CAAC;QAC/B,OAAO;YACL,KAAK,EAAE,eAAe;YACtB,KAAK,EAAE,eAAe;SACvB,CAAC;IACJ,CAAC;CACF,CAAC,CAAC,CAAC;AAEJ,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;AACjC,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;AACjC,MAAM,sBAAsB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;AACvC,MAAM,mBAAmB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;AACpC,MAAM,iBAAiB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;AAClC,EAAE,CAAC,IAAI,CAAC,sBAAsB,EAAE,GAAG,EAAE,CAAC,CAAC;IACrC,cAAc,EAAE,UAAU,GAAG,IAAe;QAC1C,sBAAsB,CAAC,GAAG,IAAI,CAAC,CAAC;QAChC,OAAO;YACL,KAAK,EAAE,gBAAgB;YACvB,KAAK,EAAE,gBAAgB;YACvB,eAAe,EAAE,mBAAmB;YACpC,aAAa,EAAE,iBAAiB;SACjC,CAAC;IACJ,CAAC;CACF,CAAC,CAAC,CAAC;AAEJ,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,IAAI,OAAoB,CAAC;IAEzB,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,CAAC,aAAa,EAAE,CAAC;QACnB,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uEAAuE,EAAE,KAAK,IAAI,EAAE;QACrF,MAAM,UAAU,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3B,MAAM,iBAAiB,GAAG,IAAI,WAAW,CAAC,cAAc,EAAE,UAAU,CAAC,CAAC;QACtE,eAAe,CAAC,kBAAkB,CAAC,KAAK,SAAS,CAAC;YAChD,KAAK,CAAC,CAAC,EAAE,CAAC;QACZ,CAAC,CAAC,CAAC;QACH,gBAAgB,CAAC,kBAAkB,CAAC,KAAK,SAAS,CAAC;YACjD,KAAK,CAAC,CAAC,EAAE,CAAC;QACZ,CAAC,CAAC,CAAC;QAEH,MAAM,iBAAiB,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC,IAAI,EAAE,CAAC;QACtE,MAAM,iBAAiB,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC,IAAI,EAAE,CAAC;QAE5D,MAAM,CAAC,qBAAqB,CAAC,CAAC,oBAAoB,CAAC,cAAc,EAAE,UAAU,CAAC,CAAC;QAC/E,MAAM,CAAC,sBAAsB,CAAC,CAAC,oBAAoB,CAAC,UAAU,CAAC,CAAC;IAClE,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;QACrB,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;YAC3B,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;gBAC5D,MAAM,WAAW,GAAkB;oBACjC;wBACE,GAAG,EAAE,yCAAyC;wBAC9C,IAAI,EAAE,WAAW;wBACjB,OAAO,EAAE,UAAU;wBACnB,aAAa,EAAE,UAAU;wBACzB,KAAK,EAAE,QAAQ;qBAChB;iBACF,CAAC;gBAEF,eAAe,CAAC,kBAAkB,CAAC,KAAK,SAAS,CAAC;oBAChD,KAAK,MAAM,MAAM,IAAI,WAAW,EAAE,CAAC;wBACjC,MAAM,MAAM,CAAC;oBACf,CAAC;gBACH,CAAC,CAAC,CAAC;gBAEH,MAAM,OAAO,GAAkB,EAAE,CAAC;gBAClC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;gBAEjE,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,SAAS,EAAE,CAAC;oBACrC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACvB,CAAC;gBAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;gBAChC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;YACjD,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;gBAC1D,yCAAyC;gBACzC,eAAe,CAAC,kBAAkB,CAAC,KAAK,SAAS,CAAC;oBAChD,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;gBACtC,CAAC,CAAC,CAAC;gBAEH,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;gBAEjE,MAAM,MAAM,CAAC,KAAK,IAAI,EAAE;oBACtB,6DAA6D;oBAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;wBAChC,uBAAuB;oBACzB,CAAC;gBACH,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;YACzC,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;YAC/B,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;gBAC7D,MAAM,WAAW,GAAkB;oBACjC,EAAE,GAAG,EAAE,gCAAgC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE;oBAClH,EAAE,GAAG,EAAE,8BAA8B,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE;iBAC3G,CAAC;gBAEF,gBAAgB,CAAC,kBAAkB,CAAC,KAAK,SAAS,CAAC;oBACjD,KAAK,MAAM,MAAM,IAAI,WAAW,EAAE,CAAC;wBACjC,MAAM,MAAM,CAAC;oBACf,CAAC;gBACH,CAAC,CAAC,CAAC;gBAEH,MAAM,OAAO,GAAkB,EAAE,CAAC;gBAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,0BAA0B,CAAC,EAAE,CAAC;oBACrE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACvB,CAAC;gBAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAClC,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;YACrB,EAAE,CAAC,IAAI,CAAC;gBACN;oBACE,IAAI,EAAE,QAAQ;oBACd,GAAG,EAAE,+BAA+B;oBACpC,KAAK,EAAE,eAAe;oBACtB,KAAK,EAAE,eAAe;oBACtB,aAAa,EAAE,UAAmB;iBACnC;gBACD;oBACE,IAAI,EAAE,SAAS;oBACf,GAAG,EAAE,qBAAqB;oBAC1B,KAAK,EAAE,gBAAgB;oBACvB,KAAK,EAAE,gBAAgB;oBACvB,aAAa,EAAE,MAAe;iBAC/B;aACF,CAAC,CAAC,6DAA6D,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,aAAa,EAAE,EAAE,EAAE;gBAC/G,MAAM,OAAO,GAAG,OAAO,CAAC,aAAa,EAAQ,CAAC;gBAC9C,MAAM,QAAQ,GAAG,OAAO,CAAC,aAAa,EAAQ,CAAC;gBAC/C,KAAK,CAAC,kBAAkB,CAAC,KAAK,SAAS,CAAC;oBACtC,OAAO,CAAC,OAAO,EAAE,CAAC;oBAClB,MAAM,QAAQ,CAAC,OAAO,CAAC;oBACvB,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;gBAC1E,CAAC,CAAC,CAAC;gBAEH,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;gBACvC,MAAM,OAAO,CAAC,OAAO,CAAC;gBACtB,OAAO,CAAC,KAAK,EAAE,CAAC;gBAEhB,MAAM,CAAC,KAAK,CAAC,CAAC,oBAAoB,EAAE,CAAC;gBACrC,QAAQ,CAAC,OAAO,EAAE,CAAC;gBACnB,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YAC5D,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;gBACzD,OAAO,CAAC,KAAK,EAAE,CAAC;gBAEhB,gBAAgB,CAAC,kBAAkB,CAAC,KAAK,SAAS,CAAC;oBACjD,MAAM,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;gBACxH,CAAC,CAAC,CAAC;gBAEH,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;gBACvD,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;gBAEtC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC/B,MAAM,CAAC,gBAAgB,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;YAClD,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;YAC/B,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;gBAClC,MAAM,YAAY,GAAG;oBACnB,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;iBAClF,CAAC;gBAEF,OAAO,CAAC,eAAe,CAAC,YAAY,CAAC,CAAC;gBAEtC,kCAAkC;gBAClC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC1B,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;gBAC3D,MAAM,YAAY,GAAG;oBACnB,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;iBAClF,CAAC;gBAEF,OAAO,CAAC,eAAe,CAAC,YAAY,CAAC,CAAC;gBAEtC,gBAAgB,CAAC,kBAAkB,CAAC,KAAK,SAAS,CAAC;oBACjD,MAAM,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;oBACtH,MAAM,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;gBACxH,CAAC,CAAC,CAAC;gBAEH,6DAA6D;gBAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;oBAC3D,uBAAuB;gBACzB,CAAC;gBAED,MAAM,CAAC,mBAAmB,CAAC,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;YACjE,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -5,7 +5,7 @@ export declare class GitHubCrawler extends BaseCrawler {
|
|
|
5
5
|
private readonly API_BASE;
|
|
6
6
|
private readonly MARKDOWN_EXTENSIONS;
|
|
7
7
|
private readonly DOCUMENTATION_PATHS;
|
|
8
|
-
constructor(
|
|
8
|
+
constructor(githubToken?: string | undefined, onProgress?: (progress: number, description: string) => void);
|
|
9
9
|
crawl(url: string): AsyncGenerator<CrawlResult, void, unknown>;
|
|
10
10
|
private parseGitHubUrl;
|
|
11
11
|
private findDocumentationDirs;
|
package/build/crawler/github.js
CHANGED
|
@@ -8,8 +8,8 @@ export class GitHubCrawler extends BaseCrawler {
|
|
|
8
8
|
API_BASE = 'https://api.github.com';
|
|
9
9
|
MARKDOWN_EXTENSIONS = ['.md', '.mdx', '.markdown'];
|
|
10
10
|
DOCUMENTATION_PATHS = ['docs', 'doc', 'documentation', 'wiki', 'guide', 'guides', 'tutorial', 'tutorials'];
|
|
11
|
-
constructor(
|
|
12
|
-
super(
|
|
11
|
+
constructor(githubToken, onProgress) {
|
|
12
|
+
super(onProgress);
|
|
13
13
|
this.githubToken = githubToken;
|
|
14
14
|
}
|
|
15
15
|
async *crawl(url) {
|