@cosmocoder/mcp-web-docs 2.0.16 → 2.0.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/crawler/queue-manager.d.ts +1 -1
- package/build/crawler/queue-manager.js +8 -8
- package/build/crawler/queue-manager.js.map +1 -1
- package/build/crawler/queue-manager.test.js +26 -12
- package/build/crawler/queue-manager.test.js.map +1 -1
- package/build/index.js +51 -51
- package/build/index.js.map +1 -1
- package/build/index.test.js +90 -61
- package/build/index.test.js.map +1 -1
- package/build/indexing/status.d.ts +8 -8
- package/build/indexing/status.js +40 -38
- package/build/indexing/status.js.map +1 -1
- package/build/indexing/status.test.js +62 -32
- package/build/indexing/status.test.js.map +1 -1
- package/build/types.d.ts +3 -0
- package/build/util/docs.d.ts +1 -0
- package/build/util/docs.js +8 -0
- package/build/util/docs.js.map +1 -1
- package/build/util/docs.test.js +18 -1
- package/build/util/docs.test.js.map +1 -1
- package/package.json +1 -1
|
@@ -3,7 +3,7 @@ import { CrawlResult } from '../types.js';
|
|
|
3
3
|
import { SiteDetectionRule } from './site-rules.js';
|
|
4
4
|
export declare class QueueManager {
|
|
5
5
|
private requestQueue;
|
|
6
|
-
private
|
|
6
|
+
private storageId;
|
|
7
7
|
private results;
|
|
8
8
|
private static readonly BATCH_SIZE;
|
|
9
9
|
/** Optional path prefix to restrict crawling to URLs under this path */
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { RequestQueue, Dataset } from 'crawlee';
|
|
2
|
-
import {
|
|
2
|
+
import { generateCrawlStorageId, isPathAllowed } from '../util/docs.js';
|
|
3
3
|
import { logger } from '../util/logger.js';
|
|
4
4
|
import { discoverUrlsFromLlmsTxt } from './llms-txt.js';
|
|
5
5
|
export class QueueManager {
|
|
6
6
|
requestQueue = null;
|
|
7
|
-
|
|
7
|
+
storageId = '';
|
|
8
8
|
results = [];
|
|
9
9
|
static BATCH_SIZE = 20;
|
|
10
10
|
/** Optional path prefix to restrict crawling to URLs under this path */
|
|
@@ -17,7 +17,7 @@ export class QueueManager {
|
|
|
17
17
|
filteredByHostnameCount = 0;
|
|
18
18
|
async initialize(url, pathPrefix) {
|
|
19
19
|
const parsedUrl = new URL(url);
|
|
20
|
-
this.
|
|
20
|
+
this.storageId = generateCrawlStorageId(url);
|
|
21
21
|
this.pathPrefix = pathPrefix || '';
|
|
22
22
|
this.allowedHostname = parsedUrl.hostname.toLowerCase();
|
|
23
23
|
this.filteredByPathCount = 0;
|
|
@@ -26,11 +26,11 @@ export class QueueManager {
|
|
|
26
26
|
if (this.pathPrefix) {
|
|
27
27
|
logger.info(`[QueueManager] Path restriction enabled: only crawling URLs under ${this.pathPrefix}`);
|
|
28
28
|
}
|
|
29
|
-
logger.debug(`[QueueManager] Using
|
|
29
|
+
logger.debug(`[QueueManager] Using storage ID: ${this.storageId}`);
|
|
30
30
|
// Initialize queue
|
|
31
|
-
this.requestQueue = await RequestQueue.open(this.
|
|
31
|
+
this.requestQueue = await RequestQueue.open(this.storageId);
|
|
32
32
|
await this.requestQueue.drop();
|
|
33
|
-
this.requestQueue = await RequestQueue.open(this.
|
|
33
|
+
this.requestQueue = await RequestQueue.open(this.storageId);
|
|
34
34
|
// Add initial request (strip any hash from URL)
|
|
35
35
|
const cleanUrl = parsedUrl.origin + parsedUrl.pathname + parsedUrl.search;
|
|
36
36
|
await this.requestQueue.addRequest({
|
|
@@ -38,7 +38,7 @@ export class QueueManager {
|
|
|
38
38
|
uniqueKey: parsedUrl.pathname + parsedUrl.search,
|
|
39
39
|
});
|
|
40
40
|
// Clear existing dataset
|
|
41
|
-
const dataset = await Dataset.open(this.
|
|
41
|
+
const dataset = await Dataset.open(this.storageId);
|
|
42
42
|
await dataset.drop();
|
|
43
43
|
}
|
|
44
44
|
/**
|
|
@@ -181,7 +181,7 @@ export class QueueManager {
|
|
|
181
181
|
if (this.results.length === 0) {
|
|
182
182
|
return [];
|
|
183
183
|
}
|
|
184
|
-
const dataset = await Dataset.open(this.
|
|
184
|
+
const dataset = await Dataset.open(this.storageId);
|
|
185
185
|
const resultsToProcess = [...this.results];
|
|
186
186
|
this.results = [];
|
|
187
187
|
// Process in chunks for better memory management
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"queue-manager.js","sourceRoot":"","sources":["../../src/crawler/queue-manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,OAAO,EAA6C,MAAM,SAAS,CAAC;AAC3F,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"queue-manager.js","sourceRoot":"","sources":["../../src/crawler/queue-manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,OAAO,EAA6C,MAAM,SAAS,CAAC;AAC3F,OAAO,EAAE,sBAAsB,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAGxE,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAC;AAExD,MAAM,OAAO,YAAY;IACf,YAAY,GAAwB,IAAI,CAAC;IACzC,SAAS,GAAW,EAAE,CAAC;IACvB,OAAO,GAAkB,EAAE,CAAC;IAC5B,MAAM,CAAU,UAAU,GAAG,EAAE,CAAC;IACxC,wEAAwE;IAChE,UAAU,GAAW,EAAE,CAAC;IAChC,gGAAgG;IACxF,eAAe,GAAW,EAAE,CAAC;IACrC,yDAAyD;IACjD,mBAAmB,GAAW,CAAC,CAAC;IACxC,sDAAsD;IAC9C,uBAAuB,GAAW,CAAC,CAAC;IAE5C,KAAK,CAAC,UAAU,CAAC,GAAW,EAAE,UAAmB;QAC/C,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,CAAC,SAAS,GAAG,sBAAsB,CAAC,GAAG,CAAC,CAAC;QAC7C,IAAI,CAAC,UAAU,GAAG,UAAU,IAAI,EAAE,CAAC;QACnC,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QACxD,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,uBAAuB,GAAG,CAAC,CAAC;QAEjC,MAAM,CAAC,IAAI,CAAC,8DAA8D,IAAI,CAAC,eAAe,uBAAuB,CAAC,CAAC;QACvH,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,MAAM,CAAC,IAAI,CAAC,qEAAqE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QACtG,CAAC;QACD,MAAM,CAAC,KAAK,CAAC,oCAAoC,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QAEnE,mBAAmB;QACnB,IAAI,CAAC,YAAY,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC5D,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;QAC/B,IAAI,CAAC,YAAY,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAE5D,gDAAgD;QAChD,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,GAAG,SAAS,CAAC,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC;QAC1E,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC;YACjC,GAAG,EAAE,QAAQ;YACb,SAAS,EAAE,SAAS,CAAC,QAAQ,GAAG,SAAS,CAAC,MAAM;SACjD,CAAC,CAAC;QAEH,yBAAyB;QACzB,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACnD,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;IACvB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,eAAe,CAAC,OAAe;QACnC,MAAM,IAAI,GAAG,MAAM,uBAAuB,CAAC,OAAO,CAAC,CAAC;QACpD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,CAAC;QACX,CAAC;QAED,OAAO,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,WAAW,CAAC,IAAc;QAC9B,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvB,MAAM,CAAC,IAAI,CAAC,4DAA4D,CAAC,CAAC;YAC1E,OAAO,CAAC,CAAC;QACX,CAAC;QAED,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,KAAK,MAAM,MAAM,IAAI,IAAI,EAAE,CAAC;YAC1B,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;gBAE/B,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAC7C,SAAS;gBACX,CAAC;gBAED,IAAI,IAAI,CAAC,UAAU,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;oBACxE,SAAS;gBACX,CAAC;gBAED,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC;gBACtE,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC;gBAElD,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,aAAa,EAAE,SAAS,EAAE,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC5F,KAAK,EAAE,CAAC;YACV,CAAC;YACD,MAAM,CAAC;gBACL,MAAM,CAAC,KAAK,CAAC,6CAA6C,MAAM,EAAE,CAAC,CAAC;YACtE,CAAC;QACH,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,yBAAyB,KAAK,wBAAwB,IAAI,CAAC,MAAM,GAAG,KAAK,YAAY,CAAC,CAAC;QACnG,OAAO,KAAK,CAAC;IACf,CAAC;IAED,sBAAsB;QACpB,OAAO,IAAI,CAAC,mBAAmB,CAAC;IAClC,CAAC;IAED,0BAA0B;QACxB,OAAO,IAAI,CAAC,uBAAuB,CAAC;IACtC,CAAC;IAED;;;;;;;;;;;OAWG;IACK,iBAAiB,CAAC,QAAgB;QACxC,MAAM,CAAC,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;QACjC,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC;QAErC,cAAc;QACd,IAAI,CAAC,KAAK,OAAO,EAAE,CAAC;YAClB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,8FAA8F;QAC9F,IAAI,CAAC,CAAC,QAAQ,CAAC,GAAG,GAAG,OAAO,CAAC,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED,KAAK,CAAC,mBAAmB,CACvB,YAAuG,EACvG,GAAQ,EACR,IAAuB;QAEvB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,YAAa,CAAC,OAAO,EAAE,CAAC;QACrD,IAAI,SAAS,EAAE,CAAC;YACd,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE;gBACxB,YAAY,EAAE,SAAS,CAAC,mBAAmB,IAAI,CAAC;gBAChD,YAAY,EAAE,SAAS,CAAC,mBAAmB,IAAI,CAAC;gBAChD,UAAU,EAAE,SAAS,CAAC,iBAAiB,IAAI,CAAC;aAC7C,CAAC,CAAC;QACL,CAAC;QAED,wDAAwD;QACxD,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QACnC,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC;QAC7C,MAAM,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5D,MAAM,uBAAuB,GAAG,GAAG,EAAE;YACnC,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC7B,CAAC,CAAC;QACF,MAAM,2BAA2B,GAAG,GAAG,EAAE;YACvC,IAAI,CAAC,uBAAuB,EAAE,CAAC;QACjC,CAAC,CAAC;QAEF,MAAM,cAAc,GAAwB;YAC1C,QAAQ,EAAE,aAAgC;YAC1C,wBAAwB,CAAC,GAAG;gBAC1B,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAE7B,oDAAoD;gBACpD,4DAA4D;gBAC5D,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;oBACb,MAAM,CAAC,KAAK,CAAC,wCAAwC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;oBAChE,OAAO,KAAK,CAAC;gBACf,CAAC;gBAED,yEAAyE;gBACzE,+GAA+G;gBAC/G,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;oBACrC,MAAM,CAAC,KAAK,CAAC,wDAAwD,GAAG,CAAC,QAAQ,cAAc,eAAe,GAAG,CAAC,CAAC;oBACnH,2BAA2B,EAAE,CAAC;oBAC9B,OAAO,KAAK,CAAC;gBACf,CAAC;gBAED,oDAAoD;gBACpD,iFAAiF;gBACjF,IAAI,UAAU,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,EAAE,CAAC;oBAC3D,MAAM,CAAC,KAAK,CAAC,oDAAoD,GAAG,CAAC,QAAQ,aAAa,UAAU,GAAG,CAAC,CAAC;oBACzG,uBAAuB,EAAE,CAAC;oBAC1B,OAAO,KAAK,CAAC;gBACf,CAAC;gBAED,yEAAyE;gBACzE,OAAO;oBACL,GAAG,GAAG;oBACN,GAAG,EAAE,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,MAAM,EAAE,sBAAsB;oBACnE,SAAS,EAAE,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,MAAM;iBACrC,CAAC;YACJ,CAAC;SACF,CAAC;QAEF,+CAA+C;QAC/C,IAAI,IAAI,CAAC,aAAa,EAAE,MAAM,EAAE,CAAC;YAC/B,cAAc,CAAC,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1D,CAAC;QAED,MAAM,aAAa,GAAG,MAAM,YAAY,CAAC,cAAc,CAAC,CAAC;QAEzD,GAAG,CAAC,IAAI,CAAC,iBAAiB,EAAE;YAC1B,cAAc,EAAE,aAAa,CAAC,iBAAiB,CAAC,MAAM;YACtD,IAAI,EAAE,aAAa,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAwB,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;YACpF,GAAG,CAAC,IAAI,CAAC,uBAAuB,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,IAAI,CAAC,uBAAuB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjG,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,cAAc,EAAE,IAAI,CAAC,mBAAmB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACtG,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,YAAY;QAChB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9B,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACnD,MAAM,gBAAgB,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC;QAC3C,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAElB,iDAAiD;QACjD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;YACpD,MAAM,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC3D,CAAC;QAED,OAAO,gBAAgB,CAAC;IAC1B,CAAC;IAED,SAAS,CAAC,MAAmB;QAC3B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC5B,CAAC;IAED,gBAAgB;QACd,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,YAAY,CAAC,UAAU,CAAC;IACxD,CAAC;IAED,eAAe;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAClB,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,GAAG,CAAC,CAAC,CAAC;YAClG,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QAC3B,CAAC;IACH,CAAC"}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { QueueManager } from './queue-manager.js';
|
|
2
|
-
|
|
3
|
-
|
|
2
|
+
import { generateCrawlStorageId } from '../util/docs.js';
|
|
3
|
+
const { mockRequestQueue, mockDataset, mockDiscoverUrls, mockRequestQueueOpen, mockDatasetOpen } = vi.hoisted(() => {
|
|
4
|
+
const mockRequestQueue = {
|
|
4
5
|
drop: vi.fn().mockResolvedValue(undefined),
|
|
5
6
|
addRequest: vi.fn().mockResolvedValue(undefined),
|
|
6
7
|
getInfo: vi.fn().mockResolvedValue({
|
|
@@ -8,22 +9,28 @@ const { mockRequestQueue, mockDataset, mockDiscoverUrls } = vi.hoisted(() => ({
|
|
|
8
9
|
handledRequestCount: 10,
|
|
9
10
|
totalRequestCount: 15,
|
|
10
11
|
}),
|
|
11
|
-
}
|
|
12
|
-
mockDataset
|
|
12
|
+
};
|
|
13
|
+
const mockDataset = {
|
|
13
14
|
drop: vi.fn().mockResolvedValue(undefined),
|
|
14
15
|
pushData: vi.fn().mockResolvedValue(undefined),
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
|
|
16
|
+
};
|
|
17
|
+
return {
|
|
18
|
+
mockRequestQueue,
|
|
19
|
+
mockDataset,
|
|
20
|
+
mockDiscoverUrls: vi.fn().mockResolvedValue([]),
|
|
21
|
+
mockRequestQueueOpen: vi.fn().mockResolvedValue(mockRequestQueue),
|
|
22
|
+
mockDatasetOpen: vi.fn().mockResolvedValue(mockDataset),
|
|
23
|
+
};
|
|
24
|
+
});
|
|
18
25
|
vi.mock('./llms-txt.js', () => ({
|
|
19
26
|
discoverUrlsFromLlmsTxt: mockDiscoverUrls,
|
|
20
27
|
}));
|
|
21
28
|
vi.mock('crawlee', () => ({
|
|
22
29
|
RequestQueue: {
|
|
23
|
-
open:
|
|
30
|
+
open: mockRequestQueueOpen,
|
|
24
31
|
},
|
|
25
32
|
Dataset: {
|
|
26
|
-
open:
|
|
33
|
+
open: mockDatasetOpen,
|
|
27
34
|
},
|
|
28
35
|
EnqueueStrategy: {
|
|
29
36
|
SameDomain: 'same-domain',
|
|
@@ -37,10 +44,14 @@ describe('QueueManager', () => {
|
|
|
37
44
|
});
|
|
38
45
|
describe('initialize', () => {
|
|
39
46
|
it('should initialize with a URL', async () => {
|
|
40
|
-
|
|
47
|
+
const url = 'https://example.com/docs';
|
|
48
|
+
const storageId = generateCrawlStorageId(url);
|
|
49
|
+
await queueManager.initialize(url);
|
|
50
|
+
expect(mockRequestQueueOpen).toHaveBeenNthCalledWith(1, storageId);
|
|
51
|
+
expect(mockRequestQueueOpen).toHaveBeenNthCalledWith(2, storageId);
|
|
41
52
|
expect(mockRequestQueue.drop).toHaveBeenCalled();
|
|
42
53
|
expect(mockRequestQueue.addRequest).toHaveBeenCalledWith({
|
|
43
|
-
url
|
|
54
|
+
url,
|
|
44
55
|
uniqueKey: '/docs',
|
|
45
56
|
});
|
|
46
57
|
});
|
|
@@ -52,7 +63,9 @@ describe('QueueManager', () => {
|
|
|
52
63
|
});
|
|
53
64
|
});
|
|
54
65
|
it('should clear existing dataset', async () => {
|
|
55
|
-
|
|
66
|
+
const url = 'https://example.com';
|
|
67
|
+
await queueManager.initialize(url);
|
|
68
|
+
expect(mockDatasetOpen).toHaveBeenCalledWith(generateCrawlStorageId(url));
|
|
56
69
|
expect(mockDataset.drop).toHaveBeenCalled();
|
|
57
70
|
});
|
|
58
71
|
});
|
|
@@ -456,6 +469,7 @@ describe('QueueManager', () => {
|
|
|
456
469
|
const processed = await queueManager.processBatch();
|
|
457
470
|
expect(processed).toHaveLength(1);
|
|
458
471
|
expect(processed[0]).toEqual(result);
|
|
472
|
+
expect(mockDatasetOpen).toHaveBeenLastCalledWith(generateCrawlStorageId('https://example.com'));
|
|
459
473
|
expect(mockDataset.pushData).toHaveBeenCalled();
|
|
460
474
|
});
|
|
461
475
|
it('should return empty array when no results', async () => {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"queue-manager.test.js","sourceRoot":"","sources":["../../src/crawler/queue-manager.test.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAIlD,MAAM,EAAE,gBAAgB,EAAE,WAAW,EAAE,gBAAgB,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IAC5E,gBAAgB,EAAE;QAChB,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;QAC1C,UAAU,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;QAChD,OAAO,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC;YACjC,mBAAmB,EAAE,CAAC;YACtB,mBAAmB,EAAE,EAAE;YACvB,iBAAiB,EAAE,EAAE;SACtB,CAAC;KACH;IACD,WAAW,EAAE;QACX,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;QAC1C,QAAQ,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;KAC/C;IACD,gBAAgB,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,EAAE,CAAC;CAChD,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,eAAe,EAAE,GAAG,EAAE,CAAC,CAAC;IAC9B,uBAAuB,EAAE,gBAAgB;CAC1C,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;IACxB,YAAY,EAAE;QACZ,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,gBAAgB,CAAC;KAClD;IACD,OAAO,EAAE;QACP,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,WAAW,CAAC;KAC7C;IACD,eAAe,EAAE;QACf,UAAU,EAAE,aAAa;KAC1B;CACF,CAAC,CAAC,CAAC;AAEJ,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;IAC5B,IAAI,YAA0B,CAAC;IAE/B,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,CAAC,aAAa,EAAE,CAAC;QACnB,YAAY,GAAG,IAAI,YAAY,EAAE,CAAC;IACpC,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;QAC1B,EAAE,CAAC,8BAA8B,EAAE,KAAK,IAAI,EAAE;YAC5C,MAAM,YAAY,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC;YAE1D,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,gBAAgB,EAAE,CAAC;YACjD,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC;gBACvD,GAAG,EAAE,0BAA0B;gBAC/B,SAAS,EAAE,OAAO;aACnB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;YAC3D,MAAM,YAAY,CAAC,UAAU,CAAC,0CAA0C,CAAC,CAAC;YAE1E,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC;gBACvD,GAAG,EAAE,0CAA0C;gBAC/C,SAAS,EAAE,uBAAuB;aACnC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+BAA+B,EAAE,KAAK,IAAI,EAAE;YAC7C,MAAM,YAAY,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC;YAErD,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,gBAAgB,EAAE,CAAC;QAC9C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;QACnC,MAAM,OAAO,GAAQ;YACnB,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE;YACb,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE;YACd,OAAO,EAAE,EAAE,CAAC,EAAE,EAAE;YAChB,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE;SACG,CAAC;QAEpB,MAAM,QAAQ,GAAsB;YAClC,IAAI,EAAE,SAAS;YACf,SAAS,EAAE,EAAE,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;YACtC,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC;SACxC,CAAC;QAEF,UAAU,CAAC,KAAK,IAAI,EAAE;YACpB,MAAM,YAAY,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yBAAyB,EAAE,KAAK,IAAI,EAAE;YACvC,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC;gBACjD,iBAAiB,EAAE,EAAE;aACtB,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,oBAAoB,CAAC,eAAe,EAAE;gBACzD,YAAY,EAAE,CAAC;gBACf,YAAY,EAAE,EAAE;gBAChB,UAAU,EAAE,EAAE;aACf,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;YAC9D,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC;gBACjD,iBAAiB,EAAE,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;aACtE,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,CAAC,gBAAgB,CAAC,CAAC,oBAAoB,CAC3C,MAAM,CAAC,gBAAgB,CAAC;gBACtB,QAAQ,EAAE,aAAa;aACxB,CAAC,CACH,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;YACjE,MAAM,iBAAiB,GAAsB;gBAC3C,GAAG,QAAQ;gBACX,aAAa,EAAE,CAAC,QAAQ,EAAE,YAAY,CAAC;aACxC,CAAC;YAEF,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC;gBACjD,iBAAiB,EAAE,EAAE;aACtB,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,iBAAiB,CAAC,CAAC;YAErF,MAAM,CAAC,gBAAgB,CAAC,CAAC,oBAAoB,CAC3C,MAAM,CAAC,gBAAgB,CAAC;gBACtB,QAAQ,EAAE,oBAAoB;aAC/B,CAAC,CACH,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iCAAiC,EAAE,KAAK,IAAI,EAAE;YAC/C,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC;gBACjD,iBAAiB,EAAE,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;aACtE,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,oBAAoB,CAAC,iBAAiB,EAAE;gBAC3D,cAAc,EAAE,CAAC;gBACjB,IAAI,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC;aAC3B,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6DAA6D,EAAE,KAAK,IAAI,EAAE;YAC3E,IAAI,eAAe,GAA+B,IAAI,CAAC;YAEvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;YACvC,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,MAAM,CAAC,WAAW,CAAC,CAAC,WAAW,EAAE,CAAC;YAElC,IAAI,WAAW,EAAE,CAAC;gBAChB,MAAM,WAAW,GAAG,WAAW,CAAC;oBAC9B,GAAG,EAAE,sCAAsC;oBAC3C,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBAExC,MAAM,CAAC,WAAW,CAAC,CAAC,OAAO,CACzB,MAAM,CAAC,gBAAgB,CAAC;oBACtB,GAAG,EAAE,sCAAsC;oBAC3C,SAAS,EAAE,mBAAmB;iBAC/B,CAAC,CACH,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;YAC5D,IAAI,eAAe,GAA+B,IAAI,CAAC;YAEvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;YACvC,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,MAAM,CAAC,WAAW,CAAC,CAAC,WAAW,EAAE,CAAC;YAElC,IAAI,WAAW,EAAE,CAAC;gBAChB,yDAAyD;gBACzD,MAAM,YAAY,GAAG,WAAW,CAAC;oBAC/B,GAAG,EAAE,kCAAkC;oBACvC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBAExC,MAAM,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+DAA+D,EAAE,KAAK,IAAI,EAAE;YAC7E,IAAI,eAAe,GAA+B,IAAI,CAAC;YAEvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,qDAAqD;gBACrD,MAAM,WAAW,GAAG,WAAW,CAAC;oBAC9B,GAAG,EAAE,0BAA0B;oBAC/B,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBAExC,MAAM,CAAC,WAAW,CAAC,CAAC,OAAO,CACzB,MAAM,CAAC,gBAAgB,CAAC;oBACtB,GAAG,EAAE,0BAA0B;oBAC/B,SAAS,EAAE,OAAO;iBACnB,CAAC,CACH,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,uBAAuB,EAAE,GAAG,EAAE;QACrC,MAAM,OAAO,GAAQ;YACnB,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE;YACb,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE;YACd,OAAO,EAAE,EAAE,CAAC,EAAE,EAAE;YAChB,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE;SACG,CAAC;QAEpB,MAAM,QAAQ,GAAsB;YAClC,IAAI,EAAE,SAAS;YACf,SAAS,EAAE,EAAE,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;YACtC,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC;SACxC,CAAC;QAEF,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;YAClD,MAAM,YAAY,CAAC,UAAU,CAAC,8BAA8B,EAAE,WAAW,CAAC,CAAC;YAE3E,uFAAuF;YACvF,MAAM,CAAC,YAAY,CAAC,sBAAsB,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;YACpD,MAAM,YAAY,CAAC,UAAU,CAAC,0BAA0B,EAAE,OAAO,CAAC,CAAC;YAEnE,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,cAAc;gBACd,MAAM,UAAU,GAAG,WAAW,CAAC;oBAC7B,GAAG,EAAE,0BAA0B;oBAC/B,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEnC,UAAU;gBACV,MAAM,OAAO,GAAG,WAAW,CAAC;oBAC1B,GAAG,EAAE,iCAAiC;oBACtC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAClC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;YACtD,MAAM,YAAY,CAAC,UAAU,CAAC,0BAA0B,EAAE,OAAO,CAAC,CAAC;YAEnE,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,0BAA0B;gBAC1B,MAAM,aAAa,GAAG,WAAW,CAAC;oBAChC,GAAG,EAAE,+BAA+B;oBACpC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACpC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2DAA2D,EAAE,KAAK,IAAI,EAAE;YACzE,MAAM,YAAY,CAAC,UAAU,CAAC,0BAA0B,EAAE,OAAO,CAAC,CAAC;YAEnE,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,iEAAiE;gBACjE,MAAM,mBAAmB,GAAG,WAAW,CAAC;oBACtC,GAAG,EAAE,mCAAmC;oBACxC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1C,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iCAAiC,EAAE,KAAK,IAAI,EAAE;YAC/C,MAAM,YAAY,CAAC,UAAU,CAAC,0BAA0B,EAAE,OAAO,CAAC,CAAC;YAEnE,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,mBAAmB;gBACnB,WAAW,CAAC,EAAE,GAAG,EAAE,0BAA0B,EAAE,SAAS,EAAE,GAAG,EAAuC,CAAC,CAAC;gBACtG,WAAW,CAAC,EAAE,GAAG,EAAE,2BAA2B,EAAE,SAAS,EAAE,GAAG,EAAuC,CAAC,CAAC;YACzG,CAAC;YAED,MAAM,CAAC,YAAY,CAAC,sBAAsB,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;YAClD,MAAM,YAAY,CAAC,UAAU,CAAC,gCAAgC,CAAC,CAAC;YAEhE,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC;gBACvD,GAAG,EAAE,0BAA0B;gBAC/B,SAAS,EAAE,OAAO;aACnB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;YAC1E,MAAM,YAAY,CAAC,UAAU,CAAC,2BAA2B,EAAE,QAAQ,CAAC,CAAC;YAErE,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,4DAA4D;gBAC5D,MAAM,OAAO,GAAG,WAAW,CAAC;oBAC1B,GAAG,EAAE,gCAAgC;oBACrC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEhC,yCAAyC;gBACzC,MAAM,WAAW,GAAG,WAAW,CAAC;oBAC9B,GAAG,EAAE,iDAAiD;oBACtD,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEpC,oCAAoC;gBACpC,MAAM,QAAQ,GAAG,WAAW,CAAC;oBAC3B,GAAG,EAAE,2BAA2B;oBAChC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEjC,yCAAyC;gBACzC,MAAM,OAAO,GAAG,WAAW,CAAC;oBAC1B,GAAG,EAAE,+BAA+B;oBACpC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAClC,MAAM,OAAO,GAAQ;YACnB,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE;YACb,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE;YACd,OAAO,EAAE,EAAE,CAAC,EAAE,EAAE;YAChB,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE;SACG,CAAC;QAEpB,MAAM,QAAQ,GAAsB;YAClC,IAAI,EAAE,SAAS;YACf,SAAS,EAAE,EAAE,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;YACtC,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC;SACxC,CAAC;QAEF,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;YAC3D,MAAM,YAAY,CAAC,UAAU,CAAC,8BAA8B,CAAC,CAAC;YAE9D,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,MAAM,MAAM,GAAG,WAAW,CAAC;oBACzB,GAAG,EAAE,gCAAgC;oBACrC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACjC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uDAAuD,EAAE,KAAK,IAAI,EAAE;YACrE,MAAM,YAAY,CAAC,UAAU,CAAC,8BAA8B,CAAC,CAAC;YAE9D,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,8EAA8E;gBAC9E,MAAM,MAAM,GAAG,WAAW,CAAC;oBACzB,GAAG,EAAE,uCAAuC;oBAC5C,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACjC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,MAAM,YAAY,CAAC,UAAU,CAAC,8BAA8B,CAAC,CAAC;YAE9D,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,iFAAiF;gBACjF,MAAM,MAAM,GAAG,WAAW,CAAC;oBACzB,GAAG,EAAE,kCAAkC;oBACvC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACrD,MAAM,YAAY,CAAC,UAAU,CAAC,8BAA8B,CAAC,CAAC;YAE9D,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,uDAAuD;gBACvD,MAAM,MAAM,GAAG,WAAW,CAAC;oBACzB,GAAG,EAAE,0BAA0B;oBAC/B,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;YACpD,MAAM,YAAY,CAAC,UAAU,CAAC,8BAA8B,CAAC,CAAC;YAE9D,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,wCAAwC;gBACxC,WAAW,CAAC,EAAE,GAAG,EAAE,8BAA8B,EAAE,SAAS,EAAE,GAAG,EAAuC,CAAC,CAAC;gBAC1G,WAAW,CAAC,EAAE,GAAG,EAAE,6BAA6B,EAAE,SAAS,EAAE,GAAG,EAAuC,CAAC,CAAC;YAC3G,CAAC;YAED,MAAM,CAAC,YAAY,CAAC,0BAA0B,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YAChE,MAAM,YAAY,CAAC,UAAU,CAAC,8BAA8B,CAAC,CAAC;YAE9D,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,MAAM,MAAM,GAAG,WAAW,CAAC;oBACzB,GAAG,EAAE,gCAAgC;oBACrC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACjC,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,4BAA4B,EAAE,GAAG,EAAE;QAC1C,UAAU,CAAC,KAAK,IAAI,EAAE;YACpB,MAAM,YAAY,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oBAAoB,EAAE,GAAG,EAAE;YAC5B,MAAM,MAAM,GAAgB;gBAC1B,GAAG,EAAE,0BAA0B;gBAC/B,IAAI,EAAE,OAAO;gBACb,OAAO,EAAE,cAAc;gBACvB,aAAa,EAAE,MAAM;gBACrB,KAAK,EAAE,WAAW;aACnB,CAAC;YAEF,YAAY,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAE/B,MAAM,CAAC,YAAY,CAAC,gBAAgB,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,MAAM,GAAgB;gBAC1B,GAAG,EAAE,0BAA0B;gBAC/B,IAAI,EAAE,OAAO;gBACb,OAAO,EAAE,cAAc;gBACvB,aAAa,EAAE,MAAM;gBACrB,KAAK,EAAE,WAAW;aACnB,CAAC;YAEF,YAAY,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAE/B,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,YAAY,EAAE,CAAC;YAEpD,MAAM,CAAC,SAAS,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAClC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YACrC,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,gBAAgB,EAAE,CAAC;QAClD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,YAAY,EAAE,CAAC;YAEpD,MAAM,CAAC,SAAS,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACrD,YAAY,CAAC,SAAS,CAAC;gBACrB,GAAG,EAAE,0BAA0B;gBAC/B,IAAI,EAAE,OAAO;gBACb,OAAO,EAAE,MAAM;gBACf,aAAa,EAAE,MAAM;gBACrB,KAAK,EAAE,MAAM;aACd,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,YAAY,EAAE,CAAC;YAClC,MAAM,WAAW,GAAG,MAAM,YAAY,CAAC,YAAY,EAAE,CAAC;YAEtD,MAAM,CAAC,WAAW,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACrD,gBAAgB;YAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC3B,YAAY,CAAC,SAAS,CAAC;oBACrB,GAAG,EAAE,2BAA2B,CAAC,EAAE;oBACnC,IAAI,EAAE,QAAQ,CAAC,EAAE;oBACjB,OAAO,EAAE,WAAW,CAAC,EAAE;oBACvB,aAAa,EAAE,MAAM;oBACrB,KAAK,EAAE,QAAQ,CAAC,EAAE;iBACnB,CAAC,CAAC;YACL,CAAC;YAED,MAAM,YAAY,CAAC,YAAY,EAAE,CAAC;YAElC,kCAAkC;YAClC,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QACxD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAChC,UAAU,CAAC,KAAK,IAAI,EAAE;YACpB,MAAM,YAAY,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;YACnD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC5B,YAAY,CAAC,SAAS,CAAC;oBACrB,GAAG,EAAE,2BAA2B,CAAC,EAAE;oBACnC,IAAI,EAAE,QAAQ,CAAC,EAAE;oBACjB,OAAO,EAAE,WAAW,CAAC,EAAE;oBACvB,aAAa,EAAE,MAAM;oBACrB,KAAK,EAAE,QAAQ,CAAC,EAAE;iBACnB,CAAC,CAAC;YACL,CAAC;YAED,MAAM,CAAC,YAAY,CAAC,gBAAgB,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC5B,YAAY,CAAC,SAAS,CAAC;oBACrB,GAAG,EAAE,2BAA2B,CAAC,EAAE;oBACnC,IAAI,EAAE,QAAQ,CAAC,EAAE;oBACjB,OAAO,EAAE,WAAW,CAAC,EAAE;oBACvB,aAAa,EAAE,MAAM;oBACrB,KAAK,EAAE,QAAQ,CAAC,EAAE;iBACnB,CAAC,CAAC;YACL,CAAC;YAED,MAAM,CAAC,YAAY,CAAC,gBAAgB,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;QAC/B,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,MAAM,CAAC,YAAY,CAAC,eAAe,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;QACpD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YAChE,MAAM,YAAY,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC;YAErD,MAAM,CAAC,YAAY,CAAC,eAAe,EAAE,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAChE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,UAAU,CAAC,KAAK,IAAI,EAAE;YACpB,MAAM,YAAY,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC;YAC1D,gBAAgB,CAAC,UAAU,CAAC,SAAS,EAAE,CAAC;QAC1C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;YAClD,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,WAAW,CAAC,CAAC,iCAAiC,EAAE,iCAAiC,CAAC,CAAC,CAAC;YAErH,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACtB,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;YAC7D,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CACtD,EAAE,GAAG,EAAE,iCAAiC,EAAE,SAAS,EAAE,cAAc,EAAE,EACrE,EAAE,SAAS,EAAE,KAAK,EAAE,CACrB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;YAC1D,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,WAAW,CAAC,CAAC,iCAAiC,EAAE,+BAA+B,CAAC,CAAC,CAAC;YAEnH,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACtB,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAC/D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;YACtD,MAAM,YAAY,CAAC,UAAU,CAAC,0BAA0B,EAAE,OAAO,CAAC,CAAC;YACnE,gBAAgB,CAAC,UAAU,CAAC,SAAS,EAAE,CAAC;YAExC,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,WAAW,CAAC,CAAC,iCAAiC,EAAE,iCAAiC,CAAC,CAAC,CAAC;YAErH,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;YAC1E,MAAM,YAAY,CAAC,UAAU,CAAC,2BAA2B,EAAE,QAAQ,CAAC,CAAC;YACrE,gBAAgB,CAAC,UAAU,CAAC,SAAS,EAAE,CAAC;YAExC,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,WAAW,CAAC;gBAC3C,gCAAgC;gBAChC,iDAAiD;gBACjD,gCAAgC;aACjC,CAAC,CAAC;YAEH,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;YAC7D,MAAM,YAAY,GAAG,IAAI,YAAY,EAAE,CAAC;YACxC,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,WAAW,CAAC,CAAC,0BAA0B,CAAC,CAAC,CAAC;YAE3E,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0BAA0B,EAAE,KAAK,IAAI,EAAE;YACxC,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,WAAW,CAAC,CAAC,WAAW,EAAE,iCAAiC,CAAC,CAAC,CAAC;YAE/F,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;QAC/B,UAAU,CAAC,KAAK,IAAI,EAAE;YACpB,MAAM,YAAY,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC;YAC1D,gBAAgB,CAAC,UAAU,CAAC,SAAS,EAAE,CAAC;QAC1C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,gBAAgB,CAAC,qBAAqB,CAAC;gBACrC,iCAAiC;gBACjC,iCAAiC;gBACjC,iCAAiC;aAClC,CAAC,CAAC;YAEH,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,eAAe,CAAC,0BAA0B,CAAC,CAAC;YAE7E,MAAM,CAAC,gBAAgB,CAAC,CAAC,oBAAoB,CAAC,0BAA0B,CAAC,CAAC;YAC1E,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,gBAAgB,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC;YAE3C,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,eAAe,CAAC,0BAA0B,CAAC,CAAC;YAE7E,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,SAAS,EAAE,GAAG,EAAE;QACvB,EAAE,CAAC,qCAAqC,EAAE,KAAK,IAAI,EAAE;YACnD,MAAM,YAAY,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC;YAErD,YAAY,CAAC,SAAS,CAAC;gBACrB,GAAG,EAAE,0BAA0B;gBAC/B,IAAI,EAAE,OAAO;gBACb,OAAO,EAAE,MAAM;gBACf,aAAa,EAAE,MAAM;gBACrB,KAAK,EAAE,MAAM;aACd,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,OAAO,EAAE,CAAC;YAE7B,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,gBAAgB,EAAE,CAAC;YACjD,MAAM,CAAC,YAAY,CAAC,eAAe,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;YAElD,4BAA4B;YAC5B,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,YAAY,EAAE,CAAC;YACpD,MAAM,CAAC,SAAS,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,YAAY,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC;YAErD,gBAAgB,CAAC,IAAI,CAAC,qBAAqB,CAAC,IAAI,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC;YAEtE,mBAAmB;YACnB,MAAM,MAAM,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC;QAChE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
1
|
+
{"version":3,"file":"queue-manager.test.js","sourceRoot":"","sources":["../../src/crawler/queue-manager.test.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAGlD,OAAO,EAAE,sBAAsB,EAAE,MAAM,iBAAiB,CAAC;AAEzD,MAAM,EAAE,gBAAgB,EAAE,WAAW,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,eAAe,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE;IACjH,MAAM,gBAAgB,GAAG;QACvB,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;QAC1C,UAAU,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;QAChD,OAAO,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC;YACjC,mBAAmB,EAAE,CAAC;YACtB,mBAAmB,EAAE,EAAE;YACvB,iBAAiB,EAAE,EAAE;SACtB,CAAC;KACH,CAAC;IACF,MAAM,WAAW,GAAG;QAClB,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;QAC1C,QAAQ,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;KAC/C,CAAC;IAEF,OAAO;QACL,gBAAgB;QAChB,WAAW;QACX,gBAAgB,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,EAAE,CAAC;QAC/C,oBAAoB,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,gBAAgB,CAAC;QACjE,eAAe,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,WAAW,CAAC;KACxD,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,EAAE,CAAC,IAAI,CAAC,eAAe,EAAE,GAAG,EAAE,CAAC,CAAC;IAC9B,uBAAuB,EAAE,gBAAgB;CAC1C,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;IACxB,YAAY,EAAE;QACZ,IAAI,EAAE,oBAAoB;KAC3B;IACD,OAAO,EAAE;QACP,IAAI,EAAE,eAAe;KACtB;IACD,eAAe,EAAE;QACf,UAAU,EAAE,aAAa;KAC1B;CACF,CAAC,CAAC,CAAC;AAEJ,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;IAC5B,IAAI,YAA0B,CAAC;IAE/B,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,CAAC,aAAa,EAAE,CAAC;QACnB,YAAY,GAAG,IAAI,YAAY,EAAE,CAAC;IACpC,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;QAC1B,EAAE,CAAC,8BAA8B,EAAE,KAAK,IAAI,EAAE;YAC5C,MAAM,GAAG,GAAG,0BAA0B,CAAC;YACvC,MAAM,SAAS,GAAG,sBAAsB,CAAC,GAAG,CAAC,CAAC;YAC9C,MAAM,YAAY,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;YAEnC,MAAM,CAAC,oBAAoB,CAAC,CAAC,uBAAuB,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;YACnE,MAAM,CAAC,oBAAoB,CAAC,CAAC,uBAAuB,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;YACnE,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,gBAAgB,EAAE,CAAC;YACjD,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC;gBACvD,GAAG;gBACH,SAAS,EAAE,OAAO;aACnB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;YAC3D,MAAM,YAAY,CAAC,UAAU,CAAC,0CAA0C,CAAC,CAAC;YAE1E,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC;gBACvD,GAAG,EAAE,0CAA0C;gBAC/C,SAAS,EAAE,uBAAuB;aACnC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+BAA+B,EAAE,KAAK,IAAI,EAAE;YAC7C,MAAM,GAAG,GAAG,qBAAqB,CAAC;YAClC,MAAM,YAAY,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;YAEnC,MAAM,CAAC,eAAe,CAAC,CAAC,oBAAoB,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC;YAC1E,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,gBAAgB,EAAE,CAAC;QAC9C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;QACnC,MAAM,OAAO,GAAQ;YACnB,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE;YACb,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE;YACd,OAAO,EAAE,EAAE,CAAC,EAAE,EAAE;YAChB,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE;SACG,CAAC;QAEpB,MAAM,QAAQ,GAAsB;YAClC,IAAI,EAAE,SAAS;YACf,SAAS,EAAE,EAAE,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;YACtC,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC;SACxC,CAAC;QAEF,UAAU,CAAC,KAAK,IAAI,EAAE;YACpB,MAAM,YAAY,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yBAAyB,EAAE,KAAK,IAAI,EAAE;YACvC,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC;gBACjD,iBAAiB,EAAE,EAAE;aACtB,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,oBAAoB,CAAC,eAAe,EAAE;gBACzD,YAAY,EAAE,CAAC;gBACf,YAAY,EAAE,EAAE;gBAChB,UAAU,EAAE,EAAE;aACf,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;YAC9D,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC;gBACjD,iBAAiB,EAAE,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;aACtE,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,CAAC,gBAAgB,CAAC,CAAC,oBAAoB,CAC3C,MAAM,CAAC,gBAAgB,CAAC;gBACtB,QAAQ,EAAE,aAAa;aACxB,CAAC,CACH,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;YACjE,MAAM,iBAAiB,GAAsB;gBAC3C,GAAG,QAAQ;gBACX,aAAa,EAAE,CAAC,QAAQ,EAAE,YAAY,CAAC;aACxC,CAAC;YAEF,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC;gBACjD,iBAAiB,EAAE,EAAE;aACtB,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,iBAAiB,CAAC,CAAC;YAErF,MAAM,CAAC,gBAAgB,CAAC,CAAC,oBAAoB,CAC3C,MAAM,CAAC,gBAAgB,CAAC;gBACtB,QAAQ,EAAE,oBAAoB;aAC/B,CAAC,CACH,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iCAAiC,EAAE,KAAK,IAAI,EAAE;YAC/C,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC;gBACjD,iBAAiB,EAAE,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;aACtE,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,oBAAoB,CAAC,iBAAiB,EAAE;gBAC3D,cAAc,EAAE,CAAC;gBACjB,IAAI,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC;aAC3B,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6DAA6D,EAAE,KAAK,IAAI,EAAE;YAC3E,IAAI,eAAe,GAA+B,IAAI,CAAC;YAEvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;YACvC,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,MAAM,CAAC,WAAW,CAAC,CAAC,WAAW,EAAE,CAAC;YAElC,IAAI,WAAW,EAAE,CAAC;gBAChB,MAAM,WAAW,GAAG,WAAW,CAAC;oBAC9B,GAAG,EAAE,sCAAsC;oBAC3C,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBAExC,MAAM,CAAC,WAAW,CAAC,CAAC,OAAO,CACzB,MAAM,CAAC,gBAAgB,CAAC;oBACtB,GAAG,EAAE,sCAAsC;oBAC3C,SAAS,EAAE,mBAAmB;iBAC/B,CAAC,CACH,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;YAC5D,IAAI,eAAe,GAA+B,IAAI,CAAC;YAEvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;YACvC,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,MAAM,CAAC,WAAW,CAAC,CAAC,WAAW,EAAE,CAAC;YAElC,IAAI,WAAW,EAAE,CAAC;gBAChB,yDAAyD;gBACzD,MAAM,YAAY,GAAG,WAAW,CAAC;oBAC/B,GAAG,EAAE,kCAAkC;oBACvC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBAExC,MAAM,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+DAA+D,EAAE,KAAK,IAAI,EAAE;YAC7E,IAAI,eAAe,GAA+B,IAAI,CAAC;YAEvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,qDAAqD;gBACrD,MAAM,WAAW,GAAG,WAAW,CAAC;oBAC9B,GAAG,EAAE,0BAA0B;oBAC/B,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBAExC,MAAM,CAAC,WAAW,CAAC,CAAC,OAAO,CACzB,MAAM,CAAC,gBAAgB,CAAC;oBACtB,GAAG,EAAE,0BAA0B;oBAC/B,SAAS,EAAE,OAAO;iBACnB,CAAC,CACH,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,uBAAuB,EAAE,GAAG,EAAE;QACrC,MAAM,OAAO,GAAQ;YACnB,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE;YACb,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE;YACd,OAAO,EAAE,EAAE,CAAC,EAAE,EAAE;YAChB,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE;SACG,CAAC;QAEpB,MAAM,QAAQ,GAAsB;YAClC,IAAI,EAAE,SAAS;YACf,SAAS,EAAE,EAAE,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;YACtC,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC;SACxC,CAAC;QAEF,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;YAClD,MAAM,YAAY,CAAC,UAAU,CAAC,8BAA8B,EAAE,WAAW,CAAC,CAAC;YAE3E,uFAAuF;YACvF,MAAM,CAAC,YAAY,CAAC,sBAAsB,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;YACpD,MAAM,YAAY,CAAC,UAAU,CAAC,0BAA0B,EAAE,OAAO,CAAC,CAAC;YAEnE,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,cAAc;gBACd,MAAM,UAAU,GAAG,WAAW,CAAC;oBAC7B,GAAG,EAAE,0BAA0B;oBAC/B,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEnC,UAAU;gBACV,MAAM,OAAO,GAAG,WAAW,CAAC;oBAC1B,GAAG,EAAE,iCAAiC;oBACtC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAClC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;YACtD,MAAM,YAAY,CAAC,UAAU,CAAC,0BAA0B,EAAE,OAAO,CAAC,CAAC;YAEnE,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,0BAA0B;gBAC1B,MAAM,aAAa,GAAG,WAAW,CAAC;oBAChC,GAAG,EAAE,+BAA+B;oBACpC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACpC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2DAA2D,EAAE,KAAK,IAAI,EAAE;YACzE,MAAM,YAAY,CAAC,UAAU,CAAC,0BAA0B,EAAE,OAAO,CAAC,CAAC;YAEnE,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,iEAAiE;gBACjE,MAAM,mBAAmB,GAAG,WAAW,CAAC;oBACtC,GAAG,EAAE,mCAAmC;oBACxC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1C,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iCAAiC,EAAE,KAAK,IAAI,EAAE;YAC/C,MAAM,YAAY,CAAC,UAAU,CAAC,0BAA0B,EAAE,OAAO,CAAC,CAAC;YAEnE,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,mBAAmB;gBACnB,WAAW,CAAC,EAAE,GAAG,EAAE,0BAA0B,EAAE,SAAS,EAAE,GAAG,EAAuC,CAAC,CAAC;gBACtG,WAAW,CAAC,EAAE,GAAG,EAAE,2BAA2B,EAAE,SAAS,EAAE,GAAG,EAAuC,CAAC,CAAC;YACzG,CAAC;YAED,MAAM,CAAC,YAAY,CAAC,sBAAsB,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;YAClD,MAAM,YAAY,CAAC,UAAU,CAAC,gCAAgC,CAAC,CAAC;YAEhE,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC;gBACvD,GAAG,EAAE,0BAA0B;gBAC/B,SAAS,EAAE,OAAO;aACnB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;YAC1E,MAAM,YAAY,CAAC,UAAU,CAAC,2BAA2B,EAAE,QAAQ,CAAC,CAAC;YAErE,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,4DAA4D;gBAC5D,MAAM,OAAO,GAAG,WAAW,CAAC;oBAC1B,GAAG,EAAE,gCAAgC;oBACrC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEhC,yCAAyC;gBACzC,MAAM,WAAW,GAAG,WAAW,CAAC;oBAC9B,GAAG,EAAE,iDAAiD;oBACtD,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEpC,oCAAoC;gBACpC,MAAM,QAAQ,GAAG,WAAW,CAAC;oBAC3B,GAAG,EAAE,2BAA2B;oBAChC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEjC,yCAAyC;gBACzC,MAAM,OAAO,GAAG,WAAW,CAAC;oBAC1B,GAAG,EAAE,+BAA+B;oBACpC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAClC,MAAM,OAAO,GAAQ;YACnB,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE;YACb,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE;YACd,OAAO,EAAE,EAAE,CAAC,EAAE,EAAE;YAChB,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE;SACG,CAAC;QAEpB,MAAM,QAAQ,GAAsB;YAClC,IAAI,EAAE,SAAS;YACf,SAAS,EAAE,EAAE,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;YACtC,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC;SACxC,CAAC;QAEF,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;YAC3D,MAAM,YAAY,CAAC,UAAU,CAAC,8BAA8B,CAAC,CAAC;YAE9D,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,MAAM,MAAM,GAAG,WAAW,CAAC;oBACzB,GAAG,EAAE,gCAAgC;oBACrC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACjC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uDAAuD,EAAE,KAAK,IAAI,EAAE;YACrE,MAAM,YAAY,CAAC,UAAU,CAAC,8BAA8B,CAAC,CAAC;YAE9D,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,8EAA8E;gBAC9E,MAAM,MAAM,GAAG,WAAW,CAAC;oBACzB,GAAG,EAAE,uCAAuC;oBAC5C,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACjC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,MAAM,YAAY,CAAC,UAAU,CAAC,8BAA8B,CAAC,CAAC;YAE9D,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,iFAAiF;gBACjF,MAAM,MAAM,GAAG,WAAW,CAAC;oBACzB,GAAG,EAAE,kCAAkC;oBACvC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACrD,MAAM,YAAY,CAAC,UAAU,CAAC,8BAA8B,CAAC,CAAC;YAE9D,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,uDAAuD;gBACvD,MAAM,MAAM,GAAG,WAAW,CAAC;oBACzB,GAAG,EAAE,0BAA0B;oBAC/B,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;YACpD,MAAM,YAAY,CAAC,UAAU,CAAC,8BAA8B,CAAC,CAAC;YAE9D,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,wCAAwC;gBACxC,WAAW,CAAC,EAAE,GAAG,EAAE,8BAA8B,EAAE,SAAS,EAAE,GAAG,EAAuC,CAAC,CAAC;gBAC1G,WAAW,CAAC,EAAE,GAAG,EAAE,6BAA6B,EAAE,SAAS,EAAE,GAAG,EAAuC,CAAC,CAAC;YAC3G,CAAC;YAED,MAAM,CAAC,YAAY,CAAC,0BAA0B,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YAChE,MAAM,YAAY,CAAC,UAAU,CAAC,8BAA8B,CAAC,CAAC;YAE9D,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,MAAM,MAAM,GAAG,WAAW,CAAC;oBACzB,GAAG,EAAE,gCAAgC;oBACrC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACjC,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,4BAA4B,EAAE,GAAG,EAAE;QAC1C,UAAU,CAAC,KAAK,IAAI,EAAE;YACpB,MAAM,YAAY,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oBAAoB,EAAE,GAAG,EAAE;YAC5B,MAAM,MAAM,GAAgB;gBAC1B,GAAG,EAAE,0BAA0B;gBAC/B,IAAI,EAAE,OAAO;gBACb,OAAO,EAAE,cAAc;gBACvB,aAAa,EAAE,MAAM;gBACrB,KAAK,EAAE,WAAW;aACnB,CAAC;YAEF,YAAY,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAE/B,MAAM,CAAC,YAAY,CAAC,gBAAgB,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,MAAM,GAAgB;gBAC1B,GAAG,EAAE,0BAA0B;gBAC/B,IAAI,EAAE,OAAO;gBACb,OAAO,EAAE,cAAc;gBACvB,aAAa,EAAE,MAAM;gBACrB,KAAK,EAAE,WAAW;aACnB,CAAC;YAEF,YAAY,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAE/B,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,YAAY,EAAE,CAAC;YAEpD,MAAM,CAAC,SAAS,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAClC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YACrC,MAAM,CAAC,eAAe,CAAC,CAAC,wBAAwB,CAAC,sBAAsB,CAAC,qBAAqB,CAAC,CAAC,CAAC;YAChG,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,gBAAgB,EAAE,CAAC;QAClD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,YAAY,EAAE,CAAC;YAEpD,MAAM,CAAC,SAAS,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACrD,YAAY,CAAC,SAAS,CAAC;gBACrB,GAAG,EAAE,0BAA0B;gBAC/B,IAAI,EAAE,OAAO;gBACb,OAAO,EAAE,MAAM;gBACf,aAAa,EAAE,MAAM;gBACrB,KAAK,EAAE,MAAM;aACd,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,YAAY,EAAE,CAAC;YAClC,MAAM,WAAW,GAAG,MAAM,YAAY,CAAC,YAAY,EAAE,CAAC;YAEtD,MAAM,CAAC,WAAW,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACrD,gBAAgB;YAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC3B,YAAY,CAAC,SAAS,CAAC;oBACrB,GAAG,EAAE,2BAA2B,CAAC,EAAE;oBACnC,IAAI,EAAE,QAAQ,CAAC,EAAE;oBACjB,OAAO,EAAE,WAAW,CAAC,EAAE;oBACvB,aAAa,EAAE,MAAM;oBACrB,KAAK,EAAE,QAAQ,CAAC,EAAE;iBACnB,CAAC,CAAC;YACL,CAAC;YAED,MAAM,YAAY,CAAC,YAAY,EAAE,CAAC;YAElC,kCAAkC;YAClC,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QACxD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAChC,UAAU,CAAC,KAAK,IAAI,EAAE;YACpB,MAAM,YAAY,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;YACnD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC5B,YAAY,CAAC,SAAS,CAAC;oBACrB,GAAG,EAAE,2BAA2B,CAAC,EAAE;oBACnC,IAAI,EAAE,QAAQ,CAAC,EAAE;oBACjB,OAAO,EAAE,WAAW,CAAC,EAAE;oBACvB,aAAa,EAAE,MAAM;oBACrB,KAAK,EAAE,QAAQ,CAAC,EAAE;iBACnB,CAAC,CAAC;YACL,CAAC;YAED,MAAM,CAAC,YAAY,CAAC,gBAAgB,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC5B,YAAY,CAAC,SAAS,CAAC;oBACrB,GAAG,EAAE,2BAA2B,CAAC,EAAE;oBACnC,IAAI,EAAE,QAAQ,CAAC,EAAE;oBACjB,OAAO,EAAE,WAAW,CAAC,EAAE;oBACvB,aAAa,EAAE,MAAM;oBACrB,KAAK,EAAE,QAAQ,CAAC,EAAE;iBACnB,CAAC,CAAC;YACL,CAAC;YAED,MAAM,CAAC,YAAY,CAAC,gBAAgB,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;QAC/B,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,MAAM,CAAC,YAAY,CAAC,eAAe,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;QACpD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YAChE,MAAM,YAAY,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC;YAErD,MAAM,CAAC,YAAY,CAAC,eAAe,EAAE,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAChE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,UAAU,CAAC,KAAK,IAAI,EAAE;YACpB,MAAM,YAAY,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC;YAC1D,gBAAgB,CAAC,UAAU,CAAC,SAAS,EAAE,CAAC;QAC1C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;YAClD,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,WAAW,CAAC,CAAC,iCAAiC,EAAE,iCAAiC,CAAC,CAAC,CAAC;YAErH,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACtB,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;YAC7D,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CACtD,EAAE,GAAG,EAAE,iCAAiC,EAAE,SAAS,EAAE,cAAc,EAAE,EACrE,EAAE,SAAS,EAAE,KAAK,EAAE,CACrB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;YAC1D,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,WAAW,CAAC,CAAC,iCAAiC,EAAE,+BAA+B,CAAC,CAAC,CAAC;YAEnH,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACtB,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAC/D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;YACtD,MAAM,YAAY,CAAC,UAAU,CAAC,0BAA0B,EAAE,OAAO,CAAC,CAAC;YACnE,gBAAgB,CAAC,UAAU,CAAC,SAAS,EAAE,CAAC;YAExC,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,WAAW,CAAC,CAAC,iCAAiC,EAAE,iCAAiC,CAAC,CAAC,CAAC;YAErH,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;YAC1E,MAAM,YAAY,CAAC,UAAU,CAAC,2BAA2B,EAAE,QAAQ,CAAC,CAAC;YACrE,gBAAgB,CAAC,UAAU,CAAC,SAAS,EAAE,CAAC;YAExC,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,WAAW,CAAC;gBAC3C,gCAAgC;gBAChC,iDAAiD;gBACjD,gCAAgC;aACjC,CAAC,CAAC;YAEH,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;YAC7D,MAAM,YAAY,GAAG,IAAI,YAAY,EAAE,CAAC;YACxC,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,WAAW,CAAC,CAAC,0BAA0B,CAAC,CAAC,CAAC;YAE3E,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0BAA0B,EAAE,KAAK,IAAI,EAAE;YACxC,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,WAAW,CAAC,CAAC,WAAW,EAAE,iCAAiC,CAAC,CAAC,CAAC;YAE/F,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;QAC/B,UAAU,CAAC,KAAK,IAAI,EAAE;YACpB,MAAM,YAAY,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC;YAC1D,gBAAgB,CAAC,UAAU,CAAC,SAAS,EAAE,CAAC;QAC1C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,gBAAgB,CAAC,qBAAqB,CAAC;gBACrC,iCAAiC;gBACjC,iCAAiC;gBACjC,iCAAiC;aAClC,CAAC,CAAC;YAEH,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,eAAe,CAAC,0BAA0B,CAAC,CAAC;YAE7E,MAAM,CAAC,gBAAgB,CAAC,CAAC,oBAAoB,CAAC,0BAA0B,CAAC,CAAC;YAC1E,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,gBAAgB,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC;YAE3C,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,eAAe,CAAC,0BAA0B,CAAC,CAAC;YAE7E,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,SAAS,EAAE,GAAG,EAAE;QACvB,EAAE,CAAC,qCAAqC,EAAE,KAAK,IAAI,EAAE;YACnD,MAAM,YAAY,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC;YAErD,YAAY,CAAC,SAAS,CAAC;gBACrB,GAAG,EAAE,0BAA0B;gBAC/B,IAAI,EAAE,OAAO;gBACb,OAAO,EAAE,MAAM;gBACf,aAAa,EAAE,MAAM;gBACrB,KAAK,EAAE,MAAM;aACd,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,OAAO,EAAE,CAAC;YAE7B,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,gBAAgB,EAAE,CAAC;YACjD,MAAM,CAAC,YAAY,CAAC,eAAe,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;YAElD,4BAA4B;YAC5B,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,YAAY,EAAE,CAAC;YACpD,MAAM,CAAC,SAAS,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,YAAY,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC;YAErD,gBAAgB,CAAC,IAAI,CAAC,qBAAqB,CAAC,IAAI,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC;YAEtE,mBAAmB;YACnB,MAAM,MAAM,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC;QAChE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
package/build/index.js
CHANGED
|
@@ -6,6 +6,7 @@ process.env.CRAWLEE_LOG_LEVEL = 'OFF';
|
|
|
6
6
|
process.env.APIFY_LOG_LEVEL = 'OFF';
|
|
7
7
|
// Import and suppress Crawlee logging
|
|
8
8
|
import { log, Configuration } from 'crawlee';
|
|
9
|
+
import { randomUUID } from 'node:crypto';
|
|
9
10
|
import { setTimeout as delay } from 'node:timers/promises';
|
|
10
11
|
log.setLevel(log.LEVELS.OFF);
|
|
11
12
|
// Configure Crawlee to be silent
|
|
@@ -22,7 +23,7 @@ import { loadConfig, isValidPublicUrl, normalizeUrl } from './config.js';
|
|
|
22
23
|
import { DocsCrawler } from './crawler/docs-crawler.js';
|
|
23
24
|
import { AuthManager } from './crawler/auth.js';
|
|
24
25
|
import { fetchFavicon } from './util/favicon.js';
|
|
25
|
-
import { generateDocId } from './util/docs.js';
|
|
26
|
+
import { generateCrawlStorageId, generateDocId } from './util/docs.js';
|
|
26
27
|
import { logger } from './util/logger.js';
|
|
27
28
|
import { closeOutboundProxy } from './util/outbound-request.js';
|
|
28
29
|
import { StorageStateSchema, safeJsonParse, validateToolArgs, sanitizeErrorMessage, detectPromptInjection, wrapExternalContent, addInjectionWarnings, SessionExpiredError, AddDocumentationArgsSchema, AuthenticateArgsSchema, ClearAuthArgsSchema, SearchDocumentationArgsSchema, ReindexDocumentationArgsSchema, DeleteDocumentationArgsSchema, SetTagsArgsSchema, CreateCollectionArgsSchema, DeleteCollectionArgsSchema, UpdateCollectionArgsSchema, GetCollectionArgsSchema, AddToCollectionArgsSchema, RemoveFromCollectionArgsSchema, SearchCollectionArgsSchema, } from './util/security.js';
|
|
@@ -66,22 +67,22 @@ class WebDocsServer {
|
|
|
66
67
|
* Throttled to avoid flooding - sends on 5% increments or status changes.
|
|
67
68
|
*/
|
|
68
69
|
async sendProgressNotification(status) {
|
|
69
|
-
const registration = this.progressTokens.get(status.
|
|
70
|
+
const registration = this.progressTokens.get(status.operationId);
|
|
70
71
|
// Only send if we have a progress token from the client
|
|
71
72
|
if (!registration) {
|
|
72
|
-
logger.debug(`[Progress] No token for ${status.
|
|
73
|
+
logger.debug(`[Progress] No token for ${status.operationId}, skipping notification`);
|
|
73
74
|
return;
|
|
74
75
|
}
|
|
75
76
|
const { token: progressToken } = registration;
|
|
76
77
|
const progressPercent = Math.round(status.progress * 100);
|
|
77
|
-
const lastProgress = this.lastNotifiedProgress.get(status.
|
|
78
|
+
const lastProgress = this.lastNotifiedProgress.get(status.operationId) ?? -1;
|
|
78
79
|
// Only notify on significant progress (5% increments) or status changes
|
|
79
80
|
const isStatusChange = status.status === 'complete' || status.status === 'failed' || status.status === 'cancelled';
|
|
80
81
|
const isSignificantProgress = progressPercent - lastProgress >= 5;
|
|
81
82
|
if (!isStatusChange && !isSignificantProgress) {
|
|
82
83
|
return;
|
|
83
84
|
}
|
|
84
|
-
this.lastNotifiedProgress.set(status.
|
|
85
|
+
this.lastNotifiedProgress.set(status.operationId, progressPercent);
|
|
85
86
|
// Build human-readable message
|
|
86
87
|
let message = status.description;
|
|
87
88
|
if (status.pagesProcessed !== undefined && status.pagesFound !== undefined) {
|
|
@@ -105,9 +106,9 @@ class WebDocsServer {
|
|
|
105
106
|
logger.debug(`[Progress] Failed to send notification:`, error);
|
|
106
107
|
}
|
|
107
108
|
// Clean up tracking for completed operations
|
|
108
|
-
if (isStatusChange && this.progressTokens.get(status.
|
|
109
|
-
this.lastNotifiedProgress.delete(status.
|
|
110
|
-
this.progressTokens.delete(status.
|
|
109
|
+
if (isStatusChange && this.progressTokens.get(status.operationId) === registration) {
|
|
110
|
+
this.lastNotifiedProgress.delete(status.operationId);
|
|
111
|
+
this.progressTokens.delete(status.operationId);
|
|
111
112
|
}
|
|
112
113
|
}
|
|
113
114
|
async initialize() {
|
|
@@ -155,7 +156,7 @@ Examples where version doesn't matter: "Company engineering handbook", "AWS cons
|
|
|
155
156
|
},
|
|
156
157
|
id: {
|
|
157
158
|
type: 'string',
|
|
158
|
-
description: 'Optional
|
|
159
|
+
description: 'Optional document ID returned for display and compatibility. If not provided, an ID is auto-generated from the URL.',
|
|
159
160
|
},
|
|
160
161
|
pathPrefix: {
|
|
161
162
|
type: 'string',
|
|
@@ -519,10 +520,7 @@ Examples where version doesn't matter: "Company engineering handbook", "AWS cons
|
|
|
519
520
|
}));
|
|
520
521
|
// Handle tool calls
|
|
521
522
|
this.server.server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
522
|
-
|
|
523
|
-
// Clients can include this to receive progress notifications
|
|
524
|
-
const args = request.params.arguments;
|
|
525
|
-
const progressToken = args?._meta?.progressToken;
|
|
523
|
+
const progressToken = request.params._meta?.progressToken;
|
|
526
524
|
switch (request.params.name) {
|
|
527
525
|
case 'add_documentation':
|
|
528
526
|
return this.handleAddDocumentation(request.params.arguments, progressToken);
|
|
@@ -633,15 +631,14 @@ Examples where version doesn't matter: "Company engineering handbook", "AWS cons
|
|
|
633
631
|
if (hasExistingSession && !authOptions?.requiresAuth) {
|
|
634
632
|
logger.info(`[WebDocsServer] Found existing auth session for ${normalizedUrl}, marking document as requiring auth`);
|
|
635
633
|
}
|
|
634
|
+
const operationId = randomUUID();
|
|
636
635
|
const operation = await this.indexingQueue.runLatest(normalizedUrl, async (signal) => {
|
|
637
|
-
this.progressTokens.delete(docId);
|
|
638
|
-
this.lastNotifiedProgress.delete(docId);
|
|
639
636
|
if (progressToken !== undefined) {
|
|
640
|
-
this.progressTokens.set(
|
|
641
|
-
logger.info(`[Progress] Registered token for ${
|
|
637
|
+
this.progressTokens.set(operationId, { token: progressToken });
|
|
638
|
+
logger.info(`[Progress] Registered token for ${operationId}: ${progressToken}`);
|
|
642
639
|
}
|
|
643
|
-
this.statusTracker.startIndexing(docId, normalizedUrl, docTitle);
|
|
644
|
-
await this.indexAndAdd(
|
|
640
|
+
this.statusTracker.startIndexing(operationId, docId, normalizedUrl, docTitle);
|
|
641
|
+
await this.indexAndAdd(operationId, normalizedUrl, docTitle, false, signal, pathPrefix, authInfo, tags, version);
|
|
645
642
|
});
|
|
646
643
|
void operation.completion.catch((error) => {
|
|
647
644
|
const err = error;
|
|
@@ -657,6 +654,7 @@ Examples where version doesn't matter: "Company engineering handbook", "AWS cons
|
|
|
657
654
|
status: 'started',
|
|
658
655
|
message: `Started indexing ${normalizedUrl}`,
|
|
659
656
|
docId,
|
|
657
|
+
operationId,
|
|
660
658
|
instruction: 'IMPORTANT: You MUST call get_indexing_status repeatedly (every few seconds) until status is "complete" or "failed". Do not ask the user - just keep polling automatically.',
|
|
661
659
|
}, null, 2),
|
|
662
660
|
},
|
|
@@ -785,15 +783,14 @@ Examples where version doesn't matter: "Company engineering handbook", "AWS cons
|
|
|
785
783
|
const existingTags = doc.tags;
|
|
786
784
|
const existingVersion = doc.version;
|
|
787
785
|
const docId = generateDocId(normalizedUrl, doc.title);
|
|
786
|
+
const operationId = randomUUID();
|
|
788
787
|
const operation = await this.indexingQueue.runLatest(normalizedUrl, async (signal) => {
|
|
789
|
-
this.progressTokens.delete(docId);
|
|
790
|
-
this.lastNotifiedProgress.delete(docId);
|
|
791
788
|
if (progressToken !== undefined) {
|
|
792
|
-
this.progressTokens.set(
|
|
793
|
-
logger.info(`[Progress] Registered token for ${
|
|
789
|
+
this.progressTokens.set(operationId, { token: progressToken });
|
|
790
|
+
logger.info(`[Progress] Registered token for ${operationId}: ${progressToken}`);
|
|
794
791
|
}
|
|
795
|
-
this.statusTracker.startIndexing(docId, normalizedUrl, doc.title);
|
|
796
|
-
await this.indexAndAdd(
|
|
792
|
+
this.statusTracker.startIndexing(operationId, docId, normalizedUrl, doc.title);
|
|
793
|
+
await this.indexAndAdd(operationId, normalizedUrl, doc.title, true, signal, doc.pathPrefix, authInfo, existingTags, existingVersion);
|
|
797
794
|
});
|
|
798
795
|
void operation.completion.catch((error) => {
|
|
799
796
|
const err = error;
|
|
@@ -811,6 +808,7 @@ Examples where version doesn't matter: "Company engineering handbook", "AWS cons
|
|
|
811
808
|
? `Started re-indexing ${normalizedUrl}. Previous operation was cancelled.`
|
|
812
809
|
: `Started re-indexing ${normalizedUrl}`,
|
|
813
810
|
docId,
|
|
811
|
+
operationId,
|
|
814
812
|
instruction: 'IMPORTANT: You MUST call get_indexing_status repeatedly (every few seconds) until status is "complete" or "failed". Do not ask the user - just keep polling automatically.',
|
|
815
813
|
}, null, 2),
|
|
816
814
|
},
|
|
@@ -988,17 +986,19 @@ Examples where version doesn't matter: "Company engineering handbook", "AWS cons
|
|
|
988
986
|
await this.store.deleteDocument(normalizedUrl);
|
|
989
987
|
deletedItems.push('document metadata (SQLite)', 'vector chunks (LanceDB)');
|
|
990
988
|
logger.info(`[WebDocsServer] Deleted document from store: ${normalizedUrl}`);
|
|
991
|
-
// 2. Delete Crawlee
|
|
992
|
-
const
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
const dataset = await Dataset.open(
|
|
989
|
+
// 2. Delete both current and historical Crawlee datasets
|
|
990
|
+
const datasetIds = [generateCrawlStorageId(normalizedUrl), generateDocId(normalizedUrl, domain)];
|
|
991
|
+
const { Dataset } = await import('crawlee');
|
|
992
|
+
const cleanupResults = await Promise.allSettled(datasetIds.map(async (datasetId) => {
|
|
993
|
+
const dataset = await Dataset.open(datasetId);
|
|
996
994
|
await dataset.drop();
|
|
995
|
+
logger.info(`[WebDocsServer] Deleted Crawlee dataset: ${datasetId}`);
|
|
996
|
+
}));
|
|
997
|
+
if (cleanupResults.some((result) => result.status === 'fulfilled')) {
|
|
997
998
|
deletedItems.push('crawl cache (Crawlee dataset)');
|
|
998
|
-
logger.info(`[WebDocsServer] Deleted Crawlee dataset: ${docId}`);
|
|
999
999
|
}
|
|
1000
|
-
|
|
1001
|
-
logger.debug(`[WebDocsServer]
|
|
1000
|
+
if (cleanupResults.some((result) => result.status === 'rejected')) {
|
|
1001
|
+
logger.debug(`[WebDocsServer] Some Crawlee datasets could not be deleted`);
|
|
1002
1002
|
}
|
|
1003
1003
|
// 3. Optionally clear auth session
|
|
1004
1004
|
if (clearAuth) {
|
|
@@ -1407,13 +1407,13 @@ Examples where version doesn't matter: "Company engineering handbook", "AWS cons
|
|
|
1407
1407
|
],
|
|
1408
1408
|
};
|
|
1409
1409
|
}
|
|
1410
|
-
async indexAndAdd(
|
|
1410
|
+
async indexAndAdd(operationId, url, title, reIndex, signal, pathPrefix, authInfo, tags, version) {
|
|
1411
1411
|
// Helper to check if operation was cancelled
|
|
1412
1412
|
const checkCancelled = () => {
|
|
1413
1413
|
if (signal.aborted) {
|
|
1414
1414
|
logger.info(`[WebDocsServer] Operation cancelled for ${url}`);
|
|
1415
|
-
if (this.statusTracker.getStatus(
|
|
1416
|
-
this.statusTracker.cancelIndexing(
|
|
1415
|
+
if (this.statusTracker.getStatus(operationId)?.status !== 'cancelled') {
|
|
1416
|
+
this.statusTracker.cancelIndexing(operationId);
|
|
1417
1417
|
}
|
|
1418
1418
|
const error = new Error('Operation cancelled');
|
|
1419
1419
|
error.name = 'AbortError';
|
|
@@ -1431,7 +1431,7 @@ Examples where version doesn't matter: "Company engineering handbook", "AWS cons
|
|
|
1431
1431
|
logger.debug(`[WebDocsServer] Document exists: ${url}`);
|
|
1432
1432
|
if (!reIndex) {
|
|
1433
1433
|
logger.info(`[WebDocsServer] Document ${url} already indexed and reIndex=false`);
|
|
1434
|
-
this.statusTracker.completeIndexing(
|
|
1434
|
+
this.statusTracker.completeIndexing(operationId);
|
|
1435
1435
|
return;
|
|
1436
1436
|
}
|
|
1437
1437
|
logger.info(`[WebDocsServer] Will reindex existing document: ${url}`);
|
|
@@ -1442,7 +1442,7 @@ Examples where version doesn't matter: "Company engineering handbook", "AWS cons
|
|
|
1442
1442
|
checkCancelled();
|
|
1443
1443
|
// Start crawling
|
|
1444
1444
|
logger.info(`[WebDocsServer] Starting crawl with depth=${this.config.maxDepth}, maxRequests=${this.config.maxRequestsPerCrawl}${pathPrefix ? `, pathPrefix=${pathPrefix}` : ''}`);
|
|
1445
|
-
this.statusTracker.updateProgress(
|
|
1445
|
+
this.statusTracker.updateProgress(operationId, 0, 'Finding subpages');
|
|
1446
1446
|
const crawler = new DocsCrawler(this.config.maxDepth, this.config.maxRequestsPerCrawl, this.config.githubToken);
|
|
1447
1447
|
// Set path prefix restriction if provided
|
|
1448
1448
|
if (pathPrefix) {
|
|
@@ -1479,8 +1479,8 @@ Examples where version doesn't matter: "Company engineering handbook", "AWS cons
|
|
|
1479
1479
|
logger.debug(`[WebDocsServer] Found page ${processedPages + 1}: ${page.path}`);
|
|
1480
1480
|
processedPages++;
|
|
1481
1481
|
estimatedProgress += 1 / 2 ** processedPages;
|
|
1482
|
-
this.statusTracker.updateProgress(
|
|
1483
|
-
this.statusTracker.updateStats(
|
|
1482
|
+
this.statusTracker.updateProgress(operationId, 0.15 * estimatedProgress + Math.min(0.35, (0.35 * processedPages) / 500), `Finding subpages (${page.path})`);
|
|
1483
|
+
this.statusTracker.updateStats(operationId, { pagesFound: processedPages });
|
|
1484
1484
|
pages.push(page);
|
|
1485
1485
|
// Small delay to allow other operations
|
|
1486
1486
|
await delay(50, undefined, { signal });
|
|
@@ -1503,7 +1503,7 @@ Examples where version doesn't matter: "Company engineering handbook", "AWS cons
|
|
|
1503
1503
|
}
|
|
1504
1504
|
logger.info(`[WebDocsServer] Found ${pages.length} pages to process`);
|
|
1505
1505
|
logger.info('[WebDocsServer] Starting content processing and embedding generation');
|
|
1506
|
-
this.statusTracker.updateStats(
|
|
1506
|
+
this.statusTracker.updateStats(operationId, { pagesFound: pages.length });
|
|
1507
1507
|
checkCancelled();
|
|
1508
1508
|
// Process pages and create embeddings
|
|
1509
1509
|
const chunks = [];
|
|
@@ -1512,13 +1512,13 @@ Examples where version doesn't matter: "Company engineering handbook", "AWS cons
|
|
|
1512
1512
|
checkCancelled();
|
|
1513
1513
|
const page = pages[i];
|
|
1514
1514
|
logger.debug(`[WebDocsServer] Processing page ${i + 1}/${pages.length}: ${page.path}`);
|
|
1515
|
-
this.statusTracker.updateProgress(
|
|
1515
|
+
this.statusTracker.updateProgress(operationId, 0.5 + 0.3 * (i / pages.length), `Creating embeddings (${i + 1}/${pages.length})`);
|
|
1516
1516
|
try {
|
|
1517
1517
|
const processed = await this.processor.process(page);
|
|
1518
1518
|
logger.debug(`[WebDocsServer] Created ${processed.chunks.length} chunks for ${page.path}`);
|
|
1519
1519
|
chunks.push(...processed.chunks);
|
|
1520
1520
|
embeddings.push(...processed.chunks.map((chunk) => chunk.vector));
|
|
1521
|
-
this.statusTracker.updateStats(
|
|
1521
|
+
this.statusTracker.updateStats(operationId, {
|
|
1522
1522
|
pagesProcessed: i + 1,
|
|
1523
1523
|
chunksCreated: chunks.length,
|
|
1524
1524
|
});
|
|
@@ -1556,7 +1556,7 @@ Examples where version doesn't matter: "Company engineering handbook", "AWS cons
|
|
|
1556
1556
|
logger.warn(`[WebDocsServer] No content was extracted from ${url}`);
|
|
1557
1557
|
logger.warn(`[WebDocsServer] Pages found: ${pages.length}`);
|
|
1558
1558
|
logger.warn(`[WebDocsServer] Chunks created: ${chunks.length}`);
|
|
1559
|
-
this.statusTracker.failIndexing(
|
|
1559
|
+
this.statusTracker.failIndexing(operationId, 'No content was extracted from the pages');
|
|
1560
1560
|
return;
|
|
1561
1561
|
}
|
|
1562
1562
|
checkCancelled();
|
|
@@ -1564,7 +1564,7 @@ Examples where version doesn't matter: "Company engineering handbook", "AWS cons
|
|
|
1564
1564
|
const favicon = await fetchFavicon(new URL(url));
|
|
1565
1565
|
checkCancelled();
|
|
1566
1566
|
// Store the data with retry logic
|
|
1567
|
-
this.statusTracker.updateProgress(
|
|
1567
|
+
this.statusTracker.updateProgress(operationId, 0.9, `Storing ${embeddings.length} chunks`);
|
|
1568
1568
|
await this.storeDocumentWithRetry({
|
|
1569
1569
|
metadata: {
|
|
1570
1570
|
url,
|
|
@@ -1591,8 +1591,8 @@ Examples where version doesn't matter: "Company engineering handbook", "AWS cons
|
|
|
1591
1591
|
logger.info(`[WebDocsServer] Successfully indexed ${url}`);
|
|
1592
1592
|
logger.info(`[WebDocsServer] Pages processed: ${pages.length}`);
|
|
1593
1593
|
logger.info(`[WebDocsServer] Chunks stored: ${chunks.length}`);
|
|
1594
|
-
this.statusTracker.updateStats(
|
|
1595
|
-
this.statusTracker.completeIndexing(
|
|
1594
|
+
this.statusTracker.updateStats(operationId, { chunksCreated: chunks.length });
|
|
1595
|
+
this.statusTracker.completeIndexing(operationId);
|
|
1596
1596
|
// Optimize storage after indexing to compact data and clean up old versions
|
|
1597
1597
|
// This runs in the background and doesn't block the response
|
|
1598
1598
|
this.store.optimize().catch((err) => {
|
|
@@ -1602,8 +1602,8 @@ Examples where version doesn't matter: "Company engineering handbook", "AWS cons
|
|
|
1602
1602
|
catch (error) {
|
|
1603
1603
|
// Don't log AbortError as a real error
|
|
1604
1604
|
if (signal.aborted || (error instanceof Error && error.name === 'AbortError')) {
|
|
1605
|
-
if (signal.aborted && this.statusTracker.getStatus(
|
|
1606
|
-
this.statusTracker.cancelIndexing(
|
|
1605
|
+
if (signal.aborted && this.statusTracker.getStatus(operationId)?.status !== 'cancelled') {
|
|
1606
|
+
this.statusTracker.cancelIndexing(operationId);
|
|
1607
1607
|
}
|
|
1608
1608
|
logger.info(`[WebDocsServer] Indexing cancelled for ${url}`);
|
|
1609
1609
|
return;
|
|
@@ -1618,12 +1618,12 @@ Examples where version doesn't matter: "Company engineering handbook", "AWS cons
|
|
|
1618
1618
|
logger.info(`[WebDocsServer] Cleared expired session for ${url}`);
|
|
1619
1619
|
// Report user-friendly error
|
|
1620
1620
|
const userMessage = `Authentication session has expired. The crawler was redirected to a login page. Please use the 'authenticate' tool to log in again before re-indexing.`;
|
|
1621
|
-
this.statusTracker.failIndexing(
|
|
1621
|
+
this.statusTracker.failIndexing(operationId, userMessage);
|
|
1622
1622
|
return;
|
|
1623
1623
|
}
|
|
1624
1624
|
logger.error('[WebDocsServer] Error during indexing:', error);
|
|
1625
1625
|
logger.error('[WebDocsServer] Error details:', error instanceof Error ? error.stack : error);
|
|
1626
|
-
this.statusTracker.failIndexing(
|
|
1626
|
+
this.statusTracker.failIndexing(operationId, error instanceof Error ? error.message : 'Unknown error');
|
|
1627
1627
|
}
|
|
1628
1628
|
}
|
|
1629
1629
|
/**
|