@crawlee/utils 4.0.0-beta.40 → 4.0.0-beta.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/internals/robots.d.ts +4 -1
- package/internals/robots.d.ts.map +1 -1
- package/internals/robots.js +7 -5
- package/internals/robots.js.map +1 -1
- package/internals/sitemap.d.ts +9 -1
- package/internals/sitemap.d.ts.map +1 -1
- package/internals/sitemap.js +10 -11
- package/internals/sitemap.js.map +1 -1
- package/internals/system-info/cpu-info.d.ts +5 -1
- package/internals/system-info/cpu-info.d.ts.map +1 -1
- package/internals/system-info/cpu-info.js +17 -12
- package/internals/system-info/cpu-info.js.map +1 -1
- package/internals/system-info/memory-info.d.ts +5 -1
- package/internals/system-info/memory-info.d.ts.map +1 -1
- package/internals/system-info/memory-info.js +3 -5
- package/internals/system-info/memory-info.js.map +1 -1
- package/package.json +4 -5
package/internals/robots.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { BaseHttpClient } from '@crawlee/types';
|
|
1
|
+
import type { BaseHttpClient, CrawleeLogger } from '@crawlee/types';
|
|
2
2
|
import { Sitemap } from './sitemap.js';
|
|
3
3
|
/**
|
|
4
4
|
* Loads and queries information from a [robots.txt file](https://en.wikipedia.org/wiki/Robots.txt).
|
|
@@ -21,6 +21,7 @@ import { Sitemap } from './sitemap.js';
|
|
|
21
21
|
export declare class RobotsTxtFile {
|
|
22
22
|
private robots;
|
|
23
23
|
private proxyUrl?;
|
|
24
|
+
private logger?;
|
|
24
25
|
private constructor();
|
|
25
26
|
/**
|
|
26
27
|
* Determine the location of a robots.txt file for a URL and fetch it.
|
|
@@ -35,6 +36,7 @@ export declare class RobotsTxtFile {
|
|
|
35
36
|
timeoutMillis?: number;
|
|
36
37
|
proxyUrl?: string;
|
|
37
38
|
httpClient?: BaseHttpClient;
|
|
39
|
+
logger?: CrawleeLogger;
|
|
38
40
|
}): Promise<RobotsTxtFile>;
|
|
39
41
|
/**
|
|
40
42
|
* Allows providing the URL and robots.txt content explicitly instead of loading it from the target site.
|
|
@@ -48,6 +50,7 @@ export declare class RobotsTxtFile {
|
|
|
48
50
|
timeoutMillis?: number;
|
|
49
51
|
proxyUrl?: string;
|
|
50
52
|
httpClient?: BaseHttpClient;
|
|
53
|
+
logger?: CrawleeLogger;
|
|
51
54
|
}): Promise<RobotsTxtFile>;
|
|
52
55
|
/**
|
|
53
56
|
* Check if a URL should be crawled by robots.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"robots.d.ts","sourceRoot":"","sources":["../../src/internals/robots.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"robots.d.ts","sourceRoot":"","sources":["../../src/internals/robots.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAIpE,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC;;;;;;;;;;;;;;;;;GAiBG;AACH,qBAAa,aAAa;IAElB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,QAAQ,CAAC;IACjB,OAAO,CAAC,MAAM,CAAC;IAHnB,OAAO;IAMP;;;;;;;OAOG;WACU,IAAI,CACb,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE;QACN,MAAM,CAAC,EAAE,WAAW,CAAC;QACrB,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,UAAU,CAAC,EAAE,cAAc,CAAC;QAC5B,MAAM,CAAC,EAAE,aAAa,CAAC;KAC1B,GACF,OAAO,CAAC,aAAa,CAAC;IAQzB;;;;;OAKG;IACH,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,aAAa;qBAKpD,IAAI,CACvB,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE;QACN,MAAM,CAAC,EAAE,WAAW,CAAC;QACrB,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,UAAU,CAAC,EAAE,cAAc,CAAC;QAC5B,MAAM,CAAC,EAAE,aAAa,CAAC;KAC1B,GACF,OAAO,CAAC,aAAa,CAAC;IAgCzB;;;;OAIG;IACH,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,SAAM,GAAG,OAAO;IAIhD;;OAEG;IACH,WAAW,IAAI,MAAM,EAAE;IAIvB;;OAEG;IACG,aAAa,IAAI,OAAO,CAAC,OAAO,CAAC;IAIvC;;OAEG;IACG,qBAAqB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;CAGnD;AAGD,OAAO,EAAE,aAAa,IAAI,UAAU,EAAE,CAAC"}
|
package/internals/robots.js
CHANGED
|
@@ -22,9 +22,11 @@ import { Sitemap } from './sitemap.js';
|
|
|
22
22
|
export class RobotsTxtFile {
|
|
23
23
|
robots;
|
|
24
24
|
proxyUrl;
|
|
25
|
-
|
|
25
|
+
logger;
|
|
26
|
+
constructor(robots, proxyUrl, logger) {
|
|
26
27
|
this.robots = robots;
|
|
27
28
|
this.proxyUrl = proxyUrl;
|
|
29
|
+
this.logger = logger;
|
|
28
30
|
}
|
|
29
31
|
/**
|
|
30
32
|
* Determine the location of a robots.txt file for a URL and fetch it.
|
|
@@ -51,7 +53,7 @@ export class RobotsTxtFile {
|
|
|
51
53
|
return new RobotsTxtFile(robotsParser(url, content), proxyUrl);
|
|
52
54
|
}
|
|
53
55
|
static async load(url, options) {
|
|
54
|
-
const { proxyUrl, httpClient = new FetchHttpClient() } = options || {};
|
|
56
|
+
const { proxyUrl, logger, httpClient = new FetchHttpClient() } = options || {};
|
|
55
57
|
const response = await httpClient.sendRequest(new Request(url, { method: 'GET' }), {
|
|
56
58
|
proxyUrl,
|
|
57
59
|
timeoutMillis: options?.timeoutMillis,
|
|
@@ -68,10 +70,10 @@ export class RobotsTxtFile {
|
|
|
68
70
|
getSitemaps() {
|
|
69
71
|
return [];
|
|
70
72
|
},
|
|
71
|
-
}, proxyUrl);
|
|
73
|
+
}, proxyUrl, logger);
|
|
72
74
|
}
|
|
73
75
|
// @ts-ignore
|
|
74
|
-
return new RobotsTxtFile(robotsParser(url.toString(), await response.text()), proxyUrl);
|
|
76
|
+
return new RobotsTxtFile(robotsParser(url.toString(), await response.text()), proxyUrl, logger);
|
|
75
77
|
}
|
|
76
78
|
/**
|
|
77
79
|
* Check if a URL should be crawled by robots.
|
|
@@ -91,7 +93,7 @@ export class RobotsTxtFile {
|
|
|
91
93
|
* Parse all the sitemaps referenced in the robots file.
|
|
92
94
|
*/
|
|
93
95
|
async parseSitemaps() {
|
|
94
|
-
return Sitemap.load(this.robots.getSitemaps(), this.proxyUrl);
|
|
96
|
+
return Sitemap.load(this.robots.getSitemaps(), this.proxyUrl, { logger: this.logger });
|
|
95
97
|
}
|
|
96
98
|
/**
|
|
97
99
|
* Get all URLs from all the sitemaps referenced in the robots file. A shorthand for `(await robots.parseSitemaps()).urls`.
|
package/internals/robots.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"robots.js","sourceRoot":"","sources":["../../src/internals/robots.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAGvD,OAAO,YAAY,MAAM,eAAe,CAAC;AAEzC,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,OAAO,aAAa;IAEV;IACA;
|
|
1
|
+
{"version":3,"file":"robots.js","sourceRoot":"","sources":["../../src/internals/robots.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAGvD,OAAO,YAAY,MAAM,eAAe,CAAC;AAEzC,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,OAAO,aAAa;IAEV;IACA;IACA;IAHZ,YACY,MAAgD,EAChD,QAAiB,EACjB,MAAsB;QAFtB,WAAM,GAAN,MAAM,CAA0C;QAChD,aAAQ,GAAR,QAAQ,CAAS;QACjB,WAAM,GAAN,MAAM,CAAgB;IAC/B,CAAC;IAEJ;;;;;;;OAOG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CACb,GAAW,EACX,OAMC;QAED,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACtC,gBAAgB,CAAC,QAAQ,GAAG,aAAa,CAAC;QAC1C,gBAAgB,CAAC,MAAM,GAAG,EAAE,CAAC;QAE7B,OAAO,aAAa,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,EAAE,OAAO,CAAC,CAAC;IACpE,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,IAAI,CAAC,GAAW,EAAE,OAAe,EAAE,QAAiB;QACvD,aAAa;QACb,OAAO,IAAI,aAAa,CAAC,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC;IACnE,CAAC;IAES,MAAM,CAAC,KAAK,CAAC,IAAI,CACvB,GAAW,EACX,OAMC;QAED,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,GAAG,IAAI,eAAe,EAAE,EAAE,GAAG,OAAO,IAAI,EAAE,CAAC;QAE/E,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,WAAW,CAAC,IAAI,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE;YAC/E,QAAQ;YACR,aAAa,EAAE,OAAO,EAAE,aAAa;YACrC,MAAM,EAAE,OAAO,EAAE,MAAM;SAC1B,CAAC,CAAC;QAEH,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;YAClD,MAAM,IAAI,KAAK,CAAC,kCAAkC,GAAG,UAAU,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QACtF,CAAC;QAED,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YAC1B,OAAO,IAAI,aAAa,CACpB;gBACI,SAAS;oBACL,OAAO,IAAI,CAAC;gBAChB,CAAC;gBACD,WAAW;oBACP,OAAO,EAAE,CAAC;gBACd,CAAC;aACJ,EACD,QAAQ,EACR,MAAM,CACT,CAAC;QACN,CAAC;QAED,aAAa;QACb,OAAO,IAAI,aAAa,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;IACpG,CAAC;IAED;;;;OAIG;IACH,SAAS,CAAC,GAAW,EAAE,SAAS,GAAG,GAAG;QAClC,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,EAAE,SAAS,CAAC,IAAI,IAAI,CAAC,CAAC,+FAA+F;IACzJ,CAAC;IAED;;OAEG;IACH,WAAW;QACP,OAAO,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa;QACf,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAC3F,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,qBAAqB;QACvB,OAAO,CAAC,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC,IAAI,CAAC;IAC7C,CAAC;CACJ;AAED,+BAA+B;AAC/B,OAAO,EAAE,aAAa,IAAI,UAAU,EAAE,CAAC"}
|
package/internals/sitemap.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { BaseHttpClient } from '@crawlee/types';
|
|
1
|
+
import type { BaseHttpClient, CrawleeLogger } from '@crawlee/types';
|
|
2
2
|
interface SitemapUrlData {
|
|
3
3
|
loc: string;
|
|
4
4
|
lastmod?: Date;
|
|
@@ -47,6 +47,10 @@ export interface ParseSitemapOptions {
|
|
|
47
47
|
* Custom HTTP client to be used for fetching sitemaps.
|
|
48
48
|
*/
|
|
49
49
|
httpClient?: BaseHttpClient;
|
|
50
|
+
/**
|
|
51
|
+
* Optional logger for reporting warnings during sitemap parsing.
|
|
52
|
+
*/
|
|
53
|
+
logger?: CrawleeLogger;
|
|
50
54
|
}
|
|
51
55
|
export declare function parseSitemap<T extends ParseSitemapOptions>(initialSources: SitemapSource[], proxyUrl?: string, options?: T): AsyncIterable<T['emitNestedSitemaps'] extends true ? SitemapUrl | NestedSitemap : SitemapUrl>;
|
|
52
56
|
/**
|
|
@@ -119,6 +123,10 @@ export declare function discoverValidSitemaps(urls: string[], options?: {
|
|
|
119
123
|
* HTTP client to be used for network requests.
|
|
120
124
|
*/
|
|
121
125
|
httpClient?: BaseHttpClient;
|
|
126
|
+
/**
|
|
127
|
+
* Optional logger for reporting warnings during sitemap discovery.
|
|
128
|
+
*/
|
|
129
|
+
logger?: CrawleeLogger;
|
|
122
130
|
}): AsyncIterable<string>;
|
|
123
131
|
export {};
|
|
124
132
|
//# sourceMappingURL=sitemap.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sitemap.d.ts","sourceRoot":"","sources":["../../src/internals/sitemap.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"sitemap.d.ts","sourceRoot":"","sources":["../../src/internals/sitemap.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAQpE,UAAU,cAAc;IACpB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,CAAC,EAAE,IAAI,CAAC;IACf,UAAU,CAAC,EAAE,QAAQ,GAAG,QAAQ,GAAG,OAAO,GAAG,QAAQ,GAAG,SAAS,GAAG,QAAQ,GAAG,OAAO,CAAC;IACvF,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,MAAM,UAAU,GAAG,cAAc,GAAG;IACtC,gBAAgB,EAAE,MAAM,CAAC;CAC5B,CAAC;AAEF,UAAU,aAAa;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,gBAAgB,EAAE,IAAI,CAAC;CAC1B;AAED,KAAK,aAAa,GAAG,CAAC;IAAE,IAAI,EAAE,KAAK,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,KAAK,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,GAAG;IAAE,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AA8I5G,MAAM,WAAW,mBAAmB;IAChC;;OAEG;IACH,kBAAkB,CAAC,EAAE,IAAI,GAAG,KAAK,CAAC;IAClC;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B;;OAEG;IACH,UAAU,CAAC,EAAE,cAAc,CAAC;IAC5B;;OAEG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED,wBAAuB,YAAY,CAAC,CAAC,SAAS,mBAAmB,EAC7D,cAAc,EAAE,aAAa,EAAE,EAC/B,QAAQ,CAAC,EAAE,MAAM,EACjB,OAAO,CAAC,EAAE,CAAC,GACZ,aAAa,CAAC,CAAC,CAAC,oBAAoB,CAAC,SAAS,IAAI,GAAG,UAAU,GAAG,aAAa,GAAG,UAAU,CAAC,CAgK/F;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,OAAO;IACJ,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE;gBAAd,IAAI,EAAE,MAAM,EAAE;IAEnC;;;;;OAKG;WACU,cAAc,CACvB,GAAG,EAAE,MAAM,EACX,QAAQ,CAAC,EAAE,MAAM,EACjB,mBAAmB,CAAC,EAAE,mBAAmB,GAC1C,OAAO,CAAC,OAAO,CAAC;IAenB;;;;OAIG;WACU,IAAI,CACb,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,EACvB,QAAQ,CAAC,EAAE,MAAM,EACjB,mBAAmB,CAAC,EAAE,mBAAmB,GAC1C,OAAO,CAAC,OAAO,CAAC;IAQnB;;;;OAIG;WACU,aAAa,CACtB,OAAO,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EACjB,mBAAmB,CAAC,EAAE,mBAAmB,GAC1C,OAAO,CAAC,OAAO,CAAC;qBAII,KAAK,CACxB,OAAO,EAAE,aAAa,EAAE,EACxB,QAAQ,CAAC,EAAE,MAAM,EACjB,mBAAmB,CAAC,EAAE,mBAAmB,GAC1C,OAAO,CAAC,OAAO,CAAC;CAgBtB;AAED;;;;;;GAMG;AACH,wBAAuB,qBAAqB,CACxC,IAAI,EAAE,MAAM,EAAE,EACd,OAAO,GAAE;IACL;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;;OAKG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;OAIG;IACH,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB;;;OAGG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;;OAEG;IACH,UAAU,CAAC,EAAE,cAAc,CAAC;IAC5B;;OAEG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC;CACrB,GACP,aAAa,CAAC,MAAM,CAAC,CAkIvB"}
|
package/internals/sitemap.js
CHANGED
|
@@ -6,7 +6,6 @@ import { FetchHttpClient } from '@crawlee/http-client';
|
|
|
6
6
|
import { fileTypeStream } from 'file-type';
|
|
7
7
|
import sax from 'sax';
|
|
8
8
|
import MIMEType from 'whatwg-mimetype';
|
|
9
|
-
import log from '@apify/log';
|
|
10
9
|
import { mergeAsyncIterables } from './iterables.js';
|
|
11
10
|
import { RobotsFile } from './robots.js';
|
|
12
11
|
class SitemapTxtParser extends Transform {
|
|
@@ -125,7 +124,7 @@ class SitemapXmlParser extends Transform {
|
|
|
125
124
|
}
|
|
126
125
|
}
|
|
127
126
|
export async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
128
|
-
const { httpClient = new FetchHttpClient(), emitNestedSitemaps = false, maxDepth = Infinity, sitemapRetries = 3, timeoutMillis: timeout = 30000, reportNetworkErrors = true, } = options ?? {};
|
|
127
|
+
const { httpClient = new FetchHttpClient(), emitNestedSitemaps = false, maxDepth = Infinity, sitemapRetries = 3, timeoutMillis: timeout = 30000, reportNetworkErrors = true, logger, } = options ?? {};
|
|
129
128
|
const sources = [...initialSources];
|
|
130
129
|
const visitedSitemapUrls = new Set();
|
|
131
130
|
const createParser = (contentType = '', url) => {
|
|
@@ -147,7 +146,6 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
|
147
146
|
while (sources.length > 0) {
|
|
148
147
|
const source = sources.shift();
|
|
149
148
|
if ((source?.depth ?? 0) > maxDepth) {
|
|
150
|
-
log.debug(`Skipping sitemap ${source.type === 'url' ? source.url : ''} because it reached max depth ${maxDepth}.`);
|
|
151
149
|
continue;
|
|
152
150
|
}
|
|
153
151
|
let items = null;
|
|
@@ -214,14 +212,14 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
|
214
212
|
}
|
|
215
213
|
}
|
|
216
214
|
catch (e) {
|
|
217
|
-
|
|
215
|
+
logger?.warning(`Malformed sitemap content: ${sitemapUrl}, ${retriesLeft === 0 ? 'no retries left.' : 'retrying...'} (${e})`);
|
|
218
216
|
}
|
|
219
217
|
}
|
|
220
218
|
}
|
|
221
219
|
else if (source.type === 'raw') {
|
|
222
220
|
items = pipeline(Readable.from([source.content]), createParser('text/xml'), (error) => {
|
|
223
221
|
if (error !== undefined) {
|
|
224
|
-
|
|
222
|
+
logger?.warning(`Malformed sitemap content: ${error}`);
|
|
225
223
|
}
|
|
226
224
|
});
|
|
227
225
|
}
|
|
@@ -303,7 +301,7 @@ export class Sitemap {
|
|
|
303
301
|
}
|
|
304
302
|
}
|
|
305
303
|
catch (e) {
|
|
306
|
-
|
|
304
|
+
parseSitemapOptions?.logger?.warning(`Sitemap.load: Failed to load sitemap, returning empty result. (${e})`);
|
|
307
305
|
return new Sitemap([]);
|
|
308
306
|
}
|
|
309
307
|
return new Sitemap(urls);
|
|
@@ -317,7 +315,7 @@ export class Sitemap {
|
|
|
317
315
|
* @returns An async iterable with the discovered sitemap URLs.
|
|
318
316
|
*/
|
|
319
317
|
export async function* discoverValidSitemaps(urls, options = {}) {
|
|
320
|
-
const { proxyUrl, timeoutMillis = 60_000, signal: externalSignal, requestTimeoutMillis = 20_000, httpClient = new FetchHttpClient(), } = options;
|
|
318
|
+
const { proxyUrl, timeoutMillis = 60_000, signal: externalSignal, requestTimeoutMillis = 20_000, httpClient = new FetchHttpClient(), logger, } = options;
|
|
321
319
|
const controller = new AbortController();
|
|
322
320
|
const timeoutHandle = setTimeout(() => controller.abort(), timeoutMillis);
|
|
323
321
|
const onExternalAbort = () => controller.abort();
|
|
@@ -365,6 +363,7 @@ export async function* discoverValidSitemaps(urls, options = {}) {
|
|
|
365
363
|
timeoutMillis: requestTimeoutMillis,
|
|
366
364
|
signal,
|
|
367
365
|
httpClient,
|
|
366
|
+
logger,
|
|
368
367
|
});
|
|
369
368
|
for (const sitemapUrl of robotsFile.getSitemaps()) {
|
|
370
369
|
if (addSitemapUrl(sitemapUrl)) {
|
|
@@ -373,7 +372,7 @@ export async function* discoverValidSitemaps(urls, options = {}) {
|
|
|
373
372
|
}
|
|
374
373
|
}
|
|
375
374
|
catch (err) {
|
|
376
|
-
|
|
375
|
+
logger?.warning(`Failed to fetch robots.txt file for ${hostname}`, { error: err });
|
|
377
376
|
}
|
|
378
377
|
const sitemapUrl = domainUrls.find((url) => /sitemap\.(?:xml|txt)(?:\.gz)?$/i.test(url));
|
|
379
378
|
if (sitemapUrl !== undefined) {
|
|
@@ -394,9 +393,9 @@ export async function* discoverValidSitemaps(urls, options = {}) {
|
|
|
394
393
|
}
|
|
395
394
|
}
|
|
396
395
|
}
|
|
397
|
-
catch (
|
|
398
|
-
|
|
399
|
-
error
|
|
396
|
+
catch (error) {
|
|
397
|
+
logger?.debug(`Failed to check sitemap candidate ${candidateSitemapUrl} for ${hostname}`, {
|
|
398
|
+
error,
|
|
400
399
|
});
|
|
401
400
|
}
|
|
402
401
|
}
|
package/internals/sitemap.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sitemap.js","sourceRoot":"","sources":["../../src/internals/sitemap.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACzE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAEzC,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAEvD,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,GAAG,MAAM,KAAK,CAAC;AACtB,OAAO,QAAQ,MAAM,iBAAiB,CAAC;AAEvC,OAAO,GAAG,MAAM,YAAY,CAAC;AAE7B,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAqBzC,MAAM,gBAAiB,SAAQ,SAAS;IAC5B,OAAO,GAAkB,IAAI,aAAa,CAAC,MAAM,CAAC,CAAC;IACnD,MAAM,GAAG,EAAE,CAAC;IAEpB;QACI,KAAK,CAAC;YACF,kBAAkB,EAAE,IAAI;YACxB,SAAS,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE;gBACtC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC;gBACrD,QAAQ,EAAE,CAAC;YACf,CAAC;YACD,KAAK,EAAE,CAAC,QAAQ,EAAE,EAAE;gBAChB,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,CAAC;gBAC7C,QAAQ,EAAE,CAAC;YACf,CAAC;SACJ,CAAC,CAAC;IACP,CAAC;IAEO,aAAa,CAAC,KAAa,EAAE,QAAiB;QAClD,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC;QAErB,IAAI,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACzC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM;iBACpB,KAAK,CAAC,IAAI,CAAC;iBACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;iBAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAEvC,IAAI,QAAQ,EAAE,CAAC;gBACX,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;oBACtB,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAwB,CAAC,CAAC;gBAC/D,CAAC;gBAED,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;YACrB,CAAC;iBAAM,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACnC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAwB,CAAC,CAAC;gBAC/D,CAAC;gBAED,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC;YAChC,CAAC;QACL,CAAC;IACL,CAAC;CACJ;AAED,MAAM,gBAAiB,SAAQ,SAAS;IAC5B,OAAO,GAAkB,IAAI,aAAa,CAAC,MAAM,CAAC,CAAC;IACnD,MAAM,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IAEjC,WAAW,CAA6B;IACxC,UAAU,GAAmD,SAAS,CAAC;IACvE,GAAG,GAAwB,EAAE,CAAC;IAEtC;QACI,KAAK,CAAC;YACF,kBAAkB,EAAE,IAAI;YACxB,SAAS,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE;gBACtC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;gBAC7C,QAAQ,EAAE,CAAC;YACf,CAAC;YACD,KAAK,EAAE,CAAC,QAAQ,EAAE,EAAE;gBAChB,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;gBAChC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAClB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC5B,CAAC;gBAED,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;gBAClB,QAAQ,EAAE,CAAC;YACf,CAAC;SACJ,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClD,IAAI,CAAC,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEpD,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5C,IAAI,CAAC,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE7C,IAAI,CAAC,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClD,CAAC;IAEO,SAAS,CAAC,IAAgC;QAC9C,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YACjC,IACI,IAAI,CAAC,IAAI,KAAK,KAAK;gBACnB,IAAI,CAAC,IAAI,KAAK,SAAS;gBACvB,IAAI,CAAC,IAAI,KAAK,UAAU;gBACxB,IAAI,CAAC,IAAI,KAAK,YAAY,EAC5B,CAAC;gBACC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC;YAChC,CAAC;QACL,CAAC;QACD,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACzB,IAAI,CAAC,WAAW,GAAG,QAAQ,CAAC;QAChC,CAAC;QACD,IAAI,IAAI,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;YAC/B,IAAI,CAAC,WAAW,GAAG,cAAc,CAAC;QACtC,CAAC;IACL,CAAC;IAEO,UAAU,CAAC,IAAY;QAC3B,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,UAAU,IAAI,IAAI,KAAK,YAAY,EAAE,CAAC;YACvF,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAChC,CAAC;QAED,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,SAAS,EAAE,CAAC;YAC/C,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAwB,CAAC,CAAC;YACjF,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC;QAClB,CAAC;IACL,CAAC;IAEO,MAAM,CAAC,IAAY;QACvB,IAAI,IAAI,CAAC,UAAU,KAAK,KAAK,EAAE,CAAC;YAC5B,IAAI,IAAI,CAAC,WAAW,KAAK,cAAc,EAAE,CAAC;gBACtC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,EAAwB,CAAC,CAAC;YAC9E,CAAC;YAED,IAAI,IAAI,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;gBAChC,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC;gBAChB,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YAC/B,CAAC;QACL,CAAC;QAED,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAEnB,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YAChC,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC;QACtC,CAAC;QAED,IAAI,IAAI,CAAC,UAAU,KAAK,UAAU,EAAE,CAAC;YACjC,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QACrC,CAAC;QAED,IAAI,IAAI,CAAC,UAAU,KAAK,YAAY,EAAE,CAAC;YACnC,IAAI,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;gBACvF,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,IAAgC,CAAC;YAC3D,CAAC;QACL,CAAC;IACL,CAAC;CACJ;AA8BD,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,YAAY,CAC/B,cAA+B,EAC/B,QAAiB,EACjB,OAAW;IAEX,MAAM,EACF,UAAU,GAAG,IAAI,eAAe,EAAE,EAClC,kBAAkB,GAAG,KAAK,EAC1B,QAAQ,GAAG,QAAQ,EACnB,cAAc,GAAG,CAAC,EAClB,aAAa,EAAE,OAAO,GAAG,KAAK,EAC9B,mBAAmB,GAAG,IAAI,GAC7B,GAAG,OAAO,IAAI,EAAE,CAAC;IAElB,MAAM,OAAO,GAAG,CAAC,GAAG,cAAc,CAAC,CAAC;IACpC,MAAM,kBAAkB,GAAG,IAAI,GAAG,EAAU,CAAC;IAE7C,MAAM,YAAY,GAAG,CAAC,WAAW,GAAG,EAAE,EAAE,GAAS,EAAU,EAAE;QACzD,IAAI,QAAyB,CAAC;QAE9B,IAAI,CAAC;YACD,QAAQ,GAAG,IAAI,QAAQ,CAAC,WAAW,CAAC,CAAC;QACzC,CAAC;QAAC,MAAM,CAAC;YACL,QAAQ,GAAG,IAAI,CAAC;QACpB,CAAC;QAED,IAAI,QAAQ,EAAE,KAAK,EAAE,IAAI,GAAG,EAAE,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACtD,OAAO,IAAI,gBAAgB,EAAE,CAAC;QAClC,CAAC;QAED,IAAI,QAAQ,EAAE,OAAO,KAAK,YAAY,IAAI,GAAG,EAAE,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACvE,OAAO,IAAI,gBAAgB,EAAE,CAAC;QAClC,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,mDAAmD,WAAW,WAAW,GAAG,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAC;IACjH,CAAC,CAAC;IAEF,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,EAAG,CAAC;QAEhC,IAAI,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,CAAC,GAAG,QAAQ,EAAE,CAAC;YAClC,GAAG,CAAC,KAAK,CACL,oBAAoB,MAAM,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,iCAAiC,QAAQ,GAAG,CAC1G,CAAC;YACF,SAAS;QACb,CAAC;QAED,IAAI,KAAK,GAAsC,IAAI,CAAC;QAEpD,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;YACxB,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACvC,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC;YAC9C,IAAI,WAAW,GAAG,cAAc,GAAG,CAAC,CAAC;YAErC,OAAO,WAAW,EAAE,GAAG,CAAC,EAAE,CAAC;gBACvB,IAAI,CAAC;oBACD,IAAI,eAAgC,CAAC;oBAErC,IAAI,CAAC;wBACD,eAAe,GAAG,MAAM,UAAU,CAAC,WAAW,CAC1C,IAAI,OAAO,CAAC,UAAU,EAAE;4BACpB,MAAM,EAAE,KAAK;4BACb,OAAO,EAAE;gCACL,MAAM,EAAE,KAAK;6BAChB;yBACJ,CAAC,EACF;4BACI,QAAQ;4BACR,aAAa,EAAE,OAAO;yBACzB,CACJ,CAAC;oBACN,CAAC;oBAAC,OAAO,KAAU,EAAE,CAAC;wBAClB,eAAe,GAAG,IAAI,CAAC;oBAC3B,CAAC;oBAED,IAAI,KAAK,GAAsD,IAAI,CAAC;oBAEpE,IAAI,eAAe,IAAI,eAAe,CAAC,MAAM,IAAI,GAAG,IAAI,eAAe,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;wBACnF,IAAI,WAAW,GAAG,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;wBAE9D,IAAI,eAAe,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;4BAChC,MAAM;wBACV,CAAC;wBACD,MAAM,cAAc,GAAG,MAAM,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,eAAe,CAAC,IAAW,CAAC,CAAC,CAAC;wBAC3F,IAAI,cAAc,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;4BACxC,WAAW,GAAG,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC;wBAC/C,CAAC;wBAED,IAAI,SAAS,GAAG,KAAK,CAAC;wBAEtB,IACI,WAAW,KAAK,SAAS;4BACrB,CAAC,CAAC,WAAW,KAAK,kBAAkB;4BACpC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAC3C,CAAC;4BACC,SAAS,GAAG,IAAI,CAAC;4BAEjB,IAAI,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gCACtC,UAAU,CAAC,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,UAAU,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;4BAC3F,CAAC;wBACL,CAAC;wBAED,KAAK,GAAG,QAAQ,CACZ,cAAc,EACd,SAAS,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,IAAI,WAAW,EAAE,EAC9C,YAAY,CAAC,WAAW,IAAI,SAAS,EAAE,UAAU,CAAC,EAClD,CAAC,CAAC,EAAE,EAAE;4BACF,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;gCAChC,KAAK,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;4BACzC,CAAC;wBACL,CAAC,CACJ,CAAC;oBACN,CAAC;yBAAM,CAAC;wBACJ,KAAK,GAAG;4BACJ,IAAI,EAAE,OAAO;4BACb,KAAK,EAAE,IAAI,KAAK,CACZ,4BAA4B,UAAU,kBAAkB,eAAe,EAAE,MAAM,EAAE,CACpF;yBACJ,CAAC;oBACN,CAAC;oBAED,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;wBACjB,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,KAAK,OAAO,IAAI,CAAC,mBAAmB,CAAC;wBACzE,IAAI,CAAC,iBAAiB,EAAE,CAAC;4BACrB,MAAM,KAAK,CAAC,KAAK,CAAC;wBACtB,CAAC;oBACL,CAAC;yBAAM,CAAC;wBACJ,MAAM;oBACV,CAAC;gBACL,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACT,GAAG,CAAC,OAAO,CACP,8BAA8B,UAAU,KAAK,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,aAAa,KAAK,CAAC,GAAG,CAC/G,CAAC;gBACN,CAAC;YACL,CAAC;QACL,CAAC;aAAM,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;YAC/B,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,EAAE;gBAClF,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;oBACtB,GAAG,CAAC,OAAO,CAAC,8BAA8B,KAAK,EAAE,CAAC,CAAC;gBACvD,CAAC;YACL,CAAC,CAAC,CAAC;QACP,CAAC;QAED,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACjB,SAAS;QACb,CAAC;QAED,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YAC7B,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBAClE,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAC7E,IAAI,kBAAkB,EAAE,CAAC;oBACrB,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,gBAAgB,EAAE,IAAI,EAAS,CAAC;gBAC3D,CAAC;YACL,CAAC;YAED,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;gBACtB,MAAM;oBACF,GAAG,IAAI;oBACP,gBAAgB,EACZ,MAAM,CAAC,IAAI,KAAK,KAAK;wBACjB,CAAC,CAAC,MAAM,CAAC,GAAG;wBACZ,CAAC,CAAC,SAAS,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE;iBACpF,CAAC;YACN,CAAC;QACL,CAAC;IACL,CAAC;AACL,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,OAAO;IACK;IAArB,YAAqB,IAAc;QAAd,SAAI,GAAJ,IAAI,CAAU;IAAG,CAAC;IAEvC;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,cAAc,CACvB,GAAW,EACX,QAAiB,EACjB,mBAAyC;QAEzC,MAAM,WAAW,GAAa,EAAE,CAAC;QAEjC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAChC,UAAU,CAAC,MAAM,GAAG,EAAE,CAAC;QAEvB,UAAU,CAAC,QAAQ,GAAG,cAAc,CAAC;QACrC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC;QAExC,UAAU,CAAC,QAAQ,GAAG,cAAc,CAAC;QACrC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC;QAExC,OAAO,OAAO,CAAC,IAAI,CAAC,WAAW,EAAE,QAAQ,EAAE,EAAE,mBAAmB,EAAE,KAAK,EAAE,GAAG,mBAAmB,EAAE,CAAC,CAAC;IACvG,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CACb,IAAuB,EACvB,QAAiB,EACjB,mBAAyC;QAEzC,OAAO,MAAM,IAAI,CAAC,KAAK,CACnB,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,EAC1E,QAAQ,EACR,mBAAmB,CACtB,CAAC;IACN,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,KAAK,CAAC,aAAa,CACtB,OAAe,EACf,QAAiB,EACjB,mBAAyC;QAEzC,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,mBAAmB,CAAC,CAAC;IACvF,CAAC;IAES,MAAM,CAAC,KAAK,CAAC,KAAK,CACxB,OAAwB,EACxB,QAAiB,EACjB,mBAAyC;QAEzC,MAAM,IAAI,GAAa,EAAE,CAAC;QAE1B,IAAI,CAAC;YACD,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,YAAY,CAAC,OAAO,EAAE,QAAQ,EAAE,mBAAmB,CAAC,EAAE,CAAC;gBAC5E,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACxB,CAAC;QACL,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,GAAG,CAAC,OAAO,CAAC,kEAAkE,CAAC,GAAG,CAAC,CAAC;YACpF,OAAO,IAAI,OAAO,CAAC,EAAE,CAAC,CAAC;QAC3B,CAAC;QAED,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;CACJ;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,qBAAqB,CACxC,IAAc,EACd,UA2BI,EAAE;IAEN,MAAM,EACF,QAAQ,EACR,aAAa,GAAG,MAAM,EACtB,MAAM,EAAE,cAAc,EACtB,oBAAoB,GAAG,MAAM,EAC7B,UAAU,GAAG,IAAI,eAAe,EAAE,GACrC,GAAG,OAAO,CAAC;IACZ,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IAEzC,MAAM,aAAa,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,aAAa,CAAC,CAAC;IAC1E,MAAM,eAAe,GAAG,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IACjD,IAAI,cAAc,EAAE,CAAC;QACjB,IAAI,cAAc,CAAC,OAAO,EAAE,CAAC;YACzB,UAAU,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;aAAM,CAAC;YACJ,cAAc,CAAC,gBAAgB,CAAC,OAAO,EAAE,eAAe,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9E,CAAC;IACL,CAAC;IAED,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC;IACjC,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IAEtC,MAAM,aAAa,GAAG,CAAC,GAAW,EAAsB,EAAE;QACtD,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC;QAEpC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAErB,IAAI,WAAW,CAAC,IAAI,GAAG,UAAU,EAAE,CAAC;YAChC,OAAO,GAAG,CAAC;QACf,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC,CAAC;IAEF,MAAM,SAAS,GAAG,KAAK,EAAE,GAAW,EAAoB,EAAE;QACtD,IAAI,CAAC,UAAU,EAAE,CAAC;YACd,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,IAAI,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,WAAW,CAAC,IAAI,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,EAAE;gBAChF,QAAQ;gBACR,aAAa,EAAE,oBAAoB;gBACnC,MAAM;aACT,CAAC,CAAC;YACH,OAAO,QAAQ,CAAC,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,CAAC;QAC3D,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,6BAA6B,GAAG,KAAK,SAAS,CAAC,EAAE,QAAgB,EAAE,UAAoB;QACzF,IAAI,CAAC,QAAQ,EAAE,CAAC;YACZ,OAAO;QACX,CAAC;QAED,IAAI,CAAC;YACD,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE;gBACpD,QAAQ;gBACR,aAAa,EAAE,oBAAoB;gBACnC,MAAM;gBACN,UAAU;aACb,CAAC,CAAC;YACH,KAAK,MAAM,UAAU,IAAI,UAAU,CAAC,WAAW,EAAE,EAAE,CAAC;gBAChD,IAAI,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC5B,MAAM,UAAU,CAAC;gBACrB,CAAC;YACL,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,GAAG,CAAC,OAAO,CAAC,uCAAuC,QAAQ,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;QACnF,CAAC;QAED,MAAM,UAAU,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,iCAAiC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QAEzF,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;YAC3B,IAAI,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC5B,MAAM,UAAU,CAAC;YACrB,CAAC;QACL,CAAC;aAAM,CAAC;YACJ,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;YACxC,MAAM,wBAAwB,GAAG,CAAC,cAAc,EAAE,cAAc,EAAE,oBAAoB,CAAC,CAAC;YACxF,KAAK,MAAM,QAAQ,IAAI,wBAAwB,EAAE,CAAC;gBAC9C,QAAQ,CAAC,QAAQ,GAAG,QAAQ,CAAC;gBAC7B,MAAM,mBAAmB,GAAG,QAAQ,CAAC,QAAQ,EAAE,CAAC;gBAEhD,IAAI,CAAC;oBACD,IAAI,MAAM,SAAS,CAAC,mBAAmB,CAAC,EAAE,CAAC;wBACvC,IAAI,aAAa,CAAC,mBAAmB,CAAC,EAAE,CAAC;4BACrC,MAAM,mBAAmB,CAAC;wBAC9B,CAAC;oBACL,CAAC;gBACL,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACX,GAAG,CAAC,KAAK,CAAC,qCAAqC,mBAAmB,QAAQ,QAAQ,EAAE,EAAE;wBAClF,KAAK,EAAE,GAAG;qBACb,CAAC,CAAC;gBACP,CAAC;YACL,CAAC;QACL,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAC3B,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;QACT,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,EAAE,QAAQ,IAAI,EAAE,CAAC;QAC9C,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QACrB,GAAG,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxB,OAAO,GAAG,CAAC;IACf,CAAC,EACD,EAA8B,CACjC,CAAC;IAEF,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,UAAU,CAAC,EAAE,EAAE,CACzE,6BAA6B,CAAC,QAAQ,EAAE,UAAU,CAAC,CACtD,CAAC;IAEF,MAAM,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;IAEzC,IAAI,CAAC;QACD,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,mBAAmB,CAAC,GAAG,SAAS,CAAC,EAAE,CAAC;YACxD,IAAI,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC1B,SAAS;YACb,CAAC;YACD,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACxB,MAAM,GAAG,CAAC;QACd,CAAC;IACL,CAAC;YAAS,CAAC;QACP,YAAY,CAAC,aAAa,CAAC,CAAC;QAC5B,cAAc,EAAE,mBAAmB,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IAClE,CAAC;AACL,CAAC"}
|
|
1
|
+
{"version":3,"file":"sitemap.js","sourceRoot":"","sources":["../../src/internals/sitemap.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACzE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAEzC,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAEvD,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,GAAG,MAAM,KAAK,CAAC;AACtB,OAAO,QAAQ,MAAM,iBAAiB,CAAC;AAEvC,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAqBzC,MAAM,gBAAiB,SAAQ,SAAS;IAC5B,OAAO,GAAkB,IAAI,aAAa,CAAC,MAAM,CAAC,CAAC;IACnD,MAAM,GAAG,EAAE,CAAC;IAEpB;QACI,KAAK,CAAC;YACF,kBAAkB,EAAE,IAAI;YACxB,SAAS,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE;gBACtC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC;gBACrD,QAAQ,EAAE,CAAC;YACf,CAAC;YACD,KAAK,EAAE,CAAC,QAAQ,EAAE,EAAE;gBAChB,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,CAAC;gBAC7C,QAAQ,EAAE,CAAC;YACf,CAAC;SACJ,CAAC,CAAC;IACP,CAAC;IAEO,aAAa,CAAC,KAAa,EAAE,QAAiB;QAClD,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC;QAErB,IAAI,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACzC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM;iBACpB,KAAK,CAAC,IAAI,CAAC;iBACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;iBAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAEvC,IAAI,QAAQ,EAAE,CAAC;gBACX,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;oBACtB,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAwB,CAAC,CAAC;gBAC/D,CAAC;gBAED,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;YACrB,CAAC;iBAAM,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACnC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAwB,CAAC,CAAC;gBAC/D,CAAC;gBAED,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC;YAChC,CAAC;QACL,CAAC;IACL,CAAC;CACJ;AAED,MAAM,gBAAiB,SAAQ,SAAS;IAC5B,OAAO,GAAkB,IAAI,aAAa,CAAC,MAAM,CAAC,CAAC;IACnD,MAAM,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IAEjC,WAAW,CAA6B;IACxC,UAAU,GAAmD,SAAS,CAAC;IACvE,GAAG,GAAwB,EAAE,CAAC;IAEtC;QACI,KAAK,CAAC;YACF,kBAAkB,EAAE,IAAI;YACxB,SAAS,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE;gBACtC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;gBAC7C,QAAQ,EAAE,CAAC;YACf,CAAC;YACD,KAAK,EAAE,CAAC,QAAQ,EAAE,EAAE;gBAChB,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;gBAChC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAClB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC5B,CAAC;gBAED,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;gBAClB,QAAQ,EAAE,CAAC;YACf,CAAC;SACJ,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClD,IAAI,CAAC,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEpD,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5C,IAAI,CAAC,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE7C,IAAI,CAAC,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClD,CAAC;IAEO,SAAS,CAAC,IAAgC;QAC9C,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YACjC,IACI,IAAI,CAAC,IAAI,KAAK,KAAK;gBACnB,IAAI,CAAC,IAAI,KAAK,SAAS;gBACvB,IAAI,CAAC,IAAI,KAAK,UAAU;gBACxB,IAAI,CAAC,IAAI,KAAK,YAAY,EAC5B,CAAC;gBACC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC;YAChC,CAAC;QACL,CAAC;QACD,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACzB,IAAI,CAAC,WAAW,GAAG,QAAQ,CAAC;QAChC,CAAC;QACD,IAAI,IAAI,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;YAC/B,IAAI,CAAC,WAAW,GAAG,cAAc,CAAC;QACtC,CAAC;IACL,CAAC;IAEO,UAAU,CAAC,IAAY;QAC3B,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,UAAU,IAAI,IAAI,KAAK,YAAY,EAAE,CAAC;YACvF,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAChC,CAAC;QAED,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,SAAS,EAAE,CAAC;YAC/C,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAwB,CAAC,CAAC;YACjF,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC;QAClB,CAAC;IACL,CAAC;IAEO,MAAM,CAAC,IAAY;QACvB,IAAI,IAAI,CAAC,UAAU,KAAK,KAAK,EAAE,CAAC;YAC5B,IAAI,IAAI,CAAC,WAAW,KAAK,cAAc,EAAE,CAAC;gBACtC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,EAAwB,CAAC,CAAC;YAC9E,CAAC;YAED,IAAI,IAAI,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;gBAChC,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC;gBAChB,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YAC/B,CAAC;QACL,CAAC;QAED,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAEnB,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YAChC,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC;QACtC,CAAC;QAED,IAAI,IAAI,CAAC,UAAU,KAAK,UAAU,EAAE,CAAC;YACjC,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QACrC,CAAC;QAED,IAAI,IAAI,CAAC,UAAU,KAAK,YAAY,EAAE,CAAC;YACnC,IAAI,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;gBACvF,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,IAAgC,CAAC;YAC3D,CAAC;QACL,CAAC;IACL,CAAC;CACJ;AAkCD,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,YAAY,CAC/B,cAA+B,EAC/B,QAAiB,EACjB,OAAW;IAEX,MAAM,EACF,UAAU,GAAG,IAAI,eAAe,EAAE,EAClC,kBAAkB,GAAG,KAAK,EAC1B,QAAQ,GAAG,QAAQ,EACnB,cAAc,GAAG,CAAC,EAClB,aAAa,EAAE,OAAO,GAAG,KAAK,EAC9B,mBAAmB,GAAG,IAAI,EAC1B,MAAM,GACT,GAAG,OAAO,IAAI,EAAE,CAAC;IAElB,MAAM,OAAO,GAAG,CAAC,GAAG,cAAc,CAAC,CAAC;IACpC,MAAM,kBAAkB,GAAG,IAAI,GAAG,EAAU,CAAC;IAE7C,MAAM,YAAY,GAAG,CAAC,WAAW,GAAG,EAAE,EAAE,GAAS,EAAU,EAAE;QACzD,IAAI,QAAyB,CAAC;QAE9B,IAAI,CAAC;YACD,QAAQ,GAAG,IAAI,QAAQ,CAAC,WAAW,CAAC,CAAC;QACzC,CAAC;QAAC,MAAM,CAAC;YACL,QAAQ,GAAG,IAAI,CAAC;QACpB,CAAC;QAED,IAAI,QAAQ,EAAE,KAAK,EAAE,IAAI,GAAG,EAAE,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACtD,OAAO,IAAI,gBAAgB,EAAE,CAAC;QAClC,CAAC;QAED,IAAI,QAAQ,EAAE,OAAO,KAAK,YAAY,IAAI,GAAG,EAAE,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACvE,OAAO,IAAI,gBAAgB,EAAE,CAAC;QAClC,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,mDAAmD,WAAW,WAAW,GAAG,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAC;IACjH,CAAC,CAAC;IAEF,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,EAAG,CAAC;QAEhC,IAAI,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,CAAC,GAAG,QAAQ,EAAE,CAAC;YAClC,SAAS;QACb,CAAC;QAED,IAAI,KAAK,GAAsC,IAAI,CAAC;QAEpD,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;YACxB,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACvC,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC;YAC9C,IAAI,WAAW,GAAG,cAAc,GAAG,CAAC,CAAC;YAErC,OAAO,WAAW,EAAE,GAAG,CAAC,EAAE,CAAC;gBACvB,IAAI,CAAC;oBACD,IAAI,eAAgC,CAAC;oBAErC,IAAI,CAAC;wBACD,eAAe,GAAG,MAAM,UAAU,CAAC,WAAW,CAC1C,IAAI,OAAO,CAAC,UAAU,EAAE;4BACpB,MAAM,EAAE,KAAK;4BACb,OAAO,EAAE;gCACL,MAAM,EAAE,KAAK;6BAChB;yBACJ,CAAC,EACF;4BACI,QAAQ;4BACR,aAAa,EAAE,OAAO;yBACzB,CACJ,CAAC;oBACN,CAAC;oBAAC,OAAO,KAAU,EAAE,CAAC;wBAClB,eAAe,GAAG,IAAI,CAAC;oBAC3B,CAAC;oBAED,IAAI,KAAK,GAAsD,IAAI,CAAC;oBAEpE,IAAI,eAAe,IAAI,eAAe,CAAC,MAAM,IAAI,GAAG,IAAI,eAAe,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;wBACnF,IAAI,WAAW,GAAG,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;wBAE9D,IAAI,eAAe,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;4BAChC,MAAM;wBACV,CAAC;wBACD,MAAM,cAAc,GAAG,MAAM,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,eAAe,CAAC,IAAW,CAAC,CAAC,CAAC;wBAC3F,IAAI,cAAc,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;4BACxC,WAAW,GAAG,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC;wBAC/C,CAAC;wBAED,IAAI,SAAS,GAAG,KAAK,CAAC;wBAEtB,IACI,WAAW,KAAK,SAAS;4BACrB,CAAC,CAAC,WAAW,KAAK,kBAAkB;4BACpC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAC3C,CAAC;4BACC,SAAS,GAAG,IAAI,CAAC;4BAEjB,IAAI,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gCACtC,UAAU,CAAC,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,UAAU,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;4BAC3F,CAAC;wBACL,CAAC;wBAED,KAAK,GAAG,QAAQ,CACZ,cAAc,EACd,SAAS,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,IAAI,WAAW,EAAE,EAC9C,YAAY,CAAC,WAAW,IAAI,SAAS,EAAE,UAAU,CAAC,EAClD,CAAC,CAAC,EAAE,EAAE;4BACF,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;gCAChC,KAAK,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;4BACzC,CAAC;wBACL,CAAC,CACJ,CAAC;oBACN,CAAC;yBAAM,CAAC;wBACJ,KAAK,GAAG;4BACJ,IAAI,EAAE,OAAO;4BACb,KAAK,EAAE,IAAI,KAAK,CACZ,4BAA4B,UAAU,kBAAkB,eAAe,EAAE,MAAM,EAAE,CACpF;yBACJ,CAAC;oBACN,CAAC;oBAED,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;wBACjB,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,KAAK,OAAO,IAAI,CAAC,mBAAmB,CAAC;wBACzE,IAAI,CAAC,iBAAiB,EAAE,CAAC;4BACrB,MAAM,KAAK,CAAC,KAAK,CAAC;wBACtB,CAAC;oBACL,CAAC;yBAAM,CAAC;wBACJ,MAAM;oBACV,CAAC;gBACL,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACT,MAAM,EAAE,OAAO,CACX,8BAA8B,UAAU,KAAK,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,aAAa,KAAK,CAAC,GAAG,CAC/G,CAAC;gBACN,CAAC;YACL,CAAC;QACL,CAAC;aAAM,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;YAC/B,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,EAAE;gBAClF,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;oBACtB,MAAM,EAAE,OAAO,CAAC,8BAA8B,KAAK,EAAE,CAAC,CAAC;gBAC3D,CAAC;YACL,CAAC,CAAC,CAAC;QACP,CAAC;QAED,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACjB,SAAS;QACb,CAAC;QAED,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YAC7B,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBAClE,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAC7E,IAAI,kBAAkB,EAAE,CAAC;oBACrB,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,gBAAgB,EAAE,IAAI,EAAS,CAAC;gBAC3D,CAAC;YACL,CAAC;YAED,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;gBACtB,MAAM;oBACF,GAAG,IAAI;oBACP,gBAAgB,EACZ,MAAM,CAAC,IAAI,KAAK,KAAK;wBACjB,CAAC,CAAC,MAAM,CAAC,GAAG;wBACZ,CAAC,CAAC,SAAS,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE;iBACpF,CAAC;YACN,CAAC;QACL,CAAC;IACL,CAAC;AACL,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,OAAO;IACK;IAArB,YAAqB,IAAc;QAAd,SAAI,GAAJ,IAAI,CAAU;IAAG,CAAC;IAEvC;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,cAAc,CACvB,GAAW,EACX,QAAiB,EACjB,mBAAyC;QAEzC,MAAM,WAAW,GAAa,EAAE,CAAC;QAEjC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAChC,UAAU,CAAC,MAAM,GAAG,EAAE,CAAC;QAEvB,UAAU,CAAC,QAAQ,GAAG,cAAc,CAAC;QACrC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC;QAExC,UAAU,CAAC,QAAQ,GAAG,cAAc,CAAC;QACrC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC;QAExC,OAAO,OAAO,CAAC,IAAI,CAAC,WAAW,EAAE,QAAQ,EAAE,EAAE,mBAAmB,EAAE,KAAK,EAAE,GAAG,mBAAmB,EAAE,CAAC,CAAC;IACvG,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CACb,IAAuB,EACvB,QAAiB,EACjB,mBAAyC;QAEzC,OAAO,MAAM,IAAI,CAAC,KAAK,CACnB,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,EAC1E,QAAQ,EACR,mBAAmB,CACtB,CAAC;IACN,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,KAAK,CAAC,aAAa,CACtB,OAAe,EACf,QAAiB,EACjB,mBAAyC;QAEzC,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,mBAAmB,CAAC,CAAC;IACvF,CAAC;IAES,MAAM,CAAC,KAAK,CAAC,KAAK,CACxB,OAAwB,EACxB,QAAiB,EACjB,mBAAyC;QAEzC,MAAM,IAAI,GAAa,EAAE,CAAC;QAE1B,IAAI,CAAC;YACD,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,YAAY,CAAC,OAAO,EAAE,QAAQ,EAAE,mBAAmB,CAAC,EAAE,CAAC;gBAC5E,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACxB,CAAC;QACL,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,mBAAmB,EAAE,MAAM,EAAE,OAAO,CAChC,kEAAkE,CAAC,GAAG,CACzE,CAAC;YACF,OAAO,IAAI,OAAO,CAAC,EAAE,CAAC,CAAC;QAC3B,CAAC;QAED,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;CACJ;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,qBAAqB,CACxC,IAAc,EACd,UA+BI,EAAE;IAEN,MAAM,EACF,QAAQ,EACR,aAAa,GAAG,MAAM,EACtB,MAAM,EAAE,cAAc,EACtB,oBAAoB,GAAG,MAAM,EAC7B,UAAU,GAAG,IAAI,eAAe,EAAE,EAClC,MAAM,GACT,GAAG,OAAO,CAAC;IACZ,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IAEzC,MAAM,aAAa,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,aAAa,CAAC,CAAC;IAC1E,MAAM,eAAe,GAAG,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IACjD,IAAI,cAAc,EAAE,CAAC;QACjB,IAAI,cAAc,CAAC,OAAO,EAAE,CAAC;YACzB,UAAU,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;aAAM,CAAC;YACJ,cAAc,CAAC,gBAAgB,CAAC,OAAO,EAAE,eAAe,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9E,CAAC;IACL,CAAC;IAED,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC;IACjC,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IAEtC,MAAM,aAAa,GAAG,CAAC,GAAW,EAAsB,EAAE;QACtD,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC;QAEpC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAErB,IAAI,WAAW,CAAC,IAAI,GAAG,UAAU,EAAE,CAAC;YAChC,OAAO,GAAG,CAAC;QACf,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC,CAAC;IAEF,MAAM,SAAS,GAAG,KAAK,EAAE,GAAW,EAAoB,EAAE;QACtD,IAAI,CAAC,UAAU,EAAE,CAAC;YACd,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,IAAI,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,WAAW,CAAC,IAAI,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,EAAE;gBAChF,QAAQ;gBACR,aAAa,EAAE,oBAAoB;gBACnC,MAAM;aACT,CAAC,CAAC;YACH,OAAO,QAAQ,CAAC,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,CAAC;QAC3D,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,6BAA6B,GAAG,KAAK,SAAS,CAAC,EAAE,QAAgB,EAAE,UAAoB;QACzF,IAAI,CAAC,QAAQ,EAAE,CAAC;YACZ,OAAO;QACX,CAAC;QAED,IAAI,CAAC;YACD,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE;gBACpD,QAAQ;gBACR,aAAa,EAAE,oBAAoB;gBACnC,MAAM;gBACN,UAAU;gBACV,MAAM;aACT,CAAC,CAAC;YACH,KAAK,MAAM,UAAU,IAAI,UAAU,CAAC,WAAW,EAAE,EAAE,CAAC;gBAChD,IAAI,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC5B,MAAM,UAAU,CAAC;gBACrB,CAAC;YACL,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,MAAM,EAAE,OAAO,CAAC,uCAAuC,QAAQ,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;QACvF,CAAC;QAED,MAAM,UAAU,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,iCAAiC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QAEzF,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;YAC3B,IAAI,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC5B,MAAM,UAAU,CAAC;YACrB,CAAC;QACL,CAAC;aAAM,CAAC;YACJ,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;YACxC,MAAM,wBAAwB,GAAG,CAAC,cAAc,EAAE,cAAc,EAAE,oBAAoB,CAAC,CAAC;YACxF,KAAK,MAAM,QAAQ,IAAI,wBAAwB,EAAE,CAAC;gBAC9C,QAAQ,CAAC,QAAQ,GAAG,QAAQ,CAAC;gBAC7B,MAAM,mBAAmB,GAAG,QAAQ,CAAC,QAAQ,EAAE,CAAC;gBAEhD,IAAI,CAAC;oBACD,IAAI,MAAM,SAAS,CAAC,mBAAmB,CAAC,EAAE,CAAC;wBACvC,IAAI,aAAa,CAAC,mBAAmB,CAAC,EAAE,CAAC;4BACrC,MAAM,mBAAmB,CAAC;wBAC9B,CAAC;oBACL,CAAC;gBACL,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACb,MAAM,EAAE,KAAK,CAAC,qCAAqC,mBAAmB,QAAQ,QAAQ,EAAE,EAAE;wBACtF,KAAK;qBACR,CAAC,CAAC;gBACP,CAAC;YACL,CAAC;QACL,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAC3B,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;QACT,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,EAAE,QAAQ,IAAI,EAAE,CAAC;QAC9C,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QACrB,GAAG,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxB,OAAO,GAAG,CAAC;IACf,CAAC,EACD,EAA8B,CACjC,CAAC;IAEF,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,UAAU,CAAC,EAAE,EAAE,CACzE,6BAA6B,CAAC,QAAQ,EAAE,UAAU,CAAC,CACtD,CAAC;IAEF,MAAM,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;IAEzC,IAAI,CAAC;QACD,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,mBAAmB,CAAC,GAAG,SAAS,CAAC,EAAE,CAAC;YACxD,IAAI,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC1B,SAAS;YACb,CAAC;YACD,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACxB,MAAM,GAAG,CAAC;QACd,CAAC;IACL,CAAC;YAAS,CAAC;QACP,YAAY,CAAC,aAAa,CAAC,CAAC;QAC5B,cAAc,EAAE,mBAAmB,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IAClE,CAAC;AACL,CAAC"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { CrawleeLogger } from '@crawlee/types';
|
|
1
2
|
/**
|
|
2
3
|
* Gets the "bare metal" cpu load.
|
|
3
4
|
* Used in
|
|
@@ -60,5 +61,8 @@ export declare function sampleCpuUsage(cGroupsVersion: string): Promise<CpuSampl
|
|
|
60
61
|
* @returns a number between 0 and 1 for the cpu load
|
|
61
62
|
* @internal
|
|
62
63
|
*/
|
|
63
|
-
export declare function getCurrentCpuTicksV2(
|
|
64
|
+
export declare function getCurrentCpuTicksV2(options?: {
|
|
65
|
+
containerized?: boolean;
|
|
66
|
+
logger?: CrawleeLogger;
|
|
67
|
+
}): Promise<number>;
|
|
64
68
|
//# sourceMappingURL=cpu-info.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cpu-info.d.ts","sourceRoot":"","sources":["../../../src/internals/system-info/cpu-info.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"cpu-info.d.ts","sourceRoot":"","sources":["../../../src/internals/system-info/cpu-info.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAyBpD;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,WAejC;AAcD;;;;;;;GAOG;AACH,wBAAsB,WAAW,CAAC,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAahF;AAED;;;;;GAKG;AACH,wBAAsB,YAAY,CAAC,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAU1E;AAED;;;;;;GAMG;AACH,wBAAsB,oBAAoB,CAAC,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAkBlF;AAED;;;;;GAKG;AACH,wBAAsB,iBAAiB,oBAiBtC;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;CACvB;AAED;;;;;GAKG;AACH,wBAAsB,cAAc,CAAC,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,CAM/E;AAID;;;;;;GAMG;AACH,wBAAsB,oBAAoB,CACtC,OAAO,GAAE;IAAE,aAAa,CAAC,EAAE,OAAO,CAAC;IAAC,MAAM,CAAC,EAAE,aAAa,CAAA;CAAO,GAClE,OAAO,CAAC,MAAM,CAAC,CAsDjB"}
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { execSync } from 'node:child_process';
|
|
2
2
|
import { readFile } from 'node:fs/promises';
|
|
3
3
|
import os from 'node:os';
|
|
4
|
-
import log from '@apify/log';
|
|
5
4
|
import { getCgroupsVersion } from '../general.js';
|
|
6
5
|
const CPU_FILE_PATHS = {
|
|
7
6
|
STAT: {
|
|
@@ -44,17 +43,15 @@ export function getCurrentCpuTicks() {
|
|
|
44
43
|
return totalTicksDelta ? 1 - idleTicksDelta / totalTicksDelta : 0;
|
|
45
44
|
}
|
|
46
45
|
/**
|
|
47
|
-
* Reads the linux tick rate
|
|
48
|
-
* @returns the number of ticks per second
|
|
46
|
+
* Reads the linux tick rate.
|
|
47
|
+
* @returns the number of ticks per second, or `null` if detection failed
|
|
49
48
|
*/
|
|
50
49
|
function getClockTicks() {
|
|
51
50
|
try {
|
|
52
|
-
|
|
53
|
-
return parseInt(result, 10);
|
|
51
|
+
return parseInt(execSync('getconf CLK_TCK').toString().trim(), 10);
|
|
54
52
|
}
|
|
55
|
-
catch
|
|
56
|
-
|
|
57
|
-
return 100;
|
|
53
|
+
catch {
|
|
54
|
+
return null;
|
|
58
55
|
}
|
|
59
56
|
}
|
|
60
57
|
/**
|
|
@@ -167,7 +164,8 @@ let previousSample = { containerUsage: 0, systemUsage: 0 };
|
|
|
167
164
|
* @returns a number between 0 and 1 for the cpu load
|
|
168
165
|
* @internal
|
|
169
166
|
*/
|
|
170
|
-
export async function getCurrentCpuTicksV2(
|
|
167
|
+
export async function getCurrentCpuTicksV2(options = {}) {
|
|
168
|
+
const { containerized = false, logger } = options;
|
|
171
169
|
try {
|
|
172
170
|
// if not containerized
|
|
173
171
|
if (!containerized) {
|
|
@@ -175,13 +173,20 @@ export async function getCurrentCpuTicksV2(containerized = false) {
|
|
|
175
173
|
return getCurrentCpuTicks();
|
|
176
174
|
}
|
|
177
175
|
if (!CLOCK_TICKS_CHECKED) {
|
|
178
|
-
|
|
176
|
+
const ticks = getClockTicks();
|
|
177
|
+
if (ticks === null) {
|
|
178
|
+
CLOCK_TICKS_PER_SECOND = 100;
|
|
179
|
+
logger?.warning('Failed to get clock ticks; defaulting to 100. CPU metrics may be inaccurate.');
|
|
180
|
+
}
|
|
181
|
+
else {
|
|
182
|
+
CLOCK_TICKS_PER_SECOND = ticks;
|
|
183
|
+
}
|
|
179
184
|
CLOCK_TICKS_CHECKED = true;
|
|
180
185
|
}
|
|
181
186
|
const cgroupsVersion = await getCgroupsVersion();
|
|
182
187
|
// if cgroup is not detected, return bare metal cpu limit
|
|
183
188
|
if (cgroupsVersion === null) {
|
|
184
|
-
|
|
189
|
+
logger?.warningOnce('Your environment is containerized, but your system does not support cgroups.\n' +
|
|
185
190
|
"If you're running containers with limited cpu, cpu auto-scaling will not work properly.");
|
|
186
191
|
return getCurrentCpuTicks();
|
|
187
192
|
}
|
|
@@ -204,7 +209,7 @@ export async function getCurrentCpuTicksV2(containerized = false) {
|
|
|
204
209
|
}
|
|
205
210
|
catch (err) {
|
|
206
211
|
// if anything fails, default to bare metal metrics
|
|
207
|
-
|
|
212
|
+
logger?.warning('Cpu snapshot failed, falling back to bare-metal metrics.', { error: err });
|
|
208
213
|
return getCurrentCpuTicks();
|
|
209
214
|
}
|
|
210
215
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cpu-info.js","sourceRoot":"","sources":["../../../src/internals/system-info/cpu-info.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"cpu-info.js","sourceRoot":"","sources":["../../../src/internals/system-info/cpu-info.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,MAAM,SAAS,CAAC;AAIzB,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAElD,MAAM,cAAc,GAAG;IACnB,IAAI,EAAE;QACF,EAAE,EAAE,sCAAsC;QAC1C,EAAE,EAAE,yBAAyB;KAChC;IACD,KAAK,EAAE;QACH,EAAE,EAAE,qCAAqC;QACzC,EAAE,EAAE,wBAAwB;KAC/B;IACD,MAAM,EAAE;QACJ,EAAE,EAAE,sCAAsC;QAC1C,EAAE,EAAE,wBAAwB;KAC/B;CACJ,CAAC;AAEF,IAAI,sBAAsB,GAAG,GAAG,CAAC;AACjC,IAAI,mBAAmB,GAAG,KAAK,CAAC;AAEhC,MAAM,sBAAsB,GAAG,GAAG,CAAC;AAEnC,MAAM,aAAa,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;AAC5C;;;;;;;;GAQG;AACH,MAAM,UAAU,kBAAkB;IAC9B,MAAM,SAAS,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC;IAC5B,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAC1B,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;QACT,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC1C,OAAO;YACH,IAAI,EAAE,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI;YAC/B,KAAK,EAAE,GAAG,CAAC,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC;SAC9D,CAAC;IACN,CAAC,EACD,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CACxB,CAAC;IACF,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,GAAG,aAAc,CAAC,IAAI,CAAC;IACxD,MAAM,eAAe,GAAG,KAAK,CAAC,KAAK,GAAG,aAAc,CAAC,KAAK,CAAC;IAC3D,OAAO,eAAe,CAAC,CAAC,CAAC,CAAC,GAAG,cAAc,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC;AACtE,CAAC;AAED;;;GAGG;AACH,SAAS,aAAa;IAClB,IAAI,CAAC;QACD,OAAO,QAAQ,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IACvE,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,IAAI,CAAC;IAChB,CAAC;AACL,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,cAAsB;IACpD,IAAI,cAAc,KAAK,IAAI,EAAE,CAAC;QAC1B,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,cAAc,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QACjE,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;QAC5C,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;IACvC,CAAC;IACD,YAAY;IACZ,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,cAAc,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IAC/D,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACzC,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,KAAK,EAAE,CAAC;QACrB,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAClC,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,cAAsB;IACrD,IAAI,cAAc,KAAK,IAAI,EAAE,CAAC;QAC1B,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,cAAc,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QAClE,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;QAC5C,OAAO,KAAK,CAAC;IACjB,CAAC;IACD,YAAY;IACZ,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,cAAc,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IAChE,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACzC,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAClC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,cAAsB;IAC7D,IAAI,cAAc,KAAK,IAAI,EAAE,CAAC;QAC1B,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,cAAc,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QAC5D,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IAC/B,CAAC;IACD,YAAY;IACZ,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,cAAc,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IAC5D,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/B,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACvC,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,YAAY,EAAE,CAAC;YAC5B,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAC7B,MAAM;QACV,CAAC;IACL,CAAC;IACD,uCAAuC;IACvC,OAAO,SAAS,GAAG,IAAI,CAAC;AAC5B,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB;IACnC,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IACtD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1B,6DAA6D;YAC7D,iDAAiD;YACjD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAC5C,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACvB,UAAU,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC;YAC/B,CAAC;YACD,sCAAsC;YACtC,OAAO,CAAC,UAAU,GAAG,sBAAsB,CAAC,GAAG,sBAAsB,CAAC;QAC1E,CAAC;IACL,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,uBAAuB;AAC3D,CAAC;AAUD;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,cAAsB;IACvD,MAAM,CAAC,cAAc,EAAE,WAAW,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QACpD,oBAAoB,CAAC,cAAc,CAAC;QACpC,iBAAiB,EAAE;KACtB,CAAC,CAAC;IACH,OAAO,EAAE,cAAc,EAAE,WAAW,EAAE,CAAC;AAC3C,CAAC;AAED,IAAI,cAAc,GAAc,EAAE,cAAc,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC;AAEtE;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACtC,UAA+D,EAAE;IAEjE,MAAM,EAAE,aAAa,GAAG,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAElD,IAAI,CAAC;QACD,uBAAuB;QACvB,IAAI,CAAC,aAAa,EAAE,CAAC;YACjB,uBAAuB;YACvB,OAAO,kBAAkB,EAAE,CAAC;QAChC,CAAC;QACD,IAAI,CAAC,mBAAmB,EAAE,CAAC;YACvB,MAAM,KAAK,GAAG,aAAa,EAAE,CAAC;YAC9B,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBACjB,sBAAsB,GAAG,GAAG,CAAC;gBAC7B,MAAM,EAAE,OAAO,CAAC,8EAA8E,CAAC,CAAC;YACpG,CAAC;iBAAM,CAAC;gBACJ,sBAAsB,GAAG,KAAK,CAAC;YACnC,CAAC;YACD,mBAAmB,GAAG,IAAI,CAAC;QAC/B,CAAC;QACD,MAAM,cAAc,GAAG,MAAM,iBAAiB,EAAE,CAAC;QACjD,yDAAyD;QACzD,IAAI,cAAc,KAAK,IAAI,EAAE,CAAC;YAC1B,MAAM,EAAE,WAAW,CACf,gFAAgF;gBAC5E,yFAAyF,CAChG,CAAC;YACF,OAAO,kBAAkB,EAAE,CAAC;QAChC,CAAC;QACD,yFAAyF;QACzF,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC,cAAe,CAAC,CAAC;QACjD,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACjB,wCAAwC;YACxC,OAAO,kBAAkB,EAAE,CAAC;QAChC,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,cAAe,CAAC,CAAC;QACnD,kFAAkF;QAClF,MAAM,YAAY,GAAG,KAAK,GAAG,MAAM,CAAC;QAEpC,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,cAAe,CAAC,CAAC;QAErD,MAAM,cAAc,GAAG,MAAM,CAAC,cAAc,GAAG,cAAc,CAAC,cAAc,CAAC;QAC7E,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,GAAG,cAAc,CAAC,WAAW,CAAC;QAEpE,cAAc,GAAG,MAAM,CAAC;QAExB,MAAM,OAAO,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC;QAEjC,sCAAsC;QACtC,OAAO,CAAC,CAAC,cAAc,GAAG,WAAW,CAAC,GAAG,OAAO,CAAC,GAAG,YAAY,CAAC;IACrE,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACX,mDAAmD;QACnD,MAAM,EAAE,OAAO,CAAC,0DAA0D,EAAE,EAAE,KAAK,EAAE,GAAY,EAAE,CAAC,CAAC;QACrG,OAAO,kBAAkB,EAAE,CAAC;IAChC,CAAC;AACL,CAAC"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { CrawleeLogger } from '@crawlee/types';
|
|
1
2
|
/**
|
|
2
3
|
* Describes memory usage of the process.
|
|
3
4
|
*/
|
|
@@ -24,5 +25,8 @@ export interface MemoryInfo {
|
|
|
24
25
|
* @returns An object containing the free and used memory metrics.
|
|
25
26
|
* @internal
|
|
26
27
|
*/
|
|
27
|
-
export declare function getMemoryInfo(
|
|
28
|
+
export declare function getMemoryInfo(options?: {
|
|
29
|
+
containerized?: boolean;
|
|
30
|
+
logger?: CrawleeLogger;
|
|
31
|
+
}): Promise<MemoryInfo>;
|
|
28
32
|
//# sourceMappingURL=memory-info.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"memory-info.d.ts","sourceRoot":"","sources":["../../../src/internals/system-info/memory-info.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"memory-info.d.ts","sourceRoot":"","sources":["../../../src/internals/system-info/memory-info.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAgBpD;;GAEG;AACH,MAAM,WAAW,UAAU;IACvB,wDAAwD;IACxD,UAAU,EAAE,MAAM,CAAC;IAEnB,uDAAuD;IACvD,SAAS,EAAE,MAAM,CAAC;IAElB,uDAAuD;IACvD,SAAS,EAAE,MAAM,CAAC;IAElB,wDAAwD;IACxD,gBAAgB,EAAE,MAAM,CAAC;IAEzB,8EAA8E;IAC9E,mBAAmB,EAAE,MAAM,CAAC;CAC/B;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,aAAa,CAC/B,OAAO,GAAE;IAAE,aAAa,CAAC,EAAE,OAAO,CAAC;IAAC,MAAM,CAAC,EAAE,aAAa,CAAA;CAAO,GAClE,OAAO,CAAC,UAAU,CAAC,CA8FrB"}
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { execSync } from 'node:child_process';
|
|
2
2
|
import { readFile } from 'node:fs/promises';
|
|
3
3
|
import { freemem, totalmem } from 'node:os';
|
|
4
|
-
import log from '@apify/log';
|
|
5
4
|
import { getCgroupsVersion, isLambda } from '../general.js';
|
|
6
5
|
import { psTree } from './ps-tree.js';
|
|
7
6
|
const MEMORY_FILE_PATHS = {
|
|
@@ -25,7 +24,8 @@ const MEMORY_FILE_PATHS = {
|
|
|
25
24
|
* @returns An object containing the free and used memory metrics.
|
|
26
25
|
* @internal
|
|
27
26
|
*/
|
|
28
|
-
export async function getMemoryInfo(
|
|
27
|
+
export async function getMemoryInfo(options = {}) {
|
|
28
|
+
const { containerized = false, logger } = options;
|
|
29
29
|
let mainProcessBytes = -1;
|
|
30
30
|
let childProcessesBytes = 0;
|
|
31
31
|
// lambda does *not* have `ps` and other command line tools
|
|
@@ -60,7 +60,6 @@ export async function getMemoryInfo(containerized = false) {
|
|
|
60
60
|
totalBytes = parseInt(process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE, 10) * 1000000;
|
|
61
61
|
usedBytes = mainProcessBytes + childProcessesBytes;
|
|
62
62
|
freeBytes = totalBytes - usedBytes;
|
|
63
|
-
log.debug(`lambda size of ${totalBytes} with ${freeBytes} free bytes`);
|
|
64
63
|
}
|
|
65
64
|
else if (containerized) {
|
|
66
65
|
// When running inside a container, use container memory limits
|
|
@@ -93,8 +92,7 @@ export async function getMemoryInfo(containerized = false) {
|
|
|
93
92
|
freeBytes = totalBytes - usedBytes;
|
|
94
93
|
}
|
|
95
94
|
catch (err) {
|
|
96
|
-
|
|
97
|
-
log.deprecated('Your environment is containerized, but your system does not support memory cgroups. ' +
|
|
95
|
+
logger?.warningOnce('Your environment is containerized, but your system does not support memory cgroups. ' +
|
|
98
96
|
"If you're running containers with limited memory, memory auto-scaling will not work properly.\n\n" +
|
|
99
97
|
`Cause: ${err.message}`);
|
|
100
98
|
totalBytes = totalmem();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"memory-info.js","sourceRoot":"","sources":["../../../src/internals/system-info/memory-info.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"memory-info.js","sourceRoot":"","sources":["../../../src/internals/system-info/memory-info.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAI5C,OAAO,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC5D,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAEtC,MAAM,iBAAiB,GAAG;IACtB,KAAK,EAAE;QACH,EAAE,EAAE,6CAA6C;QACjD,EAAE,EAAE,2BAA2B;KAClC;IACD,IAAI,EAAE;QACF,EAAE,EAAE,6CAA6C;QACjD,EAAE,EAAE,+BAA+B;KACtC;CACJ,CAAC;AAsBF;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAC/B,UAA+D,EAAE;IAEjE,MAAM,EAAE,aAAa,GAAG,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAClD,IAAI,gBAAgB,GAAG,CAAC,CAAC,CAAC;IAC1B,IAAI,mBAAmB,GAAG,CAAC,CAAC;IAE5B,2DAA2D;IAC3D,oCAAoC;IACpC,IAAI,QAAQ,EAAE,EAAE,CAAC;QACb,oBAAoB;QACpB,gBAAgB,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,GAAG,CAAC;QAE7C,8CAA8C;QAC9C,MAAM,OAAO,GAAG,QAAQ,CAAC,mBAAmB,CAAC,CAAC,QAAQ,EAAE,CAAC;QACzD,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;QACnE,uFAAuF;QACvF,qGAAqG;QACrG,mBAAmB,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,IAAI,GAAG,gBAAgB,CAAC;IAChE,CAAC;SAAM,CAAC;QACJ,sCAAsC;QACtC,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAElD,SAAS,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;YACtB,yCAAyC;YACzC,IAAI,GAAG,CAAC,GAAG,KAAK,GAAG,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC;gBAC/B,gBAAgB,GAAG,GAAG,CAAC,GAAG,CAAC;gBAC3B,OAAO;YACX,CAAC;YACD,mBAAmB,IAAI,GAAG,CAAC,GAAG,CAAC;QACnC,CAAC,CAAC,CAAC;IACP,CAAC;IAED,IAAI,UAAkB,CAAC;IACvB,IAAI,SAAiB,CAAC;IACtB,IAAI,SAAiB,CAAC;IAEtB,IAAI,QAAQ,EAAE,EAAE,CAAC;QACb,sCAAsC;QACtC,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,+BAAgC,EAAE,EAAE,CAAC,GAAG,OAAO,CAAC;QAClF,SAAS,GAAG,gBAAgB,GAAG,mBAAmB,CAAC;QACnD,SAAS,GAAG,UAAU,GAAG,SAAS,CAAC;IACvC,CAAC;SAAM,IAAI,aAAa,EAAE,CAAC;QACvB,+DAA+D;QAE/D,MAAM,cAAc,GAAG,MAAM,iBAAiB,EAAE,CAAC;QAEjD,IAAI,CAAC;YACD,IAAI,cAAc,KAAK,IAAI,EAAE,CAAC;gBAC1B,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;YAC5C,CAAC;YACD,IAAI,CAAC,aAAa,EAAE,YAAY,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;gBAClD,QAAQ,CAAC,iBAAiB,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;gBACzD,QAAQ,CAAC,iBAAiB,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;aAC3D,CAAC,CAAC;YAEH,gHAAgH;YAChH,aAAa,GAAG,aAAa,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;YAC5D,YAAY,GAAG,YAAY,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;YAE1D,4DAA4D;YAC5D,oIAAoI;YACpI,IAAI,aAAa,KAAK,KAAK,EAAE,CAAC;gBAC1B,UAAU,GAAG,QAAQ,EAAE,CAAC;gBACxB,yFAAyF;gBACzF,8CAA8C;YAClD,CAAC;iBAAM,CAAC;gBACJ,UAAU,GAAG,QAAQ,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;gBACzC,MAAM,gCAAgC,GAAG,UAAU,GAAG,MAAM,CAAC,gBAAgB,CAAC;gBAC9E,IAAI,gCAAgC;oBAAE,UAAU,GAAG,QAAQ,EAAE,CAAC;YAClE,CAAC;YACD,SAAS,GAAG,QAAQ,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;YACvC,SAAS,GAAG,UAAU,GAAG,SAAS,CAAC;QACvC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,MAAM,EAAE,WAAW,CACf,sFAAsF;gBAClF,mGAAmG;gBACnG,UAAW,GAAa,CAAC,OAAO,EAAE,CACzC,CAAC;YACF,UAAU,GAAG,QAAQ,EAAE,CAAC;YACxB,SAAS,GAAG,OAAO,EAAE,CAAC;YACtB,SAAS,GAAG,UAAU,GAAG,SAAS,CAAC;QACvC,CAAC;IACL,CAAC;SAAM,CAAC;QACJ,UAAU,GAAG,QAAQ,EAAE,CAAC;QACxB,SAAS,GAAG,OAAO,EAAE,CAAC;QACtB,SAAS,GAAG,UAAU,GAAG,SAAS,CAAC;IACvC,CAAC;IAED,OAAO;QACH,UAAU;QACV,SAAS;QACT,SAAS;QACT,gBAAgB;QAChB,mBAAmB;KACtB,CAAC;AACN,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/utils",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.41",
|
|
4
4
|
"description": "A set of shared utilities that can be used by crawlers",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -41,10 +41,9 @@
|
|
|
41
41
|
"copy": "tsx ../../scripts/copy.ts"
|
|
42
42
|
},
|
|
43
43
|
"dependencies": {
|
|
44
|
-
"@apify/log": "^2.5.18",
|
|
45
44
|
"@apify/ps-tree": "^1.2.0",
|
|
46
|
-
"@crawlee/http-client": "4.0.0-beta.
|
|
47
|
-
"@crawlee/types": "4.0.0-beta.
|
|
45
|
+
"@crawlee/http-client": "4.0.0-beta.41",
|
|
46
|
+
"@crawlee/types": "4.0.0-beta.41",
|
|
48
47
|
"@types/sax": "^1.2.7",
|
|
49
48
|
"cheerio": "^1.0.0",
|
|
50
49
|
"domhandler": "^5.0.3",
|
|
@@ -62,5 +61,5 @@
|
|
|
62
61
|
}
|
|
63
62
|
}
|
|
64
63
|
},
|
|
65
|
-
"gitHead": "
|
|
64
|
+
"gitHead": "0f3d1a433e1a27647d99ad5dc81a605e82505d1a"
|
|
66
65
|
}
|