@crawlee/utils 3.13.5 → 4.0.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +15 -16
- package/index.d.ts.map +1 -1
- package/index.js +15 -22
- package/index.js.map +1 -1
- package/internals/blocked.js +4 -7
- package/internals/blocked.js.map +1 -1
- package/internals/cheerio.d.ts +4 -5
- package/internals/cheerio.d.ts.map +1 -1
- package/internals/cheerio.js +9 -17
- package/internals/cheerio.js.map +1 -1
- package/internals/chunk.js +1 -4
- package/internals/chunk.js.map +1 -1
- package/internals/debug.js +5 -9
- package/internals/debug.js.map +1 -1
- package/internals/extract-urls.js +17 -23
- package/internals/extract-urls.js.map +1 -1
- package/internals/general.js +19 -31
- package/internals/general.js.map +1 -1
- package/internals/memory-info.js +22 -26
- package/internals/memory-info.js.map +1 -1
- package/internals/open_graph_parser.js +3 -6
- package/internals/open_graph_parser.js.map +1 -1
- package/internals/robots.d.ts +1 -1
- package/internals/robots.d.ts.map +1 -1
- package/internals/robots.js +16 -26
- package/internals/robots.js.map +1 -1
- package/internals/sitemap.d.ts.map +1 -1
- package/internals/sitemap.js +28 -73
- package/internals/sitemap.js.map +1 -1
- package/internals/social.js +37 -46
- package/internals/social.js.map +1 -1
- package/internals/systemInfoV2/cpu-info.js +26 -36
- package/internals/systemInfoV2/cpu-info.js.map +1 -1
- package/internals/systemInfoV2/memory-info.js +22 -26
- package/internals/systemInfoV2/memory-info.js.map +1 -1
- package/internals/systemInfoV2/ps-tree.js +5 -9
- package/internals/systemInfoV2/ps-tree.js.map +1 -1
- package/internals/typedefs.js +2 -6
- package/internals/typedefs.js.map +1 -1
- package/internals/url.js +1 -4
- package/internals/url.js.map +1 -1
- package/package.json +13 -22
- package/tsconfig.build.tsbuildinfo +1 -1
- package/index.mjs +0 -36
- package/internals/gotScraping.d.ts +0 -5
- package/internals/gotScraping.d.ts.map +0 -1
- package/internals/gotScraping.js +0 -10
- package/internals/gotScraping.js.map +0 -1
|
@@ -1,9 +1,5 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
exports.psTree = psTree;
|
|
4
|
-
const tslib_1 = require("tslib");
|
|
5
|
-
const node_child_process_1 = require("node:child_process");
|
|
6
|
-
const readline = tslib_1.__importStar(require("node:readline"));
|
|
1
|
+
import { spawn } from 'node:child_process';
|
|
2
|
+
import * as readline from 'node:readline';
|
|
7
3
|
/**
|
|
8
4
|
* Returns a promise that resolves with an array of ProcessInfo objects representing
|
|
9
5
|
* the children of the given PID.
|
|
@@ -13,21 +9,21 @@ const readline = tslib_1.__importStar(require("node:readline"));
|
|
|
13
9
|
* Defaults to false.
|
|
14
10
|
* @internal
|
|
15
11
|
*/
|
|
16
|
-
async function psTree(pid, includeRoot = false) {
|
|
12
|
+
export async function psTree(pid, includeRoot = false) {
|
|
17
13
|
return new Promise((resolve, reject) => {
|
|
18
14
|
if (typeof pid === 'number') {
|
|
19
15
|
pid = pid.toString();
|
|
20
16
|
}
|
|
21
17
|
let processLister;
|
|
22
18
|
if (process.platform === 'win32') {
|
|
23
|
-
processLister =
|
|
19
|
+
processLister = spawn('powershell', [
|
|
24
20
|
'-NoProfile',
|
|
25
21
|
'-Command',
|
|
26
22
|
'Get-CimInstance Win32_Process | Format-Table ProcessId,ParentProcessId,WorkingSetSize,Name',
|
|
27
23
|
]);
|
|
28
24
|
}
|
|
29
25
|
else {
|
|
30
|
-
processLister =
|
|
26
|
+
processLister = spawn('ps', ['-A', '-o', 'ppid,pid,stat,rss,comm']);
|
|
31
27
|
}
|
|
32
28
|
processLister.on('error', reject);
|
|
33
29
|
if (!processLister.stdout) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ps-tree.js","sourceRoot":"","sources":["../../../src/internals/systemInfoV2/ps-tree.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"ps-tree.js","sourceRoot":"","sources":["../../../src/internals/systemInfoV2/ps-tree.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,KAAK,QAAQ,MAAM,eAAe,CAAC;AAe1C;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,GAAoB,EAAE,WAAW,GAAG,KAAK;IAClE,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACnC,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;YAC1B,GAAG,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC;QACzB,CAAC;QAED,IAAI,aAA2B,CAAC;QAChC,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;YAC/B,aAAa,GAAG,KAAK,CAAC,YAAY,EAAE;gBAChC,YAAY;gBACZ,UAAU;gBACV,4FAA4F;aAC/F,CAAC,CAAC;QACP,CAAC;aAAM,CAAC;YACJ,aAAa,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,wBAAwB,CAAC,CAAC,CAAC;QACxE,CAAC;QAED,aAAa,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAElC,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC;YACxB,MAAM,CAAC,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC,CAAC;YAClD,OAAO;QACX,CAAC;QAED,8DAA8D;QAC9D,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC;YAChC,KAAK,EAAE,aAAa,CAAC,MAAM;SAC9B,CAAC,CAAC;QAEH,MAAM,IAAI,GAAkB,EAAE,CAAC;QAC/B,IAAI,OAAO,GAAoB,IAAI,CAAC;QAEpC,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAY,EAAE,EAAE;YAC3B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YAC5B,IAAI,OAAO,KAAK,EAAE,EAAE,CAAC;gBACjB,OAAO,CAAC,oBAAoB;YAChC,CAAC;YAED,4DAA4D;YAC5D,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACpC,IAAI,OAAO,KAAK,IAAI,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;gBAClE,OAAO;YACX,CAAC;YAED,2DAA2D;YAC3D,IAAI,CAAC,OAAO,EAAE,CAAC;gBACX,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;gBACtC,OAAO;YACX,CAAC;YAED,4CAA4C;YAC5C,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC;YAE/B,gCAAgC;YAChC,MAAM,GAAG,GAAyB,EAAE,CAAC;YACrC,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;YAC7B,qEAAqE;YACrE,gGAAgG;YAChG,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;gBAC3C,IAAI,KAAa,CAAC;gBAClB,IAAI,KAAK,KAAK,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC5B,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC9B,CAAC;qBAAM,CAAC;oBACJ,KAAK,GAAG,OAAO,CAAC,KAAK,EAAG,CAAC;gBAC7B,CAAC;gBAED,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;oBACnB,GAAG,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;oBACzC,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;wBAC/B,sDAAsD;wBACtD,GAAG,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC;oBACxB,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACJ,GAAG,CAAC,MAA+B,CAAC,GAAG,KAAK,CAAC;gBACjD,CAAC;YACL,CAAC;YAED,4DAA4D;YAC5D,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;gBAC/B,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC;YACpB,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,GAAkB,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YAChB,MAAM,OAAO,GAA+B,EAAE,CAAC;YAC/C,MAAM,QAAQ,GAAkB,EAAE,CAAC;YAEnC,8BAA8B;YAC9B,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;YAEpB,qCAAqC;YACrC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;gBAClB,+EAA+E;gBAC/E,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,IAAI,IAAI,CAAC,OAAO,KAAK,gBAAgB,EAAE,CAAC;oBAC7D,OAAO;gBACX,CAAC;gBACD,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;oBACrB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;oBACzB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACxB,CAAC;qBAAM,IAAI,WAAW,IAAI,GAAG,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC;oBACzC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACxB,CAAC;YACL,CAAC,CAAC,CAAC;YACH,OAAO,CAAC,QAAQ,CAAC,CAAC;QACtB,CAAC,CAAC,CAAC;QAEH,+CAA+C;QAC/C,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;;;;;;;;;;GAWG;AACH,SAAS,eAAe,CAAC,GAAW;IAChC,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;QAC/B,+CAA+C;QAC/C,IAAI,GAAG,KAAK,MAAM;YAAE,OAAO,SAAS,CAAC;QACrC,OAAO,GAAG,CAAC;IACf,CAAC;IAED,QAAQ,GAAG,EAAE,CAAC;QACV,KAAK,MAAM;YACP,OAAO,SAAS,CAAC;QACrB,KAAK,iBAAiB;YAClB,OAAO,MAAM,CAAC;QAClB,KAAK,WAAW;YACZ,OAAO,KAAK,CAAC;QACjB,KAAK,QAAQ;YACT,OAAO,MAAM,CAAC;QAClB,KAAK,gBAAgB;YACjB,OAAO,KAAK,CAAC;QACjB;YACI,MAAM,IAAI,KAAK,CAAC,mCAAmC,GAAG,EAAE,CAAC,CAAC;IAClE,CAAC;AACL,CAAC"}
|
package/internals/typedefs.js
CHANGED
|
@@ -1,13 +1,9 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.entries = entries;
|
|
4
|
-
exports.keys = keys;
|
|
5
1
|
/** @ignore */
|
|
6
|
-
function entries(obj) {
|
|
2
|
+
export function entries(obj) {
|
|
7
3
|
return Object.entries(obj);
|
|
8
4
|
}
|
|
9
5
|
/** @ignore */
|
|
10
|
-
function keys(obj) {
|
|
6
|
+
export function keys(obj) {
|
|
11
7
|
return Object.keys(obj);
|
|
12
8
|
}
|
|
13
9
|
//# sourceMappingURL=typedefs.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"typedefs.js","sourceRoot":"","sources":["../../src/internals/typedefs.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"typedefs.js","sourceRoot":"","sources":["../../src/internals/typedefs.ts"],"names":[],"mappings":"AAAA,cAAc;AACd,MAAM,UAAU,OAAO,CAAe,GAAM;IACxC,OAAO,MAAM,CAAC,OAAO,CAAC,GAAG,CAA4B,CAAC;AAC1D,CAAC;AAED,cAAc;AACd,MAAM,UAAU,IAAI,CAAe,GAAM;IACrC,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAgB,CAAC;AAC3C,CAAC"}
|
package/internals/url.js
CHANGED
|
@@ -1,6 +1,3 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.applySearchParams = applySearchParams;
|
|
4
1
|
/**
|
|
5
2
|
* Appends search (query string) parameters to a URL, replacing the original value (if any).
|
|
6
3
|
*
|
|
@@ -8,7 +5,7 @@ exports.applySearchParams = applySearchParams;
|
|
|
8
5
|
* @param searchParams The search parameters to be appended.
|
|
9
6
|
* @internal
|
|
10
7
|
*/
|
|
11
|
-
function applySearchParams(url, searchParams) {
|
|
8
|
+
export function applySearchParams(url, searchParams) {
|
|
12
9
|
if (searchParams === undefined) {
|
|
13
10
|
return;
|
|
14
11
|
}
|
package/internals/url.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"url.js","sourceRoot":"","sources":["../../src/internals/url.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"url.js","sourceRoot":"","sources":["../../src/internals/url.ts"],"names":[],"mappings":"AAEA;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB,CAAC,GAAQ,EAAE,YAAsC;IAC9E,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO;IACX,CAAC;IAED,IAAI,OAAO,YAAY,KAAK,QAAQ,EAAE,CAAC;QACnC,GAAG,CAAC,MAAM,GAAG,YAAY,CAAC;QAC1B,OAAO;IACX,CAAC;IAED,IAAI,eAAgC,CAAC;IAErC,IAAI,YAAY,YAAY,eAAe,EAAE,CAAC;QAC1C,eAAe,GAAG,YAAY,CAAC;IACnC,CAAC;SAAM,CAAC;QACJ,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;QACxC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;YACzD,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gBACtB,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAChC,CAAC;iBAAM,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBACxB,eAAe,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACpC,CAAC;iBAAM,CAAC;gBACJ,eAAe,CAAC,MAAM,CAAC,GAAG,EAAE,KAAe,CAAC,CAAC;YACjD,CAAC;QACL,CAAC;IACL,CAAC;IAED,GAAG,CAAC,MAAM,GAAG,eAAe,CAAC,QAAQ,EAAE,CAAC;AAC5C,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,19 +1,13 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/utils",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "4.0.0-beta.0",
|
|
4
4
|
"description": "A set of shared utilities that can be used by crawlers",
|
|
5
5
|
"engines": {
|
|
6
|
-
"node": ">=
|
|
6
|
+
"node": ">=22.0.0"
|
|
7
7
|
},
|
|
8
|
-
"
|
|
9
|
-
"module": "./index.mjs",
|
|
10
|
-
"types": "./index.d.ts",
|
|
8
|
+
"type": "module",
|
|
11
9
|
"exports": {
|
|
12
|
-
".":
|
|
13
|
-
"import": "./index.mjs",
|
|
14
|
-
"require": "./index.js",
|
|
15
|
-
"types": "./index.d.ts"
|
|
16
|
-
},
|
|
10
|
+
".": "./index.js",
|
|
17
11
|
"./package.json": "./package.json"
|
|
18
12
|
},
|
|
19
13
|
"keywords": [
|
|
@@ -43,26 +37,23 @@
|
|
|
43
37
|
"scripts": {
|
|
44
38
|
"build": "yarn clean && yarn compile && yarn copy",
|
|
45
39
|
"clean": "rimraf ./dist",
|
|
46
|
-
"compile": "tsc -p tsconfig.build.json
|
|
40
|
+
"compile": "tsc -p tsconfig.build.json",
|
|
47
41
|
"copy": "tsx ../../scripts/copy.ts"
|
|
48
42
|
},
|
|
49
43
|
"dependencies": {
|
|
50
|
-
"@apify/log": "^2.
|
|
44
|
+
"@apify/log": "^2.5.18",
|
|
51
45
|
"@apify/ps-tree": "^1.2.0",
|
|
52
|
-
"@crawlee/types": "
|
|
46
|
+
"@crawlee/types": "4.0.0-beta.0",
|
|
53
47
|
"@types/sax": "^1.2.7",
|
|
54
|
-
"cheerio": "1.0.0
|
|
55
|
-
"file-type": "^20.
|
|
56
|
-
"got-scraping": "^4.
|
|
57
|
-
"ow": "^0.
|
|
48
|
+
"cheerio": "^1.0.0",
|
|
49
|
+
"file-type": "^20.5.0",
|
|
50
|
+
"got-scraping": "^4.1.1",
|
|
51
|
+
"ow": "^2.0.0",
|
|
58
52
|
"robots-parser": "^3.0.1",
|
|
59
53
|
"sax": "^1.4.1",
|
|
60
|
-
"tslib": "^2.
|
|
54
|
+
"tslib": "^2.8.1",
|
|
61
55
|
"whatwg-mimetype": "^4.0.0"
|
|
62
56
|
},
|
|
63
|
-
"devDependencies": {
|
|
64
|
-
"@types/whatwg-mimetype": "^3.0.2"
|
|
65
|
-
},
|
|
66
57
|
"lerna": {
|
|
67
58
|
"command": {
|
|
68
59
|
"publish": {
|
|
@@ -70,5 +61,5 @@
|
|
|
70
61
|
}
|
|
71
62
|
}
|
|
72
63
|
},
|
|
73
|
-
"gitHead": "
|
|
64
|
+
"gitHead": "927bdafa403ff347327158b01d20b817378168a7"
|
|
74
65
|
}
|