@crawlee/utils 3.12.3-beta.3 → 3.12.3-beta.31

package/README.md

@@ -80,6 +80,28 @@ await crawler.run(['https://crawlee.dev']);
 
 By default, Crawlee stores data to `./storage` in the current working directory. You can override this directory via Crawlee configuration. For details, see [Configuration guide](https://crawlee.dev/docs/guides/configuration), [Request storage](https://crawlee.dev/docs/guides/request-storage) and [Result storage](https://crawlee.dev/docs/guides/result-storage).
 
+### Installing pre-release versions
+
+We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
+
+```bash
+npm install crawlee@3.12.3-beta.13
+```
+
+If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
+
+```json
+{
+    "overrides": {
+       "apify": {
+           "@crawlee/core": "3.12.3-beta.13",
+           "@crawlee/types": "3.12.3-beta.13",
+           "@crawlee/utils": "3.12.3-beta.13"
+       }
+    }
+}
+```
+
 ## 🛠 Features
 
 - Single interface for **HTTP and headless browser** crawling

package/index.d.ts

@@ -12,5 +12,7 @@ export * from './internals/gotScraping';
 export * from './internals/robots';
 export * from './internals/sitemap';
 export * from './internals/url';
+export { getCurrentCpuTicksV2 } from './internals/systemInfoV2/cpu-info';
+export { getMemoryInfoV2 } from './internals/systemInfoV2/memory-info';
 export { Dictionary, Awaitable, Constructor } from '@crawlee/types';
 //# sourceMappingURL=index.d.ts.map

package/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,qBAAqB,CAAC;AACpC,cAAc,qBAAqB,CAAC;AACpC,cAAc,mBAAmB,CAAC;AAClC,cAAc,0BAA0B,CAAC;AACzC,cAAc,qBAAqB,CAAC;AACpC,cAAc,yBAAyB,CAAC;AACxC,cAAc,mBAAmB,CAAC;AAClC,OAAO,KAAK,MAAM,MAAM,oBAAoB,CAAC;AAC7C,cAAc,sBAAsB,CAAC;AACrC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,yBAAyB,CAAC;AACxC,cAAc,oBAAoB,CAAC;AACnC,cAAc,qBAAqB,CAAC;AACpC,cAAc,iBAAiB,CAAC;AAEhC,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,qBAAqB,CAAC;AACpC,cAAc,qBAAqB,CAAC;AACpC,cAAc,mBAAmB,CAAC;AAClC,cAAc,0BAA0B,CAAC;AACzC,cAAc,qBAAqB,CAAC;AACpC,cAAc,yBAAyB,CAAC;AACxC,cAAc,mBAAmB,CAAC;AAClC,OAAO,KAAK,MAAM,MAAM,oBAAoB,CAAC;AAC7C,cAAc,sBAAsB,CAAC;AACrC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,yBAAyB,CAAC;AACxC,cAAc,oBAAoB,CAAC;AACnC,cAAc,qBAAqB,CAAC;AACpC,cAAc,iBAAiB,CAAC;AAEhC,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,EAAE,eAAe,EAAE,MAAM,sCAAsC,CAAC;AAEvE,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC"}

package/index.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.social = void 0;
+exports.getMemoryInfoV2 = exports.getCurrentCpuTicksV2 = exports.social = void 0;
 const tslib_1 = require("tslib");
 tslib_1.__exportStar(require("./internals/blocked"), exports);
 tslib_1.__exportStar(require("./internals/cheerio"), exports);
@@ -16,4 +16,8 @@ tslib_1.__exportStar(require("./internals/gotScraping"), exports);
 tslib_1.__exportStar(require("./internals/robots"), exports);
 tslib_1.__exportStar(require("./internals/sitemap"), exports);
 tslib_1.__exportStar(require("./internals/url"), exports);
+var cpu_info_1 = require("./internals/systemInfoV2/cpu-info");
+Object.defineProperty(exports, "getCurrentCpuTicksV2", { enumerable: true, get: function () { return cpu_info_1.getCurrentCpuTicksV2; } });
+var memory_info_1 = require("./internals/systemInfoV2/memory-info");
+Object.defineProperty(exports, "getMemoryInfoV2", { enumerable: true, get: function () { return memory_info_1.getMemoryInfoV2; } });
 //# sourceMappingURL=index.js.map

package/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;AAAA,8DAAoC;AACpC,8DAAoC;AACpC,4DAAkC;AAClC,mEAAyC;AACzC,8DAAoC;AACpC,kEAAwC;AACxC,4DAAkC;AAClC,qEAA6C;AAC7C,+DAAqC;AACrC,wEAA8C;AAC9C,kEAAwC;AACxC,6DAAmC;AACnC,8DAAoC;AACpC,0DAAgC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;AAAA,8DAAoC;AACpC,8DAAoC;AACpC,4DAAkC;AAClC,mEAAyC;AACzC,8DAAoC;AACpC,kEAAwC;AACxC,4DAAkC;AAClC,qEAA6C;AAC7C,+DAAqC;AACrC,wEAA8C;AAC9C,kEAAwC;AACxC,6DAAmC;AACnC,8DAAoC;AACpC,0DAAgC;AAEhC,8DAAyE;AAAhE,gHAAA,oBAAoB,OAAA;AAC7B,oEAAuE;AAA9D,8GAAA,eAAe,OAAA"}

package/index.mjs

@@ -16,10 +16,15 @@ export const entries = mod.entries;
 export const expandShadowRoots = mod.expandShadowRoots;
 export const extractUrls = mod.extractUrls;
 export const extractUrlsFromCheerio = mod.extractUrlsFromCheerio;
+export const getCgroupsVersion = mod.getCgroupsVersion;
+export const getCurrentCpuTicksV2 = mod.getCurrentCpuTicksV2;
 export const getMemoryInfo = mod.getMemoryInfo;
+export const getMemoryInfoV2 = mod.getMemoryInfoV2;
 export const gotScraping = mod.gotScraping;
 export const htmlToText = mod.htmlToText;
+export const isContainerized = mod.isContainerized;
 export const isDocker = mod.isDocker;
+export const isLambda = mod.isLambda;
 export const keys = mod.keys;
 export const parseOpenGraph = mod.parseOpenGraph;
 export const parseSitemap = mod.parseSitemap;

package/internals/general.d.ts

@@ -12,6 +12,16 @@ export declare const URL_WITH_COMMAS_REGEX: RegExp;
  * Returns a `Promise` that resolves to true if the code is running in a Docker container.
  */
 export declare function isDocker(forceReset?: boolean): Promise<boolean>;
+/**
+ * Detects if crawlee is running in a containerized environment.
+ */
+export declare function isContainerized(): Promise<boolean>;
+export declare function isLambda(): boolean;
+/**
+ * gets the cgroup version by checking for a file at /sys/fs/cgroup/memory
+ * @returns "V1" or "V2" for the version of cgroup or null if cgroup is not found.
+ */
+export declare function getCgroupsVersion(forceReset?: boolean): Promise<"V1" | "V2" | null>;
 /**
  * Computes a weighted average of an array of numbers, complemented by an array of weights.
  * @ignore

package/internals/general.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"general.d.ts","sourceRoot":"","sources":["../../src/internals/general.ts"],"names":[],"mappings":"AAGA;;;GAGG;AACH,eAAO,MAAM,mBAAmB,QAC+G,CAAC;AAEhJ;;;GAGG;AACH,eAAO,MAAM,qBAAqB,QAC8G,CAAC;AAoBjJ;;GAEG;AACH,wBAAsB,QAAQ,CAAC,UAAU,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAKrE;AAED;;;GAGG;AACH,wBAAgB,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,MAAM,CAW7E;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,KAAK,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE1D;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,YAAY,EAAE,MAAM,GAAG,MAAM,CAQjE;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,QAAQ,GAAG,MAAM,CA8B5D"}
+{"version":3,"file":"general.d.ts","sourceRoot":"","sources":["../../src/internals/general.ts"],"names":[],"mappings":"AAGA;;;GAGG;AACH,eAAO,MAAM,mBAAmB,QAC+G,CAAC;AAEhJ;;;GAGG;AACH,eAAO,MAAM,qBAAqB,QAC8G,CAAC;AAoBjJ;;GAEG;AACH,wBAAsB,QAAQ,CAAC,UAAU,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAKrE;AAID;;GAEG;AACH,wBAAsB,eAAe,qBA0BpC;AAED,wBAAgB,QAAQ,YAEvB;AAGD;;;GAGG;AACH,wBAAsB,iBAAiB,CAAC,UAAU,CAAC,EAAE,OAAO,+BAoB3D;AAED;;;GAGG;AACH,wBAAgB,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,MAAM,CAW7E;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,KAAK,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE1D;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,YAAY,EAAE,MAAM,GAAG,MAAM,CAQjE;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,QAAQ,GAAG,MAAM,CA8B5D"}

package/internals/general.js

@@ -2,6 +2,9 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.URL_WITH_COMMAS_REGEX = exports.URL_NO_COMMAS_REGEX = void 0;
 exports.isDocker = isDocker;
+exports.isContainerized = isContainerized;
+exports.isLambda = isLambda;
+exports.getCgroupsVersion = getCgroupsVersion;
 exports.weightedAvg = weightedAvg;
 exports.sleep = sleep;
 exports.snakeCaseToCamelCase = snakeCaseToCamelCase;
@@ -41,6 +44,63 @@ async function isDocker(forceReset) {
         isDockerPromiseCache = createIsDockerPromise();
     return isDockerPromiseCache;
 }
+let isContainerizedResult;
+/**
+ * Detects if crawlee is running in a containerized environment.
+ */
+async function isContainerized() {
+    // Value is very unlikley to change. Cache the result after the first execution.
+    if (isContainerizedResult !== undefined) {
+        return isContainerizedResult;
+    }
+    // return false if running in aws lambda
+    if (isLambda()) {
+        isContainerizedResult = false;
+        return isContainerizedResult;
+    }
+    const dockerenvCheck = promises_1.default
+        .stat('/.dockerenv')
+        .then(() => true)
+        .catch(() => false);
+    const cgroupCheck = promises_1.default
+        .readFile('/proc/self/cgroup', 'utf8')
+        .then((content) => content.includes('docker'))
+        .catch(() => false);
+    const [dockerenvResult, cgroupResult] = await Promise.all([dockerenvCheck, cgroupCheck]);
+    isContainerizedResult = dockerenvResult || cgroupResult || !!process.env.KUBERNETES_SERVICE_HOST;
+    return isContainerizedResult;
+}
+function isLambda() {
+    return !!process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE;
+}
+let _cgroupsVersion;
+/**
+ * gets the cgroup version by checking for a file at /sys/fs/cgroup/memory
+ * @returns "V1" or "V2" for the version of cgroup or null if cgroup is not found.
+ */
+async function getCgroupsVersion(forceReset) {
+    // Parameter forceReset is just internal for unit tests.
+    if (_cgroupsVersion !== undefined && !forceReset) {
+        return _cgroupsVersion;
+    }
+    try {
+        // If this directory does not exists, cgroups are not available
+        await promises_1.default.access('/sys/fs/cgroup/');
+    }
+    catch (e) {
+        _cgroupsVersion = null;
+        return null;
+    }
+    _cgroupsVersion = 'V1';
+    try {
+        // If this directory does not exists, assume the container is using cgroups V2
+        await promises_1.default.access('/sys/fs/cgroup/memory/');
+    }
+    catch (e) {
+        _cgroupsVersion = 'V2';
+    }
+    return _cgroupsVersion;
+}
 /**
  * Computes a weighted average of an array of numbers, complemented by an array of weights.
  * @ignore

package/internals/general.js.map

@@ -1 +1 @@
-{"version":3,"file":"general.js","sourceRoot":"","sources":["../../src/internals/general.ts"],"names":[],"mappings":";;;AAsCA,4BAKC;AAMD,kCAWC;AAkBD,sBAEC;AAMD,oDAQC;AAMD,8CA8BC;;AAlID,wEAAkC;AAClC,mDAAkD;AAElD;;;GAGG;AACU,QAAA,mBAAmB,GAC5B,2IAA2I,CAAC;AAEhJ;;;GAGG;AACU,QAAA,qBAAqB,GAC9B,4IAA4I,CAAC;AAEjJ,IAAI,oBAAkD,CAAC;AAEvD,KAAK,UAAU,qBAAqB;IAChC,MAAM,QAAQ,GAAG,kBAAE;SACd,IAAI,CAAC,aAAa,CAAC;SACnB,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;SAChB,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;IAExB,MAAM,QAAQ,GAAG,kBAAE;SACd,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;SACrC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;SAC7C,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;IAExB,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;IAEnE,OAAO,OAAO,IAAI,OAAO,CAAC;AAC9B,CAAC;AAED;;GAEG;AACI,KAAK,UAAU,QAAQ,CAAC,UAAoB;IAC/C,wDAAwD;IACxD,IAAI,CAAC,oBAAoB,IAAI,UAAU;QAAE,oBAAoB,GAAG,qBAAqB,EAAE,CAAC;IAExF,OAAO,oBAAoB,CAAC;AAChC,CAAC;AAED;;;GAGG;AACH,SAAgB,WAAW,CAAC,SAAmB,EAAE,UAAoB;IACjE,MAAM,MAAM,GAAG,SAAS;SACnB,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;QACd,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,GAAG,GAAG,KAAK,GAAG,MAAM,CAAC;QAE3B,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IACzB,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAE1D,OAAO,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;AACjC,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACI,KAAK,UAAU,KAAK,CAAC,MAAe;IACvC,OAAO,IAAA,qBAAU,EAAC,MAAM,IAAI,SAAS,CAAC,CAAC;AAC3C,CAAC;AAED;;;GAGG;AACH,SAAgB,oBAAoB,CAAC,YAAoB;IACrD,OAAO,YAAY;SACd,WAAW,EAAE;SACb,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QACjB,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC3E,CAAC,CAAC;SACD,IAAI,CAAC,EAAE,CAAC,CAAC;AAClB,CAAC;AAED;;;GAGG;AACH,SAAgB,iBAAiB,CAAC,QAAkB;IAChD,oCAAoC;IACpC,SAAS,gBAAgB,CAAC,UAAe;QACrC,IAAI,UAAU,GAAG,EAAE,CAAC;QAEpB,KAAK,MAAM,EAAE,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;YACrC,UAAU,IAAI,EAAE,CAAC,SAAS,IAAI,EAAE,CAAC,SAAS,IAAI,EAAE,CAAC;QACrD,CAAC;QAED,OAAO,UAAU,CAAC;IACtB,CAAC;IAED,oDAAoD;IACpD,SAAS,yBAAyB,CAAC,WAAgB;QAC/C,KAAK,MAAM,EAAE,IAAI,WAAW,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC;YACjD,IAAI,EAAE,CAAC,UAAU,EAAE,CAAC;gBAChB,yBAAyB,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC;gBACzC,IAAI,OAAO,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,EAAE,uBAAuB,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;gBAErE,IAAI,CAAC,CAAC,OAAO,EAAE,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;oBACzB,OAAO,GAAG,gBAAgB,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;gBACpD,CAAC;gBACD,EAAE,CAAC,SAAS,IAAI,OAAO,CAAC;YAC5B,CAAC;QACL,CAAC;IACL,CAAC;IAED,yBAAyB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAEzC,OAAO,QAAQ,CAAC,eAAe,CAAC,SAAS,CAAC;AAC9C,CAAC"}
+{"version":3,"file":"general.js","sourceRoot":"","sources":["../../src/internals/general.ts"],"names":[],"mappings":";;;AAsCA,4BAKC;AAOD,0CA0BC;AAED,4BAEC;AAOD,8CAoBC;AAMD,kCAWC;AAkBD,sBAEC;AAMD,oDAQC;AAMD,8CA8BC;;AAlMD,wEAAkC;AAClC,mDAAkD;AAElD;;;GAGG;AACU,QAAA,mBAAmB,GAC5B,2IAA2I,CAAC;AAEhJ;;;GAGG;AACU,QAAA,qBAAqB,GAC9B,4IAA4I,CAAC;AAEjJ,IAAI,oBAAkD,CAAC;AAEvD,KAAK,UAAU,qBAAqB;IAChC,MAAM,QAAQ,GAAG,kBAAE;SACd,IAAI,CAAC,aAAa,CAAC;SACnB,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;SAChB,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;IAExB,MAAM,QAAQ,GAAG,kBAAE;SACd,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;SACrC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;SAC7C,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;IAExB,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;IAEnE,OAAO,OAAO,IAAI,OAAO,CAAC;AAC9B,CAAC;AAED;;GAEG;AACI,KAAK,UAAU,QAAQ,CAAC,UAAoB;IAC/C,wDAAwD;IACxD,IAAI,CAAC,oBAAoB,IAAI,UAAU;QAAE,oBAAoB,GAAG,qBAAqB,EAAE,CAAC;IAExF,OAAO,oBAAoB,CAAC;AAChC,CAAC;AAED,IAAI,qBAA0C,CAAC;AAE/C;;GAEG;AACI,KAAK,UAAU,eAAe;IACjC,gFAAgF;IAChF,IAAI,qBAAqB,KAAK,SAAS,EAAE,CAAC;QACtC,OAAO,qBAAqB,CAAC;IACjC,CAAC;IAED,wCAAwC;IACxC,IAAI,QAAQ,EAAE,EAAE,CAAC;QACb,qBAAqB,GAAG,KAAK,CAAC;QAC9B,OAAO,qBAAqB,CAAC;IACjC,CAAC;IAED,MAAM,cAAc,GAAG,kBAAE;SACpB,IAAI,CAAC,aAAa,CAAC;SACnB,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;SAChB,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;IAExB,MAAM,WAAW,GAAG,kBAAE;SACjB,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;SACrC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;SAC7C,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;IAExB,MAAM,CAAC,eAAe,EAAE,YAAY,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC,CAAC;IAEzF,qBAAqB,GAAG,eAAe,IAAI,YAAY,IAAI,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC;IACjG,OAAO,qBAAqB,CAAC;AACjC,CAAC;AAED,SAAgB,QAAQ;IACpB,OAAO,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC;AACzD,CAAC;AAED,IAAI,eAAmC,CAAC;AACxC;;;GAGG;AACI,KAAK,UAAU,iBAAiB,CAAC,UAAoB;IACxD,wDAAwD;IACxD,IAAI,eAAe,KAAK,SAAS,IAAI,CAAC,UAAU,EAAE,CAAC;QAC/C,OAAO,eAAe,CAAC;IAC3B,CAAC;IACD,IAAI,CAAC;QACD,+DAA+D;QAC/D,MAAM,kBAAE,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;IACvC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,eAAe,GAAG,IAAI,CAAC;QACvB,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,eAAe,GAAG,IAAI,CAAC;IACvB,IAAI,CAAC;QACD,8EAA8E;QAC9E,MAAM,kBAAE,CAAC,MAAM,CAAC,wBAAwB,CAAC,CAAC;IAC9C,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,eAAe,GAAG,IAAI,CAAC;IAC3B,CAAC;IACD,OAAO,eAAe,CAAC;AAC3B,CAAC;AAED;;;GAGG;AACH,SAAgB,WAAW,CAAC,SAAmB,EAAE,UAAoB;IACjE,MAAM,MAAM,GAAG,SAAS;SACnB,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;QACd,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,GAAG,GAAG,KAAK,GAAG,MAAM,CAAC;QAE3B,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IACzB,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAE1D,OAAO,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;AACjC,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACI,KAAK,UAAU,KAAK,CAAC,MAAe;IACvC,OAAO,IAAA,qBAAU,EAAC,MAAM,IAAI,SAAS,CAAC,CAAC;AAC3C,CAAC;AAED;;;GAGG;AACH,SAAgB,oBAAoB,CAAC,YAAoB;IACrD,OAAO,YAAY;SACd,WAAW,EAAE;SACb,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QACjB,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC3E,CAAC,CAAC;SACD,IAAI,CAAC,EAAE,CAAC,CAAC;AAClB,CAAC;AAED;;;GAGG;AACH,SAAgB,iBAAiB,CAAC,QAAkB;IAChD,oCAAoC;IACpC,SAAS,gBAAgB,CAAC,UAAe;QACrC,IAAI,UAAU,GAAG,EAAE,CAAC;QAEpB,KAAK,MAAM,EAAE,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;YACrC,UAAU,IAAI,EAAE,CAAC,SAAS,IAAI,EAAE,CAAC,SAAS,IAAI,EAAE,CAAC;QACrD,CAAC;QAED,OAAO,UAAU,CAAC;IACtB,CAAC;IAED,oDAAoD;IACpD,SAAS,yBAAyB,CAAC,WAAgB;QAC/C,KAAK,MAAM,EAAE,IAAI,WAAW,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC;YACjD,IAAI,EAAE,CAAC,UAAU,EAAE,CAAC;gBAChB,yBAAyB,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC;gBACzC,IAAI,OAAO,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,EAAE,uBAAuB,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;gBAErE,IAAI,CAAC,CAAC,OAAO,EAAE,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;oBACzB,OAAO,GAAG,gBAAgB,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;gBACpD,CAAC;gBACD,EAAE,CAAC,SAAS,IAAI,OAAO,CAAC;YAC5B,CAAC;QACL,CAAC;IACL,CAAC;IAED,yBAAyB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAEzC,OAAO,QAAQ,CAAC,eAAe,CAAC,SAAS,CAAC;AAC9C,CAAC"}

package/internals/systemInfoV2/cpu-info.d.ts

@@ -0,0 +1,64 @@
+/**
+ * Gets the "bare metal" cpu load.
+ * Used in
+ *  - AWS Lambda
+ *  - Containers without a cGroup quota
+ *  - Uncontainerized environments
+ * @returns a number between 0 and 1 for the cpu load
+ * @internal
+ */
+export declare function getCurrentCpuTicks(): number;
+/**
+ * Reads the cgroup cpu quota.
+ * In V1, a quota of -1 means “unlimited.”
+ * In V2, a first field of "max" means unlimited.
+ * @param cgroupsVersion the cGroup version
+ * @returns The Cpu Quota
+ * @internal
+ */
+export declare function getCpuQuota(cgroupsVersion: string): Promise<number | null>;
+/**
+ * Reads the cgroup cpu period.
+ * @param cgroupsVersion the cGroup version
+ * @returns The Cpu quota period
+ * @internal
+ */
+export declare function getCpuPeriod(cgroupsVersion: string): Promise<number>;
+/**
+ * Reads the cgroup cpu usage of the container
+ *
+ * @param cgroupsVersion the cGroup version
+ * @returns the cpu usage
+ * @internal
+ */
+export declare function getContainerCpuUsage(cgroupsVersion: string): Promise<number>;
+/**
+ * Reads the cgroup cpu usage of the system from cgroup
+ *
+ * @returns the cpu usage
+ * @internal
+ */
+export declare function getSystemCpuUsage(): Promise<number>;
+/**
+ * a cpu sample with the container usage and system usage
+ */
+export interface CpuSample {
+    containerUsage: number;
+    systemUsage: number;
+}
+/**
+ * Takes a CPU usage sample for both the container and the system.
+ *
+ * @returns An object containing the container and system CPU usage.
+ * @internal
+ */
+export declare function sampleCpuUsage(cGroupsVersion: string): Promise<CpuSample>;
+/**
+ * Gets the cpu usage of the system.
+ * If the crawler is running in a containerized environment, crawlee will check for a cgroup enforced cpu limit.
+ * If a cgroup limit is found, it will be taken as the maximum load against which the current load will be gauged.
+ * @returns a number between 0 and 1 for the cpu load
+ * @internal
+ */
+export declare function getCurrentCpuTicksV2(containerized?: boolean): Promise<number>;
+//# sourceMappingURL=cpu-info.d.ts.map

package/internals/systemInfoV2/cpu-info.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cpu-info.d.ts","sourceRoot":"","sources":["../../../src/internals/systemInfoV2/cpu-info.ts"],"names":[],"mappings":"AA6BA;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,WAejC;AAgBD;;;;;;;GAOG;AACH,wBAAsB,WAAW,CAAC,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAahF;AAED;;;;;GAKG;AACH,wBAAsB,YAAY,CAAC,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAU1E;AAED;;;;;;GAMG;AACH,wBAAsB,oBAAoB,CAAC,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAkBlF;AAED;;;;;GAKG;AACH,wBAAsB,iBAAiB,oBAiBtC;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;CACvB;AAED;;;;;GAKG;AACH,wBAAsB,cAAc,CAAC,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,CAM/E;AAID;;;;;;GAMG;AACH,wBAAsB,oBAAoB,CAAC,aAAa,UAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CA8CjF"}

package/internals/systemInfoV2/cpu-info.js

@@ -0,0 +1,221 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getCurrentCpuTicks = getCurrentCpuTicks;
+exports.getCpuQuota = getCpuQuota;
+exports.getCpuPeriod = getCpuPeriod;
+exports.getContainerCpuUsage = getContainerCpuUsage;
+exports.getSystemCpuUsage = getSystemCpuUsage;
+exports.sampleCpuUsage = sampleCpuUsage;
+exports.getCurrentCpuTicksV2 = getCurrentCpuTicksV2;
+const tslib_1 = require("tslib");
+const node_child_process_1 = require("node:child_process");
+const promises_1 = require("node:fs/promises");
+const node_os_1 = tslib_1.__importDefault(require("node:os"));
+const log_1 = tslib_1.__importDefault(require("@apify/log"));
+const general_1 = require("../general");
+const CPU_FILE_PATHS = {
+    STAT: {
+        V1: '/sys/fs/cgroup/cpuacct/cpuacct.usage',
+        V2: '/sys/fs/cgroup/cpu.stat',
+    },
+    QUOTA: {
+        V1: '/sys/fs/cgroup/cpu/cpu.cfs_quota_us',
+        V2: '/sys/fs/cgroup/cpu.max',
+    },
+    PERIOD: {
+        V1: '/sys/fs/cgroup/cpu/cpu.cfs_period_us',
+        V2: '/sys/fs/cgroup/cpu.max',
+    },
+};
+let CLOCK_TICKS_PER_SECOND = 100;
+let CLOCK_TICKS_CHECKED = false;
+const NANOSECONDS_PER_SECOND = 1e9;
+const previousTicks = { idle: 0, total: 0 };
+/**
+ * Gets the "bare metal" cpu load.
+ * Used in
+ *  - AWS Lambda
+ *  - Containers without a cGroup quota
+ *  - Uncontainerized environments
+ * @returns a number between 0 and 1 for the cpu load
+ * @internal
+ */
+function getCurrentCpuTicks() {
+    const cpusCores = node_os_1.default.cpus();
+    const ticks = cpusCores.reduce((acc, cpu) => {
+        const cpuTimes = Object.values(cpu.times);
+        return {
+            idle: acc.idle + cpu.times.idle,
+            total: acc.total + cpuTimes.reduce((sum, num) => sum + num),
+        };
+    }, { idle: 0, total: 0 });
+    const idleTicksDelta = ticks.idle - previousTicks.idle;
+    const totalTicksDelta = ticks.total - previousTicks.total;
+    return totalTicksDelta ? 1 - idleTicksDelta / totalTicksDelta : 0;
+}
+/**
+ * Reads the linux tick rate
+ * @returns the number of ticks per second
+ */
+function getClockTicks() {
+    try {
+        const result = (0, node_child_process_1.execSync)('getconf CLK_TCK').toString().trim();
+        return parseInt(result, 10);
+    }
+    catch (err) {
+        log_1.default.warningOnce('Failed to get clock ticks; defaulting to 100');
+        return 100;
+    }
+}
+/**
+ * Reads the cgroup cpu quota.
+ * In V1, a quota of -1 means “unlimited.”
+ * In V2, a first field of "max" means unlimited.
+ * @param cgroupsVersion the cGroup version
+ * @returns The Cpu Quota
+ * @internal
+ */
+async function getCpuQuota(cgroupsVersion) {
+    if (cgroupsVersion === 'V1') {
+        const quotaStr = await (0, promises_1.readFile)(CPU_FILE_PATHS.QUOTA.V1, 'utf8');
+        const quota = parseInt(quotaStr.trim(), 10);
+        return quota === -1 ? null : quota;
+    }
+    // cgroup v2
+    const maxStr = await (0, promises_1.readFile)(CPU_FILE_PATHS.QUOTA.V2, 'utf8');
+    const parts = maxStr.trim().split(/\s+/);
+    if (parts[0] === 'max') {
+        return null;
+    }
+    return parseInt(parts[0], 10);
+}
+/**
+ * Reads the cgroup cpu period.
+ * @param cgroupsVersion the cGroup version
+ * @returns The Cpu quota period
+ * @internal
+ */
+async function getCpuPeriod(cgroupsVersion) {
+    if (cgroupsVersion === 'V1') {
+        const quotaStr = await (0, promises_1.readFile)(CPU_FILE_PATHS.PERIOD.V1, 'utf8');
+        const quota = parseInt(quotaStr.trim(), 10);
+        return quota;
+    }
+    // cgroup v2
+    const maxStr = await (0, promises_1.readFile)(CPU_FILE_PATHS.PERIOD.V2, 'utf8');
+    const parts = maxStr.trim().split(/\s+/);
+    return parseInt(parts[1], 10);
+}
+/**
+ * Reads the cgroup cpu usage of the container
+ *
+ * @param cgroupsVersion the cGroup version
+ * @returns the cpu usage
+ * @internal
+ */
+async function getContainerCpuUsage(cgroupsVersion) {
+    if (cgroupsVersion === 'V1') {
+        const data = await (0, promises_1.readFile)(CPU_FILE_PATHS.STAT.V1, 'utf8');
+        return Number(data.trim());
+    }
+    // cgroup v2
+    const data = await (0, promises_1.readFile)(CPU_FILE_PATHS.STAT.V2, 'utf8');
+    const lines = data.split('\n');
+    let usageUsec = 0;
+    for (const line of lines) {
+        const parts = line.trim().split(/\s+/);
+        if (parts[0] === 'usage_usec') {
+            usageUsec = Number(parts[1]);
+            break;
+        }
+    }
+    // Convert microseconds to nanoseconds.
+    return usageUsec * 1000;
+}
+/**
+ * Reads the cgroup cpu usage of the system from cgroup
+ *
+ * @returns the cpu usage
+ * @internal
+ */
+async function getSystemCpuUsage() {
+    const statData = await (0, promises_1.readFile)('/proc/stat', 'utf8');
+    const lines = statData.split('\n');
+    for (const line of lines) {
+        if (line.startsWith('cpu ')) {
+            // Split the line and extract the first seven numeric fields:
+            // user, nice, system, idle, iowait, irq, softirq
+            const parts = line.split(/\s+/).slice(1, 8);
+            let totalTicks = 0;
+            for (const part of parts) {
+                totalTicks += Number(part);
+            }
+            // Convert clock ticks to nanoseconds.
+            return (totalTicks * NANOSECONDS_PER_SECOND) / CLOCK_TICKS_PER_SECOND;
+        }
+    }
+    throw new Error('no cpu line'); // shouldnt ever happen
+}
+/**
+ * Takes a CPU usage sample for both the container and the system.
+ *
+ * @returns An object containing the container and system CPU usage.
+ * @internal
+ */
+async function sampleCpuUsage(cGroupsVersion) {
+    const [containerUsage, systemUsage] = await Promise.all([
+        getContainerCpuUsage(cGroupsVersion),
+        getSystemCpuUsage(),
+    ]);
+    return { containerUsage, systemUsage };
+}
+let previousSample = { containerUsage: 0, systemUsage: 0 };
+/**
+ * Gets the cpu usage of the system.
+ * If the crawler is running in a containerized environment, crawlee will check for a cgroup enforced cpu limit.
+ * If a cgroup limit is found, it will be taken as the maximum load against which the current load will be gauged.
+ * @returns a number between 0 and 1 for the cpu load
+ * @internal
+ */
+async function getCurrentCpuTicksV2(containerized = false) {
+    try {
+        // if not containerized
+        if (!containerized) {
+            // bare metal cpu limit
+            return getCurrentCpuTicks();
+        }
+        if (!CLOCK_TICKS_CHECKED) {
+            CLOCK_TICKS_PER_SECOND = getClockTicks();
+            CLOCK_TICKS_CHECKED = true;
+        }
+        const cgroupsVersion = await (0, general_1.getCgroupsVersion)();
+        // if cgroup is not detected, return bare metal cpu limit
+        if (cgroupsVersion === null) {
+            log_1.default.deprecated('Your environment is containerized, but your system does not support cgroups.\n' +
+                "If you're running containers with limited cpu, cpu auto-scaling will not work properly.");
+            return getCurrentCpuTicks();
+        }
+        // cgroup aware cpu limit. If no limits are set, default to returning getCurrentCpuTicks.
+        const quota = await getCpuQuota(cgroupsVersion);
+        if (quota === null) {
+            // no cgroup limit, return host cpu load
+            return getCurrentCpuTicks();
+        }
+        const period = await getCpuPeriod(cgroupsVersion);
+        // eg. having a 200000us quots per 100000us means the cGroup can fully use 2 cores
+        const cpuAllowance = quota / period;
+        const sample = await sampleCpuUsage(cgroupsVersion);
+        const containerDelta = sample.containerUsage - previousSample.containerUsage;
+        const systemDelta = sample.systemUsage - previousSample.systemUsage;
+        previousSample = sample;
+        const numCpus = node_os_1.default.cpus().length;
+        // Calculate the CPU usage percentage.
+        return ((containerDelta / systemDelta) * numCpus) / cpuAllowance;
+    }
+    catch (err) {
+        // if anything fails, default to bare metal metrics
+        log_1.default.exception(err, 'Cpu snapshot failed.');
+        return getCurrentCpuTicks();
+    }
+}
+//# sourceMappingURL=cpu-info.js.map

package/internals/systemInfoV2/cpu-info.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cpu-info.js","sourceRoot":"","sources":["../../../src/internals/systemInfoV2/cpu-info.ts"],"names":[],"mappings":";;AAsCA,gDAeC;AAwBD,kCAaC;AAQD,oCAUC;AASD,oDAkBC;AAQD,8CAiBC;AAgBD,wCAMC;AAWD,oDA8CC;;AA/OD,2DAA8C;AAC9C,+CAA4C;AAC5C,8DAAyB;AAEzB,6DAA6B;AAE7B,wCAA+C;AAE/C,MAAM,cAAc,GAAG;IACnB,IAAI,EAAE;QACF,EAAE,EAAE,sCAAsC;QAC1C,EAAE,EAAE,yBAAyB;KAChC;IACD,KAAK,EAAE;QACH,EAAE,EAAE,qCAAqC;QACzC,EAAE,EAAE,wBAAwB;KAC/B;IACD,MAAM,EAAE;QACJ,EAAE,EAAE,sCAAsC;QAC1C,EAAE,EAAE,wBAAwB;KAC/B;CACJ,CAAC;AAEF,IAAI,sBAAsB,GAAG,GAAG,CAAC;AACjC,IAAI,mBAAmB,GAAG,KAAK,CAAC;AAEhC,MAAM,sBAAsB,GAAG,GAAG,CAAC;AAEnC,MAAM,aAAa,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;AAC5C;;;;;;;;GAQG;AACH,SAAgB,kBAAkB;IAC9B,MAAM,SAAS,GAAG,iBAAE,CAAC,IAAI,EAAE,CAAC;IAC5B,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAC1B,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;QACT,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC1C,OAAO;YACH,IAAI,EAAE,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI;YAC/B,KAAK,EAAE,GAAG,CAAC,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC;SAC9D,CAAC;IACN,CAAC,EACD,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CACxB,CAAC;IACF,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,GAAG,aAAc,CAAC,IAAI,CAAC;IACxD,MAAM,eAAe,GAAG,KAAK,CAAC,KAAK,GAAG,aAAc,CAAC,KAAK,CAAC;IAC3D,OAAO,eAAe,CAAC,CAAC,CAAC,CAAC,GAAG,cAAc,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC;AACtE,CAAC;AAED;;;GAGG;AACH,SAAS,aAAa;IAClB,IAAI,CAAC;QACD,MAAM,MAAM,GAAG,IAAA,6BAAQ,EAAC,iBAAiB,CAAC,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC;QAC7D,OAAO,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAChC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACX,aAAG,CAAC,WAAW,CAAC,8CAA8C,CAAC,CAAC;QAChE,OAAO,GAAG,CAAC;IACf,CAAC;AACL,CAAC;AAED;;;;;;;GAOG;AACI,KAAK,UAAU,WAAW,CAAC,cAAsB;IACpD,IAAI,cAAc,KAAK,IAAI,EAAE,CAAC;QAC1B,MAAM,QAAQ,GAAG,MAAM,IAAA,mBAAQ,EAAC,cAAc,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QACjE,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;QAC5C,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;IACvC,CAAC;IACD,YAAY;IACZ,MAAM,MAAM,GAAG,MAAM,IAAA,mBAAQ,EAAC,cAAc,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IAC/D,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACzC,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,KAAK,EAAE,CAAC;QACrB,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAClC,CAAC;AAED;;;;;GAKG;AACI,KAAK,UAAU,YAAY,CAAC,cAAsB;IACrD,IAAI,cAAc,KAAK,IAAI,EAAE,CAAC;QAC1B,MAAM,QAAQ,GAAG,MAAM,IAAA,mBAAQ,EAAC,cAAc,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QAClE,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;QAC5C,OAAO,KAAK,CAAC;IACjB,CAAC;IACD,YAAY;IACZ,MAAM,MAAM,GAAG,MAAM,IAAA,mBAAQ,EAAC,cAAc,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IAChE,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACzC,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAClC,CAAC;AAED;;;;;;GAMG;AACI,KAAK,UAAU,oBAAoB,CAAC,cAAsB;IAC7D,IAAI,cAAc,KAAK,IAAI,EAAE,CAAC;QAC1B,MAAM,IAAI,GAAG,MAAM,IAAA,mBAAQ,EAAC,cAAc,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QAC5D,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IAC/B,CAAC;IACD,YAAY;IACZ,MAAM,IAAI,GAAG,MAAM,IAAA,mBAAQ,EAAC,cAAc,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IAC5D,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/B,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACvC,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,YAAY,EAAE,CAAC;YAC5B,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAC7B,MAAM;QACV,CAAC;IACL,CAAC;IACD,uCAAuC;IACvC,OAAO,SAAS,GAAG,IAAI,CAAC;AAC5B,CAAC;AAED;;;;;GAKG;AACI,KAAK,UAAU,iBAAiB;IACnC,MAAM,QAAQ,GAAG,MAAM,IAAA,mBAAQ,EAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IACtD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1B,6DAA6D;YAC7D,iDAAiD;YACjD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAC5C,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACvB,UAAU,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC;YAC/B,CAAC;YACD,sCAAsC;YACtC,OAAO,CAAC,UAAU,GAAG,sBAAsB,CAAC,GAAG,sBAAsB,CAAC;QAC1E,CAAC;IACL,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,uBAAuB;AAC3D,CAAC;AAUD;;;;;GAKG;AACI,KAAK,UAAU,cAAc,CAAC,cAAsB;IACvD,MAAM,CAAC,cAAc,EAAE,WAAW,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QACpD,oBAAoB,CAAC,cAAc,CAAC;QACpC,iBAAiB,EAAE;KACtB,CAAC,CAAC;IACH,OAAO,EAAE,cAAc,EAAE,WAAW,EAAE,CAAC;AAC3C,CAAC;AAED,IAAI,cAAc,GAAc,EAAE,cAAc,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC;AAEtE;;;;;;GAMG;AACI,KAAK,UAAU,oBAAoB,CAAC,aAAa,GAAG,KAAK;IAC5D,IAAI,CAAC;QACD,uBAAuB;QACvB,IAAI,CAAC,aAAa,EAAE,CAAC;YACjB,uBAAuB;YACvB,OAAO,kBAAkB,EAAE,CAAC;QAChC,CAAC;QACD,IAAI,CAAC,mBAAmB,EAAE,CAAC;YACvB,sBAAsB,GAAG,aAAa,EAAE,CAAC;YACzC,mBAAmB,GAAG,IAAI,CAAC;QAC/B,CAAC;QACD,MAAM,cAAc,GAAG,MAAM,IAAA,2BAAiB,GAAE,CAAC;QACjD,yDAAyD;QACzD,IAAI,cAAc,KAAK,IAAI,EAAE,CAAC;YAC1B,aAAG,CAAC,UAAU,CACV,gFAAgF;gBAC5E,yFAAyF,CAChG,CAAC;YACF,OAAO,kBAAkB,EAAE,CAAC;QAChC,CAAC;QACD,yFAAyF;QACzF,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC,cAAe,CAAC,CAAC;QACjD,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACjB,wCAAwC;YACxC,OAAO,kBAAkB,EAAE,CAAC;QAChC,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,cAAe,CAAC,CAAC;QACnD,kFAAkF;QAClF,MAAM,YAAY,GAAG,KAAK,GAAG,MAAM,CAAC;QAEpC,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,cAAe,CAAC,CAAC;QAErD,MAAM,cAAc,GAAG,MAAM,CAAC,cAAc,GAAG,cAAc,CAAC,cAAc,CAAC;QAC7E,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,GAAG,cAAc,CAAC,WAAW,CAAC;QAEpE,cAAc,GAAG,MAAM,CAAC;QAExB,MAAM,OAAO,GAAG,iBAAE,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC;QAEjC,sCAAsC;QACtC,OAAO,CAAC,CAAC,cAAc,GAAG,WAAW,CAAC,GAAG,OAAO,CAAC,GAAG,YAAY,CAAC;IACrE,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACX,mDAAmD;QACnD,aAAG,CAAC,SAAS,CAAC,GAAY,EAAE,sBAAsB,CAAC,CAAC;QACpD,OAAO,kBAAkB,EAAE,CAAC;IAChC,CAAC;AACL,CAAC"}

package/internals/systemInfoV2/memory-info.d.ts

@@ -0,0 +1,28 @@
+/**
+ * Describes memory usage of the process.
+ */
+export interface MemoryInfo {
+    /** Total memory available in the system or container */
+    totalBytes: number;
+    /** Amount of free memory in the system or container */
+    freeBytes: number;
+    /** Amount of memory used (= totalBytes - freeBytes) */
+    usedBytes: number;
+    /** Amount of memory used the current Node.js process */
+    mainProcessBytes: number;
+    /** Amount of memory used by child processes of the current Node.js process */
+    childProcessesBytes: number;
+}
+/**
+ * Returns memory statistics of the process and the system, see {@apilink MemoryInfo}.
+ *
+ * If the process runs inside of a container, the `getMemoryInfo` gets container memory limits,
+ * otherwise it gets system memory limits.
+ *
+ * Beware that the function is quite inefficient because it spawns a new process.
+ * Therefore you shouldn't call it too often, like more than once per second.
+ * @returns An object containing the free and used memory metrics.
+ * @internal
+ */
+export declare function getMemoryInfoV2(containerized?: boolean): Promise<MemoryInfo>;
+//# sourceMappingURL=memory-info.d.ts.map

package/internals/systemInfoV2/memory-info.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"memory-info.d.ts","sourceRoot":"","sources":["../../../src/internals/systemInfoV2/memory-info.ts"],"names":[],"mappings":"AAoBA;;GAEG;AACH,MAAM,WAAW,UAAU;IACvB,wDAAwD;IACxD,UAAU,EAAE,MAAM,CAAC;IAEnB,uDAAuD;IACvD,SAAS,EAAE,MAAM,CAAC;IAElB,uDAAuD;IACvD,SAAS,EAAE,MAAM,CAAC;IAElB,wDAAwD;IACxD,gBAAgB,EAAE,MAAM,CAAC;IAEzB,8EAA8E;IAC9E,mBAAmB,EAAE,MAAM,CAAC;CAC/B;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,eAAe,CAAC,aAAa,UAAQ,GAAG,OAAO,CAAC,UAAU,CAAC,CAgGhF"}

package/internals/systemInfoV2/memory-info.js

@@ -0,0 +1,122 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getMemoryInfoV2 = getMemoryInfoV2;
+const tslib_1 = require("tslib");
+const node_child_process_1 = require("node:child_process");
+const promises_1 = require("node:fs/promises");
+const node_os_1 = require("node:os");
+const log_1 = tslib_1.__importDefault(require("@apify/log"));
+const general_1 = require("../general");
+const ps_tree_1 = require("./ps-tree");
+const MEMORY_FILE_PATHS = {
+    TOTAL: {
+        V1: '/sys/fs/cgroup/memory/memory.limit_in_bytes',
+        V2: '/sys/fs/cgroup/memory.max',
+    },
+    USED: {
+        V1: '/sys/fs/cgroup/memory/memory.usage_in_bytes',
+        V2: '/sys/fs/cgroup/memory.current',
+    },
+};
+/**
+ * Returns memory statistics of the process and the system, see {@apilink MemoryInfo}.
+ *
+ * If the process runs inside of a container, the `getMemoryInfo` gets container memory limits,
+ * otherwise it gets system memory limits.
+ *
+ * Beware that the function is quite inefficient because it spawns a new process.
+ * Therefore you shouldn't call it too often, like more than once per second.
+ * @returns An object containing the free and used memory metrics.
+ * @internal
+ */
+async function getMemoryInfoV2(containerized = false) {
+    let mainProcessBytes = -1;
+    let childProcessesBytes = 0;
+    // lambda does *not* have `ps` and other command line tools
+    // required to extract memory usage.
+    if ((0, general_1.isLambda)()) {
+        // reported in bytes
+        mainProcessBytes = process.memoryUsage().rss;
+        // https://stackoverflow.com/a/55914335/129415
+        const memInfo = (0, node_child_process_1.execSync)('cat /proc/meminfo').toString();
+        const values = memInfo.split(/[\n: ]/).filter((val) => val.trim());
+        // /proc/meminfo reports in kb, not bytes, the total used memory is reported by meminfo
+        // subtract memory used by the main node process in order to infer memory used by any child processes
+        childProcessesBytes = +values[19] * 1000 - mainProcessBytes;
+    }
+    else {
+        // Query both root and child processes
+        const processes = await (0, ps_tree_1.psTree)(process.pid, true);
+        processes.forEach((rec) => {
+            // Obtain main process' memory separately
+            if (rec.PID === `${process.pid}`) {
+                mainProcessBytes = rec.RSS;
+                return;
+            }
+            childProcessesBytes += rec.RSS;
+        });
+    }
+    let totalBytes;
+    let usedBytes;
+    let freeBytes;
+    if ((0, general_1.isLambda)()) {
+        // memory size is defined in megabytes
+        totalBytes = parseInt(process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE, 10) * 1000000;
+        usedBytes = mainProcessBytes + childProcessesBytes;
+        freeBytes = totalBytes - usedBytes;
+        log_1.default.debug(`lambda size of ${totalBytes} with ${freeBytes} free bytes`);
+    }
+    else if (containerized) {
+        // When running inside a container, use container memory limits
+        const cgroupsVersion = await (0, general_1.getCgroupsVersion)();
+        try {
+            if (cgroupsVersion === null) {
+                throw new Error('cgroup not available');
+            }
+            let [totalBytesStr, usedBytesStr] = await Promise.all([
+                (0, promises_1.readFile)(MEMORY_FILE_PATHS.TOTAL[cgroupsVersion], 'utf8'),
+                (0, promises_1.readFile)(MEMORY_FILE_PATHS.USED[cgroupsVersion], 'utf8'),
+            ]);
+            // Cgroups V2 files contains newline character. Getting rid of it for better handling in later part of the code.
+            totalBytesStr = totalBytesStr.replace(/[^a-zA-Z0-9 ]/g, '');
+            usedBytesStr = usedBytesStr.replace(/[^a-zA-Z0-9 ]/g, '');
+            // Cgroups V2 contains 'max' string if memory is not limited
+            // See https://git.kernel.org/pub/scm/linux/kernel/git/tj/cgroup.git/tree/Documentation/admin-guide/cgroup-v2.rst (see "memory.max")
+            if (totalBytesStr === 'max') {
+                totalBytes = (0, node_os_1.totalmem)();
+                // Cgroups V1 is set to number related to platform and page size if memory is not limited
+                // See https://unix.stackexchange.com/q/420906
+            }
+            else {
+                totalBytes = parseInt(totalBytesStr, 10);
+                const containerRunsWithUnlimitedMemory = totalBytes > Number.MAX_SAFE_INTEGER;
+                if (containerRunsWithUnlimitedMemory)
+                    totalBytes = (0, node_os_1.totalmem)();
+            }
+            usedBytes = parseInt(usedBytesStr, 10);
+            freeBytes = totalBytes - usedBytes;
+        }
+        catch (err) {
+            // log.deprecated logs a warning only once
+            log_1.default.deprecated('Your environment is containerized, but your system does not support memory cgroups. ' +
+                "If you're running containers with limited memory, memory auto-scaling will not work properly.\n\n" +
+                `Cause: ${err.message}`);
+            totalBytes = (0, node_os_1.totalmem)();
+            freeBytes = (0, node_os_1.freemem)();
+            usedBytes = totalBytes - freeBytes;
+        }
+    }
+    else {
+        totalBytes = (0, node_os_1.totalmem)();
+        freeBytes = (0, node_os_1.freemem)();
+        usedBytes = totalBytes - freeBytes;
+    }
+    return {
+        totalBytes,
+        freeBytes,
+        usedBytes,
+        mainProcessBytes,
+        childProcessesBytes,
+    };
+}
+//# sourceMappingURL=memory-info.js.map