crawlee-one 1.0.0

package/dist/cjs/utils/valueMonitor.js

@@ -0,0 +1,91 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createSizeMonitor = exports.createValueMonitor = void 0;
+/**
+ * ValueMonitor checks and caches a (maybe remote) value, and triggers callbacks
+ * based on the value.
+ *
+ * You can think of it as a light replacement for reactivity (Vue) feature.
+ */
+const createValueMonitor = (fetchValueFn, options) => {
+    var _a;
+    const ttlInMs = (_a = options === null || options === void 0 ? void 0 : options.ttlInMs) !== null && _a !== void 0 ? _a : 5000;
+    let currValue = null;
+    let lastFetchTimestamp = null;
+    const callbacks = [];
+    const registerCallback = (callback) => {
+        callbacks.push(callback);
+        return () => deregisterCallback(callback);
+    };
+    const deregisterCallback = (callback) => {
+        const index = callbacks.indexOf(callback);
+        callbacks.splice(index, 1);
+    };
+    const isStale = () => {
+        if (lastFetchTimestamp == null)
+            return true;
+        const now = new Date().getTime();
+        const expiry = lastFetchTimestamp + ttlInMs;
+        return now >= expiry;
+    };
+    const refreshValue = () => __awaiter(void 0, void 0, void 0, function* () {
+        const oldValue = currValue;
+        const newValue = (currValue = yield fetchValueFn(oldValue));
+        lastFetchTimestamp = new Date().getTime();
+        for (const cb of callbacks) {
+            yield cb(newValue, oldValue);
+        }
+        return currValue;
+    });
+    const getValue = () => {
+        return isStale() ? refreshValue() : currValue;
+    };
+    return {
+        value: getValue,
+        isStale,
+        refresh: refreshValue,
+        onValue: registerCallback,
+    };
+};
+exports.createValueMonitor = createValueMonitor;
+/**
+ * Variant of ValueMonitor that is used for monitoring size of stores like Apify Dataset or RequestQueue.
+ * - Pass an array of items to `shortenToSize` to shorten the array to the size
+ *   that still fits the store.
+ */
+const createSizeMonitor = (maxSize, sizeGetter, onMaxSizeReached, options) => {
+    const valueMonitor = (0, exports.createValueMonitor)(sizeGetter, options);
+    const prepareSize = () => __awaiter(void 0, void 0, void 0, function* () {
+        const size = yield valueMonitor.value();
+        if (typeof size === 'number' && size >= 0 && !valueMonitor.isStale())
+            return size;
+        const newSize = yield valueMonitor.refresh();
+        return newSize;
+    });
+    const shortenToSize = (arr) => __awaiter(void 0, void 0, void 0, function* () {
+        const currSize = yield prepareSize();
+        return arr.slice(0, maxSize - currSize);
+    });
+    // Define handlers for when the value is updated
+    valueMonitor.onValue((newSize) => __awaiter(void 0, void 0, void 0, function* () {
+        if (newSize >= maxSize)
+            yield onMaxSizeReached();
+    }));
+    const isFull = () => __awaiter(void 0, void 0, void 0, function* () {
+        const currSize = yield prepareSize();
+        return currSize >= maxSize;
+    });
+    return Object.assign(Object.assign({}, valueMonitor), { shortenToSize,
+        isFull });
+};
+exports.createSizeMonitor = createSizeMonitor;
+//# sourceMappingURL=valueMonitor.js.map

package/dist/cjs/utils/valueMonitor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"valueMonitor.js","sourceRoot":"","sources":["../../../src/utils/valueMonitor.ts"],"names":[],"mappings":";;;;;;;;;;;;AAQA;;;;;GAKG;AACI,MAAM,kBAAkB,GAAG,CAChC,YAAqD,EACrD,OAA6B,EAC7B,EAAE;;IACF,MAAM,OAAO,GAAG,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,OAAO,mCAAI,IAAI,CAAC;IAEzC,IAAI,SAAS,GAAa,IAAI,CAAC;IAC/B,IAAI,kBAAkB,GAAkB,IAAI,CAAC;IAC7C,MAAM,SAAS,GAAuB,EAAE,CAAC;IAEzC,MAAM,gBAAgB,GAAG,CAAC,QAA0B,EAAE,EAAE;QACtD,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACzB,OAAO,GAAG,EAAE,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;IAC5C,CAAC,CAAC;IAEF,MAAM,kBAAkB,GAAG,CAAC,QAA0B,EAAE,EAAE;QACxD,MAAM,KAAK,GAAG,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC1C,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IAC7B,CAAC,CAAC;IAEF,MAAM,OAAO,GAAG,GAAG,EAAE;QACnB,IAAI,kBAAkB,IAAI,IAAI;YAAE,OAAO,IAAI,CAAC;QAE5C,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;QACjC,MAAM,MAAM,GAAG,kBAAkB,GAAG,OAAO,CAAC;QAC5C,OAAO,GAAG,IAAI,MAAM,CAAC;IACvB,CAAC,CAAC;IAEF,MAAM,YAAY,GAAG,GAAS,EAAE;QAC9B,MAAM,QAAQ,GAAG,SAAS,CAAC;QAC3B,MAAM,QAAQ,GAAG,CAAC,SAAS,GAAG,MAAM,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC5D,kBAAkB,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;QAE1C,KAAK,MAAM,EAAE,IAAI,SAAS,EAAE;YAC1B,MAAM,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;SAC9B;QAED,OAAO,SAAc,CAAC;IACxB,CAAC,CAAA,CAAC;IAEF,MAAM,QAAQ,GAAG,GAAG,EAAE;QACpB,OAAO,OAAO,EAAE,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IAChD,CAAC,CAAC;IAEF,OAAO;QACL,KAAK,EAAE,QAAQ;QACf,OAAO;QACP,OAAO,EAAE,YAAY;QACrB,OAAO,EAAE,gBAAgB;KAC1B,CAAC;AACJ,CAAC,CAAC;AAlDW,QAAA,kBAAkB,sBAkD7B;AAEF;;;;GAIG;AACI,MAAM,iBAAiB,GAAG,CAC/B,OAAe,EACf,UAAsC,EACtC,gBAA0C,EAC1C,OAA6B,EAC7B,EAAE;IACF,MAAM,YAAY,GAAG,IAAA,0BAAkB,EAAS,UAAU,EAAE,OAAO,CAAC,CAAC;IAErE,MAAM,WAAW,GAAG,GAAS,EAAE;QAC7B,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,KAAK,EAAE,CAAC;QACxC,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE;YAAE,OAAO,IAAI,CAAC;QAClF,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,OAAO,EAAE,CAAC;QAC7C,OAAO,OAAO,CAAC;IACjB,CAAC,CAAA,CAAC;IAEF,MAAM,aAAa,GAAG,CAAU,GAAQ,EAAE,EAAE;QAC1C,MAAM,QAAQ,GAAG,MAAM,WAAW,EAAE,CAAC;QACrC,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,GAAG,QAAQ,CAAC,CAAC;IAC1C,CAAC,CAAA,CAAC;IAEF,gDAAgD;IAChD,YAAY,CAAC,OAAO,CAAC,CAAO,OAAO,EAAE,EAAE;QACrC,IAAI,OAAO,IAAI,OAAO;YAAE,MAAM,gBAAgB,EAAE,CAAC;IACnD,CAAC,CAAA,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,GAAS,EAAE;QACxB,MAAM,QAAQ,GAAG,MAAM,WAAW,EAAE,CAAC;QACrC,OAAO,QAAQ,IAAI,OAAO,CAAC;IAC7B,CAAC,CAAA,CAAC;IAEF,uCACK,YAAY,KACf,aAAa;QACb,MAAM,IACN;AACJ,CAAC,CAAC;AAnCW,QAAA,iBAAiB,qBAmC5B","sourcesContent":["import type { MaybePromise } from './types';\n\nexport type ValueCallback<T> = (value: T, oldValue: T | null) => MaybePromise<any>;\nexport interface ValueMonitorOptions {\n  /** How long (in milliseconds) after fetching the value can we use it before we have to re-fetch it. */\n  ttlInMs?: number;\n}\n\n/**\n * ValueMonitor checks and caches a (maybe remote) value, and triggers callbacks\n * based on the value.\n *\n * You can think of it as a light replacement for reactivity (Vue) feature.\n */\nexport const createValueMonitor = <T>(\n  fetchValueFn: (oldValue: T | null) => MaybePromise<T>,\n  options?: ValueMonitorOptions\n) => {\n  const ttlInMs = options?.ttlInMs ?? 5000;\n\n  let currValue: T | null = null;\n  let lastFetchTimestamp: number | null = null;\n  const callbacks: ValueCallback<T>[] = [];\n\n  const registerCallback = (callback: ValueCallback<T>) => {\n    callbacks.push(callback);\n    return () => deregisterCallback(callback);\n  };\n\n  const deregisterCallback = (callback: ValueCallback<T>) => {\n    const index = callbacks.indexOf(callback);\n    callbacks.splice(index, 1);\n  };\n\n  const isStale = () => {\n    if (lastFetchTimestamp == null) return true;\n\n    const now = new Date().getTime();\n    const expiry = lastFetchTimestamp + ttlInMs;\n    return now >= expiry;\n  };\n\n  const refreshValue = async () => {\n    const oldValue = currValue;\n    const newValue = (currValue = await fetchValueFn(oldValue));\n    lastFetchTimestamp = new Date().getTime();\n\n    for (const cb of callbacks) {\n      await cb(newValue, oldValue);\n    }\n\n    return currValue as T;\n  };\n\n  const getValue = () => {\n    return isStale() ? refreshValue() : currValue;\n  };\n\n  return {\n    value: getValue,\n    isStale,\n    refresh: refreshValue,\n    onValue: registerCallback,\n  };\n};\n\n/**\n * Variant of ValueMonitor that is used for monitoring size of stores like Apify Dataset or RequestQueue.\n * - Pass an array of items to `shortenToSize` to shorten the array to the size\n *   that still fits the store.\n */\nexport const createSizeMonitor = (\n  maxSize: number,\n  sizeGetter: () => MaybePromise<number>,\n  onMaxSizeReached: () => MaybePromise<void>,\n  options?: ValueMonitorOptions\n) => {\n  const valueMonitor = createValueMonitor<number>(sizeGetter, options);\n\n  const prepareSize = async () => {\n    const size = await valueMonitor.value();\n    if (typeof size === 'number' && size >= 0 && !valueMonitor.isStale()) return size;\n    const newSize = await valueMonitor.refresh();\n    return newSize;\n  };\n\n  const shortenToSize = async <T>(arr: T[]) => {\n    const currSize = await prepareSize();\n    return arr.slice(0, maxSize - currSize);\n  };\n\n  // Define handlers for when the value is updated\n  valueMonitor.onValue(async (newSize) => {\n    if (newSize >= maxSize) await onMaxSizeReached();\n  });\n\n  const isFull = async () => {\n    const currSize = await prepareSize();\n    return currSize >= maxSize;\n  };\n\n  return {\n    ...valueMonitor,\n    shortenToSize,\n    isFull,\n  };\n};\n"]}

package/package.json

@@ -0,0 +1,85 @@
+{
+  "name": "crawlee-one",
+  "version": "1.0.0",
+  "private": false,
+  "description": "Crawlee One is a framework built on top of Crawlee and Apify for writing robust and highly configurable web scrapers",
+  "author": "Juro Oravec <juraj.oravec.josefson@gmail.com>",
+  "homepage": "https://github.com/jurooravec/crawlee-one#readme",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/jurooravec/crawlee-one.git"
+  },
+  "bugs": {
+    "url": "https://github.com/jurooravec/crawlee-one/issues"
+  },
+  "license": "MIT",
+  "keywords": [
+    "crawlee",
+    "crawlee-one",
+    "apify",
+    "actor",
+    "crawler",
+    "scraper",
+    "template"
+  ],
+  "files": [
+    "dist"
+  ],
+  "exports": {
+    ".": "./dist/cjs/index.js",
+    "./apify": "./dist/cjs/lib/integrations/apify.js"
+  },
+  "module": "dist/esm/index.js",
+  "types": "dist/cjs/index.d.js",
+  "bin": "dist/cjs/cli/index.js",
+  "engines": {
+    "node": ">=16.0.0"
+  },
+  "scripts": {
+    "start": "npm run start:dev",
+    "start:prod": "node dist/cjs/index.js",
+    "start:dev": "ts-node ./src/index.ts",
+    "build": "npm run build:cjs",
+    "build:all": "npm run build:cjs && npm run build:esm",
+    "build:cjs": "tsc -p tsconfig.cjs.json",
+    "build:esm": "tsc -p tsconfig.esm.json",
+    "lint": "eslint ./src --ext .ts",
+    "lint:fix": "eslint ./src --ext .ts --fix",
+    "test": "vitest run",
+    "coverage": "vitest run --coverage"
+  },
+  "devDependencies": {
+    "@types/lodash": "^4.14.194",
+    "@typescript-eslint/eslint-plugin": "^4.33.0",
+    "@typescript-eslint/parser": "^4.33.0",
+    "eslint": "^7.32.0",
+    "eslint-config-prettier": "^6.15.0",
+    "eslint-plugin-prettier": "^3.4.1",
+    "playwright": "^1.32.3",
+    "prettier": "^2.5.1",
+    "ts-node": "^10.9.1",
+    "typescript": "^5.0.4",
+    "vitest": "^0.30.1"
+  },
+  "peerDependencies": {
+    "@sentry/node": "^7.50.0",
+    "actor-spec": "^0.4.1",
+    "apify": "^3.1.x",
+    "apify-actor-config": "^1.0.3",
+    "apify-client": "^2.7.x",
+    "cheerio": "^1.0.0-rc.12",
+    "crawlee": "^3.3.1",
+    "joi": "^17.9.2",
+    "playwright": "^1.32.x",
+    "vitest": "^0.30.1"
+  },
+  "dependencies": {
+    "commander": "^10.0.1",
+    "dotenv": "^16.0.3",
+    "eta": "^2.0.1",
+    "glob": "^10.2.2",
+    "lodash": "^4.17.21",
+    "millify": "^6.1.0",
+    "pkginfo": "^0.4.1"
+  }
+}