apify 2.3.1-beta.4 → 3.0.0-alpha.2

package/README.md

@@ -1,4 +1,4 @@
-# Apify SDK: The scalable web crawling and scraping library for JavaScript
+# Crawlee: The scalable web crawling and scraping library for JavaScript
 
 <!-- Mirror this part to src/index.js -->
 
@@ -24,10 +24,10 @@ Thanks to tools like [Playwright](https://github.com/microsoft/playwright), [Pup
 [Cheerio](https://www.npmjs.com/package/cheerio), it is easy to write Node.js code to extract data from web pages. But
 eventually things will get complicated. For example, when you try to:
 
--   Perform a deep crawl of an entire website using a persistent queue of URLs.
--   Run your scraping code on a list of 100k URLs in a CSV file, without losing any data when your code crashes.
--   Rotate proxies to hide your browser origin and keep user-like sessions.
--   Disable browser fingerprinting protections used by websites.
+- Perform a deep crawl of an entire website using a persistent queue of URLs.
+- Run your scraping code on a list of 100k URLs in a CSV file, without losing any data when your code crashes.
+- Rotate proxies to hide your browser origin and keep user-like sessions.
+- Disable browser fingerprinting protections used by websites.
 
 Python has [Scrapy](https://scrapy.org/) for these tasks, but there was no such library for **JavaScript, the language of
 the web**. The use of JavaScript is natural, since the same language is used to write the scripts as well as the data extraction code running in a
@@ -92,6 +92,7 @@ If you want to learn more, proceed to the [Getting Started](https://sdk.apify.co
 tutorial that will take you step by step through creating your first scraper.
 
 ### Local stand-alone usage
+
 Apify SDK requires [Node.js](https://nodejs.org/en/) 15.10 or later.
 Add Apify SDK to any Node.js project by running:
 

package/package.json

@@ -1,132 +1,73 @@
 {
-  "name": "apify",
-  "version": "2.3.1-beta.4",
-  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
-  "engines": {
-    "node": ">=15.10.0"
-  },
-  "main": "build/index.js",
-  "types": "build/index.d.ts",
-  "keywords": [
-    "apify",
-    "headless",
-    "chrome",
-    "puppeteer",
-    "crawler",
-    "scraper"
-  ],
-  "author": {
-    "name": "Apify",
-    "email": "support@apify.com",
-    "url": "https://apify.com"
-  },
-  "contributors": [
-    "Jan Curn <jan@apify.com>",
-    "Marek Trunkat <marek@apify.com>",
-    "Ondra Urban <ondra@apify.com>"
-  ],
-  "license": "Apache-2.0",
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/apify/apify-js"
-  },
-  "bugs": {
-    "url": "https://github.com/apify/apify-js/issues"
-  },
-  "homepage": "https://sdk.apify.com/",
-  "files": [
-    "build"
-  ],
-  "scripts": {
-    "build": "npm run clean && tsc -p tsconfig.json && node ./tools/typescript_fixes.js",
-    "build-docs": "npm run build && node ./website/tools/build_docs.js",
-    "build-readme": "node ./tools/build_readme.js",
-    "test": "npm run build && jest",
-    "prepare": "npm run build",
-    "prepublishOnly": "(test $CI || (echo \"Publishing is reserved to CI!\"; exit 1))",
-    "clean": "rimraf ./build",
-    "lint": "eslint ./src ./test",
-    "lint:fix": "eslint ./src ./test --ext .js,.jsx --fix"
-  },
-  "dependencies": {
-    "@apify/consts": "^1.3.0",
-    "@apify/datastructures": "^1.0.1",
-    "@apify/log": "^1.1.1",
-    "@apify/ps-tree": "^1.1.4",
-    "@apify/storage-local": "^2.0.2",
-    "@apify/timeout": "^0.2.1",
-    "@apify/utilities": "^1.1.2",
-    "apify-client": "^2.2.0",
-    "browser-pool": "^3.1.0",
-    "cheerio": "1.0.0-rc.10",
-    "content-type": "^1.0.4",
-    "got-scraping": "^3.2.7",
-    "htmlparser2": "^7.0.0",
-    "iconv-lite": "^0.6.3",
-    "jquery": "^3.6.0",
-    "mime-types": "^2.1.31",
-    "ow": "^0.28.0",
-    "rimraf": "^3.0.2",
-    "semver": "^7.3.5",
-    "stream-json": "^1.7.2",
-    "tough-cookie": "^4.0.0",
-    "underscore": "^1.13.0",
-    "ws": "^7.5.3"
-  },
-  "peerDependencies": {
-    "playwright": "^1.11.0",
-    "puppeteer": ">= 9.x <= 13.x"
-  },
-  "peerDependenciesMeta": {
-    "playwright": {
-      "optional": true
+    "name": "apify",
+    "version": "3.0.0-alpha.2",
+    "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
+    "engines": {
+        "node": ">=16.0.0"
     },
-    "puppeteer": {
-      "optional": true
+    "types": "index.d.ts",
+    "exports": {
+        ".": {
+            "import": "./index.mjs",
+            "require": "./index.js"
+        }
+    },
+    "keywords": [
+        "apify",
+        "headless",
+        "chrome",
+        "puppeteer",
+        "crawler",
+        "scraper"
+    ],
+    "author": {
+        "name": "Apify",
+        "email": "support@apify.com",
+        "url": "https://apify.com"
+    },
+    "contributors": [
+        "Jan Curn <jan@apify.com>",
+        "Marek Trunkat <marek@apify.com>",
+        "Ondra Urban <ondra@apify.com>"
+    ],
+    "license": "Apache-2.0",
+    "repository": {
+        "type": "git",
+        "url": "git+https://github.com/apify/apify-js"
+    },
+    "bugs": {
+        "url": "https://github.com/apify/apify-js/issues"
+    },
+    "homepage": "https://sdk.apify.com/",
+    "files": [
+        "dist"
+    ],
+    "scripts": {
+        "build": "npm run clean && npm run compile && npm run copy && npm run fixApifyExport",
+        "clean": "rimraf ./dist",
+        "compile": "tsc -p tsconfig.build.json && gen-esm-wrapper ./index.js ./index.mjs",
+        "copy": "ts-node -T ../../scripts/copy.ts",
+        "fixApifyExport": "ts-node -T ../../scripts/temp_fix_apify_exports.ts"
+    },
+    "publishConfig": {
+        "access": "public"
+    },
+    "dependencies": {
+        "@crawlee/core": "^3.0.0-alpha.2",
+        "@crawlee/utils": "^3.0.0-alpha.2",
+        "semver": "^7.3.5",
+        "apify-client": "^2.2.1-beta.5"
+    },
+    "peerDependencies": {
+        "playwright": "^1.11.0",
+        "puppeteer": ">= 9.x <= 13.x"
+    },
+    "peerDependenciesMeta": {
+        "playwright": {
+            "optional": true
+        },
+        "puppeteer": {
+            "optional": true
+        }
     }
-  },
-  "devDependencies": {
-    "@apify/eslint-config": "^0.3.0",
-    "@apify/tsconfig": "^0.1.0",
-    "@babel/cli": "^7.14.8",
-    "@babel/core": "^7.14.8",
-    "@babel/eslint-parser": "^7.14.7",
-    "@babel/plugin-syntax-class-properties": "^7.12.13",
-    "@babel/preset-env": "^7.14.8",
-    "@babel/register": "^7.14.5",
-    "@types/content-type": "^1.1.5",
-    "@types/domhandler": "^2.4.2",
-    "@types/fs-extra": "^9.0.12",
-    "@types/htmlparser2": "^3.10.3",
-    "@types/jest": "^27.0.1",
-    "@types/node": "^16.0.0",
-    "@types/ps-tree": "^1.1.1",
-    "@types/request-promise-native": "^1.0.18",
-    "@types/rimraf": "^3.0.2",
-    "@types/semver": "^7.3.7",
-    "@types/sinon": "^10.0.0",
-    "@types/tough-cookie": "^4.0.1",
-    "@types/underscore": "^1.11.3",
-    "@types/ws": "^7.4.7",
-    "basic-auth-parser": "^0.0.2",
-    "body-parser": "^1.19.0",
-    "eslint": "^8.2.0",
-    "express": "^4.17.1",
-    "fpcollect": "^1.0.4",
-    "fpscanner": "^0.1.5",
-    "fs-extra": "^10.0.0",
-    "jest": "^27.1.1",
-    "jest-circus": "^27.1.1",
-    "playwright": "1.21.0",
-    "portastic": "^1.0.1",
-    "proxy": "^1.0.2",
-    "proxy-chain": "^2.0.1-beta.0",
-    "puppeteer": "13.5.2",
-    "sinon": "^12.0.0",
-    "sinon-stub-promise": "^4.0.0",
-    "ts-jest": "^27.0.5",
-    "typescript": "4.5.5",
-    "ua-parser-js": "^1.0.0",
-    "why-is-node-running": "^2.2.0"
-  }
 }

package/build/actor.d.ts

@@ -1,113 +0,0 @@
-export function getEnv(): ApifyEnv;
-export function main(userFunc: UserFunc): void;
-export function call(actId: string, input?: {
-    [x: string]: any;
-} | undefined, options?: {
-    contentType?: string | undefined;
-    token?: string | undefined;
-    memoryMbytes?: number | undefined;
-    timeoutSecs?: number | undefined;
-    build?: string | undefined;
-    waitSecs?: number | undefined;
-    fetchOutput?: boolean | undefined;
-    disableBodyParser?: boolean | undefined;
-    webhooks?: AdhocWebhook[] | undefined;
-} | undefined): Promise<ActorRun>;
-export function callTask(taskId: string, input?: {
-    [x: string]: any;
-} | undefined, options?: {
-    token?: string | undefined;
-    memoryMbytes?: number | undefined;
-    timeoutSecs?: number | undefined;
-    build?: string | undefined;
-    waitSecs?: number | undefined;
-    webhooks?: AdhocWebhook[] | undefined;
-} | undefined): Promise<ActorRun>;
-export function metamorph(targetActorId: string, input?: {
-    [x: string]: any;
-} | undefined, options?: {
-    contentType?: string | undefined;
-    build?: string | undefined;
-} | undefined): Promise<void>;
-export function addWebhook(options: {
-    eventTypes: EventTypes;
-    requestUrl: string;
-    payloadTemplate?: string | undefined;
-    idempotencyKey?: string | undefined;
-}): Promise<WebhookRun | undefined>;
-/**
- * Parsed representation of the `APIFY_XXX` environmental variables.
- * This object is returned by the {@link ApifygetEnv } function.
- */
-export type ApifyEnv = {
-    /**
-     * ID of the actor (APIFY_ACTOR_ID)
-     */
-    actorId: string | null;
-    /**
-     * ID of the actor run (APIFY_ACTOR_RUN_ID)
-     */
-    actorRunId: string | null;
-    /**
-     * ID of the actor task (APIFY_ACTOR_TASK_ID)
-     */
-    actorTaskId: string | null;
-    /**
-     * ID of the user who started the actor - note that it might be
-     * different than the owner ofthe actor (APIFY_USER_ID)
-     */
-    userId: string | null;
-    /**
-     * Authentication token representing privileges given to the actor run,
-     * it can be passed to various Apify APIs (APIFY_TOKEN)
-     */
-    token: string | null;
-    /**
-     * Date when the actor was started (APIFY_STARTED_AT)
-     */
-    startedAt: Date | null;
-    /**
-     * Date when the actor will time out (APIFY_TIMEOUT_AT)
-     */
-    timeoutAt: Date | null;
-    /**
-     * ID of the key-value store where input and output data of this
-     * actor is stored (APIFY_DEFAULT_KEY_VALUE_STORE_ID)
-     */
-    defaultKeyValueStoreId: string | null;
-    /**
-     * ID of the dataset where input and output data of this
-     * actor is stored (APIFY_DEFAULT_DATASET_ID)
-     */
-    defaultDatasetId: string | null;
-    /**
-     * Amount of memory allocated for the actor,
-     * in megabytes (APIFY_MEMORY_MBYTES)
-     */
-    memoryMbytes: number | null;
-};
-export type EventTypes = Array<('ACTOR.RUN.SUCCEEDED' | 'ACTOR.RUN.ABORTED' | 'ACTOR.RUN.CREATED' | 'ACTOR.RUN.FAILED' | 'ACTOR.RUN.TIMED_OUT')>;
-export type UserFunc = () => Promise<void>;
-export type AdhocWebhook = {
-    eventTypes: EventTypes;
-    requestUrl: string;
-    idempotencyKey?: string | undefined;
-    payloadTemplate?: string | undefined;
-};
-export type WebhookRun = {
-    id: string;
-    createdAt: string;
-    modifiedAt: string;
-    userId: string;
-    isAdHoc: boolean;
-    eventTypes: EventTypes;
-    condition: any;
-    ignoreSslErrors: boolean;
-    doNotRetry: boolean;
-    requestUrl: string;
-    payloadTemplate: string;
-    lastDispatch: any;
-    stats: any;
-};
-import { ActorRun } from "./typedefs";
-//# sourceMappingURL=actor.d.ts.map

package/build/actor.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"actor.d.ts","sourceRoot":"","sources":["../src/actor.js"],"names":[],"mappings":"AA0EO,0BAFM,QAAQ,CAqBpB;AA0EM,+BAPI,QAAQ,QA8DlB;AAkFM,4BA5CI,MAAM;;;;;;;;;;;;gBAqCJ,QAAQ,QAAQ,CAAC,CAyE7B;AAgEM,iCAlCI,MAAM;;;;;;;;;gBA2BJ,QAAQ,QAAQ,CAAC,CAqE7B;AAkCM,yCAtBI,MAAM;;;;;gBAgBJ,QAAQ,IAAI,CAAC,CAmCzB;AAmDM;IAxBwB,UAAU;IAGb,UAAU,EAA3B,MAAM;IAEW,eAAe;IAOf,cAAc;IAK9B,QAAQ,UAAU,GAAC,SAAS,CAAC,CAqCxC;;;;;;;;;aAvlBa,MAAM,GAAC,IAAI;;;;gBACX,MAAM,GAAC,IAAI;;;;iBACX,MAAM,GAAC,IAAI;;;;;YACX,MAAM,GAAC,IAAI;;;;;WAEX,MAAM,GAAC,IAAI;;;;eAEX,IAAI,GAAC,IAAI;;;;eACT,IAAI,GAAC,IAAI;;;;;4BACT,MAAM,GAAC,IAAI;;;;;sBAEX,MAAM,GAAC,IAAI;;;;;kBAEX,MAAM,GAAC,IAAI;;yBAKZ,MAAM,CAAC,qBAAqB,GAAG,mBAAmB,GAAG,mBAAmB,GAAG,kBAAkB,GAAG,qBAAqB,CAAC,CAAC;6BAsCxH,QAAQ,IAAI,CAAC;;;gBAkIX,MAAM;;;;;QAgVN,MAAM;eACN,MAAM;gBACN,MAAM;YACN,MAAM;aACN,OAAO;;;qBAGP,OAAO;gBACP,OAAO;gBACP,MAAM;qBACN,MAAM"}