apify-cli 0.18.2-beta.9 → 0.19.0

package/README.md

@@ -299,7 +299,7 @@ USAGE
   $ apify actor
 ```
 
-_See code: [src/commands/actor/index.js](https://github.com/apify/apify-cli/blob/v0.18.2/src/commands/actor/index.js)_
+_See code: [src/commands/actor/index.js](https://github.com/apify/apify-cli/blob/v0.19.0/src/commands/actor/index.js)_
 
 ## `apify actor:get-input`
 
@@ -310,7 +310,7 @@ USAGE
   $ apify actor:get-input
 ```
 
-_See code: [src/commands/actor/get-input.js](https://github.com/apify/apify-cli/blob/v0.18.2/src/commands/actor/get-input.js)_
+_See code: [src/commands/actor/get-input.js](https://github.com/apify/apify-cli/blob/v0.19.0/src/commands/actor/get-input.js)_
 
 ## `apify actor:get-value KEY`
 
@@ -324,7 +324,7 @@ ARGUMENTS
   KEY  Key of the record in key-value store
 ```
 
-_See code: [src/commands/actor/get-value.js](https://github.com/apify/apify-cli/blob/v0.18.2/src/commands/actor/get-value.js)_
+_See code: [src/commands/actor/get-value.js](https://github.com/apify/apify-cli/blob/v0.19.0/src/commands/actor/get-value.js)_
 
 ## `apify actor:push-data [ITEM]`
 
@@ -345,7 +345,7 @@ DESCRIPTION
   $ cat ./test.json | apify actor:push-data
 ```
 
-_See code: [src/commands/actor/push-data.js](https://github.com/apify/apify-cli/blob/v0.18.2/src/commands/actor/push-data.js)_
+_See code: [src/commands/actor/push-data.js](https://github.com/apify/apify-cli/blob/v0.19.0/src/commands/actor/push-data.js)_
 
 ## `apify actor:set-value KEY [VALUE]`
 
@@ -375,7 +375,7 @@ DESCRIPTION
   $ cat ./my-text-file.txt | apify actor:set-value KEY --contentType text/plain
 ```
 
-_See code: [src/commands/actor/set-value.js](https://github.com/apify/apify-cli/blob/v0.18.2/src/commands/actor/set-value.js)_
+_See code: [src/commands/actor/set-value.js](https://github.com/apify/apify-cli/blob/v0.19.0/src/commands/actor/set-value.js)_
 
 ## `apify call [ACTID]`
 
@@ -400,7 +400,7 @@ DESCRIPTION
   takes input for the Actor from the default local key-value store by default.
 ```
 
-_See code: [src/commands/call.js](https://github.com/apify/apify-cli/blob/v0.18.2/src/commands/call.js)_
+_See code: [src/commands/call.js](https://github.com/apify/apify-cli/blob/v0.19.0/src/commands/call.js)_
 
 ## `apify create [ACTORNAME]`
 
@@ -422,7 +422,7 @@ OPTIONS
   --skip-dependency-install  Skip installing actor dependencies.
 ```
 
-_See code: [src/commands/create.js](https://github.com/apify/apify-cli/blob/v0.18.2/src/commands/create.js)_
+_See code: [src/commands/create.js](https://github.com/apify/apify-cli/blob/v0.19.0/src/commands/create.js)_
 
 ## `apify info`
 
@@ -436,7 +436,7 @@ DESCRIPTION
   The information is printed to the console.
 ```
 
-_See code: [src/commands/info.js](https://github.com/apify/apify-cli/blob/v0.18.2/src/commands/info.js)_
+_See code: [src/commands/info.js](https://github.com/apify/apify-cli/blob/v0.19.0/src/commands/info.js)_
 
 ## `apify init [ACTORNAME]`
 
@@ -449,6 +449,10 @@ USAGE
 ARGUMENTS
   ACTORNAME  Name of the actor. If not provided, you will be prompted for it.
 
+OPTIONS
+  -y, --yes  Automatic yes to prompts; assume "yes" as answer to all prompts. Note that in some cases, the command may
+             still ask for confirmation.
+
 DESCRIPTION
   The command only creates the ".actor/actor.json" file and the "storage" directory in the current directory, but will 
   not touch anything else.
@@ -456,7 +460,7 @@ DESCRIPTION
   WARNING: The directory at "storage" will be overwritten if it already exists.
 ```
 
-_See code: [src/commands/init.js](https://github.com/apify/apify-cli/blob/v0.18.2/src/commands/init.js)_
+_See code: [src/commands/init.js](https://github.com/apify/apify-cli/blob/v0.19.0/src/commands/init.js)_
 
 ## `apify login`
 
@@ -474,7 +478,7 @@ DESCRIPTION
   "apify" commands. To log out, call "apify logout".
 ```
 
-_See code: [src/commands/login.js](https://github.com/apify/apify-cli/blob/v0.18.2/src/commands/login.js)_
+_See code: [src/commands/login.js](https://github.com/apify/apify-cli/blob/v0.19.0/src/commands/login.js)_
 
 ## `apify logout`
 
@@ -489,7 +493,7 @@ DESCRIPTION
    call "apify login".
 ```
 
-_See code: [src/commands/logout.js](https://github.com/apify/apify-cli/blob/v0.18.2/src/commands/logout.js)_
+_See code: [src/commands/logout.js](https://github.com/apify/apify-cli/blob/v0.19.0/src/commands/logout.js)_
 
 ## `apify pull [ACTORID]`
 
@@ -507,7 +511,7 @@ OPTIONS
   -v, --version=version  Actor version number which will be pulled, e.g. 1.2. Default: the highest version
 ```
 
-_See code: [src/commands/pull.js](https://github.com/apify/apify-cli/blob/v0.18.2/src/commands/pull.js)_
+_See code: [src/commands/pull.js](https://github.com/apify/apify-cli/blob/v0.19.0/src/commands/pull.js)_
 
 ## `apify push [ACTORID]`
 
@@ -545,7 +549,7 @@ DESCRIPTION
   WARNING: If the target Actor already exists in your Apify account, it will be overwritten!
 ```
 
-_See code: [src/commands/push.js](https://github.com/apify/apify-cli/blob/v0.18.2/src/commands/push.js)_
+_See code: [src/commands/push.js](https://github.com/apify/apify-cli/blob/v0.19.0/src/commands/push.js)_
 
 ## `apify run`
 
@@ -575,7 +579,7 @@ DESCRIPTION
   package.json file. You can set up your own main file or environment variables by changing it.
 ```
 
-_See code: [src/commands/run.js](https://github.com/apify/apify-cli/blob/v0.18.2/src/commands/run.js)_
+_See code: [src/commands/run.js](https://github.com/apify/apify-cli/blob/v0.19.0/src/commands/run.js)_
 
 ## `apify secrets`
 
@@ -603,7 +607,7 @@ DESCRIPTION
    of the actor.
 ```
 
-_See code: [src/commands/secrets/index.js](https://github.com/apify/apify-cli/blob/v0.18.2/src/commands/secrets/index.js)_
+_See code: [src/commands/secrets/index.js](https://github.com/apify/apify-cli/blob/v0.19.0/src/commands/secrets/index.js)_
 
 ## `apify secrets:add NAME VALUE`
 
@@ -621,7 +625,7 @@ DESCRIPTION
   The secrets are stored to a file at ~/.apify
 ```
 
-_See code: [src/commands/secrets/add.js](https://github.com/apify/apify-cli/blob/v0.18.2/src/commands/secrets/add.js)_
+_See code: [src/commands/secrets/add.js](https://github.com/apify/apify-cli/blob/v0.19.0/src/commands/secrets/add.js)_
 
 ## `apify secrets:rm NAME`
 
@@ -635,7 +639,7 @@ ARGUMENTS
   NAME  Name of the secret
 ```
 
-_See code: [src/commands/secrets/rm.js](https://github.com/apify/apify-cli/blob/v0.18.2/src/commands/secrets/rm.js)_
+_See code: [src/commands/secrets/rm.js](https://github.com/apify/apify-cli/blob/v0.19.0/src/commands/secrets/rm.js)_
 
 ## `apify vis [PATH]`
 
@@ -659,5 +663,5 @@ DESCRIPTION
   You can also pass any custom path to your input schema to have it validated instead.
 ```
 
-_See code: [src/commands/vis.js](https://github.com/apify/apify-cli/blob/v0.18.2/src/commands/vis.js)_
+_See code: [src/commands/vis.js](https://github.com/apify/apify-cli/blob/v0.19.0/src/commands/vis.js)_
 <!-- commandsstop -->

package/oclif.manifest.json

@@ -1 +1 @@
-{"version":"0.18.2","commands":{"call":{"id":"call","description":"Runs a specific actor remotely on the Apify cloud platform.\nThe Actor is run under your current Apify account. Therefore you need to be logged in by calling \"apify login\". It takes input for the Actor from the default local key-value store by default.","pluginName":"apify-cli","pluginType":"core","aliases":[],"flags":{"build":{"name":"build","type":"option","char":"b","description":"Tag or number of the build to run (e.g. \"latest\" or \"1.2.34\").","required":false},"timeout":{"name":"timeout","type":"option","char":"t","description":"Timeout for the actor run in seconds. Zero value means there is no timeout.","required":false},"memory":{"name":"memory","type":"option","char":"m","description":"Amount of memory allocated for the actor run, in megabytes.","required":false},"wait-for-finish":{"name":"wait-for-finish","type":"option","char":"w","description":"Seconds for waiting to run to finish, if no value passed, it waits forever.","required":false}},"args":[{"name":"actId","description":"Name or ID of the actor to run (e.g. \"apify/hello-world\" or \"E2jjCZBezvAZnX8Rb\"). If not provided, the command runs the remote actor specified in the \".actor/actor.json\" file.","required":false}]},"check-version":{"id":"check-version","description":"Checks that installed Apify CLI version is up to date.","pluginName":"apify-cli","pluginType":"core","hidden":true,"aliases":["cv"],"flags":{"enforce-update":{"name":"enforce-update","type":"boolean","char":"e","description":"[Optional] Enforce version update from NPM","required":false,"allowNo":false}},"args":[]},"create":{"id":"create","description":"Creates a new actor project directory from a selected boilerplate template.","pluginName":"apify-cli","pluginType":"core","aliases":[],"flags":{"template":{"name":"template","type":"option","char":"t","description":"Template for the actor. If not provided, the command will prompt for it.\nVisit https://raw.githubusercontent.com/apify/actor-templates/master/templates/manifest.json to find available template names.","required":false},"skip-dependency-install":{"name":"skip-dependency-install","type":"boolean","description":"Skip installing actor dependencies.","required":false,"allowNo":false},"template-archive-url":{"name":"template-archive-url","type":"option","description":"Actor template archive url. Useful for developing new templates.","hidden":true,"required":false}},"args":[{"name":"actorName","description":"Name of the actor and its directory","required":false}]},"edit-input-schema":{"id":"edit-input-schema","description":"Lets you edit your input schema that would be used on the platform in a visual input schema editor.","pluginName":"apify-cli","pluginType":"core","hidden":true,"aliases":["eis"],"flags":{},"args":[{"name":"path","description":"Optional path to your INPUT_SCHEMA.json file. If not provided default platform location for input schema is used.","required":false}]},"info":{"id":"info","description":"Displays information about the currently active Apify account.\nThe information is printed to the console.","pluginName":"apify-cli","pluginType":"core","aliases":[],"flags":{},"args":[]},"init":{"id":"init","description":"Initializes a new actor project in an existing directory.\nThe command only creates the \".actor/actor.json\" file and the \"storage\" directory in the current directory, but will not touch anything else.\n\nWARNING: The directory at \"storage\" will be overwritten if it already exists.","pluginName":"apify-cli","pluginType":"core","aliases":[],"flags":{},"args":[{"name":"actorName","description":"Name of the actor. If not provided, you will be prompted for it.","required":false}]},"login-new":{"id":"login-new","description":"Logs in to your Apify account using your API token.\nThe API token and other account information is stored in the ~/.apify directory, from where it is read by all other \"apify\" commands. To log out, call \"apify logout\".","pluginName":"apify-cli","pluginType":"core","hidden":true,"aliases":[],"flags":{"token":{"name":"token","type":"option","char":"t","description":"[Optional] Apify API token","required":false}},"args":[]},"login":{"id":"login","description":"Logs in to your Apify account using a provided API token.\nThe API token and other account information is stored in the ~/.apify directory, from where it is read by all other \"apify\" commands. To log out, call \"apify logout\".","pluginName":"apify-cli","pluginType":"core","aliases":[],"flags":{"token":{"name":"token","type":"option","char":"t","description":"[Optional] Apify API token","required":false}},"args":[]},"logout":{"id":"logout","description":"Logs out of your Apify account.\nThe command deletes the API token and all other account information stored in the ~/.apify directory. To log in again, call \"apify login\".","pluginName":"apify-cli","pluginType":"core","aliases":[],"flags":{},"args":[]},"pull":{"id":"pull","description":"Pulls an Actor from the Apify platform to the current directory. If it is defined as Git repository, it will be cloned. If it is defined as Web IDE, it will fetch the files.","pluginName":"apify-cli","pluginType":"core","aliases":[],"flags":{"version":{"name":"version","type":"option","char":"v","description":"Actor version number which will be pulled, e.g. 1.2. Default: the highest version","required":false}},"args":[{"name":"actorId","description":"Name or ID of the actor to run (e.g. \"apify/hello-world\" or \"E2jjCZBezvAZnX8Rb\"). If not provided, the command will update the Actor in the current directory based on its name in \".actor/actor.json\" file.","required":false}]},"push":{"id":"push","description":"Uploads the actor to the Apify platform and builds it there.\nThe Actor settings are read from the \".actor/actor.json\" file in the current directory, but they can be overridden using command-line options.\nNOTE: If the source files are smaller than 3 MB then they are uploaded as \n\"Multiple source files\", otherwise they are uploaded as \"Zip file\".\n\nWARNING: If the target Actor already exists in your Apify account, it will be overwritten!","pluginName":"apify-cli","pluginType":"core","aliases":[],"flags":{"version-number":{"name":"version-number","type":"option","description":"DEPRECATED: Use flag version instead. Actor version number to which the files should be pushed. By default, it is taken from the \".actor/actor.json\" file.","required":false},"version":{"name":"version","type":"option","char":"v","description":"Actor version number to which the files should be pushed. By default, it is taken from the \".actor/actor.json\" file.","required":false},"build-tag":{"name":"build-tag","type":"option","char":"b","description":"Build tag to be applied to the successful Actor build. By default, it is taken from the \".actor/actor.json\" file","required":false},"wait-for-finish":{"name":"wait-for-finish","type":"option","char":"w","description":"Seconds for waiting to build to finish, if no value passed, it waits forever.","required":false},"no-prompt":{"name":"no-prompt","type":"boolean","description":"Do not prompt for opening the actor details in a browser. This will also not open the browser automatically.","required":false,"allowNo":false}},"args":[{"name":"actorId","description":"Name or ID of the Actor to push (e.g. \"apify/hello-world\" or \"E2jjCZBezvAZnX8Rb\"). If not provided, the command will create or modify the actor with the name specified in \".actor/actor.json\" file.","required":false}]},"run":{"id":"run","description":"Runs the actor locally in the current directory.\nIt sets various APIFY_XYZ environment variables in order to provide a working execution environment for the actor. For example, this causes the actor input, as well as all other data in key-value stores, datasets or request queues to be stored in the \"storage\" directory, rather than on the Apify platform.\n\nNOTE: You can override the command's default behavior for Node.js actors by overriding the \"start\" script in the package.json file. You can set up your own main file or environment variables by changing it.","pluginName":"apify-cli","pluginType":"core","aliases":[],"flags":{"purge":{"name":"purge","type":"boolean","char":"p","description":"Shortcut that combines the --purge-queue, --purge-dataset and --purge-key-value-store options.","required":false,"allowNo":false},"purge-queue":{"name":"purge-queue","type":"boolean","description":"Deletes the local directory containing the default request queue before the run starts.","required":false,"allowNo":false},"purge-dataset":{"name":"purge-dataset","type":"boolean","description":"Deletes the local directory containing the default dataset before the run starts.","required":false,"allowNo":false},"purge-key-value-store":{"name":"purge-key-value-store","type":"boolean","description":"Deletes all records from the default key-value store in the local directory before the run starts, except for the \"INPUT\" key.","required":false,"allowNo":false}},"args":[]},"vis":{"id":"vis","description":"Validates input schema and prints errors found.\nThe input schema for the actor is used from these locations in order of preference.\nThe first one found is validated as it would be the one used on the Apify platform.\n1. Directly embedded object in \".actor/actor.json\" under 'input' key\n2. Path to JSON file referenced in \".actor/actor.json\" under 'input' key\n3. JSON file at .actor/INPUT_SCHEMA.json\n4. JSON file at INPUT_SCHEMA.json\n\nYou can also pass any custom path to your input schema to have it validated instead.\n","pluginName":"apify-cli","pluginType":"core","aliases":[],"flags":{},"args":[{"name":"path","description":"Optional path to your INPUT_SCHEMA.json file. If not provided ./INPUT_SCHEMA.json is used.","required":false}]},"actor:get-input":{"id":"actor:get-input","description":"Gets the actor input value from the default key-value store associated with the actor run.","pluginName":"apify-cli","pluginType":"core","aliases":[],"flags":{},"args":[]},"actor:get-value":{"id":"actor:get-value","description":"Gets a value from the default key-value store associated with the actor run.","pluginName":"apify-cli","pluginType":"core","aliases":[],"flags":{},"args":[{"name":"key","description":"Key of the record in key-value store","required":true}]},"actor":{"id":"actor","description":"Commands are designed to be used in actor runs. All commands are in PoC state, do not use in production environments.\n","pluginName":"apify-cli","pluginType":"core","aliases":[],"flags":{},"args":[]},"actor:push-data":{"id":"actor:push-data","description":"Stores an object or an array of objects to the default dataset of the actor run.\nIt is possible to pass data using item argument or stdin.\nPassing data using argument:\n$ apify actor:push-data {\"foo\": \"bar\"}\nPassing data using stdin with pipe:\n$ cat ./test.json | apify actor:push-data\n","pluginName":"apify-cli","pluginType":"core","aliases":[],"flags":{},"args":[{"name":"item","description":"JSON string with one object or array of objects containing data to be stored in the default dataset.","required":false}]},"actor:set-value":{"id":"actor:set-value","description":"Sets or removes record into the default KeyValueStore associated with the actor run.\nIt is possible to pass data using argument or stdin.\nPassing data using argument:\n$ apify actor:set-value KEY my-value\nPassing data using stdin with pipe:\n$ cat ./my-text-file.txt | apify actor:set-value KEY --contentType text/plain\n","pluginName":"apify-cli","pluginType":"core","aliases":[],"flags":{"contentType":{"name":"contentType","type":"option","char":"c","description":"Specifies a custom MIME content type of the record. By default \"application/json\" is used.","required":false}},"args":[{"name":"key","description":"Key of the record in key-value store.","required":true},{"name":"value","description":"Record data, which can be one of the following values:\n- If empty, the record in the key-value store is deleted.\n- If no `contentType` flag is specified, value is expected to be any JSON string value.\n- If options.contentType is set, value is taken as is.","required":false}]},"secrets:add":{"id":"secrets:add","description":"Adds a new secret value.\nThe secrets are stored to a file at ~/.apify","pluginName":"apify-cli","pluginType":"core","aliases":[],"flags":{},"args":[{"name":"name","description":"Name of the secret","required":true},{"name":"value","description":"Value of the secret","required":true}]},"secrets":{"id":"secrets","description":"Manages secret values for actor environment variables.\n\nExample:\n$ apify secrets:add mySecret TopSecretValue123\n\nNow the \"mySecret\" value can be used in an environment variable defined in \".actor/actor.json\" file by adding the \"@\" prefix:\n\n{\n  \"actorSpecification\": 1,\n  \"name\": \"my_actor\",\n  \"environmentVariables\": { \"SECRET_ENV_VAR\": \"@mySecret\" },\n  \"version\": \"0.1\n}\n\nWhen the actor is pushed to Apify cloud, the \"SECRET_ENV_VAR\" and its value is stored as a secret environment variable of the actor.","pluginName":"apify-cli","pluginType":"core","aliases":[],"flags":{},"args":[]},"secrets:rm":{"id":"secrets:rm","description":"Removes the secret.","pluginName":"apify-cli","pluginType":"core","aliases":[],"flags":{},"args":[{"name":"name","description":"Name of the secret","required":true}]}}}
+{"version":"0.19.0","commands":{"call":{"id":"call","description":"Runs a specific actor remotely on the Apify cloud platform.\nThe Actor is run under your current Apify account. Therefore you need to be logged in by calling \"apify login\". It takes input for the Actor from the default local key-value store by default.","pluginName":"apify-cli","pluginType":"core","aliases":[],"flags":{"build":{"name":"build","type":"option","char":"b","description":"Tag or number of the build to run (e.g. \"latest\" or \"1.2.34\").","required":false},"timeout":{"name":"timeout","type":"option","char":"t","description":"Timeout for the actor run in seconds. Zero value means there is no timeout.","required":false},"memory":{"name":"memory","type":"option","char":"m","description":"Amount of memory allocated for the actor run, in megabytes.","required":false},"wait-for-finish":{"name":"wait-for-finish","type":"option","char":"w","description":"Seconds for waiting to run to finish, if no value passed, it waits forever.","required":false}},"args":[{"name":"actId","description":"Name or ID of the actor to run (e.g. \"apify/hello-world\" or \"E2jjCZBezvAZnX8Rb\"). If not provided, the command runs the remote actor specified in the \".actor/actor.json\" file.","required":false}]},"check-version":{"id":"check-version","description":"Checks that installed Apify CLI version is up to date.","pluginName":"apify-cli","pluginType":"core","hidden":true,"aliases":["cv"],"flags":{"enforce-update":{"name":"enforce-update","type":"boolean","char":"e","description":"[Optional] Enforce version update from NPM","required":false,"allowNo":false}},"args":[]},"create":{"id":"create","description":"Creates a new actor project directory from a selected boilerplate template.","pluginName":"apify-cli","pluginType":"core","aliases":[],"flags":{"template":{"name":"template","type":"option","char":"t","description":"Template for the actor. If not provided, the command will prompt for it.\nVisit https://raw.githubusercontent.com/apify/actor-templates/master/templates/manifest.json to find available template names.","required":false},"skip-dependency-install":{"name":"skip-dependency-install","type":"boolean","description":"Skip installing actor dependencies.","required":false,"allowNo":false},"template-archive-url":{"name":"template-archive-url","type":"option","description":"Actor template archive url. Useful for developing new templates.","hidden":true,"required":false}},"args":[{"name":"actorName","description":"Name of the actor and its directory","required":false}]},"edit-input-schema":{"id":"edit-input-schema","description":"Lets you edit your input schema that would be used on the platform in a visual input schema editor.","pluginName":"apify-cli","pluginType":"core","hidden":true,"aliases":["eis"],"flags":{},"args":[{"name":"path","description":"Optional path to your INPUT_SCHEMA.json file. If not provided default platform location for input schema is used.","required":false}]},"info":{"id":"info","description":"Displays information about the currently active Apify account.\nThe information is printed to the console.","pluginName":"apify-cli","pluginType":"core","aliases":[],"flags":{},"args":[]},"init-wrap-scrapy":{"id":"init-wrap-scrapy","description":"Wraps your existing Scrapy project to work like an Apify Actor.\n\nIt adds the following features:\n- Automatic retry of failed requests\n- Automatic proxy rotation\n- Automatic user agent rotation\n...\n","pluginName":"apify-cli","pluginType":"core","hidden":true,"aliases":[],"flags":{},"args":[{"name":"path","description":"Optional path to your scrapy project. If not provided, the current directory is used.","required":false}]},"init":{"id":"init","description":"Initializes a new actor project in an existing directory.\nThe command only creates the \".actor/actor.json\" file and the \"storage\" directory in the current directory, but will not touch anything else.\n\nWARNING: The directory at \"storage\" will be overwritten if it already exists.","pluginName":"apify-cli","pluginType":"core","aliases":[],"flags":{"yes":{"name":"yes","type":"boolean","char":"y","description":"Automatic yes to prompts; assume \"yes\" as answer to all prompts. Note that in some cases, the command may still ask for confirmation.","required":false,"allowNo":false}},"args":[{"name":"actorName","description":"Name of the actor. If not provided, you will be prompted for it.","required":false}]},"login-new":{"id":"login-new","description":"Logs in to your Apify account using your API token.\nThe API token and other account information is stored in the ~/.apify directory, from where it is read by all other \"apify\" commands. To log out, call \"apify logout\".","pluginName":"apify-cli","pluginType":"core","hidden":true,"aliases":[],"flags":{"token":{"name":"token","type":"option","char":"t","description":"[Optional] Apify API token","required":false}},"args":[]},"login":{"id":"login","description":"Logs in to your Apify account using a provided API token.\nThe API token and other account information is stored in the ~/.apify directory, from where it is read by all other \"apify\" commands. To log out, call \"apify logout\".","pluginName":"apify-cli","pluginType":"core","aliases":[],"flags":{"token":{"name":"token","type":"option","char":"t","description":"[Optional] Apify API token","required":false}},"args":[]},"logout":{"id":"logout","description":"Logs out of your Apify account.\nThe command deletes the API token and all other account information stored in the ~/.apify directory. To log in again, call \"apify login\".","pluginName":"apify-cli","pluginType":"core","aliases":[],"flags":{},"args":[]},"pull":{"id":"pull","description":"Pulls an Actor from the Apify platform to the current directory. If it is defined as Git repository, it will be cloned. If it is defined as Web IDE, it will fetch the files.","pluginName":"apify-cli","pluginType":"core","aliases":[],"flags":{"version":{"name":"version","type":"option","char":"v","description":"Actor version number which will be pulled, e.g. 1.2. Default: the highest version","required":false}},"args":[{"name":"actorId","description":"Name or ID of the actor to run (e.g. \"apify/hello-world\" or \"E2jjCZBezvAZnX8Rb\"). If not provided, the command will update the Actor in the current directory based on its name in \".actor/actor.json\" file.","required":false}]},"push":{"id":"push","description":"Uploads the actor to the Apify platform and builds it there.\nThe Actor settings are read from the \".actor/actor.json\" file in the current directory, but they can be overridden using command-line options.\nNOTE: If the source files are smaller than 3 MB then they are uploaded as \n\"Multiple source files\", otherwise they are uploaded as \"Zip file\".\n\nWARNING: If the target Actor already exists in your Apify account, it will be overwritten!","pluginName":"apify-cli","pluginType":"core","aliases":[],"flags":{"version-number":{"name":"version-number","type":"option","description":"DEPRECATED: Use flag version instead. Actor version number to which the files should be pushed. By default, it is taken from the \".actor/actor.json\" file.","required":false},"version":{"name":"version","type":"option","char":"v","description":"Actor version number to which the files should be pushed. By default, it is taken from the \".actor/actor.json\" file.","required":false},"build-tag":{"name":"build-tag","type":"option","char":"b","description":"Build tag to be applied to the successful Actor build. By default, it is taken from the \".actor/actor.json\" file","required":false},"wait-for-finish":{"name":"wait-for-finish","type":"option","char":"w","description":"Seconds for waiting to build to finish, if no value passed, it waits forever.","required":false},"no-prompt":{"name":"no-prompt","type":"boolean","description":"Do not prompt for opening the actor details in a browser. This will also not open the browser automatically.","required":false,"allowNo":false}},"args":[{"name":"actorId","description":"Name or ID of the Actor to push (e.g. \"apify/hello-world\" or \"E2jjCZBezvAZnX8Rb\"). If not provided, the command will create or modify the actor with the name specified in \".actor/actor.json\" file.","required":false}]},"run":{"id":"run","description":"Runs the actor locally in the current directory.\nIt sets various APIFY_XYZ environment variables in order to provide a working execution environment for the actor. For example, this causes the actor input, as well as all other data in key-value stores, datasets or request queues to be stored in the \"storage\" directory, rather than on the Apify platform.\n\nNOTE: You can override the command's default behavior for Node.js actors by overriding the \"start\" script in the package.json file. You can set up your own main file or environment variables by changing it.","pluginName":"apify-cli","pluginType":"core","aliases":[],"flags":{"purge":{"name":"purge","type":"boolean","char":"p","description":"Shortcut that combines the --purge-queue, --purge-dataset and --purge-key-value-store options.","required":false,"allowNo":false},"purge-queue":{"name":"purge-queue","type":"boolean","description":"Deletes the local directory containing the default request queue before the run starts.","required":false,"allowNo":false},"purge-dataset":{"name":"purge-dataset","type":"boolean","description":"Deletes the local directory containing the default dataset before the run starts.","required":false,"allowNo":false},"purge-key-value-store":{"name":"purge-key-value-store","type":"boolean","description":"Deletes all records from the default key-value store in the local directory before the run starts, except for the \"INPUT\" key.","required":false,"allowNo":false}},"args":[]},"vis":{"id":"vis","description":"Validates input schema and prints errors found.\nThe input schema for the actor is used from these locations in order of preference.\nThe first one found is validated as it would be the one used on the Apify platform.\n1. Directly embedded object in \".actor/actor.json\" under 'input' key\n2. Path to JSON file referenced in \".actor/actor.json\" under 'input' key\n3. JSON file at .actor/INPUT_SCHEMA.json\n4. JSON file at INPUT_SCHEMA.json\n\nYou can also pass any custom path to your input schema to have it validated instead.\n","pluginName":"apify-cli","pluginType":"core","aliases":[],"flags":{},"args":[{"name":"path","description":"Optional path to your INPUT_SCHEMA.json file. If not provided ./INPUT_SCHEMA.json is used.","required":false}]},"actor:get-input":{"id":"actor:get-input","description":"Gets the actor input value from the default key-value store associated with the actor run.","pluginName":"apify-cli","pluginType":"core","aliases":[],"flags":{},"args":[]},"actor:get-value":{"id":"actor:get-value","description":"Gets a value from the default key-value store associated with the actor run.","pluginName":"apify-cli","pluginType":"core","aliases":[],"flags":{},"args":[{"name":"key","description":"Key of the record in key-value store","required":true}]},"actor":{"id":"actor","description":"Commands are designed to be used in actor runs. All commands are in PoC state, do not use in production environments.\n","pluginName":"apify-cli","pluginType":"core","aliases":[],"flags":{},"args":[]},"actor:push-data":{"id":"actor:push-data","description":"Stores an object or an array of objects to the default dataset of the actor run.\nIt is possible to pass data using item argument or stdin.\nPassing data using argument:\n$ apify actor:push-data {\"foo\": \"bar\"}\nPassing data using stdin with pipe:\n$ cat ./test.json | apify actor:push-data\n","pluginName":"apify-cli","pluginType":"core","aliases":[],"flags":{},"args":[{"name":"item","description":"JSON string with one object or array of objects containing data to be stored in the default dataset.","required":false}]},"actor:set-value":{"id":"actor:set-value","description":"Sets or removes record into the default KeyValueStore associated with the actor run.\nIt is possible to pass data using argument or stdin.\nPassing data using argument:\n$ apify actor:set-value KEY my-value\nPassing data using stdin with pipe:\n$ cat ./my-text-file.txt | apify actor:set-value KEY --contentType text/plain\n","pluginName":"apify-cli","pluginType":"core","aliases":[],"flags":{"contentType":{"name":"contentType","type":"option","char":"c","description":"Specifies a custom MIME content type of the record. By default \"application/json\" is used.","required":false}},"args":[{"name":"key","description":"Key of the record in key-value store.","required":true},{"name":"value","description":"Record data, which can be one of the following values:\n- If empty, the record in the key-value store is deleted.\n- If no `contentType` flag is specified, value is expected to be any JSON string value.\n- If options.contentType is set, value is taken as is.","required":false}]},"secrets:add":{"id":"secrets:add","description":"Adds a new secret value.\nThe secrets are stored to a file at ~/.apify","pluginName":"apify-cli","pluginType":"core","aliases":[],"flags":{},"args":[{"name":"name","description":"Name of the secret","required":true},{"name":"value","description":"Value of the secret","required":true}]},"secrets":{"id":"secrets","description":"Manages secret values for actor environment variables.\n\nExample:\n$ apify secrets:add mySecret TopSecretValue123\n\nNow the \"mySecret\" value can be used in an environment variable defined in \".actor/actor.json\" file by adding the \"@\" prefix:\n\n{\n  \"actorSpecification\": 1,\n  \"name\": \"my_actor\",\n  \"environmentVariables\": { \"SECRET_ENV_VAR\": \"@mySecret\" },\n  \"version\": \"0.1\n}\n\nWhen the actor is pushed to Apify cloud, the \"SECRET_ENV_VAR\" and its value is stored as a secret environment variable of the actor.","pluginName":"apify-cli","pluginType":"core","aliases":[],"flags":{},"args":[]},"secrets:rm":{"id":"secrets:rm","description":"Removes the secret.","pluginName":"apify-cli","pluginType":"core","aliases":[],"flags":{},"args":[{"name":"name","description":"Name of the secret","required":true}]}}}

package/package.json

@@ -1,114 +1,117 @@
 {
-  "name": "apify-cli",
-  "version": "0.18.2-beta.9",
-  "description": "Apify command-line interface helps you create, develop, build and run Apify actors, and manage the Apify cloud platform.",
-  "main": "index.js",
-  "scripts": {
-    "test": "cross-env APIFY_CLI_SKIP_UPDATE_CHECK=1 mocha --timeout 180000 --recursive",
-    "test-python": "npm run test -- --grep '\\[python\\]'",
-    "lint": "eslint src test",
-    "lint:fix": "eslint src test --fix",
-    "commands-md": "npm run manifest && oclif-dev readme",
-    "prepare-release": "npm run commands-md && npm run prod-shrinkwrap",
-    "prod-shrinkwrap": "rm -rf node_modules && npm install --production && npm shrinkwrap",
-    "manifest": "rm -f oclif.manifest.json && oclif-dev manifest",
-    "postinstall": "node \"./src/bin/run\" check-version && node ./src/lib/community"
-  },
-  "files": [
-    "src",
-    "oclif.manifest.json",
-    "index.js",
-    "package.json"
-  ],
-  "bin": {
-    "apify": "./src/bin/run"
-  },
-  "contributors": [
-    "Jakub Drobník <jakub.drobnik@apify.com>",
-    "Jan Curn <jan@apify.com>"
-  ],
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/apify/apify-cli.git"
-  },
-  "keywords": [
-    "apify",
-    "client",
-    "node",
-    "command",
-    "line",
-    "bash"
-  ],
-  "author": {
-    "name": "Apify",
-    "email": "support@apify.com",
-    "url": "https://www.apify.com"
-  },
-  "license": "Apache-2.0",
-  "bugs": {
-    "url": "https://github.com/apify/apify-cli/issues"
-  },
-  "homepage": "https://github.com/apify/apify-cli#readme",
-  "engines": {
-    "node": ">=16.0.0"
-  },
-  "dependencies": {
-    "@apify/actor-templates": "^0.1.4",
-    "@apify/consts": "^2.23.0",
-    "@apify/input_schema": "^3.5.8",
-    "@apify/utilities": "^2.9.3",
-    "@crawlee/memory-storage": "^3.5.8",
-    "@oclif/command": "^1.8.36",
-    "@oclif/config": "^1.18.17",
-    "@oclif/errors": "^1.3.6",
-    "@oclif/plugin-commands": "^2.1.0",
-    "@oclif/plugin-help": "^5.1.12",
-    "adm-zip": "^0.5.10",
-    "ajv": "^8.12.0",
-    "apify-client": "^2.8.2",
-    "archiver-promise": "^1.0.0",
-    "axios": "^1.6.1",
-    "chalk": "^4.1.2",
-    "computer-name": "^0.1.0",
-    "cors": "^2.8.5",
-    "detect-indent": "^6.1.0",
-    "escape-string-regexp": "^4.0.0",
-    "express": "^4.18.2",
-    "globby": "^11.1.0",
-    "inquirer": "^7.3.3",
-    "is-ci": "^3.0.1",
-    "is-online": "^10.0.0",
-    "istextorbinary": "^8.0.0",
-    "jju": "^1.4.0",
-    "load-json-file": "^6.2.0",
-    "mime": "^3.0.0",
-    "mixpanel": "^0.18.0",
-    "open": "^8.4.0",
-    "ow": "^0.28.2",
-    "rimraf": "^3.0.2",
-    "semver": "^7.5.4",
-    "tiged": "^2.12.5",
-    "underscore": "^1.13.6",
-    "write-json-file": "^4.3.0"
-  },
-  "devDependencies": {
-    "@apify/eslint-config": "^0.4.0",
-    "@oclif/dev-cli": "^1.26.0",
-    "@oclif/test": "^2.1.0",
-    "chai": "^4.3.4",
-    "chai-match": "^1.1.1",
-    "cross-env": "^7.0.3",
-    "eslint": "^8.53.0",
-    "mocha": "^10.0.0",
-    "sinon": "^17.0.0"
-  },
-  "oclif": {
-    "bin": "apify",
-    "commands": "./src/commands",
-    "hooks": {
-      "init": [
-        "./src/hooks/init"
-      ]
+    "name": "apify-cli",
+    "version": "0.19.0",
+    "description": "Apify command-line interface helps you create, develop, build and run Apify actors, and manage the Apify cloud platform.",
+    "main": "index.js",
+    "scripts": {
+        "test": "cross-env APIFY_CLI_SKIP_UPDATE_CHECK=1 mocha --timeout 180000 --recursive",
+        "test-python": "npm run test -- --grep '\\[python\\]'",
+        "lint": "eslint src test",
+        "lint:fix": "eslint src test --fix",
+        "commands-md": "npm run manifest && oclif-dev readme",
+        "prepare-release": "npm run commands-md && npm run prod-shrinkwrap",
+        "prod-shrinkwrap": "rm -rf node_modules && npm install --production && npm shrinkwrap",
+        "manifest": "rm -f oclif.manifest.json && oclif-dev manifest",
+        "postinstall": "node \"./src/bin/run\" check-version && node ./src/lib/community"
+    },
+    "files": [
+        "src",
+        "oclif.manifest.json",
+        "index.js",
+        "package.json"
+    ],
+    "bin": {
+        "apify": "./src/bin/run"
+    },
+    "contributors": [
+        "Jakub Drobník <jakub.drobnik@apify.com>",
+        "Jan Curn <jan@apify.com>"
+    ],
+    "repository": {
+        "type": "git",
+        "url": "git+https://github.com/apify/apify-cli.git"
+    },
+    "keywords": [
+        "apify",
+        "client",
+        "node",
+        "command",
+        "line",
+        "bash"
+    ],
+    "author": {
+        "name": "Apify",
+        "email": "support@apify.com",
+        "url": "https://www.apify.com"
+    },
+    "license": "Apache-2.0",
+    "bugs": {
+        "url": "https://github.com/apify/apify-cli/issues"
+    },
+    "homepage": "https://github.com/apify/apify-cli#readme",
+    "engines": {
+        "node": ">=16.0.0"
+    },
+    "dependencies": {
+        "@apify/actor-templates": "^0.1.4",
+        "@apify/consts": "^2.23.0",
+        "@apify/input_schema": "^3.5.8",
+        "@apify/utilities": "^2.9.3",
+        "@crawlee/memory-storage": "^3.5.8",
+        "@oclif/command": "^1.8.36",
+        "@oclif/config": "^1.18.17",
+        "@oclif/errors": "^1.3.6",
+        "@oclif/plugin-commands": "^2.1.0",
+        "@oclif/plugin-help": "^5.1.12",
+        "@root/walk": "^1.1.0",
+        "adm-zip": "^0.5.10",
+        "ajv": "^8.12.0",
+        "apify-client": "^2.8.2",
+        "archiver-promise": "^1.0.0",
+        "axios": "^1.6.1",
+        "chalk": "^4.1.2",
+        "computer-name": "^0.1.0",
+        "configparser": "^0.3.10",
+        "cors": "^2.8.5",
+        "detect-indent": "^6.1.0",
+        "escape-string-regexp": "^4.0.0",
+        "express": "^4.18.2",
+        "globby": "^11.1.0",
+        "handlebars": "^4.7.8",
+        "inquirer": "^7.3.3",
+        "is-ci": "^3.0.1",
+        "is-online": "^10.0.0",
+        "istextorbinary": "^8.0.0",
+        "jju": "^1.4.0",
+        "load-json-file": "^6.2.0",
+        "mime": "^3.0.0",
+        "mixpanel": "^0.18.0",
+        "open": "^8.4.0",
+        "ow": "^0.28.2",
+        "rimraf": "^3.0.2",
+        "semver": "^7.5.4",
+        "tiged": "^2.12.5",
+        "underscore": "^1.13.6",
+        "write-json-file": "^4.3.0"
+    },
+    "devDependencies": {
+        "@apify/eslint-config": "^0.4.0",
+        "@oclif/dev-cli": "^1.26.0",
+        "@oclif/test": "^2.1.0",
+        "chai": "^4.3.4",
+        "chai-match": "^1.1.1",
+        "cross-env": "^7.0.3",
+        "eslint": "^8.53.0",
+        "mocha": "^10.0.0",
+        "sinon": "^17.0.0"
+    },
+    "oclif": {
+        "bin": "apify",
+        "commands": "./src/commands",
+        "hooks": {
+            "init": [
+                "./src/hooks/init"
+            ]
+        }
     }
-  }
 }

package/src/commands/create.js

@@ -1,17 +1,13 @@
 const fs = require('fs');
 const path = require('path');
-const { finished } = require('stream');
-const { promisify } = require('util');
 
 const actorTemplates = require('@apify/actor-templates');
 const { flags: flagsHelper } = require('@oclif/command');
-const AdmZip = require('adm-zip');
 const semver = require('semver');
 
 const { ApifyCommand } = require('../lib/apify_command');
 const { EMPTY_LOCAL_CONFIG, LOCAL_CONFIG_PATH, PYTHON_VENV_PATH, SUPPORTED_NODEJS_VERSION } = require('../lib/consts');
 const {
-    httpsGet,
     ensureValidActorName,
     getTemplateDefinition,
     enhanceReadmeWithLocalSuffix,
@@ -31,6 +27,7 @@ const {
     detectNodeVersion,
     isNodeVersionSupported,
     detectNpmVersion,
+    downloadAndUnzip,
 } = require('../lib/utils');
 
 class CreateCommand extends ApifyCommand {
@@ -81,12 +78,7 @@ class CreateCommand extends ApifyCommand {
             throw err;
         }
 
-        const zipStream = await httpsGet(templateArchiveUrl);
-        const chunks = [];
-        zipStream.on('data', (chunk) => chunks.push(chunk));
-        await promisify(finished)(zipStream);
-        const zip = new AdmZip(Buffer.concat(chunks));
-        zip.extractAllTo(actFolderDir, true);
+        await downloadAndUnzip({ url: templateArchiveUrl, pathTo: actFolderDir });
 
         // There may be .actor/actor.json file in used template - let's try to load it and change the name prop value to actorName
         const localConfig = await getJsonFileContent(path.join(actFolderDir, LOCAL_CONFIG_PATH));

package/src/commands/init-wrap-scrapy.js

@@ -0,0 +1,34 @@
+const { ApifyCommand } = require('../lib/apify_command');
+const { info } = require('../lib/outputs');
+const { wrapScrapyProject } = require('../lib/scrapy-wrapper/index');
+
+class WrapScrapyCommand extends ApifyCommand {
+    async run() {
+        const { args } = this.parse(WrapScrapyCommand);
+
+        await wrapScrapyProject({ projectPath: args.path });
+
+        info('Scrapy project wrapped successfully.');
+    }
+}
+
+WrapScrapyCommand.hidden = true;
+
+WrapScrapyCommand.description = `Wraps your existing Scrapy project to work like an Apify Actor.
+
+It adds the following features:
+- Automatic retry of failed requests
+- Automatic proxy rotation
+- Automatic user agent rotation
+...
+`;
+
+WrapScrapyCommand.args = [
+    {
+        name: 'path',
+        required: false,
+        description: 'Optional path to your scrapy project. If not provided, the current directory is used.',
+    },
+];
+
+module.exports = WrapScrapyCommand;

package/src/commands/init.js

@@ -1,19 +1,35 @@
 const path = require('path');
 
+const { flags: flagsHelper } = require('@oclif/command');
 const inquirer = require('inquirer');
 
 const { ApifyCommand } = require('../lib/apify_command');
-const { EMPTY_LOCAL_CONFIG, DEFAULT_LOCAL_STORAGE_DIR, LOCAL_CONFIG_PATH } = require('../lib/consts');
+const { EMPTY_LOCAL_CONFIG, DEFAULT_LOCAL_STORAGE_DIR, LOCAL_CONFIG_PATH, LANGUAGE, PROJECT_TYPES } = require('../lib/consts');
 const { createPrefilledInputFileFromInputSchema } = require('../lib/input_schema');
 const outputs = require('../lib/outputs');
-const { setLocalConfig, setLocalEnv, getLocalConfig, getLocalConfigOrThrow } = require('../lib/utils');
+const { ProjectAnalyzer } = require('../lib/project_analyzer');
+const { wrapScrapyProject } = require('../lib/scrapy-wrapper');
+const { setLocalConfig, setLocalEnv, getLocalConfig, getLocalConfigOrThrow, detectLocalActorLanguage } = require('../lib/utils');
 
 class InitCommand extends ApifyCommand {
     async run() {
-        const { args } = this.parse(InitCommand);
+        const { args, flags } = this.parse(InitCommand);
         let { actorName } = args;
         const cwd = process.cwd();
 
+        if (ProjectAnalyzer.getProjectType(cwd) === PROJECT_TYPES.SCRAPY) {
+            outputs.info('The current directory looks like a Scrapy project. Using automatic project wrapping.');
+            this.telemetryData.actorWrapper = PROJECT_TYPES.SCRAPY;
+
+            return wrapScrapyProject({ projectPath: cwd });
+        }
+
+        if (!flags.yes && detectLocalActorLanguage(cwd).language === LANGUAGE.UNKNOWN) {
+            outputs.warning('The current directory does not look like a Node.js or Python project.');
+            const { c } = await inquirer.prompt([{ name: 'c', message: 'Do you want to continue?', type: 'confirm' }]);
+            if (!c) return;
+        }
+
         if (getLocalConfig()) {
             outputs.warning(`Skipping creation of "${LOCAL_CONFIG_PATH}", the file already exists in the current directory.`);
         } else {
@@ -45,4 +61,12 @@ InitCommand.args = [
     },
 ];
 
+InitCommand.flags = {
+    yes: flagsHelper.boolean({
+        char: 'y',
+        description: 'Automatic yes to prompts; assume "yes" as answer to all prompts. Note that in some cases, the command may still ask for confirmation.',
+        required: false,
+    }),
+};
+
 module.exports = InitCommand;

package/src/commands/run.js

@@ -7,9 +7,11 @@ const loadJson = require('load-json-file');
 const semver = require('semver');
 
 const { ApifyCommand } = require('../lib/apify_command');
-const { LEGACY_LOCAL_STORAGE_DIR, DEFAULT_LOCAL_STORAGE_DIR, SUPPORTED_NODEJS_VERSION, LANGUAGE } = require('../lib/consts');
+const { LEGACY_LOCAL_STORAGE_DIR, DEFAULT_LOCAL_STORAGE_DIR, SUPPORTED_NODEJS_VERSION, LANGUAGE, PROJECT_TYPES } = require('../lib/consts');
 const execWithLog = require('../lib/exec');
 const { error, info, warning } = require('../lib/outputs');
+const { ProjectAnalyzer } = require('../lib/project_analyzer');
+const { ScrapyProjectAnalyzer } = require('../lib/scrapy-wrapper/ScrapyProjectAnalyzer');
 const { replaceSecretsValue } = require('../lib/secrets');
 const {
     getLocalUserInfo, purgeDefaultQueue, purgeDefaultKeyValueStore,
@@ -29,11 +31,12 @@ class RunCommand extends ApifyCommand {
 
         const packageJsonExists = fs.existsSync(packageJsonPath);
         const mainPyExists = fs.existsSync(mainPyPath);
+        const isScrapyProject = ProjectAnalyzer.getProjectType(cwd) === PROJECT_TYPES.SCRAPY;
 
-        if (!packageJsonExists && !mainPyExists) {
+        if (!packageJsonExists && !mainPyExists && !isScrapyProject) {
             throw new Error(
                 'Actor is of an unknown format.'
-                + ` Make sure either the 'package.json' file or 'src/__main__.py' file exists.`,
+                + ` Make sure either the 'package.json' file or 'src/__main__.py' file exists or you are in a migrated Scrapy project.`,
             );
         }
 
@@ -122,7 +125,16 @@ class RunCommand extends ApifyCommand {
             if (pythonVersion) {
                 if (isPythonVersionSupported(pythonVersion)) {
                     const pythonCommand = getPythonCommand(cwd);
-                    await execWithLog(pythonCommand, ['-m', 'src'], { env });
+                    if (isScrapyProject) {
+                        const project = new ScrapyProjectAnalyzer(cwd);
+                        project.loadScrapyCfg();
+                        if (!project.configuration.hasKey('apify', 'mainpy_location')) {
+                            throw new Error(`This Scrapy project's configuration does not contain Apify settings. Did you forget to run "apify init"?`);
+                        }
+                        await execWithLog(pythonCommand, ['-m', project.configuration.get('apify', 'mainpy_location')], { env });
+                    } else {
+                        await execWithLog(pythonCommand, ['-m', 'src'], { env });
+                    }
                 } else {
                     error(`Python actors require Python 3.8 or higher, but you have Python ${pythonVersion}!`);
                     error('Please install Python 3.8 or higher to be able to run Python actors locally.');

package/src/lib/consts.js

@@ -23,6 +23,11 @@ exports.LANGUAGE = {
     UNKNOWN: 'n/a',
 };
 
+exports.PROJECT_TYPES = {
+    SCRAPY: 'scrapy',
+    UNKNOWN: 'unknown',
+};
+
 exports.COMMANDS_WITHIN_ACTOR = ['init', 'run', 'push', 'pull', 'call'];
 
 exports.CHECK_VERSION_EVERY_MILLIS = 24 * 60 * 60 * 1000; // Once a day

package/src/lib/create-utils.js

@@ -1,5 +1,4 @@
 const fs = require('fs');
-const https = require('https');
 const { pipeline } = require('stream');
 const { promisify } = require('util');
 
@@ -9,29 +8,10 @@ const inquirer = require('inquirer');
 const {
     warning,
 } = require('./outputs');
-const { validateActorName } = require('./utils');
+const { validateActorName, httpsGet } = require('./utils');
 
 const PROGRAMMING_LANGUAGES = ['JavaScript', 'TypeScript', 'Python'];
 
-/**
- * @param {string} url
- * @returns {Promise<unknown>}
- */
-exports.httpsGet = async (url) => {
-    return new Promise((resolve, reject) => {
-        https.get(url, (response) => {
-            // Handle redirects
-            if (response.statusCode === 301 || response.statusCode === 302) {
-                resolve(exports.httpsGet(response.headers.location));
-                // Destroy the response to close the HTTP connection, otherwise this hangs for a long time with Node 19+ (due to HTTP keep-alive).
-                response.destroy();
-            } else {
-                resolve(response);
-            }
-        }).on('error', reject);
-    });
-};
-
 /**
  * @param {string} maybeActorName
  * @returns {Promise<string>}
@@ -78,7 +58,7 @@ exports.enhanceReadmeWithLocalSuffix = async (readmePath, manifestPromise) => {
     if (manifest instanceof Error) throw manifest;
 
     try {
-        const suffixStream = await this.httpsGet(manifest.localReadmeSuffixUrl);
+        const suffixStream = await httpsGet(manifest.localReadmeSuffixUrl);
         const readmeStream = fs.createWriteStream(readmePath, { flags: 'a' });
         readmeStream.write('\n\n');
         await promisify(pipeline)(suffixStream, readmeStream);

package/src/lib/project_analyzer.js

@@ -0,0 +1,24 @@
+const { PROJECT_TYPES } = require('./consts');
+const { ScrapyProjectAnalyzer } = require('./scrapy-wrapper/ScrapyProjectAnalyzer');
+
+const analyzers = [
+    {
+        type: PROJECT_TYPES.SCRAPY,
+        analyzer: ScrapyProjectAnalyzer,
+    },
+];
+
+class ProjectAnalyzer {
+    static getProjectType(pathname) {
+        const analyzer = analyzers.find((a) => {
+            if (!a.analyzer.isApplicable) {
+                throw new Error(`Analyzer ${a.analyzer} does not have isApplicable method.`);
+            }
+
+            return a.analyzer.isApplicable(pathname);
+        });
+        return analyzer?.type || PROJECT_TYPES.UNKNOWN;
+    }
+}
+
+module.exports = { ProjectAnalyzer };

package/src/lib/scrapy-wrapper/ScrapyProjectAnalyzer.js

@@ -0,0 +1,90 @@
+const { readdirSync } = require('fs');
+const fs = require('fs');
+const path = require('path');
+
+const ConfigParser = require('configparser');
+const inquirer = require('inquirer');
+
+const { SpiderFileAnalyzer } = require('./SpiderFileAnalyzer');
+
+class ScrapyProjectAnalyzer {
+    constructor(pathname) {
+        this.pathname = pathname;
+        this.configuration = null;
+        this.settings = null;
+        this.loadScrapyCfg();
+    }
+
+    static isApplicable(pathname) {
+        return fs.existsSync(path.join(pathname, 'scrapy.cfg'));
+    }
+
+    async init() {
+        await this.loadSettings();
+    }
+
+    loadScrapyCfg() {
+        const config = new ConfigParser();
+        const scrapyCfgPath = path.resolve(path.join(this.pathname, 'scrapy.cfg'));
+
+        if (!fs.existsSync(scrapyCfgPath)) {
+            throw new Error(`scrapy.cfg not found in "${scrapyCfgPath}". 
+Are you sure there is a Scrapy project there?`);
+        }
+
+        config.read(scrapyCfgPath);
+        this.configuration = config;
+    }
+
+    async loadSettings() {
+        const assumedBotName = this.configuration.get('settings', 'default').split('.')[0];
+
+        const settings = await inquirer.prompt([
+            {
+                type: 'input',
+                name: 'BOT_NAME',
+                message: 'Enter the Scrapy BOT_NAME (see settings.py):',
+                default: assumedBotName,
+            },
+            {
+                type: 'input',
+                name: 'SPIDER_MODULES',
+                message: 'What folder are the Scrapy spider modules stored in? (see SPIDER_MODULES in settings.py):',
+                default: [`${assumedBotName}.spiders`],
+            },
+        ]);
+
+        if (typeof settings.SPIDER_MODULES === 'string') settings.SPIDER_MODULES = [settings.SPIDER_MODULES];
+
+        this.settings = settings;
+    }
+
+    getName() {
+        return this.settings?.BOT_NAME;
+    }
+
+    getAvailableSpiders() {
+        const spiderPaths = this.settings?.SPIDER_MODULES;
+
+        if (!spiderPaths) {
+            throw new Error('SPIDER_MODULES path not found in settings.');
+        }
+
+        const spiders = [];
+
+        for (const spiderPath of spiderPaths) {
+            const spidersDir = path.join(this.pathname, spiderPath.replaceAll('.', '/'));
+
+            const files = readdirSync(spidersDir, { withFileTypes: true });
+            for (const file of files) {
+                if (file.isFile() && file.name.endsWith('.py') && file.name !== '__init__.py') {
+                    spiders.push(...(new SpiderFileAnalyzer(path.join(spidersDir, file.name)).getSpiders()));
+                }
+            }
+        }
+
+        return spiders;
+    }
+}
+
+module.exports = { ScrapyProjectAnalyzer };

package/src/lib/scrapy-wrapper/Spider.js

@@ -0,0 +1,10 @@
+class Spider {
+    constructor(data) {
+        this.name = data.name;
+        this.class_name = data.class_name;
+        this.start_urls = data.start_urls;
+        this.pathname = data.pathname;
+    }
+}
+
+module.exports = { Spider };

package/src/lib/scrapy-wrapper/SpiderFileAnalyzer.js

@@ -0,0 +1,26 @@
+const fs = require('fs');
+
+const { Spider } = require('./Spider');
+
+class SpiderFileAnalyzer {
+    constructor(pathname) {
+        this.pathname = pathname;
+    }
+
+    getSpiders() {
+        const file = fs.readFileSync(this.pathname, 'utf8');
+
+        const regex = /class\s+(\w+)/g;
+        const spiders = [];
+
+        let match = regex.exec(file);
+        while (match) {
+            spiders.push(new Spider({ class_name: match[1], pathname: this.pathname }));
+            match = regex.exec(file);
+        }
+
+        return spiders;
+    }
+}
+
+module.exports = { SpiderFileAnalyzer };

package/src/lib/scrapy-wrapper/index.js

@@ -0,0 +1,139 @@
+const fs = require('fs');
+const path = require('path');
+
+const { fetchManifest, wrapperManifestUrl } = require('@apify/actor-templates');
+const { walk } = require('@root/walk');
+const ConfigParser = require('configparser');
+const handlebars = require('handlebars');
+const inquirer = require('inquirer');
+
+const { ScrapyProjectAnalyzer } = require('./ScrapyProjectAnalyzer');
+const outputs = require('../outputs');
+const { downloadAndUnzip } = require('../utils');
+
+/**
+ * Files that should be concatenated instead of copied (and overwritten).
+ */
+const concatenableFiles = ['.dockerignore', '.gitignore'];
+
+async function merge(fromPath, toPath, options = { bindings: {} }) {
+    await walk(fromPath, async (err, pathname, dirent) => {
+        if (pathname === fromPath) return;
+        const relPath = path.relative(fromPath, pathname);
+        const toRelPath = relPath.split(path.sep).map((part) => {
+            if (part.startsWith('{') && part.endsWith('}')) {
+                part = part.replace('{', '').replace('}', '');
+                const binding = options.bindings[part];
+                if (!binding) {
+                    throw new Error(`Binding for ${part} not found.`);
+                }
+                return binding;
+            }
+            return part;
+        }).join(path.sep);
+
+        const targetPath = path.join(toPath, toRelPath);
+
+        if (dirent.isDirectory()) {
+            if (!fs.existsSync(targetPath)) {
+                fs.mkdirSync(targetPath);
+            }
+            return merge(pathname, targetPath);
+        }
+
+        if (relPath.includes('.template')) {
+            fs.writeFileSync(
+                path.join(
+                    toPath,
+                    toRelPath.replace('.template', ''),
+                ),
+                handlebars.compile(fs.readFileSync(pathname, 'utf8'))(options.bindings));
+        } else if (fs.existsSync(targetPath) && concatenableFiles.includes(path.basename(toRelPath))) {
+            fs.appendFileSync(targetPath, fs.readFileSync(pathname));
+        } else {
+            fs.copyFileSync(pathname, targetPath);
+        }
+    });
+}
+
+async function wrapScrapyProject({ projectPath }) {
+    if (!projectPath) projectPath = '.';
+
+    const analyzer = new ScrapyProjectAnalyzer(projectPath);
+
+    if (analyzer.configuration.hasSection('apify')) {
+        throw new Error(`The Scrapy project configuration already contains Apify settings. Are you sure you didn't already wrap this project?`);
+    }
+
+    await analyzer.init();
+
+    const { spiderIndex } = await inquirer.prompt([
+        {
+            type: 'list',
+            name: 'spiderIndex',
+            message: 'Pick the Scrapy spider you want to wrap:',
+            choices: analyzer.getAvailableSpiders().map((spider, i) => ({
+                name: `${spider.class_name} (${spider.pathname})`,
+                value: i,
+            })),
+        },
+    ]);
+
+    function translatePathToRelativeModuleName(pathname) {
+        const relPath = path.relative(projectPath, pathname);
+
+        return `.${relPath.split(path.sep).slice(1).join('.').replace('.py', '')}`;
+    }
+
+    const templateBindings = {
+        botName: analyzer.settings.BOT_NAME,
+        scrapy_settings_module: analyzer.configuration.get('settings', 'default'),
+        apify_module_path: `${analyzer.settings.BOT_NAME}.apify`,
+        spider_class_name: analyzer.getAvailableSpiders()[spiderIndex].class_name,
+        spider_module_name: `${translatePathToRelativeModuleName(analyzer.getAvailableSpiders()[spiderIndex].pathname)}`,
+        projectFolder: analyzer.settings.BOT_NAME,
+    };
+
+    const manifest = await fetchManifest(wrapperManifestUrl);
+
+    outputs.info('Downloading the latest Scrapy wrapper template...');
+
+    const { archiveUrl } = manifest.templates.find(({ id }) => id === 'python-scrapy');
+    const templatePath = path.join(__dirname, 'templates', 'python-scrapy');
+
+    if (fs.existsSync(templatePath)) fs.rmSync(templatePath, { recursive: true });
+
+    await downloadAndUnzip({
+        url: archiveUrl,
+        pathTo: templatePath,
+    });
+
+    outputs.info('Wrapping the Scrapy project...');
+
+    merge(
+        path.join(__dirname, 'templates', 'python-scrapy'),
+        projectPath,
+        {
+            bindings: templateBindings,
+        },
+    );
+
+    const apifyConf = new ConfigParser();
+    apifyConf.addSection('apify');
+    apifyConf.set('apify', 'mainpy_location', analyzer.settings.BOT_NAME);
+
+    const s = fs.createWriteStream(path.join(projectPath, 'scrapy.cfg'), { flags: 'a' });
+
+    await new Promise((r) => {
+        s.on('open', (fd) => {
+            s.write('\n', () => {
+                apifyConf.write(fd);
+                r();
+            });
+        });
+    });
+
+    outputs.success('The Scrapy project has been wrapped successfully.');
+}
+
+module.exports = { wrapScrapyProject };

package/src/lib/utils.js

@@ -5,6 +5,8 @@ const {
 const fs = require('fs');
 const https = require('https');
 const path = require('path');
+const { finished } = require('stream');
+const { promisify } = require('util');
 
 const {
     ACT_JOB_TERMINAL_STATUSES,
@@ -16,6 +18,7 @@ const {
     LOCAL_STORAGE_SUBDIRS,
     SOURCE_FILE_FORMATS,
 } = require('@apify/consts');
+const AdmZip = require('adm-zip');
 const { ApifyClient } = require('apify-client');
 const archiver = require('archiver-promise');
 const escapeStringRegexp = require('escape-string-regexp');
@@ -40,6 +43,7 @@ const {
     SUPPORTED_NODEJS_VERSION,
     MINIMUM_SUPPORTED_PYTHON_VERSION,
     LANGUAGE,
+    PROJECT_TYPES,
 } = require('./consts');
 const {
     ensureFolderExistsSync,
@@ -49,6 +53,26 @@ const {
 const {
     info,
 } = require('./outputs');
+const { ProjectAnalyzer } = require('./project_analyzer');
+
+/**
+ * @param {string} url
+ * @returns {Promise<unknown>}
+ */
+const httpsGet = async (url) => {
+    return new Promise((resolve, reject) => {
+        https.get(url, (response) => {
+            // Handle redirects
+            if (response.statusCode === 301 || response.statusCode === 302) {
+                resolve(httpsGet(response.headers.location));
+                // Destroy the response to close the HTTP connection, otherwise this hangs for a long time with Node 19+ (due to HTTP keep-alive).
+                response.destroy();
+            } else {
+                resolve(response);
+            }
+        }).on('error', reject);
+    });
+};
 
 // Properties from apify.json file that will me migrated to actor specs in .actor/actor.json
 const MIGRATED_APIFY_JSON_PROPERTIES = ['name', 'version', 'buildTag'];
@@ -581,8 +605,8 @@ const detectNpmVersion = () => {
 
 const detectLocalActorLanguage = () => {
     const cwd = process.cwd();
-    const isActorInNode = fs.existsSync(path.join(process.cwd(), 'package.json'));
-    const isActorInPython = fs.existsSync(path.join(process.cwd(), 'src/__main__.py'));
+    const isActorInNode = fs.existsSync(path.join(cwd, 'package.json'));
+    const isActorInPython = fs.existsSync(path.join(cwd, 'src/__main__.py')) || ProjectAnalyzer.getProjectType(cwd) === PROJECT_TYPES.SCRAPY;
     const result = {};
     if (isActorInNode) {
         result.language = LANGUAGE.NODEJS;
@@ -596,7 +620,17 @@ const detectLocalActorLanguage = () => {
     return result;
 };
 
+const downloadAndUnzip = async ({ url, pathTo }) => {
+    const zipStream = await httpsGet(url);
+    const chunks = [];
+    zipStream.on('data', (chunk) => chunks.push(chunk));
+    await promisify(finished)(zipStream);
+    const zip = new AdmZip(Buffer.concat(chunks));
+    zip.extractAllTo(pathTo, true);
+};
+
 module.exports = {
+    httpsGet,
     getLoggedClientOrThrow,
     getLocalConfig,
     setLocalConfig,
@@ -630,4 +664,5 @@ module.exports = {
     isNodeVersionSupported,
     detectNpmVersion,
     detectLocalActorLanguage,
+    downloadAndUnzip,
 };