apify-cli 0.19.2-beta.0 → 0.19.2-beta.10

package/package.json

@@ -1,11 +1,11 @@
 {
   "name": "apify-cli",
-  "version": "0.19.2-beta.0",
+  "version": "0.19.2-beta.10",
   "description": "Apify command-line interface helps you create, develop, build and run Apify actors, and manage the Apify cloud platform.",
   "main": "index.js",
   "scripts": {
-    "test": "cross-env APIFY_CLI_SKIP_UPDATE_CHECK=1 mocha --timeout 180000 --recursive",
-    "test-python": "npm run test -- --grep '\\[python\\]'",
+    "test": "cross-env APIFY_CLI_SKIP_UPDATE_CHECK=1 vitest run",
+    "test-python": "cross-env APIFY_CLI_SKIP_UPDATE_CHECK=1 vitest run -t '.*\\[python\\]'",
     "lint": "eslint src test",
     "lint:fix": "eslint src test --fix",
     "commands-md": "npm run manifest && oclif-dev readme",
@@ -66,9 +66,9 @@
     "@root/walk": "^1.1.0",
     "adm-zip": "^0.5.10",
     "ajv": "^8.12.0",
-    "apify-client": "^2.8.2",
+    "apify-client": "^2.9.0",
     "archiver-promise": "^1.0.0",
-    "axios": "^1.6.1",
+    "axios": "^1.6.7",
     "chalk": "^4.1.2",
     "computer-name": "^0.1.0",
     "configparser": "^0.3.10",
@@ -98,12 +98,9 @@
     "@apify/eslint-config": "^0.4.0",
     "@oclif/dev-cli": "^1.26.0",
     "@oclif/test": "^2.1.0",
-    "chai": "^4.3.4",
-    "chai-match": "^1.1.1",
     "cross-env": "^7.0.3",
     "eslint": "^8.53.0",
-    "mocha": "^10.0.0",
-    "sinon": "^17.0.0"
+    "vitest": "^1.0.4"
   },
   "oclif": {
     "bin": "apify",

package/src/commands/create.js

@@ -70,7 +70,7 @@ class CreateCommand extends ApifyCommand {
         try {
             fs.mkdirSync(actFolderDir);
         } catch (err) {
-            if (err.code && err.code === 'EEXIST') {
+            if (err?.code === 'EEXIST') {
                 outputs.error(`Cannot create new actor, directory '${actorName}' already exists. `
                     + 'You can use "apify init" to create a local actor environment inside an existing directory.');
                 return;
@@ -162,6 +162,7 @@ class CreateCommand extends ApifyCommand {
 
         if (dependenciesInstalled) {
             outputs.success(`Actor '${actorName}' was created. To run it, run "cd ${actorName}" and "apify run".`);
+            outputs.success('To run your code in the cloud, run "apify push" and deploy your code to Apify Console.');
             if (messages?.postCreate) {
                 outputs.info(messages?.postCreate);
             }

package/src/commands/init.js

@@ -9,7 +9,7 @@ const { createPrefilledInputFileFromInputSchema } = require('../lib/input_schema
 const outputs = require('../lib/outputs');
 const { ProjectAnalyzer } = require('../lib/project_analyzer');
 const { wrapScrapyProject } = require('../lib/scrapy-wrapper');
-const { setLocalConfig, setLocalEnv, getLocalConfig, getLocalConfigOrThrow, detectLocalActorLanguage } = require('../lib/utils');
+const { setLocalConfig, setLocalEnv, getLocalConfig, getLocalConfigOrThrow, detectLocalActorLanguage, validateActorName } = require('../lib/utils');
 
 class InitCommand extends ApifyCommand {
     async run() {
@@ -34,8 +34,19 @@ class InitCommand extends ApifyCommand {
             outputs.warning(`Skipping creation of "${LOCAL_CONFIG_PATH}", the file already exists in the current directory.`);
         } else {
             if (!actorName) {
-                const answer = await inquirer.prompt([{ name: 'actName', message: 'Actor name:', default: path.basename(cwd) }]);
-                ({ actName: actorName } = answer);
+                let response = null;
+
+                while (!response) {
+                    try {
+                        const answer = await inquirer.prompt([{ name: 'actName', message: 'Actor name:', default: path.basename(cwd) }]);
+                        validateActorName(answer.actName);
+                        response = answer;
+                    } catch (err) {
+                        outputs.error(err.message);
+                    }
+                }
+
+                ({ actName: actorName } = response);
             }
             // Migrate apify.json to .actor/actor.json
             const localConfig = { ...EMPTY_LOCAL_CONFIG, ...await getLocalConfigOrThrow() };

package/src/commands/run.js

@@ -16,7 +16,7 @@ const { replaceSecretsValue } = require('../lib/secrets');
 const {
     getLocalUserInfo, purgeDefaultQueue, purgeDefaultKeyValueStore,
     purgeDefaultDataset, getLocalConfigOrThrow, getNpmCmd, checkIfStorageIsEmpty,
-    detectLocalActorLanguage, isPythonVersionSupported, getPythonCommand, isNodeVersionSupported,
+    detectLocalActorLanguage, isPythonVersionSupported, getPythonCommand, isNodeVersionSupported, getLocalStorageDir,
 } = require('../lib/utils');
 
 class RunCommand extends ApifyCommand {
@@ -29,9 +29,12 @@ class RunCommand extends ApifyCommand {
         const packageJsonPath = path.join(cwd, 'package.json');
         const mainPyPath = path.join(cwd, 'src/__main__.py');
 
+        const projectType = ProjectAnalyzer.getProjectType(cwd);
+        const actualStoragePath = getLocalStorageDir();
+
         const packageJsonExists = fs.existsSync(packageJsonPath);
         const mainPyExists = fs.existsSync(mainPyPath);
-        const isScrapyProject = ProjectAnalyzer.getProjectType(cwd) === PROJECT_TYPES.SCRAPY;
+        const isScrapyProject = projectType === PROJECT_TYPES.SCRAPY;
 
         if (!packageJsonExists && !mainPyExists && !isScrapyProject) {
             throw new Error(
@@ -40,25 +43,43 @@ class RunCommand extends ApifyCommand {
             );
         }
 
-        if (fs.existsSync(LEGACY_LOCAL_STORAGE_DIR) && !fs.existsSync(DEFAULT_LOCAL_STORAGE_DIR)) {
-            fs.renameSync(LEGACY_LOCAL_STORAGE_DIR, DEFAULT_LOCAL_STORAGE_DIR);
-            warning("The legacy 'apify_storage' directory was renamed to 'storage' to align it with Apify SDK v3."
+        if (fs.existsSync(LEGACY_LOCAL_STORAGE_DIR) && !fs.existsSync(actualStoragePath)) {
+            fs.renameSync(LEGACY_LOCAL_STORAGE_DIR, actualStoragePath);
+            warning(`The legacy 'apify_storage' directory was renamed to '${actualStoragePath}' to align it with Apify SDK v3.`
                 + ' Contents were left intact.');
         }
 
+        let CRAWLEE_PURGE_ON_START = '0';
+
         // Purge stores
         if (flags.purge) {
-            await Promise.all([purgeDefaultQueue(), purgeDefaultKeyValueStore(), purgeDefaultDataset()]);
-            info('All default local stores were purged.');
+            switch (projectType) {
+                case PROJECT_TYPES.CRAWLEE: {
+                    CRAWLEE_PURGE_ON_START = '1';
+                    break;
+                }
+                case PROJECT_TYPES.PRE_CRAWLEE_APIFY_SDK: {
+                    await Promise.all([purgeDefaultQueue(), purgeDefaultKeyValueStore(), purgeDefaultDataset()]);
+                    info('All default local stores were purged.');
+                    break;
+                }
+                default: {
+                    // TODO: Python SDK too
+                }
+            }
         }
+
+        // TODO: deprecate these flags
         if (flags.purgeQueue) {
             await purgeDefaultQueue();
             info('Default local request queue was purged.');
         }
+
         if (flags.purgeDataset) {
             await purgeDefaultDataset();
             info('Default local dataset was purged.');
         }
+
         if (flags.purgeKeyValueStore) {
             await purgeDefaultKeyValueStore();
             info('Default local key-value store was purged.');
@@ -74,7 +95,9 @@ class RunCommand extends ApifyCommand {
 
         // Attach env vars from local config files
         const localEnvVars = {
-            [APIFY_ENV_VARS.LOCAL_STORAGE_DIR]: DEFAULT_LOCAL_STORAGE_DIR,
+            [APIFY_ENV_VARS.LOCAL_STORAGE_DIR]: actualStoragePath,
+            CRAWLEE_STORAGE_DIR: actualStoragePath,
+            CRAWLEE_PURGE_ON_START,
         };
         if (proxy && proxy.password) localEnvVars[APIFY_ENV_VARS.PROXY_PASSWORD] = proxy.password;
         if (userId) localEnvVars[APIFY_ENV_VARS.USER_ID] = userId;

package/src/lib/consts.js

@@ -25,6 +25,8 @@ exports.LANGUAGE = {
 
 exports.PROJECT_TYPES = {
     SCRAPY: 'scrapy',
+    CRAWLEE: 'crawlee',
+    PRE_CRAWLEE_APIFY_SDK: 'apify',
     UNKNOWN: 'unknown',
 };
 

package/src/lib/project_analyzer.js

@@ -1,4 +1,6 @@
 const { PROJECT_TYPES } = require('./consts');
+const { CrawleeAnalyzer } = require('./projects/CrawleeAnalyzer');
+const { OldApifySDKAnalyzer } = require('./projects/OldApifySDKAnalyzer');
 const { ScrapyProjectAnalyzer } = require('./scrapy-wrapper/ScrapyProjectAnalyzer');
 
 const analyzers = [
@@ -6,6 +8,14 @@ const analyzers = [
         type: PROJECT_TYPES.SCRAPY,
         analyzer: ScrapyProjectAnalyzer,
     },
+    {
+        type: PROJECT_TYPES.CRAWLEE,
+        analyzer: CrawleeAnalyzer,
+    },
+    {
+        type: PROJECT_TYPES.PRE_CRAWLEE_APIFY_SDK,
+        analyzer: OldApifySDKAnalyzer,
+    },
 ];
 
 class ProjectAnalyzer {
@@ -17,6 +27,7 @@ class ProjectAnalyzer {
 
             return a.analyzer.isApplicable(pathname);
         });
+
         return analyzer?.type || PROJECT_TYPES.UNKNOWN;
     }
 }

package/src/lib/projects/CrawleeAnalyzer.js

@@ -0,0 +1,37 @@
+const { existsSync, readFileSync } = require('fs');
+const { join } = require('path');
+
+const CRAWLEE_PACKAGES = [
+    'crawlee',
+    '@crawlee/core',
+    '@crawlee/puppeteer',
+    '@crawlee/playwright',
+    '@crawlee/cheerio',
+    '@crawlee/jsdom',
+    '@crawlee/linkedom',
+    '@crawlee/http',
+    '@crawlee/browser',
+    '@crawlee/basic',
+];
+
+class CrawleeAnalyzer {
+    static isApplicable(pathname) {
+        const hasPackageJson = existsSync(join(pathname, 'package.json'));
+
+        if (!hasPackageJson) {
+            return false;
+        }
+
+        const packageJson = readFileSync(join(pathname, 'package.json'), 'utf8');
+
+        try {
+            const packageJsonParsed = JSON.parse(packageJson);
+
+            return CRAWLEE_PACKAGES.some((pkg) => packageJsonParsed?.dependencies?.[pkg] !== undefined);
+        } catch (err) {
+            return false;
+        }
+    }
+}
+
+exports.CrawleeAnalyzer = CrawleeAnalyzer;

package/src/lib/projects/OldApifySDKAnalyzer.js

@@ -0,0 +1,61 @@
+const { existsSync, readFileSync } = require('fs');
+const { join } = require('path');
+
+const { lt } = require('semver');
+
+const VERSION_WHEN_APIFY_MOVED_TO_CRAWLEE = '3.0.0';
+const CRAWLEE_PACKAGES = [
+    'crawlee',
+    '@crawlee/core',
+    '@crawlee/puppeteer',
+    '@crawlee/playwright',
+    '@crawlee/cheerio',
+    '@crawlee/jsdom',
+    '@crawlee/linkedom',
+    '@crawlee/http',
+    '@crawlee/browser',
+    '@crawlee/basic',
+];
+
+class OldApifySDKAnalyzer {
+    static isApplicable(pathname) {
+        const hasPackageJson = existsSync(join(pathname, 'package.json'));
+
+        if (!hasPackageJson) {
+            return false;
+        }
+
+        const packageJson = readFileSync(join(pathname, 'package.json'), 'utf8');
+
+        try {
+            const packageJsonParsed = JSON.parse(packageJson);
+
+            // If they have crawlee as a dependency, likely to use crawlee
+            if (CRAWLEE_PACKAGES.some((pkg) => packageJsonParsed?.dependencies?.[pkg] !== undefined)) {
+                return false;
+            }
+
+            const apifyVersion = packageJsonParsed?.dependencies?.apify;
+            if (!apifyVersion) {
+                return false;
+            }
+
+            // We cannot infer
+            if (apifyVersion === '*') {
+                return false;
+            }
+
+            let actualVersion = apifyVersion;
+
+            if (apifyVersion.startsWith('~') || apifyVersion.startsWith('^')) {
+                actualVersion = apifyVersion.slice(1);
+            }
+
+            return lt(actualVersion, VERSION_WHEN_APIFY_MOVED_TO_CRAWLEE);
+        } catch (err) {
+            return false;
+        }
+    }
+}
+
+exports.OldApifySDKAnalyzer = OldApifySDKAnalyzer;

package/src/lib/scrapy-wrapper/index.js

@@ -9,7 +9,7 @@ const inquirer = require('inquirer');
 
 const { ScrapyProjectAnalyzer } = require('./ScrapyProjectAnalyzer');
 const outputs = require('../outputs');
-const { downloadAndUnzip } = require('../utils');
+const { downloadAndUnzip, sanitizeActorName } = require('../utils');
 
 /**
  * Files that should be concatenated instead of copied (and overwritten).
@@ -86,7 +86,7 @@ async function wrapScrapyProject({ projectPath }) {
     }
 
     const templateBindings = {
-        botName: analyzer.settings.BOT_NAME,
+        botName: sanitizeActorName(analyzer.settings.BOT_NAME),
         scrapy_settings_module: analyzer.configuration.get('settings', 'default'),
         apify_module_path: `${analyzer.settings.BOT_NAME}.apify`,
         spider_class_name: analyzer.getAvailableSpiders()[spiderIndex].class_name,

package/src/lib/utils.js

@@ -21,6 +21,7 @@ const {
 const AdmZip = require('adm-zip');
 const { ApifyClient } = require('apify-client');
 const archiver = require('archiver-promise');
+const axios = require('axios');
 const escapeStringRegexp = require('escape-string-regexp');
 const globby = require('globby');
 const inquirer = require('inquirer');
@@ -80,7 +81,7 @@ const MIGRATED_APIFY_JSON_PROPERTIES = ['name', 'version', 'buildTag'];
 const getLocalStorageDir = () => {
     const envVar = APIFY_ENV_VARS.LOCAL_STORAGE_DIR;
 
-    return process.env[envVar] || DEFAULT_LOCAL_STORAGE_DIR;
+    return process.env[envVar] || process.env.CRAWLEE_STORAGE_DIR || DEFAULT_LOCAL_STORAGE_DIR;
 };
 const getLocalKeyValueStorePath = (storeId) => {
     const envVar = ACTOR_ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID;
@@ -139,7 +140,14 @@ const getApifyClientOptions = (token, apiBaseUrl) => {
         token,
         baseUrl: apiBaseUrl || process.env.APIFY_CLIENT_BASE_URL,
         requestInterceptors: [(config) => {
-            config.headers = { ...APIFY_CLIENT_DEFAULT_HEADERS, ...config.headers };
+            if (!config.headers) {
+                config.headers = new axios.AxiosHeaders();
+            }
+
+            for (const [key, value] of Object.entries(APIFY_CLIENT_DEFAULT_HEADERS)) {
+                config.headers[key] = value;
+            }
+
             return config;
         }],
     };
@@ -536,6 +544,19 @@ const validateActorName = (actorName) => {
     }
 };
 
+const sanitizeActorName = (actorName) => {
+    let sanitizedName = actorName
+        .replaceAll(/[^a-zA-Z0-9-]/g, '-');
+
+    if (sanitizedName.length < ACTOR_NAME.MIN_LENGTH) {
+        sanitizedName = `${sanitizedName}-apify-actor`;
+    }
+
+    sanitizedName = sanitizedName.replaceAll(/^-+/g, '').replaceAll(/-+$/g, '');
+
+    return sanitizedName.slice(0, ACTOR_NAME.MAX_LENGTH);
+};
+
 const getPythonCommand = (directory) => {
     const pythonVenvPath = /^win/.test(process.platform)
         ? 'Scripts/python.exe'
@@ -665,4 +686,5 @@ module.exports = {
     detectNpmVersion,
     detectLocalActorLanguage,
     downloadAndUnzip,
+    sanitizeActorName,
 };