apify-cli 0.18.2-beta.1 → 0.18.2-beta.11

package/src/lib/apify_command.js

@@ -1,11 +1,13 @@
-const { Command } = require('@oclif/command');
 const { finished } = require('stream');
 const { promisify } = require('util');
-const { argsToCamelCase } = require('./utils');
+
+const { Command } = require('@oclif/command');
+
+const { LANGUAGE, COMMANDS_WITHIN_ACTOR } = require('./consts');
 const { maybeTrackTelemetry } = require('./telemetry');
-const { detectInstallationType } = require('./version_check');
+const { argsToCamelCase } = require('./utils');
 const { detectLocalActorLanguage } = require('./utils');
-const { LANGUAGE, COMMANDS_WITHIN_ACTOR } = require('./consts');
+const { detectInstallationType } = require('./version_check');
 
 /**
  * Adding parsing flags to oclif Command class

package/src/lib/consts.js

@@ -1,5 +1,6 @@
 const os = require('os');
 const path = require('path');
+
 const { KEY_VALUE_STORE_KEYS, META_ORIGINS } = require('@apify/consts');
 
 exports.DEFAULT_LOCAL_STORAGE_DIR = 'storage';
@@ -22,6 +23,11 @@ exports.LANGUAGE = {
     UNKNOWN: 'n/a',
 };
 
+exports.PROJECT_TYPES = {
+    SCRAPY: 'scrapy',
+    UNKNOWN: 'unknown',
+};
+
 exports.COMMANDS_WITHIN_ACTOR = ['init', 'run', 'push', 'pull', 'call'];
 
 exports.CHECK_VERSION_EVERY_MILLIS = 24 * 60 * 60 * 1000; // Once a day

package/src/lib/create-utils.js

@@ -1,36 +1,17 @@
-const chalk = require('chalk');
-const inquirer = require('inquirer');
-const https = require('https');
+const fs = require('fs');
 const { pipeline } = require('stream');
 const { promisify } = require('util');
-const fs = require('fs');
 
-const { validateActorName } = require('./utils');
+const chalk = require('chalk');
+const inquirer = require('inquirer');
+
 const {
     warning,
 } = require('./outputs');
+const { validateActorName, httpsGet } = require('./utils');
 
 const PROGRAMMING_LANGUAGES = ['JavaScript', 'TypeScript', 'Python'];
 
-/**
- * @param {string} url
- * @returns {Promise<unknown>}
- */
-exports.httpsGet = async (url) => {
-    return new Promise((resolve, reject) => {
-        https.get(url, (response) => {
-            // Handle redirects
-            if (response.statusCode === 301 || response.statusCode === 302) {
-                resolve(exports.httpsGet(response.headers.location));
-                // Destroy the response to close the HTTP connection, otherwise this hangs for a long time with Node 19+ (due to HTTP keep-alive).
-                response.destroy();
-            } else {
-                resolve(response);
-            }
-        }).on('error', reject);
-    });
-};
-
 /**
  * @param {string} maybeActorName
  * @returns {Promise<string>}
@@ -77,7 +58,7 @@ exports.enhanceReadmeWithLocalSuffix = async (readmePath, manifestPromise) => {
     if (manifest instanceof Error) throw manifest;
 
     try {
-        const suffixStream = await this.httpsGet(manifest.localReadmeSuffixUrl);
+        const suffixStream = await httpsGet(manifest.localReadmeSuffixUrl);
         const readmeStream = fs.createWriteStream(readmePath, { flags: 'a' });
         readmeStream.write('\n\n');
         await promisify(pipeline)(suffixStream, readmeStream);

package/src/lib/exec.js

@@ -1,4 +1,5 @@
 const { spawn } = require('child_process');
+
 const outputs = require('./outputs');
 
 /**

package/src/lib/files.js

@@ -1,8 +1,9 @@
-const loadJson = require('load-json-file');
-const writeJson = require('write-json-file');
 const fs = require('fs');
 const path = require('path');
+
+const loadJson = require('load-json-file');
 const rimraf = require('rimraf');
+const writeJson = require('write-json-file');
 
 const updateLocalJson = async (jsonFilePath, updateAttrs = {}, nestedObjectAttr = null) => {
     const currentObject = await loadJson(jsonFilePath);

package/src/lib/input_schema.js

@@ -1,13 +1,15 @@
 const fs = require('fs');
 const path = require('path');
-const Ajv = require('ajv');
+
+const { KEY_VALUE_STORE_KEYS } = require('@apify/consts');
 const { validateInputSchema } = require('@apify/input_schema');
+const Ajv = require('ajv');
 const _ = require('underscore');
-const { KEY_VALUE_STORE_KEYS } = require('@apify/consts');
 const writeJsonFile = require('write-json-file');
+
 const { ACTOR_SPECIFICATION_FOLDER } = require('./consts');
-const { getLocalConfig, getJsonFileContent, getLocalKeyValueStorePath } = require('./utils');
 const { warning } = require('./outputs');
+const { getLocalConfig, getJsonFileContent, getLocalKeyValueStorePath } = require('./utils');
 
 const DEFAULT_INPUT_SCHEMA_PATHS = [
     '.actor/INPUT_SCHEMA.json',

package/src/lib/local_state.js

@@ -1,5 +1,6 @@
 const loadJson = require('load-json-file');
 const writeJson = require('write-json-file');
+
 const {
     STATE_FILE_PATH,
 } = require('./consts');

package/src/lib/project_analyzer.js

@@ -0,0 +1,24 @@
+const { PROJECT_TYPES } = require('./consts');
+const { ScrapyProjectAnalyzer } = require('./scrapy-wrapper/ScrapyProjectAnalyzer');
+
+const analyzers = [
+    {
+        type: PROJECT_TYPES.SCRAPY,
+        analyzer: ScrapyProjectAnalyzer,
+    },
+];
+
+class ProjectAnalyzer {
+    static getProjectType(pathname) {
+        const analyzer = analyzers.find((a) => {
+            if (!a.analyzer.isApplicable) {
+                throw new Error(`Analyzer ${a.analyzer} does not have isApplicable method.`);
+            }
+
+            return a.analyzer.isApplicable(pathname);
+        });
+        return analyzer?.type || PROJECT_TYPES.UNKNOWN;
+    }
+}
+
+module.exports = { ProjectAnalyzer };

package/src/lib/scrapy-wrapper/ScrapyProjectAnalyzer.js

@@ -0,0 +1,90 @@
+const { readdirSync } = require('fs');
+const fs = require('fs');
+const path = require('path');
+
+const ConfigParser = require('configparser');
+const inquirer = require('inquirer');
+
+const { SpiderFileAnalyzer } = require('./SpiderFileAnalyzer');
+
+class ScrapyProjectAnalyzer {
+    constructor(pathname) {
+        this.pathname = pathname;
+        this.configuration = null;
+        this.settings = null;
+        this.loadScrapyCfg();
+    }
+
+    static isApplicable(pathname) {
+        return fs.existsSync(path.join(pathname, 'scrapy.cfg'));
+    }
+
+    async init() {
+        await this.loadSettings();
+    }
+
+    loadScrapyCfg() {
+        const config = new ConfigParser();
+        const scrapyCfgPath = path.resolve(path.join(this.pathname, 'scrapy.cfg'));
+
+        if (!fs.existsSync(scrapyCfgPath)) {
+            throw new Error(`scrapy.cfg not found in "${scrapyCfgPath}". 
+Are you sure there is a Scrapy project there?`);
+        }
+
+        config.read(scrapyCfgPath);
+        this.configuration = config;
+    }
+
+    async loadSettings() {
+        const assumedBotName = this.configuration.get('settings', 'default').split('.')[0];
+
+        const settings = await inquirer.prompt([
+            {
+                type: 'input',
+                name: 'BOT_NAME',
+                message: 'Enter the Scrapy BOT_NAME (see settings.py):',
+                default: assumedBotName,
+            },
+            {
+                type: 'input',
+                name: 'SPIDER_MODULES',
+                message: 'What folder are the Scrapy spider modules stored in? (see SPIDER_MODULES in settings.py):',
+                default: [`${assumedBotName}.spiders`],
+            },
+        ]);
+
+        if (typeof settings.SPIDER_MODULES === 'string') settings.SPIDER_MODULES = [settings.SPIDER_MODULES];
+
+        this.settings = settings;
+    }
+
+    getName() {
+        return this.settings?.BOT_NAME;
+    }
+
+    getAvailableSpiders() {
+        const spiderPaths = this.settings?.SPIDER_MODULES;
+
+        if (!spiderPaths) {
+            throw new Error('SPIDER_MODULES path not found in settings.');
+        }
+
+        const spiders = [];
+
+        for (const spiderPath of spiderPaths) {
+            const spidersDir = path.join(this.pathname, spiderPath.replaceAll('.', '/'));
+
+            const files = readdirSync(spidersDir, { withFileTypes: true });
+            for (const file of files) {
+                if (file.isFile() && file.name.endsWith('.py') && file.name !== '__init__.py') {
+                    spiders.push(...(new SpiderFileAnalyzer(path.join(spidersDir, file.name)).getSpiders()));
+                }
+            }
+        }
+
+        return spiders;
+    }
+}
+
+module.exports = { ScrapyProjectAnalyzer };

package/src/lib/scrapy-wrapper/Spider.js

@@ -0,0 +1,10 @@
+class Spider {
+    constructor(data) {
+        this.name = data.name;
+        this.class_name = data.class_name;
+        this.start_urls = data.start_urls;
+        this.pathname = data.pathname;
+    }
+}
+
+module.exports = { Spider };

package/src/lib/scrapy-wrapper/SpiderFileAnalyzer.js

@@ -0,0 +1,26 @@
+const fs = require('fs');
+
+const { Spider } = require('./Spider');
+
+class SpiderFileAnalyzer {
+    constructor(pathname) {
+        this.pathname = pathname;
+    }
+
+    getSpiders() {
+        const file = fs.readFileSync(this.pathname, 'utf8');
+
+        const regex = /class\s+(\w+)/g;
+        const spiders = [];
+
+        let match = regex.exec(file);
+        while (match) {
+            spiders.push(new Spider({ class_name: match[1], pathname: this.pathname }));
+            match = regex.exec(file);
+        }
+
+        return spiders;
+    }
+}
+
+module.exports = { SpiderFileAnalyzer };

package/src/lib/scrapy-wrapper/index.js

@@ -0,0 +1,139 @@
+const fs = require('fs');
+const path = require('path');
+
+const { fetchManifest, wrapperManifestUrl } = require('@apify/actor-templates');
+const { walk } = require('@root/walk');
+const ConfigParser = require('configparser');
+const handlebars = require('handlebars');
+const inquirer = require('inquirer');
+
+const { ScrapyProjectAnalyzer } = require('./ScrapyProjectAnalyzer');
+const outputs = require('../outputs');
+const { downloadAndUnzip } = require('../utils');
+
+/**
+ * Files that should be concatenated instead of copied (and overwritten).
+ */
+const concatenableFiles = ['.dockerignore', '.gitignore'];
+
+async function merge(fromPath, toPath, options = { bindings: {} }) {
+    await walk(fromPath, async (err, pathname, dirent) => {
+        if (pathname === fromPath) return;
+        const relPath = path.relative(fromPath, pathname);
+        const toRelPath = relPath.split(path.sep).map((part) => {
+            if (part.startsWith('{') && part.endsWith('}')) {
+                part = part.replace('{', '').replace('}', '');
+                const binding = options.bindings[part];
+                if (!binding) {
+                    throw new Error(`Binding for ${part} not found.`);
+                }
+                return binding;
+            }
+            return part;
+        }).join(path.sep);
+
+        const targetPath = path.join(toPath, toRelPath);
+
+        if (dirent.isDirectory()) {
+            if (!fs.existsSync(targetPath)) {
+                fs.mkdirSync(targetPath);
+            }
+            return merge(pathname, targetPath);
+        }
+
+        if (relPath.includes('.template')) {
+            fs.writeFileSync(
+                path.join(
+                    toPath,
+                    toRelPath.replace('.template', ''),
+                ),
+                handlebars.compile(fs.readFileSync(pathname, 'utf8'))(options.bindings));
+        } else if (fs.existsSync(targetPath) && concatenableFiles.includes(path.basename(toRelPath))) {
+            fs.appendFileSync(targetPath, fs.readFileSync(pathname));
+        } else {
+            fs.copyFileSync(pathname, targetPath);
+        }
+    });
+}
+
+async function wrapScrapyProject({ projectPath }) {
+    if (!projectPath) projectPath = '.';
+
+    const analyzer = new ScrapyProjectAnalyzer(projectPath);
+
+    if (analyzer.configuration.hasSection('apify')) {
+        throw new Error(`The Scrapy project configuration already contains Apify settings. Are you sure you didn't already wrap this project?`);
+    }
+
+    await analyzer.init();
+
+    const { spiderIndex } = await inquirer.prompt([
+        {
+            type: 'list',
+            name: 'spiderIndex',
+            message: 'Pick the Scrapy spider you want to wrap:',
+            choices: analyzer.getAvailableSpiders().map((spider, i) => ({
+                name: `${spider.class_name} (${spider.pathname})`,
+                value: i,
+            })),
+        },
+    ]);
+
+    function translatePathToRelativeModuleName(pathname) {
+        const relPath = path.relative(projectPath, pathname);
+
+        return `.${relPath.split(path.sep).slice(1).join('.').replace('.py', '')}`;
+    }
+
+    const templateBindings = {
+        botName: analyzer.settings.BOT_NAME,
+        scrapy_settings_module: analyzer.configuration.get('settings', 'default'),
+        apify_module_path: `${analyzer.settings.BOT_NAME}.apify`,
+        spider_class_name: analyzer.getAvailableSpiders()[spiderIndex].class_name,
+        spider_module_name: `${translatePathToRelativeModuleName(analyzer.getAvailableSpiders()[spiderIndex].pathname)}`,
+        projectFolder: analyzer.settings.BOT_NAME,
+    };
+
+    const manifest = await fetchManifest(wrapperManifestUrl);
+
+    outputs.info('Downloading the latest Scrapy wrapper template...');
+
+    const { archiveUrl } = manifest.templates.find(({ id }) => id === 'python-scrapy');
+    const templatePath = path.join(__dirname, 'templates', 'python-scrapy');
+
+    if (fs.existsSync(templatePath)) fs.rmSync(templatePath, { recursive: true });
+
+    await downloadAndUnzip({
+        url: archiveUrl,
+        pathTo: templatePath,
+    });
+
+    outputs.info('Wrapping the Scrapy project...');
+
+    merge(
+        path.join(__dirname, 'templates', 'python-scrapy'),
+        projectPath,
+        {
+            bindings: templateBindings,
+        },
+    );
+
+    const apifyConf = new ConfigParser();
+    apifyConf.addSection('apify');
+    apifyConf.set('apify', 'mainpy_location', analyzer.settings.BOT_NAME);
+
+    const s = fs.createWriteStream(path.join(projectPath, 'scrapy.cfg'), { flags: 'a' });
+
+    await new Promise((r) => {
+        s.on('open', (fd) => {
+            s.write('\n', () => {
+                apifyConf.write(fd);
+                r();
+            });
+        });
+    });
+
+    outputs.success('The Scrapy project has been wrapped successfully.');
+}
+
+module.exports = { wrapScrapyProject };

package/src/lib/secrets.js

@@ -1,6 +1,7 @@
 const loadJson = require('load-json-file');
-const writeJson = require('write-json-file');
 const _ = require('underscore');
+const writeJson = require('write-json-file');
+
 const { SECRETS_FILE_PATH } = require('./consts');
 const { warning } = require('./outputs');
 
@@ -62,7 +63,6 @@ const replaceSecretsValue = (env, secrets) => {
             if (secrets[secretKey]) {
                 updatedEnv[key] = secrets[secretKey];
             } else {
-                // eslint-disable-next-line max-len
                 warning(`Value for ${secretKey} not found in local secrets. Set it by calling "apify secrets:add ${secretKey} [SECRET_VALUE]"`);
             }
         } else {

package/src/lib/telemetry.js

@@ -1,8 +1,10 @@
-const Mixpanel = require('mixpanel');
 const { promisify } = require('util');
+
+const { cryptoRandomObjectId } = require('@apify/utilities');
 const loadJson = require('load-json-file');
+const Mixpanel = require('mixpanel');
 const writeJson = require('write-json-file');
-const { cryptoRandomObjectId } = require('@apify/utilities');
+
 const { MIXPANEL_TOKEN, TELEMETRY_FILE_PATH } = require('./consts');
 const outputs = require('./outputs');
 const { getLocalUserInfo } = require('./utils');

package/src/lib/utils.js

@@ -1,13 +1,13 @@
-const path = require('path');
+const {
+    execSync,
+    spawnSync,
+} = require('child_process');
 const fs = require('fs');
-const mime = require('mime');
-const { getEncoding } = require('istextorbinary');
-const _ = require('underscore');
-const globby = require('globby');
-const archiver = require('archiver-promise');
-const loadJson = require('load-json-file');
-const writeJson = require('write-json-file');
-const inquirer = require('inquirer');
+const https = require('https');
+const path = require('path');
+const { finished } = require('stream');
+const { promisify } = require('util');
+
 const {
     ACT_JOB_TERMINAL_STATUSES,
     ACTOR_ENV_VARS,
@@ -18,14 +18,19 @@ const {
     LOCAL_STORAGE_SUBDIRS,
     SOURCE_FILE_FORMATS,
 } = require('@apify/consts');
-const https = require('https');
+const AdmZip = require('adm-zip');
 const { ApifyClient } = require('apify-client');
-const {
-    execSync,
-    spawnSync,
-} = require('child_process');
-const semver = require('semver');
+const archiver = require('archiver-promise');
 const escapeStringRegexp = require('escape-string-regexp');
+const globby = require('globby');
+const inquirer = require('inquirer');
+const { getEncoding } = require('istextorbinary');
+const loadJson = require('load-json-file');
+const mime = require('mime');
+const semver = require('semver');
+const _ = require('underscore');
+const writeJson = require('write-json-file');
+
 const {
     GLOBAL_CONFIGS_FOLDER,
     AUTH_FILE_PATH,
@@ -38,6 +43,7 @@ const {
     SUPPORTED_NODEJS_VERSION,
     MINIMUM_SUPPORTED_PYTHON_VERSION,
     LANGUAGE,
+    PROJECT_TYPES,
 } = require('./consts');
 const {
     ensureFolderExistsSync,
@@ -47,6 +53,26 @@ const {
 const {
     info,
 } = require('./outputs');
+const { ProjectAnalyzer } = require('./project_analyzer');
+
+/**
+ * @param {string} url
+ * @returns {Promise<unknown>}
+ */
+const httpsGet = async (url) => {
+    return new Promise((resolve, reject) => {
+        https.get(url, (response) => {
+            // Handle redirects
+            if (response.statusCode === 301 || response.statusCode === 302) {
+                resolve(httpsGet(response.headers.location));
+                // Destroy the response to close the HTTP connection, otherwise this hangs for a long time with Node 19+ (due to HTTP keep-alive).
+                response.destroy();
+            } else {
+                resolve(response);
+            }
+        }).on('error', reject);
+    });
+};
 
 // Properties from apify.json file that will me migrated to actor specs in .actor/actor.json
 const MIGRATED_APIFY_JSON_PROPERTIES = ['name', 'version', 'buildTag'];
@@ -175,7 +201,7 @@ const getLocalConfigOrThrow = async () => {
         const answer = await inquirer.prompt([{
             name: 'isConfirm',
             type: 'confirm',
-            // eslint-disable-next-line max-len
+
             message: `The new version of Apify CLI uses the "${LOCAL_CONFIG_PATH}" instead of the "apify.json" file. Since we have found both files in your actor directory, "apify.json" will be renamed to "apify.json.deprecated". Going forward, all commands will use "${LOCAL_CONFIG_PATH}". You can read about the differences between the old and the new config at https://github.com/apify/apify-cli/blob/master/MIGRATIONS.md. Do you want to continue?`,
         }]);
         if (!answer.isConfirm) {
@@ -579,8 +605,8 @@ const detectNpmVersion = () => {
 
 const detectLocalActorLanguage = () => {
     const cwd = process.cwd();
-    const isActorInNode = fs.existsSync(path.join(process.cwd(), 'package.json'));
-    const isActorInPython = fs.existsSync(path.join(process.cwd(), 'src/__main__.py'));
+    const isActorInNode = fs.existsSync(path.join(cwd, 'package.json'));
+    const isActorInPython = fs.existsSync(path.join(cwd, 'src/__main__.py')) || ProjectAnalyzer.getProjectType(cwd) === PROJECT_TYPES.SCRAPY;
     const result = {};
     if (isActorInNode) {
         result.language = LANGUAGE.NODEJS;
@@ -594,7 +620,17 @@ const detectLocalActorLanguage = () => {
     return result;
 };
 
+const downloadAndUnzip = async ({ url, pathTo }) => {
+    const zipStream = await httpsGet(url);
+    const chunks = [];
+    zipStream.on('data', (chunk) => chunks.push(chunk));
+    await promisify(finished)(zipStream);
+    const zip = new AdmZip(Buffer.concat(chunks));
+    zip.extractAllTo(pathTo, true);
+};
+
 module.exports = {
+    httpsGet,
     getLoggedClientOrThrow,
     getLocalConfig,
     setLocalConfig,
@@ -628,4 +664,5 @@ module.exports = {
     isNodeVersionSupported,
     detectNpmVersion,
     detectLocalActorLanguage,
+    downloadAndUnzip,
 };

package/src/lib/version_check.js

@@ -1,19 +1,14 @@
 const fs = require('fs');
 const process = require('process');
+
 const axios = require('axios');
 const chalk = require('chalk');
 const semver = require('semver');
-const {
-    CHECK_VERSION_EVERY_MILLIS,
-} = require('./consts');
-const {
-    warning,
-    info,
-} = require('./outputs');
-const {
-    getLocalState,
-    extendLocalState,
-} = require('./local_state');
+
+const { CHECK_VERSION_EVERY_MILLIS } = require('./consts');
+const { getLocalState, extendLocalState } = require('./local_state');
+const { warning, info } = require('./outputs');
+const { version: CURRENT_APIFY_CLI_VERSION } = require('../../package.json');
 
 const INSTALLATION_TYPE = {
     HOMEBREW: 'HOMEBREW',
@@ -30,8 +25,6 @@ const SKIP_UPDATE_CHECK = (
     && !['0', 'false'].includes(process.env.APIFY_CLI_SKIP_UPDATE_CHECK.toLowerCase())
 );
 
-const CURRENT_APIFY_CLI_VERSION = require('../../package.json').version;
-
 /**
  * Detect through which package manager the Apify CLI was installed.
  * @returns {INSTALLATION_TYPE} The installation type of the CLI.