apify-cli 0.19.3-beta.4 → 1.0.0-beta.0

package/src/commands/create.js

@@ -1,203 +0,0 @@
-const fs = require('fs');
-const path = require('path');
-
-const actorTemplates = require('@apify/actor-templates');
-const { flags: flagsHelper } = require('@oclif/command');
-const semver = require('semver');
-
-const { ApifyCommand } = require('../lib/apify_command');
-const { EMPTY_LOCAL_CONFIG, LOCAL_CONFIG_PATH, PYTHON_VENV_PATH, SUPPORTED_NODEJS_VERSION } = require('../lib/consts');
-const {
-    ensureValidActorName,
-    getTemplateDefinition,
-    enhanceReadmeWithLocalSuffix,
-} = require('../lib/create-utils');
-const execWithLog = require('../lib/exec');
-const { updateLocalJson } = require('../lib/files');
-const { createPrefilledInputFileFromInputSchema } = require('../lib/input_schema');
-const outputs = require('../lib/outputs');
-const {
-    setLocalConfig,
-    setLocalEnv,
-    getNpmCmd,
-    getJsonFileContent,
-    detectPythonVersion,
-    isPythonVersionSupported,
-    getPythonCommand,
-    detectNodeVersion,
-    isNodeVersionSupported,
-    detectNpmVersion,
-    downloadAndUnzip,
-} = require('../lib/utils');
-
-class CreateCommand extends ApifyCommand {
-    async run() {
-        const { flags, args } = this.parse(CreateCommand);
-        let { actorName } = args;
-        const {
-            template: templateName,
-            skipDependencyInstall,
-        } = flags;
-
-        // --template-archive-url is an internal, undocumented flag that's used
-        // for testing of templates that are not yet published in the manifest
-        let { templateArchiveUrl } = flags;
-        let skipOptionalDeps = false;
-
-        // Start fetching manifest immediately to prevent
-        // annoying delays that sometimes happen on CLI startup.
-        const manifestPromise = actorTemplates.fetchManifest().catch((err) => {
-            return new Error(`Could not fetch template list from server. Cause: ${err?.message}`);
-        });
-
-        actorName = await ensureValidActorName(actorName);
-        let messages = null;
-
-        this.telemetryData.fromArchiveUrl = !!templateArchiveUrl;
-
-        if (!templateArchiveUrl) {
-            const templateDefinition = await getTemplateDefinition(templateName, manifestPromise);
-            ({ archiveUrl: templateArchiveUrl, skipOptionalDeps, messages } = templateDefinition);
-            this.telemetryData.templateId = templateDefinition.id;
-            this.telemetryData.templateName = templateDefinition.name;
-            this.telemetryData.templateLanguage = templateDefinition.category;
-        }
-
-        const cwd = process.cwd();
-        const actFolderDir = path.join(cwd, actorName);
-
-        // Create actor directory structure
-        try {
-            fs.mkdirSync(actFolderDir);
-        } catch (err) {
-            if (err?.code === 'EEXIST') {
-                outputs.error(`Cannot create new Actor, directory '${actorName}' already exists. `
-                    + 'You can use "apify init" to create a local Actor environment inside an existing directory.');
-                return;
-            }
-            throw err;
-        }
-
-        await downloadAndUnzip({ url: templateArchiveUrl, pathTo: actFolderDir });
-
-        // There may be .actor/actor.json file in used template - let's try to load it and change the name prop value to actorName
-        const localConfig = await getJsonFileContent(path.join(actFolderDir, LOCAL_CONFIG_PATH));
-        await setLocalConfig(Object.assign(localConfig || EMPTY_LOCAL_CONFIG, { name: actorName }), actFolderDir);
-        await setLocalEnv(actFolderDir);
-
-        // Create prefilled INPUT.json file from the input schema prefills
-        await createPrefilledInputFileFromInputSchema(actFolderDir);
-
-        const packageJsonPath = path.join(actFolderDir, 'package.json');
-        const requirementsTxtPath = path.join(actFolderDir, 'requirements.txt');
-        const readmePath = path.join(actFolderDir, 'README.md');
-
-        // Add localReadmeSuffix which is fetched from manifest to README.md
-        // The suffix contains local development instructions
-        await enhanceReadmeWithLocalSuffix(readmePath, manifestPromise);
-
-        let dependenciesInstalled = false;
-        if (!skipDependencyInstall) {
-            if (fs.existsSync(packageJsonPath)) {
-                const currentNodeVersion = detectNodeVersion();
-                const minimumSupportedNodeVersion = semver.minVersion(SUPPORTED_NODEJS_VERSION);
-                if (currentNodeVersion) {
-                    if (!isNodeVersionSupported(currentNodeVersion)) {
-                        outputs.warning(`You are running Node.js version ${currentNodeVersion}, which is no longer supported. `
-                            + `Please upgrade to Node.js version ${minimumSupportedNodeVersion} or later.`);
-                    }
-                    // If the actor is a Node.js actor (has package.json), run `npm install`
-                    await updateLocalJson(packageJsonPath, { name: actorName });
-                    // Run npm install in actor dir.
-                    // For efficiency, don't install Puppeteer for templates that don't use it
-                    const cmdArgs = ['install'];
-                    if (skipOptionalDeps) {
-                        const currentNpmVersion = detectNpmVersion();
-                        if (semver.gte(currentNpmVersion, '7.0.0')) {
-                            cmdArgs.push('--omit=optional');
-                        } else {
-                            cmdArgs.push('--no-optional');
-                        }
-                    }
-                    await execWithLog(getNpmCmd(), cmdArgs, { cwd: actFolderDir });
-                    dependenciesInstalled = true;
-                } else {
-                    outputs.error(`No Node.js detected! Please install Node.js ${minimumSupportedNodeVersion} or higher`
-                        + ' to be able to run Node.js actors locally.');
-                }
-            } else if (fs.existsSync(requirementsTxtPath)) {
-                const pythonVersion = detectPythonVersion(actFolderDir);
-                if (pythonVersion) {
-                    if (isPythonVersionSupported(pythonVersion)) {
-                        const venvPath = path.join(actFolderDir, '.venv');
-                        outputs.info(`Python version ${pythonVersion} detected.`);
-                        outputs.info(`Creating a virtual environment in "${venvPath}" and installing dependencies from "requirements.txt"...`);
-                        let pythonCommand = getPythonCommand(actFolderDir);
-                        if (!process.env.VIRTUAL_ENV) {
-                            // If Python is not running in a virtual environment, create a new one
-                            await execWithLog(pythonCommand, ['-m', 'venv', '--prompt', '.', PYTHON_VENV_PATH], { cwd: actFolderDir });
-                            // regenerate the `pythonCommand` after we create the virtual environment
-                            pythonCommand = getPythonCommand(actFolderDir);
-                        }
-                        await execWithLog(
-                            pythonCommand,
-                            ['-m', 'pip', 'install', '--no-cache-dir', '--no-warn-script-location', '--upgrade', 'pip', 'setuptools', 'wheel'],
-                            { cwd: actFolderDir },
-                        );
-                        await execWithLog(
-                            pythonCommand,
-                            ['-m', 'pip', 'install', '--no-cache-dir', '--no-warn-script-location', '-r', 'requirements.txt'],
-                            { cwd: actFolderDir },
-                        );
-                        dependenciesInstalled = true;
-                    } else {
-                        outputs.warning(`Python actors require Python 3.8 or higher, but you have Python ${pythonVersion}!`);
-                        outputs.warning('Please install Python 3.8 or higher to be able to run Python actors locally.');
-                    }
-                } else {
-                    outputs.warning('No Python detected! Please install Python 3.8 or higher to be able to run Python actors locally.');
-                }
-            }
-        }
-
-        if (dependenciesInstalled) {
-            outputs.success(`Actor '${actorName}' was created. To run it, run "cd ${actorName}" and "apify run".`);
-            outputs.success('To run your code in the cloud, run "apify push" and deploy your code to Apify Console.');
-            if (messages?.postCreate) {
-                outputs.info(messages?.postCreate);
-            }
-        } else {
-            outputs.success(`Actor '${actorName}' was created. Please install its dependencies to be able to run it using "apify run".`);
-        }
-    }
-}
-
-CreateCommand.description = 'Creates a new Actor project directory from a selected boilerplate template.';
-
-CreateCommand.flags = {
-    template: flagsHelper.string({
-        char: 't',
-        description: 'Template for the Actor. If not provided, the command will prompt for it.\n'
-            + `Visit ${actorTemplates.manifestUrl} to find available template names.`,
-        required: false,
-    }),
-    'skip-dependency-install': flagsHelper.boolean({
-        description: 'Skip installing Actor dependencies.',
-        required: false,
-    }),
-    'template-archive-url': flagsHelper.string({
-        description: 'Actor template archive url. Useful for developing new templates.',
-        required: false,
-        hidden: true,
-    }),
-};
-
-CreateCommand.args = [
-    {
-        name: 'actorName',
-        required: false,
-        description: 'Name of the Actor and its directory',
-    },
-];
-
-module.exports = CreateCommand;

package/src/commands/info.js

@@ -1,24 +0,0 @@
-const chalk = require('chalk');
-
-const { ApifyCommand } = require('../lib/apify_command');
-const { getLoggedClientOrThrow, getLocalUserInfo } = require('../lib/utils');
-
-class InfoCommand extends ApifyCommand {
-    async run() {
-        await getLoggedClientOrThrow();
-        const info = await getLocalUserInfo();
-
-        if (info) {
-            const niceInfo = {
-                username: info.username,
-                userId: info.id,
-            };
-            Object.keys(niceInfo).forEach((key) => console.log(`${chalk.gray(key)}: ${chalk.bold(niceInfo[key])}`));
-        }
-    }
-}
-
-InfoCommand.description = 'Displays information about the currently active Apify account.\n'
-    + 'The information is printed to the console.';
-
-module.exports = InfoCommand;

package/src/commands/init-wrap-scrapy.js

@@ -1,34 +0,0 @@
-const { ApifyCommand } = require('../lib/apify_command');
-const { info } = require('../lib/outputs');
-const { wrapScrapyProject } = require('../lib/scrapy-wrapper/index');
-
-class WrapScrapyCommand extends ApifyCommand {
-    async run() {
-        const { args } = this.parse(WrapScrapyCommand);
-
-        await wrapScrapyProject({ projectPath: args.path });
-
-        info('Scrapy project wrapped successfully.');
-    }
-}
-
-WrapScrapyCommand.hidden = true;
-
-WrapScrapyCommand.description = `Wraps your existing Scrapy project to work like an Apify Actor.
-
-It adds the following features:
-- Automatic retry of failed requests
-- Automatic proxy rotation
-- Automatic user agent rotation
-...
-`;
-
-WrapScrapyCommand.args = [
-    {
-        name: 'path',
-        required: false,
-        description: 'Optional path to your scrapy project. If not provided, the current directory is used.',
-    },
-];
-
-module.exports = WrapScrapyCommand;

package/src/commands/init.js

@@ -1,83 +0,0 @@
-const path = require('path');
-
-const { flags: flagsHelper } = require('@oclif/command');
-const inquirer = require('inquirer');
-
-const { ApifyCommand } = require('../lib/apify_command');
-const { EMPTY_LOCAL_CONFIG, DEFAULT_LOCAL_STORAGE_DIR, LOCAL_CONFIG_PATH, LANGUAGE, PROJECT_TYPES } = require('../lib/consts');
-const { createPrefilledInputFileFromInputSchema } = require('../lib/input_schema');
-const outputs = require('../lib/outputs');
-const { ProjectAnalyzer } = require('../lib/project_analyzer');
-const { wrapScrapyProject } = require('../lib/scrapy-wrapper');
-const { setLocalConfig, setLocalEnv, getLocalConfig, getLocalConfigOrThrow, detectLocalActorLanguage, validateActorName } = require('../lib/utils');
-
-class InitCommand extends ApifyCommand {
-    async run() {
-        const { args, flags } = this.parse(InitCommand);
-        let { actorName } = args;
-        const cwd = process.cwd();
-
-        if (ProjectAnalyzer.getProjectType(cwd) === PROJECT_TYPES.SCRAPY) {
-            outputs.info('The current directory looks like a Scrapy project. Using automatic project wrapping.');
-            this.telemetryData.actorWrapper = PROJECT_TYPES.SCRAPY;
-
-            return wrapScrapyProject({ projectPath: cwd });
-        }
-
-        if (!flags.yes && detectLocalActorLanguage(cwd).language === LANGUAGE.UNKNOWN) {
-            outputs.warning('The current directory does not look like a Node.js or Python project.');
-            const { c } = await inquirer.prompt([{ name: 'c', message: 'Do you want to continue?', type: 'confirm' }]);
-            if (!c) return;
-        }
-
-        if (getLocalConfig()) {
-            outputs.warning(`Skipping creation of "${LOCAL_CONFIG_PATH}", the file already exists in the current directory.`);
-        } else {
-            if (!actorName) {
-                let response = null;
-
-                while (!response) {
-                    try {
-                        const answer = await inquirer.prompt([{ name: 'actName', message: 'Actor name:', default: path.basename(cwd) }]);
-                        validateActorName(answer.actName);
-                        response = answer;
-                    } catch (err) {
-                        outputs.error(err.message);
-                    }
-                }
-
-                ({ actName: actorName } = response);
-            }
-            // Migrate apify.json to .actor/actor.json
-            const localConfig = { ...EMPTY_LOCAL_CONFIG, ...await getLocalConfigOrThrow() };
-            await setLocalConfig(Object.assign(localConfig, { name: actorName }), cwd);
-        }
-        await setLocalEnv(cwd);
-        // Create prefilled INPUT.json file from the input schema prefills
-        await createPrefilledInputFileFromInputSchema(cwd);
-        outputs.success('The Actor has been initialized in the current directory.');
-    }
-}
-
-InitCommand.description = 'Initializes a new Actor project in an existing directory.\n'
-    + `The command only creates the "${LOCAL_CONFIG_PATH}" file and the "${DEFAULT_LOCAL_STORAGE_DIR}" directory in the current directory, `
-    + 'but will not touch anything else.\n\n'
-    + `WARNING: The directory at "${DEFAULT_LOCAL_STORAGE_DIR}" will be overwritten if it already exists.`;
-
-InitCommand.args = [
-    {
-        name: 'actorName',
-        required: false,
-        description: 'Name of the Actor. If not provided, you will be prompted for it.',
-    },
-];
-
-InitCommand.flags = {
-    yes: flagsHelper.boolean({
-        char: 'y',
-        description: 'Automatic yes to prompts; assume "yes" as answer to all prompts. Note that in some cases, the command may still ask for confirmation.',
-        required: false,
-    }),
-};
-
-module.exports = InitCommand;

package/src/commands/login.js

@@ -1,40 +0,0 @@
-const { flags: flagsHelper } = require('@oclif/command');
-const inquirer = require('inquirer');
-
-const { ApifyCommand } = require('../lib/apify_command');
-const { success, error } = require('../lib/outputs');
-const { useApifyIdentity } = require('../lib/telemetry');
-const { getLoggedClient } = require('../lib/utils');
-const { getLocalUserInfo } = require('../lib/utils');
-
-class LoginCommand extends ApifyCommand {
-    async run() {
-        const { flags } = this.parse(LoginCommand);
-        let { token } = flags;
-        if (!token) {
-            console.log('Enter your Apify API token. You can find it at https://console.apify.com/account#/integrations');
-            const tokenPrompt = await inquirer.prompt([{ name: 'token', message: 'token:', type: 'password' }]);
-            ({ token } = tokenPrompt);
-        }
-        const isUserLogged = await getLoggedClient(token);
-        const userInfo = getLocalUserInfo();
-        await useApifyIdentity(userInfo.id);
-        return isUserLogged
-            ? success(`You are logged in to Apify as ${userInfo.username || userInfo.id}!`)
-            : error('Login to Apify failed, the provided API token is not valid.');
-    }
-}
-
-LoginCommand.description = 'Logs in to your Apify account using a provided API token.\nThe API token and other account '
-    + 'information is stored in the ~/.apify directory, from where it is read by all other "apify" commands. '
-    + 'To log out, call "apify logout".';
-
-LoginCommand.flags = {
-    token: flagsHelper.string({
-        char: 't',
-        description: '[Optional] Apify API token',
-        required: false,
-    }),
-};
-
-module.exports = LoginCommand;

package/src/commands/logout.js

@@ -1,18 +0,0 @@
-const { ApifyCommand } = require('../lib/apify_command');
-const { AUTH_FILE_PATH } = require('../lib/consts');
-const { rimrafPromised } = require('../lib/files');
-const { success } = require('../lib/outputs');
-const { regenerateLocalDistinctId } = require('../lib/telemetry');
-
-class LogoutCommand extends ApifyCommand {
-    async run() {
-        await rimrafPromised(AUTH_FILE_PATH);
-        regenerateLocalDistinctId();
-        success('You are logged out from your Apify account.');
-    }
-}
-
-LogoutCommand.description = 'Logs out of your Apify account.\nThe command deletes the API token and all other '
-    + 'account information stored in the ~/.apify directory. To log in again, call "apify login".';
-
-module.exports = LogoutCommand;

package/src/commands/pull.js

@@ -1,154 +0,0 @@
-const fs = require('fs');
-const path = require('path');
-
-const { flags: flagsHelper } = require('@oclif/command');
-const AdmZip = require('adm-zip');
-const { get } = require('axios');
-const jju = require('jju');
-const semverGt = require('semver/functions/gt');
-const tiged = require('tiged');
-
-const { ApifyCommand } = require('../lib/apify_command');
-const { LOCAL_CONFIG_PATH } = require('../lib/consts');
-const { success, error } = require('../lib/outputs');
-const { getLoggedClientOrThrow, getLocalConfigOrThrow, getLocalUserInfo } = require('../lib/utils');
-
-const extractGitHubZip = async (url, directoryPath) => {
-    const { data } = await get(url, { responseType: 'arraybuffer' });
-
-    const zipFile = new AdmZip(Buffer.from(data, 'binary'));
-
-    zipFile.extractEntryTo(zipFile.getEntries()[0].entryName, directoryPath, false);
-};
-
-class PullCommand extends ApifyCommand {
-    async run() {
-        const { args, flags } = this.parse(PullCommand);
-        const localConfig = await getLocalConfigOrThrow();
-        const userInfo = await getLocalUserInfo();
-        const apifyClient = await getLoggedClientOrThrow();
-        const cwd = process.cwd();
-
-        const isActorAutomaticallyDetected = !args?.actorId;
-        const usernameOrId = userInfo.username || userInfo.id;
-
-        const actorId = args?.actorId || localConfig?.id || (localConfig.name ? `${usernameOrId}/${localConfig.name}` : undefined);
-
-        if (!actorId) throw new Error('Cannot find Actor in this directory.');
-
-        let actor;
-        try {
-            actor = await apifyClient.actor(actorId).get();
-        } catch {
-            throw new Error(`Cannot find Actor with ID/name '${actorId}' in your account.`);
-        }
-
-        if (!actor) throw new Error(`Cannot find Actor with ID/name '${actorId}' in your account.`);
-
-        const { name, versions } = actor;
-
-        let correctVersion = null;
-        if (flags?.version) {
-            correctVersion = versions.find((version) => version.versionNumber === flags?.version);
-            if (!correctVersion) {
-                throw new Error(`Cannot find version ${flags?.version} of Actor ${actorId}.`);
-            }
-        }
-
-        if (!correctVersion) {
-            correctVersion = versions.reduce((match, curr) => {
-                if (semverGt(`${curr.versionNumber}.0`, `${match.versionNumber}.0`)) return curr;
-                return match;
-            });
-        }
-
-        const dirpath = isActorAutomaticallyDetected ? cwd : path.join(cwd, name);
-        fs.mkdirSync(dirpath, { recursive: true });
-
-        if (!isActorAutomaticallyDetected && !(fs.readdirSync(dirpath).length === 0)) {
-            error(`Directory ${dirpath} is not empty. Please empty it or choose another directory.`);
-            return;
-        }
-
-        switch (correctVersion.sourceType) {
-            case 'TARBALL': {
-                await extractGitHubZip(correctVersion.tarballUrl, dirpath);
-
-                break;
-            }
-            case 'SOURCE_FILES': {
-                const { sourceFiles } = correctVersion;
-                for (const file of sourceFiles) {
-                    const folderPath = path.dirname(file.name);
-                    fs.mkdirSync(`${dirpath}/${folderPath}`, { recursive: true }, null);
-
-                    if (!file.folder) {
-                        const fileContent = file.format === 'BASE64' ? Buffer.from(file.content, 'base64') : file.content;
-
-                        if (file.name === LOCAL_CONFIG_PATH) {
-                            const actorJson = jju.parse(fileContent);
-                            actorJson.name = actor.name;
-                            fs.writeFileSync(`${dirpath}/${file.name}`, jju.update(fileContent, actorJson));
-                        } else {
-                            fs.writeFileSync(`${dirpath}/${file.name}`, fileContent);
-                        }
-                    }
-                }
-
-                break;
-            }
-            case 'GIT_REPO': {
-                // e.g. https://github.com/jakubbalada/Datasety.git#master:RejstrikPolitickychStran
-                const { gitRepoUrl } = correctVersion;
-                const [repoUrl, branchDirPart] = gitRepoUrl.split('#');
-
-                let branch;
-                let dir;
-                if (branchDirPart) [branch, dir] = branchDirPart.split(':');
-                let branchDirRepoUrl = repoUrl;
-                if (dir) branchDirRepoUrl += `/${dir}`;
-                if (branch) branchDirRepoUrl += `#${branch}`;
-
-                const emitter = tiged(branchDirRepoUrl);
-                try {
-                    await emitter.clone(dirpath);
-                } catch (err) {
-                    throw new Error(`Failed to pull Actor from ${gitRepoUrl}. ${err.message}`);
-                }
-
-                break;
-            }
-            case 'GITHUB_GIST': {
-                await extractGitHubZip(`${correctVersion.gitHubGistUrl}/archive/master.zip`, dirpath);
-
-                break;
-            }
-            default:
-                throw new Error(`Unknown source type: ${correctVersion.sourceType}`);
-        }
-
-        success(isActorAutomaticallyDetected ? `Actor ${name} updated at ${dirpath}/` : `Pulled to ${dirpath}/`);
-    }
-}
-
-PullCommand.description = 'Pulls an Actor from the Apify platform to the current directory. '
-    + 'If it is defined as Git repository, it will be cloned. If it is defined as Web IDE, it will fetch the files.';
-
-PullCommand.flags = {
-    version: flagsHelper.string({
-        char: 'v',
-        description: 'Actor version number which will be pulled, e.g. 1.2. Default: the highest version',
-        required: false,
-    }),
-};
-
-PullCommand.args = [
-    {
-        name: 'actorId',
-        required: false,
-        description: 'Name or ID of the actor to run (e.g. "apify/hello-world" or "E2jjCZBezvAZnX8Rb"). '
-            + 'If not provided, the command will update the Actor in the current directory based on its name in ".actor/actor.json" file.',
-    },
-];
-
-module.exports = PullCommand;