apify-cli 0.19.5 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +200 -0
- package/README.md +273 -167
- package/bin/dev.cmd +3 -0
- package/bin/dev.js +21 -0
- package/bin/dev.sh +4 -0
- package/bin/run.js +23 -0
- package/dist/.tsbuildinfo +1 -0
- package/dist/commands/actor/get-input.d.ts +6 -0
- package/dist/commands/actor/get-input.d.ts.map +1 -0
- package/dist/commands/actor/get-input.js +14 -0
- package/dist/commands/actor/get-input.js.map +1 -0
- package/dist/commands/actor/get-value.d.ts +9 -0
- package/dist/commands/actor/get-value.d.ts.map +1 -0
- package/dist/commands/actor/get-value.js +27 -0
- package/dist/commands/actor/get-value.js.map +1 -0
- package/dist/commands/actor/index.d.ts +6 -0
- package/dist/commands/actor/index.d.ts.map +1 -0
- package/dist/commands/actor/index.js +13 -0
- package/dist/commands/actor/index.js.map +1 -0
- package/dist/commands/actor/push-data.d.ts +11 -0
- package/dist/commands/actor/push-data.d.ts.map +1 -0
- package/dist/commands/actor/push-data.js +56 -0
- package/dist/commands/actor/push-data.js.map +1 -0
- package/dist/commands/actor/set-value.d.ts +15 -0
- package/dist/commands/actor/set-value.d.ts.map +1 -0
- package/dist/commands/actor/set-value.js +76 -0
- package/dist/commands/actor/set-value.js.map +1 -0
- package/dist/commands/call.d.ts +19 -0
- package/dist/commands/call.d.ts.map +1 -0
- package/dist/commands/call.js +193 -0
- package/dist/commands/call.js.map +1 -0
- package/dist/commands/check-version.d.ts +11 -0
- package/dist/commands/check-version.d.ts.map +1 -0
- package/dist/commands/check-version.js +39 -0
- package/dist/commands/check-version.js.map +1 -0
- package/dist/commands/create.d.ts +15 -0
- package/dist/commands/create.d.ts.map +1 -0
- package/dist/commands/create.js +201 -0
- package/dist/commands/create.js.map +1 -0
- package/dist/commands/edit-input-schema.d.ts +11 -0
- package/dist/commands/edit-input-schema.d.ts.map +1 -0
- package/{src → dist}/commands/edit-input-schema.js +81 -85
- package/dist/commands/edit-input-schema.js.map +1 -0
- package/dist/commands/info.d.ts +6 -0
- package/dist/commands/info.d.ts.map +1 -0
- package/dist/commands/info.js +26 -0
- package/dist/commands/info.js.map +1 -0
- package/dist/commands/init-wrap-scrapy.d.ts +10 -0
- package/dist/commands/init-wrap-scrapy.d.ts.map +1 -0
- package/dist/commands/init-wrap-scrapy.js +41 -0
- package/dist/commands/init-wrap-scrapy.js.map +1 -0
- package/dist/commands/init.d.ts +12 -0
- package/dist/commands/init.d.ts.map +1 -0
- package/dist/commands/init.js +88 -0
- package/dist/commands/init.js.map +1 -0
- package/dist/commands/login.d.ts +10 -0
- package/dist/commands/login.d.ts.map +1 -0
- package/dist/commands/login.js +174 -0
- package/dist/commands/login.js.map +1 -0
- package/dist/commands/logout.d.ts +6 -0
- package/dist/commands/logout.d.ts.map +1 -0
- package/dist/commands/logout.js +20 -0
- package/dist/commands/logout.js.map +1 -0
- package/dist/commands/pull.d.ts +12 -0
- package/dist/commands/pull.d.ts.map +1 -0
- package/dist/commands/pull.js +148 -0
- package/dist/commands/pull.js.map +1 -0
- package/dist/commands/push.d.ts +18 -0
- package/dist/commands/push.d.ts.map +1 -0
- package/dist/commands/push.js +298 -0
- package/dist/commands/push.js.map +1 -0
- package/dist/commands/run.d.ts +16 -0
- package/dist/commands/run.d.ts.map +1 -0
- package/dist/commands/run.js +383 -0
- package/dist/commands/run.js.map +1 -0
- package/dist/commands/secrets/add.d.ts +10 -0
- package/dist/commands/secrets/add.d.ts.map +1 -0
- package/dist/commands/secrets/add.js +31 -0
- package/dist/commands/secrets/add.js.map +1 -0
- package/dist/commands/secrets/index.d.ts +6 -0
- package/dist/commands/secrets/index.d.ts.map +1 -0
- package/dist/commands/secrets/index.js +24 -0
- package/dist/commands/secrets/index.js.map +1 -0
- package/dist/commands/secrets/rm.d.ts +9 -0
- package/dist/commands/secrets/rm.d.ts.map +1 -0
- package/dist/commands/secrets/rm.js +27 -0
- package/dist/commands/secrets/rm.js.map +1 -0
- package/dist/commands/task/index.d.ts +6 -0
- package/dist/commands/task/index.d.ts.map +1 -0
- package/dist/commands/task/index.js +13 -0
- package/dist/commands/task/index.js.map +1 -0
- package/dist/commands/task/run.d.ts +16 -0
- package/dist/commands/task/run.d.ts.map +1 -0
- package/dist/commands/task/run.js +91 -0
- package/dist/commands/task/run.js.map +1 -0
- package/dist/commands/validate-schema.d.ts +10 -0
- package/dist/commands/validate-schema.d.ts.map +1 -0
- package/dist/commands/validate-schema.js +58 -0
- package/dist/commands/validate-schema.js.map +1 -0
- package/dist/hooks/deprecations.d.ts +4 -0
- package/dist/hooks/deprecations.d.ts.map +1 -0
- package/dist/hooks/deprecations.js +18 -0
- package/dist/hooks/deprecations.js.map +1 -0
- package/dist/hooks/init.d.ts +7 -0
- package/dist/hooks/init.d.ts.map +1 -0
- package/dist/hooks/init.js +17 -0
- package/dist/hooks/init.js.map +1 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -0
- package/dist/lib/actor.d.ts +27 -0
- package/dist/lib/actor.d.ts.map +1 -0
- package/{src → dist}/lib/actor.js +22 -45
- package/dist/lib/actor.js.map +1 -0
- package/dist/lib/apify-oclif-help.d.ts +15 -0
- package/dist/lib/apify-oclif-help.d.ts.map +1 -0
- package/dist/lib/apify-oclif-help.js +19 -0
- package/dist/lib/apify-oclif-help.js.map +1 -0
- package/dist/lib/apify_command.d.ts +21 -0
- package/dist/lib/apify_command.d.ts.map +1 -0
- package/dist/lib/apify_command.js +104 -0
- package/dist/lib/apify_command.js.map +1 -0
- package/dist/lib/commands/resolve-input.d.ts +5 -0
- package/dist/lib/commands/resolve-input.d.ts.map +1 -0
- package/dist/lib/commands/resolve-input.js +29 -0
- package/dist/lib/commands/resolve-input.js.map +1 -0
- package/dist/lib/commands/run-on-cloud.d.ts +20 -0
- package/dist/lib/commands/run-on-cloud.d.ts.map +1 -0
- package/dist/lib/commands/run-on-cloud.js +89 -0
- package/dist/lib/commands/run-on-cloud.js.map +1 -0
- package/dist/lib/community.d.ts +2 -0
- package/dist/lib/community.d.ts.map +1 -0
- package/dist/lib/community.js +4 -0
- package/dist/lib/community.js.map +1 -0
- package/dist/lib/consts.d.ts +54 -0
- package/dist/lib/consts.d.ts.map +1 -0
- package/dist/lib/consts.js +65 -0
- package/dist/lib/consts.js.map +1 -0
- package/dist/lib/create-utils.d.ts +8 -0
- package/dist/lib/create-utils.d.ts.map +1 -0
- package/dist/lib/create-utils.js +148 -0
- package/dist/lib/create-utils.js.map +1 -0
- package/dist/lib/exec.d.ts +3 -0
- package/dist/lib/exec.d.ts.map +1 -0
- package/{src → dist}/lib/exec.js +16 -17
- package/dist/lib/exec.js.map +1 -0
- package/dist/lib/files.d.ts +14 -0
- package/dist/lib/files.d.ts.map +1 -0
- package/dist/lib/files.js +58 -0
- package/dist/lib/files.js.map +1 -0
- package/dist/lib/input_schema.d.ts +28 -0
- package/dist/lib/input_schema.d.ts.map +1 -0
- package/dist/lib/input_schema.js +126 -0
- package/dist/lib/input_schema.js.map +1 -0
- package/dist/lib/local_state.d.ts +11 -0
- package/dist/lib/local_state.d.ts.map +1 -0
- package/dist/lib/local_state.js +27 -0
- package/dist/lib/local_state.js.map +1 -0
- package/dist/lib/outputs.d.ts +19 -0
- package/dist/lib/outputs.d.ts.map +1 -0
- package/dist/lib/outputs.js +45 -0
- package/dist/lib/outputs.js.map +1 -0
- package/dist/lib/project_analyzer.d.ts +4 -0
- package/dist/lib/project_analyzer.d.ts.map +1 -0
- package/{src → dist}/lib/project_analyzer.js +6 -11
- package/dist/lib/project_analyzer.js.map +1 -0
- package/dist/lib/projects/CrawleeAnalyzer.d.ts +4 -0
- package/dist/lib/projects/CrawleeAnalyzer.d.ts.map +1 -0
- package/{src → dist}/lib/projects/CrawleeAnalyzer.js +7 -24
- package/dist/lib/projects/CrawleeAnalyzer.js.map +1 -0
- package/dist/lib/projects/OldApifySDKAnalyzer.d.ts +4 -0
- package/dist/lib/projects/OldApifySDKAnalyzer.d.ts.map +1 -0
- package/{src → dist}/lib/projects/OldApifySDKAnalyzer.js +9 -32
- package/dist/lib/projects/OldApifySDKAnalyzer.js.map +1 -0
- package/dist/lib/projects/scrapy/ScrapyProjectAnalyzer.d.ts +17 -0
- package/dist/lib/projects/scrapy/ScrapyProjectAnalyzer.d.ts.map +1 -0
- package/{src/lib/scrapy-wrapper → dist/lib/projects/scrapy}/ScrapyProjectAnalyzer.js +33 -36
- package/dist/lib/projects/scrapy/ScrapyProjectAnalyzer.js.map +1 -0
- package/dist/lib/projects/scrapy/Spider.d.ts +14 -0
- package/dist/lib/projects/scrapy/Spider.d.ts.map +1 -0
- package/dist/lib/projects/scrapy/Spider.js +33 -0
- package/dist/lib/projects/scrapy/Spider.js.map +1 -0
- package/dist/lib/projects/scrapy/SpiderFileAnalyzer.d.ts +7 -0
- package/dist/lib/projects/scrapy/SpiderFileAnalyzer.d.ts.map +1 -0
- package/dist/lib/projects/scrapy/SpiderFileAnalyzer.js +25 -0
- package/dist/lib/projects/scrapy/SpiderFileAnalyzer.js.map +1 -0
- package/dist/lib/projects/scrapy/wrapScrapyProject.d.ts +4 -0
- package/dist/lib/projects/scrapy/wrapScrapyProject.d.ts.map +1 -0
- package/{src/lib/scrapy-wrapper/index.js → dist/lib/projects/scrapy/wrapScrapyProject.js} +43 -73
- package/dist/lib/projects/scrapy/wrapScrapyProject.js.map +1 -0
- package/dist/lib/projects/shared.d.ts +2 -0
- package/dist/lib/projects/shared.d.ts.map +1 -0
- package/dist/lib/projects/shared.js +13 -0
- package/dist/lib/projects/shared.js.map +1 -0
- package/dist/lib/secrets.d.ts +22 -0
- package/dist/lib/secrets.d.ts.map +1 -0
- package/{src → dist}/lib/secrets.js +34 -42
- package/dist/lib/secrets.js.map +1 -0
- package/dist/lib/telemetry.d.ts +21 -0
- package/dist/lib/telemetry.d.ts.map +1 -0
- package/dist/lib/telemetry.js +87 -0
- package/dist/lib/telemetry.js.map +1 -0
- package/dist/lib/types.d.ts +11 -0
- package/dist/lib/types.d.ts.map +1 -0
- package/dist/lib/types.js +2 -0
- package/dist/lib/types.js.map +1 -0
- package/dist/lib/utils.d.ts +102 -0
- package/dist/lib/utils.d.ts.map +1 -0
- package/dist/lib/utils.js +577 -0
- package/dist/lib/utils.js.map +1 -0
- package/dist/lib/version_check.d.ts +14 -0
- package/dist/lib/version_check.d.ts.map +1 -0
- package/{src → dist}/lib/version_check.js +31 -60
- package/dist/lib/version_check.js.map +1 -0
- package/oclif.manifest.json +867 -1
- package/package.json +151 -109
- package/index.js +0 -1
- package/src/bin/run +0 -16
- package/src/commands/actor/get-input.js +0 -12
- package/src/commands/actor/get-value.js +0 -23
- package/src/commands/actor/index.js +0 -13
- package/src/commands/actor/push-data.js +0 -44
- package/src/commands/actor/set-value.js +0 -63
- package/src/commands/call.js +0 -131
- package/src/commands/check-version.js +0 -28
- package/src/commands/create.js +0 -203
- package/src/commands/info.js +0 -24
- package/src/commands/init-wrap-scrapy.js +0 -34
- package/src/commands/init.js +0 -83
- package/src/commands/login-new.js +0 -169
- package/src/commands/login.js +0 -40
- package/src/commands/logout.js +0 -18
- package/src/commands/pull.js +0 -154
- package/src/commands/push.js +0 -234
- package/src/commands/run.js +0 -206
- package/src/commands/secrets/add.js +0 -28
- package/src/commands/secrets/index.js +0 -24
- package/src/commands/secrets/rm.js +0 -23
- package/src/commands/vis.js +0 -50
- package/src/hooks/init.js +0 -16
- package/src/lib/apify_command.js +0 -82
- package/src/lib/community.js +0 -3
- package/src/lib/consts.js +0 -69
- package/src/lib/create-utils.js +0 -187
- package/src/lib/files.js +0 -76
- package/src/lib/input_schema.js +0 -114
- package/src/lib/local_state.js +0 -39
- package/src/lib/outputs.js +0 -27
- package/src/lib/scrapy-wrapper/Spider.js +0 -10
- package/src/lib/scrapy-wrapper/SpiderFileAnalyzer.js +0 -26
- package/src/lib/telemetry.js +0 -104
- package/src/lib/utils.js +0 -707
- /package/{src/bin → bin}/run.cmd +0 -0
package/src/commands/pull.js
DELETED
|
@@ -1,154 +0,0 @@
|
|
|
1
|
-
const fs = require('fs');
|
|
2
|
-
const path = require('path');
|
|
3
|
-
|
|
4
|
-
const { flags: flagsHelper } = require('@oclif/command');
|
|
5
|
-
const AdmZip = require('adm-zip');
|
|
6
|
-
const { get } = require('axios');
|
|
7
|
-
const jju = require('jju');
|
|
8
|
-
const semverGt = require('semver/functions/gt');
|
|
9
|
-
const tiged = require('tiged');
|
|
10
|
-
|
|
11
|
-
const { ApifyCommand } = require('../lib/apify_command');
|
|
12
|
-
const { LOCAL_CONFIG_PATH } = require('../lib/consts');
|
|
13
|
-
const { success, error } = require('../lib/outputs');
|
|
14
|
-
const { getLoggedClientOrThrow, getLocalConfigOrThrow, getLocalUserInfo } = require('../lib/utils');
|
|
15
|
-
|
|
16
|
-
const extractGitHubZip = async (url, directoryPath) => {
|
|
17
|
-
const { data } = await get(url, { responseType: 'arraybuffer' });
|
|
18
|
-
|
|
19
|
-
const zipFile = new AdmZip(Buffer.from(data, 'binary'));
|
|
20
|
-
|
|
21
|
-
zipFile.extractEntryTo(zipFile.getEntries()[0].entryName, directoryPath, false);
|
|
22
|
-
};
|
|
23
|
-
|
|
24
|
-
class PullCommand extends ApifyCommand {
|
|
25
|
-
async run() {
|
|
26
|
-
const { args, flags } = this.parse(PullCommand);
|
|
27
|
-
const localConfig = await getLocalConfigOrThrow();
|
|
28
|
-
const userInfo = await getLocalUserInfo();
|
|
29
|
-
const apifyClient = await getLoggedClientOrThrow();
|
|
30
|
-
const cwd = process.cwd();
|
|
31
|
-
|
|
32
|
-
const isActorAutomaticallyDetected = !args?.actorId;
|
|
33
|
-
const usernameOrId = userInfo.username || userInfo.id;
|
|
34
|
-
|
|
35
|
-
const actorId = args?.actorId || localConfig?.id || (localConfig.name ? `${usernameOrId}/${localConfig.name}` : undefined);
|
|
36
|
-
|
|
37
|
-
if (!actorId) throw new Error('Cannot find Actor in this directory.');
|
|
38
|
-
|
|
39
|
-
let actor;
|
|
40
|
-
try {
|
|
41
|
-
actor = await apifyClient.actor(actorId).get();
|
|
42
|
-
} catch {
|
|
43
|
-
throw new Error(`Cannot find Actor with ID/name '${actorId}' in your account.`);
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
if (!actor) throw new Error(`Cannot find Actor with ID/name '${actorId}' in your account.`);
|
|
47
|
-
|
|
48
|
-
const { name, versions } = actor;
|
|
49
|
-
|
|
50
|
-
let correctVersion = null;
|
|
51
|
-
if (flags?.version) {
|
|
52
|
-
correctVersion = versions.find((version) => version.versionNumber === flags?.version);
|
|
53
|
-
if (!correctVersion) {
|
|
54
|
-
throw new Error(`Cannot find version ${flags?.version} of Actor ${actorId}.`);
|
|
55
|
-
}
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
if (!correctVersion) {
|
|
59
|
-
correctVersion = versions.reduce((match, curr) => {
|
|
60
|
-
if (semverGt(`${curr.versionNumber}.0`, `${match.versionNumber}.0`)) return curr;
|
|
61
|
-
return match;
|
|
62
|
-
});
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
const dirpath = isActorAutomaticallyDetected ? cwd : path.join(cwd, name);
|
|
66
|
-
fs.mkdirSync(dirpath, { recursive: true });
|
|
67
|
-
|
|
68
|
-
if (!isActorAutomaticallyDetected && !(fs.readdirSync(dirpath).length === 0)) {
|
|
69
|
-
error(`Directory ${dirpath} is not empty. Please empty it or choose another directory.`);
|
|
70
|
-
return;
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
switch (correctVersion.sourceType) {
|
|
74
|
-
case 'TARBALL': {
|
|
75
|
-
await extractGitHubZip(correctVersion.tarballUrl, dirpath);
|
|
76
|
-
|
|
77
|
-
break;
|
|
78
|
-
}
|
|
79
|
-
case 'SOURCE_FILES': {
|
|
80
|
-
const { sourceFiles } = correctVersion;
|
|
81
|
-
for (const file of sourceFiles) {
|
|
82
|
-
const folderPath = path.dirname(file.name);
|
|
83
|
-
fs.mkdirSync(`${dirpath}/${folderPath}`, { recursive: true }, null);
|
|
84
|
-
|
|
85
|
-
if (!file.folder) {
|
|
86
|
-
const fileContent = file.format === 'BASE64' ? Buffer.from(file.content, 'base64') : file.content;
|
|
87
|
-
|
|
88
|
-
if (file.name === LOCAL_CONFIG_PATH) {
|
|
89
|
-
const actorJson = jju.parse(fileContent);
|
|
90
|
-
actorJson.name = actor.name;
|
|
91
|
-
fs.writeFileSync(`${dirpath}/${file.name}`, jju.update(fileContent, actorJson));
|
|
92
|
-
} else {
|
|
93
|
-
fs.writeFileSync(`${dirpath}/${file.name}`, fileContent);
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
break;
|
|
99
|
-
}
|
|
100
|
-
case 'GIT_REPO': {
|
|
101
|
-
// e.g. https://github.com/jakubbalada/Datasety.git#master:RejstrikPolitickychStran
|
|
102
|
-
const { gitRepoUrl } = correctVersion;
|
|
103
|
-
const [repoUrl, branchDirPart] = gitRepoUrl.split('#');
|
|
104
|
-
|
|
105
|
-
let branch;
|
|
106
|
-
let dir;
|
|
107
|
-
if (branchDirPart) [branch, dir] = branchDirPart.split(':');
|
|
108
|
-
let branchDirRepoUrl = repoUrl;
|
|
109
|
-
if (dir) branchDirRepoUrl += `/${dir}`;
|
|
110
|
-
if (branch) branchDirRepoUrl += `#${branch}`;
|
|
111
|
-
|
|
112
|
-
const emitter = tiged(branchDirRepoUrl);
|
|
113
|
-
try {
|
|
114
|
-
await emitter.clone(dirpath);
|
|
115
|
-
} catch (err) {
|
|
116
|
-
throw new Error(`Failed to pull Actor from ${gitRepoUrl}. ${err.message}`);
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
break;
|
|
120
|
-
}
|
|
121
|
-
case 'GITHUB_GIST': {
|
|
122
|
-
await extractGitHubZip(`${correctVersion.gitHubGistUrl}/archive/master.zip`, dirpath);
|
|
123
|
-
|
|
124
|
-
break;
|
|
125
|
-
}
|
|
126
|
-
default:
|
|
127
|
-
throw new Error(`Unknown source type: ${correctVersion.sourceType}`);
|
|
128
|
-
}
|
|
129
|
-
|
|
130
|
-
success(isActorAutomaticallyDetected ? `Actor ${name} updated at ${dirpath}/` : `Pulled to ${dirpath}/`);
|
|
131
|
-
}
|
|
132
|
-
}
|
|
133
|
-
|
|
134
|
-
PullCommand.description = 'Pulls an Actor from the Apify platform to the current directory. '
|
|
135
|
-
+ 'If it is defined as Git repository, it will be cloned. If it is defined as Web IDE, it will fetch the files.';
|
|
136
|
-
|
|
137
|
-
PullCommand.flags = {
|
|
138
|
-
version: flagsHelper.string({
|
|
139
|
-
char: 'v',
|
|
140
|
-
description: 'Actor version number which will be pulled, e.g. 1.2. Default: the highest version',
|
|
141
|
-
required: false,
|
|
142
|
-
}),
|
|
143
|
-
};
|
|
144
|
-
|
|
145
|
-
PullCommand.args = [
|
|
146
|
-
{
|
|
147
|
-
name: 'actorId',
|
|
148
|
-
required: false,
|
|
149
|
-
description: 'Name or ID of the actor to run (e.g. "apify/hello-world" or "E2jjCZBezvAZnX8Rb"). '
|
|
150
|
-
+ 'If not provided, the command will update the Actor in the current directory based on its name in ".actor/actor.json" file.',
|
|
151
|
-
},
|
|
152
|
-
];
|
|
153
|
-
|
|
154
|
-
module.exports = PullCommand;
|
package/src/commands/push.js
DELETED
|
@@ -1,234 +0,0 @@
|
|
|
1
|
-
const fs = require('fs');
|
|
2
|
-
|
|
3
|
-
const actorTemplates = require('@apify/actor-templates');
|
|
4
|
-
const { ACT_JOB_STATUSES, ACT_SOURCE_TYPES,
|
|
5
|
-
MAX_MULTIFILE_BYTES } = require('@apify/consts');
|
|
6
|
-
const { flags: flagsHelper } = require('@oclif/command');
|
|
7
|
-
const inquirer = require('inquirer');
|
|
8
|
-
const isCI = require('is-ci');
|
|
9
|
-
const open = require('open');
|
|
10
|
-
|
|
11
|
-
const { ApifyCommand } = require('../lib/apify_command');
|
|
12
|
-
const { UPLOADS_STORE_NAME, LOCAL_CONFIG_PATH } = require('../lib/consts');
|
|
13
|
-
const { sumFilesSizeInBytes } = require('../lib/files');
|
|
14
|
-
const outputs = require('../lib/outputs');
|
|
15
|
-
const { transformEnvToEnvVars } = require('../lib/secrets');
|
|
16
|
-
const { createActZip, getLoggedClientOrThrow,
|
|
17
|
-
outputJobLog, getLocalUserInfo, getActorLocalFilePaths,
|
|
18
|
-
createSourceFiles, getLocalConfigOrThrow } = require('../lib/utils');
|
|
19
|
-
|
|
20
|
-
const TEMP_ZIP_FILE_NAME = 'temp_file.zip';
|
|
21
|
-
const DEFAULT_RUN_OPTIONS = {
|
|
22
|
-
build: 'latest',
|
|
23
|
-
memoryMbytes: 4096,
|
|
24
|
-
timeoutSecs: 3600,
|
|
25
|
-
};
|
|
26
|
-
const DEFAULT_ACTOR_VERSION_NUMBER = '0.0';
|
|
27
|
-
|
|
28
|
-
// It would be better to use `version-0.0` or similar,
|
|
29
|
-
// or even have no default tag, but the platform complains when
|
|
30
|
-
// actor does not have a build with a `latest` tag, so until
|
|
31
|
-
// that changes, we have to add it.
|
|
32
|
-
const DEFAULT_BUILD_TAG = 'latest';
|
|
33
|
-
|
|
34
|
-
class PushCommand extends ApifyCommand {
|
|
35
|
-
async run() {
|
|
36
|
-
const { args, flags } = this.parse(PushCommand);
|
|
37
|
-
const apifyClient = await getLoggedClientOrThrow();
|
|
38
|
-
const localConfig = await getLocalConfigOrThrow();
|
|
39
|
-
const userInfo = await getLocalUserInfo();
|
|
40
|
-
const isOrganizationLoggedIn = !!userInfo.organizationOwnerUserId;
|
|
41
|
-
const redirectUrlPart = isOrganizationLoggedIn ? `/organization/${userInfo.id}` : '';
|
|
42
|
-
|
|
43
|
-
let actorId;
|
|
44
|
-
let actor;
|
|
45
|
-
// User can override actor version and build tag, attributes in localConfig will remain same.
|
|
46
|
-
const version = flags.version || flags.versionNumber || localConfig.version || DEFAULT_ACTOR_VERSION_NUMBER;
|
|
47
|
-
let buildTag = flags.buildTag || localConfig.buildTag;
|
|
48
|
-
// We can't add the default build tag to everything. If a user creates a new
|
|
49
|
-
// version, e.g. for testing, but forgets to add a tag, it would use the default
|
|
50
|
-
// tag and their production runs might be affected ❌
|
|
51
|
-
// TODO: revisit this when we have better build tagging system on platform.
|
|
52
|
-
if (!buildTag && version === DEFAULT_ACTOR_VERSION_NUMBER) {
|
|
53
|
-
buildTag = DEFAULT_BUILD_TAG;
|
|
54
|
-
}
|
|
55
|
-
const waitForFinishMillis = Number.isNaN(flags.waitForFinish)
|
|
56
|
-
? undefined
|
|
57
|
-
: parseInt(flags.waitForFinish, 10) * 1000;
|
|
58
|
-
// User can override actorId of pushing actor.
|
|
59
|
-
// It causes that we push actor to this id but attributes in localConfig will remain same.
|
|
60
|
-
const forceActorId = args.actorId;
|
|
61
|
-
if (forceActorId) {
|
|
62
|
-
actor = await apifyClient.actor(forceActorId).get();
|
|
63
|
-
if (!actor) throw new Error(`Cannot find actor with ID '${forceActorId}' in your account.`);
|
|
64
|
-
actorId = actor.id;
|
|
65
|
-
} else {
|
|
66
|
-
const usernameOrId = userInfo.username || userInfo.id;
|
|
67
|
-
actor = await apifyClient.actor(`${usernameOrId}/${localConfig.name}`).get();
|
|
68
|
-
if (actor) {
|
|
69
|
-
actorId = actor.id;
|
|
70
|
-
} else {
|
|
71
|
-
const { templates } = await actorTemplates.fetchManifest();
|
|
72
|
-
const actorTemplate = templates.find((t) => t.name === localConfig.template);
|
|
73
|
-
const defaultRunOptions = actorTemplate?.defaultRunOptions || DEFAULT_RUN_OPTIONS;
|
|
74
|
-
const newActor = {
|
|
75
|
-
name: localConfig.name,
|
|
76
|
-
defaultRunOptions,
|
|
77
|
-
versions: [{
|
|
78
|
-
versionNumber: version,
|
|
79
|
-
buildTag,
|
|
80
|
-
sourceType: ACT_SOURCE_TYPES.SOURCE_FILES,
|
|
81
|
-
sourceFiles: [],
|
|
82
|
-
}],
|
|
83
|
-
};
|
|
84
|
-
actor = await apifyClient.actors().create(newActor);
|
|
85
|
-
actorId = actor.id;
|
|
86
|
-
outputs.info(`Created actor with name ${localConfig.name} on Apify.`);
|
|
87
|
-
}
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
outputs.info(`Deploying actor '${localConfig.name}' to Apify.`);
|
|
91
|
-
|
|
92
|
-
const filePathsToPush = await getActorLocalFilePaths();
|
|
93
|
-
const filesSize = await sumFilesSizeInBytes(filePathsToPush);
|
|
94
|
-
const actorClient = apifyClient.actor(actorId);
|
|
95
|
-
|
|
96
|
-
let sourceType;
|
|
97
|
-
let sourceFiles;
|
|
98
|
-
let tarballUrl;
|
|
99
|
-
if (filesSize < MAX_MULTIFILE_BYTES) {
|
|
100
|
-
sourceFiles = await createSourceFiles(filePathsToPush);
|
|
101
|
-
sourceType = ACT_SOURCE_TYPES.SOURCE_FILES;
|
|
102
|
-
} else {
|
|
103
|
-
// Create zip
|
|
104
|
-
outputs.run('Zipping actor files');
|
|
105
|
-
await createActZip(TEMP_ZIP_FILE_NAME, filePathsToPush);
|
|
106
|
-
|
|
107
|
-
// Upload it to Apify.keyValueStores
|
|
108
|
-
const store = await apifyClient.keyValueStores().getOrCreate(UPLOADS_STORE_NAME);
|
|
109
|
-
const key = `${actor.name}-${version}.zip`;
|
|
110
|
-
const buffer = fs.readFileSync(TEMP_ZIP_FILE_NAME);
|
|
111
|
-
await apifyClient.keyValueStore(store.id).setRecord({
|
|
112
|
-
key,
|
|
113
|
-
value: buffer,
|
|
114
|
-
contentType: 'application/zip',
|
|
115
|
-
});
|
|
116
|
-
fs.unlinkSync(TEMP_ZIP_FILE_NAME);
|
|
117
|
-
tarballUrl = `${apifyClient.baseUrl}/key-value-stores/${store.id}/records/${key}?disableRedirect=true`;
|
|
118
|
-
sourceType = ACT_SOURCE_TYPES.TARBALL;
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
// Update actor version
|
|
122
|
-
const actorCurrentVersion = await actorClient.version(version).get();
|
|
123
|
-
const envVars = localConfig.environmentVariables
|
|
124
|
-
? transformEnvToEnvVars(localConfig.environmentVariables)
|
|
125
|
-
: undefined;
|
|
126
|
-
if (actorCurrentVersion) {
|
|
127
|
-
const actorVersionModifier = { tarballUrl, sourceFiles, buildTag, sourceType, envVars };
|
|
128
|
-
await actorClient.version(version).update(actorVersionModifier);
|
|
129
|
-
outputs.run(`Updated version ${version} for ${actor.name} actor.`);
|
|
130
|
-
} else {
|
|
131
|
-
const actorNewVersion = {
|
|
132
|
-
versionNumber: version,
|
|
133
|
-
tarballUrl,
|
|
134
|
-
sourceFiles,
|
|
135
|
-
buildTag,
|
|
136
|
-
sourceType,
|
|
137
|
-
envVars,
|
|
138
|
-
};
|
|
139
|
-
await actorClient.versions().create({
|
|
140
|
-
versionNumber: version,
|
|
141
|
-
...actorNewVersion,
|
|
142
|
-
});
|
|
143
|
-
outputs.run(`Created version ${version} for ${actor.name} actor.`);
|
|
144
|
-
}
|
|
145
|
-
|
|
146
|
-
// Build actor on Apify and wait for build to finish
|
|
147
|
-
outputs.run(`Building actor ${actor.name}`);
|
|
148
|
-
let build = await actorClient.build(version, {
|
|
149
|
-
useCache: true,
|
|
150
|
-
waitForFinish: 2, // NOTE: We need to wait some time to Apify open stream and we can create connection
|
|
151
|
-
});
|
|
152
|
-
|
|
153
|
-
try {
|
|
154
|
-
await outputJobLog(build, waitForFinishMillis);
|
|
155
|
-
} catch (err) {
|
|
156
|
-
outputs.warning('Can not get log:');
|
|
157
|
-
console.error(err);
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
build = await apifyClient.build(build.id).get();
|
|
161
|
-
|
|
162
|
-
outputs.link('Actor build detail', `https://console.apify.com${redirectUrlPart}/actors/${build.actId}#/builds/${build.buildNumber}`);
|
|
163
|
-
|
|
164
|
-
// Disable open browser on CI, or if user passed --no-prompt flag
|
|
165
|
-
if (!isCI && !flags.noPrompt) {
|
|
166
|
-
const shouldOpenBrowser = await inquirer.prompt([
|
|
167
|
-
{ type: 'confirm', name: 'continue', message: 'Do you want to open the actor detail in your browser?', default: true },
|
|
168
|
-
]);
|
|
169
|
-
|
|
170
|
-
if (shouldOpenBrowser.continue) {
|
|
171
|
-
open(`https://console.apify.com${redirectUrlPart}/actors/${build.actId}`);
|
|
172
|
-
}
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
if (build.status === ACT_JOB_STATUSES.SUCCEEDED) {
|
|
176
|
-
outputs.success('Actor was deployed to Apify cloud and built there.');
|
|
177
|
-
} else if (build.status === ACT_JOB_STATUSES.READY) {
|
|
178
|
-
outputs.warning('Build is waiting for allocation.');
|
|
179
|
-
} else if (build.status === ACT_JOB_STATUSES.RUNNING) {
|
|
180
|
-
outputs.warning('Build is still running.');
|
|
181
|
-
} else if (build.status === ACT_JOB_STATUSES.ABORTED || build.status === ACT_JOB_STATUSES.ABORTING) {
|
|
182
|
-
outputs.warning('Build was aborted!');
|
|
183
|
-
} else if (build.status === ACT_JOB_STATUSES.TIMED_OUT || build.status === ACT_JOB_STATUSES.TIMING_OUT) {
|
|
184
|
-
outputs.warning('Build timed out!');
|
|
185
|
-
} else {
|
|
186
|
-
outputs.error('Build failed!');
|
|
187
|
-
}
|
|
188
|
-
}
|
|
189
|
-
}
|
|
190
|
-
|
|
191
|
-
PushCommand.description = 'Uploads the actor to the Apify platform and builds it there.\n'
|
|
192
|
-
+ `The Actor settings are read from the "${LOCAL_CONFIG_PATH}" file in the current directory, but they can be overridden using command-line options.\n`
|
|
193
|
-
+ `NOTE: If the source files are smaller than ${MAX_MULTIFILE_BYTES / (1024 ** 2)} MB then they are uploaded as \n`
|
|
194
|
-
+ '"Multiple source files", otherwise they are uploaded as "Zip file".\n\n'
|
|
195
|
-
+ 'WARNING: If the target Actor already exists in your Apify account, it will be overwritten!';
|
|
196
|
-
|
|
197
|
-
PushCommand.flags = {
|
|
198
|
-
'version-number': flagsHelper.string({
|
|
199
|
-
description: 'DEPRECATED: Use flag version instead. Actor version number to which the files should be pushed. '
|
|
200
|
-
+ `By default, it is taken from the "${LOCAL_CONFIG_PATH}" file.`,
|
|
201
|
-
required: false,
|
|
202
|
-
}),
|
|
203
|
-
version: flagsHelper.string({
|
|
204
|
-
char: 'v',
|
|
205
|
-
description: `Actor version number to which the files should be pushed. By default, it is taken from the "${LOCAL_CONFIG_PATH}" file.`,
|
|
206
|
-
required: false,
|
|
207
|
-
}),
|
|
208
|
-
'build-tag': flagsHelper.string({
|
|
209
|
-
char: 'b',
|
|
210
|
-
description: `Build tag to be applied to the successful Actor build. By default, it is taken from the "${LOCAL_CONFIG_PATH}" file`,
|
|
211
|
-
required: false,
|
|
212
|
-
}),
|
|
213
|
-
'wait-for-finish': flagsHelper.string({
|
|
214
|
-
char: 'w',
|
|
215
|
-
description: 'Seconds for waiting to build to finish, if no value passed, it waits forever.',
|
|
216
|
-
required: false,
|
|
217
|
-
}),
|
|
218
|
-
'no-prompt': flagsHelper.boolean({
|
|
219
|
-
description: 'Do not prompt for opening the actor details in a browser. This will also not open the browser automatically.',
|
|
220
|
-
default: false,
|
|
221
|
-
required: false,
|
|
222
|
-
}),
|
|
223
|
-
};
|
|
224
|
-
|
|
225
|
-
PushCommand.args = [
|
|
226
|
-
{
|
|
227
|
-
name: 'actorId',
|
|
228
|
-
required: false,
|
|
229
|
-
description: 'Name or ID of the Actor to push (e.g. "apify/hello-world" or "E2jjCZBezvAZnX8Rb"). '
|
|
230
|
-
+ `If not provided, the command will create or modify the actor with the name specified in "${LOCAL_CONFIG_PATH}" file.`,
|
|
231
|
-
},
|
|
232
|
-
];
|
|
233
|
-
|
|
234
|
-
module.exports = PushCommand;
|
package/src/commands/run.js
DELETED
|
@@ -1,206 +0,0 @@
|
|
|
1
|
-
const fs = require('fs');
|
|
2
|
-
const path = require('path');
|
|
3
|
-
|
|
4
|
-
const { APIFY_ENV_VARS } = require('@apify/consts');
|
|
5
|
-
const { flags: flagsHelper } = require('@oclif/command');
|
|
6
|
-
const loadJson = require('load-json-file');
|
|
7
|
-
const semver = require('semver');
|
|
8
|
-
|
|
9
|
-
const { ApifyCommand } = require('../lib/apify_command');
|
|
10
|
-
const { LEGACY_LOCAL_STORAGE_DIR, DEFAULT_LOCAL_STORAGE_DIR, SUPPORTED_NODEJS_VERSION, LANGUAGE, PROJECT_TYPES } = require('../lib/consts');
|
|
11
|
-
const execWithLog = require('../lib/exec');
|
|
12
|
-
const { error, info, warning } = require('../lib/outputs');
|
|
13
|
-
const { ProjectAnalyzer } = require('../lib/project_analyzer');
|
|
14
|
-
const { ScrapyProjectAnalyzer } = require('../lib/scrapy-wrapper/ScrapyProjectAnalyzer');
|
|
15
|
-
const { replaceSecretsValue } = require('../lib/secrets');
|
|
16
|
-
const {
|
|
17
|
-
getLocalUserInfo, purgeDefaultQueue, purgeDefaultKeyValueStore,
|
|
18
|
-
purgeDefaultDataset, getLocalConfigOrThrow, getNpmCmd, checkIfStorageIsEmpty,
|
|
19
|
-
detectLocalActorLanguage, isPythonVersionSupported, getPythonCommand, isNodeVersionSupported, getLocalStorageDir,
|
|
20
|
-
} = require('../lib/utils');
|
|
21
|
-
|
|
22
|
-
class RunCommand extends ApifyCommand {
|
|
23
|
-
async run() {
|
|
24
|
-
const { flags } = this.parse(RunCommand);
|
|
25
|
-
const { proxy, id: userId, token } = getLocalUserInfo();
|
|
26
|
-
const localConfig = await getLocalConfigOrThrow();
|
|
27
|
-
const cwd = process.cwd();
|
|
28
|
-
|
|
29
|
-
const packageJsonPath = path.join(cwd, 'package.json');
|
|
30
|
-
const mainPyPath = path.join(cwd, 'src/__main__.py');
|
|
31
|
-
|
|
32
|
-
const projectType = ProjectAnalyzer.getProjectType(cwd);
|
|
33
|
-
const actualStoragePath = getLocalStorageDir();
|
|
34
|
-
|
|
35
|
-
const packageJsonExists = fs.existsSync(packageJsonPath);
|
|
36
|
-
const mainPyExists = fs.existsSync(mainPyPath);
|
|
37
|
-
const isScrapyProject = projectType === PROJECT_TYPES.SCRAPY;
|
|
38
|
-
const { language, languageVersion } = detectLocalActorLanguage();
|
|
39
|
-
|
|
40
|
-
if (!packageJsonExists && !mainPyExists && !isScrapyProject) {
|
|
41
|
-
throw new Error(
|
|
42
|
-
'Actor is of an unknown format.'
|
|
43
|
-
+ ` Make sure either the 'package.json' file or 'src/__main__.py' file exists or you are in a migrated Scrapy project.`,
|
|
44
|
-
);
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
if (fs.existsSync(LEGACY_LOCAL_STORAGE_DIR) && !fs.existsSync(actualStoragePath)) {
|
|
48
|
-
fs.renameSync(LEGACY_LOCAL_STORAGE_DIR, actualStoragePath);
|
|
49
|
-
warning(`The legacy 'apify_storage' directory was renamed to '${actualStoragePath}' to align it with Apify SDK v3.`
|
|
50
|
-
+ ' Contents were left intact.');
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
let CRAWLEE_PURGE_ON_START = '0';
|
|
54
|
-
|
|
55
|
-
// Purge stores
|
|
56
|
-
if (flags.purge) {
|
|
57
|
-
switch (projectType) {
|
|
58
|
-
case PROJECT_TYPES.PRE_CRAWLEE_APIFY_SDK: {
|
|
59
|
-
await Promise.all([purgeDefaultQueue(), purgeDefaultKeyValueStore(), purgeDefaultDataset()]);
|
|
60
|
-
info('All default local stores were purged.');
|
|
61
|
-
break;
|
|
62
|
-
}
|
|
63
|
-
case PROJECT_TYPES.CRAWLEE:
|
|
64
|
-
default: {
|
|
65
|
-
CRAWLEE_PURGE_ON_START = '1';
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
if (language === LANGUAGE.PYTHON) {
|
|
70
|
-
await Promise.all([purgeDefaultQueue(), purgeDefaultKeyValueStore(), purgeDefaultDataset()]);
|
|
71
|
-
info('All default local stores were purged.');
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
// TODO: deprecate these flags
|
|
76
|
-
if (flags.purgeQueue && !flags.purge) {
|
|
77
|
-
await purgeDefaultQueue();
|
|
78
|
-
info('Default local request queue was purged.');
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
if (flags.purgeDataset && !flags.purge) {
|
|
82
|
-
await purgeDefaultDataset();
|
|
83
|
-
info('Default local dataset was purged.');
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
if (flags.purgeKeyValueStore && !flags.purge) {
|
|
87
|
-
await purgeDefaultKeyValueStore();
|
|
88
|
-
info('Default local key-value store was purged.');
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
if (!flags.purge) {
|
|
92
|
-
const isStorageEmpty = await checkIfStorageIsEmpty();
|
|
93
|
-
if (!isStorageEmpty) {
|
|
94
|
-
warning('The storage directory contains a previous state, the actor will continue where it left off. '
|
|
95
|
-
+ 'To start from the initial state, use --purge parameter to clean the storage directory.');
|
|
96
|
-
}
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
// Attach env vars from local config files
|
|
100
|
-
const localEnvVars = {
|
|
101
|
-
[APIFY_ENV_VARS.LOCAL_STORAGE_DIR]: actualStoragePath,
|
|
102
|
-
CRAWLEE_STORAGE_DIR: actualStoragePath,
|
|
103
|
-
CRAWLEE_PURGE_ON_START,
|
|
104
|
-
};
|
|
105
|
-
if (proxy && proxy.password) localEnvVars[APIFY_ENV_VARS.PROXY_PASSWORD] = proxy.password;
|
|
106
|
-
if (userId) localEnvVars[APIFY_ENV_VARS.USER_ID] = userId;
|
|
107
|
-
if (token) localEnvVars[APIFY_ENV_VARS.TOKEN] = token;
|
|
108
|
-
if (localConfig.environmentVariables) {
|
|
109
|
-
const updatedEnv = replaceSecretsValue(localConfig.environmentVariables);
|
|
110
|
-
Object.assign(localEnvVars, updatedEnv);
|
|
111
|
-
}
|
|
112
|
-
// NOTE: User can overwrite env vars
|
|
113
|
-
const env = Object.assign(localEnvVars, process.env);
|
|
114
|
-
|
|
115
|
-
if (!userId) {
|
|
116
|
-
warning('You are not logged in with your Apify Account. Some features like Apify Proxy will not work. Call "apify login" to fix that.');
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
if (language === LANGUAGE.NODEJS) { // Actor is written in Node.js
|
|
120
|
-
const currentNodeVersion = languageVersion;
|
|
121
|
-
const minimumSupportedNodeVersion = semver.minVersion(SUPPORTED_NODEJS_VERSION);
|
|
122
|
-
if (currentNodeVersion) {
|
|
123
|
-
const serverJsFile = path.join(cwd, 'server.js');
|
|
124
|
-
const packageJson = await loadJson(packageJsonPath);
|
|
125
|
-
if ((!packageJson.scripts || !packageJson.scripts.start) && !fs.existsSync(serverJsFile)) {
|
|
126
|
-
throw new Error('The "npm start" script was not found in package.json. Please set it up for your project. '
|
|
127
|
-
+ 'For more information about that call "apify help run".');
|
|
128
|
-
}
|
|
129
|
-
|
|
130
|
-
// --max-http-header-size=80000
|
|
131
|
-
// Increases default size of headers. The original limit was 80kb, but from node 10+ they decided to lower it to 8kb.
|
|
132
|
-
// However they did not think about all the sites there with large headers,
|
|
133
|
-
// so we put back the old limit of 80kb, which seems to work just fine.
|
|
134
|
-
if (isNodeVersionSupported(currentNodeVersion)) {
|
|
135
|
-
env.NODE_OPTIONS = env.NODE_OPTIONS ? `${env.NODE_OPTIONS} --max-http-header-size=80000` : '--max-http-header-size=80000';
|
|
136
|
-
} else {
|
|
137
|
-
warning(`You are running Node.js version ${currentNodeVersion}, which is no longer supported. `
|
|
138
|
-
+ `Please upgrade to Node.js version ${minimumSupportedNodeVersion} or later.`);
|
|
139
|
-
}
|
|
140
|
-
this.telemetryData.actorNodejsVersion = currentNodeVersion;
|
|
141
|
-
this.telemetryData.actorLanguage = LANGUAGE.NODEJS;
|
|
142
|
-
await execWithLog(getNpmCmd(), ['start'], { env });
|
|
143
|
-
} else {
|
|
144
|
-
error(`No Node.js detected! Please install Node.js ${minimumSupportedNodeVersion} or higher to be able to run Node.js actors locally.`);
|
|
145
|
-
}
|
|
146
|
-
} else if (language === LANGUAGE.PYTHON) {
|
|
147
|
-
const pythonVersion = languageVersion;
|
|
148
|
-
this.telemetryData.actorPythonVersion = pythonVersion;
|
|
149
|
-
this.telemetryData.actorLanguage = LANGUAGE.PYTHON;
|
|
150
|
-
if (pythonVersion) {
|
|
151
|
-
if (isPythonVersionSupported(pythonVersion)) {
|
|
152
|
-
const pythonCommand = getPythonCommand(cwd);
|
|
153
|
-
let executableLocation = 'src';
|
|
154
|
-
|
|
155
|
-
if (isScrapyProject) {
|
|
156
|
-
const project = new ScrapyProjectAnalyzer(cwd);
|
|
157
|
-
project.loadScrapyCfg();
|
|
158
|
-
if (project.configuration.hasKey('apify', 'mainpy_location')) {
|
|
159
|
-
executableLocation = project.configuration.get('apify', 'mainpy_location');
|
|
160
|
-
}
|
|
161
|
-
}
|
|
162
|
-
|
|
163
|
-
await execWithLog(pythonCommand, ['-m', executableLocation], { env });
|
|
164
|
-
} else {
|
|
165
|
-
error(`Python actors require Python 3.8 or higher, but you have Python ${pythonVersion}!`);
|
|
166
|
-
error('Please install Python 3.8 or higher to be able to run Python actors locally.');
|
|
167
|
-
}
|
|
168
|
-
} else {
|
|
169
|
-
error('No Python detected! Please install Python 3.8 or higher to be able to run Python actors locally.');
|
|
170
|
-
}
|
|
171
|
-
}
|
|
172
|
-
}
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
// TODO: we should describe which env vars are set here:
|
|
176
|
-
|
|
177
|
-
RunCommand.description = 'Runs the actor locally in the current directory.\n'
|
|
178
|
-
+ 'It sets various APIFY_XYZ environment variables '
|
|
179
|
-
+ 'in order to provide a working execution environment for the actor. For example, this causes '
|
|
180
|
-
+ 'the actor input, as well as all other data in key-value stores, '
|
|
181
|
-
+ `datasets or request queues to be stored in the "${DEFAULT_LOCAL_STORAGE_DIR}" directory, `
|
|
182
|
-
+ 'rather than on the Apify platform.\n\n'
|
|
183
|
-
+ 'NOTE: You can override the command\'s default behavior for Node.js actors by overriding the "start" script in the package.json file. '
|
|
184
|
-
+ 'You can set up your own main file or environment variables by changing it.';
|
|
185
|
-
|
|
186
|
-
RunCommand.flags = {
|
|
187
|
-
purge: flagsHelper.boolean({
|
|
188
|
-
char: 'p',
|
|
189
|
-
description: 'Shortcut that combines the --purge-queue, --purge-dataset and --purge-key-value-store options.',
|
|
190
|
-
required: false,
|
|
191
|
-
}),
|
|
192
|
-
'purge-queue': flagsHelper.boolean({
|
|
193
|
-
description: 'Deletes the local directory containing the default request queue before the run starts.',
|
|
194
|
-
required: false,
|
|
195
|
-
}),
|
|
196
|
-
'purge-dataset': flagsHelper.boolean({
|
|
197
|
-
description: 'Deletes the local directory containing the default dataset before the run starts.',
|
|
198
|
-
required: false,
|
|
199
|
-
}),
|
|
200
|
-
'purge-key-value-store': flagsHelper.boolean({
|
|
201
|
-
description: 'Deletes all records from the default key-value store in the local directory before the run starts, except for the "INPUT" key.',
|
|
202
|
-
required: false,
|
|
203
|
-
}),
|
|
204
|
-
};
|
|
205
|
-
|
|
206
|
-
module.exports = RunCommand;
|
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
const { ApifyCommand } = require('../../lib/apify_command');
|
|
2
|
-
const { addSecret } = require('../../lib/secrets');
|
|
3
|
-
|
|
4
|
-
class AddCommand extends ApifyCommand {
|
|
5
|
-
async run() {
|
|
6
|
-
const { args } = this.parse(AddCommand);
|
|
7
|
-
const { name, value } = args;
|
|
8
|
-
|
|
9
|
-
addSecret(name, value);
|
|
10
|
-
}
|
|
11
|
-
}
|
|
12
|
-
|
|
13
|
-
AddCommand.description = 'Adds a new secret value.\nThe secrets are stored to a file at ~/.apify';
|
|
14
|
-
|
|
15
|
-
AddCommand.args = [
|
|
16
|
-
{
|
|
17
|
-
name: 'name',
|
|
18
|
-
required: true,
|
|
19
|
-
description: 'Name of the secret',
|
|
20
|
-
},
|
|
21
|
-
{
|
|
22
|
-
name: 'value',
|
|
23
|
-
required: true,
|
|
24
|
-
description: 'Value of the secret',
|
|
25
|
-
},
|
|
26
|
-
];
|
|
27
|
-
|
|
28
|
-
module.exports = AddCommand;
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
const { Command } = require('@oclif/command');
|
|
2
|
-
|
|
3
|
-
const { LOCAL_CONFIG_PATH } = require('../../lib/consts');
|
|
4
|
-
const { showHelpForCommand } = require('../../lib/utils');
|
|
5
|
-
|
|
6
|
-
class SecretsIndexCommand extends Command {
|
|
7
|
-
async run() {
|
|
8
|
-
showHelpForCommand('secrets');
|
|
9
|
-
}
|
|
10
|
-
}
|
|
11
|
-
|
|
12
|
-
SecretsIndexCommand.description = 'Manages secret values for actor environment variables.\n\n'
|
|
13
|
-
+ 'Example:\n'
|
|
14
|
-
+ '$ apify secrets:add mySecret TopSecretValue123\n\n'
|
|
15
|
-
+ `Now the "mySecret" value can be used in an environment variable defined in "${LOCAL_CONFIG_PATH}" file by adding the "@" prefix:\n\n`
|
|
16
|
-
+ '{\n'
|
|
17
|
-
+ ' "actorSpecification": 1,\n'
|
|
18
|
-
+ ' "name": "my_actor",\n'
|
|
19
|
-
+ ' "environmentVariables": { "SECRET_ENV_VAR": "@mySecret" },\n'
|
|
20
|
-
+ ' "version": "0.1\n'
|
|
21
|
-
+ '}\n\n'
|
|
22
|
-
+ 'When the actor is pushed to Apify cloud, the "SECRET_ENV_VAR" and its value is stored as a secret environment variable of the actor.';
|
|
23
|
-
|
|
24
|
-
module.exports = SecretsIndexCommand;
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
const { ApifyCommand } = require('../../lib/apify_command');
|
|
2
|
-
const { removeSecret } = require('../../lib/secrets');
|
|
3
|
-
|
|
4
|
-
class RmCommand extends ApifyCommand {
|
|
5
|
-
async run() {
|
|
6
|
-
const { args } = this.parse(RmCommand);
|
|
7
|
-
const { name } = args;
|
|
8
|
-
|
|
9
|
-
removeSecret(name);
|
|
10
|
-
}
|
|
11
|
-
}
|
|
12
|
-
|
|
13
|
-
RmCommand.description = 'Removes the secret.';
|
|
14
|
-
|
|
15
|
-
RmCommand.args = [
|
|
16
|
-
{
|
|
17
|
-
name: 'name',
|
|
18
|
-
required: true,
|
|
19
|
-
description: 'Name of the secret',
|
|
20
|
-
},
|
|
21
|
-
];
|
|
22
|
-
|
|
23
|
-
module.exports = RmCommand;
|