apify-cli 0.19.4 → 0.20.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +200 -0
- package/README.md +273 -167
- package/bin/dev.cmd +3 -0
- package/bin/dev.js +21 -0
- package/bin/dev.sh +4 -0
- package/bin/run.js +23 -0
- package/dist/.tsbuildinfo +1 -0
- package/dist/commands/actor/get-input.d.ts +6 -0
- package/dist/commands/actor/get-input.d.ts.map +1 -0
- package/dist/commands/actor/get-input.js +14 -0
- package/dist/commands/actor/get-input.js.map +1 -0
- package/dist/commands/actor/get-value.d.ts +9 -0
- package/dist/commands/actor/get-value.d.ts.map +1 -0
- package/dist/commands/actor/get-value.js +27 -0
- package/dist/commands/actor/get-value.js.map +1 -0
- package/dist/commands/actor/index.d.ts +6 -0
- package/dist/commands/actor/index.d.ts.map +1 -0
- package/dist/commands/actor/index.js +13 -0
- package/dist/commands/actor/index.js.map +1 -0
- package/dist/commands/actor/push-data.d.ts +11 -0
- package/dist/commands/actor/push-data.d.ts.map +1 -0
- package/dist/commands/actor/push-data.js +56 -0
- package/dist/commands/actor/push-data.js.map +1 -0
- package/dist/commands/actor/set-value.d.ts +15 -0
- package/dist/commands/actor/set-value.d.ts.map +1 -0
- package/dist/commands/actor/set-value.js +76 -0
- package/dist/commands/actor/set-value.js.map +1 -0
- package/dist/commands/call.d.ts +19 -0
- package/dist/commands/call.d.ts.map +1 -0
- package/dist/commands/call.js +193 -0
- package/dist/commands/call.js.map +1 -0
- package/dist/commands/check-version.d.ts +11 -0
- package/dist/commands/check-version.d.ts.map +1 -0
- package/dist/commands/check-version.js +39 -0
- package/dist/commands/check-version.js.map +1 -0
- package/dist/commands/create.d.ts +15 -0
- package/dist/commands/create.d.ts.map +1 -0
- package/dist/commands/create.js +201 -0
- package/dist/commands/create.js.map +1 -0
- package/dist/commands/edit-input-schema.d.ts +11 -0
- package/dist/commands/edit-input-schema.d.ts.map +1 -0
- package/{src → dist}/commands/edit-input-schema.js +81 -85
- package/dist/commands/edit-input-schema.js.map +1 -0
- package/dist/commands/info.d.ts +6 -0
- package/dist/commands/info.d.ts.map +1 -0
- package/dist/commands/info.js +26 -0
- package/dist/commands/info.js.map +1 -0
- package/dist/commands/init-wrap-scrapy.d.ts +10 -0
- package/dist/commands/init-wrap-scrapy.d.ts.map +1 -0
- package/dist/commands/init-wrap-scrapy.js +41 -0
- package/dist/commands/init-wrap-scrapy.js.map +1 -0
- package/dist/commands/init.d.ts +12 -0
- package/dist/commands/init.d.ts.map +1 -0
- package/dist/commands/init.js +88 -0
- package/dist/commands/init.js.map +1 -0
- package/dist/commands/login.d.ts +10 -0
- package/dist/commands/login.d.ts.map +1 -0
- package/dist/commands/login.js +174 -0
- package/dist/commands/login.js.map +1 -0
- package/dist/commands/logout.d.ts +6 -0
- package/dist/commands/logout.d.ts.map +1 -0
- package/dist/commands/logout.js +20 -0
- package/dist/commands/logout.js.map +1 -0
- package/dist/commands/pull.d.ts +12 -0
- package/dist/commands/pull.d.ts.map +1 -0
- package/dist/commands/pull.js +148 -0
- package/dist/commands/pull.js.map +1 -0
- package/dist/commands/push.d.ts +18 -0
- package/dist/commands/push.d.ts.map +1 -0
- package/dist/commands/push.js +298 -0
- package/dist/commands/push.js.map +1 -0
- package/dist/commands/run.d.ts +16 -0
- package/dist/commands/run.d.ts.map +1 -0
- package/dist/commands/run.js +383 -0
- package/dist/commands/run.js.map +1 -0
- package/dist/commands/secrets/add.d.ts +10 -0
- package/dist/commands/secrets/add.d.ts.map +1 -0
- package/dist/commands/secrets/add.js +31 -0
- package/dist/commands/secrets/add.js.map +1 -0
- package/dist/commands/secrets/index.d.ts +6 -0
- package/dist/commands/secrets/index.d.ts.map +1 -0
- package/dist/commands/secrets/index.js +24 -0
- package/dist/commands/secrets/index.js.map +1 -0
- package/dist/commands/secrets/rm.d.ts +9 -0
- package/dist/commands/secrets/rm.d.ts.map +1 -0
- package/dist/commands/secrets/rm.js +27 -0
- package/dist/commands/secrets/rm.js.map +1 -0
- package/dist/commands/task/index.d.ts +6 -0
- package/dist/commands/task/index.d.ts.map +1 -0
- package/dist/commands/task/index.js +13 -0
- package/dist/commands/task/index.js.map +1 -0
- package/dist/commands/task/run.d.ts +16 -0
- package/dist/commands/task/run.d.ts.map +1 -0
- package/dist/commands/task/run.js +91 -0
- package/dist/commands/task/run.js.map +1 -0
- package/dist/commands/validate-schema.d.ts +10 -0
- package/dist/commands/validate-schema.d.ts.map +1 -0
- package/dist/commands/validate-schema.js +58 -0
- package/dist/commands/validate-schema.js.map +1 -0
- package/dist/hooks/deprecations.d.ts +4 -0
- package/dist/hooks/deprecations.d.ts.map +1 -0
- package/dist/hooks/deprecations.js +18 -0
- package/dist/hooks/deprecations.js.map +1 -0
- package/dist/hooks/init.d.ts +7 -0
- package/dist/hooks/init.d.ts.map +1 -0
- package/dist/hooks/init.js +17 -0
- package/dist/hooks/init.js.map +1 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -0
- package/dist/lib/actor.d.ts +27 -0
- package/dist/lib/actor.d.ts.map +1 -0
- package/{src → dist}/lib/actor.js +22 -45
- package/dist/lib/actor.js.map +1 -0
- package/dist/lib/apify-oclif-help.d.ts +15 -0
- package/dist/lib/apify-oclif-help.d.ts.map +1 -0
- package/dist/lib/apify-oclif-help.js +19 -0
- package/dist/lib/apify-oclif-help.js.map +1 -0
- package/dist/lib/apify_command.d.ts +21 -0
- package/dist/lib/apify_command.d.ts.map +1 -0
- package/dist/lib/apify_command.js +104 -0
- package/dist/lib/apify_command.js.map +1 -0
- package/dist/lib/commands/resolve-input.d.ts +5 -0
- package/dist/lib/commands/resolve-input.d.ts.map +1 -0
- package/dist/lib/commands/resolve-input.js +29 -0
- package/dist/lib/commands/resolve-input.js.map +1 -0
- package/dist/lib/commands/run-on-cloud.d.ts +20 -0
- package/dist/lib/commands/run-on-cloud.d.ts.map +1 -0
- package/dist/lib/commands/run-on-cloud.js +89 -0
- package/dist/lib/commands/run-on-cloud.js.map +1 -0
- package/dist/lib/community.d.ts +2 -0
- package/dist/lib/community.d.ts.map +1 -0
- package/dist/lib/community.js +4 -0
- package/dist/lib/community.js.map +1 -0
- package/dist/lib/consts.d.ts +54 -0
- package/dist/lib/consts.d.ts.map +1 -0
- package/dist/lib/consts.js +65 -0
- package/dist/lib/consts.js.map +1 -0
- package/dist/lib/create-utils.d.ts +8 -0
- package/dist/lib/create-utils.d.ts.map +1 -0
- package/dist/lib/create-utils.js +148 -0
- package/dist/lib/create-utils.js.map +1 -0
- package/dist/lib/exec.d.ts +3 -0
- package/dist/lib/exec.d.ts.map +1 -0
- package/dist/lib/exec.js +40 -0
- package/dist/lib/exec.js.map +1 -0
- package/dist/lib/files.d.ts +14 -0
- package/dist/lib/files.d.ts.map +1 -0
- package/dist/lib/files.js +58 -0
- package/dist/lib/files.js.map +1 -0
- package/dist/lib/input_schema.d.ts +28 -0
- package/dist/lib/input_schema.d.ts.map +1 -0
- package/dist/lib/input_schema.js +126 -0
- package/dist/lib/input_schema.js.map +1 -0
- package/dist/lib/local_state.d.ts +11 -0
- package/dist/lib/local_state.d.ts.map +1 -0
- package/dist/lib/local_state.js +27 -0
- package/dist/lib/local_state.js.map +1 -0
- package/dist/lib/outputs.d.ts +19 -0
- package/dist/lib/outputs.d.ts.map +1 -0
- package/dist/lib/outputs.js +45 -0
- package/dist/lib/outputs.js.map +1 -0
- package/dist/lib/project_analyzer.d.ts +4 -0
- package/dist/lib/project_analyzer.d.ts.map +1 -0
- package/{src → dist}/lib/project_analyzer.js +6 -11
- package/dist/lib/project_analyzer.js.map +1 -0
- package/dist/lib/projects/CrawleeAnalyzer.d.ts +4 -0
- package/dist/lib/projects/CrawleeAnalyzer.d.ts.map +1 -0
- package/{src → dist}/lib/projects/CrawleeAnalyzer.js +7 -24
- package/dist/lib/projects/CrawleeAnalyzer.js.map +1 -0
- package/dist/lib/projects/OldApifySDKAnalyzer.d.ts +4 -0
- package/dist/lib/projects/OldApifySDKAnalyzer.d.ts.map +1 -0
- package/{src → dist}/lib/projects/OldApifySDKAnalyzer.js +9 -32
- package/dist/lib/projects/OldApifySDKAnalyzer.js.map +1 -0
- package/dist/lib/projects/scrapy/ScrapyProjectAnalyzer.d.ts +17 -0
- package/dist/lib/projects/scrapy/ScrapyProjectAnalyzer.d.ts.map +1 -0
- package/{src/lib/scrapy-wrapper → dist/lib/projects/scrapy}/ScrapyProjectAnalyzer.js +33 -36
- package/dist/lib/projects/scrapy/ScrapyProjectAnalyzer.js.map +1 -0
- package/dist/lib/projects/scrapy/Spider.d.ts +14 -0
- package/dist/lib/projects/scrapy/Spider.d.ts.map +1 -0
- package/dist/lib/projects/scrapy/Spider.js +33 -0
- package/dist/lib/projects/scrapy/Spider.js.map +1 -0
- package/dist/lib/projects/scrapy/SpiderFileAnalyzer.d.ts +7 -0
- package/dist/lib/projects/scrapy/SpiderFileAnalyzer.d.ts.map +1 -0
- package/dist/lib/projects/scrapy/SpiderFileAnalyzer.js +25 -0
- package/dist/lib/projects/scrapy/SpiderFileAnalyzer.js.map +1 -0
- package/dist/lib/projects/scrapy/wrapScrapyProject.d.ts +4 -0
- package/dist/lib/projects/scrapy/wrapScrapyProject.d.ts.map +1 -0
- package/{src/lib/scrapy-wrapper/index.js → dist/lib/projects/scrapy/wrapScrapyProject.js} +43 -73
- package/dist/lib/projects/scrapy/wrapScrapyProject.js.map +1 -0
- package/dist/lib/projects/shared.d.ts +2 -0
- package/dist/lib/projects/shared.d.ts.map +1 -0
- package/dist/lib/projects/shared.js +13 -0
- package/dist/lib/projects/shared.js.map +1 -0
- package/dist/lib/secrets.d.ts +22 -0
- package/dist/lib/secrets.d.ts.map +1 -0
- package/{src → dist}/lib/secrets.js +34 -42
- package/dist/lib/secrets.js.map +1 -0
- package/dist/lib/telemetry.d.ts +21 -0
- package/dist/lib/telemetry.d.ts.map +1 -0
- package/dist/lib/telemetry.js +87 -0
- package/dist/lib/telemetry.js.map +1 -0
- package/dist/lib/types.d.ts +11 -0
- package/dist/lib/types.d.ts.map +1 -0
- package/dist/lib/types.js +2 -0
- package/dist/lib/types.js.map +1 -0
- package/dist/lib/utils.d.ts +102 -0
- package/dist/lib/utils.d.ts.map +1 -0
- package/dist/lib/utils.js +577 -0
- package/dist/lib/utils.js.map +1 -0
- package/dist/lib/version_check.d.ts +14 -0
- package/dist/lib/version_check.d.ts.map +1 -0
- package/{src → dist}/lib/version_check.js +31 -60
- package/dist/lib/version_check.js.map +1 -0
- package/oclif.manifest.json +867 -1
- package/package.json +151 -109
- package/index.js +0 -1
- package/src/bin/run +0 -16
- package/src/commands/actor/get-input.js +0 -12
- package/src/commands/actor/get-value.js +0 -23
- package/src/commands/actor/index.js +0 -13
- package/src/commands/actor/push-data.js +0 -44
- package/src/commands/actor/set-value.js +0 -63
- package/src/commands/call.js +0 -131
- package/src/commands/check-version.js +0 -28
- package/src/commands/create.js +0 -203
- package/src/commands/info.js +0 -24
- package/src/commands/init-wrap-scrapy.js +0 -34
- package/src/commands/init.js +0 -83
- package/src/commands/login-new.js +0 -169
- package/src/commands/login.js +0 -40
- package/src/commands/logout.js +0 -18
- package/src/commands/pull.js +0 -154
- package/src/commands/push.js +0 -234
- package/src/commands/run.js +0 -206
- package/src/commands/secrets/add.js +0 -28
- package/src/commands/secrets/index.js +0 -24
- package/src/commands/secrets/rm.js +0 -23
- package/src/commands/vis.js +0 -50
- package/src/hooks/init.js +0 -16
- package/src/lib/apify_command.js +0 -82
- package/src/lib/community.js +0 -3
- package/src/lib/consts.js +0 -69
- package/src/lib/create-utils.js +0 -187
- package/src/lib/exec.js +0 -36
- package/src/lib/files.js +0 -76
- package/src/lib/input_schema.js +0 -114
- package/src/lib/local_state.js +0 -39
- package/src/lib/outputs.js +0 -27
- package/src/lib/scrapy-wrapper/Spider.js +0 -10
- package/src/lib/scrapy-wrapper/SpiderFileAnalyzer.js +0 -26
- package/src/lib/telemetry.js +0 -104
- package/src/lib/utils.js +0 -690
- /package/{src/bin → bin}/run.cmd +0 -0
package/src/lib/utils.js
DELETED
|
@@ -1,690 +0,0 @@
|
|
|
1
|
-
const {
|
|
2
|
-
execSync,
|
|
3
|
-
spawnSync,
|
|
4
|
-
} = require('child_process');
|
|
5
|
-
const fs = require('fs');
|
|
6
|
-
const https = require('https');
|
|
7
|
-
const path = require('path');
|
|
8
|
-
const { finished } = require('stream');
|
|
9
|
-
const { promisify } = require('util');
|
|
10
|
-
|
|
11
|
-
const {
|
|
12
|
-
ACT_JOB_TERMINAL_STATUSES,
|
|
13
|
-
ACTOR_ENV_VARS,
|
|
14
|
-
LOCAL_ACTOR_ENV_VARS,
|
|
15
|
-
ACTOR_NAME,
|
|
16
|
-
APIFY_ENV_VARS,
|
|
17
|
-
KEY_VALUE_STORE_KEYS,
|
|
18
|
-
LOCAL_STORAGE_SUBDIRS,
|
|
19
|
-
SOURCE_FILE_FORMATS,
|
|
20
|
-
} = require('@apify/consts');
|
|
21
|
-
const AdmZip = require('adm-zip');
|
|
22
|
-
const { ApifyClient } = require('apify-client');
|
|
23
|
-
const archiver = require('archiver-promise');
|
|
24
|
-
const axios = require('axios');
|
|
25
|
-
const escapeStringRegexp = require('escape-string-regexp');
|
|
26
|
-
const globby = require('globby');
|
|
27
|
-
const inquirer = require('inquirer');
|
|
28
|
-
const { getEncoding } = require('istextorbinary');
|
|
29
|
-
const loadJson = require('load-json-file');
|
|
30
|
-
const mime = require('mime');
|
|
31
|
-
const semver = require('semver');
|
|
32
|
-
const _ = require('underscore');
|
|
33
|
-
const writeJson = require('write-json-file');
|
|
34
|
-
|
|
35
|
-
const {
|
|
36
|
-
GLOBAL_CONFIGS_FOLDER,
|
|
37
|
-
AUTH_FILE_PATH,
|
|
38
|
-
INPUT_FILE_REG_EXP,
|
|
39
|
-
DEFAULT_LOCAL_STORAGE_DIR,
|
|
40
|
-
LOCAL_CONFIG_PATH,
|
|
41
|
-
DEPRECATED_LOCAL_CONFIG_NAME,
|
|
42
|
-
ACTOR_SPECIFICATION_VERSION,
|
|
43
|
-
APIFY_CLIENT_DEFAULT_HEADERS,
|
|
44
|
-
SUPPORTED_NODEJS_VERSION,
|
|
45
|
-
MINIMUM_SUPPORTED_PYTHON_VERSION,
|
|
46
|
-
LANGUAGE,
|
|
47
|
-
PROJECT_TYPES,
|
|
48
|
-
} = require('./consts');
|
|
49
|
-
const {
|
|
50
|
-
ensureFolderExistsSync,
|
|
51
|
-
rimrafPromised,
|
|
52
|
-
deleteFile,
|
|
53
|
-
} = require('./files');
|
|
54
|
-
const {
|
|
55
|
-
info,
|
|
56
|
-
} = require('./outputs');
|
|
57
|
-
const { ProjectAnalyzer } = require('./project_analyzer');
|
|
58
|
-
|
|
59
|
-
/**
|
|
60
|
-
* @param {string} url
|
|
61
|
-
* @returns {Promise<unknown>}
|
|
62
|
-
*/
|
|
63
|
-
const httpsGet = async (url) => {
|
|
64
|
-
return new Promise((resolve, reject) => {
|
|
65
|
-
https.get(url, (response) => {
|
|
66
|
-
// Handle redirects
|
|
67
|
-
if (response.statusCode === 301 || response.statusCode === 302) {
|
|
68
|
-
resolve(httpsGet(response.headers.location));
|
|
69
|
-
// Destroy the response to close the HTTP connection, otherwise this hangs for a long time with Node 19+ (due to HTTP keep-alive).
|
|
70
|
-
response.destroy();
|
|
71
|
-
} else {
|
|
72
|
-
resolve(response);
|
|
73
|
-
}
|
|
74
|
-
}).on('error', reject);
|
|
75
|
-
});
|
|
76
|
-
};
|
|
77
|
-
|
|
78
|
-
// Properties from apify.json file that will me migrated to actor specs in .actor/actor.json
|
|
79
|
-
const MIGRATED_APIFY_JSON_PROPERTIES = ['name', 'version', 'buildTag'];
|
|
80
|
-
|
|
81
|
-
const getLocalStorageDir = () => {
|
|
82
|
-
const envVar = APIFY_ENV_VARS.LOCAL_STORAGE_DIR;
|
|
83
|
-
|
|
84
|
-
return process.env[envVar] || process.env.CRAWLEE_STORAGE_DIR || DEFAULT_LOCAL_STORAGE_DIR;
|
|
85
|
-
};
|
|
86
|
-
const getLocalKeyValueStorePath = (storeId) => {
|
|
87
|
-
const envVar = ACTOR_ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID;
|
|
88
|
-
const storeDir = storeId || process.env[envVar] || LOCAL_ACTOR_ENV_VARS[envVar];
|
|
89
|
-
|
|
90
|
-
return path.join(getLocalStorageDir(), LOCAL_STORAGE_SUBDIRS.keyValueStores, storeDir);
|
|
91
|
-
};
|
|
92
|
-
const getLocalDatasetPath = (storeId) => {
|
|
93
|
-
const envVar = ACTOR_ENV_VARS.DEFAULT_DATASET_ID;
|
|
94
|
-
const storeDir = storeId || process.env[envVar] || LOCAL_ACTOR_ENV_VARS[envVar];
|
|
95
|
-
|
|
96
|
-
return path.join(getLocalStorageDir(), LOCAL_STORAGE_SUBDIRS.datasets, storeDir);
|
|
97
|
-
};
|
|
98
|
-
const getLocalRequestQueuePath = (storeId) => {
|
|
99
|
-
const envVar = ACTOR_ENV_VARS.DEFAULT_REQUEST_QUEUE_ID;
|
|
100
|
-
const storeDir = storeId || process.env[envVar] || LOCAL_ACTOR_ENV_VARS[envVar];
|
|
101
|
-
|
|
102
|
-
return path.join(getLocalStorageDir(), LOCAL_STORAGE_SUBDIRS.requestQueues, storeDir);
|
|
103
|
-
};
|
|
104
|
-
|
|
105
|
-
/**
|
|
106
|
-
* Returns object from auth file or empty object.
|
|
107
|
-
* @return {*}
|
|
108
|
-
*/
|
|
109
|
-
const getLocalUserInfo = () => {
|
|
110
|
-
try {
|
|
111
|
-
return loadJson.sync(AUTH_FILE_PATH) || {};
|
|
112
|
-
} catch (e) {
|
|
113
|
-
return {};
|
|
114
|
-
}
|
|
115
|
-
};
|
|
116
|
-
|
|
117
|
-
/**
|
|
118
|
-
* Gets instance of ApifyClient for user otherwise throws error
|
|
119
|
-
* @return {Promise<boolean|*>}
|
|
120
|
-
*/
|
|
121
|
-
const getLoggedClientOrThrow = async () => {
|
|
122
|
-
const loggedClient = await getLoggedClient();
|
|
123
|
-
if (!loggedClient) {
|
|
124
|
-
throw new Error('You are not logged in with your Apify account. Call "apify login" to fix that.');
|
|
125
|
-
}
|
|
126
|
-
return loggedClient;
|
|
127
|
-
};
|
|
128
|
-
|
|
129
|
-
/**
|
|
130
|
-
* Returns options for ApifyClient
|
|
131
|
-
* @param {String|null|undefined} token
|
|
132
|
-
* @returns {Object}
|
|
133
|
-
*/
|
|
134
|
-
const getApifyClientOptions = (token, apiBaseUrl) => {
|
|
135
|
-
if (!token && fs.existsSync(GLOBAL_CONFIGS_FOLDER) && fs.existsSync(AUTH_FILE_PATH)) {
|
|
136
|
-
({ token } = loadJson.sync(AUTH_FILE_PATH));
|
|
137
|
-
}
|
|
138
|
-
|
|
139
|
-
return {
|
|
140
|
-
token,
|
|
141
|
-
baseUrl: apiBaseUrl || process.env.APIFY_CLIENT_BASE_URL,
|
|
142
|
-
requestInterceptors: [(config) => {
|
|
143
|
-
if (!config.headers) {
|
|
144
|
-
config.headers = new axios.AxiosHeaders();
|
|
145
|
-
}
|
|
146
|
-
|
|
147
|
-
for (const [key, value] of Object.entries(APIFY_CLIENT_DEFAULT_HEADERS)) {
|
|
148
|
-
config.headers[key] = value;
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
return config;
|
|
152
|
-
}],
|
|
153
|
-
};
|
|
154
|
-
};
|
|
155
|
-
|
|
156
|
-
/**
|
|
157
|
-
* Gets instance of ApifyClient for token or for params from global auth file.
|
|
158
|
-
* NOTE: It refreshes global auth file each run
|
|
159
|
-
* @param [token]
|
|
160
|
-
* @return {Promise<*>}
|
|
161
|
-
*/
|
|
162
|
-
const getLoggedClient = async (token, apiBaseUrl) => {
|
|
163
|
-
if (!token && fs.existsSync(GLOBAL_CONFIGS_FOLDER) && fs.existsSync(AUTH_FILE_PATH)) {
|
|
164
|
-
({ token } = loadJson.sync(AUTH_FILE_PATH));
|
|
165
|
-
}
|
|
166
|
-
|
|
167
|
-
const apifyClient = new ApifyClient(getApifyClientOptions(token, apiBaseUrl));
|
|
168
|
-
let userInfo;
|
|
169
|
-
try {
|
|
170
|
-
userInfo = await apifyClient.user('me').get();
|
|
171
|
-
} catch (e) {
|
|
172
|
-
return false;
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
// Always refresh Auth file
|
|
176
|
-
if (!fs.existsSync(GLOBAL_CONFIGS_FOLDER)) fs.mkdirSync(GLOBAL_CONFIGS_FOLDER);
|
|
177
|
-
writeJson.sync(AUTH_FILE_PATH, { token: apifyClient.token, ...userInfo });
|
|
178
|
-
return apifyClient;
|
|
179
|
-
};
|
|
180
|
-
|
|
181
|
-
const getLocalConfigPath = () => path.join(process.cwd(), LOCAL_CONFIG_PATH);
|
|
182
|
-
|
|
183
|
-
/**
|
|
184
|
-
* @deprecated Use getLocalConfigPath
|
|
185
|
-
* @returns {string}
|
|
186
|
-
*/
|
|
187
|
-
const getDeprecatedLocalConfigPath = () => path.join(process.cwd(), DEPRECATED_LOCAL_CONFIG_NAME);
|
|
188
|
-
|
|
189
|
-
const getJsonFileContent = (filePath) => {
|
|
190
|
-
if (!fs.existsSync(filePath)) {
|
|
191
|
-
return;
|
|
192
|
-
}
|
|
193
|
-
return loadJson.sync(filePath);
|
|
194
|
-
};
|
|
195
|
-
|
|
196
|
-
const getLocalConfig = () => getJsonFileContent(getLocalConfigPath());
|
|
197
|
-
|
|
198
|
-
/**
|
|
199
|
-
* @deprecated Use getLocalConfig
|
|
200
|
-
* @returns {string}
|
|
201
|
-
*/
|
|
202
|
-
const getDeprecatedLocalConfig = () => getJsonFileContent(getDeprecatedLocalConfigPath());
|
|
203
|
-
|
|
204
|
-
const getLocalConfigOrThrow = async () => {
|
|
205
|
-
let localConfig = getLocalConfig();
|
|
206
|
-
let deprecatedLocalConfig = getDeprecatedLocalConfig();
|
|
207
|
-
|
|
208
|
-
if (localConfig && deprecatedLocalConfig) {
|
|
209
|
-
const answer = await inquirer.prompt([{
|
|
210
|
-
name: 'isConfirm',
|
|
211
|
-
type: 'confirm',
|
|
212
|
-
|
|
213
|
-
message: `The new version of Apify CLI uses the "${LOCAL_CONFIG_PATH}" instead of the "apify.json" file. Since we have found both files in your actor directory, "apify.json" will be renamed to "apify.json.deprecated". Going forward, all commands will use "${LOCAL_CONFIG_PATH}". You can read about the differences between the old and the new config at https://github.com/apify/apify-cli/blob/master/MIGRATIONS.md. Do you want to continue?`,
|
|
214
|
-
}]);
|
|
215
|
-
if (!answer.isConfirm) {
|
|
216
|
-
throw new Error('Command can not run with old "apify.json" file present in your actor directory., Please, either rename or remove it.');
|
|
217
|
-
}
|
|
218
|
-
try {
|
|
219
|
-
fs.renameSync(getDeprecatedLocalConfigPath(), `${getDeprecatedLocalConfigPath()}.deprecated`);
|
|
220
|
-
// eslint-disable-next-line max-len
|
|
221
|
-
info(`The "apify.json" file has been renamed to "apify.json.deprecated". The deprecated file is no longer used by the CLI or Apify Console. If you do not need it for some specific purpose, it can be safely deleted.`);
|
|
222
|
-
} catch (e) {
|
|
223
|
-
throw new Error('Failed to rename deprecated "apify.json".');
|
|
224
|
-
}
|
|
225
|
-
}
|
|
226
|
-
|
|
227
|
-
if (!localConfig && !deprecatedLocalConfig) {
|
|
228
|
-
return {};
|
|
229
|
-
}
|
|
230
|
-
|
|
231
|
-
// If apify.json exists migrate it to .actor/actor.json
|
|
232
|
-
if (!localConfig && deprecatedLocalConfig) {
|
|
233
|
-
const answer = await inquirer.prompt([{
|
|
234
|
-
name: 'isConfirm',
|
|
235
|
-
type: 'confirm',
|
|
236
|
-
// eslint-disable-next-line max-len
|
|
237
|
-
message: `The new version of Apify CLI uses the "${LOCAL_CONFIG_PATH}" instead of the "apify.json" file. Your "apify.json" file will be automatically updated to the new format under "${LOCAL_CONFIG_PATH}". The original file will be renamed by adding the ".deprecated" suffix. Do you want to continue?`,
|
|
238
|
-
}]);
|
|
239
|
-
if (!answer.isConfirm) {
|
|
240
|
-
throw new Error('Command can not run with old apify.json structure. Either let the CLI auto-update it or follow the guide on https://github.com/apify/apify-cli/blob/master/MIGRATIONS.md and update it manually.');
|
|
241
|
-
}
|
|
242
|
-
try {
|
|
243
|
-
// Check if apify.json contains old deprecated structure. If so, updates it.
|
|
244
|
-
if (_.isObject(deprecatedLocalConfig.version)) {
|
|
245
|
-
deprecatedLocalConfig = updateLocalConfigStructure(deprecatedLocalConfig);
|
|
246
|
-
}
|
|
247
|
-
localConfig = {
|
|
248
|
-
actorSpecification: ACTOR_SPECIFICATION_VERSION,
|
|
249
|
-
environmentVariables: deprecatedLocalConfig.env || undefined,
|
|
250
|
-
..._.pick(deprecatedLocalConfig, MIGRATED_APIFY_JSON_PROPERTIES),
|
|
251
|
-
};
|
|
252
|
-
|
|
253
|
-
writeJson.sync(getLocalConfigPath(), localConfig);
|
|
254
|
-
fs.renameSync(getDeprecatedLocalConfigPath(), `${getDeprecatedLocalConfigPath()}.deprecated`);
|
|
255
|
-
// eslint-disable-next-line max-len
|
|
256
|
-
info(`The "apify.json" file has been migrated to "${LOCAL_CONFIG_PATH}" and the original file renamed to "apify.json.deprecated". The deprecated file is no longer used by the CLI or Apify Console. If you do not need it for some specific purpose, it can be safely deleted. Do not forget to commit the new file to your Git repository.`);
|
|
257
|
-
} catch (e) {
|
|
258
|
-
throw new Error(`Can not update "${LOCAL_CONFIG_PATH}" structure. Follow guide on https://github.com/apify/apify-cli/blob/master/MIGRATIONS.md and update it manually.`);
|
|
259
|
-
}
|
|
260
|
-
}
|
|
261
|
-
|
|
262
|
-
return localConfig;
|
|
263
|
-
};
|
|
264
|
-
|
|
265
|
-
const setLocalConfig = async (localConfig, actDir) => {
|
|
266
|
-
actDir = actDir || process.cwd();
|
|
267
|
-
writeJson.sync(path.join(actDir, LOCAL_CONFIG_PATH), localConfig);
|
|
268
|
-
};
|
|
269
|
-
|
|
270
|
-
const GITIGNORE_REQUIRED_CONTENTS = [getLocalStorageDir(), 'node_modules', '.venv'];
|
|
271
|
-
|
|
272
|
-
const setLocalEnv = async (actDir) => {
|
|
273
|
-
// Create folders for emulation Apify stores
|
|
274
|
-
const keyValueStorePath = getLocalKeyValueStorePath();
|
|
275
|
-
ensureFolderExistsSync(actDir, getLocalDatasetPath());
|
|
276
|
-
ensureFolderExistsSync(actDir, getLocalRequestQueuePath());
|
|
277
|
-
ensureFolderExistsSync(actDir, keyValueStorePath);
|
|
278
|
-
|
|
279
|
-
// Create or update gitignore
|
|
280
|
-
const gitignorePath = path.join(actDir, '.gitignore');
|
|
281
|
-
let gitignoreContents = '';
|
|
282
|
-
if (fs.existsSync(gitignorePath)) {
|
|
283
|
-
gitignoreContents = fs.readFileSync(gitignorePath, { encoding: 'utf-8' });
|
|
284
|
-
}
|
|
285
|
-
|
|
286
|
-
const gitignoreAdditions = [];
|
|
287
|
-
for (const gitignoreRequirement of GITIGNORE_REQUIRED_CONTENTS) {
|
|
288
|
-
if (!RegExp(`^${escapeStringRegexp(gitignoreRequirement)}$`, 'mg').test(gitignoreContents)) {
|
|
289
|
-
gitignoreAdditions.push(gitignoreRequirement);
|
|
290
|
-
}
|
|
291
|
-
}
|
|
292
|
-
|
|
293
|
-
if (gitignoreAdditions.length > 0) {
|
|
294
|
-
if (gitignoreContents.length > 0) {
|
|
295
|
-
gitignoreAdditions.unshift('# Added by Apify CLI');
|
|
296
|
-
fs.writeFileSync(gitignorePath, `\n${gitignoreAdditions.join('\n')}\n`, { flag: 'a' });
|
|
297
|
-
} else {
|
|
298
|
-
fs.writeFileSync(gitignorePath, `${gitignoreAdditions.join('\n')}\n`, { flag: 'w' });
|
|
299
|
-
}
|
|
300
|
-
}
|
|
301
|
-
};
|
|
302
|
-
|
|
303
|
-
/**
|
|
304
|
-
* Convert Object with kebab-case keys to camelCased keys
|
|
305
|
-
*
|
|
306
|
-
* @param object
|
|
307
|
-
* @return {{}}
|
|
308
|
-
*/
|
|
309
|
-
const argsToCamelCase = (object) => {
|
|
310
|
-
const camelCasedObject = {};
|
|
311
|
-
Object.keys(object).forEach((arg) => {
|
|
312
|
-
const camelCasedArg = arg.replace(/-(.)/g, ($1) => $1.toUpperCase()).replace(/-/g, '');
|
|
313
|
-
camelCasedObject[camelCasedArg] = object[arg];
|
|
314
|
-
});
|
|
315
|
-
return camelCasedObject;
|
|
316
|
-
};
|
|
317
|
-
|
|
318
|
-
// Detect whether file is binary from its MIME type, or if not available, contents
|
|
319
|
-
const getSourceFileFormat = (filePath, fileContent) => {
|
|
320
|
-
// Try to detect the MIME type from the file path
|
|
321
|
-
// .tgz files don't have a MIME type defined, this fixes it
|
|
322
|
-
mime.define({ 'application/gzip': ['tgz'] }, true);
|
|
323
|
-
// Default mime-type for .ts(x) files is video/mp2t. But in our usecases they're almost always TypeScript, which we want to treat as text
|
|
324
|
-
mime.define({ 'text/typescript': ['ts', 'tsx', 'mts'] }, true);
|
|
325
|
-
|
|
326
|
-
const contentType = mime.getType(filePath);
|
|
327
|
-
if (contentType) {
|
|
328
|
-
const format = (
|
|
329
|
-
contentType.startsWith('text/')
|
|
330
|
-
|| contentType.includes('javascript')
|
|
331
|
-
|| contentType.includes('json')
|
|
332
|
-
|| contentType.includes('xml')
|
|
333
|
-
|| contentType.includes('application/node') // .cjs files
|
|
334
|
-
|| contentType.includes('application/toml') // for example pyproject.toml files
|
|
335
|
-
|| contentType.includes('application/x-httpd-php') // .php files
|
|
336
|
-
)
|
|
337
|
-
? SOURCE_FILE_FORMATS.TEXT
|
|
338
|
-
: SOURCE_FILE_FORMATS.BASE64;
|
|
339
|
-
return format;
|
|
340
|
-
}
|
|
341
|
-
|
|
342
|
-
// If the MIME type detection failed, try to detect the file encoding from the file content
|
|
343
|
-
const encoding = getEncoding(fileContent);
|
|
344
|
-
return encoding === 'binary' ? SOURCE_FILE_FORMATS.BASE64 : SOURCE_FILE_FORMATS.TEXT;
|
|
345
|
-
};
|
|
346
|
-
|
|
347
|
-
const createSourceFiles = async (paths) => {
|
|
348
|
-
return paths.map((filePath) => {
|
|
349
|
-
const file = fs.readFileSync(filePath);
|
|
350
|
-
const format = getSourceFileFormat(filePath, file);
|
|
351
|
-
return {
|
|
352
|
-
name: filePath,
|
|
353
|
-
format,
|
|
354
|
-
content: format === SOURCE_FILE_FORMATS.TEXT
|
|
355
|
-
? file.toString('utf8')
|
|
356
|
-
: file.toString('base64'),
|
|
357
|
-
};
|
|
358
|
-
});
|
|
359
|
-
};
|
|
360
|
-
|
|
361
|
-
/**
|
|
362
|
-
* Get actor local files, omit files defined in .gitignore and .git folder
|
|
363
|
-
* All dot files(.file) and folders(.folder/) are included.
|
|
364
|
-
*/
|
|
365
|
-
const getActorLocalFilePaths = () => globby(['*', '**/**'], {
|
|
366
|
-
ignore: ['.git/**', 'apify_storage', 'node_modules', 'storage', 'crawlee_storage'],
|
|
367
|
-
gitignore: true,
|
|
368
|
-
dot: true,
|
|
369
|
-
});
|
|
370
|
-
|
|
371
|
-
/**
|
|
372
|
-
* Create zip file with all actor files specified with pathsToZip
|
|
373
|
-
* @param zipName
|
|
374
|
-
* @param pathsToZip
|
|
375
|
-
* @return {Promise<void>}
|
|
376
|
-
*/
|
|
377
|
-
const createActZip = async (zipName, pathsToZip) => {
|
|
378
|
-
// NOTE: There can be a zip from a previous unfinished operation.
|
|
379
|
-
if (fs.existsSync(zipName)) await deleteFile(zipName);
|
|
380
|
-
|
|
381
|
-
const archive = archiver(zipName);
|
|
382
|
-
|
|
383
|
-
const archiveFilesPromises = [];
|
|
384
|
-
pathsToZip.forEach((globPath) => archiveFilesPromises.push(archive.glob(globPath)));
|
|
385
|
-
await Promise.all(archiveFilesPromises);
|
|
386
|
-
|
|
387
|
-
await archive.finalize();
|
|
388
|
-
};
|
|
389
|
-
|
|
390
|
-
/**
|
|
391
|
-
* Get actor input from local store
|
|
392
|
-
* @return {{body: *, contentType: string}}
|
|
393
|
-
*/
|
|
394
|
-
const getLocalInput = () => {
|
|
395
|
-
const defaultLocalStorePath = getLocalKeyValueStorePath();
|
|
396
|
-
const files = fs.readdirSync(defaultLocalStorePath);
|
|
397
|
-
const inputName = files.find((file) => !!file.match(INPUT_FILE_REG_EXP));
|
|
398
|
-
|
|
399
|
-
// No input file
|
|
400
|
-
if (!inputName) return;
|
|
401
|
-
|
|
402
|
-
const input = fs.readFileSync(path.join(defaultLocalStorePath, inputName));
|
|
403
|
-
const contentType = mime.getType(inputName);
|
|
404
|
-
return { body: input, contentType };
|
|
405
|
-
};
|
|
406
|
-
|
|
407
|
-
const purgeDefaultQueue = async () => {
|
|
408
|
-
const defaultQueuesPath = getLocalRequestQueuePath();
|
|
409
|
-
if (fs.existsSync(getLocalStorageDir()) && fs.existsSync(defaultQueuesPath)) {
|
|
410
|
-
await rimrafPromised(defaultQueuesPath);
|
|
411
|
-
}
|
|
412
|
-
};
|
|
413
|
-
|
|
414
|
-
const purgeDefaultDataset = async () => {
|
|
415
|
-
const defaultDatasetPath = getLocalDatasetPath();
|
|
416
|
-
if (fs.existsSync(getLocalStorageDir()) && fs.existsSync(defaultDatasetPath)) {
|
|
417
|
-
await rimrafPromised(defaultDatasetPath);
|
|
418
|
-
}
|
|
419
|
-
};
|
|
420
|
-
|
|
421
|
-
const purgeDefaultKeyValueStore = async () => {
|
|
422
|
-
const defaultKeyValueStorePath = getLocalKeyValueStorePath();
|
|
423
|
-
if (!fs.existsSync(getLocalStorageDir()) || !fs.existsSync(defaultKeyValueStorePath)) {
|
|
424
|
-
return;
|
|
425
|
-
}
|
|
426
|
-
const filesToDelete = fs.readdirSync(defaultKeyValueStorePath);
|
|
427
|
-
|
|
428
|
-
const deletePromises = [];
|
|
429
|
-
filesToDelete.forEach((file) => {
|
|
430
|
-
if (!file.match(INPUT_FILE_REG_EXP)) deletePromises.push(deleteFile(path.join(defaultKeyValueStorePath, file)));
|
|
431
|
-
});
|
|
432
|
-
|
|
433
|
-
await Promise.all(deletePromises);
|
|
434
|
-
};
|
|
435
|
-
|
|
436
|
-
const outputJobLog = async (job, timeout) => {
|
|
437
|
-
const { id: logId, status } = job;
|
|
438
|
-
// In case job was already done just output log
|
|
439
|
-
if (ACT_JOB_TERMINAL_STATUSES.includes(status)) {
|
|
440
|
-
const apifyClient = new ApifyClient({ baseUrl: process.env.APIFY_CLIENT_BASE_URL });
|
|
441
|
-
const log = await apifyClient.log(logId).get();
|
|
442
|
-
process.stdout.write(log);
|
|
443
|
-
}
|
|
444
|
-
|
|
445
|
-
// In other case stream it to stdout
|
|
446
|
-
// TODO: Use apifyClient.log(logId).stream() instead of http request stream.
|
|
447
|
-
return new Promise((resolve, reject) => {
|
|
448
|
-
const req = https.get(`https://api.apify.com/v2/logs/${logId}?stream=1`);
|
|
449
|
-
let res;
|
|
450
|
-
|
|
451
|
-
req.on('response', (response) => {
|
|
452
|
-
res = response;
|
|
453
|
-
response.on('data', (chunk) => process.stdout.write(chunk.toString()));
|
|
454
|
-
response.on('error', (err) => {
|
|
455
|
-
reject(err);
|
|
456
|
-
});
|
|
457
|
-
});
|
|
458
|
-
req.on('error', (err) => {
|
|
459
|
-
reject(err);
|
|
460
|
-
});
|
|
461
|
-
req.on('close', () => {
|
|
462
|
-
resolve('finished');
|
|
463
|
-
});
|
|
464
|
-
|
|
465
|
-
if (timeout) {
|
|
466
|
-
setTimeout(() => {
|
|
467
|
-
if (res) res.removeAllListeners();
|
|
468
|
-
if (req) {
|
|
469
|
-
req.removeAllListeners();
|
|
470
|
-
req.abort();
|
|
471
|
-
}
|
|
472
|
-
resolve('timeouts');
|
|
473
|
-
}, timeout);
|
|
474
|
-
}
|
|
475
|
-
});
|
|
476
|
-
};
|
|
477
|
-
|
|
478
|
-
/**
|
|
479
|
-
* Returns npm command for current os
|
|
480
|
-
* NOTE: For window we have to returns npm.cmd instead of npm, otherwise it doesn't work
|
|
481
|
-
* @return {string}
|
|
482
|
-
*/
|
|
483
|
-
const getNpmCmd = () => {
|
|
484
|
-
return /^win/.test(process.platform) ? 'npm.cmd' : 'npm';
|
|
485
|
-
};
|
|
486
|
-
|
|
487
|
-
/**
|
|
488
|
-
* Returns true if apify storage is empty (expect INPUT.*)
|
|
489
|
-
* @return {Promise<boolean>}
|
|
490
|
-
*/
|
|
491
|
-
const checkIfStorageIsEmpty = async () => {
|
|
492
|
-
const filesWithoutInput = await globby([
|
|
493
|
-
`${getLocalStorageDir()}/**`,
|
|
494
|
-
// Omit INPUT.* file
|
|
495
|
-
`!${getLocalKeyValueStorePath()}/${KEY_VALUE_STORE_KEYS.INPUT}.*`,
|
|
496
|
-
]);
|
|
497
|
-
return filesWithoutInput.length === 0;
|
|
498
|
-
};
|
|
499
|
-
|
|
500
|
-
/**
|
|
501
|
-
* Show help for command
|
|
502
|
-
* NOTE: This is not nice, but I can not find other way..
|
|
503
|
-
* @param command
|
|
504
|
-
*/
|
|
505
|
-
const showHelpForCommand = (command) => {
|
|
506
|
-
execSync(`apify ${command} --help`, { stdio: [0, 1, 2] });
|
|
507
|
-
};
|
|
508
|
-
|
|
509
|
-
/**
|
|
510
|
-
* Migration for deprecated structure of apify.json to latest.
|
|
511
|
-
* @param localConfig
|
|
512
|
-
*/
|
|
513
|
-
const updateLocalConfigStructure = (localConfig) => {
|
|
514
|
-
const updatedLocalConfig = {
|
|
515
|
-
name: localConfig.name,
|
|
516
|
-
template: localConfig.template,
|
|
517
|
-
version: localConfig.version.versionNumber,
|
|
518
|
-
buildTag: localConfig.version.buildTag,
|
|
519
|
-
env: null,
|
|
520
|
-
};
|
|
521
|
-
if (localConfig.version.envVars && localConfig.version.envVars.length) {
|
|
522
|
-
const env = {};
|
|
523
|
-
localConfig.version.envVars.forEach((envVar) => {
|
|
524
|
-
if (envVar.name && envVar.value) env[envVar.name] = envVar.value;
|
|
525
|
-
});
|
|
526
|
-
updatedLocalConfig.env = env;
|
|
527
|
-
}
|
|
528
|
-
return updatedLocalConfig;
|
|
529
|
-
};
|
|
530
|
-
|
|
531
|
-
/**
|
|
532
|
-
* Validates actor name, if finds issue throws error.
|
|
533
|
-
* @param actorName
|
|
534
|
-
*/
|
|
535
|
-
const validateActorName = (actorName) => {
|
|
536
|
-
if (!ACTOR_NAME.REGEX.test(actorName)) {
|
|
537
|
-
throw new Error('The Actor name must be a DNS hostname-friendly string (e.g. my-newest-actor).');
|
|
538
|
-
}
|
|
539
|
-
if (actorName.length < ACTOR_NAME.MIN_LENGTH) {
|
|
540
|
-
throw new Error('The Actor name must be at least 3 characters long.');
|
|
541
|
-
}
|
|
542
|
-
if (actorName.length > ACTOR_NAME.MAX_LENGTH) {
|
|
543
|
-
throw new Error('The Actor name must be a maximum of 30 characters long.');
|
|
544
|
-
}
|
|
545
|
-
};
|
|
546
|
-
|
|
547
|
-
const sanitizeActorName = (actorName) => {
|
|
548
|
-
let sanitizedName = actorName
|
|
549
|
-
.replaceAll(/[^a-zA-Z0-9-]/g, '-');
|
|
550
|
-
|
|
551
|
-
if (sanitizedName.length < ACTOR_NAME.MIN_LENGTH) {
|
|
552
|
-
sanitizedName = `${sanitizedName}-apify-actor`;
|
|
553
|
-
}
|
|
554
|
-
|
|
555
|
-
sanitizedName = sanitizedName.replaceAll(/^-+/g, '').replaceAll(/-+$/g, '');
|
|
556
|
-
|
|
557
|
-
return sanitizedName.slice(0, ACTOR_NAME.MAX_LENGTH);
|
|
558
|
-
};
|
|
559
|
-
|
|
560
|
-
const getPythonCommand = (directory) => {
|
|
561
|
-
const pythonVenvPath = /^win/.test(process.platform)
|
|
562
|
-
? 'Scripts/python.exe'
|
|
563
|
-
: 'bin/python3';
|
|
564
|
-
|
|
565
|
-
let fullPythonVenvPath;
|
|
566
|
-
if (process.env.VIRTUAL_ENV) {
|
|
567
|
-
fullPythonVenvPath = path.join(process.env.VIRTUAL_ENV, pythonVenvPath);
|
|
568
|
-
} else {
|
|
569
|
-
fullPythonVenvPath = path.join(directory, '.venv', pythonVenvPath);
|
|
570
|
-
}
|
|
571
|
-
|
|
572
|
-
if (fs.existsSync(fullPythonVenvPath)) {
|
|
573
|
-
return fullPythonVenvPath;
|
|
574
|
-
}
|
|
575
|
-
|
|
576
|
-
return /^win/.test(process.platform)
|
|
577
|
-
? 'python'
|
|
578
|
-
: 'python3';
|
|
579
|
-
};
|
|
580
|
-
|
|
581
|
-
const detectPythonVersion = (directory) => {
|
|
582
|
-
const pythonCommand = getPythonCommand(directory);
|
|
583
|
-
try {
|
|
584
|
-
const spawnResult = spawnSync(pythonCommand, ['-c', 'import platform; print(platform.python_version())'], { encoding: 'utf-8' });
|
|
585
|
-
if (!spawnResult.error && spawnResult.stdout) {
|
|
586
|
-
return spawnResult.stdout.trim();
|
|
587
|
-
}
|
|
588
|
-
} catch {
|
|
589
|
-
return undefined;
|
|
590
|
-
}
|
|
591
|
-
};
|
|
592
|
-
|
|
593
|
-
const isPythonVersionSupported = (installedPythonVersion) => {
|
|
594
|
-
return semver.satisfies(installedPythonVersion, `^${MINIMUM_SUPPORTED_PYTHON_VERSION}`);
|
|
595
|
-
};
|
|
596
|
-
|
|
597
|
-
const detectNodeVersion = () => {
|
|
598
|
-
try {
|
|
599
|
-
const spawnResult = spawnSync('node', ['--version'], { encoding: 'utf-8' });
|
|
600
|
-
if (!spawnResult.error && spawnResult.stdout) {
|
|
601
|
-
return spawnResult.stdout.trim().replace(/^v/, '');
|
|
602
|
-
}
|
|
603
|
-
} catch {
|
|
604
|
-
return undefined;
|
|
605
|
-
}
|
|
606
|
-
};
|
|
607
|
-
|
|
608
|
-
const isNodeVersionSupported = (installedNodeVersion) => {
|
|
609
|
-
// SUPPORTED_NODEJS_VERSION can be a version range,
|
|
610
|
-
// we need to get the minimum supported version from that range to be able to compare them
|
|
611
|
-
const minimumSupportedNodeVersion = semver.minVersion(SUPPORTED_NODEJS_VERSION);
|
|
612
|
-
return semver.gte(installedNodeVersion, minimumSupportedNodeVersion);
|
|
613
|
-
};
|
|
614
|
-
|
|
615
|
-
const detectNpmVersion = () => {
|
|
616
|
-
const npmCommand = getNpmCmd();
|
|
617
|
-
try {
|
|
618
|
-
const spawnResult = spawnSync(npmCommand, ['--version'], { encoding: 'utf-8' });
|
|
619
|
-
if (!spawnResult.error && spawnResult.stdout) {
|
|
620
|
-
return spawnResult.stdout.trim().replace(/^v/, '');
|
|
621
|
-
}
|
|
622
|
-
} catch {
|
|
623
|
-
return undefined;
|
|
624
|
-
}
|
|
625
|
-
};
|
|
626
|
-
|
|
627
|
-
const detectLocalActorLanguage = () => {
|
|
628
|
-
const cwd = process.cwd();
|
|
629
|
-
const isActorInNode = fs.existsSync(path.join(cwd, 'package.json'));
|
|
630
|
-
const isActorInPython = fs.existsSync(path.join(cwd, 'src/__main__.py')) || ProjectAnalyzer.getProjectType(cwd) === PROJECT_TYPES.SCRAPY;
|
|
631
|
-
const result = {};
|
|
632
|
-
if (isActorInNode) {
|
|
633
|
-
result.language = LANGUAGE.NODEJS;
|
|
634
|
-
result.languageVersion = detectNodeVersion();
|
|
635
|
-
} else if (isActorInPython) {
|
|
636
|
-
result.language = LANGUAGE.PYTHON;
|
|
637
|
-
result.languageVersion = detectPythonVersion(cwd);
|
|
638
|
-
} else {
|
|
639
|
-
result.language = LANGUAGE.UNKNOWN;
|
|
640
|
-
}
|
|
641
|
-
return result;
|
|
642
|
-
};
|
|
643
|
-
|
|
644
|
-
const downloadAndUnzip = async ({ url, pathTo }) => {
|
|
645
|
-
const zipStream = await httpsGet(url);
|
|
646
|
-
const chunks = [];
|
|
647
|
-
zipStream.on('data', (chunk) => chunks.push(chunk));
|
|
648
|
-
await promisify(finished)(zipStream);
|
|
649
|
-
const zip = new AdmZip(Buffer.concat(chunks));
|
|
650
|
-
zip.extractAllTo(pathTo, true);
|
|
651
|
-
};
|
|
652
|
-
|
|
653
|
-
module.exports = {
|
|
654
|
-
httpsGet,
|
|
655
|
-
getLoggedClientOrThrow,
|
|
656
|
-
getLocalConfig,
|
|
657
|
-
setLocalConfig,
|
|
658
|
-
setLocalEnv,
|
|
659
|
-
argsToCamelCase,
|
|
660
|
-
getLoggedClient,
|
|
661
|
-
createActZip,
|
|
662
|
-
getLocalUserInfo,
|
|
663
|
-
getLocalConfigOrThrow,
|
|
664
|
-
getLocalInput,
|
|
665
|
-
purgeDefaultQueue,
|
|
666
|
-
purgeDefaultDataset,
|
|
667
|
-
purgeDefaultKeyValueStore,
|
|
668
|
-
outputJobLog,
|
|
669
|
-
getLocalKeyValueStorePath,
|
|
670
|
-
getLocalDatasetPath,
|
|
671
|
-
getLocalRequestQueuePath,
|
|
672
|
-
getNpmCmd,
|
|
673
|
-
checkIfStorageIsEmpty,
|
|
674
|
-
getLocalStorageDir,
|
|
675
|
-
showHelpForCommand,
|
|
676
|
-
getActorLocalFilePaths,
|
|
677
|
-
createSourceFiles,
|
|
678
|
-
validateActorName,
|
|
679
|
-
getJsonFileContent,
|
|
680
|
-
getApifyClientOptions,
|
|
681
|
-
detectPythonVersion,
|
|
682
|
-
isPythonVersionSupported,
|
|
683
|
-
getPythonCommand,
|
|
684
|
-
detectNodeVersion,
|
|
685
|
-
isNodeVersionSupported,
|
|
686
|
-
detectNpmVersion,
|
|
687
|
-
detectLocalActorLanguage,
|
|
688
|
-
downloadAndUnzip,
|
|
689
|
-
sanitizeActorName,
|
|
690
|
-
};
|
/package/{src/bin → bin}/run.cmd
RENAMED
|
File without changes
|