apify-cli 0.17.0 → 0.18.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/npm-shrinkwrap.json +7 -7
- package/package.json +111 -111
- package/src/commands/create.js +10 -1
- package/src/commands/init.js +3 -0
- package/src/commands/run.js +5 -5
- package/src/lib/actor.js +5 -11
- package/src/lib/create-utils.js +1 -0
- package/src/lib/input_schema.js +43 -1
- package/src/lib/utils.js +18 -23
package/npm-shrinkwrap.json
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
"license": "Apache-2.0",
|
|
12
12
|
"dependencies": {
|
|
13
13
|
"@apify/actor-templates": "^0.1.3",
|
|
14
|
-
"@apify/consts": "^2.
|
|
14
|
+
"@apify/consts": "^2.20.0",
|
|
15
15
|
"@apify/input_schema": "^3.3.0",
|
|
16
16
|
"@apify/utilities": "^2.2.3",
|
|
17
17
|
"@crawlee/memory-storage": "^3.0.3",
|
|
@@ -72,9 +72,9 @@
|
|
|
72
72
|
"integrity": "sha512-sEASkPNvZ2ZWqSyUftJYhZo5EyWOqSnuVyQOzNDzvnYqN7Ev/H4f4tHas84BxWRdI8GPsIXLnArg0p4b6QrhJw=="
|
|
73
73
|
},
|
|
74
74
|
"node_modules/@apify/consts": {
|
|
75
|
-
"version": "2.
|
|
76
|
-
"resolved": "https://registry.npmjs.org/@apify/consts/-/consts-2.
|
|
77
|
-
"integrity": "sha512-
|
|
75
|
+
"version": "2.20.0",
|
|
76
|
+
"resolved": "https://registry.npmjs.org/@apify/consts/-/consts-2.20.0.tgz",
|
|
77
|
+
"integrity": "sha512-1vvMsSN2WG7b4zMLMj/LI6PIT94XH4Nr15mizEe+qi8xob5Y+g5LxOBR4qQQ1GlunsGBlrXo0v3dWMcVd4lRLQ=="
|
|
78
78
|
},
|
|
79
79
|
"node_modules/@apify/eslint-config": {
|
|
80
80
|
"version": "0.2.4",
|
|
@@ -8151,9 +8151,9 @@
|
|
|
8151
8151
|
"integrity": "sha512-sEASkPNvZ2ZWqSyUftJYhZo5EyWOqSnuVyQOzNDzvnYqN7Ev/H4f4tHas84BxWRdI8GPsIXLnArg0p4b6QrhJw=="
|
|
8152
8152
|
},
|
|
8153
8153
|
"@apify/consts": {
|
|
8154
|
-
"version": "2.
|
|
8155
|
-
"resolved": "https://registry.npmjs.org/@apify/consts/-/consts-2.
|
|
8156
|
-
"integrity": "sha512-
|
|
8154
|
+
"version": "2.20.0",
|
|
8155
|
+
"resolved": "https://registry.npmjs.org/@apify/consts/-/consts-2.20.0.tgz",
|
|
8156
|
+
"integrity": "sha512-1vvMsSN2WG7b4zMLMj/LI6PIT94XH4Nr15mizEe+qi8xob5Y+g5LxOBR4qQQ1GlunsGBlrXo0v3dWMcVd4lRLQ=="
|
|
8157
8157
|
},
|
|
8158
8158
|
"@apify/eslint-config": {
|
|
8159
8159
|
"version": "0.2.4",
|
package/package.json
CHANGED
|
@@ -1,114 +1,114 @@
|
|
|
1
1
|
{
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
}
|
|
2
|
+
"name": "apify-cli",
|
|
3
|
+
"version": "0.18.0-beta.0",
|
|
4
|
+
"description": "Apify command-line interface helps you create, develop, build and run Apify actors, and manage the Apify cloud platform.",
|
|
5
|
+
"main": "index.js",
|
|
6
|
+
"scripts": {
|
|
7
|
+
"test": "cross-env APIFY_CLI_SKIP_UPDATE_CHECK=1 mocha --timeout 180000 --recursive",
|
|
8
|
+
"test-python": "npm run test -- --grep '\\[python\\]'",
|
|
9
|
+
"lint": "eslint src test",
|
|
10
|
+
"lint:fix": "eslint src test --fix",
|
|
11
|
+
"commands-md": "npm run manifest && oclif-dev readme",
|
|
12
|
+
"prepare-release": "npm run commands-md && npm run prod-shrinkwrap",
|
|
13
|
+
"prod-shrinkwrap": "rm -rf node_modules && npm install --production && npm shrinkwrap",
|
|
14
|
+
"manifest": "rm -f oclif.manifest.json && oclif-dev manifest",
|
|
15
|
+
"postinstall": "node \"./src/bin/run\" check-version && node ./src/lib/community"
|
|
16
|
+
},
|
|
17
|
+
"files": [
|
|
18
|
+
"src",
|
|
19
|
+
"oclif.manifest.json",
|
|
20
|
+
"index.js",
|
|
21
|
+
"package.json"
|
|
22
|
+
],
|
|
23
|
+
"bin": {
|
|
24
|
+
"apify": "./src/bin/run"
|
|
25
|
+
},
|
|
26
|
+
"contributors": [
|
|
27
|
+
"Jakub Drobník <jakub.drobnik@apify.com>",
|
|
28
|
+
"Jan Curn <jan@apify.com>"
|
|
29
|
+
],
|
|
30
|
+
"repository": {
|
|
31
|
+
"type": "git",
|
|
32
|
+
"url": "git+https://github.com/apify/apify-cli.git"
|
|
33
|
+
},
|
|
34
|
+
"keywords": [
|
|
35
|
+
"apify",
|
|
36
|
+
"client",
|
|
37
|
+
"node",
|
|
38
|
+
"command",
|
|
39
|
+
"line",
|
|
40
|
+
"bash"
|
|
41
|
+
],
|
|
42
|
+
"author": {
|
|
43
|
+
"name": "Apify",
|
|
44
|
+
"email": "support@apify.com",
|
|
45
|
+
"url": "https://www.apify.com"
|
|
46
|
+
},
|
|
47
|
+
"license": "Apache-2.0",
|
|
48
|
+
"bugs": {
|
|
49
|
+
"url": "https://github.com/apify/apify-cli/issues"
|
|
50
|
+
},
|
|
51
|
+
"homepage": "https://github.com/apify/apify-cli#readme",
|
|
52
|
+
"engines": {
|
|
53
|
+
"node": ">=16.0.0"
|
|
54
|
+
},
|
|
55
|
+
"dependencies": {
|
|
56
|
+
"@apify/actor-templates": "^0.1.3",
|
|
57
|
+
"@apify/consts": "^2.20.0",
|
|
58
|
+
"@apify/input_schema": "^3.3.0",
|
|
59
|
+
"@apify/utilities": "^2.2.3",
|
|
60
|
+
"@crawlee/memory-storage": "^3.0.3",
|
|
61
|
+
"@oclif/command": "^1.8.16",
|
|
62
|
+
"@oclif/config": "^1.18.3",
|
|
63
|
+
"@oclif/errors": "^1.3.5",
|
|
64
|
+
"@oclif/plugin-commands": "^2.1.0",
|
|
65
|
+
"@oclif/plugin-help": "^5.1.12",
|
|
66
|
+
"adm-zip": "^0.5.10",
|
|
67
|
+
"ajv": "^8.10.0",
|
|
68
|
+
"apify-client": "^2.2.0",
|
|
69
|
+
"archiver-promise": "^1.0.0",
|
|
70
|
+
"axios": "^0.21.1",
|
|
71
|
+
"chalk": "^4.1.2",
|
|
72
|
+
"computer-name": "^0.1.0",
|
|
73
|
+
"cors": "^2.8.5",
|
|
74
|
+
"detect-indent": "^6.1.0",
|
|
75
|
+
"escape-string-regexp": "^4.0.0",
|
|
76
|
+
"express": "^4.18.2",
|
|
77
|
+
"globby": "^11.1.0",
|
|
78
|
+
"inquirer": "^7.3.3",
|
|
79
|
+
"is-ci": "^3.0.1",
|
|
80
|
+
"is-online": "^10.0.0",
|
|
81
|
+
"istextorbinary": "^6.0.0",
|
|
82
|
+
"jju": "^1.4.0",
|
|
83
|
+
"load-json-file": "^6.2.0",
|
|
84
|
+
"mime": "^2.6.0",
|
|
85
|
+
"mixpanel": "^0.17.0",
|
|
86
|
+
"open": "^8.4.0",
|
|
87
|
+
"ow": "^0.28.1",
|
|
88
|
+
"rimraf": "^3.0.2",
|
|
89
|
+
"semver": "^7.3.5",
|
|
90
|
+
"tiged": "^2.12.4",
|
|
91
|
+
"underscore": "^1.13.2",
|
|
92
|
+
"write-json-file": "^4.3.0"
|
|
93
|
+
},
|
|
94
|
+
"devDependencies": {
|
|
95
|
+
"@apify/eslint-config": "^0.2.3",
|
|
96
|
+
"@oclif/dev-cli": "^1.26.0",
|
|
97
|
+
"@oclif/test": "^2.1.0",
|
|
98
|
+
"chai": "^4.3.4",
|
|
99
|
+
"chai-match": "^1.1.1",
|
|
100
|
+
"cross-env": "^7.0.3",
|
|
101
|
+
"eslint": "^8.11.0",
|
|
102
|
+
"mocha": "^10.0.0",
|
|
103
|
+
"sinon": "^9.2.4"
|
|
104
|
+
},
|
|
105
|
+
"oclif": {
|
|
106
|
+
"bin": "apify",
|
|
107
|
+
"commands": "./src/commands",
|
|
108
|
+
"hooks": {
|
|
109
|
+
"init": [
|
|
110
|
+
"./src/hooks/init"
|
|
111
|
+
]
|
|
113
112
|
}
|
|
113
|
+
}
|
|
114
114
|
}
|
package/src/commands/create.js
CHANGED
|
@@ -23,7 +23,13 @@ const {
|
|
|
23
23
|
detectNpmVersion,
|
|
24
24
|
} = require('../lib/utils');
|
|
25
25
|
const { EMPTY_LOCAL_CONFIG, LOCAL_CONFIG_PATH, PYTHON_VENV_PATH, SUPPORTED_NODEJS_VERSION } = require('../lib/consts');
|
|
26
|
-
const {
|
|
26
|
+
const {
|
|
27
|
+
httpsGet,
|
|
28
|
+
ensureValidActorName,
|
|
29
|
+
getTemplateDefinition,
|
|
30
|
+
enhanceReadmeWithLocalSuffix,
|
|
31
|
+
} = require('../lib/create-utils');
|
|
32
|
+
const { createPrefilledInputFileFromInputSchema } = require('../lib/input_schema');
|
|
27
33
|
|
|
28
34
|
class CreateCommand extends ApifyCommand {
|
|
29
35
|
async run() {
|
|
@@ -85,6 +91,9 @@ class CreateCommand extends ApifyCommand {
|
|
|
85
91
|
await setLocalConfig(Object.assign(localConfig || EMPTY_LOCAL_CONFIG, { name: actorName }), actFolderDir);
|
|
86
92
|
await setLocalEnv(actFolderDir);
|
|
87
93
|
|
|
94
|
+
// Create prefilled INPUT.json file from the input schema prefills
|
|
95
|
+
await createPrefilledInputFileFromInputSchema(actFolderDir);
|
|
96
|
+
|
|
88
97
|
const packageJsonPath = path.join(actFolderDir, 'package.json');
|
|
89
98
|
const requirementsTxtPath = path.join(actFolderDir, 'requirements.txt');
|
|
90
99
|
const readmePath = path.join(actFolderDir, 'README.md');
|
package/src/commands/init.js
CHANGED
|
@@ -4,6 +4,7 @@ const { ApifyCommand } = require('../lib/apify_command');
|
|
|
4
4
|
const outputs = require('../lib/outputs');
|
|
5
5
|
const { setLocalConfig, setLocalEnv, getLocalConfig, getLocalConfigOrThrow } = require('../lib/utils');
|
|
6
6
|
const { EMPTY_LOCAL_CONFIG, DEFAULT_LOCAL_STORAGE_DIR, LOCAL_CONFIG_PATH } = require('../lib/consts');
|
|
7
|
+
const { createPrefilledInputFileFromInputSchema } = require('../lib/input_schema');
|
|
7
8
|
|
|
8
9
|
class InitCommand extends ApifyCommand {
|
|
9
10
|
async run() {
|
|
@@ -23,6 +24,8 @@ class InitCommand extends ApifyCommand {
|
|
|
23
24
|
await setLocalConfig(Object.assign(localConfig, { name: actorName }), cwd);
|
|
24
25
|
}
|
|
25
26
|
await setLocalEnv(cwd);
|
|
27
|
+
// Create prefilled INPUT.json file from the input schema prefills
|
|
28
|
+
await createPrefilledInputFileFromInputSchema(cwd);
|
|
26
29
|
outputs.success('The Apify actor has been initialized in the current directory.');
|
|
27
30
|
}
|
|
28
31
|
}
|
package/src/commands/run.js
CHANGED
|
@@ -2,7 +2,7 @@ const { flags: flagsHelper } = require('@oclif/command');
|
|
|
2
2
|
const fs = require('fs');
|
|
3
3
|
const path = require('path');
|
|
4
4
|
const loadJson = require('load-json-file');
|
|
5
|
-
const {
|
|
5
|
+
const { APIFY_ENV_VARS } = require('@apify/consts');
|
|
6
6
|
const semver = require('semver');
|
|
7
7
|
const execWithLog = require('../lib/exec');
|
|
8
8
|
const { LEGACY_LOCAL_STORAGE_DIR, DEFAULT_LOCAL_STORAGE_DIR, SUPPORTED_NODEJS_VERSION, LANGUAGE } = require('../lib/consts');
|
|
@@ -69,11 +69,11 @@ class RunCommand extends ApifyCommand {
|
|
|
69
69
|
|
|
70
70
|
// Attach env vars from local config files
|
|
71
71
|
const localEnvVars = {
|
|
72
|
-
[
|
|
72
|
+
[APIFY_ENV_VARS.LOCAL_STORAGE_DIR]: DEFAULT_LOCAL_STORAGE_DIR,
|
|
73
73
|
};
|
|
74
|
-
if (proxy && proxy.password) localEnvVars[
|
|
75
|
-
if (userId) localEnvVars[
|
|
76
|
-
if (token) localEnvVars[
|
|
74
|
+
if (proxy && proxy.password) localEnvVars[APIFY_ENV_VARS.PROXY_PASSWORD] = proxy.password;
|
|
75
|
+
if (userId) localEnvVars[APIFY_ENV_VARS.USER_ID] = userId;
|
|
76
|
+
if (token) localEnvVars[APIFY_ENV_VARS.TOKEN] = token;
|
|
77
77
|
if (localConfig.environmentVariables) {
|
|
78
78
|
const updatedEnv = replaceSecretsValue(localConfig.environmentVariables);
|
|
79
79
|
Object.assign(localEnvVars, updatedEnv);
|
package/src/lib/actor.js
CHANGED
|
@@ -3,7 +3,7 @@ const { pipeline } = require('stream');
|
|
|
3
3
|
const { promisify } = require('util');
|
|
4
4
|
const { default: ow } = require('ow');
|
|
5
5
|
const { MemoryStorage } = require('@crawlee/memory-storage');
|
|
6
|
-
const {
|
|
6
|
+
const { ACTOR_ENV_VARS, LOCAL_ACTOR_ENV_VARS, APIFY_ENV_VARS, KEY_VALUE_STORE_KEYS } = require('@apify/consts');
|
|
7
7
|
const { getLocalUserInfo, getLocalStorageDir, getApifyClientOptions } = require('./utils');
|
|
8
8
|
|
|
9
9
|
const pipelinePromise = promisify(pipeline);
|
|
@@ -31,7 +31,7 @@ const getApifyStorageClient = (options = {}, forceCloud = false) => {
|
|
|
31
31
|
}
|
|
32
32
|
|
|
33
33
|
// NOTE: Token in env var overrides token in local auth file.
|
|
34
|
-
let apifyToken = process.env[
|
|
34
|
+
let apifyToken = process.env[APIFY_ENV_VARS.TOKEN];
|
|
35
35
|
if (!apifyToken) {
|
|
36
36
|
const localUserInfo = getLocalUserInfo();
|
|
37
37
|
if (!localUserInfo || !localUserInfo.token) {
|
|
@@ -53,15 +53,9 @@ const getApifyStorageClient = (options = {}, forceCloud = false) => {
|
|
|
53
53
|
* @return {string}
|
|
54
54
|
*/
|
|
55
55
|
const getDefaultStorageId = (storeType) => {
|
|
56
|
-
const
|
|
57
|
-
const envVarName = ENV_VARS[`DEFAULT_${storeType}_ID`];
|
|
58
|
-
const storeId = process.env[envVarName];
|
|
59
|
-
if (isRunningOnApify && !storeId) {
|
|
60
|
-
throw new Error(`Default storage ID is not set. You can set it using the environment `
|
|
61
|
-
+ `variable ${envVarName} or use local storage with setting ${ENV_VARS.LOCAL_STORAGE_DIR} variable.`);
|
|
62
|
-
}
|
|
56
|
+
const envVarName = ACTOR_ENV_VARS[`DEFAULT_${storeType}_ID`];
|
|
63
57
|
|
|
64
|
-
return
|
|
58
|
+
return process.env[envVarName] || LOCAL_ACTOR_ENV_VARS[envVarName];
|
|
65
59
|
};
|
|
66
60
|
|
|
67
61
|
/**
|
|
@@ -82,7 +76,7 @@ const outputRecordFromDefaultStore = async (key) => {
|
|
|
82
76
|
};
|
|
83
77
|
|
|
84
78
|
const outputInputFromDefaultStore = async () => {
|
|
85
|
-
return outputRecordFromDefaultStore(process.env[
|
|
79
|
+
return outputRecordFromDefaultStore(process.env[ACTOR_ENV_VARS.INPUT_KEY] || KEY_VALUE_STORE_KEYS.INPUT);
|
|
86
80
|
};
|
|
87
81
|
|
|
88
82
|
module.exports = {
|
package/src/lib/create-utils.js
CHANGED
package/src/lib/input_schema.js
CHANGED
|
@@ -1,7 +1,13 @@
|
|
|
1
1
|
const fs = require('fs');
|
|
2
2
|
const path = require('path');
|
|
3
|
+
const Ajv = require('ajv');
|
|
4
|
+
const { validateInputSchema } = require('@apify/input_schema');
|
|
5
|
+
const _ = require('underscore');
|
|
6
|
+
const { KEY_VALUE_STORE_KEYS } = require('@apify/consts');
|
|
7
|
+
const writeJsonFile = require('write-json-file');
|
|
3
8
|
const { ACTOR_SPECIFICATION_FOLDER } = require('./consts');
|
|
4
|
-
const { getLocalConfig, getJsonFileContent } = require('./utils');
|
|
9
|
+
const { getLocalConfig, getJsonFileContent, getLocalKeyValueStorePath } = require('./utils');
|
|
10
|
+
const { warning } = require('./outputs');
|
|
5
11
|
|
|
6
12
|
const DEFAULT_INPUT_SCHEMA_PATHS = [
|
|
7
13
|
'.actor/INPUT_SCHEMA.json',
|
|
@@ -65,6 +71,42 @@ const readInputSchema = async (forcePath) => {
|
|
|
65
71
|
};
|
|
66
72
|
};
|
|
67
73
|
|
|
74
|
+
/**
|
|
75
|
+
* Goes to the Actor directory and creates INPUT.json file from the input schema prefills.
|
|
76
|
+
* @param {string} actorFolderDir
|
|
77
|
+
*/
|
|
78
|
+
const createPrefilledInputFileFromInputSchema = async (actorFolderDir) => {
|
|
79
|
+
const currentDir = process.cwd();
|
|
80
|
+
let inputFile = {};
|
|
81
|
+
try {
|
|
82
|
+
process.chdir(actorFolderDir);
|
|
83
|
+
const { inputSchema } = await readInputSchema();
|
|
84
|
+
|
|
85
|
+
if (inputSchema) {
|
|
86
|
+
/**
|
|
87
|
+
* TODO: The logic is copied from @apify-packages/actor -> getPrefillFromInputSchema
|
|
88
|
+
* It is not possible to install the package here because it is private
|
|
89
|
+
* We should move it to @apify/input_schema and use it from there.
|
|
90
|
+
*/
|
|
91
|
+
const validator = new Ajv({ strict: false });
|
|
92
|
+
validateInputSchema(validator, inputSchema);
|
|
93
|
+
|
|
94
|
+
inputFile = _.mapObject(inputSchema.properties, (fieldSchema) => ((fieldSchema.type === 'boolean' || fieldSchema.editor === 'hidden')
|
|
95
|
+
? fieldSchema.default
|
|
96
|
+
: fieldSchema.prefill
|
|
97
|
+
));
|
|
98
|
+
}
|
|
99
|
+
} catch (err) {
|
|
100
|
+
warning(`Could not create default input based on input schema, creating empty input instead. Cause: ${err.message}`);
|
|
101
|
+
} finally {
|
|
102
|
+
const keyValueStorePath = getLocalKeyValueStorePath();
|
|
103
|
+
const inputJsonPath = path.join(actorFolderDir, keyValueStorePath, `${KEY_VALUE_STORE_KEYS.INPUT}.json`);
|
|
104
|
+
await writeJsonFile(inputJsonPath, inputFile);
|
|
105
|
+
process.chdir(currentDir);
|
|
106
|
+
}
|
|
107
|
+
};
|
|
108
|
+
|
|
68
109
|
module.exports = {
|
|
69
110
|
readInputSchema,
|
|
111
|
+
createPrefilledInputFileFromInputSchema,
|
|
70
112
|
};
|
package/src/lib/utils.js
CHANGED
|
@@ -9,13 +9,14 @@ const loadJson = require('load-json-file');
|
|
|
9
9
|
const writeJson = require('write-json-file');
|
|
10
10
|
const inquirer = require('inquirer');
|
|
11
11
|
const {
|
|
12
|
-
LOCAL_STORAGE_SUBDIRS,
|
|
13
|
-
ENV_VARS,
|
|
14
|
-
LOCAL_ENV_VARS,
|
|
15
|
-
KEY_VALUE_STORE_KEYS,
|
|
16
12
|
ACT_JOB_TERMINAL_STATUSES,
|
|
17
|
-
|
|
13
|
+
ACTOR_ENV_VARS,
|
|
14
|
+
LOCAL_ACTOR_ENV_VARS,
|
|
18
15
|
ACTOR_NAME,
|
|
16
|
+
APIFY_ENV_VARS,
|
|
17
|
+
KEY_VALUE_STORE_KEYS,
|
|
18
|
+
LOCAL_STORAGE_SUBDIRS,
|
|
19
|
+
SOURCE_FILE_FORMATS,
|
|
19
20
|
} = require('@apify/consts');
|
|
20
21
|
const https = require('https');
|
|
21
22
|
const { ApifyClient } = require('apify-client');
|
|
@@ -51,25 +52,25 @@ const {
|
|
|
51
52
|
const MIGRATED_APIFY_JSON_PROPERTIES = ['name', 'version', 'buildTag'];
|
|
52
53
|
|
|
53
54
|
const getLocalStorageDir = () => {
|
|
54
|
-
const envVar =
|
|
55
|
+
const envVar = APIFY_ENV_VARS.LOCAL_STORAGE_DIR;
|
|
55
56
|
|
|
56
57
|
return process.env[envVar] || DEFAULT_LOCAL_STORAGE_DIR;
|
|
57
58
|
};
|
|
58
59
|
const getLocalKeyValueStorePath = (storeId) => {
|
|
59
|
-
const envVar =
|
|
60
|
-
const storeDir = storeId || process.env[envVar] ||
|
|
60
|
+
const envVar = ACTOR_ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID;
|
|
61
|
+
const storeDir = storeId || process.env[envVar] || LOCAL_ACTOR_ENV_VARS[envVar];
|
|
61
62
|
|
|
62
63
|
return path.join(getLocalStorageDir(), LOCAL_STORAGE_SUBDIRS.keyValueStores, storeDir);
|
|
63
64
|
};
|
|
64
65
|
const getLocalDatasetPath = (storeId) => {
|
|
65
|
-
const envVar =
|
|
66
|
-
const storeDir = storeId || process.env[envVar] ||
|
|
66
|
+
const envVar = ACTOR_ENV_VARS.DEFAULT_DATASET_ID;
|
|
67
|
+
const storeDir = storeId || process.env[envVar] || LOCAL_ACTOR_ENV_VARS[envVar];
|
|
67
68
|
|
|
68
69
|
return path.join(getLocalStorageDir(), LOCAL_STORAGE_SUBDIRS.datasets, storeDir);
|
|
69
70
|
};
|
|
70
71
|
const getLocalRequestQueuePath = (storeId) => {
|
|
71
|
-
const envVar =
|
|
72
|
-
const storeDir = storeId || process.env[envVar] ||
|
|
72
|
+
const envVar = ACTOR_ENV_VARS.DEFAULT_REQUEST_QUEUE_ID;
|
|
73
|
+
const storeDir = storeId || process.env[envVar] || LOCAL_ACTOR_ENV_VARS[envVar];
|
|
73
74
|
|
|
74
75
|
return path.join(getLocalStorageDir(), LOCAL_STORAGE_SUBDIRS.requestQueues, storeDir);
|
|
75
76
|
};
|
|
@@ -237,7 +238,6 @@ const GITIGNORE_REQUIRED_CONTENTS = [getLocalStorageDir(), 'node_modules', '.ven
|
|
|
237
238
|
const setLocalEnv = async (actDir) => {
|
|
238
239
|
// Create folders for emulation Apify stores
|
|
239
240
|
const keyValueStorePath = getLocalKeyValueStorePath();
|
|
240
|
-
const inputJsonPath = path.join(actDir, keyValueStorePath, `${KEY_VALUE_STORE_KEYS.INPUT}.json`);
|
|
241
241
|
ensureFolderExistsSync(actDir, getLocalDatasetPath());
|
|
242
242
|
ensureFolderExistsSync(actDir, getLocalRequestQueuePath());
|
|
243
243
|
ensureFolderExistsSync(actDir, keyValueStorePath);
|
|
@@ -264,11 +264,6 @@ const setLocalEnv = async (actDir) => {
|
|
|
264
264
|
fs.writeFileSync(gitignorePath, `${gitignoreAdditions.join('\n')}\n`, { flag: 'w' });
|
|
265
265
|
}
|
|
266
266
|
}
|
|
267
|
-
|
|
268
|
-
// Create an empty INPUT.json file if it does not exist.
|
|
269
|
-
if (!fs.existsSync(inputJsonPath)) {
|
|
270
|
-
writeJson.sync(inputJsonPath, {});
|
|
271
|
-
}
|
|
272
267
|
};
|
|
273
268
|
|
|
274
269
|
/**
|
|
@@ -365,14 +360,14 @@ const createActZip = async (zipName, pathsToZip) => {
|
|
|
365
360
|
const getLocalInput = () => {
|
|
366
361
|
const defaultLocalStorePath = getLocalKeyValueStorePath();
|
|
367
362
|
const files = fs.readdirSync(defaultLocalStorePath);
|
|
368
|
-
const
|
|
363
|
+
const inputName = files.find((file) => !!file.match(INPUT_FILE_REG_EXP));
|
|
369
364
|
|
|
370
365
|
// No input file
|
|
371
|
-
if (!
|
|
366
|
+
if (!inputName) return;
|
|
372
367
|
|
|
373
|
-
const
|
|
374
|
-
const contentType = mime.getType(
|
|
375
|
-
return { body:
|
|
368
|
+
const input = fs.readFileSync(path.join(defaultLocalStorePath, inputName));
|
|
369
|
+
const contentType = mime.getType(inputName);
|
|
370
|
+
return { body: input, contentType };
|
|
376
371
|
};
|
|
377
372
|
|
|
378
373
|
const purgeDefaultQueue = async () => {
|