apify-cli 0.19.4 → 0.20.0-beta.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (255) hide show
  1. package/CHANGELOG.md +200 -0
  2. package/README.md +273 -167
  3. package/bin/dev.cmd +3 -0
  4. package/bin/dev.js +21 -0
  5. package/bin/dev.sh +4 -0
  6. package/bin/run.js +23 -0
  7. package/dist/.tsbuildinfo +1 -0
  8. package/dist/commands/actor/get-input.d.ts +6 -0
  9. package/dist/commands/actor/get-input.d.ts.map +1 -0
  10. package/dist/commands/actor/get-input.js +14 -0
  11. package/dist/commands/actor/get-input.js.map +1 -0
  12. package/dist/commands/actor/get-value.d.ts +9 -0
  13. package/dist/commands/actor/get-value.d.ts.map +1 -0
  14. package/dist/commands/actor/get-value.js +27 -0
  15. package/dist/commands/actor/get-value.js.map +1 -0
  16. package/dist/commands/actor/index.d.ts +6 -0
  17. package/dist/commands/actor/index.d.ts.map +1 -0
  18. package/dist/commands/actor/index.js +13 -0
  19. package/dist/commands/actor/index.js.map +1 -0
  20. package/dist/commands/actor/push-data.d.ts +11 -0
  21. package/dist/commands/actor/push-data.d.ts.map +1 -0
  22. package/dist/commands/actor/push-data.js +56 -0
  23. package/dist/commands/actor/push-data.js.map +1 -0
  24. package/dist/commands/actor/set-value.d.ts +15 -0
  25. package/dist/commands/actor/set-value.d.ts.map +1 -0
  26. package/dist/commands/actor/set-value.js +76 -0
  27. package/dist/commands/actor/set-value.js.map +1 -0
  28. package/dist/commands/call.d.ts +19 -0
  29. package/dist/commands/call.d.ts.map +1 -0
  30. package/dist/commands/call.js +193 -0
  31. package/dist/commands/call.js.map +1 -0
  32. package/dist/commands/check-version.d.ts +11 -0
  33. package/dist/commands/check-version.d.ts.map +1 -0
  34. package/dist/commands/check-version.js +39 -0
  35. package/dist/commands/check-version.js.map +1 -0
  36. package/dist/commands/create.d.ts +15 -0
  37. package/dist/commands/create.d.ts.map +1 -0
  38. package/dist/commands/create.js +201 -0
  39. package/dist/commands/create.js.map +1 -0
  40. package/dist/commands/edit-input-schema.d.ts +11 -0
  41. package/dist/commands/edit-input-schema.d.ts.map +1 -0
  42. package/{src → dist}/commands/edit-input-schema.js +81 -85
  43. package/dist/commands/edit-input-schema.js.map +1 -0
  44. package/dist/commands/info.d.ts +6 -0
  45. package/dist/commands/info.d.ts.map +1 -0
  46. package/dist/commands/info.js +26 -0
  47. package/dist/commands/info.js.map +1 -0
  48. package/dist/commands/init-wrap-scrapy.d.ts +10 -0
  49. package/dist/commands/init-wrap-scrapy.d.ts.map +1 -0
  50. package/dist/commands/init-wrap-scrapy.js +41 -0
  51. package/dist/commands/init-wrap-scrapy.js.map +1 -0
  52. package/dist/commands/init.d.ts +12 -0
  53. package/dist/commands/init.d.ts.map +1 -0
  54. package/dist/commands/init.js +88 -0
  55. package/dist/commands/init.js.map +1 -0
  56. package/dist/commands/login.d.ts +10 -0
  57. package/dist/commands/login.d.ts.map +1 -0
  58. package/dist/commands/login.js +174 -0
  59. package/dist/commands/login.js.map +1 -0
  60. package/dist/commands/logout.d.ts +6 -0
  61. package/dist/commands/logout.d.ts.map +1 -0
  62. package/dist/commands/logout.js +20 -0
  63. package/dist/commands/logout.js.map +1 -0
  64. package/dist/commands/pull.d.ts +12 -0
  65. package/dist/commands/pull.d.ts.map +1 -0
  66. package/dist/commands/pull.js +148 -0
  67. package/dist/commands/pull.js.map +1 -0
  68. package/dist/commands/push.d.ts +18 -0
  69. package/dist/commands/push.d.ts.map +1 -0
  70. package/dist/commands/push.js +298 -0
  71. package/dist/commands/push.js.map +1 -0
  72. package/dist/commands/run.d.ts +16 -0
  73. package/dist/commands/run.d.ts.map +1 -0
  74. package/dist/commands/run.js +383 -0
  75. package/dist/commands/run.js.map +1 -0
  76. package/dist/commands/secrets/add.d.ts +10 -0
  77. package/dist/commands/secrets/add.d.ts.map +1 -0
  78. package/dist/commands/secrets/add.js +31 -0
  79. package/dist/commands/secrets/add.js.map +1 -0
  80. package/dist/commands/secrets/index.d.ts +6 -0
  81. package/dist/commands/secrets/index.d.ts.map +1 -0
  82. package/dist/commands/secrets/index.js +24 -0
  83. package/dist/commands/secrets/index.js.map +1 -0
  84. package/dist/commands/secrets/rm.d.ts +9 -0
  85. package/dist/commands/secrets/rm.d.ts.map +1 -0
  86. package/dist/commands/secrets/rm.js +27 -0
  87. package/dist/commands/secrets/rm.js.map +1 -0
  88. package/dist/commands/task/index.d.ts +6 -0
  89. package/dist/commands/task/index.d.ts.map +1 -0
  90. package/dist/commands/task/index.js +13 -0
  91. package/dist/commands/task/index.js.map +1 -0
  92. package/dist/commands/task/run.d.ts +16 -0
  93. package/dist/commands/task/run.d.ts.map +1 -0
  94. package/dist/commands/task/run.js +91 -0
  95. package/dist/commands/task/run.js.map +1 -0
  96. package/dist/commands/validate-schema.d.ts +10 -0
  97. package/dist/commands/validate-schema.d.ts.map +1 -0
  98. package/dist/commands/validate-schema.js +58 -0
  99. package/dist/commands/validate-schema.js.map +1 -0
  100. package/dist/hooks/deprecations.d.ts +4 -0
  101. package/dist/hooks/deprecations.d.ts.map +1 -0
  102. package/dist/hooks/deprecations.js +18 -0
  103. package/dist/hooks/deprecations.js.map +1 -0
  104. package/dist/hooks/init.d.ts +7 -0
  105. package/dist/hooks/init.d.ts.map +1 -0
  106. package/dist/hooks/init.js +17 -0
  107. package/dist/hooks/init.js.map +1 -0
  108. package/dist/index.d.ts +2 -0
  109. package/dist/index.d.ts.map +1 -0
  110. package/dist/index.js +3 -0
  111. package/dist/index.js.map +1 -0
  112. package/dist/lib/actor.d.ts +27 -0
  113. package/dist/lib/actor.d.ts.map +1 -0
  114. package/{src → dist}/lib/actor.js +22 -45
  115. package/dist/lib/actor.js.map +1 -0
  116. package/dist/lib/apify-oclif-help.d.ts +15 -0
  117. package/dist/lib/apify-oclif-help.d.ts.map +1 -0
  118. package/dist/lib/apify-oclif-help.js +19 -0
  119. package/dist/lib/apify-oclif-help.js.map +1 -0
  120. package/dist/lib/apify_command.d.ts +21 -0
  121. package/dist/lib/apify_command.d.ts.map +1 -0
  122. package/dist/lib/apify_command.js +104 -0
  123. package/dist/lib/apify_command.js.map +1 -0
  124. package/dist/lib/commands/resolve-input.d.ts +5 -0
  125. package/dist/lib/commands/resolve-input.d.ts.map +1 -0
  126. package/dist/lib/commands/resolve-input.js +29 -0
  127. package/dist/lib/commands/resolve-input.js.map +1 -0
  128. package/dist/lib/commands/run-on-cloud.d.ts +20 -0
  129. package/dist/lib/commands/run-on-cloud.d.ts.map +1 -0
  130. package/dist/lib/commands/run-on-cloud.js +89 -0
  131. package/dist/lib/commands/run-on-cloud.js.map +1 -0
  132. package/dist/lib/community.d.ts +2 -0
  133. package/dist/lib/community.d.ts.map +1 -0
  134. package/dist/lib/community.js +4 -0
  135. package/dist/lib/community.js.map +1 -0
  136. package/dist/lib/consts.d.ts +54 -0
  137. package/dist/lib/consts.d.ts.map +1 -0
  138. package/dist/lib/consts.js +65 -0
  139. package/dist/lib/consts.js.map +1 -0
  140. package/dist/lib/create-utils.d.ts +8 -0
  141. package/dist/lib/create-utils.d.ts.map +1 -0
  142. package/dist/lib/create-utils.js +148 -0
  143. package/dist/lib/create-utils.js.map +1 -0
  144. package/dist/lib/exec.d.ts +3 -0
  145. package/dist/lib/exec.d.ts.map +1 -0
  146. package/dist/lib/exec.js +40 -0
  147. package/dist/lib/exec.js.map +1 -0
  148. package/dist/lib/files.d.ts +14 -0
  149. package/dist/lib/files.d.ts.map +1 -0
  150. package/dist/lib/files.js +58 -0
  151. package/dist/lib/files.js.map +1 -0
  152. package/dist/lib/input_schema.d.ts +28 -0
  153. package/dist/lib/input_schema.d.ts.map +1 -0
  154. package/dist/lib/input_schema.js +126 -0
  155. package/dist/lib/input_schema.js.map +1 -0
  156. package/dist/lib/local_state.d.ts +11 -0
  157. package/dist/lib/local_state.d.ts.map +1 -0
  158. package/dist/lib/local_state.js +27 -0
  159. package/dist/lib/local_state.js.map +1 -0
  160. package/dist/lib/outputs.d.ts +19 -0
  161. package/dist/lib/outputs.d.ts.map +1 -0
  162. package/dist/lib/outputs.js +45 -0
  163. package/dist/lib/outputs.js.map +1 -0
  164. package/dist/lib/project_analyzer.d.ts +4 -0
  165. package/dist/lib/project_analyzer.d.ts.map +1 -0
  166. package/{src → dist}/lib/project_analyzer.js +6 -11
  167. package/dist/lib/project_analyzer.js.map +1 -0
  168. package/dist/lib/projects/CrawleeAnalyzer.d.ts +4 -0
  169. package/dist/lib/projects/CrawleeAnalyzer.d.ts.map +1 -0
  170. package/{src → dist}/lib/projects/CrawleeAnalyzer.js +7 -24
  171. package/dist/lib/projects/CrawleeAnalyzer.js.map +1 -0
  172. package/dist/lib/projects/OldApifySDKAnalyzer.d.ts +4 -0
  173. package/dist/lib/projects/OldApifySDKAnalyzer.d.ts.map +1 -0
  174. package/{src → dist}/lib/projects/OldApifySDKAnalyzer.js +9 -32
  175. package/dist/lib/projects/OldApifySDKAnalyzer.js.map +1 -0
  176. package/dist/lib/projects/scrapy/ScrapyProjectAnalyzer.d.ts +17 -0
  177. package/dist/lib/projects/scrapy/ScrapyProjectAnalyzer.d.ts.map +1 -0
  178. package/{src/lib/scrapy-wrapper → dist/lib/projects/scrapy}/ScrapyProjectAnalyzer.js +33 -36
  179. package/dist/lib/projects/scrapy/ScrapyProjectAnalyzer.js.map +1 -0
  180. package/dist/lib/projects/scrapy/Spider.d.ts +14 -0
  181. package/dist/lib/projects/scrapy/Spider.d.ts.map +1 -0
  182. package/dist/lib/projects/scrapy/Spider.js +33 -0
  183. package/dist/lib/projects/scrapy/Spider.js.map +1 -0
  184. package/dist/lib/projects/scrapy/SpiderFileAnalyzer.d.ts +7 -0
  185. package/dist/lib/projects/scrapy/SpiderFileAnalyzer.d.ts.map +1 -0
  186. package/dist/lib/projects/scrapy/SpiderFileAnalyzer.js +25 -0
  187. package/dist/lib/projects/scrapy/SpiderFileAnalyzer.js.map +1 -0
  188. package/dist/lib/projects/scrapy/wrapScrapyProject.d.ts +4 -0
  189. package/dist/lib/projects/scrapy/wrapScrapyProject.d.ts.map +1 -0
  190. package/{src/lib/scrapy-wrapper/index.js → dist/lib/projects/scrapy/wrapScrapyProject.js} +43 -73
  191. package/dist/lib/projects/scrapy/wrapScrapyProject.js.map +1 -0
  192. package/dist/lib/projects/shared.d.ts +2 -0
  193. package/dist/lib/projects/shared.d.ts.map +1 -0
  194. package/dist/lib/projects/shared.js +13 -0
  195. package/dist/lib/projects/shared.js.map +1 -0
  196. package/dist/lib/secrets.d.ts +22 -0
  197. package/dist/lib/secrets.d.ts.map +1 -0
  198. package/{src → dist}/lib/secrets.js +34 -42
  199. package/dist/lib/secrets.js.map +1 -0
  200. package/dist/lib/telemetry.d.ts +21 -0
  201. package/dist/lib/telemetry.d.ts.map +1 -0
  202. package/dist/lib/telemetry.js +87 -0
  203. package/dist/lib/telemetry.js.map +1 -0
  204. package/dist/lib/types.d.ts +11 -0
  205. package/dist/lib/types.d.ts.map +1 -0
  206. package/dist/lib/types.js +2 -0
  207. package/dist/lib/types.js.map +1 -0
  208. package/dist/lib/utils.d.ts +102 -0
  209. package/dist/lib/utils.d.ts.map +1 -0
  210. package/dist/lib/utils.js +577 -0
  211. package/dist/lib/utils.js.map +1 -0
  212. package/dist/lib/version_check.d.ts +14 -0
  213. package/dist/lib/version_check.d.ts.map +1 -0
  214. package/{src → dist}/lib/version_check.js +31 -60
  215. package/dist/lib/version_check.js.map +1 -0
  216. package/oclif.manifest.json +867 -1
  217. package/package.json +151 -109
  218. package/index.js +0 -1
  219. package/src/bin/run +0 -16
  220. package/src/commands/actor/get-input.js +0 -12
  221. package/src/commands/actor/get-value.js +0 -23
  222. package/src/commands/actor/index.js +0 -13
  223. package/src/commands/actor/push-data.js +0 -44
  224. package/src/commands/actor/set-value.js +0 -63
  225. package/src/commands/call.js +0 -131
  226. package/src/commands/check-version.js +0 -28
  227. package/src/commands/create.js +0 -203
  228. package/src/commands/info.js +0 -24
  229. package/src/commands/init-wrap-scrapy.js +0 -34
  230. package/src/commands/init.js +0 -83
  231. package/src/commands/login-new.js +0 -169
  232. package/src/commands/login.js +0 -40
  233. package/src/commands/logout.js +0 -18
  234. package/src/commands/pull.js +0 -154
  235. package/src/commands/push.js +0 -234
  236. package/src/commands/run.js +0 -206
  237. package/src/commands/secrets/add.js +0 -28
  238. package/src/commands/secrets/index.js +0 -24
  239. package/src/commands/secrets/rm.js +0 -23
  240. package/src/commands/vis.js +0 -50
  241. package/src/hooks/init.js +0 -16
  242. package/src/lib/apify_command.js +0 -82
  243. package/src/lib/community.js +0 -3
  244. package/src/lib/consts.js +0 -69
  245. package/src/lib/create-utils.js +0 -187
  246. package/src/lib/exec.js +0 -36
  247. package/src/lib/files.js +0 -76
  248. package/src/lib/input_schema.js +0 -114
  249. package/src/lib/local_state.js +0 -39
  250. package/src/lib/outputs.js +0 -27
  251. package/src/lib/scrapy-wrapper/Spider.js +0 -10
  252. package/src/lib/scrapy-wrapper/SpiderFileAnalyzer.js +0 -26
  253. package/src/lib/telemetry.js +0 -104
  254. package/src/lib/utils.js +0 -690
  255. /package/{src/bin → bin}/run.cmd +0 -0
package/src/lib/utils.js DELETED
@@ -1,690 +0,0 @@
1
- const {
2
- execSync,
3
- spawnSync,
4
- } = require('child_process');
5
- const fs = require('fs');
6
- const https = require('https');
7
- const path = require('path');
8
- const { finished } = require('stream');
9
- const { promisify } = require('util');
10
-
11
- const {
12
- ACT_JOB_TERMINAL_STATUSES,
13
- ACTOR_ENV_VARS,
14
- LOCAL_ACTOR_ENV_VARS,
15
- ACTOR_NAME,
16
- APIFY_ENV_VARS,
17
- KEY_VALUE_STORE_KEYS,
18
- LOCAL_STORAGE_SUBDIRS,
19
- SOURCE_FILE_FORMATS,
20
- } = require('@apify/consts');
21
- const AdmZip = require('adm-zip');
22
- const { ApifyClient } = require('apify-client');
23
- const archiver = require('archiver-promise');
24
- const axios = require('axios');
25
- const escapeStringRegexp = require('escape-string-regexp');
26
- const globby = require('globby');
27
- const inquirer = require('inquirer');
28
- const { getEncoding } = require('istextorbinary');
29
- const loadJson = require('load-json-file');
30
- const mime = require('mime');
31
- const semver = require('semver');
32
- const _ = require('underscore');
33
- const writeJson = require('write-json-file');
34
-
35
- const {
36
- GLOBAL_CONFIGS_FOLDER,
37
- AUTH_FILE_PATH,
38
- INPUT_FILE_REG_EXP,
39
- DEFAULT_LOCAL_STORAGE_DIR,
40
- LOCAL_CONFIG_PATH,
41
- DEPRECATED_LOCAL_CONFIG_NAME,
42
- ACTOR_SPECIFICATION_VERSION,
43
- APIFY_CLIENT_DEFAULT_HEADERS,
44
- SUPPORTED_NODEJS_VERSION,
45
- MINIMUM_SUPPORTED_PYTHON_VERSION,
46
- LANGUAGE,
47
- PROJECT_TYPES,
48
- } = require('./consts');
49
- const {
50
- ensureFolderExistsSync,
51
- rimrafPromised,
52
- deleteFile,
53
- } = require('./files');
54
- const {
55
- info,
56
- } = require('./outputs');
57
- const { ProjectAnalyzer } = require('./project_analyzer');
58
-
59
- /**
60
- * @param {string} url
61
- * @returns {Promise<unknown>}
62
- */
63
- const httpsGet = async (url) => {
64
- return new Promise((resolve, reject) => {
65
- https.get(url, (response) => {
66
- // Handle redirects
67
- if (response.statusCode === 301 || response.statusCode === 302) {
68
- resolve(httpsGet(response.headers.location));
69
- // Destroy the response to close the HTTP connection, otherwise this hangs for a long time with Node 19+ (due to HTTP keep-alive).
70
- response.destroy();
71
- } else {
72
- resolve(response);
73
- }
74
- }).on('error', reject);
75
- });
76
- };
77
-
78
- // Properties from apify.json file that will me migrated to actor specs in .actor/actor.json
79
- const MIGRATED_APIFY_JSON_PROPERTIES = ['name', 'version', 'buildTag'];
80
-
81
- const getLocalStorageDir = () => {
82
- const envVar = APIFY_ENV_VARS.LOCAL_STORAGE_DIR;
83
-
84
- return process.env[envVar] || process.env.CRAWLEE_STORAGE_DIR || DEFAULT_LOCAL_STORAGE_DIR;
85
- };
86
- const getLocalKeyValueStorePath = (storeId) => {
87
- const envVar = ACTOR_ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID;
88
- const storeDir = storeId || process.env[envVar] || LOCAL_ACTOR_ENV_VARS[envVar];
89
-
90
- return path.join(getLocalStorageDir(), LOCAL_STORAGE_SUBDIRS.keyValueStores, storeDir);
91
- };
92
- const getLocalDatasetPath = (storeId) => {
93
- const envVar = ACTOR_ENV_VARS.DEFAULT_DATASET_ID;
94
- const storeDir = storeId || process.env[envVar] || LOCAL_ACTOR_ENV_VARS[envVar];
95
-
96
- return path.join(getLocalStorageDir(), LOCAL_STORAGE_SUBDIRS.datasets, storeDir);
97
- };
98
- const getLocalRequestQueuePath = (storeId) => {
99
- const envVar = ACTOR_ENV_VARS.DEFAULT_REQUEST_QUEUE_ID;
100
- const storeDir = storeId || process.env[envVar] || LOCAL_ACTOR_ENV_VARS[envVar];
101
-
102
- return path.join(getLocalStorageDir(), LOCAL_STORAGE_SUBDIRS.requestQueues, storeDir);
103
- };
104
-
105
- /**
106
- * Returns object from auth file or empty object.
107
- * @return {*}
108
- */
109
- const getLocalUserInfo = () => {
110
- try {
111
- return loadJson.sync(AUTH_FILE_PATH) || {};
112
- } catch (e) {
113
- return {};
114
- }
115
- };
116
-
117
- /**
118
- * Gets instance of ApifyClient for user otherwise throws error
119
- * @return {Promise<boolean|*>}
120
- */
121
- const getLoggedClientOrThrow = async () => {
122
- const loggedClient = await getLoggedClient();
123
- if (!loggedClient) {
124
- throw new Error('You are not logged in with your Apify account. Call "apify login" to fix that.');
125
- }
126
- return loggedClient;
127
- };
128
-
129
- /**
130
- * Returns options for ApifyClient
131
- * @param {String|null|undefined} token
132
- * @returns {Object}
133
- */
134
- const getApifyClientOptions = (token, apiBaseUrl) => {
135
- if (!token && fs.existsSync(GLOBAL_CONFIGS_FOLDER) && fs.existsSync(AUTH_FILE_PATH)) {
136
- ({ token } = loadJson.sync(AUTH_FILE_PATH));
137
- }
138
-
139
- return {
140
- token,
141
- baseUrl: apiBaseUrl || process.env.APIFY_CLIENT_BASE_URL,
142
- requestInterceptors: [(config) => {
143
- if (!config.headers) {
144
- config.headers = new axios.AxiosHeaders();
145
- }
146
-
147
- for (const [key, value] of Object.entries(APIFY_CLIENT_DEFAULT_HEADERS)) {
148
- config.headers[key] = value;
149
- }
150
-
151
- return config;
152
- }],
153
- };
154
- };
155
-
156
- /**
157
- * Gets instance of ApifyClient for token or for params from global auth file.
158
- * NOTE: It refreshes global auth file each run
159
- * @param [token]
160
- * @return {Promise<*>}
161
- */
162
- const getLoggedClient = async (token, apiBaseUrl) => {
163
- if (!token && fs.existsSync(GLOBAL_CONFIGS_FOLDER) && fs.existsSync(AUTH_FILE_PATH)) {
164
- ({ token } = loadJson.sync(AUTH_FILE_PATH));
165
- }
166
-
167
- const apifyClient = new ApifyClient(getApifyClientOptions(token, apiBaseUrl));
168
- let userInfo;
169
- try {
170
- userInfo = await apifyClient.user('me').get();
171
- } catch (e) {
172
- return false;
173
- }
174
-
175
- // Always refresh Auth file
176
- if (!fs.existsSync(GLOBAL_CONFIGS_FOLDER)) fs.mkdirSync(GLOBAL_CONFIGS_FOLDER);
177
- writeJson.sync(AUTH_FILE_PATH, { token: apifyClient.token, ...userInfo });
178
- return apifyClient;
179
- };
180
-
181
- const getLocalConfigPath = () => path.join(process.cwd(), LOCAL_CONFIG_PATH);
182
-
183
- /**
184
- * @deprecated Use getLocalConfigPath
185
- * @returns {string}
186
- */
187
- const getDeprecatedLocalConfigPath = () => path.join(process.cwd(), DEPRECATED_LOCAL_CONFIG_NAME);
188
-
189
- const getJsonFileContent = (filePath) => {
190
- if (!fs.existsSync(filePath)) {
191
- return;
192
- }
193
- return loadJson.sync(filePath);
194
- };
195
-
196
- const getLocalConfig = () => getJsonFileContent(getLocalConfigPath());
197
-
198
- /**
199
- * @deprecated Use getLocalConfig
200
- * @returns {string}
201
- */
202
- const getDeprecatedLocalConfig = () => getJsonFileContent(getDeprecatedLocalConfigPath());
203
-
204
- const getLocalConfigOrThrow = async () => {
205
- let localConfig = getLocalConfig();
206
- let deprecatedLocalConfig = getDeprecatedLocalConfig();
207
-
208
- if (localConfig && deprecatedLocalConfig) {
209
- const answer = await inquirer.prompt([{
210
- name: 'isConfirm',
211
- type: 'confirm',
212
-
213
- message: `The new version of Apify CLI uses the "${LOCAL_CONFIG_PATH}" instead of the "apify.json" file. Since we have found both files in your actor directory, "apify.json" will be renamed to "apify.json.deprecated". Going forward, all commands will use "${LOCAL_CONFIG_PATH}". You can read about the differences between the old and the new config at https://github.com/apify/apify-cli/blob/master/MIGRATIONS.md. Do you want to continue?`,
214
- }]);
215
- if (!answer.isConfirm) {
216
- throw new Error('Command can not run with old "apify.json" file present in your actor directory., Please, either rename or remove it.');
217
- }
218
- try {
219
- fs.renameSync(getDeprecatedLocalConfigPath(), `${getDeprecatedLocalConfigPath()}.deprecated`);
220
- // eslint-disable-next-line max-len
221
- info(`The "apify.json" file has been renamed to "apify.json.deprecated". The deprecated file is no longer used by the CLI or Apify Console. If you do not need it for some specific purpose, it can be safely deleted.`);
222
- } catch (e) {
223
- throw new Error('Failed to rename deprecated "apify.json".');
224
- }
225
- }
226
-
227
- if (!localConfig && !deprecatedLocalConfig) {
228
- return {};
229
- }
230
-
231
- // If apify.json exists migrate it to .actor/actor.json
232
- if (!localConfig && deprecatedLocalConfig) {
233
- const answer = await inquirer.prompt([{
234
- name: 'isConfirm',
235
- type: 'confirm',
236
- // eslint-disable-next-line max-len
237
- message: `The new version of Apify CLI uses the "${LOCAL_CONFIG_PATH}" instead of the "apify.json" file. Your "apify.json" file will be automatically updated to the new format under "${LOCAL_CONFIG_PATH}". The original file will be renamed by adding the ".deprecated" suffix. Do you want to continue?`,
238
- }]);
239
- if (!answer.isConfirm) {
240
- throw new Error('Command can not run with old apify.json structure. Either let the CLI auto-update it or follow the guide on https://github.com/apify/apify-cli/blob/master/MIGRATIONS.md and update it manually.');
241
- }
242
- try {
243
- // Check if apify.json contains old deprecated structure. If so, updates it.
244
- if (_.isObject(deprecatedLocalConfig.version)) {
245
- deprecatedLocalConfig = updateLocalConfigStructure(deprecatedLocalConfig);
246
- }
247
- localConfig = {
248
- actorSpecification: ACTOR_SPECIFICATION_VERSION,
249
- environmentVariables: deprecatedLocalConfig.env || undefined,
250
- ..._.pick(deprecatedLocalConfig, MIGRATED_APIFY_JSON_PROPERTIES),
251
- };
252
-
253
- writeJson.sync(getLocalConfigPath(), localConfig);
254
- fs.renameSync(getDeprecatedLocalConfigPath(), `${getDeprecatedLocalConfigPath()}.deprecated`);
255
- // eslint-disable-next-line max-len
256
- info(`The "apify.json" file has been migrated to "${LOCAL_CONFIG_PATH}" and the original file renamed to "apify.json.deprecated". The deprecated file is no longer used by the CLI or Apify Console. If you do not need it for some specific purpose, it can be safely deleted. Do not forget to commit the new file to your Git repository.`);
257
- } catch (e) {
258
- throw new Error(`Can not update "${LOCAL_CONFIG_PATH}" structure. Follow guide on https://github.com/apify/apify-cli/blob/master/MIGRATIONS.md and update it manually.`);
259
- }
260
- }
261
-
262
- return localConfig;
263
- };
264
-
265
- const setLocalConfig = async (localConfig, actDir) => {
266
- actDir = actDir || process.cwd();
267
- writeJson.sync(path.join(actDir, LOCAL_CONFIG_PATH), localConfig);
268
- };
269
-
270
- const GITIGNORE_REQUIRED_CONTENTS = [getLocalStorageDir(), 'node_modules', '.venv'];
271
-
272
- const setLocalEnv = async (actDir) => {
273
- // Create folders for emulation Apify stores
274
- const keyValueStorePath = getLocalKeyValueStorePath();
275
- ensureFolderExistsSync(actDir, getLocalDatasetPath());
276
- ensureFolderExistsSync(actDir, getLocalRequestQueuePath());
277
- ensureFolderExistsSync(actDir, keyValueStorePath);
278
-
279
- // Create or update gitignore
280
- const gitignorePath = path.join(actDir, '.gitignore');
281
- let gitignoreContents = '';
282
- if (fs.existsSync(gitignorePath)) {
283
- gitignoreContents = fs.readFileSync(gitignorePath, { encoding: 'utf-8' });
284
- }
285
-
286
- const gitignoreAdditions = [];
287
- for (const gitignoreRequirement of GITIGNORE_REQUIRED_CONTENTS) {
288
- if (!RegExp(`^${escapeStringRegexp(gitignoreRequirement)}$`, 'mg').test(gitignoreContents)) {
289
- gitignoreAdditions.push(gitignoreRequirement);
290
- }
291
- }
292
-
293
- if (gitignoreAdditions.length > 0) {
294
- if (gitignoreContents.length > 0) {
295
- gitignoreAdditions.unshift('# Added by Apify CLI');
296
- fs.writeFileSync(gitignorePath, `\n${gitignoreAdditions.join('\n')}\n`, { flag: 'a' });
297
- } else {
298
- fs.writeFileSync(gitignorePath, `${gitignoreAdditions.join('\n')}\n`, { flag: 'w' });
299
- }
300
- }
301
- };
302
-
303
- /**
304
- * Convert Object with kebab-case keys to camelCased keys
305
- *
306
- * @param object
307
- * @return {{}}
308
- */
309
- const argsToCamelCase = (object) => {
310
- const camelCasedObject = {};
311
- Object.keys(object).forEach((arg) => {
312
- const camelCasedArg = arg.replace(/-(.)/g, ($1) => $1.toUpperCase()).replace(/-/g, '');
313
- camelCasedObject[camelCasedArg] = object[arg];
314
- });
315
- return camelCasedObject;
316
- };
317
-
318
- // Detect whether file is binary from its MIME type, or if not available, contents
319
- const getSourceFileFormat = (filePath, fileContent) => {
320
- // Try to detect the MIME type from the file path
321
- // .tgz files don't have a MIME type defined, this fixes it
322
- mime.define({ 'application/gzip': ['tgz'] }, true);
323
- // Default mime-type for .ts(x) files is video/mp2t. But in our usecases they're almost always TypeScript, which we want to treat as text
324
- mime.define({ 'text/typescript': ['ts', 'tsx', 'mts'] }, true);
325
-
326
- const contentType = mime.getType(filePath);
327
- if (contentType) {
328
- const format = (
329
- contentType.startsWith('text/')
330
- || contentType.includes('javascript')
331
- || contentType.includes('json')
332
- || contentType.includes('xml')
333
- || contentType.includes('application/node') // .cjs files
334
- || contentType.includes('application/toml') // for example pyproject.toml files
335
- || contentType.includes('application/x-httpd-php') // .php files
336
- )
337
- ? SOURCE_FILE_FORMATS.TEXT
338
- : SOURCE_FILE_FORMATS.BASE64;
339
- return format;
340
- }
341
-
342
- // If the MIME type detection failed, try to detect the file encoding from the file content
343
- const encoding = getEncoding(fileContent);
344
- return encoding === 'binary' ? SOURCE_FILE_FORMATS.BASE64 : SOURCE_FILE_FORMATS.TEXT;
345
- };
346
-
347
- const createSourceFiles = async (paths) => {
348
- return paths.map((filePath) => {
349
- const file = fs.readFileSync(filePath);
350
- const format = getSourceFileFormat(filePath, file);
351
- return {
352
- name: filePath,
353
- format,
354
- content: format === SOURCE_FILE_FORMATS.TEXT
355
- ? file.toString('utf8')
356
- : file.toString('base64'),
357
- };
358
- });
359
- };
360
-
361
- /**
362
- * Get actor local files, omit files defined in .gitignore and .git folder
363
- * All dot files(.file) and folders(.folder/) are included.
364
- */
365
- const getActorLocalFilePaths = () => globby(['*', '**/**'], {
366
- ignore: ['.git/**', 'apify_storage', 'node_modules', 'storage', 'crawlee_storage'],
367
- gitignore: true,
368
- dot: true,
369
- });
370
-
371
- /**
372
- * Create zip file with all actor files specified with pathsToZip
373
- * @param zipName
374
- * @param pathsToZip
375
- * @return {Promise<void>}
376
- */
377
- const createActZip = async (zipName, pathsToZip) => {
378
- // NOTE: There can be a zip from a previous unfinished operation.
379
- if (fs.existsSync(zipName)) await deleteFile(zipName);
380
-
381
- const archive = archiver(zipName);
382
-
383
- const archiveFilesPromises = [];
384
- pathsToZip.forEach((globPath) => archiveFilesPromises.push(archive.glob(globPath)));
385
- await Promise.all(archiveFilesPromises);
386
-
387
- await archive.finalize();
388
- };
389
-
390
- /**
391
- * Get actor input from local store
392
- * @return {{body: *, contentType: string}}
393
- */
394
- const getLocalInput = () => {
395
- const defaultLocalStorePath = getLocalKeyValueStorePath();
396
- const files = fs.readdirSync(defaultLocalStorePath);
397
- const inputName = files.find((file) => !!file.match(INPUT_FILE_REG_EXP));
398
-
399
- // No input file
400
- if (!inputName) return;
401
-
402
- const input = fs.readFileSync(path.join(defaultLocalStorePath, inputName));
403
- const contentType = mime.getType(inputName);
404
- return { body: input, contentType };
405
- };
406
-
407
- const purgeDefaultQueue = async () => {
408
- const defaultQueuesPath = getLocalRequestQueuePath();
409
- if (fs.existsSync(getLocalStorageDir()) && fs.existsSync(defaultQueuesPath)) {
410
- await rimrafPromised(defaultQueuesPath);
411
- }
412
- };
413
-
414
- const purgeDefaultDataset = async () => {
415
- const defaultDatasetPath = getLocalDatasetPath();
416
- if (fs.existsSync(getLocalStorageDir()) && fs.existsSync(defaultDatasetPath)) {
417
- await rimrafPromised(defaultDatasetPath);
418
- }
419
- };
420
-
421
- const purgeDefaultKeyValueStore = async () => {
422
- const defaultKeyValueStorePath = getLocalKeyValueStorePath();
423
- if (!fs.existsSync(getLocalStorageDir()) || !fs.existsSync(defaultKeyValueStorePath)) {
424
- return;
425
- }
426
- const filesToDelete = fs.readdirSync(defaultKeyValueStorePath);
427
-
428
- const deletePromises = [];
429
- filesToDelete.forEach((file) => {
430
- if (!file.match(INPUT_FILE_REG_EXP)) deletePromises.push(deleteFile(path.join(defaultKeyValueStorePath, file)));
431
- });
432
-
433
- await Promise.all(deletePromises);
434
- };
435
-
436
- const outputJobLog = async (job, timeout) => {
437
- const { id: logId, status } = job;
438
- // In case job was already done just output log
439
- if (ACT_JOB_TERMINAL_STATUSES.includes(status)) {
440
- const apifyClient = new ApifyClient({ baseUrl: process.env.APIFY_CLIENT_BASE_URL });
441
- const log = await apifyClient.log(logId).get();
442
- process.stdout.write(log);
443
- }
444
-
445
- // In other case stream it to stdout
446
- // TODO: Use apifyClient.log(logId).stream() instead of http request stream.
447
- return new Promise((resolve, reject) => {
448
- const req = https.get(`https://api.apify.com/v2/logs/${logId}?stream=1`);
449
- let res;
450
-
451
- req.on('response', (response) => {
452
- res = response;
453
- response.on('data', (chunk) => process.stdout.write(chunk.toString()));
454
- response.on('error', (err) => {
455
- reject(err);
456
- });
457
- });
458
- req.on('error', (err) => {
459
- reject(err);
460
- });
461
- req.on('close', () => {
462
- resolve('finished');
463
- });
464
-
465
- if (timeout) {
466
- setTimeout(() => {
467
- if (res) res.removeAllListeners();
468
- if (req) {
469
- req.removeAllListeners();
470
- req.abort();
471
- }
472
- resolve('timeouts');
473
- }, timeout);
474
- }
475
- });
476
- };
477
-
478
- /**
479
- * Returns npm command for current os
480
- * NOTE: For window we have to returns npm.cmd instead of npm, otherwise it doesn't work
481
- * @return {string}
482
- */
483
- const getNpmCmd = () => {
484
- return /^win/.test(process.platform) ? 'npm.cmd' : 'npm';
485
- };
486
-
487
- /**
488
- * Returns true if apify storage is empty (expect INPUT.*)
489
- * @return {Promise<boolean>}
490
- */
491
- const checkIfStorageIsEmpty = async () => {
492
- const filesWithoutInput = await globby([
493
- `${getLocalStorageDir()}/**`,
494
- // Omit INPUT.* file
495
- `!${getLocalKeyValueStorePath()}/${KEY_VALUE_STORE_KEYS.INPUT}.*`,
496
- ]);
497
- return filesWithoutInput.length === 0;
498
- };
499
-
500
- /**
501
- * Show help for command
502
- * NOTE: This is not nice, but I can not find other way..
503
- * @param command
504
- */
505
- const showHelpForCommand = (command) => {
506
- execSync(`apify ${command} --help`, { stdio: [0, 1, 2] });
507
- };
508
-
509
- /**
510
- * Migration for deprecated structure of apify.json to latest.
511
- * @param localConfig
512
- */
513
- const updateLocalConfigStructure = (localConfig) => {
514
- const updatedLocalConfig = {
515
- name: localConfig.name,
516
- template: localConfig.template,
517
- version: localConfig.version.versionNumber,
518
- buildTag: localConfig.version.buildTag,
519
- env: null,
520
- };
521
- if (localConfig.version.envVars && localConfig.version.envVars.length) {
522
- const env = {};
523
- localConfig.version.envVars.forEach((envVar) => {
524
- if (envVar.name && envVar.value) env[envVar.name] = envVar.value;
525
- });
526
- updatedLocalConfig.env = env;
527
- }
528
- return updatedLocalConfig;
529
- };
530
-
531
- /**
532
- * Validates actor name, if finds issue throws error.
533
- * @param actorName
534
- */
535
- const validateActorName = (actorName) => {
536
- if (!ACTOR_NAME.REGEX.test(actorName)) {
537
- throw new Error('The Actor name must be a DNS hostname-friendly string (e.g. my-newest-actor).');
538
- }
539
- if (actorName.length < ACTOR_NAME.MIN_LENGTH) {
540
- throw new Error('The Actor name must be at least 3 characters long.');
541
- }
542
- if (actorName.length > ACTOR_NAME.MAX_LENGTH) {
543
- throw new Error('The Actor name must be a maximum of 30 characters long.');
544
- }
545
- };
546
-
547
- const sanitizeActorName = (actorName) => {
548
- let sanitizedName = actorName
549
- .replaceAll(/[^a-zA-Z0-9-]/g, '-');
550
-
551
- if (sanitizedName.length < ACTOR_NAME.MIN_LENGTH) {
552
- sanitizedName = `${sanitizedName}-apify-actor`;
553
- }
554
-
555
- sanitizedName = sanitizedName.replaceAll(/^-+/g, '').replaceAll(/-+$/g, '');
556
-
557
- return sanitizedName.slice(0, ACTOR_NAME.MAX_LENGTH);
558
- };
559
-
560
- const getPythonCommand = (directory) => {
561
- const pythonVenvPath = /^win/.test(process.platform)
562
- ? 'Scripts/python.exe'
563
- : 'bin/python3';
564
-
565
- let fullPythonVenvPath;
566
- if (process.env.VIRTUAL_ENV) {
567
- fullPythonVenvPath = path.join(process.env.VIRTUAL_ENV, pythonVenvPath);
568
- } else {
569
- fullPythonVenvPath = path.join(directory, '.venv', pythonVenvPath);
570
- }
571
-
572
- if (fs.existsSync(fullPythonVenvPath)) {
573
- return fullPythonVenvPath;
574
- }
575
-
576
- return /^win/.test(process.platform)
577
- ? 'python'
578
- : 'python3';
579
- };
580
-
581
- const detectPythonVersion = (directory) => {
582
- const pythonCommand = getPythonCommand(directory);
583
- try {
584
- const spawnResult = spawnSync(pythonCommand, ['-c', 'import platform; print(platform.python_version())'], { encoding: 'utf-8' });
585
- if (!spawnResult.error && spawnResult.stdout) {
586
- return spawnResult.stdout.trim();
587
- }
588
- } catch {
589
- return undefined;
590
- }
591
- };
592
-
593
- const isPythonVersionSupported = (installedPythonVersion) => {
594
- return semver.satisfies(installedPythonVersion, `^${MINIMUM_SUPPORTED_PYTHON_VERSION}`);
595
- };
596
-
597
- const detectNodeVersion = () => {
598
- try {
599
- const spawnResult = spawnSync('node', ['--version'], { encoding: 'utf-8' });
600
- if (!spawnResult.error && spawnResult.stdout) {
601
- return spawnResult.stdout.trim().replace(/^v/, '');
602
- }
603
- } catch {
604
- return undefined;
605
- }
606
- };
607
-
608
- const isNodeVersionSupported = (installedNodeVersion) => {
609
- // SUPPORTED_NODEJS_VERSION can be a version range,
610
- // we need to get the minimum supported version from that range to be able to compare them
611
- const minimumSupportedNodeVersion = semver.minVersion(SUPPORTED_NODEJS_VERSION);
612
- return semver.gte(installedNodeVersion, minimumSupportedNodeVersion);
613
- };
614
-
615
- const detectNpmVersion = () => {
616
- const npmCommand = getNpmCmd();
617
- try {
618
- const spawnResult = spawnSync(npmCommand, ['--version'], { encoding: 'utf-8' });
619
- if (!spawnResult.error && spawnResult.stdout) {
620
- return spawnResult.stdout.trim().replace(/^v/, '');
621
- }
622
- } catch {
623
- return undefined;
624
- }
625
- };
626
-
627
- const detectLocalActorLanguage = () => {
628
- const cwd = process.cwd();
629
- const isActorInNode = fs.existsSync(path.join(cwd, 'package.json'));
630
- const isActorInPython = fs.existsSync(path.join(cwd, 'src/__main__.py')) || ProjectAnalyzer.getProjectType(cwd) === PROJECT_TYPES.SCRAPY;
631
- const result = {};
632
- if (isActorInNode) {
633
- result.language = LANGUAGE.NODEJS;
634
- result.languageVersion = detectNodeVersion();
635
- } else if (isActorInPython) {
636
- result.language = LANGUAGE.PYTHON;
637
- result.languageVersion = detectPythonVersion(cwd);
638
- } else {
639
- result.language = LANGUAGE.UNKNOWN;
640
- }
641
- return result;
642
- };
643
-
644
- const downloadAndUnzip = async ({ url, pathTo }) => {
645
- const zipStream = await httpsGet(url);
646
- const chunks = [];
647
- zipStream.on('data', (chunk) => chunks.push(chunk));
648
- await promisify(finished)(zipStream);
649
- const zip = new AdmZip(Buffer.concat(chunks));
650
- zip.extractAllTo(pathTo, true);
651
- };
652
-
653
- module.exports = {
654
- httpsGet,
655
- getLoggedClientOrThrow,
656
- getLocalConfig,
657
- setLocalConfig,
658
- setLocalEnv,
659
- argsToCamelCase,
660
- getLoggedClient,
661
- createActZip,
662
- getLocalUserInfo,
663
- getLocalConfigOrThrow,
664
- getLocalInput,
665
- purgeDefaultQueue,
666
- purgeDefaultDataset,
667
- purgeDefaultKeyValueStore,
668
- outputJobLog,
669
- getLocalKeyValueStorePath,
670
- getLocalDatasetPath,
671
- getLocalRequestQueuePath,
672
- getNpmCmd,
673
- checkIfStorageIsEmpty,
674
- getLocalStorageDir,
675
- showHelpForCommand,
676
- getActorLocalFilePaths,
677
- createSourceFiles,
678
- validateActorName,
679
- getJsonFileContent,
680
- getApifyClientOptions,
681
- detectPythonVersion,
682
- isPythonVersionSupported,
683
- getPythonCommand,
684
- detectNodeVersion,
685
- isNodeVersionSupported,
686
- detectNpmVersion,
687
- detectLocalActorLanguage,
688
- downloadAndUnzip,
689
- sanitizeActorName,
690
- };
File without changes