apify 2.0.8-beta.6 → 2.1.0-beta.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (217) hide show
  1. package/README.md +3 -0
  2. package/{types → build}/actor.d.ts +1 -0
  3. package/build/actor.d.ts.map +1 -0
  4. package/build/actor.js +13 -12
  5. package/build/actor.js.map +1 -0
  6. package/{types → build}/apify.d.ts +3 -0
  7. package/build/apify.d.ts.map +1 -0
  8. package/build/apify.js +4 -2
  9. package/build/apify.js.map +1 -0
  10. package/{types → build}/autoscaling/autoscaled_pool.d.ts +1 -0
  11. package/build/autoscaling/autoscaled_pool.d.ts.map +1 -0
  12. package/build/autoscaling/autoscaled_pool.js +6 -4
  13. package/build/autoscaling/autoscaled_pool.js.map +1 -0
  14. package/{types → build}/autoscaling/snapshotter.d.ts +1 -0
  15. package/build/autoscaling/snapshotter.d.ts.map +1 -0
  16. package/build/autoscaling/snapshotter.js +7 -5
  17. package/build/autoscaling/snapshotter.js.map +1 -0
  18. package/{types → build}/autoscaling/system_status.d.ts +1 -0
  19. package/build/autoscaling/system_status.d.ts.map +1 -0
  20. package/build/autoscaling/system_status.js +4 -2
  21. package/build/autoscaling/system_status.js.map +1 -0
  22. package/{types → build}/browser_launchers/browser_launcher.d.ts +5 -0
  23. package/build/browser_launchers/browser_launcher.d.ts.map +1 -0
  24. package/build/browser_launchers/browser_launcher.js +15 -8
  25. package/build/browser_launchers/browser_launcher.js.map +1 -0
  26. package/{types → build}/browser_launchers/browser_plugin.d.ts +1 -0
  27. package/build/browser_launchers/browser_plugin.d.ts.map +1 -0
  28. package/build/browser_launchers/browser_plugin.js +1 -0
  29. package/build/browser_launchers/browser_plugin.js.map +1 -0
  30. package/{types → build}/browser_launchers/playwright_launcher.d.ts +6 -0
  31. package/build/browser_launchers/playwright_launcher.d.ts.map +1 -0
  32. package/build/browser_launchers/playwright_launcher.js +13 -6
  33. package/build/browser_launchers/playwright_launcher.js.map +1 -0
  34. package/{types → build}/browser_launchers/puppeteer_launcher.d.ts +1 -0
  35. package/build/browser_launchers/puppeteer_launcher.d.ts.map +1 -0
  36. package/build/browser_launchers/puppeteer_launcher.js +17 -10
  37. package/build/browser_launchers/puppeteer_launcher.js.map +1 -0
  38. package/{types → build}/cache_container.d.ts +1 -0
  39. package/build/cache_container.d.ts.map +1 -0
  40. package/build/cache_container.js +1 -0
  41. package/build/cache_container.js.map +1 -0
  42. package/{types → build}/configuration.d.ts +1 -0
  43. package/build/configuration.d.ts.map +1 -0
  44. package/build/configuration.js +88 -65
  45. package/build/configuration.js.map +1 -0
  46. package/{types → build}/constants.d.ts +1 -0
  47. package/build/constants.d.ts.map +1 -0
  48. package/build/constants.js +1 -0
  49. package/build/constants.js.map +1 -0
  50. package/{types → build}/crawlers/basic_crawler.d.ts +1 -0
  51. package/build/crawlers/basic_crawler.d.ts.map +1 -0
  52. package/build/crawlers/basic_crawler.js +35 -28
  53. package/build/crawlers/basic_crawler.js.map +1 -0
  54. package/{types → build}/crawlers/browser_crawler.d.ts +1 -0
  55. package/build/crawlers/browser_crawler.d.ts.map +1 -0
  56. package/build/crawlers/browser_crawler.js +26 -19
  57. package/build/crawlers/browser_crawler.js.map +1 -0
  58. package/{types → build}/crawlers/cheerio_crawler.d.ts +1 -0
  59. package/build/crawlers/cheerio_crawler.d.ts.map +1 -0
  60. package/build/crawlers/cheerio_crawler.js +41 -34
  61. package/build/crawlers/cheerio_crawler.js.map +1 -0
  62. package/{types → build}/crawlers/crawler_extension.d.ts +1 -0
  63. package/build/crawlers/crawler_extension.d.ts.map +1 -0
  64. package/build/crawlers/crawler_extension.js +3 -1
  65. package/build/crawlers/crawler_extension.js.map +1 -0
  66. package/{types → build}/crawlers/crawler_utils.d.ts +1 -0
  67. package/build/crawlers/crawler_utils.d.ts.map +1 -0
  68. package/build/crawlers/crawler_utils.js +1 -0
  69. package/build/crawlers/crawler_utils.js.map +1 -0
  70. package/{types → build}/crawlers/playwright_crawler.d.ts +2 -0
  71. package/build/crawlers/playwright_crawler.d.ts.map +1 -0
  72. package/build/crawlers/playwright_crawler.js +15 -8
  73. package/build/crawlers/playwright_crawler.js.map +1 -0
  74. package/{types → build}/crawlers/puppeteer_crawler.d.ts +1 -0
  75. package/build/crawlers/puppeteer_crawler.d.ts.map +1 -0
  76. package/build/crawlers/puppeteer_crawler.js +15 -8
  77. package/build/crawlers/puppeteer_crawler.js.map +1 -0
  78. package/{types → build}/crawlers/statistics.d.ts +1 -0
  79. package/build/crawlers/statistics.d.ts.map +1 -0
  80. package/build/crawlers/statistics.js +5 -3
  81. package/build/crawlers/statistics.js.map +1 -0
  82. package/{types → build}/enqueue_links/click_elements.d.ts +1 -0
  83. package/build/enqueue_links/click_elements.d.ts.map +1 -0
  84. package/build/enqueue_links/click_elements.js +4 -2
  85. package/build/enqueue_links/click_elements.js.map +1 -0
  86. package/{types → build}/enqueue_links/enqueue_links.d.ts +2 -0
  87. package/build/enqueue_links/enqueue_links.d.ts.map +1 -0
  88. package/build/enqueue_links/enqueue_links.js +4 -2
  89. package/build/enqueue_links/enqueue_links.js.map +1 -0
  90. package/{types → build}/enqueue_links/shared.d.ts +1 -0
  91. package/build/enqueue_links/shared.d.ts.map +1 -0
  92. package/build/enqueue_links/shared.js +6 -4
  93. package/build/enqueue_links/shared.js.map +1 -0
  94. package/{types → build}/errors.d.ts +1 -0
  95. package/build/errors.d.ts.map +1 -0
  96. package/build/errors.js +1 -0
  97. package/build/errors.js.map +1 -0
  98. package/{types → build}/events.d.ts +1 -0
  99. package/build/events.d.ts.map +1 -0
  100. package/build/events.js +5 -3
  101. package/build/events.js.map +1 -0
  102. package/{types → build}/index.d.ts +1 -0
  103. package/build/index.d.ts.map +1 -0
  104. package/build/index.js +4 -12
  105. package/build/index.js.map +1 -0
  106. package/{types → build}/main.d.ts +3 -1
  107. package/build/main.d.ts.map +1 -0
  108. package/build/main.js +11 -9
  109. package/build/main.js.map +1 -0
  110. package/{types → build}/playwright_utils.d.ts +3 -3
  111. package/build/playwright_utils.d.ts.map +1 -0
  112. package/build/playwright_utils.js +25 -18
  113. package/build/playwright_utils.js.map +1 -0
  114. package/{types → build}/proxy_configuration.d.ts +1 -0
  115. package/build/proxy_configuration.d.ts.map +1 -0
  116. package/build/proxy_configuration.js +8 -2
  117. package/build/proxy_configuration.js.map +1 -0
  118. package/{types → build}/pseudo_url.d.ts +1 -0
  119. package/build/pseudo_url.d.ts.map +1 -0
  120. package/build/pseudo_url.js +7 -5
  121. package/build/pseudo_url.js.map +1 -0
  122. package/{types → build}/puppeteer_request_interception.d.ts +1 -0
  123. package/build/puppeteer_request_interception.d.ts.map +1 -0
  124. package/build/puppeteer_request_interception.js +15 -7
  125. package/build/puppeteer_request_interception.js.map +1 -0
  126. package/{types → build}/puppeteer_utils.d.ts +1 -0
  127. package/build/puppeteer_utils.d.ts.map +1 -0
  128. package/build/puppeteer_utils.js +23 -19
  129. package/build/puppeteer_utils.js.map +1 -0
  130. package/{types → build}/request.d.ts +1 -0
  131. package/build/request.d.ts.map +1 -0
  132. package/build/request.js +8 -6
  133. package/build/request.js.map +1 -0
  134. package/{types → build}/request_list.d.ts +17 -3
  135. package/build/request_list.d.ts.map +1 -0
  136. package/build/request_list.js +36 -16
  137. package/build/request_list.js.map +1 -0
  138. package/{types → build}/serialization.d.ts +1 -0
  139. package/build/serialization.d.ts.map +1 -0
  140. package/build/serialization.js +14 -12
  141. package/build/serialization.js.map +1 -0
  142. package/{types → build}/session_pool/errors.d.ts +1 -0
  143. package/build/session_pool/errors.d.ts.map +1 -0
  144. package/build/session_pool/errors.js +1 -0
  145. package/build/session_pool/errors.js.map +1 -0
  146. package/{types → build}/session_pool/events.d.ts +1 -0
  147. package/build/session_pool/events.d.ts.map +1 -0
  148. package/build/session_pool/events.js +1 -0
  149. package/build/session_pool/events.js.map +1 -0
  150. package/{types → build}/session_pool/session.d.ts +1 -0
  151. package/build/session_pool/session.d.ts.map +1 -0
  152. package/build/session_pool/session.js +5 -3
  153. package/build/session_pool/session.js.map +1 -0
  154. package/{types → build}/session_pool/session_pool.d.ts +1 -0
  155. package/build/session_pool/session_pool.d.ts.map +1 -0
  156. package/build/session_pool/session_pool.js +5 -3
  157. package/build/session_pool/session_pool.js.map +1 -0
  158. package/{types → build}/session_pool/session_utils.d.ts +1 -0
  159. package/build/session_pool/session_utils.d.ts.map +1 -0
  160. package/build/session_pool/session_utils.js +1 -0
  161. package/build/session_pool/session_utils.js.map +1 -0
  162. package/{types → build}/stealth/hiding_tricks.d.ts +1 -0
  163. package/build/stealth/hiding_tricks.d.ts.map +1 -0
  164. package/build/stealth/hiding_tricks.js +1 -0
  165. package/build/stealth/hiding_tricks.js.map +1 -0
  166. package/{types → build}/stealth/stealth.d.ts +1 -0
  167. package/build/stealth/stealth.d.ts.map +1 -0
  168. package/build/stealth/stealth.js +6 -4
  169. package/build/stealth/stealth.js.map +1 -0
  170. package/{types → build}/storages/dataset.d.ts +1 -0
  171. package/build/storages/dataset.d.ts.map +1 -0
  172. package/build/storages/dataset.js +7 -5
  173. package/build/storages/dataset.js.map +1 -0
  174. package/{types → build}/storages/key_value_store.d.ts +1 -0
  175. package/build/storages/key_value_store.d.ts.map +1 -0
  176. package/build/storages/key_value_store.js +4 -2
  177. package/build/storages/key_value_store.js.map +1 -0
  178. package/{types → build}/storages/request_queue.d.ts +1 -0
  179. package/build/storages/request_queue.d.ts.map +1 -0
  180. package/build/storages/request_queue.js +7 -5
  181. package/build/storages/request_queue.js.map +1 -0
  182. package/{types → build}/storages/storage_manager.d.ts +1 -0
  183. package/build/storages/storage_manager.d.ts.map +1 -0
  184. package/build/storages/storage_manager.js +3 -1
  185. package/build/storages/storage_manager.js.map +1 -0
  186. package/build/tsconfig.tsbuildinfo +1 -0
  187. package/{types → build}/typedefs.d.ts +1 -0
  188. package/build/typedefs.d.ts.map +1 -0
  189. package/build/typedefs.js +1 -0
  190. package/build/typedefs.js.map +1 -0
  191. package/{types → build}/utils.d.ts +3 -0
  192. package/build/utils.d.ts.map +1 -0
  193. package/build/utils.js +104 -46
  194. package/build/utils.js.map +1 -0
  195. package/{types → build}/utils_log.d.ts +1 -0
  196. package/build/utils_log.d.ts.map +1 -0
  197. package/build/utils_log.js +3 -1
  198. package/build/utils_log.js.map +1 -0
  199. package/{types → build}/utils_request.d.ts +1 -0
  200. package/build/utils_request.d.ts.map +1 -0
  201. package/build/utils_request.js +5 -3
  202. package/build/utils_request.js.map +1 -0
  203. package/{types → build}/utils_social.d.ts +1 -0
  204. package/build/utils_social.d.ts.map +1 -0
  205. package/build/utils_social.js +23 -11
  206. package/build/utils_social.js.map +1 -0
  207. package/{types → build}/validators.d.ts +1 -0
  208. package/build/validators.d.ts.map +1 -0
  209. package/build/validators.js +3 -1
  210. package/build/validators.js.map +1 -0
  211. package/package.json +26 -29
  212. package/types/playwright/index.d.ts +0 -22
  213. package/types/playwright/types/protocol.d.ts +0 -17846
  214. package/types/playwright/types/structs.d.ts +0 -50
  215. package/types/playwright/types/types.d.ts +0 -13078
  216. package/types-apify/apify-http-request.d.ts +0 -8
  217. package/types-apify/ps-tree.d.ts +0 -4
package/build/utils.js CHANGED
@@ -1,28 +1,31 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.publicUtils = exports.waitForRunToFinish = exports.parseContentTypeFromResponse = exports.printOutdatedSdkWarning = exports.snakeCaseToCamelCase = exports.sleep = exports.isAtHome = exports.addTimeoutToPromise = exports.getTypicalChromeExecutablePath = exports.getFirstKey = exports.getMemoryInfo = exports.weightedAvg = exports.isDocker = exports.addCharsetToContentType = exports.apifyClient = exports.logSystemInfo = exports.newClient = void 0;
4
- const psTree = require("@apify/ps-tree");
3
+ exports.publicUtils = exports.purgeLocalStorage = exports.waitForRunToFinish = exports.parseContentTypeFromResponse = exports.printOutdatedSdkWarning = exports.snakeCaseToCamelCase = exports.sleep = exports.isAtHome = exports.addTimeoutToPromise = exports.getTypicalChromeExecutablePath = exports.getFirstKey = exports.getMemoryInfo = exports.weightedAvg = exports.isDocker = exports.addCharsetToContentType = exports.apifyClient = exports.logSystemInfo = exports.newClient = void 0;
4
+ const tslib_1 = require("tslib");
5
+ const ps_tree_1 = (0, tslib_1.__importDefault)(require("@apify/ps-tree"));
5
6
  // eslint-disable-next-line import/no-extraneous-dependencies
6
7
  const child_process_1 = require("child_process");
7
- const ApifyClient = require("apify-client");
8
+ const apify_client_1 = (0, tslib_1.__importDefault)(require("apify-client"));
8
9
  const package_json_1 = require("apify-client/package.json");
9
10
  const consts_1 = require("@apify/consts");
10
- const cheerio = require("cheerio");
11
- const contentTypeParser = require("content-type");
12
- const fs = require("fs");
13
- const mime = require("mime-types");
14
- const os = require("os");
11
+ const cheerio_1 = (0, tslib_1.__importDefault)(require("cheerio"));
12
+ const content_type_1 = (0, tslib_1.__importDefault)(require("content-type"));
13
+ const fs_1 = (0, tslib_1.__importDefault)(require("fs"));
14
+ const mime_types_1 = (0, tslib_1.__importDefault)(require("mime-types"));
15
+ const os_1 = (0, tslib_1.__importDefault)(require("os"));
15
16
  // eslint-disable-next-line import/no-extraneous-dependencies
16
- const ow_1 = require("ow");
17
- const path = require("path");
18
- const semver = require("semver");
19
- const _ = require("underscore");
17
+ const ow_1 = (0, tslib_1.__importDefault)(require("ow"));
18
+ const path_1 = (0, tslib_1.__importDefault)(require("path"));
19
+ const semver_1 = (0, tslib_1.__importDefault)(require("semver"));
20
+ const underscore_1 = (0, tslib_1.__importDefault)(require("underscore"));
20
21
  const url_1 = require("url");
21
- const util = require("util");
22
+ const util_1 = (0, tslib_1.__importDefault)(require("util"));
23
+ const rimraf_1 = (0, tslib_1.__importDefault)(require("rimraf"));
22
24
  const package_json_2 = require("../package.json");
23
- const utils_log_1 = require("./utils_log");
24
- const requestUtils = require("./utils_request");
25
+ const utils_log_1 = (0, tslib_1.__importDefault)(require("./utils_log"));
26
+ const utils_request_1 = require("./utils_request");
25
27
  /* eslint-enable no-unused-vars,import/named,import/no-duplicates,import/order */
28
+ const rimrafp = util_1.default.promisify(rimraf_1.default);
26
29
  /**
27
30
  * Default regular expression to match URLs in a string that may be plain text, JSON, CSV or other. It supports common URL characters
28
31
  * and does not support URLs containing commas or spaces. The URLs also may contain Unicode letters (not symbols).
@@ -35,7 +38,19 @@ const URL_NO_COMMAS_REGEX = RegExp('https?://(www\\.)?[\\p{L}0-9][-\\p{L}0-9@:%.
35
38
  * @memberOf utils
36
39
  */
37
40
  const URL_WITH_COMMAS_REGEX = RegExp('https?://(www\\.)?[\\p{L}0-9][-\\p{L}0-9@:%._\\+~#=]{0,254}[\\p{L}0-9]\\.[a-z]{2,63}(:\\d{1,5})?(/[-\\p{L}0-9@:%_\\+,.~#?&//=\\(\\)]*)?', 'giu'); // eslint-disable-line
38
- const psTreePromised = util.promisify(psTree);
41
+ const MEMORY_FILE_PATHS = {
42
+ TOTAL: {
43
+ V1: '/sys/fs/cgroup/memory/memory.limit_in_bytes',
44
+ V2: '/sys/fs/cgroup/memory.max',
45
+ },
46
+ USED: {
47
+ V1: '/sys/fs/cgroup/memory/memory.usage_in_bytes',
48
+ V2: '/sys/fs/cgroup/memory.current',
49
+ },
50
+ };
51
+ // Set encoding to utf-8 so fs.readFile returns string instead of buffer
52
+ const MEMORY_FILE_ENCODING = 'utf-8';
53
+ const psTreePromised = util_1.default.promisify(ps_tree_1.default);
39
54
  /**
40
55
  * Returns a new instance of the Apify API client. The `ApifyClient` class is provided
41
56
  * by the <a href="https://www.npmjs.com/package/apify-client" target="_blank">apify-client</a>
@@ -60,7 +75,7 @@ const newClient = (options = {}) => {
60
75
  minDelayBetweenRetriesMillis: ow_1.default.optional.number,
61
76
  }));
62
77
  const { baseUrl = process.env[consts_1.ENV_VARS.API_BASE_URL], token = process.env[consts_1.ENV_VARS.TOKEN], } = options;
63
- return new ApifyClient({
78
+ return new apify_client_1.default({
64
79
  ...options,
65
80
  baseUrl,
66
81
  token,
@@ -74,7 +89,7 @@ const logSystemInfo = () => {
74
89
  utils_log_1.default.info('System info', {
75
90
  apifyVersion: package_json_2.version,
76
91
  apifyClientVersion: package_json_1.version,
77
- osType: os.type(),
92
+ osType: os_1.default.type(),
78
93
  nodeVersion: process.version,
79
94
  });
80
95
  };
@@ -98,21 +113,21 @@ exports.apifyClient = (0, exports.newClient)();
98
113
  const addCharsetToContentType = (contentType) => {
99
114
  if (!contentType)
100
115
  return contentType;
101
- const parsed = contentTypeParser.parse(contentType);
116
+ const parsed = content_type_1.default.parse(contentType);
102
117
  if (parsed.parameters.charset)
103
118
  return contentType;
104
119
  parsed.parameters.charset = 'utf-8';
105
- return contentTypeParser.format(parsed);
120
+ return content_type_1.default.format(parsed);
106
121
  };
107
122
  exports.addCharsetToContentType = addCharsetToContentType;
108
123
  let isDockerPromiseCache;
109
124
  const createIsDockerPromise = () => {
110
- const promise1 = util
111
- .promisify(fs.stat)('/.dockerenv')
125
+ const promise1 = util_1.default
126
+ .promisify(fs_1.default.stat)('/.dockerenv')
112
127
  .then(() => true)
113
128
  .catch(() => false);
114
- const promise2 = util
115
- .promisify(fs.readFile)('/proc/self/cgroup', 'utf8')
129
+ const promise2 = util_1.default
130
+ .promisify(fs_1.default.readFile)('/proc/self/cgroup', 'utf8')
116
131
  .then((content) => content.indexOf('docker') !== -1)
117
132
  .catch(() => false);
118
133
  return Promise
@@ -233,17 +248,38 @@ const getMemoryInfo = async () => {
233
248
  else if (isDockerVar) {
234
249
  // When running inside Docker container, use container memory limits
235
250
  // This must be promisified here so that we can mock it.
236
- const readPromised = util.promisify(fs.readFile);
251
+ const readPromised = util_1.default.promisify(fs_1.default.readFile);
252
+ const accessPromised = util_1.default.promisify(fs_1.default.access);
253
+ // Check wheter cgroups V1 or V2 is used
254
+ let cgroupsVersion = 'V1';
255
+ try {
256
+ // If this directory does not exists, assume docker is using cgroups V2
257
+ await accessPromised('/sys/fs/cgroup/memory/', fs_1.default.constants.R_OK);
258
+ }
259
+ catch (err) {
260
+ cgroupsVersion = 'V2';
261
+ }
237
262
  try {
238
- const [totalBytesStr, usedBytesStr] = await Promise.all([
239
- readPromised('/sys/fs/cgroup/memory/memory.limit_in_bytes'),
240
- readPromised('/sys/fs/cgroup/memory/memory.usage_in_bytes'),
263
+ let [totalBytesStr, usedBytesStr] = await Promise.all([
264
+ readPromised(MEMORY_FILE_PATHS.TOTAL[cgroupsVersion], MEMORY_FILE_ENCODING),
265
+ readPromised(MEMORY_FILE_PATHS.USED[cgroupsVersion], MEMORY_FILE_ENCODING),
241
266
  ]);
242
- totalBytes = parseInt(totalBytesStr, 10);
243
- // https://unix.stackexchange.com/q/420906
244
- const containerRunsWithUnlimitedMemory = totalBytes > Number.MAX_SAFE_INTEGER;
245
- if (containerRunsWithUnlimitedMemory)
246
- totalBytes = os.totalmem();
267
+ // Cgroups V2 files contains newline character. Getting rid of it for better handling in later part of the code.
268
+ totalBytesStr = totalBytesStr.replace(/[^a-zA-Z0-9 ]/g, '');
269
+ usedBytesStr = usedBytesStr.replace(/[^a-zA-Z0-9 ]/g, '');
270
+ // Cgroups V2 contains 'max' string if memory is not limited
271
+ // See https://git.kernel.org/pub/scm/linux/kernel/git/tj/cgroup.git/tree/Documentation/admin-guide/cgroup-v2.rst (see "memory.max")
272
+ if (totalBytesStr === 'max') {
273
+ totalBytes = os_1.default.totalmem();
274
+ // Cgroups V1 is set to number related to platform and page size if memory is not limited
275
+ // See https://unix.stackexchange.com/q/420906
276
+ }
277
+ else {
278
+ totalBytes = parseInt(totalBytesStr, 10);
279
+ const containerRunsWithUnlimitedMemory = totalBytes > Number.MAX_SAFE_INTEGER;
280
+ if (containerRunsWithUnlimitedMemory)
281
+ totalBytes = os_1.default.totalmem();
282
+ }
247
283
  usedBytes = parseInt(usedBytesStr, 10);
248
284
  freeBytes = totalBytes - usedBytes;
249
285
  }
@@ -252,14 +288,14 @@ const getMemoryInfo = async () => {
252
288
  utils_log_1.default.deprecated('Your environment is Docker, but your system does not support memory cgroups. '
253
289
  + 'If you\'re running containers with limited memory, memory auto-scaling will not work properly.\n\n'
254
290
  + `Cause: ${err.message}`);
255
- totalBytes = os.totalmem();
256
- freeBytes = os.freemem();
291
+ totalBytes = os_1.default.totalmem();
292
+ freeBytes = os_1.default.freemem();
257
293
  usedBytes = totalBytes - freeBytes;
258
294
  }
259
295
  }
260
296
  else {
261
- totalBytes = os.totalmem();
262
- freeBytes = os.freemem();
297
+ totalBytes = os_1.default.totalmem();
298
+ freeBytes = os_1.default.freemem();
263
299
  usedBytes = totalBytes - freeBytes;
264
300
  }
265
301
  return {
@@ -289,7 +325,7 @@ exports.getFirstKey = getFirstKey;
289
325
  * @ignore
290
326
  */
291
327
  const getTypicalChromeExecutablePath = () => {
292
- switch (os.platform()) {
328
+ switch (os_1.default.platform()) {
293
329
  case 'darwin': return '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome';
294
330
  case 'win32': return 'C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe';
295
331
  default: return '/usr/bin/google-chrome';
@@ -374,8 +410,7 @@ const downloadListOfUrls = async (options) => {
374
410
  urlRegExp: ow_1.default.optional.regExp,
375
411
  }));
376
412
  const { url, encoding = 'utf8', urlRegExp = URL_NO_COMMAS_REGEX } = options;
377
- const { requestAsBrowser } = requestUtils;
378
- const { body: string } = await requestAsBrowser({ url, encoding });
413
+ const { body: string } = await (0, utils_request_1.requestAsBrowser)({ url, encoding });
379
414
  return extractUrls({ string, urlRegExp });
380
415
  };
381
416
  /**
@@ -440,7 +475,7 @@ const htmlToText = (html) => {
440
475
  * @type {cheerio.Root}
441
476
  * @ignore
442
477
  */
443
- const $ = typeof html === 'function' ? html : cheerio.load(html, { decodeEntities: true });
478
+ const $ = typeof html === 'function' ? html : cheerio_1.default.load(html, { decodeEntities: true });
444
479
  let text = '';
445
480
  const process = (elems) => {
446
481
  const len = elems ? elems.length : 0;
@@ -507,7 +542,7 @@ const createRequestDebugInfo = (request, response = {}, additionalFields = {}) =
507
542
  retryCount: request.retryCount,
508
543
  errorMessages: request.errorMessages,
509
544
  // Puppeteer response has .status() funtion and NodeJS response ,statusCode property.
510
- statusCode: _.isFunction(response.status) ? response.status() : response.statusCode,
545
+ statusCode: underscore_1.default.isFunction(response.status) ? response.status() : response.statusCode,
511
546
  ...additionalFields,
512
547
  };
513
548
  };
@@ -539,7 +574,7 @@ const printOutdatedSdkWarning = () => {
539
574
  if (process.env[consts_1.ENV_VARS.DISABLE_OUTDATED_WARNING])
540
575
  return;
541
576
  const latestApifyVersion = process.env[consts_1.ENV_VARS.SDK_LATEST_VERSION];
542
- if (!latestApifyVersion || !semver.lt(package_json_2.version, latestApifyVersion))
577
+ if (!latestApifyVersion || !semver_1.default.lt(package_json_2.version, latestApifyVersion))
543
578
  return;
544
579
  // eslint-disable-next-line
545
580
  utils_log_1.default.warning(`You are using an outdated version (${package_json_2.version}) of Apify SDK. We recommend you to update to the latest version (${latestApifyVersion}).
@@ -561,7 +596,7 @@ const parseContentTypeFromResponse = (response) => {
561
596
  let parsedContentType;
562
597
  if (headers['content-type']) {
563
598
  try {
564
- parsedContentType = contentTypeParser.parse(headers['content-type']);
599
+ parsedContentType = content_type_1.default.parse(headers['content-type']);
565
600
  }
566
601
  catch (err) {
567
602
  // Can not parse content type from Content-Type header. Try to parse it from file extension.
@@ -570,9 +605,9 @@ const parseContentTypeFromResponse = (response) => {
570
605
  // Parse content type from file extension as fallback
571
606
  if (!parsedContentType) {
572
607
  const parsedUrl = new url_1.URL(url);
573
- const contentTypeFromExtname = mime.contentType(path.extname(parsedUrl.pathname))
608
+ const contentTypeFromExtname = mime_types_1.default.contentType(path_1.default.extname(parsedUrl.pathname))
574
609
  || 'application/octet-stream; charset=utf-8'; // Fallback content type, specified in https://tools.ietf.org/html/rfc7231#section-3.1.1.5
575
- parsedContentType = contentTypeParser.parse(contentTypeFromExtname);
610
+ parsedContentType = content_type_1.default.parse(contentTypeFromExtname);
576
611
  }
577
612
  return {
578
613
  type: parsedContentType.type,
@@ -641,6 +676,27 @@ const waitForRunToFinish = async (options) => {
641
676
  return run;
642
677
  };
643
678
  exports.waitForRunToFinish = waitForRunToFinish;
679
+ /**
680
+ * Cleans up the local storage folder created when testing locally.
681
+ * This is useful in the event you are debugging your code locally.
682
+ *
683
+ * Be careful as this will remove the folder you provide and everything in it!
684
+ *
685
+ * @param {string} [folder] The folder to clean up
686
+ * @returns {Promise<void>}
687
+ * @memberOf utils
688
+ * @name purgeLocalStorage
689
+ * @function
690
+ */
691
+ const purgeLocalStorage = async (folder) => {
692
+ // If the user did not provide a folder, try to get it from the env variables, or the default one
693
+ if (!folder) {
694
+ folder = process.env[consts_1.ENV_VARS.LOCAL_STORAGE_DIR] || 'apify_storage';
695
+ }
696
+ // Clear the folder
697
+ await rimrafp(folder);
698
+ };
699
+ exports.purgeLocalStorage = purgeLocalStorage;
644
700
  /**
645
701
  * A namespace that contains various utilities.
646
702
  *
@@ -666,4 +722,6 @@ exports.publicUtils = {
666
722
  URL_WITH_COMMAS_REGEX,
667
723
  createRequestDebugInfo,
668
724
  waitForRunToFinish: exports.waitForRunToFinish,
725
+ purgeLocalStorage: exports.purgeLocalStorage,
669
726
  };
727
+ //# sourceMappingURL=utils.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.js"],"names":[],"mappings":";;;;AAAA,0EAAoC;AACpC,6DAA6D;AAC7D,iDAAyC;AACzC,6EAAuC;AACvC,4DAA0E;AAC1E,0CAAoE;AACpE,mEAA8B;AAC9B,6EAA6C;AAC7C,yDAAoB;AACpB,yEAA8B;AAC9B,yDAAoB;AACpB,6DAA6D;AAC7D,yDAAoB;AACpB,6DAAwB;AACxB,iEAA4B;AAC5B,yEAA2B;AAC3B,6BAA0B;AAC1B,6DAAwB;AACxB,iEAA4B;AAM5B,kDAA0D;AAC1D,yEAA8B;AAC9B,mDAAmD;AAInD,iFAAiF;AAEjF,MAAM,OAAO,GAAG,cAAI,CAAC,SAAS,CAAC,gBAAM,CAAC,CAAC;AAEvC;;;;GAIG;AACH,MAAM,mBAAmB,GAAG,MAAM,CAAC,wIAAwI,EAAE,KAAK,CAAC,CAAC,CAAC,sBAAsB;AAC3M;;;;GAIG;AACH,MAAM,qBAAqB,GAAG,MAAM,CAAC,yIAAyI,EAAE,KAAK,CAAC,CAAC,CAAC,sBAAsB;AAE9M,MAAM,iBAAiB,GAAG;IACtB,KAAK,EAAE;QACH,EAAE,EAAE,6CAA6C;QACjD,EAAE,EAAE,2BAA2B;KAClC;IACD,IAAI,EAAE;QACF,EAAE,EAAE,6CAA6C;QACjD,EAAE,EAAE,+BAA+B;KACtC;CACJ,CAAC;AAEF,wEAAwE;AACxE,MAAM,oBAAoB,GAAG,OAAO,CAAC;AAErC,MAAM,cAAc,GAAG,cAAI,CAAC,SAAS,CAAC,iBAAM,CAAC,CAAC;AAE9C;;;;;;;;;;;;;;;GAeG;AACI,MAAM,SAAS,GAAG,CAAC,OAAO,GAAG,EAAE,EAAE,EAAE;IACtC,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG;QAC/B,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC9B,4BAA4B,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;KACnD,CAAC,CAAC,CAAC;IACJ,MAAM,EACF,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAQ,CAAC,YAAY,CAAC,EAC5C,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAQ,CAAC,KAAK,CAAC,GACtC,GAAG,OAAO,CAAC;IAEZ,OAAO,IAAI,sBAAW,CAAC;QACnB,GAAG,OAAO;QACV,OAAO;QACP,KAAK;KACR,CAAC,CAAC;AACP,CAAC,CAAC;AAjBW,QAAA,SAAS,aAiBpB;AAEF;;GAEG;AACI,MAAM,aAAa,GAAG,GAAG,EAAE;IAC9B,mBAAG,CAAC,IAAI,CAAC,aAAa,EAAE;QACpB,YAAY,EAAZ,sBAAY;QACZ,kBAAkB,EAAlB,sBAAkB;QAClB,MAAM,EAAE,YAAE,CAAC,IAAI,EAAE;QACjB,WAAW,EAAE,OAAO,CAAC,OAAO;KAC/B,CAAC,CAAC;AACP,CAAC,CAAC;AAPW,QAAA,aAAa,iBAOxB;AAEF;;;;;;GAMG;AACU,QAAA,WAAW,GAAG,IAAA,iBAAS,GAAE,CAAC;AAEvC;;;;;;;GAOG;AACI,MAAM,uBAAuB,GAAG,CAAC,WAAW,EAAE,EAAE;IACnD,IAAI,CAAC,WAAW;QAAE,OAAO,WAAW,CAAC;IAErC,MAAM,MAAM,GAAG,sBAAiB,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IAEpD,IAAI,MAAM,CAAC,UAAU,CAAC,OAAO;QAAE,OAAO,WAAW,CAAC;IAElD,MAAM,CAAC,UAAU,CAAC,OAAO,GAAG,OAAO,CAAC;IAEpC,OAAO,sBAAiB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;AAC5C,CAAC,CAAC;AAVW,QAAA,uBAAuB,2BAUlC;AAEF,IAAI,oBAAoB,CAAC;AACzB,MAAM,qBAAqB,GAAG,GAAG,EAAE;IAC/B,MAAM,QAAQ,GAAG,cAAI;SAChB,SAAS,CAAC,YAAE,CAAC,IAAI,CAAC,CAAC,aAAa,CAAC;SACjC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;SAChB,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;IAExB,MAAM,QAAQ,GAAG,cAAI;SAChB,SAAS,CAAC,YAAE,CAAC,QAAQ,CAAC,CAAC,mBAAmB,EAAE,MAAM,CAAC;SACnD,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;SACnD,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;IAExB,OAAO,OAAO;SACT,GAAG,CAAC,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;SACzB,IAAI,CAAC,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,OAAO,IAAI,OAAO,CAAC,CAAC;AAC1D,CAAC,CAAC;AAEF;;;;;;;;;GASG;AACI,MAAM,QAAQ,GAAG,CAAC,UAAU,EAAE,EAAE;IACnC,wDAAwD;IACxD,IAAI,CAAC,oBAAoB,IAAI,UAAU;QAAE,oBAAoB,GAAG,qBAAqB,EAAE,CAAC;IAExF,OAAO,oBAAoB,CAAC;AAChC,CAAC,CAAC;AALW,QAAA,QAAQ,YAKnB;AAEF;;;;;;;;GAQG;AACI,MAAM,WAAW,GAAG,CAAC,SAAS,EAAE,UAAU,EAAE,EAAE;IACjD,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;QACtC,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,GAAG,GAAG,KAAK,GAAG,MAAM,CAAC,CAAC,gCAAgC;QAE5D,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IACzB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAExD,OAAO,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;AACjC,CAAC,CAAC;AATW,QAAA,WAAW,eAStB;AAEF;;;;;;;;;GASG;AAEH;;;;;;;;;;;;;;GAcG;AACI,MAAM,aAAa,GAAG,KAAK,IAAI,EAAE;IACpC,2DAA2D;IAC3D,oCAAoC;IACpC,MAAM,mBAAmB,GAAG,OAAO,CAAC,QAAQ,KAAK,OAAO;WACjD,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC;IAErD,sDAAsD;IACtD,MAAM,WAAW,GAAG,CAAC,mBAAmB,IAAI,CAAC,MAAM,MAAM,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;IAE9E,IAAI,gBAAgB,GAAG,CAAC,CAAC,CAAC;IAC1B,IAAI,mBAAmB,GAAG,CAAC,CAAC;IAE5B,IAAI,mBAAmB,EAAE;QACrB,oBAAoB;QACpB,gBAAgB,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,GAAG,CAAC;QAE7C,8CAA8C;QAC9C,mBAAmB,GAAG,IAAA,wBAAQ,EAAC,mBAAmB,CAAC;aAC9C,QAAQ,EAAE;aACV,KAAK,CAAC,QAAQ,CAAC;aACf,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;YAChC,mCAAmC;cACjC,IAAI;YACN,+CAA+C;YAC/C,+CAA+C;YAC/C,uDAAuD;cACrD,gBAAgB,CAAC;KAC1B;SAAM;QACH,sCAAsC;QACtC,MAAM,SAAS,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAE1D,SAAS,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;YACtB,0EAA0E;YAC1E,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI,IAAI,GAAG,CAAC,OAAO,KAAK,UAAU,EAAE;gBACpD,OAAO;aACV;YACD,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACpC,yCAAyC;YACzC,IAAI,GAAG,CAAC,GAAG,KAAK,GAAG,OAAO,CAAC,GAAG,EAAE,EAAE;gBAC9B,gBAAgB,GAAG,KAAK,CAAC;gBACzB,OAAO;aACV;YACD,mBAAmB,IAAI,KAAK,CAAC;QACjC,CAAC,CAAC,CAAC;KACN;IAED,IAAI,UAAU,CAAC;IACf,IAAI,SAAS,CAAC;IACd,IAAI,SAAS,CAAC;IAEd,IAAI,mBAAmB,EAAE;QACrB,sCAAsC;QACtC,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,+BAA+B,EAAE,EAAE,CAAC,GAAG,OAAO,CAAC;QACjF,SAAS,GAAG,gBAAgB,GAAG,mBAAmB,CAAC;QACnD,SAAS,GAAG,UAAU,GAAG,SAAS,CAAC;QAEnC,mBAAG,CAAC,KAAK,CAAC,kBAAkB,UAAU,SAAS,SAAS,aAAa,CAAC,CAAC;KAC1E;SAAM,IAAI,WAAW,EAAE;QACpB,oEAAoE;QACpE,wDAAwD;QACxD,MAAM,YAAY,GAAG,cAAI,CAAC,SAAS,CAAC,YAAE,CAAC,QAAQ,CAAC,CAAC;QACjD,MAAM,cAAc,GAAG,cAAI,CAAC,SAAS,CAAC,YAAE,CAAC,MAAM,CAAC,CAAC;QAEjD,wCAAwC;QACxC,IAAI,cAAc,GAAG,IAAI,CAAC;QAC1B,IAAI;YACA,uEAAuE;YACvE,MAAM,cAAc,CAAC,wBAAwB,EAAE,YAAE,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;SACrE;QAAC,OAAO,GAAG,EAAE;YACV,cAAc,GAAG,IAAI,CAAC;SACzB;QAED,IAAI;YACA,IAAI,CAAC,aAAa,EAAE,YAAY,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;gBAClD,YAAY,CAAC,iBAAiB,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,oBAAoB,CAAC;gBAC3E,YAAY,CAAC,iBAAiB,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,oBAAoB,CAAC;aAC7E,CAAC,CAAC;YAEH,gHAAgH;YAChH,aAAa,GAAG,aAAa,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;YAC5D,YAAY,GAAG,YAAY,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;YAE1D,4DAA4D;YAC5D,oIAAoI;YACpI,IAAI,aAAa,KAAK,KAAK,EAAE;gBACzB,UAAU,GAAG,YAAE,CAAC,QAAQ,EAAE,CAAC;gBAC/B,yFAAyF;gBACzF,8CAA8C;aAC7C;iBAAM;gBACH,UAAU,GAAG,QAAQ,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;gBACzC,MAAM,gCAAgC,GAAG,UAAU,GAAG,MAAM,CAAC,gBAAgB,CAAC;gBAC9E,IAAI,gCAAgC;oBAAE,UAAU,GAAG,YAAE,CAAC,QAAQ,EAAE,CAAC;aACpE;YACD,SAAS,GAAG,QAAQ,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;YACvC,SAAS,GAAG,UAAU,GAAG,SAAS,CAAC;SACtC;QAAC,OAAO,GAAG,EAAE;YACV,0CAA0C;YAC1C,mBAAG,CAAC,UAAU,CAAC,+EAA+E;kBACxF,oGAAoG;kBACpG,UAAU,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;YAC/B,UAAU,GAAG,YAAE,CAAC,QAAQ,EAAE,CAAC;YAC3B,SAAS,GAAG,YAAE,CAAC,OAAO,EAAE,CAAC;YACzB,SAAS,GAAG,UAAU,GAAG,SAAS,CAAC;SACtC;KACJ;SAAM;QACH,UAAU,GAAG,YAAE,CAAC,QAAQ,EAAE,CAAC;QAC3B,SAAS,GAAG,YAAE,CAAC,OAAO,EAAE,CAAC;QACzB,SAAS,GAAG,UAAU,GAAG,SAAS,CAAC;KACtC;IAED,OAAO;QACH,UAAU;QACV,SAAS;QACT,SAAS;QACT,gBAAgB;QAChB,mBAAmB;KACtB,CAAC;AACN,CAAC,CAAC;AArHW,QAAA,aAAa,iBAqHxB;AAEF;;;;GAIG;AACI,MAAM,WAAW,GAAG,CAAC,IAAI,EAAE,EAAE;IAChC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,EAAE,yDAAyD;QAC/E,OAAO,GAAG,CAAC;KACd;AACL,CAAC,CAAC;AAJW,QAAA,WAAW,eAItB;AAEF;;;;;GAKG;AACI,MAAM,8BAA8B,GAAG,GAAG,EAAE;IAC/C,QAAQ,YAAE,CAAC,QAAQ,EAAE,EAAE;QACnB,KAAK,QAAQ,CAAC,CAAC,OAAO,8DAA8D,CAAC;QACrF,KAAK,OAAO,CAAC,CAAC,OAAO,4DAA4D,CAAC;QAClF,OAAO,CAAC,CAAC,OAAO,wBAAwB,CAAC;KAC5C;AACL,CAAC,CAAC;AANW,QAAA,8BAA8B,kCAMzC;AAEF;;;;;;;;;GASG;AACI,MAAM,mBAAmB,GAAG,CAAC,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,EAAE;IACxE,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACnC,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,OAAO,CAAC,CAAC;QACxB,IAAA,YAAE,EAAC,aAAa,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QAC7B,IAAA,YAAE,EAAC,YAAY,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QAC5B,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,YAAY,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC;QACjF,OAAO;aACF,IAAI,CAAC,OAAO,CAAC;aACb,KAAK,CAAC,MAAM,CAAC;aACb,OAAO,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;AACP,CAAC,CAAC;AAXW,QAAA,mBAAmB,uBAW9B;AAEF;;;;;;;;GAQG;AACI,MAAM,QAAQ,GAAG,GAAG,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAQ,CAAC,UAAU,CAAC,CAAC;AAApD,QAAA,QAAQ,YAA4C;AAEjE;;;;;;;;;;;;;;;;;;;GAmBG;AACI,MAAM,KAAK,GAAG,CAAC,MAAM,EAAE,EAAE;IAC5B,OAAO,IAAI,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC;AACzD,CAAC,CAAC;AAFW,QAAA,KAAK,SAEhB;AAEF;;;;;;;;;;;;GAYG;AACH,MAAM,kBAAkB,GAAG,KAAK,EAAE,OAAO,EAAE,EAAE;IACzC,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,GAAG;QAClB,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;KAChC,CAAC,CAAC,CAAC;IACJ,MAAM,EAAE,GAAG,EAAE,QAAQ,GAAG,MAAM,EAAE,SAAS,GAAG,mBAAmB,EAAE,GAAG,OAAO,CAAC;IAC5E,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,IAAA,gCAAgB,EAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC;IACnE,OAAO,WAAW,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;AAC9C,CAAC,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,WAAW,GAAG,CAAC,OAAO,EAAE,EAAE;IAC5B,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,MAAM,EAAE,YAAE,CAAC,MAAM;QACjB,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;KAChC,CAAC,CAAC,CAAC;IACJ,MAAM,EAAE,MAAM,EAAE,SAAS,GAAG,mBAAmB,EAAE,GAAG,OAAO,CAAC;IAC5D,OAAO,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;AACzC,CAAC,CAAC;AAEF,4HAA4H;AAC5H,MAAM,eAAe,GAAG,uCAAuC,CAAC;AAChE,MAAM,gBAAgB,GAAG,sGAAsG,CAAC;AAEhI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,MAAM,UAAU,GAAG,CAAC,IAAI,EAAE,EAAE;IACxB,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,CAAC;IAErB,+EAA+E;IAC/E,qEAAqE;IACrE,+EAA+E;IAC/E,oGAAoG;IACpG,iDAAiD;IACjD;;;OAGG;IACH,MAAM,CAAC,GAAG,OAAO,IAAI,KAAK,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,iBAAO,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3F,IAAI,IAAI,GAAG,EAAE,CAAC;IAEd,MAAM,OAAO,GAAG,CAAC,KAAK,EAAE,EAAE;QACtB,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE;YAC1B,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE;gBACtB,qDAAqD;gBACrD,IAAI,KAAK,CAAC;gBACV,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,KAAK,KAAK;oBAAE,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC;;oBAC/D,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;gBAC5C,+EAA+E;gBAC/E,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;oBAAE,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBAC3E,IAAI,IAAI,KAAK,CAAC;aACjB;iBAAM,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE;gBACtE,qCAAqC;aACxC;iBAAM,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE;gBAC9B,IAAI,IAAI,IAAI,CAAC;aAChB;iBAAM,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE;gBAC9B,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACvB,IAAI,IAAI,IAAI,CAAC;aAChB;iBAAM;gBACH,2EAA2E;gBAC3E,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACvD,IAAI,UAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;oBAAE,IAAI,IAAI,IAAI,CAAC;gBACtD,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACvB,IAAI,UAAU,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;oBAAE,IAAI,IAAI,IAAI,CAAC;aACxD;SACJ;IACL,CAAC,CAAC;IAEF,kFAAkF;IAClF,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC;IACxB,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAE7C,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC;AACvB,CAAC,CAAC;AAEF;;;;;;;;;;GAUG;AACH,MAAM,sBAAsB,GAAG,CAAC,OAAO,EAAE,QAAQ,GAAG,EAAE,EAAE,gBAAgB,GAAG,EAAE,EAAE,EAAE;IAC7E,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;IACvB,IAAA,YAAE,EAAC,QAAQ,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;IACxB,IAAA,YAAE,EAAC,gBAAgB,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;IAEhC,OAAO;QACH,SAAS,EAAE,OAAO,CAAC,EAAE;QACrB,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,qFAAqF;QACrF,UAAU,EAAE,oBAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU;QACnF,GAAG,gBAAgB;KACtB,CAAC;AACN,CAAC,CAAC;AAEF;;;;;;GAMG;AACI,MAAM,oBAAoB,GAAG,CAAC,YAAY,EAAE,EAAE;IACjD,OAAO,YAAY;SACd,WAAW,EAAE;SACb,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QACjB,OAAO,KAAK,GAAG,CAAC;YACZ,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;YAC9C,CAAC,CAAC,IAAI,CAAC;IACf,CAAC,CAAC;SACD,IAAI,CAAC,EAAE,CAAC,CAAC;AAClB,CAAC,CAAC;AAVW,QAAA,oBAAoB,wBAU/B;AAEF;;;;GAIG;AACI,MAAM,uBAAuB,GAAG,GAAG,EAAE;IACxC,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAQ,CAAC,wBAAwB,CAAC;QAAE,OAAO;IAC3D,MAAM,kBAAkB,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAQ,CAAC,kBAAkB,CAAC,CAAC;IACpE,IAAI,CAAC,kBAAkB,IAAI,CAAC,gBAAM,CAAC,EAAE,CAAC,sBAAY,EAAE,kBAAkB,CAAC;QAAE,OAAO;IAEhF,2BAA2B;IAC3B,mBAAG,CAAC,OAAO,CAAC,sCAAsC,sBAAY,qEAAqE,kBAAkB;iIACxB,CAAC,CAAC;AACnI,CAAC,CAAC;AARW,QAAA,uBAAuB,2BAQlC;AAEF;;;;;GAKG;AACI,MAAM,4BAA4B,GAAG,CAAC,QAAQ,EAAE,EAAE;IACrD,IAAA,YAAE,EAAC,QAAQ,EAAE,YAAE,CAAC,MAAM,CAAC,YAAY,CAAC;QAChC,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,GAAG;QAClB,OAAO,EAAE,YAAE,CAAC,MAAM;KACrB,CAAC,CAAC,CAAC;IAEJ,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAC;IAClC,IAAI,iBAAiB,CAAC;IAEtB,IAAI,OAAO,CAAC,cAAc,CAAC,EAAE;QACzB,IAAI;YACA,iBAAiB,GAAG,sBAAiB,CAAC,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC;SACxE;QAAC,OAAO,GAAG,EAAE;YACV,4FAA4F;SAC/F;KACJ;IAED,qDAAqD;IACrD,IAAI,CAAC,iBAAiB,EAAE;QACpB,MAAM,SAAS,GAAG,IAAI,SAAG,CAAC,GAAG,CAAC,CAAC;QAC/B,MAAM,sBAAsB,GAAG,oBAAI,CAAC,WAAW,CAAC,cAAI,CAAC,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;eAC1E,yCAAyC,CAAC,CAAC,0FAA0F;QAC5I,iBAAiB,GAAG,sBAAiB,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;KACvE;IAED,OAAO;QACH,IAAI,EAAE,iBAAiB,CAAC,IAAI;QAC5B,OAAO,EAAE,iBAAiB,CAAC,UAAU,CAAC,OAAO;KAChD,CAAC;AACN,CAAC,CAAC;AA7BW,QAAA,4BAA4B,gCA6BvC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACI,MAAM,kBAAkB,GAAG,KAAK,EAAE,OAAO,EAAE,EAAE;IAChD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,OAAO,EAAE,YAAE,CAAC,MAAM;QAClB,KAAK,EAAE,YAAE,CAAC,MAAM;QAChB,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;KAC/B,CAAC,CAAC,CAAC;IAEJ,MAAM,EACF,OAAO,EACP,KAAK,EACL,QAAQ,GACX,GAAG,OAAO,CAAC;IACZ,IAAI,GAAG,CAAC;IAER,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,YAAY,GAAG,GAAG,EAAE;QACtB,IAAI,QAAQ,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,IAAI,QAAQ;YAAE,OAAO,KAAK,CAAC;QAC1E,IAAI,GAAG,IAAI,kCAAyB,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC;YAAE,OAAO,KAAK,CAAC;QAExE,OAAO,IAAI,CAAC;IAChB,CAAC,CAAC;IAEF,OAAO,YAAY,EAAE,EAAE;QACnB,MAAM,aAAa,GAAG,QAAQ;YAC1B,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC;YACxD,CAAC,CAAC,MAAM,CAAC;QAEb,GAAG,GAAG,MAAM,mBAAW,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,aAAa,CAAC,EAAE,QAAQ,EAAE,aAAa,EAAE,CAAC,CAAC;QAEvF,mEAAmE;QACnE,qEAAqE;QACrE,IAAI,CAAC,GAAG;YAAE,MAAM,IAAA,aAAK,EAAC,GAAG,CAAC,CAAC;KAC9B;IAED,IAAI,CAAC,GAAG,EAAE;QACN,MAAM,IAAI,KAAK,CAAC,mFAAmF,CAAC,CAAC;KACxG;IAED,OAAO,GAAG,CAAC;AACf,CAAC,CAAC;AAvCW,QAAA,kBAAkB,sBAuC7B;AAEF;;;;;;;;;;;GAWG;AACI,MAAM,iBAAiB,GAAG,KAAK,EAAE,MAAM,EAAE,EAAE;IAC9C,iGAAiG;IACjG,IAAI,CAAC,MAAM,EAAE;QACT,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAQ,CAAC,iBAAiB,CAAC,IAAI,eAAe,CAAC;KACvE;IAED,mBAAmB;IACnB,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;AAC1B,CAAC,CAAC;AARW,QAAA,iBAAiB,qBAQ5B;AAEF;;;;;;;;;;;;;;GAcG;AACU,QAAA,WAAW,GAAG;IACvB,QAAQ,EAAR,gBAAQ;IACR,KAAK,EAAL,aAAK;IACL,kBAAkB;IAClB,WAAW;IACX,UAAU;IACV,mBAAmB;IACnB,qBAAqB;IACrB,sBAAsB;IACtB,kBAAkB,EAAlB,0BAAkB;IAClB,iBAAiB,EAAjB,yBAAiB;CACpB,CAAC"}
@@ -38,3 +38,4 @@ import { LoggerJson } from "@apify/log/logger_json";
38
38
  import { LoggerText } from "@apify/log/logger_text";
39
39
  import log from "@apify/log";
40
40
  export { Log, LogLevel, Logger, LoggerJson, LoggerText };
41
+ //# sourceMappingURL=utils_log.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"utils_log.d.ts","sourceRoot":"","sources":["../src/utils_log.js"],"names":[],"mappings":""}
@@ -5,6 +5,7 @@
5
5
  // instead of '@apify/log'
6
6
  Object.defineProperty(exports, "__esModule", { value: true });
7
7
  exports.LoggerText = exports.LoggerJson = exports.Logger = exports.LogLevel = exports.Log = void 0;
8
+ const tslib_1 = require("tslib");
8
9
  /**
9
10
  * @typedef {object} LoggerOptions
10
11
  * @property {number} [level=4] Sets the log level to the given value, preventing messages from less important log levels
@@ -179,10 +180,11 @@ exports.LoggerText = exports.LoggerJson = exports.Logger = exports.LogLevel = ex
179
180
  * @memberOf log
180
181
  */
181
182
  // variable can't be the same from the namespace above
182
- const log_1 = require("@apify/log");
183
+ const log_1 = (0, tslib_1.__importStar)(require("@apify/log"));
183
184
  Object.defineProperty(exports, "Log", { enumerable: true, get: function () { return log_1.Log; } });
184
185
  Object.defineProperty(exports, "LogLevel", { enumerable: true, get: function () { return log_1.LogLevel; } });
185
186
  Object.defineProperty(exports, "Logger", { enumerable: true, get: function () { return log_1.Logger; } });
186
187
  Object.defineProperty(exports, "LoggerJson", { enumerable: true, get: function () { return log_1.LoggerJson; } });
187
188
  Object.defineProperty(exports, "LoggerText", { enumerable: true, get: function () { return log_1.LoggerText; } });
188
189
  exports.default = log_1.default;
190
+ //# sourceMappingURL=utils_log.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"utils_log.js","sourceRoot":"","sources":["../src/utils_log.js"],"names":[],"mappings":";AAAA,uFAAuF;AACvF,oEAAoE;AACpE,iFAAiF;AACjF,0BAA0B;;;;AAE1B;;;;;;;;;;;GAWG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8DG;AAEH;;;;;;GAMG;AAEH;;;;;;;;;;;;;GAaG;AAEH;;;;;;;;GAQG;AAEH;;;;;;GAMG;AAEH;;;;;;GAMG;AAEH;;;;;;GAMG;AAEH;;;;;;;;;;GAUG;AAEH;;;;;;;;GAQG;AAEH;;;;;;;GAOG;AAEH;;;;;;;;GAQG;AAEH;;;;;;;;;GASG;AAEH,sDAAsD;AACtD,+DAA+F;AAEtF,oFAFK,SAAG,OAEL;AAAiB,yFAFK,cAAQ,OAEL;AAAE,uFAFK,YAAM,OAEL;AAAE,2FAFK,gBAAU,OAEL;AAAE,2FAFK,gBAAU,OAEL;AACrE,kBAAe,aAAG,CAAC"}
@@ -75,3 +75,4 @@ export type RequestAsBrowserOptions = {
75
75
  };
76
76
  export type AbortFunction = (response: IncomingMessage) => boolean;
77
77
  import { IncomingMessage } from "http";
78
+ //# sourceMappingURL=utils_request.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"utils_request.d.ts","sourceRoot":"","sources":["../src/utils_request.js"],"names":[],"mappings":";AA2GO,2CATI,uBAAuB,GAEtB,QAAQ,sBAAsB,CAAC,CAyH1C;qCApNY,CAAC,eAAe,SAAc;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC;;;;;SAK9C,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;uCA4CT,eAAe,KACb,OAAO"}
@@ -1,9 +1,10 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.requestAsBrowser = void 0;
4
+ const tslib_1 = require("tslib");
4
5
  const got_scraping_1 = require("got-scraping");
5
- const ow_1 = require("ow");
6
- const utils_log_1 = require("./utils_log");
6
+ const ow_1 = (0, tslib_1.__importDefault)(require("ow"));
7
+ const utils_log_1 = (0, tslib_1.__importDefault)(require("./utils_log"));
7
8
  /* eslint-enable no-unused-vars,import/named,import/order */
8
9
  /**
9
10
  * @typedef {(IncomingMessage & Readable & { body: string })} RequestAsBrowserResult
@@ -223,7 +224,7 @@ function fixUrl(url) {
223
224
  if (chunks.length) {
224
225
  const encoded = encodeURI(Buffer.from(chunks, 'hex').toString());
225
226
  url = `${url.slice(0, start)}${encoded}${url.slice(index)}`;
226
- index += encoded.length;
227
+ index = start + encoded.length;
227
228
  chunks = '';
228
229
  }
229
230
  index = url.indexOf('%', index);
@@ -421,3 +422,4 @@ function addResponsePropertiesToStream(stream, response) {
421
422
  }
422
423
  return stream;
423
424
  }
425
+ //# sourceMappingURL=utils_request.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"utils_request.js","sourceRoot":"","sources":["../src/utils_request.js"],"names":[],"mappings":";;;;AAAA,+CAA2C;AAC3C,yDAAoB;AACpB,yEAA8B;AAI9B,4DAA4D;AAE5D;;GAEG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AAEH;;;;GAIG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4CG;AACI,MAAM,gBAAgB,GAAG,KAAK,EAAE,OAAO,GAAG,EAAE,EAAE,EAAE;IACnD,oBAAoB,CAAC,OAAO,CAAC,CAAC;IAC9B,IAAA,YAAE,EAAC,OAAO,EAAE,yBAAyB,EAAE,YAAE,CAAC,MAAM,CAAC,YAAY,CAAC;QAC1D,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC;QAC9C,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG;QAChC,YAAY,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QAC1C,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QACzC,gBAAgB,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QACrC,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QACnC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QACpC,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QAC1C,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QAC7B,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC/B,iBAAiB,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QACtC,sBAAsB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC1C,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QAC3B,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QAC/B,YAAY,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;KACnC,CAAC,CAAC,CAAC;IAEJ,IAAA,YAAE,EAAC,OAAO,EAAE,yBAAyB,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACjE,SAAS,EAAE,wBAAwB,CAAC,IAAI,CAAC;QACzC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,uDAAuD,KAAK,0BAA0B;KAC7G,CAAC,CAAC,CAAC,CAAC;IAEL,uFAAuF;IACvF,+FAA+F;IAC/F,2FAA2F;IAC3F,oEAAoE;IACpE,MAAM,EACF,OAAO,EAAE,gEAAgE;IACzE,IAAI,EACJ,sBAAsB,EACtB,YAAY,GAAG,IAAI,EACnB,WAAW,GAAG,IAAI,EAClB,gBAAgB,GAAG,KAAK,EACxB,aAAa,GAAG,GAAG,EAAE,CAAC,KAAK,EAC3B,eAAe,GAAG,IAAI,EACtB,qBAAqB,GAAG,IAAI,EAC5B,QAAQ,GAAG,IAAI,EACf,WAAW,GAAG,EAAE,EAChB,iBAAiB,GAAG,KAAK,EACzB,MAAM,GAAG,KAAK,EACd,UAAU,GAAG,IAAI,EACjB,gBAAgB,EAAE,6FAA6F;IAC/G,GAAG,SAAS,EACf,GAAG,OAAO,CAAC;IAEZ,MAAM,kBAAkB,GAAG;QACvB,kBAAkB,EAAE,qBAAqB;QACzC,KAAK,EAAE,QAAQ;QACf,OAAO,EAAE,EAAE,OAAO,EAAE,WAAW,GAAG,IAAI,EAAE;QACxC,eAAe,EAAE,iBAAiB;QAClC,QAAQ,EAAE,MAAM;QAChB,UAAU,EAAE,UAAU;QACtB,wEAAwE;QACxE,0DAA0D;QAC1D,6EAA6E;QAC7E,GAAG,SAAS;QACZ,KAAK,EAAE;YACH,GAAG,SAAS,CAAC,KAAK;YAClB,kBAAkB,EAAE,CAAC,eAAe;SACvC;KACJ,CAAC;IAEF,8BAA8B;IAC9B,kBAAkB,CAAC,GAAG,GAAG,MAAM,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;IAExD,qBAAqB;IACrB,sBAAsB,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC;IACpD,mBAAmB,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAC;IAC9C,yBAAyB,CAAC,kBAAkB,CAAC,CAAC;IAC9C,iBAAiB,CAAC,aAAa,EAAE,kBAAkB,CAAC,CAAC;IACrD,IAAI,CAAC,sBAAsB,EAAE;QACzB,oEAAoE;QACpE,kBAAkB,CAAC,sBAAsB,GAAG;YACxC,OAAO,EAAE,gBAAgB,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YACpD,OAAO,EAAE,CAAC,GAAG,YAAY,IAAI,WAAW,EAAE,CAAC;SAC9C,CAAC;KACL;SAAM;QACH,kBAAkB,CAAC,sBAAsB,GAAG,sBAAsB,CAAC;KACtE;IAED,8BAA8B;IAC9B,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE;QAC9B,OAAO,IAAA,0BAAW,EAAC,kBAAkB,CAAC,CAAC;KAC1C;IAED,0DAA0D;IAC1D,MAAM,YAAY,GAAG,IAAA,0BAAW,EAAC,kBAAkB,CAAC,CAAC;IAErD,yBAAyB,CAAC,YAAY,EAAE,kBAAkB,CAAC,CAAC;IAE5D,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACnC,YAAY;aACP,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC;aACnB,EAAE,CAAC,UAAU,EAAE,CAAC,GAAG,EAAE,EAAE;YACpB,IAAI;gBACA,MAAM,WAAW,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;gBACvC,IAAI,WAAW,EAAE;oBACb,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,eAAe,kBAAkB,CAAC,GAAG,gCAAgC,CAAC,CAAC;oBAC7F,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;oBAC1B,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;iBACtB;aACJ;YAAC,OAAO,CAAC,EAAE;gBACR,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBACxB,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;aACpB;YAED,6BAA6B,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC;YAEjD,OAAO,OAAO,CAAC,YAAY,CAAC,CAAC;QACjC,CAAC,CAAC,CAAC;IACX,CAAC,CAAC,CAAC;AACP,CAAC,CAAC;AAlHW,QAAA,gBAAgB,oBAkH3B;AAEF;;;;;;;;GAQG;AACH,SAAS,MAAM,CAAC,GAAG;IACf,MAAM,QAAQ,GAAG,wBAAwB,CAAC;IAC1C,IAAI,MAAM,GAAG,EAAE,CAAC;IAEhB,IAAI,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC7B,OAAO,KAAK,KAAK,CAAC,CAAC,IAAI,KAAK,GAAG,GAAG,CAAC,MAAM,EAAE;QACvC,MAAM,KAAK,GAAG,KAAK,CAAC;QAEpB,GAAG;YACC,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;YAE/C,IAAI,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE;gBAC1E,MAAM,IAAI,MAAM,CAAC;gBAEjB,KAAK,IAAI,CAAC,CAAC;aACd;iBAAM;gBACH,MAAM,IAAI,IAAI,CAAC,CAAC,IAAI;gBAEpB,KAAK,IAAI,CAAC,CAAC;gBACX,MAAM;aACT;SACJ,QAAQ,GAAG,CAAC,KAAK,CAAC,KAAK,GAAG,EAAE;QAE7B,IAAI,MAAM,CAAC,MAAM,EAAE;YACf,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;YACjE,GAAG,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,GAAG,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;YAE5D,KAAK,GAAG,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC;YAC/B,MAAM,GAAG,EAAE,CAAC;SACf;QAED,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;KACnC;IAED,OAAO,GAAG,CAAC;AACf,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,wBAAwB,CAAC,uBAAuB;IACrD,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,uBAAuB,CAAC;IAC9D,oEAAoE;IACpE,wCAAwC;IACxC,MAAM,QAAQ,GAAG,OAAO,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC;IAE9D,MAAM,YAAY,GAAG,CAAC,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;IACrD,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC;IAEjE,iDAAiD;IACjD,OAAO,QAAQ,CAAC,MAAM,IAAI,CAAC,CAAC;AAChC,CAAC;AAED;;;;;;GAMG;AACH,SAAS,sBAAsB,CAAC,OAAO,EAAE,kBAAkB;IACvD,IAAI,OAAO,KAAK,SAAS;QAAE,kBAAkB,CAAC,IAAI,GAAG,OAAO,CAAC;AACjE,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,mBAAmB,CAAC,IAAI,EAAE,kBAAkB;IACjD,6DAA6D;IAC7D,2DAA2D;IAC3D,MAAM,kBAAkB,GAAG,gDAAgD;UACrE,sEAAsE;UACtE,gEAAgE,CAAC;IACvE,IAAI,IAAI,KAAK,IAAI,EAAE;QACf,mBAAG,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC;QACnC,kBAAkB,CAAC,YAAY,GAAG,MAAM,CAAC;QACzC,kBAAkB,CAAC,KAAK,CAAC,OAAO,GAAG,SAAS,CAAC;KAChD;SAAM,IAAI,IAAI,KAAK,KAAK,EAAE;QACvB,mBAAG,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC;QACnC,kEAAkE;KACrE;SAAM;QACH,oEAAoE;QACpE,kBAAkB,CAAC,IAAI,GAAG,IAAI,CAAC;KAClC;AACL,CAAC;AAED;;;;;;GAMG;AACH,SAAS,yBAAyB,CAAC,kBAAkB;IACjD,IAAI,kBAAkB,CAAC,KAAK,IAAI,kBAAkB,CAAC,OAAO,EAAE;QACxD,kBAAkB,CAAC,OAAO,GAAG,EAAE,GAAG,kBAAkB,CAAC,OAAO,EAAE,CAAC;QAE/D,8DAA8D;QAC9D,KAAK,MAAM,GAAG,IAAI,kBAAkB,CAAC,OAAO,EAAE;YAC1C,MAAM,IAAI,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;YAE/B,IAAI,IAAI,KAAK,YAAY,IAAI,IAAI,KAAK,MAAM,EAAE;gBAC1C,OAAO,kBAAkB,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;aAC1C;SACJ;KACJ;AACL,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,iBAAiB,CAAC,aAAa,EAAE,kBAAkB;IACxD,uEAAuE;IACvE,wDAAwD;IACxD,IAAI,kBAAkB,CAAC,QAAQ;QAAE,OAAO;IAExC,MAAM,SAAS,GAAG,CAAC,QAAQ,EAAE,EAAE;QAC3B,MAAM,WAAW,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC;QAC5C,IAAI,WAAW,EAAE;YACb,MAAM,IAAI,KAAK,CAAC,eAAe,kBAAkB,CAAC,GAAG,gCAAgC,CAAC,CAAC;SAC1F;QACD,OAAO,QAAQ,CAAC;IACpB,CAAC,CAAC;IAEF,MAAM,EAAE,KAAK,EAAE,GAAG,kBAAkB,CAAC;IACrC,MAAM,UAAU,GAAG;QACf,GAAG,KAAK;QACR,aAAa,EAAE;YACX,GAAG,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;YACzC,SAAS;SACZ;KACJ,CAAC;IAEF,kBAAkB,CAAC,KAAK,GAAG,UAAU,CAAC;AAC1C,CAAC;AAED;;;;GAIG;AACH,SAAS,yBAAyB,CAAC,YAAY,EAAE,kBAAkB;IAC/D,MAAM,EAAE,MAAM,EAAE,GAAG,kBAAkB,CAAC;IACtC,MAAM,WAAW,GAAG,kBAAkB,CAAC,IAAI,KAAK,SAAS;WAClD,kBAAkB,CAAC,IAAI,KAAK,SAAS;WACrC,kBAAkB,CAAC,IAAI,KAAK,SAAS,CAAC;IAE7C,IAAI,MAAM,IAAI,MAAM,CAAC,WAAW,EAAE,KAAK,KAAK,IAAI,WAAW,EAAE;QACzD,YAAY,CAAC,GAAG,EAAE,CAAC;KACtB;AACL,CAAC;AAED;;;;GAIG;AACH,SAAS,oBAAoB,CAAC,OAAO;IACjC,MAAM,iBAAiB,GAAG;QACtB,4EAA4E;QAC5E,CAAC,cAAc,EAAE,gCAAgC,CAAC;QAClD,CAAC,aAAa,EAAE,gCAAgC,CAAC;QACjD,CAAC,kBAAkB,EAAE,gCAAgC,CAAC;QACtD,CAAC,SAAS,EAAE,MAAM,CAAC;QACnB,CAAC,UAAU,EAAE,OAAO,CAAC;QACrB,CAAC,QAAQ,EAAE,UAAU,CAAC;QACtB,CAAC,YAAY,EAAE,YAAY,CAAC;QAC5B,CAAC,mBAAmB,EAAE,iBAAiB,CAAC;QACxC,CAAC,aAAa,EAAE,iBAAiB,CAAC;QAClC,CAAC,iBAAiB,EAAE,0BAA0B,CAAC;QAC/C,CAAC,eAAe,CAAC,EAAE,uBAAuB;KAC7C,CAAC;IAEF,KAAK,MAAM,CAAC,gBAAgB,EAAE,SAAS,CAAC,IAAI,iBAAiB,EAAE;QAC3D,IAAI,OAAO,CAAC,gBAAgB,CAAC,KAAK,SAAS,EAAE;YACzC,wFAAwF;YACxF,MAAM,cAAc,GAAG,2FAA2F;kBAC5G,2GAA2G;kBAC3G,sGAAsG,CAAC;YAC7G,mBAAG,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;YAE/B,IAAI,gBAAgB,KAAK,eAAe,EAAE;gBACtC,mBAAG,CAAC,UAAU,CAAC,YAAY,gBAAgB,kBAAkB;sBACvD,uEAAuE;sBACvE,wGAAwG,CAAC,CAAC;aACnH;iBAAM;gBACH,mBAAG,CAAC,UAAU,CAAC,YAAY,gBAAgB,iCAAiC,SAAS,YAAY,CAAC,CAAC;aACtG;SACJ;KACJ;AACL,CAAC;AAED;;;;;;;;;;;GAWG;AACH,SAAS,6BAA6B,CAAC,MAAM,EAAE,QAAQ;IACnD,MAAM,UAAU,GAAG;QACf,YAAY,EAAE,eAAe,EAAE,SAAS;QACxC,UAAU,EAAE,aAAa,EAAE,YAAY;QACvC,aAAa,EAAE,UAAU,EAAE,KAAK;QAChC,SAAS;KACZ,CAAC;IAEF,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;QACpB,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;QACxD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAElD,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE;QAC3B,IAAI,CAAC,CAAC,IAAI,IAAI,MAAM,CAAC,EAAE;YACnB,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;SACjC;KACJ;IAED,OAAO,MAAM,CAAC;AAClB,CAAC"}
@@ -178,3 +178,4 @@ declare let YOUTUBE_REGEX: RegExp;
178
178
  /** @type RegExp */
179
179
  declare let YOUTUBE_REGEX_GLOBAL: RegExp;
180
180
  export {};
181
+ //# sourceMappingURL=utils_social.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"utils_social.d.ts","sourceRoot":"","sources":["../src/utils_social.js"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;YAkgBc,MAAM,EAAE;YACR,MAAM,EAAE;qBACR,MAAM,EAAE;eACR,MAAM,EAAE;cACR,MAAM,EAAE;gBACR,MAAM,EAAE;eACR,MAAM,EAAE;cACR,MAAM,EAAE;;AA3etB;;;;;;;GAOG;AACH,sCALW,MAAM,GACL,MAAM,EAAE,CAOnB;AAED;;;;;;;;GAQG;AACH,sCALW,MAAM,EAAE,GACP,MAAM,EAAE,CAgBnB;AA6DD;;;;;;;;GAQG;AACH,sCALW,MAAM,GACL,MAAM,EAAE,CAqBnB;AAED;;;;;;;GAOG;AACH,sCALW,MAAM,EAAE,GACP,MAAM,EAAE,CAgBnB;AAmUD;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,4CARW,MAAM,SACN,MAAE,IAAI,GAGL,aAAa,CAsDxB;AAllBD;;;;;GAKG;AACH,2BAHU,MAAM,CAG+C;AAE/D;;;;;GAKG;AACH,kCAHU,MAAM,CAGgD;AA4KhE,mBAAmB;AACnB,4BADU,MAAM,CACG;AACnB,mBAAmB;AACnB,mCADU,MAAM,CACU;AAC1B,mBAAmB;AACnB,6BADU,MAAM,CACI;AACpB,mBAAmB;AACnB,oCADU,MAAM,CACW;AAC3B,mBAAmB;AACnB,2BADU,MAAM,CACE;AAClB,mBAAmB;AACnB,kCADU,MAAM,CACS;AACzB,mBAAmB;AACnB,4BADU,MAAM,CACG;AACnB,mBAAmB;AACnB,mCADU,MAAM,CACU;AAC1B,mBAAmB;AACnB,2BADU,MAAM,CACE;AAClB,mBAAmB;AACnB,kCADU,MAAM,CACS"}
@@ -1,10 +1,11 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.socialUtils = void 0;
4
+ const tslib_1 = require("tslib");
4
5
  /* eslint-disable no-continue */
5
- const _ = require("underscore");
6
- const cheerio = require("cheerio");
7
- const utils_log_1 = require("./utils_log");
6
+ const underscore_1 = (0, tslib_1.__importDefault)(require("underscore"));
7
+ const cheerio_1 = (0, tslib_1.__importDefault)(require("cheerio"));
8
+ const utils_log_1 = (0, tslib_1.__importDefault)(require("./utils_log"));
8
9
  const utils_1 = require("./utils");
9
10
  // TODO: We could support URLs like https://www.linkedin.com/company/some-company-inc
10
11
  // Regex inspired by https://zapier.com/blog/extract-links-email-phone-regex/
@@ -34,7 +35,7 @@ const EMAIL_URL_PREFIX_REGEX = /^mailto:/i;
34
35
  * @memberOf social
35
36
  */
36
37
  const emailsFromText = (text) => {
37
- if (!_.isString(text))
38
+ if (!underscore_1.default.isString(text))
38
39
  return [];
39
40
  return text.match(EMAIL_REGEX_GLOBAL) || [];
40
41
  };
@@ -120,7 +121,7 @@ const SKIP_PHONE_REGEX = new RegExp(`^(${SKIP_PHONE_REGEXS.join('|')})$`, 'i');
120
121
  * @memberOf social
121
122
  */
122
123
  const phonesFromText = (text) => {
123
- if (!_.isString(text))
124
+ if (!underscore_1.default.isString(text))
124
125
  return [];
125
126
  let phones = text.match(PHONE_REGEX_GLOBAL) || [];
126
127
  phones = phones.filter((phone) => {
@@ -176,7 +177,7 @@ const FACEBOOK_RESERVED_PATHS = 'rsrc\\.php|apps|groups|events|l\\.php|friends|i
176
177
  // eslint-disable-next-line max-len, quotes
177
178
  const FACEBOOK_REGEX_STRING = `(?<!\\w)(?:http(?:s)?:\\/\\/)?(?:www.)?(?:facebook.com|fb.com)\\/(?!(?:${FACEBOOK_RESERVED_PATHS})(?:[\\'\\"\\?\\.\\/]|$))(profile\\.php\\?id\\=[0-9]{3,20}|(?!profile\\.php)[a-z0-9\\.]{5,51})(?![a-z0-9\\.])(?:/)?`;
178
179
  // eslint-disable-next-line max-len, quotes
179
- const YOUTUBE_REGEX_STRING = '(?:https?:\\/\\/)?(?:youtu\\.be\\/|(?:www\\.|m\\.)?youtube\\.com\\/(?:watch|v|embed)(?:\\.php)?(?:\\?.*v=|\\/))([a-zA-Z0-9\\-_]+)';
180
+ const YOUTUBE_REGEX_STRING = '(?:https?:\\/\\/)?(?:youtu\\.be\\/|(?:www\\.|m\\.)?youtube\\.com\\/(?:watch|v|embed|user|c(?:hannel)?)(?:\\.php)?(?:\\?[^ ]*v=|\\/))([a-zA-Z0-9\\-_]+)';
180
181
  /** @type RegExp */
181
182
  let LINKEDIN_REGEX;
182
183
  /** @type RegExp */
@@ -410,13 +411,18 @@ try {
410
411
  */
411
412
  FACEBOOK_REGEX_GLOBAL = new RegExp(FACEBOOK_REGEX_STRING, 'ig');
412
413
  /**
413
- * Regular expression to exactly match a single Youtube video URL.
414
+ * Regular expression to exactly match a single Youtube channel, user or video URL.
414
415
  * It has the following form: `/^...$/i` and matches URLs such as:
415
416
  * ```
416
417
  * https://www.youtube.com/watch?v=kM7YfhfkiEE
417
418
  * https://youtu.be/kM7YfhfkiEE
419
+ * https://www.youtube.com/c/TrapNation
420
+ * https://www.youtube.com/channel/UCklie6BM0fhFvzWYqQVoCTA
421
+ * https://www.youtube.com/user/pewdiepie
418
422
  * ```
419
423
  *
424
+ * Please note that this won't match URLs like https://www.youtube.com/pewdiepie that redirect to /user or /channel.
425
+ *
420
426
  * Example usage:
421
427
  * ```
422
428
  * if (Apify.utils.social.YOUTUBE_REGEX.test('https://www.youtube.com/watch?v=kM7YfhfkiEE')) {
@@ -428,13 +434,18 @@ try {
428
434
  */
429
435
  YOUTUBE_REGEX = new RegExp(`^${YOUTUBE_REGEX_STRING}$`, 'i');
430
436
  /**
431
- * Regular expression to find multiple Youtube video URLs in a text or HTML.
437
+ * Regular expression to find multiple Youtube channel, user or video URLs in a text or HTML.
432
438
  * It has the following form: `/.../ig` and matches URLs such as:
433
439
  * ```
434
440
  * https://www.youtube.com/watch?v=kM7YfhfkiEE
435
441
  * https://youtu.be/kM7YfhfkiEE
442
+ * https://www.youtube.com/c/TrapNation
443
+ * https://www.youtube.com/channel/UCklie6BM0fhFvzWYqQVoCTA
444
+ * https://www.youtube.com/user/pewdiepie
436
445
  * ```
437
446
  *
447
+ * Please note that this won't match URLs like https://www.youtube.com/pewdiepie that redirect to /user or /channel.
448
+ *
438
449
  * Example usage:
439
450
  * ```
440
451
  * const matches = text.match(Apify.utils.social.YOUTUBE_REGEX_GLOBAL);
@@ -527,9 +538,9 @@ const parseHandlesFromHtml = (html, data = null) => {
527
538
  youtubes: [],
528
539
  };
529
540
  // TODO: maybe extract phone numbers from JSON+LD
530
- if (!_.isString(html))
541
+ if (!underscore_1.default.isString(html))
531
542
  return result;
532
- const $ = cheerio.load(html, { decodeEntities: true });
543
+ const $ = cheerio_1.default.load(html, { decodeEntities: true });
533
544
  if (data)
534
545
  data.$ = $;
535
546
  const text = utils_1.publicUtils.htmlToText($);
@@ -557,7 +568,7 @@ const parseHandlesFromHtml = (html, data = null) => {
557
568
  // eslint-disable-next-line guard-for-in, no-restricted-syntax
558
569
  for (const key in result) {
559
570
  result[key].sort();
560
- result[key] = _.uniq(result[key], true);
571
+ result[key] = underscore_1.default.uniq(result[key], true);
561
572
  }
562
573
  return result;
563
574
  };
@@ -593,3 +604,4 @@ exports.socialUtils = {
593
604
  YOUTUBE_REGEX,
594
605
  YOUTUBE_REGEX_GLOBAL,
595
606
  };
607
+ //# sourceMappingURL=utils_social.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"utils_social.js","sourceRoot":"","sources":["../src/utils_social.js"],"names":[],"mappings":";;;;AAAA,gCAAgC;AAChC,yEAA2B;AAC3B,mEAA8B;AAC9B,yEAA8B;AAC9B,mCAAsC;AAEtC,qFAAqF;AAErF,6EAA6E;AAC7E,mCAAmC;AACnC,MAAM,kBAAkB,GAAG,ubAAub,CAAC;AAEnd;;;;;GAKG;AACH,MAAM,WAAW,GAAG,IAAI,MAAM,CAAC,IAAI,kBAAkB,GAAG,EAAE,GAAG,CAAC,CAAC;AAE/D;;;;;GAKG;AACH,MAAM,kBAAkB,GAAG,IAAI,MAAM,CAAC,kBAAkB,EAAE,IAAI,CAAC,CAAC;AAEhE,MAAM,sBAAsB,GAAG,WAAW,CAAC;AAE3C;;;;;;;GAOG;AACH,MAAM,cAAc,GAAG,CAAC,IAAI,EAAE,EAAE;IAC5B,IAAI,CAAC,oBAAC,CAAC,QAAQ,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IACjC,OAAO,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC;AAChD,CAAC,CAAC;AAEF;;;;;;;;GAQG;AACH,MAAM,cAAc,GAAG,CAAC,IAAI,EAAE,EAAE;IAC5B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAEnF,MAAM,MAAM,GAAG,EAAE,CAAC;IAClB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACpB,IAAI,CAAC,GAAG;YAAE,SAAS;QACnB,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,SAAS;QAEhD,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,sBAAsB,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAC7D,IAAI,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;KACnD;IACD,OAAO,MAAM,CAAC;AAClB,CAAC,CAAC;AAEF,4GAA4G;AAC5G,MAAM,sBAAsB,GAAG,2CAA2C,CAAC;AAE3E,6HAA6H;AAC7H,iHAAiH;AACjH,mEAAmE;AACnE,qHAAqH;AACrH,MAAM,oBAAoB,GAAG;IACzB,YAAY;IACZ,aAAa;IAEb,6FAA6F;IAC7F,2EAA2E;IAE3E,sCAAsC;IACtC,2EAA2E;IAE3E,mBAAmB;IACnB,6CAA6C;IAC7C,eAAe;IACf,kCAAkC;IAClC,UAAU;IACV,uBAAuB;IAEvB,mBAAmB;IACnB,mDAAmD;IACnD,eAAe;IACf,sCAAsC;IACtC,UAAU;IACV,yBAAyB;IAEzB,mBAAmB;IACnB,6CAA6C;IAC7C,eAAe;IACf,kCAAkC;IAClC,WAAW;IACX,uBAAuB;CAC1B,CAAC;AAEF,gDAAgD;AAChD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,oBAAoB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;IAClD,oBAAoB,CAAC,CAAC,CAAC,GAAG,YAAY,oBAAoB,CAAC,CAAC,CAAC,EAAE,CAAC;CACnE;AAED,2DAA2D;AAC3D,uGAAuG;AACvG,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAE3B,oEAAoE;AACpE,4EAA4E;AAC5E,MAAM,iBAAiB,GAAG;IACtB,aAAa;IACb,8BAA8B;CACjC,CAAC;AAEF,MAAM,kBAAkB,GAAG,IAAI,MAAM,CAAC,IAAI,oBAAoB,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;AACnF,MAAM,WAAW,GAAG,IAAI,MAAM,CAAC,KAAK,oBAAoB,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;AAC7E,MAAM,gBAAgB,GAAG,IAAI,MAAM,CAAC,KAAK,iBAAiB,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;AAE/E;;;;;;;;GAQG;AACH,MAAM,cAAc,GAAG,CAAC,IAAI,EAAE,EAAE;IAC5B,IAAI,CAAC,oBAAC,CAAC,QAAQ,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IAEjC,IAAI,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC;IAClD,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE;QAC7B,IAAI,CAAC,KAAK;YAAE,OAAO,KAAK,CAAC;QAEzB,wDAAwD;QACxD,IAAI,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,gBAAgB;YAAE,OAAO,KAAK,CAAC;QAElE,gDAAgD;QAChD,IAAI,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QAE/C,OAAO,IAAI,CAAC;IAChB,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAClB,CAAC,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,cAAc,GAAG,CAAC,IAAI,EAAE,EAAE;IAC5B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAEnF,MAAM,MAAM,GAAG,EAAE,CAAC;IAClB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACpB,IAAI,CAAC,GAAG;YAAE,SAAS;QACnB,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,SAAS;QAEhD,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,sBAAsB,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAC7D,IAAI,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;KACnD;IACD,OAAO,MAAM,CAAC;AAClB,CAAC,CAAC;AAEF,sCAAsC;AACtC,uGAAuG;AACvG,kGAAkG;AAClG,wFAAwF;AAExF,mCAAmC;AACnC,MAAM,qBAAqB,GAAG,oHAAoH,CAAC;AAEnJ,yHAAyH;AACzH,mCAAmC;AACnC,MAAM,sBAAsB,GAAG,wHAAwH,CAAC;AAExJ,MAAM,sBAAsB,GAAG,sGAAsG,CAAC;AACtI,2CAA2C;AAC3C,MAAM,oBAAoB,GAAG,kEAAkE,sBAAsB,+DAA+D,CAAC;AAErL,2CAA2C;AAC3C,MAAM,uBAAuB,GAAG,kWAAkW,CAAC;AACnY,2CAA2C;AAC3C,MAAM,qBAAqB,GAAG,0EAA0E,uBAAuB,qHAAqH,CAAC;AACrP,2CAA2C;AAC3C,MAAM,oBAAoB,GAAG,wJAAwJ,CAAC;AAEtL,mBAAmB;AACnB,IAAI,cAAc,CAAC;AACnB,mBAAmB;AACnB,IAAI,qBAAqB,CAAC;AAC1B,mBAAmB;AACnB,IAAI,eAAe,CAAC;AACpB,mBAAmB;AACnB,IAAI,sBAAsB,CAAC;AAC3B,mBAAmB;AACnB,IAAI,aAAa,CAAC;AAClB,mBAAmB;AACnB,IAAI,oBAAoB,CAAC;AACzB,mBAAmB;AACnB,IAAI,cAAc,CAAC;AACnB,mBAAmB;AACnB,IAAI,qBAAqB,CAAC;AAC1B,mBAAmB;AACnB,IAAI,aAAa,CAAC;AAClB,mBAAmB;AACnB,IAAI,oBAAoB,CAAC;AAEzB,IAAI;IACA;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,cAAc,GAAG,IAAI,MAAM,CAAC,IAAI,qBAAqB,GAAG,EAAE,GAAG,CAAC,CAAC;IAE/D;;;;;;;;;;;;;;;;;;;;;;;;;;OA0BG;IACH,qBAAqB,GAAG,IAAI,MAAM,CAAC,qBAAqB,EAAE,IAAI,CAAC,CAAC;IAEhE;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,eAAe,GAAG,IAAI,MAAM,CAAC,IAAI,sBAAsB,GAAG,EAAE,GAAG,CAAC,CAAC;IAEjE;;;;;;;;;;;;;;;;;;;;;;;;;;OA0BG;IACH,sBAAsB,GAAG,IAAI,MAAM,CAAC,sBAAsB,EAAE,IAAI,CAAC,CAAC;IAElE;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,aAAa,GAAG,IAAI,MAAM,CAAC,IAAI,oBAAoB,GAAG,EAAE,GAAG,CAAC,CAAC;IAE7D;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;IACH,oBAAoB,GAAG,IAAI,MAAM,CAAC,oBAAoB,EAAE,IAAI,CAAC,CAAC;IAE9D;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACH,cAAc,GAAG,IAAI,MAAM,CAAC,IAAI,qBAAqB,GAAG,EAAE,GAAG,CAAC,CAAC;IAE/D;;;;;;;;;;;;;;;;;;;;;;;;;;OA0BG;IACH,qBAAqB,GAAG,IAAI,MAAM,CAAC,qBAAqB,EAAE,IAAI,CAAC,CAAC;IAEhE;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,aAAa,GAAG,IAAI,MAAM,CAAC,IAAI,oBAAoB,GAAG,EAAE,GAAG,CAAC,CAAC;IAE7D;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,oBAAoB,GAAG,IAAI,MAAM,CAAC,oBAAoB,EAAE,IAAI,CAAC,CAAC;CACjE;AAAC,OAAO,CAAC,EAAE;IACR,sFAAsF;IACtF,mCAAmC;IACnC,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,EAAE;QACvD,mCAAmC;QACnC,mBAAG,CAAC,OAAO,CAAC,4BAA4B,OAAO,CAAC,OAAO,+JAA+J,CAAC,CAAC;KAC3N;SAAM;QACH,MAAM,CAAC,CAAC;KACX;CACJ;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,MAAM,oBAAoB,GAAG,CAAC,IAAI,EAAE,IAAI,GAAG,IAAI,EAAE,EAAE;IAC/C,MAAM,MAAM,GAAG;QACX,MAAM,EAAE,EAAE;QACV,MAAM,EAAE,EAAE;QACV,eAAe,EAAE,EAAE;QACnB,SAAS,EAAE,EAAE;QACb,QAAQ,EAAE,EAAE;QACZ,UAAU,EAAE,EAAE;QACd,SAAS,EAAE,EAAE;QACb,QAAQ,EAAE,EAAE;KACf,CAAC;IAEF,iDAAiD;IAEjD,IAAI,CAAC,oBAAC,CAAC,QAAQ,CAAC,IAAI,CAAC;QAAE,OAAO,MAAM,CAAC;IAErC,MAAM,CAAC,GAAG,iBAAO,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC;IACvD,IAAI,IAAI;QAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IAErB,MAAM,IAAI,GAAG,mBAAW,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IACvC,IAAI,IAAI;QAAE,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IAE3B,mCAAmC;IACnC,MAAM,QAAQ,GAAG,EAAE,CAAC;IACpB,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;QAC9B,IAAI,IAAI;YAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,MAAM,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC;IACtE,MAAM,CAAC,MAAM,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;IACzC,MAAM,CAAC,eAAe,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;IAE9C,8EAA8E;IAC9E,2EAA2E;IAC3E,mBAAmB;IACnB,mDAAmD;IACnD,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,qBAAqB,CAAC,IAAI,EAAE,CAAC;IAC3D,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,oBAAoB,CAAC,IAAI,EAAE,CAAC;IACzD,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,IAAI,EAAE,CAAC;IAC7D,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,qBAAqB,CAAC,IAAI,EAAE,CAAC;IAC3D,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,oBAAoB,CAAC,IAAI,EAAE,CAAC;IAEzD,+BAA+B;IAC/B,8DAA8D;IAC9D,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE;QACtB,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACnB,MAAM,CAAC,GAAG,CAAC,GAAG,oBAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;KAC3C;IAED,OAAO,MAAM,CAAC;AAClB,CAAC,CAAC;AAEF;;;;;;;;;;;;GAYG;AACU,QAAA,WAAW,GAAG;IACvB,cAAc;IACd,cAAc;IACd,cAAc;IACd,cAAc;IACd,oBAAoB;IAEpB,WAAW;IACX,kBAAkB;IAElB,cAAc;IACd,qBAAqB;IAErB,eAAe;IACf,sBAAsB;IAEtB,aAAa;IACb,oBAAoB;IAEpB,cAAc;IACd,qBAAqB;IAErB,aAAa;IACb,oBAAoB;CACvB,CAAC"}
@@ -20,3 +20,4 @@ export namespace validators {
20
20
  message: (label: any) => string;
21
21
  };
22
22
  }
23
+ //# sourceMappingURL=validators.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"validators.d.ts","sourceRoot":"","sources":["../src/validators.js"],"names":[],"mappings":";IAIiB;;;MAGX;IACkB;;;MAGlB;IACW;;;MAGX;IACY;;;MAGZ;IACS;;;MAGT"}
@@ -1,7 +1,8 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.validators = void 0;
4
- const ow_1 = require("ow");
4
+ const tslib_1 = require("tslib");
5
+ const ow_1 = (0, tslib_1.__importDefault)(require("ow"));
5
6
  exports.validators = {
6
7
  // Naming it browser page for future proofing with Playwright
7
8
  browserPage: (value) => ({
@@ -25,3 +26,4 @@ exports.validators = {
25
26
  message: (label) => `Expected argument '${label}' to be a PseudoUrl, got something else.`,
26
27
  }),
27
28
  };
29
+ //# sourceMappingURL=validators.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"validators.js","sourceRoot":"","sources":["../src/validators.js"],"names":[],"mappings":";;;;AAAA,yDAAoB;AAEP,QAAA,UAAU,GAAG;IACtB,6DAA6D;IAC7D,WAAW,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACrB,SAAS,EAAE,YAAE,CAAC,OAAO,CAAC,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;QAC9E,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,sBAAsB,KAAK,+CAA+C;KACjG,CAAC;IACF,kBAAkB,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC5B,SAAS,EAAE,YAAE,CAAC,OAAO,CAAC,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;QACzE,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,sBAAsB,KAAK,mDAAmD;KACrG,CAAC;IACF,WAAW,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACrB,SAAS,EAAE,YAAE,CAAC,OAAO,CAAC,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,EAAE,cAAc,CAAC,CAAC;QACnF,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,sBAAsB,KAAK,4CAA4C;KAC9F,CAAC;IACF,YAAY,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACtB,SAAS,EAAE,YAAE,CAAC,OAAO,CAAC,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,EAAE,YAAY,CAAC,CAAC;QACjF,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,sBAAsB,KAAK,6CAA6C;KAC/F,CAAC;IACF,SAAS,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACnB,SAAS,EAAE,YAAE,CAAC,OAAO,CAAC,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC;QAC3E,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,sBAAsB,KAAK,0CAA0C;KAC5F,CAAC;CACL,CAAC"}