@detiq/crawler 1.1.0 → 1.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/job-queue.ts +3 -0
package/package.json
CHANGED
package/src/job-queue.ts
CHANGED
|
@@ -1689,6 +1689,7 @@ async function processDiscoverJob(jobId: string, config: any, opts?: ProcessJobO
|
|
|
1689
1689
|
const maxDiscoverUrls: number = config.maxDiscoverUrls ?? 2000;
|
|
1690
1690
|
const proxyUrl: string | undefined = config.proxyUrl ?? crawlCfg.proxyUrl;
|
|
1691
1691
|
const screenshotDir = os.tmpdir();
|
|
1692
|
+
const discoverStorageConfig = await getTenantStorageConfig(tenantId).catch(() => null);
|
|
1692
1693
|
|
|
1693
1694
|
await onProgress(3, 'Seeding from sitemap/robots.txt…');
|
|
1694
1695
|
|
|
@@ -1714,6 +1715,7 @@ async function processDiscoverJob(jobId: string, config: any, opts?: ProcessJobO
|
|
|
1714
1715
|
maxScreens: maxDiscoverUrls,
|
|
1715
1716
|
proxyUrl,
|
|
1716
1717
|
useFirefox: crawlCfg.useFirefox,
|
|
1718
|
+
storageConfig: discoverStorageConfig,
|
|
1717
1719
|
abortSignal: controller.signal,
|
|
1718
1720
|
jobId,
|
|
1719
1721
|
onUrlsDiscovered: async (urls: string[]) => {
|
|
@@ -1794,6 +1796,7 @@ async function processDiscoverJob(jobId: string, config: any, opts?: ProcessJobO
|
|
|
1794
1796
|
proxyUrl,
|
|
1795
1797
|
chromeProfilePath: crawlCfg.chromeProfilePath,
|
|
1796
1798
|
useFirefox: crawlCfg.useFirefox,
|
|
1799
|
+
storageConfig: discoverStorageConfig,
|
|
1797
1800
|
captchaSolverApiKey: crawlCfg.captchaSolverApiKey,
|
|
1798
1801
|
captchaSolverProvider: crawlCfg.captchaSolverProvider,
|
|
1799
1802
|
mailslurpApiKey: crawlCfg.mailslurpApiKey,
|