@detiq/crawler 1.1.21 → 1.1.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Dockerfile +5 -0
- package/dist/crawler.js +20 -7
- package/dist/crawler.js.map +2 -2
- package/dist/job-queue.js +7 -1
- package/dist/job-queue.js.map +2 -2
- package/package.json +1 -1
- package/src/crawler.ts +27 -6
- package/src/job-queue.ts +98 -86
- package/types/job-queue.d.ts +3 -1
package/dist/job-queue.js
CHANGED
|
@@ -85,6 +85,7 @@ const WORKER_ID = `${os.hostname()}:${process.pid}`;
|
|
|
85
85
|
const STALE_JOB_TIMEOUT_MINUTES = Number(process.env.JOB_STALE_TIMEOUT_MINUTES) || 30;
|
|
86
86
|
const WORKER_POLL_INTERVAL_MS = Number(process.env.JOB_WORKER_POLL_INTERVAL_MS) || 15e3;
|
|
87
87
|
const WORKER_CONCURRENCY = Number(process.env.JOB_WORKER_CONCURRENCY) || 8;
|
|
88
|
+
const MAX_CONCURRENT_DISCOVER = Number(process.env.MAX_CONCURRENT_DISCOVER) || 2;
|
|
88
89
|
const JOB_HEARTBEAT_INTERVAL_MS = Number(process.env.JOB_HEARTBEAT_INTERVAL_MS) || 15e3;
|
|
89
90
|
const TEST_EXEC_CONCURRENCY = Number(process.env.TEST_EXEC_CONCURRENCY) || 5;
|
|
90
91
|
let jobDispatchTable = null;
|
|
@@ -98,12 +99,17 @@ async function claimSpecificJob(jobId) {
|
|
|
98
99
|
return rows[0] ?? null;
|
|
99
100
|
}
|
|
100
101
|
async function claimNextQueuedJob() {
|
|
102
|
+
const maxDiscover = MAX_CONCURRENT_DISCOVER;
|
|
101
103
|
const rows = await prismaAdmin.$queryRaw`
|
|
102
104
|
UPDATE "PlatformJob"
|
|
103
105
|
SET status = 'RUNNING', "startedAt" = NOW(), "heartbeatAt" = NOW(), "workerId" = ${WORKER_ID}
|
|
104
106
|
WHERE id = (
|
|
105
107
|
SELECT id FROM "PlatformJob"
|
|
106
108
|
WHERE status = 'QUEUED'
|
|
109
|
+
AND (
|
|
110
|
+
type != 'DISCOVER'
|
|
111
|
+
OR (SELECT COUNT(*) FROM "PlatformJob" WHERE status = 'RUNNING' AND type = 'DISCOVER') < ${maxDiscover}
|
|
112
|
+
)
|
|
107
113
|
ORDER BY "queuedAt" ASC
|
|
108
114
|
FOR UPDATE SKIP LOCKED
|
|
109
115
|
LIMIT 1
|
|
@@ -1530,7 +1536,7 @@ async function processDiscoverJob(jobId, config, opts) {
|
|
|
1530
1536
|
}
|
|
1531
1537
|
logger.info({ count: allUrls.size }, "[discover] Phase 3 auth spider done");
|
|
1532
1538
|
}
|
|
1533
|
-
const discoveredUrls = Array.from(allUrls);
|
|
1539
|
+
const discoveredUrls = Array.from(allUrls).filter((u) => u.startsWith("http"));
|
|
1534
1540
|
const existingCfg = await AppBrain.getCrawlConfig(tenantId, projectId).catch(() => ({}));
|
|
1535
1541
|
await AppBrain.saveCrawlConfig(tenantId, projectId, {
|
|
1536
1542
|
...existingCfg,
|