@detiq/crawler 1.1.28 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/crawler.js +9 -0
- package/dist/crawler.js.map +2 -2
- package/dist/failure-analysis-agent.js +2 -1
- package/dist/failure-analysis-agent.js.map +2 -2
- package/dist/job-queue.js +177 -195
- package/dist/job-queue.js.map +2 -2
- package/package.json +1 -1
- package/src/job-queue.test.ts +69 -2
- package/src/job-queue.ts +202 -160
- package/types/job-queue.d.ts +12 -0
package/dist/crawler.js
CHANGED
|
@@ -6123,6 +6123,15 @@ async function runAuthDiscoveryPhase(page, opts, origin, onProgress) {
|
|
|
6123
6123
|
if (pageTitle === homeTitle && elemCount === homeElemCount) continue;
|
|
6124
6124
|
}
|
|
6125
6125
|
const finalUrl = page.url?.() ?? current.url;
|
|
6126
|
+
if (current.source === "seed_path" && finalUrl !== current.url) {
|
|
6127
|
+
try {
|
|
6128
|
+
const origPath = new URL(current.url).pathname;
|
|
6129
|
+
const finalPath = new URL(finalUrl).pathname;
|
|
6130
|
+
const isAuthSeed = /login|sign-?in|signin|auth|sso|forgot-?password|reset-?password|2fa|mfa/i.test(origPath);
|
|
6131
|
+
if (isAuthSeed && finalPath !== origPath) continue;
|
|
6132
|
+
} catch {
|
|
6133
|
+
}
|
|
6134
|
+
}
|
|
6126
6135
|
const captureKey = normalizeAuthUrl(finalUrl);
|
|
6127
6136
|
if (!capturedKeys.has(captureKey)) {
|
|
6128
6137
|
const result = await captureAuthDiscoveryScreen(page, opts, current.url, current.source, ordinal++);
|