@detiq/crawler 1.1.20 → 1.1.21

package/dist/crawler.js

@@ -374,6 +374,7 @@ const TwoFASchema = z.object({
   has2FA: z.boolean(),
   fieldLabel: z.string().optional()
 });
+const INLINE_AUTH_WALL_RE = /your login session (was|has been) expired|session (has )?expired[^.]*click|please (log ?in|sign ?in) to continue|you (are|were) logged out|your session has timed out/i;
 async function crawlProject(opts, onProgress) {
   const tuning = await getCrawlTuning();
   if (opts.screenshotDir) await fs.mkdir(opts.screenshotDir, { recursive: true });
@@ -1681,7 +1682,6 @@ async function crawlProject(opts, onProgress) {
           const name = pathParts.map((p) => p.charAt(0).toUpperCase() + p.slice(1).replace(/[-_]/g, " ")).join(" \u2014 ") || "Home";
           const screen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, name, url, opts.jobId, discoveryMethodFor(url));
           visited.set(canonUrl, screen.id);
-          const INLINE_AUTH_WALL_RE = /your login session (was|has been) expired|session (has )?expired[^.]*click|please (log ?in|sign ?in) to continue|you (are|were) logged out|your session has timed out/i;
           const lwBodyText = await workerPage.evaluate(() => document.body?.innerText ?? "").catch(() => "");
           if (INLINE_AUTH_WALL_RE.test(lwBodyText)) {
             logger.warn({ url, screenId: screen.id }, "[lite-worker] Inline session-expired content detected \u2014 marking requiresAuth");
@@ -2875,9 +2875,9 @@ async function crawlProject(opts, onProgress) {
             })();
           }
         }
-        const INLINE_AUTH_WALL_RE = /your login session (was|has been) expired|session (has )?expired[^.]*click|please (log ?in|sign ?in) to continue|you (are|were) logged out|your session has timed out/i;
+        const INLINE_AUTH_WALL_RE2 = /your login session (was|has been) expired|session (has )?expired[^.]*click|please (log ?in|sign ?in) to continue|you (are|were) logged out|your session has timed out/i;
         const pageBodyText = await page.evaluate(() => document.body?.innerText ?? "").catch(() => "");
-        if (INLINE_AUTH_WALL_RE.test(pageBodyText)) {
+        if (INLINE_AUTH_WALL_RE2.test(pageBodyText)) {
           logger.warn({ url, screenId: screen.id }, "[crawler] Inline session-expired content detected \u2014 skipping DOM snapshot, marking requiresAuth");
           AppBrain.updateScreenRequiresAuth(opts.tenantId, screen.id, true).catch(() => {
           });
@@ -3614,6 +3614,13 @@ ${snippet}`,
               await AppBrain.saveElement(opts.tenantId, opts.projectId, existingScreenId, el.meaning, el.role, el.expectedData, el.notes, void 0, true, el.boundingRect, el.ariaState, el.parentLandmark).catch(() => {
               });
             }
+          } else {
+            const p3aBodyText = await page.evaluate(() => document.body?.innerText ?? "").catch(() => "");
+            if (INLINE_AUTH_WALL_RE.test(p3aBodyText)) {
+              logger.warn({ url: revisitUrl, screenId: existingScreenId }, "[P3a] Re-visit got 0 elements \u2014 inline session-expired content, marking requiresAuth");
+              AppBrain.updateScreenRequiresAuth(opts.tenantId, existingScreenId, true).catch(() => {
+              });
+            }
           }
           const revisitLinks = await discoverLinks(stagehand, origin, page, linkLLM ?? void 0, false, rawPage ?? void 0).catch(() => []);
           for (const href of revisitLinks) {
@@ -3753,6 +3760,14 @@ ${snippet}`,
                 await AppBrain.saveElement(opts.tenantId, opts.projectId, gapScreen.id, el.meaning, el.role, el.expectedData, el.notes, void 0, true, el.boundingRect, el.ariaState, el.parentLandmark).catch(() => {
                 });
               }
+              if (gapElems.length === 0) {
+                const gapBodyText = await page.evaluate(() => document.body?.innerText ?? "").catch(() => "");
+                if (INLINE_AUTH_WALL_RE.test(gapBodyText)) {
+                  logger.warn({ url: gapUrl, screenId: gapScreen.id }, "[P6] Gap URL got 0 elements \u2014 inline session-expired content, marking requiresAuth");
+                  AppBrain.updateScreenRequiresAuth(opts.tenantId, gapScreen.id, true).catch(() => {
+                  });
+                }
+              }
               visited.set(canonicalize(gapUrl), gapScreen.id);
               discovered.push({ url: gapUrl, screenId: gapScreen.id, elements: gapElems.length, version: gapScreen.version });
               for (const lm of gapLinkMaps) {
@@ -5867,6 +5882,16 @@ async function captureAuthDiscoveryScreen(page, opts, requestedUrl, source, ordi
     await AppBrain.updateScreenDomSnapshot(opts.tenantId, screen.id, authDomPath, authDomCloudUrl).catch(() => {
     });
   }
+  const authDiscoveryElements = await readScreen(page).catch(() => []);
+  if (authDiscoveryElements.length > 0) {
+    await AppBrain.clearScreenElements(opts.tenantId, opts.projectId, screen.id).catch(() => {
+    });
+    for (const el of authDiscoveryElements) {
+      await AppBrain.saveElement(opts.tenantId, opts.projectId, screen.id, el.label, el.role, el.expectedData, void 0, void 0, true, void 0, void 0, void 0).catch(() => {
+      });
+    }
+    logger.info({ screenId: screen.id, elements: authDiscoveryElements.length }, "[crawler] auth-discovery: saved elements");
+  }
   await AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
     screenId: screen.id,
     jobId: opts.jobId,