@detiq/crawler 0.2.14 → 1.0.0

package/dist/crawler.js

@@ -3,7 +3,7 @@ import { chromium } from "rebrowser-playwright-core";
 import { chromium as stealthChromium } from "playwright-extra";
 import StealthPlugin from "puppeteer-extra-plugin-stealth";
 import { execSync } from "node:child_process";
-import { readFileSync } from "node:fs";
+import { readFileSync, existsSync as existsSyncFs } from "node:fs";
 import { createRequire } from "node:module";
 stealthChromium.use(StealthPlugin());
 import { z } from "zod";
@@ -316,6 +316,7 @@ async function crawlProject(opts, onProgress) {
   if (opts.screenshotDir) await fs.mkdir(opts.screenshotDir, { recursive: true });
   const uploadToCloud = async (localPath, key) => {
     if (!opts.storageConfig) return void 0;
+    if (!existsSyncFs(localPath)) return void 0;
     try {
       const { uploadScreenshotToTenantStorage } = await import("@detiq/app-brain");
       return await uploadScreenshotToTenantStorage(opts.storageConfig, localPath, key);
@@ -1024,7 +1025,7 @@ async function crawlProject(opts, onProgress) {
     }
     let checkpointSuccessCount = 0;
     let maxSeenTotal = 0;
-    let lastEmittedPct = 5;
+    let lastEmittedPct = 20;
     const crawlLoopStart = Date.now();
     const startUrl = page.url();
     const resumeUrls = opts.resumeUrls ?? [];
@@ -1426,7 +1427,7 @@ async function crawlProject(opts, onProgress) {
               throw new Error("[AUTH_FAILURE] Re-authentication failed. Session is unrecoverable. Re-capture the session in project settings and retry the crawl.");
             }
           } else {
-            logger.warn("[crawler] Session expired mid-crawl, no credentials to recover \u2014 stopping BFS");
+            logger.warn("[crawler] Session expired mid-crawl, no credentials to recover \u2014 skipping URL");
             AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
               jobId: opts.jobId,
               type: "AUTH_FAILURE",
@@ -3661,15 +3662,17 @@ async function captureAuthDiscoveryScreen(page, opts, requestedUrl, source, ordi
   const screen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, name, screenUrl, opts.jobId, source === "entry" ? "ENTRY_POINT" : "SEED_URL").catch(() => null);
   if (!screen) return { captured: false, finalUrl, flowType };
   let authCloudUrl;
-  if (opts.storageConfig) {
+  if (opts.storageConfig && existsSyncFs(shotPath)) {
     try {
       const { uploadScreenshotToTenantStorage } = await import("@detiq/app-brain");
       authCloudUrl = await uploadScreenshotToTenantStorage(opts.storageConfig, shotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(shotPath)}`);
     } catch {
     }
   }
-  await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, "DESKTOP", authCloudUrl).catch(() => {
-  });
+  if (existsSyncFs(shotPath)) {
+    await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, "DESKTOP", authCloudUrl).catch(() => {
+    });
+  }
   await AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
     screenId: screen.id,
     jobId: opts.jobId,