@detiq/crawler 0.2.9 → 0.2.11

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@detiq/crawler",
-  "version": "0.2.9",
+  "version": "0.2.11",
   "private": false,
   "publishConfig": {
     "access": "public",

package/src/crawler.ts

@@ -20,7 +20,7 @@ import StealthPlugin from 'puppeteer-extra-plugin-stealth';
 // Option 3: Camoufox — Firefox-based browser with built-in fingerprint randomisation.
 // Loaded dynamically (not at startup) to avoid ESM/CJS conflict when API imports crawler.
 import { spawn, execSync, type ChildProcess } from 'node:child_process';
-import { readFileSync } from 'node:fs';
+import { readFileSync, existsSync as existsSyncFs } from 'node:fs';
 import { createRequire } from 'node:module';
 
 stealthChromium.use(StealthPlugin());
@@ -510,6 +510,7 @@ export async function crawlProject(opts: {
   // Upload screenshot to cloud storage if configured, returns cloudUrl or undefined
   const uploadToCloud = async (localPath: string, key: string): Promise<string | undefined> => {
     if (!opts.storageConfig) return undefined;
+    if (!existsSyncFs(localPath)) return undefined;
     try {
       const { uploadScreenshotToTenantStorage } = await import('@detiq/app-brain');
       return await uploadScreenshotToTenantStorage(opts.storageConfig as any, localPath, key);
@@ -4313,13 +4314,15 @@ async function captureAuthDiscoveryScreen(
   const screen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, name, screenUrl, opts.jobId, source === 'entry' ? 'ENTRY_POINT' : 'SEED_URL').catch(() => null);
   if (!screen) return { captured: false, finalUrl, flowType };
   let authCloudUrl: string | undefined;
-  if (opts.storageConfig) {
+  if (opts.storageConfig && existsSyncFs(shotPath)) {
     try {
       const { uploadScreenshotToTenantStorage } = await import('@detiq/app-brain');
       authCloudUrl = await uploadScreenshotToTenantStorage(opts.storageConfig as any, shotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(shotPath)}`);
     } catch { /* non-fatal */ }
   }
-  await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, 'DESKTOP', authCloudUrl).catch(() => { });
+  if (existsSyncFs(shotPath)) {
+    await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, 'DESKTOP', authCloudUrl).catch(() => { });
+  }
   await AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
     screenId: screen.id,
     jobId: opts.jobId,

package/src/job-queue.ts

@@ -383,7 +383,10 @@ export async function enqueueJob(type: string, config: any): Promise<string> {
   if (type === 'CRAWL' && process.env.JOB_QUEUE_BACKEND === 'sqs') {
     const { sendJobToSqs } = await import('./sqs-queue.js');
     await sendJobToSqs(job.id, tenantId, config);
-  } else {
+  } else if (process.env.DISABLE_WORKER !== 'true') {
+    // Skip inline dispatch when DISABLE_WORKER=true — the dedicated crawler process
+    // will pick this up via its poll loop. Inline dispatch in the API container
+    // would launch Playwright browsers and crash the API.
     setImmediate(() => claimSpecificJob(job.id).then((claimed) => claimed && dispatchClaimedJob(claimed)));
   }