@detiq/crawler 0.2.10 → 0.2.12

package/Dockerfile

@@ -16,13 +16,25 @@ RUN apt-get update && apt-get install -y curl && \
     apt-get clean && rm -rf /var/lib/apt/lists/*
 
 # Workspace manifests (layer cache)
+# Copy all workspace package.json files so npm ci can resolve the full lockfile.
+# Packages not used by the crawler (api, web, etc.) are stubs; their source is not copied later.
 COPY package.json package-lock.json ./
-COPY packages/core/package.json         packages/core/
-COPY packages/database/package.json     packages/database/
-COPY packages/app-brain/package.json    packages/app-brain/
+COPY packages/agent-cli/package.json    packages/agent-cli/
 COPY packages/agents/package.json       packages/agents/
+COPY packages/api/package.json          packages/api/
+COPY packages/app-brain/package.json    packages/app-brain/
+COPY packages/chrome-extension/package.json packages/chrome-extension/
+COPY packages/connectors/package.json   packages/connectors/
+COPY packages/core/package.json         packages/core/
 COPY packages/crawler/package.json      packages/crawler/
-RUN npm ci
+COPY packages/database/package.json     packages/database/
+COPY packages/github-action/package.json packages/github-action/
+COPY packages/reranker/package.json     packages/reranker/
+COPY packages/sdk/package.json          packages/sdk/
+COPY packages/session-receiver/package.json packages/session-receiver/
+COPY packages/vscode-extension/package.json packages/vscode-extension/
+COPY packages/web/package.json          packages/web/
+RUN npm ci --include=dev
 
 # Re-install chromium after npm ci so the browser revision matches the
 # exact playwright version resolved by the lockfile (not the base image's
@@ -31,16 +43,20 @@ RUN npx playwright install chromium
 
 # Source
 COPY tsconfig.base.json ./
-COPY packages/core       packages/core
-COPY packages/database   packages/database
-COPY packages/app-brain  packages/app-brain
-COPY packages/agents     packages/agents
-COPY packages/crawler    packages/crawler
+COPY packages/database        packages/database
+COPY packages/reranker        packages/reranker
+COPY packages/core            packages/core
+COPY packages/connectors      packages/connectors
+COPY packages/app-brain       packages/app-brain
+COPY packages/agents          packages/agents
+COPY packages/crawler         packages/crawler
 
-# Build
+# Build order: database → reranker → core → connectors → app-brain → agents → crawler
 RUN npx prisma generate --schema packages/database/prisma/schema.prisma && \
-    npm run -w @detiq/core build && \
     npm run -w @detiq/database build && \
+    npm run -w @detiq/reranker build && \
+    npm run -w @detiq/core build && \
+    npm run -w @detiq/connectors build && \
     npm run -w @detiq/app-brain build && \
     npm run -w @detiq/agents build && \
     npm run -w @detiq/crawler build

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@detiq/crawler",
-  "version": "0.2.10",
+  "version": "0.2.12",
   "private": false,
   "publishConfig": {
     "access": "public",

package/src/crawler.ts

@@ -20,7 +20,7 @@ import StealthPlugin from 'puppeteer-extra-plugin-stealth';
 // Option 3: Camoufox — Firefox-based browser with built-in fingerprint randomisation.
 // Loaded dynamically (not at startup) to avoid ESM/CJS conflict when API imports crawler.
 import { spawn, execSync, type ChildProcess } from 'node:child_process';
-import { readFileSync } from 'node:fs';
+import { readFileSync, existsSync as existsSyncFs } from 'node:fs';
 import { createRequire } from 'node:module';
 
 stealthChromium.use(StealthPlugin());
@@ -510,6 +510,7 @@ export async function crawlProject(opts: {
   // Upload screenshot to cloud storage if configured, returns cloudUrl or undefined
   const uploadToCloud = async (localPath: string, key: string): Promise<string | undefined> => {
     if (!opts.storageConfig) return undefined;
+    if (!existsSyncFs(localPath)) return undefined;
     try {
       const { uploadScreenshotToTenantStorage } = await import('@detiq/app-brain');
       return await uploadScreenshotToTenantStorage(opts.storageConfig as any, localPath, key);
@@ -1339,7 +1340,7 @@ export async function crawlProject(opts: {
     }
     let checkpointSuccessCount = 0;
     let maxSeenTotal = 0; // track high-water mark so progress % never goes backwards
-    let lastEmittedPct = 5; // clamp so progress never decreases even when queue grows
+    let lastEmittedPct = 20; // floor above auth-discovery + login phases (max ~19%)
     const crawlLoopStart = Date.now();
     const startUrl = page.url();
     // Resume from previous segment's pending queue if provided
@@ -4313,13 +4314,15 @@ async function captureAuthDiscoveryScreen(
   const screen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, name, screenUrl, opts.jobId, source === 'entry' ? 'ENTRY_POINT' : 'SEED_URL').catch(() => null);
   if (!screen) return { captured: false, finalUrl, flowType };
   let authCloudUrl: string | undefined;
-  if (opts.storageConfig) {
+  if (opts.storageConfig && existsSyncFs(shotPath)) {
     try {
       const { uploadScreenshotToTenantStorage } = await import('@detiq/app-brain');
       authCloudUrl = await uploadScreenshotToTenantStorage(opts.storageConfig as any, shotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(shotPath)}`);
     } catch { /* non-fatal */ }
   }
-  await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, 'DESKTOP', authCloudUrl).catch(() => { });
+  if (existsSyncFs(shotPath)) {
+    await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, 'DESKTOP', authCloudUrl).catch(() => { });
+  }
   await AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
     screenId: screen.id,
     jobId: opts.jobId,

package/src/job-queue.ts

@@ -1555,21 +1555,24 @@ async function processDiscoverJob(jobId: string, config: any, opts?: ProcessJobO
   // This lets the frontend poll and see a growing list before the job finishes.
   const allUrls = new Set<string>();
   let lastPartialFlush = 0;
+  let lastFlushedPct = 0; // high-water mark — progress never decreases
 
   const flushPartialUrls = async (pct: number, step: string) => {
     const now = Date.now();
     // Flush to DB at most every 5 seconds to avoid excessive writes
     if (now - lastPartialFlush < 5_000 && allUrls.size < 5) return;
+    const monotonic = Math.max(pct, lastFlushedPct);
+    lastFlushedPct = monotonic;
     lastPartialFlush = now;
     try {
       await withTenant(tenantId, (tx) =>
         tx.platformJob.update({
           where: { id: jobId },
-          data: { progress: pct, result: { step, partialUrls: Array.from(allUrls) } as any, heartbeatAt: new Date() },
+          data: { progress: monotonic, result: { step, partialUrls: Array.from(allUrls) } as any, heartbeatAt: new Date() },
         })
       );
     } catch { /* non-fatal */ }
-    await opts?.onProgress?.(pct);
+    await opts?.onProgress?.(monotonic);
   };
 
   const onProgress = async (pct: number, step: string) => {

package/src/tls-fingerprint.ts

@@ -82,7 +82,7 @@ export async function probeTlsFingerprint(host: string, port = 443): Promise<{
         protocol: socket.getProtocol() ?? 'unknown',
         cipher: socket.getCipher()?.name ?? 'unknown',
         authorized: socket.authorized,
-        subjectCN: cert?.subject?.CN ?? null,
+        subjectCN: cert?.subject?.CN?.toString() ?? null,
       });
       socket.destroy();
     });