@detiq/crawler 0.2.11 → 0.2.13

package/Dockerfile

@@ -16,13 +16,25 @@ RUN apt-get update && apt-get install -y curl && \
     apt-get clean && rm -rf /var/lib/apt/lists/*
 
 # Workspace manifests (layer cache)
+# Copy all workspace package.json files so npm ci can resolve the full lockfile.
+# Packages not used by the crawler (api, web, etc.) are stubs; their source is not copied later.
 COPY package.json package-lock.json ./
-COPY packages/core/package.json         packages/core/
-COPY packages/database/package.json     packages/database/
-COPY packages/app-brain/package.json    packages/app-brain/
+COPY packages/agent-cli/package.json    packages/agent-cli/
 COPY packages/agents/package.json       packages/agents/
+COPY packages/api/package.json          packages/api/
+COPY packages/app-brain/package.json    packages/app-brain/
+COPY packages/chrome-extension/package.json packages/chrome-extension/
+COPY packages/connectors/package.json   packages/connectors/
+COPY packages/core/package.json         packages/core/
 COPY packages/crawler/package.json      packages/crawler/
-RUN npm ci
+COPY packages/database/package.json     packages/database/
+COPY packages/github-action/package.json packages/github-action/
+COPY packages/reranker/package.json     packages/reranker/
+COPY packages/sdk/package.json          packages/sdk/
+COPY packages/session-receiver/package.json packages/session-receiver/
+COPY packages/vscode-extension/package.json packages/vscode-extension/
+COPY packages/web/package.json          packages/web/
+RUN npm ci --include=dev
 
 # Re-install chromium after npm ci so the browser revision matches the
 # exact playwright version resolved by the lockfile (not the base image's
@@ -31,16 +43,20 @@ RUN npx playwright install chromium
 
 # Source
 COPY tsconfig.base.json ./
-COPY packages/core       packages/core
-COPY packages/database   packages/database
-COPY packages/app-brain  packages/app-brain
-COPY packages/agents     packages/agents
-COPY packages/crawler    packages/crawler
+COPY packages/database        packages/database
+COPY packages/reranker        packages/reranker
+COPY packages/core            packages/core
+COPY packages/connectors      packages/connectors
+COPY packages/app-brain       packages/app-brain
+COPY packages/agents          packages/agents
+COPY packages/crawler         packages/crawler
 
-# Build
+# Build order: database → reranker → core → connectors → app-brain → agents → crawler
 RUN npx prisma generate --schema packages/database/prisma/schema.prisma && \
-    npm run -w @detiq/core build && \
     npm run -w @detiq/database build && \
+    npm run -w @detiq/reranker build && \
+    npm run -w @detiq/core build && \
+    npm run -w @detiq/connectors build && \
     npm run -w @detiq/app-brain build && \
     npm run -w @detiq/agents build && \
     npm run -w @detiq/crawler build

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@detiq/crawler",
-  "version": "0.2.11",
+  "version": "0.2.13",
   "private": false,
   "publishConfig": {
     "access": "public",

package/src/crawler.ts

@@ -1340,7 +1340,7 @@ export async function crawlProject(opts: {
     }
     let checkpointSuccessCount = 0;
     let maxSeenTotal = 0; // track high-water mark so progress % never goes backwards
-    let lastEmittedPct = 5; // clamp so progress never decreases even when queue grows
+    let lastEmittedPct = 20; // floor above auth-discovery + login phases (max ~19%)
     const crawlLoopStart = Date.now();
     const startUrl = page.url();
     // Resume from previous segment's pending queue if provided

package/src/job-queue.ts

@@ -1551,25 +1551,34 @@ async function processDiscoverJob(jobId: string, config: any, opts?: ProcessJobO
   const controller = new AbortController();
   runningControllers.set(jobId, controller);
 
+  // Independent heartbeat timer — keeps heartbeatAt fresh even when no URLs are
+  // arriving (e.g. during auth login, page-load waits, 2FA handling).
+  const heartbeatTimer = setInterval(() => {
+    void heartbeatJob(tenantId, jobId);
+  }, JOB_HEARTBEAT_INTERVAL_MS);
+
   // Shared accumulator — updated by onUrlsDiscovered callback as each page is visited.
   // This lets the frontend poll and see a growing list before the job finishes.
   const allUrls = new Set<string>();
   let lastPartialFlush = 0;
+  let lastFlushedPct = 0; // high-water mark — progress never decreases
 
   const flushPartialUrls = async (pct: number, step: string) => {
     const now = Date.now();
     // Flush to DB at most every 5 seconds to avoid excessive writes
     if (now - lastPartialFlush < 5_000 && allUrls.size < 5) return;
+    const monotonic = Math.max(pct, lastFlushedPct);
+    lastFlushedPct = monotonic;
     lastPartialFlush = now;
     try {
       await withTenant(tenantId, (tx) =>
         tx.platformJob.update({
           where: { id: jobId },
-          data: { progress: pct, result: { step, partialUrls: Array.from(allUrls) } as any, heartbeatAt: new Date() },
+          data: { progress: monotonic, result: { step, partialUrls: Array.from(allUrls) } as any, heartbeatAt: new Date() },
         })
       );
     } catch { /* non-fatal */ }
-    await opts?.onProgress?.(pct);
+    await opts?.onProgress?.(monotonic);
   };
 
   const onProgress = async (pct: number, step: string) => {
@@ -1773,6 +1782,7 @@ async function processDiscoverJob(jobId: string, config: any, opts?: ProcessJobO
       })
     ).catch(console.error);
   } finally {
+    clearInterval(heartbeatTimer);
     runningControllers.delete(jobId);
   }
 }

package/src/tls-fingerprint.ts

@@ -82,7 +82,7 @@ export async function probeTlsFingerprint(host: string, port = 443): Promise<{
         protocol: socket.getProtocol() ?? 'unknown',
         cipher: socket.getCipher()?.name ?? 'unknown',
         authorized: socket.authorized,
-        subjectCN: cert?.subject?.CN ?? null,
+        subjectCN: cert?.subject?.CN?.toString() ?? null,
       });
       socket.destroy();
     });