@detiq/crawler 1.0.4 → 1.0.6

package/dist/index.d.ts

@@ -33,3 +33,5 @@ export type { ProxyTier, ProxyTierConfig } from './proxy.js';
 export { RedisCrawlFrontier, isRedisFrontierEnabled } from './redis-frontier.js';
 export { startSpan, endSpan, withSpan, increment, flushMetrics } from './telemetry.js';
 export type { Span, FinishedSpan } from './telemetry.js';
+export { analyzeFailure } from './failure-analysis-agent.js';
+export type { FailureContext } from './failure-analysis-agent.js';

package/dist/index.js

@@ -24,11 +24,13 @@ import { extractMarkdown } from "./markdown-extract.js";
 import { TieredProxyManager } from "./proxy.js";
 import { RedisCrawlFrontier, isRedisFrontierEnabled } from "./redis-frontier.js";
 import { startSpan, endSpan, withSpan, increment, flushMetrics } from "./telemetry.js";
+import { analyzeFailure } from "./failure-analysis-agent.js";
 export {
   CHROME_TLS_HEADERS,
   RedisCrawlFrontier,
   TieredProxyManager,
   adBlockerProfile,
+  analyzeFailure,
   chromeTlsAgent,
   closeLiveView,
   completeLiveView,

package/dist/index.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../src/index.ts"],
-  "sourcesContent": ["export * from './crawler.js';\nexport * from './visual-diff-verdict.js';\nexport * from './auth.js';\nexport * from './screen-reader.js';\nexport * from './job-queue.js';\nexport { startSqsWorker } from './sqs-queue.js';\nexport * from './session-capture.js';\nexport * from './session-export.js';\nexport * from './test-executor.js';\nexport { startLiveView, getFrame, liveViewNavigate, completeLiveView, closeLiveView, sendInput } from './live-view.js';\nexport { waitForEmailOTP, getInboxEmail } from './email-otp.js';\nexport { getSsoToken, getSsoClientCredentialsToken } from './sso-token.js';\nexport type { SsoConfig, SsoProvider } from './sso-token.js';\nexport * from './engine-router.js';\nexport * from './profiles/domain-profile.js';\nexport * from './policies/crawl-safety.js';\nexport * from './extractors/artifact-envelope.js';\nexport * from './network-observer.js';\nexport { createChromeTlsAgent, probeTlsFingerprint, chromeTlsAgent, CHROME_TLS_HEADERS } from './tls-fingerprint.js';\nexport type { TlsFingerprintOptions } from './tls-fingerprint.js';\nexport { prepareExtensionProfile, adBlockerProfile } from './browser-extension.js';\nexport type { ExtensionSource, ExtensionProfile } from './browser-extension.js';\nexport { createRedisQueue, maybeCreateRedisQueue, isRedisQueueEnabled } from './redis-queue.js';\nexport type { RedisQueueJob, RedisQueueClient } from './redis-queue.js';\nexport { discoverProjectUrls, discoverSitemapUrls } from './sitemap.js';\nexport type { UrlCandidate } from './sitemap.js';\nexport { httpCrawl } from './http-engine.js';\nexport { extractMarkdown } from './markdown-extract.js';\nexport type { ExtractedMarkdown } from './markdown-extract.js';\nexport type { HttpScreen, HttpCrawlOpts, HttpCrawlResult, HttpOutputFormat } from './http-engine.js';\nexport { TieredProxyManager } from './proxy.js';\nexport type { ProxyTier, ProxyTierConfig } from './proxy.js';\nexport { RedisCrawlFrontier, isRedisFrontierEnabled } from './redis-frontier.js';\nexport { startSpan, endSpan, withSpan, increment, flushMetrics } from './telemetry.js';\nexport type { Span, FinishedSpan } from './telemetry.js';\n"],
-  "mappings": "AAAA,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,SAAS,sBAAsB;AAC/B,cAAc;AACd,cAAc;AACd,cAAc;AACd,SAAS,eAAe,UAAU,kBAAkB,kBAAkB,eAAe,iBAAiB;AACtG,SAAS,iBAAiB,qBAAqB;AAC/C,SAAS,aAAa,oCAAoC;AAE1D,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,SAAS,sBAAsB,qBAAqB,gBAAgB,0BAA0B;AAE9F,SAAS,yBAAyB,wBAAwB;AAE1D,SAAS,kBAAkB,uBAAuB,2BAA2B;AAE7E,SAAS,qBAAqB,2BAA2B;AAEzD,SAAS,iBAAiB;AAC1B,SAAS,uBAAuB;AAGhC,SAAS,0BAA0B;AAEnC,SAAS,oBAAoB,8BAA8B;AAC3D,SAAS,WAAW,SAAS,UAAU,WAAW,oBAAoB;",
+  "sourcesContent": ["export * from './crawler.js';\nexport * from './visual-diff-verdict.js';\nexport * from './auth.js';\nexport * from './screen-reader.js';\nexport * from './job-queue.js';\nexport { startSqsWorker } from './sqs-queue.js';\nexport * from './session-capture.js';\nexport * from './session-export.js';\nexport * from './test-executor.js';\nexport { startLiveView, getFrame, liveViewNavigate, completeLiveView, closeLiveView, sendInput } from './live-view.js';\nexport { waitForEmailOTP, getInboxEmail } from './email-otp.js';\nexport { getSsoToken, getSsoClientCredentialsToken } from './sso-token.js';\nexport type { SsoConfig, SsoProvider } from './sso-token.js';\nexport * from './engine-router.js';\nexport * from './profiles/domain-profile.js';\nexport * from './policies/crawl-safety.js';\nexport * from './extractors/artifact-envelope.js';\nexport * from './network-observer.js';\nexport { createChromeTlsAgent, probeTlsFingerprint, chromeTlsAgent, CHROME_TLS_HEADERS } from './tls-fingerprint.js';\nexport type { TlsFingerprintOptions } from './tls-fingerprint.js';\nexport { prepareExtensionProfile, adBlockerProfile } from './browser-extension.js';\nexport type { ExtensionSource, ExtensionProfile } from './browser-extension.js';\nexport { createRedisQueue, maybeCreateRedisQueue, isRedisQueueEnabled } from './redis-queue.js';\nexport type { RedisQueueJob, RedisQueueClient } from './redis-queue.js';\nexport { discoverProjectUrls, discoverSitemapUrls } from './sitemap.js';\nexport type { UrlCandidate } from './sitemap.js';\nexport { httpCrawl } from './http-engine.js';\nexport { extractMarkdown } from './markdown-extract.js';\nexport type { ExtractedMarkdown } from './markdown-extract.js';\nexport type { HttpScreen, HttpCrawlOpts, HttpCrawlResult, HttpOutputFormat } from './http-engine.js';\nexport { TieredProxyManager } from './proxy.js';\nexport type { ProxyTier, ProxyTierConfig } from './proxy.js';\nexport { RedisCrawlFrontier, isRedisFrontierEnabled } from './redis-frontier.js';\nexport { startSpan, endSpan, withSpan, increment, flushMetrics } from './telemetry.js';\nexport type { Span, FinishedSpan } from './telemetry.js';\nexport { analyzeFailure } from './failure-analysis-agent.js';\nexport type { FailureContext } from './failure-analysis-agent.js';\n"],
+  "mappings": "AAAA,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,SAAS,sBAAsB;AAC/B,cAAc;AACd,cAAc;AACd,cAAc;AACd,SAAS,eAAe,UAAU,kBAAkB,kBAAkB,eAAe,iBAAiB;AACtG,SAAS,iBAAiB,qBAAqB;AAC/C,SAAS,aAAa,oCAAoC;AAE1D,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,SAAS,sBAAsB,qBAAqB,gBAAgB,0BAA0B;AAE9F,SAAS,yBAAyB,wBAAwB;AAE1D,SAAS,kBAAkB,uBAAuB,2BAA2B;AAE7E,SAAS,qBAAqB,2BAA2B;AAEzD,SAAS,iBAAiB;AAC1B,SAAS,uBAAuB;AAGhC,SAAS,0BAA0B;AAEnC,SAAS,oBAAoB,8BAA8B;AAC3D,SAAS,WAAW,SAAS,UAAU,WAAW,oBAAoB;AAEtE,SAAS,sBAAsB;",
   "names": []
 }

package/dist/job-queue.d.ts

@@ -84,6 +84,13 @@ export declare function claimNextQueuedJob(): Promise<ClaimedJob | null>;
  *  died (crash, restart, OOM) without ever marking them DONE/FAILED. Idempotent and
  *  safe to run from every worker process; only affects rows past the timeout. */
 export declare function requeueStaleRunningJobs(): Promise<void>;
+/**
+ * Requeues any RUNNING jobs claimed by a different worker (i.e., a previous
+ * process instance). Call once at startup before the poll loop begins — this
+ * provides immediate recovery instead of waiting the full STALE_JOB_TIMEOUT_MINUTES.
+ * Safe to call from multiple replicas simultaneously (each uses its own WORKER_ID).
+ */
+export declare function requeueOrphanedJobs(): Promise<void>;
 /**
  * Starts the durable job worker poll loop. This is what makes job processing
  * survive a restart — enqueueJob()'s immediate same-process claim attempt is

package/dist/job-queue.js

@@ -130,6 +130,26 @@ async function requeueStaleRunningJobs() {
     logger.warn({ jobIds: rows.map((r) => r.id), staleTimeoutMinutes: STALE_JOB_TIMEOUT_MINUTES }, "[job-worker] requeued stale RUNNING job(s)");
   }
 }
+async function requeueOrphanedJobs() {
+  const rows = await prismaAdmin.$queryRaw`
+    UPDATE "PlatformJob"
+    SET status = 'QUEUED',
+        "workerId" = NULL,
+        result = jsonb_set(
+          COALESCE(result::jsonb, '{}'::jsonb),
+          '{step}',
+          to_jsonb('Worker restarted — queued for recovery'::text),
+          true
+        )
+    WHERE status = 'RUNNING'
+      AND "workerId" IS NOT NULL
+      AND "workerId" != ${WORKER_ID}
+    RETURNING id
+  `;
+  if (rows.length > 0) {
+    logger.warn({ jobIds: rows.map((r) => r.id), workerId: WORKER_ID }, "[job-worker] requeued orphaned RUNNING job(s) from dead worker");
+  }
+}
 async function dispatchClaimedJob(job) {
   const dispatcher = jobDispatchTable?.[job.type];
   if (!dispatcher) {
@@ -181,6 +201,9 @@ function startJobWorkerLoop(opts) {
     }
   };
   workerLoopTimer = setInterval(tick, pollIntervalMs);
+  requeueOrphanedJobs().catch(
+    (err) => logger.error({ err: String(err) }, "[job-worker] startup orphan sweep failed")
+  );
   void tick();
 }
 function stopJobWorkerLoop() {
@@ -238,14 +261,24 @@ async function getJob(tenantId, jobId) {
   );
 }
 async function listJobs(tenantId, projectId) {
-  return withTenant(
-    tenantId,
-    (tx) => tx.platformJob.findMany({
+  return withTenant(tenantId, async (tx) => {
+    const jobs = await tx.platformJob.findMany({
       where: { tenantId, projectId },
       orderBy: { queuedAt: "desc" },
       take: 50
-    })
-  );
+    });
+    const jobIds = jobs.map((j) => j.id);
+    if (jobIds.length === 0) return jobs;
+    const modelRows = await tx.lLMCallLog.groupBy({
+      by: ["jobId"],
+      where: { tenantId, jobId: { in: jobIds } },
+      _max: { model: true, calledAt: true }
+    });
+    const modelByJobId = new Map(
+      modelRows.filter((r) => r.jobId && r._max?.model).map((r) => [r.jobId, r._max.model])
+    );
+    return jobs.map((j) => ({ ...j, llmModel: modelByJobId.get(j.id) ?? null }));
+  });
 }
 async function cancelJob(tenantId, jobId) {
   const controller = runningControllers.get(jobId);
@@ -374,13 +407,14 @@ async function investigateScreenChanges(tenantId, projectId, changedScreens) {
 async function processJob(jobId, config, opts) {
   const { tenantId, projectId } = config;
   const jobStartedAt = /* @__PURE__ */ new Date();
-  await withTenant(
+  const claimed = await withTenant(
     tenantId,
-    (tx) => tx.platformJob.update({
-      where: { id: jobId },
+    (tx) => tx.platformJob.updateMany({
+      where: { id: jobId, status: { not: "CANCELLED" } },
       data: { status: "RUNNING", startedAt: jobStartedAt, heartbeatAt: jobStartedAt, workerId: WORKER_ID }
     })
   );
+  if (claimed.count === 0) return;
   const controller = new AbortController();
   runningControllers.set(jobId, controller);
   const heartbeatTimer = setInterval(() => {
@@ -2265,6 +2299,7 @@ export {
   listJobs,
   processDataRetentionPurge,
   processJob,
+  requeueOrphanedJobs,
   requeueStaleRunningJobs,
   rerunJob,
   resumeJob,