@detiq/crawler 1.1.26 → 1.1.27

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@detiq/crawler",
-  "version": "1.1.26",
+  "version": "1.1.27",
   "private": false,
   "publishConfig": {
     "access": "public",

package/src/crawler.ts

@@ -373,7 +373,6 @@ async function launchChromeCDP(
       '--remote-allow-origins=*',
       '--no-first-run',
       '--no-default-browser-check',
-      '--window-size=1280,900',
     ],
   });
 
@@ -634,6 +633,10 @@ export async function crawlProject(opts: {
   /** When false, disables the puppeteer-extra stealth plugin (playwright-extra) for this crawl.
    *  Default true — stealth is applied unless explicitly set to false. */
   stealth?: boolean;
+  /** Desktop viewport width in pixels (default: 1440). */
+  viewportWidth?: number;
+  /** Desktop viewport height in pixels (default: 900). */
+  viewportHeight?: number;
   /** Called every 10 successful screen captures with current visited URLs and remaining queue — used for checkpoint persistence. */
   onCheckpoint?: (visitedUrls: string[], pendingQueue: string[]) => Promise<void>;
   /** Discovery mode only: called after each page with the current set of discovered original URLs — enables streaming partial results to callers. */
@@ -829,7 +832,7 @@ export async function crawlProject(opts: {
     '--disable-blink-features=AutomationControlled',
     '--disable-http2',
     '--lang=en-US,en',
-    '--window-size=1366,768',
+    `--window-size=${opts.viewportWidth ?? 1440},${opts.viewportHeight ?? 900}`,
     '--disable-features=IsolateOrigins,site-per-process',
   ];
 

package/src/failure-analysis-agent.ts

@@ -36,7 +36,8 @@ Error: ${ctx.error}${consoleSection}${networkSection}`;
     if (ctx.screenshotPath) {
       try {
         const img = readFileSync(ctx.screenshotPath);
-        const result = await askTrackedWithImages('', userText, [{ base64: img.toString('base64'), mediaType: 'image/png' }], llmCfg);
+        const imgSeed = deterministicSeed(`failure-img:${ctx.tenantId ?? ''}:${(ctx.stepDescription ?? '').slice(0, 100)}:${ctx.error.slice(0, 200)}`);
+        const result = await askTrackedWithImages('', userText, [{ base64: img.toString('base64'), mediaType: 'image/png' }], llmCfg, { temperature: 0, seed: imgSeed });
         return result.text || null;
       } catch { /* fall through to text-only */ }
     }

package/src/job-queue.ts

@@ -18,40 +18,18 @@ import crypto from 'node:crypto';
 import { withTenant, prismaAdmin } from '@detiq/database';
 import { crawlJobsTotal, crawlJobDurationSeconds } from '@detiq/core';
 import { logger } from './logger.js';
+import { AppBrain, getProjectStorageConfig, buildArtifactKey, uploadArtifact, getTenantStorageConfig, uploadScreenshotToTenantStorage, uploadFileToTenantStorage, notifyRunCompletion, recordLLMCall, downloadBufferFromTenantStorage, dispatchToTenantAdmins } from '@detiq/app-brain';
 
 async function notifyAdmins(tenantId: string, data: {
   type: string; title: string; body: string; metadata?: Record<string, unknown>
 }) {
   try {
-    // Notification/Membership both have FORCE ROW LEVEL SECURITY — a bare
-    // prisma call with no app.current_tenant set gets silently filtered to
-    // zero rows (SELECT) or rejected (INSERT), not an error, so this call
-    // site had been a complete no-op with no visible failure until fixed.
-    const members = await withTenant(tenantId, (tx) =>
-      tx.membership.findMany({
-        where: { tenantId, role: { in: ['OWNER', 'ADMIN'] } },
-        select: { userId: true },
-      })
-    ) as any[];
-    await withTenant(tenantId, (tx) =>
-      tx.notification.createMany({
-        data: members.map((m: { userId: string }) => ({
-          id: `notif_${crypto.randomUUID()}`,
-          tenantId,
-          userId: m.userId,
-          type: data.type,
-          title: data.title,
-          body: data.body,
-          metadata: data.metadata as any,
-        })),
-      })
-    );
+    await dispatchToTenantAdmins(tenantId, data as any);
   } catch (e) {
     console.error('[notify] failed:', e);
   }
 }
-import { AppBrain, getProjectStorageConfig, buildArtifactKey, uploadArtifact, getTenantStorageConfig, uploadScreenshotToTenantStorage, uploadFileToTenantStorage, notifyRunCompletion, recordLLMCall } from '@detiq/app-brain';
-import { generateTaxonomySkillFiles, groupScreensWithAI, generateExecutionPlan, refreshTestCasesSkillFile, refreshTestDataSkillFile, refreshTestScriptSkillFile, refreshRequirementsCoverageSkillFile, refreshTestExecutionHistorySkillFile, refreshVisualRegressionHistorySkillFile, refreshLocatorStrategyWithHealingData, refreshFieldCatalogSkillFile, detectFieldsFromHtml, mergeIntoCatalog } from '@detiq/agents';
+import { generateTaxonomySkillFiles, groupScreensWithAI, generateExecutionPlan, refreshTestCasesSkillFile, refreshTestDataSkillFile, refreshTestScriptSkillFile, refreshRequirementsCoverageSkillFile, refreshTestExecutionHistorySkillFile, refreshVisualRegressionHistorySkillFile, refreshLocatorStrategyWithHealingData, refreshFieldCatalogSkillFile, detectFieldsFromHtml, mergeIntoCatalog, generateArtifactIntelligence } from '@detiq/agents';
 import type { ExecutionStep } from '@detiq/agents';
 import { crawlProject, domFallbackElements } from './crawler.js';
 import { httpCrawl } from './http-engine.js';
@@ -1688,7 +1666,7 @@ Each test case must reference its reqRef (the requirement ID like REQ-001).`;
 
     let parsed: { cases: any[] } = { cases: [] };
     try {
-      const reqSeed = deterministicSeed(`${tenantId}:${projectId}:req-cases:${reqs.map((r: any) => r.reqId).sort().join('|').slice(0, 300)}`);
+      const reqSeed = deterministicSeed(`req-cases:${reqs.map((r: any) => r.reqId).sort().join('|').slice(0, 300)}`);
       const { text, usage } = await askTracked(system, user, llmCfg, undefined, undefined, { temperature: 0, seed: reqSeed });
       if (usage) recordLLMCall({ tenantId, projectId, jobId, agentType: 'generate_tests_from_reqs', label: 'test_case_generation', usage }).catch(() => { });
       const json = text.match(/\{[\s\S]*\}/)?.[0];
@@ -2561,8 +2539,13 @@ async function processGenerateScreenTestsJob(jobId: string, config: any) {
     const crawlCfg = await AppBrain.getCrawlConfig(tenantId, projectId).catch(() => null);
     const deviceProfiles: string[] = (crawlCfg as any)?.deviceProfiles ?? ['DESKTOP'];
 
+    const existingAiCases: number = await withTenant(tenantId, (tx) =>
+      (tx as any).testCase.count({ where: { tenantId, screenId, source: 'AI_GENERATED' } })
+    );
+    const generationMode = existingAiCases > 0 ? 'replace' : undefined;
+
     for (const viewport of deviceProfiles) {
-      await DualContextOrchestrator.execute({ tenantId, projectId, targetScreenId: screenId, jobId, viewport });
+      await DualContextOrchestrator.execute({ tenantId, projectId, targetScreenId: screenId, jobId, viewport, ...(generationMode ? { mode: generationMode } : {}) });
     }
 
     await updateJobProgress(tenantId, jobId, 80, 'Test cases generated');
@@ -3197,13 +3180,111 @@ async function processGenerateScriptJob(jobId: string, config: any): Promise<voi
   }
 }
 
+function queueImportedArtifactSkillRefresh(tenantId: string, projectId: string, reason: string): void {
+  setImmediate(() => {
+    import('@detiq/agents')
+      .then((agentsModule) => {
+        const agents = agentsModule as any;
+        return Promise.allSettled([
+          agents.refreshArtifactKnowledgeSkillFile(tenantId, projectId),
+          agents.refreshRequirementsCoverageSkillFile(tenantId, projectId),
+        ]);
+      })
+      .then((results) => {
+        results?.forEach((result) => {
+          if (result.status === 'rejected') {
+            console.warn(`[artifacts] skill refresh failed after ${reason}:`, result.reason?.message ?? result.reason);
+          }
+        });
+      })
+      .catch((e) => console.warn(`[artifacts] skill refresh failed after ${reason}:`, e?.message ?? e));
+  });
+}
+
 // Registers the type→processor dispatch table used by the durable worker (both the
 // enqueueJob() fast-path kick and startJobWorkerLoop()'s poll). Declared at module
 // bottom so it can reference every process*Job function by name; function
 // declarations are hoisted, so this could technically live anywhere in the file,
 // but keeping it here alongside the last definition keeps the mapping easy to audit
 // against the actual functions above.
+const CONTENT_DETECTED_DOC_TYPES = new Set(['FUNCTIONIZE_TC', 'SELENIUM_SCRIPT', 'ENV_CONFIG']);
+const AI_DOC_TYPES = new Set([
+  'REQUIREMENTS', 'USER_STORY', 'TEST_CASES', 'TEST_DATA', 'AUTOMATION_SCRIPT', 'API_SPEC', 'ARCHITECTURE', 'REPORT', 'OTHER',
+  'SPREADSHEET', 'DATASHEET',
+]);
+const AI_DOC_TYPE_ALIASES: Record<string, string> = {
+  REQUIREMENT: 'REQUIREMENTS', SPEC: 'REQUIREMENTS', SPECIFICATION: 'REQUIREMENTS',
+  USER_STORIES: 'USER_STORY', STORY: 'USER_STORY',
+  TEST_CASE: 'TEST_CASES', TESTCASES: 'TEST_CASES',
+  TESTDATA: 'TEST_DATA', SCRIPT: 'AUTOMATION_SCRIPT', AUTOMATION: 'AUTOMATION_SCRIPT',
+};
+function normalizeAiDocType(raw: unknown): string {
+  const key = String(raw ?? '').trim().toUpperCase().replace(/[\s-]+/g, '_');
+  const mapped = AI_DOC_TYPE_ALIASES[key] ?? key;
+  return AI_DOC_TYPES.has(mapped) ? mapped : 'OTHER';
+}
+
+async function processArtifactIntelJob(jobId: string, config: any) {
+  const { tenantId, projectId, artifactId, artifactName, isReprocess } = config;
+  try {
+    await updateJobProgress(tenantId, jobId, 10, 'Downloading artifact content');
+    
+    const artifact = await withTenant(tenantId, (tx) =>
+      (tx as any).projectArtifact.findUnique({ where: { id: artifactId }, select: { storageKey: true, docType: true } })
+    ) as any;
+    
+    if (!artifact || !artifact.storageKey) {
+      await updateJobProgress(tenantId, jobId, 100, 'Artifact deleted or missing content, skipping.');
+      await withTenant(tenantId, (tx) =>
+        (tx as any).platformJob.update({ where: { id: jobId }, data: { status: 'DONE', finishedAt: new Date() } })
+      );
+      return;
+    }
+
+    const storageCfg = await getTenantStorageConfig(tenantId);
+    if (!storageCfg) {
+      throw new Error('Tenant storage configuration missing.');
+    }
+    const contentBuffer = await downloadBufferFromTenantStorage(storageCfg, artifact.storageKey);
+    const content = contentBuffer.toString('utf8');
+
+    await updateJobProgress(tenantId, jobId, 40, 'Generating intelligence');
+
+    const intel = await generateArtifactIntelligence(tenantId, projectId, artifactId, content, artifactName, jobId);
+    
+    const finalDocType = artifact.docType && CONTENT_DETECTED_DOC_TYPES.has(artifact.docType)
+      ? artifact.docType
+      : normalizeAiDocType(intel.docType);
+
+    if (finalDocType !== intel.docType) {
+      await withTenant(tenantId, (tx) =>
+        (tx as any).projectArtifact.update({ where: { id: artifactId }, data: { docType: finalDocType } })
+      ).catch((err: any) => console.warn(`[artifact-intel] docType fix-up failed for "${artifactName}":`, err?.message));
+      intel.docType = finalDocType;
+    }
+
+    await updateJobProgress(tenantId, jobId, 90, 'Saving results');
+    
+    await withTenant(tenantId, (tx) =>
+      (tx as any).platformJob.update({ where: { id: jobId }, data: { result: { artifactId, artifactName, summary: intel.summary, keywords: intel.keywords, docType: intel.docType }, status: 'DONE', finishedAt: new Date(), progress: 100 } })
+    );
+
+    queueImportedArtifactSkillRefresh(tenantId, projectId, isReprocess ? 'artifact-reprocess' : 'artifact-intel');
+  } catch (err: any) {
+    const msg = String(err?.message ?? err);
+    logger.error({ jobId, err: msg }, '[artifact-intel] failed');
+    await withTenant(tenantId, (tx) =>
+      (tx as any).projectArtifact.update({ where: { id: artifactId }, data: { status: 'ERROR', summary: 'AI analysis failed' } })
+    ).catch(() => { });
+    await withTenant(tenantId, (tx) =>
+      (tx as any).platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', finishedAt: new Date(), error: msg.slice(0, 1000) } })
+    ).catch(() => { });
+    queueImportedArtifactSkillRefresh(tenantId, projectId, 'artifact-intel-failed');
+  }
+}
+
 jobDispatchTable = {
+  ARTIFACT_INTEL: processArtifactIntelJob,
   CRAWL: processJob,
   CAPTURE_STEPS: processCaptureStepsJob,
   DISCOVER: processDiscoverJob,
@@ -3726,8 +3807,6 @@ async function processBatchGenerateAlignedJob(jobId: string, config: any) {
     ) as any;
     if (!conn) throw new Error(`RepoConnection ${repoConnectionId} not found`);
 
-    const { decrypt } = await import('@detiq/core');
-
     const {
       generateScriptAligned, deriveRepoTargetPath, validateGeneratedScript,
       mapWithConcurrency, SCRIPT_GEN_CONCURRENCY,
@@ -3992,7 +4071,7 @@ function buildPRBody(scripts: any[], projectId: string): string {
 // ── NEW_PAGES_PIPELINE — post-crawl: detect new screens, generate tests + scripts ──
 
 async function processNewPagesPipelineJob(jobId: string, config: any) {
-  const { tenantId, projectId, crawlJobId, repoConnectionId, autoPR = false } = config;
+  const { tenantId, projectId, crawlJobId: _crawlJobId, repoConnectionId, autoPR = false } = config;
   try {
     await updateJobProgress(tenantId, jobId, 5, 'Detecting new screens');
 

package/types/crawler.d.ts

@@ -167,6 +167,10 @@ export declare function crawlProject(opts: {
     /** When false, disables the puppeteer-extra stealth plugin (playwright-extra) for this crawl.
      *  Default true — stealth is applied unless explicitly set to false. */
     stealth?: boolean;
+    /** Desktop viewport width in pixels (default: 1440). */
+    viewportWidth?: number;
+    /** Desktop viewport height in pixels (default: 900). */
+    viewportHeight?: number;
     /** Called every 10 successful screen captures with current visited URLs and remaining queue — used for checkpoint persistence. */
     onCheckpoint?: (visitedUrls: string[], pendingQueue: string[]) => Promise<void>;
     /** Discovery mode only: called after each page with the current set of discovered original URLs — enables streaming partial results to callers. */

package/types/job-queue.d.ts

@@ -13,6 +13,13 @@
  * (JOB_QUEUE_BACKEND=sqs, see sqs-queue.ts) for horizontally-scaled crawler
  * fleets; every other job type always goes through the DB-backed queue.
  */
+export declare enum FailureType {
+    TRANSIENT = "TRANSIENT",// network timeout, rate limit, 429, 503
+    INFRASTRUCTURE = "INFRASTRUCTURE",// DB connection, OOM, worker crash
+    LOGICAL = "LOGICAL",// bad input, business rule violation, 400/422
+    FATAL = "FATAL"
+}
+export declare function classifyFailure(error: string | Error | unknown): FailureType;
 export declare function generateTOTP(secret: string): string;
 export interface PlatformJobConfig {
     tenantId: string;
@@ -114,6 +121,7 @@ export interface PlatformJobConfig {
     /** Set on role-crawl sub-jobs — profile name tagged on discovered screens via roleAccess column. */
     roleProfileName?: string;
 }
+export declare function getActiveExecutionCount(): number;
 type ClaimedJob = {
     id: string;
     tenantId: string;
@@ -144,7 +152,9 @@ export declare function requeueOrphanedJobs(): Promise<void>;
 /** Dispatches a claimed job to its type's processor. Last-resort safety net — every
  *  process*Job function already catches and records its own failures. */
 export declare function dispatchClaimedJob(job: ClaimedJob): Promise<void>;
-/** Re-promotes FAILED jobs to QUEUED if they are eligible for retry based on per-type limits. */
+/** Re-promotes FAILED jobs to RETRY_WAIT if they are eligible for retry based on per-type limits.
+ *  RETRY_WAIT makes the waiting period visible in the UI. A separate sweep then promotes
+ *  RETRY_WAIT → QUEUED once nextRetryAt <= NOW(). */
 export declare function retryEligibleFailedJobs(): Promise<void>;
 /** Promotes FAILED jobs to DLQ if they exceed their per-type retry limits. */
 export declare function promoteToDeadLetterQueue(): Promise<void>;
@@ -169,7 +179,7 @@ export declare function createTrackedJob(type: string, config: {
     [k: string]: any;
 }): Promise<string>;
 /** Mark a tracked job DONE or FAILED. Safe to call in finally — swallows errors.
- * Gap 30: When marking FAILED, sets nextRetryAt (1 min) so the retry worker picks it up promptly. */
+ * GAP-14/15: Uses failure-type classification for per-type backoff on FAILED. */
 export declare function finishTrackedJob(tenantId: string, jobId: string, error?: string): Promise<void>;
 /** Enqueue a generic platform job — returns the job ID immediately. */
 export declare function enqueueJob(type: string, config: any): Promise<string>;
@@ -184,8 +194,10 @@ export declare function getJob(tenantId: string, jobId: string): Promise<{
     status: string;
     tenantId: string;
     projectId: string;
+    inline: boolean;
     config: import("@prisma/client/runtime/library").JsonValue;
     retryCount: number;
+    failureType: string | null;
     queuedAt: Date;
     startedAt: Date | null;
     finishedAt: Date | null;
@@ -199,6 +211,18 @@ export declare function getJob(tenantId: string, jobId: string): Promise<{
     nextRetryAt: Date | null;
     dlq: boolean;
     dlqReason: string | null;
+    leaseExpiresAt: Date | null;
+    fencingToken: string | null;
+    checkpoint: import("@prisma/client/runtime/library").JsonValue | null;
+    checkpointAt: Date | null;
+    parentJobId: string | null;
+    rootJobId: string | null;
+    currentStep: string | null;
+    requeuedAt: Date | null;
+    dlqAt: Date | null;
+    dlqRedriveCount: number;
+    submittedBy: string | null;
+    priority: number;
 } | null>;
 /** List recent jobs for a project. */
 export declare function listJobs(tenantId: string, projectId: string): Promise<{
@@ -211,8 +235,10 @@ export declare function listJobs(tenantId: string, projectId: string): Promise<{
     status: string;
     tenantId: string;
     projectId: string;
+    inline: boolean;
     config: import("@prisma/client/runtime/library").JsonValue;
     retryCount: number;
+    failureType: string | null;
     queuedAt: Date;
     startedAt: Date | null;
     finishedAt: Date | null;
@@ -226,8 +252,23 @@ export declare function listJobs(tenantId: string, projectId: string): Promise<{
     nextRetryAt: Date | null;
     dlq: boolean;
     dlqReason: string | null;
+    leaseExpiresAt: Date | null;
+    fencingToken: string | null;
+    checkpoint: import("@prisma/client/runtime/library").JsonValue | null;
+    checkpointAt: Date | null;
+    parentJobId: string | null;
+    rootJobId: string | null;
+    currentStep: string | null;
+    requeuedAt: Date | null;
+    dlqAt: Date | null;
+    dlqRedriveCount: number;
+    submittedBy: string | null;
+    priority: number;
 }[]>;
-/** Cancel a RUNNING or QUEUED job. Aborts the in-process crawl and marks DB CANCELLED. */
+/** Cancel a RUNNING or QUEUED job.
+ *  Sets status to CANCEL_REQUESTED (GAP-08) so the worker's abort handler can
+ *  detect the request and set CANCELLED + finishedAt after the underlying work stops.
+ *  The in-process AbortController is also triggered for same-process jobs. */
 export declare function cancelJob(tenantId: string, jobId: string): Promise<void>;
 /** Re-enqueue a job using the same config as a previous job. */
 export declare function rerunJob(tenantId: string, jobId: string): Promise<string>;