@detiq/crawler 1.1.11 → 1.1.13

package/src/job-queue.ts

@@ -51,13 +51,15 @@ async function notifyAdmins(tenantId: string, data: {
   }
 }
 import { AppBrain, getProjectStorageConfig, buildArtifactKey, uploadArtifact, getTenantStorageConfig, uploadScreenshotToTenantStorage, uploadFileToTenantStorage, notifyRunCompletion, recordLLMCall } from '@detiq/app-brain';
-import { generateTaxonomySkillFiles, groupScreensWithAI, generateExecutionPlan, refreshTestCasesSkillFile, refreshTestDataSkillFile, refreshTestScriptSkillFile, refreshRequirementsCoverageSkillFile, detectFieldsFromHtml, mergeIntoCatalog } from '@detiq/agents';
-import { crawlProject } from './crawler.js';
+import { generateTaxonomySkillFiles, groupScreensWithAI, generateExecutionPlan, refreshTestCasesSkillFile, refreshTestDataSkillFile, refreshTestScriptSkillFile, refreshRequirementsCoverageSkillFile, refreshTestExecutionHistorySkillFile, refreshVisualRegressionHistorySkillFile, refreshLocatorStrategyWithHealingData, refreshFieldCatalogSkillFile, detectFieldsFromHtml, mergeIntoCatalog } from '@detiq/agents';
+import type { ExecutionStep } from '@detiq/agents';
+import { crawlProject, domFallbackElements } from './crawler.js';
 import { httpCrawl } from './http-engine.js';
 import { hashContent, normalizeText } from './incremental-diff.js';
 import { discoverProjectUrls } from './sitemap.js';
-import { executeTestPlan } from './test-executor.js';
+import { executeExecutionStep, executeTestPlan } from './test-executor.js';
 import { runNativeCrawl } from './native-crawler.js';
+import { elementsToPlaywrightLocators } from './locator-extractor.js';
 import { jira, confluence, figma } from '@detiq/connectors';
 import { decrypt, config as platformConfig, dispatchWebhook, isFeatureEnabled, interpolatePlaceholders } from '@detiq/core';
 import { createHmac } from 'node:crypto';
@@ -106,6 +108,7 @@ function makeJobName(type: string): string {
   const date = now.toISOString().slice(0, 10); // YYYY-MM-DD
   const ts = now.getTime();
   const label = type === 'CRAWL' ? 'Crawl'
+    : type === 'CAPTURE_STEPS' ? 'CaptureSteps'
     : type === 'NATIVE_CRAWL' ? 'NativeCrawl'
     : type === 'GENERATE_TESTS' ? 'GenerateTests'
     : type === 'VISUAL_REGRESSION' ? 'VisualRegression'
@@ -380,12 +383,18 @@ export async function createTrackedJob(
   return job.id;
 }
 
-/** Mark a tracked job DONE or FAILED. Safe to call in finally — swallows errors. */
+/** Mark a tracked job DONE or FAILED. Safe to call in finally — swallows errors.
+ * Gap 30: When marking FAILED, sets nextRetryAt (1 min) so the retry worker picks it up promptly. */
 export async function finishTrackedJob(tenantId: string, jobId: string, error?: string): Promise<void> {
+  const nextRetryAt = error ? new Date(Date.now() + 60_000) : undefined;
   await withTenant(tenantId, (tx) =>
     tx.platformJob.update({
       where: { id: jobId },
-      data: { status: error ? 'FAILED' : 'DONE', finishedAt: new Date(), ...(error ? { error } : { progress: 100 }) },
+      data: {
+        status: error ? 'FAILED' : 'DONE',
+        finishedAt: new Date(),
+        ...(error ? { error, nextRetryAt } : { progress: 100 }),
+      },
     })
   ).catch(() => {});
 }
@@ -468,7 +477,7 @@ export async function cancelJob(tenantId: string, jobId: string): Promise<void>
   await withTenant(tenantId, (tx) =>
     tx.platformJob.update({
       where: { id: jobId },
-      data: { status: 'CANCELLED', finishedAt: new Date() },
+      data: { status: 'CANCELLED', finishedAt: new Date(), heartbeatAt: null, workerId: null },
     })
   );
 }
@@ -634,7 +643,7 @@ export async function processJob(jobId: string, config: any, opts?: ProcessJobOp
     await opts?.onProgress?.(pct);
   };
 
-  const DEFAULT_MAX_SCREENS = 150;
+  const DEFAULT_MAX_SCREENS = parseInt(process.env.CRAWL_DEFAULT_MAX_SCREENS ?? '') || 150;
 
   try {
     // Per-project maxScreens + deviceProfiles + seedUrls + interactive probing + sitemap seeding + traversal strategy from crawlConfig override job config defaults.
@@ -783,6 +792,8 @@ export async function processJob(jobId: string, config: any, opts?: ProcessJobOp
             }
             if (fieldDetectAdded > 0 || detectLimit > 0) {
               logger.info({ screens: detectLimit, added: fieldDetectAdded, projectId }, '[field-detect] catalog populated from crawl');
+              // Refresh FIELD_CATALOG skill file after catalog is populated (GAP-SF-20)
+              refreshFieldCatalogSkillFile(tenantId, projectId).catch(console.error);
             }
           } catch (fdErr) {
             logger.warn({ err: fdErr }, '[field-detect] post-crawl field detection failed');
@@ -1524,6 +1535,7 @@ async function processRunTestSuiteJob(jobId: string, config: any) {
       { passed, failed, total: testCases.length },
       runResults,
     ).catch(console.error);
+    refreshTestExecutionHistorySkillFile(tenantId, projectId).catch(console.error);
   } catch (err: any) {
     const msg = String(err?.message ?? err).slice(0, 500);
     logger.error({ jobId, testRunId, err: msg }, 'processRunTestSuiteJob failed');
@@ -1563,6 +1575,10 @@ async function processGenerateSkillFilesJob(jobId: string, config: any) {
       refreshTestDataSkillFile(tenantId, projectId),
       refreshTestScriptSkillFile(tenantId, projectId),
       refreshRequirementsCoverageSkillFile(tenantId, projectId),
+      refreshTestExecutionHistorySkillFile(tenantId, projectId),
+      refreshVisualRegressionHistorySkillFile(tenantId, projectId),
+      refreshLocatorStrategyWithHealingData(tenantId, projectId),
+      refreshFieldCatalogSkillFile(tenantId, projectId),
     ]);
     await withTenant(tenantId, (tx) =>
       tx.platformJob.update({ where: { id: jobId }, data: { status: 'DONE', progress: 100, finishedAt: new Date() } })
@@ -1601,7 +1617,7 @@ export async function processDataRetentionPurge(): Promise<void> {
       orderBy: { dataRetentionDays: 'asc' },
       select: { dataRetentionDays: true },
     });
-    const retentionDays = setting?.dataRetentionDays ?? 365;
+    const retentionDays = setting?.dataRetentionDays ?? (parseInt(process.env.DEFAULT_DATA_RETENTION_DAYS ?? '') || 365);
     const cutoff = new Date(Date.now() - retentionDays * 24 * 60 * 60 * 1000);
 
     const result = await prismaAdmin.auditLog.deleteMany({
@@ -2205,6 +2221,260 @@ async function processGenerateScreenTestsJob(jobId: string, config: any) {
   }
 }
 
+type CaptureStepStatus = 'PASSED' | 'FAILED' | 'HEALED' | 'SKIPPED';
+type CaptureStepResult = {
+  stepNumber: number;
+  description: string;
+  status: CaptureStepStatus;
+  screenshotPath?: string;
+  screenshotUrl?: string;
+  screenId?: string;
+  url?: string;
+  elementsCount?: number;
+  healed?: boolean;
+  healTier?: 'mechanical' | 'ai';
+  healReasoning?: string;
+  error?: string;
+};
+
+function captureStepText(step: unknown): string {
+  if (typeof step === 'string') return step;
+  if (!step || typeof step !== 'object') return '';
+  const obj = step as Record<string, unknown>;
+  return [obj.action, obj.text, obj.label, obj.data, obj.expected]
+    .filter((v): v is string => typeof v === 'string' && v.trim().length > 0)
+    .join(' -> ');
+}
+
+function firstQuoted(text: string): string | undefined {
+  return (text.match(/"([^"]+)"/)?.[1] ?? text.match(/'([^']+)'/)?.[1])?.trim();
+}
+
+function fallbackExecutionPlan(rawSteps: unknown[]): ExecutionStep[] {
+  const plan: ExecutionStep[] = [];
+  for (const raw of rawSteps) {
+    const text = captureStepText(raw).trim();
+    if (!text) continue;
+    const stepNumber = plan.length + 1;
+    const url = text.match(/https?:\/\/[^\s"']+/)?.[0];
+
+    if (/^navigate\b|^go to\b|^open\b/i.test(text) && url) {
+      plan.push({ stepNumber, description: text, action: 'navigate', url });
+      continue;
+    }
+
+    const urlContains = text.match(/verify url contains:\s*["']?([^"']+)["']?/i);
+    if (urlContains) {
+      plan.push({ stepNumber, description: text, action: 'assert_url', value: urlContains[1].trim() });
+      continue;
+    }
+
+    const inputMatch = text.match(/(?:enter|type|fill)\s+["']([^"']+)["']\s+(?:in|into)\s+(.+?)(?:\s+field|\s+input|\s+box|\s+area)?(?:\s*→.*)?$/i);
+    if (inputMatch) {
+      const selector = inputMatch[2].replace(/^(the\s+)?/i, '').replace(/\s+(field|input|box|area)$/i, '').trim();
+      plan.push({ stepNumber, description: text, action: 'fill', selectorStrategy: 'label', selector, value: inputMatch[1].trim() });
+      continue;
+    }
+
+    const quoted = firstQuoted(text);
+    if (/^(click|tap|navigate menu|check|select)\b/i.test(text) && quoted) {
+      plan.push({ stepNumber, description: text, action: 'click', selectorStrategy: 'text', selector: quoted });
+      continue;
+    }
+    if (/^(verify|assert|expect)\b/i.test(text) && quoted) {
+      plan.push({ stepNumber, description: text, action: 'assert_visible', selectorStrategy: 'text', selector: quoted });
+      continue;
+    }
+    if (/execute script/i.test(text)) {
+      const script = text.split(':').slice(1).join(':').replace(/\s*→.*$/, '').trim();
+      if (script) plan.push({ stepNumber, description: text, action: 'script', value: script });
+    }
+  }
+  return plan;
+}
+
+function screenNameFromUrl(url: string, fallback: string): string {
+  try {
+    const u = new URL(url);
+    return u.pathname.replace(/\/+$/, '').split('/').filter(Boolean).pop() || u.hostname || fallback;
+  } catch {
+    return fallback;
+  }
+}
+
+async function processCaptureStepsJob(jobId: string, config: any) {
+  const { tenantId, projectId, testCaseId } = config as { tenantId: string; projectId: string; testCaseId: string };
+  const screenshotDir = config.screenshotDir || platformConfig.screenshotDir || process.env.SCREENSHOT_DIR || '/tmp/zeta-screenshots';
+  await fs.mkdir(screenshotDir, { recursive: true });
+
+  await withTenant(tenantId, (tx) =>
+    tx.platformJob.update({ where: { id: jobId }, data: { status: 'RUNNING', startedAt: new Date(), progress: 2 } })
+  );
+
+  let browser: any;
+  try {
+    await updateJobProgress(tenantId, jobId, 5, 'Loading test case');
+    const [tc, crawlCfg, tenantStorageCfg] = await Promise.all([
+      AppBrain.getTestCaseForExecution(tenantId, testCaseId) as Promise<any>,
+      AppBrain.getCrawlConfig(tenantId, projectId).catch(() => null),
+      getTenantStorageConfig(tenantId).catch(() => null),
+    ]);
+    if (!tc) throw new Error('Test case not found');
+
+    const rawSteps = Array.isArray(tc.steps) ? tc.steps : [];
+    const stepTexts = rawSteps.map(captureStepText).filter(Boolean);
+    if (stepTexts.length === 0) throw new Error('Test case has no steps to capture');
+
+    const baseUrl = config.baseUrl || tc?.screen?.project?.appUrl || crawlCfg?.appUrl || '';
+    let plan = await generateExecutionPlan(tenantId, { ...tc, steps: stepTexts }, baseUrl).catch(() => []);
+    plan = Array.isArray(plan)
+      ? plan.filter(Boolean).map((step: any, idx: number) => ({ ...step, stepNumber: idx + 1 }))
+      : [];
+    if (plan.length === 0) plan = fallbackExecutionPlan(rawSteps);
+    if (plan.length === 0) throw new Error('Could not generate execution plan from steps');
+
+    const { chromium } = await import('playwright');
+    browser = await chromium.launch({
+      headless: true,
+      ...(process.env.PLAYWRIGHT_EXECUTABLE_PATH ? { executablePath: process.env.PLAYWRIGHT_EXECUTABLE_PATH } : {}),
+    });
+    let parsedStorageState: any;
+    try {
+      parsedStorageState = config.storageState ? JSON.parse(config.storageState) : crawlCfg?.storageState ? JSON.parse(crawlCfg.storageState) : undefined;
+    } catch {
+      parsedStorageState = undefined;
+    }
+    const context = await browser.newContext({ viewport: { width: 1280, height: 800 }, ...(parsedStorageState ? { storageState: parsedStorageState } : {}) });
+    const page = await context.newPage();
+    const aiHealingEnabled = await isFeatureEnabled(tenantId, 'AI_SELF_HEALING').catch(() => false);
+
+    const stepResults: CaptureStepResult[] = [];
+    const screens: Array<{ stepNumber: number; screenId: string; url: string; screenshotPath?: string; screenshotUrl?: string; isNewScreen: boolean }> = [];
+    const locators: any[] = [];
+    const seenLocatorExpressions = new Set<string>();
+    const seenScreenKeys = new Set<string>();
+    let lastStateKey = '';
+
+    for (let i = 0; i < plan.length; i++) {
+      const step = plan[i];
+      const description = step.description || stepTexts[i] || `Step ${i + 1}`;
+      const stepNumber = i + 1;
+      const result: CaptureStepResult = { stepNumber, description, status: 'FAILED' };
+      await withTenant(tenantId, (tx) =>
+        tx.platformJob.update({
+          where: { id: jobId },
+          data: {
+            progress: Math.min(90, Math.round((i / Math.max(plan.length, 1)) * 90)),
+            result: { currentStep: { stepNumber, description } } as any,
+            heartbeatAt: new Date(),
+          },
+        })
+      ).catch(() => {});
+
+      try {
+        const heal = await executeExecutionStep(page, step, aiHealingEnabled ? { healCtx: { tenantId, projectId } } : {});
+        if (heal.healed) {
+          result.healed = true;
+          result.healTier = heal.healedVia;
+          result.healReasoning = heal.healReasoning;
+        }
+        await page.waitForLoadState('networkidle', { timeout: parseInt(process.env.TEST_NETWORK_IDLE_TIMEOUT_MS ?? '2000', 10) }).catch(() => {});
+        result.status = result.healed ? 'HEALED' : 'PASSED';
+      } catch (err: any) {
+        result.status = step.optional ? 'SKIPPED' : 'FAILED';
+        result.error = String(err?.message ?? err).slice(0, 300);
+      }
+
+      const currentUrl = page.url();
+      result.url = currentUrl;
+      const ssFilename = `${testCaseId}-capture-step-${stepNumber}${result.status === 'FAILED' ? '-err' : ''}.png`;
+      const ssPath = path.join(screenshotDir, ssFilename);
+      await page.screenshot({ path: ssPath, fullPage: false }).catch(() => {});
+      result.screenshotPath = ssPath;
+
+      let cloudUrl: string | undefined;
+      if (tenantStorageCfg) {
+        const key = buildArtifactKey(tenantId, projectId, 'screenshots', ssFilename, tenantStorageCfg.prefix);
+        cloudUrl = (await uploadScreenshotToTenantStorage(tenantStorageCfg, ssPath, key).catch(() => undefined)) ?? undefined;
+        if (cloudUrl) result.screenshotUrl = cloudUrl;
+      }
+
+      const elements = await domFallbackElements(page).catch(() => []);
+      result.elementsCount = elements.length;
+      const stateHash = hashContent(elements.map((el) => `${el.role}:${el.meaning}:${el.notes ?? ''}`).join('|'));
+      const screenKey = `${currentUrl || 'about:blank'}:${stateHash}`;
+      const shouldPersistScreen = currentUrl && currentUrl !== 'about:blank' && (i === 0 || screenKey !== lastStateKey || !seenScreenKeys.has(screenKey));
+      lastStateKey = screenKey;
+
+      if (shouldPersistScreen) {
+        const screenName = screenNameFromUrl(currentUrl, tc.title ?? `Step ${stepNumber}`);
+        const screen = await AppBrain.saveScreen(tenantId, projectId, screenName, currentUrl, jobId, i === 0 ? 'ENTRY_POINT' : 'LINK_FOLLOW').catch(() => null) as any;
+        if (screen?.id) {
+          result.screenId = screen.id;
+          await AppBrain.saveScreenshot(tenantId, projectId, screen.id, ssPath, screen.version ?? 1, 'DESKTOP', cloudUrl).catch(() => {});
+          for (const el of elements) {
+            await AppBrain.saveElement(
+              tenantId, projectId, screen.id,
+              el.meaning, el.role, el.expectedData, el.notes,
+              undefined, true, el.boundingRect, el.ariaState, el.parentLandmark,
+            ).catch(() => {});
+          }
+          for (const loc of elementsToPlaywrightLocators(elements as any)) {
+            if (seenLocatorExpressions.has(loc.expression)) continue;
+            seenLocatorExpressions.add(loc.expression);
+            locators.push(loc);
+          }
+          if (!seenScreenKeys.has(screenKey)) {
+            screens.push({ stepNumber, screenId: screen.id, url: currentUrl, screenshotPath: ssPath, screenshotUrl: cloudUrl, isNewScreen: true });
+            seenScreenKeys.add(screenKey);
+          }
+        }
+      }
+
+      stepResults.push(result);
+      if (result.status === 'FAILED') break;
+    }
+
+    if (locators.length > 0) {
+      await AppBrain.saveLocatorsJson(tenantId, testCaseId, locators as any).catch(() => {});
+    }
+
+    const firstScreen = screens[0];
+    await withTenant(tenantId, (tx) =>
+      (tx as any).testCase.update({
+        where: { id: testCaseId },
+        data: {
+          ...(firstScreen?.screenId ? { screenId: firstScreen.screenId } : {}),
+          captureStepScreenshots: stepResults as any,
+        },
+      })
+    ).catch(() => {});
+
+    const finalResult = {
+      stepResults,
+      screens,
+      locators,
+      totalSteps: plan.length,
+      capturedScreens: screens.length,
+    };
+    await withTenant(tenantId, (tx) =>
+      tx.platformJob.update({
+        where: { id: jobId },
+        data: { status: 'DONE', progress: 100, result: finalResult as any, finishedAt: new Date(), heartbeatAt: new Date() },
+      })
+    );
+  } catch (err: any) {
+    const error = String(err?.message ?? err);
+    await withTenant(tenantId, (tx) =>
+      tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', error, finishedAt: new Date(), heartbeatAt: new Date() } })
+    ).catch(() => {});
+    logger.error({ jobId, testCaseId, err: error }, 'processCaptureStepsJob failed');
+    throw err;
+  } finally {
+    await browser?.close?.().catch(() => {});
+  }
+}
+
 // Registers the type→processor dispatch table used by the durable worker (both the
 // enqueueJob() fast-path kick and startJobWorkerLoop()'s poll). Declared at module
 // bottom so it can reference every process*Job function by name; function
@@ -2213,6 +2483,7 @@ async function processGenerateScreenTestsJob(jobId: string, config: any) {
 // against the actual functions above.
 jobDispatchTable = {
   CRAWL: processJob,
+  CAPTURE_STEPS: processCaptureStepsJob,
   DISCOVER: processDiscoverJob,
   MONITOR: processMonitorJob,
   NATIVE_CRAWL: processNativeCrawlJob,
@@ -2240,7 +2511,7 @@ async function processRepoScanJob(jobId: string, config: any) {
     const {
       fetchRepoTree, fetchFileContent, detectFramework,
       extractTestIds, extractStyleProfile,
-      buildPageObjectInventory,
+      buildPageObjectInventory, resolveFixtureAliasesFromImports,
     } = await import('@detiq/agents');
 
     const conn = await withTenant(tenantId, (tx) =>
@@ -2260,6 +2531,9 @@ async function processRepoScanJob(jobId: string, config: any) {
     await updateJobProgress(tenantId, jobId, 15, 'Fetching repo tree');
     // Use HEAD so it always resolves to the actual default branch regardless of stored value
     const treeResult = await fetchRepoTree(token, conn.repoFullName, 'HEAD');
+    if (treeResult.truncated) {
+      console.warn(`[repo-scan] ${conn.repoFullName}: tree truncated (>100k blobs) — detection may be incomplete`);
+    }
 
     const frameworks = detectFramework(treeResult.paths);
     const primary = frameworks[0];
@@ -2295,6 +2569,21 @@ async function processRepoScanJob(jobId: string, config: any) {
 
     await updateJobProgress(tenantId, jobId, 40, `Found ${testPaths.length} test files`);
 
+    if (testPaths.length === 0) {
+      logger.warn({ jobId, repoConnectionId, framework }, '[repo-scan] no test files matched glob — scan complete with 0 results');
+      await withTenant(tenantId, (tx) =>
+        tx.gitHubRepoConnection.update({
+          where: { id: repoConnectionId },
+          data: { scanStatus: 'DONE', testFileCount: 0, extractedIdCount: 0, lastScannedAt: new Date(), lastSyncAt: new Date() },
+        })
+      );
+      await updateJobProgress(tenantId, jobId, 100, 'Done: 0 test files found');
+      await withTenant(tenantId, (tx) =>
+        tx.platformJob.update({ where: { id: jobId }, data: { status: 'DONE', finishedAt: new Date(), result: { testFileCount: 0, extractedIdCount: 0 } as any } })
+      );
+      return;
+    }
+
     // Extract test IDs from each file
     const allExtracted: any[] = [];
     const styleSampleFiles: Array<{ path: string; content: string }> = [];
@@ -2307,7 +2596,11 @@ async function processRepoScanJob(jobId: string, config: any) {
       const content = await fetchFileContent(token, conn.repoFullName, filePath);
       if (!content) continue;
 
-      const extracted = extractTestIds(content, filePath, framework);
+      // G-07: multi-file fixture tracing — resolve non-obvious aliases before extraction
+      const extraAliases = await resolveFixtureAliasesFromImports(
+        content, token, conn.repoFullName, filePath,
+      ).catch(() => new Set<string>());
+      const extracted = extractTestIds(content, filePath, framework, extraAliases);
       allExtracted.push(...extracted);
 
       await withTenant(tenantId, (tx) =>
@@ -2348,31 +2641,57 @@ async function processRepoScanJob(jobId: string, config: any) {
       const lang = framework.includes('java') ? 'java'
         : (framework.includes('py') || framework === 'pytest') ? 'python' : 'js';
 
-      const configFilesToFetch: Array<{ path: string; type: string }> = [];
+      const configFilesToFetch: Array<{ path: string; type: string; dirPrefix: string }> = [];
       const tryAdd = (candidates: string[], type: string) => {
-        for (const c of candidates) { if (pathSet.has(c)) { configFilesToFetch.push({ path: c, type }); break; } }
+        for (const c of candidates) { if (pathSet.has(c)) { configFilesToFetch.push({ path: c, type, dirPrefix: '' }); break; } }
+      };
+      // Monorepo fallback: when root-level config is absent, find shallowest match anywhere in tree.
+      // The dirPrefix (e.g. 'packages/e2e/') is prepended to testDir so paths are repo-root-relative.
+      const scanTree = (pattern: RegExp, type: string) => {
+        if (configFilesToFetch.some(f => f.type === type)) return;
+        const all = (treeResult.paths as string[]).filter(p => pattern.test(p)).sort((a, b) => a.split('/').length - b.split('/').length);
+        if (all[0]) {
+          const lastSlash = all[0].lastIndexOf('/');
+          const dirPrefix = lastSlash > 0 ? all[0].slice(0, lastSlash + 1) : '';
+          configFilesToFetch.push({ path: all[0], type, dirPrefix });
+        }
       };
 
       if (lang === 'js') {
-        if (framework.startsWith('playwright')) tryAdd(['playwright.config.ts', 'playwright.config.js'], 'playwright');
-        if (framework.startsWith('cypress')) tryAdd(['cypress.config.ts', 'cypress.config.js'], 'cypress');
+        if (framework.startsWith('playwright')) {
+          tryAdd(['playwright.config.ts', 'playwright.config.js'], 'playwright');
+          scanTree(/(?:^|\/)playwright\.config\.(ts|js|mts)$/, 'playwright');
+        }
+        if (framework.startsWith('cypress')) {
+          tryAdd(['cypress.config.ts', 'cypress.config.js'], 'cypress');
+          scanTree(/(?:^|\/)cypress\.config\.(ts|js)$/, 'cypress');
+        }
         if (framework === 'jest-ts' || framework === 'jest-js') {
           tryAdd(['jest.config.ts', 'jest.config.js', 'jest.config.mjs'], 'jest');
+          scanTree(/(?:^|\/)jest\.config\.(ts|js|mjs|cjs)$/, 'jest');
           tryAdd(['package.json'], 'package-json');
         }
-        if (framework === 'webdriverio') tryAdd(['wdio.conf.ts', 'wdio.conf.js', 'wdio.config.ts', 'wdio.config.js'], 'wdio');
+        if (framework === 'webdriverio') {
+          tryAdd(['wdio.conf.ts', 'wdio.conf.js', 'wdio.config.ts', 'wdio.config.js'], 'wdio');
+          scanTree(/(?:^|\/)wdio\.(?:conf|config)\.(ts|js|mjs)$/, 'wdio');
+        }
         if (framework === 'cucumber' || framework === 'gherkin') {
           tryAdd(['.cucumber.js', '.cucumber.cjs', 'cucumber.json', '.cucumber.yaml', '.cucumber.yml'], 'cucumber');
           tryAdd(['package.json'], 'package-json');
         }
         tryAdd(['tsconfig.json'], 'tsconfig');
+        scanTree(/(?:^|\/)tsconfig(?:\.base)?\.json$/, 'tsconfig');
       } else if (lang === 'python') {
         tryAdd(['pytest.ini'], 'pytest-ini');
+        scanTree(/(?:^|\/)pytest\.ini$/, 'pytest-ini');
         tryAdd(['pyproject.toml'], 'pyproject-toml');
+        scanTree(/(?:^|\/)pyproject\.toml$/, 'pyproject-toml');
         tryAdd(['setup.cfg'], 'setup-cfg');
       } else if (lang === 'java') {
         tryAdd(['pom.xml'], 'pom-xml');
+        scanTree(/(?:^|\/)pom\.xml$/, 'pom-xml');
         tryAdd(['build.gradle.kts', 'build.gradle'], 'build-gradle');
+        scanTree(/(?:^|\/)build\.gradle(?:\.kts)?$/, 'build-gradle');
         tryAdd(['testng.xml', 'src/test/resources/testng.xml'], 'testng-xml');
       }
 
@@ -2384,7 +2703,7 @@ async function processRepoScanJob(jobId: string, config: any) {
       };
 
       const fetched = await Promise.all(
-        configFilesToFetch.map(async ({ path: cfgPath, type }) => {
+        configFilesToFetch.map(async ({ path: cfgPath, type, dirPrefix }) => {
           const content = await fetchFileContent(token, conn.repoFullName, cfgPath).catch(() => null);
           if (!content) return null;
           const r: typeof configPaths = {};
@@ -2437,6 +2756,15 @@ async function processRepoScanJob(jobId: string, config: any) {
             if (pm) { const fp = pm[1].match(/['"`]([^'"`]+)['"`]/)?.[1]; if (fp) { const gr = globRoot(fp); if (gr) r.testDir = gr; } }
             if (!r.testDir) { try { const p = JSON.parse(content) as any; const paths = p?.default?.paths ?? p?.paths; if (Array.isArray(paths) && paths[0]) { const gr = globRoot(String(paths[0])); if (gr) r.testDir = gr; } } catch { /* not JSON */ } }
           }
+          // Monorepo workspace prefix: config found at e.g. packages/frontend/playwright.config.ts.
+          // testDir from config is relative to that file ('tests' or 'e2e'). Prepend the directory
+          // so the final path is repo-root-relative ('packages/frontend/tests').
+          if (dirPrefix && r.testDir && !r.testDir.startsWith(dirPrefix)) {
+            r.testDir = dirPrefix + r.testDir;
+          }
+          if (dirPrefix && r.javaTestSourceDir && !r.javaTestSourceDir.startsWith(dirPrefix)) {
+            r.javaTestSourceDir = dirPrefix + r.javaTestSourceDir;
+          }
           return r;
         })
       );
@@ -2477,7 +2805,8 @@ async function processRepoScanJob(jobId: string, config: any) {
               .slice(0, 60)
           : [];
         const allPathsForProfile = [...testPaths, ...mainSourcePaths, ...sourceFilePaths].slice(0, 200);
-        const freshProfile = await extractStyleProfile(styleSampleFiles, framework, language, tenantId, projectId, jobId, allPathsForProfile);
+        // Pass full tree paths separately so detectDataDirs can find testdata/*.json, fixtures/*.yaml etc.
+        const freshProfile = await extractStyleProfile(styleSampleFiles, framework, language, tenantId, projectId, jobId, allPathsForProfile, treeResult.paths);
         if (freshProfile) styleProfile = freshProfile;
       } catch { /* non-fatal */ }
     }
@@ -2587,6 +2916,8 @@ async function processRepoScanJob(jobId: string, config: any) {
         pageObjectInventory,
         testPaths,
         extractedTestCount: allExtracted.length,
+        treeWasTruncated: treeResult.truncated,
+        extractedTests: allExtracted,
       });
     } catch (e: any) {
       logger.warn({ jobId, repoConnectionId, err: e?.message }, '[repo-scan] repo skill files generation failed (non-fatal)');
@@ -2598,7 +2929,8 @@ async function processRepoScanJob(jobId: string, config: any) {
     );
     aggregateJobTokens(tenantId, jobId, projectId, jobStart).catch(console.error);
   } catch (err: any) {
-    const msg = String(err?.message ?? err);
+    const cause = err?.cause?.message ?? (typeof err?.cause === 'string' ? err.cause : '');
+    const msg = `${String(err?.message ?? err)}${cause ? ` — ${cause}` : ''}`;
     logger.error({ jobId, repoConnectionId, err: msg }, '[repo-scan] failed');
     await withTenant(tenantId, (tx) =>
       tx.gitHubRepoConnection.update({ where: { id: repoConnectionId }, data: { scanStatus: 'FAILED', scanError: msg.slice(0, 500) } })

package/src/locator-extractor.ts

@@ -56,6 +56,7 @@ export function elementToPlaywrightLocator(element: {
   text?: string | null;
   notes?: unknown;
   selector?: string | null;
+  ariaState?: Record<string, unknown> | null;
 }): PlaywrightLocator | null {
   const attrs = parseNotes(element.notes);
   const testId = attrs['data-testid'];
@@ -66,6 +67,12 @@ export function elementToPlaywrightLocator(element: {
   const text = (element.text ?? '').trim();
   const meaning = (element.meaning ?? '').trim();
   const role = mapRole(element.role);
+  const ariaState: Record<string, string> = {};
+  if (element.ariaState && typeof element.ariaState === 'object') {
+    for (const [k, v] of Object.entries(element.ariaState)) {
+      if (typeof v === 'string') ariaState[k] = v;
+    }
+  }
 
   const label = meaning || text || testId || ariaLabel || placeholder || id || name || 'element';
   const key = toKey(label);
@@ -106,7 +113,19 @@ export function elementToPlaywrightLocator(element: {
     }
   }
 
-  // Priority 3: checkbox/radio → getByLabel
+  // Priority 3: checkbox/radio → getByRole with optional checked state from ariaState
+  if ((role === 'checkbox' || role === 'radio') && meaning) {
+    const isChecked = ariaState['checked'];
+    const stateOpt = isChecked === 'true' ? `, { name: ${JSON.stringify(meaning)}, checked: true }` : isChecked === 'false' ? `, { name: ${JSON.stringify(meaning)}, checked: false }` : `, { name: ${JSON.stringify(meaning)} }`;
+    return {
+      key,
+      expression: `page.getByRole(${JSON.stringify(role)}${stateOpt})`,
+      elementType: role,
+      confidence: 'high',
+      source: 'role-name',
+      description: meaning,
+    };
+  }
   if ((role === 'checkbox' || role === 'radio') && ariaLabel) {
     return {
       key,
@@ -118,11 +137,29 @@ export function elementToPlaywrightLocator(element: {
     };
   }
 
-  // Priority 4: button/link with meaningful name → getByRole
+  // Priority 3b: tab/option/treeitem → getByRole with optional selected state
+  const tabLikeRoles = ['tab', 'option', 'treeitem', 'menuitem', 'menuitemcheckbox', 'menuitemradio', 'gridcell'];
+  if (tabLikeRoles.includes(role ?? '') && meaning) {
+    const isSelected = ariaState['selected'];
+    const isCurrent = ariaState['current'];
+    const stateOpt = isSelected === 'true' ? `, { name: ${JSON.stringify(meaning)}, selected: true }` : isCurrent ? `, { name: ${JSON.stringify(meaning)} }` : `, { name: ${JSON.stringify(meaning)} }`;
+    return {
+      key,
+      expression: `page.getByRole(${JSON.stringify(role)}${stateOpt})`,
+      elementType: role ?? 'element',
+      confidence: 'high',
+      source: 'role-name',
+      description: meaning,
+    };
+  }
+
+  // Priority 4: button/link with meaningful name → getByRole; include pressed state for toggles
   if ((role === 'button' || role === 'link') && meaning) {
+    const isPressed = ariaState['pressed'];
+    const stateOpt = isPressed === 'true' ? `, { name: ${JSON.stringify(meaning)}, pressed: true }` : isPressed === 'false' ? `, { name: ${JSON.stringify(meaning)}, pressed: false }` : `, { name: ${JSON.stringify(meaning)} }`;
     return {
       key,
-      expression: `page.getByRole(${JSON.stringify(role)}, { name: ${JSON.stringify(meaning)} })`,
+      expression: `page.getByRole(${JSON.stringify(role)}${stateOpt})`,
       elementType: role,
       confidence: 'high',
       source: 'role-name',
@@ -194,6 +231,7 @@ export function elementsToPlaywrightLocators(elements: Array<{
   notes?: unknown;
   selector?: string | null;
   isInteractive?: boolean | null;
+  ariaState?: Record<string, unknown> | null;
 }>): PlaywrightLocator[] {
   const CONFIDENCE_RANK = { high: 0, medium: 1, low: 2 };
   const byKey = new Map<string, PlaywrightLocator>();

package/src/policies/crawl-safety.ts

@@ -6,7 +6,13 @@ export interface CrawlSafetyDecision {
   severity: 'info' | 'warn' | 'block';
 }
 
-const DESTRUCTIVE_ACTION = /\b(delete|remove|destroy|deactivate|disable|cancel|refund|void|transfer|withdraw|pay now|place order|submit order|confirm payment|send money|publish|approve|reject|bulk update|drop)\b/i;
+const _extraDestructive = process.env.CRAWL_EXTRA_DESTRUCTIVE_PATTERNS
+  ? process.env.CRAWL_EXTRA_DESTRUCTIVE_PATTERNS.split(',').map((p) => p.trim()).filter(Boolean)
+  : [];
+const DESTRUCTIVE_ACTION = new RegExp(
+  `\\b(delete|remove|destroy|deactivate|disable|cancel|refund|void|transfer|withdraw|pay now|place order|submit order|confirm payment|send money|publish|approve|reject|bulk update|drop${_extraDestructive.length ? `|${_extraDestructive.join('|')}` : ''})\\b`,
+  'i',
+);
 const SECRET_LIKE = /\b(api[_ -]?key|secret|token|password|authorization|cookie|set-cookie)\b/i;
 
 export function assessActionSafety(label: string, profile: DomainProfile): CrawlSafetyDecision {