@detiq/crawler 1.1.10 → 1.1.12

package/src/job-queue.ts

@@ -51,13 +51,15 @@ async function notifyAdmins(tenantId: string, data: {
   }
 }
 import { AppBrain, getProjectStorageConfig, buildArtifactKey, uploadArtifact, getTenantStorageConfig, uploadScreenshotToTenantStorage, uploadFileToTenantStorage, notifyRunCompletion, recordLLMCall } from '@detiq/app-brain';
-import { generateTaxonomySkillFiles, groupScreensWithAI, generateExecutionPlan, refreshTestCasesSkillFile, refreshTestDataSkillFile, refreshTestScriptSkillFile, refreshRequirementsCoverageSkillFile, detectFieldsFromHtml, mergeIntoCatalog } from '@detiq/agents';
-import { crawlProject } from './crawler.js';
+import { generateTaxonomySkillFiles, groupScreensWithAI, generateExecutionPlan, refreshTestCasesSkillFile, refreshTestDataSkillFile, refreshTestScriptSkillFile, refreshRequirementsCoverageSkillFile, refreshTestExecutionHistorySkillFile, refreshVisualRegressionHistorySkillFile, refreshLocatorStrategyWithHealingData, refreshFieldCatalogSkillFile, detectFieldsFromHtml, mergeIntoCatalog } from '@detiq/agents';
+import type { ExecutionStep } from '@detiq/agents';
+import { crawlProject, domFallbackElements } from './crawler.js';
 import { httpCrawl } from './http-engine.js';
 import { hashContent, normalizeText } from './incremental-diff.js';
 import { discoverProjectUrls } from './sitemap.js';
-import { executeTestPlan } from './test-executor.js';
+import { executeExecutionStep, executeTestPlan } from './test-executor.js';
 import { runNativeCrawl } from './native-crawler.js';
+import { elementsToPlaywrightLocators } from './locator-extractor.js';
 import { jira, confluence, figma } from '@detiq/connectors';
 import { decrypt, config as platformConfig, dispatchWebhook, isFeatureEnabled, interpolatePlaceholders } from '@detiq/core';
 import { createHmac } from 'node:crypto';
@@ -106,6 +108,7 @@ function makeJobName(type: string): string {
   const date = now.toISOString().slice(0, 10); // YYYY-MM-DD
   const ts = now.getTime();
   const label = type === 'CRAWL' ? 'Crawl'
+    : type === 'CAPTURE_STEPS' ? 'CaptureSteps'
     : type === 'NATIVE_CRAWL' ? 'NativeCrawl'
     : type === 'GENERATE_TESTS' ? 'GenerateTests'
     : type === 'VISUAL_REGRESSION' ? 'VisualRegression'
@@ -783,6 +786,8 @@ export async function processJob(jobId: string, config: any, opts?: ProcessJobOp
             }
             if (fieldDetectAdded > 0 || detectLimit > 0) {
               logger.info({ screens: detectLimit, added: fieldDetectAdded, projectId }, '[field-detect] catalog populated from crawl');
+              // Refresh FIELD_CATALOG skill file after catalog is populated (GAP-SF-20)
+              refreshFieldCatalogSkillFile(tenantId, projectId).catch(console.error);
             }
           } catch (fdErr) {
             logger.warn({ err: fdErr }, '[field-detect] post-crawl field detection failed');
@@ -1524,6 +1529,7 @@ async function processRunTestSuiteJob(jobId: string, config: any) {
       { passed, failed, total: testCases.length },
       runResults,
     ).catch(console.error);
+    refreshTestExecutionHistorySkillFile(tenantId, projectId).catch(console.error);
   } catch (err: any) {
     const msg = String(err?.message ?? err).slice(0, 500);
     logger.error({ jobId, testRunId, err: msg }, 'processRunTestSuiteJob failed');
@@ -1563,6 +1569,10 @@ async function processGenerateSkillFilesJob(jobId: string, config: any) {
       refreshTestDataSkillFile(tenantId, projectId),
       refreshTestScriptSkillFile(tenantId, projectId),
       refreshRequirementsCoverageSkillFile(tenantId, projectId),
+      refreshTestExecutionHistorySkillFile(tenantId, projectId),
+      refreshVisualRegressionHistorySkillFile(tenantId, projectId),
+      refreshLocatorStrategyWithHealingData(tenantId, projectId),
+      refreshFieldCatalogSkillFile(tenantId, projectId),
     ]);
     await withTenant(tenantId, (tx) =>
       tx.platformJob.update({ where: { id: jobId }, data: { status: 'DONE', progress: 100, finishedAt: new Date() } })
@@ -2205,6 +2215,260 @@ async function processGenerateScreenTestsJob(jobId: string, config: any) {
   }
 }
 
+type CaptureStepStatus = 'PASSED' | 'FAILED' | 'HEALED' | 'SKIPPED';
+type CaptureStepResult = {
+  stepNumber: number;
+  description: string;
+  status: CaptureStepStatus;
+  screenshotPath?: string;
+  screenshotUrl?: string;
+  screenId?: string;
+  url?: string;
+  elementsCount?: number;
+  healed?: boolean;
+  healTier?: 'mechanical' | 'ai';
+  healReasoning?: string;
+  error?: string;
+};
+
+function captureStepText(step: unknown): string {
+  if (typeof step === 'string') return step;
+  if (!step || typeof step !== 'object') return '';
+  const obj = step as Record<string, unknown>;
+  return [obj.action, obj.text, obj.label, obj.data, obj.expected]
+    .filter((v): v is string => typeof v === 'string' && v.trim().length > 0)
+    .join(' -> ');
+}
+
+function firstQuoted(text: string): string | undefined {
+  return (text.match(/"([^"]+)"/)?.[1] ?? text.match(/'([^']+)'/)?.[1])?.trim();
+}
+
+function fallbackExecutionPlan(rawSteps: unknown[]): ExecutionStep[] {
+  const plan: ExecutionStep[] = [];
+  for (const raw of rawSteps) {
+    const text = captureStepText(raw).trim();
+    if (!text) continue;
+    const stepNumber = plan.length + 1;
+    const url = text.match(/https?:\/\/[^\s"']+/)?.[0];
+
+    if (/^navigate\b|^go to\b|^open\b/i.test(text) && url) {
+      plan.push({ stepNumber, description: text, action: 'navigate', url });
+      continue;
+    }
+
+    const urlContains = text.match(/verify url contains:\s*["']?([^"']+)["']?/i);
+    if (urlContains) {
+      plan.push({ stepNumber, description: text, action: 'assert_url', value: urlContains[1].trim() });
+      continue;
+    }
+
+    const inputMatch = text.match(/(?:enter|type|fill)\s+["']([^"']+)["']\s+(?:in|into)\s+(.+?)(?:\s+field|\s+input|\s+box|\s+area)?(?:\s*→.*)?$/i);
+    if (inputMatch) {
+      const selector = inputMatch[2].replace(/^(the\s+)?/i, '').replace(/\s+(field|input|box|area)$/i, '').trim();
+      plan.push({ stepNumber, description: text, action: 'fill', selectorStrategy: 'label', selector, value: inputMatch[1].trim() });
+      continue;
+    }
+
+    const quoted = firstQuoted(text);
+    if (/^(click|tap|navigate menu|check|select)\b/i.test(text) && quoted) {
+      plan.push({ stepNumber, description: text, action: 'click', selectorStrategy: 'text', selector: quoted });
+      continue;
+    }
+    if (/^(verify|assert|expect)\b/i.test(text) && quoted) {
+      plan.push({ stepNumber, description: text, action: 'assert_visible', selectorStrategy: 'text', selector: quoted });
+      continue;
+    }
+    if (/execute script/i.test(text)) {
+      const script = text.split(':').slice(1).join(':').replace(/\s*→.*$/, '').trim();
+      if (script) plan.push({ stepNumber, description: text, action: 'script', value: script });
+    }
+  }
+  return plan;
+}
+
+function screenNameFromUrl(url: string, fallback: string): string {
+  try {
+    const u = new URL(url);
+    return u.pathname.replace(/\/+$/, '').split('/').filter(Boolean).pop() || u.hostname || fallback;
+  } catch {
+    return fallback;
+  }
+}
+
+async function processCaptureStepsJob(jobId: string, config: any) {
+  const { tenantId, projectId, testCaseId } = config as { tenantId: string; projectId: string; testCaseId: string };
+  const screenshotDir = config.screenshotDir || platformConfig.screenshotDir || process.env.SCREENSHOT_DIR || '/tmp/zeta-screenshots';
+  await fs.mkdir(screenshotDir, { recursive: true });
+
+  await withTenant(tenantId, (tx) =>
+    tx.platformJob.update({ where: { id: jobId }, data: { status: 'RUNNING', startedAt: new Date(), progress: 2 } })
+  );
+
+  let browser: any;
+  try {
+    await updateJobProgress(tenantId, jobId, 5, 'Loading test case');
+    const [tc, crawlCfg, tenantStorageCfg] = await Promise.all([
+      AppBrain.getTestCaseForExecution(tenantId, testCaseId) as Promise<any>,
+      AppBrain.getCrawlConfig(tenantId, projectId).catch(() => null),
+      getTenantStorageConfig(tenantId).catch(() => null),
+    ]);
+    if (!tc) throw new Error('Test case not found');
+
+    const rawSteps = Array.isArray(tc.steps) ? tc.steps : [];
+    const stepTexts = rawSteps.map(captureStepText).filter(Boolean);
+    if (stepTexts.length === 0) throw new Error('Test case has no steps to capture');
+
+    const baseUrl = config.baseUrl || tc?.screen?.project?.appUrl || crawlCfg?.appUrl || '';
+    let plan = await generateExecutionPlan(tenantId, { ...tc, steps: stepTexts }, baseUrl).catch(() => []);
+    plan = Array.isArray(plan)
+      ? plan.filter(Boolean).map((step: any, idx: number) => ({ ...step, stepNumber: idx + 1 }))
+      : [];
+    if (plan.length === 0) plan = fallbackExecutionPlan(rawSteps);
+    if (plan.length === 0) throw new Error('Could not generate execution plan from steps');
+
+    const { chromium } = await import('playwright');
+    browser = await chromium.launch({
+      headless: true,
+      ...(process.env.PLAYWRIGHT_EXECUTABLE_PATH ? { executablePath: process.env.PLAYWRIGHT_EXECUTABLE_PATH } : {}),
+    });
+    let parsedStorageState: any;
+    try {
+      parsedStorageState = config.storageState ? JSON.parse(config.storageState) : crawlCfg?.storageState ? JSON.parse(crawlCfg.storageState) : undefined;
+    } catch {
+      parsedStorageState = undefined;
+    }
+    const context = await browser.newContext({ viewport: { width: 1280, height: 800 }, ...(parsedStorageState ? { storageState: parsedStorageState } : {}) });
+    const page = await context.newPage();
+    const aiHealingEnabled = await isFeatureEnabled(tenantId, 'AI_SELF_HEALING').catch(() => false);
+
+    const stepResults: CaptureStepResult[] = [];
+    const screens: Array<{ stepNumber: number; screenId: string; url: string; screenshotPath?: string; screenshotUrl?: string; isNewScreen: boolean }> = [];
+    const locators: any[] = [];
+    const seenLocatorExpressions = new Set<string>();
+    const seenScreenKeys = new Set<string>();
+    let lastStateKey = '';
+
+    for (let i = 0; i < plan.length; i++) {
+      const step = plan[i];
+      const description = step.description || stepTexts[i] || `Step ${i + 1}`;
+      const stepNumber = i + 1;
+      const result: CaptureStepResult = { stepNumber, description, status: 'FAILED' };
+      await withTenant(tenantId, (tx) =>
+        tx.platformJob.update({
+          where: { id: jobId },
+          data: {
+            progress: Math.min(90, Math.round((i / Math.max(plan.length, 1)) * 90)),
+            result: { currentStep: { stepNumber, description } } as any,
+            heartbeatAt: new Date(),
+          },
+        })
+      ).catch(() => {});
+
+      try {
+        const heal = await executeExecutionStep(page, step, aiHealingEnabled ? { healCtx: { tenantId, projectId } } : {});
+        if (heal.healed) {
+          result.healed = true;
+          result.healTier = heal.healedVia;
+          result.healReasoning = heal.healReasoning;
+        }
+        await page.waitForLoadState('networkidle', { timeout: parseInt(process.env.TEST_NETWORK_IDLE_TIMEOUT_MS ?? '2000', 10) }).catch(() => {});
+        result.status = result.healed ? 'HEALED' : 'PASSED';
+      } catch (err: any) {
+        result.status = step.optional ? 'SKIPPED' : 'FAILED';
+        result.error = String(err?.message ?? err).slice(0, 300);
+      }
+
+      const currentUrl = page.url();
+      result.url = currentUrl;
+      const ssFilename = `${testCaseId}-capture-step-${stepNumber}${result.status === 'FAILED' ? '-err' : ''}.png`;
+      const ssPath = path.join(screenshotDir, ssFilename);
+      await page.screenshot({ path: ssPath, fullPage: false }).catch(() => {});
+      result.screenshotPath = ssPath;
+
+      let cloudUrl: string | undefined;
+      if (tenantStorageCfg) {
+        const key = buildArtifactKey(tenantId, projectId, 'screenshots', ssFilename, tenantStorageCfg.prefix);
+        cloudUrl = (await uploadScreenshotToTenantStorage(tenantStorageCfg, ssPath, key).catch(() => undefined)) ?? undefined;
+        if (cloudUrl) result.screenshotUrl = cloudUrl;
+      }
+
+      const elements = await domFallbackElements(page).catch(() => []);
+      result.elementsCount = elements.length;
+      const stateHash = hashContent(elements.map((el) => `${el.role}:${el.meaning}:${el.notes ?? ''}`).join('|'));
+      const screenKey = `${currentUrl || 'about:blank'}:${stateHash}`;
+      const shouldPersistScreen = currentUrl && currentUrl !== 'about:blank' && (i === 0 || screenKey !== lastStateKey || !seenScreenKeys.has(screenKey));
+      lastStateKey = screenKey;
+
+      if (shouldPersistScreen) {
+        const screenName = screenNameFromUrl(currentUrl, tc.title ?? `Step ${stepNumber}`);
+        const screen = await AppBrain.saveScreen(tenantId, projectId, screenName, currentUrl, jobId, i === 0 ? 'ENTRY_POINT' : 'LINK_FOLLOW').catch(() => null) as any;
+        if (screen?.id) {
+          result.screenId = screen.id;
+          await AppBrain.saveScreenshot(tenantId, projectId, screen.id, ssPath, screen.version ?? 1, 'DESKTOP', cloudUrl).catch(() => {});
+          for (const el of elements) {
+            await AppBrain.saveElement(
+              tenantId, projectId, screen.id,
+              el.meaning, el.role, el.expectedData, el.notes,
+              undefined, true, el.boundingRect, el.ariaState, el.parentLandmark,
+            ).catch(() => {});
+          }
+          for (const loc of elementsToPlaywrightLocators(elements as any)) {
+            if (seenLocatorExpressions.has(loc.expression)) continue;
+            seenLocatorExpressions.add(loc.expression);
+            locators.push(loc);
+          }
+          if (!seenScreenKeys.has(screenKey)) {
+            screens.push({ stepNumber, screenId: screen.id, url: currentUrl, screenshotPath: ssPath, screenshotUrl: cloudUrl, isNewScreen: true });
+            seenScreenKeys.add(screenKey);
+          }
+        }
+      }
+
+      stepResults.push(result);
+      if (result.status === 'FAILED') break;
+    }
+
+    if (locators.length > 0) {
+      await AppBrain.saveLocatorsJson(tenantId, testCaseId, locators as any).catch(() => {});
+    }
+
+    const firstScreen = screens[0];
+    await withTenant(tenantId, (tx) =>
+      (tx as any).testCase.update({
+        where: { id: testCaseId },
+        data: {
+          ...(firstScreen?.screenId ? { screenId: firstScreen.screenId } : {}),
+          captureStepScreenshots: stepResults as any,
+        },
+      })
+    ).catch(() => {});
+
+    const finalResult = {
+      stepResults,
+      screens,
+      locators,
+      totalSteps: plan.length,
+      capturedScreens: screens.length,
+    };
+    await withTenant(tenantId, (tx) =>
+      tx.platformJob.update({
+        where: { id: jobId },
+        data: { status: 'DONE', progress: 100, result: finalResult as any, finishedAt: new Date(), heartbeatAt: new Date() },
+      })
+    );
+  } catch (err: any) {
+    const error = String(err?.message ?? err);
+    await withTenant(tenantId, (tx) =>
+      tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', error, finishedAt: new Date(), heartbeatAt: new Date() } })
+    ).catch(() => {});
+    logger.error({ jobId, testCaseId, err: error }, 'processCaptureStepsJob failed');
+    throw err;
+  } finally {
+    await browser?.close?.().catch(() => {});
+  }
+}
+
 // Registers the type→processor dispatch table used by the durable worker (both the
 // enqueueJob() fast-path kick and startJobWorkerLoop()'s poll). Declared at module
 // bottom so it can reference every process*Job function by name; function
@@ -2213,6 +2477,7 @@ async function processGenerateScreenTestsJob(jobId: string, config: any) {
 // against the actual functions above.
 jobDispatchTable = {
   CRAWL: processJob,
+  CAPTURE_STEPS: processCaptureStepsJob,
   DISCOVER: processDiscoverJob,
   MONITOR: processMonitorJob,
   NATIVE_CRAWL: processNativeCrawlJob,
@@ -2240,7 +2505,7 @@ async function processRepoScanJob(jobId: string, config: any) {
     const {
       fetchRepoTree, fetchFileContent, detectFramework,
       extractTestIds, extractStyleProfile,
-      buildPageObjectInventory,
+      buildPageObjectInventory, resolveFixtureAliasesFromImports,
     } = await import('@detiq/agents');
 
     const conn = await withTenant(tenantId, (tx) =>
@@ -2260,6 +2525,9 @@ async function processRepoScanJob(jobId: string, config: any) {
     await updateJobProgress(tenantId, jobId, 15, 'Fetching repo tree');
     // Use HEAD so it always resolves to the actual default branch regardless of stored value
     const treeResult = await fetchRepoTree(token, conn.repoFullName, 'HEAD');
+    if (treeResult.truncated) {
+      console.warn(`[repo-scan] ${conn.repoFullName}: tree truncated (>100k blobs) — detection may be incomplete`);
+    }
 
     const frameworks = detectFramework(treeResult.paths);
     const primary = frameworks[0];
@@ -2295,6 +2563,21 @@ async function processRepoScanJob(jobId: string, config: any) {
 
     await updateJobProgress(tenantId, jobId, 40, `Found ${testPaths.length} test files`);
 
+    if (testPaths.length === 0) {
+      logger.warn({ jobId, repoConnectionId, framework }, '[repo-scan] no test files matched glob — scan complete with 0 results');
+      await withTenant(tenantId, (tx) =>
+        tx.gitHubRepoConnection.update({
+          where: { id: repoConnectionId },
+          data: { scanStatus: 'DONE', testFileCount: 0, extractedIdCount: 0, lastScannedAt: new Date(), lastSyncAt: new Date() },
+        })
+      );
+      await updateJobProgress(tenantId, jobId, 100, 'Done: 0 test files found');
+      await withTenant(tenantId, (tx) =>
+        tx.platformJob.update({ where: { id: jobId }, data: { status: 'DONE', finishedAt: new Date(), result: { testFileCount: 0, extractedIdCount: 0 } as any } })
+      );
+      return;
+    }
+
     // Extract test IDs from each file
     const allExtracted: any[] = [];
     const styleSampleFiles: Array<{ path: string; content: string }> = [];
@@ -2307,7 +2590,11 @@ async function processRepoScanJob(jobId: string, config: any) {
       const content = await fetchFileContent(token, conn.repoFullName, filePath);
       if (!content) continue;
 
-      const extracted = extractTestIds(content, filePath, framework);
+      // G-07: multi-file fixture tracing — resolve non-obvious aliases before extraction
+      const extraAliases = await resolveFixtureAliasesFromImports(
+        content, token, conn.repoFullName, filePath,
+      ).catch(() => new Set<string>());
+      const extracted = extractTestIds(content, filePath, framework, extraAliases);
       allExtracted.push(...extracted);
 
       await withTenant(tenantId, (tx) =>
@@ -2333,6 +2620,161 @@ async function processRepoScanJob(jobId: string, config: any) {
       }
     }
 
+    // Fetch and parse framework config files (playwright.config.ts, cypress.config.ts, tsconfig.json)
+    // Fetch and parse framework config files for all supported frameworks.
+    // Pure regex/JSON — no LLM. Results override LLM inference for path detection.
+    // Covers: playwright, cypress, jest, pytest, junit/maven/gradle, selenium-java/py,
+    //         webdriverio, cucumber — all frameworks in DetectedFramework.
+    const configPaths: {
+      testDir?: string; fixturesFolder?: string; baseURL?: string;
+      tsconfigPaths?: Record<string, string[]>; tsconfigBaseUrl?: string;
+      javaTestSourceDir?: string; jestModuleNameMapper?: Record<string, string>;
+    } = {};
+    try {
+      const pathSet = new Set(treeResult.paths as string[]);
+      const lang = framework.includes('java') ? 'java'
+        : (framework.includes('py') || framework === 'pytest') ? 'python' : 'js';
+
+      const configFilesToFetch: Array<{ path: string; type: string; dirPrefix: string }> = [];
+      const tryAdd = (candidates: string[], type: string) => {
+        for (const c of candidates) { if (pathSet.has(c)) { configFilesToFetch.push({ path: c, type, dirPrefix: '' }); break; } }
+      };
+      // Monorepo fallback: when root-level config is absent, find shallowest match anywhere in tree.
+      // The dirPrefix (e.g. 'packages/e2e/') is prepended to testDir so paths are repo-root-relative.
+      const scanTree = (pattern: RegExp, type: string) => {
+        if (configFilesToFetch.some(f => f.type === type)) return;
+        const all = (treeResult.paths as string[]).filter(p => pattern.test(p)).sort((a, b) => a.split('/').length - b.split('/').length);
+        if (all[0]) {
+          const lastSlash = all[0].lastIndexOf('/');
+          const dirPrefix = lastSlash > 0 ? all[0].slice(0, lastSlash + 1) : '';
+          configFilesToFetch.push({ path: all[0], type, dirPrefix });
+        }
+      };
+
+      if (lang === 'js') {
+        if (framework.startsWith('playwright')) {
+          tryAdd(['playwright.config.ts', 'playwright.config.js'], 'playwright');
+          scanTree(/(?:^|\/)playwright\.config\.(ts|js|mts)$/, 'playwright');
+        }
+        if (framework.startsWith('cypress')) {
+          tryAdd(['cypress.config.ts', 'cypress.config.js'], 'cypress');
+          scanTree(/(?:^|\/)cypress\.config\.(ts|js)$/, 'cypress');
+        }
+        if (framework === 'jest-ts' || framework === 'jest-js') {
+          tryAdd(['jest.config.ts', 'jest.config.js', 'jest.config.mjs'], 'jest');
+          scanTree(/(?:^|\/)jest\.config\.(ts|js|mjs|cjs)$/, 'jest');
+          tryAdd(['package.json'], 'package-json');
+        }
+        if (framework === 'webdriverio') {
+          tryAdd(['wdio.conf.ts', 'wdio.conf.js', 'wdio.config.ts', 'wdio.config.js'], 'wdio');
+          scanTree(/(?:^|\/)wdio\.(?:conf|config)\.(ts|js|mjs)$/, 'wdio');
+        }
+        if (framework === 'cucumber' || framework === 'gherkin') {
+          tryAdd(['.cucumber.js', '.cucumber.cjs', 'cucumber.json', '.cucumber.yaml', '.cucumber.yml'], 'cucumber');
+          tryAdd(['package.json'], 'package-json');
+        }
+        tryAdd(['tsconfig.json'], 'tsconfig');
+        scanTree(/(?:^|\/)tsconfig(?:\.base)?\.json$/, 'tsconfig');
+      } else if (lang === 'python') {
+        tryAdd(['pytest.ini'], 'pytest-ini');
+        scanTree(/(?:^|\/)pytest\.ini$/, 'pytest-ini');
+        tryAdd(['pyproject.toml'], 'pyproject-toml');
+        scanTree(/(?:^|\/)pyproject\.toml$/, 'pyproject-toml');
+        tryAdd(['setup.cfg'], 'setup-cfg');
+      } else if (lang === 'java') {
+        tryAdd(['pom.xml'], 'pom-xml');
+        scanTree(/(?:^|\/)pom\.xml$/, 'pom-xml');
+        tryAdd(['build.gradle.kts', 'build.gradle'], 'build-gradle');
+        scanTree(/(?:^|\/)build\.gradle(?:\.kts)?$/, 'build-gradle');
+        tryAdd(['testng.xml', 'src/test/resources/testng.xml'], 'testng-xml');
+      }
+
+      // Inline parsers — mirror parseFrameworkConfig from @detiq/agents (kept inline to avoid build dep)
+      const normDir = (s: string) => s.replace(/^\.\//, '').replace(/\/$/, '');
+      const globRoot = (pattern: string): string | null => {
+        const root = pattern.replace(/\/\*\*.*$/, '').replace(/\/\*[^*].*$/, '');
+        return (root && !root.startsWith('*') && !root.startsWith('{')) ? normDir(root) : null;
+      };
+
+      const fetched = await Promise.all(
+        configFilesToFetch.map(async ({ path: cfgPath, type, dirPrefix }) => {
+          const content = await fetchFileContent(token, conn.repoFullName, cfgPath).catch(() => null);
+          if (!content) return null;
+          const r: typeof configPaths = {};
+
+          if (type === 'playwright') {
+            const m = content.match(/\btestDir\s*:\s*['"`]([^'"`]+)['"`]/);
+            if (m) r.testDir = normDir(m[1]);
+            const bm = content.match(/\bbaseURL\s*:\s*['"`]([^'"`]+)['"`]/);
+            if (bm) r.baseURL = bm[1];
+          } else if (type === 'cypress') {
+            const sm = content.match(/\bspecPattern\s*:\s*['"`]([^'"`]+)['"`]/);
+            if (sm) { const gr = globRoot(sm[1]); if (gr) r.testDir = gr; }
+            const fm = content.match(/\bfixturesFolder\s*:\s*['"`]([^'"`]+)['"`]/);
+            if (fm) r.fixturesFolder = normDir(fm[1]);
+            const bm = content.match(/\bbaseUrl\s*:\s*['"`]([^'"`]+)['"`]/);
+            if (bm) r.baseURL = bm[1];
+          } else if (type === 'jest') {
+            const rm = content.match(/\broots\s*:\s*\[([^\]]+)\]/);
+            if (rm) { const fr = rm[1].match(/['"`]([^'"`]+)['"`]/)?.[1]; if (fr) r.testDir = fr.replace(/<rootDir>\/?/, '').replace(/\/$/, '') || 'src'; }
+            if (!r.testDir) { const tm = content.match(/\btestMatch\s*:\s*\[([^\]]+)\]/); if (tm) { const fp = tm[1].match(/['"`]([^'"`]+)['"`]/)?.[1]; if (fp) { const gr = globRoot(fp.replace(/<rootDir>\/?/, '')); if (gr) r.testDir = gr; } } }
+            const mm = content.match(/\bmoduleNameMapper\s*:\s*\{([^}]+)\}/s);
+            if (mm) { const map: Record<string, string> = {}; for (const [, k, v] of mm[1].matchAll(/['"`]([^'"`]+)['"`]\s*:\s*['"`]([^'"`]+)['"`]/g)) map[k] = v; if (Object.keys(map).length) r.jestModuleNameMapper = map; }
+          } else if (type === 'package-json') {
+            try { const p = JSON.parse(content) as any; const j = p?.jest ?? {}; if (Array.isArray(j.roots) && j.roots[0]) r.testDir = String(j.roots[0]).replace(/<rootDir>\/?/, '').replace(/\/$/, '') || 'src'; if (!r.testDir && Array.isArray(j.testMatch) && j.testMatch[0]) { const gr = globRoot(String(j.testMatch[0]).replace(/<rootDir>\/?/, '')); if (gr) r.testDir = gr; } if (j.moduleNameMapper && typeof j.moduleNameMapper === 'object') r.jestModuleNameMapper = j.moduleNameMapper; } catch { /* ignore */ }
+          } else if (type === 'tsconfig') {
+            try { const stripped = content.replace(/\/\/[^\n]*/g, '').replace(/,(\s*[}\]])/g, '$1'); const p = JSON.parse(stripped) as any; const opts = p?.compilerOptions ?? {}; if (opts.paths && typeof opts.paths === 'object') r.tsconfigPaths = opts.paths; if (typeof opts.baseUrl === 'string') r.tsconfigBaseUrl = opts.baseUrl; } catch { /* ignore */ }
+          } else if (type === 'pytest-ini') {
+            const m = content.match(/^\s*testpaths\s*=\s*(.+)$/m);
+            if (m) { const first = m[1].trim().split(/\s+/)[0]; if (first) r.testDir = normDir(first); }
+          } else if (type === 'pyproject-toml') {
+            const m = content.match(/testpaths\s*=\s*\[([^\]]+)\]/);
+            if (m) { const first = m[1].match(/['"]([^'"]+)['"]/)?.[1]; if (first) r.testDir = normDir(first); }
+            if (!r.testDir) { const m2 = content.match(/testpaths\s*=\s*['"]([^'"]+)['"]/); if (m2) r.testDir = normDir(m2[1]); }
+          } else if (type === 'setup-cfg') {
+            const m = content.match(/^\s*testpaths\s*=\s*(.+)$/m);
+            if (m) { const first = m[1].trim().split(/[\s,]+/)[0]; if (first) r.testDir = normDir(first); }
+          } else if (type === 'pom-xml') {
+            const m = content.match(/<testSourceDirectory>\s*([^<]+)\s*<\/testSourceDirectory>/);
+            if (m) r.javaTestSourceDir = normDir(m[1].trim());
+          } else if (type === 'build-gradle') {
+            const m = content.match(/srcDir[s]?\s*[=([]?\s*['"`]([^'"`]+)['"`]/) ?? content.match(/srcDir\s*\(\s*['"`]([^'"`]+)['"`]\s*\)/);
+            if (m) r.javaTestSourceDir = normDir(m[1]);
+          } else if (type === 'wdio') {
+            const sm = content.match(/\bspecs\s*:\s*\[([^\]]+)\]/);
+            if (sm) { const fp = sm[1].match(/['"`]([^'"`]+)['"`]/)?.[1]; if (fp) { const gr = globRoot(fp); if (gr) r.testDir = gr; } }
+            const bm = content.match(/\bbaseUrl\s*:\s*['"`]([^'"`]+)['"`]/);
+            if (bm) r.baseURL = bm[1];
+          } else if (type === 'cucumber') {
+            const pm = content.match(/\bpaths\s*:\s*\[([^\]]+)\]/);
+            if (pm) { const fp = pm[1].match(/['"`]([^'"`]+)['"`]/)?.[1]; if (fp) { const gr = globRoot(fp); if (gr) r.testDir = gr; } }
+            if (!r.testDir) { try { const p = JSON.parse(content) as any; const paths = p?.default?.paths ?? p?.paths; if (Array.isArray(paths) && paths[0]) { const gr = globRoot(String(paths[0])); if (gr) r.testDir = gr; } } catch { /* not JSON */ } }
+          }
+          // Monorepo workspace prefix: config found at e.g. packages/frontend/playwright.config.ts.
+          // testDir from config is relative to that file ('tests' or 'e2e'). Prepend the directory
+          // so the final path is repo-root-relative ('packages/frontend/tests').
+          if (dirPrefix && r.testDir && !r.testDir.startsWith(dirPrefix)) {
+            r.testDir = dirPrefix + r.testDir;
+          }
+          if (dirPrefix && r.javaTestSourceDir && !r.javaTestSourceDir.startsWith(dirPrefix)) {
+            r.javaTestSourceDir = dirPrefix + r.javaTestSourceDir;
+          }
+          return r;
+        })
+      );
+
+      for (const parsed of fetched) {
+        if (!parsed) continue;
+        if (parsed.testDir && !configPaths.testDir) configPaths.testDir = parsed.testDir;
+        if (parsed.fixturesFolder && !configPaths.fixturesFolder) configPaths.fixturesFolder = parsed.fixturesFolder;
+        if (parsed.baseURL && !configPaths.baseURL) configPaths.baseURL = parsed.baseURL;
+        if (parsed.tsconfigPaths && !configPaths.tsconfigPaths) configPaths.tsconfigPaths = parsed.tsconfigPaths;
+        if (parsed.tsconfigBaseUrl && !configPaths.tsconfigBaseUrl) configPaths.tsconfigBaseUrl = parsed.tsconfigBaseUrl;
+        if (parsed.javaTestSourceDir && !configPaths.javaTestSourceDir) configPaths.javaTestSourceDir = parsed.javaTestSourceDir;
+        if (parsed.jestModuleNameMapper && !configPaths.jestModuleNameMapper) configPaths.jestModuleNameMapper = parsed.jestModuleNameMapper;
+      }
+    } catch { /* non-fatal */ }
+
     // Extract style profile for aligned generation
     // For Java, include main source paths so LLM can detect page object location in src/main/java/
     // Fall back to existing styleProfile from DB if extraction fails or returns null.
@@ -2357,11 +2799,28 @@ async function processRepoScanJob(jobId: string, config: any) {
               .slice(0, 60)
           : [];
         const allPathsForProfile = [...testPaths, ...mainSourcePaths, ...sourceFilePaths].slice(0, 200);
-        const freshProfile = await extractStyleProfile(styleSampleFiles, framework, language, tenantId, projectId, jobId, allPathsForProfile);
+        // Pass full tree paths separately so detectDataDirs can find testdata/*.json, fixtures/*.yaml etc.
+        const freshProfile = await extractStyleProfile(styleSampleFiles, framework, language, tenantId, projectId, jobId, allPathsForProfile, treeResult.paths);
         if (freshProfile) styleProfile = freshProfile;
       } catch { /* non-fatal */ }
     }
 
+    // Merge config file ground-truth into styleProfile (Priority 0 — highest, no LLM inference needed).
+    // Java: javaTestSourceDir also sets testLocation (equivalent field for Java).
+    if (styleProfile && Object.keys(configPaths).length > 0) {
+      if (configPaths.testDir) styleProfile.configTestDir = configPaths.testDir;
+      if (configPaths.javaTestSourceDir) {
+        styleProfile.configTestDir = configPaths.javaTestSourceDir;
+        // Also set testLocation if not already present from LLM scan
+        if (!styleProfile.testLocation) styleProfile.testLocation = configPaths.javaTestSourceDir;
+      }
+      if (configPaths.fixturesFolder) styleProfile.configFixturesFolder = configPaths.fixturesFolder;
+      if (configPaths.baseURL) styleProfile.configBaseURL = configPaths.baseURL;
+      if (configPaths.tsconfigPaths) styleProfile.tsconfigPaths = configPaths.tsconfigPaths;
+      if (configPaths.tsconfigBaseUrl) styleProfile.tsconfigBaseUrl = configPaths.tsconfigBaseUrl;
+      if (configPaths.jestModuleNameMapper) styleProfile.jestModuleNameMapper = configPaths.jestModuleNameMapper;
+    }
+
     // Extract page object inventory — finds POM/page classes in non-test files
     // Fall back to existing inventory from DB if fetch fails.
     let pageObjectInventory: any[] = (conn.pageObjectInventory as any[]) ?? [];
@@ -2451,6 +2910,8 @@ async function processRepoScanJob(jobId: string, config: any) {
         pageObjectInventory,
         testPaths,
         extractedTestCount: allExtracted.length,
+        treeWasTruncated: treeResult.truncated,
+        extractedTests: allExtracted,
       });
     } catch (e: any) {
       logger.warn({ jobId, repoConnectionId, err: e?.message }, '[repo-scan] repo skill files generation failed (non-fatal)');
@@ -2462,7 +2923,8 @@ async function processRepoScanJob(jobId: string, config: any) {
     );
     aggregateJobTokens(tenantId, jobId, projectId, jobStart).catch(console.error);
   } catch (err: any) {
-    const msg = String(err?.message ?? err);
+    const cause = err?.cause?.message ?? (typeof err?.cause === 'string' ? err.cause : '');
+    const msg = `${String(err?.message ?? err)}${cause ? ` — ${cause}` : ''}`;
     logger.error({ jobId, repoConnectionId, err: msg }, '[repo-scan] failed');
     await withTenant(tenantId, (tx) =>
       tx.gitHubRepoConnection.update({ where: { id: repoConnectionId }, data: { scanStatus: 'FAILED', scanError: msg.slice(0, 500) } })