@detiq/crawler 1.1.10 → 1.1.12

package/dist/job-queue.js

@@ -30,13 +30,14 @@ async function notifyAdmins(tenantId, data) {
   }
 }
 import { AppBrain, buildArtifactKey, getTenantStorageConfig, uploadScreenshotToTenantStorage, uploadFileToTenantStorage, notifyRunCompletion, recordLLMCall } from "@detiq/app-brain";
-import { generateTaxonomySkillFiles, groupScreensWithAI, generateExecutionPlan, refreshTestCasesSkillFile, refreshTestDataSkillFile, refreshTestScriptSkillFile, refreshRequirementsCoverageSkillFile, detectFieldsFromHtml, mergeIntoCatalog } from "@detiq/agents";
-import { crawlProject } from "./crawler.js";
+import { generateTaxonomySkillFiles, groupScreensWithAI, generateExecutionPlan, refreshTestCasesSkillFile, refreshTestDataSkillFile, refreshTestScriptSkillFile, refreshRequirementsCoverageSkillFile, refreshTestExecutionHistorySkillFile, refreshVisualRegressionHistorySkillFile, refreshLocatorStrategyWithHealingData, refreshFieldCatalogSkillFile, detectFieldsFromHtml, mergeIntoCatalog } from "@detiq/agents";
+import { crawlProject, domFallbackElements } from "./crawler.js";
 import { httpCrawl } from "./http-engine.js";
 import { hashContent, normalizeText } from "./incremental-diff.js";
 import { discoverProjectUrls } from "./sitemap.js";
-import { executeTestPlan } from "./test-executor.js";
+import { executeExecutionStep, executeTestPlan } from "./test-executor.js";
 import { runNativeCrawl } from "./native-crawler.js";
+import { elementsToPlaywrightLocators } from "./locator-extractor.js";
 import { jira, confluence, figma } from "@detiq/connectors";
 import { decrypt, config as platformConfig, dispatchWebhook, isFeatureEnabled, interpolatePlaceholders } from "@detiq/core";
 import { createHmac } from "node:crypto";
@@ -76,7 +77,7 @@ function makeJobName(type) {
   const now = /* @__PURE__ */ new Date();
   const date = now.toISOString().slice(0, 10);
   const ts = now.getTime();
-  const label = type === "CRAWL" ? "Crawl" : type === "NATIVE_CRAWL" ? "NativeCrawl" : type === "GENERATE_TESTS" ? "GenerateTests" : type === "VISUAL_REGRESSION" ? "VisualRegression" : type === "AI_EDIT" ? "AIEdit" : type === "API_TESTS" ? "ApiTests" : type === "SECURITY_TESTS" ? "SecurityTests" : type === "INTEGRATION_TESTS" ? "IntegrationTests" : type === "GENERATE_TESTS_FROM_REQS" ? "GenerateTestsFromReqs" : type === "GENERATE_DATA" ? "GenerateData" : type === "GENERATE_SCRIPT" ? "GenerateScript" : type === "GENERATE_TEST_PLAN" ? "GenerateTestPlan" : type === "IMPORT_TESTS" ? "ImportTests" : type === "GENERATE_SCREEN_TESTS" ? "GenerateScreenTests" : type === "BATCH_GENERATE_ALIGNED" ? "BatchGenerateAligned" : type === "CREATE_PR" ? "CreatePR" : type === "NEW_PAGES_PIPELINE" ? "NewPagesPipeline" : type.charAt(0) + type.slice(1).toLowerCase().replace(/_([a-z])/g, (_, c) => c.toUpperCase());
+  const label = type === "CRAWL" ? "Crawl" : type === "CAPTURE_STEPS" ? "CaptureSteps" : type === "NATIVE_CRAWL" ? "NativeCrawl" : type === "GENERATE_TESTS" ? "GenerateTests" : type === "VISUAL_REGRESSION" ? "VisualRegression" : type === "AI_EDIT" ? "AIEdit" : type === "API_TESTS" ? "ApiTests" : type === "SECURITY_TESTS" ? "SecurityTests" : type === "INTEGRATION_TESTS" ? "IntegrationTests" : type === "GENERATE_TESTS_FROM_REQS" ? "GenerateTestsFromReqs" : type === "GENERATE_DATA" ? "GenerateData" : type === "GENERATE_SCRIPT" ? "GenerateScript" : type === "GENERATE_TEST_PLAN" ? "GenerateTestPlan" : type === "IMPORT_TESTS" ? "ImportTests" : type === "GENERATE_SCREEN_TESTS" ? "GenerateScreenTests" : type === "BATCH_GENERATE_ALIGNED" ? "BatchGenerateAligned" : type === "CREATE_PR" ? "CreatePR" : type === "NEW_PAGES_PIPELINE" ? "NewPagesPipeline" : type.charAt(0) + type.slice(1).toLowerCase().replace(/_([a-z])/g, (_, c) => c.toUpperCase());
   return `${label}_${date}_${ts}`;
 }
 const runningControllers = /* @__PURE__ */ new Map();
@@ -548,6 +549,7 @@ async function processJob(jobId, config, opts) {
             }
             if (fieldDetectAdded > 0 || detectLimit > 0) {
               logger.info({ screens: detectLimit, added: fieldDetectAdded, projectId }, "[field-detect] catalog populated from crawl");
+              refreshFieldCatalogSkillFile(tenantId, projectId).catch(console.error);
             }
           } catch (fdErr) {
             logger.warn({ err: fdErr }, "[field-detect] post-crawl field detection failed");
@@ -1234,6 +1236,7 @@ async function processRunTestSuiteJob(jobId, config) {
       { passed, failed, total: testCases.length },
       runResults
     ).catch(console.error);
+    refreshTestExecutionHistorySkillFile(tenantId, projectId).catch(console.error);
   } catch (err) {
     const msg = String(err?.message ?? err).slice(0, 500);
     logger.error({ jobId, testRunId, err: msg }, "processRunTestSuiteJob failed");
@@ -1273,7 +1276,11 @@ async function processGenerateSkillFilesJob(jobId, config) {
       refreshTestCasesSkillFile(tenantId, projectId),
       refreshTestDataSkillFile(tenantId, projectId),
       refreshTestScriptSkillFile(tenantId, projectId),
-      refreshRequirementsCoverageSkillFile(tenantId, projectId)
+      refreshRequirementsCoverageSkillFile(tenantId, projectId),
+      refreshTestExecutionHistorySkillFile(tenantId, projectId),
+      refreshVisualRegressionHistorySkillFile(tenantId, projectId),
+      refreshLocatorStrategyWithHealingData(tenantId, projectId),
+      refreshFieldCatalogSkillFile(tenantId, projectId)
     ]);
     await withTenant(
       tenantId,
@@ -1803,8 +1810,241 @@ async function processGenerateScreenTestsJob(jobId, config) {
     });
   }
 }
+function captureStepText(step) {
+  if (typeof step === "string") return step;
+  if (!step || typeof step !== "object") return "";
+  const obj = step;
+  return [obj.action, obj.text, obj.label, obj.data, obj.expected].filter((v) => typeof v === "string" && v.trim().length > 0).join(" -> ");
+}
+function firstQuoted(text) {
+  return (text.match(/"([^"]+)"/)?.[1] ?? text.match(/'([^']+)'/)?.[1])?.trim();
+}
+function fallbackExecutionPlan(rawSteps) {
+  const plan = [];
+  for (const raw of rawSteps) {
+    const text = captureStepText(raw).trim();
+    if (!text) continue;
+    const stepNumber = plan.length + 1;
+    const url = text.match(/https?:\/\/[^\s"']+/)?.[0];
+    if (/^navigate\b|^go to\b|^open\b/i.test(text) && url) {
+      plan.push({ stepNumber, description: text, action: "navigate", url });
+      continue;
+    }
+    const urlContains = text.match(/verify url contains:\s*["']?([^"']+)["']?/i);
+    if (urlContains) {
+      plan.push({ stepNumber, description: text, action: "assert_url", value: urlContains[1].trim() });
+      continue;
+    }
+    const inputMatch = text.match(/(?:enter|type|fill)\s+["']([^"']+)["']\s+(?:in|into)\s+(.+?)(?:\s+field|\s+input|\s+box|\s+area)?(?:\s*→.*)?$/i);
+    if (inputMatch) {
+      const selector = inputMatch[2].replace(/^(the\s+)?/i, "").replace(/\s+(field|input|box|area)$/i, "").trim();
+      plan.push({ stepNumber, description: text, action: "fill", selectorStrategy: "label", selector, value: inputMatch[1].trim() });
+      continue;
+    }
+    const quoted = firstQuoted(text);
+    if (/^(click|tap|navigate menu|check|select)\b/i.test(text) && quoted) {
+      plan.push({ stepNumber, description: text, action: "click", selectorStrategy: "text", selector: quoted });
+      continue;
+    }
+    if (/^(verify|assert|expect)\b/i.test(text) && quoted) {
+      plan.push({ stepNumber, description: text, action: "assert_visible", selectorStrategy: "text", selector: quoted });
+      continue;
+    }
+    if (/execute script/i.test(text)) {
+      const script = text.split(":").slice(1).join(":").replace(/\s*→.*$/, "").trim();
+      if (script) plan.push({ stepNumber, description: text, action: "script", value: script });
+    }
+  }
+  return plan;
+}
+function screenNameFromUrl(url, fallback) {
+  try {
+    const u = new URL(url);
+    return u.pathname.replace(/\/+$/, "").split("/").filter(Boolean).pop() || u.hostname || fallback;
+  } catch {
+    return fallback;
+  }
+}
+async function processCaptureStepsJob(jobId, config) {
+  const { tenantId, projectId, testCaseId } = config;
+  const screenshotDir = config.screenshotDir || platformConfig.screenshotDir || process.env.SCREENSHOT_DIR || "/tmp/zeta-screenshots";
+  await fs.mkdir(screenshotDir, { recursive: true });
+  await withTenant(
+    tenantId,
+    (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "RUNNING", startedAt: /* @__PURE__ */ new Date(), progress: 2 } })
+  );
+  let browser;
+  try {
+    await updateJobProgress(tenantId, jobId, 5, "Loading test case");
+    const [tc, crawlCfg, tenantStorageCfg] = await Promise.all([
+      AppBrain.getTestCaseForExecution(tenantId, testCaseId),
+      AppBrain.getCrawlConfig(tenantId, projectId).catch(() => null),
+      getTenantStorageConfig(tenantId).catch(() => null)
+    ]);
+    if (!tc) throw new Error("Test case not found");
+    const rawSteps = Array.isArray(tc.steps) ? tc.steps : [];
+    const stepTexts = rawSteps.map(captureStepText).filter(Boolean);
+    if (stepTexts.length === 0) throw new Error("Test case has no steps to capture");
+    const baseUrl = config.baseUrl || tc?.screen?.project?.appUrl || crawlCfg?.appUrl || "";
+    let plan = await generateExecutionPlan(tenantId, { ...tc, steps: stepTexts }, baseUrl).catch(() => []);
+    plan = Array.isArray(plan) ? plan.filter(Boolean).map((step, idx) => ({ ...step, stepNumber: idx + 1 })) : [];
+    if (plan.length === 0) plan = fallbackExecutionPlan(rawSteps);
+    if (plan.length === 0) throw new Error("Could not generate execution plan from steps");
+    const { chromium } = await import("playwright");
+    browser = await chromium.launch({
+      headless: true,
+      ...process.env.PLAYWRIGHT_EXECUTABLE_PATH ? { executablePath: process.env.PLAYWRIGHT_EXECUTABLE_PATH } : {}
+    });
+    let parsedStorageState;
+    try {
+      parsedStorageState = config.storageState ? JSON.parse(config.storageState) : crawlCfg?.storageState ? JSON.parse(crawlCfg.storageState) : void 0;
+    } catch {
+      parsedStorageState = void 0;
+    }
+    const context = await browser.newContext({ viewport: { width: 1280, height: 800 }, ...parsedStorageState ? { storageState: parsedStorageState } : {} });
+    const page = await context.newPage();
+    const aiHealingEnabled = await isFeatureEnabled(tenantId, "AI_SELF_HEALING").catch(() => false);
+    const stepResults = [];
+    const screens = [];
+    const locators = [];
+    const seenLocatorExpressions = /* @__PURE__ */ new Set();
+    const seenScreenKeys = /* @__PURE__ */ new Set();
+    let lastStateKey = "";
+    for (let i = 0; i < plan.length; i++) {
+      const step = plan[i];
+      const description = step.description || stepTexts[i] || `Step ${i + 1}`;
+      const stepNumber = i + 1;
+      const result = { stepNumber, description, status: "FAILED" };
+      await withTenant(
+        tenantId,
+        (tx) => tx.platformJob.update({
+          where: { id: jobId },
+          data: {
+            progress: Math.min(90, Math.round(i / Math.max(plan.length, 1) * 90)),
+            result: { currentStep: { stepNumber, description } },
+            heartbeatAt: /* @__PURE__ */ new Date()
+          }
+        })
+      ).catch(() => {
+      });
+      try {
+        const heal = await executeExecutionStep(page, step, aiHealingEnabled ? { healCtx: { tenantId, projectId } } : {});
+        if (heal.healed) {
+          result.healed = true;
+          result.healTier = heal.healedVia;
+          result.healReasoning = heal.healReasoning;
+        }
+        await page.waitForLoadState("networkidle", { timeout: parseInt(process.env.TEST_NETWORK_IDLE_TIMEOUT_MS ?? "2000", 10) }).catch(() => {
+        });
+        result.status = result.healed ? "HEALED" : "PASSED";
+      } catch (err) {
+        result.status = step.optional ? "SKIPPED" : "FAILED";
+        result.error = String(err?.message ?? err).slice(0, 300);
+      }
+      const currentUrl = page.url();
+      result.url = currentUrl;
+      const ssFilename = `${testCaseId}-capture-step-${stepNumber}${result.status === "FAILED" ? "-err" : ""}.png`;
+      const ssPath = path.join(screenshotDir, ssFilename);
+      await page.screenshot({ path: ssPath, fullPage: false }).catch(() => {
+      });
+      result.screenshotPath = ssPath;
+      let cloudUrl;
+      if (tenantStorageCfg) {
+        const key = buildArtifactKey(tenantId, projectId, "screenshots", ssFilename, tenantStorageCfg.prefix);
+        cloudUrl = await uploadScreenshotToTenantStorage(tenantStorageCfg, ssPath, key).catch(() => void 0) ?? void 0;
+        if (cloudUrl) result.screenshotUrl = cloudUrl;
+      }
+      const elements = await domFallbackElements(page).catch(() => []);
+      result.elementsCount = elements.length;
+      const stateHash = hashContent(elements.map((el) => `${el.role}:${el.meaning}:${el.notes ?? ""}`).join("|"));
+      const screenKey = `${currentUrl || "about:blank"}:${stateHash}`;
+      const shouldPersistScreen = currentUrl && currentUrl !== "about:blank" && (i === 0 || screenKey !== lastStateKey || !seenScreenKeys.has(screenKey));
+      lastStateKey = screenKey;
+      if (shouldPersistScreen) {
+        const screenName = screenNameFromUrl(currentUrl, tc.title ?? `Step ${stepNumber}`);
+        const screen = await AppBrain.saveScreen(tenantId, projectId, screenName, currentUrl, jobId, i === 0 ? "ENTRY_POINT" : "LINK_FOLLOW").catch(() => null);
+        if (screen?.id) {
+          result.screenId = screen.id;
+          await AppBrain.saveScreenshot(tenantId, projectId, screen.id, ssPath, screen.version ?? 1, "DESKTOP", cloudUrl).catch(() => {
+          });
+          for (const el of elements) {
+            await AppBrain.saveElement(
+              tenantId,
+              projectId,
+              screen.id,
+              el.meaning,
+              el.role,
+              el.expectedData,
+              el.notes,
+              void 0,
+              true,
+              el.boundingRect,
+              el.ariaState,
+              el.parentLandmark
+            ).catch(() => {
+            });
+          }
+          for (const loc of elementsToPlaywrightLocators(elements)) {
+            if (seenLocatorExpressions.has(loc.expression)) continue;
+            seenLocatorExpressions.add(loc.expression);
+            locators.push(loc);
+          }
+          if (!seenScreenKeys.has(screenKey)) {
+            screens.push({ stepNumber, screenId: screen.id, url: currentUrl, screenshotPath: ssPath, screenshotUrl: cloudUrl, isNewScreen: true });
+            seenScreenKeys.add(screenKey);
+          }
+        }
+      }
+      stepResults.push(result);
+      if (result.status === "FAILED") break;
+    }
+    if (locators.length > 0) {
+      await AppBrain.saveLocatorsJson(tenantId, testCaseId, locators).catch(() => {
+      });
+    }
+    const firstScreen = screens[0];
+    await withTenant(
+      tenantId,
+      (tx) => tx.testCase.update({
+        where: { id: testCaseId },
+        data: {
+          ...firstScreen?.screenId ? { screenId: firstScreen.screenId } : {},
+          captureStepScreenshots: stepResults
+        }
+      })
+    ).catch(() => {
+    });
+    const finalResult = {
+      stepResults,
+      screens,
+      locators,
+      totalSteps: plan.length,
+      capturedScreens: screens.length
+    };
+    await withTenant(
+      tenantId,
+      (tx) => tx.platformJob.update({
+        where: { id: jobId },
+        data: { status: "DONE", progress: 100, result: finalResult, finishedAt: /* @__PURE__ */ new Date(), heartbeatAt: /* @__PURE__ */ new Date() }
+      })
+    );
+  } catch (err) {
+    const error = String(err?.message ?? err);
+    await withTenant(
+      tenantId,
+      (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", error, finishedAt: /* @__PURE__ */ new Date(), heartbeatAt: /* @__PURE__ */ new Date() } })
+    ).catch(() => {
+    });
+    logger.error({ jobId, testCaseId, err: error }, "processCaptureStepsJob failed");
+    throw err;
+  } finally {
+    await browser?.close?.().catch(() => {
+    });
+  }
+}
 jobDispatchTable = {
   CRAWL: processJob,
+  CAPTURE_STEPS: processCaptureStepsJob,
   DISCOVER: processDiscoverJob,
   MONITOR: processMonitorJob,
   NATIVE_CRAWL: processNativeCrawlJob,
@@ -1833,7 +2073,8 @@ async function processRepoScanJob(jobId, config) {
       detectFramework,
       extractTestIds,
       extractStyleProfile,
-      buildPageObjectInventory
+      buildPageObjectInventory,
+      resolveFixtureAliasesFromImports
     } = await import("@detiq/agents");
     const conn = await withTenant(
       tenantId,
@@ -1848,6 +2089,9 @@ async function processRepoScanJob(jobId, config) {
     );
     await updateJobProgress(tenantId, jobId, 15, "Fetching repo tree");
     const treeResult = await fetchRepoTree(token, conn.repoFullName, "HEAD");
+    if (treeResult.truncated) {
+      console.warn(`[repo-scan] ${conn.repoFullName}: tree truncated (>100k blobs) \u2014 detection may be incomplete`);
+    }
     const frameworks = detectFramework(treeResult.paths);
     const primary = frameworks[0];
     await updateJobProgress(tenantId, jobId, 30, `Detected framework: ${primary?.framework ?? "unknown"}`);
@@ -1865,6 +2109,22 @@ async function processRepoScanJob(jobId, config) {
       return p.endsWith(".spec.ts") || p.endsWith(".spec.js") || p.endsWith(".test.ts") || p.endsWith(".test.js") || p.endsWith("_test.py") || p.includes("/test_");
     }).slice(0, 500);
     await updateJobProgress(tenantId, jobId, 40, `Found ${testPaths.length} test files`);
+    if (testPaths.length === 0) {
+      logger.warn({ jobId, repoConnectionId, framework }, "[repo-scan] no test files matched glob \u2014 scan complete with 0 results");
+      await withTenant(
+        tenantId,
+        (tx) => tx.gitHubRepoConnection.update({
+          where: { id: repoConnectionId },
+          data: { scanStatus: "DONE", testFileCount: 0, extractedIdCount: 0, lastScannedAt: /* @__PURE__ */ new Date(), lastSyncAt: /* @__PURE__ */ new Date() }
+        })
+      );
+      await updateJobProgress(tenantId, jobId, 100, "Done: 0 test files found");
+      await withTenant(
+        tenantId,
+        (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "DONE", finishedAt: /* @__PURE__ */ new Date(), result: { testFileCount: 0, extractedIdCount: 0 } } })
+      );
+      return;
+    }
     const allExtracted = [];
     const styleSampleFiles = [];
     let processed = 0;
@@ -1872,7 +2132,13 @@ async function processRepoScanJob(jobId, config) {
     for (const filePath of testPaths) {
       const content = await fetchFileContent(token, conn.repoFullName, filePath);
       if (!content) continue;
-      const extracted = extractTestIds(content, filePath, framework);
+      const extraAliases = await resolveFixtureAliasesFromImports(
+        content,
+        token,
+        conn.repoFullName,
+        filePath
+      ).catch(() => /* @__PURE__ */ new Set());
+      const extracted = extractTestIds(content, filePath, framework, extraAliases);
       allExtracted.push(...extracted);
       await withTenant(
         tenantId,
@@ -1895,6 +2161,214 @@ async function processRepoScanJob(jobId, config) {
         await updateJobProgress(tenantId, jobId, 40 + Math.floor(processed / testPaths.length * 40), `Processed ${processed}/${testPaths.length} files`);
       }
     }
+    const configPaths = {};
+    try {
+      const pathSet = new Set(treeResult.paths);
+      const lang = framework.includes("java") ? "java" : framework.includes("py") || framework === "pytest" ? "python" : "js";
+      const configFilesToFetch = [];
+      const tryAdd = (candidates, type) => {
+        for (const c of candidates) {
+          if (pathSet.has(c)) {
+            configFilesToFetch.push({ path: c, type, dirPrefix: "" });
+            break;
+          }
+        }
+      };
+      const scanTree = (pattern, type) => {
+        if (configFilesToFetch.some((f) => f.type === type)) return;
+        const all = treeResult.paths.filter((p) => pattern.test(p)).sort((a, b) => a.split("/").length - b.split("/").length);
+        if (all[0]) {
+          const lastSlash = all[0].lastIndexOf("/");
+          const dirPrefix = lastSlash > 0 ? all[0].slice(0, lastSlash + 1) : "";
+          configFilesToFetch.push({ path: all[0], type, dirPrefix });
+        }
+      };
+      if (lang === "js") {
+        if (framework.startsWith("playwright")) {
+          tryAdd(["playwright.config.ts", "playwright.config.js"], "playwright");
+          scanTree(/(?:^|\/)playwright\.config\.(ts|js|mts)$/, "playwright");
+        }
+        if (framework.startsWith("cypress")) {
+          tryAdd(["cypress.config.ts", "cypress.config.js"], "cypress");
+          scanTree(/(?:^|\/)cypress\.config\.(ts|js)$/, "cypress");
+        }
+        if (framework === "jest-ts" || framework === "jest-js") {
+          tryAdd(["jest.config.ts", "jest.config.js", "jest.config.mjs"], "jest");
+          scanTree(/(?:^|\/)jest\.config\.(ts|js|mjs|cjs)$/, "jest");
+          tryAdd(["package.json"], "package-json");
+        }
+        if (framework === "webdriverio") {
+          tryAdd(["wdio.conf.ts", "wdio.conf.js", "wdio.config.ts", "wdio.config.js"], "wdio");
+          scanTree(/(?:^|\/)wdio\.(?:conf|config)\.(ts|js|mjs)$/, "wdio");
+        }
+        if (framework === "cucumber" || framework === "gherkin") {
+          tryAdd([".cucumber.js", ".cucumber.cjs", "cucumber.json", ".cucumber.yaml", ".cucumber.yml"], "cucumber");
+          tryAdd(["package.json"], "package-json");
+        }
+        tryAdd(["tsconfig.json"], "tsconfig");
+        scanTree(/(?:^|\/)tsconfig(?:\.base)?\.json$/, "tsconfig");
+      } else if (lang === "python") {
+        tryAdd(["pytest.ini"], "pytest-ini");
+        scanTree(/(?:^|\/)pytest\.ini$/, "pytest-ini");
+        tryAdd(["pyproject.toml"], "pyproject-toml");
+        scanTree(/(?:^|\/)pyproject\.toml$/, "pyproject-toml");
+        tryAdd(["setup.cfg"], "setup-cfg");
+      } else if (lang === "java") {
+        tryAdd(["pom.xml"], "pom-xml");
+        scanTree(/(?:^|\/)pom\.xml$/, "pom-xml");
+        tryAdd(["build.gradle.kts", "build.gradle"], "build-gradle");
+        scanTree(/(?:^|\/)build\.gradle(?:\.kts)?$/, "build-gradle");
+        tryAdd(["testng.xml", "src/test/resources/testng.xml"], "testng-xml");
+      }
+      const normDir = (s) => s.replace(/^\.\//, "").replace(/\/$/, "");
+      const globRoot = (pattern) => {
+        const root = pattern.replace(/\/\*\*.*$/, "").replace(/\/\*[^*].*$/, "");
+        return root && !root.startsWith("*") && !root.startsWith("{") ? normDir(root) : null;
+      };
+      const fetched = await Promise.all(
+        configFilesToFetch.map(async ({ path: cfgPath, type, dirPrefix }) => {
+          const content = await fetchFileContent(token, conn.repoFullName, cfgPath).catch(() => null);
+          if (!content) return null;
+          const r = {};
+          if (type === "playwright") {
+            const m = content.match(/\btestDir\s*:\s*['"`]([^'"`]+)['"`]/);
+            if (m) r.testDir = normDir(m[1]);
+            const bm = content.match(/\bbaseURL\s*:\s*['"`]([^'"`]+)['"`]/);
+            if (bm) r.baseURL = bm[1];
+          } else if (type === "cypress") {
+            const sm = content.match(/\bspecPattern\s*:\s*['"`]([^'"`]+)['"`]/);
+            if (sm) {
+              const gr = globRoot(sm[1]);
+              if (gr) r.testDir = gr;
+            }
+            const fm = content.match(/\bfixturesFolder\s*:\s*['"`]([^'"`]+)['"`]/);
+            if (fm) r.fixturesFolder = normDir(fm[1]);
+            const bm = content.match(/\bbaseUrl\s*:\s*['"`]([^'"`]+)['"`]/);
+            if (bm) r.baseURL = bm[1];
+          } else if (type === "jest") {
+            const rm = content.match(/\broots\s*:\s*\[([^\]]+)\]/);
+            if (rm) {
+              const fr = rm[1].match(/['"`]([^'"`]+)['"`]/)?.[1];
+              if (fr) r.testDir = fr.replace(/<rootDir>\/?/, "").replace(/\/$/, "") || "src";
+            }
+            if (!r.testDir) {
+              const tm = content.match(/\btestMatch\s*:\s*\[([^\]]+)\]/);
+              if (tm) {
+                const fp = tm[1].match(/['"`]([^'"`]+)['"`]/)?.[1];
+                if (fp) {
+                  const gr = globRoot(fp.replace(/<rootDir>\/?/, ""));
+                  if (gr) r.testDir = gr;
+                }
+              }
+            }
+            const mm = content.match(/\bmoduleNameMapper\s*:\s*\{([^}]+)\}/s);
+            if (mm) {
+              const map = {};
+              for (const [, k, v] of mm[1].matchAll(/['"`]([^'"`]+)['"`]\s*:\s*['"`]([^'"`]+)['"`]/g)) map[k] = v;
+              if (Object.keys(map).length) r.jestModuleNameMapper = map;
+            }
+          } else if (type === "package-json") {
+            try {
+              const p = JSON.parse(content);
+              const j = p?.jest ?? {};
+              if (Array.isArray(j.roots) && j.roots[0]) r.testDir = String(j.roots[0]).replace(/<rootDir>\/?/, "").replace(/\/$/, "") || "src";
+              if (!r.testDir && Array.isArray(j.testMatch) && j.testMatch[0]) {
+                const gr = globRoot(String(j.testMatch[0]).replace(/<rootDir>\/?/, ""));
+                if (gr) r.testDir = gr;
+              }
+              if (j.moduleNameMapper && typeof j.moduleNameMapper === "object") r.jestModuleNameMapper = j.moduleNameMapper;
+            } catch {
+            }
+          } else if (type === "tsconfig") {
+            try {
+              const stripped = content.replace(/\/\/[^\n]*/g, "").replace(/,(\s*[}\]])/g, "$1");
+              const p = JSON.parse(stripped);
+              const opts = p?.compilerOptions ?? {};
+              if (opts.paths && typeof opts.paths === "object") r.tsconfigPaths = opts.paths;
+              if (typeof opts.baseUrl === "string") r.tsconfigBaseUrl = opts.baseUrl;
+            } catch {
+            }
+          } else if (type === "pytest-ini") {
+            const m = content.match(/^\s*testpaths\s*=\s*(.+)$/m);
+            if (m) {
+              const first = m[1].trim().split(/\s+/)[0];
+              if (first) r.testDir = normDir(first);
+            }
+          } else if (type === "pyproject-toml") {
+            const m = content.match(/testpaths\s*=\s*\[([^\]]+)\]/);
+            if (m) {
+              const first = m[1].match(/['"]([^'"]+)['"]/)?.[1];
+              if (first) r.testDir = normDir(first);
+            }
+            if (!r.testDir) {
+              const m2 = content.match(/testpaths\s*=\s*['"]([^'"]+)['"]/);
+              if (m2) r.testDir = normDir(m2[1]);
+            }
+          } else if (type === "setup-cfg") {
+            const m = content.match(/^\s*testpaths\s*=\s*(.+)$/m);
+            if (m) {
+              const first = m[1].trim().split(/[\s,]+/)[0];
+              if (first) r.testDir = normDir(first);
+            }
+          } else if (type === "pom-xml") {
+            const m = content.match(/<testSourceDirectory>\s*([^<]+)\s*<\/testSourceDirectory>/);
+            if (m) r.javaTestSourceDir = normDir(m[1].trim());
+          } else if (type === "build-gradle") {
+            const m = content.match(/srcDir[s]?\s*[=([]?\s*['"`]([^'"`]+)['"`]/) ?? content.match(/srcDir\s*\(\s*['"`]([^'"`]+)['"`]\s*\)/);
+            if (m) r.javaTestSourceDir = normDir(m[1]);
+          } else if (type === "wdio") {
+            const sm = content.match(/\bspecs\s*:\s*\[([^\]]+)\]/);
+            if (sm) {
+              const fp = sm[1].match(/['"`]([^'"`]+)['"`]/)?.[1];
+              if (fp) {
+                const gr = globRoot(fp);
+                if (gr) r.testDir = gr;
+              }
+            }
+            const bm = content.match(/\bbaseUrl\s*:\s*['"`]([^'"`]+)['"`]/);
+            if (bm) r.baseURL = bm[1];
+          } else if (type === "cucumber") {
+            const pm = content.match(/\bpaths\s*:\s*\[([^\]]+)\]/);
+            if (pm) {
+              const fp = pm[1].match(/['"`]([^'"`]+)['"`]/)?.[1];
+              if (fp) {
+                const gr = globRoot(fp);
+                if (gr) r.testDir = gr;
+              }
+            }
+            if (!r.testDir) {
+              try {
+                const p = JSON.parse(content);
+                const paths = p?.default?.paths ?? p?.paths;
+                if (Array.isArray(paths) && paths[0]) {
+                  const gr = globRoot(String(paths[0]));
+                  if (gr) r.testDir = gr;
+                }
+              } catch {
+              }
+            }
+          }
+          if (dirPrefix && r.testDir && !r.testDir.startsWith(dirPrefix)) {
+            r.testDir = dirPrefix + r.testDir;
+          }
+          if (dirPrefix && r.javaTestSourceDir && !r.javaTestSourceDir.startsWith(dirPrefix)) {
+            r.javaTestSourceDir = dirPrefix + r.javaTestSourceDir;
+          }
+          return r;
+        })
+      );
+      for (const parsed of fetched) {
+        if (!parsed) continue;
+        if (parsed.testDir && !configPaths.testDir) configPaths.testDir = parsed.testDir;
+        if (parsed.fixturesFolder && !configPaths.fixturesFolder) configPaths.fixturesFolder = parsed.fixturesFolder;
+        if (parsed.baseURL && !configPaths.baseURL) configPaths.baseURL = parsed.baseURL;
+        if (parsed.tsconfigPaths && !configPaths.tsconfigPaths) configPaths.tsconfigPaths = parsed.tsconfigPaths;
+        if (parsed.tsconfigBaseUrl && !configPaths.tsconfigBaseUrl) configPaths.tsconfigBaseUrl = parsed.tsconfigBaseUrl;
+        if (parsed.javaTestSourceDir && !configPaths.javaTestSourceDir) configPaths.javaTestSourceDir = parsed.javaTestSourceDir;
+        if (parsed.jestModuleNameMapper && !configPaths.jestModuleNameMapper) configPaths.jestModuleNameMapper = parsed.jestModuleNameMapper;
+      }
+    } catch {
+    }
     let styleProfile = conn.styleProfile ?? null;
     if (styleSampleFiles.length > 0) {
       try {
@@ -1902,11 +2376,23 @@ async function processRepoScanJob(jobId, config) {
         const testPathSet = new Set(testPaths);
         const sourceFilePaths = language === "typescript" || language === "javascript" ? treeResult.paths.filter((p) => (p.endsWith(".ts") || p.endsWith(".js")) && !testPathSet.has(p) && !p.endsWith(".d.ts") && !p.includes("node_modules/") && !p.includes("dist/") && !p.includes(".config.")).slice(0, 60) : [];
         const allPathsForProfile = [...testPaths, ...mainSourcePaths, ...sourceFilePaths].slice(0, 200);
-        const freshProfile = await extractStyleProfile(styleSampleFiles, framework, language, tenantId, projectId, jobId, allPathsForProfile);
+        const freshProfile = await extractStyleProfile(styleSampleFiles, framework, language, tenantId, projectId, jobId, allPathsForProfile, treeResult.paths);
         if (freshProfile) styleProfile = freshProfile;
       } catch {
       }
     }
+    if (styleProfile && Object.keys(configPaths).length > 0) {
+      if (configPaths.testDir) styleProfile.configTestDir = configPaths.testDir;
+      if (configPaths.javaTestSourceDir) {
+        styleProfile.configTestDir = configPaths.javaTestSourceDir;
+        if (!styleProfile.testLocation) styleProfile.testLocation = configPaths.javaTestSourceDir;
+      }
+      if (configPaths.fixturesFolder) styleProfile.configFixturesFolder = configPaths.fixturesFolder;
+      if (configPaths.baseURL) styleProfile.configBaseURL = configPaths.baseURL;
+      if (configPaths.tsconfigPaths) styleProfile.tsconfigPaths = configPaths.tsconfigPaths;
+      if (configPaths.tsconfigBaseUrl) styleProfile.tsconfigBaseUrl = configPaths.tsconfigBaseUrl;
+      if (configPaths.jestModuleNameMapper) styleProfile.jestModuleNameMapper = configPaths.jestModuleNameMapper;
+    }
     let pageObjectInventory = conn.pageObjectInventory ?? [];
     try {
       const freshInventory = await buildPageObjectInventory(token, conn.repoFullName, treeResult.paths, framework);
@@ -1985,7 +2471,9 @@ async function processRepoScanJob(jobId, config) {
         styleProfile: resolvedStyleProfile,
         pageObjectInventory,
         testPaths,
-        extractedTestCount: allExtracted.length
+        extractedTestCount: allExtracted.length,
+        treeWasTruncated: treeResult.truncated,
+        extractedTests: allExtracted
       });
     } catch (e) {
       logger.warn({ jobId, repoConnectionId, err: e?.message }, "[repo-scan] repo skill files generation failed (non-fatal)");
@@ -1997,7 +2485,8 @@ async function processRepoScanJob(jobId, config) {
     );
     aggregateJobTokens(tenantId, jobId, projectId, jobStart).catch(console.error);
   } catch (err) {
-    const msg = String(err?.message ?? err);
+    const cause = err?.cause?.message ?? (typeof err?.cause === "string" ? err.cause : "");
+    const msg = `${String(err?.message ?? err)}${cause ? ` \u2014 ${cause}` : ""}`;
     logger.error({ jobId, repoConnectionId, err: msg }, "[repo-scan] failed");
     await withTenant(
       tenantId,