@detiq/crawler 1.1.10 → 1.1.11

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@detiq/crawler",
-  "version": "1.1.10",
+  "version": "1.1.11",
   "private": false,
   "publishConfig": {
     "access": "public",

package/src/job-queue.ts

@@ -2333,6 +2333,126 @@ async function processRepoScanJob(jobId: string, config: any) {
       }
     }
 
+    // Fetch and parse framework config files (playwright.config.ts, cypress.config.ts, tsconfig.json)
+    // Fetch and parse framework config files for all supported frameworks.
+    // Pure regex/JSON — no LLM. Results override LLM inference for path detection.
+    // Covers: playwright, cypress, jest, pytest, junit/maven/gradle, selenium-java/py,
+    //         webdriverio, cucumber — all frameworks in DetectedFramework.
+    const configPaths: {
+      testDir?: string; fixturesFolder?: string; baseURL?: string;
+      tsconfigPaths?: Record<string, string[]>; tsconfigBaseUrl?: string;
+      javaTestSourceDir?: string; jestModuleNameMapper?: Record<string, string>;
+    } = {};
+    try {
+      const pathSet = new Set(treeResult.paths as string[]);
+      const lang = framework.includes('java') ? 'java'
+        : (framework.includes('py') || framework === 'pytest') ? 'python' : 'js';
+
+      const configFilesToFetch: Array<{ path: string; type: string }> = [];
+      const tryAdd = (candidates: string[], type: string) => {
+        for (const c of candidates) { if (pathSet.has(c)) { configFilesToFetch.push({ path: c, type }); break; } }
+      };
+
+      if (lang === 'js') {
+        if (framework.startsWith('playwright')) tryAdd(['playwright.config.ts', 'playwright.config.js'], 'playwright');
+        if (framework.startsWith('cypress')) tryAdd(['cypress.config.ts', 'cypress.config.js'], 'cypress');
+        if (framework === 'jest-ts' || framework === 'jest-js') {
+          tryAdd(['jest.config.ts', 'jest.config.js', 'jest.config.mjs'], 'jest');
+          tryAdd(['package.json'], 'package-json');
+        }
+        if (framework === 'webdriverio') tryAdd(['wdio.conf.ts', 'wdio.conf.js', 'wdio.config.ts', 'wdio.config.js'], 'wdio');
+        if (framework === 'cucumber' || framework === 'gherkin') {
+          tryAdd(['.cucumber.js', '.cucumber.cjs', 'cucumber.json', '.cucumber.yaml', '.cucumber.yml'], 'cucumber');
+          tryAdd(['package.json'], 'package-json');
+        }
+        tryAdd(['tsconfig.json'], 'tsconfig');
+      } else if (lang === 'python') {
+        tryAdd(['pytest.ini'], 'pytest-ini');
+        tryAdd(['pyproject.toml'], 'pyproject-toml');
+        tryAdd(['setup.cfg'], 'setup-cfg');
+      } else if (lang === 'java') {
+        tryAdd(['pom.xml'], 'pom-xml');
+        tryAdd(['build.gradle.kts', 'build.gradle'], 'build-gradle');
+        tryAdd(['testng.xml', 'src/test/resources/testng.xml'], 'testng-xml');
+      }
+
+      // Inline parsers — mirror parseFrameworkConfig from @detiq/agents (kept inline to avoid build dep)
+      const normDir = (s: string) => s.replace(/^\.\//, '').replace(/\/$/, '');
+      const globRoot = (pattern: string): string | null => {
+        const root = pattern.replace(/\/\*\*.*$/, '').replace(/\/\*[^*].*$/, '');
+        return (root && !root.startsWith('*') && !root.startsWith('{')) ? normDir(root) : null;
+      };
+
+      const fetched = await Promise.all(
+        configFilesToFetch.map(async ({ path: cfgPath, type }) => {
+          const content = await fetchFileContent(token, conn.repoFullName, cfgPath).catch(() => null);
+          if (!content) return null;
+          const r: typeof configPaths = {};
+
+          if (type === 'playwright') {
+            const m = content.match(/\btestDir\s*:\s*['"`]([^'"`]+)['"`]/);
+            if (m) r.testDir = normDir(m[1]);
+            const bm = content.match(/\bbaseURL\s*:\s*['"`]([^'"`]+)['"`]/);
+            if (bm) r.baseURL = bm[1];
+          } else if (type === 'cypress') {
+            const sm = content.match(/\bspecPattern\s*:\s*['"`]([^'"`]+)['"`]/);
+            if (sm) { const gr = globRoot(sm[1]); if (gr) r.testDir = gr; }
+            const fm = content.match(/\bfixturesFolder\s*:\s*['"`]([^'"`]+)['"`]/);
+            if (fm) r.fixturesFolder = normDir(fm[1]);
+            const bm = content.match(/\bbaseUrl\s*:\s*['"`]([^'"`]+)['"`]/);
+            if (bm) r.baseURL = bm[1];
+          } else if (type === 'jest') {
+            const rm = content.match(/\broots\s*:\s*\[([^\]]+)\]/);
+            if (rm) { const fr = rm[1].match(/['"`]([^'"`]+)['"`]/)?.[1]; if (fr) r.testDir = fr.replace(/<rootDir>\/?/, '').replace(/\/$/, '') || 'src'; }
+            if (!r.testDir) { const tm = content.match(/\btestMatch\s*:\s*\[([^\]]+)\]/); if (tm) { const fp = tm[1].match(/['"`]([^'"`]+)['"`]/)?.[1]; if (fp) { const gr = globRoot(fp.replace(/<rootDir>\/?/, '')); if (gr) r.testDir = gr; } } }
+            const mm = content.match(/\bmoduleNameMapper\s*:\s*\{([^}]+)\}/s);
+            if (mm) { const map: Record<string, string> = {}; for (const [, k, v] of mm[1].matchAll(/['"`]([^'"`]+)['"`]\s*:\s*['"`]([^'"`]+)['"`]/g)) map[k] = v; if (Object.keys(map).length) r.jestModuleNameMapper = map; }
+          } else if (type === 'package-json') {
+            try { const p = JSON.parse(content) as any; const j = p?.jest ?? {}; if (Array.isArray(j.roots) && j.roots[0]) r.testDir = String(j.roots[0]).replace(/<rootDir>\/?/, '').replace(/\/$/, '') || 'src'; if (!r.testDir && Array.isArray(j.testMatch) && j.testMatch[0]) { const gr = globRoot(String(j.testMatch[0]).replace(/<rootDir>\/?/, '')); if (gr) r.testDir = gr; } if (j.moduleNameMapper && typeof j.moduleNameMapper === 'object') r.jestModuleNameMapper = j.moduleNameMapper; } catch { /* ignore */ }
+          } else if (type === 'tsconfig') {
+            try { const stripped = content.replace(/\/\/[^\n]*/g, '').replace(/,(\s*[}\]])/g, '$1'); const p = JSON.parse(stripped) as any; const opts = p?.compilerOptions ?? {}; if (opts.paths && typeof opts.paths === 'object') r.tsconfigPaths = opts.paths; if (typeof opts.baseUrl === 'string') r.tsconfigBaseUrl = opts.baseUrl; } catch { /* ignore */ }
+          } else if (type === 'pytest-ini') {
+            const m = content.match(/^\s*testpaths\s*=\s*(.+)$/m);
+            if (m) { const first = m[1].trim().split(/\s+/)[0]; if (first) r.testDir = normDir(first); }
+          } else if (type === 'pyproject-toml') {
+            const m = content.match(/testpaths\s*=\s*\[([^\]]+)\]/);
+            if (m) { const first = m[1].match(/['"]([^'"]+)['"]/)?.[1]; if (first) r.testDir = normDir(first); }
+            if (!r.testDir) { const m2 = content.match(/testpaths\s*=\s*['"]([^'"]+)['"]/); if (m2) r.testDir = normDir(m2[1]); }
+          } else if (type === 'setup-cfg') {
+            const m = content.match(/^\s*testpaths\s*=\s*(.+)$/m);
+            if (m) { const first = m[1].trim().split(/[\s,]+/)[0]; if (first) r.testDir = normDir(first); }
+          } else if (type === 'pom-xml') {
+            const m = content.match(/<testSourceDirectory>\s*([^<]+)\s*<\/testSourceDirectory>/);
+            if (m) r.javaTestSourceDir = normDir(m[1].trim());
+          } else if (type === 'build-gradle') {
+            const m = content.match(/srcDir[s]?\s*[=([]?\s*['"`]([^'"`]+)['"`]/) ?? content.match(/srcDir\s*\(\s*['"`]([^'"`]+)['"`]\s*\)/);
+            if (m) r.javaTestSourceDir = normDir(m[1]);
+          } else if (type === 'wdio') {
+            const sm = content.match(/\bspecs\s*:\s*\[([^\]]+)\]/);
+            if (sm) { const fp = sm[1].match(/['"`]([^'"`]+)['"`]/)?.[1]; if (fp) { const gr = globRoot(fp); if (gr) r.testDir = gr; } }
+            const bm = content.match(/\bbaseUrl\s*:\s*['"`]([^'"`]+)['"`]/);
+            if (bm) r.baseURL = bm[1];
+          } else if (type === 'cucumber') {
+            const pm = content.match(/\bpaths\s*:\s*\[([^\]]+)\]/);
+            if (pm) { const fp = pm[1].match(/['"`]([^'"`]+)['"`]/)?.[1]; if (fp) { const gr = globRoot(fp); if (gr) r.testDir = gr; } }
+            if (!r.testDir) { try { const p = JSON.parse(content) as any; const paths = p?.default?.paths ?? p?.paths; if (Array.isArray(paths) && paths[0]) { const gr = globRoot(String(paths[0])); if (gr) r.testDir = gr; } } catch { /* not JSON */ } }
+          }
+          return r;
+        })
+      );
+
+      for (const parsed of fetched) {
+        if (!parsed) continue;
+        if (parsed.testDir && !configPaths.testDir) configPaths.testDir = parsed.testDir;
+        if (parsed.fixturesFolder && !configPaths.fixturesFolder) configPaths.fixturesFolder = parsed.fixturesFolder;
+        if (parsed.baseURL && !configPaths.baseURL) configPaths.baseURL = parsed.baseURL;
+        if (parsed.tsconfigPaths && !configPaths.tsconfigPaths) configPaths.tsconfigPaths = parsed.tsconfigPaths;
+        if (parsed.tsconfigBaseUrl && !configPaths.tsconfigBaseUrl) configPaths.tsconfigBaseUrl = parsed.tsconfigBaseUrl;
+        if (parsed.javaTestSourceDir && !configPaths.javaTestSourceDir) configPaths.javaTestSourceDir = parsed.javaTestSourceDir;
+        if (parsed.jestModuleNameMapper && !configPaths.jestModuleNameMapper) configPaths.jestModuleNameMapper = parsed.jestModuleNameMapper;
+      }
+    } catch { /* non-fatal */ }
+
     // Extract style profile for aligned generation
     // For Java, include main source paths so LLM can detect page object location in src/main/java/
     // Fall back to existing styleProfile from DB if extraction fails or returns null.
@@ -2362,6 +2482,22 @@ async function processRepoScanJob(jobId: string, config: any) {
       } catch { /* non-fatal */ }
     }
 
+    // Merge config file ground-truth into styleProfile (Priority 0 — highest, no LLM inference needed).
+    // Java: javaTestSourceDir also sets testLocation (equivalent field for Java).
+    if (styleProfile && Object.keys(configPaths).length > 0) {
+      if (configPaths.testDir) styleProfile.configTestDir = configPaths.testDir;
+      if (configPaths.javaTestSourceDir) {
+        styleProfile.configTestDir = configPaths.javaTestSourceDir;
+        // Also set testLocation if not already present from LLM scan
+        if (!styleProfile.testLocation) styleProfile.testLocation = configPaths.javaTestSourceDir;
+      }
+      if (configPaths.fixturesFolder) styleProfile.configFixturesFolder = configPaths.fixturesFolder;
+      if (configPaths.baseURL) styleProfile.configBaseURL = configPaths.baseURL;
+      if (configPaths.tsconfigPaths) styleProfile.tsconfigPaths = configPaths.tsconfigPaths;
+      if (configPaths.tsconfigBaseUrl) styleProfile.tsconfigBaseUrl = configPaths.tsconfigBaseUrl;
+      if (configPaths.jestModuleNameMapper) styleProfile.jestModuleNameMapper = configPaths.jestModuleNameMapper;
+    }
+
     // Extract page object inventory — finds POM/page classes in non-test files
     // Fall back to existing inventory from DB if fetch fails.
     let pageObjectInventory: any[] = (conn.pageObjectInventory as any[]) ?? [];