@detiq/crawler 1.1.9 → 1.1.11

package/dist/websocket-detector.js

@@ -1,72 +1,60 @@
-/**
- * WebSocket detection — intercepts WS connections opened during page crawl.
- * Records connection URLs, message schemas, and message counts.
- * This lets ZeTa discover real-time API contracts (chat, notifications, presence)
- * that would be invisible to HTTP-only crawlers.
- */
 function inferProtocol(sample) {
-    try {
-        const obj = JSON.parse(sample);
-        if (obj?.jsonrpc)
-            return 'json-rpc';
-        if (obj?.type === 'connection_init' || obj?.type === 'subscribe')
-            return 'graphql-ws';
-        if (typeof obj?.sid === 'string' || sample.startsWith('0{'))
-            return 'socket.io';
-        return 'json';
-    }
-    catch {
-        return 'raw';
-    }
+  try {
+    const obj = JSON.parse(sample);
+    if (obj?.jsonrpc) return "json-rpc";
+    if (obj?.type === "connection_init" || obj?.type === "subscribe") return "graphql-ws";
+    if (typeof obj?.sid === "string" || sample.startsWith("0{")) return "socket.io";
+    return "json";
+  } catch {
+    return "raw";
+  }
 }
-export function attachWebSocketDetector(cdpSession) {
-    const connections = new Map();
-    const onCreated = (evt) => {
-        const { requestId, url, timestamp } = evt ?? {};
-        if (!requestId || !url)
-            return;
-        connections.set(requestId, {
-            url,
-            framesSent: 0,
-            framesReceived: 0,
-            samples: [],
-            protocol: 'unknown',
-            openedAt: new Date(timestamp * 1000).toISOString(),
-        });
-    };
-    const onFrame = (evt, direction) => {
-        const { requestId, response } = evt ?? {};
-        if (!requestId)
-            return;
-        const conn = connections.get(requestId);
-        if (!conn)
-            return;
-        if (direction === 'sent')
-            conn.framesSent++;
-        else
-            conn.framesReceived++;
-        if (conn.samples.length < 10 && response?.payloadData) {
-            const payload = String(response.payloadData).slice(0, 500);
-            conn.samples.push({ direction, payload, ts: new Date().toISOString() });
-            if (conn.protocol === 'unknown')
-                conn.protocol = inferProtocol(payload);
-        }
-    };
-    const onSent = (evt) => onFrame(evt, 'sent');
-    const onReceived = (evt) => onFrame(evt, 'received');
-    if (cdpSession?.on) {
-        cdpSession.on('Network.webSocketCreated', onCreated);
-        cdpSession.on('Network.webSocketFrameSent', onSent);
-        cdpSession.on('Network.webSocketFrameReceived', onReceived);
+function attachWebSocketDetector(cdpSession) {
+  const connections = /* @__PURE__ */ new Map();
+  const onCreated = (evt) => {
+    const { requestId, url, timestamp } = evt ?? {};
+    if (!requestId || !url) return;
+    connections.set(requestId, {
+      url,
+      framesSent: 0,
+      framesReceived: 0,
+      samples: [],
+      protocol: "unknown",
+      openedAt: new Date(timestamp * 1e3).toISOString()
+    });
+  };
+  const onFrame = (evt, direction) => {
+    const { requestId, response } = evt ?? {};
+    if (!requestId) return;
+    const conn = connections.get(requestId);
+    if (!conn) return;
+    if (direction === "sent") conn.framesSent++;
+    else conn.framesReceived++;
+    if (conn.samples.length < 10 && response?.payloadData) {
+      const payload = String(response.payloadData).slice(0, 500);
+      conn.samples.push({ direction, payload, ts: (/* @__PURE__ */ new Date()).toISOString() });
+      if (conn.protocol === "unknown") conn.protocol = inferProtocol(payload);
+    }
+  };
+  const onSent = (evt) => onFrame(evt, "sent");
+  const onReceived = (evt) => onFrame(evt, "received");
+  if (cdpSession?.on) {
+    cdpSession.on("Network.webSocketCreated", onCreated);
+    cdpSession.on("Network.webSocketFrameSent", onSent);
+    cdpSession.on("Network.webSocketFrameReceived", onReceived);
+  }
+  return {
+    getConnections: () => [...connections.values()],
+    detach() {
+      if (cdpSession?.off) {
+        cdpSession.off("Network.webSocketCreated", onCreated);
+        cdpSession.off("Network.webSocketFrameSent", onSent);
+        cdpSession.off("Network.webSocketFrameReceived", onReceived);
+      }
     }
-    return {
-        getConnections: () => [...connections.values()],
-        detach() {
-            if (cdpSession?.off) {
-                cdpSession.off('Network.webSocketCreated', onCreated);
-                cdpSession.off('Network.webSocketFrameSent', onSent);
-                cdpSession.off('Network.webSocketFrameReceived', onReceived);
-            }
-        },
-    };
+  };
 }
+export {
+  attachWebSocketDetector
+};
+//# sourceMappingURL=websocket-detector.js.map

package/dist/worker.js

@@ -1,42 +1,33 @@
-/**
- * Standalone worker process entrypoint for a dedicated crawler/worker container,
- * separate from the API server (see infrastructure/ecs-task-definitions/crawler.json).
- *
- * index.ts is a side-effect-free library barrel imported by @detiq/api — it must NOT
- * start any background loop itself, or importing a single helper from @detiq/crawler
- * would silently spin up a second worker inside the API process too. This file is
- * the actual entrypoint that keeps a dedicated worker container's process alive and
- * consuming jobs: `node dist/worker.js`.
- */
-import { startJobWorkerLoop, processDataRetentionPurge } from './job-queue.js';
-import { logger } from './logger.js';
+import { startJobWorkerLoop, processDataRetentionPurge } from "./job-queue.js";
+import { logger } from "./logger.js";
 async function main() {
-    logger.info('[worker] starting standalone crawler worker process');
-    // Durable DB-backed queue — claims every non-CRAWL job type, and CRAWL too when
-    // JOB_QUEUE_BACKEND isn't 'sqs'. Safe to run in many worker replicas at once.
-    startJobWorkerLoop();
-    // Daily data-retention purge: runs immediately on startup then every 24 h.
-    // Only runs in the dedicated worker container (this file), not in the API
-    // process, so there is exactly one purge sweep regardless of API replica count.
-    const runPurge = () => {
-        processDataRetentionPurge().catch((err) => {
-            logger.error({ err: String(err) }, '[worker] data-retention purge failed');
-        });
-    };
-    runPurge();
-    setInterval(runPurge, 24 * 60 * 60 * 1000);
-    // Optional: also consume the SQS transport for horizontally-scaled CRAWL fleets.
-    if (process.env.JOB_QUEUE_BACKEND === 'sqs') {
-        const { startSqsWorker } = await import('./sqs-queue.js');
-        startSqsWorker().catch((err) => {
-            logger.error({ err: String(err) }, '[worker] SQS worker crashed');
-            process.exit(1);
-        });
-    }
+  logger.info("[worker] starting standalone crawler worker process");
+  startJobWorkerLoop();
+  const runPurge = () => {
+    processDataRetentionPurge().catch((err) => {
+      logger.error({ err: String(err) }, "[worker] data-retention purge failed");
+    });
+  };
+  runPurge();
+  setInterval(runPurge, 24 * 60 * 60 * 1e3);
+  if (process.env.JOB_QUEUE_BACKEND === "sqs") {
+    const { startSqsWorker } = await import("./sqs-queue.js");
+    startSqsWorker().catch((err) => {
+      logger.error({ err: String(err) }, "[worker] SQS worker crashed");
+      process.exit(1);
+    });
+  }
 }
-process.on('SIGTERM', () => { logger.info('[worker] SIGTERM received, exiting'); process.exit(0); });
-process.on('SIGINT', () => { logger.info('[worker] SIGINT received, exiting'); process.exit(0); });
+process.on("SIGTERM", () => {
+  logger.info("[worker] SIGTERM received, exiting");
+  process.exit(0);
+});
+process.on("SIGINT", () => {
+  logger.info("[worker] SIGINT received, exiting");
+  process.exit(0);
+});
 main().catch((err) => {
-    logger.error({ err: String(err) }, '[worker] fatal startup error');
-    process.exit(1);
+  logger.error({ err: String(err) }, "[worker] fatal startup error");
+  process.exit(1);
 });
+//# sourceMappingURL=worker.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@detiq/crawler",
-  "version": "1.1.9",
+  "version": "1.1.11",
   "private": false,
   "publishConfig": {
     "access": "public",

package/src/job-queue.ts

@@ -2333,6 +2333,126 @@ async function processRepoScanJob(jobId: string, config: any) {
       }
     }
 
+    // Fetch and parse framework config files (playwright.config.ts, cypress.config.ts, tsconfig.json)
+    // Fetch and parse framework config files for all supported frameworks.
+    // Pure regex/JSON — no LLM. Results override LLM inference for path detection.
+    // Covers: playwright, cypress, jest, pytest, junit/maven/gradle, selenium-java/py,
+    //         webdriverio, cucumber — all frameworks in DetectedFramework.
+    const configPaths: {
+      testDir?: string; fixturesFolder?: string; baseURL?: string;
+      tsconfigPaths?: Record<string, string[]>; tsconfigBaseUrl?: string;
+      javaTestSourceDir?: string; jestModuleNameMapper?: Record<string, string>;
+    } = {};
+    try {
+      const pathSet = new Set(treeResult.paths as string[]);
+      const lang = framework.includes('java') ? 'java'
+        : (framework.includes('py') || framework === 'pytest') ? 'python' : 'js';
+
+      const configFilesToFetch: Array<{ path: string; type: string }> = [];
+      const tryAdd = (candidates: string[], type: string) => {
+        for (const c of candidates) { if (pathSet.has(c)) { configFilesToFetch.push({ path: c, type }); break; } }
+      };
+
+      if (lang === 'js') {
+        if (framework.startsWith('playwright')) tryAdd(['playwright.config.ts', 'playwright.config.js'], 'playwright');
+        if (framework.startsWith('cypress')) tryAdd(['cypress.config.ts', 'cypress.config.js'], 'cypress');
+        if (framework === 'jest-ts' || framework === 'jest-js') {
+          tryAdd(['jest.config.ts', 'jest.config.js', 'jest.config.mjs'], 'jest');
+          tryAdd(['package.json'], 'package-json');
+        }
+        if (framework === 'webdriverio') tryAdd(['wdio.conf.ts', 'wdio.conf.js', 'wdio.config.ts', 'wdio.config.js'], 'wdio');
+        if (framework === 'cucumber' || framework === 'gherkin') {
+          tryAdd(['.cucumber.js', '.cucumber.cjs', 'cucumber.json', '.cucumber.yaml', '.cucumber.yml'], 'cucumber');
+          tryAdd(['package.json'], 'package-json');
+        }
+        tryAdd(['tsconfig.json'], 'tsconfig');
+      } else if (lang === 'python') {
+        tryAdd(['pytest.ini'], 'pytest-ini');
+        tryAdd(['pyproject.toml'], 'pyproject-toml');
+        tryAdd(['setup.cfg'], 'setup-cfg');
+      } else if (lang === 'java') {
+        tryAdd(['pom.xml'], 'pom-xml');
+        tryAdd(['build.gradle.kts', 'build.gradle'], 'build-gradle');
+        tryAdd(['testng.xml', 'src/test/resources/testng.xml'], 'testng-xml');
+      }
+
+      // Inline parsers — mirror parseFrameworkConfig from @detiq/agents (kept inline to avoid build dep)
+      const normDir = (s: string) => s.replace(/^\.\//, '').replace(/\/$/, '');
+      const globRoot = (pattern: string): string | null => {
+        const root = pattern.replace(/\/\*\*.*$/, '').replace(/\/\*[^*].*$/, '');
+        return (root && !root.startsWith('*') && !root.startsWith('{')) ? normDir(root) : null;
+      };
+
+      const fetched = await Promise.all(
+        configFilesToFetch.map(async ({ path: cfgPath, type }) => {
+          const content = await fetchFileContent(token, conn.repoFullName, cfgPath).catch(() => null);
+          if (!content) return null;
+          const r: typeof configPaths = {};
+
+          if (type === 'playwright') {
+            const m = content.match(/\btestDir\s*:\s*['"`]([^'"`]+)['"`]/);
+            if (m) r.testDir = normDir(m[1]);
+            const bm = content.match(/\bbaseURL\s*:\s*['"`]([^'"`]+)['"`]/);
+            if (bm) r.baseURL = bm[1];
+          } else if (type === 'cypress') {
+            const sm = content.match(/\bspecPattern\s*:\s*['"`]([^'"`]+)['"`]/);
+            if (sm) { const gr = globRoot(sm[1]); if (gr) r.testDir = gr; }
+            const fm = content.match(/\bfixturesFolder\s*:\s*['"`]([^'"`]+)['"`]/);
+            if (fm) r.fixturesFolder = normDir(fm[1]);
+            const bm = content.match(/\bbaseUrl\s*:\s*['"`]([^'"`]+)['"`]/);
+            if (bm) r.baseURL = bm[1];
+          } else if (type === 'jest') {
+            const rm = content.match(/\broots\s*:\s*\[([^\]]+)\]/);
+            if (rm) { const fr = rm[1].match(/['"`]([^'"`]+)['"`]/)?.[1]; if (fr) r.testDir = fr.replace(/<rootDir>\/?/, '').replace(/\/$/, '') || 'src'; }
+            if (!r.testDir) { const tm = content.match(/\btestMatch\s*:\s*\[([^\]]+)\]/); if (tm) { const fp = tm[1].match(/['"`]([^'"`]+)['"`]/)?.[1]; if (fp) { const gr = globRoot(fp.replace(/<rootDir>\/?/, '')); if (gr) r.testDir = gr; } } }
+            const mm = content.match(/\bmoduleNameMapper\s*:\s*\{([^}]+)\}/s);
+            if (mm) { const map: Record<string, string> = {}; for (const [, k, v] of mm[1].matchAll(/['"`]([^'"`]+)['"`]\s*:\s*['"`]([^'"`]+)['"`]/g)) map[k] = v; if (Object.keys(map).length) r.jestModuleNameMapper = map; }
+          } else if (type === 'package-json') {
+            try { const p = JSON.parse(content) as any; const j = p?.jest ?? {}; if (Array.isArray(j.roots) && j.roots[0]) r.testDir = String(j.roots[0]).replace(/<rootDir>\/?/, '').replace(/\/$/, '') || 'src'; if (!r.testDir && Array.isArray(j.testMatch) && j.testMatch[0]) { const gr = globRoot(String(j.testMatch[0]).replace(/<rootDir>\/?/, '')); if (gr) r.testDir = gr; } if (j.moduleNameMapper && typeof j.moduleNameMapper === 'object') r.jestModuleNameMapper = j.moduleNameMapper; } catch { /* ignore */ }
+          } else if (type === 'tsconfig') {
+            try { const stripped = content.replace(/\/\/[^\n]*/g, '').replace(/,(\s*[}\]])/g, '$1'); const p = JSON.parse(stripped) as any; const opts = p?.compilerOptions ?? {}; if (opts.paths && typeof opts.paths === 'object') r.tsconfigPaths = opts.paths; if (typeof opts.baseUrl === 'string') r.tsconfigBaseUrl = opts.baseUrl; } catch { /* ignore */ }
+          } else if (type === 'pytest-ini') {
+            const m = content.match(/^\s*testpaths\s*=\s*(.+)$/m);
+            if (m) { const first = m[1].trim().split(/\s+/)[0]; if (first) r.testDir = normDir(first); }
+          } else if (type === 'pyproject-toml') {
+            const m = content.match(/testpaths\s*=\s*\[([^\]]+)\]/);
+            if (m) { const first = m[1].match(/['"]([^'"]+)['"]/)?.[1]; if (first) r.testDir = normDir(first); }
+            if (!r.testDir) { const m2 = content.match(/testpaths\s*=\s*['"]([^'"]+)['"]/); if (m2) r.testDir = normDir(m2[1]); }
+          } else if (type === 'setup-cfg') {
+            const m = content.match(/^\s*testpaths\s*=\s*(.+)$/m);
+            if (m) { const first = m[1].trim().split(/[\s,]+/)[0]; if (first) r.testDir = normDir(first); }
+          } else if (type === 'pom-xml') {
+            const m = content.match(/<testSourceDirectory>\s*([^<]+)\s*<\/testSourceDirectory>/);
+            if (m) r.javaTestSourceDir = normDir(m[1].trim());
+          } else if (type === 'build-gradle') {
+            const m = content.match(/srcDir[s]?\s*[=([]?\s*['"`]([^'"`]+)['"`]/) ?? content.match(/srcDir\s*\(\s*['"`]([^'"`]+)['"`]\s*\)/);
+            if (m) r.javaTestSourceDir = normDir(m[1]);
+          } else if (type === 'wdio') {
+            const sm = content.match(/\bspecs\s*:\s*\[([^\]]+)\]/);
+            if (sm) { const fp = sm[1].match(/['"`]([^'"`]+)['"`]/)?.[1]; if (fp) { const gr = globRoot(fp); if (gr) r.testDir = gr; } }
+            const bm = content.match(/\bbaseUrl\s*:\s*['"`]([^'"`]+)['"`]/);
+            if (bm) r.baseURL = bm[1];
+          } else if (type === 'cucumber') {
+            const pm = content.match(/\bpaths\s*:\s*\[([^\]]+)\]/);
+            if (pm) { const fp = pm[1].match(/['"`]([^'"`]+)['"`]/)?.[1]; if (fp) { const gr = globRoot(fp); if (gr) r.testDir = gr; } }
+            if (!r.testDir) { try { const p = JSON.parse(content) as any; const paths = p?.default?.paths ?? p?.paths; if (Array.isArray(paths) && paths[0]) { const gr = globRoot(String(paths[0])); if (gr) r.testDir = gr; } } catch { /* not JSON */ } }
+          }
+          return r;
+        })
+      );
+
+      for (const parsed of fetched) {
+        if (!parsed) continue;
+        if (parsed.testDir && !configPaths.testDir) configPaths.testDir = parsed.testDir;
+        if (parsed.fixturesFolder && !configPaths.fixturesFolder) configPaths.fixturesFolder = parsed.fixturesFolder;
+        if (parsed.baseURL && !configPaths.baseURL) configPaths.baseURL = parsed.baseURL;
+        if (parsed.tsconfigPaths && !configPaths.tsconfigPaths) configPaths.tsconfigPaths = parsed.tsconfigPaths;
+        if (parsed.tsconfigBaseUrl && !configPaths.tsconfigBaseUrl) configPaths.tsconfigBaseUrl = parsed.tsconfigBaseUrl;
+        if (parsed.javaTestSourceDir && !configPaths.javaTestSourceDir) configPaths.javaTestSourceDir = parsed.javaTestSourceDir;
+        if (parsed.jestModuleNameMapper && !configPaths.jestModuleNameMapper) configPaths.jestModuleNameMapper = parsed.jestModuleNameMapper;
+      }
+    } catch { /* non-fatal */ }
+
     // Extract style profile for aligned generation
     // For Java, include main source paths so LLM can detect page object location in src/main/java/
     // Fall back to existing styleProfile from DB if extraction fails or returns null.
@@ -2362,6 +2482,22 @@ async function processRepoScanJob(jobId: string, config: any) {
       } catch { /* non-fatal */ }
     }
 
+    // Merge config file ground-truth into styleProfile (Priority 0 — highest, no LLM inference needed).
+    // Java: javaTestSourceDir also sets testLocation (equivalent field for Java).
+    if (styleProfile && Object.keys(configPaths).length > 0) {
+      if (configPaths.testDir) styleProfile.configTestDir = configPaths.testDir;
+      if (configPaths.javaTestSourceDir) {
+        styleProfile.configTestDir = configPaths.javaTestSourceDir;
+        // Also set testLocation if not already present from LLM scan
+        if (!styleProfile.testLocation) styleProfile.testLocation = configPaths.javaTestSourceDir;
+      }
+      if (configPaths.fixturesFolder) styleProfile.configFixturesFolder = configPaths.fixturesFolder;
+      if (configPaths.baseURL) styleProfile.configBaseURL = configPaths.baseURL;
+      if (configPaths.tsconfigPaths) styleProfile.tsconfigPaths = configPaths.tsconfigPaths;
+      if (configPaths.tsconfigBaseUrl) styleProfile.tsconfigBaseUrl = configPaths.tsconfigBaseUrl;
+      if (configPaths.jestModuleNameMapper) styleProfile.jestModuleNameMapper = configPaths.jestModuleNameMapper;
+    }
+
     // Extract page object inventory — finds POM/page classes in non-test files
     // Fall back to existing inventory from DB if fetch fails.
     let pageObjectInventory: any[] = (conn.pageObjectInventory as any[]) ?? [];