@taodigital/crawler 1.2.0 → 1.2.2

package/dist/job-queue.js

@@ -460,25 +460,34 @@ async function getJob(tenantId, jobId) {
     (tx) => tx.platformJob.findFirst({ where: { id: jobId, tenantId } })
   );
 }
-async function listJobs(tenantId, projectId) {
-  return withTenant(tenantId, async (tx) => {
-    const jobs = await tx.platformJob.findMany({
+async function countJobs(tenantId, projectId) {
+  return withTenant(tenantId, (tx) => tx.platformJob.count({ where: { tenantId, projectId } }));
+}
+async function listJobs(tenantId, projectId, { skip = 0, take = 50 } = {}) {
+  const jobs = await withTenant(
+    tenantId,
+    (tx) => tx.platformJob.findMany({
       where: { tenantId, projectId },
       orderBy: { queuedAt: "desc" },
-      take: 50
-    });
+      skip,
+      take
+    })
+  );
+  if (jobs.length === 0) return jobs;
+  let modelByJobId = /* @__PURE__ */ new Map();
+  try {
     const jobIds = jobs.map((j) => j.id);
-    if (jobIds.length === 0) return jobs;
-    const modelRows = await tx.lLMCallLog.groupBy({
+    const modelRows = await prismaAdmin.lLMCallLog.groupBy({
       by: ["jobId"],
       where: { tenantId, jobId: { in: jobIds } },
       _max: { model: true, calledAt: true }
     });
-    const modelByJobId = new Map(
+    modelByJobId = new Map(
       modelRows.filter((r) => r.jobId && r._max?.model).map((r) => [r.jobId, r._max.model])
     );
-    return jobs.map((j) => ({ ...j, llmModel: modelByJobId.get(j.id) ?? null }));
-  });
+  } catch {
+  }
+  return jobs.map((j) => ({ ...j, llmModel: modelByJobId.get(j.id) ?? null }));
 }
 async function cancelJob(tenantId, jobId) {
   const controller = runningControllers.get(jobId);
@@ -2355,6 +2364,15 @@ async function processMultiRoleCrawlJob(jobId, config) {
     );
     const savedCrawlConfig = proj?.crawlConfig ?? {};
     const credentialProfiles = config.credentialProfiles ?? [];
+    const multiRoleWsProxy = await AppBrain.getTenantProxyConfig(tenantId).catch(() => null);
+    const multiRoleResolvedProxyUrl = (() => {
+      const projectProxy = savedCrawlConfig?.proxyUrl;
+      if (projectProxy === "agent" || multiRoleWsProxy?.mode === "agent") return "agent";
+      if (projectProxy === "auto" || multiRoleWsProxy?.mode === "auto") return "auto";
+      if (projectProxy && projectProxy !== "auto" && projectProxy !== "agent") return projectProxy;
+      if (multiRoleWsProxy?.mode === "custom" && multiRoleWsProxy.proxyUrl) return multiRoleWsProxy.proxyUrl;
+      return void 0;
+    })();
     await updateJobProgress(tenantId, jobId, 10, "Enqueuing primary crawl\u2026");
     const primaryJobId = await enqueueJob("CRAWL", {
       tenantId,
@@ -2366,7 +2384,7 @@ async function processMultiRoleCrawlJob(jobId, config) {
       startUrls: savedCrawlConfig.seedUrls?.length ? savedCrawlConfig.seedUrls : void 0,
       deviceProfiles: savedCrawlConfig.deviceProfiles ?? ["DESKTOP"],
       loginInstructions: savedCrawlConfig.loginInstructions,
-      proxyUrl: savedCrawlConfig.proxyUrl,
+      proxyUrl: multiRoleResolvedProxyUrl,
       useSitemap: true,
       sitemapUrl: savedCrawlConfig.sitemapUrl
     });
@@ -2415,7 +2433,7 @@ async function processMultiRoleCrawlJob(jobId, config) {
         storageState: savedCrawlConfig.storageState,
         deviceProfiles: savedCrawlConfig.deviceProfiles ?? ["DESKTOP"],
         loginInstructions: savedCrawlConfig.loginInstructions,
-        proxyUrl: savedCrawlConfig.proxyUrl
+        proxyUrl: multiRoleResolvedProxyUrl
       });
       roleJobs.push({ roleName: profile.name, jobId: roleJobId });
       logger.info({ roleJobId, roleName: profile.name, projectId }, "[multi-role-crawl] ROLE_CRAWL enqueued");
@@ -2494,6 +2512,15 @@ async function processRoleCrawlJob(jobId, config) {
         (tx) => tx.project.findUnique({ where: { id: projectId }, select: { crawlConfig: true } })
       );
       const savedCrawlConfig = proj?.crawlConfig ?? {};
+      const roleCrawlWsProxy = await AppBrain.getTenantProxyConfig(tenantId).catch(() => null);
+      const roleCrawlResolvedProxyUrl = (() => {
+        const projectProxy = config.proxyUrl ?? savedCrawlConfig?.proxyUrl;
+        if (projectProxy === "agent" || roleCrawlWsProxy?.mode === "agent") return "agent";
+        if (projectProxy === "auto" || roleCrawlWsProxy?.mode === "auto") return "auto";
+        if (projectProxy && projectProxy !== "auto" && projectProxy !== "agent") return projectProxy;
+        if (roleCrawlWsProxy?.mode === "custom" && roleCrawlWsProxy.proxyUrl) return roleCrawlWsProxy.proxyUrl;
+        return void 0;
+      })();
       const runtimeCreds = config.credentials ? {
         ...config.credentials,
         ...config.credentials.totpSecret ? { getOneTimeCode: async () => generateTOTP(config.credentials.totpSecret) } : {}
@@ -2503,7 +2530,7 @@ async function processRoleCrawlJob(jobId, config) {
         ...config,
         credentials: runtimeCreds,
         storageState: config.storageState ?? savedCrawlConfig.storageState,
-        proxyUrl: config.proxyUrl ?? savedCrawlConfig.proxyUrl,
+        proxyUrl: roleCrawlResolvedProxyUrl,
         loginInstructions: config.loginInstructions ?? savedCrawlConfig.loginInstructions,
         abortSignal: controller.signal,
         jobId,
@@ -3024,14 +3051,20 @@ async function processRepoScanJob(jobId, config) {
     const isJavaTestFile = (p) => p.endsWith(".java") && (/[A-Z]\w*Tests?\.java$/.test(p) || /\/tests?\//.test(p) && /[A-Z][a-z]/.test(p.split("/").pop() ?? ""));
     const testGlobs = minimatch ? Array.from(/* @__PURE__ */ new Set([
       testGlob,
-      ...testGlob === "**/*.spec.ts" ? ["**/*.test.ts", "**/*.cy.ts"] : [],
-      ...testGlob === "**/*.spec.js" ? ["**/*.test.js", "**/*.cy.js"] : []
+      // TS variants: add all common suffixes whenever the primary glob is any *.spec.ts or *.test.ts
+      ...testGlob.endsWith(".spec.ts") ? ["**/*.test.ts", "**/*.test.tsx", "**/*.spec.tsx", "**/*.cy.ts", "**/*.e2e.ts", "**/*.integration.ts", "**/*.smoke.ts", "**/*.steps.ts"] : [],
+      ...testGlob.endsWith(".test.ts") ? ["**/*.spec.ts", "**/*.test.tsx", "**/*.spec.tsx", "**/*.cy.ts", "**/*.e2e.ts", "**/*.integration.ts", "**/*.smoke.ts", "**/*.steps.ts"] : [],
+      // JS variants
+      ...testGlob.endsWith(".spec.js") ? ["**/*.test.js", "**/*.cy.js", "**/*.e2e.js", "**/*.integration.js", "**/*.smoke.js", "**/*.steps.js"] : [],
+      ...testGlob.endsWith(".test.js") ? ["**/*.spec.js", "**/*.cy.js", "**/*.e2e.js", "**/*.integration.js", "**/*.smoke.js", "**/*.steps.js"] : [],
+      // Gherkin feature files are always relevant regardless of primary glob
+      ...framework === "cucumber" || framework === "gherkin" ? ["**/*.feature"] : []
     ])) : [];
     const testPaths = treeResult.paths.filter((p) => {
       if (language === "java") return isJavaTestFile(p);
       if (language === "python") return isPythonTestFile(p);
       if (testGlobs.length) return testGlobs.some((g) => minimatch(p, g, { matchBase: true }));
-      return p.endsWith(".spec.ts") || p.endsWith(".spec.js") || p.endsWith(".test.ts") || p.endsWith(".test.js") || p.endsWith(".cy.ts") || p.endsWith(".cy.js") || p.endsWith("_test.py") || p.includes("/test_");
+      return p.endsWith(".spec.ts") || p.endsWith(".spec.js") || p.endsWith(".test.ts") || p.endsWith(".test.js") || p.endsWith(".cy.ts") || p.endsWith(".cy.js") || p.endsWith(".e2e.ts") || p.endsWith(".e2e.js") || p.endsWith(".integration.ts") || p.endsWith(".integration.js") || p.endsWith(".smoke.ts") || p.endsWith(".steps.ts") || p.endsWith(".feature") || p.endsWith("_test.py") || p.includes("/test_");
     }).slice(0, 2e3);
     await updateJobProgress(tenantId, jobId, 40, `Found ${testPaths.length} test files`);
     if (testPaths.length === 0) {
@@ -3299,8 +3332,16 @@ async function processRepoScanJob(jobId, config) {
       try {
         const mainSourcePaths = language === "java" ? treeResult.paths.filter((p) => /^src\/main\/java\//.test(p)).slice(0, 50) : [];
         const testPathSet = new Set(testPaths);
-        const sourceFilePaths = language === "typescript" || language === "javascript" ? treeResult.paths.filter((p) => (p.endsWith(".ts") || p.endsWith(".js")) && !testPathSet.has(p) && !p.endsWith(".d.ts") && !p.includes("node_modules/") && !p.includes("dist/") && !p.includes(".config.")).slice(0, 60) : [];
-        const allPathsForProfile = [...testPaths, ...mainSourcePaths, ...sourceFilePaths].slice(0, 200);
+        const isFrameworkSupportFile = (p) => /\.(page|fixture|helper|util|utils|data|actions|assertions|locator|locators|steps)\.(ts|js|tsx|jsx)$/.test(p);
+        const isGenericSourceFile = (p) => (p.endsWith(".ts") || p.endsWith(".js") || p.endsWith(".tsx") || p.endsWith(".jsx")) && !p.endsWith(".d.ts") && !p.includes("node_modules/") && !p.includes("dist/") && !p.includes(".next/");
+        const candidateSourcePaths = treeResult.paths.filter(
+          (p) => isGenericSourceFile(p) && !testPathSet.has(p)
+        );
+        const supportFilePaths = candidateSourcePaths.filter(isFrameworkSupportFile).slice(0, 80);
+        const supportPathSet = new Set(supportFilePaths);
+        const otherSourcePaths = candidateSourcePaths.filter((p) => !supportPathSet.has(p)).slice(0, 60);
+        const sourceFilePaths = language === "typescript" || language === "javascript" ? [...supportFilePaths, ...otherSourcePaths] : [];
+        const allPathsForProfile = [...testPaths, ...mainSourcePaths, ...sourceFilePaths].slice(0, 400);
         const freshProfile = await extractStyleProfile(styleSampleFiles, framework, language, tenantId, projectId, jobId, allPathsForProfile, treeResult.paths);
         if (freshProfile) styleProfile = freshProfile;
       } catch {
@@ -3787,6 +3828,7 @@ export {
   claimNextQueuedJob,
   claimSpecificJob,
   classifyFailure,
+  countJobs,
   createTrackedJob,
   dispatchClaimedJob,
   enqueueJob,