@detiq/crawler 1.1.9 → 1.1.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/dist/accessibility-auditor.js +54 -74
  2. package/dist/ai-login.js +227 -249
  3. package/dist/auth-token-rotator.js +57 -67
  4. package/dist/auth.js +299 -336
  5. package/dist/browser-extension.js +104 -142
  6. package/dist/captcha-solver.js +98 -125
  7. package/dist/color-contrast.js +77 -89
  8. package/dist/content-chunker.js +126 -126
  9. package/dist/content-dedup.js +54 -71
  10. package/dist/cookie-consent.js +68 -83
  11. package/dist/cookie-jar.js +53 -60
  12. package/dist/crawler.js +4494 -5353
  13. package/dist/custom-js-injector.js +45 -61
  14. package/dist/dark-mode-capture.js +67 -81
  15. package/dist/dead-link-detector.js +47 -55
  16. package/dist/element-hash.js +8 -12
  17. package/dist/email-otp.js +154 -195
  18. package/dist/engine-router.js +87 -98
  19. package/dist/export/bulk-export.js +70 -55
  20. package/dist/export/vector-export.js +77 -105
  21. package/dist/extractors/advanced.js +249 -266
  22. package/dist/extractors/artifact-envelope.js +59 -58
  23. package/dist/failure-analysis-agent.js +37 -33
  24. package/dist/feature-flag-detector.js +51 -66
  25. package/dist/form-detector.js +55 -75
  26. package/dist/frontier.js +76 -101
  27. package/dist/gap-pass.js +68 -89
  28. package/dist/graphql-detector.js +62 -72
  29. package/dist/http-engine.js +282 -330
  30. package/dist/incremental-diff.js +54 -87
  31. package/dist/index.js +67 -28
  32. package/dist/input-fuzzer.js +93 -97
  33. package/dist/interactive-probe.js +715 -827
  34. package/dist/job-queue.js +2151 -2169
  35. package/dist/js-coverage.js +52 -56
  36. package/dist/js-router-detector.js +86 -117
  37. package/dist/language-detector.js +51 -63
  38. package/dist/link-classifier.js +32 -50
  39. package/dist/live-view.js +91 -108
  40. package/dist/locator-extractor.js +151 -187
  41. package/dist/logger.js +9 -7
  42. package/dist/markdown-extract.js +48 -61
  43. package/dist/native-crawler.js +780 -946
  44. package/dist/network-observer.js +135 -156
  45. package/dist/network-throttle.js +29 -40
  46. package/dist/openapi-generator.js +77 -92
  47. package/dist/pdf-extractor.js +72 -90
  48. package/dist/performance-metrics.js +80 -102
  49. package/dist/phash.js +125 -144
  50. package/dist/pii-scrubber.js +65 -68
  51. package/dist/playwright-recorder.js +70 -58
  52. package/dist/policies/crawl-safety.js +31 -34
  53. package/dist/profiles/domain-profile.js +130 -125
  54. package/dist/proxy.js +146 -172
  55. package/dist/rate-limiter.js +58 -65
  56. package/dist/redirect-tracker.js +60 -64
  57. package/dist/redis-frontier.js +116 -136
  58. package/dist/redis-queue.js +157 -144
  59. package/dist/response-cache.js +127 -140
  60. package/dist/responsive-detector.js +42 -46
  61. package/dist/retry-policy.js +46 -50
  62. package/dist/robots-txt.js +69 -83
  63. package/dist/screen-reader.js +45 -70
  64. package/dist/screenshot-masker.js +59 -77
  65. package/dist/seo-checker.js +88 -107
  66. package/dist/session-capture.js +192 -212
  67. package/dist/session-export.js +107 -133
  68. package/dist/sitemap-writer.js +38 -41
  69. package/dist/sitemap.js +141 -202
  70. package/dist/skyvern.js +123 -148
  71. package/dist/sqs-queue.js +42 -42
  72. package/dist/sso-token.js +74 -90
  73. package/dist/stealth/human-behavior.js +156 -173
  74. package/dist/telemetry.js +68 -82
  75. package/dist/test-executor.js +309 -335
  76. package/dist/tls-fingerprint.js +77 -88
  77. package/dist/url-pattern-completer.js +71 -90
  78. package/dist/visual-diff-verdict.js +23 -38
  79. package/dist/webhook-notifier.js +71 -69
  80. package/dist/websocket-detector.js +56 -68
  81. package/dist/worker.js +29 -38
  82. package/package.json +1 -1
  83. package/src/job-queue.ts +136 -0
package/dist/job-queue.js CHANGED
@@ -1,161 +1,103 @@
1
- /**
2
- * ZeTa-AI Background Job Queue
3
- *
4
- * Postgres-backed durable queue. Each job becomes a PlatformJob record.
5
- * enqueueJob() writes it QUEUED and, for same-process responsiveness, kicks
6
- * an immediate best-effort claim attempt — but durability comes from
7
- * startJobWorkerLoop()'s poll: it atomically claims QUEUED jobs (Postgres
8
- * `FOR UPDATE SKIP LOCKED`, safe across any number of concurrent worker
9
- * processes/containers) and requeues jobs whose heartbeat went stale,
10
- * so a crashed/restarted worker never silently loses a job.
11
- *
12
- * CRAWL-type jobs additionally support AWS SQS as an alternate transport
13
- * (JOB_QUEUE_BACKEND=sqs, see sqs-queue.ts) for horizontally-scaled crawler
14
- * fleets; every other job type always goes through the DB-backed queue.
15
- */
16
- import crypto from 'node:crypto';
17
- import { withTenant, prismaAdmin } from '@detiq/database';
18
- import { crawlJobsTotal, crawlJobDurationSeconds } from '@detiq/core';
19
- import { logger } from './logger.js';
1
+ import crypto from "node:crypto";
2
+ import { withTenant, prismaAdmin } from "@detiq/database";
3
+ import { crawlJobsTotal, crawlJobDurationSeconds } from "@detiq/core";
4
+ import { logger } from "./logger.js";
20
5
  async function notifyAdmins(tenantId, data) {
21
- try {
22
- // Notification/Membership both have FORCE ROW LEVEL SECURITY — a bare
23
- // prisma call with no app.current_tenant set gets silently filtered to
24
- // zero rows (SELECT) or rejected (INSERT), not an error, so this call
25
- // site had been a complete no-op with no visible failure until fixed.
26
- const members = await withTenant(tenantId, (tx) => tx.membership.findMany({
27
- where: { tenantId, role: { in: ['OWNER', 'ADMIN'] } },
28
- select: { userId: true },
29
- }));
30
- await withTenant(tenantId, (tx) => tx.notification.createMany({
31
- data: members.map((m) => ({
32
- id: `notif_${crypto.randomUUID()}`,
33
- tenantId,
34
- userId: m.userId,
35
- type: data.type,
36
- title: data.title,
37
- body: data.body,
38
- metadata: data.metadata,
39
- })),
40
- }));
41
- }
42
- catch (e) {
43
- console.error('[notify] failed:', e);
44
- }
6
+ try {
7
+ const members = await withTenant(
8
+ tenantId,
9
+ (tx) => tx.membership.findMany({
10
+ where: { tenantId, role: { in: ["OWNER", "ADMIN"] } },
11
+ select: { userId: true }
12
+ })
13
+ );
14
+ await withTenant(
15
+ tenantId,
16
+ (tx) => tx.notification.createMany({
17
+ data: members.map((m) => ({
18
+ id: `notif_${crypto.randomUUID()}`,
19
+ tenantId,
20
+ userId: m.userId,
21
+ type: data.type,
22
+ title: data.title,
23
+ body: data.body,
24
+ metadata: data.metadata
25
+ }))
26
+ })
27
+ );
28
+ } catch (e) {
29
+ console.error("[notify] failed:", e);
30
+ }
45
31
  }
46
- import { AppBrain, buildArtifactKey, getTenantStorageConfig, uploadScreenshotToTenantStorage, uploadFileToTenantStorage, notifyRunCompletion, recordLLMCall } from '@detiq/app-brain';
47
- import { generateTaxonomySkillFiles, groupScreensWithAI, generateExecutionPlan, refreshTestCasesSkillFile, refreshTestDataSkillFile, refreshTestScriptSkillFile, refreshRequirementsCoverageSkillFile, detectFieldsFromHtml, mergeIntoCatalog } from '@detiq/agents';
48
- import { crawlProject } from './crawler.js';
49
- import { httpCrawl } from './http-engine.js';
50
- import { hashContent, normalizeText } from './incremental-diff.js';
51
- import { discoverProjectUrls } from './sitemap.js';
52
- import { executeTestPlan } from './test-executor.js';
53
- import { runNativeCrawl } from './native-crawler.js';
54
- import { jira, confluence, figma } from '@detiq/connectors';
55
- import { decrypt, config as platformConfig, dispatchWebhook, isFeatureEnabled, interpolatePlaceholders } from '@detiq/core';
56
- import { createHmac } from 'node:crypto';
57
- import fs from 'node:fs/promises';
58
- import path from 'node:path';
59
- import os from 'node:os';
60
- // RFC 6238 TOTP — no external dependency, uses Node built-in crypto.
61
- export function generateTOTP(secret) {
62
- const BASE32 = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567';
63
- const clean = secret.toUpperCase().replace(/\s/g, '').replace(/=/g, '');
64
- let bits = 0, value = 0;
65
- const bytes = [];
66
- for (const ch of clean) {
67
- const idx = BASE32.indexOf(ch);
68
- if (idx === -1)
69
- continue;
70
- value = (value << 5) | idx;
71
- bits += 5;
72
- if (bits >= 8) {
73
- bits -= 8;
74
- bytes.push((value >> bits) & 0xff);
75
- }
32
+ import { AppBrain, buildArtifactKey, getTenantStorageConfig, uploadScreenshotToTenantStorage, uploadFileToTenantStorage, notifyRunCompletion, recordLLMCall } from "@detiq/app-brain";
33
+ import { generateTaxonomySkillFiles, groupScreensWithAI, generateExecutionPlan, refreshTestCasesSkillFile, refreshTestDataSkillFile, refreshTestScriptSkillFile, refreshRequirementsCoverageSkillFile, detectFieldsFromHtml, mergeIntoCatalog } from "@detiq/agents";
34
+ import { crawlProject } from "./crawler.js";
35
+ import { httpCrawl } from "./http-engine.js";
36
+ import { hashContent, normalizeText } from "./incremental-diff.js";
37
+ import { discoverProjectUrls } from "./sitemap.js";
38
+ import { executeTestPlan } from "./test-executor.js";
39
+ import { runNativeCrawl } from "./native-crawler.js";
40
+ import { jira, confluence, figma } from "@detiq/connectors";
41
+ import { decrypt, config as platformConfig, dispatchWebhook, isFeatureEnabled, interpolatePlaceholders } from "@detiq/core";
42
+ import { createHmac } from "node:crypto";
43
+ import fs from "node:fs/promises";
44
+ import path from "node:path";
45
+ import os from "node:os";
46
+ function generateTOTP(secret) {
47
+ const BASE32 = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";
48
+ const clean = secret.toUpperCase().replace(/\s/g, "").replace(/=/g, "");
49
+ let bits = 0, value = 0;
50
+ const bytes = [];
51
+ for (const ch of clean) {
52
+ const idx = BASE32.indexOf(ch);
53
+ if (idx === -1) continue;
54
+ value = value << 5 | idx;
55
+ bits += 5;
56
+ if (bits >= 8) {
57
+ bits -= 8;
58
+ bytes.push(value >> bits & 255);
76
59
  }
77
- const counter = Math.floor(Date.now() / 30000);
78
- const buf = Buffer.alloc(8);
79
- buf.writeBigInt64BE(BigInt(counter));
80
- const hmac = createHmac('sha1', Buffer.from(bytes)).update(buf).digest();
81
- const offset = hmac[hmac.length - 1] & 0xf;
82
- const code = ((hmac[offset] & 0x7f) << 24) | ((hmac[offset + 1] & 0xff) << 16) |
83
- ((hmac[offset + 2] & 0xff) << 8) | (hmac[offset + 3] & 0xff);
84
- return String(code % 1_000_000).padStart(6, '0');
60
+ }
61
+ const counter = Math.floor(Date.now() / 3e4);
62
+ const buf = Buffer.alloc(8);
63
+ buf.writeBigInt64BE(BigInt(counter));
64
+ const hmac = createHmac("sha1", Buffer.from(bytes)).update(buf).digest();
65
+ const offset = hmac[hmac.length - 1] & 15;
66
+ const code = (hmac[offset] & 127) << 24 | (hmac[offset + 1] & 255) << 16 | (hmac[offset + 2] & 255) << 8 | hmac[offset + 3] & 255;
67
+ return String(code % 1e6).padStart(6, "0");
85
68
  }
86
69
  function deriveFallbackTargetPath(ref, framework) {
87
- const slug = (ref ?? 'test').toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '') || 'test';
88
- const base = framework.startsWith('cypress') ? 'cypress/e2e/'
89
- : (framework === 'playwright-py' || framework === 'pytest' || framework === 'selenium-py') ? 'tests/'
90
- : (framework === 'playwright-java' || framework === 'selenium-java' || framework === 'junit') ? 'src/test/java/'
91
- : 'tests/';
92
- const ext = (framework === 'playwright-py' || framework === 'pytest' || framework === 'selenium-py') ? '_test.py'
93
- : (framework === 'playwright-java' || framework === 'selenium-java' || framework === 'junit') ? 'Test.java'
94
- : framework.includes('ts') ? '.spec.ts'
95
- : '.spec.js';
96
- return `${base}${slug}${ext}`;
70
+ const slug = (ref ?? "test").toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "") || "test";
71
+ const base = framework.startsWith("cypress") ? "cypress/e2e/" : framework === "playwright-py" || framework === "pytest" || framework === "selenium-py" ? "tests/" : framework === "playwright-java" || framework === "selenium-java" || framework === "junit" ? "src/test/java/" : "tests/";
72
+ const ext = framework === "playwright-py" || framework === "pytest" || framework === "selenium-py" ? "_test.py" : framework === "playwright-java" || framework === "selenium-java" || framework === "junit" ? "Test.java" : framework.includes("ts") ? ".spec.ts" : ".spec.js";
73
+ return `${base}${slug}${ext}`;
97
74
  }
98
75
  function makeJobName(type) {
99
- const now = new Date();
100
- const date = now.toISOString().slice(0, 10); // YYYY-MM-DD
101
- const ts = now.getTime();
102
- const label = type === 'CRAWL' ? 'Crawl'
103
- : type === 'NATIVE_CRAWL' ? 'NativeCrawl'
104
- : type === 'GENERATE_TESTS' ? 'GenerateTests'
105
- : type === 'VISUAL_REGRESSION' ? 'VisualRegression'
106
- : type === 'AI_EDIT' ? 'AIEdit'
107
- : type === 'API_TESTS' ? 'ApiTests'
108
- : type === 'SECURITY_TESTS' ? 'SecurityTests'
109
- : type === 'INTEGRATION_TESTS' ? 'IntegrationTests'
110
- : type === 'GENERATE_TESTS_FROM_REQS' ? 'GenerateTestsFromReqs'
111
- : type === 'GENERATE_DATA' ? 'GenerateData'
112
- : type === 'GENERATE_SCRIPT' ? 'GenerateScript'
113
- : type === 'GENERATE_TEST_PLAN' ? 'GenerateTestPlan'
114
- : type === 'IMPORT_TESTS' ? 'ImportTests'
115
- : type === 'GENERATE_SCREEN_TESTS' ? 'GenerateScreenTests'
116
- : type === 'BATCH_GENERATE_ALIGNED' ? 'BatchGenerateAligned'
117
- : type === 'CREATE_PR' ? 'CreatePR'
118
- : type === 'NEW_PAGES_PIPELINE' ? 'NewPagesPipeline'
119
- : type.charAt(0) + type.slice(1).toLowerCase().replace(/_([a-z])/g, (_, c) => c.toUpperCase());
120
- return `${label}_${date}_${ts}`;
76
+ const now = /* @__PURE__ */ new Date();
77
+ const date = now.toISOString().slice(0, 10);
78
+ const ts = now.getTime();
79
+ const label = type === "CRAWL" ? "Crawl" : type === "NATIVE_CRAWL" ? "NativeCrawl" : type === "GENERATE_TESTS" ? "GenerateTests" : type === "VISUAL_REGRESSION" ? "VisualRegression" : type === "AI_EDIT" ? "AIEdit" : type === "API_TESTS" ? "ApiTests" : type === "SECURITY_TESTS" ? "SecurityTests" : type === "INTEGRATION_TESTS" ? "IntegrationTests" : type === "GENERATE_TESTS_FROM_REQS" ? "GenerateTestsFromReqs" : type === "GENERATE_DATA" ? "GenerateData" : type === "GENERATE_SCRIPT" ? "GenerateScript" : type === "GENERATE_TEST_PLAN" ? "GenerateTestPlan" : type === "IMPORT_TESTS" ? "ImportTests" : type === "GENERATE_SCREEN_TESTS" ? "GenerateScreenTests" : type === "BATCH_GENERATE_ALIGNED" ? "BatchGenerateAligned" : type === "CREATE_PR" ? "CreatePR" : type === "NEW_PAGES_PIPELINE" ? "NewPagesPipeline" : type.charAt(0) + type.slice(1).toLowerCase().replace(/_([a-z])/g, (_, c) => c.toUpperCase());
80
+ return `${label}_${date}_${ts}`;
121
81
  }
122
- /** In-memory map of jobId → AbortController for RUNNING jobs. */
123
- const runningControllers = new Map();
124
- // ─── Durable job worker ──────────────────────────────────────────────────────
125
- //
126
- // A minimal Postgres-backed work queue: atomic claim via `FOR UPDATE SKIP
127
- // LOCKED` (safe for any number of concurrent pollers, in-process or across
128
- // containers), plus a stale-job sweep keyed off a liveness heartbeat rather
129
- // than wall-clock job age, so a slow-but-alive job is never requeued and
130
- // double-processed. See startJobWorkerLoop() below.
82
+ const runningControllers = /* @__PURE__ */ new Map();
131
83
  const WORKER_ID = `${os.hostname()}:${process.pid}`;
132
84
  const STALE_JOB_TIMEOUT_MINUTES = Number(process.env.JOB_STALE_TIMEOUT_MINUTES) || 30;
133
- const WORKER_POLL_INTERVAL_MS = Number(process.env.JOB_WORKER_POLL_INTERVAL_MS) || 15_000;
85
+ const WORKER_POLL_INTERVAL_MS = Number(process.env.JOB_WORKER_POLL_INTERVAL_MS) || 15e3;
134
86
  const WORKER_CONCURRENCY = Number(process.env.JOB_WORKER_CONCURRENCY) || 8;
135
- const JOB_HEARTBEAT_INTERVAL_MS = Number(process.env.JOB_HEARTBEAT_INTERVAL_MS) || 15_000;
136
- // How many headless browsers a single RUN_TEST_SUITE job launches at once —
137
- // distinct from WORKER_CONCURRENCY (how many *jobs* the poll loop claims at
138
- // once). Bounded by default to avoid exhausting a container's memory; the
139
- // primary scale-out path for running more tests at once is still horizontal
140
- // (more worker containers / SQS), not raising this past what one container
141
- // can safely hold — see startJobWorkerLoop's doc comment.
87
+ const JOB_HEARTBEAT_INTERVAL_MS = Number(process.env.JOB_HEARTBEAT_INTERVAL_MS) || 15e3;
142
88
  const TEST_EXEC_CONCURRENCY = Number(process.env.TEST_EXEC_CONCURRENCY) || 5;
143
- /** Dispatch table populated at module bottom, once every process*Job function is declared. */
144
89
  let jobDispatchTable = null;
145
- /** Atomically claims one specific QUEUED job for this worker. Returns null if it was already claimed. */
146
- export async function claimSpecificJob(jobId) {
147
- const rows = await prismaAdmin.$queryRaw `
90
+ async function claimSpecificJob(jobId) {
91
+ const rows = await prismaAdmin.$queryRaw`
148
92
  UPDATE "PlatformJob"
149
93
  SET status = 'RUNNING', "startedAt" = NOW(), "heartbeatAt" = NOW(), "workerId" = ${WORKER_ID}
150
94
  WHERE id = ${jobId} AND status = 'QUEUED'
151
95
  RETURNING id, "tenantId" AS "tenantId", "projectId" AS "projectId", type, config
152
96
  `;
153
- return rows[0] ?? null;
97
+ return rows[0] ?? null;
154
98
  }
155
- /** Atomically claims the oldest QUEUED job across all tenants. SKIP LOCKED makes this
156
- * race-safe against any number of other pollers doing the same thing concurrently. */
157
- export async function claimNextQueuedJob() {
158
- const rows = await prismaAdmin.$queryRaw `
99
+ async function claimNextQueuedJob() {
100
+ const rows = await prismaAdmin.$queryRaw`
159
101
  UPDATE "PlatformJob"
160
102
  SET status = 'RUNNING', "startedAt" = NOW(), "heartbeatAt" = NOW(), "workerId" = ${WORKER_ID}
161
103
  WHERE id = (
@@ -167,13 +109,10 @@ export async function claimNextQueuedJob() {
167
109
  )
168
110
  RETURNING id, "tenantId" AS "tenantId", "projectId" AS "projectId", type, config
169
111
  `;
170
- return rows[0] ?? null;
112
+ return rows[0] ?? null;
171
113
  }
172
- /** Requeues RUNNING jobs whose heartbeat has gone stale — the worker that claimed them
173
- * died (crash, restart, OOM) without ever marking them DONE/FAILED. Idempotent and
174
- * safe to run from every worker process; only affects rows past the timeout. */
175
- export async function requeueStaleRunningJobs() {
176
- const rows = await prismaAdmin.$queryRaw `
114
+ async function requeueStaleRunningJobs() {
115
+ const rows = await prismaAdmin.$queryRaw`
177
116
  UPDATE "PlatformJob"
178
117
  SET status = 'QUEUED',
179
118
  "workerId" = NULL,
@@ -187,18 +126,12 @@ export async function requeueStaleRunningJobs() {
187
126
  AND COALESCE("heartbeatAt", "startedAt") < NOW() - (${STALE_JOB_TIMEOUT_MINUTES} * interval '1 minute')
188
127
  RETURNING id
189
128
  `;
190
- if (rows.length > 0) {
191
- logger.warn({ jobIds: rows.map((r) => r.id), staleTimeoutMinutes: STALE_JOB_TIMEOUT_MINUTES }, '[job-worker] requeued stale RUNNING job(s)');
192
- }
129
+ if (rows.length > 0) {
130
+ logger.warn({ jobIds: rows.map((r) => r.id), staleTimeoutMinutes: STALE_JOB_TIMEOUT_MINUTES }, "[job-worker] requeued stale RUNNING job(s)");
131
+ }
193
132
  }
194
- /**
195
- * Requeues any RUNNING jobs claimed by a different worker (i.e., a previous
196
- * process instance). Call once at startup before the poll loop begins — this
197
- * provides immediate recovery instead of waiting the full STALE_JOB_TIMEOUT_MINUTES.
198
- * Safe to call from multiple replicas simultaneously (each uses its own WORKER_ID).
199
- */
200
- export async function requeueOrphanedJobs() {
201
- const rows = await prismaAdmin.$queryRaw `
133
+ async function requeueOrphanedJobs() {
134
+ const rows = await prismaAdmin.$queryRaw`
202
135
  UPDATE "PlatformJob"
203
136
  SET status = 'QUEUED',
204
137
  "workerId" = NULL,
@@ -213,522 +146,464 @@ export async function requeueOrphanedJobs() {
213
146
  AND "workerId" != ${WORKER_ID}
214
147
  RETURNING id
215
148
  `;
216
- if (rows.length > 0) {
217
- logger.warn({ jobIds: rows.map((r) => r.id), workerId: WORKER_ID }, '[job-worker] requeued orphaned RUNNING job(s) from dead worker');
218
- }
149
+ if (rows.length > 0) {
150
+ logger.warn({ jobIds: rows.map((r) => r.id), workerId: WORKER_ID }, "[job-worker] requeued orphaned RUNNING job(s) from dead worker");
151
+ }
219
152
  }
220
- /** Dispatches a claimed job to its type's processor. Last-resort safety net — every
221
- * process*Job function already catches and records its own failures. */
222
153
  async function dispatchClaimedJob(job) {
223
- const dispatcher = jobDispatchTable?.[job.type];
224
- if (!dispatcher) {
225
- logger.error({ jobId: job.id, type: job.type }, '[job-worker] no dispatcher registered for job type');
226
- await prismaAdmin.platformJob.update({
227
- where: { id: job.id },
228
- data: { status: 'FAILED', error: `Unknown job type: ${job.type}`, finishedAt: new Date() },
229
- }).catch(() => { });
230
- return;
231
- }
232
- const stopTimer = crawlJobDurationSeconds.startTimer({ type: job.type });
154
+ const dispatcher = jobDispatchTable?.[job.type];
155
+ if (!dispatcher) {
156
+ logger.error({ jobId: job.id, type: job.type }, "[job-worker] no dispatcher registered for job type");
157
+ await prismaAdmin.platformJob.update({
158
+ where: { id: job.id },
159
+ data: { status: "FAILED", error: `Unknown job type: ${job.type}`, finishedAt: /* @__PURE__ */ new Date() }
160
+ }).catch(() => {
161
+ });
162
+ return;
163
+ }
164
+ const stopTimer = crawlJobDurationSeconds.startTimer({ type: job.type });
165
+ try {
166
+ await dispatcher(job.id, job.config);
167
+ stopTimer();
168
+ const finished = await prismaAdmin.platformJob.findUnique({ where: { id: job.id }, select: { status: true } }).catch(() => null);
169
+ crawlJobsTotal.inc({ type: job.type, status: finished?.status === "FAILED" ? "failed" : "success" });
170
+ } catch (err) {
171
+ stopTimer();
172
+ crawlJobsTotal.inc({ type: job.type, status: "failed" });
173
+ logger.error({ jobId: job.id, type: job.type, err: String(err?.message ?? err) }, "[job-worker] job threw uncaught error");
174
+ await prismaAdmin.platformJob.update({
175
+ where: { id: job.id },
176
+ data: { status: "FAILED", error: String(err?.message ?? err), finishedAt: /* @__PURE__ */ new Date() }
177
+ }).catch(() => {
178
+ });
179
+ }
180
+ }
181
+ let workerLoopTimer = null;
182
+ function startJobWorkerLoop(opts) {
183
+ if (workerLoopTimer) return;
184
+ const pollIntervalMs = opts?.pollIntervalMs ?? WORKER_POLL_INTERVAL_MS;
185
+ const concurrency = opts?.concurrency ?? WORKER_CONCURRENCY;
186
+ logger.info({ workerId: WORKER_ID, pollIntervalMs, concurrency }, "[job-worker] starting");
187
+ const tick = async () => {
233
188
  try {
234
- await dispatcher(job.id, job.config);
235
- stopTimer();
236
- // dispatcher() resolving doesn't mean the job succeeded — every process*Job
237
- // function catches and records its own failures internally without
238
- // rethrowing (see this function's doc comment). Read back the actual
239
- // final status so the metric reflects reality, not just "didn't throw".
240
- const finished = await prismaAdmin.platformJob.findUnique({ where: { id: job.id }, select: { status: true } }).catch(() => null);
241
- crawlJobsTotal.inc({ type: job.type, status: finished?.status === 'FAILED' ? 'failed' : 'success' });
189
+ await requeueStaleRunningJobs();
190
+ } catch (err) {
191
+ logger.error({ err: String(err) }, "[job-worker] stale-job sweep failed");
242
192
  }
243
- catch (err) {
244
- stopTimer();
245
- crawlJobsTotal.inc({ type: job.type, status: 'failed' });
246
- logger.error({ jobId: job.id, type: job.type, err: String(err?.message ?? err) }, '[job-worker] job threw uncaught error');
247
- await prismaAdmin.platformJob.update({
248
- where: { id: job.id },
249
- data: { status: 'FAILED', error: String(err?.message ?? err), finishedAt: new Date() },
250
- }).catch(() => { });
193
+ try {
194
+ for (let i = 0; i < concurrency; i++) {
195
+ const job = await claimNextQueuedJob();
196
+ if (!job) break;
197
+ void dispatchClaimedJob(job);
198
+ }
199
+ } catch (err) {
200
+ logger.error({ err: String(err) }, "[job-worker] claim loop failed");
251
201
  }
202
+ };
203
+ workerLoopTimer = setInterval(tick, pollIntervalMs);
204
+ requeueOrphanedJobs().catch(
205
+ (err) => logger.error({ err: String(err) }, "[job-worker] startup orphan sweep failed")
206
+ );
207
+ void tick();
252
208
  }
253
- let workerLoopTimer = null;
254
- /**
255
- * Starts the durable job worker poll loop. This is what makes job processing
256
- * survive a restart — enqueueJob()'s immediate same-process claim attempt is
257
- * only a latency optimization; if the process dies before that fires (or
258
- * mid-job), this loop's next tick claims/requeues it. Call once at process
259
- * startup (API server and/or a dedicated crawler worker process — safe to
260
- * run in both, and safe to run in many container replicas at once).
261
- */
262
- export function startJobWorkerLoop(opts) {
263
- if (workerLoopTimer)
264
- return; // already running in this process
265
- const pollIntervalMs = opts?.pollIntervalMs ?? WORKER_POLL_INTERVAL_MS;
266
- const concurrency = opts?.concurrency ?? WORKER_CONCURRENCY;
267
- logger.info({ workerId: WORKER_ID, pollIntervalMs, concurrency }, '[job-worker] starting');
268
- const tick = async () => {
269
- try {
270
- await requeueStaleRunningJobs();
271
- }
272
- catch (err) {
273
- logger.error({ err: String(err) }, '[job-worker] stale-job sweep failed');
274
- }
275
- try {
276
- for (let i = 0; i < concurrency; i++) {
277
- const job = await claimNextQueuedJob();
278
- if (!job)
279
- break;
280
- void dispatchClaimedJob(job); // not awaited — runs concurrently while the loop keeps polling
281
- }
282
- }
283
- catch (err) {
284
- logger.error({ err: String(err) }, '[job-worker] claim loop failed');
285
- }
286
- };
287
- workerLoopTimer = setInterval(tick, pollIntervalMs);
288
- // On startup: immediately requeue jobs orphaned by the previous process instance.
289
- // This gives instant recovery rather than waiting STALE_JOB_TIMEOUT_MINUTES (30 min).
290
- requeueOrphanedJobs().catch((err) => logger.error({ err: String(err) }, '[job-worker] startup orphan sweep failed'));
291
- void tick();
209
+ function stopJobWorkerLoop() {
210
+ if (workerLoopTimer) clearInterval(workerLoopTimer);
211
+ workerLoopTimer = null;
292
212
  }
293
- /** Stops the poll loop. Mainly for tests. */
294
- export function stopJobWorkerLoop() {
295
- if (workerLoopTimer)
296
- clearInterval(workerLoopTimer);
297
- workerLoopTimer = null;
213
+ async function createTrackedJob(type, config) {
214
+ const name = makeJobName(type);
215
+ const job = await withTenant(
216
+ config.tenantId,
217
+ (tx) => tx.platformJob.create({
218
+ data: { tenantId: config.tenantId, projectId: config.projectId, type, name, status: "RUNNING", startedAt: /* @__PURE__ */ new Date(), config }
219
+ })
220
+ );
221
+ return job.id;
298
222
  }
299
- /** Create a job record that tracks a synchronous operation (telemetry only, no background processor). */
300
- export async function createTrackedJob(type, config) {
301
- const name = makeJobName(type);
302
- const job = await withTenant(config.tenantId, (tx) => tx.platformJob.create({
303
- data: { tenantId: config.tenantId, projectId: config.projectId, type, name, status: 'RUNNING', startedAt: new Date(), config: config },
304
- }));
305
- return job.id;
306
- }
307
- /** Mark a tracked job DONE or FAILED. Safe to call in finally — swallows errors. */
308
- export async function finishTrackedJob(tenantId, jobId, error) {
309
- await withTenant(tenantId, (tx) => tx.platformJob.update({
310
- where: { id: jobId },
311
- data: { status: error ? 'FAILED' : 'DONE', finishedAt: new Date(), ...(error ? { error } : { progress: 100 }) },
312
- })).catch(() => { });
223
+ async function finishTrackedJob(tenantId, jobId, error) {
224
+ await withTenant(
225
+ tenantId,
226
+ (tx) => tx.platformJob.update({
227
+ where: { id: jobId },
228
+ data: { status: error ? "FAILED" : "DONE", finishedAt: /* @__PURE__ */ new Date(), ...error ? { error } : { progress: 100 } }
229
+ })
230
+ ).catch(() => {
231
+ });
313
232
  }
314
- /** Enqueue a generic platform job — returns the job ID immediately. */
315
- export async function enqueueJob(type, config) {
316
- const { tenantId, projectId } = config;
317
- const name = makeJobName(type);
318
- const job = await withTenant(tenantId, (tx) => tx.platformJob.create({
319
- data: {
320
- tenantId,
321
- projectId,
322
- type,
323
- name,
324
- status: 'QUEUED',
325
- config: config,
326
- },
327
- }));
328
- // Durability comes from startJobWorkerLoop()'s poll — this is only a same-process
329
- // latency optimization so a job doesn't sit idle until the next poll tick. The
330
- // claim is atomic (WHERE status='QUEUED'), so it's safe even if the poll loop's
331
- // own tick fires concurrently and claims this same job first.
332
- if (type === 'CRAWL' && process.env.JOB_QUEUE_BACKEND === 'sqs') {
333
- const { sendJobToSqs } = await import('./sqs-queue.js');
334
- await sendJobToSqs(job.id, tenantId, config);
335
- }
336
- else if (process.env.DISABLE_WORKER !== 'true') {
337
- // Skip inline dispatch when DISABLE_WORKER=true — the dedicated crawler process
338
- // will pick this up via its poll loop. Inline dispatch in the API container
339
- // would launch Playwright browsers and crash the API.
340
- setImmediate(() => claimSpecificJob(job.id).then((claimed) => claimed && dispatchClaimedJob(claimed)));
341
- }
342
- return job.id;
233
+ async function enqueueJob(type, config) {
234
+ const { tenantId, projectId } = config;
235
+ const name = makeJobName(type);
236
+ const job = await withTenant(
237
+ tenantId,
238
+ (tx) => tx.platformJob.create({
239
+ data: {
240
+ tenantId,
241
+ projectId,
242
+ type,
243
+ name,
244
+ status: "QUEUED",
245
+ config
246
+ }
247
+ })
248
+ );
249
+ if (type === "CRAWL" && process.env.JOB_QUEUE_BACKEND === "sqs") {
250
+ const { sendJobToSqs } = await import("./sqs-queue.js");
251
+ await sendJobToSqs(job.id, tenantId, config);
252
+ } else if (process.env.DISABLE_WORKER !== "true") {
253
+ setImmediate(() => claimSpecificJob(job.id).then((claimed) => claimed && dispatchClaimedJob(claimed)));
254
+ }
255
+ return job.id;
343
256
  }
344
- /** Get the current state of a platform job. */
345
- export async function getJob(tenantId, jobId) {
346
- return withTenant(tenantId, (tx) => tx.platformJob.findFirst({ where: { id: jobId, tenantId } }));
257
+ async function getJob(tenantId, jobId) {
258
+ return withTenant(
259
+ tenantId,
260
+ (tx) => tx.platformJob.findFirst({ where: { id: jobId, tenantId } })
261
+ );
347
262
  }
348
- /** List recent jobs for a project. */
349
- export async function listJobs(tenantId, projectId) {
350
- return withTenant(tenantId, async (tx) => {
351
- const jobs = await tx.platformJob.findMany({
352
- where: { tenantId, projectId },
353
- orderBy: { queuedAt: 'desc' },
354
- take: 50,
355
- });
356
- // Attach the most-recently-used model for each job from LLMCallLog
357
- const jobIds = jobs.map((j) => j.id);
358
- if (jobIds.length === 0)
359
- return jobs;
360
- const modelRows = await tx.lLMCallLog.groupBy({
361
- by: ['jobId'],
362
- where: { tenantId, jobId: { in: jobIds } },
363
- _max: { model: true, calledAt: true },
364
- });
365
- const modelByJobId = new Map(modelRows
366
- .filter((r) => r.jobId && r._max?.model)
367
- .map((r) => [r.jobId, r._max.model]));
368
- return jobs.map((j) => ({ ...j, llmModel: modelByJobId.get(j.id) ?? null }));
263
+ async function listJobs(tenantId, projectId) {
264
+ return withTenant(tenantId, async (tx) => {
265
+ const jobs = await tx.platformJob.findMany({
266
+ where: { tenantId, projectId },
267
+ orderBy: { queuedAt: "desc" },
268
+ take: 50
269
+ });
270
+ const jobIds = jobs.map((j) => j.id);
271
+ if (jobIds.length === 0) return jobs;
272
+ const modelRows = await tx.lLMCallLog.groupBy({
273
+ by: ["jobId"],
274
+ where: { tenantId, jobId: { in: jobIds } },
275
+ _max: { model: true, calledAt: true }
369
276
  });
277
+ const modelByJobId = new Map(
278
+ modelRows.filter((r) => r.jobId && r._max?.model).map((r) => [r.jobId, r._max.model])
279
+ );
280
+ return jobs.map((j) => ({ ...j, llmModel: modelByJobId.get(j.id) ?? null }));
281
+ });
370
282
  }
371
- /** Cancel a RUNNING or QUEUED job. Aborts the in-process crawl and marks DB CANCELLED. */
372
- export async function cancelJob(tenantId, jobId) {
373
- // Abort in-memory controller if job is running in the same process
374
- const controller = runningControllers.get(jobId);
375
- if (controller) {
376
- controller.abort();
377
- runningControllers.delete(jobId);
378
- }
379
- await withTenant(tenantId, (tx) => tx.platformJob.update({
380
- where: { id: jobId },
381
- data: { status: 'CANCELLED', finishedAt: new Date() },
382
- }));
283
+ async function cancelJob(tenantId, jobId) {
284
+ const controller = runningControllers.get(jobId);
285
+ if (controller) {
286
+ controller.abort();
287
+ runningControllers.delete(jobId);
288
+ }
289
+ await withTenant(
290
+ tenantId,
291
+ (tx) => tx.platformJob.update({
292
+ where: { id: jobId },
293
+ data: { status: "CANCELLED", finishedAt: /* @__PURE__ */ new Date() }
294
+ })
295
+ );
383
296
  }
384
- /** Re-enqueue a job using the same config as a previous job. */
385
- export async function rerunJob(tenantId, jobId) {
386
- const job = await withTenant(tenantId, (tx) => tx.platformJob.findUnique({ where: { id: jobId } }));
387
- if (!job)
388
- throw new Error('Job not found');
297
+ async function rerunJob(tenantId, jobId) {
298
+ const job = await withTenant(
299
+ tenantId,
300
+ (tx) => tx.platformJob.findUnique({ where: { id: jobId } })
301
+ );
302
+ if (!job) throw new Error("Job not found");
303
+ return enqueueJob(job.type, { ...job.config, tenantId });
304
+ }
305
+ const jobCheckpoints = /* @__PURE__ */ new Map();
306
+ async function resumeJob(tenantId, jobId) {
307
+ const job = await withTenant(
308
+ tenantId,
309
+ (tx) => tx.platformJob.findUnique({ where: { id: jobId } })
310
+ );
311
+ if (!job) throw new Error("Job not found");
312
+ const checkpoint = job.result?.checkpoint;
313
+ const visitedUrls = checkpoint?.visitedUrls ?? [];
314
+ const pendingQueue = checkpoint?.pendingQueue ?? [];
315
+ if (pendingQueue.length === 0 && visitedUrls.length === 0) {
389
316
  return enqueueJob(job.type, { ...job.config, tenantId });
317
+ }
318
+ return enqueueJob(job.type, {
319
+ ...job.config,
320
+ tenantId,
321
+ skipUrls: visitedUrls.length > 0 ? visitedUrls : void 0,
322
+ resumeUrls: pendingQueue.length > 0 ? pendingQueue : void 0
323
+ });
390
324
  }
391
- // In-memory checkpoint store: updated every 10 screens, saved to DB result on failure.
392
- const jobCheckpoints = new Map();
393
- export async function resumeJob(tenantId, jobId) {
394
- const job = await withTenant(tenantId, (tx) => tx.platformJob.findUnique({ where: { id: jobId } }));
395
- if (!job)
396
- throw new Error('Job not found');
397
- const checkpoint = job.result?.checkpoint;
398
- const visitedUrls = checkpoint?.visitedUrls ?? [];
399
- const pendingQueue = checkpoint?.pendingQueue ?? [];
400
- if (pendingQueue.length === 0 && visitedUrls.length === 0) {
401
- // No checkpoint saved — full rerun
402
- return enqueueJob(job.type, { ...job.config, tenantId });
403
- }
404
- return enqueueJob(job.type, {
405
- ...job.config,
406
- tenantId,
407
- skipUrls: visitedUrls.length > 0 ? visitedUrls : undefined,
408
- resumeUrls: pendingQueue.length > 0 ? pendingQueue : undefined,
325
+ async function updateJobProgress(tenantId, jobId, progress, step) {
326
+ try {
327
+ await withTenant(tenantId, async (tx) => {
328
+ const current = await tx.platformJob.findUnique({
329
+ where: { id: jobId },
330
+ select: { status: true }
331
+ });
332
+ if (current?.status === "CANCELLED") {
333
+ runningControllers.get(jobId)?.abort();
334
+ runningControllers.delete(jobId);
335
+ return;
336
+ }
337
+ await tx.platformJob.update({
338
+ where: { id: jobId },
339
+ data: { progress, result: { step }, heartbeatAt: /* @__PURE__ */ new Date() }
340
+ });
409
341
  });
342
+ } catch {
343
+ }
410
344
  }
411
- /** Update progress percentage and current step label on a running job (non-fatal). */
412
- export async function updateJobProgress(tenantId, jobId, progress, step) {
413
- try {
414
- await withTenant(tenantId, async (tx) => {
415
- // DB-flag cancellation: works for both inline and SQS backends.
416
- // When a user cancels, status is set to 'CANCELLED' in DB.
417
- // This check fires at each progress update (every few seconds during crawl).
418
- const current = await tx.platformJob.findUnique({
419
- where: { id: jobId },
420
- select: { status: true },
421
- });
422
- if (current?.status === 'CANCELLED') {
423
- runningControllers.get(jobId)?.abort();
424
- runningControllers.delete(jobId);
425
- return; // abort signal propagates; crawlProject exits on next signal check
426
- }
427
- await tx.platformJob.update({
428
- where: { id: jobId },
429
- data: { progress, result: { step }, heartbeatAt: new Date() },
430
- });
431
- });
432
- }
433
- catch { /* non-fatal — progress updates best-effort */ }
434
- }
435
- /** Bump only worker liveness. This intentionally does not touch progress/result. */
436
345
  async function heartbeatJob(tenantId, jobId) {
437
- try {
438
- await withTenant(tenantId, async (tx) => {
439
- const current = await tx.platformJob.findUnique({
440
- where: { id: jobId },
441
- select: { status: true },
442
- });
443
- if (current?.status === 'CANCELLED') {
444
- runningControllers.get(jobId)?.abort();
445
- runningControllers.delete(jobId);
446
- return;
447
- }
448
- if (current?.status !== 'RUNNING')
449
- return;
450
- await tx.platformJob.update({
451
- where: { id: jobId },
452
- data: { heartbeatAt: new Date(), workerId: WORKER_ID },
453
- });
454
- });
455
- }
456
- catch { /* non-fatal — heartbeat updates best-effort */ }
346
+ try {
347
+ await withTenant(tenantId, async (tx) => {
348
+ const current = await tx.platformJob.findUnique({
349
+ where: { id: jobId },
350
+ select: { status: true }
351
+ });
352
+ if (current?.status === "CANCELLED") {
353
+ runningControllers.get(jobId)?.abort();
354
+ runningControllers.delete(jobId);
355
+ return;
356
+ }
357
+ if (current?.status !== "RUNNING") return;
358
+ await tx.platformJob.update({
359
+ where: { id: jobId },
360
+ data: { heartbeatAt: /* @__PURE__ */ new Date(), workerId: WORKER_ID }
361
+ });
362
+ });
363
+ } catch {
364
+ }
457
365
  }
458
- /** Background: for each changed screen, search Jira + Confluence for related tickets/pages. */
459
366
  async function investigateScreenChanges(tenantId, projectId, changedScreens) {
460
- const [jiraCfg, confluenceCfg] = await Promise.all([
461
- AppBrain.getConnectorConfig(tenantId, projectId, 'JIRA'),
462
- AppBrain.getConnectorConfig(tenantId, projectId, 'CONFLUENCE'),
463
- ]);
464
- for (const screen of changedScreens) {
465
- const searchTerm = screen.url ?? screen.name;
466
- const changeSummary = `Screen "${screen.name}" changed: +${screen.addedElements.length} elements, -${screen.removedElements.length} elements`;
467
- const detail = {
468
- screenId: screen.id,
469
- screenName: screen.name,
470
- url: screen.url,
471
- addedElements: screen.addedElements,
472
- removedElements: screen.removedElements,
473
- jiraTickets: [],
474
- confluencePages: [],
475
- };
476
- if (jiraCfg?.baseUrl && jiraCfg?.email && jiraCfg?.apiToken) {
477
- try {
478
- detail.jiraTickets = await jira.searchIssues({ baseUrl: jiraCfg.baseUrl, email: jiraCfg.email, apiToken: jiraCfg.apiToken, projectKey: jiraCfg.projectKey }, searchTerm);
479
- }
480
- catch (e) {
481
- logger.warn({ searchTerm, err: e }, 'Jira search failed');
482
- }
483
- }
484
- if (confluenceCfg?.baseUrl && confluenceCfg?.email && confluenceCfg?.apiToken) {
485
- try {
486
- detail.confluencePages = await confluence.searchPages({ baseUrl: confluenceCfg.baseUrl, email: confluenceCfg.email, apiToken: confluenceCfg.apiToken, spaceKey: confluenceCfg.spaceKey }, searchTerm);
487
- }
488
- catch (e) {
489
- logger.warn({ searchTerm, err: e }, 'Confluence search failed');
490
- }
491
- }
492
- await AppBrain.logChange(tenantId, projectId, 'MODIFIED', changeSummary, detail);
493
- logger.info({ screen: screen.name, jiraTickets: detail.jiraTickets.length, confluencePages: detail.confluencePages.length }, 'Logged screen change investigation');
367
+ const [jiraCfg, confluenceCfg] = await Promise.all([
368
+ AppBrain.getConnectorConfig(tenantId, projectId, "JIRA"),
369
+ AppBrain.getConnectorConfig(tenantId, projectId, "CONFLUENCE")
370
+ ]);
371
+ for (const screen of changedScreens) {
372
+ const searchTerm = screen.url ?? screen.name;
373
+ const changeSummary = `Screen "${screen.name}" changed: +${screen.addedElements.length} elements, -${screen.removedElements.length} elements`;
374
+ const detail = {
375
+ screenId: screen.id,
376
+ screenName: screen.name,
377
+ url: screen.url,
378
+ addedElements: screen.addedElements,
379
+ removedElements: screen.removedElements,
380
+ jiraTickets: [],
381
+ confluencePages: []
382
+ };
383
+ if (jiraCfg?.baseUrl && jiraCfg?.email && jiraCfg?.apiToken) {
384
+ try {
385
+ detail.jiraTickets = await jira.searchIssues(
386
+ { baseUrl: jiraCfg.baseUrl, email: jiraCfg.email, apiToken: jiraCfg.apiToken, projectKey: jiraCfg.projectKey },
387
+ searchTerm
388
+ );
389
+ } catch (e) {
390
+ logger.warn({ searchTerm, err: e }, "Jira search failed");
391
+ }
494
392
  }
393
+ if (confluenceCfg?.baseUrl && confluenceCfg?.email && confluenceCfg?.apiToken) {
394
+ try {
395
+ detail.confluencePages = await confluence.searchPages(
396
+ { baseUrl: confluenceCfg.baseUrl, email: confluenceCfg.email, apiToken: confluenceCfg.apiToken, spaceKey: confluenceCfg.spaceKey },
397
+ searchTerm
398
+ );
399
+ } catch (e) {
400
+ logger.warn({ searchTerm, err: e }, "Confluence search failed");
401
+ }
402
+ }
403
+ await AppBrain.logChange(tenantId, projectId, "MODIFIED", changeSummary, detail);
404
+ logger.info({ screen: screen.name, jiraTickets: detail.jiraTickets.length, confluencePages: detail.confluencePages.length }, "Logged screen change investigation");
405
+ }
495
406
  }
496
- export async function processJob(jobId, config, opts) {
497
- const { tenantId, projectId } = config;
498
- // Mark as RUNNING — skip if already CANCELLED (race: user cancelled before worker picked it up).
499
- const jobStartedAt = new Date();
500
- const claimed = await withTenant(tenantId, (tx) => tx.platformJob.updateMany({
501
- where: { id: jobId, status: { not: 'CANCELLED' } },
502
- data: { status: 'RUNNING', startedAt: jobStartedAt, heartbeatAt: jobStartedAt, workerId: WORKER_ID },
503
- }));
504
- if (claimed.count === 0)
505
- return;
506
- const controller = new AbortController();
507
- runningControllers.set(jobId, controller);
508
- const heartbeatTimer = setInterval(() => {
509
- void heartbeatJob(tenantId, jobId);
510
- }, JOB_HEARTBEAT_INTERVAL_MS);
511
- const onProgress = async (pct, step) => {
512
- await updateJobProgress(tenantId, jobId, pct, step);
513
- await opts?.onProgress?.(pct);
514
- };
515
- const DEFAULT_MAX_SCREENS = 150;
516
- try {
517
- // Per-project maxScreens + deviceProfiles + seedUrls + interactive probing + sitemap seeding + traversal strategy from crawlConfig override job config defaults.
518
- if (!config.maxScreens || !config.deviceProfiles || config.interactive === undefined || config.useSitemap === undefined || !config.traversalStrategy) {
519
- const proj = await withTenant(tenantId, (tx) => tx.project.findUnique({ where: { id: projectId }, select: { crawlConfig: true } }));
520
- const perProject = proj?.crawlConfig?.maxScreens;
521
- if (!config.maxScreens)
522
- config.maxScreens = perProject || DEFAULT_MAX_SCREENS;
523
- if (!config.deviceProfiles)
524
- config.deviceProfiles = proj?.crawlConfig?.deviceProfiles ?? ['DESKTOP'];
525
- if (config.interactive === undefined)
526
- config.interactive = proj?.crawlConfig?.interactive ?? false;
527
- if (config.useSitemap === undefined)
528
- config.useSitemap = proj?.crawlConfig?.useSitemap ?? false;
529
- if (!config.sitemapUrl)
530
- config.sitemapUrl = proj?.crawlConfig?.sitemapUrl ?? undefined;
531
- if (!config.traversalStrategy)
532
- config.traversalStrategy = proj?.crawlConfig?.traversalStrategy ?? 'BFS';
533
- if (!config.bestFirstKeywords)
534
- config.bestFirstKeywords = proj?.crawlConfig?.bestFirstKeywords ?? undefined;
535
- // segmentSize from crawl config — default unlimited (9_999_999) so normal sites run in one job;
536
- // only segment if the project explicitly sets a segmentSize in its crawl config.
537
- if (!config.segmentSize)
538
- config.segmentSize = proj?.crawlConfig?.segmentSize ?? 9_999_999;
539
- // Load saved seedUrls as non-selective seeds (link discovery still runs from them).
540
- // Skip when a selective recrawl is requested (selectiveUrls set) or seeds already provided.
541
- if (!config.selectiveUrls?.length && (!config.startUrls || config.startUrls.length === 0)) {
542
- const saved = proj?.crawlConfig?.seedUrls ?? [];
543
- if (saved.length > 0)
544
- config.startUrls = saved;
545
- }
546
- if (config.interactive === undefined)
547
- config.interactive = proj?.crawlConfig?.interactive ?? false;
407
+ async function processJob(jobId, config, opts) {
408
+ const { tenantId, projectId } = config;
409
+ const jobStartedAt = /* @__PURE__ */ new Date();
410
+ const claimed = await withTenant(
411
+ tenantId,
412
+ (tx) => tx.platformJob.updateMany({
413
+ where: { id: jobId, status: { not: "CANCELLED" } },
414
+ data: { status: "RUNNING", startedAt: jobStartedAt, heartbeatAt: jobStartedAt, workerId: WORKER_ID }
415
+ })
416
+ );
417
+ if (claimed.count === 0) return;
418
+ const controller = new AbortController();
419
+ runningControllers.set(jobId, controller);
420
+ const heartbeatTimer = setInterval(() => {
421
+ void heartbeatJob(tenantId, jobId);
422
+ }, JOB_HEARTBEAT_INTERVAL_MS);
423
+ const onProgress = async (pct, step) => {
424
+ await updateJobProgress(tenantId, jobId, pct, step);
425
+ await opts?.onProgress?.(pct);
426
+ };
427
+ const DEFAULT_MAX_SCREENS = 150;
428
+ try {
429
+ if (!config.maxScreens || !config.deviceProfiles || config.interactive === void 0 || config.useSitemap === void 0 || !config.traversalStrategy) {
430
+ const proj = await withTenant(
431
+ tenantId,
432
+ (tx) => tx.project.findUnique({ where: { id: projectId }, select: { crawlConfig: true } })
433
+ );
434
+ const perProject = proj?.crawlConfig?.maxScreens;
435
+ if (!config.maxScreens) config.maxScreens = perProject || DEFAULT_MAX_SCREENS;
436
+ if (!config.deviceProfiles) config.deviceProfiles = proj?.crawlConfig?.deviceProfiles ?? ["DESKTOP"];
437
+ if (config.interactive === void 0) config.interactive = proj?.crawlConfig?.interactive ?? false;
438
+ if (config.useSitemap === void 0) config.useSitemap = proj?.crawlConfig?.useSitemap ?? false;
439
+ if (!config.sitemapUrl) config.sitemapUrl = proj?.crawlConfig?.sitemapUrl ?? void 0;
440
+ if (!config.traversalStrategy) config.traversalStrategy = proj?.crawlConfig?.traversalStrategy ?? "BFS";
441
+ if (!config.bestFirstKeywords) config.bestFirstKeywords = proj?.crawlConfig?.bestFirstKeywords ?? void 0;
442
+ if (!config.segmentSize) config.segmentSize = proj?.crawlConfig?.segmentSize ?? 9999999;
443
+ if (!config.selectiveUrls?.length && (!config.startUrls || config.startUrls.length === 0)) {
444
+ const saved = proj?.crawlConfig?.seedUrls ?? [];
445
+ if (saved.length > 0) config.startUrls = saved;
446
+ }
447
+ if (config.interactive === void 0) config.interactive = proj?.crawlConfig?.interactive ?? false;
448
+ }
449
+ const runtimeCreds = config.credentials ? {
450
+ ...config.credentials,
451
+ ...config.credentials.totpSecret ? { getOneTimeCode: async () => generateTOTP(config.credentials.totpSecret) } : {}
452
+ } : void 0;
453
+ const storageConfigForCrawl = await getTenantStorageConfig(tenantId).catch(() => null);
454
+ const result = await crawlProject({
455
+ ...config,
456
+ credentials: runtimeCreds,
457
+ abortSignal: controller.signal,
458
+ jobId,
459
+ storageConfig: storageConfigForCrawl,
460
+ onCheckpoint: async (visitedUrls, pendingQueue) => {
461
+ jobCheckpoints.set(jobId, { visitedUrls, pendingQueue });
462
+ }
463
+ }, onProgress);
464
+ jobCheckpoints.delete(jobId);
465
+ const pendingUrls = result?.pendingUrls ?? [];
466
+ if (pendingUrls.length > 0) {
467
+ logger.info({ pendingUrls: pendingUrls.length }, "Crawl finished with pending URLs \u2014 not auto-queuing (segmented crawls disabled)");
468
+ }
469
+ await withTenant(
470
+ tenantId,
471
+ (tx) => tx.platformJob.update({
472
+ where: { id: jobId },
473
+ data: {
474
+ status: "DONE",
475
+ result: { ...result },
476
+ progress: 100,
477
+ finishedAt: /* @__PURE__ */ new Date()
548
478
  }
549
- // Build runtime credentials: convert stored totpSecret → getOneTimeCode function
550
- const runtimeCreds = config.credentials
551
- ? {
552
- ...config.credentials,
553
- ...(config.credentials.totpSecret
554
- ? { getOneTimeCode: async () => generateTOTP(config.credentials.totpSecret) }
555
- : {}),
479
+ })
480
+ );
481
+ clearInterval(heartbeatTimer);
482
+ runningControllers.delete(jobId);
483
+ {
484
+ const crawledUrls = (result.discovered ?? []).filter((d) => !!d.screenId && !!d.url).map((d) => d.url);
485
+ if (crawledUrls.length > 0) {
486
+ (async () => {
487
+ const cfg = await AppBrain.getCrawlConfig(tenantId, projectId);
488
+ const existing = cfg?.seedUrls ?? [];
489
+ const merged = Array.from(/* @__PURE__ */ new Set([...existing, ...crawledUrls]));
490
+ if (merged.length > existing.length) {
491
+ await AppBrain.saveCrawlConfig(tenantId, projectId, { seedUrls: merged });
492
+ logger.info({ added: merged.length - existing.length, total: merged.length }, "[crawl] auto-seeded crawled URLs into crawlConfig.seedUrls");
493
+ }
494
+ })().catch((err) => logger.warn({ err: err?.message }, "[crawl] auto-seed seedUrls failed (non-fatal)"));
495
+ }
496
+ notifyAdmins(tenantId, {
497
+ type: "crawl.done",
498
+ title: "Crawl completed",
499
+ body: `Crawl for project finished successfully.`,
500
+ metadata: { projectId, jobId }
501
+ }).catch(console.error);
502
+ dispatchWebhook(tenantId, projectId, "crawl.done", {
503
+ jobId,
504
+ screensFound: result?.screensFound ?? result?.screens?.length ?? 0
505
+ }).catch(console.error);
506
+ }
507
+ setTimeout(() => {
508
+ aggregateJobTokens(tenantId, jobId, projectId, jobStartedAt).catch(console.error);
509
+ }, 8e3);
510
+ setImmediate(async () => {
511
+ try {
512
+ const projectMap = await AppBrain.getProjectMap(tenantId, projectId);
513
+ if (projectMap?.screens?.length) {
514
+ const screenCount = projectMap.screens.length;
515
+ if (screenCount > 300) {
516
+ await withTenant(
517
+ tenantId,
518
+ (tx) => tx.project.update({ where: { id: projectId }, data: { oversized: true } })
519
+ );
520
+ }
521
+ await generateTaxonomySkillFiles(tenantId, projectId, projectMap.screens, result.apiEndpoints, result.performanceMetrics);
522
+ try {
523
+ const { readFile } = await import("node:fs/promises");
524
+ const { existsSync } = await import("node:fs");
525
+ const screensWithDom = projectMap.screens.filter((s) => s.domSnapshotPath);
526
+ const detectLimit = Math.min(screensWithDom.length, 20);
527
+ let fieldDetectAdded = 0;
528
+ for (let si = 0; si < detectLimit; si++) {
529
+ const screen = screensWithDom[si];
530
+ try {
531
+ let htmlContent = null;
532
+ const domPath = screen.domSnapshotPath;
533
+ if (domPath.startsWith("https://") || domPath.startsWith("http://")) {
534
+ const res = await fetch(domPath);
535
+ if (res.ok) htmlContent = await res.text();
536
+ } else if (existsSync(domPath)) {
537
+ htmlContent = await readFile(domPath, "utf8");
538
+ }
539
+ if (!htmlContent) continue;
540
+ const detected = await detectFieldsFromHtml({ tenantId, projectId, html: htmlContent });
541
+ if (detected.length > 0) {
542
+ const merged = await mergeIntoCatalog({ tenantId, projectId, detected });
543
+ fieldDetectAdded += merged.added;
544
+ }
545
+ } catch (sErr) {
546
+ logger.warn({ screenId: screen.id, err: sErr }, "[field-detect] skipped screen");
547
+ }
556
548
  }
557
- : undefined;
558
- const storageConfigForCrawl = await getTenantStorageConfig(tenantId).catch(() => null);
559
- const result = await crawlProject({
560
- ...config, credentials: runtimeCreds, abortSignal: controller.signal, jobId,
561
- storageConfig: storageConfigForCrawl,
562
- onCheckpoint: async (visitedUrls, pendingQueue) => {
563
- jobCheckpoints.set(jobId, { visitedUrls, pendingQueue });
564
- },
565
- }, onProgress);
566
- jobCheckpoints.delete(jobId);
567
- const pendingUrls = result?.pendingUrls ?? [];
568
- if (pendingUrls.length > 0) {
569
- logger.info({ pendingUrls: pendingUrls.length }, 'Crawl finished with pending URLs — not auto-queuing (segmented crawls disabled)');
570
- }
571
- await withTenant(tenantId, (tx) => tx.platformJob.update({
572
- where: { id: jobId },
573
- data: {
574
- status: 'DONE',
575
- result: { ...result },
576
- progress: 100,
577
- finishedAt: new Date(),
578
- },
579
- }));
580
- clearInterval(heartbeatTimer);
581
- runningControllers.delete(jobId);
582
- {
583
- // Auto-seed: merge all successfully crawled URLs into crawlConfig.seedUrls so future
584
- // crawls and discovery agents start from a complete picture of the app.
585
- const crawledUrls = (result.discovered ?? [])
586
- .filter((d) => !!d.screenId && !!d.url)
587
- .map((d) => d.url);
588
- if (crawledUrls.length > 0) {
589
- (async () => {
590
- const cfg = await AppBrain.getCrawlConfig(tenantId, projectId);
591
- const existing = cfg?.seedUrls ?? [];
592
- const merged = Array.from(new Set([...existing, ...crawledUrls]));
593
- if (merged.length > existing.length) {
594
- await AppBrain.saveCrawlConfig(tenantId, projectId, { seedUrls: merged });
595
- logger.info({ added: merged.length - existing.length, total: merged.length }, '[crawl] auto-seeded crawled URLs into crawlConfig.seedUrls');
596
- }
597
- })().catch((err) => logger.warn({ err: err?.message }, '[crawl] auto-seed seedUrls failed (non-fatal)'));
549
+ if (fieldDetectAdded > 0 || detectLimit > 0) {
550
+ logger.info({ screens: detectLimit, added: fieldDetectAdded, projectId }, "[field-detect] catalog populated from crawl");
598
551
  }
599
- notifyAdmins(tenantId, {
600
- type: 'crawl.done',
601
- title: 'Crawl completed',
602
- body: `Crawl for project finished successfully.`,
603
- metadata: { projectId, jobId },
604
- }).catch(console.error);
605
- dispatchWebhook(tenantId, projectId, 'crawl.done', {
606
- jobId,
607
- screensFound: result?.screensFound ?? result?.screens?.length ?? 0,
608
- }).catch(console.error);
609
- }
610
- // Aggregate token usage from LLM calls made during this job.
611
- // Delay 8s to let concurrent recordLLMCall writes settle before querying LLMCallLog.
612
- setTimeout(() => {
613
- aggregateJobTokens(tenantId, jobId, projectId, jobStartedAt).catch(console.error);
614
- }, 8000);
615
- // Post-crawl: generate / refresh all taxonomy skill files + investigate changes.
616
- setImmediate(async () => {
617
- try {
618
- const projectMap = await AppBrain.getProjectMap(tenantId, projectId);
619
- if (projectMap?.screens?.length) {
620
- // Mark project oversized when > 300 screens — activates VectorSearchGrounding
621
- const screenCount = projectMap.screens.length;
622
- if (screenCount > 300) {
623
- await withTenant(tenantId, (tx) => tx.project.update({ where: { id: projectId }, data: { oversized: true } }));
624
- }
625
- await generateTaxonomySkillFiles(tenantId, projectId, projectMap.screens, result.apiEndpoints, result.performanceMetrics);
626
- // Auto-detect AppField catalog from crawled DOM snapshots.
627
- // Runs before test generation so getFieldsForCase() returns typed fields (FORM mode).
628
- // Capped at 20 screens and only where domSnapshotPath exists to bound LLM cost.
629
- try {
630
- const { readFile } = await import('node:fs/promises');
631
- const { existsSync } = await import('node:fs');
632
- const screensWithDom = projectMap.screens.filter((s) => s.domSnapshotPath);
633
- const detectLimit = Math.min(screensWithDom.length, 20);
634
- let fieldDetectAdded = 0;
635
- for (let si = 0; si < detectLimit; si++) {
636
- const screen = screensWithDom[si];
637
- try {
638
- let htmlContent = null;
639
- const domPath = screen.domSnapshotPath;
640
- if (domPath.startsWith('https://') || domPath.startsWith('http://')) {
641
- const res = await fetch(domPath);
642
- if (res.ok)
643
- htmlContent = await res.text();
644
- }
645
- else if (existsSync(domPath)) {
646
- htmlContent = await readFile(domPath, 'utf8');
647
- }
648
- if (!htmlContent)
649
- continue;
650
- const detected = await detectFieldsFromHtml({ tenantId, projectId, html: htmlContent });
651
- if (detected.length > 0) {
652
- const merged = await mergeIntoCatalog({ tenantId, projectId, detected });
653
- fieldDetectAdded += merged.added;
654
- }
655
- }
656
- catch (sErr) {
657
- logger.warn({ screenId: screen.id, err: sErr }, '[field-detect] skipped screen');
658
- }
659
- }
660
- if (fieldDetectAdded > 0 || detectLimit > 0) {
661
- logger.info({ screens: detectLimit, added: fieldDetectAdded, projectId }, '[field-detect] catalog populated from crawl');
662
- }
663
- }
664
- catch (fdErr) {
665
- logger.warn({ err: fdErr }, '[field-detect] post-crawl field detection failed');
666
- }
667
- // Cloud storage: upload screenshots to tenant S3/R2 if configured
668
- const tenantStorageCfg = await getTenantStorageConfig(tenantId).catch(() => null);
669
- if (tenantStorageCfg) {
670
- let uploaded = 0;
671
- for (const screen of projectMap.screens) {
672
- for (const shot of (screen.screenshots ?? [])) {
673
- if (!shot.storagePath || shot.cloudUrl)
674
- continue;
675
- try {
676
- const filename = `${screen.id}-v${shot.version}.png`;
677
- const key = buildArtifactKey(tenantId, projectId, 'screenshots', filename, tenantStorageCfg.prefix);
678
- const cloudUrl = await uploadScreenshotToTenantStorage(tenantStorageCfg, shot.storagePath, key);
679
- await AppBrain.updateScreenshotCloudUrl(tenantId, shot.id, cloudUrl);
680
- uploaded++;
681
- }
682
- catch (uploadErr) {
683
- logger.warn({ screenId: screen.id, err: uploadErr }, 'Failed to upload screenshot to cloud storage');
684
- }
685
- }
686
- }
687
- if (uploaded > 0)
688
- logger.info({ uploaded, provider: tenantStorageCfg.provider }, 'Uploaded screenshots to cloud storage');
689
- // Upload DOM snapshots and markdown to tenant storage
690
- const { existsSync } = await import('node:fs');
691
- let domUploaded = 0;
692
- let mdUploaded = 0;
693
- for (const screen of projectMap.screens) {
694
- // DOM snapshot
695
- const domPath = screen.domSnapshotPath;
696
- if (domPath && !domPath.startsWith('https://') && existsSync(domPath)) {
697
- try {
698
- const key = buildArtifactKey(tenantId, projectId, 'dom-snapshots', `${screen.id}.json`, tenantStorageCfg.prefix);
699
- const cloudUrl = await uploadFileToTenantStorage(tenantStorageCfg, domPath, key, 'application/json');
700
- await AppBrain.updateScreenDomSnapshot(tenantId, screen.id, cloudUrl);
701
- domUploaded++;
702
- }
703
- catch (err) {
704
- logger.warn({ screenId: screen.id, err }, 'Failed to upload DOM snapshot to cloud storage');
705
- }
706
- }
707
- // Markdown extract
708
- const mdPath = screen.markdownPath;
709
- if (mdPath && !mdPath.startsWith('https://') && existsSync(mdPath)) {
710
- try {
711
- const key = buildArtifactKey(tenantId, projectId, 'markdown', `${screen.id}.md`, tenantStorageCfg.prefix);
712
- const cloudUrl = await uploadFileToTenantStorage(tenantStorageCfg, mdPath, key, 'text/markdown');
713
- await AppBrain.updateScreenMarkdownPath(tenantId, screen.id, cloudUrl);
714
- mdUploaded++;
715
- }
716
- catch (err) {
717
- logger.warn({ screenId: screen.id, err }, 'Failed to upload markdown to cloud storage');
718
- }
719
- }
720
- }
721
- if (domUploaded > 0)
722
- logger.info({ domUploaded, provider: tenantStorageCfg.provider }, 'Uploaded DOM snapshots to cloud storage');
723
- if (mdUploaded > 0)
724
- logger.info({ mdUploaded, provider: tenantStorageCfg.provider }, 'Uploaded markdown to cloud storage');
725
- }
726
- // AI screen grouping — clusters screens into semantic feature areas
727
- await groupScreensWithAI(tenantId, projectId, projectMap.screens, jobId, jobStartedAt);
728
- // Auto-flag stale tests: mark PENDING_REVIEW when screen elements changed since test was generated
729
- try {
730
- await withTenant(tenantId, async (tx) => {
731
- const staleTests = await tx.$queryRaw `
552
+ } catch (fdErr) {
553
+ logger.warn({ err: fdErr }, "[field-detect] post-crawl field detection failed");
554
+ }
555
+ const tenantStorageCfg = await getTenantStorageConfig(tenantId).catch(() => null);
556
+ if (tenantStorageCfg) {
557
+ let uploaded = 0;
558
+ for (const screen of projectMap.screens) {
559
+ for (const shot of screen.screenshots ?? []) {
560
+ if (!shot.storagePath || shot.cloudUrl) continue;
561
+ try {
562
+ const filename = `${screen.id}-v${shot.version}.png`;
563
+ const key = buildArtifactKey(tenantId, projectId, "screenshots", filename, tenantStorageCfg.prefix);
564
+ const cloudUrl = await uploadScreenshotToTenantStorage(tenantStorageCfg, shot.storagePath, key);
565
+ await AppBrain.updateScreenshotCloudUrl(tenantId, shot.id, cloudUrl);
566
+ uploaded++;
567
+ } catch (uploadErr) {
568
+ logger.warn({ screenId: screen.id, err: uploadErr }, "Failed to upload screenshot to cloud storage");
569
+ }
570
+ }
571
+ }
572
+ if (uploaded > 0) logger.info({ uploaded, provider: tenantStorageCfg.provider }, "Uploaded screenshots to cloud storage");
573
+ const { existsSync } = await import("node:fs");
574
+ let domUploaded = 0;
575
+ let mdUploaded = 0;
576
+ for (const screen of projectMap.screens) {
577
+ const domPath = screen.domSnapshotPath;
578
+ if (domPath && !domPath.startsWith("https://") && existsSync(domPath)) {
579
+ try {
580
+ const key = buildArtifactKey(tenantId, projectId, "dom-snapshots", `${screen.id}.json`, tenantStorageCfg.prefix);
581
+ const cloudUrl = await uploadFileToTenantStorage(tenantStorageCfg, domPath, key, "application/json");
582
+ await AppBrain.updateScreenDomSnapshot(tenantId, screen.id, cloudUrl);
583
+ domUploaded++;
584
+ } catch (err) {
585
+ logger.warn({ screenId: screen.id, err }, "Failed to upload DOM snapshot to cloud storage");
586
+ }
587
+ }
588
+ const mdPath = screen.markdownPath;
589
+ if (mdPath && !mdPath.startsWith("https://") && existsSync(mdPath)) {
590
+ try {
591
+ const key = buildArtifactKey(tenantId, projectId, "markdown", `${screen.id}.md`, tenantStorageCfg.prefix);
592
+ const cloudUrl = await uploadFileToTenantStorage(tenantStorageCfg, mdPath, key, "text/markdown");
593
+ await AppBrain.updateScreenMarkdownPath(tenantId, screen.id, cloudUrl);
594
+ mdUploaded++;
595
+ } catch (err) {
596
+ logger.warn({ screenId: screen.id, err }, "Failed to upload markdown to cloud storage");
597
+ }
598
+ }
599
+ }
600
+ if (domUploaded > 0) logger.info({ domUploaded, provider: tenantStorageCfg.provider }, "Uploaded DOM snapshots to cloud storage");
601
+ if (mdUploaded > 0) logger.info({ mdUploaded, provider: tenantStorageCfg.provider }, "Uploaded markdown to cloud storage");
602
+ }
603
+ await groupScreensWithAI(tenantId, projectId, projectMap.screens, jobId, jobStartedAt);
604
+ try {
605
+ await withTenant(tenantId, async (tx) => {
606
+ const staleTests = await tx.$queryRaw`
732
607
  SELECT tc.id
733
608
  FROM "TestCase" tc
734
609
  JOIN "Screen" s ON tc."screenId" = s.id
@@ -740,69 +615,65 @@ export async function processJob(jobId, config, opts) {
740
615
  AND tc.status != 'PENDING_REVIEW'
741
616
  AND tc.status != 'RETIRED'
742
617
  `;
743
- if (staleTests.length > 0) {
744
- const ids = staleTests.map((t) => t.id);
745
- await tx.testCase.updateMany({
746
- where: { id: { in: ids } },
747
- data: { status: 'PENDING_REVIEW' },
748
- });
749
- logger.info({ count: staleTests.length, projectId }, 'Auto-flagged stale test cases as PENDING_REVIEW');
750
- }
751
- });
752
- }
753
- catch (staleErr) {
754
- logger.warn({ err: staleErr }, 'Failed to auto-flag stale tests');
755
- }
756
- const changed = projectMap.screens.filter((s) => s.hasChange);
757
- if (changed.length > 0) {
758
- await investigateScreenChanges(tenantId, projectId, changed);
759
- }
760
- }
761
- }
762
- catch (skillErr) {
763
- logger.error({ projectId, err: skillErr }, 'Taxonomy generation failed');
764
- }
765
- });
766
- }
767
- catch (err) {
768
- clearInterval(heartbeatTimer);
769
- runningControllers.delete(jobId);
770
- const aborted = String(err).includes('CRAWL_ABORTED');
771
- const checkpoint = jobCheckpoints.get(jobId);
772
- jobCheckpoints.delete(jobId);
773
- await withTenant(tenantId, (tx) => tx.platformJob.update({
774
- where: { id: jobId },
775
- data: {
776
- status: aborted ? 'CANCELLED' : 'FAILED',
777
- error: aborted ? 'Stopped by user' : String(err),
778
- finishedAt: new Date(),
779
- ...(checkpoint ? { result: { checkpoint } } : {}),
780
- },
781
- }));
782
- // Notify on crawl failure (fire-and-forget)
783
- if (!aborted) {
784
- notifyAdmins(tenantId, {
785
- type: 'crawl.failed',
786
- title: 'Crawl failed',
787
- body: `Crawl for project failed: ${String(err)}`,
788
- metadata: { projectId, jobId, error: String(err) },
789
- }).catch(console.error);
790
- // Dispatch outbound webhooks (fire-and-forget)
791
- dispatchWebhook(tenantId, projectId, 'crawl.failed', {
792
- jobId,
793
- error: String(err),
794
- }).catch(console.error);
618
+ if (staleTests.length > 0) {
619
+ const ids = staleTests.map((t) => t.id);
620
+ await tx.testCase.updateMany({
621
+ where: { id: { in: ids } },
622
+ data: { status: "PENDING_REVIEW" }
623
+ });
624
+ logger.info({ count: staleTests.length, projectId }, "Auto-flagged stale test cases as PENDING_REVIEW");
625
+ }
626
+ });
627
+ } catch (staleErr) {
628
+ logger.warn({ err: staleErr }, "Failed to auto-flag stale tests");
629
+ }
630
+ const changed = projectMap.screens.filter((s) => s.hasChange);
631
+ if (changed.length > 0) {
632
+ await investigateScreenChanges(tenantId, projectId, changed);
633
+ }
795
634
  }
635
+ } catch (skillErr) {
636
+ logger.error({ projectId, err: skillErr }, "Taxonomy generation failed");
637
+ }
638
+ });
639
+ } catch (err) {
640
+ clearInterval(heartbeatTimer);
641
+ runningControllers.delete(jobId);
642
+ const aborted = String(err).includes("CRAWL_ABORTED");
643
+ const checkpoint = jobCheckpoints.get(jobId);
644
+ jobCheckpoints.delete(jobId);
645
+ await withTenant(
646
+ tenantId,
647
+ (tx) => tx.platformJob.update({
648
+ where: { id: jobId },
649
+ data: {
650
+ status: aborted ? "CANCELLED" : "FAILED",
651
+ error: aborted ? "Stopped by user" : String(err),
652
+ finishedAt: /* @__PURE__ */ new Date(),
653
+ ...checkpoint ? { result: { checkpoint } } : {}
654
+ }
655
+ })
656
+ );
657
+ if (!aborted) {
658
+ notifyAdmins(tenantId, {
659
+ type: "crawl.failed",
660
+ title: "Crawl failed",
661
+ body: `Crawl for project failed: ${String(err)}`,
662
+ metadata: { projectId, jobId, error: String(err) }
663
+ }).catch(console.error);
664
+ dispatchWebhook(tenantId, projectId, "crawl.failed", {
665
+ jobId,
666
+ error: String(err)
667
+ }).catch(console.error);
796
668
  }
797
- finally {
798
- runningControllers.delete(jobId);
799
- }
669
+ } finally {
670
+ runningControllers.delete(jobId);
671
+ }
800
672
  }
801
- /** After a job finishes, sum LLMCallLog entries for that project during the job window. */
802
- export async function aggregateJobTokens(tenantId, jobId, projectId, startedAt) {
803
- try {
804
- await withTenant(tenantId, async (tx) => {
805
- const rows = await tx.$queryRaw `
673
+ async function aggregateJobTokens(tenantId, jobId, projectId, startedAt) {
674
+ try {
675
+ await withTenant(tenantId, async (tx) => {
676
+ const rows = await tx.$queryRaw`
806
677
  SELECT
807
678
  COALESCE(SUM("promptTokens"), 0)::INT AS "totalPromptTokens",
808
679
  COALESCE(SUM("completionTokens"), 0)::INT AS "totalCompletionTokens",
@@ -813,252 +684,297 @@ export async function aggregateJobTokens(tenantId, jobId, projectId, startedAt)
813
684
  WHERE "tenantId" = ${tenantId}
814
685
  AND "jobId" = ${jobId}
815
686
  `;
816
- const row = rows[0];
817
- if (!row || !row.llmCallCount)
818
- return;
819
- await tx.platformJob.update({
820
- where: { id: jobId },
821
- data: {
822
- totalPromptTokens: row.totalPromptTokens,
823
- totalCompletionTokens: row.totalCompletionTokens,
824
- totalCacheReadTokens: row.totalCacheReadTokens,
825
- totalEstimatedCostUsd: Number(row.totalEstimatedCostUsd),
826
- llmCallCount: row.llmCallCount,
827
- },
828
- });
829
- });
830
- }
831
- catch (err) {
832
- console.warn('[job-tokens] aggregation failed:', err);
833
- }
687
+ const row = rows[0];
688
+ if (!row || !row.llmCallCount) return;
689
+ await tx.platformJob.update({
690
+ where: { id: jobId },
691
+ data: {
692
+ totalPromptTokens: row.totalPromptTokens,
693
+ totalCompletionTokens: row.totalCompletionTokens,
694
+ totalCacheReadTokens: row.totalCacheReadTokens,
695
+ totalEstimatedCostUsd: Number(row.totalEstimatedCostUsd),
696
+ llmCallCount: row.llmCallCount
697
+ }
698
+ });
699
+ });
700
+ } catch (err) {
701
+ console.warn("[job-tokens] aggregation failed:", err);
702
+ }
834
703
  }
835
704
  async function processApiTestsJob(jobId, config) {
836
- const { tenantId, projectId } = config;
837
- const jobStartedAt = new Date();
838
- await withTenant(tenantId, (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: 'RUNNING', startedAt: jobStartedAt } }));
839
- try {
840
- const { generateApiTests, refreshCoverageMatrix, refreshTestCaseInventory } = await import('@detiq/agents');
841
- await updateJobProgress(tenantId, jobId, 20, 'Loading API contracts and endpoints…');
842
- const result = await generateApiTests({ tenantId, projectId, apiEndpoints: config.apiEndpoints });
843
- await updateJobProgress(tenantId, jobId, 90, `Generated ${result.cases?.length ?? 0} API test cases`);
844
- refreshCoverageMatrix(tenantId, projectId).catch((e) => console.warn('[coverage-matrix]', e));
845
- refreshTestCaseInventory(tenantId, projectId).catch((e) => console.warn('[tc-inventory]', e));
846
- refreshTestCasesSkillFile(tenantId, projectId).catch(console.error);
847
- await withTenant(tenantId, (tx) => tx.platformJob.update({
848
- where: { id: jobId },
849
- data: {
850
- status: 'DONE', progress: 100, finishedAt: new Date(),
851
- result: { cases: result.cases?.length ?? 0, suiteName: result.suiteName },
852
- },
853
- }));
854
- aggregateJobTokens(tenantId, jobId, projectId, jobStartedAt).catch(console.error);
855
- }
856
- catch (err) {
857
- await withTenant(tenantId, (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', error: String(err), finishedAt: new Date() } }));
858
- }
705
+ const { tenantId, projectId } = config;
706
+ const jobStartedAt = /* @__PURE__ */ new Date();
707
+ await withTenant(
708
+ tenantId,
709
+ (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "RUNNING", startedAt: jobStartedAt } })
710
+ );
711
+ try {
712
+ const { generateApiTests, refreshCoverageMatrix, refreshTestCaseInventory } = await import("@detiq/agents");
713
+ await updateJobProgress(tenantId, jobId, 20, "Loading API contracts and endpoints\u2026");
714
+ const result = await generateApiTests({ tenantId, projectId, apiEndpoints: config.apiEndpoints });
715
+ await updateJobProgress(tenantId, jobId, 90, `Generated ${result.cases?.length ?? 0} API test cases`);
716
+ refreshCoverageMatrix(tenantId, projectId).catch((e) => console.warn("[coverage-matrix]", e));
717
+ refreshTestCaseInventory(tenantId, projectId).catch((e) => console.warn("[tc-inventory]", e));
718
+ refreshTestCasesSkillFile(tenantId, projectId).catch(console.error);
719
+ await withTenant(
720
+ tenantId,
721
+ (tx) => tx.platformJob.update({
722
+ where: { id: jobId },
723
+ data: {
724
+ status: "DONE",
725
+ progress: 100,
726
+ finishedAt: /* @__PURE__ */ new Date(),
727
+ result: { cases: result.cases?.length ?? 0, suiteName: result.suiteName }
728
+ }
729
+ })
730
+ );
731
+ aggregateJobTokens(tenantId, jobId, projectId, jobStartedAt).catch(console.error);
732
+ } catch (err) {
733
+ await withTenant(
734
+ tenantId,
735
+ (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", error: String(err), finishedAt: /* @__PURE__ */ new Date() } })
736
+ );
737
+ }
859
738
  }
860
739
  async function processSecurityTestsJob(jobId, config) {
861
- const { tenantId, projectId } = config;
862
- const jobStartedAt = new Date();
863
- await withTenant(tenantId, (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: 'RUNNING', startedAt: jobStartedAt } }));
864
- try {
865
- const { generateSecurityTests, refreshCoverageMatrix, refreshTestCaseInventory } = await import('@detiq/agents');
866
- await updateJobProgress(tenantId, jobId, 20, 'Mapping OWASP attack surface…');
867
- const result = await generateSecurityTests({ tenantId, projectId, apiEndpoints: config.apiEndpoints });
868
- await updateJobProgress(tenantId, jobId, 90, `Generated ${result.cases?.length ?? 0} security test cases`);
869
- refreshCoverageMatrix(tenantId, projectId).catch((e) => console.warn('[coverage-matrix]', e));
870
- refreshTestCaseInventory(tenantId, projectId).catch((e) => console.warn('[tc-inventory]', e));
871
- refreshTestCasesSkillFile(tenantId, projectId).catch(console.error);
872
- await withTenant(tenantId, (tx) => tx.platformJob.update({
873
- where: { id: jobId },
874
- data: {
875
- status: 'DONE', progress: 100, finishedAt: new Date(),
876
- result: { cases: result.cases?.length ?? 0, suiteName: result.suiteName },
877
- },
878
- }));
879
- aggregateJobTokens(tenantId, jobId, projectId, jobStartedAt).catch(console.error);
880
- }
881
- catch (err) {
882
- await withTenant(tenantId, (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', error: String(err), finishedAt: new Date() } }));
883
- }
740
+ const { tenantId, projectId } = config;
741
+ const jobStartedAt = /* @__PURE__ */ new Date();
742
+ await withTenant(
743
+ tenantId,
744
+ (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "RUNNING", startedAt: jobStartedAt } })
745
+ );
746
+ try {
747
+ const { generateSecurityTests, refreshCoverageMatrix, refreshTestCaseInventory } = await import("@detiq/agents");
748
+ await updateJobProgress(tenantId, jobId, 20, "Mapping OWASP attack surface\u2026");
749
+ const result = await generateSecurityTests({ tenantId, projectId, apiEndpoints: config.apiEndpoints });
750
+ await updateJobProgress(tenantId, jobId, 90, `Generated ${result.cases?.length ?? 0} security test cases`);
751
+ refreshCoverageMatrix(tenantId, projectId).catch((e) => console.warn("[coverage-matrix]", e));
752
+ refreshTestCaseInventory(tenantId, projectId).catch((e) => console.warn("[tc-inventory]", e));
753
+ refreshTestCasesSkillFile(tenantId, projectId).catch(console.error);
754
+ await withTenant(
755
+ tenantId,
756
+ (tx) => tx.platformJob.update({
757
+ where: { id: jobId },
758
+ data: {
759
+ status: "DONE",
760
+ progress: 100,
761
+ finishedAt: /* @__PURE__ */ new Date(),
762
+ result: { cases: result.cases?.length ?? 0, suiteName: result.suiteName }
763
+ }
764
+ })
765
+ );
766
+ aggregateJobTokens(tenantId, jobId, projectId, jobStartedAt).catch(console.error);
767
+ } catch (err) {
768
+ await withTenant(
769
+ tenantId,
770
+ (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", error: String(err), finishedAt: /* @__PURE__ */ new Date() } })
771
+ );
772
+ }
884
773
  }
885
774
  async function processIntegrationTestsJob(jobId, config) {
886
- const { tenantId, projectId } = config;
887
- const jobStartedAt = new Date();
888
- await withTenant(tenantId, (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: 'RUNNING', startedAt: jobStartedAt } }));
889
- try {
890
- const { generateIntegrationTests, refreshCoverageMatrix, refreshTestCaseInventory } = await import('@detiq/agents');
891
- await updateJobProgress(tenantId, jobId, 20, 'Analysing API contracts and data models…');
892
- const result = await generateIntegrationTests({ tenantId, projectId, apiEndpoints: config.apiEndpoints });
893
- await updateJobProgress(tenantId, jobId, 90, `Generated ${result.cases?.length ?? 0} integration test cases`);
894
- refreshCoverageMatrix(tenantId, projectId).catch((e) => console.warn('[coverage-matrix]', e));
895
- refreshTestCaseInventory(tenantId, projectId).catch((e) => console.warn('[tc-inventory]', e));
896
- refreshTestCasesSkillFile(tenantId, projectId).catch(console.error);
897
- await withTenant(tenantId, (tx) => tx.platformJob.update({
898
- where: { id: jobId },
899
- data: {
900
- status: 'DONE', progress: 100, finishedAt: new Date(),
901
- result: { cases: result.cases?.length ?? 0, suiteName: result.suiteName },
902
- },
903
- }));
904
- aggregateJobTokens(tenantId, jobId, projectId, jobStartedAt).catch(console.error);
905
- }
906
- catch (err) {
907
- await withTenant(tenantId, (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', error: String(err), finishedAt: new Date() } }));
908
- }
775
+ const { tenantId, projectId } = config;
776
+ const jobStartedAt = /* @__PURE__ */ new Date();
777
+ await withTenant(
778
+ tenantId,
779
+ (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "RUNNING", startedAt: jobStartedAt } })
780
+ );
781
+ try {
782
+ const { generateIntegrationTests, refreshCoverageMatrix, refreshTestCaseInventory } = await import("@detiq/agents");
783
+ await updateJobProgress(tenantId, jobId, 20, "Analysing API contracts and data models\u2026");
784
+ const result = await generateIntegrationTests({ tenantId, projectId, apiEndpoints: config.apiEndpoints });
785
+ await updateJobProgress(tenantId, jobId, 90, `Generated ${result.cases?.length ?? 0} integration test cases`);
786
+ refreshCoverageMatrix(tenantId, projectId).catch((e) => console.warn("[coverage-matrix]", e));
787
+ refreshTestCaseInventory(tenantId, projectId).catch((e) => console.warn("[tc-inventory]", e));
788
+ refreshTestCasesSkillFile(tenantId, projectId).catch(console.error);
789
+ await withTenant(
790
+ tenantId,
791
+ (tx) => tx.platformJob.update({
792
+ where: { id: jobId },
793
+ data: {
794
+ status: "DONE",
795
+ progress: 100,
796
+ finishedAt: /* @__PURE__ */ new Date(),
797
+ result: { cases: result.cases?.length ?? 0, suiteName: result.suiteName }
798
+ }
799
+ })
800
+ );
801
+ aggregateJobTokens(tenantId, jobId, projectId, jobStartedAt).catch(console.error);
802
+ } catch (err) {
803
+ await withTenant(
804
+ tenantId,
805
+ (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", error: String(err), finishedAt: /* @__PURE__ */ new Date() } })
806
+ );
807
+ }
909
808
  }
910
809
  async function processFigmaImportJob(jobId, config) {
911
- const { tenantId, projectId, fileKey } = config;
912
- await withTenant(tenantId, (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: 'RUNNING', startedAt: new Date() } }));
913
- try {
914
- const connector = await withTenant(tenantId, (tx) => tx.connector.findFirst({ where: { projectId, kind: 'FIGMA', tenantId } }));
915
- if (!connector)
916
- throw new Error('Figma connector not configured. Add your Figma token in Project Settings → Integrations.');
917
- const cfg = connector.config;
918
- const accessToken = cfg.accessToken_enc ? decrypt(cfg.accessToken_enc) : '';
919
- if (!accessToken)
920
- throw new Error('Figma access token missing. Edit the Figma connector and add your Personal Access Token.');
921
- const resolvedFileKey = fileKey ?? cfg.fileKey ?? '';
922
- if (!resolvedFileKey)
923
- throw new Error('No Figma file key. Paste the full Figma file URL as the App URL.');
924
- const creds = { accessToken, fileKey: resolvedFileKey };
925
- await updateJobProgress(tenantId, jobId, 5, 'Listing frames from Figma file…');
926
- const frames = await figma.listScreens(creds, resolvedFileKey);
927
- if (frames.length === 0) {
928
- await withTenant(tenantId, (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: 'DONE', progress: 100, result: { screensImported: 0 }, finishedAt: new Date() } }));
929
- return;
930
- }
931
- await updateJobProgress(tenantId, jobId, 15, `Fetching images for ${frames.length} frames…`);
932
- const nodeIds = frames.map((f) => f.id);
933
- const imageUrls = await figma.getScreenImages(creds, resolvedFileKey, nodeIds, 1);
934
- const screenshotDir = path.join(platformConfig.screenshotDir, projectId);
935
- await fs.mkdir(screenshotDir, { recursive: true });
936
- const tenantStorageCfg = await getTenantStorageConfig(tenantId).catch(() => null);
937
- let saved = 0;
938
- for (const frame of frames) {
939
- try {
940
- const pct = Math.round(15 + ((saved / frames.length) * 80));
941
- await updateJobProgress(tenantId, jobId, pct, `Saving frame ${saved + 1}/${frames.length}: ${frame.name}`);
942
- const screen = await AppBrain.saveScreen(tenantId, projectId, frame.name, `figma://file/${resolvedFileKey}/node/${frame.id}`);
943
- const imgUrl = imageUrls[frame.id];
944
- if (imgUrl) {
945
- const imgRes = await fetch(imgUrl);
946
- if (imgRes.ok) {
947
- const buf = Buffer.from(await imgRes.arrayBuffer());
948
- const filename = `${screen.id}-v1.png`;
949
- const storagePath = path.join(screenshotDir, filename);
950
- await fs.writeFile(storagePath, buf);
951
- let cloudUrl;
952
- if (tenantStorageCfg) {
953
- try {
954
- const key = buildArtifactKey(tenantId, projectId, 'screenshots', filename, tenantStorageCfg.prefix);
955
- cloudUrl = (await uploadScreenshotToTenantStorage(tenantStorageCfg, storagePath, key)) ?? undefined;
956
- }
957
- catch (e) {
958
- console.warn('[figma-import] cloud upload failed:', e);
959
- }
960
- }
961
- await AppBrain.saveScreenshot(tenantId, projectId, screen.id, storagePath, 1, 'DESKTOP', cloudUrl);
962
- }
963
- }
964
- // Extract elements from Figma frame node → save for grounding + test generation
965
- if (frame.node) {
966
- const figmaElements = figma.extractElements(frame.node);
967
- for (const el of figmaElements) {
968
- const role = el.type === 'COMPONENT' || el.type === 'COMPONENT_SET' ? 'COMPONENT'
969
- : el.type === 'INSTANCE' ? 'COMPONENT'
970
- : el.type === 'TEXT' ? 'TEXT'
971
- : el.type === 'FRAME' || el.type === 'GROUP' ? 'REGION'
972
- : el.type;
973
- const boundingRect = el.boundingBox
974
- ? { x: Math.round(el.boundingBox.x), y: Math.round(el.boundingBox.y), width: Math.round(el.boundingBox.width), height: Math.round(el.boundingBox.height) }
975
- : undefined;
976
- await AppBrain.saveElement(tenantId, projectId, screen.id, el.name, role, undefined, el.description ?? undefined, undefined, el.isInteractive, boundingRect, undefined, undefined);
977
- }
978
- }
979
- saved++;
980
- }
981
- catch (err) {
982
- console.warn(`[figma-import] Failed to save frame:`, err);
810
+ const { tenantId, projectId, fileKey } = config;
811
+ await withTenant(
812
+ tenantId,
813
+ (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "RUNNING", startedAt: /* @__PURE__ */ new Date() } })
814
+ );
815
+ try {
816
+ const connector = await withTenant(
817
+ tenantId,
818
+ (tx) => tx.connector.findFirst({ where: { projectId, kind: "FIGMA", tenantId } })
819
+ );
820
+ if (!connector) throw new Error("Figma connector not configured. Add your Figma token in Project Settings \u2192 Integrations.");
821
+ const cfg = connector.config;
822
+ const accessToken = cfg.accessToken_enc ? decrypt(cfg.accessToken_enc) : "";
823
+ if (!accessToken) throw new Error("Figma access token missing. Edit the Figma connector and add your Personal Access Token.");
824
+ const resolvedFileKey = fileKey ?? cfg.fileKey ?? "";
825
+ if (!resolvedFileKey) throw new Error("No Figma file key. Paste the full Figma file URL as the App URL.");
826
+ const creds = { accessToken, fileKey: resolvedFileKey };
827
+ await updateJobProgress(tenantId, jobId, 5, "Listing frames from Figma file\u2026");
828
+ const frames = await figma.listScreens(creds, resolvedFileKey);
829
+ if (frames.length === 0) {
830
+ await withTenant(
831
+ tenantId,
832
+ (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "DONE", progress: 100, result: { screensImported: 0 }, finishedAt: /* @__PURE__ */ new Date() } })
833
+ );
834
+ return;
835
+ }
836
+ await updateJobProgress(tenantId, jobId, 15, `Fetching images for ${frames.length} frames\u2026`);
837
+ const nodeIds = frames.map((f) => f.id);
838
+ const imageUrls = await figma.getScreenImages(creds, resolvedFileKey, nodeIds, 1);
839
+ const screenshotDir = path.join(platformConfig.screenshotDir, projectId);
840
+ await fs.mkdir(screenshotDir, { recursive: true });
841
+ const tenantStorageCfg = await getTenantStorageConfig(tenantId).catch(() => null);
842
+ let saved = 0;
843
+ for (const frame of frames) {
844
+ try {
845
+ const pct = Math.round(15 + saved / frames.length * 80);
846
+ await updateJobProgress(tenantId, jobId, pct, `Saving frame ${saved + 1}/${frames.length}: ${frame.name}`);
847
+ const screen = await AppBrain.saveScreen(tenantId, projectId, frame.name, `figma://file/${resolvedFileKey}/node/${frame.id}`);
848
+ const imgUrl = imageUrls[frame.id];
849
+ if (imgUrl) {
850
+ const imgRes = await fetch(imgUrl);
851
+ if (imgRes.ok) {
852
+ const buf = Buffer.from(await imgRes.arrayBuffer());
853
+ const filename = `${screen.id}-v1.png`;
854
+ const storagePath = path.join(screenshotDir, filename);
855
+ await fs.writeFile(storagePath, buf);
856
+ let cloudUrl;
857
+ if (tenantStorageCfg) {
858
+ try {
859
+ const key = buildArtifactKey(tenantId, projectId, "screenshots", filename, tenantStorageCfg.prefix);
860
+ cloudUrl = await uploadScreenshotToTenantStorage(tenantStorageCfg, storagePath, key) ?? void 0;
861
+ } catch (e) {
862
+ console.warn("[figma-import] cloud upload failed:", e);
863
+ }
983
864
  }
865
+ await AppBrain.saveScreenshot(tenantId, projectId, screen.id, storagePath, 1, "DESKTOP", cloudUrl);
866
+ }
984
867
  }
985
- await withTenant(tenantId, (tx) => tx.platformJob.update({
986
- where: { id: jobId },
987
- data: { status: 'DONE', progress: 100, result: { screensImported: saved, totalFrames: frames.length }, finishedAt: new Date() },
988
- }));
989
- // Post-import: run AI grouping + taxonomy skill files (same as post-crawl)
990
- setImmediate(async () => {
991
- try {
992
- const projectMap = await AppBrain.getProjectMap(tenantId, projectId);
993
- if (projectMap?.screens?.length) {
994
- await generateTaxonomySkillFiles(tenantId, projectId, projectMap.screens);
995
- await groupScreensWithAI(tenantId, projectId, projectMap.screens, jobId, new Date());
996
- }
997
- }
998
- catch (err) {
999
- console.error('[figma-import] post-import grouping failed:', err);
1000
- }
1001
- });
1002
- }
1003
- catch (err) {
1004
- await withTenant(tenantId, (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', error: String(err?.message ?? err), finishedAt: new Date() } }));
868
+ if (frame.node) {
869
+ const figmaElements = figma.extractElements(frame.node);
870
+ for (const el of figmaElements) {
871
+ const role = el.type === "COMPONENT" || el.type === "COMPONENT_SET" ? "COMPONENT" : el.type === "INSTANCE" ? "COMPONENT" : el.type === "TEXT" ? "TEXT" : el.type === "FRAME" || el.type === "GROUP" ? "REGION" : el.type;
872
+ const boundingRect = el.boundingBox ? { x: Math.round(el.boundingBox.x), y: Math.round(el.boundingBox.y), width: Math.round(el.boundingBox.width), height: Math.round(el.boundingBox.height) } : void 0;
873
+ await AppBrain.saveElement(
874
+ tenantId,
875
+ projectId,
876
+ screen.id,
877
+ el.name,
878
+ role,
879
+ void 0,
880
+ el.description ?? void 0,
881
+ void 0,
882
+ el.isInteractive,
883
+ boundingRect,
884
+ void 0,
885
+ void 0
886
+ );
887
+ }
888
+ }
889
+ saved++;
890
+ } catch (err) {
891
+ console.warn(`[figma-import] Failed to save frame:`, err);
892
+ }
1005
893
  }
894
+ await withTenant(
895
+ tenantId,
896
+ (tx) => tx.platformJob.update({
897
+ where: { id: jobId },
898
+ data: { status: "DONE", progress: 100, result: { screensImported: saved, totalFrames: frames.length }, finishedAt: /* @__PURE__ */ new Date() }
899
+ })
900
+ );
901
+ setImmediate(async () => {
902
+ try {
903
+ const projectMap = await AppBrain.getProjectMap(tenantId, projectId);
904
+ if (projectMap?.screens?.length) {
905
+ await generateTaxonomySkillFiles(tenantId, projectId, projectMap.screens);
906
+ await groupScreensWithAI(tenantId, projectId, projectMap.screens, jobId, /* @__PURE__ */ new Date());
907
+ }
908
+ } catch (err) {
909
+ console.error("[figma-import] post-import grouping failed:", err);
910
+ }
911
+ });
912
+ } catch (err) {
913
+ await withTenant(
914
+ tenantId,
915
+ (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", error: String(err?.message ?? err), finishedAt: /* @__PURE__ */ new Date() } })
916
+ );
917
+ }
1006
918
  }
1007
919
  async function processNativeCrawlJob(jobId, config) {
1008
- const { tenantId, projectId } = config;
1009
- const jobStartedAt = new Date();
1010
- try {
1011
- const build = await withTenant(tenantId, (tx) => tx.nativeAppBuild.findFirst({
1012
- where: { tenantId, projectId },
1013
- orderBy: { uploadedAt: 'desc' },
1014
- }));
1015
- if (!build)
1016
- throw new Error('No native app build found for project');
1017
- // AAB requires bundletool conversion — fail early with a clear message
1018
- if (build.fileFormat === 'aab') {
1019
- throw new Error('AAB format cannot be installed directly by Appium. Convert to APK using bundletool first, then re-upload the APK.');
920
+ const { tenantId, projectId } = config;
921
+ const jobStartedAt = /* @__PURE__ */ new Date();
922
+ try {
923
+ const build = await withTenant(
924
+ tenantId,
925
+ (tx) => tx.nativeAppBuild.findFirst({
926
+ where: { tenantId, projectId },
927
+ orderBy: { uploadedAt: "desc" }
928
+ })
929
+ );
930
+ if (!build) throw new Error("No native app build found for project");
931
+ if (build.fileFormat === "aab") {
932
+ throw new Error(
933
+ "AAB format cannot be installed directly by Appium. Convert to APK using bundletool first, then re-upload the APK."
934
+ );
935
+ }
936
+ const deviceFarm = config.deviceFarm ?? "local_emulator";
937
+ await withTenant(
938
+ tenantId,
939
+ (tx) => tx.platformJob.update({
940
+ where: { id: jobId },
941
+ data: {
942
+ deviceModel: config.deviceModel ?? null,
943
+ osVersion: config.osVersion ?? null,
944
+ deviceFarm
1020
945
  }
1021
- // Persist device targeting info on the job record
1022
- const deviceFarm = (config.deviceFarm ?? 'local_emulator');
1023
- await withTenant(tenantId, (tx) => tx.platformJob.update({
1024
- where: { id: jobId },
1025
- data: {
1026
- deviceModel: config.deviceModel ?? null,
1027
- osVersion: config.osVersion ?? null,
1028
- deviceFarm,
1029
- },
1030
- })).catch(() => { });
1031
- const workspaceDeviceFarm = await AppBrain.getTenantDeviceFarmConfig(tenantId).catch(() => null);
1032
- await runNativeCrawl({
1033
- tenantId,
1034
- projectId,
1035
- jobId,
1036
- platform: build.platform,
1037
- appS3Path: build.s3Path,
1038
- packageName: build.packageName ?? undefined,
1039
- deviceModel: config.deviceModel,
1040
- osVersion: config.osVersion,
1041
- deviceFarm,
1042
- appiumServerUrl: workspaceDeviceFarm?.appiumServerUrl,
1043
- browserstackUsername: workspaceDeviceFarm?.browserstackUsername,
1044
- browserstackAccessKey: workspaceDeviceFarm?.browserstackAccessKey,
1045
- awsAccessKeyId: workspaceDeviceFarm?.awsAccessKeyId,
1046
- awsSecretAccessKey: workspaceDeviceFarm?.awsSecretAccessKey,
1047
- awsRegion: workspaceDeviceFarm?.awsRegion,
1048
- awsDeviceFarmProjectArn: workspaceDeviceFarm?.awsDeviceFarmProjectArn,
1049
- });
1050
- // Post-crawl: generate taxonomy skill files + AI screen grouping (mirrors web crawl processJob)
1051
- setImmediate(async () => {
1052
- try {
1053
- const projectMap = await AppBrain.getProjectMap(tenantId, projectId);
1054
- const screens = (projectMap?.screens ?? []);
1055
- if (screens.length > 0) {
1056
- await generateTaxonomySkillFiles(tenantId, projectId, screens);
1057
- await groupScreensWithAI(tenantId, projectId, screens, jobId, jobStartedAt);
1058
- // Auto-flag stale tests: mark PENDING_REVIEW when screen elements changed since test was generated
1059
- try {
1060
- await withTenant(tenantId, async (tx) => {
1061
- const staleTests = await tx.$queryRaw `
946
+ })
947
+ ).catch(() => {
948
+ });
949
+ const workspaceDeviceFarm = await AppBrain.getTenantDeviceFarmConfig(tenantId).catch(() => null);
950
+ await runNativeCrawl({
951
+ tenantId,
952
+ projectId,
953
+ jobId,
954
+ platform: build.platform,
955
+ appS3Path: build.s3Path,
956
+ packageName: build.packageName ?? void 0,
957
+ deviceModel: config.deviceModel,
958
+ osVersion: config.osVersion,
959
+ deviceFarm,
960
+ appiumServerUrl: workspaceDeviceFarm?.appiumServerUrl,
961
+ browserstackUsername: workspaceDeviceFarm?.browserstackUsername,
962
+ browserstackAccessKey: workspaceDeviceFarm?.browserstackAccessKey,
963
+ awsAccessKeyId: workspaceDeviceFarm?.awsAccessKeyId,
964
+ awsSecretAccessKey: workspaceDeviceFarm?.awsSecretAccessKey,
965
+ awsRegion: workspaceDeviceFarm?.awsRegion,
966
+ awsDeviceFarmProjectArn: workspaceDeviceFarm?.awsDeviceFarmProjectArn
967
+ });
968
+ setImmediate(async () => {
969
+ try {
970
+ const projectMap = await AppBrain.getProjectMap(tenantId, projectId);
971
+ const screens = projectMap?.screens ?? [];
972
+ if (screens.length > 0) {
973
+ await generateTaxonomySkillFiles(tenantId, projectId, screens);
974
+ await groupScreensWithAI(tenantId, projectId, screens, jobId, jobStartedAt);
975
+ try {
976
+ await withTenant(tenantId, async (tx) => {
977
+ const staleTests = await tx.$queryRaw`
1062
978
  SELECT tc.id
1063
979
  FROM "TestCase" tc
1064
980
  JOIN "Screen" s ON tc."screenId" = s.id
@@ -1070,1296 +986,1362 @@ async function processNativeCrawlJob(jobId, config) {
1070
986
  AND tc.status != 'PENDING_REVIEW'
1071
987
  AND tc.status != 'RETIRED'
1072
988
  `;
1073
- if (staleTests.length > 0) {
1074
- const ids = staleTests.map((t) => t.id);
1075
- await tx.testCase.updateMany({
1076
- where: { id: { in: ids } },
1077
- data: { status: 'PENDING_REVIEW' },
1078
- });
1079
- logger.info({ count: staleTests.length, projectId }, 'Auto-flagged stale test cases as PENDING_REVIEW');
1080
- }
1081
- });
1082
- }
1083
- catch (staleErr) {
1084
- logger.warn({ err: staleErr }, 'Failed to auto-flag stale tests');
1085
- }
1086
- }
1087
- }
1088
- catch (e) {
1089
- logger.error({ err: e }, 'post-native-crawl processing failed');
1090
- }
1091
- });
1092
- }
1093
- catch (err) {
1094
- await withTenant(tenantId, (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', error: String(err?.message ?? err), finishedAt: new Date() } })).catch(() => { });
1095
- }
1096
- }
1097
- async function processGenerateTestsFromReqsJob(jobId, config) {
1098
- const { tenantId, projectId, reqIds, suiteId, suiteName } = config;
1099
- try {
1100
- await updateJobProgress(tenantId, jobId, 5, 'Loading requirements');
1101
- const { askTracked, deterministicSeed, getTaskAgentLLMConfig } = await import('@detiq/agents');
1102
- const llmCfg = await getTaskAgentLLMConfig(tenantId, 'testGeneration');
1103
- const all = await AppBrain.listRequirements(tenantId, projectId, {});
1104
- const reqs = reqIds?.length
1105
- ? all.filter((r) => reqIds.includes(r.reqId))
1106
- : all.filter((r) => r.status === 'unbuilt' || r.status === 'pending');
1107
- if (reqs.length === 0) {
1108
- await updateJobProgress(tenantId, jobId, 100, 'No pending requirements found');
1109
- await withTenant(tenantId, (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: 'DONE', finishedAt: new Date(), result: { created: 0, skipped: 0 } } }));
1110
- return;
1111
- }
1112
- // Resolve or create suite
1113
- let resolvedSuiteId = suiteId;
1114
- if (!resolvedSuiteId) {
1115
- const name = suiteName?.trim() || 'Requirements Tests';
1116
- const suite = await withTenant(tenantId, async (tx) => {
1117
- const existing = await tx.testSuite.findFirst({ where: { tenantId, projectId, name } });
1118
- if (existing)
1119
- return existing;
1120
- return tx.testSuite.create({ data: { tenantId, projectId, name } });
989
+ if (staleTests.length > 0) {
990
+ const ids = staleTests.map((t) => t.id);
991
+ await tx.testCase.updateMany({
992
+ where: { id: { in: ids } },
993
+ data: { status: "PENDING_REVIEW" }
994
+ });
995
+ logger.info({ count: staleTests.length, projectId }, "Auto-flagged stale test cases as PENDING_REVIEW");
996
+ }
1121
997
  });
1122
- resolvedSuiteId = suite.id;
998
+ } catch (staleErr) {
999
+ logger.warn({ err: staleErr }, "Failed to auto-flag stale tests");
1000
+ }
1123
1001
  }
1124
- await updateJobProgress(tenantId, jobId, 20, `Generating tests for ${reqs.length} requirements`);
1125
- const reqBlock = reqs.map((r) => [
1126
- `## ${r.reqId}: ${r.title}`,
1127
- `Priority: ${r.priority} | Type: ${r.type}`,
1128
- r.acceptanceCriteria?.length ? `Acceptance Criteria:\n${r.acceptanceCriteria.map((c) => `- ${c}`).join('\n')}` : '',
1129
- r.businessRules?.length ? `Business Rules:\n${r.businessRules.map((b) => `- ${b}`).join('\n')}` : '',
1130
- ].filter(Boolean).join('\n')).join('\n\n');
1131
- const system = `You are ZeTa-AI's test case generator. Convert requirements into structured test cases.
1002
+ } catch (e) {
1003
+ logger.error({ err: e }, "post-native-crawl processing failed");
1004
+ }
1005
+ });
1006
+ } catch (err) {
1007
+ await withTenant(
1008
+ tenantId,
1009
+ (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", error: String(err?.message ?? err), finishedAt: /* @__PURE__ */ new Date() } })
1010
+ ).catch(() => {
1011
+ });
1012
+ }
1013
+ }
1014
+ async function processGenerateTestsFromReqsJob(jobId, config) {
1015
+ const { tenantId, projectId, reqIds, suiteId, suiteName } = config;
1016
+ try {
1017
+ await updateJobProgress(tenantId, jobId, 5, "Loading requirements");
1018
+ const { askTracked, deterministicSeed, getTaskAgentLLMConfig } = await import("@detiq/agents");
1019
+ const llmCfg = await getTaskAgentLLMConfig(tenantId, "testGeneration");
1020
+ const all = await AppBrain.listRequirements(tenantId, projectId, {});
1021
+ const reqs = reqIds?.length ? all.filter((r) => reqIds.includes(r.reqId)) : all.filter((r) => r.status === "unbuilt" || r.status === "pending");
1022
+ if (reqs.length === 0) {
1023
+ await updateJobProgress(tenantId, jobId, 100, "No pending requirements found");
1024
+ await withTenant(
1025
+ tenantId,
1026
+ (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "DONE", finishedAt: /* @__PURE__ */ new Date(), result: { created: 0, skipped: 0 } } })
1027
+ );
1028
+ return;
1029
+ }
1030
+ let resolvedSuiteId = suiteId;
1031
+ if (!resolvedSuiteId) {
1032
+ const name = suiteName?.trim() || "Requirements Tests";
1033
+ const suite = await withTenant(tenantId, async (tx) => {
1034
+ const existing = await tx.testSuite.findFirst({ where: { tenantId, projectId, name } });
1035
+ if (existing) return existing;
1036
+ return tx.testSuite.create({ data: { tenantId, projectId, name } });
1037
+ });
1038
+ resolvedSuiteId = suite.id;
1039
+ }
1040
+ await updateJobProgress(tenantId, jobId, 20, `Generating tests for ${reqs.length} requirements`);
1041
+ const reqBlock = reqs.map((r) => [
1042
+ `## ${r.reqId}: ${r.title}`,
1043
+ `Priority: ${r.priority} | Type: ${r.type}`,
1044
+ r.acceptanceCriteria?.length ? `Acceptance Criteria:
1045
+ ${r.acceptanceCriteria.map((c) => `- ${c}`).join("\n")}` : "",
1046
+ r.businessRules?.length ? `Business Rules:
1047
+ ${r.businessRules.map((b) => `- ${b}`).join("\n")}` : ""
1048
+ ].filter(Boolean).join("\n")).join("\n\n");
1049
+ const system = `You are ZeTa-AI's test case generator. Convert requirements into structured test cases.
1132
1050
  Return ONLY valid JSON: { "cases": [ { "title": string, "kind": string, "priority": string, "preconditions": string, "steps": [{"action": string, "data": string, "expected": string}], "expectedResult": string, "tags": string[], "reqRef": string } ] }
1133
1051
  Kind must be one of: POSITIVE, NEGATIVE, EDGE, SECURITY, ACCESSIBILITY, INTEGRATION, E2E, SMOKE, REGRESSION.
1134
1052
  Priority must be one of: CRITICAL, HIGH, MEDIUM, LOW.
1135
1053
  Generate 1-3 test cases per requirement (positive + negative + edge if applicable).
1136
1054
  Each test case must reference its reqRef (the requirement ID like REQ-001).`;
1137
- const user = `Generate test cases for these requirements:\n\n${reqBlock}`;
1138
- let parsed = { cases: [] };
1139
- try {
1140
- const reqSeed = deterministicSeed(`${tenantId}:${projectId}:req-cases:${reqs.map((r) => r.reqId).sort().join('|').slice(0, 300)}`);
1141
- const { text, usage } = await askTracked(system, user, llmCfg, undefined, undefined, { temperature: 0, seed: reqSeed });
1142
- if (usage)
1143
- recordLLMCall({ tenantId, projectId, jobId, agentType: 'generate_tests_from_reqs', label: 'test_case_generation', usage }).catch(() => { });
1144
- const json = text.match(/\{[\s\S]*\}/)?.[0];
1145
- if (json)
1146
- parsed = JSON.parse(json);
1147
- }
1148
- catch (e) {
1149
- throw new Error('AI failed to generate test cases: ' + (e.message ?? 'unknown'));
1150
- }
1151
- await updateJobProgress(tenantId, jobId, 70, `Saving ${parsed.cases?.length ?? 0} test cases`);
1152
- const VALID_KINDS = new Set(['POSITIVE', 'NEGATIVE', 'EDGE', 'STATE_TRANSITION', 'ACCESSIBILITY', 'SECURITY', 'INTEGRATION', 'E2E', 'API', 'EXPLORATORY', 'UAT', 'PERFORMANCE', 'COMPATIBILITY', 'L10N', 'USABILITY', 'LOAD', 'UNIT', 'MUTATION', 'AB_TEST', 'ALPHA_BETA', 'SYSTEM', 'SMOKE', 'REGRESSION', 'SANITY']);
1153
- const VALID_PRIORITIES = new Set(['CRITICAL', 'HIGH', 'MEDIUM', 'LOW']);
1154
- const normalized = (parsed.cases ?? []).map((c) => ({
1155
- ref: c.reqRef ?? null,
1156
- traceTo: c.reqRef ?? null,
1157
- title: c.title || 'Untitled',
1158
- kind: VALID_KINDS.has((c.kind ?? '').toUpperCase()) ? c.kind.toUpperCase() : 'POSITIVE',
1159
- priority: VALID_PRIORITIES.has((c.priority ?? '').toUpperCase()) ? c.priority.toUpperCase() : 'MEDIUM',
1160
- preconditions: c.preconditions ?? '',
1161
- steps: Array.isArray(c.steps) ? c.steps : [],
1162
- expectedResult: c.expectedResult ?? '',
1163
- tags: Array.isArray(c.tags) ? c.tags : [],
1164
- }));
1165
- const { created, skipped } = await AppBrain.importTestCasesFromArtifact(tenantId, projectId, resolvedSuiteId, normalized, { duplicateMode: 'skip' });
1166
- // Mark requirements as covered
1167
- const coveredReqIds = new Set(normalized.map((c) => c.ref).filter(Boolean));
1168
- await Promise.all(reqs
1169
- .filter((r) => coveredReqIds.has(r.reqId))
1170
- .map((r) => AppBrain.updateRequirement(tenantId, projectId, r.reqId, { status: 'covered' }).catch(() => { })));
1171
- await withTenant(tenantId, (tx) => tx.platformJob.update({
1172
- where: { id: jobId },
1173
- data: { status: 'DONE', finishedAt: new Date(), result: { created: created.length, skipped, suiteId: resolvedSuiteId }, progress: 100 },
1174
- }));
1175
- }
1176
- catch (e) {
1177
- const msg = e?.message ?? 'Unknown error';
1178
- logger.error({ jobId, err: msg }, 'processGenerateTestsFromReqsJob failed');
1179
- await withTenant(tenantId, (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', finishedAt: new Date(), error: msg } })).catch(() => { });
1055
+ const user = `Generate test cases for these requirements:
1056
+
1057
+ ${reqBlock}`;
1058
+ let parsed = { cases: [] };
1059
+ try {
1060
+ const reqSeed = deterministicSeed(`${tenantId}:${projectId}:req-cases:${reqs.map((r) => r.reqId).sort().join("|").slice(0, 300)}`);
1061
+ const { text, usage } = await askTracked(system, user, llmCfg, void 0, void 0, { temperature: 0, seed: reqSeed });
1062
+ if (usage) recordLLMCall({ tenantId, projectId, jobId, agentType: "generate_tests_from_reqs", label: "test_case_generation", usage }).catch(() => {
1063
+ });
1064
+ const json = text.match(/\{[\s\S]*\}/)?.[0];
1065
+ if (json) parsed = JSON.parse(json);
1066
+ } catch (e) {
1067
+ throw new Error("AI failed to generate test cases: " + (e.message ?? "unknown"));
1180
1068
  }
1069
+ await updateJobProgress(tenantId, jobId, 70, `Saving ${parsed.cases?.length ?? 0} test cases`);
1070
+ const VALID_KINDS = /* @__PURE__ */ new Set(["POSITIVE", "NEGATIVE", "EDGE", "STATE_TRANSITION", "ACCESSIBILITY", "SECURITY", "INTEGRATION", "E2E", "API", "EXPLORATORY", "UAT", "PERFORMANCE", "COMPATIBILITY", "L10N", "USABILITY", "LOAD", "UNIT", "MUTATION", "AB_TEST", "ALPHA_BETA", "SYSTEM", "SMOKE", "REGRESSION", "SANITY"]);
1071
+ const VALID_PRIORITIES = /* @__PURE__ */ new Set(["CRITICAL", "HIGH", "MEDIUM", "LOW"]);
1072
+ const normalized = (parsed.cases ?? []).map((c) => ({
1073
+ ref: c.reqRef ?? null,
1074
+ traceTo: c.reqRef ?? null,
1075
+ title: c.title || "Untitled",
1076
+ kind: VALID_KINDS.has((c.kind ?? "").toUpperCase()) ? c.kind.toUpperCase() : "POSITIVE",
1077
+ priority: VALID_PRIORITIES.has((c.priority ?? "").toUpperCase()) ? c.priority.toUpperCase() : "MEDIUM",
1078
+ preconditions: c.preconditions ?? "",
1079
+ steps: Array.isArray(c.steps) ? c.steps : [],
1080
+ expectedResult: c.expectedResult ?? "",
1081
+ tags: Array.isArray(c.tags) ? c.tags : []
1082
+ }));
1083
+ const { created, skipped } = await AppBrain.importTestCasesFromArtifact(
1084
+ tenantId,
1085
+ projectId,
1086
+ resolvedSuiteId,
1087
+ normalized,
1088
+ { duplicateMode: "skip" }
1089
+ );
1090
+ const coveredReqIds = new Set(normalized.map((c) => c.ref).filter(Boolean));
1091
+ await Promise.all(
1092
+ reqs.filter((r) => coveredReqIds.has(r.reqId)).map((r) => AppBrain.updateRequirement(tenantId, projectId, r.reqId, { status: "covered" }).catch(() => {
1093
+ }))
1094
+ );
1095
+ await withTenant(
1096
+ tenantId,
1097
+ (tx) => tx.platformJob.update({
1098
+ where: { id: jobId },
1099
+ data: { status: "DONE", finishedAt: /* @__PURE__ */ new Date(), result: { created: created.length, skipped, suiteId: resolvedSuiteId }, progress: 100 }
1100
+ })
1101
+ );
1102
+ } catch (e) {
1103
+ const msg = e?.message ?? "Unknown error";
1104
+ logger.error({ jobId, err: msg }, "processGenerateTestsFromReqsJob failed");
1105
+ await withTenant(
1106
+ tenantId,
1107
+ (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", finishedAt: /* @__PURE__ */ new Date(), error: msg } })
1108
+ ).catch(() => {
1109
+ });
1110
+ }
1181
1111
  }
1182
- /**
1183
- * Runs every APPROVED test case in a project (optionally scoped to one suite)
1184
- * against the agent-free Playwright executor (test-executor.ts) and aggregates
1185
- * the results into the TestRun row created synchronously by POST /run-tests
1186
- * (see ci.ts) — this is what makes the CI gate (GET /test-results/latest)
1187
- * finally reflect what CI itself triggered, instead of a stale or nonexistent
1188
- * result from an unrelated, manually-triggered, agent-connected suite run.
1189
- */
1190
1112
  async function processRunTestSuiteJob(jobId, config) {
1191
- const { tenantId, projectId, testRunId, suiteId, storageState, browser, shardIndex, totalShards } = config;
1192
- const runStart = Date.now();
1193
- await withTenant(tenantId, (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: 'RUNNING', startedAt: new Date() } }));
1194
- try {
1195
- let testCases = await AppBrain.listTestCases(tenantId, projectId, {
1196
- status: 'APPROVED',
1197
- ...(suiteId ? { suiteId } : {}),
1198
- });
1199
- // Shard filtering: when running a distributed matrix (CI-triggered multi-browser/shard),
1200
- // each worker only executes its own slice — testCases[shardIndex::totalShards].
1201
- if (typeof totalShards === 'number' && totalShards > 1 && typeof shardIndex === 'number') {
1202
- testCases = testCases.filter((_, i) => i % totalShards === shardIndex);
1203
- }
1204
- if (testCases.length === 0) {
1205
- // Nothing approved to run isn't a CI failure — same as any test runner
1206
- // reporting "0 tests, 0 failures" when a suite is empty.
1207
- await AppBrain.completeTestRun(tenantId, testRunId, { total: 0, passed: 0, failed: 0, skipped: 0, duration: 0 });
1208
- await withTenant(tenantId, (tx) => tx.platformJob.update({
1209
- where: { id: jobId },
1210
- data: { status: 'DONE', progress: 100, finishedAt: new Date(), result: { total: 0 } },
1211
- }));
1212
- return;
1213
- }
1214
- const aiHealingEnabled = await isFeatureEnabled(tenantId, 'AI_SELF_HEALING');
1215
- const SCREENSHOT_DIR = process.env.SCREENSHOT_DIR ?? '/tmp/zeta-screenshots';
1216
- const screenshotDir = config.screenshotDir || platformConfig.screenshotDir || SCREENSHOT_DIR;
1217
- const VALID_BROWSERS = ['chromium', 'firefox', 'webkit'];
1218
- const validBrowser = VALID_BROWSERS.includes(browser) ? browser : 'chromium';
1219
- let passed = 0;
1220
- let failed = 0;
1221
- const runResults = [];
1222
- for (let i = 0; i < testCases.length; i += TEST_EXEC_CONCURRENCY) {
1223
- const batch = testCases.slice(i, i + TEST_EXEC_CONCURRENCY);
1224
- await Promise.all(batch.map(async (tc) => {
1225
- try {
1226
- const full = await AppBrain.getTestCaseForExecution(tenantId, tc.id);
1227
- const baseUrl = full?.screen?.project?.appUrl ?? '';
1228
- const execProjectId = full?.screen?.project?.id ?? projectId;
1229
- const dataRows = full?.dataSet?.rows ?? [];
1230
- // Data-driven: run once per row (substituting placeholders). If no data set, run once with no substitution.
1231
- const runs = dataRows.length > 0 ? dataRows : [null];
1232
- for (const row of runs) {
1233
- let tcForPlan = full ?? tc;
1234
- if (row !== null) {
1235
- // Clone the test case with interpolated steps and title
1236
- const rawSteps = Array.isArray(tcForPlan.steps) ? tcForPlan.steps : [];
1237
- tcForPlan = {
1238
- ...tcForPlan,
1239
- steps: rawSteps.map((s) => interpolatePlaceholders(s, row.data)),
1240
- title: interpolatePlaceholders(tcForPlan.title ?? '', row.data),
1241
- preconditions: tcForPlan.preconditions
1242
- ? interpolatePlaceholders(tcForPlan.preconditions, row.data)
1243
- : tcForPlan.preconditions,
1244
- };
1245
- }
1246
- const plan = await generateExecutionPlan(tenantId, tcForPlan, baseUrl);
1247
- const result = await executeTestPlan(plan, {
1248
- screenshotDir,
1249
- testRunId,
1250
- maxRuns: 1,
1251
- browser: validBrowser,
1252
- storageState,
1253
- healCtx: aiHealingEnabled ? { tenantId, projectId: execProjectId } : undefined,
1254
- });
1255
- const status = result.status === 'PASSED' ? 'PASSED' : result.status === 'ERROR' ? 'ERROR' : 'FAILED';
1256
- if (status === 'PASSED')
1257
- passed++;
1258
- else
1259
- failed++;
1260
- runResults.push({ testCaseId: tc.id, status });
1261
- await AppBrain.saveTestResult(tenantId, {
1262
- runId: testRunId,
1263
- testCaseId: tc.id,
1264
- status,
1265
- duration: result.totalDuration,
1266
- ...(result.rootCauseSummary
1267
- ? { failureAnalysis: result.rootCauseSummary, failureAnalysisAt: new Date() }
1268
- : {}),
1269
- ...(row !== null ? { dataRowId: row.id, dataRowIndex: row.rowIndex } : {}),
1270
- ...((result.consoleLogs?.length || result.networkErrors?.length) ? {
1271
- logs: {
1272
- consoleLogs: result.consoleLogs ?? [],
1273
- networkErrors: result.networkErrors ?? [],
1274
- },
1275
- } : {}),
1276
- });
1277
- }
1278
- await AppBrain.recordFlakinessOutcome(tenantId, tc.id).catch(() => { });
1279
- }
1280
- catch (err) {
1281
- failed++;
1282
- runResults.push({ testCaseId: tc.id, status: 'ERROR' });
1283
- await AppBrain.saveTestResult(tenantId, {
1284
- runId: testRunId,
1285
- testCaseId: tc.id,
1286
- status: 'ERROR',
1287
- error: String(err?.message ?? err).slice(0, 500),
1288
- }).catch(() => { });
1113
+ const { tenantId, projectId, testRunId, suiteId, storageState, browser, shardIndex, totalShards } = config;
1114
+ const runStart = Date.now();
1115
+ await withTenant(
1116
+ tenantId,
1117
+ (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "RUNNING", startedAt: /* @__PURE__ */ new Date() } })
1118
+ );
1119
+ try {
1120
+ let testCases = await AppBrain.listTestCases(tenantId, projectId, {
1121
+ status: "APPROVED",
1122
+ ...suiteId ? { suiteId } : {}
1123
+ });
1124
+ if (typeof totalShards === "number" && totalShards > 1 && typeof shardIndex === "number") {
1125
+ testCases = testCases.filter((_, i) => i % totalShards === shardIndex);
1126
+ }
1127
+ if (testCases.length === 0) {
1128
+ await AppBrain.completeTestRun(tenantId, testRunId, { total: 0, passed: 0, failed: 0, skipped: 0, duration: 0 });
1129
+ await withTenant(
1130
+ tenantId,
1131
+ (tx) => tx.platformJob.update({
1132
+ where: { id: jobId },
1133
+ data: { status: "DONE", progress: 100, finishedAt: /* @__PURE__ */ new Date(), result: { total: 0 } }
1134
+ })
1135
+ );
1136
+ return;
1137
+ }
1138
+ const aiHealingEnabled = await isFeatureEnabled(tenantId, "AI_SELF_HEALING");
1139
+ const SCREENSHOT_DIR = process.env.SCREENSHOT_DIR ?? "/tmp/zeta-screenshots";
1140
+ const screenshotDir = config.screenshotDir || platformConfig.screenshotDir || SCREENSHOT_DIR;
1141
+ const VALID_BROWSERS = ["chromium", "firefox", "webkit"];
1142
+ const validBrowser = VALID_BROWSERS.includes(browser) ? browser : "chromium";
1143
+ let passed = 0;
1144
+ let failed = 0;
1145
+ const runResults = [];
1146
+ for (let i = 0; i < testCases.length; i += TEST_EXEC_CONCURRENCY) {
1147
+ const batch = testCases.slice(i, i + TEST_EXEC_CONCURRENCY);
1148
+ await Promise.all(batch.map(async (tc) => {
1149
+ try {
1150
+ const full = await AppBrain.getTestCaseForExecution(tenantId, tc.id);
1151
+ const baseUrl = full?.screen?.project?.appUrl ?? "";
1152
+ const execProjectId = full?.screen?.project?.id ?? projectId;
1153
+ const dataRows = full?.dataSet?.rows ?? [];
1154
+ const runs = dataRows.length > 0 ? dataRows : [null];
1155
+ for (const row of runs) {
1156
+ let tcForPlan = full ?? tc;
1157
+ if (row !== null) {
1158
+ const rawSteps = Array.isArray(tcForPlan.steps) ? tcForPlan.steps : [];
1159
+ tcForPlan = {
1160
+ ...tcForPlan,
1161
+ steps: rawSteps.map((s) => interpolatePlaceholders(s, row.data)),
1162
+ title: interpolatePlaceholders(tcForPlan.title ?? "", row.data),
1163
+ preconditions: tcForPlan.preconditions ? interpolatePlaceholders(tcForPlan.preconditions, row.data) : tcForPlan.preconditions
1164
+ };
1165
+ }
1166
+ const plan = await generateExecutionPlan(tenantId, tcForPlan, baseUrl);
1167
+ const result = await executeTestPlan(plan, {
1168
+ screenshotDir,
1169
+ testRunId,
1170
+ maxRuns: 1,
1171
+ browser: validBrowser,
1172
+ storageState,
1173
+ healCtx: aiHealingEnabled ? { tenantId, projectId: execProjectId } : void 0
1174
+ });
1175
+ const status = result.status === "PASSED" ? "PASSED" : result.status === "ERROR" ? "ERROR" : "FAILED";
1176
+ if (status === "PASSED") passed++;
1177
+ else failed++;
1178
+ runResults.push({ testCaseId: tc.id, status });
1179
+ await AppBrain.saveTestResult(tenantId, {
1180
+ runId: testRunId,
1181
+ testCaseId: tc.id,
1182
+ status,
1183
+ duration: result.totalDuration,
1184
+ ...result.rootCauseSummary ? { failureAnalysis: result.rootCauseSummary, failureAnalysisAt: /* @__PURE__ */ new Date() } : {},
1185
+ ...row !== null ? { dataRowId: row.id, dataRowIndex: row.rowIndex } : {},
1186
+ ...result.consoleLogs?.length || result.networkErrors?.length ? {
1187
+ logs: {
1188
+ consoleLogs: result.consoleLogs ?? [],
1189
+ networkErrors: result.networkErrors ?? []
1289
1190
  }
1290
- }));
1291
- await updateJobProgress(tenantId, jobId, Math.round((Math.min(i + TEST_EXEC_CONCURRENCY, testCases.length) / testCases.length) * 100), `Ran ${Math.min(i + TEST_EXEC_CONCURRENCY, testCases.length)}/${testCases.length} test cases`);
1191
+ } : {}
1192
+ });
1193
+ }
1194
+ await AppBrain.recordFlakinessOutcome(tenantId, tc.id).catch(() => {
1195
+ });
1196
+ } catch (err) {
1197
+ failed++;
1198
+ runResults.push({ testCaseId: tc.id, status: "ERROR" });
1199
+ await AppBrain.saveTestResult(tenantId, {
1200
+ runId: testRunId,
1201
+ testCaseId: tc.id,
1202
+ status: "ERROR",
1203
+ error: String(err?.message ?? err).slice(0, 500)
1204
+ }).catch(() => {
1205
+ });
1292
1206
  }
1293
- await AppBrain.completeTestRun(tenantId, testRunId, {
1294
- total: testCases.length,
1295
- passed,
1296
- failed,
1297
- skipped: 0,
1298
- duration: Date.now() - runStart,
1299
- });
1300
- await withTenant(tenantId, (tx) => tx.platformJob.update({
1301
- where: { id: jobId },
1302
- data: { status: 'DONE', progress: 100, finishedAt: new Date(), result: { total: testCases.length, passed, failed } },
1303
- }));
1304
- // Same full fan-out as the interactive web-driven run path (in-app,
1305
- // generic webhook, suite-owner failure emails, GitHub check-run, Slack,
1306
- // Teams, alert emails on failure) — a CI-triggered run has nobody
1307
- // watching the SSE stream, so this is the only place these fire from.
1308
- notifyRunCompletion(tenantId, projectId, testRunId, failed === 0 ? 'COMPLETED' : 'FAILED', { passed, failed, total: testCases.length }, runResults).catch(console.error);
1309
- }
1310
- catch (err) {
1311
- const msg = String(err?.message ?? err).slice(0, 500);
1312
- logger.error({ jobId, testRunId, err: msg }, 'processRunTestSuiteJob failed');
1313
- await AppBrain.failTestRun(tenantId, testRunId, msg).catch(() => { });
1314
- await withTenant(tenantId, (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', finishedAt: new Date(), error: msg } })).catch(() => { });
1207
+ }));
1208
+ await updateJobProgress(
1209
+ tenantId,
1210
+ jobId,
1211
+ Math.round(Math.min(i + TEST_EXEC_CONCURRENCY, testCases.length) / testCases.length * 100),
1212
+ `Ran ${Math.min(i + TEST_EXEC_CONCURRENCY, testCases.length)}/${testCases.length} test cases`
1213
+ );
1315
1214
  }
1215
+ await AppBrain.completeTestRun(tenantId, testRunId, {
1216
+ total: testCases.length,
1217
+ passed,
1218
+ failed,
1219
+ skipped: 0,
1220
+ duration: Date.now() - runStart
1221
+ });
1222
+ await withTenant(
1223
+ tenantId,
1224
+ (tx) => tx.platformJob.update({
1225
+ where: { id: jobId },
1226
+ data: { status: "DONE", progress: 100, finishedAt: /* @__PURE__ */ new Date(), result: { total: testCases.length, passed, failed } }
1227
+ })
1228
+ );
1229
+ notifyRunCompletion(
1230
+ tenantId,
1231
+ projectId,
1232
+ testRunId,
1233
+ failed === 0 ? "COMPLETED" : "FAILED",
1234
+ { passed, failed, total: testCases.length },
1235
+ runResults
1236
+ ).catch(console.error);
1237
+ } catch (err) {
1238
+ const msg = String(err?.message ?? err).slice(0, 500);
1239
+ logger.error({ jobId, testRunId, err: msg }, "processRunTestSuiteJob failed");
1240
+ await AppBrain.failTestRun(tenantId, testRunId, msg).catch(() => {
1241
+ });
1242
+ await withTenant(
1243
+ tenantId,
1244
+ (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", finishedAt: /* @__PURE__ */ new Date(), error: msg } })
1245
+ ).catch(() => {
1246
+ });
1247
+ }
1316
1248
  }
1317
1249
  async function processGenerateSkillFilesJob(jobId, config) {
1318
- const { tenantId, projectId } = config;
1319
- await withTenant(tenantId, (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: 'RUNNING', startedAt: new Date() } }));
1320
- try {
1321
- await updateJobProgress(tenantId, jobId, 10, 'Loading project data…');
1322
- const projectMap = await AppBrain.getProjectMap(tenantId, projectId);
1323
- if (!projectMap)
1324
- throw new Error('Project not found.');
1325
- const screens = (projectMap.screens ?? []);
1326
- // Manual regeneration has no fresh crawl result — reuse the metrics and
1327
- // endpoints persisted on the last completed crawl job so the Performance
1328
- // Baseline and API Contract aren't wiped to "no data captured".
1329
- const lastCrawl = await withTenant(tenantId, (tx) => tx.platformJob.findFirst({
1330
- where: { projectId, type: 'CRAWL', status: 'DONE' },
1331
- orderBy: { finishedAt: 'desc' },
1332
- select: { result: true },
1333
- }));
1334
- const lastResult = (lastCrawl?.result ?? {});
1335
- await updateJobProgress(tenantId, jobId, 25, `Generating ${screens.length} app intelligence skill files…`);
1336
- await generateTaxonomySkillFiles(tenantId, projectId, screens, lastResult.apiEndpoints, lastResult.performanceMetrics);
1337
- await updateJobProgress(tenantId, jobId, 85, 'Refreshing testing intelligence skill files…');
1338
- await Promise.allSettled([
1339
- refreshTestCasesSkillFile(tenantId, projectId),
1340
- refreshTestDataSkillFile(tenantId, projectId),
1341
- refreshTestScriptSkillFile(tenantId, projectId),
1342
- refreshRequirementsCoverageSkillFile(tenantId, projectId),
1343
- ]);
1344
- await withTenant(tenantId, (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: 'DONE', progress: 100, finishedAt: new Date() } }));
1345
- }
1346
- catch (err) {
1347
- await withTenant(tenantId, (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', error: String(err?.message ?? err), finishedAt: new Date() } }));
1348
- }
1250
+ const { tenantId, projectId } = config;
1251
+ await withTenant(
1252
+ tenantId,
1253
+ (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "RUNNING", startedAt: /* @__PURE__ */ new Date() } })
1254
+ );
1255
+ try {
1256
+ await updateJobProgress(tenantId, jobId, 10, "Loading project data\u2026");
1257
+ const projectMap = await AppBrain.getProjectMap(tenantId, projectId);
1258
+ if (!projectMap) throw new Error("Project not found.");
1259
+ const screens = projectMap.screens ?? [];
1260
+ const lastCrawl = await withTenant(
1261
+ tenantId,
1262
+ (tx) => tx.platformJob.findFirst({
1263
+ where: { projectId, type: "CRAWL", status: "DONE" },
1264
+ orderBy: { finishedAt: "desc" },
1265
+ select: { result: true }
1266
+ })
1267
+ );
1268
+ const lastResult = lastCrawl?.result ?? {};
1269
+ await updateJobProgress(tenantId, jobId, 25, `Generating ${screens.length} app intelligence skill files\u2026`);
1270
+ await generateTaxonomySkillFiles(tenantId, projectId, screens, lastResult.apiEndpoints, lastResult.performanceMetrics);
1271
+ await updateJobProgress(tenantId, jobId, 85, "Refreshing testing intelligence skill files\u2026");
1272
+ await Promise.allSettled([
1273
+ refreshTestCasesSkillFile(tenantId, projectId),
1274
+ refreshTestDataSkillFile(tenantId, projectId),
1275
+ refreshTestScriptSkillFile(tenantId, projectId),
1276
+ refreshRequirementsCoverageSkillFile(tenantId, projectId)
1277
+ ]);
1278
+ await withTenant(
1279
+ tenantId,
1280
+ (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "DONE", progress: 100, finishedAt: /* @__PURE__ */ new Date() } })
1281
+ );
1282
+ } catch (err) {
1283
+ await withTenant(
1284
+ tenantId,
1285
+ (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", error: String(err?.message ?? err), finishedAt: /* @__PURE__ */ new Date() } })
1286
+ );
1287
+ }
1349
1288
  }
1350
- // DATA_RETENTION_PURGE — deletes AuditLog rows older than each tenant's
1351
- // dataRetentionDays setting (default 365), and LLMCallLog rows older than 90
1352
- // days (fixed, no per-tenant override). Runs once daily via the cron in
1353
- // worker.ts. The job has no tenantId/projectId — it sweeps all tenants so
1354
- // enqueueJob() is not used; the worker.ts cron calls this directly.
1355
- export async function processDataRetentionPurge() {
1356
- logger.info('[data-retention] starting purge run');
1357
- // LLMCallLog: fixed 90-day retention across all tenants (uses calledAt, not createdAt)
1358
- const llmCutoff = new Date(Date.now() - 90 * 24 * 60 * 60 * 1000);
1359
- const llmResult = await prismaAdmin.lLMCallLog.deleteMany({
1360
- where: { calledAt: { lt: llmCutoff } },
1289
+ async function processDataRetentionPurge() {
1290
+ logger.info("[data-retention] starting purge run");
1291
+ const llmCutoff = new Date(Date.now() - 90 * 24 * 60 * 60 * 1e3);
1292
+ const llmResult = await prismaAdmin.lLMCallLog.deleteMany({
1293
+ where: { calledAt: { lt: llmCutoff } }
1294
+ });
1295
+ logger.info({ deleted: llmResult.count }, "[data-retention] LLMCallLog purged");
1296
+ const tenants = await prismaAdmin.tenant.findMany({ select: { id: true } });
1297
+ let auditTotal = 0;
1298
+ for (const { id: tenantId } of tenants) {
1299
+ const setting = await prismaAdmin.tenantSetting.findFirst({
1300
+ where: { tenantId },
1301
+ orderBy: { dataRetentionDays: "asc" },
1302
+ select: { dataRetentionDays: true }
1361
1303
  });
1362
- logger.info({ deleted: llmResult.count }, '[data-retention] LLMCallLog purged');
1363
- // AuditLog: per-tenant retention via TenantSetting.dataRetentionDays
1364
- const tenants = await prismaAdmin.tenant.findMany({ select: { id: true } });
1365
- let auditTotal = 0;
1366
- for (const { id: tenantId } of tenants) {
1367
- // Find the most restrictive (lowest) dataRetentionDays across this tenant's
1368
- // settings rows, falling back to 365 if no setting row exists.
1369
- const setting = await prismaAdmin.tenantSetting.findFirst({
1370
- where: { tenantId },
1371
- orderBy: { dataRetentionDays: 'asc' },
1372
- select: { dataRetentionDays: true },
1373
- });
1374
- const retentionDays = setting?.dataRetentionDays ?? 365;
1375
- const cutoff = new Date(Date.now() - retentionDays * 24 * 60 * 60 * 1000);
1376
- const result = await prismaAdmin.auditLog.deleteMany({
1377
- where: { tenantId, createdAt: { lt: cutoff } },
1378
- });
1379
- auditTotal += result.count;
1380
- }
1381
- logger.info({ deleted: auditTotal, tenants: tenants.length }, '[data-retention] AuditLog purged');
1382
- // AuthSession: fixed 12-month retention — IP addresses are PII under GDPR.
1383
- // Only purge revoked sessions; active (non-revoked) sessions are kept.
1384
- const SESSION_RETENTION_DAYS = 365;
1385
- const sessionCutoff = new Date(Date.now() - SESSION_RETENTION_DAYS * 24 * 60 * 60_000);
1386
- const sessionResult = await prismaAdmin.authSession.deleteMany({
1387
- where: {
1388
- createdAt: { lt: sessionCutoff },
1389
- revokedAt: { not: null },
1390
- },
1304
+ const retentionDays = setting?.dataRetentionDays ?? 365;
1305
+ const cutoff = new Date(Date.now() - retentionDays * 24 * 60 * 60 * 1e3);
1306
+ const result = await prismaAdmin.auditLog.deleteMany({
1307
+ where: { tenantId, createdAt: { lt: cutoff } }
1391
1308
  });
1392
- logger.info({ deleted: sessionResult.count }, '[data-retention] AuthSession purged');
1309
+ auditTotal += result.count;
1310
+ }
1311
+ logger.info({ deleted: auditTotal, tenants: tenants.length }, "[data-retention] AuditLog purged");
1312
+ const SESSION_RETENTION_DAYS = 365;
1313
+ const sessionCutoff = new Date(Date.now() - SESSION_RETENTION_DAYS * 24 * 60 * 6e4);
1314
+ const sessionResult = await prismaAdmin.authSession.deleteMany({
1315
+ where: {
1316
+ createdAt: { lt: sessionCutoff },
1317
+ revokedAt: { not: null }
1318
+ }
1319
+ });
1320
+ logger.info({ deleted: sessionResult.count }, "[data-retention] AuthSession purged");
1393
1321
  }
1394
- // ── URL Discovery Job ─────────────────────────────────────────────────────────
1395
- // Three-phase URL discovery: sitemap seed → unauthenticated BFS spider → authenticated BFS
1396
- // spider (when crawl credentials exist). Persists discovered URLs to crawlConfig and marks the
1397
- // job DONE with the full URL list so the UI can poll for results.
1398
1322
  async function processDiscoverJob(jobId, config, opts) {
1399
- const { tenantId, projectId } = config;
1400
- const controller = new AbortController();
1401
- runningControllers.set(jobId, controller);
1402
- // Independent heartbeat timer — keeps heartbeatAt fresh even when no URLs are
1403
- // arriving (e.g. during auth login, page-load waits, 2FA handling).
1404
- const heartbeatTimer = setInterval(() => {
1405
- void heartbeatJob(tenantId, jobId);
1406
- }, JOB_HEARTBEAT_INTERVAL_MS);
1407
- // Shared accumulator — updated by onUrlsDiscovered callback as each page is visited.
1408
- // This lets the frontend poll and see a growing list before the job finishes.
1409
- const allUrls = new Set();
1410
- let lastPartialFlush = 0;
1411
- let lastFlushedPct = 0; // high-water mark — progress never decreases
1412
- const flushPartialUrls = async (pct, step) => {
1413
- const now = Date.now();
1414
- // Flush to DB at most every 5 seconds to avoid excessive writes
1415
- if (now - lastPartialFlush < 5_000 && allUrls.size < 5)
1416
- return;
1417
- const monotonic = Math.max(pct, lastFlushedPct);
1418
- lastFlushedPct = monotonic;
1419
- lastPartialFlush = now;
1420
- try {
1421
- await withTenant(tenantId, (tx) => tx.platformJob.update({
1422
- where: { id: jobId },
1423
- data: { progress: monotonic, result: { step, partialUrls: Array.from(allUrls) }, heartbeatAt: new Date() },
1424
- }));
1425
- }
1426
- catch { /* non-fatal */ }
1427
- await opts?.onProgress?.(monotonic);
1428
- };
1429
- const onProgress = async (pct, step) => {
1430
- await updateJobProgress(tenantId, jobId, pct, step);
1431
- await opts?.onProgress?.(pct);
1432
- };
1323
+ const { tenantId, projectId } = config;
1324
+ const controller = new AbortController();
1325
+ runningControllers.set(jobId, controller);
1326
+ const heartbeatTimer = setInterval(() => {
1327
+ void heartbeatJob(tenantId, jobId);
1328
+ }, JOB_HEARTBEAT_INTERVAL_MS);
1329
+ const allUrls = /* @__PURE__ */ new Set();
1330
+ let lastPartialFlush = 0;
1331
+ let lastFlushedPct = 0;
1332
+ const flushPartialUrls = async (pct, step) => {
1333
+ const now = Date.now();
1334
+ if (now - lastPartialFlush < 5e3 && allUrls.size < 5) return;
1335
+ const monotonic = Math.max(pct, lastFlushedPct);
1336
+ lastFlushedPct = monotonic;
1337
+ lastPartialFlush = now;
1433
1338
  try {
1434
- // Mark RUNNING inside the try block so any DB error is caught by the internal
1435
- // handler below (which writes result.error). If this were before the try block,
1436
- // a DB failure would bubble up to dispatchClaimedJob's catch, which writes to
1437
- // the `error` column instead of `result.error` — the frontend would then show
1438
- // "Unknown error" rather than the actual message.
1439
- await withTenant(tenantId, (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: 'RUNNING', startedAt: new Date() } }));
1440
- // Load decrypted project crawl config for credentials/proxy/sitemap settings.
1441
- // Reading Project.crawlConfig directly leaves encrypted fields as *_enc, so
1442
- // discovery would see username but miss password/storageState/authToken.
1443
- const crawlCfg = await AppBrain.getCrawlConfig(tenantId, projectId).catch(() => null) ?? {};
1444
- const appUrl = config.appUrl ?? crawlCfg.appUrl;
1445
- if (!appUrl)
1446
- throw new Error('No app URL configured. Save the crawl settings first.');
1447
- const maxDiscoverUrls = config.maxDiscoverUrls ?? 2000;
1448
- const proxyUrl = config.proxyUrl ?? crawlCfg.proxyUrl;
1449
- const screenshotDir = os.tmpdir();
1450
- const discoverStorageConfig = await getTenantStorageConfig(tenantId).catch(() => null);
1451
- await onProgress(3, 'Seeding from sitemap/robots.txt…');
1452
- // Phase 1 — sitemap + robots.txt seed (fast, no browser)
1453
- const sitemapResult = await discoverProjectUrls(appUrl, {
1454
- sitemapUrl: config.sitemapUrl ?? crawlCfg.sitemapUrl,
1455
- proxyUrl,
1456
- }).catch(() => ({ candidates: [], sitemapFound: false }));
1457
- const sitemapUrls = sitemapResult.candidates.map((c) => c.url);
1458
- // Seed accumulator with sitemap URLs so they show immediately
1459
- for (const u of sitemapUrls)
1460
- allUrls.add(u);
1461
- logger.info({ count: sitemapUrls.length, sitemapFound: sitemapResult.sitemapFound }, '[discover] Phase 1 sitemap done');
1462
- // Phase 2 — unauthenticated BFS spider
1463
- await onProgress(8, `Unauthenticated spider from ${appUrl}…`);
1464
- const unauthResult = await crawlProject({
1465
- tenantId,
1466
- projectId,
1467
- appUrl,
1468
- discoveryMode: true,
1469
- startUrls: sitemapUrls.length > 0 ? sitemapUrls : undefined,
1470
- screenshotDir,
1471
- maxScreens: maxDiscoverUrls,
1472
- proxyUrl,
1473
- useFirefox: crawlCfg.useFirefox,
1474
- storageConfig: discoverStorageConfig,
1475
- abortSignal: controller.signal,
1476
- jobId,
1477
- onUrlsDiscovered: async (urls) => {
1478
- for (const u of urls)
1479
- allUrls.add(u);
1480
- const pct = 8 + Math.round((allUrls.size / Math.max(1, maxDiscoverUrls)) * 42);
1481
- await flushPartialUrls(Math.min(pct, 49), `Discovered ${allUrls.size} URLs…`);
1482
- },
1483
- }, async (pct, step) => {
1484
- await flushPartialUrls(8 + Math.round(pct * 0.42), `Unauthenticated: ${step}`);
1485
- }).catch((err) => {
1486
- if (err?.message === 'CRAWL_ABORTED')
1487
- throw err;
1488
- logger.warn({ err: err?.message }, '[discover] Phase 2 unauthenticated spider failed — continuing with partial results');
1489
- return { discoveredUrls: [] };
1490
- });
1491
- for (const u of (unauthResult.discoveredUrls ?? []))
1492
- allUrls.add(u);
1493
- logger.info({ count: allUrls.size }, '[discover] Phase 2 unauth spider done');
1494
- // Phase 3 — authenticated BFS spider (only when credentials configured)
1495
- // getCrawlConfig returns flat fields (username/password/totpSecret) — NOT nested under .credentials
1496
- let resolvedSso = {};
1497
- if (crawlCfg.ssoProviderId) {
1498
- const ssoProviders = await AppBrain.getWorkspaceSsoProviders(tenantId).catch(() => []);
1499
- const ssoProvider = ssoProviders.find((p) => p.id === crawlCfg.ssoProviderId);
1500
- if (ssoProvider) {
1501
- resolvedSso = {
1502
- ssoProvider: ssoProvider.provider,
1503
- ssoDomain: ssoProvider.domain,
1504
- ssoClientId: ssoProvider.clientId,
1505
- ssoClientSecret: ssoProvider.clientSecret,
1506
- ssoScope: ssoProvider.scope,
1507
- };
1508
- }
1509
- }
1510
- const hasCreds = !!(crawlCfg.username ||
1511
- crawlCfg.storageState ||
1512
- crawlCfg.authToken ||
1513
- crawlCfg.chromeProfilePath ||
1514
- (crawlCfg.httpBasicUsername && crawlCfg.httpBasicPassword));
1515
- if (hasCreds && !controller.signal.aborted) {
1516
- await onProgress(52, 'Authenticated spider…');
1517
- const runtimeCreds = crawlCfg.username
1518
- ? {
1519
- username: crawlCfg.username,
1520
- password: crawlCfg.password ?? '',
1521
- ...(crawlCfg.totpSecret
1522
- ? { getOneTimeCode: async () => generateTOTP(crawlCfg.totpSecret) }
1523
- : {}),
1524
- }
1525
- : undefined;
1526
- const authResult = await crawlProject({
1527
- tenantId,
1528
- projectId,
1529
- appUrl,
1530
- discoveryMode: true,
1531
- credentials: runtimeCreds,
1532
- storageState: crawlCfg.storageState,
1533
- httpBasicUsername: crawlCfg.httpBasicUsername,
1534
- httpBasicPassword: crawlCfg.httpBasicPassword,
1535
- authToken: crawlCfg.authToken,
1536
- authTokenHeader: crawlCfg.authTokenHeader,
1537
- authTokenPrefix: crawlCfg.authTokenPrefix,
1538
- startUrls: crawlCfg.postLoginUrl
1539
- ? [crawlCfg.postLoginUrl, ...(sitemapUrls.length > 0 ? sitemapUrls : [])]
1540
- : (sitemapUrls.length > 0 ? sitemapUrls : undefined),
1541
- screenshotDir,
1542
- maxScreens: maxDiscoverUrls,
1543
- proxyUrl,
1544
- chromeProfilePath: crawlCfg.chromeProfilePath,
1545
- useFirefox: crawlCfg.useFirefox,
1546
- storageConfig: discoverStorageConfig,
1547
- captchaSolverApiKey: crawlCfg.captchaSolverApiKey,
1548
- captchaSolverProvider: crawlCfg.captchaSolverProvider,
1549
- mailslurpApiKey: crawlCfg.mailslurpApiKey,
1550
- mailslurpInboxId: crawlCfg.mailslurpInboxId,
1551
- loginInstructions: crawlCfg.loginInstructions,
1552
- ...resolvedSso,
1553
- abortSignal: controller.signal,
1554
- jobId,
1555
- onUrlsDiscovered: async (urls) => {
1556
- for (const u of urls)
1557
- allUrls.add(u);
1558
- const pct = 52 + Math.round((allUrls.size / Math.max(1, maxDiscoverUrls)) * 38);
1559
- await flushPartialUrls(Math.min(pct, 89), `Discovered ${allUrls.size} URLs (authenticated)…`);
1560
- },
1561
- }, async (pct, step) => {
1562
- await flushPartialUrls(52 + Math.round(pct * 0.38), `Authenticated: ${step}`);
1563
- }).catch((err) => {
1564
- if (err?.message === 'CRAWL_ABORTED')
1565
- throw err;
1566
- logger.warn({ err: err?.message }, '[discover] Phase 3 authenticated spider failed — using unauthenticated results');
1567
- return { discoveredUrls: [] };
1568
- });
1569
- for (const u of ((authResult.discoveredUrls) ?? [])) {
1570
- allUrls.add(u);
1571
- }
1572
- logger.info({ count: allUrls.size }, '[discover] Phase 3 auth spider done');
1573
- }
1574
- // Persist discovered URLs to project crawl config
1575
- const discoveredUrls = Array.from(allUrls);
1576
- const existingCfg = await AppBrain.getCrawlConfig(tenantId, projectId).catch(() => ({}));
1577
- await AppBrain.saveCrawlConfig(tenantId, projectId, {
1578
- ...existingCfg,
1579
- discoveredUrls,
1580
- discoveredAt: new Date().toISOString(),
1581
- });
1582
- await onProgress(100, `Done — ${discoveredUrls.length} URLs discovered`);
1583
- await withTenant(tenantId, (tx) => tx.platformJob.update({
1584
- where: { id: jobId },
1585
- data: {
1586
- status: 'DONE',
1587
- result: {
1588
- discoveredUrls,
1589
- count: discoveredUrls.length,
1590
- sitemapFound: sitemapResult.sitemapFound,
1591
- },
1592
- progress: 100,
1593
- finishedAt: new Date(),
1594
- },
1595
- }));
1596
- notifyAdmins(tenantId, {
1597
- type: 'discovery.done',
1598
- title: 'URL Discovery completed',
1599
- body: `Found ${discoveredUrls.length} URLs for the project.`,
1600
- metadata: { projectId, jobId },
1601
- }).catch(console.error);
1602
- }
1603
- catch (err) {
1604
- logger.error({ err: err?.message, jobId }, '[discover] Job failed');
1605
- await withTenant(tenantId, (tx) => tx.platformJob.update({
1606
- where: { id: jobId },
1607
- data: {
1608
- status: err?.message === 'CRAWL_ABORTED' ? 'CANCELLED' : 'FAILED',
1609
- result: { error: err?.message ?? String(err) },
1610
- finishedAt: new Date(),
1611
- },
1612
- })).catch(console.error);
1339
+ await withTenant(
1340
+ tenantId,
1341
+ (tx) => tx.platformJob.update({
1342
+ where: { id: jobId },
1343
+ data: { progress: monotonic, result: { step, partialUrls: Array.from(allUrls) }, heartbeatAt: /* @__PURE__ */ new Date() }
1344
+ })
1345
+ );
1346
+ } catch {
1613
1347
  }
1614
- finally {
1615
- clearInterval(heartbeatTimer);
1616
- runningControllers.delete(jobId);
1348
+ await opts?.onProgress?.(monotonic);
1349
+ };
1350
+ const onProgress = async (pct, step) => {
1351
+ await updateJobProgress(tenantId, jobId, pct, step);
1352
+ await opts?.onProgress?.(pct);
1353
+ };
1354
+ try {
1355
+ await withTenant(
1356
+ tenantId,
1357
+ (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "RUNNING", startedAt: /* @__PURE__ */ new Date() } })
1358
+ );
1359
+ const crawlCfg = await AppBrain.getCrawlConfig(tenantId, projectId).catch(() => null) ?? {};
1360
+ const appUrl = config.appUrl ?? crawlCfg.appUrl;
1361
+ if (!appUrl) throw new Error("No app URL configured. Save the crawl settings first.");
1362
+ const maxDiscoverUrls = config.maxDiscoverUrls ?? 2e3;
1363
+ const proxyUrl = config.proxyUrl ?? crawlCfg.proxyUrl;
1364
+ const screenshotDir = os.tmpdir();
1365
+ const discoverStorageConfig = await getTenantStorageConfig(tenantId).catch(() => null);
1366
+ await onProgress(3, "Seeding from sitemap/robots.txt\u2026");
1367
+ const sitemapResult = await discoverProjectUrls(appUrl, {
1368
+ sitemapUrl: config.sitemapUrl ?? crawlCfg.sitemapUrl,
1369
+ proxyUrl
1370
+ }).catch(() => ({ candidates: [], sitemapFound: false }));
1371
+ const sitemapUrls = sitemapResult.candidates.map((c) => c.url);
1372
+ for (const u of sitemapUrls) allUrls.add(u);
1373
+ logger.info({ count: sitemapUrls.length, sitemapFound: sitemapResult.sitemapFound }, "[discover] Phase 1 sitemap done");
1374
+ await onProgress(8, `Unauthenticated spider from ${appUrl}\u2026`);
1375
+ const unauthResult = await crawlProject({
1376
+ tenantId,
1377
+ projectId,
1378
+ appUrl,
1379
+ discoveryMode: true,
1380
+ startUrls: sitemapUrls.length > 0 ? sitemapUrls : void 0,
1381
+ screenshotDir,
1382
+ maxScreens: maxDiscoverUrls,
1383
+ proxyUrl,
1384
+ useFirefox: crawlCfg.useFirefox,
1385
+ storageConfig: discoverStorageConfig,
1386
+ abortSignal: controller.signal,
1387
+ jobId,
1388
+ onUrlsDiscovered: async (urls) => {
1389
+ for (const u of urls) allUrls.add(u);
1390
+ const pct = 8 + Math.round(allUrls.size / Math.max(1, maxDiscoverUrls) * 42);
1391
+ await flushPartialUrls(Math.min(pct, 49), `Discovered ${allUrls.size} URLs\u2026`);
1392
+ }
1393
+ }, async (pct, step) => {
1394
+ await flushPartialUrls(8 + Math.round(pct * 0.42), `Unauthenticated: ${step}`);
1395
+ }).catch((err) => {
1396
+ if (err?.message === "CRAWL_ABORTED") throw err;
1397
+ logger.warn({ err: err?.message }, "[discover] Phase 2 unauthenticated spider failed \u2014 continuing with partial results");
1398
+ return { discoveredUrls: [] };
1399
+ });
1400
+ for (const u of unauthResult.discoveredUrls ?? []) allUrls.add(u);
1401
+ logger.info({ count: allUrls.size }, "[discover] Phase 2 unauth spider done");
1402
+ let resolvedSso = {};
1403
+ if (crawlCfg.ssoProviderId) {
1404
+ const ssoProviders = await AppBrain.getWorkspaceSsoProviders(tenantId).catch(() => []);
1405
+ const ssoProvider = ssoProviders.find((p) => p.id === crawlCfg.ssoProviderId);
1406
+ if (ssoProvider) {
1407
+ resolvedSso = {
1408
+ ssoProvider: ssoProvider.provider,
1409
+ ssoDomain: ssoProvider.domain,
1410
+ ssoClientId: ssoProvider.clientId,
1411
+ ssoClientSecret: ssoProvider.clientSecret,
1412
+ ssoScope: ssoProvider.scope
1413
+ };
1414
+ }
1617
1415
  }
1618
- }
1619
- // ── Site Monitor Job ─────────────────────────────────────────────────────────
1620
- // Lightweight scheduled content check: fetches configured URLs, diffs content
1621
- // hashes against stored baselines, fires content.changed webhooks on drift.
1622
- // Baselines are stored in project.crawlConfig.monitorBaselines (url→hash map).
1623
- async function processMonitorJob(jobId, config, opts) {
1624
- const { tenantId, projectId } = config;
1625
- const controller = new AbortController();
1626
- runningControllers.set(jobId, controller);
1627
- try {
1628
- await withTenant(tenantId, (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: 'RUNNING', startedAt: new Date() } }));
1629
- const proj = await withTenant(tenantId, (tx) => tx.project.findUnique({ where: { id: projectId }, select: { crawlConfig: true } }));
1630
- const crawlCfg = proj?.crawlConfig ?? {};
1631
- const monitorUrls = config.monitorUrls ?? crawlCfg.monitorUrls ?? [];
1632
- if (monitorUrls.length === 0) {
1633
- throw new Error('No monitor URLs configured. Add monitorUrls to crawl config or job config.');
1634
- }
1635
- await updateJobProgress(tenantId, jobId, 10, `Monitoring ${monitorUrls.length} URL(s)...`);
1636
- // Baseline: stored baselines from previous monitor run take priority;
1637
- // project screens' elementHash is the fallback for first-run.
1638
- const storedBaselines = crawlCfg.monitorBaselines ?? {};
1639
- const projectScreens = await withTenant(tenantId, (tx) => tx.screen.findMany({ where: { projectId }, select: { url: true, elementHash: true } })).catch(() => []);
1640
- const screenHashMap = {};
1641
- for (const s of projectScreens) {
1642
- if (s.url && s.elementHash)
1643
- screenHashMap[s.url] = s.elementHash;
1416
+ const hasCreds = !!(crawlCfg.username || crawlCfg.storageState || crawlCfg.authToken || crawlCfg.chromeProfilePath || crawlCfg.httpBasicUsername && crawlCfg.httpBasicPassword);
1417
+ if (hasCreds && !controller.signal.aborted) {
1418
+ await onProgress(52, "Authenticated spider\u2026");
1419
+ const runtimeCreds = crawlCfg.username ? {
1420
+ username: crawlCfg.username,
1421
+ password: crawlCfg.password ?? "",
1422
+ ...crawlCfg.totpSecret ? { getOneTimeCode: async () => generateTOTP(crawlCfg.totpSecret) } : {}
1423
+ } : void 0;
1424
+ const authResult = await crawlProject({
1425
+ tenantId,
1426
+ projectId,
1427
+ appUrl,
1428
+ discoveryMode: true,
1429
+ credentials: runtimeCreds,
1430
+ storageState: crawlCfg.storageState,
1431
+ httpBasicUsername: crawlCfg.httpBasicUsername,
1432
+ httpBasicPassword: crawlCfg.httpBasicPassword,
1433
+ authToken: crawlCfg.authToken,
1434
+ authTokenHeader: crawlCfg.authTokenHeader,
1435
+ authTokenPrefix: crawlCfg.authTokenPrefix,
1436
+ startUrls: crawlCfg.postLoginUrl ? [crawlCfg.postLoginUrl, ...sitemapUrls.length > 0 ? sitemapUrls : []] : sitemapUrls.length > 0 ? sitemapUrls : void 0,
1437
+ screenshotDir,
1438
+ maxScreens: maxDiscoverUrls,
1439
+ proxyUrl,
1440
+ chromeProfilePath: crawlCfg.chromeProfilePath,
1441
+ useFirefox: crawlCfg.useFirefox,
1442
+ storageConfig: discoverStorageConfig,
1443
+ captchaSolverApiKey: crawlCfg.captchaSolverApiKey,
1444
+ captchaSolverProvider: crawlCfg.captchaSolverProvider,
1445
+ mailslurpApiKey: crawlCfg.mailslurpApiKey,
1446
+ mailslurpInboxId: crawlCfg.mailslurpInboxId,
1447
+ loginInstructions: crawlCfg.loginInstructions,
1448
+ ...resolvedSso,
1449
+ abortSignal: controller.signal,
1450
+ jobId,
1451
+ onUrlsDiscovered: async (urls) => {
1452
+ for (const u of urls) allUrls.add(u);
1453
+ const pct = 52 + Math.round(allUrls.size / Math.max(1, maxDiscoverUrls) * 38);
1454
+ await flushPartialUrls(Math.min(pct, 89), `Discovered ${allUrls.size} URLs (authenticated)\u2026`);
1644
1455
  }
1645
- const results = [];
1646
- const newBaselines = { ...storedBaselines };
1647
- for (let i = 0; i < monitorUrls.length; i++) {
1648
- if (controller.signal.aborted)
1649
- break;
1650
- const url = monitorUrls[i];
1651
- const pct = 10 + Math.floor((i / monitorUrls.length) * 80);
1652
- await updateJobProgress(tenantId, jobId, pct, `Checking ${url}...`);
1653
- opts?.onProgress?.(pct);
1654
- try {
1655
- const crawlResult = await httpCrawl({ appUrl: url, maxScreens: 1, respectRobotsTxt: false, timeoutMs: parseInt(process.env.CRAWL_TIMEOUT_MS ?? '20000', 10) });
1656
- const screen = crawlResult.screens[0];
1657
- if (!screen) {
1658
- results.push({ url, changed: false, reason: 'fetch_empty', currentHash: '', previousHash: null });
1659
- continue;
1660
- }
1661
- const currentHash = hashContent(normalizeText(screen.markdown));
1662
- const previousHash = storedBaselines[url] ?? screenHashMap[url] ?? null;
1663
- newBaselines[url] = currentHash;
1664
- if (!previousHash) {
1665
- results.push({ url, changed: false, reason: 'new_baseline', currentHash, previousHash: null });
1666
- }
1667
- else if (previousHash === currentHash) {
1668
- results.push({ url, changed: false, reason: 'unchanged', currentHash, previousHash });
1669
- }
1670
- else {
1671
- results.push({ url, changed: true, reason: 'content_changed', currentHash, previousHash });
1672
- }
1673
- }
1674
- catch (fetchErr) {
1675
- results.push({ url, changed: false, reason: `error:${fetchErr?.message ?? String(fetchErr)}`, currentHash: '', previousHash: null });
1676
- }
1456
+ }, async (pct, step) => {
1457
+ await flushPartialUrls(52 + Math.round(pct * 0.38), `Authenticated: ${step}`);
1458
+ }).catch((err) => {
1459
+ if (err?.message === "CRAWL_ABORTED") throw err;
1460
+ logger.warn({ err: err?.message }, "[discover] Phase 3 authenticated spider failed \u2014 using unauthenticated results");
1461
+ return { discoveredUrls: [] };
1462
+ });
1463
+ for (const u of authResult.discoveredUrls ?? []) {
1464
+ allUrls.add(u);
1465
+ }
1466
+ logger.info({ count: allUrls.size }, "[discover] Phase 3 auth spider done");
1467
+ }
1468
+ const discoveredUrls = Array.from(allUrls);
1469
+ const existingCfg = await AppBrain.getCrawlConfig(tenantId, projectId).catch(() => ({}));
1470
+ await AppBrain.saveCrawlConfig(tenantId, projectId, {
1471
+ ...existingCfg,
1472
+ discoveredUrls,
1473
+ discoveredAt: (/* @__PURE__ */ new Date()).toISOString()
1474
+ });
1475
+ await onProgress(100, `Done \u2014 ${discoveredUrls.length} URLs discovered`);
1476
+ await withTenant(
1477
+ tenantId,
1478
+ (tx) => tx.platformJob.update({
1479
+ where: { id: jobId },
1480
+ data: {
1481
+ status: "DONE",
1482
+ result: {
1483
+ discoveredUrls,
1484
+ count: discoveredUrls.length,
1485
+ sitemapFound: sitemapResult.sitemapFound
1486
+ },
1487
+ progress: 100,
1488
+ finishedAt: /* @__PURE__ */ new Date()
1677
1489
  }
1678
- // Persist updated baselines so next run has fresh reference points.
1679
- await withTenant(tenantId, (tx) => tx.project.update({
1680
- where: { id: projectId },
1681
- data: { crawlConfig: { ...crawlCfg, monitorBaselines: newBaselines } },
1682
- })).catch((e) => logger.warn({ err: e }, '[monitor] baseline persist failed'));
1683
- const changedUrls = results.filter(r => r.changed);
1684
- if (changedUrls.length > 0) {
1685
- dispatchWebhook(tenantId, projectId, 'content.changed', {
1686
- jobId,
1687
- changedUrls: changedUrls.map(r => ({ url: r.url, previousHash: r.previousHash, currentHash: r.currentHash })),
1688
- totalChecked: results.length,
1689
- changedCount: changedUrls.length,
1690
- }).catch(console.error);
1490
+ })
1491
+ );
1492
+ notifyAdmins(tenantId, {
1493
+ type: "discovery.done",
1494
+ title: "URL Discovery completed",
1495
+ body: `Found ${discoveredUrls.length} URLs for the project.`,
1496
+ metadata: { projectId, jobId }
1497
+ }).catch(console.error);
1498
+ } catch (err) {
1499
+ logger.error({ err: err?.message, jobId }, "[discover] Job failed");
1500
+ await withTenant(
1501
+ tenantId,
1502
+ (tx) => tx.platformJob.update({
1503
+ where: { id: jobId },
1504
+ data: {
1505
+ status: err?.message === "CRAWL_ABORTED" ? "CANCELLED" : "FAILED",
1506
+ result: { error: err?.message ?? String(err) },
1507
+ finishedAt: /* @__PURE__ */ new Date()
1691
1508
  }
1692
- await updateJobProgress(tenantId, jobId, 100, 'Monitor check complete');
1693
- await withTenant(tenantId, (tx) => tx.platformJob.update({
1694
- where: { id: jobId },
1695
- data: {
1696
- status: 'DONE',
1697
- progress: 100,
1698
- finishedAt: new Date(),
1699
- result: { results, changedCount: changedUrls.length, totalChecked: results.length },
1700
- },
1701
- }));
1702
- }
1703
- catch (err) {
1704
- await withTenant(tenantId, (tx) => tx.platformJob.update({
1705
- where: { id: jobId },
1706
- data: { status: 'FAILED', error: String(err?.message ?? err), finishedAt: new Date() },
1707
- }));
1708
- }
1709
- finally {
1710
- runningControllers.delete(jobId);
1711
- }
1509
+ })
1510
+ ).catch(console.error);
1511
+ } finally {
1512
+ clearInterval(heartbeatTimer);
1513
+ runningControllers.delete(jobId);
1514
+ }
1712
1515
  }
1713
- // ─── IMPORT_TESTS job processor ──────────────────────────────────────────────
1714
- // Async counterpart to the synchronous POST /artifacts/:id/import-tests route.
1715
- // Enqueued by POST /modernization/bulk-import for each artifact in a batch.
1716
- // Config: { tenantId, projectId, artifactId, suiteId?, autoLabel?, sessionId? }
1717
- async function processImportTestsJob(jobId, config) {
1718
- const { tenantId, projectId, artifactId, autoLabel, sourcePlatform } = config;
1719
- const jobStart = new Date();
1720
- let suiteId = config.suiteId ?? 'new';
1721
- await updateJobProgress(tenantId, jobId, 5, 'Loading artifact');
1722
- const artifact = await withTenant(tenantId, (tx) => tx.projectArtifact.findUnique({ where: { id: artifactId } }));
1723
- if (!artifact || artifact.tenantId !== tenantId) {
1724
- throw new Error(`Artifact ${artifactId} not found`);
1516
+ async function processMonitorJob(jobId, config, opts) {
1517
+ const { tenantId, projectId } = config;
1518
+ const controller = new AbortController();
1519
+ runningControllers.set(jobId, controller);
1520
+ try {
1521
+ await withTenant(
1522
+ tenantId,
1523
+ (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "RUNNING", startedAt: /* @__PURE__ */ new Date() } })
1524
+ );
1525
+ const proj = await withTenant(
1526
+ tenantId,
1527
+ (tx) => tx.project.findUnique({ where: { id: projectId }, select: { crawlConfig: true } })
1528
+ );
1529
+ const crawlCfg = proj?.crawlConfig ?? {};
1530
+ const monitorUrls = config.monitorUrls ?? crawlCfg.monitorUrls ?? [];
1531
+ if (monitorUrls.length === 0) {
1532
+ throw new Error("No monitor URLs configured. Add monitorUrls to crawl config or job config.");
1725
1533
  }
1726
- const content = artifact.content ?? '';
1727
- const filename = artifact.originalName ?? artifact.name ?? '';
1728
- const kind = artifact.kind ?? '';
1729
- // Detect format using same logic as the synchronous route
1730
- function detectFormat(fn, k, c) {
1731
- if (fn.endsWith('.feature'))
1732
- return 'GHERKIN';
1733
- if (fn.match(/\.(xlsx|xls|csv)$/i) || k === 'SPREADSHEET')
1734
- return 'EXCEL';
1735
- if (fn.endsWith('.txt') && c && c.split('\n').some((l) => l.trim().startsWith('|')))
1736
- return 'EXCEL';
1737
- return 'AI_EXTRACT';
1534
+ await updateJobProgress(tenantId, jobId, 10, `Monitoring ${monitorUrls.length} URL(s)...`);
1535
+ const storedBaselines = crawlCfg.monitorBaselines ?? {};
1536
+ const projectScreens = await withTenant(
1537
+ tenantId,
1538
+ (tx) => tx.screen.findMany({ where: { projectId }, select: { url: true, elementHash: true } })
1539
+ ).catch(() => []);
1540
+ const screenHashMap = {};
1541
+ for (const s of projectScreens) {
1542
+ if (s.url && s.elementHash) screenHashMap[s.url] = s.elementHash;
1738
1543
  }
1739
- const format = detectFormat(filename, kind, content);
1740
- await updateJobProgress(tenantId, jobId, 15, `Parsing ${filename}`);
1741
- const { parseTestCases, labelTestCasesBatch } = await import('@detiq/agents');
1742
- let res = await parseTestCases(content, format, tenantId, artifact.columnMapping ?? undefined, undefined, undefined, sourcePlatform);
1743
- if (res.cases.length === 0 && format !== 'AI_EXTRACT') {
1744
- await updateJobProgress(tenantId, jobId, 25, 'Retrying with AI extraction');
1745
- try {
1746
- res = await parseTestCases(content, 'AI_EXTRACT', tenantId, undefined, undefined, undefined, sourcePlatform);
1544
+ const results = [];
1545
+ const newBaselines = { ...storedBaselines };
1546
+ for (let i = 0; i < monitorUrls.length; i++) {
1547
+ if (controller.signal.aborted) break;
1548
+ const url = monitorUrls[i];
1549
+ const pct = 10 + Math.floor(i / monitorUrls.length * 80);
1550
+ await updateJobProgress(tenantId, jobId, pct, `Checking ${url}...`);
1551
+ opts?.onProgress?.(pct);
1552
+ try {
1553
+ const crawlResult = await httpCrawl({ appUrl: url, maxScreens: 1, respectRobotsTxt: false, timeoutMs: parseInt(process.env.CRAWL_TIMEOUT_MS ?? "20000", 10) });
1554
+ const screen = crawlResult.screens[0];
1555
+ if (!screen) {
1556
+ results.push({ url, changed: false, reason: "fetch_empty", currentHash: "", previousHash: null });
1557
+ continue;
1747
1558
  }
1748
- catch {
1749
- // AI unavailable (no LLM configured) — proceed with 0 cases
1559
+ const currentHash = hashContent(normalizeText(screen.markdown));
1560
+ const previousHash = storedBaselines[url] ?? screenHashMap[url] ?? null;
1561
+ newBaselines[url] = currentHash;
1562
+ if (!previousHash) {
1563
+ results.push({ url, changed: false, reason: "new_baseline", currentHash, previousHash: null });
1564
+ } else if (previousHash === currentHash) {
1565
+ results.push({ url, changed: false, reason: "unchanged", currentHash, previousHash });
1566
+ } else {
1567
+ results.push({ url, changed: true, reason: "content_changed", currentHash, previousHash });
1750
1568
  }
1569
+ } catch (fetchErr) {
1570
+ results.push({ url, changed: false, reason: `error:${fetchErr?.message ?? String(fetchErr)}`, currentHash: "", previousHash: null });
1571
+ }
1751
1572
  }
1752
- const parsedCases = res.cases;
1753
- if (parsedCases.length === 0) {
1754
- await withTenant(tenantId, (tx) => tx.platformJob.update({
1755
- where: { id: jobId },
1756
- data: { status: 'DONE', progress: 100, finishedAt: new Date(), result: { imported: 0, skipped: 0, labeled: 0 } },
1757
- }));
1758
- return;
1759
- }
1760
- await updateJobProgress(tenantId, jobId, 35, `Parsed ${parsedCases.length} cases — creating suite`);
1761
- // Auto-create suite from filename when suiteId === 'new'
1762
- if (!suiteId || suiteId === 'new') {
1763
- const suiteName = filename.replace(/\.[^/.]+$/, '') || 'Imported Tests';
1764
- const suite = await withTenant(tenantId, (tx) => tx.testSuite.create({
1765
- data: { tenantId, projectId, name: suiteName },
1766
- }));
1767
- suiteId = suite.id;
1573
+ await withTenant(
1574
+ tenantId,
1575
+ (tx) => tx.project.update({
1576
+ where: { id: projectId },
1577
+ data: { crawlConfig: { ...crawlCfg, monitorBaselines: newBaselines } }
1578
+ })
1579
+ ).catch((e) => logger.warn({ err: e }, "[monitor] baseline persist failed"));
1580
+ const changedUrls = results.filter((r) => r.changed);
1581
+ if (changedUrls.length > 0) {
1582
+ dispatchWebhook(tenantId, projectId, "content.changed", {
1583
+ jobId,
1584
+ changedUrls: changedUrls.map((r) => ({ url: r.url, previousHash: r.previousHash, currentHash: r.currentHash })),
1585
+ totalChecked: results.length,
1586
+ changedCount: changedUrls.length
1587
+ }).catch(console.error);
1768
1588
  }
1769
- await updateJobProgress(tenantId, jobId, 50, `Importing ${parsedCases.length} cases`);
1770
- const { created, skipped } = await AppBrain.importTestCasesFromArtifact(tenantId, projectId, suiteId, parsedCases, { duplicateMode: 'skip', artifactId });
1771
- let labeled = 0;
1772
- if (autoLabel && created.length > 0) {
1773
- await updateJobProgress(tenantId, jobId, 75, `AI labeling ${created.length} cases`);
1774
- try {
1775
- const labelInput = created.map((c) => ({
1776
- id: c.id, title: c.title,
1777
- description: c.description ?? '',
1778
- stepsText: (Array.isArray(c.steps) ? c.steps.join(' ') : String(c.steps ?? '')).slice(0, 200),
1779
- }));
1780
- const labelResults = await labelTestCasesBatch(labelInput, tenantId);
1781
- // Persist labels back to each test case
1782
- await Promise.all(labelResults.map((lr) => withTenant(tenantId, (tx) => tx.testCase.update({
1783
- where: { id: lr.id },
1784
- data: {
1785
- priority: lr.priority ?? undefined,
1786
- kind: lr.kind ?? undefined,
1787
- tags: lr.tags ?? undefined,
1788
- status: lr.needsReview ? 'PENDING_REVIEW' : undefined,
1789
- },
1790
- })).catch(() => { }) // non-fatal per-case
1791
- ));
1792
- labeled = labelResults.length;
1793
- }
1794
- catch {
1795
- // non-fatal — import succeeded even if labeling fails
1589
+ await updateJobProgress(tenantId, jobId, 100, "Monitor check complete");
1590
+ await withTenant(
1591
+ tenantId,
1592
+ (tx) => tx.platformJob.update({
1593
+ where: { id: jobId },
1594
+ data: {
1595
+ status: "DONE",
1596
+ progress: 100,
1597
+ finishedAt: /* @__PURE__ */ new Date(),
1598
+ result: { results, changedCount: changedUrls.length, totalChecked: results.length }
1796
1599
  }
1600
+ })
1601
+ );
1602
+ } catch (err) {
1603
+ await withTenant(
1604
+ tenantId,
1605
+ (tx) => tx.platformJob.update({
1606
+ where: { id: jobId },
1607
+ data: { status: "FAILED", error: String(err?.message ?? err), finishedAt: /* @__PURE__ */ new Date() }
1608
+ })
1609
+ );
1610
+ } finally {
1611
+ runningControllers.delete(jobId);
1612
+ }
1613
+ }
1614
+ async function processImportTestsJob(jobId, config) {
1615
+ const { tenantId, projectId, artifactId, autoLabel, sourcePlatform } = config;
1616
+ const jobStart = /* @__PURE__ */ new Date();
1617
+ let suiteId = config.suiteId ?? "new";
1618
+ await updateJobProgress(tenantId, jobId, 5, "Loading artifact");
1619
+ const artifact = await withTenant(
1620
+ tenantId,
1621
+ (tx) => tx.projectArtifact.findUnique({ where: { id: artifactId } })
1622
+ );
1623
+ if (!artifact || artifact.tenantId !== tenantId) {
1624
+ throw new Error(`Artifact ${artifactId} not found`);
1625
+ }
1626
+ const content = artifact.content ?? "";
1627
+ const filename = artifact.originalName ?? artifact.name ?? "";
1628
+ const kind = artifact.kind ?? "";
1629
+ function detectFormat(fn, k, c) {
1630
+ if (fn.endsWith(".feature")) return "GHERKIN";
1631
+ if (fn.match(/\.(xlsx|xls|csv)$/i) || k === "SPREADSHEET") return "EXCEL";
1632
+ if (fn.endsWith(".txt") && c && c.split("\n").some((l) => l.trim().startsWith("|"))) return "EXCEL";
1633
+ return "AI_EXTRACT";
1634
+ }
1635
+ const format = detectFormat(filename, kind, content);
1636
+ await updateJobProgress(tenantId, jobId, 15, `Parsing ${filename}`);
1637
+ const { parseTestCases, labelTestCasesBatch } = await import("@detiq/agents");
1638
+ let res = await parseTestCases(content, format, tenantId, artifact.columnMapping ?? void 0, void 0, void 0, sourcePlatform);
1639
+ if (res.cases.length === 0 && format !== "AI_EXTRACT") {
1640
+ await updateJobProgress(tenantId, jobId, 25, "Retrying with AI extraction");
1641
+ try {
1642
+ res = await parseTestCases(content, "AI_EXTRACT", tenantId, void 0, void 0, void 0, sourcePlatform);
1643
+ } catch {
1797
1644
  }
1798
- // Auto-generate AI test data for imported cases that have recorded form fields (testDataInline)
1799
- const casesNeedingData = created.filter((c) => {
1800
- const inline = c.testDataInline;
1801
- return inline && typeof inline === 'object' && Object.keys(inline).length > 0;
1802
- });
1803
- if (casesNeedingData.length > 0) {
1804
- const cap = Math.min(casesNeedingData.length, 20);
1805
- await updateJobProgress(tenantId, jobId, 90, `Generating test data for ${cap} cases`);
1806
- const { generateTestData, refreshTestDataSkillFile } = await import('@detiq/agents');
1807
- for (let i = 0; i < cap; i++) {
1808
- const c = casesNeedingData[i];
1809
- await generateTestData({ tenantId, projectId, testCaseId: c.id, jobId }).catch((e) => {
1810
- logger.warn({ err: e?.message, testCaseId: c.id }, '[import] test data generation failed');
1811
- });
1812
- }
1813
- refreshTestDataSkillFile(tenantId, projectId).catch(console.error);
1645
+ }
1646
+ const parsedCases = res.cases;
1647
+ if (parsedCases.length === 0) {
1648
+ await withTenant(
1649
+ tenantId,
1650
+ (tx) => tx.platformJob.update({
1651
+ where: { id: jobId },
1652
+ data: { status: "DONE", progress: 100, finishedAt: /* @__PURE__ */ new Date(), result: { imported: 0, skipped: 0, labeled: 0 } }
1653
+ })
1654
+ );
1655
+ return;
1656
+ }
1657
+ await updateJobProgress(tenantId, jobId, 35, `Parsed ${parsedCases.length} cases \u2014 creating suite`);
1658
+ if (!suiteId || suiteId === "new") {
1659
+ const suiteName = filename.replace(/\.[^/.]+$/, "") || "Imported Tests";
1660
+ const suite = await withTenant(
1661
+ tenantId,
1662
+ (tx) => tx.testSuite.create({
1663
+ data: { tenantId, projectId, name: suiteName }
1664
+ })
1665
+ );
1666
+ suiteId = suite.id;
1667
+ }
1668
+ await updateJobProgress(tenantId, jobId, 50, `Importing ${parsedCases.length} cases`);
1669
+ const { created, skipped } = await AppBrain.importTestCasesFromArtifact(
1670
+ tenantId,
1671
+ projectId,
1672
+ suiteId,
1673
+ parsedCases,
1674
+ { duplicateMode: "skip", artifactId }
1675
+ );
1676
+ let labeled = 0;
1677
+ if (autoLabel && created.length > 0) {
1678
+ await updateJobProgress(tenantId, jobId, 75, `AI labeling ${created.length} cases`);
1679
+ try {
1680
+ const labelInput = created.map((c) => ({
1681
+ id: c.id,
1682
+ title: c.title,
1683
+ description: c.description ?? "",
1684
+ stepsText: (Array.isArray(c.steps) ? c.steps.join(" ") : String(c.steps ?? "")).slice(0, 200)
1685
+ }));
1686
+ const labelResults = await labelTestCasesBatch(labelInput, tenantId);
1687
+ await Promise.all(
1688
+ labelResults.map(
1689
+ (lr) => withTenant(
1690
+ tenantId,
1691
+ (tx) => tx.testCase.update({
1692
+ where: { id: lr.id },
1693
+ data: {
1694
+ priority: lr.priority ?? void 0,
1695
+ kind: lr.kind ?? void 0,
1696
+ tags: lr.tags ?? void 0,
1697
+ status: lr.needsReview ? "PENDING_REVIEW" : void 0
1698
+ }
1699
+ })
1700
+ ).catch(() => {
1701
+ })
1702
+ // non-fatal per-case
1703
+ )
1704
+ );
1705
+ labeled = labelResults.length;
1706
+ } catch {
1814
1707
  }
1815
- if (created.length > 0) {
1816
- refreshTestCasesSkillFile(tenantId, projectId).catch(console.error);
1708
+ }
1709
+ const casesNeedingData = created.filter((c) => {
1710
+ const inline = c.testDataInline;
1711
+ return inline && typeof inline === "object" && Object.keys(inline).length > 0;
1712
+ });
1713
+ if (casesNeedingData.length > 0) {
1714
+ const cap = Math.min(casesNeedingData.length, 20);
1715
+ await updateJobProgress(tenantId, jobId, 90, `Generating test data for ${cap} cases`);
1716
+ const { generateTestData, refreshTestDataSkillFile: refreshTestDataSkillFile2 } = await import("@detiq/agents");
1717
+ for (let i = 0; i < cap; i++) {
1718
+ const c = casesNeedingData[i];
1719
+ await generateTestData({ tenantId, projectId, testCaseId: c.id, jobId }).catch((e) => {
1720
+ logger.warn({ err: e?.message, testCaseId: c.id }, "[import] test data generation failed");
1721
+ });
1817
1722
  }
1818
- await updateJobProgress(tenantId, jobId, 98, `Done — ${created.length} imported`);
1819
- // Write final result and mark job DONE
1820
- await withTenant(tenantId, (tx) => tx.platformJob.update({
1821
- where: { id: jobId },
1822
- data: {
1823
- status: 'DONE',
1824
- progress: 100,
1825
- finishedAt: new Date(),
1826
- result: {
1827
- imported: created.length,
1828
- skipped: skipped ?? 0,
1829
- labeled,
1830
- suiteId,
1831
- artifactId,
1832
- artifactName: filename,
1833
- },
1834
- },
1835
- }));
1836
- aggregateJobTokens(tenantId, jobId, projectId, jobStart).catch(console.error);
1723
+ refreshTestDataSkillFile2(tenantId, projectId).catch(console.error);
1724
+ }
1725
+ if (created.length > 0) {
1726
+ refreshTestCasesSkillFile(tenantId, projectId).catch(console.error);
1727
+ }
1728
+ await updateJobProgress(tenantId, jobId, 98, `Done \u2014 ${created.length} imported`);
1729
+ await withTenant(
1730
+ tenantId,
1731
+ (tx) => tx.platformJob.update({
1732
+ where: { id: jobId },
1733
+ data: {
1734
+ status: "DONE",
1735
+ progress: 100,
1736
+ finishedAt: /* @__PURE__ */ new Date(),
1737
+ result: {
1738
+ imported: created.length,
1739
+ skipped: skipped ?? 0,
1740
+ labeled,
1741
+ suiteId,
1742
+ artifactId,
1743
+ artifactName: filename
1744
+ }
1745
+ }
1746
+ })
1747
+ );
1748
+ aggregateJobTokens(tenantId, jobId, projectId, jobStart).catch(console.error);
1837
1749
  }
1838
- // Generate test cases for a single screen, enqueued by NEW_PAGES_PIPELINE.
1839
- // Uses the same DualContextOrchestrator path as the in-process GENERATE_TESTS route.
1840
1750
  async function processGenerateScreenTestsJob(jobId, config) {
1841
- const { tenantId, projectId, screenId, repoConnectionId, triggerAlignedScripts } = config;
1842
- const jobStartedAt = new Date();
1843
- try {
1844
- await withTenant(tenantId, (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: 'RUNNING', startedAt: jobStartedAt } }));
1845
- const { DualContextOrchestrator } = await import('@detiq/agents');
1846
- await updateJobProgress(tenantId, jobId, 10, 'Generating test cases for screen');
1847
- const crawlCfg = await AppBrain.getCrawlConfig(tenantId, projectId).catch(() => null);
1848
- const deviceProfiles = crawlCfg?.deviceProfiles ?? ['DESKTOP'];
1849
- for (const viewport of deviceProfiles) {
1850
- await DualContextOrchestrator.execute({ tenantId, projectId, targetScreenId: screenId, jobId, viewport });
1851
- }
1852
- await updateJobProgress(tenantId, jobId, 80, 'Test cases generated');
1853
- // Optionally kick framework-aligned script generation for this screen's test cases
1854
- if (triggerAlignedScripts && repoConnectionId) {
1855
- const conn = await withTenant(tenantId, (tx) => tx.gitHubRepoConnection.findFirst({ where: { id: repoConnectionId, tenantId } }));
1856
- if (conn?.styleProfile) {
1857
- const { generateScriptAligned, mapWithConcurrency, SCRIPT_GEN_CONCURRENCY } = await import('@detiq/agents');
1858
- const cases = await withTenant(tenantId, (tx) => tx.testCase.findMany({ where: { tenantId, projectId, screenId, status: 'APPROVED' }, select: { id: true }, take: 50 }));
1859
- await mapWithConcurrency(cases, SCRIPT_GEN_CONCURRENCY, (tc) => generateScriptAligned({
1860
- tenantId,
1861
- projectId,
1862
- testCaseId: tc.id,
1863
- styleProfile: conn.styleProfile,
1864
- pageObjectInventory: conn.pageObjectInventory ?? undefined,
1865
- }).catch(() => { }));
1866
- }
1867
- }
1868
- await withTenant(tenantId, (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: 'DONE', progress: 100, finishedAt: new Date() } }));
1869
- aggregateJobTokens(tenantId, jobId, projectId, jobStartedAt).catch(console.error);
1751
+ const { tenantId, projectId, screenId, repoConnectionId, triggerAlignedScripts } = config;
1752
+ const jobStartedAt = /* @__PURE__ */ new Date();
1753
+ try {
1754
+ await withTenant(
1755
+ tenantId,
1756
+ (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "RUNNING", startedAt: jobStartedAt } })
1757
+ );
1758
+ const { DualContextOrchestrator } = await import("@detiq/agents");
1759
+ await updateJobProgress(tenantId, jobId, 10, "Generating test cases for screen");
1760
+ const crawlCfg = await AppBrain.getCrawlConfig(tenantId, projectId).catch(() => null);
1761
+ const deviceProfiles = crawlCfg?.deviceProfiles ?? ["DESKTOP"];
1762
+ for (const viewport of deviceProfiles) {
1763
+ await DualContextOrchestrator.execute({ tenantId, projectId, targetScreenId: screenId, jobId, viewport });
1870
1764
  }
1871
- catch (err) {
1872
- const msg = err?.message ?? String(err);
1873
- logger.error({ jobId, screenId, err: msg }, 'processGenerateScreenTestsJob failed');
1874
- await withTenant(tenantId, (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', error: msg.slice(0, 500), finishedAt: new Date() } })).catch(() => { });
1765
+ await updateJobProgress(tenantId, jobId, 80, "Test cases generated");
1766
+ if (triggerAlignedScripts && repoConnectionId) {
1767
+ const conn = await withTenant(
1768
+ tenantId,
1769
+ (tx) => tx.gitHubRepoConnection.findFirst({ where: { id: repoConnectionId, tenantId } })
1770
+ );
1771
+ if (conn?.styleProfile) {
1772
+ const { generateScriptAligned, mapWithConcurrency, SCRIPT_GEN_CONCURRENCY } = await import("@detiq/agents");
1773
+ const cases = await withTenant(
1774
+ tenantId,
1775
+ (tx) => tx.testCase.findMany({ where: { tenantId, projectId, screenId, status: "APPROVED" }, select: { id: true }, take: 50 })
1776
+ );
1777
+ await mapWithConcurrency(
1778
+ cases,
1779
+ SCRIPT_GEN_CONCURRENCY,
1780
+ (tc) => generateScriptAligned({
1781
+ tenantId,
1782
+ projectId,
1783
+ testCaseId: tc.id,
1784
+ styleProfile: conn.styleProfile,
1785
+ pageObjectInventory: conn.pageObjectInventory ?? void 0
1786
+ }).catch(() => {
1787
+ })
1788
+ );
1789
+ }
1875
1790
  }
1791
+ await withTenant(
1792
+ tenantId,
1793
+ (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "DONE", progress: 100, finishedAt: /* @__PURE__ */ new Date() } })
1794
+ );
1795
+ aggregateJobTokens(tenantId, jobId, projectId, jobStartedAt).catch(console.error);
1796
+ } catch (err) {
1797
+ const msg = err?.message ?? String(err);
1798
+ logger.error({ jobId, screenId, err: msg }, "processGenerateScreenTestsJob failed");
1799
+ await withTenant(
1800
+ tenantId,
1801
+ (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", error: msg.slice(0, 500), finishedAt: /* @__PURE__ */ new Date() } })
1802
+ ).catch(() => {
1803
+ });
1804
+ }
1876
1805
  }
1877
- // Registers the type→processor dispatch table used by the durable worker (both the
1878
- // enqueueJob() fast-path kick and startJobWorkerLoop()'s poll). Declared at module
1879
- // bottom so it can reference every process*Job function by name; function
1880
- // declarations are hoisted, so this could technically live anywhere in the file,
1881
- // but keeping it here alongside the last definition keeps the mapping easy to audit
1882
- // against the actual functions above.
1883
1806
  jobDispatchTable = {
1884
- CRAWL: processJob,
1885
- DISCOVER: processDiscoverJob,
1886
- MONITOR: processMonitorJob,
1887
- NATIVE_CRAWL: processNativeCrawlJob,
1888
- API_TESTS: processApiTestsJob,
1889
- SECURITY_TESTS: processSecurityTestsJob,
1890
- INTEGRATION_TESTS: processIntegrationTestsJob,
1891
- FIGMA_IMPORT: processFigmaImportJob,
1892
- GENERATE_SKILL_FILES: processGenerateSkillFilesJob,
1893
- GENERATE_TESTS_FROM_REQS: processGenerateTestsFromReqsJob,
1894
- GENERATE_SCREEN_TESTS: processGenerateScreenTestsJob,
1895
- RUN_TEST_SUITE: processRunTestSuiteJob,
1896
- REPO_SCAN: processRepoScanJob,
1897
- IMPORT_TESTS: processImportTestsJob,
1898
- BATCH_GENERATE_ALIGNED: processBatchGenerateAlignedJob,
1899
- CREATE_PR: processCreatePRJob,
1900
- NEW_PAGES_PIPELINE: processNewPagesPipelineJob,
1807
+ CRAWL: processJob,
1808
+ DISCOVER: processDiscoverJob,
1809
+ MONITOR: processMonitorJob,
1810
+ NATIVE_CRAWL: processNativeCrawlJob,
1811
+ API_TESTS: processApiTestsJob,
1812
+ SECURITY_TESTS: processSecurityTestsJob,
1813
+ INTEGRATION_TESTS: processIntegrationTestsJob,
1814
+ FIGMA_IMPORT: processFigmaImportJob,
1815
+ GENERATE_SKILL_FILES: processGenerateSkillFilesJob,
1816
+ GENERATE_TESTS_FROM_REQS: processGenerateTestsFromReqsJob,
1817
+ GENERATE_SCREEN_TESTS: processGenerateScreenTestsJob,
1818
+ RUN_TEST_SUITE: processRunTestSuiteJob,
1819
+ REPO_SCAN: processRepoScanJob,
1820
+ IMPORT_TESTS: processImportTestsJob,
1821
+ BATCH_GENERATE_ALIGNED: processBatchGenerateAlignedJob,
1822
+ CREATE_PR: processCreatePRJob,
1823
+ NEW_PAGES_PIPELINE: processNewPagesPipelineJob
1901
1824
  };
1902
1825
  async function processRepoScanJob(jobId, config) {
1903
- const { repoConnectionId, tenantId, projectId } = config;
1904
- const jobStart = new Date();
1826
+ const { repoConnectionId, tenantId, projectId } = config;
1827
+ const jobStart = /* @__PURE__ */ new Date();
1828
+ try {
1829
+ await updateJobProgress(tenantId, jobId, 5, "Starting repo scan");
1830
+ const {
1831
+ fetchRepoTree,
1832
+ fetchFileContent,
1833
+ detectFramework,
1834
+ extractTestIds,
1835
+ extractStyleProfile,
1836
+ buildPageObjectInventory
1837
+ } = await import("@detiq/agents");
1838
+ const conn = await withTenant(
1839
+ tenantId,
1840
+ (tx) => tx.gitHubRepoConnection.findFirst({ where: { id: repoConnectionId, tenantId } })
1841
+ );
1842
+ if (!conn) throw new Error(`RepoConnection ${repoConnectionId} not found`);
1843
+ const { decrypt: decrypt2 } = await import("@detiq/core");
1844
+ const token = decrypt2(conn.tokenEnc);
1845
+ await withTenant(
1846
+ tenantId,
1847
+ (tx) => tx.gitHubRepoConnection.update({ where: { id: repoConnectionId }, data: { scanStatus: "SCANNING", scanError: null } })
1848
+ );
1849
+ await updateJobProgress(tenantId, jobId, 15, "Fetching repo tree");
1850
+ const treeResult = await fetchRepoTree(token, conn.repoFullName, "HEAD");
1851
+ const frameworks = detectFramework(treeResult.paths);
1852
+ const primary = frameworks[0];
1853
+ await updateJobProgress(tenantId, jobId, 30, `Detected framework: ${primary?.framework ?? "unknown"}`);
1854
+ const storedGlob = conn.testGlob || "";
1855
+ const testGlob = storedGlob && storedGlob !== "**/*.spec.ts" ? storedGlob : primary?.testGlob || storedGlob || "**/*.spec.ts";
1856
+ const framework = primary?.framework ?? "unknown";
1857
+ const language = primary?.language ?? "typescript";
1858
+ const { minimatch } = await import("minimatch").catch(() => ({ minimatch: null }));
1859
+ const isPythonTestFile = (p) => p.endsWith(".py") && !p.endsWith("__init__.py") && !p.endsWith("conftest.py") && (p.endsWith("_test.py") || /\/test_[^/]+\.py$/.test(p) || /^test_[^/]+\.py$/.test(p) || p.includes("/tests/") || p.includes("/test/"));
1860
+ const isJavaTestFile = (p) => p.endsWith(".java") && (/[A-Z]\w*Tests?\.java$/.test(p) || /\/tests?\//.test(p) && /[A-Z][a-z]/.test(p.split("/").pop() ?? ""));
1861
+ const testPaths = treeResult.paths.filter((p) => {
1862
+ if (language === "java") return isJavaTestFile(p);
1863
+ if (language === "python") return isPythonTestFile(p);
1864
+ if (minimatch) return minimatch(p, testGlob, { matchBase: true });
1865
+ return p.endsWith(".spec.ts") || p.endsWith(".spec.js") || p.endsWith(".test.ts") || p.endsWith(".test.js") || p.endsWith("_test.py") || p.includes("/test_");
1866
+ }).slice(0, 500);
1867
+ await updateJobProgress(tenantId, jobId, 40, `Found ${testPaths.length} test files`);
1868
+ const allExtracted = [];
1869
+ const styleSampleFiles = [];
1870
+ let processed = 0;
1871
+ await withTenant(tenantId, (tx) => tx.repoTestFile.deleteMany({ where: { repoConnectionId } }));
1872
+ for (const filePath of testPaths) {
1873
+ const content = await fetchFileContent(token, conn.repoFullName, filePath);
1874
+ if (!content) continue;
1875
+ const extracted = extractTestIds(content, filePath, framework);
1876
+ allExtracted.push(...extracted);
1877
+ await withTenant(
1878
+ tenantId,
1879
+ (tx) => tx.repoTestFile.create({
1880
+ data: {
1881
+ tenantId,
1882
+ projectId,
1883
+ repoConnectionId,
1884
+ filePath,
1885
+ framework,
1886
+ extractedIds: extracted.map((e) => ({ name: e.name, normalizedName: e.normalizedName, lineNumber: e.lineNumber })),
1887
+ rawSample: content.slice(0, 2e3),
1888
+ lastScannedAt: /* @__PURE__ */ new Date()
1889
+ }
1890
+ })
1891
+ );
1892
+ if (styleSampleFiles.length < 5) styleSampleFiles.push({ path: filePath, content: content.slice(0, 3e3) });
1893
+ processed++;
1894
+ if (processed % 10 === 0) {
1895
+ await updateJobProgress(tenantId, jobId, 40 + Math.floor(processed / testPaths.length * 40), `Processed ${processed}/${testPaths.length} files`);
1896
+ }
1897
+ }
1898
+ let styleProfile = conn.styleProfile ?? null;
1899
+ if (styleSampleFiles.length > 0) {
1900
+ try {
1901
+ const mainSourcePaths = language === "java" ? treeResult.paths.filter((p) => /^src\/main\/java\//.test(p)).slice(0, 50) : [];
1902
+ const testPathSet = new Set(testPaths);
1903
+ const sourceFilePaths = language === "typescript" || language === "javascript" ? treeResult.paths.filter((p) => (p.endsWith(".ts") || p.endsWith(".js")) && !testPathSet.has(p) && !p.endsWith(".d.ts") && !p.includes("node_modules/") && !p.includes("dist/") && !p.includes(".config.")).slice(0, 60) : [];
1904
+ const allPathsForProfile = [...testPaths, ...mainSourcePaths, ...sourceFilePaths].slice(0, 200);
1905
+ const freshProfile = await extractStyleProfile(styleSampleFiles, framework, language, tenantId, projectId, jobId, allPathsForProfile);
1906
+ if (freshProfile) styleProfile = freshProfile;
1907
+ } catch {
1908
+ }
1909
+ }
1910
+ let pageObjectInventory = conn.pageObjectInventory ?? [];
1905
1911
  try {
1906
- await updateJobProgress(tenantId, jobId, 5, 'Starting repo scan');
1907
- const { fetchRepoTree, fetchFileContent, detectFramework, extractTestIds, extractStyleProfile, buildPageObjectInventory, } = await import('@detiq/agents');
1908
- const conn = await withTenant(tenantId, (tx) => tx.gitHubRepoConnection.findFirst({ where: { id: repoConnectionId, tenantId } }));
1909
- if (!conn)
1910
- throw new Error(`RepoConnection ${repoConnectionId} not found`);
1911
- // Decrypt token
1912
- const { decrypt } = await import('@detiq/core');
1913
- const token = decrypt(conn.tokenEnc);
1914
- // Mark scanning
1915
- await withTenant(tenantId, (tx) => tx.gitHubRepoConnection.update({ where: { id: repoConnectionId }, data: { scanStatus: 'SCANNING', scanError: null } }));
1916
- await updateJobProgress(tenantId, jobId, 15, 'Fetching repo tree');
1917
- // Use HEAD so it always resolves to the actual default branch regardless of stored value
1918
- const treeResult = await fetchRepoTree(token, conn.repoFullName, 'HEAD');
1919
- const frameworks = detectFramework(treeResult.paths);
1920
- const primary = frameworks[0];
1921
- await updateJobProgress(tenantId, jobId, 30, `Detected framework: ${primary?.framework ?? 'unknown'}`);
1922
- // Prefer framework-detected glob when stored glob is the TS default (never explicitly set)
1923
- const storedGlob = conn.testGlob || '';
1924
- const testGlob = (storedGlob && storedGlob !== '**/*.spec.ts')
1925
- ? storedGlob
1926
- : (primary?.testGlob || storedGlob || '**/*.spec.ts');
1927
- const framework = primary?.framework ?? 'unknown';
1928
- const language = primary?.language ?? 'typescript';
1929
- // Filter test files
1930
- const { minimatch } = await import('minimatch').catch(() => ({ minimatch: null }));
1931
- const isPythonTestFile = (p) => p.endsWith('.py') && !p.endsWith('__init__.py') && !p.endsWith('conftest.py') &&
1932
- (p.endsWith('_test.py') || /\/test_[^/]+\.py$/.test(p) || /^test_[^/]+\.py$/.test(p) ||
1933
- p.includes('/tests/') || p.includes('/test/'));
1934
- // Java: testGlob matches all *.java — further filter to actual test classes only
1935
- // (excludes helpers, base classes, utilities that don't follow *Test.java/*Tests.java naming)
1936
- const isJavaTestFile = (p) => p.endsWith('.java') && (/[A-Z]\w*Tests?\.java$/.test(p) ||
1937
- (/\/tests?\//.test(p) && /[A-Z][a-z]/.test(p.split('/').pop() ?? '')));
1938
- const testPaths = treeResult.paths.filter((p) => {
1939
- if (language === 'java')
1940
- return isJavaTestFile(p);
1941
- if (language === 'python')
1942
- return isPythonTestFile(p);
1943
- if (minimatch)
1944
- return minimatch(p, testGlob, { matchBase: true });
1945
- return p.endsWith('.spec.ts') || p.endsWith('.spec.js') || p.endsWith('.test.ts') || p.endsWith('.test.js') || p.endsWith('_test.py') || p.includes('/test_');
1946
- }).slice(0, 500); // cap at 500 files
1947
- await updateJobProgress(tenantId, jobId, 40, `Found ${testPaths.length} test files`);
1948
- // Extract test IDs from each file
1949
- const allExtracted = [];
1950
- const styleSampleFiles = [];
1951
- let processed = 0;
1952
- // Delete old test files for this connection
1953
- await withTenant(tenantId, (tx) => tx.repoTestFile.deleteMany({ where: { repoConnectionId } }));
1954
- for (const filePath of testPaths) {
1955
- const content = await fetchFileContent(token, conn.repoFullName, filePath);
1956
- if (!content)
1957
- continue;
1958
- const extracted = extractTestIds(content, filePath, framework);
1959
- allExtracted.push(...extracted);
1960
- await withTenant(tenantId, (tx) => tx.repoTestFile.create({
1961
- data: {
1962
- tenantId,
1963
- projectId,
1964
- repoConnectionId,
1965
- filePath,
1966
- framework,
1967
- extractedIds: extracted.map((e) => ({ name: e.name, normalizedName: e.normalizedName, lineNumber: e.lineNumber })),
1968
- rawSample: content.slice(0, 2000),
1969
- lastScannedAt: new Date(),
1970
- },
1971
- }));
1972
- if (styleSampleFiles.length < 5)
1973
- styleSampleFiles.push({ path: filePath, content: content.slice(0, 3000) });
1974
- processed++;
1975
- if (processed % 10 === 0) {
1976
- await updateJobProgress(tenantId, jobId, 40 + Math.floor((processed / testPaths.length) * 40), `Processed ${processed}/${testPaths.length} files`);
1977
- }
1978
- }
1979
- // Extract style profile for aligned generation
1980
- // For Java, include main source paths so LLM can detect page object location in src/main/java/
1981
- // Fall back to existing styleProfile from DB if extraction fails or returns null.
1982
- let styleProfile = conn.styleProfile ?? null;
1983
- if (styleSampleFiles.length > 0) {
1984
- try {
1985
- const mainSourcePaths = language === 'java'
1986
- ? treeResult.paths.filter((p) => /^src\/main\/java\//.test(p)).slice(0, 50)
1987
- : [];
1988
- // For TS/JS, also include non-test source files so LLM can detect pageObjectLocation
1989
- // (e.g. src/advantage/pages/ in playwright-sample-project)
1990
- const testPathSet = new Set(testPaths);
1991
- const sourceFilePaths = (language === 'typescript' || language === 'javascript')
1992
- ? treeResult.paths
1993
- .filter((p) => (p.endsWith('.ts') || p.endsWith('.js')) &&
1994
- !testPathSet.has(p) &&
1995
- !p.endsWith('.d.ts') &&
1996
- !p.includes('node_modules/') &&
1997
- !p.includes('dist/') &&
1998
- !p.includes('.config.'))
1999
- .slice(0, 60)
2000
- : [];
2001
- const allPathsForProfile = [...testPaths, ...mainSourcePaths, ...sourceFilePaths].slice(0, 200);
2002
- const freshProfile = await extractStyleProfile(styleSampleFiles, framework, language, tenantId, projectId, jobId, allPathsForProfile);
2003
- if (freshProfile)
2004
- styleProfile = freshProfile;
2005
- }
2006
- catch { /* non-fatal */ }
2007
- }
2008
- // Extract page object inventory — finds POM/page classes in non-test files
2009
- // Fall back to existing inventory from DB if fetch fails.
2010
- let pageObjectInventory = conn.pageObjectInventory ?? [];
2011
- try {
2012
- const freshInventory = await buildPageObjectInventory(token, conn.repoFullName, treeResult.paths, framework);
2013
- if (freshInventory.length > 0) {
2014
- pageObjectInventory = freshInventory;
2015
- await updateJobProgress(tenantId, jobId, 82, `Found ${pageObjectInventory.length} page objects`);
2016
- }
2017
- }
2018
- catch { /* non-fatal */ }
2019
- // Run coverage mapping
2020
- await updateJobProgress(tenantId, jobId, 85, 'Running coverage mapping');
2021
- const { buildCoverageMap } = await import('@detiq/agents');
2022
- const testCases = await withTenant(tenantId, (tx) => tx.testCase.findMany({ where: { projectId, tenantId, status: { not: 'RETIRED' } }, select: { id: true, title: true, ref: true } }));
2023
- const mappings = await buildCoverageMap(testCases, allExtracted, tenantId);
2024
- // Upsert CoverageMapping records (skip manualOverride=true)
2025
- for (const m of mappings) {
2026
- const existing = await withTenant(tenantId, (tx) => tx.coverageMapping.findUnique({ where: { testCaseId: m.testCaseId } }));
2027
- if (existing?.manualOverride)
2028
- continue;
2029
- await withTenant(tenantId, (tx) => tx.coverageMapping.upsert({
2030
- where: { testCaseId: m.testCaseId },
2031
- create: { tenantId, projectId, testCaseId: m.testCaseId, repoConnectionId, status: m.status, matchedFilePath: m.matchedFilePath ?? null, scriptTestName: m.scriptTestName ?? null, lineNumber: m.lineNumber ?? null, confidence: m.confidence, matchMethod: m.matchMethod },
2032
- update: { status: m.status, matchedFilePath: m.matchedFilePath ?? null, scriptTestName: m.scriptTestName ?? null, lineNumber: m.lineNumber ?? null, confidence: m.confidence, matchMethod: m.matchMethod },
2033
- }));
2034
- }
2035
- const mapped = mappings.filter((m) => m.status === 'MAPPED').length;
2036
- const unmapped = mappings.filter((m) => m.status === 'UNMAPPED').length;
2037
- const ambiguous = mappings.filter((m) => m.status === 'AMBIGUOUS').length;
2038
- const coveragePercent = testCases.length > 0 ? (mapped / testCases.length) * 100 : 0;
2039
- // Write coverage snapshot
2040
- await withTenant(tenantId, (tx) => tx.coverageSnapshot.create({
2041
- data: { tenantId, projectId, totalCases: testCases.length, mappedCases: mapped, unmappedCases: unmapped, ambiguousCases: ambiguous, coveragePercent },
2042
- }));
2043
- // For Java, if styleProfile still missing testStructurePattern, infer two-tier from tree paths.
2044
- // This covers the case where both extractStyleProfile and buildPageObjectInventory failed.
2045
- const resolvedStyleProfile = (() => {
2046
- const sp = styleProfile ?? {};
2047
- if (framework === 'selenium-java' && !sp.testStructurePattern) {
2048
- const hasMavenMain = treeResult.paths.some((p) => /^src\/main\/java\//.test(p));
2049
- if (hasMavenMain)
2050
- return { ...sp, testStructurePattern: 'two-tier' };
2051
- }
2052
- return sp;
2053
- })();
2054
- // Update repo connection with results
2055
- await withTenant(tenantId, (tx) => tx.gitHubRepoConnection.update({
2056
- where: { id: repoConnectionId },
2057
- data: {
2058
- scanStatus: 'DONE',
2059
- detectedFramework: framework,
2060
- configFilePath: primary?.configPath ?? null,
2061
- testGlob: testGlob,
2062
- testFileCount: testPaths.length,
2063
- extractedIdCount: allExtracted.length,
2064
- styleProfile: resolvedStyleProfile,
2065
- pageObjectInventory: pageObjectInventory.length > 0 ? pageObjectInventory : undefined,
2066
- treeSnapshot: JSON.stringify(treeResult.paths).slice(0, 100000),
2067
- lastScannedAt: new Date(),
2068
- lastSyncAt: new Date(),
2069
- },
2070
- }));
2071
- try {
2072
- const { generateRepoSkillFiles } = await import('@detiq/agents');
2073
- await generateRepoSkillFiles(tenantId, projectId, {
2074
- repoFullName: conn.repoFullName,
2075
- framework,
2076
- language,
2077
- testGlob,
2078
- styleProfile: resolvedStyleProfile,
2079
- pageObjectInventory,
2080
- testPaths,
2081
- extractedTestCount: allExtracted.length,
2082
- });
2083
- }
2084
- catch (e) {
2085
- logger.warn({ jobId, repoConnectionId, err: e?.message }, '[repo-scan] repo skill files generation failed (non-fatal)');
2086
- }
2087
- await updateJobProgress(tenantId, jobId, 100, `Done: ${mapped} mapped, ${unmapped} unmapped`);
2088
- await withTenant(tenantId, (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: 'DONE', finishedAt: new Date(), result: { testFileCount: testPaths.length, extractedIdCount: allExtracted.length, mapped, unmapped, ambiguous, coveragePercent } } }));
2089
- aggregateJobTokens(tenantId, jobId, projectId, jobStart).catch(console.error);
1912
+ const freshInventory = await buildPageObjectInventory(token, conn.repoFullName, treeResult.paths, framework);
1913
+ if (freshInventory.length > 0) {
1914
+ pageObjectInventory = freshInventory;
1915
+ await updateJobProgress(tenantId, jobId, 82, `Found ${pageObjectInventory.length} page objects`);
1916
+ }
1917
+ } catch {
1918
+ }
1919
+ await updateJobProgress(tenantId, jobId, 85, "Running coverage mapping");
1920
+ const { buildCoverageMap } = await import("@detiq/agents");
1921
+ const testCases = await withTenant(
1922
+ tenantId,
1923
+ (tx) => tx.testCase.findMany({ where: { projectId, tenantId, status: { not: "RETIRED" } }, select: { id: true, title: true, ref: true } })
1924
+ );
1925
+ const mappings = await buildCoverageMap(testCases, allExtracted, tenantId);
1926
+ for (const m of mappings) {
1927
+ const existing = await withTenant(
1928
+ tenantId,
1929
+ (tx) => tx.coverageMapping.findUnique({ where: { testCaseId: m.testCaseId } })
1930
+ );
1931
+ if (existing?.manualOverride) continue;
1932
+ await withTenant(
1933
+ tenantId,
1934
+ (tx) => tx.coverageMapping.upsert({
1935
+ where: { testCaseId: m.testCaseId },
1936
+ create: { tenantId, projectId, testCaseId: m.testCaseId, repoConnectionId, status: m.status, matchedFilePath: m.matchedFilePath ?? null, scriptTestName: m.scriptTestName ?? null, lineNumber: m.lineNumber ?? null, confidence: m.confidence, matchMethod: m.matchMethod },
1937
+ update: { status: m.status, matchedFilePath: m.matchedFilePath ?? null, scriptTestName: m.scriptTestName ?? null, lineNumber: m.lineNumber ?? null, confidence: m.confidence, matchMethod: m.matchMethod }
1938
+ })
1939
+ );
2090
1940
  }
2091
- catch (err) {
2092
- const msg = String(err?.message ?? err);
2093
- logger.error({ jobId, repoConnectionId, err: msg }, '[repo-scan] failed');
2094
- await withTenant(tenantId, (tx) => tx.gitHubRepoConnection.update({ where: { id: repoConnectionId }, data: { scanStatus: 'FAILED', scanError: msg.slice(0, 500) } })).catch(() => { });
2095
- await withTenant(tenantId, (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', finishedAt: new Date(), error: msg.slice(0, 1000) } })).catch(() => { });
1941
+ const mapped = mappings.filter((m) => m.status === "MAPPED").length;
1942
+ const unmapped = mappings.filter((m) => m.status === "UNMAPPED").length;
1943
+ const ambiguous = mappings.filter((m) => m.status === "AMBIGUOUS").length;
1944
+ const coveragePercent = testCases.length > 0 ? mapped / testCases.length * 100 : 0;
1945
+ await withTenant(
1946
+ tenantId,
1947
+ (tx) => tx.coverageSnapshot.create({
1948
+ data: { tenantId, projectId, totalCases: testCases.length, mappedCases: mapped, unmappedCases: unmapped, ambiguousCases: ambiguous, coveragePercent }
1949
+ })
1950
+ );
1951
+ const resolvedStyleProfile = (() => {
1952
+ const sp = styleProfile ?? {};
1953
+ if (framework === "selenium-java" && !sp.testStructurePattern) {
1954
+ const hasMavenMain = treeResult.paths.some((p) => /^src\/main\/java\//.test(p));
1955
+ if (hasMavenMain) return { ...sp, testStructurePattern: "two-tier" };
1956
+ }
1957
+ return sp;
1958
+ })();
1959
+ await withTenant(
1960
+ tenantId,
1961
+ (tx) => tx.gitHubRepoConnection.update({
1962
+ where: { id: repoConnectionId },
1963
+ data: {
1964
+ scanStatus: "DONE",
1965
+ detectedFramework: framework,
1966
+ configFilePath: primary?.configPath ?? null,
1967
+ testGlob,
1968
+ testFileCount: testPaths.length,
1969
+ extractedIdCount: allExtracted.length,
1970
+ styleProfile: resolvedStyleProfile,
1971
+ pageObjectInventory: pageObjectInventory.length > 0 ? pageObjectInventory : void 0,
1972
+ treeSnapshot: JSON.stringify(treeResult.paths).slice(0, 1e5),
1973
+ lastScannedAt: /* @__PURE__ */ new Date(),
1974
+ lastSyncAt: /* @__PURE__ */ new Date()
1975
+ }
1976
+ })
1977
+ );
1978
+ try {
1979
+ const { generateRepoSkillFiles } = await import("@detiq/agents");
1980
+ await generateRepoSkillFiles(tenantId, projectId, {
1981
+ repoFullName: conn.repoFullName,
1982
+ framework,
1983
+ language,
1984
+ testGlob,
1985
+ styleProfile: resolvedStyleProfile,
1986
+ pageObjectInventory,
1987
+ testPaths,
1988
+ extractedTestCount: allExtracted.length
1989
+ });
1990
+ } catch (e) {
1991
+ logger.warn({ jobId, repoConnectionId, err: e?.message }, "[repo-scan] repo skill files generation failed (non-fatal)");
2096
1992
  }
1993
+ await updateJobProgress(tenantId, jobId, 100, `Done: ${mapped} mapped, ${unmapped} unmapped`);
1994
+ await withTenant(
1995
+ tenantId,
1996
+ (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "DONE", finishedAt: /* @__PURE__ */ new Date(), result: { testFileCount: testPaths.length, extractedIdCount: allExtracted.length, mapped, unmapped, ambiguous, coveragePercent } } })
1997
+ );
1998
+ aggregateJobTokens(tenantId, jobId, projectId, jobStart).catch(console.error);
1999
+ } catch (err) {
2000
+ const msg = String(err?.message ?? err);
2001
+ logger.error({ jobId, repoConnectionId, err: msg }, "[repo-scan] failed");
2002
+ await withTenant(
2003
+ tenantId,
2004
+ (tx) => tx.gitHubRepoConnection.update({ where: { id: repoConnectionId }, data: { scanStatus: "FAILED", scanError: msg.slice(0, 500) } })
2005
+ ).catch(() => {
2006
+ });
2007
+ await withTenant(
2008
+ tenantId,
2009
+ (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", finishedAt: /* @__PURE__ */ new Date(), error: msg.slice(0, 1e3) } })
2010
+ ).catch(() => {
2011
+ });
2012
+ }
2097
2013
  }
2098
- // ── BATCH_GENERATE_ALIGNED — generate framework-aligned scripts for UNMAPPED cases ──
2099
2014
  async function processBatchGenerateAlignedJob(jobId, config) {
2100
- const { tenantId, projectId, repoConnectionId, scope = 'UNMAPPED', testCaseIds, force = false } = config;
2101
- try {
2102
- await updateJobProgress(tenantId, jobId, 5, 'Loading repo connection');
2103
- const conn = await withTenant(tenantId, (tx) => tx.gitHubRepoConnection.findFirst({ where: { id: repoConnectionId, tenantId } }));
2104
- if (!conn)
2105
- throw new Error(`RepoConnection ${repoConnectionId} not found`);
2106
- const { decrypt } = await import('@detiq/core');
2107
- const { generateScriptAligned, deriveRepoTargetPath, validateGeneratedScript, mapWithConcurrency, SCRIPT_GEN_CONCURRENCY, } = await import('@detiq/agents');
2108
- const styleProfile = conn.styleProfile;
2109
- const pageObjectInventory = conn.pageObjectInventory ?? [];
2110
- const framework = conn.detectedFramework ?? 'playwright-ts';
2111
- const language = styleProfile?.language ?? 'typescript';
2112
- // Resolve which test cases to process
2113
- let caseIds;
2114
- if (testCaseIds && testCaseIds.length > 0) {
2115
- caseIds = testCaseIds;
2116
- }
2117
- else if (scope === 'UNMAPPED') {
2118
- const mappings = await withTenant(tenantId, (tx) => tx.coverageMapping.findMany({
2119
- where: { tenantId, projectId, repoConnectionId, status: 'UNMAPPED' },
2120
- select: { testCaseId: true },
2121
- }));
2122
- caseIds = mappings.map((m) => m.testCaseId);
2123
- }
2124
- else {
2125
- const cases = await withTenant(tenantId, (tx) => tx.testCase.findMany({
2126
- where: { tenantId, projectId, status: { not: 'RETIRED' } },
2127
- select: { id: true },
2128
- }));
2129
- caseIds = cases.map((c) => c.id);
2015
+ const { tenantId, projectId, repoConnectionId, scope = "UNMAPPED", testCaseIds, force = false } = config;
2016
+ try {
2017
+ await updateJobProgress(tenantId, jobId, 5, "Loading repo connection");
2018
+ const conn = await withTenant(
2019
+ tenantId,
2020
+ (tx) => tx.gitHubRepoConnection.findFirst({ where: { id: repoConnectionId, tenantId } })
2021
+ );
2022
+ if (!conn) throw new Error(`RepoConnection ${repoConnectionId} not found`);
2023
+ const { decrypt: decrypt2 } = await import("@detiq/core");
2024
+ const {
2025
+ generateScriptAligned,
2026
+ deriveRepoTargetPath,
2027
+ validateGeneratedScript,
2028
+ mapWithConcurrency,
2029
+ SCRIPT_GEN_CONCURRENCY
2030
+ } = await import("@detiq/agents");
2031
+ const styleProfile = conn.styleProfile;
2032
+ const pageObjectInventory = conn.pageObjectInventory ?? [];
2033
+ const framework = conn.detectedFramework ?? "playwright-ts";
2034
+ const language = styleProfile?.language ?? "typescript";
2035
+ let caseIds;
2036
+ if (testCaseIds && testCaseIds.length > 0) {
2037
+ caseIds = testCaseIds;
2038
+ } else if (scope === "UNMAPPED") {
2039
+ const mappings = await withTenant(
2040
+ tenantId,
2041
+ (tx) => tx.coverageMapping.findMany({
2042
+ where: { tenantId, projectId, repoConnectionId, status: "UNMAPPED" },
2043
+ select: { testCaseId: true }
2044
+ })
2045
+ );
2046
+ caseIds = mappings.map((m) => m.testCaseId);
2047
+ } else {
2048
+ const cases = await withTenant(
2049
+ tenantId,
2050
+ (tx) => tx.testCase.findMany({
2051
+ where: { tenantId, projectId, status: { not: "RETIRED" } },
2052
+ select: { id: true }
2053
+ })
2054
+ );
2055
+ caseIds = cases.map((c) => c.id);
2056
+ }
2057
+ await updateJobProgress(tenantId, jobId, 15, `Generating scripts for ${caseIds.length} test cases`);
2058
+ let done = 0;
2059
+ const errors = [];
2060
+ await mapWithConcurrency(caseIds, SCRIPT_GEN_CONCURRENCY, async (tcId) => {
2061
+ try {
2062
+ if (!force) {
2063
+ const existing = await withTenant(
2064
+ tenantId,
2065
+ (tx) => tx.testScript.findFirst({ where: { testCaseId: tcId } })
2066
+ );
2067
+ if (existing) {
2068
+ done++;
2069
+ return;
2070
+ }
2130
2071
  }
2131
- await updateJobProgress(tenantId, jobId, 15, `Generating scripts for ${caseIds.length} test cases`);
2132
- let done = 0;
2133
- const errors = [];
2134
- // One LLM call per case — bounded worker pool instead of a strictly serial loop,
2135
- // which made large batches take an hour or more.
2136
- await mapWithConcurrency(caseIds, SCRIPT_GEN_CONCURRENCY, async (tcId) => {
2137
- try {
2138
- // Skip if script already exists and not forcing
2139
- if (!force) {
2140
- const existing = await withTenant(tenantId, (tx) => tx.testScript.findFirst({ where: { testCaseId: tcId } }));
2141
- if (existing) {
2142
- done++;
2143
- return;
2144
- }
2145
- }
2146
- // Get screen URL and ref for target path derivation
2147
- const tc = await withTenant(tenantId, (tx) => tx.testCase.findUnique({
2148
- where: { id: tcId },
2149
- select: { id: true, ref: true, title: true, screen: { select: { url: true } } },
2150
- }));
2151
- const screenUrl = tc?.screen?.url ?? '';
2152
- const repoTargetPath = styleProfile
2153
- ? deriveRepoTargetPath(screenUrl, styleProfile, framework)
2154
- : deriveFallbackTargetPath(tc?.ref ?? tc?.title ?? tcId, framework);
2155
- const result = await generateScriptAligned({
2156
- tenantId, projectId, testCaseId: tcId,
2157
- styleProfile,
2158
- pageObjectInventory,
2159
- repoTargetPath,
2160
- save: true,
2161
- });
2162
- // Validate the generated script
2163
- if (result.code) {
2164
- const validation = await validateGeneratedScript(result.code, framework, language);
2165
- await withTenant(tenantId, (tx) => tx.testScript.updateMany({
2166
- where: { testCaseId: tcId },
2167
- data: {
2168
- validationStatus: validation.ok ? 'PASS' : 'FAIL',
2169
- validationErrors: validation.ok ? null : validation.errors.slice(0, 2000),
2170
- },
2171
- }));
2172
- }
2173
- done++;
2174
- }
2175
- catch (e) {
2176
- errors.push(`${tcId}: ${String(e.message).slice(0, 100)}`);
2177
- }
2178
- if (done % 5 === 0) {
2179
- await updateJobProgress(tenantId, jobId, 15 + Math.floor((done / caseIds.length) * 80), `Generated ${done}/${caseIds.length}`);
2180
- }
2072
+ const tc = await withTenant(
2073
+ tenantId,
2074
+ (tx) => tx.testCase.findUnique({
2075
+ where: { id: tcId },
2076
+ select: { id: true, ref: true, title: true, screen: { select: { url: true } } }
2077
+ })
2078
+ );
2079
+ const screenUrl = tc?.screen?.url ?? "";
2080
+ const repoTargetPath = styleProfile ? deriveRepoTargetPath(screenUrl, styleProfile, framework) : deriveFallbackTargetPath(tc?.ref ?? tc?.title ?? tcId, framework);
2081
+ const result = await generateScriptAligned({
2082
+ tenantId,
2083
+ projectId,
2084
+ testCaseId: tcId,
2085
+ styleProfile,
2086
+ pageObjectInventory,
2087
+ repoTargetPath,
2088
+ save: true
2181
2089
  });
2182
- refreshTestScriptSkillFile(tenantId, projectId).catch(console.error);
2183
- await withTenant(tenantId, (tx) => tx.platformJob.update({
2184
- where: { id: jobId },
2185
- data: { status: 'DONE', finishedAt: new Date(), result: { done, errors: errors.slice(0, 20), total: caseIds.length } },
2186
- }));
2187
- }
2188
- catch (err) {
2189
- const msg = String(err?.message ?? err);
2190
- logger.error({ jobId, err: msg }, '[batch-generate-aligned] failed');
2191
- await withTenant(tenantId, (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', finishedAt: new Date(), error: msg.slice(0, 1000) } })).catch(() => { });
2192
- }
2090
+ if (result.code) {
2091
+ const validation = await validateGeneratedScript(result.code, framework, language);
2092
+ await withTenant(
2093
+ tenantId,
2094
+ (tx) => tx.testScript.updateMany({
2095
+ where: { testCaseId: tcId },
2096
+ data: {
2097
+ validationStatus: validation.ok ? "PASS" : "FAIL",
2098
+ validationErrors: validation.ok ? null : validation.errors.slice(0, 2e3)
2099
+ }
2100
+ })
2101
+ );
2102
+ }
2103
+ done++;
2104
+ } catch (e) {
2105
+ errors.push(`${tcId}: ${String(e.message).slice(0, 100)}`);
2106
+ }
2107
+ if (done % 5 === 0) {
2108
+ await updateJobProgress(tenantId, jobId, 15 + Math.floor(done / caseIds.length * 80), `Generated ${done}/${caseIds.length}`);
2109
+ }
2110
+ });
2111
+ refreshTestScriptSkillFile(tenantId, projectId).catch(console.error);
2112
+ await withTenant(
2113
+ tenantId,
2114
+ (tx) => tx.platformJob.update({
2115
+ where: { id: jobId },
2116
+ data: { status: "DONE", finishedAt: /* @__PURE__ */ new Date(), result: { done, errors: errors.slice(0, 20), total: caseIds.length } }
2117
+ })
2118
+ );
2119
+ } catch (err) {
2120
+ const msg = String(err?.message ?? err);
2121
+ logger.error({ jobId, err: msg }, "[batch-generate-aligned] failed");
2122
+ await withTenant(
2123
+ tenantId,
2124
+ (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", finishedAt: /* @__PURE__ */ new Date(), error: msg.slice(0, 1e3) } })
2125
+ ).catch(() => {
2126
+ });
2127
+ }
2193
2128
  }
2194
- // ── CREATE_PR — push generated scripts to GitHub as a PR ─────────────────────
2195
2129
  async function processCreatePRJob(jobId, config) {
2196
- const { tenantId, projectId, repoConnectionId, testScriptIds, crawlJobId } = config;
2197
- try {
2198
- await updateJobProgress(tenantId, jobId, 5, 'Loading connection');
2199
- const conn = await withTenant(tenantId, (tx) => tx.gitHubRepoConnection.findFirst({ where: { id: repoConnectionId, tenantId } }));
2200
- if (!conn)
2201
- throw new Error(`RepoConnection ${repoConnectionId} not found`);
2202
- const { decrypt } = await import('@detiq/core');
2203
- const token = decrypt(conn.tokenEnc);
2204
- const { pushScriptsAsPR } = await import('@detiq/agents');
2205
- const framework = conn.detectedFramework ?? 'playwright-ts';
2206
- // Collect scripts + their target paths
2207
- let scripts;
2208
- if (testScriptIds && testScriptIds.length > 0) {
2209
- scripts = await withTenant(tenantId, (tx) => tx.testScript.findMany({
2210
- where: { id: { in: testScriptIds }, tenantId },
2211
- include: { testCase: { select: { title: true, ref: true, repoTargetPath: true } } },
2212
- }));
2213
- }
2214
- else {
2215
- // All scripts that haven't been pushed yet
2216
- scripts = await withTenant(tenantId, (tx) => tx.testScript.findMany({
2217
- where: { tenantId, projectId, pushedToPR: false, validationStatus: { in: ['PASS', 'PENDING'] } },
2218
- include: { testCase: { select: { title: true, ref: true, repoTargetPath: true } } },
2219
- }));
2220
- }
2221
- if (scripts.length === 0)
2222
- throw new Error('No generated scripts found to push.');
2223
- const files = scripts.map((s) => {
2224
- const path = s.testCase?.repoTargetPath
2225
- ?? deriveFallbackTargetPath(s.testCase?.ref ?? s.testCase?.title ?? s.id, framework);
2226
- return { path, content: s.code };
2227
- });
2228
- if (files.length === 0)
2229
- throw new Error('No scripts to push.');
2230
- await updateJobProgress(tenantId, jobId, 20, `Pushing ${files.length} scripts to GitHub`);
2231
- const now = new Date().toISOString().slice(0, 10);
2232
- const branchName = `zeta/new-coverage-${now}-${jobId.slice(0, 8)}`;
2233
- const prBody = buildPRBody(scripts, projectId);
2234
- const prResult = await pushScriptsAsPR({
2235
- token,
2236
- repo: conn.repoFullName,
2237
- baseBranch: conn.defaultBranch,
2238
- branchName,
2239
- files,
2240
- prTitle: `ZeTa: Add test coverage for ${files.length} new scenario${files.length === 1 ? '' : 's'}`,
2241
- prBody,
2242
- draft: false,
2243
- });
2244
- // Record the PR
2245
- const pr = await withTenant(tenantId, (tx) => tx.gitHubPR.create({
2246
- data: {
2247
- tenantId, projectId,
2248
- repoConnectionId,
2249
- prNumber: prResult.prNumber,
2250
- prUrl: prResult.prUrl,
2251
- branch: prResult.branch,
2252
- headSha: prResult.headSha,
2253
- status: 'OPEN',
2254
- crawlJobId: crawlJobId ?? null,
2255
- scriptCount: files.length,
2256
- prBody: prBody.slice(0, 10000),
2257
- },
2258
- }));
2259
- // Mark scripts as pushed
2260
- const scriptIds = scripts.map((s) => s.id);
2261
- await withTenant(tenantId, (tx) => tx.testScript.updateMany({
2262
- where: { id: { in: scriptIds } },
2263
- data: { pushedToPR: true, pushedPRId: pr.id },
2264
- }));
2265
- await notifyAdmins(tenantId, {
2266
- type: 'GITHUB_PR_CREATED',
2267
- title: 'ZeTa created a GitHub PR',
2268
- body: `${files.length} new test script${files.length === 1 ? '' : 's'} pushed. PR #${prResult.prNumber}: ${prResult.prUrl}`,
2269
- metadata: { prUrl: prResult.prUrl, prNumber: prResult.prNumber, scriptCount: files.length },
2270
- });
2271
- await updateJobProgress(tenantId, jobId, 100, `PR #${prResult.prNumber} created`);
2272
- await withTenant(tenantId, (tx) => tx.platformJob.update({
2273
- where: { id: jobId },
2274
- data: { status: 'DONE', finishedAt: new Date(), result: { prUrl: prResult.prUrl, prNumber: prResult.prNumber, scriptCount: files.length } },
2275
- }));
2276
- }
2277
- catch (err) {
2278
- const msg = String(err?.message ?? err);
2279
- logger.error({ jobId, err: msg }, '[create-pr] failed');
2280
- await withTenant(tenantId, (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', finishedAt: new Date(), error: msg.slice(0, 1000) } })).catch(() => { });
2130
+ const { tenantId, projectId, repoConnectionId, testScriptIds, crawlJobId } = config;
2131
+ try {
2132
+ await updateJobProgress(tenantId, jobId, 5, "Loading connection");
2133
+ const conn = await withTenant(
2134
+ tenantId,
2135
+ (tx) => tx.gitHubRepoConnection.findFirst({ where: { id: repoConnectionId, tenantId } })
2136
+ );
2137
+ if (!conn) throw new Error(`RepoConnection ${repoConnectionId} not found`);
2138
+ const { decrypt: decrypt2 } = await import("@detiq/core");
2139
+ const token = decrypt2(conn.tokenEnc);
2140
+ const { pushScriptsAsPR } = await import("@detiq/agents");
2141
+ const framework = conn.detectedFramework ?? "playwright-ts";
2142
+ let scripts;
2143
+ if (testScriptIds && testScriptIds.length > 0) {
2144
+ scripts = await withTenant(
2145
+ tenantId,
2146
+ (tx) => tx.testScript.findMany({
2147
+ where: { id: { in: testScriptIds }, tenantId },
2148
+ include: { testCase: { select: { title: true, ref: true, repoTargetPath: true } } }
2149
+ })
2150
+ );
2151
+ } else {
2152
+ scripts = await withTenant(
2153
+ tenantId,
2154
+ (tx) => tx.testScript.findMany({
2155
+ where: { tenantId, projectId, pushedToPR: false, validationStatus: { in: ["PASS", "PENDING"] } },
2156
+ include: { testCase: { select: { title: true, ref: true, repoTargetPath: true } } }
2157
+ })
2158
+ );
2281
2159
  }
2160
+ if (scripts.length === 0) throw new Error("No generated scripts found to push.");
2161
+ const files = scripts.map((s) => {
2162
+ const path2 = s.testCase?.repoTargetPath ?? deriveFallbackTargetPath(s.testCase?.ref ?? s.testCase?.title ?? s.id, framework);
2163
+ return { path: path2, content: s.code };
2164
+ });
2165
+ if (files.length === 0) throw new Error("No scripts to push.");
2166
+ await updateJobProgress(tenantId, jobId, 20, `Pushing ${files.length} scripts to GitHub`);
2167
+ const now = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
2168
+ const branchName = `zeta/new-coverage-${now}-${jobId.slice(0, 8)}`;
2169
+ const prBody = buildPRBody(scripts, projectId);
2170
+ const prResult = await pushScriptsAsPR({
2171
+ token,
2172
+ repo: conn.repoFullName,
2173
+ baseBranch: conn.defaultBranch,
2174
+ branchName,
2175
+ files,
2176
+ prTitle: `ZeTa: Add test coverage for ${files.length} new scenario${files.length === 1 ? "" : "s"}`,
2177
+ prBody,
2178
+ draft: false
2179
+ });
2180
+ const pr = await withTenant(
2181
+ tenantId,
2182
+ (tx) => tx.gitHubPR.create({
2183
+ data: {
2184
+ tenantId,
2185
+ projectId,
2186
+ repoConnectionId,
2187
+ prNumber: prResult.prNumber,
2188
+ prUrl: prResult.prUrl,
2189
+ branch: prResult.branch,
2190
+ headSha: prResult.headSha,
2191
+ status: "OPEN",
2192
+ crawlJobId: crawlJobId ?? null,
2193
+ scriptCount: files.length,
2194
+ prBody: prBody.slice(0, 1e4)
2195
+ }
2196
+ })
2197
+ );
2198
+ const scriptIds = scripts.map((s) => s.id);
2199
+ await withTenant(
2200
+ tenantId,
2201
+ (tx) => tx.testScript.updateMany({
2202
+ where: { id: { in: scriptIds } },
2203
+ data: { pushedToPR: true, pushedPRId: pr.id }
2204
+ })
2205
+ );
2206
+ await notifyAdmins(tenantId, {
2207
+ type: "GITHUB_PR_CREATED",
2208
+ title: "ZeTa created a GitHub PR",
2209
+ body: `${files.length} new test script${files.length === 1 ? "" : "s"} pushed. PR #${prResult.prNumber}: ${prResult.prUrl}`,
2210
+ metadata: { prUrl: prResult.prUrl, prNumber: prResult.prNumber, scriptCount: files.length }
2211
+ });
2212
+ await updateJobProgress(tenantId, jobId, 100, `PR #${prResult.prNumber} created`);
2213
+ await withTenant(
2214
+ tenantId,
2215
+ (tx) => tx.platformJob.update({
2216
+ where: { id: jobId },
2217
+ data: { status: "DONE", finishedAt: /* @__PURE__ */ new Date(), result: { prUrl: prResult.prUrl, prNumber: prResult.prNumber, scriptCount: files.length } }
2218
+ })
2219
+ );
2220
+ } catch (err) {
2221
+ const msg = String(err?.message ?? err);
2222
+ logger.error({ jobId, err: msg }, "[create-pr] failed");
2223
+ await withTenant(
2224
+ tenantId,
2225
+ (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", finishedAt: /* @__PURE__ */ new Date(), error: msg.slice(0, 1e3) } })
2226
+ ).catch(() => {
2227
+ });
2228
+ }
2282
2229
  }
2283
2230
  function buildPRBody(scripts, projectId) {
2284
- const lines = [
2285
- '## ZeTa — Automated Test Coverage',
2286
- '',
2287
- `Generated by ZeTa Quality Intelligence Platform for project \`${projectId}\`.`,
2288
- '',
2289
- '### Scripts Added',
2290
- '',
2291
- '| Test Case | Target Path |',
2292
- '|---|---|',
2293
- ];
2294
- for (const s of scripts.slice(0, 30)) {
2295
- const title = s.testCase?.title ?? s.id;
2296
- const path = s.testCase?.repoTargetPath ?? '(unknown)';
2297
- lines.push(`| ${title.replace(/\|/g, '\\|').slice(0, 60)} | \`${path}\` |`);
2298
- }
2299
- if (scripts.length > 30)
2300
- lines.push(`| ... and ${scripts.length - 30} more | |`);
2301
- lines.push('', '---', '_Review each test carefully before merging. ZeTa generates scripts aligned to your existing framework conventions._');
2302
- return lines.join('\n');
2231
+ const lines = [
2232
+ "## ZeTa \u2014 Automated Test Coverage",
2233
+ "",
2234
+ `Generated by ZeTa Quality Intelligence Platform for project \`${projectId}\`.`,
2235
+ "",
2236
+ "### Scripts Added",
2237
+ "",
2238
+ "| Test Case | Target Path |",
2239
+ "|---|---|"
2240
+ ];
2241
+ for (const s of scripts.slice(0, 30)) {
2242
+ const title = s.testCase?.title ?? s.id;
2243
+ const path2 = s.testCase?.repoTargetPath ?? "(unknown)";
2244
+ lines.push(`| ${title.replace(/\|/g, "\\|").slice(0, 60)} | \`${path2}\` |`);
2245
+ }
2246
+ if (scripts.length > 30) lines.push(`| ... and ${scripts.length - 30} more | |`);
2247
+ lines.push("", "---", "_Review each test carefully before merging. ZeTa generates scripts aligned to your existing framework conventions._");
2248
+ return lines.join("\n");
2303
2249
  }
2304
- // ── NEW_PAGES_PIPELINE — post-crawl: detect new screens, generate tests + scripts ──
2305
2250
  async function processNewPagesPipelineJob(jobId, config) {
2306
- const { tenantId, projectId, crawlJobId, repoConnectionId, autoPR = false } = config;
2307
- try {
2308
- await updateJobProgress(tenantId, jobId, 5, 'Detecting new screens');
2309
- // Find repo connection (use provided or find first active)
2310
- const conn = await withTenant(tenantId, (tx) => tx.gitHubRepoConnection.findFirst({
2311
- where: repoConnectionId
2312
- ? { id: repoConnectionId, tenantId }
2313
- : { tenantId, projectId, status: 'ACTIVE' },
2314
- }));
2315
- // Get all screens last seen after the lastSyncAt of the repo connection
2316
- const sinceDate = conn?.lastSyncAt ?? new Date(0);
2317
- const newScreens = await withTenant(tenantId, (tx) => tx.screen.findMany({
2318
- where: {
2319
- tenantId,
2320
- projectId,
2321
- lastSeenAt: { gt: sinceDate },
2322
- status: { not: 'DELETED' },
2323
- },
2324
- select: { id: true, url: true, name: true },
2325
- take: 100,
2326
- }));
2327
- if (newScreens.length === 0) {
2328
- await updateJobProgress(tenantId, jobId, 100, 'No new screens to process');
2329
- await withTenant(tenantId, (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: 'DONE', finishedAt: new Date(), result: { newScreens: 0 } } }));
2330
- return;
2331
- }
2332
- await updateJobProgress(tenantId, jobId, 15, `Found ${newScreens.length} new screens`);
2333
- // For each new screen, enqueue GENERATE_TESTS if no test cases exist
2334
- const generationJobs = [];
2335
- for (const screen of newScreens) {
2336
- const caseCount = await withTenant(tenantId, (tx) => tx.testCase.count({ where: { tenantId, screenId: screen.id } }));
2337
- if (caseCount === 0) {
2338
- const jid = await enqueueJob('GENERATE_SCREEN_TESTS', {
2339
- tenantId, projectId, screenId: screen.id,
2340
- triggerAlignedScripts: conn != null,
2341
- repoConnectionId: conn?.id ?? null,
2342
- });
2343
- generationJobs.push(jid);
2344
- }
2345
- }
2346
- await updateJobProgress(tenantId, jobId, 60, `Queued ${generationJobs.length} test generation jobs`);
2347
- // If repo connected and autoPR enabled, schedule BATCH_GENERATE_ALIGNED after generation
2348
- if (conn && autoPR) {
2349
- await enqueueJob('BATCH_GENERATE_ALIGNED', {
2350
- tenantId, projectId,
2351
- repoConnectionId: conn.id,
2352
- scope: 'UNMAPPED',
2353
- });
2354
- }
2355
- await withTenant(tenantId, (tx) => tx.platformJob.update({
2356
- where: { id: jobId },
2357
- data: { status: 'DONE', finishedAt: new Date(), result: { newScreens: newScreens.length, generationJobs: generationJobs.length } },
2358
- }));
2251
+ const { tenantId, projectId, crawlJobId, repoConnectionId, autoPR = false } = config;
2252
+ try {
2253
+ await updateJobProgress(tenantId, jobId, 5, "Detecting new screens");
2254
+ const conn = await withTenant(
2255
+ tenantId,
2256
+ (tx) => tx.gitHubRepoConnection.findFirst({
2257
+ where: repoConnectionId ? { id: repoConnectionId, tenantId } : { tenantId, projectId, status: "ACTIVE" }
2258
+ })
2259
+ );
2260
+ const sinceDate = conn?.lastSyncAt ?? /* @__PURE__ */ new Date(0);
2261
+ const newScreens = await withTenant(
2262
+ tenantId,
2263
+ (tx) => tx.screen.findMany({
2264
+ where: {
2265
+ tenantId,
2266
+ projectId,
2267
+ lastSeenAt: { gt: sinceDate },
2268
+ status: { not: "DELETED" }
2269
+ },
2270
+ select: { id: true, url: true, name: true },
2271
+ take: 100
2272
+ })
2273
+ );
2274
+ if (newScreens.length === 0) {
2275
+ await updateJobProgress(tenantId, jobId, 100, "No new screens to process");
2276
+ await withTenant(
2277
+ tenantId,
2278
+ (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "DONE", finishedAt: /* @__PURE__ */ new Date(), result: { newScreens: 0 } } })
2279
+ );
2280
+ return;
2281
+ }
2282
+ await updateJobProgress(tenantId, jobId, 15, `Found ${newScreens.length} new screens`);
2283
+ const generationJobs = [];
2284
+ for (const screen of newScreens) {
2285
+ const caseCount = await withTenant(
2286
+ tenantId,
2287
+ (tx) => tx.testCase.count({ where: { tenantId, screenId: screen.id } })
2288
+ );
2289
+ if (caseCount === 0) {
2290
+ const jid = await enqueueJob("GENERATE_SCREEN_TESTS", {
2291
+ tenantId,
2292
+ projectId,
2293
+ screenId: screen.id,
2294
+ triggerAlignedScripts: conn != null,
2295
+ repoConnectionId: conn?.id ?? null
2296
+ });
2297
+ generationJobs.push(jid);
2298
+ }
2359
2299
  }
2360
- catch (err) {
2361
- const msg = String(err?.message ?? err);
2362
- logger.error({ jobId, err: msg }, '[new-pages-pipeline] failed');
2363
- await withTenant(tenantId, (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', finishedAt: new Date(), error: msg.slice(0, 1000) } })).catch(() => { });
2300
+ await updateJobProgress(tenantId, jobId, 60, `Queued ${generationJobs.length} test generation jobs`);
2301
+ if (conn && autoPR) {
2302
+ await enqueueJob("BATCH_GENERATE_ALIGNED", {
2303
+ tenantId,
2304
+ projectId,
2305
+ repoConnectionId: conn.id,
2306
+ scope: "UNMAPPED"
2307
+ });
2364
2308
  }
2309
+ await withTenant(
2310
+ tenantId,
2311
+ (tx) => tx.platformJob.update({
2312
+ where: { id: jobId },
2313
+ data: { status: "DONE", finishedAt: /* @__PURE__ */ new Date(), result: { newScreens: newScreens.length, generationJobs: generationJobs.length } }
2314
+ })
2315
+ );
2316
+ } catch (err) {
2317
+ const msg = String(err?.message ?? err);
2318
+ logger.error({ jobId, err: msg }, "[new-pages-pipeline] failed");
2319
+ await withTenant(
2320
+ tenantId,
2321
+ (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", finishedAt: /* @__PURE__ */ new Date(), error: msg.slice(0, 1e3) } })
2322
+ ).catch(() => {
2323
+ });
2324
+ }
2365
2325
  }
2326
+ export {
2327
+ aggregateJobTokens,
2328
+ cancelJob,
2329
+ claimNextQueuedJob,
2330
+ claimSpecificJob,
2331
+ createTrackedJob,
2332
+ enqueueJob,
2333
+ finishTrackedJob,
2334
+ generateTOTP,
2335
+ getJob,
2336
+ listJobs,
2337
+ processDataRetentionPurge,
2338
+ processJob,
2339
+ requeueOrphanedJobs,
2340
+ requeueStaleRunningJobs,
2341
+ rerunJob,
2342
+ resumeJob,
2343
+ startJobWorkerLoop,
2344
+ stopJobWorkerLoop,
2345
+ updateJobProgress
2346
+ };
2347
+ //# sourceMappingURL=job-queue.js.map