@detiq/crawler 1.1.9 → 1.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/accessibility-auditor.js +54 -74
- package/dist/ai-login.js +227 -249
- package/dist/auth-token-rotator.js +57 -67
- package/dist/auth.js +299 -336
- package/dist/browser-extension.js +104 -142
- package/dist/captcha-solver.js +98 -125
- package/dist/color-contrast.js +77 -89
- package/dist/content-chunker.js +126 -126
- package/dist/content-dedup.js +54 -71
- package/dist/cookie-consent.js +68 -83
- package/dist/cookie-jar.js +53 -60
- package/dist/crawler.js +4494 -5353
- package/dist/custom-js-injector.js +45 -61
- package/dist/dark-mode-capture.js +67 -81
- package/dist/dead-link-detector.js +47 -55
- package/dist/element-hash.js +8 -12
- package/dist/email-otp.js +154 -195
- package/dist/engine-router.js +87 -98
- package/dist/export/bulk-export.js +70 -55
- package/dist/export/vector-export.js +77 -105
- package/dist/extractors/advanced.js +249 -266
- package/dist/extractors/artifact-envelope.js +59 -58
- package/dist/failure-analysis-agent.js +37 -33
- package/dist/feature-flag-detector.js +51 -66
- package/dist/form-detector.js +55 -75
- package/dist/frontier.js +76 -101
- package/dist/gap-pass.js +68 -89
- package/dist/graphql-detector.js +62 -72
- package/dist/http-engine.js +282 -330
- package/dist/incremental-diff.js +54 -87
- package/dist/index.js +67 -28
- package/dist/input-fuzzer.js +93 -97
- package/dist/interactive-probe.js +715 -827
- package/dist/job-queue.js +2151 -2169
- package/dist/js-coverage.js +52 -56
- package/dist/js-router-detector.js +86 -117
- package/dist/language-detector.js +51 -63
- package/dist/link-classifier.js +32 -50
- package/dist/live-view.js +91 -108
- package/dist/locator-extractor.js +151 -187
- package/dist/logger.js +9 -7
- package/dist/markdown-extract.js +48 -61
- package/dist/native-crawler.js +780 -946
- package/dist/network-observer.js +135 -156
- package/dist/network-throttle.js +29 -40
- package/dist/openapi-generator.js +77 -92
- package/dist/pdf-extractor.js +72 -90
- package/dist/performance-metrics.js +80 -102
- package/dist/phash.js +125 -144
- package/dist/pii-scrubber.js +65 -68
- package/dist/playwright-recorder.js +70 -58
- package/dist/policies/crawl-safety.js +31 -34
- package/dist/profiles/domain-profile.js +130 -125
- package/dist/proxy.js +146 -172
- package/dist/rate-limiter.js +58 -65
- package/dist/redirect-tracker.js +60 -64
- package/dist/redis-frontier.js +116 -136
- package/dist/redis-queue.js +157 -144
- package/dist/response-cache.js +127 -140
- package/dist/responsive-detector.js +42 -46
- package/dist/retry-policy.js +46 -50
- package/dist/robots-txt.js +69 -83
- package/dist/screen-reader.js +45 -70
- package/dist/screenshot-masker.js +59 -77
- package/dist/seo-checker.js +88 -107
- package/dist/session-capture.js +192 -212
- package/dist/session-export.js +107 -133
- package/dist/sitemap-writer.js +38 -41
- package/dist/sitemap.js +141 -202
- package/dist/skyvern.js +123 -148
- package/dist/sqs-queue.js +42 -42
- package/dist/sso-token.js +74 -90
- package/dist/stealth/human-behavior.js +156 -173
- package/dist/telemetry.js +68 -82
- package/dist/test-executor.js +309 -335
- package/dist/tls-fingerprint.js +77 -88
- package/dist/url-pattern-completer.js +71 -90
- package/dist/visual-diff-verdict.js +23 -38
- package/dist/webhook-notifier.js +71 -69
- package/dist/websocket-detector.js +56 -68
- package/dist/worker.js +29 -38
- package/package.json +1 -1
- package/src/job-queue.ts +136 -0
package/dist/job-queue.js
CHANGED
|
@@ -1,161 +1,103 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
* enqueueJob() writes it QUEUED and, for same-process responsiveness, kicks
|
|
6
|
-
* an immediate best-effort claim attempt — but durability comes from
|
|
7
|
-
* startJobWorkerLoop()'s poll: it atomically claims QUEUED jobs (Postgres
|
|
8
|
-
* `FOR UPDATE SKIP LOCKED`, safe across any number of concurrent worker
|
|
9
|
-
* processes/containers) and requeues jobs whose heartbeat went stale,
|
|
10
|
-
* so a crashed/restarted worker never silently loses a job.
|
|
11
|
-
*
|
|
12
|
-
* CRAWL-type jobs additionally support AWS SQS as an alternate transport
|
|
13
|
-
* (JOB_QUEUE_BACKEND=sqs, see sqs-queue.ts) for horizontally-scaled crawler
|
|
14
|
-
* fleets; every other job type always goes through the DB-backed queue.
|
|
15
|
-
*/
|
|
16
|
-
import crypto from 'node:crypto';
|
|
17
|
-
import { withTenant, prismaAdmin } from '@detiq/database';
|
|
18
|
-
import { crawlJobsTotal, crawlJobDurationSeconds } from '@detiq/core';
|
|
19
|
-
import { logger } from './logger.js';
|
|
1
|
+
import crypto from "node:crypto";
|
|
2
|
+
import { withTenant, prismaAdmin } from "@detiq/database";
|
|
3
|
+
import { crawlJobsTotal, crawlJobDurationSeconds } from "@detiq/core";
|
|
4
|
+
import { logger } from "./logger.js";
|
|
20
5
|
async function notifyAdmins(tenantId, data) {
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
}))
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
6
|
+
try {
|
|
7
|
+
const members = await withTenant(
|
|
8
|
+
tenantId,
|
|
9
|
+
(tx) => tx.membership.findMany({
|
|
10
|
+
where: { tenantId, role: { in: ["OWNER", "ADMIN"] } },
|
|
11
|
+
select: { userId: true }
|
|
12
|
+
})
|
|
13
|
+
);
|
|
14
|
+
await withTenant(
|
|
15
|
+
tenantId,
|
|
16
|
+
(tx) => tx.notification.createMany({
|
|
17
|
+
data: members.map((m) => ({
|
|
18
|
+
id: `notif_${crypto.randomUUID()}`,
|
|
19
|
+
tenantId,
|
|
20
|
+
userId: m.userId,
|
|
21
|
+
type: data.type,
|
|
22
|
+
title: data.title,
|
|
23
|
+
body: data.body,
|
|
24
|
+
metadata: data.metadata
|
|
25
|
+
}))
|
|
26
|
+
})
|
|
27
|
+
);
|
|
28
|
+
} catch (e) {
|
|
29
|
+
console.error("[notify] failed:", e);
|
|
30
|
+
}
|
|
45
31
|
}
|
|
46
|
-
import { AppBrain, buildArtifactKey, getTenantStorageConfig, uploadScreenshotToTenantStorage, uploadFileToTenantStorage, notifyRunCompletion, recordLLMCall } from
|
|
47
|
-
import { generateTaxonomySkillFiles, groupScreensWithAI, generateExecutionPlan, refreshTestCasesSkillFile, refreshTestDataSkillFile, refreshTestScriptSkillFile, refreshRequirementsCoverageSkillFile, detectFieldsFromHtml, mergeIntoCatalog } from
|
|
48
|
-
import { crawlProject } from
|
|
49
|
-
import { httpCrawl } from
|
|
50
|
-
import { hashContent, normalizeText } from
|
|
51
|
-
import { discoverProjectUrls } from
|
|
52
|
-
import { executeTestPlan } from
|
|
53
|
-
import { runNativeCrawl } from
|
|
54
|
-
import { jira, confluence, figma } from
|
|
55
|
-
import { decrypt, config as platformConfig, dispatchWebhook, isFeatureEnabled, interpolatePlaceholders } from
|
|
56
|
-
import { createHmac } from
|
|
57
|
-
import fs from
|
|
58
|
-
import path from
|
|
59
|
-
import os from
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
bits -= 8;
|
|
74
|
-
bytes.push((value >> bits) & 0xff);
|
|
75
|
-
}
|
|
32
|
+
import { AppBrain, buildArtifactKey, getTenantStorageConfig, uploadScreenshotToTenantStorage, uploadFileToTenantStorage, notifyRunCompletion, recordLLMCall } from "@detiq/app-brain";
|
|
33
|
+
import { generateTaxonomySkillFiles, groupScreensWithAI, generateExecutionPlan, refreshTestCasesSkillFile, refreshTestDataSkillFile, refreshTestScriptSkillFile, refreshRequirementsCoverageSkillFile, detectFieldsFromHtml, mergeIntoCatalog } from "@detiq/agents";
|
|
34
|
+
import { crawlProject } from "./crawler.js";
|
|
35
|
+
import { httpCrawl } from "./http-engine.js";
|
|
36
|
+
import { hashContent, normalizeText } from "./incremental-diff.js";
|
|
37
|
+
import { discoverProjectUrls } from "./sitemap.js";
|
|
38
|
+
import { executeTestPlan } from "./test-executor.js";
|
|
39
|
+
import { runNativeCrawl } from "./native-crawler.js";
|
|
40
|
+
import { jira, confluence, figma } from "@detiq/connectors";
|
|
41
|
+
import { decrypt, config as platformConfig, dispatchWebhook, isFeatureEnabled, interpolatePlaceholders } from "@detiq/core";
|
|
42
|
+
import { createHmac } from "node:crypto";
|
|
43
|
+
import fs from "node:fs/promises";
|
|
44
|
+
import path from "node:path";
|
|
45
|
+
import os from "node:os";
|
|
46
|
+
function generateTOTP(secret) {
|
|
47
|
+
const BASE32 = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";
|
|
48
|
+
const clean = secret.toUpperCase().replace(/\s/g, "").replace(/=/g, "");
|
|
49
|
+
let bits = 0, value = 0;
|
|
50
|
+
const bytes = [];
|
|
51
|
+
for (const ch of clean) {
|
|
52
|
+
const idx = BASE32.indexOf(ch);
|
|
53
|
+
if (idx === -1) continue;
|
|
54
|
+
value = value << 5 | idx;
|
|
55
|
+
bits += 5;
|
|
56
|
+
if (bits >= 8) {
|
|
57
|
+
bits -= 8;
|
|
58
|
+
bytes.push(value >> bits & 255);
|
|
76
59
|
}
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
60
|
+
}
|
|
61
|
+
const counter = Math.floor(Date.now() / 3e4);
|
|
62
|
+
const buf = Buffer.alloc(8);
|
|
63
|
+
buf.writeBigInt64BE(BigInt(counter));
|
|
64
|
+
const hmac = createHmac("sha1", Buffer.from(bytes)).update(buf).digest();
|
|
65
|
+
const offset = hmac[hmac.length - 1] & 15;
|
|
66
|
+
const code = (hmac[offset] & 127) << 24 | (hmac[offset + 1] & 255) << 16 | (hmac[offset + 2] & 255) << 8 | hmac[offset + 3] & 255;
|
|
67
|
+
return String(code % 1e6).padStart(6, "0");
|
|
85
68
|
}
|
|
86
69
|
function deriveFallbackTargetPath(ref, framework) {
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
: 'tests/';
|
|
92
|
-
const ext = (framework === 'playwright-py' || framework === 'pytest' || framework === 'selenium-py') ? '_test.py'
|
|
93
|
-
: (framework === 'playwright-java' || framework === 'selenium-java' || framework === 'junit') ? 'Test.java'
|
|
94
|
-
: framework.includes('ts') ? '.spec.ts'
|
|
95
|
-
: '.spec.js';
|
|
96
|
-
return `${base}${slug}${ext}`;
|
|
70
|
+
const slug = (ref ?? "test").toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "") || "test";
|
|
71
|
+
const base = framework.startsWith("cypress") ? "cypress/e2e/" : framework === "playwright-py" || framework === "pytest" || framework === "selenium-py" ? "tests/" : framework === "playwright-java" || framework === "selenium-java" || framework === "junit" ? "src/test/java/" : "tests/";
|
|
72
|
+
const ext = framework === "playwright-py" || framework === "pytest" || framework === "selenium-py" ? "_test.py" : framework === "playwright-java" || framework === "selenium-java" || framework === "junit" ? "Test.java" : framework.includes("ts") ? ".spec.ts" : ".spec.js";
|
|
73
|
+
return `${base}${slug}${ext}`;
|
|
97
74
|
}
|
|
98
75
|
function makeJobName(type) {
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
: type === 'GENERATE_TESTS' ? 'GenerateTests'
|
|
105
|
-
: type === 'VISUAL_REGRESSION' ? 'VisualRegression'
|
|
106
|
-
: type === 'AI_EDIT' ? 'AIEdit'
|
|
107
|
-
: type === 'API_TESTS' ? 'ApiTests'
|
|
108
|
-
: type === 'SECURITY_TESTS' ? 'SecurityTests'
|
|
109
|
-
: type === 'INTEGRATION_TESTS' ? 'IntegrationTests'
|
|
110
|
-
: type === 'GENERATE_TESTS_FROM_REQS' ? 'GenerateTestsFromReqs'
|
|
111
|
-
: type === 'GENERATE_DATA' ? 'GenerateData'
|
|
112
|
-
: type === 'GENERATE_SCRIPT' ? 'GenerateScript'
|
|
113
|
-
: type === 'GENERATE_TEST_PLAN' ? 'GenerateTestPlan'
|
|
114
|
-
: type === 'IMPORT_TESTS' ? 'ImportTests'
|
|
115
|
-
: type === 'GENERATE_SCREEN_TESTS' ? 'GenerateScreenTests'
|
|
116
|
-
: type === 'BATCH_GENERATE_ALIGNED' ? 'BatchGenerateAligned'
|
|
117
|
-
: type === 'CREATE_PR' ? 'CreatePR'
|
|
118
|
-
: type === 'NEW_PAGES_PIPELINE' ? 'NewPagesPipeline'
|
|
119
|
-
: type.charAt(0) + type.slice(1).toLowerCase().replace(/_([a-z])/g, (_, c) => c.toUpperCase());
|
|
120
|
-
return `${label}_${date}_${ts}`;
|
|
76
|
+
const now = /* @__PURE__ */ new Date();
|
|
77
|
+
const date = now.toISOString().slice(0, 10);
|
|
78
|
+
const ts = now.getTime();
|
|
79
|
+
const label = type === "CRAWL" ? "Crawl" : type === "NATIVE_CRAWL" ? "NativeCrawl" : type === "GENERATE_TESTS" ? "GenerateTests" : type === "VISUAL_REGRESSION" ? "VisualRegression" : type === "AI_EDIT" ? "AIEdit" : type === "API_TESTS" ? "ApiTests" : type === "SECURITY_TESTS" ? "SecurityTests" : type === "INTEGRATION_TESTS" ? "IntegrationTests" : type === "GENERATE_TESTS_FROM_REQS" ? "GenerateTestsFromReqs" : type === "GENERATE_DATA" ? "GenerateData" : type === "GENERATE_SCRIPT" ? "GenerateScript" : type === "GENERATE_TEST_PLAN" ? "GenerateTestPlan" : type === "IMPORT_TESTS" ? "ImportTests" : type === "GENERATE_SCREEN_TESTS" ? "GenerateScreenTests" : type === "BATCH_GENERATE_ALIGNED" ? "BatchGenerateAligned" : type === "CREATE_PR" ? "CreatePR" : type === "NEW_PAGES_PIPELINE" ? "NewPagesPipeline" : type.charAt(0) + type.slice(1).toLowerCase().replace(/_([a-z])/g, (_, c) => c.toUpperCase());
|
|
80
|
+
return `${label}_${date}_${ts}`;
|
|
121
81
|
}
|
|
122
|
-
|
|
123
|
-
const runningControllers = new Map();
|
|
124
|
-
// ─── Durable job worker ──────────────────────────────────────────────────────
|
|
125
|
-
//
|
|
126
|
-
// A minimal Postgres-backed work queue: atomic claim via `FOR UPDATE SKIP
|
|
127
|
-
// LOCKED` (safe for any number of concurrent pollers, in-process or across
|
|
128
|
-
// containers), plus a stale-job sweep keyed off a liveness heartbeat rather
|
|
129
|
-
// than wall-clock job age, so a slow-but-alive job is never requeued and
|
|
130
|
-
// double-processed. See startJobWorkerLoop() below.
|
|
82
|
+
const runningControllers = /* @__PURE__ */ new Map();
|
|
131
83
|
const WORKER_ID = `${os.hostname()}:${process.pid}`;
|
|
132
84
|
const STALE_JOB_TIMEOUT_MINUTES = Number(process.env.JOB_STALE_TIMEOUT_MINUTES) || 30;
|
|
133
|
-
const WORKER_POLL_INTERVAL_MS = Number(process.env.JOB_WORKER_POLL_INTERVAL_MS) ||
|
|
85
|
+
const WORKER_POLL_INTERVAL_MS = Number(process.env.JOB_WORKER_POLL_INTERVAL_MS) || 15e3;
|
|
134
86
|
const WORKER_CONCURRENCY = Number(process.env.JOB_WORKER_CONCURRENCY) || 8;
|
|
135
|
-
const JOB_HEARTBEAT_INTERVAL_MS = Number(process.env.JOB_HEARTBEAT_INTERVAL_MS) ||
|
|
136
|
-
// How many headless browsers a single RUN_TEST_SUITE job launches at once —
|
|
137
|
-
// distinct from WORKER_CONCURRENCY (how many *jobs* the poll loop claims at
|
|
138
|
-
// once). Bounded by default to avoid exhausting a container's memory; the
|
|
139
|
-
// primary scale-out path for running more tests at once is still horizontal
|
|
140
|
-
// (more worker containers / SQS), not raising this past what one container
|
|
141
|
-
// can safely hold — see startJobWorkerLoop's doc comment.
|
|
87
|
+
const JOB_HEARTBEAT_INTERVAL_MS = Number(process.env.JOB_HEARTBEAT_INTERVAL_MS) || 15e3;
|
|
142
88
|
const TEST_EXEC_CONCURRENCY = Number(process.env.TEST_EXEC_CONCURRENCY) || 5;
|
|
143
|
-
/** Dispatch table populated at module bottom, once every process*Job function is declared. */
|
|
144
89
|
let jobDispatchTable = null;
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
const rows = await prismaAdmin.$queryRaw `
|
|
90
|
+
async function claimSpecificJob(jobId) {
|
|
91
|
+
const rows = await prismaAdmin.$queryRaw`
|
|
148
92
|
UPDATE "PlatformJob"
|
|
149
93
|
SET status = 'RUNNING', "startedAt" = NOW(), "heartbeatAt" = NOW(), "workerId" = ${WORKER_ID}
|
|
150
94
|
WHERE id = ${jobId} AND status = 'QUEUED'
|
|
151
95
|
RETURNING id, "tenantId" AS "tenantId", "projectId" AS "projectId", type, config
|
|
152
96
|
`;
|
|
153
|
-
|
|
97
|
+
return rows[0] ?? null;
|
|
154
98
|
}
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
export async function claimNextQueuedJob() {
|
|
158
|
-
const rows = await prismaAdmin.$queryRaw `
|
|
99
|
+
async function claimNextQueuedJob() {
|
|
100
|
+
const rows = await prismaAdmin.$queryRaw`
|
|
159
101
|
UPDATE "PlatformJob"
|
|
160
102
|
SET status = 'RUNNING', "startedAt" = NOW(), "heartbeatAt" = NOW(), "workerId" = ${WORKER_ID}
|
|
161
103
|
WHERE id = (
|
|
@@ -167,13 +109,10 @@ export async function claimNextQueuedJob() {
|
|
|
167
109
|
)
|
|
168
110
|
RETURNING id, "tenantId" AS "tenantId", "projectId" AS "projectId", type, config
|
|
169
111
|
`;
|
|
170
|
-
|
|
112
|
+
return rows[0] ?? null;
|
|
171
113
|
}
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
* safe to run from every worker process; only affects rows past the timeout. */
|
|
175
|
-
export async function requeueStaleRunningJobs() {
|
|
176
|
-
const rows = await prismaAdmin.$queryRaw `
|
|
114
|
+
async function requeueStaleRunningJobs() {
|
|
115
|
+
const rows = await prismaAdmin.$queryRaw`
|
|
177
116
|
UPDATE "PlatformJob"
|
|
178
117
|
SET status = 'QUEUED',
|
|
179
118
|
"workerId" = NULL,
|
|
@@ -187,18 +126,12 @@ export async function requeueStaleRunningJobs() {
|
|
|
187
126
|
AND COALESCE("heartbeatAt", "startedAt") < NOW() - (${STALE_JOB_TIMEOUT_MINUTES} * interval '1 minute')
|
|
188
127
|
RETURNING id
|
|
189
128
|
`;
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
129
|
+
if (rows.length > 0) {
|
|
130
|
+
logger.warn({ jobIds: rows.map((r) => r.id), staleTimeoutMinutes: STALE_JOB_TIMEOUT_MINUTES }, "[job-worker] requeued stale RUNNING job(s)");
|
|
131
|
+
}
|
|
193
132
|
}
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
* process instance). Call once at startup before the poll loop begins — this
|
|
197
|
-
* provides immediate recovery instead of waiting the full STALE_JOB_TIMEOUT_MINUTES.
|
|
198
|
-
* Safe to call from multiple replicas simultaneously (each uses its own WORKER_ID).
|
|
199
|
-
*/
|
|
200
|
-
export async function requeueOrphanedJobs() {
|
|
201
|
-
const rows = await prismaAdmin.$queryRaw `
|
|
133
|
+
async function requeueOrphanedJobs() {
|
|
134
|
+
const rows = await prismaAdmin.$queryRaw`
|
|
202
135
|
UPDATE "PlatformJob"
|
|
203
136
|
SET status = 'QUEUED',
|
|
204
137
|
"workerId" = NULL,
|
|
@@ -213,522 +146,464 @@ export async function requeueOrphanedJobs() {
|
|
|
213
146
|
AND "workerId" != ${WORKER_ID}
|
|
214
147
|
RETURNING id
|
|
215
148
|
`;
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
149
|
+
if (rows.length > 0) {
|
|
150
|
+
logger.warn({ jobIds: rows.map((r) => r.id), workerId: WORKER_ID }, "[job-worker] requeued orphaned RUNNING job(s) from dead worker");
|
|
151
|
+
}
|
|
219
152
|
}
|
|
220
|
-
/** Dispatches a claimed job to its type's processor. Last-resort safety net — every
|
|
221
|
-
* process*Job function already catches and records its own failures. */
|
|
222
153
|
async function dispatchClaimedJob(job) {
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
154
|
+
const dispatcher = jobDispatchTable?.[job.type];
|
|
155
|
+
if (!dispatcher) {
|
|
156
|
+
logger.error({ jobId: job.id, type: job.type }, "[job-worker] no dispatcher registered for job type");
|
|
157
|
+
await prismaAdmin.platformJob.update({
|
|
158
|
+
where: { id: job.id },
|
|
159
|
+
data: { status: "FAILED", error: `Unknown job type: ${job.type}`, finishedAt: /* @__PURE__ */ new Date() }
|
|
160
|
+
}).catch(() => {
|
|
161
|
+
});
|
|
162
|
+
return;
|
|
163
|
+
}
|
|
164
|
+
const stopTimer = crawlJobDurationSeconds.startTimer({ type: job.type });
|
|
165
|
+
try {
|
|
166
|
+
await dispatcher(job.id, job.config);
|
|
167
|
+
stopTimer();
|
|
168
|
+
const finished = await prismaAdmin.platformJob.findUnique({ where: { id: job.id }, select: { status: true } }).catch(() => null);
|
|
169
|
+
crawlJobsTotal.inc({ type: job.type, status: finished?.status === "FAILED" ? "failed" : "success" });
|
|
170
|
+
} catch (err) {
|
|
171
|
+
stopTimer();
|
|
172
|
+
crawlJobsTotal.inc({ type: job.type, status: "failed" });
|
|
173
|
+
logger.error({ jobId: job.id, type: job.type, err: String(err?.message ?? err) }, "[job-worker] job threw uncaught error");
|
|
174
|
+
await prismaAdmin.platformJob.update({
|
|
175
|
+
where: { id: job.id },
|
|
176
|
+
data: { status: "FAILED", error: String(err?.message ?? err), finishedAt: /* @__PURE__ */ new Date() }
|
|
177
|
+
}).catch(() => {
|
|
178
|
+
});
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
let workerLoopTimer = null;
|
|
182
|
+
function startJobWorkerLoop(opts) {
|
|
183
|
+
if (workerLoopTimer) return;
|
|
184
|
+
const pollIntervalMs = opts?.pollIntervalMs ?? WORKER_POLL_INTERVAL_MS;
|
|
185
|
+
const concurrency = opts?.concurrency ?? WORKER_CONCURRENCY;
|
|
186
|
+
logger.info({ workerId: WORKER_ID, pollIntervalMs, concurrency }, "[job-worker] starting");
|
|
187
|
+
const tick = async () => {
|
|
233
188
|
try {
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
// function catches and records its own failures internally without
|
|
238
|
-
// rethrowing (see this function's doc comment). Read back the actual
|
|
239
|
-
// final status so the metric reflects reality, not just "didn't throw".
|
|
240
|
-
const finished = await prismaAdmin.platformJob.findUnique({ where: { id: job.id }, select: { status: true } }).catch(() => null);
|
|
241
|
-
crawlJobsTotal.inc({ type: job.type, status: finished?.status === 'FAILED' ? 'failed' : 'success' });
|
|
189
|
+
await requeueStaleRunningJobs();
|
|
190
|
+
} catch (err) {
|
|
191
|
+
logger.error({ err: String(err) }, "[job-worker] stale-job sweep failed");
|
|
242
192
|
}
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
193
|
+
try {
|
|
194
|
+
for (let i = 0; i < concurrency; i++) {
|
|
195
|
+
const job = await claimNextQueuedJob();
|
|
196
|
+
if (!job) break;
|
|
197
|
+
void dispatchClaimedJob(job);
|
|
198
|
+
}
|
|
199
|
+
} catch (err) {
|
|
200
|
+
logger.error({ err: String(err) }, "[job-worker] claim loop failed");
|
|
251
201
|
}
|
|
202
|
+
};
|
|
203
|
+
workerLoopTimer = setInterval(tick, pollIntervalMs);
|
|
204
|
+
requeueOrphanedJobs().catch(
|
|
205
|
+
(err) => logger.error({ err: String(err) }, "[job-worker] startup orphan sweep failed")
|
|
206
|
+
);
|
|
207
|
+
void tick();
|
|
252
208
|
}
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
* survive a restart — enqueueJob()'s immediate same-process claim attempt is
|
|
257
|
-
* only a latency optimization; if the process dies before that fires (or
|
|
258
|
-
* mid-job), this loop's next tick claims/requeues it. Call once at process
|
|
259
|
-
* startup (API server and/or a dedicated crawler worker process — safe to
|
|
260
|
-
* run in both, and safe to run in many container replicas at once).
|
|
261
|
-
*/
|
|
262
|
-
export function startJobWorkerLoop(opts) {
|
|
263
|
-
if (workerLoopTimer)
|
|
264
|
-
return; // already running in this process
|
|
265
|
-
const pollIntervalMs = opts?.pollIntervalMs ?? WORKER_POLL_INTERVAL_MS;
|
|
266
|
-
const concurrency = opts?.concurrency ?? WORKER_CONCURRENCY;
|
|
267
|
-
logger.info({ workerId: WORKER_ID, pollIntervalMs, concurrency }, '[job-worker] starting');
|
|
268
|
-
const tick = async () => {
|
|
269
|
-
try {
|
|
270
|
-
await requeueStaleRunningJobs();
|
|
271
|
-
}
|
|
272
|
-
catch (err) {
|
|
273
|
-
logger.error({ err: String(err) }, '[job-worker] stale-job sweep failed');
|
|
274
|
-
}
|
|
275
|
-
try {
|
|
276
|
-
for (let i = 0; i < concurrency; i++) {
|
|
277
|
-
const job = await claimNextQueuedJob();
|
|
278
|
-
if (!job)
|
|
279
|
-
break;
|
|
280
|
-
void dispatchClaimedJob(job); // not awaited — runs concurrently while the loop keeps polling
|
|
281
|
-
}
|
|
282
|
-
}
|
|
283
|
-
catch (err) {
|
|
284
|
-
logger.error({ err: String(err) }, '[job-worker] claim loop failed');
|
|
285
|
-
}
|
|
286
|
-
};
|
|
287
|
-
workerLoopTimer = setInterval(tick, pollIntervalMs);
|
|
288
|
-
// On startup: immediately requeue jobs orphaned by the previous process instance.
|
|
289
|
-
// This gives instant recovery rather than waiting STALE_JOB_TIMEOUT_MINUTES (30 min).
|
|
290
|
-
requeueOrphanedJobs().catch((err) => logger.error({ err: String(err) }, '[job-worker] startup orphan sweep failed'));
|
|
291
|
-
void tick();
|
|
209
|
+
function stopJobWorkerLoop() {
|
|
210
|
+
if (workerLoopTimer) clearInterval(workerLoopTimer);
|
|
211
|
+
workerLoopTimer = null;
|
|
292
212
|
}
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
213
|
+
async function createTrackedJob(type, config) {
|
|
214
|
+
const name = makeJobName(type);
|
|
215
|
+
const job = await withTenant(
|
|
216
|
+
config.tenantId,
|
|
217
|
+
(tx) => tx.platformJob.create({
|
|
218
|
+
data: { tenantId: config.tenantId, projectId: config.projectId, type, name, status: "RUNNING", startedAt: /* @__PURE__ */ new Date(), config }
|
|
219
|
+
})
|
|
220
|
+
);
|
|
221
|
+
return job.id;
|
|
298
222
|
}
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
export async function finishTrackedJob(tenantId, jobId, error) {
|
|
309
|
-
await withTenant(tenantId, (tx) => tx.platformJob.update({
|
|
310
|
-
where: { id: jobId },
|
|
311
|
-
data: { status: error ? 'FAILED' : 'DONE', finishedAt: new Date(), ...(error ? { error } : { progress: 100 }) },
|
|
312
|
-
})).catch(() => { });
|
|
223
|
+
async function finishTrackedJob(tenantId, jobId, error) {
|
|
224
|
+
await withTenant(
|
|
225
|
+
tenantId,
|
|
226
|
+
(tx) => tx.platformJob.update({
|
|
227
|
+
where: { id: jobId },
|
|
228
|
+
data: { status: error ? "FAILED" : "DONE", finishedAt: /* @__PURE__ */ new Date(), ...error ? { error } : { progress: 100 } }
|
|
229
|
+
})
|
|
230
|
+
).catch(() => {
|
|
231
|
+
});
|
|
313
232
|
}
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
// Skip inline dispatch when DISABLE_WORKER=true — the dedicated crawler process
|
|
338
|
-
// will pick this up via its poll loop. Inline dispatch in the API container
|
|
339
|
-
// would launch Playwright browsers and crash the API.
|
|
340
|
-
setImmediate(() => claimSpecificJob(job.id).then((claimed) => claimed && dispatchClaimedJob(claimed)));
|
|
341
|
-
}
|
|
342
|
-
return job.id;
|
|
233
|
+
async function enqueueJob(type, config) {
|
|
234
|
+
const { tenantId, projectId } = config;
|
|
235
|
+
const name = makeJobName(type);
|
|
236
|
+
const job = await withTenant(
|
|
237
|
+
tenantId,
|
|
238
|
+
(tx) => tx.platformJob.create({
|
|
239
|
+
data: {
|
|
240
|
+
tenantId,
|
|
241
|
+
projectId,
|
|
242
|
+
type,
|
|
243
|
+
name,
|
|
244
|
+
status: "QUEUED",
|
|
245
|
+
config
|
|
246
|
+
}
|
|
247
|
+
})
|
|
248
|
+
);
|
|
249
|
+
if (type === "CRAWL" && process.env.JOB_QUEUE_BACKEND === "sqs") {
|
|
250
|
+
const { sendJobToSqs } = await import("./sqs-queue.js");
|
|
251
|
+
await sendJobToSqs(job.id, tenantId, config);
|
|
252
|
+
} else if (process.env.DISABLE_WORKER !== "true") {
|
|
253
|
+
setImmediate(() => claimSpecificJob(job.id).then((claimed) => claimed && dispatchClaimedJob(claimed)));
|
|
254
|
+
}
|
|
255
|
+
return job.id;
|
|
343
256
|
}
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
257
|
+
async function getJob(tenantId, jobId) {
|
|
258
|
+
return withTenant(
|
|
259
|
+
tenantId,
|
|
260
|
+
(tx) => tx.platformJob.findFirst({ where: { id: jobId, tenantId } })
|
|
261
|
+
);
|
|
347
262
|
}
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
by: ['jobId'],
|
|
362
|
-
where: { tenantId, jobId: { in: jobIds } },
|
|
363
|
-
_max: { model: true, calledAt: true },
|
|
364
|
-
});
|
|
365
|
-
const modelByJobId = new Map(modelRows
|
|
366
|
-
.filter((r) => r.jobId && r._max?.model)
|
|
367
|
-
.map((r) => [r.jobId, r._max.model]));
|
|
368
|
-
return jobs.map((j) => ({ ...j, llmModel: modelByJobId.get(j.id) ?? null }));
|
|
263
|
+
async function listJobs(tenantId, projectId) {
|
|
264
|
+
return withTenant(tenantId, async (tx) => {
|
|
265
|
+
const jobs = await tx.platformJob.findMany({
|
|
266
|
+
where: { tenantId, projectId },
|
|
267
|
+
orderBy: { queuedAt: "desc" },
|
|
268
|
+
take: 50
|
|
269
|
+
});
|
|
270
|
+
const jobIds = jobs.map((j) => j.id);
|
|
271
|
+
if (jobIds.length === 0) return jobs;
|
|
272
|
+
const modelRows = await tx.lLMCallLog.groupBy({
|
|
273
|
+
by: ["jobId"],
|
|
274
|
+
where: { tenantId, jobId: { in: jobIds } },
|
|
275
|
+
_max: { model: true, calledAt: true }
|
|
369
276
|
});
|
|
277
|
+
const modelByJobId = new Map(
|
|
278
|
+
modelRows.filter((r) => r.jobId && r._max?.model).map((r) => [r.jobId, r._max.model])
|
|
279
|
+
);
|
|
280
|
+
return jobs.map((j) => ({ ...j, llmModel: modelByJobId.get(j.id) ?? null }));
|
|
281
|
+
});
|
|
370
282
|
}
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
})
|
|
283
|
+
async function cancelJob(tenantId, jobId) {
|
|
284
|
+
const controller = runningControllers.get(jobId);
|
|
285
|
+
if (controller) {
|
|
286
|
+
controller.abort();
|
|
287
|
+
runningControllers.delete(jobId);
|
|
288
|
+
}
|
|
289
|
+
await withTenant(
|
|
290
|
+
tenantId,
|
|
291
|
+
(tx) => tx.platformJob.update({
|
|
292
|
+
where: { id: jobId },
|
|
293
|
+
data: { status: "CANCELLED", finishedAt: /* @__PURE__ */ new Date() }
|
|
294
|
+
})
|
|
295
|
+
);
|
|
383
296
|
}
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
297
|
+
async function rerunJob(tenantId, jobId) {
|
|
298
|
+
const job = await withTenant(
|
|
299
|
+
tenantId,
|
|
300
|
+
(tx) => tx.platformJob.findUnique({ where: { id: jobId } })
|
|
301
|
+
);
|
|
302
|
+
if (!job) throw new Error("Job not found");
|
|
303
|
+
return enqueueJob(job.type, { ...job.config, tenantId });
|
|
304
|
+
}
|
|
305
|
+
const jobCheckpoints = /* @__PURE__ */ new Map();
|
|
306
|
+
async function resumeJob(tenantId, jobId) {
|
|
307
|
+
const job = await withTenant(
|
|
308
|
+
tenantId,
|
|
309
|
+
(tx) => tx.platformJob.findUnique({ where: { id: jobId } })
|
|
310
|
+
);
|
|
311
|
+
if (!job) throw new Error("Job not found");
|
|
312
|
+
const checkpoint = job.result?.checkpoint;
|
|
313
|
+
const visitedUrls = checkpoint?.visitedUrls ?? [];
|
|
314
|
+
const pendingQueue = checkpoint?.pendingQueue ?? [];
|
|
315
|
+
if (pendingQueue.length === 0 && visitedUrls.length === 0) {
|
|
389
316
|
return enqueueJob(job.type, { ...job.config, tenantId });
|
|
317
|
+
}
|
|
318
|
+
return enqueueJob(job.type, {
|
|
319
|
+
...job.config,
|
|
320
|
+
tenantId,
|
|
321
|
+
skipUrls: visitedUrls.length > 0 ? visitedUrls : void 0,
|
|
322
|
+
resumeUrls: pendingQueue.length > 0 ? pendingQueue : void 0
|
|
323
|
+
});
|
|
390
324
|
}
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
skipUrls: visitedUrls.length > 0 ? visitedUrls : undefined,
|
|
408
|
-
resumeUrls: pendingQueue.length > 0 ? pendingQueue : undefined,
|
|
325
|
+
async function updateJobProgress(tenantId, jobId, progress, step) {
|
|
326
|
+
try {
|
|
327
|
+
await withTenant(tenantId, async (tx) => {
|
|
328
|
+
const current = await tx.platformJob.findUnique({
|
|
329
|
+
where: { id: jobId },
|
|
330
|
+
select: { status: true }
|
|
331
|
+
});
|
|
332
|
+
if (current?.status === "CANCELLED") {
|
|
333
|
+
runningControllers.get(jobId)?.abort();
|
|
334
|
+
runningControllers.delete(jobId);
|
|
335
|
+
return;
|
|
336
|
+
}
|
|
337
|
+
await tx.platformJob.update({
|
|
338
|
+
where: { id: jobId },
|
|
339
|
+
data: { progress, result: { step }, heartbeatAt: /* @__PURE__ */ new Date() }
|
|
340
|
+
});
|
|
409
341
|
});
|
|
342
|
+
} catch {
|
|
343
|
+
}
|
|
410
344
|
}
|
|
411
|
-
/** Update progress percentage and current step label on a running job (non-fatal). */
|
|
412
|
-
export async function updateJobProgress(tenantId, jobId, progress, step) {
|
|
413
|
-
try {
|
|
414
|
-
await withTenant(tenantId, async (tx) => {
|
|
415
|
-
// DB-flag cancellation: works for both inline and SQS backends.
|
|
416
|
-
// When a user cancels, status is set to 'CANCELLED' in DB.
|
|
417
|
-
// This check fires at each progress update (every few seconds during crawl).
|
|
418
|
-
const current = await tx.platformJob.findUnique({
|
|
419
|
-
where: { id: jobId },
|
|
420
|
-
select: { status: true },
|
|
421
|
-
});
|
|
422
|
-
if (current?.status === 'CANCELLED') {
|
|
423
|
-
runningControllers.get(jobId)?.abort();
|
|
424
|
-
runningControllers.delete(jobId);
|
|
425
|
-
return; // abort signal propagates; crawlProject exits on next signal check
|
|
426
|
-
}
|
|
427
|
-
await tx.platformJob.update({
|
|
428
|
-
where: { id: jobId },
|
|
429
|
-
data: { progress, result: { step }, heartbeatAt: new Date() },
|
|
430
|
-
});
|
|
431
|
-
});
|
|
432
|
-
}
|
|
433
|
-
catch { /* non-fatal — progress updates best-effort */ }
|
|
434
|
-
}
|
|
435
|
-
/** Bump only worker liveness. This intentionally does not touch progress/result. */
|
|
436
345
|
async function heartbeatJob(tenantId, jobId) {
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
catch { /* non-fatal — heartbeat updates best-effort */ }
|
|
346
|
+
try {
|
|
347
|
+
await withTenant(tenantId, async (tx) => {
|
|
348
|
+
const current = await tx.platformJob.findUnique({
|
|
349
|
+
where: { id: jobId },
|
|
350
|
+
select: { status: true }
|
|
351
|
+
});
|
|
352
|
+
if (current?.status === "CANCELLED") {
|
|
353
|
+
runningControllers.get(jobId)?.abort();
|
|
354
|
+
runningControllers.delete(jobId);
|
|
355
|
+
return;
|
|
356
|
+
}
|
|
357
|
+
if (current?.status !== "RUNNING") return;
|
|
358
|
+
await tx.platformJob.update({
|
|
359
|
+
where: { id: jobId },
|
|
360
|
+
data: { heartbeatAt: /* @__PURE__ */ new Date(), workerId: WORKER_ID }
|
|
361
|
+
});
|
|
362
|
+
});
|
|
363
|
+
} catch {
|
|
364
|
+
}
|
|
457
365
|
}
|
|
458
|
-
/** Background: for each changed screen, search Jira + Confluence for related tickets/pages. */
|
|
459
366
|
async function investigateScreenChanges(tenantId, projectId, changedScreens) {
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
}
|
|
484
|
-
|
|
485
|
-
try {
|
|
486
|
-
detail.confluencePages = await confluence.searchPages({ baseUrl: confluenceCfg.baseUrl, email: confluenceCfg.email, apiToken: confluenceCfg.apiToken, spaceKey: confluenceCfg.spaceKey }, searchTerm);
|
|
487
|
-
}
|
|
488
|
-
catch (e) {
|
|
489
|
-
logger.warn({ searchTerm, err: e }, 'Confluence search failed');
|
|
490
|
-
}
|
|
491
|
-
}
|
|
492
|
-
await AppBrain.logChange(tenantId, projectId, 'MODIFIED', changeSummary, detail);
|
|
493
|
-
logger.info({ screen: screen.name, jiraTickets: detail.jiraTickets.length, confluencePages: detail.confluencePages.length }, 'Logged screen change investigation');
|
|
367
|
+
const [jiraCfg, confluenceCfg] = await Promise.all([
|
|
368
|
+
AppBrain.getConnectorConfig(tenantId, projectId, "JIRA"),
|
|
369
|
+
AppBrain.getConnectorConfig(tenantId, projectId, "CONFLUENCE")
|
|
370
|
+
]);
|
|
371
|
+
for (const screen of changedScreens) {
|
|
372
|
+
const searchTerm = screen.url ?? screen.name;
|
|
373
|
+
const changeSummary = `Screen "${screen.name}" changed: +${screen.addedElements.length} elements, -${screen.removedElements.length} elements`;
|
|
374
|
+
const detail = {
|
|
375
|
+
screenId: screen.id,
|
|
376
|
+
screenName: screen.name,
|
|
377
|
+
url: screen.url,
|
|
378
|
+
addedElements: screen.addedElements,
|
|
379
|
+
removedElements: screen.removedElements,
|
|
380
|
+
jiraTickets: [],
|
|
381
|
+
confluencePages: []
|
|
382
|
+
};
|
|
383
|
+
if (jiraCfg?.baseUrl && jiraCfg?.email && jiraCfg?.apiToken) {
|
|
384
|
+
try {
|
|
385
|
+
detail.jiraTickets = await jira.searchIssues(
|
|
386
|
+
{ baseUrl: jiraCfg.baseUrl, email: jiraCfg.email, apiToken: jiraCfg.apiToken, projectKey: jiraCfg.projectKey },
|
|
387
|
+
searchTerm
|
|
388
|
+
);
|
|
389
|
+
} catch (e) {
|
|
390
|
+
logger.warn({ searchTerm, err: e }, "Jira search failed");
|
|
391
|
+
}
|
|
494
392
|
}
|
|
393
|
+
if (confluenceCfg?.baseUrl && confluenceCfg?.email && confluenceCfg?.apiToken) {
|
|
394
|
+
try {
|
|
395
|
+
detail.confluencePages = await confluence.searchPages(
|
|
396
|
+
{ baseUrl: confluenceCfg.baseUrl, email: confluenceCfg.email, apiToken: confluenceCfg.apiToken, spaceKey: confluenceCfg.spaceKey },
|
|
397
|
+
searchTerm
|
|
398
|
+
);
|
|
399
|
+
} catch (e) {
|
|
400
|
+
logger.warn({ searchTerm, err: e }, "Confluence search failed");
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
await AppBrain.logChange(tenantId, projectId, "MODIFIED", changeSummary, detail);
|
|
404
|
+
logger.info({ screen: screen.name, jiraTickets: detail.jiraTickets.length, confluencePages: detail.confluencePages.length }, "Logged screen change investigation");
|
|
405
|
+
}
|
|
495
406
|
}
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
407
|
+
async function processJob(jobId, config, opts) {
|
|
408
|
+
const { tenantId, projectId } = config;
|
|
409
|
+
const jobStartedAt = /* @__PURE__ */ new Date();
|
|
410
|
+
const claimed = await withTenant(
|
|
411
|
+
tenantId,
|
|
412
|
+
(tx) => tx.platformJob.updateMany({
|
|
413
|
+
where: { id: jobId, status: { not: "CANCELLED" } },
|
|
414
|
+
data: { status: "RUNNING", startedAt: jobStartedAt, heartbeatAt: jobStartedAt, workerId: WORKER_ID }
|
|
415
|
+
})
|
|
416
|
+
);
|
|
417
|
+
if (claimed.count === 0) return;
|
|
418
|
+
const controller = new AbortController();
|
|
419
|
+
runningControllers.set(jobId, controller);
|
|
420
|
+
const heartbeatTimer = setInterval(() => {
|
|
421
|
+
void heartbeatJob(tenantId, jobId);
|
|
422
|
+
}, JOB_HEARTBEAT_INTERVAL_MS);
|
|
423
|
+
const onProgress = async (pct, step) => {
|
|
424
|
+
await updateJobProgress(tenantId, jobId, pct, step);
|
|
425
|
+
await opts?.onProgress?.(pct);
|
|
426
|
+
};
|
|
427
|
+
const DEFAULT_MAX_SCREENS = 150;
|
|
428
|
+
try {
|
|
429
|
+
if (!config.maxScreens || !config.deviceProfiles || config.interactive === void 0 || config.useSitemap === void 0 || !config.traversalStrategy) {
|
|
430
|
+
const proj = await withTenant(
|
|
431
|
+
tenantId,
|
|
432
|
+
(tx) => tx.project.findUnique({ where: { id: projectId }, select: { crawlConfig: true } })
|
|
433
|
+
);
|
|
434
|
+
const perProject = proj?.crawlConfig?.maxScreens;
|
|
435
|
+
if (!config.maxScreens) config.maxScreens = perProject || DEFAULT_MAX_SCREENS;
|
|
436
|
+
if (!config.deviceProfiles) config.deviceProfiles = proj?.crawlConfig?.deviceProfiles ?? ["DESKTOP"];
|
|
437
|
+
if (config.interactive === void 0) config.interactive = proj?.crawlConfig?.interactive ?? false;
|
|
438
|
+
if (config.useSitemap === void 0) config.useSitemap = proj?.crawlConfig?.useSitemap ?? false;
|
|
439
|
+
if (!config.sitemapUrl) config.sitemapUrl = proj?.crawlConfig?.sitemapUrl ?? void 0;
|
|
440
|
+
if (!config.traversalStrategy) config.traversalStrategy = proj?.crawlConfig?.traversalStrategy ?? "BFS";
|
|
441
|
+
if (!config.bestFirstKeywords) config.bestFirstKeywords = proj?.crawlConfig?.bestFirstKeywords ?? void 0;
|
|
442
|
+
if (!config.segmentSize) config.segmentSize = proj?.crawlConfig?.segmentSize ?? 9999999;
|
|
443
|
+
if (!config.selectiveUrls?.length && (!config.startUrls || config.startUrls.length === 0)) {
|
|
444
|
+
const saved = proj?.crawlConfig?.seedUrls ?? [];
|
|
445
|
+
if (saved.length > 0) config.startUrls = saved;
|
|
446
|
+
}
|
|
447
|
+
if (config.interactive === void 0) config.interactive = proj?.crawlConfig?.interactive ?? false;
|
|
448
|
+
}
|
|
449
|
+
const runtimeCreds = config.credentials ? {
|
|
450
|
+
...config.credentials,
|
|
451
|
+
...config.credentials.totpSecret ? { getOneTimeCode: async () => generateTOTP(config.credentials.totpSecret) } : {}
|
|
452
|
+
} : void 0;
|
|
453
|
+
const storageConfigForCrawl = await getTenantStorageConfig(tenantId).catch(() => null);
|
|
454
|
+
const result = await crawlProject({
|
|
455
|
+
...config,
|
|
456
|
+
credentials: runtimeCreds,
|
|
457
|
+
abortSignal: controller.signal,
|
|
458
|
+
jobId,
|
|
459
|
+
storageConfig: storageConfigForCrawl,
|
|
460
|
+
onCheckpoint: async (visitedUrls, pendingQueue) => {
|
|
461
|
+
jobCheckpoints.set(jobId, { visitedUrls, pendingQueue });
|
|
462
|
+
}
|
|
463
|
+
}, onProgress);
|
|
464
|
+
jobCheckpoints.delete(jobId);
|
|
465
|
+
const pendingUrls = result?.pendingUrls ?? [];
|
|
466
|
+
if (pendingUrls.length > 0) {
|
|
467
|
+
logger.info({ pendingUrls: pendingUrls.length }, "Crawl finished with pending URLs \u2014 not auto-queuing (segmented crawls disabled)");
|
|
468
|
+
}
|
|
469
|
+
await withTenant(
|
|
470
|
+
tenantId,
|
|
471
|
+
(tx) => tx.platformJob.update({
|
|
472
|
+
where: { id: jobId },
|
|
473
|
+
data: {
|
|
474
|
+
status: "DONE",
|
|
475
|
+
result: { ...result },
|
|
476
|
+
progress: 100,
|
|
477
|
+
finishedAt: /* @__PURE__ */ new Date()
|
|
548
478
|
}
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
479
|
+
})
|
|
480
|
+
);
|
|
481
|
+
clearInterval(heartbeatTimer);
|
|
482
|
+
runningControllers.delete(jobId);
|
|
483
|
+
{
|
|
484
|
+
const crawledUrls = (result.discovered ?? []).filter((d) => !!d.screenId && !!d.url).map((d) => d.url);
|
|
485
|
+
if (crawledUrls.length > 0) {
|
|
486
|
+
(async () => {
|
|
487
|
+
const cfg = await AppBrain.getCrawlConfig(tenantId, projectId);
|
|
488
|
+
const existing = cfg?.seedUrls ?? [];
|
|
489
|
+
const merged = Array.from(/* @__PURE__ */ new Set([...existing, ...crawledUrls]));
|
|
490
|
+
if (merged.length > existing.length) {
|
|
491
|
+
await AppBrain.saveCrawlConfig(tenantId, projectId, { seedUrls: merged });
|
|
492
|
+
logger.info({ added: merged.length - existing.length, total: merged.length }, "[crawl] auto-seeded crawled URLs into crawlConfig.seedUrls");
|
|
493
|
+
}
|
|
494
|
+
})().catch((err) => logger.warn({ err: err?.message }, "[crawl] auto-seed seedUrls failed (non-fatal)"));
|
|
495
|
+
}
|
|
496
|
+
notifyAdmins(tenantId, {
|
|
497
|
+
type: "crawl.done",
|
|
498
|
+
title: "Crawl completed",
|
|
499
|
+
body: `Crawl for project finished successfully.`,
|
|
500
|
+
metadata: { projectId, jobId }
|
|
501
|
+
}).catch(console.error);
|
|
502
|
+
dispatchWebhook(tenantId, projectId, "crawl.done", {
|
|
503
|
+
jobId,
|
|
504
|
+
screensFound: result?.screensFound ?? result?.screens?.length ?? 0
|
|
505
|
+
}).catch(console.error);
|
|
506
|
+
}
|
|
507
|
+
setTimeout(() => {
|
|
508
|
+
aggregateJobTokens(tenantId, jobId, projectId, jobStartedAt).catch(console.error);
|
|
509
|
+
}, 8e3);
|
|
510
|
+
setImmediate(async () => {
|
|
511
|
+
try {
|
|
512
|
+
const projectMap = await AppBrain.getProjectMap(tenantId, projectId);
|
|
513
|
+
if (projectMap?.screens?.length) {
|
|
514
|
+
const screenCount = projectMap.screens.length;
|
|
515
|
+
if (screenCount > 300) {
|
|
516
|
+
await withTenant(
|
|
517
|
+
tenantId,
|
|
518
|
+
(tx) => tx.project.update({ where: { id: projectId }, data: { oversized: true } })
|
|
519
|
+
);
|
|
520
|
+
}
|
|
521
|
+
await generateTaxonomySkillFiles(tenantId, projectId, projectMap.screens, result.apiEndpoints, result.performanceMetrics);
|
|
522
|
+
try {
|
|
523
|
+
const { readFile } = await import("node:fs/promises");
|
|
524
|
+
const { existsSync } = await import("node:fs");
|
|
525
|
+
const screensWithDom = projectMap.screens.filter((s) => s.domSnapshotPath);
|
|
526
|
+
const detectLimit = Math.min(screensWithDom.length, 20);
|
|
527
|
+
let fieldDetectAdded = 0;
|
|
528
|
+
for (let si = 0; si < detectLimit; si++) {
|
|
529
|
+
const screen = screensWithDom[si];
|
|
530
|
+
try {
|
|
531
|
+
let htmlContent = null;
|
|
532
|
+
const domPath = screen.domSnapshotPath;
|
|
533
|
+
if (domPath.startsWith("https://") || domPath.startsWith("http://")) {
|
|
534
|
+
const res = await fetch(domPath);
|
|
535
|
+
if (res.ok) htmlContent = await res.text();
|
|
536
|
+
} else if (existsSync(domPath)) {
|
|
537
|
+
htmlContent = await readFile(domPath, "utf8");
|
|
538
|
+
}
|
|
539
|
+
if (!htmlContent) continue;
|
|
540
|
+
const detected = await detectFieldsFromHtml({ tenantId, projectId, html: htmlContent });
|
|
541
|
+
if (detected.length > 0) {
|
|
542
|
+
const merged = await mergeIntoCatalog({ tenantId, projectId, detected });
|
|
543
|
+
fieldDetectAdded += merged.added;
|
|
544
|
+
}
|
|
545
|
+
} catch (sErr) {
|
|
546
|
+
logger.warn({ screenId: screen.id, err: sErr }, "[field-detect] skipped screen");
|
|
547
|
+
}
|
|
556
548
|
}
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
const result = await crawlProject({
|
|
560
|
-
...config, credentials: runtimeCreds, abortSignal: controller.signal, jobId,
|
|
561
|
-
storageConfig: storageConfigForCrawl,
|
|
562
|
-
onCheckpoint: async (visitedUrls, pendingQueue) => {
|
|
563
|
-
jobCheckpoints.set(jobId, { visitedUrls, pendingQueue });
|
|
564
|
-
},
|
|
565
|
-
}, onProgress);
|
|
566
|
-
jobCheckpoints.delete(jobId);
|
|
567
|
-
const pendingUrls = result?.pendingUrls ?? [];
|
|
568
|
-
if (pendingUrls.length > 0) {
|
|
569
|
-
logger.info({ pendingUrls: pendingUrls.length }, 'Crawl finished with pending URLs — not auto-queuing (segmented crawls disabled)');
|
|
570
|
-
}
|
|
571
|
-
await withTenant(tenantId, (tx) => tx.platformJob.update({
|
|
572
|
-
where: { id: jobId },
|
|
573
|
-
data: {
|
|
574
|
-
status: 'DONE',
|
|
575
|
-
result: { ...result },
|
|
576
|
-
progress: 100,
|
|
577
|
-
finishedAt: new Date(),
|
|
578
|
-
},
|
|
579
|
-
}));
|
|
580
|
-
clearInterval(heartbeatTimer);
|
|
581
|
-
runningControllers.delete(jobId);
|
|
582
|
-
{
|
|
583
|
-
// Auto-seed: merge all successfully crawled URLs into crawlConfig.seedUrls so future
|
|
584
|
-
// crawls and discovery agents start from a complete picture of the app.
|
|
585
|
-
const crawledUrls = (result.discovered ?? [])
|
|
586
|
-
.filter((d) => !!d.screenId && !!d.url)
|
|
587
|
-
.map((d) => d.url);
|
|
588
|
-
if (crawledUrls.length > 0) {
|
|
589
|
-
(async () => {
|
|
590
|
-
const cfg = await AppBrain.getCrawlConfig(tenantId, projectId);
|
|
591
|
-
const existing = cfg?.seedUrls ?? [];
|
|
592
|
-
const merged = Array.from(new Set([...existing, ...crawledUrls]));
|
|
593
|
-
if (merged.length > existing.length) {
|
|
594
|
-
await AppBrain.saveCrawlConfig(tenantId, projectId, { seedUrls: merged });
|
|
595
|
-
logger.info({ added: merged.length - existing.length, total: merged.length }, '[crawl] auto-seeded crawled URLs into crawlConfig.seedUrls');
|
|
596
|
-
}
|
|
597
|
-
})().catch((err) => logger.warn({ err: err?.message }, '[crawl] auto-seed seedUrls failed (non-fatal)'));
|
|
549
|
+
if (fieldDetectAdded > 0 || detectLimit > 0) {
|
|
550
|
+
logger.info({ screens: detectLimit, added: fieldDetectAdded, projectId }, "[field-detect] catalog populated from crawl");
|
|
598
551
|
}
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
}
|
|
655
|
-
}
|
|
656
|
-
catch (sErr) {
|
|
657
|
-
logger.warn({ screenId: screen.id, err: sErr }, '[field-detect] skipped screen');
|
|
658
|
-
}
|
|
659
|
-
}
|
|
660
|
-
if (fieldDetectAdded > 0 || detectLimit > 0) {
|
|
661
|
-
logger.info({ screens: detectLimit, added: fieldDetectAdded, projectId }, '[field-detect] catalog populated from crawl');
|
|
662
|
-
}
|
|
663
|
-
}
|
|
664
|
-
catch (fdErr) {
|
|
665
|
-
logger.warn({ err: fdErr }, '[field-detect] post-crawl field detection failed');
|
|
666
|
-
}
|
|
667
|
-
// Cloud storage: upload screenshots to tenant S3/R2 if configured
|
|
668
|
-
const tenantStorageCfg = await getTenantStorageConfig(tenantId).catch(() => null);
|
|
669
|
-
if (tenantStorageCfg) {
|
|
670
|
-
let uploaded = 0;
|
|
671
|
-
for (const screen of projectMap.screens) {
|
|
672
|
-
for (const shot of (screen.screenshots ?? [])) {
|
|
673
|
-
if (!shot.storagePath || shot.cloudUrl)
|
|
674
|
-
continue;
|
|
675
|
-
try {
|
|
676
|
-
const filename = `${screen.id}-v${shot.version}.png`;
|
|
677
|
-
const key = buildArtifactKey(tenantId, projectId, 'screenshots', filename, tenantStorageCfg.prefix);
|
|
678
|
-
const cloudUrl = await uploadScreenshotToTenantStorage(tenantStorageCfg, shot.storagePath, key);
|
|
679
|
-
await AppBrain.updateScreenshotCloudUrl(tenantId, shot.id, cloudUrl);
|
|
680
|
-
uploaded++;
|
|
681
|
-
}
|
|
682
|
-
catch (uploadErr) {
|
|
683
|
-
logger.warn({ screenId: screen.id, err: uploadErr }, 'Failed to upload screenshot to cloud storage');
|
|
684
|
-
}
|
|
685
|
-
}
|
|
686
|
-
}
|
|
687
|
-
if (uploaded > 0)
|
|
688
|
-
logger.info({ uploaded, provider: tenantStorageCfg.provider }, 'Uploaded screenshots to cloud storage');
|
|
689
|
-
// Upload DOM snapshots and markdown to tenant storage
|
|
690
|
-
const { existsSync } = await import('node:fs');
|
|
691
|
-
let domUploaded = 0;
|
|
692
|
-
let mdUploaded = 0;
|
|
693
|
-
for (const screen of projectMap.screens) {
|
|
694
|
-
// DOM snapshot
|
|
695
|
-
const domPath = screen.domSnapshotPath;
|
|
696
|
-
if (domPath && !domPath.startsWith('https://') && existsSync(domPath)) {
|
|
697
|
-
try {
|
|
698
|
-
const key = buildArtifactKey(tenantId, projectId, 'dom-snapshots', `${screen.id}.json`, tenantStorageCfg.prefix);
|
|
699
|
-
const cloudUrl = await uploadFileToTenantStorage(tenantStorageCfg, domPath, key, 'application/json');
|
|
700
|
-
await AppBrain.updateScreenDomSnapshot(tenantId, screen.id, cloudUrl);
|
|
701
|
-
domUploaded++;
|
|
702
|
-
}
|
|
703
|
-
catch (err) {
|
|
704
|
-
logger.warn({ screenId: screen.id, err }, 'Failed to upload DOM snapshot to cloud storage');
|
|
705
|
-
}
|
|
706
|
-
}
|
|
707
|
-
// Markdown extract
|
|
708
|
-
const mdPath = screen.markdownPath;
|
|
709
|
-
if (mdPath && !mdPath.startsWith('https://') && existsSync(mdPath)) {
|
|
710
|
-
try {
|
|
711
|
-
const key = buildArtifactKey(tenantId, projectId, 'markdown', `${screen.id}.md`, tenantStorageCfg.prefix);
|
|
712
|
-
const cloudUrl = await uploadFileToTenantStorage(tenantStorageCfg, mdPath, key, 'text/markdown');
|
|
713
|
-
await AppBrain.updateScreenMarkdownPath(tenantId, screen.id, cloudUrl);
|
|
714
|
-
mdUploaded++;
|
|
715
|
-
}
|
|
716
|
-
catch (err) {
|
|
717
|
-
logger.warn({ screenId: screen.id, err }, 'Failed to upload markdown to cloud storage');
|
|
718
|
-
}
|
|
719
|
-
}
|
|
720
|
-
}
|
|
721
|
-
if (domUploaded > 0)
|
|
722
|
-
logger.info({ domUploaded, provider: tenantStorageCfg.provider }, 'Uploaded DOM snapshots to cloud storage');
|
|
723
|
-
if (mdUploaded > 0)
|
|
724
|
-
logger.info({ mdUploaded, provider: tenantStorageCfg.provider }, 'Uploaded markdown to cloud storage');
|
|
725
|
-
}
|
|
726
|
-
// AI screen grouping — clusters screens into semantic feature areas
|
|
727
|
-
await groupScreensWithAI(tenantId, projectId, projectMap.screens, jobId, jobStartedAt);
|
|
728
|
-
// Auto-flag stale tests: mark PENDING_REVIEW when screen elements changed since test was generated
|
|
729
|
-
try {
|
|
730
|
-
await withTenant(tenantId, async (tx) => {
|
|
731
|
-
const staleTests = await tx.$queryRaw `
|
|
552
|
+
} catch (fdErr) {
|
|
553
|
+
logger.warn({ err: fdErr }, "[field-detect] post-crawl field detection failed");
|
|
554
|
+
}
|
|
555
|
+
const tenantStorageCfg = await getTenantStorageConfig(tenantId).catch(() => null);
|
|
556
|
+
if (tenantStorageCfg) {
|
|
557
|
+
let uploaded = 0;
|
|
558
|
+
for (const screen of projectMap.screens) {
|
|
559
|
+
for (const shot of screen.screenshots ?? []) {
|
|
560
|
+
if (!shot.storagePath || shot.cloudUrl) continue;
|
|
561
|
+
try {
|
|
562
|
+
const filename = `${screen.id}-v${shot.version}.png`;
|
|
563
|
+
const key = buildArtifactKey(tenantId, projectId, "screenshots", filename, tenantStorageCfg.prefix);
|
|
564
|
+
const cloudUrl = await uploadScreenshotToTenantStorage(tenantStorageCfg, shot.storagePath, key);
|
|
565
|
+
await AppBrain.updateScreenshotCloudUrl(tenantId, shot.id, cloudUrl);
|
|
566
|
+
uploaded++;
|
|
567
|
+
} catch (uploadErr) {
|
|
568
|
+
logger.warn({ screenId: screen.id, err: uploadErr }, "Failed to upload screenshot to cloud storage");
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
}
|
|
572
|
+
if (uploaded > 0) logger.info({ uploaded, provider: tenantStorageCfg.provider }, "Uploaded screenshots to cloud storage");
|
|
573
|
+
const { existsSync } = await import("node:fs");
|
|
574
|
+
let domUploaded = 0;
|
|
575
|
+
let mdUploaded = 0;
|
|
576
|
+
for (const screen of projectMap.screens) {
|
|
577
|
+
const domPath = screen.domSnapshotPath;
|
|
578
|
+
if (domPath && !domPath.startsWith("https://") && existsSync(domPath)) {
|
|
579
|
+
try {
|
|
580
|
+
const key = buildArtifactKey(tenantId, projectId, "dom-snapshots", `${screen.id}.json`, tenantStorageCfg.prefix);
|
|
581
|
+
const cloudUrl = await uploadFileToTenantStorage(tenantStorageCfg, domPath, key, "application/json");
|
|
582
|
+
await AppBrain.updateScreenDomSnapshot(tenantId, screen.id, cloudUrl);
|
|
583
|
+
domUploaded++;
|
|
584
|
+
} catch (err) {
|
|
585
|
+
logger.warn({ screenId: screen.id, err }, "Failed to upload DOM snapshot to cloud storage");
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
const mdPath = screen.markdownPath;
|
|
589
|
+
if (mdPath && !mdPath.startsWith("https://") && existsSync(mdPath)) {
|
|
590
|
+
try {
|
|
591
|
+
const key = buildArtifactKey(tenantId, projectId, "markdown", `${screen.id}.md`, tenantStorageCfg.prefix);
|
|
592
|
+
const cloudUrl = await uploadFileToTenantStorage(tenantStorageCfg, mdPath, key, "text/markdown");
|
|
593
|
+
await AppBrain.updateScreenMarkdownPath(tenantId, screen.id, cloudUrl);
|
|
594
|
+
mdUploaded++;
|
|
595
|
+
} catch (err) {
|
|
596
|
+
logger.warn({ screenId: screen.id, err }, "Failed to upload markdown to cloud storage");
|
|
597
|
+
}
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
if (domUploaded > 0) logger.info({ domUploaded, provider: tenantStorageCfg.provider }, "Uploaded DOM snapshots to cloud storage");
|
|
601
|
+
if (mdUploaded > 0) logger.info({ mdUploaded, provider: tenantStorageCfg.provider }, "Uploaded markdown to cloud storage");
|
|
602
|
+
}
|
|
603
|
+
await groupScreensWithAI(tenantId, projectId, projectMap.screens, jobId, jobStartedAt);
|
|
604
|
+
try {
|
|
605
|
+
await withTenant(tenantId, async (tx) => {
|
|
606
|
+
const staleTests = await tx.$queryRaw`
|
|
732
607
|
SELECT tc.id
|
|
733
608
|
FROM "TestCase" tc
|
|
734
609
|
JOIN "Screen" s ON tc."screenId" = s.id
|
|
@@ -740,69 +615,65 @@ export async function processJob(jobId, config, opts) {
|
|
|
740
615
|
AND tc.status != 'PENDING_REVIEW'
|
|
741
616
|
AND tc.status != 'RETIRED'
|
|
742
617
|
`;
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
}
|
|
760
|
-
}
|
|
761
|
-
}
|
|
762
|
-
catch (skillErr) {
|
|
763
|
-
logger.error({ projectId, err: skillErr }, 'Taxonomy generation failed');
|
|
764
|
-
}
|
|
765
|
-
});
|
|
766
|
-
}
|
|
767
|
-
catch (err) {
|
|
768
|
-
clearInterval(heartbeatTimer);
|
|
769
|
-
runningControllers.delete(jobId);
|
|
770
|
-
const aborted = String(err).includes('CRAWL_ABORTED');
|
|
771
|
-
const checkpoint = jobCheckpoints.get(jobId);
|
|
772
|
-
jobCheckpoints.delete(jobId);
|
|
773
|
-
await withTenant(tenantId, (tx) => tx.platformJob.update({
|
|
774
|
-
where: { id: jobId },
|
|
775
|
-
data: {
|
|
776
|
-
status: aborted ? 'CANCELLED' : 'FAILED',
|
|
777
|
-
error: aborted ? 'Stopped by user' : String(err),
|
|
778
|
-
finishedAt: new Date(),
|
|
779
|
-
...(checkpoint ? { result: { checkpoint } } : {}),
|
|
780
|
-
},
|
|
781
|
-
}));
|
|
782
|
-
// Notify on crawl failure (fire-and-forget)
|
|
783
|
-
if (!aborted) {
|
|
784
|
-
notifyAdmins(tenantId, {
|
|
785
|
-
type: 'crawl.failed',
|
|
786
|
-
title: 'Crawl failed',
|
|
787
|
-
body: `Crawl for project failed: ${String(err)}`,
|
|
788
|
-
metadata: { projectId, jobId, error: String(err) },
|
|
789
|
-
}).catch(console.error);
|
|
790
|
-
// Dispatch outbound webhooks (fire-and-forget)
|
|
791
|
-
dispatchWebhook(tenantId, projectId, 'crawl.failed', {
|
|
792
|
-
jobId,
|
|
793
|
-
error: String(err),
|
|
794
|
-
}).catch(console.error);
|
|
618
|
+
if (staleTests.length > 0) {
|
|
619
|
+
const ids = staleTests.map((t) => t.id);
|
|
620
|
+
await tx.testCase.updateMany({
|
|
621
|
+
where: { id: { in: ids } },
|
|
622
|
+
data: { status: "PENDING_REVIEW" }
|
|
623
|
+
});
|
|
624
|
+
logger.info({ count: staleTests.length, projectId }, "Auto-flagged stale test cases as PENDING_REVIEW");
|
|
625
|
+
}
|
|
626
|
+
});
|
|
627
|
+
} catch (staleErr) {
|
|
628
|
+
logger.warn({ err: staleErr }, "Failed to auto-flag stale tests");
|
|
629
|
+
}
|
|
630
|
+
const changed = projectMap.screens.filter((s) => s.hasChange);
|
|
631
|
+
if (changed.length > 0) {
|
|
632
|
+
await investigateScreenChanges(tenantId, projectId, changed);
|
|
633
|
+
}
|
|
795
634
|
}
|
|
635
|
+
} catch (skillErr) {
|
|
636
|
+
logger.error({ projectId, err: skillErr }, "Taxonomy generation failed");
|
|
637
|
+
}
|
|
638
|
+
});
|
|
639
|
+
} catch (err) {
|
|
640
|
+
clearInterval(heartbeatTimer);
|
|
641
|
+
runningControllers.delete(jobId);
|
|
642
|
+
const aborted = String(err).includes("CRAWL_ABORTED");
|
|
643
|
+
const checkpoint = jobCheckpoints.get(jobId);
|
|
644
|
+
jobCheckpoints.delete(jobId);
|
|
645
|
+
await withTenant(
|
|
646
|
+
tenantId,
|
|
647
|
+
(tx) => tx.platformJob.update({
|
|
648
|
+
where: { id: jobId },
|
|
649
|
+
data: {
|
|
650
|
+
status: aborted ? "CANCELLED" : "FAILED",
|
|
651
|
+
error: aborted ? "Stopped by user" : String(err),
|
|
652
|
+
finishedAt: /* @__PURE__ */ new Date(),
|
|
653
|
+
...checkpoint ? { result: { checkpoint } } : {}
|
|
654
|
+
}
|
|
655
|
+
})
|
|
656
|
+
);
|
|
657
|
+
if (!aborted) {
|
|
658
|
+
notifyAdmins(tenantId, {
|
|
659
|
+
type: "crawl.failed",
|
|
660
|
+
title: "Crawl failed",
|
|
661
|
+
body: `Crawl for project failed: ${String(err)}`,
|
|
662
|
+
metadata: { projectId, jobId, error: String(err) }
|
|
663
|
+
}).catch(console.error);
|
|
664
|
+
dispatchWebhook(tenantId, projectId, "crawl.failed", {
|
|
665
|
+
jobId,
|
|
666
|
+
error: String(err)
|
|
667
|
+
}).catch(console.error);
|
|
796
668
|
}
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
669
|
+
} finally {
|
|
670
|
+
runningControllers.delete(jobId);
|
|
671
|
+
}
|
|
800
672
|
}
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
const rows = await tx.$queryRaw `
|
|
673
|
+
async function aggregateJobTokens(tenantId, jobId, projectId, startedAt) {
|
|
674
|
+
try {
|
|
675
|
+
await withTenant(tenantId, async (tx) => {
|
|
676
|
+
const rows = await tx.$queryRaw`
|
|
806
677
|
SELECT
|
|
807
678
|
COALESCE(SUM("promptTokens"), 0)::INT AS "totalPromptTokens",
|
|
808
679
|
COALESCE(SUM("completionTokens"), 0)::INT AS "totalCompletionTokens",
|
|
@@ -813,252 +684,297 @@ export async function aggregateJobTokens(tenantId, jobId, projectId, startedAt)
|
|
|
813
684
|
WHERE "tenantId" = ${tenantId}
|
|
814
685
|
AND "jobId" = ${jobId}
|
|
815
686
|
`;
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
console.warn('[job-tokens] aggregation failed:', err);
|
|
833
|
-
}
|
|
687
|
+
const row = rows[0];
|
|
688
|
+
if (!row || !row.llmCallCount) return;
|
|
689
|
+
await tx.platformJob.update({
|
|
690
|
+
where: { id: jobId },
|
|
691
|
+
data: {
|
|
692
|
+
totalPromptTokens: row.totalPromptTokens,
|
|
693
|
+
totalCompletionTokens: row.totalCompletionTokens,
|
|
694
|
+
totalCacheReadTokens: row.totalCacheReadTokens,
|
|
695
|
+
totalEstimatedCostUsd: Number(row.totalEstimatedCostUsd),
|
|
696
|
+
llmCallCount: row.llmCallCount
|
|
697
|
+
}
|
|
698
|
+
});
|
|
699
|
+
});
|
|
700
|
+
} catch (err) {
|
|
701
|
+
console.warn("[job-tokens] aggregation failed:", err);
|
|
702
|
+
}
|
|
834
703
|
}
|
|
835
704
|
async function processApiTestsJob(jobId, config) {
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
}
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
705
|
+
const { tenantId, projectId } = config;
|
|
706
|
+
const jobStartedAt = /* @__PURE__ */ new Date();
|
|
707
|
+
await withTenant(
|
|
708
|
+
tenantId,
|
|
709
|
+
(tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "RUNNING", startedAt: jobStartedAt } })
|
|
710
|
+
);
|
|
711
|
+
try {
|
|
712
|
+
const { generateApiTests, refreshCoverageMatrix, refreshTestCaseInventory } = await import("@detiq/agents");
|
|
713
|
+
await updateJobProgress(tenantId, jobId, 20, "Loading API contracts and endpoints\u2026");
|
|
714
|
+
const result = await generateApiTests({ tenantId, projectId, apiEndpoints: config.apiEndpoints });
|
|
715
|
+
await updateJobProgress(tenantId, jobId, 90, `Generated ${result.cases?.length ?? 0} API test cases`);
|
|
716
|
+
refreshCoverageMatrix(tenantId, projectId).catch((e) => console.warn("[coverage-matrix]", e));
|
|
717
|
+
refreshTestCaseInventory(tenantId, projectId).catch((e) => console.warn("[tc-inventory]", e));
|
|
718
|
+
refreshTestCasesSkillFile(tenantId, projectId).catch(console.error);
|
|
719
|
+
await withTenant(
|
|
720
|
+
tenantId,
|
|
721
|
+
(tx) => tx.platformJob.update({
|
|
722
|
+
where: { id: jobId },
|
|
723
|
+
data: {
|
|
724
|
+
status: "DONE",
|
|
725
|
+
progress: 100,
|
|
726
|
+
finishedAt: /* @__PURE__ */ new Date(),
|
|
727
|
+
result: { cases: result.cases?.length ?? 0, suiteName: result.suiteName }
|
|
728
|
+
}
|
|
729
|
+
})
|
|
730
|
+
);
|
|
731
|
+
aggregateJobTokens(tenantId, jobId, projectId, jobStartedAt).catch(console.error);
|
|
732
|
+
} catch (err) {
|
|
733
|
+
await withTenant(
|
|
734
|
+
tenantId,
|
|
735
|
+
(tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", error: String(err), finishedAt: /* @__PURE__ */ new Date() } })
|
|
736
|
+
);
|
|
737
|
+
}
|
|
859
738
|
}
|
|
860
739
|
async function processSecurityTestsJob(jobId, config) {
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
}
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
740
|
+
const { tenantId, projectId } = config;
|
|
741
|
+
const jobStartedAt = /* @__PURE__ */ new Date();
|
|
742
|
+
await withTenant(
|
|
743
|
+
tenantId,
|
|
744
|
+
(tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "RUNNING", startedAt: jobStartedAt } })
|
|
745
|
+
);
|
|
746
|
+
try {
|
|
747
|
+
const { generateSecurityTests, refreshCoverageMatrix, refreshTestCaseInventory } = await import("@detiq/agents");
|
|
748
|
+
await updateJobProgress(tenantId, jobId, 20, "Mapping OWASP attack surface\u2026");
|
|
749
|
+
const result = await generateSecurityTests({ tenantId, projectId, apiEndpoints: config.apiEndpoints });
|
|
750
|
+
await updateJobProgress(tenantId, jobId, 90, `Generated ${result.cases?.length ?? 0} security test cases`);
|
|
751
|
+
refreshCoverageMatrix(tenantId, projectId).catch((e) => console.warn("[coverage-matrix]", e));
|
|
752
|
+
refreshTestCaseInventory(tenantId, projectId).catch((e) => console.warn("[tc-inventory]", e));
|
|
753
|
+
refreshTestCasesSkillFile(tenantId, projectId).catch(console.error);
|
|
754
|
+
await withTenant(
|
|
755
|
+
tenantId,
|
|
756
|
+
(tx) => tx.platformJob.update({
|
|
757
|
+
where: { id: jobId },
|
|
758
|
+
data: {
|
|
759
|
+
status: "DONE",
|
|
760
|
+
progress: 100,
|
|
761
|
+
finishedAt: /* @__PURE__ */ new Date(),
|
|
762
|
+
result: { cases: result.cases?.length ?? 0, suiteName: result.suiteName }
|
|
763
|
+
}
|
|
764
|
+
})
|
|
765
|
+
);
|
|
766
|
+
aggregateJobTokens(tenantId, jobId, projectId, jobStartedAt).catch(console.error);
|
|
767
|
+
} catch (err) {
|
|
768
|
+
await withTenant(
|
|
769
|
+
tenantId,
|
|
770
|
+
(tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", error: String(err), finishedAt: /* @__PURE__ */ new Date() } })
|
|
771
|
+
);
|
|
772
|
+
}
|
|
884
773
|
}
|
|
885
774
|
async function processIntegrationTestsJob(jobId, config) {
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
}
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
775
|
+
const { tenantId, projectId } = config;
|
|
776
|
+
const jobStartedAt = /* @__PURE__ */ new Date();
|
|
777
|
+
await withTenant(
|
|
778
|
+
tenantId,
|
|
779
|
+
(tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "RUNNING", startedAt: jobStartedAt } })
|
|
780
|
+
);
|
|
781
|
+
try {
|
|
782
|
+
const { generateIntegrationTests, refreshCoverageMatrix, refreshTestCaseInventory } = await import("@detiq/agents");
|
|
783
|
+
await updateJobProgress(tenantId, jobId, 20, "Analysing API contracts and data models\u2026");
|
|
784
|
+
const result = await generateIntegrationTests({ tenantId, projectId, apiEndpoints: config.apiEndpoints });
|
|
785
|
+
await updateJobProgress(tenantId, jobId, 90, `Generated ${result.cases?.length ?? 0} integration test cases`);
|
|
786
|
+
refreshCoverageMatrix(tenantId, projectId).catch((e) => console.warn("[coverage-matrix]", e));
|
|
787
|
+
refreshTestCaseInventory(tenantId, projectId).catch((e) => console.warn("[tc-inventory]", e));
|
|
788
|
+
refreshTestCasesSkillFile(tenantId, projectId).catch(console.error);
|
|
789
|
+
await withTenant(
|
|
790
|
+
tenantId,
|
|
791
|
+
(tx) => tx.platformJob.update({
|
|
792
|
+
where: { id: jobId },
|
|
793
|
+
data: {
|
|
794
|
+
status: "DONE",
|
|
795
|
+
progress: 100,
|
|
796
|
+
finishedAt: /* @__PURE__ */ new Date(),
|
|
797
|
+
result: { cases: result.cases?.length ?? 0, suiteName: result.suiteName }
|
|
798
|
+
}
|
|
799
|
+
})
|
|
800
|
+
);
|
|
801
|
+
aggregateJobTokens(tenantId, jobId, projectId, jobStartedAt).catch(console.error);
|
|
802
|
+
} catch (err) {
|
|
803
|
+
await withTenant(
|
|
804
|
+
tenantId,
|
|
805
|
+
(tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", error: String(err), finishedAt: /* @__PURE__ */ new Date() } })
|
|
806
|
+
);
|
|
807
|
+
}
|
|
909
808
|
}
|
|
910
809
|
async function processFigmaImportJob(jobId, config) {
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
if (frame.node) {
|
|
966
|
-
const figmaElements = figma.extractElements(frame.node);
|
|
967
|
-
for (const el of figmaElements) {
|
|
968
|
-
const role = el.type === 'COMPONENT' || el.type === 'COMPONENT_SET' ? 'COMPONENT'
|
|
969
|
-
: el.type === 'INSTANCE' ? 'COMPONENT'
|
|
970
|
-
: el.type === 'TEXT' ? 'TEXT'
|
|
971
|
-
: el.type === 'FRAME' || el.type === 'GROUP' ? 'REGION'
|
|
972
|
-
: el.type;
|
|
973
|
-
const boundingRect = el.boundingBox
|
|
974
|
-
? { x: Math.round(el.boundingBox.x), y: Math.round(el.boundingBox.y), width: Math.round(el.boundingBox.width), height: Math.round(el.boundingBox.height) }
|
|
975
|
-
: undefined;
|
|
976
|
-
await AppBrain.saveElement(tenantId, projectId, screen.id, el.name, role, undefined, el.description ?? undefined, undefined, el.isInteractive, boundingRect, undefined, undefined);
|
|
977
|
-
}
|
|
978
|
-
}
|
|
979
|
-
saved++;
|
|
980
|
-
}
|
|
981
|
-
catch (err) {
|
|
982
|
-
console.warn(`[figma-import] Failed to save frame:`, err);
|
|
810
|
+
const { tenantId, projectId, fileKey } = config;
|
|
811
|
+
await withTenant(
|
|
812
|
+
tenantId,
|
|
813
|
+
(tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "RUNNING", startedAt: /* @__PURE__ */ new Date() } })
|
|
814
|
+
);
|
|
815
|
+
try {
|
|
816
|
+
const connector = await withTenant(
|
|
817
|
+
tenantId,
|
|
818
|
+
(tx) => tx.connector.findFirst({ where: { projectId, kind: "FIGMA", tenantId } })
|
|
819
|
+
);
|
|
820
|
+
if (!connector) throw new Error("Figma connector not configured. Add your Figma token in Project Settings \u2192 Integrations.");
|
|
821
|
+
const cfg = connector.config;
|
|
822
|
+
const accessToken = cfg.accessToken_enc ? decrypt(cfg.accessToken_enc) : "";
|
|
823
|
+
if (!accessToken) throw new Error("Figma access token missing. Edit the Figma connector and add your Personal Access Token.");
|
|
824
|
+
const resolvedFileKey = fileKey ?? cfg.fileKey ?? "";
|
|
825
|
+
if (!resolvedFileKey) throw new Error("No Figma file key. Paste the full Figma file URL as the App URL.");
|
|
826
|
+
const creds = { accessToken, fileKey: resolvedFileKey };
|
|
827
|
+
await updateJobProgress(tenantId, jobId, 5, "Listing frames from Figma file\u2026");
|
|
828
|
+
const frames = await figma.listScreens(creds, resolvedFileKey);
|
|
829
|
+
if (frames.length === 0) {
|
|
830
|
+
await withTenant(
|
|
831
|
+
tenantId,
|
|
832
|
+
(tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "DONE", progress: 100, result: { screensImported: 0 }, finishedAt: /* @__PURE__ */ new Date() } })
|
|
833
|
+
);
|
|
834
|
+
return;
|
|
835
|
+
}
|
|
836
|
+
await updateJobProgress(tenantId, jobId, 15, `Fetching images for ${frames.length} frames\u2026`);
|
|
837
|
+
const nodeIds = frames.map((f) => f.id);
|
|
838
|
+
const imageUrls = await figma.getScreenImages(creds, resolvedFileKey, nodeIds, 1);
|
|
839
|
+
const screenshotDir = path.join(platformConfig.screenshotDir, projectId);
|
|
840
|
+
await fs.mkdir(screenshotDir, { recursive: true });
|
|
841
|
+
const tenantStorageCfg = await getTenantStorageConfig(tenantId).catch(() => null);
|
|
842
|
+
let saved = 0;
|
|
843
|
+
for (const frame of frames) {
|
|
844
|
+
try {
|
|
845
|
+
const pct = Math.round(15 + saved / frames.length * 80);
|
|
846
|
+
await updateJobProgress(tenantId, jobId, pct, `Saving frame ${saved + 1}/${frames.length}: ${frame.name}`);
|
|
847
|
+
const screen = await AppBrain.saveScreen(tenantId, projectId, frame.name, `figma://file/${resolvedFileKey}/node/${frame.id}`);
|
|
848
|
+
const imgUrl = imageUrls[frame.id];
|
|
849
|
+
if (imgUrl) {
|
|
850
|
+
const imgRes = await fetch(imgUrl);
|
|
851
|
+
if (imgRes.ok) {
|
|
852
|
+
const buf = Buffer.from(await imgRes.arrayBuffer());
|
|
853
|
+
const filename = `${screen.id}-v1.png`;
|
|
854
|
+
const storagePath = path.join(screenshotDir, filename);
|
|
855
|
+
await fs.writeFile(storagePath, buf);
|
|
856
|
+
let cloudUrl;
|
|
857
|
+
if (tenantStorageCfg) {
|
|
858
|
+
try {
|
|
859
|
+
const key = buildArtifactKey(tenantId, projectId, "screenshots", filename, tenantStorageCfg.prefix);
|
|
860
|
+
cloudUrl = await uploadScreenshotToTenantStorage(tenantStorageCfg, storagePath, key) ?? void 0;
|
|
861
|
+
} catch (e) {
|
|
862
|
+
console.warn("[figma-import] cloud upload failed:", e);
|
|
863
|
+
}
|
|
983
864
|
}
|
|
865
|
+
await AppBrain.saveScreenshot(tenantId, projectId, screen.id, storagePath, 1, "DESKTOP", cloudUrl);
|
|
866
|
+
}
|
|
984
867
|
}
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
868
|
+
if (frame.node) {
|
|
869
|
+
const figmaElements = figma.extractElements(frame.node);
|
|
870
|
+
for (const el of figmaElements) {
|
|
871
|
+
const role = el.type === "COMPONENT" || el.type === "COMPONENT_SET" ? "COMPONENT" : el.type === "INSTANCE" ? "COMPONENT" : el.type === "TEXT" ? "TEXT" : el.type === "FRAME" || el.type === "GROUP" ? "REGION" : el.type;
|
|
872
|
+
const boundingRect = el.boundingBox ? { x: Math.round(el.boundingBox.x), y: Math.round(el.boundingBox.y), width: Math.round(el.boundingBox.width), height: Math.round(el.boundingBox.height) } : void 0;
|
|
873
|
+
await AppBrain.saveElement(
|
|
874
|
+
tenantId,
|
|
875
|
+
projectId,
|
|
876
|
+
screen.id,
|
|
877
|
+
el.name,
|
|
878
|
+
role,
|
|
879
|
+
void 0,
|
|
880
|
+
el.description ?? void 0,
|
|
881
|
+
void 0,
|
|
882
|
+
el.isInteractive,
|
|
883
|
+
boundingRect,
|
|
884
|
+
void 0,
|
|
885
|
+
void 0
|
|
886
|
+
);
|
|
887
|
+
}
|
|
888
|
+
}
|
|
889
|
+
saved++;
|
|
890
|
+
} catch (err) {
|
|
891
|
+
console.warn(`[figma-import] Failed to save frame:`, err);
|
|
892
|
+
}
|
|
1005
893
|
}
|
|
894
|
+
await withTenant(
|
|
895
|
+
tenantId,
|
|
896
|
+
(tx) => tx.platformJob.update({
|
|
897
|
+
where: { id: jobId },
|
|
898
|
+
data: { status: "DONE", progress: 100, result: { screensImported: saved, totalFrames: frames.length }, finishedAt: /* @__PURE__ */ new Date() }
|
|
899
|
+
})
|
|
900
|
+
);
|
|
901
|
+
setImmediate(async () => {
|
|
902
|
+
try {
|
|
903
|
+
const projectMap = await AppBrain.getProjectMap(tenantId, projectId);
|
|
904
|
+
if (projectMap?.screens?.length) {
|
|
905
|
+
await generateTaxonomySkillFiles(tenantId, projectId, projectMap.screens);
|
|
906
|
+
await groupScreensWithAI(tenantId, projectId, projectMap.screens, jobId, /* @__PURE__ */ new Date());
|
|
907
|
+
}
|
|
908
|
+
} catch (err) {
|
|
909
|
+
console.error("[figma-import] post-import grouping failed:", err);
|
|
910
|
+
}
|
|
911
|
+
});
|
|
912
|
+
} catch (err) {
|
|
913
|
+
await withTenant(
|
|
914
|
+
tenantId,
|
|
915
|
+
(tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", error: String(err?.message ?? err), finishedAt: /* @__PURE__ */ new Date() } })
|
|
916
|
+
);
|
|
917
|
+
}
|
|
1006
918
|
}
|
|
1007
919
|
async function processNativeCrawlJob(jobId, config) {
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
}
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
920
|
+
const { tenantId, projectId } = config;
|
|
921
|
+
const jobStartedAt = /* @__PURE__ */ new Date();
|
|
922
|
+
try {
|
|
923
|
+
const build = await withTenant(
|
|
924
|
+
tenantId,
|
|
925
|
+
(tx) => tx.nativeAppBuild.findFirst({
|
|
926
|
+
where: { tenantId, projectId },
|
|
927
|
+
orderBy: { uploadedAt: "desc" }
|
|
928
|
+
})
|
|
929
|
+
);
|
|
930
|
+
if (!build) throw new Error("No native app build found for project");
|
|
931
|
+
if (build.fileFormat === "aab") {
|
|
932
|
+
throw new Error(
|
|
933
|
+
"AAB format cannot be installed directly by Appium. Convert to APK using bundletool first, then re-upload the APK."
|
|
934
|
+
);
|
|
935
|
+
}
|
|
936
|
+
const deviceFarm = config.deviceFarm ?? "local_emulator";
|
|
937
|
+
await withTenant(
|
|
938
|
+
tenantId,
|
|
939
|
+
(tx) => tx.platformJob.update({
|
|
940
|
+
where: { id: jobId },
|
|
941
|
+
data: {
|
|
942
|
+
deviceModel: config.deviceModel ?? null,
|
|
943
|
+
osVersion: config.osVersion ?? null,
|
|
944
|
+
deviceFarm
|
|
1020
945
|
}
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
const projectMap = await AppBrain.getProjectMap(tenantId, projectId);
|
|
1054
|
-
const screens = (projectMap?.screens ?? []);
|
|
1055
|
-
if (screens.length > 0) {
|
|
1056
|
-
await generateTaxonomySkillFiles(tenantId, projectId, screens);
|
|
1057
|
-
await groupScreensWithAI(tenantId, projectId, screens, jobId, jobStartedAt);
|
|
1058
|
-
// Auto-flag stale tests: mark PENDING_REVIEW when screen elements changed since test was generated
|
|
1059
|
-
try {
|
|
1060
|
-
await withTenant(tenantId, async (tx) => {
|
|
1061
|
-
const staleTests = await tx.$queryRaw `
|
|
946
|
+
})
|
|
947
|
+
).catch(() => {
|
|
948
|
+
});
|
|
949
|
+
const workspaceDeviceFarm = await AppBrain.getTenantDeviceFarmConfig(tenantId).catch(() => null);
|
|
950
|
+
await runNativeCrawl({
|
|
951
|
+
tenantId,
|
|
952
|
+
projectId,
|
|
953
|
+
jobId,
|
|
954
|
+
platform: build.platform,
|
|
955
|
+
appS3Path: build.s3Path,
|
|
956
|
+
packageName: build.packageName ?? void 0,
|
|
957
|
+
deviceModel: config.deviceModel,
|
|
958
|
+
osVersion: config.osVersion,
|
|
959
|
+
deviceFarm,
|
|
960
|
+
appiumServerUrl: workspaceDeviceFarm?.appiumServerUrl,
|
|
961
|
+
browserstackUsername: workspaceDeviceFarm?.browserstackUsername,
|
|
962
|
+
browserstackAccessKey: workspaceDeviceFarm?.browserstackAccessKey,
|
|
963
|
+
awsAccessKeyId: workspaceDeviceFarm?.awsAccessKeyId,
|
|
964
|
+
awsSecretAccessKey: workspaceDeviceFarm?.awsSecretAccessKey,
|
|
965
|
+
awsRegion: workspaceDeviceFarm?.awsRegion,
|
|
966
|
+
awsDeviceFarmProjectArn: workspaceDeviceFarm?.awsDeviceFarmProjectArn
|
|
967
|
+
});
|
|
968
|
+
setImmediate(async () => {
|
|
969
|
+
try {
|
|
970
|
+
const projectMap = await AppBrain.getProjectMap(tenantId, projectId);
|
|
971
|
+
const screens = projectMap?.screens ?? [];
|
|
972
|
+
if (screens.length > 0) {
|
|
973
|
+
await generateTaxonomySkillFiles(tenantId, projectId, screens);
|
|
974
|
+
await groupScreensWithAI(tenantId, projectId, screens, jobId, jobStartedAt);
|
|
975
|
+
try {
|
|
976
|
+
await withTenant(tenantId, async (tx) => {
|
|
977
|
+
const staleTests = await tx.$queryRaw`
|
|
1062
978
|
SELECT tc.id
|
|
1063
979
|
FROM "TestCase" tc
|
|
1064
980
|
JOIN "Screen" s ON tc."screenId" = s.id
|
|
@@ -1070,1296 +986,1362 @@ async function processNativeCrawlJob(jobId, config) {
|
|
|
1070
986
|
AND tc.status != 'PENDING_REVIEW'
|
|
1071
987
|
AND tc.status != 'RETIRED'
|
|
1072
988
|
`;
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
});
|
|
1082
|
-
}
|
|
1083
|
-
catch (staleErr) {
|
|
1084
|
-
logger.warn({ err: staleErr }, 'Failed to auto-flag stale tests');
|
|
1085
|
-
}
|
|
1086
|
-
}
|
|
1087
|
-
}
|
|
1088
|
-
catch (e) {
|
|
1089
|
-
logger.error({ err: e }, 'post-native-crawl processing failed');
|
|
1090
|
-
}
|
|
1091
|
-
});
|
|
1092
|
-
}
|
|
1093
|
-
catch (err) {
|
|
1094
|
-
await withTenant(tenantId, (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', error: String(err?.message ?? err), finishedAt: new Date() } })).catch(() => { });
|
|
1095
|
-
}
|
|
1096
|
-
}
|
|
1097
|
-
async function processGenerateTestsFromReqsJob(jobId, config) {
|
|
1098
|
-
const { tenantId, projectId, reqIds, suiteId, suiteName } = config;
|
|
1099
|
-
try {
|
|
1100
|
-
await updateJobProgress(tenantId, jobId, 5, 'Loading requirements');
|
|
1101
|
-
const { askTracked, deterministicSeed, getTaskAgentLLMConfig } = await import('@detiq/agents');
|
|
1102
|
-
const llmCfg = await getTaskAgentLLMConfig(tenantId, 'testGeneration');
|
|
1103
|
-
const all = await AppBrain.listRequirements(tenantId, projectId, {});
|
|
1104
|
-
const reqs = reqIds?.length
|
|
1105
|
-
? all.filter((r) => reqIds.includes(r.reqId))
|
|
1106
|
-
: all.filter((r) => r.status === 'unbuilt' || r.status === 'pending');
|
|
1107
|
-
if (reqs.length === 0) {
|
|
1108
|
-
await updateJobProgress(tenantId, jobId, 100, 'No pending requirements found');
|
|
1109
|
-
await withTenant(tenantId, (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: 'DONE', finishedAt: new Date(), result: { created: 0, skipped: 0 } } }));
|
|
1110
|
-
return;
|
|
1111
|
-
}
|
|
1112
|
-
// Resolve or create suite
|
|
1113
|
-
let resolvedSuiteId = suiteId;
|
|
1114
|
-
if (!resolvedSuiteId) {
|
|
1115
|
-
const name = suiteName?.trim() || 'Requirements Tests';
|
|
1116
|
-
const suite = await withTenant(tenantId, async (tx) => {
|
|
1117
|
-
const existing = await tx.testSuite.findFirst({ where: { tenantId, projectId, name } });
|
|
1118
|
-
if (existing)
|
|
1119
|
-
return existing;
|
|
1120
|
-
return tx.testSuite.create({ data: { tenantId, projectId, name } });
|
|
989
|
+
if (staleTests.length > 0) {
|
|
990
|
+
const ids = staleTests.map((t) => t.id);
|
|
991
|
+
await tx.testCase.updateMany({
|
|
992
|
+
where: { id: { in: ids } },
|
|
993
|
+
data: { status: "PENDING_REVIEW" }
|
|
994
|
+
});
|
|
995
|
+
logger.info({ count: staleTests.length, projectId }, "Auto-flagged stale test cases as PENDING_REVIEW");
|
|
996
|
+
}
|
|
1121
997
|
});
|
|
1122
|
-
|
|
998
|
+
} catch (staleErr) {
|
|
999
|
+
logger.warn({ err: staleErr }, "Failed to auto-flag stale tests");
|
|
1000
|
+
}
|
|
1123
1001
|
}
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1002
|
+
} catch (e) {
|
|
1003
|
+
logger.error({ err: e }, "post-native-crawl processing failed");
|
|
1004
|
+
}
|
|
1005
|
+
});
|
|
1006
|
+
} catch (err) {
|
|
1007
|
+
await withTenant(
|
|
1008
|
+
tenantId,
|
|
1009
|
+
(tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", error: String(err?.message ?? err), finishedAt: /* @__PURE__ */ new Date() } })
|
|
1010
|
+
).catch(() => {
|
|
1011
|
+
});
|
|
1012
|
+
}
|
|
1013
|
+
}
|
|
1014
|
+
async function processGenerateTestsFromReqsJob(jobId, config) {
|
|
1015
|
+
const { tenantId, projectId, reqIds, suiteId, suiteName } = config;
|
|
1016
|
+
try {
|
|
1017
|
+
await updateJobProgress(tenantId, jobId, 5, "Loading requirements");
|
|
1018
|
+
const { askTracked, deterministicSeed, getTaskAgentLLMConfig } = await import("@detiq/agents");
|
|
1019
|
+
const llmCfg = await getTaskAgentLLMConfig(tenantId, "testGeneration");
|
|
1020
|
+
const all = await AppBrain.listRequirements(tenantId, projectId, {});
|
|
1021
|
+
const reqs = reqIds?.length ? all.filter((r) => reqIds.includes(r.reqId)) : all.filter((r) => r.status === "unbuilt" || r.status === "pending");
|
|
1022
|
+
if (reqs.length === 0) {
|
|
1023
|
+
await updateJobProgress(tenantId, jobId, 100, "No pending requirements found");
|
|
1024
|
+
await withTenant(
|
|
1025
|
+
tenantId,
|
|
1026
|
+
(tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "DONE", finishedAt: /* @__PURE__ */ new Date(), result: { created: 0, skipped: 0 } } })
|
|
1027
|
+
);
|
|
1028
|
+
return;
|
|
1029
|
+
}
|
|
1030
|
+
let resolvedSuiteId = suiteId;
|
|
1031
|
+
if (!resolvedSuiteId) {
|
|
1032
|
+
const name = suiteName?.trim() || "Requirements Tests";
|
|
1033
|
+
const suite = await withTenant(tenantId, async (tx) => {
|
|
1034
|
+
const existing = await tx.testSuite.findFirst({ where: { tenantId, projectId, name } });
|
|
1035
|
+
if (existing) return existing;
|
|
1036
|
+
return tx.testSuite.create({ data: { tenantId, projectId, name } });
|
|
1037
|
+
});
|
|
1038
|
+
resolvedSuiteId = suite.id;
|
|
1039
|
+
}
|
|
1040
|
+
await updateJobProgress(tenantId, jobId, 20, `Generating tests for ${reqs.length} requirements`);
|
|
1041
|
+
const reqBlock = reqs.map((r) => [
|
|
1042
|
+
`## ${r.reqId}: ${r.title}`,
|
|
1043
|
+
`Priority: ${r.priority} | Type: ${r.type}`,
|
|
1044
|
+
r.acceptanceCriteria?.length ? `Acceptance Criteria:
|
|
1045
|
+
${r.acceptanceCriteria.map((c) => `- ${c}`).join("\n")}` : "",
|
|
1046
|
+
r.businessRules?.length ? `Business Rules:
|
|
1047
|
+
${r.businessRules.map((b) => `- ${b}`).join("\n")}` : ""
|
|
1048
|
+
].filter(Boolean).join("\n")).join("\n\n");
|
|
1049
|
+
const system = `You are ZeTa-AI's test case generator. Convert requirements into structured test cases.
|
|
1132
1050
|
Return ONLY valid JSON: { "cases": [ { "title": string, "kind": string, "priority": string, "preconditions": string, "steps": [{"action": string, "data": string, "expected": string}], "expectedResult": string, "tags": string[], "reqRef": string } ] }
|
|
1133
1051
|
Kind must be one of: POSITIVE, NEGATIVE, EDGE, SECURITY, ACCESSIBILITY, INTEGRATION, E2E, SMOKE, REGRESSION.
|
|
1134
1052
|
Priority must be one of: CRITICAL, HIGH, MEDIUM, LOW.
|
|
1135
1053
|
Generate 1-3 test cases per requirement (positive + negative + edge if applicable).
|
|
1136
1054
|
Each test case must reference its reqRef (the requirement ID like REQ-001).`;
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
}
|
|
1151
|
-
await updateJobProgress(tenantId, jobId, 70, `Saving ${parsed.cases?.length ?? 0} test cases`);
|
|
1152
|
-
const VALID_KINDS = new Set(['POSITIVE', 'NEGATIVE', 'EDGE', 'STATE_TRANSITION', 'ACCESSIBILITY', 'SECURITY', 'INTEGRATION', 'E2E', 'API', 'EXPLORATORY', 'UAT', 'PERFORMANCE', 'COMPATIBILITY', 'L10N', 'USABILITY', 'LOAD', 'UNIT', 'MUTATION', 'AB_TEST', 'ALPHA_BETA', 'SYSTEM', 'SMOKE', 'REGRESSION', 'SANITY']);
|
|
1153
|
-
const VALID_PRIORITIES = new Set(['CRITICAL', 'HIGH', 'MEDIUM', 'LOW']);
|
|
1154
|
-
const normalized = (parsed.cases ?? []).map((c) => ({
|
|
1155
|
-
ref: c.reqRef ?? null,
|
|
1156
|
-
traceTo: c.reqRef ?? null,
|
|
1157
|
-
title: c.title || 'Untitled',
|
|
1158
|
-
kind: VALID_KINDS.has((c.kind ?? '').toUpperCase()) ? c.kind.toUpperCase() : 'POSITIVE',
|
|
1159
|
-
priority: VALID_PRIORITIES.has((c.priority ?? '').toUpperCase()) ? c.priority.toUpperCase() : 'MEDIUM',
|
|
1160
|
-
preconditions: c.preconditions ?? '',
|
|
1161
|
-
steps: Array.isArray(c.steps) ? c.steps : [],
|
|
1162
|
-
expectedResult: c.expectedResult ?? '',
|
|
1163
|
-
tags: Array.isArray(c.tags) ? c.tags : [],
|
|
1164
|
-
}));
|
|
1165
|
-
const { created, skipped } = await AppBrain.importTestCasesFromArtifact(tenantId, projectId, resolvedSuiteId, normalized, { duplicateMode: 'skip' });
|
|
1166
|
-
// Mark requirements as covered
|
|
1167
|
-
const coveredReqIds = new Set(normalized.map((c) => c.ref).filter(Boolean));
|
|
1168
|
-
await Promise.all(reqs
|
|
1169
|
-
.filter((r) => coveredReqIds.has(r.reqId))
|
|
1170
|
-
.map((r) => AppBrain.updateRequirement(tenantId, projectId, r.reqId, { status: 'covered' }).catch(() => { })));
|
|
1171
|
-
await withTenant(tenantId, (tx) => tx.platformJob.update({
|
|
1172
|
-
where: { id: jobId },
|
|
1173
|
-
data: { status: 'DONE', finishedAt: new Date(), result: { created: created.length, skipped, suiteId: resolvedSuiteId }, progress: 100 },
|
|
1174
|
-
}));
|
|
1175
|
-
}
|
|
1176
|
-
catch (e) {
|
|
1177
|
-
const msg = e?.message ?? 'Unknown error';
|
|
1178
|
-
logger.error({ jobId, err: msg }, 'processGenerateTestsFromReqsJob failed');
|
|
1179
|
-
await withTenant(tenantId, (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', finishedAt: new Date(), error: msg } })).catch(() => { });
|
|
1055
|
+
const user = `Generate test cases for these requirements:
|
|
1056
|
+
|
|
1057
|
+
${reqBlock}`;
|
|
1058
|
+
let parsed = { cases: [] };
|
|
1059
|
+
try {
|
|
1060
|
+
const reqSeed = deterministicSeed(`${tenantId}:${projectId}:req-cases:${reqs.map((r) => r.reqId).sort().join("|").slice(0, 300)}`);
|
|
1061
|
+
const { text, usage } = await askTracked(system, user, llmCfg, void 0, void 0, { temperature: 0, seed: reqSeed });
|
|
1062
|
+
if (usage) recordLLMCall({ tenantId, projectId, jobId, agentType: "generate_tests_from_reqs", label: "test_case_generation", usage }).catch(() => {
|
|
1063
|
+
});
|
|
1064
|
+
const json = text.match(/\{[\s\S]*\}/)?.[0];
|
|
1065
|
+
if (json) parsed = JSON.parse(json);
|
|
1066
|
+
} catch (e) {
|
|
1067
|
+
throw new Error("AI failed to generate test cases: " + (e.message ?? "unknown"));
|
|
1180
1068
|
}
|
|
1069
|
+
await updateJobProgress(tenantId, jobId, 70, `Saving ${parsed.cases?.length ?? 0} test cases`);
|
|
1070
|
+
const VALID_KINDS = /* @__PURE__ */ new Set(["POSITIVE", "NEGATIVE", "EDGE", "STATE_TRANSITION", "ACCESSIBILITY", "SECURITY", "INTEGRATION", "E2E", "API", "EXPLORATORY", "UAT", "PERFORMANCE", "COMPATIBILITY", "L10N", "USABILITY", "LOAD", "UNIT", "MUTATION", "AB_TEST", "ALPHA_BETA", "SYSTEM", "SMOKE", "REGRESSION", "SANITY"]);
|
|
1071
|
+
const VALID_PRIORITIES = /* @__PURE__ */ new Set(["CRITICAL", "HIGH", "MEDIUM", "LOW"]);
|
|
1072
|
+
const normalized = (parsed.cases ?? []).map((c) => ({
|
|
1073
|
+
ref: c.reqRef ?? null,
|
|
1074
|
+
traceTo: c.reqRef ?? null,
|
|
1075
|
+
title: c.title || "Untitled",
|
|
1076
|
+
kind: VALID_KINDS.has((c.kind ?? "").toUpperCase()) ? c.kind.toUpperCase() : "POSITIVE",
|
|
1077
|
+
priority: VALID_PRIORITIES.has((c.priority ?? "").toUpperCase()) ? c.priority.toUpperCase() : "MEDIUM",
|
|
1078
|
+
preconditions: c.preconditions ?? "",
|
|
1079
|
+
steps: Array.isArray(c.steps) ? c.steps : [],
|
|
1080
|
+
expectedResult: c.expectedResult ?? "",
|
|
1081
|
+
tags: Array.isArray(c.tags) ? c.tags : []
|
|
1082
|
+
}));
|
|
1083
|
+
const { created, skipped } = await AppBrain.importTestCasesFromArtifact(
|
|
1084
|
+
tenantId,
|
|
1085
|
+
projectId,
|
|
1086
|
+
resolvedSuiteId,
|
|
1087
|
+
normalized,
|
|
1088
|
+
{ duplicateMode: "skip" }
|
|
1089
|
+
);
|
|
1090
|
+
const coveredReqIds = new Set(normalized.map((c) => c.ref).filter(Boolean));
|
|
1091
|
+
await Promise.all(
|
|
1092
|
+
reqs.filter((r) => coveredReqIds.has(r.reqId)).map((r) => AppBrain.updateRequirement(tenantId, projectId, r.reqId, { status: "covered" }).catch(() => {
|
|
1093
|
+
}))
|
|
1094
|
+
);
|
|
1095
|
+
await withTenant(
|
|
1096
|
+
tenantId,
|
|
1097
|
+
(tx) => tx.platformJob.update({
|
|
1098
|
+
where: { id: jobId },
|
|
1099
|
+
data: { status: "DONE", finishedAt: /* @__PURE__ */ new Date(), result: { created: created.length, skipped, suiteId: resolvedSuiteId }, progress: 100 }
|
|
1100
|
+
})
|
|
1101
|
+
);
|
|
1102
|
+
} catch (e) {
|
|
1103
|
+
const msg = e?.message ?? "Unknown error";
|
|
1104
|
+
logger.error({ jobId, err: msg }, "processGenerateTestsFromReqsJob failed");
|
|
1105
|
+
await withTenant(
|
|
1106
|
+
tenantId,
|
|
1107
|
+
(tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", finishedAt: /* @__PURE__ */ new Date(), error: msg } })
|
|
1108
|
+
).catch(() => {
|
|
1109
|
+
});
|
|
1110
|
+
}
|
|
1181
1111
|
}
|
|
1182
|
-
/**
|
|
1183
|
-
* Runs every APPROVED test case in a project (optionally scoped to one suite)
|
|
1184
|
-
* against the agent-free Playwright executor (test-executor.ts) and aggregates
|
|
1185
|
-
* the results into the TestRun row created synchronously by POST /run-tests
|
|
1186
|
-
* (see ci.ts) — this is what makes the CI gate (GET /test-results/latest)
|
|
1187
|
-
* finally reflect what CI itself triggered, instead of a stale or nonexistent
|
|
1188
|
-
* result from an unrelated, manually-triggered, agent-connected suite run.
|
|
1189
|
-
*/
|
|
1190
1112
|
async function processRunTestSuiteJob(jobId, config) {
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
: {}),
|
|
1269
|
-
...(row !== null ? { dataRowId: row.id, dataRowIndex: row.rowIndex } : {}),
|
|
1270
|
-
...((result.consoleLogs?.length || result.networkErrors?.length) ? {
|
|
1271
|
-
logs: {
|
|
1272
|
-
consoleLogs: result.consoleLogs ?? [],
|
|
1273
|
-
networkErrors: result.networkErrors ?? [],
|
|
1274
|
-
},
|
|
1275
|
-
} : {}),
|
|
1276
|
-
});
|
|
1277
|
-
}
|
|
1278
|
-
await AppBrain.recordFlakinessOutcome(tenantId, tc.id).catch(() => { });
|
|
1279
|
-
}
|
|
1280
|
-
catch (err) {
|
|
1281
|
-
failed++;
|
|
1282
|
-
runResults.push({ testCaseId: tc.id, status: 'ERROR' });
|
|
1283
|
-
await AppBrain.saveTestResult(tenantId, {
|
|
1284
|
-
runId: testRunId,
|
|
1285
|
-
testCaseId: tc.id,
|
|
1286
|
-
status: 'ERROR',
|
|
1287
|
-
error: String(err?.message ?? err).slice(0, 500),
|
|
1288
|
-
}).catch(() => { });
|
|
1113
|
+
const { tenantId, projectId, testRunId, suiteId, storageState, browser, shardIndex, totalShards } = config;
|
|
1114
|
+
const runStart = Date.now();
|
|
1115
|
+
await withTenant(
|
|
1116
|
+
tenantId,
|
|
1117
|
+
(tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "RUNNING", startedAt: /* @__PURE__ */ new Date() } })
|
|
1118
|
+
);
|
|
1119
|
+
try {
|
|
1120
|
+
let testCases = await AppBrain.listTestCases(tenantId, projectId, {
|
|
1121
|
+
status: "APPROVED",
|
|
1122
|
+
...suiteId ? { suiteId } : {}
|
|
1123
|
+
});
|
|
1124
|
+
if (typeof totalShards === "number" && totalShards > 1 && typeof shardIndex === "number") {
|
|
1125
|
+
testCases = testCases.filter((_, i) => i % totalShards === shardIndex);
|
|
1126
|
+
}
|
|
1127
|
+
if (testCases.length === 0) {
|
|
1128
|
+
await AppBrain.completeTestRun(tenantId, testRunId, { total: 0, passed: 0, failed: 0, skipped: 0, duration: 0 });
|
|
1129
|
+
await withTenant(
|
|
1130
|
+
tenantId,
|
|
1131
|
+
(tx) => tx.platformJob.update({
|
|
1132
|
+
where: { id: jobId },
|
|
1133
|
+
data: { status: "DONE", progress: 100, finishedAt: /* @__PURE__ */ new Date(), result: { total: 0 } }
|
|
1134
|
+
})
|
|
1135
|
+
);
|
|
1136
|
+
return;
|
|
1137
|
+
}
|
|
1138
|
+
const aiHealingEnabled = await isFeatureEnabled(tenantId, "AI_SELF_HEALING");
|
|
1139
|
+
const SCREENSHOT_DIR = process.env.SCREENSHOT_DIR ?? "/tmp/zeta-screenshots";
|
|
1140
|
+
const screenshotDir = config.screenshotDir || platformConfig.screenshotDir || SCREENSHOT_DIR;
|
|
1141
|
+
const VALID_BROWSERS = ["chromium", "firefox", "webkit"];
|
|
1142
|
+
const validBrowser = VALID_BROWSERS.includes(browser) ? browser : "chromium";
|
|
1143
|
+
let passed = 0;
|
|
1144
|
+
let failed = 0;
|
|
1145
|
+
const runResults = [];
|
|
1146
|
+
for (let i = 0; i < testCases.length; i += TEST_EXEC_CONCURRENCY) {
|
|
1147
|
+
const batch = testCases.slice(i, i + TEST_EXEC_CONCURRENCY);
|
|
1148
|
+
await Promise.all(batch.map(async (tc) => {
|
|
1149
|
+
try {
|
|
1150
|
+
const full = await AppBrain.getTestCaseForExecution(tenantId, tc.id);
|
|
1151
|
+
const baseUrl = full?.screen?.project?.appUrl ?? "";
|
|
1152
|
+
const execProjectId = full?.screen?.project?.id ?? projectId;
|
|
1153
|
+
const dataRows = full?.dataSet?.rows ?? [];
|
|
1154
|
+
const runs = dataRows.length > 0 ? dataRows : [null];
|
|
1155
|
+
for (const row of runs) {
|
|
1156
|
+
let tcForPlan = full ?? tc;
|
|
1157
|
+
if (row !== null) {
|
|
1158
|
+
const rawSteps = Array.isArray(tcForPlan.steps) ? tcForPlan.steps : [];
|
|
1159
|
+
tcForPlan = {
|
|
1160
|
+
...tcForPlan,
|
|
1161
|
+
steps: rawSteps.map((s) => interpolatePlaceholders(s, row.data)),
|
|
1162
|
+
title: interpolatePlaceholders(tcForPlan.title ?? "", row.data),
|
|
1163
|
+
preconditions: tcForPlan.preconditions ? interpolatePlaceholders(tcForPlan.preconditions, row.data) : tcForPlan.preconditions
|
|
1164
|
+
};
|
|
1165
|
+
}
|
|
1166
|
+
const plan = await generateExecutionPlan(tenantId, tcForPlan, baseUrl);
|
|
1167
|
+
const result = await executeTestPlan(plan, {
|
|
1168
|
+
screenshotDir,
|
|
1169
|
+
testRunId,
|
|
1170
|
+
maxRuns: 1,
|
|
1171
|
+
browser: validBrowser,
|
|
1172
|
+
storageState,
|
|
1173
|
+
healCtx: aiHealingEnabled ? { tenantId, projectId: execProjectId } : void 0
|
|
1174
|
+
});
|
|
1175
|
+
const status = result.status === "PASSED" ? "PASSED" : result.status === "ERROR" ? "ERROR" : "FAILED";
|
|
1176
|
+
if (status === "PASSED") passed++;
|
|
1177
|
+
else failed++;
|
|
1178
|
+
runResults.push({ testCaseId: tc.id, status });
|
|
1179
|
+
await AppBrain.saveTestResult(tenantId, {
|
|
1180
|
+
runId: testRunId,
|
|
1181
|
+
testCaseId: tc.id,
|
|
1182
|
+
status,
|
|
1183
|
+
duration: result.totalDuration,
|
|
1184
|
+
...result.rootCauseSummary ? { failureAnalysis: result.rootCauseSummary, failureAnalysisAt: /* @__PURE__ */ new Date() } : {},
|
|
1185
|
+
...row !== null ? { dataRowId: row.id, dataRowIndex: row.rowIndex } : {},
|
|
1186
|
+
...result.consoleLogs?.length || result.networkErrors?.length ? {
|
|
1187
|
+
logs: {
|
|
1188
|
+
consoleLogs: result.consoleLogs ?? [],
|
|
1189
|
+
networkErrors: result.networkErrors ?? []
|
|
1289
1190
|
}
|
|
1290
|
-
|
|
1291
|
-
|
|
1191
|
+
} : {}
|
|
1192
|
+
});
|
|
1193
|
+
}
|
|
1194
|
+
await AppBrain.recordFlakinessOutcome(tenantId, tc.id).catch(() => {
|
|
1195
|
+
});
|
|
1196
|
+
} catch (err) {
|
|
1197
|
+
failed++;
|
|
1198
|
+
runResults.push({ testCaseId: tc.id, status: "ERROR" });
|
|
1199
|
+
await AppBrain.saveTestResult(tenantId, {
|
|
1200
|
+
runId: testRunId,
|
|
1201
|
+
testCaseId: tc.id,
|
|
1202
|
+
status: "ERROR",
|
|
1203
|
+
error: String(err?.message ?? err).slice(0, 500)
|
|
1204
|
+
}).catch(() => {
|
|
1205
|
+
});
|
|
1292
1206
|
}
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
await withTenant(tenantId, (tx) => tx.platformJob.update({
|
|
1301
|
-
where: { id: jobId },
|
|
1302
|
-
data: { status: 'DONE', progress: 100, finishedAt: new Date(), result: { total: testCases.length, passed, failed } },
|
|
1303
|
-
}));
|
|
1304
|
-
// Same full fan-out as the interactive web-driven run path (in-app,
|
|
1305
|
-
// generic webhook, suite-owner failure emails, GitHub check-run, Slack,
|
|
1306
|
-
// Teams, alert emails on failure) — a CI-triggered run has nobody
|
|
1307
|
-
// watching the SSE stream, so this is the only place these fire from.
|
|
1308
|
-
notifyRunCompletion(tenantId, projectId, testRunId, failed === 0 ? 'COMPLETED' : 'FAILED', { passed, failed, total: testCases.length }, runResults).catch(console.error);
|
|
1309
|
-
}
|
|
1310
|
-
catch (err) {
|
|
1311
|
-
const msg = String(err?.message ?? err).slice(0, 500);
|
|
1312
|
-
logger.error({ jobId, testRunId, err: msg }, 'processRunTestSuiteJob failed');
|
|
1313
|
-
await AppBrain.failTestRun(tenantId, testRunId, msg).catch(() => { });
|
|
1314
|
-
await withTenant(tenantId, (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', finishedAt: new Date(), error: msg } })).catch(() => { });
|
|
1207
|
+
}));
|
|
1208
|
+
await updateJobProgress(
|
|
1209
|
+
tenantId,
|
|
1210
|
+
jobId,
|
|
1211
|
+
Math.round(Math.min(i + TEST_EXEC_CONCURRENCY, testCases.length) / testCases.length * 100),
|
|
1212
|
+
`Ran ${Math.min(i + TEST_EXEC_CONCURRENCY, testCases.length)}/${testCases.length} test cases`
|
|
1213
|
+
);
|
|
1315
1214
|
}
|
|
1215
|
+
await AppBrain.completeTestRun(tenantId, testRunId, {
|
|
1216
|
+
total: testCases.length,
|
|
1217
|
+
passed,
|
|
1218
|
+
failed,
|
|
1219
|
+
skipped: 0,
|
|
1220
|
+
duration: Date.now() - runStart
|
|
1221
|
+
});
|
|
1222
|
+
await withTenant(
|
|
1223
|
+
tenantId,
|
|
1224
|
+
(tx) => tx.platformJob.update({
|
|
1225
|
+
where: { id: jobId },
|
|
1226
|
+
data: { status: "DONE", progress: 100, finishedAt: /* @__PURE__ */ new Date(), result: { total: testCases.length, passed, failed } }
|
|
1227
|
+
})
|
|
1228
|
+
);
|
|
1229
|
+
notifyRunCompletion(
|
|
1230
|
+
tenantId,
|
|
1231
|
+
projectId,
|
|
1232
|
+
testRunId,
|
|
1233
|
+
failed === 0 ? "COMPLETED" : "FAILED",
|
|
1234
|
+
{ passed, failed, total: testCases.length },
|
|
1235
|
+
runResults
|
|
1236
|
+
).catch(console.error);
|
|
1237
|
+
} catch (err) {
|
|
1238
|
+
const msg = String(err?.message ?? err).slice(0, 500);
|
|
1239
|
+
logger.error({ jobId, testRunId, err: msg }, "processRunTestSuiteJob failed");
|
|
1240
|
+
await AppBrain.failTestRun(tenantId, testRunId, msg).catch(() => {
|
|
1241
|
+
});
|
|
1242
|
+
await withTenant(
|
|
1243
|
+
tenantId,
|
|
1244
|
+
(tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", finishedAt: /* @__PURE__ */ new Date(), error: msg } })
|
|
1245
|
+
).catch(() => {
|
|
1246
|
+
});
|
|
1247
|
+
}
|
|
1316
1248
|
}
|
|
1317
1249
|
async function processGenerateSkillFilesJob(jobId, config) {
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
}
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1250
|
+
const { tenantId, projectId } = config;
|
|
1251
|
+
await withTenant(
|
|
1252
|
+
tenantId,
|
|
1253
|
+
(tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "RUNNING", startedAt: /* @__PURE__ */ new Date() } })
|
|
1254
|
+
);
|
|
1255
|
+
try {
|
|
1256
|
+
await updateJobProgress(tenantId, jobId, 10, "Loading project data\u2026");
|
|
1257
|
+
const projectMap = await AppBrain.getProjectMap(tenantId, projectId);
|
|
1258
|
+
if (!projectMap) throw new Error("Project not found.");
|
|
1259
|
+
const screens = projectMap.screens ?? [];
|
|
1260
|
+
const lastCrawl = await withTenant(
|
|
1261
|
+
tenantId,
|
|
1262
|
+
(tx) => tx.platformJob.findFirst({
|
|
1263
|
+
where: { projectId, type: "CRAWL", status: "DONE" },
|
|
1264
|
+
orderBy: { finishedAt: "desc" },
|
|
1265
|
+
select: { result: true }
|
|
1266
|
+
})
|
|
1267
|
+
);
|
|
1268
|
+
const lastResult = lastCrawl?.result ?? {};
|
|
1269
|
+
await updateJobProgress(tenantId, jobId, 25, `Generating ${screens.length} app intelligence skill files\u2026`);
|
|
1270
|
+
await generateTaxonomySkillFiles(tenantId, projectId, screens, lastResult.apiEndpoints, lastResult.performanceMetrics);
|
|
1271
|
+
await updateJobProgress(tenantId, jobId, 85, "Refreshing testing intelligence skill files\u2026");
|
|
1272
|
+
await Promise.allSettled([
|
|
1273
|
+
refreshTestCasesSkillFile(tenantId, projectId),
|
|
1274
|
+
refreshTestDataSkillFile(tenantId, projectId),
|
|
1275
|
+
refreshTestScriptSkillFile(tenantId, projectId),
|
|
1276
|
+
refreshRequirementsCoverageSkillFile(tenantId, projectId)
|
|
1277
|
+
]);
|
|
1278
|
+
await withTenant(
|
|
1279
|
+
tenantId,
|
|
1280
|
+
(tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "DONE", progress: 100, finishedAt: /* @__PURE__ */ new Date() } })
|
|
1281
|
+
);
|
|
1282
|
+
} catch (err) {
|
|
1283
|
+
await withTenant(
|
|
1284
|
+
tenantId,
|
|
1285
|
+
(tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", error: String(err?.message ?? err), finishedAt: /* @__PURE__ */ new Date() } })
|
|
1286
|
+
);
|
|
1287
|
+
}
|
|
1349
1288
|
}
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1289
|
+
async function processDataRetentionPurge() {
|
|
1290
|
+
logger.info("[data-retention] starting purge run");
|
|
1291
|
+
const llmCutoff = new Date(Date.now() - 90 * 24 * 60 * 60 * 1e3);
|
|
1292
|
+
const llmResult = await prismaAdmin.lLMCallLog.deleteMany({
|
|
1293
|
+
where: { calledAt: { lt: llmCutoff } }
|
|
1294
|
+
});
|
|
1295
|
+
logger.info({ deleted: llmResult.count }, "[data-retention] LLMCallLog purged");
|
|
1296
|
+
const tenants = await prismaAdmin.tenant.findMany({ select: { id: true } });
|
|
1297
|
+
let auditTotal = 0;
|
|
1298
|
+
for (const { id: tenantId } of tenants) {
|
|
1299
|
+
const setting = await prismaAdmin.tenantSetting.findFirst({
|
|
1300
|
+
where: { tenantId },
|
|
1301
|
+
orderBy: { dataRetentionDays: "asc" },
|
|
1302
|
+
select: { dataRetentionDays: true }
|
|
1361
1303
|
});
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
const
|
|
1365
|
-
|
|
1366
|
-
for (const { id: tenantId } of tenants) {
|
|
1367
|
-
// Find the most restrictive (lowest) dataRetentionDays across this tenant's
|
|
1368
|
-
// settings rows, falling back to 365 if no setting row exists.
|
|
1369
|
-
const setting = await prismaAdmin.tenantSetting.findFirst({
|
|
1370
|
-
where: { tenantId },
|
|
1371
|
-
orderBy: { dataRetentionDays: 'asc' },
|
|
1372
|
-
select: { dataRetentionDays: true },
|
|
1373
|
-
});
|
|
1374
|
-
const retentionDays = setting?.dataRetentionDays ?? 365;
|
|
1375
|
-
const cutoff = new Date(Date.now() - retentionDays * 24 * 60 * 60 * 1000);
|
|
1376
|
-
const result = await prismaAdmin.auditLog.deleteMany({
|
|
1377
|
-
where: { tenantId, createdAt: { lt: cutoff } },
|
|
1378
|
-
});
|
|
1379
|
-
auditTotal += result.count;
|
|
1380
|
-
}
|
|
1381
|
-
logger.info({ deleted: auditTotal, tenants: tenants.length }, '[data-retention] AuditLog purged');
|
|
1382
|
-
// AuthSession: fixed 12-month retention — IP addresses are PII under GDPR.
|
|
1383
|
-
// Only purge revoked sessions; active (non-revoked) sessions are kept.
|
|
1384
|
-
const SESSION_RETENTION_DAYS = 365;
|
|
1385
|
-
const sessionCutoff = new Date(Date.now() - SESSION_RETENTION_DAYS * 24 * 60 * 60_000);
|
|
1386
|
-
const sessionResult = await prismaAdmin.authSession.deleteMany({
|
|
1387
|
-
where: {
|
|
1388
|
-
createdAt: { lt: sessionCutoff },
|
|
1389
|
-
revokedAt: { not: null },
|
|
1390
|
-
},
|
|
1304
|
+
const retentionDays = setting?.dataRetentionDays ?? 365;
|
|
1305
|
+
const cutoff = new Date(Date.now() - retentionDays * 24 * 60 * 60 * 1e3);
|
|
1306
|
+
const result = await prismaAdmin.auditLog.deleteMany({
|
|
1307
|
+
where: { tenantId, createdAt: { lt: cutoff } }
|
|
1391
1308
|
});
|
|
1392
|
-
|
|
1309
|
+
auditTotal += result.count;
|
|
1310
|
+
}
|
|
1311
|
+
logger.info({ deleted: auditTotal, tenants: tenants.length }, "[data-retention] AuditLog purged");
|
|
1312
|
+
const SESSION_RETENTION_DAYS = 365;
|
|
1313
|
+
const sessionCutoff = new Date(Date.now() - SESSION_RETENTION_DAYS * 24 * 60 * 6e4);
|
|
1314
|
+
const sessionResult = await prismaAdmin.authSession.deleteMany({
|
|
1315
|
+
where: {
|
|
1316
|
+
createdAt: { lt: sessionCutoff },
|
|
1317
|
+
revokedAt: { not: null }
|
|
1318
|
+
}
|
|
1319
|
+
});
|
|
1320
|
+
logger.info({ deleted: sessionResult.count }, "[data-retention] AuthSession purged");
|
|
1393
1321
|
}
|
|
1394
|
-
// ── URL Discovery Job ─────────────────────────────────────────────────────────
|
|
1395
|
-
// Three-phase URL discovery: sitemap seed → unauthenticated BFS spider → authenticated BFS
|
|
1396
|
-
// spider (when crawl credentials exist). Persists discovered URLs to crawlConfig and marks the
|
|
1397
|
-
// job DONE with the full URL list so the UI can poll for results.
|
|
1398
1322
|
async function processDiscoverJob(jobId, config, opts) {
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
const
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
// Flush to DB at most every 5 seconds to avoid excessive writes
|
|
1415
|
-
if (now - lastPartialFlush < 5_000 && allUrls.size < 5)
|
|
1416
|
-
return;
|
|
1417
|
-
const monotonic = Math.max(pct, lastFlushedPct);
|
|
1418
|
-
lastFlushedPct = monotonic;
|
|
1419
|
-
lastPartialFlush = now;
|
|
1420
|
-
try {
|
|
1421
|
-
await withTenant(tenantId, (tx) => tx.platformJob.update({
|
|
1422
|
-
where: { id: jobId },
|
|
1423
|
-
data: { progress: monotonic, result: { step, partialUrls: Array.from(allUrls) }, heartbeatAt: new Date() },
|
|
1424
|
-
}));
|
|
1425
|
-
}
|
|
1426
|
-
catch { /* non-fatal */ }
|
|
1427
|
-
await opts?.onProgress?.(monotonic);
|
|
1428
|
-
};
|
|
1429
|
-
const onProgress = async (pct, step) => {
|
|
1430
|
-
await updateJobProgress(tenantId, jobId, pct, step);
|
|
1431
|
-
await opts?.onProgress?.(pct);
|
|
1432
|
-
};
|
|
1323
|
+
const { tenantId, projectId } = config;
|
|
1324
|
+
const controller = new AbortController();
|
|
1325
|
+
runningControllers.set(jobId, controller);
|
|
1326
|
+
const heartbeatTimer = setInterval(() => {
|
|
1327
|
+
void heartbeatJob(tenantId, jobId);
|
|
1328
|
+
}, JOB_HEARTBEAT_INTERVAL_MS);
|
|
1329
|
+
const allUrls = /* @__PURE__ */ new Set();
|
|
1330
|
+
let lastPartialFlush = 0;
|
|
1331
|
+
let lastFlushedPct = 0;
|
|
1332
|
+
const flushPartialUrls = async (pct, step) => {
|
|
1333
|
+
const now = Date.now();
|
|
1334
|
+
if (now - lastPartialFlush < 5e3 && allUrls.size < 5) return;
|
|
1335
|
+
const monotonic = Math.max(pct, lastFlushedPct);
|
|
1336
|
+
lastFlushedPct = monotonic;
|
|
1337
|
+
lastPartialFlush = now;
|
|
1433
1338
|
try {
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
// discovery would see username but miss password/storageState/authToken.
|
|
1443
|
-
const crawlCfg = await AppBrain.getCrawlConfig(tenantId, projectId).catch(() => null) ?? {};
|
|
1444
|
-
const appUrl = config.appUrl ?? crawlCfg.appUrl;
|
|
1445
|
-
if (!appUrl)
|
|
1446
|
-
throw new Error('No app URL configured. Save the crawl settings first.');
|
|
1447
|
-
const maxDiscoverUrls = config.maxDiscoverUrls ?? 2000;
|
|
1448
|
-
const proxyUrl = config.proxyUrl ?? crawlCfg.proxyUrl;
|
|
1449
|
-
const screenshotDir = os.tmpdir();
|
|
1450
|
-
const discoverStorageConfig = await getTenantStorageConfig(tenantId).catch(() => null);
|
|
1451
|
-
await onProgress(3, 'Seeding from sitemap/robots.txt…');
|
|
1452
|
-
// Phase 1 — sitemap + robots.txt seed (fast, no browser)
|
|
1453
|
-
const sitemapResult = await discoverProjectUrls(appUrl, {
|
|
1454
|
-
sitemapUrl: config.sitemapUrl ?? crawlCfg.sitemapUrl,
|
|
1455
|
-
proxyUrl,
|
|
1456
|
-
}).catch(() => ({ candidates: [], sitemapFound: false }));
|
|
1457
|
-
const sitemapUrls = sitemapResult.candidates.map((c) => c.url);
|
|
1458
|
-
// Seed accumulator with sitemap URLs so they show immediately
|
|
1459
|
-
for (const u of sitemapUrls)
|
|
1460
|
-
allUrls.add(u);
|
|
1461
|
-
logger.info({ count: sitemapUrls.length, sitemapFound: sitemapResult.sitemapFound }, '[discover] Phase 1 sitemap done');
|
|
1462
|
-
// Phase 2 — unauthenticated BFS spider
|
|
1463
|
-
await onProgress(8, `Unauthenticated spider from ${appUrl}…`);
|
|
1464
|
-
const unauthResult = await crawlProject({
|
|
1465
|
-
tenantId,
|
|
1466
|
-
projectId,
|
|
1467
|
-
appUrl,
|
|
1468
|
-
discoveryMode: true,
|
|
1469
|
-
startUrls: sitemapUrls.length > 0 ? sitemapUrls : undefined,
|
|
1470
|
-
screenshotDir,
|
|
1471
|
-
maxScreens: maxDiscoverUrls,
|
|
1472
|
-
proxyUrl,
|
|
1473
|
-
useFirefox: crawlCfg.useFirefox,
|
|
1474
|
-
storageConfig: discoverStorageConfig,
|
|
1475
|
-
abortSignal: controller.signal,
|
|
1476
|
-
jobId,
|
|
1477
|
-
onUrlsDiscovered: async (urls) => {
|
|
1478
|
-
for (const u of urls)
|
|
1479
|
-
allUrls.add(u);
|
|
1480
|
-
const pct = 8 + Math.round((allUrls.size / Math.max(1, maxDiscoverUrls)) * 42);
|
|
1481
|
-
await flushPartialUrls(Math.min(pct, 49), `Discovered ${allUrls.size} URLs…`);
|
|
1482
|
-
},
|
|
1483
|
-
}, async (pct, step) => {
|
|
1484
|
-
await flushPartialUrls(8 + Math.round(pct * 0.42), `Unauthenticated: ${step}`);
|
|
1485
|
-
}).catch((err) => {
|
|
1486
|
-
if (err?.message === 'CRAWL_ABORTED')
|
|
1487
|
-
throw err;
|
|
1488
|
-
logger.warn({ err: err?.message }, '[discover] Phase 2 unauthenticated spider failed — continuing with partial results');
|
|
1489
|
-
return { discoveredUrls: [] };
|
|
1490
|
-
});
|
|
1491
|
-
for (const u of (unauthResult.discoveredUrls ?? []))
|
|
1492
|
-
allUrls.add(u);
|
|
1493
|
-
logger.info({ count: allUrls.size }, '[discover] Phase 2 unauth spider done');
|
|
1494
|
-
// Phase 3 — authenticated BFS spider (only when credentials configured)
|
|
1495
|
-
// getCrawlConfig returns flat fields (username/password/totpSecret) — NOT nested under .credentials
|
|
1496
|
-
let resolvedSso = {};
|
|
1497
|
-
if (crawlCfg.ssoProviderId) {
|
|
1498
|
-
const ssoProviders = await AppBrain.getWorkspaceSsoProviders(tenantId).catch(() => []);
|
|
1499
|
-
const ssoProvider = ssoProviders.find((p) => p.id === crawlCfg.ssoProviderId);
|
|
1500
|
-
if (ssoProvider) {
|
|
1501
|
-
resolvedSso = {
|
|
1502
|
-
ssoProvider: ssoProvider.provider,
|
|
1503
|
-
ssoDomain: ssoProvider.domain,
|
|
1504
|
-
ssoClientId: ssoProvider.clientId,
|
|
1505
|
-
ssoClientSecret: ssoProvider.clientSecret,
|
|
1506
|
-
ssoScope: ssoProvider.scope,
|
|
1507
|
-
};
|
|
1508
|
-
}
|
|
1509
|
-
}
|
|
1510
|
-
const hasCreds = !!(crawlCfg.username ||
|
|
1511
|
-
crawlCfg.storageState ||
|
|
1512
|
-
crawlCfg.authToken ||
|
|
1513
|
-
crawlCfg.chromeProfilePath ||
|
|
1514
|
-
(crawlCfg.httpBasicUsername && crawlCfg.httpBasicPassword));
|
|
1515
|
-
if (hasCreds && !controller.signal.aborted) {
|
|
1516
|
-
await onProgress(52, 'Authenticated spider…');
|
|
1517
|
-
const runtimeCreds = crawlCfg.username
|
|
1518
|
-
? {
|
|
1519
|
-
username: crawlCfg.username,
|
|
1520
|
-
password: crawlCfg.password ?? '',
|
|
1521
|
-
...(crawlCfg.totpSecret
|
|
1522
|
-
? { getOneTimeCode: async () => generateTOTP(crawlCfg.totpSecret) }
|
|
1523
|
-
: {}),
|
|
1524
|
-
}
|
|
1525
|
-
: undefined;
|
|
1526
|
-
const authResult = await crawlProject({
|
|
1527
|
-
tenantId,
|
|
1528
|
-
projectId,
|
|
1529
|
-
appUrl,
|
|
1530
|
-
discoveryMode: true,
|
|
1531
|
-
credentials: runtimeCreds,
|
|
1532
|
-
storageState: crawlCfg.storageState,
|
|
1533
|
-
httpBasicUsername: crawlCfg.httpBasicUsername,
|
|
1534
|
-
httpBasicPassword: crawlCfg.httpBasicPassword,
|
|
1535
|
-
authToken: crawlCfg.authToken,
|
|
1536
|
-
authTokenHeader: crawlCfg.authTokenHeader,
|
|
1537
|
-
authTokenPrefix: crawlCfg.authTokenPrefix,
|
|
1538
|
-
startUrls: crawlCfg.postLoginUrl
|
|
1539
|
-
? [crawlCfg.postLoginUrl, ...(sitemapUrls.length > 0 ? sitemapUrls : [])]
|
|
1540
|
-
: (sitemapUrls.length > 0 ? sitemapUrls : undefined),
|
|
1541
|
-
screenshotDir,
|
|
1542
|
-
maxScreens: maxDiscoverUrls,
|
|
1543
|
-
proxyUrl,
|
|
1544
|
-
chromeProfilePath: crawlCfg.chromeProfilePath,
|
|
1545
|
-
useFirefox: crawlCfg.useFirefox,
|
|
1546
|
-
storageConfig: discoverStorageConfig,
|
|
1547
|
-
captchaSolverApiKey: crawlCfg.captchaSolverApiKey,
|
|
1548
|
-
captchaSolverProvider: crawlCfg.captchaSolverProvider,
|
|
1549
|
-
mailslurpApiKey: crawlCfg.mailslurpApiKey,
|
|
1550
|
-
mailslurpInboxId: crawlCfg.mailslurpInboxId,
|
|
1551
|
-
loginInstructions: crawlCfg.loginInstructions,
|
|
1552
|
-
...resolvedSso,
|
|
1553
|
-
abortSignal: controller.signal,
|
|
1554
|
-
jobId,
|
|
1555
|
-
onUrlsDiscovered: async (urls) => {
|
|
1556
|
-
for (const u of urls)
|
|
1557
|
-
allUrls.add(u);
|
|
1558
|
-
const pct = 52 + Math.round((allUrls.size / Math.max(1, maxDiscoverUrls)) * 38);
|
|
1559
|
-
await flushPartialUrls(Math.min(pct, 89), `Discovered ${allUrls.size} URLs (authenticated)…`);
|
|
1560
|
-
},
|
|
1561
|
-
}, async (pct, step) => {
|
|
1562
|
-
await flushPartialUrls(52 + Math.round(pct * 0.38), `Authenticated: ${step}`);
|
|
1563
|
-
}).catch((err) => {
|
|
1564
|
-
if (err?.message === 'CRAWL_ABORTED')
|
|
1565
|
-
throw err;
|
|
1566
|
-
logger.warn({ err: err?.message }, '[discover] Phase 3 authenticated spider failed — using unauthenticated results');
|
|
1567
|
-
return { discoveredUrls: [] };
|
|
1568
|
-
});
|
|
1569
|
-
for (const u of ((authResult.discoveredUrls) ?? [])) {
|
|
1570
|
-
allUrls.add(u);
|
|
1571
|
-
}
|
|
1572
|
-
logger.info({ count: allUrls.size }, '[discover] Phase 3 auth spider done');
|
|
1573
|
-
}
|
|
1574
|
-
// Persist discovered URLs to project crawl config
|
|
1575
|
-
const discoveredUrls = Array.from(allUrls);
|
|
1576
|
-
const existingCfg = await AppBrain.getCrawlConfig(tenantId, projectId).catch(() => ({}));
|
|
1577
|
-
await AppBrain.saveCrawlConfig(tenantId, projectId, {
|
|
1578
|
-
...existingCfg,
|
|
1579
|
-
discoveredUrls,
|
|
1580
|
-
discoveredAt: new Date().toISOString(),
|
|
1581
|
-
});
|
|
1582
|
-
await onProgress(100, `Done — ${discoveredUrls.length} URLs discovered`);
|
|
1583
|
-
await withTenant(tenantId, (tx) => tx.platformJob.update({
|
|
1584
|
-
where: { id: jobId },
|
|
1585
|
-
data: {
|
|
1586
|
-
status: 'DONE',
|
|
1587
|
-
result: {
|
|
1588
|
-
discoveredUrls,
|
|
1589
|
-
count: discoveredUrls.length,
|
|
1590
|
-
sitemapFound: sitemapResult.sitemapFound,
|
|
1591
|
-
},
|
|
1592
|
-
progress: 100,
|
|
1593
|
-
finishedAt: new Date(),
|
|
1594
|
-
},
|
|
1595
|
-
}));
|
|
1596
|
-
notifyAdmins(tenantId, {
|
|
1597
|
-
type: 'discovery.done',
|
|
1598
|
-
title: 'URL Discovery completed',
|
|
1599
|
-
body: `Found ${discoveredUrls.length} URLs for the project.`,
|
|
1600
|
-
metadata: { projectId, jobId },
|
|
1601
|
-
}).catch(console.error);
|
|
1602
|
-
}
|
|
1603
|
-
catch (err) {
|
|
1604
|
-
logger.error({ err: err?.message, jobId }, '[discover] Job failed');
|
|
1605
|
-
await withTenant(tenantId, (tx) => tx.platformJob.update({
|
|
1606
|
-
where: { id: jobId },
|
|
1607
|
-
data: {
|
|
1608
|
-
status: err?.message === 'CRAWL_ABORTED' ? 'CANCELLED' : 'FAILED',
|
|
1609
|
-
result: { error: err?.message ?? String(err) },
|
|
1610
|
-
finishedAt: new Date(),
|
|
1611
|
-
},
|
|
1612
|
-
})).catch(console.error);
|
|
1339
|
+
await withTenant(
|
|
1340
|
+
tenantId,
|
|
1341
|
+
(tx) => tx.platformJob.update({
|
|
1342
|
+
where: { id: jobId },
|
|
1343
|
+
data: { progress: monotonic, result: { step, partialUrls: Array.from(allUrls) }, heartbeatAt: /* @__PURE__ */ new Date() }
|
|
1344
|
+
})
|
|
1345
|
+
);
|
|
1346
|
+
} catch {
|
|
1613
1347
|
}
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1348
|
+
await opts?.onProgress?.(monotonic);
|
|
1349
|
+
};
|
|
1350
|
+
const onProgress = async (pct, step) => {
|
|
1351
|
+
await updateJobProgress(tenantId, jobId, pct, step);
|
|
1352
|
+
await opts?.onProgress?.(pct);
|
|
1353
|
+
};
|
|
1354
|
+
try {
|
|
1355
|
+
await withTenant(
|
|
1356
|
+
tenantId,
|
|
1357
|
+
(tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "RUNNING", startedAt: /* @__PURE__ */ new Date() } })
|
|
1358
|
+
);
|
|
1359
|
+
const crawlCfg = await AppBrain.getCrawlConfig(tenantId, projectId).catch(() => null) ?? {};
|
|
1360
|
+
const appUrl = config.appUrl ?? crawlCfg.appUrl;
|
|
1361
|
+
if (!appUrl) throw new Error("No app URL configured. Save the crawl settings first.");
|
|
1362
|
+
const maxDiscoverUrls = config.maxDiscoverUrls ?? 2e3;
|
|
1363
|
+
const proxyUrl = config.proxyUrl ?? crawlCfg.proxyUrl;
|
|
1364
|
+
const screenshotDir = os.tmpdir();
|
|
1365
|
+
const discoverStorageConfig = await getTenantStorageConfig(tenantId).catch(() => null);
|
|
1366
|
+
await onProgress(3, "Seeding from sitemap/robots.txt\u2026");
|
|
1367
|
+
const sitemapResult = await discoverProjectUrls(appUrl, {
|
|
1368
|
+
sitemapUrl: config.sitemapUrl ?? crawlCfg.sitemapUrl,
|
|
1369
|
+
proxyUrl
|
|
1370
|
+
}).catch(() => ({ candidates: [], sitemapFound: false }));
|
|
1371
|
+
const sitemapUrls = sitemapResult.candidates.map((c) => c.url);
|
|
1372
|
+
for (const u of sitemapUrls) allUrls.add(u);
|
|
1373
|
+
logger.info({ count: sitemapUrls.length, sitemapFound: sitemapResult.sitemapFound }, "[discover] Phase 1 sitemap done");
|
|
1374
|
+
await onProgress(8, `Unauthenticated spider from ${appUrl}\u2026`);
|
|
1375
|
+
const unauthResult = await crawlProject({
|
|
1376
|
+
tenantId,
|
|
1377
|
+
projectId,
|
|
1378
|
+
appUrl,
|
|
1379
|
+
discoveryMode: true,
|
|
1380
|
+
startUrls: sitemapUrls.length > 0 ? sitemapUrls : void 0,
|
|
1381
|
+
screenshotDir,
|
|
1382
|
+
maxScreens: maxDiscoverUrls,
|
|
1383
|
+
proxyUrl,
|
|
1384
|
+
useFirefox: crawlCfg.useFirefox,
|
|
1385
|
+
storageConfig: discoverStorageConfig,
|
|
1386
|
+
abortSignal: controller.signal,
|
|
1387
|
+
jobId,
|
|
1388
|
+
onUrlsDiscovered: async (urls) => {
|
|
1389
|
+
for (const u of urls) allUrls.add(u);
|
|
1390
|
+
const pct = 8 + Math.round(allUrls.size / Math.max(1, maxDiscoverUrls) * 42);
|
|
1391
|
+
await flushPartialUrls(Math.min(pct, 49), `Discovered ${allUrls.size} URLs\u2026`);
|
|
1392
|
+
}
|
|
1393
|
+
}, async (pct, step) => {
|
|
1394
|
+
await flushPartialUrls(8 + Math.round(pct * 0.42), `Unauthenticated: ${step}`);
|
|
1395
|
+
}).catch((err) => {
|
|
1396
|
+
if (err?.message === "CRAWL_ABORTED") throw err;
|
|
1397
|
+
logger.warn({ err: err?.message }, "[discover] Phase 2 unauthenticated spider failed \u2014 continuing with partial results");
|
|
1398
|
+
return { discoveredUrls: [] };
|
|
1399
|
+
});
|
|
1400
|
+
for (const u of unauthResult.discoveredUrls ?? []) allUrls.add(u);
|
|
1401
|
+
logger.info({ count: allUrls.size }, "[discover] Phase 2 unauth spider done");
|
|
1402
|
+
let resolvedSso = {};
|
|
1403
|
+
if (crawlCfg.ssoProviderId) {
|
|
1404
|
+
const ssoProviders = await AppBrain.getWorkspaceSsoProviders(tenantId).catch(() => []);
|
|
1405
|
+
const ssoProvider = ssoProviders.find((p) => p.id === crawlCfg.ssoProviderId);
|
|
1406
|
+
if (ssoProvider) {
|
|
1407
|
+
resolvedSso = {
|
|
1408
|
+
ssoProvider: ssoProvider.provider,
|
|
1409
|
+
ssoDomain: ssoProvider.domain,
|
|
1410
|
+
ssoClientId: ssoProvider.clientId,
|
|
1411
|
+
ssoClientSecret: ssoProvider.clientSecret,
|
|
1412
|
+
ssoScope: ssoProvider.scope
|
|
1413
|
+
};
|
|
1414
|
+
}
|
|
1617
1415
|
}
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1416
|
+
const hasCreds = !!(crawlCfg.username || crawlCfg.storageState || crawlCfg.authToken || crawlCfg.chromeProfilePath || crawlCfg.httpBasicUsername && crawlCfg.httpBasicPassword);
|
|
1417
|
+
if (hasCreds && !controller.signal.aborted) {
|
|
1418
|
+
await onProgress(52, "Authenticated spider\u2026");
|
|
1419
|
+
const runtimeCreds = crawlCfg.username ? {
|
|
1420
|
+
username: crawlCfg.username,
|
|
1421
|
+
password: crawlCfg.password ?? "",
|
|
1422
|
+
...crawlCfg.totpSecret ? { getOneTimeCode: async () => generateTOTP(crawlCfg.totpSecret) } : {}
|
|
1423
|
+
} : void 0;
|
|
1424
|
+
const authResult = await crawlProject({
|
|
1425
|
+
tenantId,
|
|
1426
|
+
projectId,
|
|
1427
|
+
appUrl,
|
|
1428
|
+
discoveryMode: true,
|
|
1429
|
+
credentials: runtimeCreds,
|
|
1430
|
+
storageState: crawlCfg.storageState,
|
|
1431
|
+
httpBasicUsername: crawlCfg.httpBasicUsername,
|
|
1432
|
+
httpBasicPassword: crawlCfg.httpBasicPassword,
|
|
1433
|
+
authToken: crawlCfg.authToken,
|
|
1434
|
+
authTokenHeader: crawlCfg.authTokenHeader,
|
|
1435
|
+
authTokenPrefix: crawlCfg.authTokenPrefix,
|
|
1436
|
+
startUrls: crawlCfg.postLoginUrl ? [crawlCfg.postLoginUrl, ...sitemapUrls.length > 0 ? sitemapUrls : []] : sitemapUrls.length > 0 ? sitemapUrls : void 0,
|
|
1437
|
+
screenshotDir,
|
|
1438
|
+
maxScreens: maxDiscoverUrls,
|
|
1439
|
+
proxyUrl,
|
|
1440
|
+
chromeProfilePath: crawlCfg.chromeProfilePath,
|
|
1441
|
+
useFirefox: crawlCfg.useFirefox,
|
|
1442
|
+
storageConfig: discoverStorageConfig,
|
|
1443
|
+
captchaSolverApiKey: crawlCfg.captchaSolverApiKey,
|
|
1444
|
+
captchaSolverProvider: crawlCfg.captchaSolverProvider,
|
|
1445
|
+
mailslurpApiKey: crawlCfg.mailslurpApiKey,
|
|
1446
|
+
mailslurpInboxId: crawlCfg.mailslurpInboxId,
|
|
1447
|
+
loginInstructions: crawlCfg.loginInstructions,
|
|
1448
|
+
...resolvedSso,
|
|
1449
|
+
abortSignal: controller.signal,
|
|
1450
|
+
jobId,
|
|
1451
|
+
onUrlsDiscovered: async (urls) => {
|
|
1452
|
+
for (const u of urls) allUrls.add(u);
|
|
1453
|
+
const pct = 52 + Math.round(allUrls.size / Math.max(1, maxDiscoverUrls) * 38);
|
|
1454
|
+
await flushPartialUrls(Math.min(pct, 89), `Discovered ${allUrls.size} URLs (authenticated)\u2026`);
|
|
1644
1455
|
}
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1456
|
+
}, async (pct, step) => {
|
|
1457
|
+
await flushPartialUrls(52 + Math.round(pct * 0.38), `Authenticated: ${step}`);
|
|
1458
|
+
}).catch((err) => {
|
|
1459
|
+
if (err?.message === "CRAWL_ABORTED") throw err;
|
|
1460
|
+
logger.warn({ err: err?.message }, "[discover] Phase 3 authenticated spider failed \u2014 using unauthenticated results");
|
|
1461
|
+
return { discoveredUrls: [] };
|
|
1462
|
+
});
|
|
1463
|
+
for (const u of authResult.discoveredUrls ?? []) {
|
|
1464
|
+
allUrls.add(u);
|
|
1465
|
+
}
|
|
1466
|
+
logger.info({ count: allUrls.size }, "[discover] Phase 3 auth spider done");
|
|
1467
|
+
}
|
|
1468
|
+
const discoveredUrls = Array.from(allUrls);
|
|
1469
|
+
const existingCfg = await AppBrain.getCrawlConfig(tenantId, projectId).catch(() => ({}));
|
|
1470
|
+
await AppBrain.saveCrawlConfig(tenantId, projectId, {
|
|
1471
|
+
...existingCfg,
|
|
1472
|
+
discoveredUrls,
|
|
1473
|
+
discoveredAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1474
|
+
});
|
|
1475
|
+
await onProgress(100, `Done \u2014 ${discoveredUrls.length} URLs discovered`);
|
|
1476
|
+
await withTenant(
|
|
1477
|
+
tenantId,
|
|
1478
|
+
(tx) => tx.platformJob.update({
|
|
1479
|
+
where: { id: jobId },
|
|
1480
|
+
data: {
|
|
1481
|
+
status: "DONE",
|
|
1482
|
+
result: {
|
|
1483
|
+
discoveredUrls,
|
|
1484
|
+
count: discoveredUrls.length,
|
|
1485
|
+
sitemapFound: sitemapResult.sitemapFound
|
|
1486
|
+
},
|
|
1487
|
+
progress: 100,
|
|
1488
|
+
finishedAt: /* @__PURE__ */ new Date()
|
|
1677
1489
|
}
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1490
|
+
})
|
|
1491
|
+
);
|
|
1492
|
+
notifyAdmins(tenantId, {
|
|
1493
|
+
type: "discovery.done",
|
|
1494
|
+
title: "URL Discovery completed",
|
|
1495
|
+
body: `Found ${discoveredUrls.length} URLs for the project.`,
|
|
1496
|
+
metadata: { projectId, jobId }
|
|
1497
|
+
}).catch(console.error);
|
|
1498
|
+
} catch (err) {
|
|
1499
|
+
logger.error({ err: err?.message, jobId }, "[discover] Job failed");
|
|
1500
|
+
await withTenant(
|
|
1501
|
+
tenantId,
|
|
1502
|
+
(tx) => tx.platformJob.update({
|
|
1503
|
+
where: { id: jobId },
|
|
1504
|
+
data: {
|
|
1505
|
+
status: err?.message === "CRAWL_ABORTED" ? "CANCELLED" : "FAILED",
|
|
1506
|
+
result: { error: err?.message ?? String(err) },
|
|
1507
|
+
finishedAt: /* @__PURE__ */ new Date()
|
|
1691
1508
|
}
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
finishedAt: new Date(),
|
|
1699
|
-
result: { results, changedCount: changedUrls.length, totalChecked: results.length },
|
|
1700
|
-
},
|
|
1701
|
-
}));
|
|
1702
|
-
}
|
|
1703
|
-
catch (err) {
|
|
1704
|
-
await withTenant(tenantId, (tx) => tx.platformJob.update({
|
|
1705
|
-
where: { id: jobId },
|
|
1706
|
-
data: { status: 'FAILED', error: String(err?.message ?? err), finishedAt: new Date() },
|
|
1707
|
-
}));
|
|
1708
|
-
}
|
|
1709
|
-
finally {
|
|
1710
|
-
runningControllers.delete(jobId);
|
|
1711
|
-
}
|
|
1509
|
+
})
|
|
1510
|
+
).catch(console.error);
|
|
1511
|
+
} finally {
|
|
1512
|
+
clearInterval(heartbeatTimer);
|
|
1513
|
+
runningControllers.delete(jobId);
|
|
1514
|
+
}
|
|
1712
1515
|
}
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
const
|
|
1723
|
-
|
|
1724
|
-
|
|
1516
|
+
async function processMonitorJob(jobId, config, opts) {
|
|
1517
|
+
const { tenantId, projectId } = config;
|
|
1518
|
+
const controller = new AbortController();
|
|
1519
|
+
runningControllers.set(jobId, controller);
|
|
1520
|
+
try {
|
|
1521
|
+
await withTenant(
|
|
1522
|
+
tenantId,
|
|
1523
|
+
(tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "RUNNING", startedAt: /* @__PURE__ */ new Date() } })
|
|
1524
|
+
);
|
|
1525
|
+
const proj = await withTenant(
|
|
1526
|
+
tenantId,
|
|
1527
|
+
(tx) => tx.project.findUnique({ where: { id: projectId }, select: { crawlConfig: true } })
|
|
1528
|
+
);
|
|
1529
|
+
const crawlCfg = proj?.crawlConfig ?? {};
|
|
1530
|
+
const monitorUrls = config.monitorUrls ?? crawlCfg.monitorUrls ?? [];
|
|
1531
|
+
if (monitorUrls.length === 0) {
|
|
1532
|
+
throw new Error("No monitor URLs configured. Add monitorUrls to crawl config or job config.");
|
|
1725
1533
|
}
|
|
1726
|
-
|
|
1727
|
-
const
|
|
1728
|
-
const
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
if (fn.endsWith('.txt') && c && c.split('\n').some((l) => l.trim().startsWith('|')))
|
|
1736
|
-
return 'EXCEL';
|
|
1737
|
-
return 'AI_EXTRACT';
|
|
1534
|
+
await updateJobProgress(tenantId, jobId, 10, `Monitoring ${monitorUrls.length} URL(s)...`);
|
|
1535
|
+
const storedBaselines = crawlCfg.monitorBaselines ?? {};
|
|
1536
|
+
const projectScreens = await withTenant(
|
|
1537
|
+
tenantId,
|
|
1538
|
+
(tx) => tx.screen.findMany({ where: { projectId }, select: { url: true, elementHash: true } })
|
|
1539
|
+
).catch(() => []);
|
|
1540
|
+
const screenHashMap = {};
|
|
1541
|
+
for (const s of projectScreens) {
|
|
1542
|
+
if (s.url && s.elementHash) screenHashMap[s.url] = s.elementHash;
|
|
1738
1543
|
}
|
|
1739
|
-
const
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1544
|
+
const results = [];
|
|
1545
|
+
const newBaselines = { ...storedBaselines };
|
|
1546
|
+
for (let i = 0; i < monitorUrls.length; i++) {
|
|
1547
|
+
if (controller.signal.aborted) break;
|
|
1548
|
+
const url = monitorUrls[i];
|
|
1549
|
+
const pct = 10 + Math.floor(i / monitorUrls.length * 80);
|
|
1550
|
+
await updateJobProgress(tenantId, jobId, pct, `Checking ${url}...`);
|
|
1551
|
+
opts?.onProgress?.(pct);
|
|
1552
|
+
try {
|
|
1553
|
+
const crawlResult = await httpCrawl({ appUrl: url, maxScreens: 1, respectRobotsTxt: false, timeoutMs: parseInt(process.env.CRAWL_TIMEOUT_MS ?? "20000", 10) });
|
|
1554
|
+
const screen = crawlResult.screens[0];
|
|
1555
|
+
if (!screen) {
|
|
1556
|
+
results.push({ url, changed: false, reason: "fetch_empty", currentHash: "", previousHash: null });
|
|
1557
|
+
continue;
|
|
1747
1558
|
}
|
|
1748
|
-
|
|
1749
|
-
|
|
1559
|
+
const currentHash = hashContent(normalizeText(screen.markdown));
|
|
1560
|
+
const previousHash = storedBaselines[url] ?? screenHashMap[url] ?? null;
|
|
1561
|
+
newBaselines[url] = currentHash;
|
|
1562
|
+
if (!previousHash) {
|
|
1563
|
+
results.push({ url, changed: false, reason: "new_baseline", currentHash, previousHash: null });
|
|
1564
|
+
} else if (previousHash === currentHash) {
|
|
1565
|
+
results.push({ url, changed: false, reason: "unchanged", currentHash, previousHash });
|
|
1566
|
+
} else {
|
|
1567
|
+
results.push({ url, changed: true, reason: "content_changed", currentHash, previousHash });
|
|
1750
1568
|
}
|
|
1569
|
+
} catch (fetchErr) {
|
|
1570
|
+
results.push({ url, changed: false, reason: `error:${fetchErr?.message ?? String(fetchErr)}`, currentHash: "", previousHash: null });
|
|
1571
|
+
}
|
|
1751
1572
|
}
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
suiteId = suite.id;
|
|
1573
|
+
await withTenant(
|
|
1574
|
+
tenantId,
|
|
1575
|
+
(tx) => tx.project.update({
|
|
1576
|
+
where: { id: projectId },
|
|
1577
|
+
data: { crawlConfig: { ...crawlCfg, monitorBaselines: newBaselines } }
|
|
1578
|
+
})
|
|
1579
|
+
).catch((e) => logger.warn({ err: e }, "[monitor] baseline persist failed"));
|
|
1580
|
+
const changedUrls = results.filter((r) => r.changed);
|
|
1581
|
+
if (changedUrls.length > 0) {
|
|
1582
|
+
dispatchWebhook(tenantId, projectId, "content.changed", {
|
|
1583
|
+
jobId,
|
|
1584
|
+
changedUrls: changedUrls.map((r) => ({ url: r.url, previousHash: r.previousHash, currentHash: r.currentHash })),
|
|
1585
|
+
totalChecked: results.length,
|
|
1586
|
+
changedCount: changedUrls.length
|
|
1587
|
+
}).catch(console.error);
|
|
1768
1588
|
}
|
|
1769
|
-
await updateJobProgress(tenantId, jobId,
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
}));
|
|
1780
|
-
const labelResults = await labelTestCasesBatch(labelInput, tenantId);
|
|
1781
|
-
// Persist labels back to each test case
|
|
1782
|
-
await Promise.all(labelResults.map((lr) => withTenant(tenantId, (tx) => tx.testCase.update({
|
|
1783
|
-
where: { id: lr.id },
|
|
1784
|
-
data: {
|
|
1785
|
-
priority: lr.priority ?? undefined,
|
|
1786
|
-
kind: lr.kind ?? undefined,
|
|
1787
|
-
tags: lr.tags ?? undefined,
|
|
1788
|
-
status: lr.needsReview ? 'PENDING_REVIEW' : undefined,
|
|
1789
|
-
},
|
|
1790
|
-
})).catch(() => { }) // non-fatal per-case
|
|
1791
|
-
));
|
|
1792
|
-
labeled = labelResults.length;
|
|
1793
|
-
}
|
|
1794
|
-
catch {
|
|
1795
|
-
// non-fatal — import succeeded even if labeling fails
|
|
1589
|
+
await updateJobProgress(tenantId, jobId, 100, "Monitor check complete");
|
|
1590
|
+
await withTenant(
|
|
1591
|
+
tenantId,
|
|
1592
|
+
(tx) => tx.platformJob.update({
|
|
1593
|
+
where: { id: jobId },
|
|
1594
|
+
data: {
|
|
1595
|
+
status: "DONE",
|
|
1596
|
+
progress: 100,
|
|
1597
|
+
finishedAt: /* @__PURE__ */ new Date(),
|
|
1598
|
+
result: { results, changedCount: changedUrls.length, totalChecked: results.length }
|
|
1796
1599
|
}
|
|
1600
|
+
})
|
|
1601
|
+
);
|
|
1602
|
+
} catch (err) {
|
|
1603
|
+
await withTenant(
|
|
1604
|
+
tenantId,
|
|
1605
|
+
(tx) => tx.platformJob.update({
|
|
1606
|
+
where: { id: jobId },
|
|
1607
|
+
data: { status: "FAILED", error: String(err?.message ?? err), finishedAt: /* @__PURE__ */ new Date() }
|
|
1608
|
+
})
|
|
1609
|
+
);
|
|
1610
|
+
} finally {
|
|
1611
|
+
runningControllers.delete(jobId);
|
|
1612
|
+
}
|
|
1613
|
+
}
|
|
1614
|
+
async function processImportTestsJob(jobId, config) {
|
|
1615
|
+
const { tenantId, projectId, artifactId, autoLabel, sourcePlatform } = config;
|
|
1616
|
+
const jobStart = /* @__PURE__ */ new Date();
|
|
1617
|
+
let suiteId = config.suiteId ?? "new";
|
|
1618
|
+
await updateJobProgress(tenantId, jobId, 5, "Loading artifact");
|
|
1619
|
+
const artifact = await withTenant(
|
|
1620
|
+
tenantId,
|
|
1621
|
+
(tx) => tx.projectArtifact.findUnique({ where: { id: artifactId } })
|
|
1622
|
+
);
|
|
1623
|
+
if (!artifact || artifact.tenantId !== tenantId) {
|
|
1624
|
+
throw new Error(`Artifact ${artifactId} not found`);
|
|
1625
|
+
}
|
|
1626
|
+
const content = artifact.content ?? "";
|
|
1627
|
+
const filename = artifact.originalName ?? artifact.name ?? "";
|
|
1628
|
+
const kind = artifact.kind ?? "";
|
|
1629
|
+
function detectFormat(fn, k, c) {
|
|
1630
|
+
if (fn.endsWith(".feature")) return "GHERKIN";
|
|
1631
|
+
if (fn.match(/\.(xlsx|xls|csv)$/i) || k === "SPREADSHEET") return "EXCEL";
|
|
1632
|
+
if (fn.endsWith(".txt") && c && c.split("\n").some((l) => l.trim().startsWith("|"))) return "EXCEL";
|
|
1633
|
+
return "AI_EXTRACT";
|
|
1634
|
+
}
|
|
1635
|
+
const format = detectFormat(filename, kind, content);
|
|
1636
|
+
await updateJobProgress(tenantId, jobId, 15, `Parsing ${filename}`);
|
|
1637
|
+
const { parseTestCases, labelTestCasesBatch } = await import("@detiq/agents");
|
|
1638
|
+
let res = await parseTestCases(content, format, tenantId, artifact.columnMapping ?? void 0, void 0, void 0, sourcePlatform);
|
|
1639
|
+
if (res.cases.length === 0 && format !== "AI_EXTRACT") {
|
|
1640
|
+
await updateJobProgress(tenantId, jobId, 25, "Retrying with AI extraction");
|
|
1641
|
+
try {
|
|
1642
|
+
res = await parseTestCases(content, "AI_EXTRACT", tenantId, void 0, void 0, void 0, sourcePlatform);
|
|
1643
|
+
} catch {
|
|
1797
1644
|
}
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1645
|
+
}
|
|
1646
|
+
const parsedCases = res.cases;
|
|
1647
|
+
if (parsedCases.length === 0) {
|
|
1648
|
+
await withTenant(
|
|
1649
|
+
tenantId,
|
|
1650
|
+
(tx) => tx.platformJob.update({
|
|
1651
|
+
where: { id: jobId },
|
|
1652
|
+
data: { status: "DONE", progress: 100, finishedAt: /* @__PURE__ */ new Date(), result: { imported: 0, skipped: 0, labeled: 0 } }
|
|
1653
|
+
})
|
|
1654
|
+
);
|
|
1655
|
+
return;
|
|
1656
|
+
}
|
|
1657
|
+
await updateJobProgress(tenantId, jobId, 35, `Parsed ${parsedCases.length} cases \u2014 creating suite`);
|
|
1658
|
+
if (!suiteId || suiteId === "new") {
|
|
1659
|
+
const suiteName = filename.replace(/\.[^/.]+$/, "") || "Imported Tests";
|
|
1660
|
+
const suite = await withTenant(
|
|
1661
|
+
tenantId,
|
|
1662
|
+
(tx) => tx.testSuite.create({
|
|
1663
|
+
data: { tenantId, projectId, name: suiteName }
|
|
1664
|
+
})
|
|
1665
|
+
);
|
|
1666
|
+
suiteId = suite.id;
|
|
1667
|
+
}
|
|
1668
|
+
await updateJobProgress(tenantId, jobId, 50, `Importing ${parsedCases.length} cases`);
|
|
1669
|
+
const { created, skipped } = await AppBrain.importTestCasesFromArtifact(
|
|
1670
|
+
tenantId,
|
|
1671
|
+
projectId,
|
|
1672
|
+
suiteId,
|
|
1673
|
+
parsedCases,
|
|
1674
|
+
{ duplicateMode: "skip", artifactId }
|
|
1675
|
+
);
|
|
1676
|
+
let labeled = 0;
|
|
1677
|
+
if (autoLabel && created.length > 0) {
|
|
1678
|
+
await updateJobProgress(tenantId, jobId, 75, `AI labeling ${created.length} cases`);
|
|
1679
|
+
try {
|
|
1680
|
+
const labelInput = created.map((c) => ({
|
|
1681
|
+
id: c.id,
|
|
1682
|
+
title: c.title,
|
|
1683
|
+
description: c.description ?? "",
|
|
1684
|
+
stepsText: (Array.isArray(c.steps) ? c.steps.join(" ") : String(c.steps ?? "")).slice(0, 200)
|
|
1685
|
+
}));
|
|
1686
|
+
const labelResults = await labelTestCasesBatch(labelInput, tenantId);
|
|
1687
|
+
await Promise.all(
|
|
1688
|
+
labelResults.map(
|
|
1689
|
+
(lr) => withTenant(
|
|
1690
|
+
tenantId,
|
|
1691
|
+
(tx) => tx.testCase.update({
|
|
1692
|
+
where: { id: lr.id },
|
|
1693
|
+
data: {
|
|
1694
|
+
priority: lr.priority ?? void 0,
|
|
1695
|
+
kind: lr.kind ?? void 0,
|
|
1696
|
+
tags: lr.tags ?? void 0,
|
|
1697
|
+
status: lr.needsReview ? "PENDING_REVIEW" : void 0
|
|
1698
|
+
}
|
|
1699
|
+
})
|
|
1700
|
+
).catch(() => {
|
|
1701
|
+
})
|
|
1702
|
+
// non-fatal per-case
|
|
1703
|
+
)
|
|
1704
|
+
);
|
|
1705
|
+
labeled = labelResults.length;
|
|
1706
|
+
} catch {
|
|
1814
1707
|
}
|
|
1815
|
-
|
|
1816
|
-
|
|
1708
|
+
}
|
|
1709
|
+
const casesNeedingData = created.filter((c) => {
|
|
1710
|
+
const inline = c.testDataInline;
|
|
1711
|
+
return inline && typeof inline === "object" && Object.keys(inline).length > 0;
|
|
1712
|
+
});
|
|
1713
|
+
if (casesNeedingData.length > 0) {
|
|
1714
|
+
const cap = Math.min(casesNeedingData.length, 20);
|
|
1715
|
+
await updateJobProgress(tenantId, jobId, 90, `Generating test data for ${cap} cases`);
|
|
1716
|
+
const { generateTestData, refreshTestDataSkillFile: refreshTestDataSkillFile2 } = await import("@detiq/agents");
|
|
1717
|
+
for (let i = 0; i < cap; i++) {
|
|
1718
|
+
const c = casesNeedingData[i];
|
|
1719
|
+
await generateTestData({ tenantId, projectId, testCaseId: c.id, jobId }).catch((e) => {
|
|
1720
|
+
logger.warn({ err: e?.message, testCaseId: c.id }, "[import] test data generation failed");
|
|
1721
|
+
});
|
|
1817
1722
|
}
|
|
1818
|
-
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
|
|
1825
|
-
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
|
|
1830
|
-
|
|
1831
|
-
|
|
1832
|
-
|
|
1833
|
-
|
|
1834
|
-
|
|
1835
|
-
|
|
1836
|
-
|
|
1723
|
+
refreshTestDataSkillFile2(tenantId, projectId).catch(console.error);
|
|
1724
|
+
}
|
|
1725
|
+
if (created.length > 0) {
|
|
1726
|
+
refreshTestCasesSkillFile(tenantId, projectId).catch(console.error);
|
|
1727
|
+
}
|
|
1728
|
+
await updateJobProgress(tenantId, jobId, 98, `Done \u2014 ${created.length} imported`);
|
|
1729
|
+
await withTenant(
|
|
1730
|
+
tenantId,
|
|
1731
|
+
(tx) => tx.platformJob.update({
|
|
1732
|
+
where: { id: jobId },
|
|
1733
|
+
data: {
|
|
1734
|
+
status: "DONE",
|
|
1735
|
+
progress: 100,
|
|
1736
|
+
finishedAt: /* @__PURE__ */ new Date(),
|
|
1737
|
+
result: {
|
|
1738
|
+
imported: created.length,
|
|
1739
|
+
skipped: skipped ?? 0,
|
|
1740
|
+
labeled,
|
|
1741
|
+
suiteId,
|
|
1742
|
+
artifactId,
|
|
1743
|
+
artifactName: filename
|
|
1744
|
+
}
|
|
1745
|
+
}
|
|
1746
|
+
})
|
|
1747
|
+
);
|
|
1748
|
+
aggregateJobTokens(tenantId, jobId, projectId, jobStart).catch(console.error);
|
|
1837
1749
|
}
|
|
1838
|
-
// Generate test cases for a single screen, enqueued by NEW_PAGES_PIPELINE.
|
|
1839
|
-
// Uses the same DualContextOrchestrator path as the in-process GENERATE_TESTS route.
|
|
1840
1750
|
async function processGenerateScreenTestsJob(jobId, config) {
|
|
1841
|
-
|
|
1842
|
-
|
|
1843
|
-
|
|
1844
|
-
|
|
1845
|
-
|
|
1846
|
-
|
|
1847
|
-
|
|
1848
|
-
|
|
1849
|
-
|
|
1850
|
-
|
|
1851
|
-
|
|
1852
|
-
|
|
1853
|
-
|
|
1854
|
-
if (triggerAlignedScripts && repoConnectionId) {
|
|
1855
|
-
const conn = await withTenant(tenantId, (tx) => tx.gitHubRepoConnection.findFirst({ where: { id: repoConnectionId, tenantId } }));
|
|
1856
|
-
if (conn?.styleProfile) {
|
|
1857
|
-
const { generateScriptAligned, mapWithConcurrency, SCRIPT_GEN_CONCURRENCY } = await import('@detiq/agents');
|
|
1858
|
-
const cases = await withTenant(tenantId, (tx) => tx.testCase.findMany({ where: { tenantId, projectId, screenId, status: 'APPROVED' }, select: { id: true }, take: 50 }));
|
|
1859
|
-
await mapWithConcurrency(cases, SCRIPT_GEN_CONCURRENCY, (tc) => generateScriptAligned({
|
|
1860
|
-
tenantId,
|
|
1861
|
-
projectId,
|
|
1862
|
-
testCaseId: tc.id,
|
|
1863
|
-
styleProfile: conn.styleProfile,
|
|
1864
|
-
pageObjectInventory: conn.pageObjectInventory ?? undefined,
|
|
1865
|
-
}).catch(() => { }));
|
|
1866
|
-
}
|
|
1867
|
-
}
|
|
1868
|
-
await withTenant(tenantId, (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: 'DONE', progress: 100, finishedAt: new Date() } }));
|
|
1869
|
-
aggregateJobTokens(tenantId, jobId, projectId, jobStartedAt).catch(console.error);
|
|
1751
|
+
const { tenantId, projectId, screenId, repoConnectionId, triggerAlignedScripts } = config;
|
|
1752
|
+
const jobStartedAt = /* @__PURE__ */ new Date();
|
|
1753
|
+
try {
|
|
1754
|
+
await withTenant(
|
|
1755
|
+
tenantId,
|
|
1756
|
+
(tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "RUNNING", startedAt: jobStartedAt } })
|
|
1757
|
+
);
|
|
1758
|
+
const { DualContextOrchestrator } = await import("@detiq/agents");
|
|
1759
|
+
await updateJobProgress(tenantId, jobId, 10, "Generating test cases for screen");
|
|
1760
|
+
const crawlCfg = await AppBrain.getCrawlConfig(tenantId, projectId).catch(() => null);
|
|
1761
|
+
const deviceProfiles = crawlCfg?.deviceProfiles ?? ["DESKTOP"];
|
|
1762
|
+
for (const viewport of deviceProfiles) {
|
|
1763
|
+
await DualContextOrchestrator.execute({ tenantId, projectId, targetScreenId: screenId, jobId, viewport });
|
|
1870
1764
|
}
|
|
1871
|
-
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
1765
|
+
await updateJobProgress(tenantId, jobId, 80, "Test cases generated");
|
|
1766
|
+
if (triggerAlignedScripts && repoConnectionId) {
|
|
1767
|
+
const conn = await withTenant(
|
|
1768
|
+
tenantId,
|
|
1769
|
+
(tx) => tx.gitHubRepoConnection.findFirst({ where: { id: repoConnectionId, tenantId } })
|
|
1770
|
+
);
|
|
1771
|
+
if (conn?.styleProfile) {
|
|
1772
|
+
const { generateScriptAligned, mapWithConcurrency, SCRIPT_GEN_CONCURRENCY } = await import("@detiq/agents");
|
|
1773
|
+
const cases = await withTenant(
|
|
1774
|
+
tenantId,
|
|
1775
|
+
(tx) => tx.testCase.findMany({ where: { tenantId, projectId, screenId, status: "APPROVED" }, select: { id: true }, take: 50 })
|
|
1776
|
+
);
|
|
1777
|
+
await mapWithConcurrency(
|
|
1778
|
+
cases,
|
|
1779
|
+
SCRIPT_GEN_CONCURRENCY,
|
|
1780
|
+
(tc) => generateScriptAligned({
|
|
1781
|
+
tenantId,
|
|
1782
|
+
projectId,
|
|
1783
|
+
testCaseId: tc.id,
|
|
1784
|
+
styleProfile: conn.styleProfile,
|
|
1785
|
+
pageObjectInventory: conn.pageObjectInventory ?? void 0
|
|
1786
|
+
}).catch(() => {
|
|
1787
|
+
})
|
|
1788
|
+
);
|
|
1789
|
+
}
|
|
1875
1790
|
}
|
|
1791
|
+
await withTenant(
|
|
1792
|
+
tenantId,
|
|
1793
|
+
(tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "DONE", progress: 100, finishedAt: /* @__PURE__ */ new Date() } })
|
|
1794
|
+
);
|
|
1795
|
+
aggregateJobTokens(tenantId, jobId, projectId, jobStartedAt).catch(console.error);
|
|
1796
|
+
} catch (err) {
|
|
1797
|
+
const msg = err?.message ?? String(err);
|
|
1798
|
+
logger.error({ jobId, screenId, err: msg }, "processGenerateScreenTestsJob failed");
|
|
1799
|
+
await withTenant(
|
|
1800
|
+
tenantId,
|
|
1801
|
+
(tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", error: msg.slice(0, 500), finishedAt: /* @__PURE__ */ new Date() } })
|
|
1802
|
+
).catch(() => {
|
|
1803
|
+
});
|
|
1804
|
+
}
|
|
1876
1805
|
}
|
|
1877
|
-
// Registers the type→processor dispatch table used by the durable worker (both the
|
|
1878
|
-
// enqueueJob() fast-path kick and startJobWorkerLoop()'s poll). Declared at module
|
|
1879
|
-
// bottom so it can reference every process*Job function by name; function
|
|
1880
|
-
// declarations are hoisted, so this could technically live anywhere in the file,
|
|
1881
|
-
// but keeping it here alongside the last definition keeps the mapping easy to audit
|
|
1882
|
-
// against the actual functions above.
|
|
1883
1806
|
jobDispatchTable = {
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
|
|
1890
|
-
|
|
1891
|
-
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1807
|
+
CRAWL: processJob,
|
|
1808
|
+
DISCOVER: processDiscoverJob,
|
|
1809
|
+
MONITOR: processMonitorJob,
|
|
1810
|
+
NATIVE_CRAWL: processNativeCrawlJob,
|
|
1811
|
+
API_TESTS: processApiTestsJob,
|
|
1812
|
+
SECURITY_TESTS: processSecurityTestsJob,
|
|
1813
|
+
INTEGRATION_TESTS: processIntegrationTestsJob,
|
|
1814
|
+
FIGMA_IMPORT: processFigmaImportJob,
|
|
1815
|
+
GENERATE_SKILL_FILES: processGenerateSkillFilesJob,
|
|
1816
|
+
GENERATE_TESTS_FROM_REQS: processGenerateTestsFromReqsJob,
|
|
1817
|
+
GENERATE_SCREEN_TESTS: processGenerateScreenTestsJob,
|
|
1818
|
+
RUN_TEST_SUITE: processRunTestSuiteJob,
|
|
1819
|
+
REPO_SCAN: processRepoScanJob,
|
|
1820
|
+
IMPORT_TESTS: processImportTestsJob,
|
|
1821
|
+
BATCH_GENERATE_ALIGNED: processBatchGenerateAlignedJob,
|
|
1822
|
+
CREATE_PR: processCreatePRJob,
|
|
1823
|
+
NEW_PAGES_PIPELINE: processNewPagesPipelineJob
|
|
1901
1824
|
};
|
|
1902
1825
|
async function processRepoScanJob(jobId, config) {
|
|
1903
|
-
|
|
1904
|
-
|
|
1826
|
+
const { repoConnectionId, tenantId, projectId } = config;
|
|
1827
|
+
const jobStart = /* @__PURE__ */ new Date();
|
|
1828
|
+
try {
|
|
1829
|
+
await updateJobProgress(tenantId, jobId, 5, "Starting repo scan");
|
|
1830
|
+
const {
|
|
1831
|
+
fetchRepoTree,
|
|
1832
|
+
fetchFileContent,
|
|
1833
|
+
detectFramework,
|
|
1834
|
+
extractTestIds,
|
|
1835
|
+
extractStyleProfile,
|
|
1836
|
+
buildPageObjectInventory
|
|
1837
|
+
} = await import("@detiq/agents");
|
|
1838
|
+
const conn = await withTenant(
|
|
1839
|
+
tenantId,
|
|
1840
|
+
(tx) => tx.gitHubRepoConnection.findFirst({ where: { id: repoConnectionId, tenantId } })
|
|
1841
|
+
);
|
|
1842
|
+
if (!conn) throw new Error(`RepoConnection ${repoConnectionId} not found`);
|
|
1843
|
+
const { decrypt: decrypt2 } = await import("@detiq/core");
|
|
1844
|
+
const token = decrypt2(conn.tokenEnc);
|
|
1845
|
+
await withTenant(
|
|
1846
|
+
tenantId,
|
|
1847
|
+
(tx) => tx.gitHubRepoConnection.update({ where: { id: repoConnectionId }, data: { scanStatus: "SCANNING", scanError: null } })
|
|
1848
|
+
);
|
|
1849
|
+
await updateJobProgress(tenantId, jobId, 15, "Fetching repo tree");
|
|
1850
|
+
const treeResult = await fetchRepoTree(token, conn.repoFullName, "HEAD");
|
|
1851
|
+
const frameworks = detectFramework(treeResult.paths);
|
|
1852
|
+
const primary = frameworks[0];
|
|
1853
|
+
await updateJobProgress(tenantId, jobId, 30, `Detected framework: ${primary?.framework ?? "unknown"}`);
|
|
1854
|
+
const storedGlob = conn.testGlob || "";
|
|
1855
|
+
const testGlob = storedGlob && storedGlob !== "**/*.spec.ts" ? storedGlob : primary?.testGlob || storedGlob || "**/*.spec.ts";
|
|
1856
|
+
const framework = primary?.framework ?? "unknown";
|
|
1857
|
+
const language = primary?.language ?? "typescript";
|
|
1858
|
+
const { minimatch } = await import("minimatch").catch(() => ({ minimatch: null }));
|
|
1859
|
+
const isPythonTestFile = (p) => p.endsWith(".py") && !p.endsWith("__init__.py") && !p.endsWith("conftest.py") && (p.endsWith("_test.py") || /\/test_[^/]+\.py$/.test(p) || /^test_[^/]+\.py$/.test(p) || p.includes("/tests/") || p.includes("/test/"));
|
|
1860
|
+
const isJavaTestFile = (p) => p.endsWith(".java") && (/[A-Z]\w*Tests?\.java$/.test(p) || /\/tests?\//.test(p) && /[A-Z][a-z]/.test(p.split("/").pop() ?? ""));
|
|
1861
|
+
const testPaths = treeResult.paths.filter((p) => {
|
|
1862
|
+
if (language === "java") return isJavaTestFile(p);
|
|
1863
|
+
if (language === "python") return isPythonTestFile(p);
|
|
1864
|
+
if (minimatch) return minimatch(p, testGlob, { matchBase: true });
|
|
1865
|
+
return p.endsWith(".spec.ts") || p.endsWith(".spec.js") || p.endsWith(".test.ts") || p.endsWith(".test.js") || p.endsWith("_test.py") || p.includes("/test_");
|
|
1866
|
+
}).slice(0, 500);
|
|
1867
|
+
await updateJobProgress(tenantId, jobId, 40, `Found ${testPaths.length} test files`);
|
|
1868
|
+
const allExtracted = [];
|
|
1869
|
+
const styleSampleFiles = [];
|
|
1870
|
+
let processed = 0;
|
|
1871
|
+
await withTenant(tenantId, (tx) => tx.repoTestFile.deleteMany({ where: { repoConnectionId } }));
|
|
1872
|
+
for (const filePath of testPaths) {
|
|
1873
|
+
const content = await fetchFileContent(token, conn.repoFullName, filePath);
|
|
1874
|
+
if (!content) continue;
|
|
1875
|
+
const extracted = extractTestIds(content, filePath, framework);
|
|
1876
|
+
allExtracted.push(...extracted);
|
|
1877
|
+
await withTenant(
|
|
1878
|
+
tenantId,
|
|
1879
|
+
(tx) => tx.repoTestFile.create({
|
|
1880
|
+
data: {
|
|
1881
|
+
tenantId,
|
|
1882
|
+
projectId,
|
|
1883
|
+
repoConnectionId,
|
|
1884
|
+
filePath,
|
|
1885
|
+
framework,
|
|
1886
|
+
extractedIds: extracted.map((e) => ({ name: e.name, normalizedName: e.normalizedName, lineNumber: e.lineNumber })),
|
|
1887
|
+
rawSample: content.slice(0, 2e3),
|
|
1888
|
+
lastScannedAt: /* @__PURE__ */ new Date()
|
|
1889
|
+
}
|
|
1890
|
+
})
|
|
1891
|
+
);
|
|
1892
|
+
if (styleSampleFiles.length < 5) styleSampleFiles.push({ path: filePath, content: content.slice(0, 3e3) });
|
|
1893
|
+
processed++;
|
|
1894
|
+
if (processed % 10 === 0) {
|
|
1895
|
+
await updateJobProgress(tenantId, jobId, 40 + Math.floor(processed / testPaths.length * 40), `Processed ${processed}/${testPaths.length} files`);
|
|
1896
|
+
}
|
|
1897
|
+
}
|
|
1898
|
+
let styleProfile = conn.styleProfile ?? null;
|
|
1899
|
+
if (styleSampleFiles.length > 0) {
|
|
1900
|
+
try {
|
|
1901
|
+
const mainSourcePaths = language === "java" ? treeResult.paths.filter((p) => /^src\/main\/java\//.test(p)).slice(0, 50) : [];
|
|
1902
|
+
const testPathSet = new Set(testPaths);
|
|
1903
|
+
const sourceFilePaths = language === "typescript" || language === "javascript" ? treeResult.paths.filter((p) => (p.endsWith(".ts") || p.endsWith(".js")) && !testPathSet.has(p) && !p.endsWith(".d.ts") && !p.includes("node_modules/") && !p.includes("dist/") && !p.includes(".config.")).slice(0, 60) : [];
|
|
1904
|
+
const allPathsForProfile = [...testPaths, ...mainSourcePaths, ...sourceFilePaths].slice(0, 200);
|
|
1905
|
+
const freshProfile = await extractStyleProfile(styleSampleFiles, framework, language, tenantId, projectId, jobId, allPathsForProfile);
|
|
1906
|
+
if (freshProfile) styleProfile = freshProfile;
|
|
1907
|
+
} catch {
|
|
1908
|
+
}
|
|
1909
|
+
}
|
|
1910
|
+
let pageObjectInventory = conn.pageObjectInventory ?? [];
|
|
1905
1911
|
try {
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
|
|
1909
|
-
|
|
1910
|
-
|
|
1911
|
-
|
|
1912
|
-
|
|
1913
|
-
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
|
|
1919
|
-
|
|
1920
|
-
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
|
|
1926
|
-
|
|
1927
|
-
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
|
|
1932
|
-
|
|
1933
|
-
|
|
1934
|
-
// Java: testGlob matches all *.java — further filter to actual test classes only
|
|
1935
|
-
// (excludes helpers, base classes, utilities that don't follow *Test.java/*Tests.java naming)
|
|
1936
|
-
const isJavaTestFile = (p) => p.endsWith('.java') && (/[A-Z]\w*Tests?\.java$/.test(p) ||
|
|
1937
|
-
(/\/tests?\//.test(p) && /[A-Z][a-z]/.test(p.split('/').pop() ?? '')));
|
|
1938
|
-
const testPaths = treeResult.paths.filter((p) => {
|
|
1939
|
-
if (language === 'java')
|
|
1940
|
-
return isJavaTestFile(p);
|
|
1941
|
-
if (language === 'python')
|
|
1942
|
-
return isPythonTestFile(p);
|
|
1943
|
-
if (minimatch)
|
|
1944
|
-
return minimatch(p, testGlob, { matchBase: true });
|
|
1945
|
-
return p.endsWith('.spec.ts') || p.endsWith('.spec.js') || p.endsWith('.test.ts') || p.endsWith('.test.js') || p.endsWith('_test.py') || p.includes('/test_');
|
|
1946
|
-
}).slice(0, 500); // cap at 500 files
|
|
1947
|
-
await updateJobProgress(tenantId, jobId, 40, `Found ${testPaths.length} test files`);
|
|
1948
|
-
// Extract test IDs from each file
|
|
1949
|
-
const allExtracted = [];
|
|
1950
|
-
const styleSampleFiles = [];
|
|
1951
|
-
let processed = 0;
|
|
1952
|
-
// Delete old test files for this connection
|
|
1953
|
-
await withTenant(tenantId, (tx) => tx.repoTestFile.deleteMany({ where: { repoConnectionId } }));
|
|
1954
|
-
for (const filePath of testPaths) {
|
|
1955
|
-
const content = await fetchFileContent(token, conn.repoFullName, filePath);
|
|
1956
|
-
if (!content)
|
|
1957
|
-
continue;
|
|
1958
|
-
const extracted = extractTestIds(content, filePath, framework);
|
|
1959
|
-
allExtracted.push(...extracted);
|
|
1960
|
-
await withTenant(tenantId, (tx) => tx.repoTestFile.create({
|
|
1961
|
-
data: {
|
|
1962
|
-
tenantId,
|
|
1963
|
-
projectId,
|
|
1964
|
-
repoConnectionId,
|
|
1965
|
-
filePath,
|
|
1966
|
-
framework,
|
|
1967
|
-
extractedIds: extracted.map((e) => ({ name: e.name, normalizedName: e.normalizedName, lineNumber: e.lineNumber })),
|
|
1968
|
-
rawSample: content.slice(0, 2000),
|
|
1969
|
-
lastScannedAt: new Date(),
|
|
1970
|
-
},
|
|
1971
|
-
}));
|
|
1972
|
-
if (styleSampleFiles.length < 5)
|
|
1973
|
-
styleSampleFiles.push({ path: filePath, content: content.slice(0, 3000) });
|
|
1974
|
-
processed++;
|
|
1975
|
-
if (processed % 10 === 0) {
|
|
1976
|
-
await updateJobProgress(tenantId, jobId, 40 + Math.floor((processed / testPaths.length) * 40), `Processed ${processed}/${testPaths.length} files`);
|
|
1977
|
-
}
|
|
1978
|
-
}
|
|
1979
|
-
// Extract style profile for aligned generation
|
|
1980
|
-
// For Java, include main source paths so LLM can detect page object location in src/main/java/
|
|
1981
|
-
// Fall back to existing styleProfile from DB if extraction fails or returns null.
|
|
1982
|
-
let styleProfile = conn.styleProfile ?? null;
|
|
1983
|
-
if (styleSampleFiles.length > 0) {
|
|
1984
|
-
try {
|
|
1985
|
-
const mainSourcePaths = language === 'java'
|
|
1986
|
-
? treeResult.paths.filter((p) => /^src\/main\/java\//.test(p)).slice(0, 50)
|
|
1987
|
-
: [];
|
|
1988
|
-
// For TS/JS, also include non-test source files so LLM can detect pageObjectLocation
|
|
1989
|
-
// (e.g. src/advantage/pages/ in playwright-sample-project)
|
|
1990
|
-
const testPathSet = new Set(testPaths);
|
|
1991
|
-
const sourceFilePaths = (language === 'typescript' || language === 'javascript')
|
|
1992
|
-
? treeResult.paths
|
|
1993
|
-
.filter((p) => (p.endsWith('.ts') || p.endsWith('.js')) &&
|
|
1994
|
-
!testPathSet.has(p) &&
|
|
1995
|
-
!p.endsWith('.d.ts') &&
|
|
1996
|
-
!p.includes('node_modules/') &&
|
|
1997
|
-
!p.includes('dist/') &&
|
|
1998
|
-
!p.includes('.config.'))
|
|
1999
|
-
.slice(0, 60)
|
|
2000
|
-
: [];
|
|
2001
|
-
const allPathsForProfile = [...testPaths, ...mainSourcePaths, ...sourceFilePaths].slice(0, 200);
|
|
2002
|
-
const freshProfile = await extractStyleProfile(styleSampleFiles, framework, language, tenantId, projectId, jobId, allPathsForProfile);
|
|
2003
|
-
if (freshProfile)
|
|
2004
|
-
styleProfile = freshProfile;
|
|
2005
|
-
}
|
|
2006
|
-
catch { /* non-fatal */ }
|
|
2007
|
-
}
|
|
2008
|
-
// Extract page object inventory — finds POM/page classes in non-test files
|
|
2009
|
-
// Fall back to existing inventory from DB if fetch fails.
|
|
2010
|
-
let pageObjectInventory = conn.pageObjectInventory ?? [];
|
|
2011
|
-
try {
|
|
2012
|
-
const freshInventory = await buildPageObjectInventory(token, conn.repoFullName, treeResult.paths, framework);
|
|
2013
|
-
if (freshInventory.length > 0) {
|
|
2014
|
-
pageObjectInventory = freshInventory;
|
|
2015
|
-
await updateJobProgress(tenantId, jobId, 82, `Found ${pageObjectInventory.length} page objects`);
|
|
2016
|
-
}
|
|
2017
|
-
}
|
|
2018
|
-
catch { /* non-fatal */ }
|
|
2019
|
-
// Run coverage mapping
|
|
2020
|
-
await updateJobProgress(tenantId, jobId, 85, 'Running coverage mapping');
|
|
2021
|
-
const { buildCoverageMap } = await import('@detiq/agents');
|
|
2022
|
-
const testCases = await withTenant(tenantId, (tx) => tx.testCase.findMany({ where: { projectId, tenantId, status: { not: 'RETIRED' } }, select: { id: true, title: true, ref: true } }));
|
|
2023
|
-
const mappings = await buildCoverageMap(testCases, allExtracted, tenantId);
|
|
2024
|
-
// Upsert CoverageMapping records (skip manualOverride=true)
|
|
2025
|
-
for (const m of mappings) {
|
|
2026
|
-
const existing = await withTenant(tenantId, (tx) => tx.coverageMapping.findUnique({ where: { testCaseId: m.testCaseId } }));
|
|
2027
|
-
if (existing?.manualOverride)
|
|
2028
|
-
continue;
|
|
2029
|
-
await withTenant(tenantId, (tx) => tx.coverageMapping.upsert({
|
|
2030
|
-
where: { testCaseId: m.testCaseId },
|
|
2031
|
-
create: { tenantId, projectId, testCaseId: m.testCaseId, repoConnectionId, status: m.status, matchedFilePath: m.matchedFilePath ?? null, scriptTestName: m.scriptTestName ?? null, lineNumber: m.lineNumber ?? null, confidence: m.confidence, matchMethod: m.matchMethod },
|
|
2032
|
-
update: { status: m.status, matchedFilePath: m.matchedFilePath ?? null, scriptTestName: m.scriptTestName ?? null, lineNumber: m.lineNumber ?? null, confidence: m.confidence, matchMethod: m.matchMethod },
|
|
2033
|
-
}));
|
|
2034
|
-
}
|
|
2035
|
-
const mapped = mappings.filter((m) => m.status === 'MAPPED').length;
|
|
2036
|
-
const unmapped = mappings.filter((m) => m.status === 'UNMAPPED').length;
|
|
2037
|
-
const ambiguous = mappings.filter((m) => m.status === 'AMBIGUOUS').length;
|
|
2038
|
-
const coveragePercent = testCases.length > 0 ? (mapped / testCases.length) * 100 : 0;
|
|
2039
|
-
// Write coverage snapshot
|
|
2040
|
-
await withTenant(tenantId, (tx) => tx.coverageSnapshot.create({
|
|
2041
|
-
data: { tenantId, projectId, totalCases: testCases.length, mappedCases: mapped, unmappedCases: unmapped, ambiguousCases: ambiguous, coveragePercent },
|
|
2042
|
-
}));
|
|
2043
|
-
// For Java, if styleProfile still missing testStructurePattern, infer two-tier from tree paths.
|
|
2044
|
-
// This covers the case where both extractStyleProfile and buildPageObjectInventory failed.
|
|
2045
|
-
const resolvedStyleProfile = (() => {
|
|
2046
|
-
const sp = styleProfile ?? {};
|
|
2047
|
-
if (framework === 'selenium-java' && !sp.testStructurePattern) {
|
|
2048
|
-
const hasMavenMain = treeResult.paths.some((p) => /^src\/main\/java\//.test(p));
|
|
2049
|
-
if (hasMavenMain)
|
|
2050
|
-
return { ...sp, testStructurePattern: 'two-tier' };
|
|
2051
|
-
}
|
|
2052
|
-
return sp;
|
|
2053
|
-
})();
|
|
2054
|
-
// Update repo connection with results
|
|
2055
|
-
await withTenant(tenantId, (tx) => tx.gitHubRepoConnection.update({
|
|
2056
|
-
where: { id: repoConnectionId },
|
|
2057
|
-
data: {
|
|
2058
|
-
scanStatus: 'DONE',
|
|
2059
|
-
detectedFramework: framework,
|
|
2060
|
-
configFilePath: primary?.configPath ?? null,
|
|
2061
|
-
testGlob: testGlob,
|
|
2062
|
-
testFileCount: testPaths.length,
|
|
2063
|
-
extractedIdCount: allExtracted.length,
|
|
2064
|
-
styleProfile: resolvedStyleProfile,
|
|
2065
|
-
pageObjectInventory: pageObjectInventory.length > 0 ? pageObjectInventory : undefined,
|
|
2066
|
-
treeSnapshot: JSON.stringify(treeResult.paths).slice(0, 100000),
|
|
2067
|
-
lastScannedAt: new Date(),
|
|
2068
|
-
lastSyncAt: new Date(),
|
|
2069
|
-
},
|
|
2070
|
-
}));
|
|
2071
|
-
try {
|
|
2072
|
-
const { generateRepoSkillFiles } = await import('@detiq/agents');
|
|
2073
|
-
await generateRepoSkillFiles(tenantId, projectId, {
|
|
2074
|
-
repoFullName: conn.repoFullName,
|
|
2075
|
-
framework,
|
|
2076
|
-
language,
|
|
2077
|
-
testGlob,
|
|
2078
|
-
styleProfile: resolvedStyleProfile,
|
|
2079
|
-
pageObjectInventory,
|
|
2080
|
-
testPaths,
|
|
2081
|
-
extractedTestCount: allExtracted.length,
|
|
2082
|
-
});
|
|
2083
|
-
}
|
|
2084
|
-
catch (e) {
|
|
2085
|
-
logger.warn({ jobId, repoConnectionId, err: e?.message }, '[repo-scan] repo skill files generation failed (non-fatal)');
|
|
2086
|
-
}
|
|
2087
|
-
await updateJobProgress(tenantId, jobId, 100, `Done: ${mapped} mapped, ${unmapped} unmapped`);
|
|
2088
|
-
await withTenant(tenantId, (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: 'DONE', finishedAt: new Date(), result: { testFileCount: testPaths.length, extractedIdCount: allExtracted.length, mapped, unmapped, ambiguous, coveragePercent } } }));
|
|
2089
|
-
aggregateJobTokens(tenantId, jobId, projectId, jobStart).catch(console.error);
|
|
1912
|
+
const freshInventory = await buildPageObjectInventory(token, conn.repoFullName, treeResult.paths, framework);
|
|
1913
|
+
if (freshInventory.length > 0) {
|
|
1914
|
+
pageObjectInventory = freshInventory;
|
|
1915
|
+
await updateJobProgress(tenantId, jobId, 82, `Found ${pageObjectInventory.length} page objects`);
|
|
1916
|
+
}
|
|
1917
|
+
} catch {
|
|
1918
|
+
}
|
|
1919
|
+
await updateJobProgress(tenantId, jobId, 85, "Running coverage mapping");
|
|
1920
|
+
const { buildCoverageMap } = await import("@detiq/agents");
|
|
1921
|
+
const testCases = await withTenant(
|
|
1922
|
+
tenantId,
|
|
1923
|
+
(tx) => tx.testCase.findMany({ where: { projectId, tenantId, status: { not: "RETIRED" } }, select: { id: true, title: true, ref: true } })
|
|
1924
|
+
);
|
|
1925
|
+
const mappings = await buildCoverageMap(testCases, allExtracted, tenantId);
|
|
1926
|
+
for (const m of mappings) {
|
|
1927
|
+
const existing = await withTenant(
|
|
1928
|
+
tenantId,
|
|
1929
|
+
(tx) => tx.coverageMapping.findUnique({ where: { testCaseId: m.testCaseId } })
|
|
1930
|
+
);
|
|
1931
|
+
if (existing?.manualOverride) continue;
|
|
1932
|
+
await withTenant(
|
|
1933
|
+
tenantId,
|
|
1934
|
+
(tx) => tx.coverageMapping.upsert({
|
|
1935
|
+
where: { testCaseId: m.testCaseId },
|
|
1936
|
+
create: { tenantId, projectId, testCaseId: m.testCaseId, repoConnectionId, status: m.status, matchedFilePath: m.matchedFilePath ?? null, scriptTestName: m.scriptTestName ?? null, lineNumber: m.lineNumber ?? null, confidence: m.confidence, matchMethod: m.matchMethod },
|
|
1937
|
+
update: { status: m.status, matchedFilePath: m.matchedFilePath ?? null, scriptTestName: m.scriptTestName ?? null, lineNumber: m.lineNumber ?? null, confidence: m.confidence, matchMethod: m.matchMethod }
|
|
1938
|
+
})
|
|
1939
|
+
);
|
|
2090
1940
|
}
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
|
|
2094
|
-
|
|
2095
|
-
|
|
1941
|
+
const mapped = mappings.filter((m) => m.status === "MAPPED").length;
|
|
1942
|
+
const unmapped = mappings.filter((m) => m.status === "UNMAPPED").length;
|
|
1943
|
+
const ambiguous = mappings.filter((m) => m.status === "AMBIGUOUS").length;
|
|
1944
|
+
const coveragePercent = testCases.length > 0 ? mapped / testCases.length * 100 : 0;
|
|
1945
|
+
await withTenant(
|
|
1946
|
+
tenantId,
|
|
1947
|
+
(tx) => tx.coverageSnapshot.create({
|
|
1948
|
+
data: { tenantId, projectId, totalCases: testCases.length, mappedCases: mapped, unmappedCases: unmapped, ambiguousCases: ambiguous, coveragePercent }
|
|
1949
|
+
})
|
|
1950
|
+
);
|
|
1951
|
+
const resolvedStyleProfile = (() => {
|
|
1952
|
+
const sp = styleProfile ?? {};
|
|
1953
|
+
if (framework === "selenium-java" && !sp.testStructurePattern) {
|
|
1954
|
+
const hasMavenMain = treeResult.paths.some((p) => /^src\/main\/java\//.test(p));
|
|
1955
|
+
if (hasMavenMain) return { ...sp, testStructurePattern: "two-tier" };
|
|
1956
|
+
}
|
|
1957
|
+
return sp;
|
|
1958
|
+
})();
|
|
1959
|
+
await withTenant(
|
|
1960
|
+
tenantId,
|
|
1961
|
+
(tx) => tx.gitHubRepoConnection.update({
|
|
1962
|
+
where: { id: repoConnectionId },
|
|
1963
|
+
data: {
|
|
1964
|
+
scanStatus: "DONE",
|
|
1965
|
+
detectedFramework: framework,
|
|
1966
|
+
configFilePath: primary?.configPath ?? null,
|
|
1967
|
+
testGlob,
|
|
1968
|
+
testFileCount: testPaths.length,
|
|
1969
|
+
extractedIdCount: allExtracted.length,
|
|
1970
|
+
styleProfile: resolvedStyleProfile,
|
|
1971
|
+
pageObjectInventory: pageObjectInventory.length > 0 ? pageObjectInventory : void 0,
|
|
1972
|
+
treeSnapshot: JSON.stringify(treeResult.paths).slice(0, 1e5),
|
|
1973
|
+
lastScannedAt: /* @__PURE__ */ new Date(),
|
|
1974
|
+
lastSyncAt: /* @__PURE__ */ new Date()
|
|
1975
|
+
}
|
|
1976
|
+
})
|
|
1977
|
+
);
|
|
1978
|
+
try {
|
|
1979
|
+
const { generateRepoSkillFiles } = await import("@detiq/agents");
|
|
1980
|
+
await generateRepoSkillFiles(tenantId, projectId, {
|
|
1981
|
+
repoFullName: conn.repoFullName,
|
|
1982
|
+
framework,
|
|
1983
|
+
language,
|
|
1984
|
+
testGlob,
|
|
1985
|
+
styleProfile: resolvedStyleProfile,
|
|
1986
|
+
pageObjectInventory,
|
|
1987
|
+
testPaths,
|
|
1988
|
+
extractedTestCount: allExtracted.length
|
|
1989
|
+
});
|
|
1990
|
+
} catch (e) {
|
|
1991
|
+
logger.warn({ jobId, repoConnectionId, err: e?.message }, "[repo-scan] repo skill files generation failed (non-fatal)");
|
|
2096
1992
|
}
|
|
1993
|
+
await updateJobProgress(tenantId, jobId, 100, `Done: ${mapped} mapped, ${unmapped} unmapped`);
|
|
1994
|
+
await withTenant(
|
|
1995
|
+
tenantId,
|
|
1996
|
+
(tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "DONE", finishedAt: /* @__PURE__ */ new Date(), result: { testFileCount: testPaths.length, extractedIdCount: allExtracted.length, mapped, unmapped, ambiguous, coveragePercent } } })
|
|
1997
|
+
);
|
|
1998
|
+
aggregateJobTokens(tenantId, jobId, projectId, jobStart).catch(console.error);
|
|
1999
|
+
} catch (err) {
|
|
2000
|
+
const msg = String(err?.message ?? err);
|
|
2001
|
+
logger.error({ jobId, repoConnectionId, err: msg }, "[repo-scan] failed");
|
|
2002
|
+
await withTenant(
|
|
2003
|
+
tenantId,
|
|
2004
|
+
(tx) => tx.gitHubRepoConnection.update({ where: { id: repoConnectionId }, data: { scanStatus: "FAILED", scanError: msg.slice(0, 500) } })
|
|
2005
|
+
).catch(() => {
|
|
2006
|
+
});
|
|
2007
|
+
await withTenant(
|
|
2008
|
+
tenantId,
|
|
2009
|
+
(tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", finishedAt: /* @__PURE__ */ new Date(), error: msg.slice(0, 1e3) } })
|
|
2010
|
+
).catch(() => {
|
|
2011
|
+
});
|
|
2012
|
+
}
|
|
2097
2013
|
}
|
|
2098
|
-
// ── BATCH_GENERATE_ALIGNED — generate framework-aligned scripts for UNMAPPED cases ──
|
|
2099
2014
|
async function processBatchGenerateAlignedJob(jobId, config) {
|
|
2100
|
-
|
|
2101
|
-
|
|
2102
|
-
|
|
2103
|
-
|
|
2104
|
-
|
|
2105
|
-
|
|
2106
|
-
|
|
2107
|
-
|
|
2108
|
-
|
|
2109
|
-
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
|
|
2113
|
-
|
|
2114
|
-
|
|
2115
|
-
|
|
2116
|
-
|
|
2117
|
-
|
|
2118
|
-
|
|
2119
|
-
|
|
2120
|
-
|
|
2121
|
-
|
|
2122
|
-
|
|
2123
|
-
|
|
2124
|
-
|
|
2125
|
-
|
|
2126
|
-
|
|
2127
|
-
|
|
2128
|
-
|
|
2129
|
-
|
|
2015
|
+
const { tenantId, projectId, repoConnectionId, scope = "UNMAPPED", testCaseIds, force = false } = config;
|
|
2016
|
+
try {
|
|
2017
|
+
await updateJobProgress(tenantId, jobId, 5, "Loading repo connection");
|
|
2018
|
+
const conn = await withTenant(
|
|
2019
|
+
tenantId,
|
|
2020
|
+
(tx) => tx.gitHubRepoConnection.findFirst({ where: { id: repoConnectionId, tenantId } })
|
|
2021
|
+
);
|
|
2022
|
+
if (!conn) throw new Error(`RepoConnection ${repoConnectionId} not found`);
|
|
2023
|
+
const { decrypt: decrypt2 } = await import("@detiq/core");
|
|
2024
|
+
const {
|
|
2025
|
+
generateScriptAligned,
|
|
2026
|
+
deriveRepoTargetPath,
|
|
2027
|
+
validateGeneratedScript,
|
|
2028
|
+
mapWithConcurrency,
|
|
2029
|
+
SCRIPT_GEN_CONCURRENCY
|
|
2030
|
+
} = await import("@detiq/agents");
|
|
2031
|
+
const styleProfile = conn.styleProfile;
|
|
2032
|
+
const pageObjectInventory = conn.pageObjectInventory ?? [];
|
|
2033
|
+
const framework = conn.detectedFramework ?? "playwright-ts";
|
|
2034
|
+
const language = styleProfile?.language ?? "typescript";
|
|
2035
|
+
let caseIds;
|
|
2036
|
+
if (testCaseIds && testCaseIds.length > 0) {
|
|
2037
|
+
caseIds = testCaseIds;
|
|
2038
|
+
} else if (scope === "UNMAPPED") {
|
|
2039
|
+
const mappings = await withTenant(
|
|
2040
|
+
tenantId,
|
|
2041
|
+
(tx) => tx.coverageMapping.findMany({
|
|
2042
|
+
where: { tenantId, projectId, repoConnectionId, status: "UNMAPPED" },
|
|
2043
|
+
select: { testCaseId: true }
|
|
2044
|
+
})
|
|
2045
|
+
);
|
|
2046
|
+
caseIds = mappings.map((m) => m.testCaseId);
|
|
2047
|
+
} else {
|
|
2048
|
+
const cases = await withTenant(
|
|
2049
|
+
tenantId,
|
|
2050
|
+
(tx) => tx.testCase.findMany({
|
|
2051
|
+
where: { tenantId, projectId, status: { not: "RETIRED" } },
|
|
2052
|
+
select: { id: true }
|
|
2053
|
+
})
|
|
2054
|
+
);
|
|
2055
|
+
caseIds = cases.map((c) => c.id);
|
|
2056
|
+
}
|
|
2057
|
+
await updateJobProgress(tenantId, jobId, 15, `Generating scripts for ${caseIds.length} test cases`);
|
|
2058
|
+
let done = 0;
|
|
2059
|
+
const errors = [];
|
|
2060
|
+
await mapWithConcurrency(caseIds, SCRIPT_GEN_CONCURRENCY, async (tcId) => {
|
|
2061
|
+
try {
|
|
2062
|
+
if (!force) {
|
|
2063
|
+
const existing = await withTenant(
|
|
2064
|
+
tenantId,
|
|
2065
|
+
(tx) => tx.testScript.findFirst({ where: { testCaseId: tcId } })
|
|
2066
|
+
);
|
|
2067
|
+
if (existing) {
|
|
2068
|
+
done++;
|
|
2069
|
+
return;
|
|
2070
|
+
}
|
|
2130
2071
|
}
|
|
2131
|
-
|
|
2132
|
-
|
|
2133
|
-
|
|
2134
|
-
|
|
2135
|
-
|
|
2136
|
-
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
|
|
2144
|
-
|
|
2145
|
-
|
|
2146
|
-
|
|
2147
|
-
|
|
2148
|
-
where: { id: tcId },
|
|
2149
|
-
select: { id: true, ref: true, title: true, screen: { select: { url: true } } },
|
|
2150
|
-
}));
|
|
2151
|
-
const screenUrl = tc?.screen?.url ?? '';
|
|
2152
|
-
const repoTargetPath = styleProfile
|
|
2153
|
-
? deriveRepoTargetPath(screenUrl, styleProfile, framework)
|
|
2154
|
-
: deriveFallbackTargetPath(tc?.ref ?? tc?.title ?? tcId, framework);
|
|
2155
|
-
const result = await generateScriptAligned({
|
|
2156
|
-
tenantId, projectId, testCaseId: tcId,
|
|
2157
|
-
styleProfile,
|
|
2158
|
-
pageObjectInventory,
|
|
2159
|
-
repoTargetPath,
|
|
2160
|
-
save: true,
|
|
2161
|
-
});
|
|
2162
|
-
// Validate the generated script
|
|
2163
|
-
if (result.code) {
|
|
2164
|
-
const validation = await validateGeneratedScript(result.code, framework, language);
|
|
2165
|
-
await withTenant(tenantId, (tx) => tx.testScript.updateMany({
|
|
2166
|
-
where: { testCaseId: tcId },
|
|
2167
|
-
data: {
|
|
2168
|
-
validationStatus: validation.ok ? 'PASS' : 'FAIL',
|
|
2169
|
-
validationErrors: validation.ok ? null : validation.errors.slice(0, 2000),
|
|
2170
|
-
},
|
|
2171
|
-
}));
|
|
2172
|
-
}
|
|
2173
|
-
done++;
|
|
2174
|
-
}
|
|
2175
|
-
catch (e) {
|
|
2176
|
-
errors.push(`${tcId}: ${String(e.message).slice(0, 100)}`);
|
|
2177
|
-
}
|
|
2178
|
-
if (done % 5 === 0) {
|
|
2179
|
-
await updateJobProgress(tenantId, jobId, 15 + Math.floor((done / caseIds.length) * 80), `Generated ${done}/${caseIds.length}`);
|
|
2180
|
-
}
|
|
2072
|
+
const tc = await withTenant(
|
|
2073
|
+
tenantId,
|
|
2074
|
+
(tx) => tx.testCase.findUnique({
|
|
2075
|
+
where: { id: tcId },
|
|
2076
|
+
select: { id: true, ref: true, title: true, screen: { select: { url: true } } }
|
|
2077
|
+
})
|
|
2078
|
+
);
|
|
2079
|
+
const screenUrl = tc?.screen?.url ?? "";
|
|
2080
|
+
const repoTargetPath = styleProfile ? deriveRepoTargetPath(screenUrl, styleProfile, framework) : deriveFallbackTargetPath(tc?.ref ?? tc?.title ?? tcId, framework);
|
|
2081
|
+
const result = await generateScriptAligned({
|
|
2082
|
+
tenantId,
|
|
2083
|
+
projectId,
|
|
2084
|
+
testCaseId: tcId,
|
|
2085
|
+
styleProfile,
|
|
2086
|
+
pageObjectInventory,
|
|
2087
|
+
repoTargetPath,
|
|
2088
|
+
save: true
|
|
2181
2089
|
});
|
|
2182
|
-
|
|
2183
|
-
|
|
2184
|
-
|
|
2185
|
-
|
|
2186
|
-
|
|
2187
|
-
|
|
2188
|
-
|
|
2189
|
-
|
|
2190
|
-
|
|
2191
|
-
|
|
2192
|
-
|
|
2090
|
+
if (result.code) {
|
|
2091
|
+
const validation = await validateGeneratedScript(result.code, framework, language);
|
|
2092
|
+
await withTenant(
|
|
2093
|
+
tenantId,
|
|
2094
|
+
(tx) => tx.testScript.updateMany({
|
|
2095
|
+
where: { testCaseId: tcId },
|
|
2096
|
+
data: {
|
|
2097
|
+
validationStatus: validation.ok ? "PASS" : "FAIL",
|
|
2098
|
+
validationErrors: validation.ok ? null : validation.errors.slice(0, 2e3)
|
|
2099
|
+
}
|
|
2100
|
+
})
|
|
2101
|
+
);
|
|
2102
|
+
}
|
|
2103
|
+
done++;
|
|
2104
|
+
} catch (e) {
|
|
2105
|
+
errors.push(`${tcId}: ${String(e.message).slice(0, 100)}`);
|
|
2106
|
+
}
|
|
2107
|
+
if (done % 5 === 0) {
|
|
2108
|
+
await updateJobProgress(tenantId, jobId, 15 + Math.floor(done / caseIds.length * 80), `Generated ${done}/${caseIds.length}`);
|
|
2109
|
+
}
|
|
2110
|
+
});
|
|
2111
|
+
refreshTestScriptSkillFile(tenantId, projectId).catch(console.error);
|
|
2112
|
+
await withTenant(
|
|
2113
|
+
tenantId,
|
|
2114
|
+
(tx) => tx.platformJob.update({
|
|
2115
|
+
where: { id: jobId },
|
|
2116
|
+
data: { status: "DONE", finishedAt: /* @__PURE__ */ new Date(), result: { done, errors: errors.slice(0, 20), total: caseIds.length } }
|
|
2117
|
+
})
|
|
2118
|
+
);
|
|
2119
|
+
} catch (err) {
|
|
2120
|
+
const msg = String(err?.message ?? err);
|
|
2121
|
+
logger.error({ jobId, err: msg }, "[batch-generate-aligned] failed");
|
|
2122
|
+
await withTenant(
|
|
2123
|
+
tenantId,
|
|
2124
|
+
(tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", finishedAt: /* @__PURE__ */ new Date(), error: msg.slice(0, 1e3) } })
|
|
2125
|
+
).catch(() => {
|
|
2126
|
+
});
|
|
2127
|
+
}
|
|
2193
2128
|
}
|
|
2194
|
-
// ── CREATE_PR — push generated scripts to GitHub as a PR ─────────────────────
|
|
2195
2129
|
async function processCreatePRJob(jobId, config) {
|
|
2196
|
-
|
|
2197
|
-
|
|
2198
|
-
|
|
2199
|
-
|
|
2200
|
-
|
|
2201
|
-
|
|
2202
|
-
|
|
2203
|
-
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2207
|
-
|
|
2208
|
-
|
|
2209
|
-
|
|
2210
|
-
|
|
2211
|
-
|
|
2212
|
-
|
|
2213
|
-
|
|
2214
|
-
|
|
2215
|
-
|
|
2216
|
-
|
|
2217
|
-
|
|
2218
|
-
|
|
2219
|
-
|
|
2220
|
-
|
|
2221
|
-
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
?? deriveFallbackTargetPath(s.testCase?.ref ?? s.testCase?.title ?? s.id, framework);
|
|
2226
|
-
return { path, content: s.code };
|
|
2227
|
-
});
|
|
2228
|
-
if (files.length === 0)
|
|
2229
|
-
throw new Error('No scripts to push.');
|
|
2230
|
-
await updateJobProgress(tenantId, jobId, 20, `Pushing ${files.length} scripts to GitHub`);
|
|
2231
|
-
const now = new Date().toISOString().slice(0, 10);
|
|
2232
|
-
const branchName = `zeta/new-coverage-${now}-${jobId.slice(0, 8)}`;
|
|
2233
|
-
const prBody = buildPRBody(scripts, projectId);
|
|
2234
|
-
const prResult = await pushScriptsAsPR({
|
|
2235
|
-
token,
|
|
2236
|
-
repo: conn.repoFullName,
|
|
2237
|
-
baseBranch: conn.defaultBranch,
|
|
2238
|
-
branchName,
|
|
2239
|
-
files,
|
|
2240
|
-
prTitle: `ZeTa: Add test coverage for ${files.length} new scenario${files.length === 1 ? '' : 's'}`,
|
|
2241
|
-
prBody,
|
|
2242
|
-
draft: false,
|
|
2243
|
-
});
|
|
2244
|
-
// Record the PR
|
|
2245
|
-
const pr = await withTenant(tenantId, (tx) => tx.gitHubPR.create({
|
|
2246
|
-
data: {
|
|
2247
|
-
tenantId, projectId,
|
|
2248
|
-
repoConnectionId,
|
|
2249
|
-
prNumber: prResult.prNumber,
|
|
2250
|
-
prUrl: prResult.prUrl,
|
|
2251
|
-
branch: prResult.branch,
|
|
2252
|
-
headSha: prResult.headSha,
|
|
2253
|
-
status: 'OPEN',
|
|
2254
|
-
crawlJobId: crawlJobId ?? null,
|
|
2255
|
-
scriptCount: files.length,
|
|
2256
|
-
prBody: prBody.slice(0, 10000),
|
|
2257
|
-
},
|
|
2258
|
-
}));
|
|
2259
|
-
// Mark scripts as pushed
|
|
2260
|
-
const scriptIds = scripts.map((s) => s.id);
|
|
2261
|
-
await withTenant(tenantId, (tx) => tx.testScript.updateMany({
|
|
2262
|
-
where: { id: { in: scriptIds } },
|
|
2263
|
-
data: { pushedToPR: true, pushedPRId: pr.id },
|
|
2264
|
-
}));
|
|
2265
|
-
await notifyAdmins(tenantId, {
|
|
2266
|
-
type: 'GITHUB_PR_CREATED',
|
|
2267
|
-
title: 'ZeTa created a GitHub PR',
|
|
2268
|
-
body: `${files.length} new test script${files.length === 1 ? '' : 's'} pushed. PR #${prResult.prNumber}: ${prResult.prUrl}`,
|
|
2269
|
-
metadata: { prUrl: prResult.prUrl, prNumber: prResult.prNumber, scriptCount: files.length },
|
|
2270
|
-
});
|
|
2271
|
-
await updateJobProgress(tenantId, jobId, 100, `PR #${prResult.prNumber} created`);
|
|
2272
|
-
await withTenant(tenantId, (tx) => tx.platformJob.update({
|
|
2273
|
-
where: { id: jobId },
|
|
2274
|
-
data: { status: 'DONE', finishedAt: new Date(), result: { prUrl: prResult.prUrl, prNumber: prResult.prNumber, scriptCount: files.length } },
|
|
2275
|
-
}));
|
|
2276
|
-
}
|
|
2277
|
-
catch (err) {
|
|
2278
|
-
const msg = String(err?.message ?? err);
|
|
2279
|
-
logger.error({ jobId, err: msg }, '[create-pr] failed');
|
|
2280
|
-
await withTenant(tenantId, (tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: 'FAILED', finishedAt: new Date(), error: msg.slice(0, 1000) } })).catch(() => { });
|
|
2130
|
+
const { tenantId, projectId, repoConnectionId, testScriptIds, crawlJobId } = config;
|
|
2131
|
+
try {
|
|
2132
|
+
await updateJobProgress(tenantId, jobId, 5, "Loading connection");
|
|
2133
|
+
const conn = await withTenant(
|
|
2134
|
+
tenantId,
|
|
2135
|
+
(tx) => tx.gitHubRepoConnection.findFirst({ where: { id: repoConnectionId, tenantId } })
|
|
2136
|
+
);
|
|
2137
|
+
if (!conn) throw new Error(`RepoConnection ${repoConnectionId} not found`);
|
|
2138
|
+
const { decrypt: decrypt2 } = await import("@detiq/core");
|
|
2139
|
+
const token = decrypt2(conn.tokenEnc);
|
|
2140
|
+
const { pushScriptsAsPR } = await import("@detiq/agents");
|
|
2141
|
+
const framework = conn.detectedFramework ?? "playwright-ts";
|
|
2142
|
+
let scripts;
|
|
2143
|
+
if (testScriptIds && testScriptIds.length > 0) {
|
|
2144
|
+
scripts = await withTenant(
|
|
2145
|
+
tenantId,
|
|
2146
|
+
(tx) => tx.testScript.findMany({
|
|
2147
|
+
where: { id: { in: testScriptIds }, tenantId },
|
|
2148
|
+
include: { testCase: { select: { title: true, ref: true, repoTargetPath: true } } }
|
|
2149
|
+
})
|
|
2150
|
+
);
|
|
2151
|
+
} else {
|
|
2152
|
+
scripts = await withTenant(
|
|
2153
|
+
tenantId,
|
|
2154
|
+
(tx) => tx.testScript.findMany({
|
|
2155
|
+
where: { tenantId, projectId, pushedToPR: false, validationStatus: { in: ["PASS", "PENDING"] } },
|
|
2156
|
+
include: { testCase: { select: { title: true, ref: true, repoTargetPath: true } } }
|
|
2157
|
+
})
|
|
2158
|
+
);
|
|
2281
2159
|
}
|
|
2160
|
+
if (scripts.length === 0) throw new Error("No generated scripts found to push.");
|
|
2161
|
+
const files = scripts.map((s) => {
|
|
2162
|
+
const path2 = s.testCase?.repoTargetPath ?? deriveFallbackTargetPath(s.testCase?.ref ?? s.testCase?.title ?? s.id, framework);
|
|
2163
|
+
return { path: path2, content: s.code };
|
|
2164
|
+
});
|
|
2165
|
+
if (files.length === 0) throw new Error("No scripts to push.");
|
|
2166
|
+
await updateJobProgress(tenantId, jobId, 20, `Pushing ${files.length} scripts to GitHub`);
|
|
2167
|
+
const now = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
|
|
2168
|
+
const branchName = `zeta/new-coverage-${now}-${jobId.slice(0, 8)}`;
|
|
2169
|
+
const prBody = buildPRBody(scripts, projectId);
|
|
2170
|
+
const prResult = await pushScriptsAsPR({
|
|
2171
|
+
token,
|
|
2172
|
+
repo: conn.repoFullName,
|
|
2173
|
+
baseBranch: conn.defaultBranch,
|
|
2174
|
+
branchName,
|
|
2175
|
+
files,
|
|
2176
|
+
prTitle: `ZeTa: Add test coverage for ${files.length} new scenario${files.length === 1 ? "" : "s"}`,
|
|
2177
|
+
prBody,
|
|
2178
|
+
draft: false
|
|
2179
|
+
});
|
|
2180
|
+
const pr = await withTenant(
|
|
2181
|
+
tenantId,
|
|
2182
|
+
(tx) => tx.gitHubPR.create({
|
|
2183
|
+
data: {
|
|
2184
|
+
tenantId,
|
|
2185
|
+
projectId,
|
|
2186
|
+
repoConnectionId,
|
|
2187
|
+
prNumber: prResult.prNumber,
|
|
2188
|
+
prUrl: prResult.prUrl,
|
|
2189
|
+
branch: prResult.branch,
|
|
2190
|
+
headSha: prResult.headSha,
|
|
2191
|
+
status: "OPEN",
|
|
2192
|
+
crawlJobId: crawlJobId ?? null,
|
|
2193
|
+
scriptCount: files.length,
|
|
2194
|
+
prBody: prBody.slice(0, 1e4)
|
|
2195
|
+
}
|
|
2196
|
+
})
|
|
2197
|
+
);
|
|
2198
|
+
const scriptIds = scripts.map((s) => s.id);
|
|
2199
|
+
await withTenant(
|
|
2200
|
+
tenantId,
|
|
2201
|
+
(tx) => tx.testScript.updateMany({
|
|
2202
|
+
where: { id: { in: scriptIds } },
|
|
2203
|
+
data: { pushedToPR: true, pushedPRId: pr.id }
|
|
2204
|
+
})
|
|
2205
|
+
);
|
|
2206
|
+
await notifyAdmins(tenantId, {
|
|
2207
|
+
type: "GITHUB_PR_CREATED",
|
|
2208
|
+
title: "ZeTa created a GitHub PR",
|
|
2209
|
+
body: `${files.length} new test script${files.length === 1 ? "" : "s"} pushed. PR #${prResult.prNumber}: ${prResult.prUrl}`,
|
|
2210
|
+
metadata: { prUrl: prResult.prUrl, prNumber: prResult.prNumber, scriptCount: files.length }
|
|
2211
|
+
});
|
|
2212
|
+
await updateJobProgress(tenantId, jobId, 100, `PR #${prResult.prNumber} created`);
|
|
2213
|
+
await withTenant(
|
|
2214
|
+
tenantId,
|
|
2215
|
+
(tx) => tx.platformJob.update({
|
|
2216
|
+
where: { id: jobId },
|
|
2217
|
+
data: { status: "DONE", finishedAt: /* @__PURE__ */ new Date(), result: { prUrl: prResult.prUrl, prNumber: prResult.prNumber, scriptCount: files.length } }
|
|
2218
|
+
})
|
|
2219
|
+
);
|
|
2220
|
+
} catch (err) {
|
|
2221
|
+
const msg = String(err?.message ?? err);
|
|
2222
|
+
logger.error({ jobId, err: msg }, "[create-pr] failed");
|
|
2223
|
+
await withTenant(
|
|
2224
|
+
tenantId,
|
|
2225
|
+
(tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", finishedAt: /* @__PURE__ */ new Date(), error: msg.slice(0, 1e3) } })
|
|
2226
|
+
).catch(() => {
|
|
2227
|
+
});
|
|
2228
|
+
}
|
|
2282
2229
|
}
|
|
2283
2230
|
function buildPRBody(scripts, projectId) {
|
|
2284
|
-
|
|
2285
|
-
|
|
2286
|
-
|
|
2287
|
-
|
|
2288
|
-
|
|
2289
|
-
|
|
2290
|
-
|
|
2291
|
-
|
|
2292
|
-
|
|
2293
|
-
|
|
2294
|
-
|
|
2295
|
-
|
|
2296
|
-
|
|
2297
|
-
|
|
2298
|
-
|
|
2299
|
-
|
|
2300
|
-
|
|
2301
|
-
|
|
2302
|
-
return lines.join('\n');
|
|
2231
|
+
const lines = [
|
|
2232
|
+
"## ZeTa \u2014 Automated Test Coverage",
|
|
2233
|
+
"",
|
|
2234
|
+
`Generated by ZeTa Quality Intelligence Platform for project \`${projectId}\`.`,
|
|
2235
|
+
"",
|
|
2236
|
+
"### Scripts Added",
|
|
2237
|
+
"",
|
|
2238
|
+
"| Test Case | Target Path |",
|
|
2239
|
+
"|---|---|"
|
|
2240
|
+
];
|
|
2241
|
+
for (const s of scripts.slice(0, 30)) {
|
|
2242
|
+
const title = s.testCase?.title ?? s.id;
|
|
2243
|
+
const path2 = s.testCase?.repoTargetPath ?? "(unknown)";
|
|
2244
|
+
lines.push(`| ${title.replace(/\|/g, "\\|").slice(0, 60)} | \`${path2}\` |`);
|
|
2245
|
+
}
|
|
2246
|
+
if (scripts.length > 30) lines.push(`| ... and ${scripts.length - 30} more | |`);
|
|
2247
|
+
lines.push("", "---", "_Review each test carefully before merging. ZeTa generates scripts aligned to your existing framework conventions._");
|
|
2248
|
+
return lines.join("\n");
|
|
2303
2249
|
}
|
|
2304
|
-
// ── NEW_PAGES_PIPELINE — post-crawl: detect new screens, generate tests + scripts ──
|
|
2305
2250
|
async function processNewPagesPipelineJob(jobId, config) {
|
|
2306
|
-
|
|
2307
|
-
|
|
2308
|
-
|
|
2309
|
-
|
|
2310
|
-
|
|
2311
|
-
|
|
2312
|
-
|
|
2313
|
-
|
|
2314
|
-
|
|
2315
|
-
|
|
2316
|
-
|
|
2317
|
-
|
|
2318
|
-
|
|
2319
|
-
|
|
2320
|
-
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
|
|
2333
|
-
|
|
2334
|
-
|
|
2335
|
-
|
|
2336
|
-
|
|
2337
|
-
|
|
2338
|
-
|
|
2339
|
-
|
|
2340
|
-
|
|
2341
|
-
|
|
2342
|
-
|
|
2343
|
-
|
|
2344
|
-
|
|
2345
|
-
|
|
2346
|
-
|
|
2347
|
-
|
|
2348
|
-
|
|
2349
|
-
|
|
2350
|
-
|
|
2351
|
-
|
|
2352
|
-
|
|
2353
|
-
|
|
2354
|
-
}
|
|
2355
|
-
await withTenant(tenantId, (tx) => tx.platformJob.update({
|
|
2356
|
-
where: { id: jobId },
|
|
2357
|
-
data: { status: 'DONE', finishedAt: new Date(), result: { newScreens: newScreens.length, generationJobs: generationJobs.length } },
|
|
2358
|
-
}));
|
|
2251
|
+
const { tenantId, projectId, crawlJobId, repoConnectionId, autoPR = false } = config;
|
|
2252
|
+
try {
|
|
2253
|
+
await updateJobProgress(tenantId, jobId, 5, "Detecting new screens");
|
|
2254
|
+
const conn = await withTenant(
|
|
2255
|
+
tenantId,
|
|
2256
|
+
(tx) => tx.gitHubRepoConnection.findFirst({
|
|
2257
|
+
where: repoConnectionId ? { id: repoConnectionId, tenantId } : { tenantId, projectId, status: "ACTIVE" }
|
|
2258
|
+
})
|
|
2259
|
+
);
|
|
2260
|
+
const sinceDate = conn?.lastSyncAt ?? /* @__PURE__ */ new Date(0);
|
|
2261
|
+
const newScreens = await withTenant(
|
|
2262
|
+
tenantId,
|
|
2263
|
+
(tx) => tx.screen.findMany({
|
|
2264
|
+
where: {
|
|
2265
|
+
tenantId,
|
|
2266
|
+
projectId,
|
|
2267
|
+
lastSeenAt: { gt: sinceDate },
|
|
2268
|
+
status: { not: "DELETED" }
|
|
2269
|
+
},
|
|
2270
|
+
select: { id: true, url: true, name: true },
|
|
2271
|
+
take: 100
|
|
2272
|
+
})
|
|
2273
|
+
);
|
|
2274
|
+
if (newScreens.length === 0) {
|
|
2275
|
+
await updateJobProgress(tenantId, jobId, 100, "No new screens to process");
|
|
2276
|
+
await withTenant(
|
|
2277
|
+
tenantId,
|
|
2278
|
+
(tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "DONE", finishedAt: /* @__PURE__ */ new Date(), result: { newScreens: 0 } } })
|
|
2279
|
+
);
|
|
2280
|
+
return;
|
|
2281
|
+
}
|
|
2282
|
+
await updateJobProgress(tenantId, jobId, 15, `Found ${newScreens.length} new screens`);
|
|
2283
|
+
const generationJobs = [];
|
|
2284
|
+
for (const screen of newScreens) {
|
|
2285
|
+
const caseCount = await withTenant(
|
|
2286
|
+
tenantId,
|
|
2287
|
+
(tx) => tx.testCase.count({ where: { tenantId, screenId: screen.id } })
|
|
2288
|
+
);
|
|
2289
|
+
if (caseCount === 0) {
|
|
2290
|
+
const jid = await enqueueJob("GENERATE_SCREEN_TESTS", {
|
|
2291
|
+
tenantId,
|
|
2292
|
+
projectId,
|
|
2293
|
+
screenId: screen.id,
|
|
2294
|
+
triggerAlignedScripts: conn != null,
|
|
2295
|
+
repoConnectionId: conn?.id ?? null
|
|
2296
|
+
});
|
|
2297
|
+
generationJobs.push(jid);
|
|
2298
|
+
}
|
|
2359
2299
|
}
|
|
2360
|
-
|
|
2361
|
-
|
|
2362
|
-
|
|
2363
|
-
|
|
2300
|
+
await updateJobProgress(tenantId, jobId, 60, `Queued ${generationJobs.length} test generation jobs`);
|
|
2301
|
+
if (conn && autoPR) {
|
|
2302
|
+
await enqueueJob("BATCH_GENERATE_ALIGNED", {
|
|
2303
|
+
tenantId,
|
|
2304
|
+
projectId,
|
|
2305
|
+
repoConnectionId: conn.id,
|
|
2306
|
+
scope: "UNMAPPED"
|
|
2307
|
+
});
|
|
2364
2308
|
}
|
|
2309
|
+
await withTenant(
|
|
2310
|
+
tenantId,
|
|
2311
|
+
(tx) => tx.platformJob.update({
|
|
2312
|
+
where: { id: jobId },
|
|
2313
|
+
data: { status: "DONE", finishedAt: /* @__PURE__ */ new Date(), result: { newScreens: newScreens.length, generationJobs: generationJobs.length } }
|
|
2314
|
+
})
|
|
2315
|
+
);
|
|
2316
|
+
} catch (err) {
|
|
2317
|
+
const msg = String(err?.message ?? err);
|
|
2318
|
+
logger.error({ jobId, err: msg }, "[new-pages-pipeline] failed");
|
|
2319
|
+
await withTenant(
|
|
2320
|
+
tenantId,
|
|
2321
|
+
(tx) => tx.platformJob.update({ where: { id: jobId }, data: { status: "FAILED", finishedAt: /* @__PURE__ */ new Date(), error: msg.slice(0, 1e3) } })
|
|
2322
|
+
).catch(() => {
|
|
2323
|
+
});
|
|
2324
|
+
}
|
|
2365
2325
|
}
|
|
2326
|
+
export {
|
|
2327
|
+
aggregateJobTokens,
|
|
2328
|
+
cancelJob,
|
|
2329
|
+
claimNextQueuedJob,
|
|
2330
|
+
claimSpecificJob,
|
|
2331
|
+
createTrackedJob,
|
|
2332
|
+
enqueueJob,
|
|
2333
|
+
finishTrackedJob,
|
|
2334
|
+
generateTOTP,
|
|
2335
|
+
getJob,
|
|
2336
|
+
listJobs,
|
|
2337
|
+
processDataRetentionPurge,
|
|
2338
|
+
processJob,
|
|
2339
|
+
requeueOrphanedJobs,
|
|
2340
|
+
requeueStaleRunningJobs,
|
|
2341
|
+
rerunJob,
|
|
2342
|
+
resumeJob,
|
|
2343
|
+
startJobWorkerLoop,
|
|
2344
|
+
stopJobWorkerLoop,
|
|
2345
|
+
updateJobProgress
|
|
2346
|
+
};
|
|
2347
|
+
//# sourceMappingURL=job-queue.js.map
|