@detiq/crawler 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Dockerfile +4 -0
- package/dist/ai-login.js +10 -2
- package/dist/ai-login.js.map +2 -2
- package/dist/crawler.js +12 -8
- package/dist/crawler.js.map +3 -3
- package/dist/failure-analysis-agent.js +2 -2
- package/dist/failure-analysis-agent.js.map +1 -1
- package/dist/interactive-probe.js +1 -1
- package/dist/interactive-probe.js.map +1 -1
- package/dist/job-queue.js +32 -23
- package/dist/job-queue.js.map +2 -2
- package/dist/live-view.js +2 -1
- package/dist/live-view.js.map +2 -2
- package/dist/native-crawler.js +3 -3
- package/dist/native-crawler.js.map +1 -1
- package/dist/session-capture.js +2 -1
- package/dist/session-capture.js.map +2 -2
- package/dist/sitemap.js +1 -1
- package/dist/sitemap.js.map +1 -1
- package/dist/skyvern.js +4 -0
- package/dist/skyvern.js.map +2 -2
- package/dist/test-executor.js +11 -2
- package/dist/test-executor.js.map +2 -2
- package/dist/visual-diff-verdict.js.map +1 -1
- package/dist/worker.js.map +1 -1
- package/package.json +5 -4
- package/src/ai-login.ts +11 -1
- package/src/crawler.ts +10 -5
- package/src/job-queue.ts +13 -1
- package/src/live-view.ts +2 -1
- package/src/session-capture.ts +2 -1
- package/src/skyvern.ts +6 -0
- package/src/test-executor.ts +11 -2
package/dist/sitemap.js.map
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../src/sitemap.ts"],
|
|
4
|
-
"sourcesContent": ["/**\n * Sitemap.xml discovery \u2014 seeds the crawl queue from a project's sitemap\n * instead of (or alongside) plain link-following, matching crawl4AI's\n * sitemap-seeding capability. Best-effort throughout: most sites don't have\n * a sitemap, or it 404s, or it's malformed \u2014 none of that should ever fail\n * a crawl, it just means sitemap seeding contributes nothing this run.\n */\nimport { XMLParser } from 'fast-xml-parser';\nimport { ProxyAgent } from 'undici';\nimport { assertCrawlUrlSafe } from '@
|
|
4
|
+
"sourcesContent": ["/**\n * Sitemap.xml discovery \u2014 seeds the crawl queue from a project's sitemap\n * instead of (or alongside) plain link-following, matching crawl4AI's\n * sitemap-seeding capability. Best-effort throughout: most sites don't have\n * a sitemap, or it 404s, or it's malformed \u2014 none of that should ever fail\n * a crawl, it just means sitemap seeding contributes nothing this run.\n */\nimport { XMLParser } from 'fast-xml-parser';\nimport { ProxyAgent } from 'undici';\nimport { assertCrawlUrlSafe } from '@detiq/core';\nimport { logger } from './logger.js';\n\nconst FETCH_TIMEOUT_MS = 10_000;\nconst MAX_SITEMAP_INDEX_DEPTH = 3; // sitemap index -> sub-sitemaps -> (no deeper nesting expected)\nconst MAX_SUB_SITEMAPS = 50; // bound fan-out from a single sitemap index\nconst MAX_URLS = 5000; // sane upper bound so a huge/malicious sitemap can't blow up memory\n\nfunction toArray<T>(x: T | T[] | undefined | null): T[] {\n if (x == null) return [];\n return Array.isArray(x) ? x : [x];\n}\n\n/** Registrable-host key: lowercased hostname with a leading `www.` stripped. */\nfunction hostKey(hostname: string): string {\n return hostname.replace(/^www\\./i, '').toLowerCase();\n}\n\n/**\n * Same-site check that tolerates www/non-www and http/https differences \u2014 a sitemap on\n * `vercel.com` is valid for a project pointed at `www.vercel.com` (and vice versa).\n */\nexport function sameSite(candidateUrl: string, siteOrigin: string): boolean {\n try {\n const originHost = /^https?:\\/\\//i.test(siteOrigin) ? new URL(siteOrigin).hostname : siteOrigin;\n return hostKey(new URL(candidateUrl).hostname) === hostKey(originHost);\n } catch {\n return false;\n }\n}\n\n/**\n * Pulls page URLs and sub-sitemap URLs out of an already-XML-parsed sitemap\n * document (either a <urlset> or a <sitemapindex>), filtered to same-origin\n * and deduped. Pure function \u2014 no I/O \u2014 for direct unit testing without a\n * real HTTP fetch.\n */\nexport function extractUrlsFromParsedSitemap(parsed: any, origin: string): { pageUrls: string[]; subSitemapUrls: string[] } {\n const rawPageUrls: string[] = [];\n let subSitemapUrls: string[] = [];\n\n if (parsed?.sitemapindex) {\n subSitemapUrls = toArray(parsed.sitemapindex.sitemap)\n .map((s: any) => s?.loc)\n .filter((loc: any): loc is string => typeof loc === 'string')\n .slice(0, MAX_SUB_SITEMAPS);\n }\n\n if (parsed?.urlset) {\n rawPageUrls.push(\n ...toArray(parsed.urlset.url)\n .map((u: any) => u?.loc)\n .filter((loc: any): loc is string => typeof loc === 'string'),\n );\n }\n\n // Same-site only (www/protocol-tolerant) \u2014 a sitemap listing external URLs shouldn't seed this project's queue.\n const sameOrigin = rawPageUrls.filter((u) => sameSite(u, origin));\n\n return { pageUrls: [...new Set(sameOrigin)].slice(0, MAX_URLS), subSitemapUrls };\n}\n\nasync function fetchAndParseXml(url: string, dispatcher?: any): Promise<any | null> {\n // Same SSRF posture as the crawl itself (assertCrawlUrlSafe, not the\n // stricter webhook-oriented assertResolvesToPublicAddress) \u2014 a sitemap\n // living on a tenant's internal/VPN-only staging server is a legitimate\n // crawl target, not an attack; only loopback/cloud-metadata are blocked.\n try {\n assertCrawlUrlSafe(url);\n } catch {\n return null;\n }\n try {\n // Redirects followed normally here (unlike the webhook-dispatch.ts\n // 'manual' pattern) \u2014 an http\u2192https or bare\u2192www redirect on the\n // sitemap is completely ordinary, and Playwright's actual page\n // navigation later in the crawl already follows redirects to whatever\n // this same site sends it to; a sitemap fetch isn't a more sensitive\n // operation than that.\n const res = await fetch(url, {\n signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),\n headers: { 'User-Agent': 'ZeTa-AI-Crawler/1.0 (+sitemap-discovery)' },\n ...(dispatcher ? { dispatcher } : {}),\n });\n if (!res.ok) return null;\n const text = await res.text();\n return new XMLParser().parse(text);\n } catch {\n return null;\n }\n}\n\n/**\n * Fetches and parses a sitemap (or sitemap index, recursed up to a small\n * depth) into a flat, deduped list of same-origin URLs.\n */\nexport async function fetchSitemapUrls(sitemapUrl: string, origin: string, depth = 0, dispatcher?: any): Promise<string[]> {\n if (depth > MAX_SITEMAP_INDEX_DEPTH) return [];\n const parsed = await fetchAndParseXml(sitemapUrl, dispatcher);\n if (!parsed) return [];\n\n const { pageUrls, subSitemapUrls } = extractUrlsFromParsedSitemap(parsed, origin);\n const collected = [...pageUrls];\n\n for (const subUrl of subSitemapUrls) {\n if (collected.length >= MAX_URLS) break;\n collected.push(...(await fetchSitemapUrls(subUrl, origin, depth + 1, dispatcher)));\n }\n\n return [...new Set(collected)].slice(0, MAX_URLS);\n}\n\n/**\n * Discovers a project's sitemap (default `${origin}/sitemap.xml`, or an\n * explicit override) and returns the URLs it lists. Never throws.\n */\nexport async function discoverSitemapUrls(appUrl: string, sitemapUrl?: string): Promise<string[]> {\n let origin: string;\n try {\n origin = new URL(appUrl).origin;\n } catch {\n return [];\n }\n const target = sitemapUrl || `${origin}/sitemap.xml`;\n try {\n return await fetchSitemapUrls(target, origin);\n } catch (err) {\n logger.warn({ err: String(err), target }, '[sitemap] discovery failed, continuing without it');\n return [];\n }\n}\n\n/**\n * Build an undici ProxyAgent from a proxy URL. Parses credentials manually because\n * DataImpulse-style usernames contain commas (country codes) that break new URL().\n * Returns undefined for no/auto proxy \u2014 the caller then fetches directly.\n */\nexport function buildProxyDispatcher(proxyUrl?: string): ProxyAgent | undefined {\n if (!proxyUrl || proxyUrl === 'auto') return undefined;\n try {\n const m = proxyUrl.match(/^(https?):\\/\\/(?:([^:@]*)(?::([^@]*))?@)?(.+)$/);\n if (!m) return new ProxyAgent(proxyUrl);\n const [, scheme, user, pass, hostPort] = m;\n const uri = `${scheme}://${hostPort}`;\n if (user) {\n const token = 'Basic ' + Buffer.from(`${decodeURIComponent(user)}:${decodeURIComponent(pass ?? '')}`).toString('base64');\n return new ProxyAgent({ uri, token });\n }\n return new ProxyAgent({ uri });\n } catch {\n return undefined;\n }\n}\n\n/** Extract same-origin `Sitemap:` directives from a robots.txt body. */\nexport function parseRobotsSitemaps(robotsTxt: string, origin: string): string[] {\n const out: string[] = [];\n for (const line of robotsTxt.split(/\\r?\\n/)) {\n const m = line.match(/^\\s*sitemap:\\s*(\\S+)/i);\n if (m && sameSite(m[1], origin)) out.push(m[1]);\n }\n return [...new Set(out)];\n}\n\nexport interface UrlCandidate {\n url: string;\n source: 'sitemap' | 'robots' | 'link';\n}\n\n/**\n * Pre-crawl URL discovery for the review-before-crawl flow: reads robots.txt Sitemap\n * directives and sitemap.xml (recursed, same-origin), proxy-aware. Returns a deduped\n * candidate list the user can review and seed before launching the Playwright crawl.\n * Never throws \u2014 bot-protected sites (Akamai/Cloudflare) simply yield an empty list.\n */\nexport async function discoverProjectUrls(\n appUrl: string,\n opts?: { sitemapUrl?: string; proxyUrl?: string },\n): Promise<{ candidates: UrlCandidate[]; sitemapFound: boolean }> {\n let origin: string;\n try { origin = new URL(appUrl).origin; } catch { return { candidates: [], sitemapFound: false }; }\n\n const dispatcher = buildProxyDispatcher(opts?.proxyUrl);\n\n // 1. robots.txt \u2192 declared sitemaps\n let robotsSitemaps: string[] = [];\n try {\n assertCrawlUrlSafe(`${origin}/robots.txt`);\n const res = await fetch(`${origin}/robots.txt`, {\n signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),\n headers: { 'User-Agent': 'ZeTa-AI-Crawler/1.0 (+url-discovery)' },\n ...(dispatcher ? { dispatcher } : {}),\n });\n if (res.ok) robotsSitemaps = parseRobotsSitemaps(await res.text(), origin);\n } catch { /* best-effort */ }\n\n // 2. sitemap targets: explicit override + robots-declared + conventional default\n const robotsSet = new Set(robotsSitemaps);\n const sitemapTargets = [...new Set([\n ...(opts?.sitemapUrl ? [opts.sitemapUrl] : []),\n ...robotsSitemaps,\n `${origin}/sitemap.xml`,\n ])];\n\n const seen = new Map<string, UrlCandidate>();\n let sitemapFound = false;\n for (const sm of sitemapTargets) {\n const urls = await fetchSitemapUrls(sm, origin, 0, dispatcher);\n if (urls.length) sitemapFound = true;\n const source: UrlCandidate['source'] = robotsSet.has(sm) ? 'robots' : 'sitemap';\n for (const u of urls) if (!seen.has(u)) seen.set(u, { url: u, source });\n }\n\n return { candidates: [...seen.values()].slice(0, MAX_URLS), sitemapFound };\n}\n"],
|
|
5
5
|
"mappings": "AAOA,SAAS,iBAAiB;AAC1B,SAAS,kBAAkB;AAC3B,SAAS,0BAA0B;AACnC,SAAS,cAAc;AAEvB,MAAM,mBAAmB;AACzB,MAAM,0BAA0B;AAChC,MAAM,mBAAmB;AACzB,MAAM,WAAW;AAEjB,SAAS,QAAW,GAAoC;AACtD,MAAI,KAAK,KAAM,QAAO,CAAC;AACvB,SAAO,MAAM,QAAQ,CAAC,IAAI,IAAI,CAAC,CAAC;AAClC;AAGA,SAAS,QAAQ,UAA0B;AACzC,SAAO,SAAS,QAAQ,WAAW,EAAE,EAAE,YAAY;AACrD;AAMO,SAAS,SAAS,cAAsB,YAA6B;AAC1E,MAAI;AACF,UAAM,aAAa,gBAAgB,KAAK,UAAU,IAAI,IAAI,IAAI,UAAU,EAAE,WAAW;AACrF,WAAO,QAAQ,IAAI,IAAI,YAAY,EAAE,QAAQ,MAAM,QAAQ,UAAU;AAAA,EACvE,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAQO,SAAS,6BAA6B,QAAa,QAAkE;AAC1H,QAAM,cAAwB,CAAC;AAC/B,MAAI,iBAA2B,CAAC;AAEhC,MAAI,QAAQ,cAAc;AACxB,qBAAiB,QAAQ,OAAO,aAAa,OAAO,EACjD,IAAI,CAAC,MAAW,GAAG,GAAG,EACtB,OAAO,CAAC,QAA4B,OAAO,QAAQ,QAAQ,EAC3D,MAAM,GAAG,gBAAgB;AAAA,EAC9B;AAEA,MAAI,QAAQ,QAAQ;AAClB,gBAAY;AAAA,MACV,GAAG,QAAQ,OAAO,OAAO,GAAG,EACzB,IAAI,CAAC,MAAW,GAAG,GAAG,EACtB,OAAO,CAAC,QAA4B,OAAO,QAAQ,QAAQ;AAAA,IAChE;AAAA,EACF;AAGA,QAAM,aAAa,YAAY,OAAO,CAAC,MAAM,SAAS,GAAG,MAAM,CAAC;AAEhE,SAAO,EAAE,UAAU,CAAC,GAAG,IAAI,IAAI,UAAU,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,eAAe;AACjF;AAEA,eAAe,iBAAiB,KAAa,YAAuC;AAKlF,MAAI;AACF,uBAAmB,GAAG;AAAA,EACxB,QAAQ;AACN,WAAO;AAAA,EACT;AACA,MAAI;AAOF,UAAM,MAAM,MAAM,MAAM,KAAK;AAAA,MAC3B,QAAQ,YAAY,QAAQ,gBAAgB;AAAA,MAC5C,SAAS,EAAE,cAAc,2CAA2C;AAAA,MACpE,GAAI,aAAa,EAAE,WAAW,IAAI,CAAC;AAAA,IACrC,CAAC;AACD,QAAI,CAAC,IAAI,GAAI,QAAO;AACpB,UAAM,OAAO,MAAM,IAAI,KAAK;AAC5B,WAAO,IAAI,UAAU,EAAE,MAAM,IAAI;AAAA,EACnC,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAMA,eAAsB,iBAAiB,YAAoB,QAAgB,QAAQ,GAAG,YAAqC;AACzH,MAAI,QAAQ,wBAAyB,QAAO,CAAC;AAC7C,QAAM,SAAS,MAAM,iBAAiB,YAAY,UAAU;AAC5D,MAAI,CAAC,OAAQ,QAAO,CAAC;AAErB,QAAM,EAAE,UAAU,eAAe,IAAI,6BAA6B,QAAQ,MAAM;AAChF,QAAM,YAAY,CAAC,GAAG,QAAQ;AAE9B,aAAW,UAAU,gBAAgB;AACnC,QAAI,UAAU,UAAU,SAAU;AAClC,cAAU,KAAK,GAAI,MAAM,iBAAiB,QAAQ,QAAQ,QAAQ,GAAG,UAAU,CAAE;AAAA,EACnF;AAEA,SAAO,CAAC,GAAG,IAAI,IAAI,SAAS,CAAC,EAAE,MAAM,GAAG,QAAQ;AAClD;AAMA,eAAsB,oBAAoB,QAAgB,YAAwC;AAChG,MAAI;AACJ,MAAI;AACF,aAAS,IAAI,IAAI,MAAM,EAAE;AAAA,EAC3B,QAAQ;AACN,WAAO,CAAC;AAAA,EACV;AACA,QAAM,SAAS,cAAc,GAAG,MAAM;AACtC,MAAI;AACF,WAAO,MAAM,iBAAiB,QAAQ,MAAM;AAAA,EAC9C,SAAS,KAAK;AACZ,WAAO,KAAK,EAAE,KAAK,OAAO,GAAG,GAAG,OAAO,GAAG,mDAAmD;AAC7F,WAAO,CAAC;AAAA,EACV;AACF;AAOO,SAAS,qBAAqB,UAA2C;AAC9E,MAAI,CAAC,YAAY,aAAa,OAAQ,QAAO;AAC7C,MAAI;AACF,UAAM,IAAI,SAAS,MAAM,gDAAgD;AACzE,QAAI,CAAC,EAAG,QAAO,IAAI,WAAW,QAAQ;AACtC,UAAM,CAAC,EAAE,QAAQ,MAAM,MAAM,QAAQ,IAAI;AACzC,UAAM,MAAM,GAAG,MAAM,MAAM,QAAQ;AACnC,QAAI,MAAM;AACR,YAAM,QAAQ,WAAW,OAAO,KAAK,GAAG,mBAAmB,IAAI,CAAC,IAAI,mBAAmB,QAAQ,EAAE,CAAC,EAAE,EAAE,SAAS,QAAQ;AACvH,aAAO,IAAI,WAAW,EAAE,KAAK,MAAM,CAAC;AAAA,IACtC;AACA,WAAO,IAAI,WAAW,EAAE,IAAI,CAAC;AAAA,EAC/B,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAGO,SAAS,oBAAoB,WAAmB,QAA0B;AAC/E,QAAM,MAAgB,CAAC;AACvB,aAAW,QAAQ,UAAU,MAAM,OAAO,GAAG;AAC3C,UAAM,IAAI,KAAK,MAAM,uBAAuB;AAC5C,QAAI,KAAK,SAAS,EAAE,CAAC,GAAG,MAAM,EAAG,KAAI,KAAK,EAAE,CAAC,CAAC;AAAA,EAChD;AACA,SAAO,CAAC,GAAG,IAAI,IAAI,GAAG,CAAC;AACzB;AAaA,eAAsB,oBACpB,QACA,MACgE;AAChE,MAAI;AACJ,MAAI;AAAE,aAAS,IAAI,IAAI,MAAM,EAAE;AAAA,EAAQ,QAAQ;AAAE,WAAO,EAAE,YAAY,CAAC,GAAG,cAAc,MAAM;AAAA,EAAG;AAEjG,QAAM,aAAa,qBAAqB,MAAM,QAAQ;AAGtD,MAAI,iBAA2B,CAAC;AAChC,MAAI;AACF,uBAAmB,GAAG,MAAM,aAAa;AACzC,UAAM,MAAM,MAAM,MAAM,GAAG,MAAM,eAAe;AAAA,MAC9C,QAAQ,YAAY,QAAQ,gBAAgB;AAAA,MAC5C,SAAS,EAAE,cAAc,uCAAuC;AAAA,MAChE,GAAI,aAAa,EAAE,WAAW,IAAI,CAAC;AAAA,IACrC,CAAC;AACD,QAAI,IAAI,GAAI,kBAAiB,oBAAoB,MAAM,IAAI,KAAK,GAAG,MAAM;AAAA,EAC3E,QAAQ;AAAA,EAAoB;AAG5B,QAAM,YAAY,IAAI,IAAI,cAAc;AACxC,QAAM,iBAAiB,CAAC,GAAG,oBAAI,IAAI;AAAA,IACjC,GAAI,MAAM,aAAa,CAAC,KAAK,UAAU,IAAI,CAAC;AAAA,IAC5C,GAAG;AAAA,IACH,GAAG,MAAM;AAAA,EACX,CAAC,CAAC;AAEF,QAAM,OAAO,oBAAI,IAA0B;AAC3C,MAAI,eAAe;AACnB,aAAW,MAAM,gBAAgB;AAC/B,UAAM,OAAO,MAAM,iBAAiB,IAAI,QAAQ,GAAG,UAAU;AAC7D,QAAI,KAAK,OAAQ,gBAAe;AAChC,UAAM,SAAiC,UAAU,IAAI,EAAE,IAAI,WAAW;AACtE,eAAW,KAAK,KAAM,KAAI,CAAC,KAAK,IAAI,CAAC,EAAG,MAAK,IAAI,GAAG,EAAE,KAAK,GAAG,OAAO,CAAC;AAAA,EACxE;AAEA,SAAO,EAAE,YAAY,CAAC,GAAG,KAAK,OAAO,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,aAAa;AAC3E;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
package/dist/skyvern.js
CHANGED
|
@@ -5,6 +5,8 @@ async function skyvernNavigate(startUrl, navigationGoals, tenantConfig, triggerR
|
|
|
5
5
|
console.warn("[skyvern] skyvernNavigate: no API config \u2014 skipping post-login vision navigation");
|
|
6
6
|
return { screenshotUrls: [], visitedUrls: [], status: "failed" };
|
|
7
7
|
}
|
|
8
|
+
const { assertResolvesToPublicAddress } = await import("@detiq/core");
|
|
9
|
+
await assertResolvesToPublicAddress(apiUrl);
|
|
8
10
|
const visitedUrls = [];
|
|
9
11
|
const screenshotUrls = [];
|
|
10
12
|
const skyvernSteps = [];
|
|
@@ -91,6 +93,8 @@ async function skyvernFallback(task, tenantConfig) {
|
|
|
91
93
|
);
|
|
92
94
|
return "failed";
|
|
93
95
|
}
|
|
96
|
+
const { assertResolvesToPublicAddress } = await import("@detiq/core");
|
|
97
|
+
await assertResolvesToPublicAddress(apiUrl);
|
|
94
98
|
try {
|
|
95
99
|
const res = await fetch(`${apiUrl}/api/v1/tasks`, {
|
|
96
100
|
method: "POST",
|
package/dist/skyvern.js.map
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../src/skyvern.ts"],
|
|
4
|
-
"sourcesContent": ["/**\n * Skyvern Vision Fallback\n *\n * Skyvern combines computer vision + LLM reasoning to navigate sites that\n * defeat DOM-based approaches: CAPTCHAs, legacy apps, visual-only login flows,\n * government portals. It's the \"break glass\" option when Stagehand fails.\n *\n * Production setup (Phase 4+):\n * 1. Self-host: https://github.com/Skyvern-AI/skyvern\n * 2. Or use cloud: https://app.skyvern.com\n * 3. Set SKYVERN_API_URL + SKYVERN_API_KEY in .env\n *\n * Pricing note: vision tasks cost more per step than DOM-based Stagehand.\n * Use only when the fast path fails.\n */\n\nexport interface SkyvernTask {\n taskDescription: string;\n url: string;\n credentials?: { username: string; password: string; oneTimeCode?: string };\n}\n\nexport interface SkyvernConfig {\n apiUrl: string;\n apiKey: string;\n llmKey?: string;\n llmApiKey?: string;\n}\n\nexport interface SkyvernStep {\n order: number;\n action_type: string;\n element_description?: string;\n reasoning?: string;\n success: boolean;\n screenshot_url?: string;\n duration_ms?: number;\n error?: string;\n}\n\nexport interface SkyvernNavigateResult {\n screenshotUrls: string[];\n visitedUrls: string[];\n status: 'success' | 'failed' | 'partial';\n steps?: SkyvernStep[];\n taskId?: string;\n durationMs?: number;\n triggerReason?: string;\n}\n\n/**\n * Gap 5: Post-login navigation via Skyvern vision for apps where DOM analysis\n * cannot discover screens (canvas, WebGL, legacy Flash-replacement stacks).\n * Skyvern drives the browser via screenshot + LLM, returning visited page list.\n */\nexport async function skyvernNavigate(\n startUrl: string,\n navigationGoals: string[],\n tenantConfig?: SkyvernConfig,\n triggerReason?: string,\n): Promise<SkyvernNavigateResult> {\n const apiUrl = tenantConfig?.apiUrl || process.env.SKYVERN_API_URL;\n const apiKey = tenantConfig?.apiKey || process.env.SKYVERN_API_KEY;\n\n if (!apiUrl || !apiKey) {\n console.warn('[skyvern] skyvernNavigate: no API config \u2014 skipping post-login vision navigation');\n return { screenshotUrls: [], visitedUrls: [], status: 'failed' };\n }\n\n const visitedUrls: string[] = [];\n const screenshotUrls: string[] = [];\n const skyvernSteps: SkyvernStep[] = [];\n let lastTaskId: string | undefined;\n let totalDurationMs = 0;\n\n for (const goal of navigationGoals) {\n try {\n const taskStart = Date.now();\n const res = await fetch(`${apiUrl}/api/v1/tasks`, {\n method: 'POST',\n headers: { 'Content-Type': 'application/json', 'x-api-key': apiKey },\n body: JSON.stringify({\n url: startUrl,\n navigation_goal: goal,\n ...(tenantConfig?.llmKey ? { llm_key: tenantConfig.llmKey } : process.env.SKYVERN_LLM_KEY ? { llm_key: process.env.SKYVERN_LLM_KEY } : {}),\n ...(tenantConfig?.llmApiKey ? { llm_api_key: tenantConfig.llmApiKey } : process.env.OPENAI_API_KEY ? { llm_api_key: process.env.OPENAI_API_KEY } : {}),\n }),\n });\n if (!res.ok) continue;\n const data: any = await res.json();\n const taskId: string = data.task_id;\n if (!taskId) continue;\n lastTaskId = taskId;\n\n // Poll for task completion (max 60s per goal)\n let done = false;\n for (let p = 0; p < 12 && !done; p++) {\n await new Promise((r) => setTimeout(r, 5000));\n const poll = await fetch(`${apiUrl}/api/v1/tasks/${taskId}`, {\n headers: { 'x-api-key': apiKey },\n }).catch(() => null);\n if (!poll?.ok) continue;\n const pollData: any = await poll.json();\n if (pollData.status === 'completed' || pollData.status === 'failed') {\n done = true;\n totalDurationMs += Date.now() - taskStart;\n if (pollData.status === 'completed') {\n if (pollData.output?.url) visitedUrls.push(pollData.output.url);\n if (pollData.output?.screenshot_url) screenshotUrls.push(pollData.output.screenshot_url);\n }\n\n // Fetch step-level detail from Skyvern\n try {\n const stepsRes = await fetch(`${apiUrl}/api/v1/tasks/${taskId}/steps`, {\n headers: { 'x-api-key': apiKey },\n }).catch(() => null);\n if (stepsRes?.ok) {\n const stepsData: any = await stepsRes.json();\n const stepsList = Array.isArray(stepsData) ? stepsData : (stepsData?.steps ?? []);\n for (const s of stepsList) {\n if (s.output?.screenshot_url) screenshotUrls.push(s.output.screenshot_url);\n skyvernSteps.push({\n order: s.step_order ?? s.order ?? 0,\n action_type: s.action_type ?? s.action ?? 'unknown',\n element_description: s.output?.element_description,\n reasoning: s.output?.llm_response?.action_description ?? s.output?.reasoning,\n success: s.status === 'completed',\n screenshot_url: s.output?.screenshot_url,\n error: s.status === 'failed' ? (s.output?.errors?.[0] ?? 'Step failed') : undefined,\n });\n }\n }\n } catch { /* non-fatal */ }\n }\n }\n } catch (err) {\n console.warn(`[skyvern] Navigation goal failed: ${goal}`, err);\n }\n }\n\n return {\n screenshotUrls,\n visitedUrls,\n status: visitedUrls.length > 0 ? 'success' : screenshotUrls.length > 0 ? 'partial' : 'failed',\n steps: skyvernSteps,\n taskId: lastTaskId,\n durationMs: totalDurationMs || undefined,\n triggerReason,\n };\n}\n\nexport async function skyvernFallback(task: SkyvernTask, tenantConfig?: SkyvernConfig): Promise<'success' | 'failed'> {\n const apiUrl = tenantConfig?.apiUrl || process.env.SKYVERN_API_URL;\n const apiKey = tenantConfig?.apiKey || process.env.SKYVERN_API_KEY;\n\n if (!apiUrl || !apiKey) {\n console.warn(\n '[skyvern] Vision fallback disabled \u2014 set SKYVERN_API_URL + SKYVERN_API_KEY in .env to enable.',\n '\\n[skyvern] Self-host: https://github.com/Skyvern-AI/skyvern'\n );\n return 'failed';\n }\n\n try {\n const res = await fetch(`${apiUrl}/api/v1/tasks`, {\n method: 'POST',\n headers: { 'Content-Type': 'application/json', 'x-api-key': apiKey },\n body: JSON.stringify({\n url: task.url,\n navigation_goal: task.taskDescription,\n ...(task.credentials && {\n navigation_payload: {\n username: task.credentials.username,\n password: task.credentials.password,\n ...(task.credentials.oneTimeCode && { otp: task.credentials.oneTimeCode }),\n },\n }),\n ...(tenantConfig?.llmKey ? { llm_key: tenantConfig.llmKey } : process.env.SKYVERN_LLM_KEY ? { llm_key: process.env.SKYVERN_LLM_KEY } : {}),\n ...(tenantConfig?.llmApiKey ? { llm_api_key: tenantConfig.llmApiKey } : process.env.OPENAI_API_KEY ? { llm_api_key: process.env.OPENAI_API_KEY } : {}),\n }),\n });\n\n if (!res.ok) {\n const body = await res.text().catch(() => '');\n console.error(`[skyvern] Task creation failed: ${res.status}`, body);\n return 'failed';\n }\n\n const data: any = await res.json();\n console.log(`[skyvern] Task queued: ${data.task_id}. Poll GET /api/v1/tasks/${data.task_id} for completion.`);\n // Full implementation: poll until status === \"completed\" | \"failed\".\n // For Phase 1 the login already happened in the browser context Skyvern manages,\n // so returning 'success' here means we can proceed in that browser session.\n return 'success';\n } catch (err) {\n console.error('[skyvern] Error:', err);\n return 'failed';\n }\n}\n"],
|
|
5
|
-
"mappings": "AAuDA,eAAsB,gBACpB,UACA,iBACA,cACA,eACgC;AAChC,QAAM,SAAS,cAAc,UAAU,QAAQ,IAAI;AACnD,QAAM,SAAS,cAAc,UAAU,QAAQ,IAAI;AAEnD,MAAI,CAAC,UAAU,CAAC,QAAQ;AACtB,YAAQ,KAAK,uFAAkF;AAC/F,WAAO,EAAE,gBAAgB,CAAC,GAAG,aAAa,CAAC,GAAG,QAAQ,SAAS;AAAA,EACjE;AAEA,QAAM,cAAwB,CAAC;AAC/B,QAAM,iBAA2B,CAAC;AAClC,QAAM,eAA8B,CAAC;AACrC,MAAI;AACJ,MAAI,kBAAkB;AAEtB,aAAW,QAAQ,iBAAiB;AAClC,QAAI;AACF,YAAM,YAAY,KAAK,IAAI;AAC3B,YAAM,MAAM,MAAM,MAAM,GAAG,MAAM,iBAAiB;AAAA,QAChD,QAAQ;AAAA,QACR,SAAS,EAAE,gBAAgB,oBAAoB,aAAa,OAAO;AAAA,QACnE,MAAM,KAAK,UAAU;AAAA,UACnB,KAAK;AAAA,UACL,iBAAiB;AAAA,UACjB,GAAI,cAAc,SAAS,EAAE,SAAS,aAAa,OAAO,IAAI,QAAQ,IAAI,kBAAkB,EAAE,SAAS,QAAQ,IAAI,gBAAgB,IAAI,CAAC;AAAA,UACxI,GAAI,cAAc,YAAY,EAAE,aAAa,aAAa,UAAU,IAAI,QAAQ,IAAI,iBAAiB,EAAE,aAAa,QAAQ,IAAI,eAAe,IAAI,CAAC;AAAA,QACtJ,CAAC;AAAA,MACH,CAAC;AACD,UAAI,CAAC,IAAI,GAAI;AACb,YAAM,OAAY,MAAM,IAAI,KAAK;AACjC,YAAM,SAAiB,KAAK;AAC5B,UAAI,CAAC,OAAQ;AACb,mBAAa;AAGb,UAAI,OAAO;AACX,eAAS,IAAI,GAAG,IAAI,MAAM,CAAC,MAAM,KAAK;AACpC,cAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,GAAI,CAAC;AAC5C,cAAM,OAAO,MAAM,MAAM,GAAG,MAAM,iBAAiB,MAAM,IAAI;AAAA,UAC3D,SAAS,EAAE,aAAa,OAAO;AAAA,QACjC,CAAC,EAAE,MAAM,MAAM,IAAI;AACnB,YAAI,CAAC,MAAM,GAAI;AACf,cAAM,WAAgB,MAAM,KAAK,KAAK;AACtC,YAAI,SAAS,WAAW,eAAe,SAAS,WAAW,UAAU;AACnE,iBAAO;AACP,6BAAmB,KAAK,IAAI,IAAI;AAChC,cAAI,SAAS,WAAW,aAAa;AACnC,gBAAI,SAAS,QAAQ,IAAK,aAAY,KAAK,SAAS,OAAO,GAAG;AAC9D,gBAAI,SAAS,QAAQ,eAAgB,gBAAe,KAAK,SAAS,OAAO,cAAc;AAAA,UACzF;AAGA,cAAI;AACF,kBAAM,WAAW,MAAM,MAAM,GAAG,MAAM,iBAAiB,MAAM,UAAU;AAAA,cACrE,SAAS,EAAE,aAAa,OAAO;AAAA,YACjC,CAAC,EAAE,MAAM,MAAM,IAAI;AACnB,gBAAI,UAAU,IAAI;AAChB,oBAAM,YAAiB,MAAM,SAAS,KAAK;AAC3C,oBAAM,YAAY,MAAM,QAAQ,SAAS,IAAI,YAAa,WAAW,SAAS,CAAC;AAC/E,yBAAW,KAAK,WAAW;AACzB,oBAAI,EAAE,QAAQ,eAAgB,gBAAe,KAAK,EAAE,OAAO,cAAc;AACzE,6BAAa,KAAK;AAAA,kBAChB,OAAO,EAAE,cAAc,EAAE,SAAS;AAAA,kBAClC,aAAa,EAAE,eAAe,EAAE,UAAU;AAAA,kBAC1C,qBAAqB,EAAE,QAAQ;AAAA,kBAC/B,WAAW,EAAE,QAAQ,cAAc,sBAAsB,EAAE,QAAQ;AAAA,kBACnE,SAAS,EAAE,WAAW;AAAA,kBACtB,gBAAgB,EAAE,QAAQ;AAAA,kBAC1B,OAAO,EAAE,WAAW,WAAY,EAAE,QAAQ,SAAS,CAAC,KAAK,gBAAiB;AAAA,gBAC5E,CAAC;AAAA,cACH;AAAA,YACF;AAAA,UACF,QAAQ;AAAA,UAAkB;AAAA,QAC5B;AAAA,MACF;AAAA,IACF,SAAS,KAAK;AACZ,cAAQ,KAAK,qCAAqC,IAAI,IAAI,GAAG;AAAA,IAC/D;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,QAAQ,YAAY,SAAS,IAAI,YAAY,eAAe,SAAS,IAAI,YAAY;AAAA,IACrF,OAAO;AAAA,IACP,QAAQ;AAAA,IACR,YAAY,mBAAmB;AAAA,IAC/B;AAAA,EACF;AACF;AAEA,eAAsB,gBAAgB,MAAmB,cAA6D;AACpH,QAAM,SAAS,cAAc,UAAU,QAAQ,IAAI;AACnD,QAAM,SAAS,cAAc,UAAU,QAAQ,IAAI;AAEnD,MAAI,CAAC,UAAU,CAAC,QAAQ;AACtB,YAAQ;AAAA,MACN;AAAA,MACA;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAEA,MAAI;AACF,UAAM,MAAM,MAAM,MAAM,GAAG,MAAM,iBAAiB;AAAA,MAChD,QAAQ;AAAA,MACR,SAAS,EAAE,gBAAgB,oBAAoB,aAAa,OAAO;AAAA,MACnE,MAAM,KAAK,UAAU;AAAA,QACnB,KAAK,KAAK;AAAA,QACV,iBAAiB,KAAK;AAAA,QACtB,GAAI,KAAK,eAAe;AAAA,UACtB,oBAAoB;AAAA,YAClB,UAAU,KAAK,YAAY;AAAA,YAC3B,UAAU,KAAK,YAAY;AAAA,YAC3B,GAAI,KAAK,YAAY,eAAe,EAAE,KAAK,KAAK,YAAY,YAAY;AAAA,UAC1E;AAAA,QACF;AAAA,QACA,GAAI,cAAc,SAAS,EAAE,SAAS,aAAa,OAAO,IAAI,QAAQ,IAAI,kBAAkB,EAAE,SAAS,QAAQ,IAAI,gBAAgB,IAAI,CAAC;AAAA,QACxI,GAAI,cAAc,YAAY,EAAE,aAAa,aAAa,UAAU,IAAI,QAAQ,IAAI,iBAAiB,EAAE,aAAa,QAAQ,IAAI,eAAe,IAAI,CAAC;AAAA,MACtJ,CAAC;AAAA,IACH,CAAC;AAED,QAAI,CAAC,IAAI,IAAI;AACX,YAAM,OAAO,MAAM,IAAI,KAAK,EAAE,MAAM,MAAM,EAAE;AAC5C,cAAQ,MAAM,mCAAmC,IAAI,MAAM,IAAI,IAAI;AACnE,aAAO;AAAA,IACT;AAEA,UAAM,OAAY,MAAM,IAAI,KAAK;AACjC,YAAQ,IAAI,0BAA0B,KAAK,OAAO,4BAA4B,KAAK,OAAO,kBAAkB;AAI5G,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,YAAQ,MAAM,oBAAoB,GAAG;AACrC,WAAO;AAAA,EACT;AACF;",
|
|
4
|
+
"sourcesContent": ["/**\n * Skyvern Vision Fallback\n *\n * Skyvern combines computer vision + LLM reasoning to navigate sites that\n * defeat DOM-based approaches: CAPTCHAs, legacy apps, visual-only login flows,\n * government portals. It's the \"break glass\" option when Stagehand fails.\n *\n * Production setup (Phase 4+):\n * 1. Self-host: https://github.com/Skyvern-AI/skyvern\n * 2. Or use cloud: https://app.skyvern.com\n * 3. Set SKYVERN_API_URL + SKYVERN_API_KEY in .env\n *\n * Pricing note: vision tasks cost more per step than DOM-based Stagehand.\n * Use only when the fast path fails.\n */\n\nexport interface SkyvernTask {\n taskDescription: string;\n url: string;\n credentials?: { username: string; password: string; oneTimeCode?: string };\n}\n\nexport interface SkyvernConfig {\n apiUrl: string;\n apiKey: string;\n llmKey?: string;\n llmApiKey?: string;\n}\n\nexport interface SkyvernStep {\n order: number;\n action_type: string;\n element_description?: string;\n reasoning?: string;\n success: boolean;\n screenshot_url?: string;\n duration_ms?: number;\n error?: string;\n}\n\nexport interface SkyvernNavigateResult {\n screenshotUrls: string[];\n visitedUrls: string[];\n status: 'success' | 'failed' | 'partial';\n steps?: SkyvernStep[];\n taskId?: string;\n durationMs?: number;\n triggerReason?: string;\n}\n\n/**\n * Gap 5: Post-login navigation via Skyvern vision for apps where DOM analysis\n * cannot discover screens (canvas, WebGL, legacy Flash-replacement stacks).\n * Skyvern drives the browser via screenshot + LLM, returning visited page list.\n */\nexport async function skyvernNavigate(\n startUrl: string,\n navigationGoals: string[],\n tenantConfig?: SkyvernConfig,\n triggerReason?: string,\n): Promise<SkyvernNavigateResult> {\n const apiUrl = tenantConfig?.apiUrl || process.env.SKYVERN_API_URL;\n const apiKey = tenantConfig?.apiKey || process.env.SKYVERN_API_KEY;\n\n if (!apiUrl || !apiKey) {\n console.warn('[skyvern] skyvernNavigate: no API config \u2014 skipping post-login vision navigation');\n return { screenshotUrls: [], visitedUrls: [], status: 'failed' };\n }\n\n const { assertResolvesToPublicAddress } = await import('@detiq/core');\n await assertResolvesToPublicAddress(apiUrl);\n\n const visitedUrls: string[] = [];\n const screenshotUrls: string[] = [];\n const skyvernSteps: SkyvernStep[] = [];\n let lastTaskId: string | undefined;\n let totalDurationMs = 0;\n\n for (const goal of navigationGoals) {\n try {\n const taskStart = Date.now();\n const res = await fetch(`${apiUrl}/api/v1/tasks`, {\n method: 'POST',\n headers: { 'Content-Type': 'application/json', 'x-api-key': apiKey },\n body: JSON.stringify({\n url: startUrl,\n navigation_goal: goal,\n ...(tenantConfig?.llmKey ? { llm_key: tenantConfig.llmKey } : process.env.SKYVERN_LLM_KEY ? { llm_key: process.env.SKYVERN_LLM_KEY } : {}),\n ...(tenantConfig?.llmApiKey ? { llm_api_key: tenantConfig.llmApiKey } : process.env.OPENAI_API_KEY ? { llm_api_key: process.env.OPENAI_API_KEY } : {}),\n }),\n });\n if (!res.ok) continue;\n const data: any = await res.json();\n const taskId: string = data.task_id;\n if (!taskId) continue;\n lastTaskId = taskId;\n\n // Poll for task completion (max 60s per goal)\n let done = false;\n for (let p = 0; p < 12 && !done; p++) {\n await new Promise((r) => setTimeout(r, 5000));\n const poll = await fetch(`${apiUrl}/api/v1/tasks/${taskId}`, {\n headers: { 'x-api-key': apiKey },\n }).catch(() => null);\n if (!poll?.ok) continue;\n const pollData: any = await poll.json();\n if (pollData.status === 'completed' || pollData.status === 'failed') {\n done = true;\n totalDurationMs += Date.now() - taskStart;\n if (pollData.status === 'completed') {\n if (pollData.output?.url) visitedUrls.push(pollData.output.url);\n if (pollData.output?.screenshot_url) screenshotUrls.push(pollData.output.screenshot_url);\n }\n\n // Fetch step-level detail from Skyvern\n try {\n const stepsRes = await fetch(`${apiUrl}/api/v1/tasks/${taskId}/steps`, {\n headers: { 'x-api-key': apiKey },\n }).catch(() => null);\n if (stepsRes?.ok) {\n const stepsData: any = await stepsRes.json();\n const stepsList = Array.isArray(stepsData) ? stepsData : (stepsData?.steps ?? []);\n for (const s of stepsList) {\n if (s.output?.screenshot_url) screenshotUrls.push(s.output.screenshot_url);\n skyvernSteps.push({\n order: s.step_order ?? s.order ?? 0,\n action_type: s.action_type ?? s.action ?? 'unknown',\n element_description: s.output?.element_description,\n reasoning: s.output?.llm_response?.action_description ?? s.output?.reasoning,\n success: s.status === 'completed',\n screenshot_url: s.output?.screenshot_url,\n error: s.status === 'failed' ? (s.output?.errors?.[0] ?? 'Step failed') : undefined,\n });\n }\n }\n } catch { /* non-fatal */ }\n }\n }\n } catch (err) {\n console.warn(`[skyvern] Navigation goal failed: ${goal}`, err);\n }\n }\n\n return {\n screenshotUrls,\n visitedUrls,\n status: visitedUrls.length > 0 ? 'success' : screenshotUrls.length > 0 ? 'partial' : 'failed',\n steps: skyvernSteps,\n taskId: lastTaskId,\n durationMs: totalDurationMs || undefined,\n triggerReason,\n };\n}\n\nexport async function skyvernFallback(task: SkyvernTask, tenantConfig?: SkyvernConfig): Promise<'success' | 'failed'> {\n const apiUrl = tenantConfig?.apiUrl || process.env.SKYVERN_API_URL;\n const apiKey = tenantConfig?.apiKey || process.env.SKYVERN_API_KEY;\n\n if (!apiUrl || !apiKey) {\n console.warn(\n '[skyvern] Vision fallback disabled \u2014 set SKYVERN_API_URL + SKYVERN_API_KEY in .env to enable.',\n '\\n[skyvern] Self-host: https://github.com/Skyvern-AI/skyvern'\n );\n return 'failed';\n }\n\n const { assertResolvesToPublicAddress } = await import('@detiq/core');\n await assertResolvesToPublicAddress(apiUrl);\n\n try {\n const res = await fetch(`${apiUrl}/api/v1/tasks`, {\n method: 'POST',\n headers: { 'Content-Type': 'application/json', 'x-api-key': apiKey },\n body: JSON.stringify({\n url: task.url,\n navigation_goal: task.taskDescription,\n ...(task.credentials && {\n navigation_payload: {\n username: task.credentials.username,\n password: task.credentials.password,\n ...(task.credentials.oneTimeCode && { otp: task.credentials.oneTimeCode }),\n },\n }),\n ...(tenantConfig?.llmKey ? { llm_key: tenantConfig.llmKey } : process.env.SKYVERN_LLM_KEY ? { llm_key: process.env.SKYVERN_LLM_KEY } : {}),\n ...(tenantConfig?.llmApiKey ? { llm_api_key: tenantConfig.llmApiKey } : process.env.OPENAI_API_KEY ? { llm_api_key: process.env.OPENAI_API_KEY } : {}),\n }),\n });\n\n if (!res.ok) {\n const body = await res.text().catch(() => '');\n console.error(`[skyvern] Task creation failed: ${res.status}`, body);\n return 'failed';\n }\n\n const data: any = await res.json();\n console.log(`[skyvern] Task queued: ${data.task_id}. Poll GET /api/v1/tasks/${data.task_id} for completion.`);\n // Full implementation: poll until status === \"completed\" | \"failed\".\n // For Phase 1 the login already happened in the browser context Skyvern manages,\n // so returning 'success' here means we can proceed in that browser session.\n return 'success';\n } catch (err) {\n console.error('[skyvern] Error:', err);\n return 'failed';\n }\n}\n"],
|
|
5
|
+
"mappings": "AAuDA,eAAsB,gBACpB,UACA,iBACA,cACA,eACgC;AAChC,QAAM,SAAS,cAAc,UAAU,QAAQ,IAAI;AACnD,QAAM,SAAS,cAAc,UAAU,QAAQ,IAAI;AAEnD,MAAI,CAAC,UAAU,CAAC,QAAQ;AACtB,YAAQ,KAAK,uFAAkF;AAC/F,WAAO,EAAE,gBAAgB,CAAC,GAAG,aAAa,CAAC,GAAG,QAAQ,SAAS;AAAA,EACjE;AAEA,QAAM,EAAE,8BAA8B,IAAI,MAAM,OAAO,aAAa;AACpE,QAAM,8BAA8B,MAAM;AAE1C,QAAM,cAAwB,CAAC;AAC/B,QAAM,iBAA2B,CAAC;AAClC,QAAM,eAA8B,CAAC;AACrC,MAAI;AACJ,MAAI,kBAAkB;AAEtB,aAAW,QAAQ,iBAAiB;AAClC,QAAI;AACF,YAAM,YAAY,KAAK,IAAI;AAC3B,YAAM,MAAM,MAAM,MAAM,GAAG,MAAM,iBAAiB;AAAA,QAChD,QAAQ;AAAA,QACR,SAAS,EAAE,gBAAgB,oBAAoB,aAAa,OAAO;AAAA,QACnE,MAAM,KAAK,UAAU;AAAA,UACnB,KAAK;AAAA,UACL,iBAAiB;AAAA,UACjB,GAAI,cAAc,SAAS,EAAE,SAAS,aAAa,OAAO,IAAI,QAAQ,IAAI,kBAAkB,EAAE,SAAS,QAAQ,IAAI,gBAAgB,IAAI,CAAC;AAAA,UACxI,GAAI,cAAc,YAAY,EAAE,aAAa,aAAa,UAAU,IAAI,QAAQ,IAAI,iBAAiB,EAAE,aAAa,QAAQ,IAAI,eAAe,IAAI,CAAC;AAAA,QACtJ,CAAC;AAAA,MACH,CAAC;AACD,UAAI,CAAC,IAAI,GAAI;AACb,YAAM,OAAY,MAAM,IAAI,KAAK;AACjC,YAAM,SAAiB,KAAK;AAC5B,UAAI,CAAC,OAAQ;AACb,mBAAa;AAGb,UAAI,OAAO;AACX,eAAS,IAAI,GAAG,IAAI,MAAM,CAAC,MAAM,KAAK;AACpC,cAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,GAAI,CAAC;AAC5C,cAAM,OAAO,MAAM,MAAM,GAAG,MAAM,iBAAiB,MAAM,IAAI;AAAA,UAC3D,SAAS,EAAE,aAAa,OAAO;AAAA,QACjC,CAAC,EAAE,MAAM,MAAM,IAAI;AACnB,YAAI,CAAC,MAAM,GAAI;AACf,cAAM,WAAgB,MAAM,KAAK,KAAK;AACtC,YAAI,SAAS,WAAW,eAAe,SAAS,WAAW,UAAU;AACnE,iBAAO;AACP,6BAAmB,KAAK,IAAI,IAAI;AAChC,cAAI,SAAS,WAAW,aAAa;AACnC,gBAAI,SAAS,QAAQ,IAAK,aAAY,KAAK,SAAS,OAAO,GAAG;AAC9D,gBAAI,SAAS,QAAQ,eAAgB,gBAAe,KAAK,SAAS,OAAO,cAAc;AAAA,UACzF;AAGA,cAAI;AACF,kBAAM,WAAW,MAAM,MAAM,GAAG,MAAM,iBAAiB,MAAM,UAAU;AAAA,cACrE,SAAS,EAAE,aAAa,OAAO;AAAA,YACjC,CAAC,EAAE,MAAM,MAAM,IAAI;AACnB,gBAAI,UAAU,IAAI;AAChB,oBAAM,YAAiB,MAAM,SAAS,KAAK;AAC3C,oBAAM,YAAY,MAAM,QAAQ,SAAS,IAAI,YAAa,WAAW,SAAS,CAAC;AAC/E,yBAAW,KAAK,WAAW;AACzB,oBAAI,EAAE,QAAQ,eAAgB,gBAAe,KAAK,EAAE,OAAO,cAAc;AACzE,6BAAa,KAAK;AAAA,kBAChB,OAAO,EAAE,cAAc,EAAE,SAAS;AAAA,kBAClC,aAAa,EAAE,eAAe,EAAE,UAAU;AAAA,kBAC1C,qBAAqB,EAAE,QAAQ;AAAA,kBAC/B,WAAW,EAAE,QAAQ,cAAc,sBAAsB,EAAE,QAAQ;AAAA,kBACnE,SAAS,EAAE,WAAW;AAAA,kBACtB,gBAAgB,EAAE,QAAQ;AAAA,kBAC1B,OAAO,EAAE,WAAW,WAAY,EAAE,QAAQ,SAAS,CAAC,KAAK,gBAAiB;AAAA,gBAC5E,CAAC;AAAA,cACH;AAAA,YACF;AAAA,UACF,QAAQ;AAAA,UAAkB;AAAA,QAC5B;AAAA,MACF;AAAA,IACF,SAAS,KAAK;AACZ,cAAQ,KAAK,qCAAqC,IAAI,IAAI,GAAG;AAAA,IAC/D;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,QAAQ,YAAY,SAAS,IAAI,YAAY,eAAe,SAAS,IAAI,YAAY;AAAA,IACrF,OAAO;AAAA,IACP,QAAQ;AAAA,IACR,YAAY,mBAAmB;AAAA,IAC/B;AAAA,EACF;AACF;AAEA,eAAsB,gBAAgB,MAAmB,cAA6D;AACpH,QAAM,SAAS,cAAc,UAAU,QAAQ,IAAI;AACnD,QAAM,SAAS,cAAc,UAAU,QAAQ,IAAI;AAEnD,MAAI,CAAC,UAAU,CAAC,QAAQ;AACtB,YAAQ;AAAA,MACN;AAAA,MACA;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAEA,QAAM,EAAE,8BAA8B,IAAI,MAAM,OAAO,aAAa;AACpE,QAAM,8BAA8B,MAAM;AAE1C,MAAI;AACF,UAAM,MAAM,MAAM,MAAM,GAAG,MAAM,iBAAiB;AAAA,MAChD,QAAQ;AAAA,MACR,SAAS,EAAE,gBAAgB,oBAAoB,aAAa,OAAO;AAAA,MACnE,MAAM,KAAK,UAAU;AAAA,QACnB,KAAK,KAAK;AAAA,QACV,iBAAiB,KAAK;AAAA,QACtB,GAAI,KAAK,eAAe;AAAA,UACtB,oBAAoB;AAAA,YAClB,UAAU,KAAK,YAAY;AAAA,YAC3B,UAAU,KAAK,YAAY;AAAA,YAC3B,GAAI,KAAK,YAAY,eAAe,EAAE,KAAK,KAAK,YAAY,YAAY;AAAA,UAC1E;AAAA,QACF;AAAA,QACA,GAAI,cAAc,SAAS,EAAE,SAAS,aAAa,OAAO,IAAI,QAAQ,IAAI,kBAAkB,EAAE,SAAS,QAAQ,IAAI,gBAAgB,IAAI,CAAC;AAAA,QACxI,GAAI,cAAc,YAAY,EAAE,aAAa,aAAa,UAAU,IAAI,QAAQ,IAAI,iBAAiB,EAAE,aAAa,QAAQ,IAAI,eAAe,IAAI,CAAC;AAAA,MACtJ,CAAC;AAAA,IACH,CAAC;AAED,QAAI,CAAC,IAAI,IAAI;AACX,YAAM,OAAO,MAAM,IAAI,KAAK,EAAE,MAAM,MAAM,EAAE;AAC5C,cAAQ,MAAM,mCAAmC,IAAI,MAAM,IAAI,IAAI;AACnE,aAAO;AAAA,IACT;AAEA,UAAM,OAAY,MAAM,IAAI,KAAK;AACjC,YAAQ,IAAI,0BAA0B,KAAK,OAAO,4BAA4B,KAAK,OAAO,kBAAkB;AAI5G,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,YAAQ,MAAM,oBAAoB,GAAG;AACrC,WAAO;AAAA,EACT;AACF;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
package/dist/test-executor.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { chromium, firefox, webkit } from "playwright";
|
|
2
|
+
import { assertCrawlUrlSafe } from "@detiq/core";
|
|
2
3
|
import { analyzeFailure } from "./failure-analysis-agent.js";
|
|
3
4
|
const LOCATOR_STRATEGIES = ["text", "label", "role", "css"];
|
|
4
5
|
function buildLocator(page, selector, strategy = "text") {
|
|
@@ -45,7 +46,7 @@ async function resolveLocator(page, step, healCtx) {
|
|
|
45
46
|
}
|
|
46
47
|
if (healCtx) {
|
|
47
48
|
try {
|
|
48
|
-
const { healElement } = await import("@
|
|
49
|
+
const { healElement } = await import("@detiq/agents");
|
|
49
50
|
const currentPageElements = await extractCurrentPageElements(page);
|
|
50
51
|
const result = await healElement({
|
|
51
52
|
tenantId: healCtx.tenantId,
|
|
@@ -93,6 +94,8 @@ async function runPlan(page, plan, options, consoleLogs, networkErrors) {
|
|
|
93
94
|
const timeout = step.timeout ?? 8e3;
|
|
94
95
|
switch (step.action) {
|
|
95
96
|
case "navigate":
|
|
97
|
+
if (!step.url) throw new Error("Navigate step missing url");
|
|
98
|
+
assertCrawlUrlSafe(step.url);
|
|
96
99
|
await page.goto(step.url, { waitUntil: "domcontentloaded", timeout });
|
|
97
100
|
break;
|
|
98
101
|
case "click": {
|
|
@@ -263,7 +266,13 @@ async function executeTestPlan(plan, options) {
|
|
|
263
266
|
});
|
|
264
267
|
}
|
|
265
268
|
});
|
|
266
|
-
|
|
269
|
+
const TEST_EXECUTION_TIMEOUT_MS = 5 * 6e4;
|
|
270
|
+
lastResult = await Promise.race([
|
|
271
|
+
runPlan(page, plan, options, consoleLogs, networkErrors),
|
|
272
|
+
new Promise(
|
|
273
|
+
(_, reject) => setTimeout(() => reject(new Error("Test execution timed out")), TEST_EXECUTION_TIMEOUT_MS)
|
|
274
|
+
)
|
|
275
|
+
]);
|
|
267
276
|
lastNetworkTimings = networkTimings;
|
|
268
277
|
await browser.close();
|
|
269
278
|
const passed = !lastResult.stepResults.some((r) => r.status === "FAILED");
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../src/test-executor.ts"],
|
|
4
|
-
"sourcesContent": ["import { chromium, firefox, webkit, type Page, type Locator } from 'playwright';\nimport type { ExecutionStep } from '@qip/agents';\nimport { analyzeFailure } from './failure-analysis-agent.js';\n\nexport interface StepResult {\n stepNumber: number;\n description: string;\n status: 'PASSED' | 'FAILED' | 'SKIPPED';\n duration: number;\n error?: string;\n screenshotPath?: string;\n healed?: boolean;\n /** 'mechanical' = same selector text, different Playwright locator strategy.\n * 'ai' = AI Self-Healing Agent matched a semantically different element\n * (the app's selector text itself changed, e.g. \"Sign In\" -> \"Log In\"). */\n healedVia?: 'mechanical' | 'ai';\n healReasoning?: string;\n healConfidence?: number;\n}\n\nexport interface ExecutionResult {\n stepResults: StepResult[];\n status: 'PASSED' | 'FAILED' | 'ERROR' | 'FLAKY';\n errorStep?: number;\n totalDuration: number;\n finalScreenshotPath?: string;\n retryCount?: number;\n consoleLogs?: Array<{ type: string; text: string; timestamp: number }>;\n networkErrors?: Array<{ url: string; status: number; method: string; timestamp: number }>;\n rootCauseSummary?: string;\n networkTimings?: Array<{ url: string; method: string; status: number; duration: number }>;\n}\n\nconst LOCATOR_STRATEGIES: ExecutionStep['selectorStrategy'][] = ['text', 'label', 'role', 'css'];\n\nfunction buildLocator(page: Page, selector: string, strategy: ExecutionStep['selectorStrategy'] = 'text'): Locator {\n switch (strategy) {\n case 'css': return page.locator(selector);\n case 'text': return page.getByText(selector, { exact: false });\n case 'role': return page.getByRole(selector as any);\n case 'label': return page.getByLabel(selector);\n default: return page.locator(selector);\n }\n}\n\n/** Scans the live page for visible interactive elements \u2014 the \"current page elements\"\n * fed to the AI Self-Healing Agent when mechanical locator strategies fail. */\nasync function extractCurrentPageElements(page: Page): Promise<Array<{ meaning: string; role: string; ariaLabel?: string; textContent?: string }>> {\n return page.evaluate(() => {\n const els = Array.from(document.querySelectorAll<HTMLElement>(\n 'a, button, input, select, textarea, [role=\"button\"], [role=\"link\"], [role=\"tab\"], [role=\"menuitem\"], [aria-label]'\n ));\n return els\n .filter((el) => {\n const rect = el.getBoundingClientRect();\n return rect.width > 0 && rect.height > 0;\n })\n .slice(0, 200)\n .map((el) => {\n const ariaLabel = el.getAttribute('aria-label') ?? undefined;\n const textContent = (el.textContent ?? '').replace(/\\s+/g, ' ').trim().slice(0, 100) || undefined;\n const role = el.getAttribute('role') ?? el.tagName.toLowerCase();\n return { meaning: ariaLabel || textContent || role, role, ariaLabel, textContent };\n });\n });\n}\n\n/**\n * Resolves a step's target element in three tiers:\n * 1. Exact selector + strategy as generated.\n * 2. Mechanical: same selector text, other Playwright locator strategies\n * (handles a strategy mismatch, not an actual UI change).\n * 3. AI: when tenantId/projectId are supplied, asks the Self-Healing Agent to find\n * the current page's best semantic match for the failed step description \u2014 the\n * tier that actually survives the app's selector text changing.\n */\nasync function resolveLocator(\n page: Page,\n step: ExecutionStep,\n healCtx?: { tenantId: string; projectId: string },\n): Promise<{ loc: Locator; healed: boolean; healedVia?: 'mechanical' | 'ai'; healReasoning?: string; healConfidence?: number } | null> {\n const { selector, selectorStrategy = 'text' } = step;\n if (!selector) return null;\n\n const primary = buildLocator(page, selector, selectorStrategy);\n const attached = await primary.first().isVisible({ timeout: 1500 }).catch(() => false);\n if (attached) return { loc: primary, healed: false };\n\n // Tier 2 \u2014 mechanical: try other strategies before giving up\n for (const strategy of LOCATOR_STRATEGIES) {\n if (strategy === selectorStrategy) continue;\n const loc = buildLocator(page, selector, strategy);\n const visible = await loc.first().isVisible({ timeout: 800 }).catch(() => false);\n if (visible) return { loc, healed: true, healedVia: 'mechanical' };\n }\n\n // Tier 3 \u2014 AI: the selector text itself may no longer match anything on the page.\n if (healCtx) {\n try {\n const { healElement } = await import('@qip/agents');\n const currentPageElements = await extractCurrentPageElements(page);\n const result = await healElement({\n tenantId: healCtx.tenantId,\n projectId: healCtx.projectId,\n failedElementMeaning: step.description || selector,\n currentPageElements,\n });\n if (result.healed && result.updatedTest) {\n const healedLoc = page.getByText(result.updatedTest.elementMeaning, { exact: false });\n const visible = await healedLoc.first().isVisible({ timeout: 1500 }).catch(() => false);\n if (visible) {\n return {\n loc: healedLoc,\n healed: true,\n healedVia: 'ai',\n healReasoning: result.reasoning,\n healConfidence: result.confidence,\n };\n }\n }\n } catch {\n // AI healing is a best-effort last resort \u2014 any failure here (LLM error, bad\n // JSON, etc.) just falls through to the caller's normal Playwright error below.\n }\n }\n\n // Return primary \u2014 caller will fail with a meaningful Playwright error\n return { loc: primary, healed: false };\n}\n\nfunction applyHealInfo(result: StepResult, r: { healed: boolean; healedVia?: 'mechanical' | 'ai'; healReasoning?: string; healConfidence?: number }): void {\n result.healed = r.healed;\n result.healedVia = r.healedVia;\n result.healReasoning = r.healReasoning;\n result.healConfidence = r.healConfidence;\n}\n\nasync function runPlan(\n page: Page,\n plan: ExecutionStep[],\n options: { screenshotDir: string; testRunId: string; onStepComplete?: (result: StepResult) => void; healCtx?: { tenantId: string; projectId: string } },\n consoleLogs: Array<{ type: string; text: string; timestamp: number }>,\n networkErrors: Array<{ url: string; status: number; method: string; timestamp: number }>,\n): Promise<{ stepResults: StepResult[]; errorStep?: number; finalScreenshotPath?: string; consoleLogs: typeof consoleLogs; networkErrors: typeof networkErrors }> {\n const stepResults: StepResult[] = [];\n let errorStep: number | undefined;\n let finalScreenshotPath: string | undefined;\n\n for (const step of plan) {\n const stepStart = Date.now();\n const result: StepResult = {\n stepNumber: step.stepNumber,\n description: step.description,\n status: 'FAILED',\n duration: 0,\n };\n\n try {\n const timeout = step.timeout ?? 8000;\n\n switch (step.action) {\n case 'navigate':\n await page.goto(step.url!, { waitUntil: 'domcontentloaded', timeout });\n break;\n\n case 'click': {\n const r = await resolveLocator(page, step, options.healCtx);\n if (!r) throw new Error('Could not resolve locator');\n await r.loc.first().click({ timeout });\n applyHealInfo(result, r);\n break;\n }\n\n case 'fill': {\n const r = await resolveLocator(page, step, options.healCtx);\n if (!r) throw new Error('Could not resolve locator');\n await r.loc.first().fill(step.value ?? '', { timeout });\n applyHealInfo(result, r);\n break;\n }\n\n case 'select': {\n const r = await resolveLocator(page, step, options.healCtx);\n if (!r) throw new Error('Could not resolve locator');\n await r.loc.first().selectOption(step.value ?? '', { timeout });\n applyHealInfo(result, r);\n break;\n }\n\n case 'hover': {\n const r = await resolveLocator(page, step, options.healCtx);\n if (!r) throw new Error('Could not resolve locator');\n await r.loc.first().hover({ timeout });\n applyHealInfo(result, r);\n break;\n }\n\n case 'press_key':\n await page.keyboard.press(step.value ?? 'Enter');\n break;\n\n case 'assert_visible': {\n const r = await resolveLocator(page, step, options.healCtx);\n if (!r) throw new Error('Could not resolve locator');\n await r.loc.first().waitFor({ state: 'visible', timeout });\n applyHealInfo(result, r);\n break;\n }\n\n case 'assert_text': {\n const r = await resolveLocator(page, step, options.healCtx);\n if (!r) throw new Error('Could not resolve locator');\n const text = await r.loc.first().textContent({ timeout });\n if (!text?.includes(step.value ?? '')) {\n throw new Error(`Expected text \"${step.value}\" not found. Got: \"${text?.slice(0, 100)}\"`);\n }\n applyHealInfo(result, r);\n break;\n }\n\n case 'assert_url': {\n const url = page.url();\n if (!url.includes(step.value ?? '')) {\n throw new Error(`Expected URL to contain \"${step.value}\". Got: \"${url}\"`);\n }\n break;\n }\n\n case 'wait':\n await page.waitForTimeout(step.timeout ?? 2000);\n break;\n\n case 'screenshot': {\n const ssPath = `${options.screenshotDir}/${options.testRunId}-step-${step.stepNumber}.png`;\n await page.screenshot({ path: ssPath, fullPage: true });\n result.screenshotPath = ssPath;\n break;\n }\n }\n\n result.status = 'PASSED';\n } catch (err: any) {\n result.status = step.optional ? 'SKIPPED' : 'FAILED';\n result.error = err.message;\n if (!step.optional) {\n errorStep = step.stepNumber;\n }\n }\n\n try {\n await page.waitForLoadState('networkidle', { timeout: 1500 }).catch(() => {});\n if (!result.screenshotPath) {\n const ssPath = `${options.screenshotDir}/${options.testRunId}-step-${step.stepNumber}.png`;\n await page.screenshot({ path: ssPath, fullPage: true });\n result.screenshotPath = ssPath;\n }\n } catch { /* Screenshot evidence is best-effort and should not fail the run. */ }\n\n result.duration = Date.now() - stepStart;\n stepResults.push(result);\n options.onStepComplete?.(result);\n\n if (result.status === 'FAILED') break;\n }\n\n try {\n const finalPath = `${options.screenshotDir}/${options.testRunId}-final.png`;\n await page.screenshot({ path: finalPath, fullPage: true });\n finalScreenshotPath = finalPath;\n } catch { /* non-fatal */ }\n\n return { stepResults, errorStep, finalScreenshotPath, consoleLogs, networkErrors };\n}\n\nfunction interpolate(template: string | undefined, data: Record<string, string>): string | undefined {\n if (!template) return template;\n return template.replace(/\\{\\{(\\w+)\\}\\}/g, (_, key) => data[key] ?? `{{${key}}}`);\n}\n\nexport async function executeTestPlan(\n plan: ExecutionStep[],\n options: {\n screenshotDir: string;\n testRunId: string;\n viewport?: { width: number; height: number };\n onStepComplete?: (result: StepResult) => void;\n /** Run the test up to this many times to detect flakiness (default 1). */\n maxRuns?: number;\n browser?: 'chromium' | 'firefox' | 'webkit';\n /** Data row for parameterized execution: substitutes {{varname}} in step fields. */\n dataRow?: Record<string, string>;\n /** Enables the AI Self-Healing Agent tier when a step's selector can't be resolved\n * even after trying alternate locator strategies. Omit to keep mechanical-only\n * healing (no LLM calls during execution). */\n healCtx?: { tenantId: string; projectId: string };\n /** Decrypted Playwright storageState JSON \u2014 restores a logged-in session so\n * steps don't have to re-authenticate. Without this, any test case behind a\n * login wall fails at the first authenticated step. */\n storageState?: string;\n },\n): Promise<ExecutionResult> {\n if (options.dataRow && Object.keys(options.dataRow).length > 0) {\n const data = options.dataRow;\n plan = plan.map((step) => ({\n ...step,\n url: interpolate(step.url, data),\n selector: interpolate(step.selector, data),\n value: interpolate(step.value, data),\n description: interpolate(step.description, data) ?? step.description,\n }));\n }\n const maxRuns = options.maxRuns ?? 1;\n const browserEngine = { chromium, firefox, webkit }[options.browser ?? 'chromium'] ?? chromium;\n const start = Date.now();\n\n const outcomes: ('PASSED' | 'FAILED')[] = [];\n let lastResult: Awaited<ReturnType<typeof runPlan>> | null = null;\n let lastNetworkTimings: Array<{ url: string; method: string; status: number; duration: number }> = [];\n\n for (let run = 0; run < maxRuns; run++) {\n const browser = await browserEngine.launch({\n headless: true,\n // Optional override for environments with a non-standard/pre-provisioned\n // browser install (e.g. a custom base image with only system Chromium).\n ...(process.env.PLAYWRIGHT_EXECUTABLE_PATH ? { executablePath: process.env.PLAYWRIGHT_EXECUTABLE_PATH } : {}),\n });\n let parsedStorageState: any;\n try {\n parsedStorageState = options.storageState ? JSON.parse(options.storageState) : undefined;\n } catch {\n parsedStorageState = undefined;\n }\n const context = await browser.newContext({\n viewport: options.viewport ?? { width: 1440, height: 900 },\n ...(parsedStorageState ? { storageState: parsedStorageState } : {}),\n });\n const page = await context.newPage();\n\n await page\n .addStyleTag({ content: '*, *::before, *::after { animation-duration: 0s !important; transition-duration: 0s !important; }' })\n .catch(() => {});\n\n const consoleLogs: Array<{ type: string; text: string; timestamp: number }> = [];\n const networkErrors: Array<{ url: string; status: number; method: string; timestamp: number }> = [];\n const networkTimings: Array<{ url: string; method: string; status: number; duration: number }> = [];\n const requestStartTimes = new Map<string, number>();\n\n page.on('console', msg => {\n if (consoleLogs.length < 100) {\n consoleLogs.push({ type: msg.type(), text: msg.text().slice(0, 500), timestamp: Date.now() });\n }\n });\n page.on('request', request => {\n const url = request.url();\n if (url.includes('/api/') || url.match(/\\/(graphql|rest|v\\d)\\//)) {\n requestStartTimes.set(request.url(), Date.now());\n }\n });\n page.on('response', response => {\n const url = response.url();\n const start = requestStartTimes.get(url);\n if (start && networkTimings.length < 200) {\n networkTimings.push({ url: url.slice(0, 300), method: response.request().method(), status: response.status(), duration: Date.now() - start });\n requestStartTimes.delete(url);\n }\n if (!response.ok() && !url.includes('favicon') && networkErrors.length < 50) {\n networkErrors.push({\n url: url.slice(0, 300),\n status: response.status(),\n method: response.request().method(),\n timestamp: Date.now(),\n });\n }\n });\n\n lastResult = await runPlan(page, plan, options, consoleLogs, networkErrors);\n lastNetworkTimings = networkTimings;\n await browser.close();\n\n const passed = !lastResult.stepResults.some((r) => r.status === 'FAILED');\n outcomes.push(passed ? 'PASSED' : 'FAILED');\n\n // If all subsequent runs are consistent with first, no need to keep going\n if (run > 0 && outcomes.every((o) => o === outcomes[0])) break;\n }\n\n const passCount = outcomes.filter((o) => o === 'PASSED').length;\n const failCount = outcomes.filter((o) => o === 'FAILED').length;\n\n let status: ExecutionResult['status'];\n if (maxRuns > 1 && passCount > 0 && failCount > 0) {\n status = 'FLAKY';\n } else if (passCount === outcomes.length) {\n status = 'PASSED';\n } else {\n status = 'FAILED';\n }\n\n let rootCauseSummary: string | undefined;\n if (status === 'FAILED' || status === 'FLAKY') {\n const failedStep = lastResult!.stepResults.find(s => s.status === 'FAILED');\n rootCauseSummary = await analyzeFailure({\n tenantId: options.healCtx?.tenantId,\n error: failedStep?.error ?? 'Unknown error',\n stepDescription: failedStep?.description,\n consoleLogs: lastResult!.consoleLogs,\n networkErrors: lastResult!.networkErrors,\n screenshotPath: lastResult!.finalScreenshotPath,\n }).catch(() => undefined) ?? undefined;\n }\n\n return {\n stepResults: lastResult!.stepResults,\n status,\n errorStep: lastResult!.errorStep,\n totalDuration: Date.now() - start,\n finalScreenshotPath: lastResult!.finalScreenshotPath,\n retryCount: outcomes.length - 1,\n consoleLogs: lastResult!.consoleLogs,\n networkErrors: lastResult!.networkErrors,\n rootCauseSummary,\n networkTimings: lastNetworkTimings,\n };\n}\n"],
|
|
5
|
-
"mappings": "AAAA,SAAS,UAAU,SAAS,cAAuC;AAEnE,SAAS,sBAAsB;AA+B/B,MAAM,qBAA0D,CAAC,QAAQ,SAAS,QAAQ,KAAK;AAE/F,SAAS,aAAa,MAAY,UAAkB,WAA8C,QAAiB;AACjH,UAAQ,UAAU;AAAA,IAChB,KAAK;AAAS,aAAO,KAAK,QAAQ,QAAQ;AAAA,IAC1C,KAAK;AAAS,aAAO,KAAK,UAAU,UAAU,EAAE,OAAO,MAAM,CAAC;AAAA,IAC9D,KAAK;AAAS,aAAO,KAAK,UAAU,QAAe;AAAA,IACnD,KAAK;AAAS,aAAO,KAAK,WAAW,QAAQ;AAAA,IAC7C;AAAc,aAAO,KAAK,QAAQ,QAAQ;AAAA,EAC5C;AACF;AAIA,eAAe,2BAA2B,MAAyG;AACjJ,SAAO,KAAK,SAAS,MAAM;AACzB,UAAM,MAAM,MAAM,KAAK,SAAS;AAAA,MAC9B;AAAA,IACF,CAAC;AACD,WAAO,IACJ,OAAO,CAAC,OAAO;AACd,YAAM,OAAO,GAAG,sBAAsB;AACtC,aAAO,KAAK,QAAQ,KAAK,KAAK,SAAS;AAAA,IACzC,CAAC,EACA,MAAM,GAAG,GAAG,EACZ,IAAI,CAAC,OAAO;AACX,YAAM,YAAY,GAAG,aAAa,YAAY,KAAK;AACnD,YAAM,eAAe,GAAG,eAAe,IAAI,QAAQ,QAAQ,GAAG,EAAE,KAAK,EAAE,MAAM,GAAG,GAAG,KAAK;AACxF,YAAM,OAAO,GAAG,aAAa,MAAM,KAAK,GAAG,QAAQ,YAAY;AAC/D,aAAO,EAAE,SAAS,aAAa,eAAe,MAAM,MAAM,WAAW,YAAY;AAAA,IACnF,CAAC;AAAA,EACL,CAAC;AACH;AAWA,eAAe,eACb,MACA,MACA,SACqI;AACrI,QAAM,EAAE,UAAU,mBAAmB,OAAO,IAAI;AAChD,MAAI,CAAC,SAAU,QAAO;AAEtB,QAAM,UAAU,aAAa,MAAM,UAAU,gBAAgB;AAC7D,QAAM,WAAW,MAAM,QAAQ,MAAM,EAAE,UAAU,EAAE,SAAS,KAAK,CAAC,EAAE,MAAM,MAAM,KAAK;AACrF,MAAI,SAAU,QAAO,EAAE,KAAK,SAAS,QAAQ,MAAM;AAGnD,aAAW,YAAY,oBAAoB;AACzC,QAAI,aAAa,iBAAkB;AACnC,UAAM,MAAM,aAAa,MAAM,UAAU,QAAQ;AACjD,UAAM,UAAU,MAAM,IAAI,MAAM,EAAE,UAAU,EAAE,SAAS,IAAI,CAAC,EAAE,MAAM,MAAM,KAAK;AAC/E,QAAI,QAAS,QAAO,EAAE,KAAK,QAAQ,MAAM,WAAW,aAAa;AAAA,EACnE;AAGA,MAAI,SAAS;AACX,QAAI;AACF,YAAM,EAAE,YAAY,IAAI,MAAM,OAAO,
|
|
4
|
+
"sourcesContent": ["import { chromium, firefox, webkit, type Page, type Locator } from 'playwright';\nimport type { ExecutionStep } from '@detiq/agents';\nimport { assertCrawlUrlSafe } from '@detiq/core';\nimport { analyzeFailure } from './failure-analysis-agent.js';\n\nexport interface StepResult {\n stepNumber: number;\n description: string;\n status: 'PASSED' | 'FAILED' | 'SKIPPED';\n duration: number;\n error?: string;\n screenshotPath?: string;\n healed?: boolean;\n /** 'mechanical' = same selector text, different Playwright locator strategy.\n * 'ai' = AI Self-Healing Agent matched a semantically different element\n * (the app's selector text itself changed, e.g. \"Sign In\" -> \"Log In\"). */\n healedVia?: 'mechanical' | 'ai';\n healReasoning?: string;\n healConfidence?: number;\n}\n\nexport interface ExecutionResult {\n stepResults: StepResult[];\n status: 'PASSED' | 'FAILED' | 'ERROR' | 'FLAKY';\n errorStep?: number;\n totalDuration: number;\n finalScreenshotPath?: string;\n retryCount?: number;\n consoleLogs?: Array<{ type: string; text: string; timestamp: number }>;\n networkErrors?: Array<{ url: string; status: number; method: string; timestamp: number }>;\n rootCauseSummary?: string;\n networkTimings?: Array<{ url: string; method: string; status: number; duration: number }>;\n}\n\nconst LOCATOR_STRATEGIES: ExecutionStep['selectorStrategy'][] = ['text', 'label', 'role', 'css'];\n\nfunction buildLocator(page: Page, selector: string, strategy: ExecutionStep['selectorStrategy'] = 'text'): Locator {\n switch (strategy) {\n case 'css': return page.locator(selector);\n case 'text': return page.getByText(selector, { exact: false });\n case 'role': return page.getByRole(selector as any);\n case 'label': return page.getByLabel(selector);\n default: return page.locator(selector);\n }\n}\n\n/** Scans the live page for visible interactive elements \u2014 the \"current page elements\"\n * fed to the AI Self-Healing Agent when mechanical locator strategies fail. */\nasync function extractCurrentPageElements(page: Page): Promise<Array<{ meaning: string; role: string; ariaLabel?: string; textContent?: string }>> {\n return page.evaluate(() => {\n const els = Array.from(document.querySelectorAll<HTMLElement>(\n 'a, button, input, select, textarea, [role=\"button\"], [role=\"link\"], [role=\"tab\"], [role=\"menuitem\"], [aria-label]'\n ));\n return els\n .filter((el) => {\n const rect = el.getBoundingClientRect();\n return rect.width > 0 && rect.height > 0;\n })\n .slice(0, 200)\n .map((el) => {\n const ariaLabel = el.getAttribute('aria-label') ?? undefined;\n const textContent = (el.textContent ?? '').replace(/\\s+/g, ' ').trim().slice(0, 100) || undefined;\n const role = el.getAttribute('role') ?? el.tagName.toLowerCase();\n return { meaning: ariaLabel || textContent || role, role, ariaLabel, textContent };\n });\n });\n}\n\n/**\n * Resolves a step's target element in three tiers:\n * 1. Exact selector + strategy as generated.\n * 2. Mechanical: same selector text, other Playwright locator strategies\n * (handles a strategy mismatch, not an actual UI change).\n * 3. AI: when tenantId/projectId are supplied, asks the Self-Healing Agent to find\n * the current page's best semantic match for the failed step description \u2014 the\n * tier that actually survives the app's selector text changing.\n */\nasync function resolveLocator(\n page: Page,\n step: ExecutionStep,\n healCtx?: { tenantId: string; projectId: string },\n): Promise<{ loc: Locator; healed: boolean; healedVia?: 'mechanical' | 'ai'; healReasoning?: string; healConfidence?: number } | null> {\n const { selector, selectorStrategy = 'text' } = step;\n if (!selector) return null;\n\n const primary = buildLocator(page, selector, selectorStrategy);\n const attached = await primary.first().isVisible({ timeout: 1500 }).catch(() => false);\n if (attached) return { loc: primary, healed: false };\n\n // Tier 2 \u2014 mechanical: try other strategies before giving up\n for (const strategy of LOCATOR_STRATEGIES) {\n if (strategy === selectorStrategy) continue;\n const loc = buildLocator(page, selector, strategy);\n const visible = await loc.first().isVisible({ timeout: 800 }).catch(() => false);\n if (visible) return { loc, healed: true, healedVia: 'mechanical' };\n }\n\n // Tier 3 \u2014 AI: the selector text itself may no longer match anything on the page.\n if (healCtx) {\n try {\n const { healElement } = await import('@detiq/agents');\n const currentPageElements = await extractCurrentPageElements(page);\n const result = await healElement({\n tenantId: healCtx.tenantId,\n projectId: healCtx.projectId,\n failedElementMeaning: step.description || selector,\n currentPageElements,\n });\n if (result.healed && result.updatedTest) {\n const healedLoc = page.getByText(result.updatedTest.elementMeaning, { exact: false });\n const visible = await healedLoc.first().isVisible({ timeout: 1500 }).catch(() => false);\n if (visible) {\n return {\n loc: healedLoc,\n healed: true,\n healedVia: 'ai',\n healReasoning: result.reasoning,\n healConfidence: result.confidence,\n };\n }\n }\n } catch {\n // AI healing is a best-effort last resort \u2014 any failure here (LLM error, bad\n // JSON, etc.) just falls through to the caller's normal Playwright error below.\n }\n }\n\n // Return primary \u2014 caller will fail with a meaningful Playwright error\n return { loc: primary, healed: false };\n}\n\nfunction applyHealInfo(result: StepResult, r: { healed: boolean; healedVia?: 'mechanical' | 'ai'; healReasoning?: string; healConfidence?: number }): void {\n result.healed = r.healed;\n result.healedVia = r.healedVia;\n result.healReasoning = r.healReasoning;\n result.healConfidence = r.healConfidence;\n}\n\nasync function runPlan(\n page: Page,\n plan: ExecutionStep[],\n options: { screenshotDir: string; testRunId: string; onStepComplete?: (result: StepResult) => void; healCtx?: { tenantId: string; projectId: string } },\n consoleLogs: Array<{ type: string; text: string; timestamp: number }>,\n networkErrors: Array<{ url: string; status: number; method: string; timestamp: number }>,\n): Promise<{ stepResults: StepResult[]; errorStep?: number; finalScreenshotPath?: string; consoleLogs: typeof consoleLogs; networkErrors: typeof networkErrors }> {\n const stepResults: StepResult[] = [];\n let errorStep: number | undefined;\n let finalScreenshotPath: string | undefined;\n\n for (const step of plan) {\n const stepStart = Date.now();\n const result: StepResult = {\n stepNumber: step.stepNumber,\n description: step.description,\n status: 'FAILED',\n duration: 0,\n };\n\n try {\n const timeout = step.timeout ?? 8000;\n\n switch (step.action) {\n case 'navigate':\n if (!step.url) throw new Error('Navigate step missing url');\n assertCrawlUrlSafe(step.url);\n await page.goto(step.url, { waitUntil: 'domcontentloaded', timeout });\n break;\n\n case 'click': {\n const r = await resolveLocator(page, step, options.healCtx);\n if (!r) throw new Error('Could not resolve locator');\n await r.loc.first().click({ timeout });\n applyHealInfo(result, r);\n break;\n }\n\n case 'fill': {\n const r = await resolveLocator(page, step, options.healCtx);\n if (!r) throw new Error('Could not resolve locator');\n await r.loc.first().fill(step.value ?? '', { timeout });\n applyHealInfo(result, r);\n break;\n }\n\n case 'select': {\n const r = await resolveLocator(page, step, options.healCtx);\n if (!r) throw new Error('Could not resolve locator');\n await r.loc.first().selectOption(step.value ?? '', { timeout });\n applyHealInfo(result, r);\n break;\n }\n\n case 'hover': {\n const r = await resolveLocator(page, step, options.healCtx);\n if (!r) throw new Error('Could not resolve locator');\n await r.loc.first().hover({ timeout });\n applyHealInfo(result, r);\n break;\n }\n\n case 'press_key':\n await page.keyboard.press(step.value ?? 'Enter');\n break;\n\n case 'assert_visible': {\n const r = await resolveLocator(page, step, options.healCtx);\n if (!r) throw new Error('Could not resolve locator');\n await r.loc.first().waitFor({ state: 'visible', timeout });\n applyHealInfo(result, r);\n break;\n }\n\n case 'assert_text': {\n const r = await resolveLocator(page, step, options.healCtx);\n if (!r) throw new Error('Could not resolve locator');\n const text = await r.loc.first().textContent({ timeout });\n if (!text?.includes(step.value ?? '')) {\n throw new Error(`Expected text \"${step.value}\" not found. Got: \"${text?.slice(0, 100)}\"`);\n }\n applyHealInfo(result, r);\n break;\n }\n\n case 'assert_url': {\n const url = page.url();\n if (!url.includes(step.value ?? '')) {\n throw new Error(`Expected URL to contain \"${step.value}\". Got: \"${url}\"`);\n }\n break;\n }\n\n case 'wait':\n await page.waitForTimeout(step.timeout ?? 2000);\n break;\n\n case 'screenshot': {\n const ssPath = `${options.screenshotDir}/${options.testRunId}-step-${step.stepNumber}.png`;\n await page.screenshot({ path: ssPath, fullPage: true });\n result.screenshotPath = ssPath;\n break;\n }\n }\n\n result.status = 'PASSED';\n } catch (err: any) {\n result.status = step.optional ? 'SKIPPED' : 'FAILED';\n result.error = err.message;\n if (!step.optional) {\n errorStep = step.stepNumber;\n }\n }\n\n try {\n await page.waitForLoadState('networkidle', { timeout: 1500 }).catch(() => {});\n if (!result.screenshotPath) {\n const ssPath = `${options.screenshotDir}/${options.testRunId}-step-${step.stepNumber}.png`;\n await page.screenshot({ path: ssPath, fullPage: true });\n result.screenshotPath = ssPath;\n }\n } catch { /* Screenshot evidence is best-effort and should not fail the run. */ }\n\n result.duration = Date.now() - stepStart;\n stepResults.push(result);\n options.onStepComplete?.(result);\n\n if (result.status === 'FAILED') break;\n }\n\n try {\n const finalPath = `${options.screenshotDir}/${options.testRunId}-final.png`;\n await page.screenshot({ path: finalPath, fullPage: true });\n finalScreenshotPath = finalPath;\n } catch { /* non-fatal */ }\n\n return { stepResults, errorStep, finalScreenshotPath, consoleLogs, networkErrors };\n}\n\nfunction interpolate(template: string | undefined, data: Record<string, string>): string | undefined {\n if (!template) return template;\n return template.replace(/\\{\\{(\\w+)\\}\\}/g, (_, key) => data[key] ?? `{{${key}}}`);\n}\n\nexport async function executeTestPlan(\n plan: ExecutionStep[],\n options: {\n screenshotDir: string;\n testRunId: string;\n viewport?: { width: number; height: number };\n onStepComplete?: (result: StepResult) => void;\n /** Run the test up to this many times to detect flakiness (default 1). */\n maxRuns?: number;\n browser?: 'chromium' | 'firefox' | 'webkit';\n /** Data row for parameterized execution: substitutes {{varname}} in step fields. */\n dataRow?: Record<string, string>;\n /** Enables the AI Self-Healing Agent tier when a step's selector can't be resolved\n * even after trying alternate locator strategies. Omit to keep mechanical-only\n * healing (no LLM calls during execution). */\n healCtx?: { tenantId: string; projectId: string };\n /** Decrypted Playwright storageState JSON \u2014 restores a logged-in session so\n * steps don't have to re-authenticate. Without this, any test case behind a\n * login wall fails at the first authenticated step. */\n storageState?: string;\n },\n): Promise<ExecutionResult> {\n if (options.dataRow && Object.keys(options.dataRow).length > 0) {\n const data = options.dataRow;\n plan = plan.map((step) => ({\n ...step,\n url: interpolate(step.url, data),\n selector: interpolate(step.selector, data),\n value: interpolate(step.value, data),\n description: interpolate(step.description, data) ?? step.description,\n }));\n }\n const maxRuns = options.maxRuns ?? 1;\n const browserEngine = { chromium, firefox, webkit }[options.browser ?? 'chromium'] ?? chromium;\n const start = Date.now();\n\n const outcomes: ('PASSED' | 'FAILED')[] = [];\n let lastResult: Awaited<ReturnType<typeof runPlan>> | null = null;\n let lastNetworkTimings: Array<{ url: string; method: string; status: number; duration: number }> = [];\n\n for (let run = 0; run < maxRuns; run++) {\n const browser = await browserEngine.launch({\n headless: true,\n // Optional override for environments with a non-standard/pre-provisioned\n // browser install (e.g. a custom base image with only system Chromium).\n ...(process.env.PLAYWRIGHT_EXECUTABLE_PATH ? { executablePath: process.env.PLAYWRIGHT_EXECUTABLE_PATH } : {}),\n });\n let parsedStorageState: any;\n try {\n parsedStorageState = options.storageState ? JSON.parse(options.storageState) : undefined;\n } catch {\n parsedStorageState = undefined;\n }\n const context = await browser.newContext({\n viewport: options.viewport ?? { width: 1440, height: 900 },\n ...(parsedStorageState ? { storageState: parsedStorageState } : {}),\n });\n const page = await context.newPage();\n\n await page\n .addStyleTag({ content: '*, *::before, *::after { animation-duration: 0s !important; transition-duration: 0s !important; }' })\n .catch(() => {});\n\n const consoleLogs: Array<{ type: string; text: string; timestamp: number }> = [];\n const networkErrors: Array<{ url: string; status: number; method: string; timestamp: number }> = [];\n const networkTimings: Array<{ url: string; method: string; status: number; duration: number }> = [];\n const requestStartTimes = new Map<string, number>();\n\n page.on('console', msg => {\n if (consoleLogs.length < 100) {\n consoleLogs.push({ type: msg.type(), text: msg.text().slice(0, 500), timestamp: Date.now() });\n }\n });\n page.on('request', request => {\n const url = request.url();\n if (url.includes('/api/') || url.match(/\\/(graphql|rest|v\\d)\\//)) {\n requestStartTimes.set(request.url(), Date.now());\n }\n });\n page.on('response', response => {\n const url = response.url();\n const start = requestStartTimes.get(url);\n if (start && networkTimings.length < 200) {\n networkTimings.push({ url: url.slice(0, 300), method: response.request().method(), status: response.status(), duration: Date.now() - start });\n requestStartTimes.delete(url);\n }\n if (!response.ok() && !url.includes('favicon') && networkErrors.length < 50) {\n networkErrors.push({\n url: url.slice(0, 300),\n status: response.status(),\n method: response.request().method(),\n timestamp: Date.now(),\n });\n }\n });\n\n const TEST_EXECUTION_TIMEOUT_MS = 5 * 60_000;\n lastResult = await Promise.race([\n runPlan(page, plan, options, consoleLogs, networkErrors),\n new Promise<never>((_, reject) =>\n setTimeout(() => reject(new Error('Test execution timed out')), TEST_EXECUTION_TIMEOUT_MS)\n ),\n ]);\n lastNetworkTimings = networkTimings;\n await browser.close();\n\n const passed = !lastResult.stepResults.some((r) => r.status === 'FAILED');\n outcomes.push(passed ? 'PASSED' : 'FAILED');\n\n // If all subsequent runs are consistent with first, no need to keep going\n if (run > 0 && outcomes.every((o) => o === outcomes[0])) break;\n }\n\n const passCount = outcomes.filter((o) => o === 'PASSED').length;\n const failCount = outcomes.filter((o) => o === 'FAILED').length;\n\n let status: ExecutionResult['status'];\n if (maxRuns > 1 && passCount > 0 && failCount > 0) {\n status = 'FLAKY';\n } else if (passCount === outcomes.length) {\n status = 'PASSED';\n } else {\n status = 'FAILED';\n }\n\n let rootCauseSummary: string | undefined;\n if (status === 'FAILED' || status === 'FLAKY') {\n const failedStep = lastResult!.stepResults.find(s => s.status === 'FAILED');\n rootCauseSummary = await analyzeFailure({\n tenantId: options.healCtx?.tenantId,\n error: failedStep?.error ?? 'Unknown error',\n stepDescription: failedStep?.description,\n consoleLogs: lastResult!.consoleLogs,\n networkErrors: lastResult!.networkErrors,\n screenshotPath: lastResult!.finalScreenshotPath,\n }).catch(() => undefined) ?? undefined;\n }\n\n return {\n stepResults: lastResult!.stepResults,\n status,\n errorStep: lastResult!.errorStep,\n totalDuration: Date.now() - start,\n finalScreenshotPath: lastResult!.finalScreenshotPath,\n retryCount: outcomes.length - 1,\n consoleLogs: lastResult!.consoleLogs,\n networkErrors: lastResult!.networkErrors,\n rootCauseSummary,\n networkTimings: lastNetworkTimings,\n };\n}\n"],
|
|
5
|
+
"mappings": "AAAA,SAAS,UAAU,SAAS,cAAuC;AAEnE,SAAS,0BAA0B;AACnC,SAAS,sBAAsB;AA+B/B,MAAM,qBAA0D,CAAC,QAAQ,SAAS,QAAQ,KAAK;AAE/F,SAAS,aAAa,MAAY,UAAkB,WAA8C,QAAiB;AACjH,UAAQ,UAAU;AAAA,IAChB,KAAK;AAAS,aAAO,KAAK,QAAQ,QAAQ;AAAA,IAC1C,KAAK;AAAS,aAAO,KAAK,UAAU,UAAU,EAAE,OAAO,MAAM,CAAC;AAAA,IAC9D,KAAK;AAAS,aAAO,KAAK,UAAU,QAAe;AAAA,IACnD,KAAK;AAAS,aAAO,KAAK,WAAW,QAAQ;AAAA,IAC7C;AAAc,aAAO,KAAK,QAAQ,QAAQ;AAAA,EAC5C;AACF;AAIA,eAAe,2BAA2B,MAAyG;AACjJ,SAAO,KAAK,SAAS,MAAM;AACzB,UAAM,MAAM,MAAM,KAAK,SAAS;AAAA,MAC9B;AAAA,IACF,CAAC;AACD,WAAO,IACJ,OAAO,CAAC,OAAO;AACd,YAAM,OAAO,GAAG,sBAAsB;AACtC,aAAO,KAAK,QAAQ,KAAK,KAAK,SAAS;AAAA,IACzC,CAAC,EACA,MAAM,GAAG,GAAG,EACZ,IAAI,CAAC,OAAO;AACX,YAAM,YAAY,GAAG,aAAa,YAAY,KAAK;AACnD,YAAM,eAAe,GAAG,eAAe,IAAI,QAAQ,QAAQ,GAAG,EAAE,KAAK,EAAE,MAAM,GAAG,GAAG,KAAK;AACxF,YAAM,OAAO,GAAG,aAAa,MAAM,KAAK,GAAG,QAAQ,YAAY;AAC/D,aAAO,EAAE,SAAS,aAAa,eAAe,MAAM,MAAM,WAAW,YAAY;AAAA,IACnF,CAAC;AAAA,EACL,CAAC;AACH;AAWA,eAAe,eACb,MACA,MACA,SACqI;AACrI,QAAM,EAAE,UAAU,mBAAmB,OAAO,IAAI;AAChD,MAAI,CAAC,SAAU,QAAO;AAEtB,QAAM,UAAU,aAAa,MAAM,UAAU,gBAAgB;AAC7D,QAAM,WAAW,MAAM,QAAQ,MAAM,EAAE,UAAU,EAAE,SAAS,KAAK,CAAC,EAAE,MAAM,MAAM,KAAK;AACrF,MAAI,SAAU,QAAO,EAAE,KAAK,SAAS,QAAQ,MAAM;AAGnD,aAAW,YAAY,oBAAoB;AACzC,QAAI,aAAa,iBAAkB;AACnC,UAAM,MAAM,aAAa,MAAM,UAAU,QAAQ;AACjD,UAAM,UAAU,MAAM,IAAI,MAAM,EAAE,UAAU,EAAE,SAAS,IAAI,CAAC,EAAE,MAAM,MAAM,KAAK;AAC/E,QAAI,QAAS,QAAO,EAAE,KAAK,QAAQ,MAAM,WAAW,aAAa;AAAA,EACnE;AAGA,MAAI,SAAS;AACX,QAAI;AACF,YAAM,EAAE,YAAY,IAAI,MAAM,OAAO,eAAe;AACpD,YAAM,sBAAsB,MAAM,2BAA2B,IAAI;AACjE,YAAM,SAAS,MAAM,YAAY;AAAA,QAC/B,UAAU,QAAQ;AAAA,QAClB,WAAW,QAAQ;AAAA,QACnB,sBAAsB,KAAK,eAAe;AAAA,QAC1C;AAAA,MACF,CAAC;AACD,UAAI,OAAO,UAAU,OAAO,aAAa;AACvC,cAAM,YAAY,KAAK,UAAU,OAAO,YAAY,gBAAgB,EAAE,OAAO,MAAM,CAAC;AACpF,cAAM,UAAU,MAAM,UAAU,MAAM,EAAE,UAAU,EAAE,SAAS,KAAK,CAAC,EAAE,MAAM,MAAM,KAAK;AACtF,YAAI,SAAS;AACX,iBAAO;AAAA,YACL,KAAK;AAAA,YACL,QAAQ;AAAA,YACR,WAAW;AAAA,YACX,eAAe,OAAO;AAAA,YACtB,gBAAgB,OAAO;AAAA,UACzB;AAAA,QACF;AAAA,MACF;AAAA,IACF,QAAQ;AAAA,IAGR;AAAA,EACF;AAGA,SAAO,EAAE,KAAK,SAAS,QAAQ,MAAM;AACvC;AAEA,SAAS,cAAc,QAAoB,GAAgH;AACzJ,SAAO,SAAS,EAAE;AAClB,SAAO,YAAY,EAAE;AACrB,SAAO,gBAAgB,EAAE;AACzB,SAAO,iBAAiB,EAAE;AAC5B;AAEA,eAAe,QACb,MACA,MACA,SACA,aACA,eACgK;AAChK,QAAM,cAA4B,CAAC;AACnC,MAAI;AACJ,MAAI;AAEJ,aAAW,QAAQ,MAAM;AACvB,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,SAAqB;AAAA,MACzB,YAAY,KAAK;AAAA,MACjB,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,UAAU;AAAA,IACZ;AAEA,QAAI;AACF,YAAM,UAAU,KAAK,WAAW;AAEhC,cAAQ,KAAK,QAAQ;AAAA,QACnB,KAAK;AACH,cAAI,CAAC,KAAK,IAAK,OAAM,IAAI,MAAM,2BAA2B;AAC1D,6BAAmB,KAAK,GAAG;AAC3B,gBAAM,KAAK,KAAK,KAAK,KAAK,EAAE,WAAW,oBAAoB,QAAQ,CAAC;AACpE;AAAA,QAEF,KAAK,SAAS;AACZ,gBAAM,IAAI,MAAM,eAAe,MAAM,MAAM,QAAQ,OAAO;AAC1D,cAAI,CAAC,EAAG,OAAM,IAAI,MAAM,2BAA2B;AACnD,gBAAM,EAAE,IAAI,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC;AACrC,wBAAc,QAAQ,CAAC;AACvB;AAAA,QACF;AAAA,QAEA,KAAK,QAAQ;AACX,gBAAM,IAAI,MAAM,eAAe,MAAM,MAAM,QAAQ,OAAO;AAC1D,cAAI,CAAC,EAAG,OAAM,IAAI,MAAM,2BAA2B;AACnD,gBAAM,EAAE,IAAI,MAAM,EAAE,KAAK,KAAK,SAAS,IAAI,EAAE,QAAQ,CAAC;AACtD,wBAAc,QAAQ,CAAC;AACvB;AAAA,QACF;AAAA,QAEA,KAAK,UAAU;AACb,gBAAM,IAAI,MAAM,eAAe,MAAM,MAAM,QAAQ,OAAO;AAC1D,cAAI,CAAC,EAAG,OAAM,IAAI,MAAM,2BAA2B;AACnD,gBAAM,EAAE,IAAI,MAAM,EAAE,aAAa,KAAK,SAAS,IAAI,EAAE,QAAQ,CAAC;AAC9D,wBAAc,QAAQ,CAAC;AACvB;AAAA,QACF;AAAA,QAEA,KAAK,SAAS;AACZ,gBAAM,IAAI,MAAM,eAAe,MAAM,MAAM,QAAQ,OAAO;AAC1D,cAAI,CAAC,EAAG,OAAM,IAAI,MAAM,2BAA2B;AACnD,gBAAM,EAAE,IAAI,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC;AACrC,wBAAc,QAAQ,CAAC;AACvB;AAAA,QACF;AAAA,QAEA,KAAK;AACH,gBAAM,KAAK,SAAS,MAAM,KAAK,SAAS,OAAO;AAC/C;AAAA,QAEF,KAAK,kBAAkB;AACrB,gBAAM,IAAI,MAAM,eAAe,MAAM,MAAM,QAAQ,OAAO;AAC1D,cAAI,CAAC,EAAG,OAAM,IAAI,MAAM,2BAA2B;AACnD,gBAAM,EAAE,IAAI,MAAM,EAAE,QAAQ,EAAE,OAAO,WAAW,QAAQ,CAAC;AACzD,wBAAc,QAAQ,CAAC;AACvB;AAAA,QACF;AAAA,QAEA,KAAK,eAAe;AAClB,gBAAM,IAAI,MAAM,eAAe,MAAM,MAAM,QAAQ,OAAO;AAC1D,cAAI,CAAC,EAAG,OAAM,IAAI,MAAM,2BAA2B;AACnD,gBAAM,OAAO,MAAM,EAAE,IAAI,MAAM,EAAE,YAAY,EAAE,QAAQ,CAAC;AACxD,cAAI,CAAC,MAAM,SAAS,KAAK,SAAS,EAAE,GAAG;AACrC,kBAAM,IAAI,MAAM,kBAAkB,KAAK,KAAK,sBAAsB,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG;AAAA,UAC1F;AACA,wBAAc,QAAQ,CAAC;AACvB;AAAA,QACF;AAAA,QAEA,KAAK,cAAc;AACjB,gBAAM,MAAM,KAAK,IAAI;AACrB,cAAI,CAAC,IAAI,SAAS,KAAK,SAAS,EAAE,GAAG;AACnC,kBAAM,IAAI,MAAM,4BAA4B,KAAK,KAAK,YAAY,GAAG,GAAG;AAAA,UAC1E;AACA;AAAA,QACF;AAAA,QAEA,KAAK;AACH,gBAAM,KAAK,eAAe,KAAK,WAAW,GAAI;AAC9C;AAAA,QAEF,KAAK,cAAc;AACjB,gBAAM,SAAS,GAAG,QAAQ,aAAa,IAAI,QAAQ,SAAS,SAAS,KAAK,UAAU;AACpF,gBAAM,KAAK,WAAW,EAAE,MAAM,QAAQ,UAAU,KAAK,CAAC;AACtD,iBAAO,iBAAiB;AACxB;AAAA,QACF;AAAA,MACF;AAEA,aAAO,SAAS;AAAA,IAClB,SAAS,KAAU;AACjB,aAAO,SAAS,KAAK,WAAW,YAAY;AAC5C,aAAO,QAAQ,IAAI;AACnB,UAAI,CAAC,KAAK,UAAU;AAClB,oBAAY,KAAK;AAAA,MACnB;AAAA,IACF;AAEA,QAAI;AACF,YAAM,KAAK,iBAAiB,eAAe,EAAE,SAAS,KAAK,CAAC,EAAE,MAAM,MAAM;AAAA,MAAC,CAAC;AAC5E,UAAI,CAAC,OAAO,gBAAgB;AAC1B,cAAM,SAAS,GAAG,QAAQ,aAAa,IAAI,QAAQ,SAAS,SAAS,KAAK,UAAU;AACpF,cAAM,KAAK,WAAW,EAAE,MAAM,QAAQ,UAAU,KAAK,CAAC;AACtD,eAAO,iBAAiB;AAAA,MAC1B;AAAA,IACF,QAAQ;AAAA,IAAwE;AAEhF,WAAO,WAAW,KAAK,IAAI,IAAI;AAC/B,gBAAY,KAAK,MAAM;AACvB,YAAQ,iBAAiB,MAAM;AAE/B,QAAI,OAAO,WAAW,SAAU;AAAA,EAClC;AAEA,MAAI;AACF,UAAM,YAAY,GAAG,QAAQ,aAAa,IAAI,QAAQ,SAAS;AAC/D,UAAM,KAAK,WAAW,EAAE,MAAM,WAAW,UAAU,KAAK,CAAC;AACzD,0BAAsB;AAAA,EACxB,QAAQ;AAAA,EAAkB;AAE1B,SAAO,EAAE,aAAa,WAAW,qBAAqB,aAAa,cAAc;AACnF;AAEA,SAAS,YAAY,UAA8B,MAAkD;AACnG,MAAI,CAAC,SAAU,QAAO;AACtB,SAAO,SAAS,QAAQ,kBAAkB,CAAC,GAAG,QAAQ,KAAK,GAAG,KAAK,KAAK,GAAG,IAAI;AACjF;AAEA,eAAsB,gBACpB,MACA,SAmB0B;AAC1B,MAAI,QAAQ,WAAW,OAAO,KAAK,QAAQ,OAAO,EAAE,SAAS,GAAG;AAC9D,UAAM,OAAO,QAAQ;AACrB,WAAO,KAAK,IAAI,CAAC,UAAU;AAAA,MACzB,GAAG;AAAA,MACH,KAAK,YAAY,KAAK,KAAK,IAAI;AAAA,MAC/B,UAAU,YAAY,KAAK,UAAU,IAAI;AAAA,MACzC,OAAO,YAAY,KAAK,OAAO,IAAI;AAAA,MACnC,aAAa,YAAY,KAAK,aAAa,IAAI,KAAK,KAAK;AAAA,IAC3D,EAAE;AAAA,EACJ;AACA,QAAM,UAAU,QAAQ,WAAW;AACnC,QAAM,gBAAgB,EAAE,UAAU,SAAS,OAAO,EAAE,QAAQ,WAAW,UAAU,KAAK;AACtF,QAAM,QAAQ,KAAK,IAAI;AAEvB,QAAM,WAAoC,CAAC;AAC3C,MAAI,aAAyD;AAC7D,MAAI,qBAA+F,CAAC;AAEpG,WAAS,MAAM,GAAG,MAAM,SAAS,OAAO;AACtC,UAAM,UAAU,MAAM,cAAc,OAAO;AAAA,MACzC,UAAU;AAAA;AAAA;AAAA,MAGV,GAAI,QAAQ,IAAI,6BAA6B,EAAE,gBAAgB,QAAQ,IAAI,2BAA2B,IAAI,CAAC;AAAA,IAC7G,CAAC;AACD,QAAI;AACJ,QAAI;AACF,2BAAqB,QAAQ,eAAe,KAAK,MAAM,QAAQ,YAAY,IAAI;AAAA,IACjF,QAAQ;AACN,2BAAqB;AAAA,IACvB;AACA,UAAM,UAAU,MAAM,QAAQ,WAAW;AAAA,MACvC,UAAU,QAAQ,YAAY,EAAE,OAAO,MAAM,QAAQ,IAAI;AAAA,MACzD,GAAI,qBAAqB,EAAE,cAAc,mBAAmB,IAAI,CAAC;AAAA,IACnE,CAAC;AACD,UAAM,OAAO,MAAM,QAAQ,QAAQ;AAEnC,UAAM,KACH,YAAY,EAAE,SAAS,oGAAoG,CAAC,EAC5H,MAAM,MAAM;AAAA,IAAC,CAAC;AAEjB,UAAM,cAAwE,CAAC;AAC/E,UAAM,gBAA2F,CAAC;AAClG,UAAM,iBAA2F,CAAC;AAClG,UAAM,oBAAoB,oBAAI,IAAoB;AAElD,SAAK,GAAG,WAAW,SAAO;AACxB,UAAI,YAAY,SAAS,KAAK;AAC5B,oBAAY,KAAK,EAAE,MAAM,IAAI,KAAK,GAAG,MAAM,IAAI,KAAK,EAAE,MAAM,GAAG,GAAG,GAAG,WAAW,KAAK,IAAI,EAAE,CAAC;AAAA,MAC9F;AAAA,IACF,CAAC;AACD,SAAK,GAAG,WAAW,aAAW;AAC5B,YAAM,MAAM,QAAQ,IAAI;AACxB,UAAI,IAAI,SAAS,OAAO,KAAK,IAAI,MAAM,wBAAwB,GAAG;AAChE,0BAAkB,IAAI,QAAQ,IAAI,GAAG,KAAK,IAAI,CAAC;AAAA,MACjD;AAAA,IACF,CAAC;AACD,SAAK,GAAG,YAAY,cAAY;AAC9B,YAAM,MAAM,SAAS,IAAI;AACzB,YAAMA,SAAQ,kBAAkB,IAAI,GAAG;AACvC,UAAIA,UAAS,eAAe,SAAS,KAAK;AACxC,uBAAe,KAAK,EAAE,KAAK,IAAI,MAAM,GAAG,GAAG,GAAG,QAAQ,SAAS,QAAQ,EAAE,OAAO,GAAG,QAAQ,SAAS,OAAO,GAAG,UAAU,KAAK,IAAI,IAAIA,OAAM,CAAC;AAC5I,0BAAkB,OAAO,GAAG;AAAA,MAC9B;AACA,UAAI,CAAC,SAAS,GAAG,KAAK,CAAC,IAAI,SAAS,SAAS,KAAK,cAAc,SAAS,IAAI;AAC3E,sBAAc,KAAK;AAAA,UACjB,KAAK,IAAI,MAAM,GAAG,GAAG;AAAA,UACrB,QAAQ,SAAS,OAAO;AAAA,UACxB,QAAQ,SAAS,QAAQ,EAAE,OAAO;AAAA,UAClC,WAAW,KAAK,IAAI;AAAA,QACtB,CAAC;AAAA,MACH;AAAA,IACF,CAAC;AAED,UAAM,4BAA4B,IAAI;AACtC,iBAAa,MAAM,QAAQ,KAAK;AAAA,MAC9B,QAAQ,MAAM,MAAM,SAAS,aAAa,aAAa;AAAA,MACvD,IAAI;AAAA,QAAe,CAAC,GAAG,WACrB,WAAW,MAAM,OAAO,IAAI,MAAM,0BAA0B,CAAC,GAAG,yBAAyB;AAAA,MAC3F;AAAA,IACF,CAAC;AACD,yBAAqB;AACrB,UAAM,QAAQ,MAAM;AAEpB,UAAM,SAAS,CAAC,WAAW,YAAY,KAAK,CAAC,MAAM,EAAE,WAAW,QAAQ;AACxE,aAAS,KAAK,SAAS,WAAW,QAAQ;AAG1C,QAAI,MAAM,KAAK,SAAS,MAAM,CAAC,MAAM,MAAM,SAAS,CAAC,CAAC,EAAG;AAAA,EAC3D;AAEA,QAAM,YAAY,SAAS,OAAO,CAAC,MAAM,MAAM,QAAQ,EAAE;AACzD,QAAM,YAAY,SAAS,OAAO,CAAC,MAAM,MAAM,QAAQ,EAAE;AAEzD,MAAI;AACJ,MAAI,UAAU,KAAK,YAAY,KAAK,YAAY,GAAG;AACjD,aAAS;AAAA,EACX,WAAW,cAAc,SAAS,QAAQ;AACxC,aAAS;AAAA,EACX,OAAO;AACL,aAAS;AAAA,EACX;AAEA,MAAI;AACJ,MAAI,WAAW,YAAY,WAAW,SAAS;AAC7C,UAAM,aAAa,WAAY,YAAY,KAAK,OAAK,EAAE,WAAW,QAAQ;AAC1E,uBAAmB,MAAM,eAAe;AAAA,MACtC,UAAU,QAAQ,SAAS;AAAA,MAC3B,OAAO,YAAY,SAAS;AAAA,MAC5B,iBAAiB,YAAY;AAAA,MAC7B,aAAa,WAAY;AAAA,MACzB,eAAe,WAAY;AAAA,MAC3B,gBAAgB,WAAY;AAAA,IAC9B,CAAC,EAAE,MAAM,MAAM,MAAS,KAAK;AAAA,EAC/B;AAEA,SAAO;AAAA,IACL,aAAa,WAAY;AAAA,IACzB;AAAA,IACA,WAAW,WAAY;AAAA,IACvB,eAAe,KAAK,IAAI,IAAI;AAAA,IAC5B,qBAAqB,WAAY;AAAA,IACjC,YAAY,SAAS,SAAS;AAAA,IAC9B,aAAa,WAAY;AAAA,IACzB,eAAe,WAAY;AAAA,IAC3B;AAAA,IACA,gBAAgB;AAAA,EAClB;AACF;",
|
|
6
6
|
"names": ["start"]
|
|
7
7
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../src/visual-diff-verdict.ts"],
|
|
4
|
-
"sourcesContent": ["/**\n * Pure visual-diff verdict logic, split out of crawler.ts so it can be unit-tested\n * without pulling in crawler.ts's full import graph (Playwright, Stagehand, and\n * transitively @
|
|
4
|
+
"sourcesContent": ["/**\n * Pure visual-diff verdict logic, split out of crawler.ts so it can be unit-tested\n * without pulling in crawler.ts's full import graph (Playwright, Stagehand, and\n * transitively @detiq/database, which instantiates a PrismaClient at module load and\n * so requires a live DATABASE_URL just to import the module).\n */\n\nexport interface VisualDiffVerdict {\n changeType: 'NONE' | 'MINOR' | 'SIGNIFICANT' | 'CRITICAL';\n recommendation: 'APPROVE' | 'REVIEW' | 'FLAG' | 'BLOCK';\n changes: Array<{ location: string; severity: string; description: string }>;\n bugReportDraft?: { title: string; severity: string; steps: string[]; expected: string; actual: string };\n aiSummary?: string;\n}\n\n/** Cheap, always-on first-pass verdict from perceptual hash distance + layout/element diffs. */\nexport function pHashVerdict(\n pHashDistance: number,\n changeLevel: 'IDENTICAL' | 'MINOR' | 'MODERATE' | 'SIGNIFICANT',\n changes: Array<{ location: string; severity: string; description: string }>,\n): VisualDiffVerdict {\n const changeType: VisualDiffVerdict['changeType'] = changeLevel === 'IDENTICAL' ? 'NONE'\n : changeLevel === 'MINOR' ? 'MINOR'\n : changeLevel === 'MODERATE' ? 'SIGNIFICANT'\n : 'CRITICAL';\n const recommendation: VisualDiffVerdict['recommendation'] = pHashDistance === 0 ? 'APPROVE'\n : pHashDistance < 5 ? 'APPROVE'\n : pHashDistance < 20 ? 'REVIEW'\n : 'FLAG';\n return { changeType, recommendation, changes };\n}\n\n/** Replaces a pHash-only verdict with the AI's semantic judgment once it's available,\n * and drafts a bug report for anything still CRITICAL/SIGNIFICANT after AI review\n * (pHash flags plenty of anti-aliasing/font-rendering noise the AI correctly discards\n * as NONE/MINOR \u2014 only what survives AI review is worth a draft). */\nexport function mergeAiVisualVerdict(\n pHashResult: VisualDiffVerdict,\n aiResult: { changeType: string; changes: any[]; recommendation: string; diffScore: number; summary?: string },\n screenName: string,\n): VisualDiffVerdict {\n if (!aiResult.changeType) return pHashResult;\n const changeType = aiResult.changeType as VisualDiffVerdict['changeType'];\n const recommendation = aiResult.recommendation as VisualDiffVerdict['recommendation'];\n const changes = Array.isArray(aiResult.changes) && aiResult.changes.length > 0 ? aiResult.changes : pHashResult.changes;\n const aiSummary = aiResult.summary ?? undefined;\n const bugReportDraft = (changeType === 'CRITICAL' || changeType === 'SIGNIFICANT')\n ? {\n title: `Visual regression on \"${screenName}\"`,\n severity: changeType === 'CRITICAL' ? 'High' : 'Medium',\n steps: ['Navigate to the screen', 'Compare current view to baseline screenshot'],\n expected: 'Screen appearance should match the stored baseline.',\n actual: aiSummary || `Screen has changed (diff score: ${aiResult.diffScore.toFixed(1)}%)`,\n }\n : undefined;\n return { changeType, recommendation, changes, bugReportDraft, aiSummary };\n}\n"],
|
|
5
5
|
"mappings": "AAgBO,SAAS,aACd,eACA,aACA,SACmB;AACnB,QAAM,aAA8C,gBAAgB,cAAc,SAC9E,gBAAgB,UAAU,UACxB,gBAAgB,aAAa,gBAC3B;AACR,QAAM,iBAAsD,kBAAkB,IAAI,YAC9E,gBAAgB,IAAI,YAClB,gBAAgB,KAAK,WACnB;AACR,SAAO,EAAE,YAAY,gBAAgB,QAAQ;AAC/C;AAMO,SAAS,qBACd,aACA,UACA,YACmB;AACnB,MAAI,CAAC,SAAS,WAAY,QAAO;AACjC,QAAM,aAAa,SAAS;AAC5B,QAAM,iBAAiB,SAAS;AAChC,QAAM,UAAU,MAAM,QAAQ,SAAS,OAAO,KAAK,SAAS,QAAQ,SAAS,IAAI,SAAS,UAAU,YAAY;AAChH,QAAM,YAAY,SAAS,WAAW;AACtC,QAAM,iBAAkB,eAAe,cAAc,eAAe,gBAChE;AAAA,IACE,OAAO,yBAAyB,UAAU;AAAA,IAC1C,UAAU,eAAe,aAAa,SAAS;AAAA,IAC/C,OAAO,CAAC,0BAA0B,6CAA6C;AAAA,IAC/E,UAAU;AAAA,IACV,QAAQ,aAAa,mCAAmC,SAAS,UAAU,QAAQ,CAAC,CAAC;AAAA,EACvF,IACA;AACJ,SAAO,EAAE,YAAY,gBAAgB,SAAS,gBAAgB,UAAU;AAC1E;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
package/dist/worker.js.map
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../src/worker.ts"],
|
|
4
|
-
"sourcesContent": ["/**\n * Standalone worker process entrypoint for a dedicated crawler/worker container,\n * separate from the API server (see infrastructure/ecs-task-definitions/crawler.json).\n *\n * index.ts is a side-effect-free library barrel imported by @
|
|
4
|
+
"sourcesContent": ["/**\n * Standalone worker process entrypoint for a dedicated crawler/worker container,\n * separate from the API server (see infrastructure/ecs-task-definitions/crawler.json).\n *\n * index.ts is a side-effect-free library barrel imported by @detiq/api \u2014 it must NOT\n * start any background loop itself, or importing a single helper from @detiq/crawler\n * would silently spin up a second worker inside the API process too. This file is\n * the actual entrypoint that keeps a dedicated worker container's process alive and\n * consuming jobs: `node dist/worker.js`.\n */\nimport { startJobWorkerLoop, processDataRetentionPurge } from './job-queue.js';\nimport { logger } from './logger.js';\n\nasync function main() {\n logger.info('[worker] starting standalone crawler worker process');\n\n // Durable DB-backed queue \u2014 claims every non-CRAWL job type, and CRAWL too when\n // JOB_QUEUE_BACKEND isn't 'sqs'. Safe to run in many worker replicas at once.\n startJobWorkerLoop();\n\n // Daily data-retention purge: runs immediately on startup then every 24 h.\n // Only runs in the dedicated worker container (this file), not in the API\n // process, so there is exactly one purge sweep regardless of API replica count.\n const runPurge = () => {\n processDataRetentionPurge().catch((err) => {\n logger.error({ err: String(err) }, '[worker] data-retention purge failed');\n });\n };\n runPurge();\n setInterval(runPurge, 24 * 60 * 60 * 1000);\n\n // Optional: also consume the SQS transport for horizontally-scaled CRAWL fleets.\n if (process.env.JOB_QUEUE_BACKEND === 'sqs') {\n const { startSqsWorker } = await import('./sqs-queue.js');\n startSqsWorker().catch((err) => {\n logger.error({ err: String(err) }, '[worker] SQS worker crashed');\n process.exit(1);\n });\n }\n}\n\nprocess.on('SIGTERM', () => { logger.info('[worker] SIGTERM received, exiting'); process.exit(0); });\nprocess.on('SIGINT', () => { logger.info('[worker] SIGINT received, exiting'); process.exit(0); });\n\nmain().catch((err) => {\n logger.error({ err: String(err) }, '[worker] fatal startup error');\n process.exit(1);\n});\n"],
|
|
5
5
|
"mappings": "AAUA,SAAS,oBAAoB,iCAAiC;AAC9D,SAAS,cAAc;AAEvB,eAAe,OAAO;AACpB,SAAO,KAAK,qDAAqD;AAIjE,qBAAmB;AAKnB,QAAM,WAAW,MAAM;AACrB,8BAA0B,EAAE,MAAM,CAAC,QAAQ;AACzC,aAAO,MAAM,EAAE,KAAK,OAAO,GAAG,EAAE,GAAG,sCAAsC;AAAA,IAC3E,CAAC;AAAA,EACH;AACA,WAAS;AACT,cAAY,UAAU,KAAK,KAAK,KAAK,GAAI;AAGzC,MAAI,QAAQ,IAAI,sBAAsB,OAAO;AAC3C,UAAM,EAAE,eAAe,IAAI,MAAM,OAAO,gBAAgB;AACxD,mBAAe,EAAE,MAAM,CAAC,QAAQ;AAC9B,aAAO,MAAM,EAAE,KAAK,OAAO,GAAG,EAAE,GAAG,6BAA6B;AAChE,cAAQ,KAAK,CAAC;AAAA,IAChB,CAAC;AAAA,EACH;AACF;AAEA,QAAQ,GAAG,WAAW,MAAM;AAAE,SAAO,KAAK,oCAAoC;AAAG,UAAQ,KAAK,CAAC;AAAG,CAAC;AACnG,QAAQ,GAAG,UAAU,MAAM;AAAE,SAAO,KAAK,mCAAmC;AAAG,UAAQ,KAAK,CAAC;AAAG,CAAC;AAEjG,KAAK,EAAE,MAAM,CAAC,QAAQ;AACpB,SAAO,MAAM,EAAE,KAAK,OAAO,GAAG,EAAE,GAAG,8BAA8B;AACjE,UAAQ,KAAK,CAAC;AAChB,CAAC;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
package/package.json
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@detiq/crawler",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.5",
|
|
4
4
|
"private": false,
|
|
5
|
+
"publishConfig": { "access": "public", "registry": "https://registry.npmjs.org/" },
|
|
5
6
|
"type": "module",
|
|
6
7
|
"main": "dist/index.js",
|
|
7
8
|
"types": "types/index.d.ts",
|
|
@@ -13,14 +14,14 @@
|
|
|
13
14
|
"dependencies": {
|
|
14
15
|
"@aws-sdk/client-sqs": "^3.1075.0",
|
|
15
16
|
"@browserbasehq/stagehand": "^3.6.0",
|
|
16
|
-
"@mozilla/readability": "^0.6.0",
|
|
17
17
|
"@detiq/agents": "*",
|
|
18
18
|
"@detiq/app-brain": "*",
|
|
19
19
|
"@detiq/core": "*",
|
|
20
20
|
"@detiq/database": "*",
|
|
21
|
+
"@mozilla/readability": "^0.6.0",
|
|
21
22
|
"axe-core": "^4.12.1",
|
|
22
23
|
"camoufox": "^0.1.19",
|
|
23
|
-
"fast-xml-parser": "^5.
|
|
24
|
+
"fast-xml-parser": "^5.10.1",
|
|
24
25
|
"imapflow": "^1.4.3",
|
|
25
26
|
"lighthouse": "^13.4.0",
|
|
26
27
|
"linkedom": "^0.18.12",
|
|
@@ -32,7 +33,7 @@
|
|
|
32
33
|
"puppeteer-extra-plugin-stealth": "^2.11.2",
|
|
33
34
|
"rebrowser-patches": "^1.0.19",
|
|
34
35
|
"rebrowser-playwright-core": "^1.52.0",
|
|
35
|
-
"sharp": "^0.
|
|
36
|
+
"sharp": "^0.35.0",
|
|
36
37
|
"turndown": "^7.2.4",
|
|
37
38
|
"webdriverio": "^9.0.0",
|
|
38
39
|
"zod": "^3.25.0"
|
package/src/ai-login.ts
CHANGED
|
@@ -126,13 +126,23 @@ export async function loginWithStagehand(
|
|
|
126
126
|
}
|
|
127
127
|
|
|
128
128
|
// If caller provided natural language login instructions, use them to navigate to the login form
|
|
129
|
+
const MAX_LOGIN_INSTRUCTIONS = 2000;
|
|
130
|
+
if (loginInstructions && loginInstructions.length > MAX_LOGIN_INSTRUCTIONS) {
|
|
131
|
+
throw new Error(`loginInstructions exceeds ${MAX_LOGIN_INSTRUCTIONS} character limit`);
|
|
132
|
+
}
|
|
129
133
|
if (loginInstructions) {
|
|
130
134
|
await logStep('custom_recipe', () => actWithMouseSim(stagehand, loginInstructions));
|
|
131
135
|
await pg.waitForLoadState('networkidle').catch(() => pg.waitForTimeout(1500));
|
|
132
136
|
}
|
|
133
137
|
|
|
134
138
|
await logStep('fill_username', () => actWithMouseSim(stagehand, `Type "${creds.username}" into the username or email input field`));
|
|
135
|
-
|
|
139
|
+
// Don't send password to LLM — use direct Playwright fill to avoid
|
|
140
|
+
// leaking credentials into the Stagehand/LLM provider's request logs.
|
|
141
|
+
await logStep('fill_password', async () => {
|
|
142
|
+
const observations = await stagehand.observe('Find the password input field');
|
|
143
|
+
const selector = observations[0]?.selector || 'input[type="password"]';
|
|
144
|
+
await pg.locator(selector).fill(creds.password);
|
|
145
|
+
});
|
|
136
146
|
await logStep('click_submit', () => actWithMouseSim(stagehand, 'Click the sign in, log in, login, or continue button to submit credentials'));
|
|
137
147
|
await pg.waitForLoadState('networkidle').catch(() => pg.waitForTimeout(2000));
|
|
138
148
|
|
package/src/crawler.ts
CHANGED
|
@@ -1303,6 +1303,7 @@ export async function crawlProject(opts: {
|
|
|
1303
1303
|
}
|
|
1304
1304
|
let checkpointSuccessCount = 0;
|
|
1305
1305
|
let maxSeenTotal = 0; // track high-water mark so progress % never goes backwards
|
|
1306
|
+
let lastEmittedPct = 20; // clamp so progress never decreases even when queue grows
|
|
1306
1307
|
const startUrl = page.url();
|
|
1307
1308
|
// Resume from previous segment's pending queue if provided
|
|
1308
1309
|
const resumeUrls = opts.resumeUrls ?? [];
|
|
@@ -1365,8 +1366,9 @@ export async function crawlProject(opts: {
|
|
|
1365
1366
|
const u = s.url as string | null;
|
|
1366
1367
|
if (!u) return false;
|
|
1367
1368
|
try {
|
|
1368
|
-
const {
|
|
1369
|
-
|
|
1369
|
+
const { pathname } = new URL(u);
|
|
1370
|
+
const stripWww = (h: string) => h.replace(/^www\./i, '').toLowerCase();
|
|
1371
|
+
if (stripWww(new URL(u).hostname) !== stripWww(new URL(startUrl).hostname)) return false;
|
|
1370
1372
|
// Don't re-seed generated dynamic routes — they produce duplicate templates.
|
|
1371
1373
|
if (isGeneratedDynamicPath(pathname)) return false;
|
|
1372
1374
|
return true;
|
|
@@ -2642,10 +2644,13 @@ export async function crawlProject(opts: {
|
|
|
2642
2644
|
|
|
2643
2645
|
discovered.push({ url, screenId: screen.id, elements: elements.length, elementFacts: elements, screenshotPath: shotPath, version: screen.version });
|
|
2644
2646
|
// Progress: 20% baseline + up to 60% across discovered screens.
|
|
2645
|
-
//
|
|
2647
|
+
// maxSeenTotal is the denominator high-water mark; lastEmittedPct prevents
|
|
2648
|
+
// the displayed % from going backwards when new link discoveries grow the queue.
|
|
2646
2649
|
maxSeenTotal = Math.max(maxSeenTotal, visited.size + queue.length);
|
|
2647
|
-
const
|
|
2648
|
-
|
|
2650
|
+
const rawPct = 20 + Math.round(Math.min(60, (visited.size / Math.max(1, maxSeenTotal)) * 60));
|
|
2651
|
+
const clampedPct = Math.max(lastEmittedPct, rawPct);
|
|
2652
|
+
lastEmittedPct = clampedPct;
|
|
2653
|
+
await onProgress?.(clampedPct, `Crawling screens (${visited.size} mapped, ${queue.length} pending)`);
|
|
2649
2654
|
logger.info({ name, elements: elements.length }, 'Screen crawled');
|
|
2650
2655
|
// Checkpoint every 10 successes so a failed job can be resumed from close to where it stopped
|
|
2651
2656
|
if (++checkpointSuccessCount % 10 === 0) {
|
package/src/job-queue.ts
CHANGED
|
@@ -393,7 +393,7 @@ export async function enqueueJob(type: string, config: any): Promise<string> {
|
|
|
393
393
|
/** Get the current state of a platform job. */
|
|
394
394
|
export async function getJob(tenantId: string, jobId: string) {
|
|
395
395
|
return withTenant(tenantId, (tx) =>
|
|
396
|
-
tx.platformJob.
|
|
396
|
+
tx.platformJob.findFirst({ where: { id: jobId, tenantId } })
|
|
397
397
|
);
|
|
398
398
|
}
|
|
399
399
|
|
|
@@ -1520,6 +1520,18 @@ export async function processDataRetentionPurge(): Promise<void> {
|
|
|
1520
1520
|
}
|
|
1521
1521
|
|
|
1522
1522
|
logger.info({ deleted: auditTotal, tenants: tenants.length }, '[data-retention] AuditLog purged');
|
|
1523
|
+
|
|
1524
|
+
// AuthSession: fixed 12-month retention — IP addresses are PII under GDPR.
|
|
1525
|
+
// Only purge revoked sessions; active (non-revoked) sessions are kept.
|
|
1526
|
+
const SESSION_RETENTION_DAYS = 365;
|
|
1527
|
+
const sessionCutoff = new Date(Date.now() - SESSION_RETENTION_DAYS * 24 * 60 * 60_000);
|
|
1528
|
+
const sessionResult = await prismaAdmin.authSession.deleteMany({
|
|
1529
|
+
where: {
|
|
1530
|
+
createdAt: { lt: sessionCutoff },
|
|
1531
|
+
revokedAt: { not: null },
|
|
1532
|
+
},
|
|
1533
|
+
});
|
|
1534
|
+
logger.info({ deleted: sessionResult.count }, '[data-retention] AuthSession purged');
|
|
1523
1535
|
}
|
|
1524
1536
|
|
|
1525
1537
|
// ── URL Discovery Job ─────────────────────────────────────────────────────────
|
package/src/live-view.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { chromium, type Browser, type BrowserContext, type Page } from 'playwright';
|
|
2
|
+
import { randomBytes } from 'node:crypto';
|
|
2
3
|
import { logger } from './logger.js';
|
|
3
4
|
|
|
4
5
|
interface LiveViewSession {
|
|
@@ -17,7 +18,7 @@ const sessions = new Map<string, LiveViewSession>();
|
|
|
17
18
|
const SESSION_TTL_MS = 15 * 60 * 1_000; // 15 min
|
|
18
19
|
|
|
19
20
|
function generateId(): string {
|
|
20
|
-
return
|
|
21
|
+
return randomBytes(16).toString('hex');
|
|
21
22
|
}
|
|
22
23
|
|
|
23
24
|
/**
|
package/src/session-capture.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { chromium } from 'playwright';
|
|
2
|
+
import { randomBytes } from 'node:crypto';
|
|
2
3
|
import { login } from './auth.js';
|
|
3
4
|
import { generateTOTP } from './job-queue.js';
|
|
4
5
|
|
|
@@ -25,7 +26,7 @@ export async function startSessionCapture(appUrl: string): Promise<string> {
|
|
|
25
26
|
throw new Error('HEADLESS_ENV');
|
|
26
27
|
}
|
|
27
28
|
|
|
28
|
-
const id =
|
|
29
|
+
const id = randomBytes(16).toString('hex');
|
|
29
30
|
const browser = await chromium.launch({ headless: false });
|
|
30
31
|
const context = await browser.newContext();
|
|
31
32
|
const page = await context.newPage();
|
package/src/skyvern.ts
CHANGED
|
@@ -67,6 +67,9 @@ export async function skyvernNavigate(
|
|
|
67
67
|
return { screenshotUrls: [], visitedUrls: [], status: 'failed' };
|
|
68
68
|
}
|
|
69
69
|
|
|
70
|
+
const { assertResolvesToPublicAddress } = await import('@detiq/core');
|
|
71
|
+
await assertResolvesToPublicAddress(apiUrl);
|
|
72
|
+
|
|
70
73
|
const visitedUrls: string[] = [];
|
|
71
74
|
const screenshotUrls: string[] = [];
|
|
72
75
|
const skyvernSteps: SkyvernStep[] = [];
|
|
@@ -161,6 +164,9 @@ export async function skyvernFallback(task: SkyvernTask, tenantConfig?: SkyvernC
|
|
|
161
164
|
return 'failed';
|
|
162
165
|
}
|
|
163
166
|
|
|
167
|
+
const { assertResolvesToPublicAddress } = await import('@detiq/core');
|
|
168
|
+
await assertResolvesToPublicAddress(apiUrl);
|
|
169
|
+
|
|
164
170
|
try {
|
|
165
171
|
const res = await fetch(`${apiUrl}/api/v1/tasks`, {
|
|
166
172
|
method: 'POST',
|
package/src/test-executor.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { chromium, firefox, webkit, type Page, type Locator } from 'playwright';
|
|
2
2
|
import type { ExecutionStep } from '@detiq/agents';
|
|
3
|
+
import { assertCrawlUrlSafe } from '@detiq/core';
|
|
3
4
|
import { analyzeFailure } from './failure-analysis-agent.js';
|
|
4
5
|
|
|
5
6
|
export interface StepResult {
|
|
@@ -160,7 +161,9 @@ async function runPlan(
|
|
|
160
161
|
|
|
161
162
|
switch (step.action) {
|
|
162
163
|
case 'navigate':
|
|
163
|
-
|
|
164
|
+
if (!step.url) throw new Error('Navigate step missing url');
|
|
165
|
+
assertCrawlUrlSafe(step.url);
|
|
166
|
+
await page.goto(step.url, { waitUntil: 'domcontentloaded', timeout });
|
|
164
167
|
break;
|
|
165
168
|
|
|
166
169
|
case 'click': {
|
|
@@ -373,7 +376,13 @@ export async function executeTestPlan(
|
|
|
373
376
|
}
|
|
374
377
|
});
|
|
375
378
|
|
|
376
|
-
|
|
379
|
+
const TEST_EXECUTION_TIMEOUT_MS = 5 * 60_000;
|
|
380
|
+
lastResult = await Promise.race([
|
|
381
|
+
runPlan(page, plan, options, consoleLogs, networkErrors),
|
|
382
|
+
new Promise<never>((_, reject) =>
|
|
383
|
+
setTimeout(() => reject(new Error('Test execution timed out')), TEST_EXECUTION_TIMEOUT_MS)
|
|
384
|
+
),
|
|
385
|
+
]);
|
|
377
386
|
lastNetworkTimings = networkTimings;
|
|
378
387
|
await browser.close();
|
|
379
388
|
|