@detiq/crawler 0.1.4 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ai-login.js +5 -1
- package/dist/ai-login.js.map +2 -2
- package/dist/crawler.js +33 -18
- package/dist/crawler.js.map +2 -2
- package/dist/test-executor.js +2 -2
- package/dist/test-executor.js.map +2 -2
- package/package.json +1 -1
- package/src/crawler.ts +29 -17
package/dist/test-executor.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { chromium, firefox, webkit } from "playwright";
|
|
2
|
-
import {
|
|
2
|
+
import { assertCrawlUrlSafe } from "@detiq/core";
|
|
3
3
|
import { analyzeFailure } from "./failure-analysis-agent.js";
|
|
4
4
|
const LOCATOR_STRATEGIES = ["text", "label", "role", "css"];
|
|
5
5
|
function buildLocator(page, selector, strategy = "text") {
|
|
@@ -95,7 +95,7 @@ async function runPlan(page, plan, options, consoleLogs, networkErrors) {
|
|
|
95
95
|
switch (step.action) {
|
|
96
96
|
case "navigate":
|
|
97
97
|
if (!step.url) throw new Error("Navigate step missing url");
|
|
98
|
-
|
|
98
|
+
assertCrawlUrlSafe(step.url);
|
|
99
99
|
await page.goto(step.url, { waitUntil: "domcontentloaded", timeout });
|
|
100
100
|
break;
|
|
101
101
|
case "click": {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../src/test-executor.ts"],
|
|
4
|
-
"sourcesContent": ["import { chromium, firefox, webkit, type Page, type Locator } from 'playwright';\nimport type { ExecutionStep } from '@detiq/agents';\nimport { assertResolvesToPublicAddress } from '@detiq/core';\nimport { analyzeFailure } from './failure-analysis-agent.js';\n\nexport interface StepResult {\n stepNumber: number;\n description: string;\n status: 'PASSED' | 'FAILED' | 'SKIPPED';\n duration: number;\n error?: string;\n screenshotPath?: string;\n healed?: boolean;\n /** 'mechanical' = same selector text, different Playwright locator strategy.\n * 'ai' = AI Self-Healing Agent matched a semantically different element\n * (the app's selector text itself changed, e.g. \"Sign In\" -> \"Log In\"). */\n healedVia?: 'mechanical' | 'ai';\n healReasoning?: string;\n healConfidence?: number;\n}\n\nexport interface ExecutionResult {\n stepResults: StepResult[];\n status: 'PASSED' | 'FAILED' | 'ERROR' | 'FLAKY';\n errorStep?: number;\n totalDuration: number;\n finalScreenshotPath?: string;\n retryCount?: number;\n consoleLogs?: Array<{ type: string; text: string; timestamp: number }>;\n networkErrors?: Array<{ url: string; status: number; method: string; timestamp: number }>;\n rootCauseSummary?: string;\n networkTimings?: Array<{ url: string; method: string; status: number; duration: number }>;\n}\n\nconst LOCATOR_STRATEGIES: ExecutionStep['selectorStrategy'][] = ['text', 'label', 'role', 'css'];\n\nfunction buildLocator(page: Page, selector: string, strategy: ExecutionStep['selectorStrategy'] = 'text'): Locator {\n switch (strategy) {\n case 'css': return page.locator(selector);\n case 'text': return page.getByText(selector, { exact: false });\n case 'role': return page.getByRole(selector as any);\n case 'label': return page.getByLabel(selector);\n default: return page.locator(selector);\n }\n}\n\n/** Scans the live page for visible interactive elements \u2014 the \"current page elements\"\n * fed to the AI Self-Healing Agent when mechanical locator strategies fail. */\nasync function extractCurrentPageElements(page: Page): Promise<Array<{ meaning: string; role: string; ariaLabel?: string; textContent?: string }>> {\n return page.evaluate(() => {\n const els = Array.from(document.querySelectorAll<HTMLElement>(\n 'a, button, input, select, textarea, [role=\"button\"], [role=\"link\"], [role=\"tab\"], [role=\"menuitem\"], [aria-label]'\n ));\n return els\n .filter((el) => {\n const rect = el.getBoundingClientRect();\n return rect.width > 0 && rect.height > 0;\n })\n .slice(0, 200)\n .map((el) => {\n const ariaLabel = el.getAttribute('aria-label') ?? undefined;\n const textContent = (el.textContent ?? '').replace(/\\s+/g, ' ').trim().slice(0, 100) || undefined;\n const role = el.getAttribute('role') ?? el.tagName.toLowerCase();\n return { meaning: ariaLabel || textContent || role, role, ariaLabel, textContent };\n });\n });\n}\n\n/**\n * Resolves a step's target element in three tiers:\n * 1. Exact selector + strategy as generated.\n * 2. Mechanical: same selector text, other Playwright locator strategies\n * (handles a strategy mismatch, not an actual UI change).\n * 3. AI: when tenantId/projectId are supplied, asks the Self-Healing Agent to find\n * the current page's best semantic match for the failed step description \u2014 the\n * tier that actually survives the app's selector text changing.\n */\nasync function resolveLocator(\n page: Page,\n step: ExecutionStep,\n healCtx?: { tenantId: string; projectId: string },\n): Promise<{ loc: Locator; healed: boolean; healedVia?: 'mechanical' | 'ai'; healReasoning?: string; healConfidence?: number } | null> {\n const { selector, selectorStrategy = 'text' } = step;\n if (!selector) return null;\n\n const primary = buildLocator(page, selector, selectorStrategy);\n const attached = await primary.first().isVisible({ timeout: 1500 }).catch(() => false);\n if (attached) return { loc: primary, healed: false };\n\n // Tier 2 \u2014 mechanical: try other strategies before giving up\n for (const strategy of LOCATOR_STRATEGIES) {\n if (strategy === selectorStrategy) continue;\n const loc = buildLocator(page, selector, strategy);\n const visible = await loc.first().isVisible({ timeout: 800 }).catch(() => false);\n if (visible) return { loc, healed: true, healedVia: 'mechanical' };\n }\n\n // Tier 3 \u2014 AI: the selector text itself may no longer match anything on the page.\n if (healCtx) {\n try {\n const { healElement } = await import('@detiq/agents');\n const currentPageElements = await extractCurrentPageElements(page);\n const result = await healElement({\n tenantId: healCtx.tenantId,\n projectId: healCtx.projectId,\n failedElementMeaning: step.description || selector,\n currentPageElements,\n });\n if (result.healed && result.updatedTest) {\n const healedLoc = page.getByText(result.updatedTest.elementMeaning, { exact: false });\n const visible = await healedLoc.first().isVisible({ timeout: 1500 }).catch(() => false);\n if (visible) {\n return {\n loc: healedLoc,\n healed: true,\n healedVia: 'ai',\n healReasoning: result.reasoning,\n healConfidence: result.confidence,\n };\n }\n }\n } catch {\n // AI healing is a best-effort last resort \u2014 any failure here (LLM error, bad\n // JSON, etc.) just falls through to the caller's normal Playwright error below.\n }\n }\n\n // Return primary \u2014 caller will fail with a meaningful Playwright error\n return { loc: primary, healed: false };\n}\n\nfunction applyHealInfo(result: StepResult, r: { healed: boolean; healedVia?: 'mechanical' | 'ai'; healReasoning?: string; healConfidence?: number }): void {\n result.healed = r.healed;\n result.healedVia = r.healedVia;\n result.healReasoning = r.healReasoning;\n result.healConfidence = r.healConfidence;\n}\n\nasync function runPlan(\n page: Page,\n plan: ExecutionStep[],\n options: { screenshotDir: string; testRunId: string; onStepComplete?: (result: StepResult) => void; healCtx?: { tenantId: string; projectId: string } },\n consoleLogs: Array<{ type: string; text: string; timestamp: number }>,\n networkErrors: Array<{ url: string; status: number; method: string; timestamp: number }>,\n): Promise<{ stepResults: StepResult[]; errorStep?: number; finalScreenshotPath?: string; consoleLogs: typeof consoleLogs; networkErrors: typeof networkErrors }> {\n const stepResults: StepResult[] = [];\n let errorStep: number | undefined;\n let finalScreenshotPath: string | undefined;\n\n for (const step of plan) {\n const stepStart = Date.now();\n const result: StepResult = {\n stepNumber: step.stepNumber,\n description: step.description,\n status: 'FAILED',\n duration: 0,\n };\n\n try {\n const timeout = step.timeout ?? 8000;\n\n switch (step.action) {\n case 'navigate':\n if (!step.url) throw new Error('Navigate step missing url');\n await assertResolvesToPublicAddress(step.url);\n await page.goto(step.url, { waitUntil: 'domcontentloaded', timeout });\n break;\n\n case 'click': {\n const r = await resolveLocator(page, step, options.healCtx);\n if (!r) throw new Error('Could not resolve locator');\n await r.loc.first().click({ timeout });\n applyHealInfo(result, r);\n break;\n }\n\n case 'fill': {\n const r = await resolveLocator(page, step, options.healCtx);\n if (!r) throw new Error('Could not resolve locator');\n await r.loc.first().fill(step.value ?? '', { timeout });\n applyHealInfo(result, r);\n break;\n }\n\n case 'select': {\n const r = await resolveLocator(page, step, options.healCtx);\n if (!r) throw new Error('Could not resolve locator');\n await r.loc.first().selectOption(step.value ?? '', { timeout });\n applyHealInfo(result, r);\n break;\n }\n\n case 'hover': {\n const r = await resolveLocator(page, step, options.healCtx);\n if (!r) throw new Error('Could not resolve locator');\n await r.loc.first().hover({ timeout });\n applyHealInfo(result, r);\n break;\n }\n\n case 'press_key':\n await page.keyboard.press(step.value ?? 'Enter');\n break;\n\n case 'assert_visible': {\n const r = await resolveLocator(page, step, options.healCtx);\n if (!r) throw new Error('Could not resolve locator');\n await r.loc.first().waitFor({ state: 'visible', timeout });\n applyHealInfo(result, r);\n break;\n }\n\n case 'assert_text': {\n const r = await resolveLocator(page, step, options.healCtx);\n if (!r) throw new Error('Could not resolve locator');\n const text = await r.loc.first().textContent({ timeout });\n if (!text?.includes(step.value ?? '')) {\n throw new Error(`Expected text \"${step.value}\" not found. Got: \"${text?.slice(0, 100)}\"`);\n }\n applyHealInfo(result, r);\n break;\n }\n\n case 'assert_url': {\n const url = page.url();\n if (!url.includes(step.value ?? '')) {\n throw new Error(`Expected URL to contain \"${step.value}\". Got: \"${url}\"`);\n }\n break;\n }\n\n case 'wait':\n await page.waitForTimeout(step.timeout ?? 2000);\n break;\n\n case 'screenshot': {\n const ssPath = `${options.screenshotDir}/${options.testRunId}-step-${step.stepNumber}.png`;\n await page.screenshot({ path: ssPath, fullPage: true });\n result.screenshotPath = ssPath;\n break;\n }\n }\n\n result.status = 'PASSED';\n } catch (err: any) {\n result.status = step.optional ? 'SKIPPED' : 'FAILED';\n result.error = err.message;\n if (!step.optional) {\n errorStep = step.stepNumber;\n }\n }\n\n try {\n await page.waitForLoadState('networkidle', { timeout: 1500 }).catch(() => {});\n if (!result.screenshotPath) {\n const ssPath = `${options.screenshotDir}/${options.testRunId}-step-${step.stepNumber}.png`;\n await page.screenshot({ path: ssPath, fullPage: true });\n result.screenshotPath = ssPath;\n }\n } catch { /* Screenshot evidence is best-effort and should not fail the run. */ }\n\n result.duration = Date.now() - stepStart;\n stepResults.push(result);\n options.onStepComplete?.(result);\n\n if (result.status === 'FAILED') break;\n }\n\n try {\n const finalPath = `${options.screenshotDir}/${options.testRunId}-final.png`;\n await page.screenshot({ path: finalPath, fullPage: true });\n finalScreenshotPath = finalPath;\n } catch { /* non-fatal */ }\n\n return { stepResults, errorStep, finalScreenshotPath, consoleLogs, networkErrors };\n}\n\nfunction interpolate(template: string | undefined, data: Record<string, string>): string | undefined {\n if (!template) return template;\n return template.replace(/\\{\\{(\\w+)\\}\\}/g, (_, key) => data[key] ?? `{{${key}}}`);\n}\n\nexport async function executeTestPlan(\n plan: ExecutionStep[],\n options: {\n screenshotDir: string;\n testRunId: string;\n viewport?: { width: number; height: number };\n onStepComplete?: (result: StepResult) => void;\n /** Run the test up to this many times to detect flakiness (default 1). */\n maxRuns?: number;\n browser?: 'chromium' | 'firefox' | 'webkit';\n /** Data row for parameterized execution: substitutes {{varname}} in step fields. */\n dataRow?: Record<string, string>;\n /** Enables the AI Self-Healing Agent tier when a step's selector can't be resolved\n * even after trying alternate locator strategies. Omit to keep mechanical-only\n * healing (no LLM calls during execution). */\n healCtx?: { tenantId: string; projectId: string };\n /** Decrypted Playwright storageState JSON \u2014 restores a logged-in session so\n * steps don't have to re-authenticate. Without this, any test case behind a\n * login wall fails at the first authenticated step. */\n storageState?: string;\n },\n): Promise<ExecutionResult> {\n if (options.dataRow && Object.keys(options.dataRow).length > 0) {\n const data = options.dataRow;\n plan = plan.map((step) => ({\n ...step,\n url: interpolate(step.url, data),\n selector: interpolate(step.selector, data),\n value: interpolate(step.value, data),\n description: interpolate(step.description, data) ?? step.description,\n }));\n }\n const maxRuns = options.maxRuns ?? 1;\n const browserEngine = { chromium, firefox, webkit }[options.browser ?? 'chromium'] ?? chromium;\n const start = Date.now();\n\n const outcomes: ('PASSED' | 'FAILED')[] = [];\n let lastResult: Awaited<ReturnType<typeof runPlan>> | null = null;\n let lastNetworkTimings: Array<{ url: string; method: string; status: number; duration: number }> = [];\n\n for (let run = 0; run < maxRuns; run++) {\n const browser = await browserEngine.launch({\n headless: true,\n // Optional override for environments with a non-standard/pre-provisioned\n // browser install (e.g. a custom base image with only system Chromium).\n ...(process.env.PLAYWRIGHT_EXECUTABLE_PATH ? { executablePath: process.env.PLAYWRIGHT_EXECUTABLE_PATH } : {}),\n });\n let parsedStorageState: any;\n try {\n parsedStorageState = options.storageState ? JSON.parse(options.storageState) : undefined;\n } catch {\n parsedStorageState = undefined;\n }\n const context = await browser.newContext({\n viewport: options.viewport ?? { width: 1440, height: 900 },\n ...(parsedStorageState ? { storageState: parsedStorageState } : {}),\n });\n const page = await context.newPage();\n\n await page\n .addStyleTag({ content: '*, *::before, *::after { animation-duration: 0s !important; transition-duration: 0s !important; }' })\n .catch(() => {});\n\n const consoleLogs: Array<{ type: string; text: string; timestamp: number }> = [];\n const networkErrors: Array<{ url: string; status: number; method: string; timestamp: number }> = [];\n const networkTimings: Array<{ url: string; method: string; status: number; duration: number }> = [];\n const requestStartTimes = new Map<string, number>();\n\n page.on('console', msg => {\n if (consoleLogs.length < 100) {\n consoleLogs.push({ type: msg.type(), text: msg.text().slice(0, 500), timestamp: Date.now() });\n }\n });\n page.on('request', request => {\n const url = request.url();\n if (url.includes('/api/') || url.match(/\\/(graphql|rest|v\\d)\\//)) {\n requestStartTimes.set(request.url(), Date.now());\n }\n });\n page.on('response', response => {\n const url = response.url();\n const start = requestStartTimes.get(url);\n if (start && networkTimings.length < 200) {\n networkTimings.push({ url: url.slice(0, 300), method: response.request().method(), status: response.status(), duration: Date.now() - start });\n requestStartTimes.delete(url);\n }\n if (!response.ok() && !url.includes('favicon') && networkErrors.length < 50) {\n networkErrors.push({\n url: url.slice(0, 300),\n status: response.status(),\n method: response.request().method(),\n timestamp: Date.now(),\n });\n }\n });\n\n const TEST_EXECUTION_TIMEOUT_MS = 5 * 60_000;\n lastResult = await Promise.race([\n runPlan(page, plan, options, consoleLogs, networkErrors),\n new Promise<never>((_, reject) =>\n setTimeout(() => reject(new Error('Test execution timed out')), TEST_EXECUTION_TIMEOUT_MS)\n ),\n ]);\n lastNetworkTimings = networkTimings;\n await browser.close();\n\n const passed = !lastResult.stepResults.some((r) => r.status === 'FAILED');\n outcomes.push(passed ? 'PASSED' : 'FAILED');\n\n // If all subsequent runs are consistent with first, no need to keep going\n if (run > 0 && outcomes.every((o) => o === outcomes[0])) break;\n }\n\n const passCount = outcomes.filter((o) => o === 'PASSED').length;\n const failCount = outcomes.filter((o) => o === 'FAILED').length;\n\n let status: ExecutionResult['status'];\n if (maxRuns > 1 && passCount > 0 && failCount > 0) {\n status = 'FLAKY';\n } else if (passCount === outcomes.length) {\n status = 'PASSED';\n } else {\n status = 'FAILED';\n }\n\n let rootCauseSummary: string | undefined;\n if (status === 'FAILED' || status === 'FLAKY') {\n const failedStep = lastResult!.stepResults.find(s => s.status === 'FAILED');\n rootCauseSummary = await analyzeFailure({\n tenantId: options.healCtx?.tenantId,\n error: failedStep?.error ?? 'Unknown error',\n stepDescription: failedStep?.description,\n consoleLogs: lastResult!.consoleLogs,\n networkErrors: lastResult!.networkErrors,\n screenshotPath: lastResult!.finalScreenshotPath,\n }).catch(() => undefined) ?? undefined;\n }\n\n return {\n stepResults: lastResult!.stepResults,\n status,\n errorStep: lastResult!.errorStep,\n totalDuration: Date.now() - start,\n finalScreenshotPath: lastResult!.finalScreenshotPath,\n retryCount: outcomes.length - 1,\n consoleLogs: lastResult!.consoleLogs,\n networkErrors: lastResult!.networkErrors,\n rootCauseSummary,\n networkTimings: lastNetworkTimings,\n };\n}\n"],
|
|
5
|
-
"mappings": "AAAA,SAAS,UAAU,SAAS,cAAuC;AAEnE,SAAS,
|
|
4
|
+
"sourcesContent": ["import { chromium, firefox, webkit, type Page, type Locator } from 'playwright';\nimport type { ExecutionStep } from '@detiq/agents';\nimport { assertCrawlUrlSafe } from '@detiq/core';\nimport { analyzeFailure } from './failure-analysis-agent.js';\n\nexport interface StepResult {\n stepNumber: number;\n description: string;\n status: 'PASSED' | 'FAILED' | 'SKIPPED';\n duration: number;\n error?: string;\n screenshotPath?: string;\n healed?: boolean;\n /** 'mechanical' = same selector text, different Playwright locator strategy.\n * 'ai' = AI Self-Healing Agent matched a semantically different element\n * (the app's selector text itself changed, e.g. \"Sign In\" -> \"Log In\"). */\n healedVia?: 'mechanical' | 'ai';\n healReasoning?: string;\n healConfidence?: number;\n}\n\nexport interface ExecutionResult {\n stepResults: StepResult[];\n status: 'PASSED' | 'FAILED' | 'ERROR' | 'FLAKY';\n errorStep?: number;\n totalDuration: number;\n finalScreenshotPath?: string;\n retryCount?: number;\n consoleLogs?: Array<{ type: string; text: string; timestamp: number }>;\n networkErrors?: Array<{ url: string; status: number; method: string; timestamp: number }>;\n rootCauseSummary?: string;\n networkTimings?: Array<{ url: string; method: string; status: number; duration: number }>;\n}\n\nconst LOCATOR_STRATEGIES: ExecutionStep['selectorStrategy'][] = ['text', 'label', 'role', 'css'];\n\nfunction buildLocator(page: Page, selector: string, strategy: ExecutionStep['selectorStrategy'] = 'text'): Locator {\n switch (strategy) {\n case 'css': return page.locator(selector);\n case 'text': return page.getByText(selector, { exact: false });\n case 'role': return page.getByRole(selector as any);\n case 'label': return page.getByLabel(selector);\n default: return page.locator(selector);\n }\n}\n\n/** Scans the live page for visible interactive elements \u2014 the \"current page elements\"\n * fed to the AI Self-Healing Agent when mechanical locator strategies fail. */\nasync function extractCurrentPageElements(page: Page): Promise<Array<{ meaning: string; role: string; ariaLabel?: string; textContent?: string }>> {\n return page.evaluate(() => {\n const els = Array.from(document.querySelectorAll<HTMLElement>(\n 'a, button, input, select, textarea, [role=\"button\"], [role=\"link\"], [role=\"tab\"], [role=\"menuitem\"], [aria-label]'\n ));\n return els\n .filter((el) => {\n const rect = el.getBoundingClientRect();\n return rect.width > 0 && rect.height > 0;\n })\n .slice(0, 200)\n .map((el) => {\n const ariaLabel = el.getAttribute('aria-label') ?? undefined;\n const textContent = (el.textContent ?? '').replace(/\\s+/g, ' ').trim().slice(0, 100) || undefined;\n const role = el.getAttribute('role') ?? el.tagName.toLowerCase();\n return { meaning: ariaLabel || textContent || role, role, ariaLabel, textContent };\n });\n });\n}\n\n/**\n * Resolves a step's target element in three tiers:\n * 1. Exact selector + strategy as generated.\n * 2. Mechanical: same selector text, other Playwright locator strategies\n * (handles a strategy mismatch, not an actual UI change).\n * 3. AI: when tenantId/projectId are supplied, asks the Self-Healing Agent to find\n * the current page's best semantic match for the failed step description \u2014 the\n * tier that actually survives the app's selector text changing.\n */\nasync function resolveLocator(\n page: Page,\n step: ExecutionStep,\n healCtx?: { tenantId: string; projectId: string },\n): Promise<{ loc: Locator; healed: boolean; healedVia?: 'mechanical' | 'ai'; healReasoning?: string; healConfidence?: number } | null> {\n const { selector, selectorStrategy = 'text' } = step;\n if (!selector) return null;\n\n const primary = buildLocator(page, selector, selectorStrategy);\n const attached = await primary.first().isVisible({ timeout: 1500 }).catch(() => false);\n if (attached) return { loc: primary, healed: false };\n\n // Tier 2 \u2014 mechanical: try other strategies before giving up\n for (const strategy of LOCATOR_STRATEGIES) {\n if (strategy === selectorStrategy) continue;\n const loc = buildLocator(page, selector, strategy);\n const visible = await loc.first().isVisible({ timeout: 800 }).catch(() => false);\n if (visible) return { loc, healed: true, healedVia: 'mechanical' };\n }\n\n // Tier 3 \u2014 AI: the selector text itself may no longer match anything on the page.\n if (healCtx) {\n try {\n const { healElement } = await import('@detiq/agents');\n const currentPageElements = await extractCurrentPageElements(page);\n const result = await healElement({\n tenantId: healCtx.tenantId,\n projectId: healCtx.projectId,\n failedElementMeaning: step.description || selector,\n currentPageElements,\n });\n if (result.healed && result.updatedTest) {\n const healedLoc = page.getByText(result.updatedTest.elementMeaning, { exact: false });\n const visible = await healedLoc.first().isVisible({ timeout: 1500 }).catch(() => false);\n if (visible) {\n return {\n loc: healedLoc,\n healed: true,\n healedVia: 'ai',\n healReasoning: result.reasoning,\n healConfidence: result.confidence,\n };\n }\n }\n } catch {\n // AI healing is a best-effort last resort \u2014 any failure here (LLM error, bad\n // JSON, etc.) just falls through to the caller's normal Playwright error below.\n }\n }\n\n // Return primary \u2014 caller will fail with a meaningful Playwright error\n return { loc: primary, healed: false };\n}\n\nfunction applyHealInfo(result: StepResult, r: { healed: boolean; healedVia?: 'mechanical' | 'ai'; healReasoning?: string; healConfidence?: number }): void {\n result.healed = r.healed;\n result.healedVia = r.healedVia;\n result.healReasoning = r.healReasoning;\n result.healConfidence = r.healConfidence;\n}\n\nasync function runPlan(\n page: Page,\n plan: ExecutionStep[],\n options: { screenshotDir: string; testRunId: string; onStepComplete?: (result: StepResult) => void; healCtx?: { tenantId: string; projectId: string } },\n consoleLogs: Array<{ type: string; text: string; timestamp: number }>,\n networkErrors: Array<{ url: string; status: number; method: string; timestamp: number }>,\n): Promise<{ stepResults: StepResult[]; errorStep?: number; finalScreenshotPath?: string; consoleLogs: typeof consoleLogs; networkErrors: typeof networkErrors }> {\n const stepResults: StepResult[] = [];\n let errorStep: number | undefined;\n let finalScreenshotPath: string | undefined;\n\n for (const step of plan) {\n const stepStart = Date.now();\n const result: StepResult = {\n stepNumber: step.stepNumber,\n description: step.description,\n status: 'FAILED',\n duration: 0,\n };\n\n try {\n const timeout = step.timeout ?? 8000;\n\n switch (step.action) {\n case 'navigate':\n if (!step.url) throw new Error('Navigate step missing url');\n assertCrawlUrlSafe(step.url);\n await page.goto(step.url, { waitUntil: 'domcontentloaded', timeout });\n break;\n\n case 'click': {\n const r = await resolveLocator(page, step, options.healCtx);\n if (!r) throw new Error('Could not resolve locator');\n await r.loc.first().click({ timeout });\n applyHealInfo(result, r);\n break;\n }\n\n case 'fill': {\n const r = await resolveLocator(page, step, options.healCtx);\n if (!r) throw new Error('Could not resolve locator');\n await r.loc.first().fill(step.value ?? '', { timeout });\n applyHealInfo(result, r);\n break;\n }\n\n case 'select': {\n const r = await resolveLocator(page, step, options.healCtx);\n if (!r) throw new Error('Could not resolve locator');\n await r.loc.first().selectOption(step.value ?? '', { timeout });\n applyHealInfo(result, r);\n break;\n }\n\n case 'hover': {\n const r = await resolveLocator(page, step, options.healCtx);\n if (!r) throw new Error('Could not resolve locator');\n await r.loc.first().hover({ timeout });\n applyHealInfo(result, r);\n break;\n }\n\n case 'press_key':\n await page.keyboard.press(step.value ?? 'Enter');\n break;\n\n case 'assert_visible': {\n const r = await resolveLocator(page, step, options.healCtx);\n if (!r) throw new Error('Could not resolve locator');\n await r.loc.first().waitFor({ state: 'visible', timeout });\n applyHealInfo(result, r);\n break;\n }\n\n case 'assert_text': {\n const r = await resolveLocator(page, step, options.healCtx);\n if (!r) throw new Error('Could not resolve locator');\n const text = await r.loc.first().textContent({ timeout });\n if (!text?.includes(step.value ?? '')) {\n throw new Error(`Expected text \"${step.value}\" not found. Got: \"${text?.slice(0, 100)}\"`);\n }\n applyHealInfo(result, r);\n break;\n }\n\n case 'assert_url': {\n const url = page.url();\n if (!url.includes(step.value ?? '')) {\n throw new Error(`Expected URL to contain \"${step.value}\". Got: \"${url}\"`);\n }\n break;\n }\n\n case 'wait':\n await page.waitForTimeout(step.timeout ?? 2000);\n break;\n\n case 'screenshot': {\n const ssPath = `${options.screenshotDir}/${options.testRunId}-step-${step.stepNumber}.png`;\n await page.screenshot({ path: ssPath, fullPage: true });\n result.screenshotPath = ssPath;\n break;\n }\n }\n\n result.status = 'PASSED';\n } catch (err: any) {\n result.status = step.optional ? 'SKIPPED' : 'FAILED';\n result.error = err.message;\n if (!step.optional) {\n errorStep = step.stepNumber;\n }\n }\n\n try {\n await page.waitForLoadState('networkidle', { timeout: 1500 }).catch(() => {});\n if (!result.screenshotPath) {\n const ssPath = `${options.screenshotDir}/${options.testRunId}-step-${step.stepNumber}.png`;\n await page.screenshot({ path: ssPath, fullPage: true });\n result.screenshotPath = ssPath;\n }\n } catch { /* Screenshot evidence is best-effort and should not fail the run. */ }\n\n result.duration = Date.now() - stepStart;\n stepResults.push(result);\n options.onStepComplete?.(result);\n\n if (result.status === 'FAILED') break;\n }\n\n try {\n const finalPath = `${options.screenshotDir}/${options.testRunId}-final.png`;\n await page.screenshot({ path: finalPath, fullPage: true });\n finalScreenshotPath = finalPath;\n } catch { /* non-fatal */ }\n\n return { stepResults, errorStep, finalScreenshotPath, consoleLogs, networkErrors };\n}\n\nfunction interpolate(template: string | undefined, data: Record<string, string>): string | undefined {\n if (!template) return template;\n return template.replace(/\\{\\{(\\w+)\\}\\}/g, (_, key) => data[key] ?? `{{${key}}}`);\n}\n\nexport async function executeTestPlan(\n plan: ExecutionStep[],\n options: {\n screenshotDir: string;\n testRunId: string;\n viewport?: { width: number; height: number };\n onStepComplete?: (result: StepResult) => void;\n /** Run the test up to this many times to detect flakiness (default 1). */\n maxRuns?: number;\n browser?: 'chromium' | 'firefox' | 'webkit';\n /** Data row for parameterized execution: substitutes {{varname}} in step fields. */\n dataRow?: Record<string, string>;\n /** Enables the AI Self-Healing Agent tier when a step's selector can't be resolved\n * even after trying alternate locator strategies. Omit to keep mechanical-only\n * healing (no LLM calls during execution). */\n healCtx?: { tenantId: string; projectId: string };\n /** Decrypted Playwright storageState JSON \u2014 restores a logged-in session so\n * steps don't have to re-authenticate. Without this, any test case behind a\n * login wall fails at the first authenticated step. */\n storageState?: string;\n },\n): Promise<ExecutionResult> {\n if (options.dataRow && Object.keys(options.dataRow).length > 0) {\n const data = options.dataRow;\n plan = plan.map((step) => ({\n ...step,\n url: interpolate(step.url, data),\n selector: interpolate(step.selector, data),\n value: interpolate(step.value, data),\n description: interpolate(step.description, data) ?? step.description,\n }));\n }\n const maxRuns = options.maxRuns ?? 1;\n const browserEngine = { chromium, firefox, webkit }[options.browser ?? 'chromium'] ?? chromium;\n const start = Date.now();\n\n const outcomes: ('PASSED' | 'FAILED')[] = [];\n let lastResult: Awaited<ReturnType<typeof runPlan>> | null = null;\n let lastNetworkTimings: Array<{ url: string; method: string; status: number; duration: number }> = [];\n\n for (let run = 0; run < maxRuns; run++) {\n const browser = await browserEngine.launch({\n headless: true,\n // Optional override for environments with a non-standard/pre-provisioned\n // browser install (e.g. a custom base image with only system Chromium).\n ...(process.env.PLAYWRIGHT_EXECUTABLE_PATH ? { executablePath: process.env.PLAYWRIGHT_EXECUTABLE_PATH } : {}),\n });\n let parsedStorageState: any;\n try {\n parsedStorageState = options.storageState ? JSON.parse(options.storageState) : undefined;\n } catch {\n parsedStorageState = undefined;\n }\n const context = await browser.newContext({\n viewport: options.viewport ?? { width: 1440, height: 900 },\n ...(parsedStorageState ? { storageState: parsedStorageState } : {}),\n });\n const page = await context.newPage();\n\n await page\n .addStyleTag({ content: '*, *::before, *::after { animation-duration: 0s !important; transition-duration: 0s !important; }' })\n .catch(() => {});\n\n const consoleLogs: Array<{ type: string; text: string; timestamp: number }> = [];\n const networkErrors: Array<{ url: string; status: number; method: string; timestamp: number }> = [];\n const networkTimings: Array<{ url: string; method: string; status: number; duration: number }> = [];\n const requestStartTimes = new Map<string, number>();\n\n page.on('console', msg => {\n if (consoleLogs.length < 100) {\n consoleLogs.push({ type: msg.type(), text: msg.text().slice(0, 500), timestamp: Date.now() });\n }\n });\n page.on('request', request => {\n const url = request.url();\n if (url.includes('/api/') || url.match(/\\/(graphql|rest|v\\d)\\//)) {\n requestStartTimes.set(request.url(), Date.now());\n }\n });\n page.on('response', response => {\n const url = response.url();\n const start = requestStartTimes.get(url);\n if (start && networkTimings.length < 200) {\n networkTimings.push({ url: url.slice(0, 300), method: response.request().method(), status: response.status(), duration: Date.now() - start });\n requestStartTimes.delete(url);\n }\n if (!response.ok() && !url.includes('favicon') && networkErrors.length < 50) {\n networkErrors.push({\n url: url.slice(0, 300),\n status: response.status(),\n method: response.request().method(),\n timestamp: Date.now(),\n });\n }\n });\n\n const TEST_EXECUTION_TIMEOUT_MS = 5 * 60_000;\n lastResult = await Promise.race([\n runPlan(page, plan, options, consoleLogs, networkErrors),\n new Promise<never>((_, reject) =>\n setTimeout(() => reject(new Error('Test execution timed out')), TEST_EXECUTION_TIMEOUT_MS)\n ),\n ]);\n lastNetworkTimings = networkTimings;\n await browser.close();\n\n const passed = !lastResult.stepResults.some((r) => r.status === 'FAILED');\n outcomes.push(passed ? 'PASSED' : 'FAILED');\n\n // If all subsequent runs are consistent with first, no need to keep going\n if (run > 0 && outcomes.every((o) => o === outcomes[0])) break;\n }\n\n const passCount = outcomes.filter((o) => o === 'PASSED').length;\n const failCount = outcomes.filter((o) => o === 'FAILED').length;\n\n let status: ExecutionResult['status'];\n if (maxRuns > 1 && passCount > 0 && failCount > 0) {\n status = 'FLAKY';\n } else if (passCount === outcomes.length) {\n status = 'PASSED';\n } else {\n status = 'FAILED';\n }\n\n let rootCauseSummary: string | undefined;\n if (status === 'FAILED' || status === 'FLAKY') {\n const failedStep = lastResult!.stepResults.find(s => s.status === 'FAILED');\n rootCauseSummary = await analyzeFailure({\n tenantId: options.healCtx?.tenantId,\n error: failedStep?.error ?? 'Unknown error',\n stepDescription: failedStep?.description,\n consoleLogs: lastResult!.consoleLogs,\n networkErrors: lastResult!.networkErrors,\n screenshotPath: lastResult!.finalScreenshotPath,\n }).catch(() => undefined) ?? undefined;\n }\n\n return {\n stepResults: lastResult!.stepResults,\n status,\n errorStep: lastResult!.errorStep,\n totalDuration: Date.now() - start,\n finalScreenshotPath: lastResult!.finalScreenshotPath,\n retryCount: outcomes.length - 1,\n consoleLogs: lastResult!.consoleLogs,\n networkErrors: lastResult!.networkErrors,\n rootCauseSummary,\n networkTimings: lastNetworkTimings,\n };\n}\n"],
|
|
5
|
+
"mappings": "AAAA,SAAS,UAAU,SAAS,cAAuC;AAEnE,SAAS,0BAA0B;AACnC,SAAS,sBAAsB;AA+B/B,MAAM,qBAA0D,CAAC,QAAQ,SAAS,QAAQ,KAAK;AAE/F,SAAS,aAAa,MAAY,UAAkB,WAA8C,QAAiB;AACjH,UAAQ,UAAU;AAAA,IAChB,KAAK;AAAS,aAAO,KAAK,QAAQ,QAAQ;AAAA,IAC1C,KAAK;AAAS,aAAO,KAAK,UAAU,UAAU,EAAE,OAAO,MAAM,CAAC;AAAA,IAC9D,KAAK;AAAS,aAAO,KAAK,UAAU,QAAe;AAAA,IACnD,KAAK;AAAS,aAAO,KAAK,WAAW,QAAQ;AAAA,IAC7C;AAAc,aAAO,KAAK,QAAQ,QAAQ;AAAA,EAC5C;AACF;AAIA,eAAe,2BAA2B,MAAyG;AACjJ,SAAO,KAAK,SAAS,MAAM;AACzB,UAAM,MAAM,MAAM,KAAK,SAAS;AAAA,MAC9B;AAAA,IACF,CAAC;AACD,WAAO,IACJ,OAAO,CAAC,OAAO;AACd,YAAM,OAAO,GAAG,sBAAsB;AACtC,aAAO,KAAK,QAAQ,KAAK,KAAK,SAAS;AAAA,IACzC,CAAC,EACA,MAAM,GAAG,GAAG,EACZ,IAAI,CAAC,OAAO;AACX,YAAM,YAAY,GAAG,aAAa,YAAY,KAAK;AACnD,YAAM,eAAe,GAAG,eAAe,IAAI,QAAQ,QAAQ,GAAG,EAAE,KAAK,EAAE,MAAM,GAAG,GAAG,KAAK;AACxF,YAAM,OAAO,GAAG,aAAa,MAAM,KAAK,GAAG,QAAQ,YAAY;AAC/D,aAAO,EAAE,SAAS,aAAa,eAAe,MAAM,MAAM,WAAW,YAAY;AAAA,IACnF,CAAC;AAAA,EACL,CAAC;AACH;AAWA,eAAe,eACb,MACA,MACA,SACqI;AACrI,QAAM,EAAE,UAAU,mBAAmB,OAAO,IAAI;AAChD,MAAI,CAAC,SAAU,QAAO;AAEtB,QAAM,UAAU,aAAa,MAAM,UAAU,gBAAgB;AAC7D,QAAM,WAAW,MAAM,QAAQ,MAAM,EAAE,UAAU,EAAE,SAAS,KAAK,CAAC,EAAE,MAAM,MAAM,KAAK;AACrF,MAAI,SAAU,QAAO,EAAE,KAAK,SAAS,QAAQ,MAAM;AAGnD,aAAW,YAAY,oBAAoB;AACzC,QAAI,aAAa,iBAAkB;AACnC,UAAM,MAAM,aAAa,MAAM,UAAU,QAAQ;AACjD,UAAM,UAAU,MAAM,IAAI,MAAM,EAAE,UAAU,EAAE,SAAS,IAAI,CAAC,EAAE,MAAM,MAAM,KAAK;AAC/E,QAAI,QAAS,QAAO,EAAE,KAAK,QAAQ,MAAM,WAAW,aAAa;AAAA,EACnE;AAGA,MAAI,SAAS;AACX,QAAI;AACF,YAAM,EAAE,YAAY,IAAI,MAAM,OAAO,eAAe;AACpD,YAAM,sBAAsB,MAAM,2BAA2B,IAAI;AACjE,YAAM,SAAS,MAAM,YAAY;AAAA,QAC/B,UAAU,QAAQ;AAAA,QAClB,WAAW,QAAQ;AAAA,QACnB,sBAAsB,KAAK,eAAe;AAAA,QAC1C;AAAA,MACF,CAAC;AACD,UAAI,OAAO,UAAU,OAAO,aAAa;AACvC,cAAM,YAAY,KAAK,UAAU,OAAO,YAAY,gBAAgB,EAAE,OAAO,MAAM,CAAC;AACpF,cAAM,UAAU,MAAM,UAAU,MAAM,EAAE,UAAU,EAAE,SAAS,KAAK,CAAC,EAAE,MAAM,MAAM,KAAK;AACtF,YAAI,SAAS;AACX,iBAAO;AAAA,YACL,KAAK;AAAA,YACL,QAAQ;AAAA,YACR,WAAW;AAAA,YACX,eAAe,OAAO;AAAA,YACtB,gBAAgB,OAAO;AAAA,UACzB;AAAA,QACF;AAAA,MACF;AAAA,IACF,QAAQ;AAAA,IAGR;AAAA,EACF;AAGA,SAAO,EAAE,KAAK,SAAS,QAAQ,MAAM;AACvC;AAEA,SAAS,cAAc,QAAoB,GAAgH;AACzJ,SAAO,SAAS,EAAE;AAClB,SAAO,YAAY,EAAE;AACrB,SAAO,gBAAgB,EAAE;AACzB,SAAO,iBAAiB,EAAE;AAC5B;AAEA,eAAe,QACb,MACA,MACA,SACA,aACA,eACgK;AAChK,QAAM,cAA4B,CAAC;AACnC,MAAI;AACJ,MAAI;AAEJ,aAAW,QAAQ,MAAM;AACvB,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,SAAqB;AAAA,MACzB,YAAY,KAAK;AAAA,MACjB,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,UAAU;AAAA,IACZ;AAEA,QAAI;AACF,YAAM,UAAU,KAAK,WAAW;AAEhC,cAAQ,KAAK,QAAQ;AAAA,QACnB,KAAK;AACH,cAAI,CAAC,KAAK,IAAK,OAAM,IAAI,MAAM,2BAA2B;AAC1D,6BAAmB,KAAK,GAAG;AAC3B,gBAAM,KAAK,KAAK,KAAK,KAAK,EAAE,WAAW,oBAAoB,QAAQ,CAAC;AACpE;AAAA,QAEF,KAAK,SAAS;AACZ,gBAAM,IAAI,MAAM,eAAe,MAAM,MAAM,QAAQ,OAAO;AAC1D,cAAI,CAAC,EAAG,OAAM,IAAI,MAAM,2BAA2B;AACnD,gBAAM,EAAE,IAAI,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC;AACrC,wBAAc,QAAQ,CAAC;AACvB;AAAA,QACF;AAAA,QAEA,KAAK,QAAQ;AACX,gBAAM,IAAI,MAAM,eAAe,MAAM,MAAM,QAAQ,OAAO;AAC1D,cAAI,CAAC,EAAG,OAAM,IAAI,MAAM,2BAA2B;AACnD,gBAAM,EAAE,IAAI,MAAM,EAAE,KAAK,KAAK,SAAS,IAAI,EAAE,QAAQ,CAAC;AACtD,wBAAc,QAAQ,CAAC;AACvB;AAAA,QACF;AAAA,QAEA,KAAK,UAAU;AACb,gBAAM,IAAI,MAAM,eAAe,MAAM,MAAM,QAAQ,OAAO;AAC1D,cAAI,CAAC,EAAG,OAAM,IAAI,MAAM,2BAA2B;AACnD,gBAAM,EAAE,IAAI,MAAM,EAAE,aAAa,KAAK,SAAS,IAAI,EAAE,QAAQ,CAAC;AAC9D,wBAAc,QAAQ,CAAC;AACvB;AAAA,QACF;AAAA,QAEA,KAAK,SAAS;AACZ,gBAAM,IAAI,MAAM,eAAe,MAAM,MAAM,QAAQ,OAAO;AAC1D,cAAI,CAAC,EAAG,OAAM,IAAI,MAAM,2BAA2B;AACnD,gBAAM,EAAE,IAAI,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC;AACrC,wBAAc,QAAQ,CAAC;AACvB;AAAA,QACF;AAAA,QAEA,KAAK;AACH,gBAAM,KAAK,SAAS,MAAM,KAAK,SAAS,OAAO;AAC/C;AAAA,QAEF,KAAK,kBAAkB;AACrB,gBAAM,IAAI,MAAM,eAAe,MAAM,MAAM,QAAQ,OAAO;AAC1D,cAAI,CAAC,EAAG,OAAM,IAAI,MAAM,2BAA2B;AACnD,gBAAM,EAAE,IAAI,MAAM,EAAE,QAAQ,EAAE,OAAO,WAAW,QAAQ,CAAC;AACzD,wBAAc,QAAQ,CAAC;AACvB;AAAA,QACF;AAAA,QAEA,KAAK,eAAe;AAClB,gBAAM,IAAI,MAAM,eAAe,MAAM,MAAM,QAAQ,OAAO;AAC1D,cAAI,CAAC,EAAG,OAAM,IAAI,MAAM,2BAA2B;AACnD,gBAAM,OAAO,MAAM,EAAE,IAAI,MAAM,EAAE,YAAY,EAAE,QAAQ,CAAC;AACxD,cAAI,CAAC,MAAM,SAAS,KAAK,SAAS,EAAE,GAAG;AACrC,kBAAM,IAAI,MAAM,kBAAkB,KAAK,KAAK,sBAAsB,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG;AAAA,UAC1F;AACA,wBAAc,QAAQ,CAAC;AACvB;AAAA,QACF;AAAA,QAEA,KAAK,cAAc;AACjB,gBAAM,MAAM,KAAK,IAAI;AACrB,cAAI,CAAC,IAAI,SAAS,KAAK,SAAS,EAAE,GAAG;AACnC,kBAAM,IAAI,MAAM,4BAA4B,KAAK,KAAK,YAAY,GAAG,GAAG;AAAA,UAC1E;AACA;AAAA,QACF;AAAA,QAEA,KAAK;AACH,gBAAM,KAAK,eAAe,KAAK,WAAW,GAAI;AAC9C;AAAA,QAEF,KAAK,cAAc;AACjB,gBAAM,SAAS,GAAG,QAAQ,aAAa,IAAI,QAAQ,SAAS,SAAS,KAAK,UAAU;AACpF,gBAAM,KAAK,WAAW,EAAE,MAAM,QAAQ,UAAU,KAAK,CAAC;AACtD,iBAAO,iBAAiB;AACxB;AAAA,QACF;AAAA,MACF;AAEA,aAAO,SAAS;AAAA,IAClB,SAAS,KAAU;AACjB,aAAO,SAAS,KAAK,WAAW,YAAY;AAC5C,aAAO,QAAQ,IAAI;AACnB,UAAI,CAAC,KAAK,UAAU;AAClB,oBAAY,KAAK;AAAA,MACnB;AAAA,IACF;AAEA,QAAI;AACF,YAAM,KAAK,iBAAiB,eAAe,EAAE,SAAS,KAAK,CAAC,EAAE,MAAM,MAAM;AAAA,MAAC,CAAC;AAC5E,UAAI,CAAC,OAAO,gBAAgB;AAC1B,cAAM,SAAS,GAAG,QAAQ,aAAa,IAAI,QAAQ,SAAS,SAAS,KAAK,UAAU;AACpF,cAAM,KAAK,WAAW,EAAE,MAAM,QAAQ,UAAU,KAAK,CAAC;AACtD,eAAO,iBAAiB;AAAA,MAC1B;AAAA,IACF,QAAQ;AAAA,IAAwE;AAEhF,WAAO,WAAW,KAAK,IAAI,IAAI;AAC/B,gBAAY,KAAK,MAAM;AACvB,YAAQ,iBAAiB,MAAM;AAE/B,QAAI,OAAO,WAAW,SAAU;AAAA,EAClC;AAEA,MAAI;AACF,UAAM,YAAY,GAAG,QAAQ,aAAa,IAAI,QAAQ,SAAS;AAC/D,UAAM,KAAK,WAAW,EAAE,MAAM,WAAW,UAAU,KAAK,CAAC;AACzD,0BAAsB;AAAA,EACxB,QAAQ;AAAA,EAAkB;AAE1B,SAAO,EAAE,aAAa,WAAW,qBAAqB,aAAa,cAAc;AACnF;AAEA,SAAS,YAAY,UAA8B,MAAkD;AACnG,MAAI,CAAC,SAAU,QAAO;AACtB,SAAO,SAAS,QAAQ,kBAAkB,CAAC,GAAG,QAAQ,KAAK,GAAG,KAAK,KAAK,GAAG,IAAI;AACjF;AAEA,eAAsB,gBACpB,MACA,SAmB0B;AAC1B,MAAI,QAAQ,WAAW,OAAO,KAAK,QAAQ,OAAO,EAAE,SAAS,GAAG;AAC9D,UAAM,OAAO,QAAQ;AACrB,WAAO,KAAK,IAAI,CAAC,UAAU;AAAA,MACzB,GAAG;AAAA,MACH,KAAK,YAAY,KAAK,KAAK,IAAI;AAAA,MAC/B,UAAU,YAAY,KAAK,UAAU,IAAI;AAAA,MACzC,OAAO,YAAY,KAAK,OAAO,IAAI;AAAA,MACnC,aAAa,YAAY,KAAK,aAAa,IAAI,KAAK,KAAK;AAAA,IAC3D,EAAE;AAAA,EACJ;AACA,QAAM,UAAU,QAAQ,WAAW;AACnC,QAAM,gBAAgB,EAAE,UAAU,SAAS,OAAO,EAAE,QAAQ,WAAW,UAAU,KAAK;AACtF,QAAM,QAAQ,KAAK,IAAI;AAEvB,QAAM,WAAoC,CAAC;AAC3C,MAAI,aAAyD;AAC7D,MAAI,qBAA+F,CAAC;AAEpG,WAAS,MAAM,GAAG,MAAM,SAAS,OAAO;AACtC,UAAM,UAAU,MAAM,cAAc,OAAO;AAAA,MACzC,UAAU;AAAA;AAAA;AAAA,MAGV,GAAI,QAAQ,IAAI,6BAA6B,EAAE,gBAAgB,QAAQ,IAAI,2BAA2B,IAAI,CAAC;AAAA,IAC7G,CAAC;AACD,QAAI;AACJ,QAAI;AACF,2BAAqB,QAAQ,eAAe,KAAK,MAAM,QAAQ,YAAY,IAAI;AAAA,IACjF,QAAQ;AACN,2BAAqB;AAAA,IACvB;AACA,UAAM,UAAU,MAAM,QAAQ,WAAW;AAAA,MACvC,UAAU,QAAQ,YAAY,EAAE,OAAO,MAAM,QAAQ,IAAI;AAAA,MACzD,GAAI,qBAAqB,EAAE,cAAc,mBAAmB,IAAI,CAAC;AAAA,IACnE,CAAC;AACD,UAAM,OAAO,MAAM,QAAQ,QAAQ;AAEnC,UAAM,KACH,YAAY,EAAE,SAAS,oGAAoG,CAAC,EAC5H,MAAM,MAAM;AAAA,IAAC,CAAC;AAEjB,UAAM,cAAwE,CAAC;AAC/E,UAAM,gBAA2F,CAAC;AAClG,UAAM,iBAA2F,CAAC;AAClG,UAAM,oBAAoB,oBAAI,IAAoB;AAElD,SAAK,GAAG,WAAW,SAAO;AACxB,UAAI,YAAY,SAAS,KAAK;AAC5B,oBAAY,KAAK,EAAE,MAAM,IAAI,KAAK,GAAG,MAAM,IAAI,KAAK,EAAE,MAAM,GAAG,GAAG,GAAG,WAAW,KAAK,IAAI,EAAE,CAAC;AAAA,MAC9F;AAAA,IACF,CAAC;AACD,SAAK,GAAG,WAAW,aAAW;AAC5B,YAAM,MAAM,QAAQ,IAAI;AACxB,UAAI,IAAI,SAAS,OAAO,KAAK,IAAI,MAAM,wBAAwB,GAAG;AAChE,0BAAkB,IAAI,QAAQ,IAAI,GAAG,KAAK,IAAI,CAAC;AAAA,MACjD;AAAA,IACF,CAAC;AACD,SAAK,GAAG,YAAY,cAAY;AAC9B,YAAM,MAAM,SAAS,IAAI;AACzB,YAAMA,SAAQ,kBAAkB,IAAI,GAAG;AACvC,UAAIA,UAAS,eAAe,SAAS,KAAK;AACxC,uBAAe,KAAK,EAAE,KAAK,IAAI,MAAM,GAAG,GAAG,GAAG,QAAQ,SAAS,QAAQ,EAAE,OAAO,GAAG,QAAQ,SAAS,OAAO,GAAG,UAAU,KAAK,IAAI,IAAIA,OAAM,CAAC;AAC5I,0BAAkB,OAAO,GAAG;AAAA,MAC9B;AACA,UAAI,CAAC,SAAS,GAAG,KAAK,CAAC,IAAI,SAAS,SAAS,KAAK,cAAc,SAAS,IAAI;AAC3E,sBAAc,KAAK;AAAA,UACjB,KAAK,IAAI,MAAM,GAAG,GAAG;AAAA,UACrB,QAAQ,SAAS,OAAO;AAAA,UACxB,QAAQ,SAAS,QAAQ,EAAE,OAAO;AAAA,UAClC,WAAW,KAAK,IAAI;AAAA,QACtB,CAAC;AAAA,MACH;AAAA,IACF,CAAC;AAED,UAAM,4BAA4B,IAAI;AACtC,iBAAa,MAAM,QAAQ,KAAK;AAAA,MAC9B,QAAQ,MAAM,MAAM,SAAS,aAAa,aAAa;AAAA,MACvD,IAAI;AAAA,QAAe,CAAC,GAAG,WACrB,WAAW,MAAM,OAAO,IAAI,MAAM,0BAA0B,CAAC,GAAG,yBAAyB;AAAA,MAC3F;AAAA,IACF,CAAC;AACD,yBAAqB;AACrB,UAAM,QAAQ,MAAM;AAEpB,UAAM,SAAS,CAAC,WAAW,YAAY,KAAK,CAAC,MAAM,EAAE,WAAW,QAAQ;AACxE,aAAS,KAAK,SAAS,WAAW,QAAQ;AAG1C,QAAI,MAAM,KAAK,SAAS,MAAM,CAAC,MAAM,MAAM,SAAS,CAAC,CAAC,EAAG;AAAA,EAC3D;AAEA,QAAM,YAAY,SAAS,OAAO,CAAC,MAAM,MAAM,QAAQ,EAAE;AACzD,QAAM,YAAY,SAAS,OAAO,CAAC,MAAM,MAAM,QAAQ,EAAE;AAEzD,MAAI;AACJ,MAAI,UAAU,KAAK,YAAY,KAAK,YAAY,GAAG;AACjD,aAAS;AAAA,EACX,WAAW,cAAc,SAAS,QAAQ;AACxC,aAAS;AAAA,EACX,OAAO;AACL,aAAS;AAAA,EACX;AAEA,MAAI;AACJ,MAAI,WAAW,YAAY,WAAW,SAAS;AAC7C,UAAM,aAAa,WAAY,YAAY,KAAK,OAAK,EAAE,WAAW,QAAQ;AAC1E,uBAAmB,MAAM,eAAe;AAAA,MACtC,UAAU,QAAQ,SAAS;AAAA,MAC3B,OAAO,YAAY,SAAS;AAAA,MAC5B,iBAAiB,YAAY;AAAA,MAC7B,aAAa,WAAY;AAAA,MACzB,eAAe,WAAY;AAAA,MAC3B,gBAAgB,WAAY;AAAA,IAC9B,CAAC,EAAE,MAAM,MAAM,MAAS,KAAK;AAAA,EAC/B;AAEA,SAAO;AAAA,IACL,aAAa,WAAY;AAAA,IACzB;AAAA,IACA,WAAW,WAAY;AAAA,IACvB,eAAe,KAAK,IAAI,IAAI;AAAA,IAC5B,qBAAqB,WAAY;AAAA,IACjC,YAAY,SAAS,SAAS;AAAA,IAC9B,aAAa,WAAY;AAAA,IACzB,eAAe,WAAY;AAAA,IAC3B;AAAA,IACA,gBAAgB;AAAA,EAClB;AACF;",
|
|
6
6
|
"names": ["start"]
|
|
7
7
|
}
|
package/package.json
CHANGED
package/src/crawler.ts
CHANGED
|
@@ -1303,7 +1303,8 @@ export async function crawlProject(opts: {
|
|
|
1303
1303
|
}
|
|
1304
1304
|
let checkpointSuccessCount = 0;
|
|
1305
1305
|
let maxSeenTotal = 0; // track high-water mark so progress % never goes backwards
|
|
1306
|
-
let lastEmittedPct =
|
|
1306
|
+
let lastEmittedPct = 5; // clamp so progress never decreases even when queue grows
|
|
1307
|
+
const crawlLoopStart = Date.now();
|
|
1307
1308
|
const startUrl = page.url();
|
|
1308
1309
|
// Resume from previous segment's pending queue if provided
|
|
1309
1310
|
const resumeUrls = opts.resumeUrls ?? [];
|
|
@@ -2135,7 +2136,13 @@ export async function crawlProject(opts: {
|
|
|
2135
2136
|
|
|
2136
2137
|
// Emit progress now — navigation is done and the page is stable.
|
|
2137
2138
|
// This fires before the 30-90s parallel screenshot+AI phase, preventing a silent gap in the UI.
|
|
2138
|
-
|
|
2139
|
+
// Phase 1 (link discovery): 5–20% over first 60s of wall-clock time so users see movement
|
|
2140
|
+
// before visited.size grows enough to drive the 20–80% phase.
|
|
2141
|
+
const elapsedDiscoverySec = (Date.now() - crawlLoopStart) / 1000;
|
|
2142
|
+
const discoveryPct = Math.round(5 + Math.min(15, (elapsedDiscoverySec / 60) * 15));
|
|
2143
|
+
const preCapturePct = Math.max(lastEmittedPct, discoveryPct);
|
|
2144
|
+
lastEmittedPct = preCapturePct;
|
|
2145
|
+
await onProgress?.(preCapturePct, `Crawling ${visited.size} of ~${visited.size + queue.length} · ${(() => { try { return new URL(url).pathname || '/'; } catch { return '/'; } })()}`);
|
|
2139
2146
|
|
|
2140
2147
|
// P3a: Async content detection — get raw DOM element count before AI extraction.
|
|
2141
2148
|
// The AI extraction takes 30-90s; any async-loaded content will be visible after it completes.
|
|
@@ -2145,6 +2152,7 @@ export async function crawlProject(opts: {
|
|
|
2145
2152
|
).catch(() => 0);
|
|
2146
2153
|
|
|
2147
2154
|
await waitForScreenshotReady(page);
|
|
2155
|
+
await onProgress?.(lastEmittedPct, `Capturing ${(() => { try { return new URL(url).pathname || '/'; } catch { return '/'; } })()}`);
|
|
2148
2156
|
|
|
2149
2157
|
// Parallelize: screenshot + element extraction + link discovery + AT capture (all work on current page state)
|
|
2150
2158
|
const [, elements, nextLinks, atElements] = await Promise.all([
|
|
@@ -2644,9 +2652,10 @@ export async function crawlProject(opts: {
|
|
|
2644
2652
|
|
|
2645
2653
|
discovered.push({ url, screenId: screen.id, elements: elements.length, elementFacts: elements, screenshotPath: shotPath, version: screen.version });
|
|
2646
2654
|
// Progress: 20% baseline + up to 60% across discovered screens.
|
|
2647
|
-
// maxSeenTotal
|
|
2648
|
-
// the
|
|
2649
|
-
|
|
2655
|
+
// Cap maxSeenTotal at maxScreens * 3 so unbounded link discovery does not
|
|
2656
|
+
// collapse the denominator and freeze progress in the 20-22% band.
|
|
2657
|
+
const effectiveMax = isFinite(maxScreens) ? Math.min(visited.size + queue.length, maxScreens * 3) : visited.size + queue.length;
|
|
2658
|
+
maxSeenTotal = Math.max(maxSeenTotal, effectiveMax);
|
|
2650
2659
|
const rawPct = 20 + Math.round(Math.min(60, (visited.size / Math.max(1, maxSeenTotal)) * 60));
|
|
2651
2660
|
const clampedPct = Math.max(lastEmittedPct, rawPct);
|
|
2652
2661
|
lastEmittedPct = clampedPct;
|
|
@@ -4018,24 +4027,27 @@ async function waitForStableDOM(page: any, maxWait = 8_000, stabilityMs = 400):
|
|
|
4018
4027
|
}
|
|
4019
4028
|
|
|
4020
4029
|
async function waitForScreenshotReady(page: any): Promise<void> {
|
|
4030
|
+
const hardDeadline = Date.now() + 20_000;
|
|
4031
|
+
const withHardCap = <T>(p: Promise<T>): Promise<T | void> =>
|
|
4032
|
+
Promise.race([p, new Promise<void>((resolve) => setTimeout(resolve, Math.max(0, hardDeadline - Date.now())))]);
|
|
4021
4033
|
try {
|
|
4022
|
-
await page.waitForLoadState('domcontentloaded', { timeout: 5_000 }).catch(() => { });
|
|
4023
|
-
await page.waitForLoadState('networkidle', { timeout: 12_000 }).catch(() => { });
|
|
4024
|
-
await page.evaluate(() => {
|
|
4034
|
+
await withHardCap(page.waitForLoadState('domcontentloaded', { timeout: 5_000 }).catch(() => { }));
|
|
4035
|
+
await withHardCap(page.waitForLoadState('networkidle', { timeout: 12_000 }).catch(() => { }));
|
|
4036
|
+
await withHardCap(page.evaluate(() => {
|
|
4025
4037
|
if (document.readyState === 'complete') return true;
|
|
4026
4038
|
return new Promise<boolean>((resolve) => {
|
|
4027
4039
|
const done = () => resolve(true);
|
|
4028
4040
|
window.addEventListener('load', done, { once: true });
|
|
4029
4041
|
setTimeout(done, 4_000);
|
|
4030
4042
|
});
|
|
4031
|
-
}).catch(() => { });
|
|
4032
|
-
await page.evaluate(async () => {
|
|
4043
|
+
}).catch(() => { }));
|
|
4044
|
+
await withHardCap(page.evaluate(async () => {
|
|
4033
4045
|
const fonts = (document as any).fonts;
|
|
4034
4046
|
if (fonts?.ready) await fonts.ready;
|
|
4035
|
-
}).catch(() => { });
|
|
4036
|
-
await scrollToTriggerLazyLoad(page);
|
|
4037
|
-
await waitForStableDOM(page,
|
|
4038
|
-
await page.evaluate(async () => {
|
|
4047
|
+
}).catch(() => { }));
|
|
4048
|
+
await withHardCap(scrollToTriggerLazyLoad(page));
|
|
4049
|
+
await withHardCap(waitForStableDOM(page, 4_000, 700));
|
|
4050
|
+
await withHardCap(page.evaluate(async () => {
|
|
4039
4051
|
const pending = Array.from(document.images).filter((img) => !img.complete);
|
|
4040
4052
|
if (pending.length === 0) return;
|
|
4041
4053
|
await Promise.race([
|
|
@@ -4045,9 +4057,9 @@ async function waitForScreenshotReady(page: any): Promise<void> {
|
|
|
4045
4057
|
}))),
|
|
4046
4058
|
new Promise<void>((resolve) => setTimeout(resolve, 4_000)),
|
|
4047
4059
|
]);
|
|
4048
|
-
}).catch(() => { });
|
|
4049
|
-
await waitForStableDOM(page,
|
|
4050
|
-
await page.waitForTimeout(250).catch(() => { });
|
|
4060
|
+
}).catch(() => { }));
|
|
4061
|
+
await withHardCap(waitForStableDOM(page, 2_000, 500));
|
|
4062
|
+
await withHardCap(page.waitForTimeout(250).catch(() => { }));
|
|
4051
4063
|
} catch { /* readiness is best-effort; never fail the crawl for a continuously mutating page */ }
|
|
4052
4064
|
}
|
|
4053
4065
|
|