@debugg-ai/debugg-ai-mcp 3.5.2 → 3.5.3

package/README.md

@@ -226,6 +226,7 @@ Response-shape changes: the bare `count` field on list responses is gone — use
 | `DEBUGGAI_API_KEY` | yes | Backend API key. Aliases: `DEBUGGAI_API_TOKEN`, `DEBUGGAI_JWT_TOKEN`. |
 | `DEBUGGAI_API_URL` | no | Backend base URL. Defaults to `https://api.debugg.ai`. |
 | `DEBUGGAI_TOKEN_TYPE` | no | `token` (default) or `bearer`. |
+| `DEBUGGAI_EVAL_TEMPLATE` | no | Override the App Evaluation workflow **slug** that `check_app_in_browser` dispatches to. Defaults to `flow/e2es/app-eval`. Dispatch pins to this slug so a backend template rename can't break it. |
 | `LOG_LEVEL` | no | `error` / `warn` / `info` (default) / `debug`. |
 | `POSTHOG_API_KEY` | no | Override the embedded telemetry project key (e.g. private fork). |
 | `DEBUGGAI_TELEMETRY_DISABLED` | no | Set to `1` / `true` / `yes` / `on` to disable telemetry entirely. |

package/dist/handlers/testPageChangesHandler.js

@@ -8,6 +8,8 @@ import { Logger } from '../utils/logger.js';
 import { handleExternalServiceError } from '../utils/errors.js';
 import { fetchImageAsBase64, imageContentBlock, resourceLinkBlock, artifactResourceLinks } from '../utils/imageUtils.js';
 import { DebuggAIServerClient } from '../services/index.js';
+import { getEvalTemplateSlug } from '../services/workflows.js';
+import { adaptVerdict } from '../services/verdictAdapter.js';
 import { TunnelProvisionError } from '../services/tunnels.js';
 import { resolveTargetUrl, buildContext, findExistingTunnel, ensureTunnel, sanitizeResponseUrls, touchTunnelById, } from '../utils/tunnelContext.js';
 import { detectRepoName } from '../utils/gitContext.js';
@@ -18,7 +20,6 @@ import { getCachedTemplateUuid, getCachedProjectUuid, invalidateTemplateCache, i
 import { isTransientWorkflowError, transientReasonTag } from '../utils/transientErrors.js';
 import { Telemetry, TelemetryEvents } from '../utils/telemetry.js';
 const logger = new Logger({ module: 'testPageChangesHandler' });
-const TEMPLATE_NAME = 'app evaluation';
 // Bead kbxy: bounded retry on known transient backend signatures (Pydantic
 // JSON parse errors, 502s, ECONNRESETs). Default 1 retry; env-overridable
 // up to 3 to balance reliability vs quota cost. Conservative: only retries
@@ -203,7 +204,10 @@ async function testPageChangesHandlerInner(input, context, rawProgressCallback)
         }
         const repoName = input.repoName || detectRepoName();
         const [templateUuid, projectUuid] = await Promise.all([
-            getCachedTemplateUuid(TEMPLATE_NAME, async () => {
+            // Cache key = the dispatch slug so the cache key and the lookup can never
+            // drift apart (bug clka: the key used to be a decoupled 'app evaluation'
+            // literal while the lookup searched a different string).
+            getCachedTemplateUuid(getEvalTemplateSlug(), async () => {
                 return client.workflows.findEvaluationTemplate();
             }),
             repoName
@@ -366,6 +370,10 @@ async function testPageChangesHandlerInner(input, context, rawProgressCallback)
             // result the agent reached.
             if (attempt > MAX_RETRIES)
                 break;
+            // A poll-deadline timeout (bead 56kd.3) is never retried — surface the
+            // partial result instead of burning another 10 minutes.
+            if (finalExecution.timedOut)
+                break;
             if (!isTransientWorkflowError(finalExecution))
                 break;
             logger.warn(`Transient backend error detected (${transientReasonTag(finalExecution) ?? 'unknown'}) — ` +
@@ -373,7 +381,6 @@ async function testPageChangesHandlerInner(input, context, rawProgressCallback)
         }
         const duration = Date.now() - startTime;
         // --- Format result ---
-        const outcome = finalExecution.state?.outcome ?? finalExecution.status;
         const nodes = finalExecution.nodeExecutions ?? [];
         // subworkflow.run is the current graph shape — carries outcome, actionHistory, screenshot
         const subworkflowNode = nodes.find(n => n.nodeType === 'subworkflow.run');
@@ -428,43 +435,41 @@ async function testPageChangesHandlerInner(input, context, rawProgressCallback)
                 reason: sw.error || undefined,
             };
         }
-        const stepsTaken = finalExecution.state?.stepsTaken ?? subworkflowNode?.outputData?.stepsTaken ?? actionTrace.length;
-        const success = finalExecution.state?.success ?? subworkflowNode?.outputData?.success ?? false;
+        // --- Relay the backend's explicit verdict (bead 56kd.2) ---
+        // ONE adapter owns the backend-field → MCP mapping (services/verdictAdapter).
+        // We consume the verdict/budget/evidence VERBATIM — no fabricated outcome,
+        // no success default-false, no synthesized 'assertion-mismatch'. A
+        // missing/unknown verdict surfaces as 'inconclusive', never as a failure.
+        // On a poll-deadline timeout (bead 56kd.3) pollExecution returns the last
+        // observed execution flagged `timedOut`; force outcome 'timeout' since there
+        // is no terminal backend verdict, and shape its partial evidence below.
+        const timedOut = finalExecution.timedOut === true;
+        const verdict = adaptVerdict(finalExecution, {
+            fallbackBudget: MAX_EXEC_STEPS,
+            outcomeOverride: timedOut ? 'timeout' : undefined,
+        });
+        // Evidence: prefer the backend's contract evidence.actionTrace; fall back to
+        // the legacy node-extracted trace while the backend contract deploys.
+        const relayActionTrace = verdict.actionTrace ?? actionTrace;
         const responsePayload = {
-            outcome,
-            success,
+            outcome: verdict.outcome,
+            success: verdict.success,
             status: finalExecution.status,
-            stepsTaken,
-            stepsBudget: MAX_EXEC_STEPS, // bead qmdd
-            stepsRemaining: Math.max(0, MAX_EXEC_STEPS - (stepsTaken ?? 0)), // bead qmdd
+            stepsTaken: verdict.stepsTaken,
+            stepsBudget: verdict.stepsBudget, // from the response (bead 56kd.2)
+            stepsRemaining: verdict.stepsRemaining,
             targetUrl: originalUrl,
             executionId: executionUuid,
             durationMs: finalExecution.durationMs ?? duration,
         };
-        // Bead jqmj: failureCategory disambiguates the three meanings of 'fail':
-        //   'agent-error'        — workflow/infra failure (Pydantic parse error,
-        //                          backend exception, transport issue). Caller's
-        //                          right move: retry-with-backoff.
-        //   'assertion-mismatch' — agent ran the scenario but page state didn't
-        //                          match expectations. Caller's right move: fix
-        //                          code or update the test description.
-        //   ('page-error' is reserved for v2 — needs a structured signal from
-        //   backend to distinguish from assertion-mismatch reliably; today's
-        //   inferrable info is too fragile.)
-        // Field is OMITTED on success (no failure to categorize).
-        if (!success) {
-            // state.error is the AGENT's narrative — it can describe assertion
-            // failures ("expected heading to contain Welcome") OR infrastructure
-            // failures ("Pydantic JSON parse error"). Without a structured signal,
-            // we only count it as 'agent-error' when paired with workflow-level
-            // failure (status='failed') or transient signature.
-            // status='failed' or errorMessage set → workflow-level / transport error.
-            const hasInfraFailure = finalExecution.status === 'failed'
-                || !!finalExecution.errorMessage;
-            responsePayload.failureCategory = hasInfraFailure ? 'agent-error' : 'assertion-mismatch';
-        }
-        if (actionTrace.length > 0)
-            responsePayload.actionTrace = actionTrace;
+        // failureCategory = the outcome verbatim (fail | inconclusive | error |
+        // timeout); OMITTED on success. No inference, no 'assertion-mismatch'.
+        if (verdict.failureCategory)
+            responsePayload.failureCategory = verdict.failureCategory;
+        if (verdict.reason)
+            responsePayload.reason = verdict.reason;
+        if (Array.isArray(relayActionTrace) && relayActionTrace.length > 0)
+            responsePayload.actionTrace = relayActionTrace;
         if (evaluation)
             responsePayload.evaluation = evaluation;
         if (finalExecution.state?.error)
@@ -505,14 +510,29 @@ async function testPageChangesHandlerInner(input, context, rawProgressCallback)
         const GIF_KEYS = ['runGif', 'gifUrl', 'gif', 'videoUrl', 'recordingUrl'];
         let screenshotEmbedded = false;
         let gifUrl = null;
+        let screenshotUrl = null;
+        // Contract evidence.screenshot (bead 56kd.2/.3) is the preferred source and
+        // is present on EVERY terminal state — including fail and timeout — so we
+        // always have the last screenshot on non-success. Base64 embeds inline; an
+        // http(s) URL is fetched below via the same path as legacy node URLs.
+        const evidenceScreenshot = verdict.screenshot;
+        if (typeof evidenceScreenshot === 'string' && evidenceScreenshot) {
+            if (/^https?:\/\//i.test(evidenceScreenshot)) {
+                screenshotUrl = evidenceScreenshot;
+            }
+            else {
+                logger.info('Embedding inline base64 screenshot from backend evidence');
+                content.push(imageContentBlock(evidenceScreenshot, 'image/png'));
+                screenshotEmbedded = true;
+            }
+        }
         // subworkflow.run carries screenshotB64 directly — no fetch needed
         const screenshotB64 = subworkflowNode?.outputData?.screenshotB64;
-        if (typeof screenshotB64 === 'string' && screenshotB64) {
+        if (!screenshotEmbedded && !screenshotUrl && typeof screenshotB64 === 'string' && screenshotB64) {
             logger.info('Embedding inline base64 screenshot from subworkflow.run');
             content.push(imageContentBlock(screenshotB64, 'image/png'));
             screenshotEmbedded = true;
         }
-        let screenshotUrl = null;
         for (const node of nodes) {
             const data = node.outputData ?? {};
             if (!screenshotEmbedded && !screenshotUrl) {

package/dist/services/verdictAdapter.js

@@ -0,0 +1,73 @@
+/**
+ * Verdict adapter (bead 56kd.2).
+ *
+ * THE ONE place that maps the backend explicit-verdict + budget + evidence
+ * contract onto the MCP relay fields. If the backend's final field shape
+ * differs, re-align it here and nowhere else.
+ *
+ * Backend contract (camelCase after axiosTransport conversion). The containers
+ * are TOP-LEVEL siblings of `state` on the execution-detail response:
+ *   execution.verdict:  { outcome: pass|fail|inconclusive|error|timeout, reason }
+ *   execution.budget:   { maxSteps, usedSteps }
+ *   execution.evidence: { screenshot, actionTrace }
+ * `verdict` is SINGULAR — distinct from the pre-existing plural `verdicts`
+ * (RunVerdict array) and the raw `outcome` string, neither of which we read.
+ *
+ * Principle: relay, never invent.
+ *   - `verdict.outcome` is relayed VERBATIM; `success` = (outcome === 'pass').
+ *   - Anything else (fail/inconclusive/error/timeout) → success:false with
+ *     `failureCategory = outcome` — NOT a fabricated 'assertion-mismatch'.
+ *   - A missing / null / unrecognized verdict maps to `inconclusive`, never to
+ *     `fail` and never to the raw execution status.
+ *   - Budget comes from the response, not a client-side constant (the constant
+ *     is only a fallback for pre-contract backends).
+ */
+/** The verdict enum the backend emits (bead sentinal-k8x1f.2). */
+export const KNOWN_OUTCOMES = ['pass', 'fail', 'inconclusive', 'error', 'timeout'];
+const KNOWN = new Set(KNOWN_OUTCOMES);
+/**
+ * Map a workflow execution onto the MCP relay verdict. Never throws.
+ */
+export function adaptVerdict(execution, opts = {}) {
+    const state = execution?.state ?? null;
+    // Contract containers live at the TOP LEVEL of the execution response
+    // (siblings of `state`), NOT nested under state.
+    const verdict = execution?.verdict ?? null;
+    const budget = execution?.budget ?? null;
+    const evidence = execution?.evidence ?? null;
+    // --- Outcome (verbatim relay, inconclusive on anything unrecognized) ---
+    let outcome;
+    if (opts.outcomeOverride) {
+        outcome = opts.outcomeOverride;
+    }
+    else {
+        // Prefer the explicit verdict; fall back to the legacy per-run outcome
+        // field; NEVER fall back to execution.status. Unknown values → inconclusive.
+        const raw = verdict?.outcome ?? state?.outcome;
+        outcome = typeof raw === 'string' && KNOWN.has(raw) ? raw : 'inconclusive';
+    }
+    const success = outcome === 'pass';
+    // --- Budget (from the response; constant is a fallback only) ---
+    const fallbackBudget = opts.fallbackBudget ?? 0;
+    const stepsBudget = typeof budget?.maxSteps === 'number' ? budget.maxSteps : fallbackBudget;
+    const stepsTaken = typeof budget?.usedSteps === 'number'
+        ? budget.usedSteps
+        : (typeof state?.stepsTaken === 'number' ? state.stepsTaken : 0);
+    const stepsRemaining = Math.max(0, stepsBudget - stepsTaken);
+    const relay = {
+        outcome,
+        success,
+        stepsBudget,
+        stepsTaken,
+        stepsRemaining,
+    };
+    if (!success)
+        relay.failureCategory = outcome;
+    if (typeof verdict?.reason === 'string' && verdict.reason)
+        relay.reason = verdict.reason;
+    if (typeof evidence?.screenshot === 'string' && evidence.screenshot)
+        relay.screenshot = evidence.screenshot;
+    if (Array.isArray(evidence?.actionTrace))
+        relay.actionTrace = evidence.actionTrace;
+    return relay;
+}

package/dist/services/workflows.js

@@ -12,6 +12,22 @@ const POLL_INTERVAL_INITIAL_MS = 1000;
 const POLL_INTERVAL_MAX_MS = 5000;
 const POLL_BACKOFF_MULTIPLIER = 1.5;
 const EXECUTION_TIMEOUT_MS = 10 * 60 * 1000; // 10 min
+/**
+ * Stable dispatch slug for the App Evaluation workflow (bug clka). Dispatch
+ * pins to this slug so a backend display-name change never breaks
+ * check_app_in_browser. Backend contract sentinal-k8x1f.8 exposes the slug on
+ * template results; until that deploy lands the resolver falls back to a
+ * name-substring search (see EVAL_TEMPLATE_NAME_FALLBACK).
+ *
+ * Override with the DEBUGGAI_EVAL_TEMPLATE env var to pin a different slug.
+ */
+export const EVAL_TEMPLATE_SLUG = 'flow/e2es/app-eval';
+/** Interim name keyword used ONLY when the backend hasn't exposed slugs yet. */
+export const EVAL_TEMPLATE_NAME_FALLBACK = 'app evaluation workflow';
+/** The slug the eval-template dispatch pins to (env-overridable). */
+export function getEvalTemplateSlug() {
+    return process.env.DEBUGGAI_EVAL_TEMPLATE || EVAL_TEMPLATE_SLUG;
+}
 export const createWorkflowsService = (tx) => {
     const service = {
         async findTemplateByName(keyword) {
@@ -41,11 +57,37 @@ export const createWorkflowsService = (tx) => {
                 `Available templates: ${seenNames.map(n => `"${n}"`).join(', ')}. ` +
                 `Ensure the backend has a template with "${keyword}" in its name.`);
         },
+        async findTemplateBySlug(slug) {
+            // Pin dispatch to the stable slug (bug clka). We pass `slug` as a query
+            // param so a slug-aware backend can filter server-side; on backends that
+            // ignore it we still walk the pages and match client-side on the `slug`
+            // field. Returns null if NO result carries a slug field (backend hasn't
+            // deployed the slug contract yet) — the caller then falls back to name
+            // search. `next`-paging mirrors findTemplateByName for the same reason
+            // (the backend caps page size).
+            const MAX_PAGES = 50;
+            for (let page = 1; page <= MAX_PAGES; page++) {
+                const response = await tx.get('api/v1/workflows/', { isTemplate: true, slug, page });
+                const templates = response?.results ?? [];
+                for (const t of templates) {
+                    if (t.slug === slug)
+                        return t;
+                }
+                if (!response?.next)
+                    break;
+            }
+            return null;
+        },
         async findEvaluationTemplate() {
-            // 'app evaluation workflow' is specific enough to skip 'App Evaluation Brain'
-            // (subworkflow, no browser lifecycle) which also contains 'app evaluation'.
-            const keyword = process.env.DEBUGGAI_EVAL_TEMPLATE || 'app evaluation workflow';
-            return service.findTemplateByName(keyword);
+            // Primary: resolve by the stable slug so a backend rename can't break us.
+            const slug = getEvalTemplateSlug();
+            const bySlug = await service.findTemplateBySlug(slug);
+            if (bySlug)
+                return bySlug;
+            // Fallback (interim, until backend contract sentinal-k8x1f.8 lands): the
+            // name is specific enough to skip 'App Evaluation Brain' (subworkflow, no
+            // browser lifecycle) which also contains 'app evaluation'.
+            return service.findTemplateByName(EVAL_TEMPLATE_NAME_FALLBACK);
         },
         async executeWorkflow(workflowUuid, contextData, env) {
             const body = { contextData };
@@ -105,11 +147,16 @@ export const createWorkflowsService = (tx) => {
             const pollStart = Date.now();
             let pollCount = 0;
             let intervalMs = POLL_INTERVAL_INITIAL_MS;
+            // Track the most recent observation so a poll-deadline timeout can return
+            // partial results (screenshot + trace) instead of discarding them (bead
+            // 56kd.3).
+            let lastExecution;
             while (Date.now() < deadline) {
                 if (signal?.aborted) {
                     throw new Error(`Polling cancelled for execution ${executionUuid}`);
                 }
                 const execution = await service.getExecution(executionUuid);
+                lastExecution = execution;
                 pollCount++;
                 if (onUpdate) {
                     await onUpdate(execution).catch(() => { });
@@ -147,6 +194,23 @@ export const createWorkflowsService = (tx) => {
                 // past terminal-state achievement on the longest runs.
                 intervalMs = Math.min(Math.round(intervalMs * POLL_BACKOFF_MULTIPLIER), POLL_INTERVAL_MAX_MS);
             }
+            // Deadline hit. Return the last observed execution flagged `timedOut` so
+            // the handler shapes a partial 'timeout' result (bead 56kd.3) rather than
+            // discarding the screenshot + trace we already captured. Only throw if we
+            // never observed anything to shape.
+            if (lastExecution) {
+                Telemetry.capture(TelemetryEvents.WORKFLOW_EXECUTED, {
+                    status: lastExecution.status,
+                    success: false,
+                    outcome: 'timeout',
+                    stepsTaken: lastExecution.state?.stepsTaken ?? 0,
+                    durationMs: Date.now() - pollStart,
+                    pollCount,
+                    finalIntervalMs: intervalMs,
+                    timedOut: true,
+                });
+                return { ...lastExecution, timedOut: true };
+            }
             throw new Error(`Execution ${executionUuid} timed out after ${EXECUTION_TIMEOUT_MS / 1000}s`);
         }
     };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@debugg-ai/debugg-ai-mcp",
-  "version": "3.5.2",
+  "version": "3.5.3",
   "description": "Zero-Config, Fully AI-Managed End-to-End Testing for all code gen platforms.",
   "type": "module",
   "bin": {