donobu 5.35.0 → 5.36.0

package/dist/esm/reporter/render.js

@@ -310,6 +310,18 @@ function extractTests(jsonData) {
                             // Ignore parse failures
                         }
                     }
+                    // Parse AI invocation wrappers from donobu-ai-invocations attachment
+                    let aiInvocations = [];
+                    const aiInvAtt = attachments.find((a) => a.name === 'donobu-ai-invocations');
+                    if (aiInvAtt?.body) {
+                        try {
+                            const decoded = Buffer.from(aiInvAtt.body, 'base64').toString('utf8');
+                            aiInvocations = JSON.parse(decoded);
+                        }
+                        catch {
+                            // Ignore parse failures
+                        }
+                    }
                     return {
                         index: i,
                         status: r.status,
@@ -334,6 +346,7 @@ function extractTests(jsonData) {
                         steps: parseStderrSteps(r.stderr ?? []),
                         stepScreenshots,
                         nativeSteps,
+                        aiInvocations,
                     };
                 });
                 // Extract flow ID from the test-flow-metadata.json attachment
@@ -613,6 +626,107 @@ function renderNativeStep(ns, childrenHtml) {
     html += `</details>`;
     return html;
 }
+const AI_KIND_LABELS = {
+    act: 'page.ai',
+    assert: 'page.ai.assert',
+    locate: 'page.ai.locate',
+};
+/**
+ * Render a single structured assertion step back as the Playwright source
+ * line that effectively executes — e.g. `expect(page.getByRole('heading',
+ * { name: 'Create an account' })).toBeVisible()`. Used to surface in the
+ * report what a cached `page.ai.assert` actually checked.
+ */
+function formatAssertionStep(step) {
+    const quote = (s) => `'${s.replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'`;
+    const matcher = step.valueIsRegex ? `/${step.value}/` : quote(step.value);
+    // Page-level assertions (no element locator)
+    if (step.locator === null) {
+        return `expect(page).${step.assertion}(${matcher})`;
+    }
+    let locatorExpr;
+    if (step.locator === 'role' && step.role) {
+        locatorExpr = `page.getByRole(${quote(step.role)}, { name: ${matcher} })`;
+    }
+    else if (step.locator === 'label') {
+        locatorExpr = `page.getByLabel(${matcher})`;
+    }
+    else {
+        locatorExpr = `page.getByText(${matcher})`;
+    }
+    locatorExpr += '.first()';
+    const attrValue = step.attributeValue ?? '';
+    switch (step.assertion) {
+        case 'toBeVisible':
+        case 'toBeEnabled':
+        case 'toBeDisabled':
+        case 'toBeChecked':
+            return `expect(${locatorExpr}).${step.assertion}()`;
+        case 'toBeHidden':
+            // Executor uses `not.toBeVisible()` for `toBeHidden`; mirror that here.
+            return `expect(${locatorExpr}).not.toBeVisible()`;
+        case 'toHaveValue':
+        case 'toContainText':
+            return `expect(${locatorExpr}).${step.assertion}(${quote(attrValue)})`;
+        case 'toHaveAttribute':
+            return `expect(${locatorExpr}).toHaveAttribute(${quote(step.value)}, ${quote(attrValue)})`;
+        default:
+            return `expect(${locatorExpr}).${step.assertion}(${matcher})`;
+    }
+}
+function renderAiInvocation(inv, childrenHtml) {
+    const statusIcon = inv.passed
+        ? '<span class="step-status-ok">&#10003;</span>'
+        : '<span class="step-status-fail">&#10007;</span>';
+    const kindBadge = `<span class="ai-invocation-badge ai-invocation-badge--${inv.kind}">${esc(AI_KIND_LABELS[inv.kind])}</span>`;
+    const cachedBadge = inv.cacheHit
+        ? '<span class="ai-cached-badge">cached</span>'
+        : '';
+    const hasError = !inv.passed && !!inv.error?.message;
+    const hasAssertSteps = !!inv.assertSteps && inv.assertSteps.length > 0;
+    const hasBody = hasError || !!childrenHtml || hasAssertSteps;
+    const renderHeader = (tag) => {
+        let header = `<${tag} class="filmstrip-header">`;
+        header += statusIcon;
+        header += `<span class="ai-invocation-title">${esc(inv.description)}</span>`;
+        header += kindBadge;
+        header += cachedBadge;
+        if (tag === 'summary') {
+            header +=
+                '<span class="native-step-chevron" aria-hidden="true">&#9656;</span>';
+        }
+        header += `</${tag}>`;
+        return header;
+    };
+    if (!hasBody) {
+        // Leaf row — no children, no error. Common for `page.ai.locate` cache
+        // hits and for any other invocation whose internal work didn't surface
+        // any captured tool calls or native steps.
+        return `<div class="filmstrip-step ai-invocation">${renderHeader('div')}</div>`;
+    }
+    // Failures always render expanded; passing wrappers with children open
+    // by default so the contents are visible without an extra click.
+    const defaultOpen = !inv.passed || !!childrenHtml || hasAssertSteps;
+    const passClass = inv.passed
+        ? 'ai-invocation--passed'
+        : 'ai-invocation--failed';
+    let html = `<details class="filmstrip-step ai-invocation ${passClass}"${defaultOpen ? ' open' : ''}>`;
+    html += renderHeader('summary');
+    if (hasError) {
+        html += `<pre class="native-step-error">${ansiToHtml(inv.error.message)}</pre>`;
+    }
+    if (hasAssertSteps) {
+        const lines = inv
+            .assertSteps.map((s) => esc(formatAssertionStep(s)))
+            .join('\n');
+        html += `<pre class="ai-assert-steps">${lines}</pre>`;
+    }
+    if (childrenHtml) {
+        html += childrenHtml;
+    }
+    html += `</details>`;
+    return html;
+}
 const AUDIT_CHECK_DEFS = [
     {
         key: 'pageLoad',
@@ -852,14 +966,15 @@ function renderFilmstripStep(ss, outputDir) {
     html += `</div>`;
     return html;
 }
-function renderSteps(steps, stepScreenshots, nativeSteps, outputDir) {
+function renderSteps(steps, stepScreenshots, nativeSteps, aiInvocations, outputDir) {
     const meaningful = steps.filter((s) => s.type === 'action' || s.type === 'result');
     const hasScreenshots = stepScreenshots.length > 0;
     const hasNative = nativeSteps.length > 0;
-    if (!meaningful.length && !hasScreenshots && !hasNative) {
+    const hasAi = aiInvocations.length > 0;
+    if (!meaningful.length && !hasScreenshots && !hasNative && !hasAi) {
         return '';
     }
-    if (hasScreenshots || hasNative) {
+    if (hasScreenshots || hasNative || hasAi) {
         const buildNativeTree = (nss) => nss.map((ns) => ({
             kind: 'native',
             ns,
@@ -868,32 +983,53 @@ function renderSteps(steps, stepScreenshots, nativeSteps, outputDir) {
             children: buildNativeTree(ns.children),
         }));
         const roots = buildNativeTree(nativeSteps);
-        // Place each Donobu screenshot under the deepest native step whose
-        // [start, end] window contains it. Falls back to top level if none.
-        const placeDonobu = (nodes, d) => {
+        // Place a node into the deepest container whose [t, tEnd] window
+        // contains its [tStart, tEnd]. Returns true on placement. Both native
+        // steps and AI invocations are eligible parents.
+        const placeNode = (nodes, leaf, tStart, tEnd) => {
             for (const n of nodes) {
-                if (n.kind !== 'native') {
+                if (n.kind !== 'native' && n.kind !== 'ai') {
                     continue;
                 }
-                if (d.ss.startedAt >= n.t && d.ss.completedAt <= n.tEnd) {
-                    if (!placeDonobu(n.children, d)) {
-                        n.children.push(d);
+                if (tStart >= n.t && tEnd <= n.tEnd) {
+                    if (!placeNode(n.children, leaf, tStart, tEnd)) {
+                        n.children.push(leaf);
                     }
                     return true;
                 }
             }
             return false;
         };
+        // AI invocations placed first, longer-window first so an outer cached
+        // `page.ai` is in place before its inner `page.ai.assert` lands.
+        const sortedInvocations = [...aiInvocations].sort((a, b) => b.endedAt - b.startedAt - (a.endedAt - a.startedAt));
+        for (const inv of sortedInvocations) {
+            const node = {
+                kind: 'ai',
+                inv,
+                t: inv.startedAt,
+                tEnd: inv.endedAt,
+                children: [],
+            };
+            if (!placeNode(roots, node, inv.startedAt, inv.endedAt)) {
+                roots.push(node);
+            }
+        }
         for (const ss of stepScreenshots) {
-            const d = { kind: 'donobu', ss, t: ss.startedAt };
-            if (!placeDonobu(roots, d)) {
+            const d = {
+                kind: 'donobu',
+                ss,
+                t: ss.startedAt,
+                tEnd: ss.completedAt,
+            };
+            if (!placeNode(roots, d, ss.startedAt, ss.completedAt)) {
                 roots.push(d);
             }
         }
         const sortTree = (nodes) => {
             nodes.sort((a, b) => a.t - b.t);
             for (const n of nodes) {
-                if (n.kind === 'native') {
+                if (n.kind === 'native' || n.kind === 'ai') {
                     sortTree(n.children);
                 }
             }
@@ -903,7 +1039,7 @@ function renderSteps(steps, stepScreenshots, nativeSteps, outputDir) {
             let c = 0;
             for (const n of nodes) {
                 c += 1;
-                if (n.kind === 'native') {
+                if (n.kind === 'native' || n.kind === 'ai') {
                     c += countNodes(n.children);
                 }
             }
@@ -913,6 +1049,12 @@ function renderSteps(steps, stepScreenshots, nativeSteps, outputDir) {
             if (node.kind === 'donobu') {
                 return renderFilmstripStep(node.ss, outputDir);
             }
+            if (node.kind === 'ai') {
+                const childrenHtml = node.children.length > 0
+                    ? `<div class="native-step-children">${node.children.map(renderNode).join('')}</div>`
+                    : '';
+                return renderAiInvocation(node.inv, childrenHtml);
+            }
             const childrenHtml = node.children.length > 0
                 ? `<div class="native-step-children">${node.children.map(renderNode).join('')}</div>`
                 : '';
@@ -1227,7 +1369,7 @@ function renderResultTimeline(results, outputDir) {
             html += `<div class="timeline-errors">${renderErrors(r.errors)}</div>`;
         }
         html += renderAttachments(r.attachments, outputDir, r.stepScreenshots);
-        html += renderSteps(r.steps, r.stepScreenshots, r.nativeSteps, outputDir);
+        html += renderSteps(r.steps, r.stepScreenshots, r.nativeSteps, r.aiInvocations, outputDir);
         html += '</div></div>';
     }
     html += '</div>';
@@ -1385,7 +1527,7 @@ function renderHtml(report, triage, outputDir) {
         }
         // 6. Steps — detailed forensics
         if (!hasMultipleResults && lastResult) {
-            detailsHtml += renderSteps(lastResult.steps, lastResult.stepScreenshots, lastResult.nativeSteps, outputDir);
+            detailsHtml += renderSteps(lastResult.steps, lastResult.stepScreenshots, lastResult.nativeSteps, lastResult.aiInvocations, outputDir);
         }
         // 7. Triage details — remediation steps (expandable)
         if (test.plan) {
@@ -1659,6 +1801,18 @@ details.native-step[open]>summary .native-step-chevron{transform:rotate(90deg)}
 .native-step-snippet{font-size:11px;font-family:var(--mono);margin:4px 0 2px 22px;overflow:hidden}
 .native-step-children{display:flex;flex-direction:column;margin:4px 0 0 10px;border-left:1px solid var(--border-subtle);padding-left:8px}
 .native-step-children>.filmstrip-step{padding-left:8px}
+
+/* AI invocation wrappers — page.ai / page.ai.assert / page.ai.locate */
+details.ai-invocation>summary{list-style:none;cursor:pointer}
+details.ai-invocation>summary::-webkit-details-marker{display:none}
+.ai-invocation-title{font-size:12px;font-weight:500;color:var(--text);font-family:var(--mono);flex:1;min-width:0;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
+.ai-invocation-badge{font-size:10px;font-weight:600;padding:1px 5px;border-radius:3px;white-space:nowrap;flex-shrink:0;font-family:var(--mono)}
+.ai-invocation-badge--act{background:rgba(168,85,247,.12);color:#c084fc}
+.ai-invocation-badge--assert{background:rgba(236,72,153,.12);color:#f472b6}
+.ai-invocation-badge--locate{background:rgba(59,130,246,.12);color:#60a5fa}
+.ai-cached-badge{font-size:10px;font-weight:600;padding:1px 5px;border-radius:3px;white-space:nowrap;flex-shrink:0;background:rgba(245,158,11,.12);color:#fbbf24}
+details.ai-invocation[open]>summary .native-step-chevron{transform:rotate(90deg)}
+.ai-assert-steps{font-size:11px;font-family:var(--mono);background:var(--bg);border:1px solid var(--border-subtle);border-radius:var(--radius);padding:8px 12px;margin:6px 0 2px 22px;color:var(--text-muted);white-space:pre-wrap;word-break:break-word;overflow-x:auto;max-height:240px;overflow-y:auto}
 .snippet-line{display:flex;padding:1px 8px;white-space:pre}
 .snippet-line--target{background:rgba(239,68,68,.10)}
 .snippet-linenum{color:var(--text-dim);min-width:40px;user-select:none}

package/dist/lib/ai/PageAi.js

@@ -148,52 +148,75 @@ class PageAi {
         return new PageAi(donobu, gptClient, new cache_1.InMemoryPageAiCache());
     }
     async ai(page, instruction, options) {
-        const descriptor = this.buildDescriptor(page, instruction, options);
-        // Keep the per-page metadata in sync with the env vars needed for this invocation so cached
-        // replays can resolve interpolations via runTool.
-        page._dnb.donobuFlowMetadata.envVars = descriptor.envVarNames;
-        const cachedEntry = descriptor.useCache
-            ? await this.cache.get(descriptor.key)
-            : null;
-        if (cachedEntry) {
-            page._dnb.donobuFlowMetadata.runMode = 'DETERMINISTIC';
-            page._dnb.envVals = descriptor.envVals;
-            try {
-                await cachedEntry.run({ page });
+        const startedAt = Date.now();
+        let cacheHit = false;
+        let thrownError = undefined;
+        try {
+            const descriptor = this.buildDescriptor(page, instruction, options);
+            // Keep the per-page metadata in sync with the env vars needed for this invocation so cached
+            // replays can resolve interpolations via runTool.
+            page._dnb.donobuFlowMetadata.envVars = descriptor.envVarNames;
+            const cachedEntry = descriptor.useCache
+                ? await this.cache.get(descriptor.key)
+                : null;
+            cacheHit = !!cachedEntry;
+            if (cachedEntry) {
+                page._dnb.donobuFlowMetadata.runMode = 'DETERMINISTIC';
+                page._dnb.envVals = descriptor.envVals;
+                try {
+                    await cachedEntry.run({ page });
+                }
+                finally {
+                    page._dnb.envVals = undefined;
+                }
+                return this.synthesizeResultFromMetadata(page, instruction, descriptor, options);
             }
-            finally {
-                page._dnb.envVals = undefined;
+            else {
+                const runResult = await this.runner.run({
+                    page,
+                    instruction,
+                    schema: descriptor.schema,
+                    jsonSchema: descriptor.jsonSchema,
+                    allowedTools: descriptor.allowedTools,
+                    maxToolCalls: descriptor.maxToolCalls,
+                    envVarNames: descriptor.envVarNames,
+                    envVals: descriptor.envVals,
+                    runMode: 'AUTONOMOUS',
+                    gptClient: options?.gptClient,
+                });
+                if (descriptor.useCache) {
+                    const preparedToolCalls = await (0, DonobuFlowsManager_1.prepareToolCallsForRerun)(
+                    // Only retain successfully run tool calls, otherwise when a cache file
+                    // with some bad calls in it runs in the future, the test will blow up
+                    // when the first bad tool call is read.
+                    runResult.donobuFlow.invokedToolCalls.filter((tc) => {
+                        return tc.outcome.isSuccessful;
+                    }), {
+                        areElementIdsVolatile: options?.volatileElementIds,
+                        disableSelectorFailover: options?.noSelectorFailover,
+                    }, this.donobu.toolRegistry);
+                    const cacheEntry = cacheEntryBuilder_1.PageAiCacheEntryBuilder.fromMetadata(descriptor.key.pageUrl, runResult.donobuFlow.metadata, preparedToolCalls);
+                    await this.cache.put(cacheEntry);
+                }
+                return runResult.parsedResult;
             }
-            return this.synthesizeResultFromMetadata(page, instruction, descriptor, options);
         }
-        else {
-            const runResult = await this.runner.run({
-                page,
-                instruction,
-                schema: descriptor.schema,
-                jsonSchema: descriptor.jsonSchema,
-                allowedTools: descriptor.allowedTools,
-                maxToolCalls: descriptor.maxToolCalls,
-                envVarNames: descriptor.envVarNames,
-                envVals: descriptor.envVals,
-                runMode: 'AUTONOMOUS',
-                gptClient: options?.gptClient,
+        catch (e) {
+            thrownError = e;
+            throw e;
+        }
+        finally {
+            page._dnb.aiInvocations.push({
+                kind: 'act',
+                description: instruction,
+                startedAt,
+                endedAt: Date.now(),
+                cacheHit,
+                passed: thrownError === undefined,
+                error: thrownError !== undefined
+                    ? { message: thrownError?.message }
+                    : undefined,
             });
-            if (descriptor.useCache) {
-                const preparedToolCalls = await (0, DonobuFlowsManager_1.prepareToolCallsForRerun)(
-                // Only retain successfully run tool calls, otherwise when a cache file
-                // with some bad calls in it runs in the future, the test will blow up
-                // when the first bad tool call is read.
-                runResult.donobuFlow.invokedToolCalls.filter((tc) => {
-                    return tc.outcome.isSuccessful;
-                }), {
-                    areElementIdsVolatile: options?.volatileElementIds,
-                    disableSelectorFailover: options?.noSelectorFailover,
-                }, this.donobu.toolRegistry);
-                const cacheEntry = cacheEntryBuilder_1.PageAiCacheEntryBuilder.fromMetadata(descriptor.key.pageUrl, runResult.donobuFlow.metadata, preparedToolCalls);
-                await this.cache.put(cacheEntry);
-            }
-            return runResult.parsedResult;
         }
     }
     /**

package/dist/lib/page/DonobuExtendedPage.d.ts

@@ -13,6 +13,7 @@ import type { FlowsPersistence } from '../../persistence/flows/FlowsPersistence'
 import type { TestsPersistence } from '../../persistence/tests/TestsPersistence';
 import type { CookieAnalyses } from '../../tools/CreateBrowserCookieReportTool';
 import type { AccessibilityResults } from '../../tools/RunAccessibilityTestTool';
+import type { PlaywrightAssertionStep } from '../ai/cache/assertCache';
 import type { PageAiCache } from '../ai/cache/cache';
 import type { LocateOptions } from '../ai/locate/locateTypes';
 import type { PageAi, PageAiNoSchemaOptions, PageAiOptions, PageAiSchemaOptions } from '../ai/PageAi';
@@ -466,6 +467,38 @@ export interface DonobuExtendedPage extends Page {
         envVals?: Record<string, string | undefined>;
         /** Sessions recorded by {@link tbd} for post-test code generation. */
         tbdSessions: TbdSession[];
+        /**
+         * Wrapping records for every `page.ai`, `page.ai.assert`, and
+         * `page.ai.locate` invocation in this test. The HTML reporter renders
+         * each as a parent node containing whichever Donobu tool calls and
+         * native Playwright steps fell inside its time window, with a
+         * `[cached]` badge driven by the per-record `cacheHit` flag.
+         *
+         * Recording happens for ALL calls (cache hit or miss) so the wrapper
+         * is visible regardless. Nested AI calls (e.g. a cached `page.ai`
+         * whose runSource calls `page.ai.assert(...)`) become nested wrappers
+         * — each carries its own cache state.
+         */
+        aiInvocations: AiInvocationRecord[];
     };
 }
+export interface AiInvocationRecord {
+    kind: 'act' | 'assert' | 'locate';
+    description: string;
+    startedAt: number;
+    endedAt: number;
+    cacheHit: boolean;
+    passed: boolean;
+    error?: {
+        message?: string;
+    };
+    /**
+     * For cached `page.ai.assert` invocations: the structured Playwright
+     * assertion steps that were replayed. The reporter formats these back
+     * into source-code lines so the report shows exactly what was checked
+     * (e.g. `expect(page.getByRole('heading', { name: '…' })).toBeVisible()`).
+     * Undefined for live assert runs, `act`, and `locate` records.
+     */
+    assertSteps?: PlaywrightAssertionStep[];
+}
 //# sourceMappingURL=DonobuExtendedPage.d.ts.map