donobu 5.35.1 → 5.36.1

package/dist/esm/lib/ai/PageAi.js

@@ -148,52 +148,75 @@ class PageAi {
         return new PageAi(donobu, gptClient, new cache_1.InMemoryPageAiCache());
     }
     async ai(page, instruction, options) {
-        const descriptor = this.buildDescriptor(page, instruction, options);
-        // Keep the per-page metadata in sync with the env vars needed for this invocation so cached
-        // replays can resolve interpolations via runTool.
-        page._dnb.donobuFlowMetadata.envVars = descriptor.envVarNames;
-        const cachedEntry = descriptor.useCache
-            ? await this.cache.get(descriptor.key)
-            : null;
-        if (cachedEntry) {
-            page._dnb.donobuFlowMetadata.runMode = 'DETERMINISTIC';
-            page._dnb.envVals = descriptor.envVals;
-            try {
-                await cachedEntry.run({ page });
+        const startedAt = Date.now();
+        let cacheHit = false;
+        let thrownError = undefined;
+        try {
+            const descriptor = this.buildDescriptor(page, instruction, options);
+            // Keep the per-page metadata in sync with the env vars needed for this invocation so cached
+            // replays can resolve interpolations via runTool.
+            page._dnb.donobuFlowMetadata.envVars = descriptor.envVarNames;
+            const cachedEntry = descriptor.useCache
+                ? await this.cache.get(descriptor.key)
+                : null;
+            cacheHit = !!cachedEntry;
+            if (cachedEntry) {
+                page._dnb.donobuFlowMetadata.runMode = 'DETERMINISTIC';
+                page._dnb.envVals = descriptor.envVals;
+                try {
+                    await cachedEntry.run({ page });
+                }
+                finally {
+                    page._dnb.envVals = undefined;
+                }
+                return this.synthesizeResultFromMetadata(page, instruction, descriptor, options);
             }
-            finally {
-                page._dnb.envVals = undefined;
+            else {
+                const runResult = await this.runner.run({
+                    page,
+                    instruction,
+                    schema: descriptor.schema,
+                    jsonSchema: descriptor.jsonSchema,
+                    allowedTools: descriptor.allowedTools,
+                    maxToolCalls: descriptor.maxToolCalls,
+                    envVarNames: descriptor.envVarNames,
+                    envVals: descriptor.envVals,
+                    runMode: 'AUTONOMOUS',
+                    gptClient: options?.gptClient,
+                });
+                if (descriptor.useCache) {
+                    const preparedToolCalls = await (0, DonobuFlowsManager_1.prepareToolCallsForRerun)(
+                    // Only retain successfully run tool calls, otherwise when a cache file
+                    // with some bad calls in it runs in the future, the test will blow up
+                    // when the first bad tool call is read.
+                    runResult.donobuFlow.invokedToolCalls.filter((tc) => {
+                        return tc.outcome.isSuccessful;
+                    }), {
+                        areElementIdsVolatile: options?.volatileElementIds,
+                        disableSelectorFailover: options?.noSelectorFailover,
+                    }, this.donobu.toolRegistry);
+                    const cacheEntry = cacheEntryBuilder_1.PageAiCacheEntryBuilder.fromMetadata(descriptor.key.pageUrl, runResult.donobuFlow.metadata, preparedToolCalls);
+                    await this.cache.put(cacheEntry);
+                }
+                return runResult.parsedResult;
             }
-            return this.synthesizeResultFromMetadata(page, instruction, descriptor, options);
         }
-        else {
-            const runResult = await this.runner.run({
-                page,
-                instruction,
-                schema: descriptor.schema,
-                jsonSchema: descriptor.jsonSchema,
-                allowedTools: descriptor.allowedTools,
-                maxToolCalls: descriptor.maxToolCalls,
-                envVarNames: descriptor.envVarNames,
-                envVals: descriptor.envVals,
-                runMode: 'AUTONOMOUS',
-                gptClient: options?.gptClient,
+        catch (e) {
+            thrownError = e;
+            throw e;
+        }
+        finally {
+            page._dnb.aiInvocations.push({
+                kind: 'act',
+                description: instruction,
+                startedAt,
+                endedAt: Date.now(),
+                cacheHit,
+                passed: thrownError === undefined,
+                error: thrownError !== undefined
+                    ? { message: thrownError?.message }
+                    : undefined,
             });
-            if (descriptor.useCache) {
-                const preparedToolCalls = await (0, DonobuFlowsManager_1.prepareToolCallsForRerun)(
-                // Only retain successfully run tool calls, otherwise when a cache file
-                // with some bad calls in it runs in the future, the test will blow up
-                // when the first bad tool call is read.
-                runResult.donobuFlow.invokedToolCalls.filter((tc) => {
-                    return tc.outcome.isSuccessful;
-                }), {
-                    areElementIdsVolatile: options?.volatileElementIds,
-                    disableSelectorFailover: options?.noSelectorFailover,
-                }, this.donobu.toolRegistry);
-                const cacheEntry = cacheEntryBuilder_1.PageAiCacheEntryBuilder.fromMetadata(descriptor.key.pageUrl, runResult.donobuFlow.metadata, preparedToolCalls);
-                await this.cache.put(cacheEntry);
-            }
-            return runResult.parsedResult;
         }
     }
     /**

package/dist/esm/lib/page/DonobuExtendedPage.d.ts

@@ -13,6 +13,7 @@ import type { FlowsPersistence } from '../../persistence/flows/FlowsPersistence'
 import type { TestsPersistence } from '../../persistence/tests/TestsPersistence';
 import type { CookieAnalyses } from '../../tools/CreateBrowserCookieReportTool';
 import type { AccessibilityResults } from '../../tools/RunAccessibilityTestTool';
+import type { PlaywrightAssertionStep } from '../ai/cache/assertCache';
 import type { PageAiCache } from '../ai/cache/cache';
 import type { LocateOptions } from '../ai/locate/locateTypes';
 import type { PageAi, PageAiNoSchemaOptions, PageAiOptions, PageAiSchemaOptions } from '../ai/PageAi';
@@ -466,6 +467,38 @@ export interface DonobuExtendedPage extends Page {
         envVals?: Record<string, string | undefined>;
         /** Sessions recorded by {@link tbd} for post-test code generation. */
         tbdSessions: TbdSession[];
+        /**
+         * Wrapping records for every `page.ai`, `page.ai.assert`, and
+         * `page.ai.locate` invocation in this test. The HTML reporter renders
+         * each as a parent node containing whichever Donobu tool calls and
+         * native Playwright steps fell inside its time window, with a
+         * `[cached]` badge driven by the per-record `cacheHit` flag.
+         *
+         * Recording happens for ALL calls (cache hit or miss) so the wrapper
+         * is visible regardless. Nested AI calls (e.g. a cached `page.ai`
+         * whose runSource calls `page.ai.assert(...)`) become nested wrappers
+         * — each carries its own cache state.
+         */
+        aiInvocations: AiInvocationRecord[];
     };
 }
+export interface AiInvocationRecord {
+    kind: 'act' | 'assert' | 'locate';
+    description: string;
+    startedAt: number;
+    endedAt: number;
+    cacheHit: boolean;
+    passed: boolean;
+    error?: {
+        message?: string;
+    };
+    /**
+     * For cached `page.ai.assert` invocations: the structured Playwright
+     * assertion steps that were replayed. The reporter formats these back
+     * into source-code lines so the report shows exactly what was checked
+     * (e.g. `expect(page.getByRole('heading', { name: '…' })).toBeVisible()`).
+     * Undefined for live assert runs, `act`, and `locate` records.
+     */
+    assertSteps?: PlaywrightAssertionStep[];
+}
 //# sourceMappingURL=DonobuExtendedPage.d.ts.map

package/dist/esm/lib/page/extendPage.js

@@ -137,6 +137,7 @@ async function extendPage(page, options) {
                 clearPageAiCache: MiscUtils_1.MiscUtils.yn(envVars_1.env.data.DONOBU_PAGE_AI_CLEAR_CACHE),
             },
             tbdSessions: [],
+            aiInvocations: [],
         };
         const showMouse = async (p) => {
             if (interactionVisualizer.defaultMessageDurationMillis > 0) {
@@ -217,120 +218,146 @@ Valid options:
     const pageAi = Object.assign(act, {
         act,
         assert: async (assertion, options) => {
-            const useCache = options?.cache !== false;
-            const clearCache = sharedState.runtimeDirectives?.clearPageAiCache ?? false;
-            const retries = options?.retries ?? 0;
-            const retryDelaySeconds = options?.retryDelaySeconds ?? 3;
-            // Distill env var names from `{{$.env.*}}` interpolations in the
-            // assertion plus any explicitly provided names/overrides. Cached
-            // Playwright steps may carry the same `{{$.env.X}}` placeholders in
-            // their `value`/`attributeValue` fields, so we resolve env data at
-            // replay time and let the executor interpolate before applying.
-            const envVarNames = (0, DonobuFlowsManager_1.distillAllowedEnvVariableNames)(assertion, [
-                ...(options?.envVars ?? []),
-                ...Object.keys(options?.envVals ?? {}),
-            ]);
-            const hasEnvRefs = envVarNames.length > 0;
-            const resolveEnvData = async () => {
-                if (!hasEnvRefs) {
-                    return undefined;
-                }
-                const envData = await sharedState.donobuStack.envDataManager.getByNames(envVarNames);
-                if (options?.envVals) {
-                    for (const [k, v] of Object.entries(options.envVals)) {
-                        if (v === undefined) {
-                            delete envData[k];
+            const aiInvocationStartedAt = Date.now();
+            let aiInvocationCacheHit = false;
+            let aiInvocationError = undefined;
+            let aiInvocationAssertSteps;
+            try {
+                const useCache = options?.cache !== false;
+                const clearCache = sharedState.runtimeDirectives?.clearPageAiCache ?? false;
+                const retries = options?.retries ?? 0;
+                const retryDelaySeconds = options?.retryDelaySeconds ?? 3;
+                // Distill env var names from `{{$.env.*}}` interpolations in the
+                // assertion plus any explicitly provided names/overrides. Cached
+                // Playwright steps may carry the same `{{$.env.X}}` placeholders in
+                // their `value`/`attributeValue` fields, so we resolve env data at
+                // replay time and let the executor interpolate before applying.
+                const envVarNames = (0, DonobuFlowsManager_1.distillAllowedEnvVariableNames)(assertion, [
+                    ...(options?.envVars ?? []),
+                    ...Object.keys(options?.envVals ?? {}),
+                ]);
+                const hasEnvRefs = envVarNames.length > 0;
+                const resolveEnvData = async () => {
+                    if (!hasEnvRefs) {
+                        return undefined;
+                    }
+                    const envData = await sharedState.donobuStack.envDataManager.getByNames(envVarNames);
+                    if (options?.envVals) {
+                        for (const [k, v] of Object.entries(options.envVals)) {
+                            if (v === undefined) {
+                                delete envData[k];
+                            }
+                            else {
+                                envData[k] = v;
+                            }
                         }
-                        else {
-                            envData[k] = v;
+                    }
+                    return envData;
+                };
+                // --- Cache lookup (when enabled and not clearing) ---
+                if (useCache && !clearCache) {
+                    const cache = getOrInitPageAiCache();
+                    const pageUrl = (0, cacheLocator_1.extractCacheKeyHostname)(page.url());
+                    const cached = await cache.getAssert({ pageUrl, assertion });
+                    if (cached) {
+                        aiInvocationCacheHit = true;
+                        aiInvocationAssertSteps = cached.steps;
+                        Logger_1.appLogger.debug(`Assert cache HIT for: "${assertion}" - running cached Playwright assertion`);
+                        const envData = await resolveEnvData();
+                        let lastError = null;
+                        for (let attempt = 0; attempt <= retries; attempt++) {
+                            if (attempt > 0) {
+                                Logger_1.appLogger.info(`Retry ${attempt} of ${retries} for cached assert`);
+                                await page.waitForTimeout(retryDelaySeconds * 1000);
+                            }
+                            try {
+                                await cached.run({ page, envData });
+                                return; // Assertion passed
+                            }
+                            catch (error) {
+                                lastError = error;
+                            }
                         }
+                        // All retry attempts exhausted
+                        throw new ToolCallFailedException_1.ToolCallFailedException(AssertTool_1.AssertTool.NAME, {
+                            isSuccessful: false,
+                            // Strip ANSI: Playwright matchers style their messages for the
+                            // terminal, but this string flows into JSON-stringified exception
+                            // messages, the LLM, and HTML/markdown reports — places where the
+                            // codes never render and just become visible junk.
+                            forLlm: `Assertion FAILED (cached) for: ${assertion}\nPlaywright Error: ${(0, ansi_1.stripAnsi)(lastError?.message ?? '')}`,
+                            metadata: {
+                                cached: true,
+                                steps: cached.steps,
+                            },
+                        });
                     }
                 }
-                return envData;
-            };
-            // --- Cache lookup (when enabled and not clearing) ---
-            if (useCache && !clearCache) {
-                const cache = getOrInitPageAiCache();
-                const pageUrl = (0, cacheLocator_1.extractCacheKeyHostname)(page.url());
-                const cached = await cache.getAssert({ pageUrl, assertion });
-                if (cached) {
-                    Logger_1.appLogger.debug(`Assert cache HIT for: "${assertion}" - running cached Playwright assertion`);
-                    const envData = await resolveEnvData();
-                    let lastError = null;
-                    for (let attempt = 0; attempt <= retries; attempt++) {
-                        if (attempt > 0) {
-                            Logger_1.appLogger.info(`Retry ${attempt} of ${retries} for cached assert`);
-                            await page.waitForTimeout(retryDelaySeconds * 1000);
-                        }
+                // --- Cache invalidation (when clearing) ---
+                if (useCache && clearCache) {
+                    const cache = getOrInitPageAiCache();
+                    const pageUrl = (0, cacheLocator_1.extractCacheKeyHostname)(page.url());
+                    await cache.deleteAssert({ pageUrl, assertion });
+                    Logger_1.appLogger.debug(`Assert cache invalidated for: "${assertion}"`);
+                }
+                // Make env vars available to runTool's envData for `{{$.env.*}}`
+                // interpolation inside `assertionToTestFor` and so AssertTool can
+                // instruct the AI to emit placeholders in cached step values. Mirrors
+                // PageAi.ai for `act`: metadata.envVars is set (overwriting), envVals
+                // is restored.
+                if (hasEnvRefs) {
+                    sharedState.donobuFlowMetadata.envVars = envVarNames;
+                }
+                const previousEnvVals = sharedState.envVals;
+                sharedState.envVals = options?.envVals;
+                let result;
+                try {
+                    // --- Cache miss or cache disabled: run AI assertion ---
+                    result = await runTool(page, AssertTool_1.AssertTool.NAME, {
+                        assertionToTestFor: assertion,
+                        retries: options?.retries,
+                        retryWaitSeconds: options?.retryDelaySeconds,
+                    }, options?.gptClient);
+                }
+                finally {
+                    sharedState.envVals = previousEnvVals;
+                }
+                if (!result.outcome.isSuccessful) {
+                    throw new ToolCallFailedException_1.ToolCallFailedException(AssertTool_1.AssertTool.NAME, result.outcome);
+                }
+                // --- Cache the Playwright assertion for future runs ---
+                if (useCache) {
+                    const steps = result.outcome.metadata?.playwrightAssertionSteps;
+                    if (Array.isArray(steps) && steps.length > 0) {
                         try {
-                            await cached.run({ page, envData });
-                            return; // Assertion passed
+                            const cache = getOrInitPageAiCache();
+                            const pageUrl = (0, cacheLocator_1.extractCacheKeyHostname)(page.url());
+                            await cache.putAssert({ pageUrl, assertion, steps });
+                            Logger_1.appLogger.debug(`Assert cache STORED for: "${assertion}"`);
                         }
                         catch (error) {
-                            lastError = error;
+                            Logger_1.appLogger.debug(`Skipping assert cache for: "${assertion}" - failed to persist: ${error.message}`);
                         }
                     }
-                    // All retry attempts exhausted
-                    throw new ToolCallFailedException_1.ToolCallFailedException(AssertTool_1.AssertTool.NAME, {
-                        isSuccessful: false,
-                        // Strip ANSI: Playwright matchers style their messages for the
-                        // terminal, but this string flows into JSON-stringified exception
-                        // messages, the LLM, and HTML/markdown reports — places where the
-                        // codes never render and just become visible junk.
-                        forLlm: `Assertion FAILED (cached) for: ${assertion}\nPlaywright Error: ${(0, ansi_1.stripAnsi)(lastError?.message ?? '')}`,
-                        metadata: {
-                            cached: true,
-                            steps: cached.steps,
-                        },
-                    });
                 }
             }
-            // --- Cache invalidation (when clearing) ---
-            if (useCache && clearCache) {
-                const cache = getOrInitPageAiCache();
-                const pageUrl = (0, cacheLocator_1.extractCacheKeyHostname)(page.url());
-                await cache.deleteAssert({ pageUrl, assertion });
-                Logger_1.appLogger.debug(`Assert cache invalidated for: "${assertion}"`);
-            }
-            // Make env vars available to runTool's envData for `{{$.env.*}}`
-            // interpolation inside `assertionToTestFor` and so AssertTool can
-            // instruct the AI to emit placeholders in cached step values. Mirrors
-            // PageAi.ai for `act`: metadata.envVars is set (overwriting), envVals
-            // is restored.
-            if (hasEnvRefs) {
-                sharedState.donobuFlowMetadata.envVars = envVarNames;
-            }
-            const previousEnvVals = sharedState.envVals;
-            sharedState.envVals = options?.envVals;
-            let result;
-            try {
-                // --- Cache miss or cache disabled: run AI assertion ---
-                result = await runTool(page, AssertTool_1.AssertTool.NAME, {
-                    assertionToTestFor: assertion,
-                    retries: options?.retries,
-                    retryWaitSeconds: options?.retryDelaySeconds,
-                }, options?.gptClient);
+            catch (e) {
+                aiInvocationError = e;
+                throw e;
             }
             finally {
-                sharedState.envVals = previousEnvVals;
-            }
-            if (!result.outcome.isSuccessful) {
-                throw new ToolCallFailedException_1.ToolCallFailedException(AssertTool_1.AssertTool.NAME, result.outcome);
-            }
-            // --- Cache the Playwright assertion for future runs ---
-            if (useCache) {
-                const steps = result.outcome.metadata?.playwrightAssertionSteps;
-                if (Array.isArray(steps) && steps.length > 0) {
-                    try {
-                        const cache = getOrInitPageAiCache();
-                        const pageUrl = (0, cacheLocator_1.extractCacheKeyHostname)(page.url());
-                        await cache.putAssert({ pageUrl, assertion, steps });
-                        Logger_1.appLogger.debug(`Assert cache STORED for: "${assertion}"`);
-                    }
-                    catch (error) {
-                        Logger_1.appLogger.debug(`Skipping assert cache for: "${assertion}" - failed to persist: ${error.message}`);
-                    }
-                }
+                sharedState.aiInvocations.push({
+                    kind: 'assert',
+                    description: assertion,
+                    startedAt: aiInvocationStartedAt,
+                    endedAt: Date.now(),
+                    cacheHit: aiInvocationCacheHit,
+                    passed: aiInvocationError === undefined,
+                    error: aiInvocationError !== undefined
+                        ? { message: aiInvocationError?.message }
+                        : undefined,
+                    assertSteps: aiInvocationAssertSteps,
+                });
             }
         },
         extract: async (schema, options) => {
@@ -405,6 +432,9 @@ Use this information to return an appropriate JSON object.`,
             return result.metadata;
         },
         locate: async (description, options) => {
+            const aiInvocationStartedAt = Date.now();
+            let aiInvocationCacheHit = false;
+            let aiInvocationError = undefined;
             const useCache = options?.cache !== false;
             const clearCache = sharedState.runtimeDirectives?.clearPageAiCache ?? false;
             const pageUrl = (0, cacheLocator_1.extractCacheKeyHostname)(page.url());
@@ -464,6 +494,7 @@ Use this information to return an appropriate JSON object.`,
                             });
                             Logger_1.appLogger.debug(`Locate cache HIT for: "${description}" — rebuilt locator from cache`);
                             Logger_1.appLogger.info(`Located: ${candidate}`);
+                            aiInvocationCacheHit = true;
                             return candidate;
                         }
                         catch {
@@ -503,7 +534,22 @@ Use this information to return an appropriate JSON object.`,
                 Logger_1.appLogger.info(`Located: ${locator}`);
                 return locator;
             }
+            catch (e) {
+                aiInvocationError = e;
+                throw e;
+            }
             finally {
+                sharedState.aiInvocations.push({
+                    kind: 'locate',
+                    description,
+                    startedAt: aiInvocationStartedAt,
+                    endedAt: Date.now(),
+                    cacheHit: aiInvocationCacheHit,
+                    passed: aiInvocationError === undefined,
+                    error: aiInvocationError !== undefined
+                        ? { message: aiInvocationError?.message }
+                        : undefined,
+                });
                 clearTimeout(timeoutId);
             }
         },

package/dist/esm/lib/test/testExtension.js

@@ -877,6 +877,24 @@ async function finalizeTest(page, testInfo, logBuffer, videoOption) {
         catch {
             // Non-fatal: native step collection failing must not affect the test result.
         }
+        // Attach AI invocation wrappers (page.ai / page.ai.assert / page.ai.locate)
+        // so the HTML reporter can render each as a parent node containing the
+        // tool calls and native steps that fell inside its time window. The
+        // `cacheHit` flag drives the `[cached]` badge on the wrapper itself —
+        // not on inner actions, since a cached `page.ai` may legitimately invoke
+        // a live `page.ai.assert` and vice versa.
+        try {
+            const aiInvocations = sharedState.aiInvocations;
+            if (aiInvocations.length > 0) {
+                await testInfo.attach('donobu-ai-invocations', {
+                    body: JSON.stringify(aiInvocations),
+                    contentType: 'application/json',
+                });
+            }
+        }
+        catch {
+            // Non-fatal.
+        }
         const browserState = await BrowserUtils_1.BrowserUtils.getBrowserStorageState(page.context());
         await sharedState.persistence.setBrowserState(sharedState.donobuFlowMetadata.id, browserState);
     }