donobu 5.32.0 → 5.34.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/esm/lib/ai/cache/assertCache.d.ts +7 -0
- package/dist/esm/lib/ai/cache/assertCache.js +23 -6
- package/dist/esm/lib/page/DonobuExtendedPage.d.ts +24 -0
- package/dist/esm/lib/page/extendPage.js +52 -7
- package/dist/esm/lib/test/testExtension.js +49 -8
- package/dist/esm/managers/ToolManager.js +3 -0
- package/dist/esm/models/ToolCallContext.d.ts +11 -0
- package/dist/esm/reporter/render.js +76 -27
- package/dist/esm/tools/AssertTool.js +32 -2
- package/dist/lib/ai/cache/assertCache.d.ts +7 -0
- package/dist/lib/ai/cache/assertCache.js +23 -6
- package/dist/lib/page/DonobuExtendedPage.d.ts +24 -0
- package/dist/lib/page/extendPage.js +52 -7
- package/dist/lib/test/testExtension.js +49 -8
- package/dist/managers/ToolManager.js +3 -0
- package/dist/models/ToolCallContext.d.ts +11 -0
- package/dist/reporter/render.js +76 -27
- package/dist/tools/AssertTool.js +32 -2
- package/package.json +1 -1
|
@@ -33,6 +33,13 @@ export declare const PlaywrightAssertionStepSchema: z.ZodObject<{
|
|
|
33
33
|
export type PlaywrightAssertionStep = z.infer<typeof PlaywrightAssertionStepSchema>;
|
|
34
34
|
export type AssertCacheExecutor = (context: {
|
|
35
35
|
page: Page;
|
|
36
|
+
/**
|
|
37
|
+
* Optional env mapping used to interpolate `{{$.env.X}}` placeholders that
|
|
38
|
+
* the AI may have embedded into step `value`/`attributeValue` fields. When
|
|
39
|
+
* absent, steps run unchanged (backwards compatible with cache entries
|
|
40
|
+
* recorded before env-aware caching).
|
|
41
|
+
*/
|
|
42
|
+
envData?: Record<string, string>;
|
|
36
43
|
}) => Promise<void>;
|
|
37
44
|
/**
|
|
38
45
|
* Builds an executor function from structured assertion steps.
|
|
@@ -5,6 +5,7 @@ exports.buildAssertExecutor = buildAssertExecutor;
|
|
|
5
5
|
exports.buildLocateExecutor = buildLocateExecutor;
|
|
6
6
|
const test_1 = require("@playwright/test");
|
|
7
7
|
const v4_1 = require("zod/v4");
|
|
8
|
+
const TemplateInterpolator_1 = require("../../../utils/TemplateInterpolator");
|
|
8
9
|
const buildLocator_1 = require("../locate/buildLocator");
|
|
9
10
|
// ---------------------------------------------------------------------------
|
|
10
11
|
// Structured assertion step schema
|
|
@@ -84,17 +85,33 @@ Common roles: 'heading', 'button', 'link', 'tab', 'tabpanel', 'dialog', 'navigat
|
|
|
84
85
|
- toContainText: set to the text substring to match within the element
|
|
85
86
|
- All other assertions: set to null`),
|
|
86
87
|
});
|
|
88
|
+
/**
|
|
89
|
+
* Resolves any `{{$.env.X}}` placeholders in a step field against the
|
|
90
|
+
* supplied env data. Returns the input verbatim when no env data is given,
|
|
91
|
+
* preserving backwards compatibility with cached entries that contain
|
|
92
|
+
* literal values only.
|
|
93
|
+
*/
|
|
94
|
+
function resolveStepField(value, envData) {
|
|
95
|
+
if (!envData || !value.includes('{{')) {
|
|
96
|
+
return value;
|
|
97
|
+
}
|
|
98
|
+
return (0, TemplateInterpolator_1.interpolateString)(value, { env: envData, calls: [] });
|
|
99
|
+
}
|
|
87
100
|
/**
|
|
88
101
|
* Builds an executor function from structured assertion steps.
|
|
89
102
|
* Each step maps to exactly one Playwright `expect` call — no string
|
|
90
103
|
* evaluation, no VM contexts.
|
|
91
104
|
*/
|
|
92
105
|
function buildAssertExecutor(steps) {
|
|
93
|
-
return async ({ page }) => {
|
|
106
|
+
return async ({ page, envData }) => {
|
|
94
107
|
for (const step of steps) {
|
|
108
|
+
const resolvedValue = resolveStepField(step.value, envData);
|
|
109
|
+
const resolvedAttrValue = step.attributeValue === null
|
|
110
|
+
? null
|
|
111
|
+
: resolveStepField(step.attributeValue, envData);
|
|
95
112
|
const matcher = step.valueIsRegex
|
|
96
|
-
? new RegExp(
|
|
97
|
-
:
|
|
113
|
+
? new RegExp(resolvedValue)
|
|
114
|
+
: resolvedValue;
|
|
98
115
|
// Page-level assertions (no element locator needed)
|
|
99
116
|
if (step.assertion === 'toHaveTitle') {
|
|
100
117
|
await (0, test_1.expect)(page).toHaveTitle(matcher);
|
|
@@ -138,13 +155,13 @@ function buildAssertExecutor(steps) {
|
|
|
138
155
|
await (0, test_1.expect)(locator).toBeChecked();
|
|
139
156
|
break;
|
|
140
157
|
case 'toHaveValue':
|
|
141
|
-
await (0, test_1.expect)(locator).toHaveValue(
|
|
158
|
+
await (0, test_1.expect)(locator).toHaveValue(resolvedAttrValue ?? '');
|
|
142
159
|
break;
|
|
143
160
|
case 'toContainText':
|
|
144
|
-
await (0, test_1.expect)(locator).toContainText(
|
|
161
|
+
await (0, test_1.expect)(locator).toContainText(resolvedAttrValue ?? '');
|
|
145
162
|
break;
|
|
146
163
|
case 'toHaveAttribute':
|
|
147
|
-
await (0, test_1.expect)(locator).toHaveAttribute(
|
|
164
|
+
await (0, test_1.expect)(locator).toHaveAttribute(resolvedValue, resolvedAttrValue ?? '');
|
|
148
165
|
break;
|
|
149
166
|
}
|
|
150
167
|
}
|
|
@@ -40,8 +40,32 @@ export type AssertOptions = {
|
|
|
40
40
|
* and generates equivalent Playwright code which is cached. Subsequent
|
|
41
41
|
* runs execute the cached code directly, skipping the AI call entirely.
|
|
42
42
|
* Defaults to `true`.
|
|
43
|
+
*
|
|
44
|
+
* Cached steps preserve `{{$.env.*}}` placeholders for any value that came
|
|
45
|
+
* from an env var, so changing an env value between runs replays the same
|
|
46
|
+
* cached steps with the new value rather than re-invoking the AI.
|
|
43
47
|
*/
|
|
44
48
|
cache?: boolean;
|
|
49
|
+
/**
|
|
50
|
+
* Explicit environment variable names (in addition to the heuristically
|
|
51
|
+
* derived ones) that the assertion may read via `{{$.env.*}}` interpolations.
|
|
52
|
+
*/
|
|
53
|
+
envVars?: string[];
|
|
54
|
+
/**
|
|
55
|
+
* Explicitly supply environment variable values that amend (or override)
|
|
56
|
+
* the environment observed by this `page.ai.assert` call. Keys are merged
|
|
57
|
+
* with any names derived from {@link AssertOptions.envVars} and from
|
|
58
|
+
* `{{$.env.*}}` interpolations in the assertion text.
|
|
59
|
+
*
|
|
60
|
+
* - A `string` value sets or overrides the variable for this invocation.
|
|
61
|
+
* - An `undefined` value *removes* the variable, even if it would
|
|
62
|
+
* otherwise be resolved from persistence.
|
|
63
|
+
*
|
|
64
|
+
* Only the **names** (keys) influence cache lookup; changing a value
|
|
65
|
+
* replays the cached steps with the new value via `{{$.env.*}}` placeholder
|
|
66
|
+
* substitution rather than busting the cache.
|
|
67
|
+
*/
|
|
68
|
+
envVals?: Record<string, string | undefined>;
|
|
45
69
|
};
|
|
46
70
|
type PageAiAct = {
|
|
47
71
|
<Schema extends z.ZodObject>(instruction: string, options?: PageAiActWithSchemaOptions<Schema>): Promise<z.infer<Schema>>;
|
|
@@ -10,6 +10,7 @@ const GptApiKeysNotSetupException_1 = require("../../exceptions/GptApiKeysNotSet
|
|
|
10
10
|
const TestNotFoundException_1 = require("../../exceptions/TestNotFoundException");
|
|
11
11
|
const ToolCallFailedException_1 = require("../../exceptions/ToolCallFailedException");
|
|
12
12
|
const ToolRequiresGptException_1 = require("../../exceptions/ToolRequiresGptException");
|
|
13
|
+
const DonobuFlowsManager_1 = require("../../managers/DonobuFlowsManager");
|
|
13
14
|
const InteractionVisualizer_1 = require("../../managers/InteractionVisualizer");
|
|
14
15
|
const ToolManager_1 = require("../../managers/ToolManager");
|
|
15
16
|
const WebTargetInspector_1 = require("../../managers/WebTargetInspector");
|
|
@@ -220,6 +221,33 @@ Valid options:
|
|
|
220
221
|
const clearCache = sharedState.runtimeDirectives?.clearPageAiCache ?? false;
|
|
221
222
|
const retries = options?.retries ?? 0;
|
|
222
223
|
const retryDelaySeconds = options?.retryDelaySeconds ?? 3;
|
|
224
|
+
// Distill env var names from `{{$.env.*}}` interpolations in the
|
|
225
|
+
// assertion plus any explicitly provided names/overrides. Cached
|
|
226
|
+
// Playwright steps may carry the same `{{$.env.X}}` placeholders in
|
|
227
|
+
// their `value`/`attributeValue` fields, so we resolve env data at
|
|
228
|
+
// replay time and let the executor interpolate before applying.
|
|
229
|
+
const envVarNames = (0, DonobuFlowsManager_1.distillAllowedEnvVariableNames)(assertion, [
|
|
230
|
+
...(options?.envVars ?? []),
|
|
231
|
+
...Object.keys(options?.envVals ?? {}),
|
|
232
|
+
]);
|
|
233
|
+
const hasEnvRefs = envVarNames.length > 0;
|
|
234
|
+
const resolveEnvData = async () => {
|
|
235
|
+
if (!hasEnvRefs) {
|
|
236
|
+
return undefined;
|
|
237
|
+
}
|
|
238
|
+
const envData = await sharedState.donobuStack.envDataManager.getByNames(envVarNames);
|
|
239
|
+
if (options?.envVals) {
|
|
240
|
+
for (const [k, v] of Object.entries(options.envVals)) {
|
|
241
|
+
if (v === undefined) {
|
|
242
|
+
delete envData[k];
|
|
243
|
+
}
|
|
244
|
+
else {
|
|
245
|
+
envData[k] = v;
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
return envData;
|
|
250
|
+
};
|
|
223
251
|
// --- Cache lookup (when enabled and not clearing) ---
|
|
224
252
|
if (useCache && !clearCache) {
|
|
225
253
|
const cache = getOrInitPageAiCache();
|
|
@@ -227,6 +255,7 @@ Valid options:
|
|
|
227
255
|
const cached = await cache.getAssert({ pageUrl, assertion });
|
|
228
256
|
if (cached) {
|
|
229
257
|
Logger_1.appLogger.debug(`Assert cache HIT for: "${assertion}" - running cached Playwright assertion`);
|
|
258
|
+
const envData = await resolveEnvData();
|
|
230
259
|
let lastError = null;
|
|
231
260
|
for (let attempt = 0; attempt <= retries; attempt++) {
|
|
232
261
|
if (attempt > 0) {
|
|
@@ -234,7 +263,7 @@ Valid options:
|
|
|
234
263
|
await page.waitForTimeout(retryDelaySeconds * 1000);
|
|
235
264
|
}
|
|
236
265
|
try {
|
|
237
|
-
await cached.run({ page });
|
|
266
|
+
await cached.run({ page, envData });
|
|
238
267
|
return; // Assertion passed
|
|
239
268
|
}
|
|
240
269
|
catch (error) {
|
|
@@ -263,12 +292,28 @@ Valid options:
|
|
|
263
292
|
await cache.deleteAssert({ pageUrl, assertion });
|
|
264
293
|
Logger_1.appLogger.debug(`Assert cache invalidated for: "${assertion}"`);
|
|
265
294
|
}
|
|
266
|
-
//
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
295
|
+
// Make env vars available to runTool's envData for `{{$.env.*}}`
|
|
296
|
+
// interpolation inside `assertionToTestFor` and so AssertTool can
|
|
297
|
+
// instruct the AI to emit placeholders in cached step values. Mirrors
|
|
298
|
+
// PageAi.ai for `act`: metadata.envVars is set (overwriting), envVals
|
|
299
|
+
// is restored.
|
|
300
|
+
if (hasEnvRefs) {
|
|
301
|
+
sharedState.donobuFlowMetadata.envVars = envVarNames;
|
|
302
|
+
}
|
|
303
|
+
const previousEnvVals = sharedState.envVals;
|
|
304
|
+
sharedState.envVals = options?.envVals;
|
|
305
|
+
let result;
|
|
306
|
+
try {
|
|
307
|
+
// --- Cache miss or cache disabled: run AI assertion ---
|
|
308
|
+
result = await runTool(page, AssertTool_1.AssertTool.NAME, {
|
|
309
|
+
assertionToTestFor: assertion,
|
|
310
|
+
retries: options?.retries,
|
|
311
|
+
retryWaitSeconds: options?.retryDelaySeconds,
|
|
312
|
+
}, options?.gptClient);
|
|
313
|
+
}
|
|
314
|
+
finally {
|
|
315
|
+
sharedState.envVals = previousEnvVals;
|
|
316
|
+
}
|
|
272
317
|
if (!result.outcome.isSuccessful) {
|
|
273
318
|
throw new ToolCallFailedException_1.ToolCallFailedException(AssertTool_1.AssertTool.NAME, result.outcome);
|
|
274
319
|
}
|
|
@@ -451,15 +451,25 @@ exports.test = test_1.test.extend({
|
|
|
451
451
|
* 3. `_steps` is fully populated by the time `finalizeTest()` runs (the
|
|
452
452
|
* test body has already completed).
|
|
453
453
|
*/
|
|
454
|
-
function collectNativeSteps(rawSteps) {
|
|
454
|
+
function collectNativeSteps(rawSteps, startTimes) {
|
|
455
455
|
const result = [];
|
|
456
456
|
for (const step of rawSteps) {
|
|
457
457
|
const cat = step.category ?? '';
|
|
458
|
+
const childRaw = Array.isArray(step.steps) ? step.steps : [];
|
|
459
|
+
const collectedChildren = collectNativeSteps(childRaw, startTimes);
|
|
458
460
|
if (cat === 'expect' || cat === 'test.step') {
|
|
461
|
+
// Start time comes from the onStepBegin payload (captured by the
|
|
462
|
+
// pwApiStepLogger fixture into startTimes). If unavailable, fall
|
|
463
|
+
// back to the step's endWallTime so the window is zero-width and
|
|
464
|
+
// sibling steps don't get falsely nested inside.
|
|
465
|
+
const endWallTime = step.endWallTime ?? Date.now();
|
|
466
|
+
const startWallTime = (typeof step.stepId === 'string' && startTimes.get(step.stepId)) ||
|
|
467
|
+
endWallTime;
|
|
459
468
|
result.push({
|
|
460
469
|
title: step.title ?? '',
|
|
461
470
|
category: cat,
|
|
462
|
-
|
|
471
|
+
startWallTime,
|
|
472
|
+
endWallTime,
|
|
463
473
|
passed: !step.error,
|
|
464
474
|
error: step.error
|
|
465
475
|
? { message: step.error.message, stack: step.error.stack }
|
|
@@ -471,15 +481,16 @@ function collectNativeSteps(rawSteps) {
|
|
|
471
481
|
column: step.location.column ?? 0,
|
|
472
482
|
}
|
|
473
483
|
: undefined,
|
|
484
|
+
children: collectedChildren,
|
|
474
485
|
});
|
|
475
486
|
}
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
result.push(...
|
|
487
|
+
else {
|
|
488
|
+
// Parent is not a kept category (e.g. pw:api) — promote any qualifying
|
|
489
|
+
// descendants so a nested expect() still appears in the report.
|
|
490
|
+
result.push(...collectedChildren);
|
|
480
491
|
}
|
|
481
492
|
}
|
|
482
|
-
result.sort((a, b) => a.
|
|
493
|
+
result.sort((a, b) => a.startWallTime - b.startWallTime);
|
|
483
494
|
return result;
|
|
484
495
|
}
|
|
485
496
|
// ---------------------------------------------------------------------------
|
|
@@ -689,9 +700,20 @@ function installPlaywrightStepLogger(testInfo) {
|
|
|
689
700
|
const originalOnStepEnd = callbacks && typeof callbacks.onStepEnd === 'function'
|
|
690
701
|
? callbacks.onStepEnd
|
|
691
702
|
: null;
|
|
703
|
+
const originalOnStepBegin = callbacks && typeof callbacks.onStepBegin === 'function'
|
|
704
|
+
? callbacks.onStepBegin
|
|
705
|
+
: null;
|
|
692
706
|
if (!callbacks || !originalOnStepEnd || !stepMap) {
|
|
693
707
|
return () => { };
|
|
694
708
|
}
|
|
709
|
+
// Stash a stepId -> wallTime map on testInfo so collectNativeSteps can
|
|
710
|
+
// look up start times when building the report hierarchy. Playwright
|
|
711
|
+
// doesn't store wallTime on the step object itself — it's only emitted
|
|
712
|
+
// via the onStepBegin payload — so we capture it here. Without this,
|
|
713
|
+
// `test.step` blocks have no recoverable start time and any preceding
|
|
714
|
+
// tool-call step gets falsely nested inside them.
|
|
715
|
+
const startTimes = new Map();
|
|
716
|
+
ti.__donobuStepStartTimes = startTimes;
|
|
695
717
|
let installed = false;
|
|
696
718
|
try {
|
|
697
719
|
callbacks.onStepEnd = function patchedOnStepEnd(payload) {
|
|
@@ -710,6 +732,21 @@ function installPlaywrightStepLogger(testInfo) {
|
|
|
710
732
|
}
|
|
711
733
|
return ret;
|
|
712
734
|
};
|
|
735
|
+
callbacks.onStepBegin = function patchedOnStepBegin(payload) {
|
|
736
|
+
const ret = originalOnStepBegin
|
|
737
|
+
? originalOnStepBegin.call(this, payload)
|
|
738
|
+
: undefined;
|
|
739
|
+
try {
|
|
740
|
+
if (typeof payload?.stepId === 'string' &&
|
|
741
|
+
typeof payload?.wallTime === 'number') {
|
|
742
|
+
startTimes.set(payload.stepId, payload.wallTime);
|
|
743
|
+
}
|
|
744
|
+
}
|
|
745
|
+
catch (err) {
|
|
746
|
+
Logger_1.appLogger.debug('Failed to record Playwright step start time', err);
|
|
747
|
+
}
|
|
748
|
+
return ret;
|
|
749
|
+
};
|
|
713
750
|
installed = true;
|
|
714
751
|
}
|
|
715
752
|
catch (err) {
|
|
@@ -718,6 +755,9 @@ function installPlaywrightStepLogger(testInfo) {
|
|
|
718
755
|
return () => {
|
|
719
756
|
if (installed) {
|
|
720
757
|
callbacks.onStepEnd = originalOnStepEnd;
|
|
758
|
+
if (originalOnStepBegin) {
|
|
759
|
+
callbacks.onStepBegin = originalOnStepBegin;
|
|
760
|
+
}
|
|
721
761
|
}
|
|
722
762
|
};
|
|
723
763
|
}
|
|
@@ -825,7 +865,8 @@ async function finalizeTest(page, testInfo, logBuffer, videoOption) {
|
|
|
825
865
|
// Attach native Playwright steps (expect assertions, test.step blocks)
|
|
826
866
|
// so the HTML report can show a unified timeline alongside AI tool calls.
|
|
827
867
|
try {
|
|
828
|
-
const
|
|
868
|
+
const startTimes = testInfo.__donobuStepStartTimes ?? new Map();
|
|
869
|
+
const nativeSteps = collectNativeSteps(testInfo._steps ?? [], startTimes);
|
|
829
870
|
if (nativeSteps.length > 0) {
|
|
830
871
|
await testInfo.attach('donobu-native-steps', {
|
|
831
872
|
body: JSON.stringify(nativeSteps),
|
|
@@ -87,6 +87,9 @@ class ToolManager {
|
|
|
87
87
|
startedAt: startedAt,
|
|
88
88
|
completedAt: null,
|
|
89
89
|
});
|
|
90
|
+
// Expose the un-interpolated parameters so tools that need to
|
|
91
|
+
// preserve `{{...}}` references (e.g. AssertTool) can read them.
|
|
92
|
+
context.rawParameters = toolParameters;
|
|
90
93
|
// Use the interpolated parameters when calling the tool.
|
|
91
94
|
toolCallResult = isFromGpt
|
|
92
95
|
? await tool.callFromGpt(context, tool.inputSchemaForGpt.parse(interpolatedParameters))
|
|
@@ -22,5 +22,16 @@ export type ToolCallContext = {
|
|
|
22
22
|
readonly invokedToolCalls: ToolCall[];
|
|
23
23
|
readonly metadata: FlowMetadata;
|
|
24
24
|
readonly toolCallId: string;
|
|
25
|
+
/**
|
|
26
|
+
* Original (un-interpolated) parameters supplied to the current tool call.
|
|
27
|
+
*
|
|
28
|
+
* `ToolManager.invokeTool` interpolates `{{...}}` expressions in the tool's
|
|
29
|
+
* parameters before invoking the tool, but a tool may need the raw text to
|
|
30
|
+
* preserve env var references in its output (e.g. `AssertTool` emits
|
|
31
|
+
* Playwright steps that retain `{{$.env.X}}` so cached replays stay correct
|
|
32
|
+
* across env value changes). Set by `ToolManager` immediately before the
|
|
33
|
+
* tool runs; absent for direct/legacy invocation paths.
|
|
34
|
+
*/
|
|
35
|
+
rawParameters?: Record<string, any>;
|
|
25
36
|
};
|
|
26
37
|
//# sourceMappingURL=ToolCallContext.d.ts.map
|
|
@@ -562,7 +562,7 @@ function renderErrors(errors) {
|
|
|
562
562
|
}
|
|
563
563
|
return html;
|
|
564
564
|
}
|
|
565
|
-
function renderNativeStep(ns) {
|
|
565
|
+
function renderNativeStep(ns, childrenHtml) {
|
|
566
566
|
const statusIcon = ns.passed
|
|
567
567
|
? '<span class="step-status-ok">✓</span>'
|
|
568
568
|
: '<span class="step-status-fail">✗</span>';
|
|
@@ -573,7 +573,8 @@ function renderNativeStep(ns) {
|
|
|
573
573
|
const snippet = ns.location?.file
|
|
574
574
|
? readSourceSnippet(ns.location.file, ns.location.line)
|
|
575
575
|
: null;
|
|
576
|
-
const
|
|
576
|
+
const hasError = !ns.passed && !!ns.error?.message;
|
|
577
|
+
const hasBody = !!snippet || hasError || !!childrenHtml;
|
|
577
578
|
const renderHeader = (tag) => {
|
|
578
579
|
let header = `<${tag} class="filmstrip-header">`;
|
|
579
580
|
header += statusIcon;
|
|
@@ -592,17 +593,23 @@ function renderNativeStep(ns) {
|
|
|
592
593
|
if (!hasBody) {
|
|
593
594
|
return `<div class="filmstrip-step native-step">${renderHeader('div')}</div>`;
|
|
594
595
|
}
|
|
595
|
-
//
|
|
596
|
-
//
|
|
596
|
+
// Failures always render expanded so the error is immediately visible.
|
|
597
|
+
// test.step blocks with nested content also default open so users see
|
|
598
|
+
// what's inside; bare passing expects with just a snippet collapse to
|
|
599
|
+
// keep tests with many assertions scannable.
|
|
600
|
+
const defaultOpen = !ns.passed || (ns.category === 'test.step' && !!childrenHtml);
|
|
597
601
|
const passClass = ns.passed ? 'native-step--passed' : 'native-step--failed';
|
|
598
|
-
let html = `<details class="filmstrip-step native-step ${passClass}"${
|
|
602
|
+
let html = `<details class="filmstrip-step native-step ${passClass}"${defaultOpen ? ' open' : ''}>`;
|
|
599
603
|
html += renderHeader('summary');
|
|
600
|
-
if (
|
|
604
|
+
if (hasError) {
|
|
601
605
|
html += `<pre class="native-step-error">${ansiToHtml(ns.error.message)}</pre>`;
|
|
602
606
|
}
|
|
603
607
|
if (snippet) {
|
|
604
608
|
html += snippet;
|
|
605
609
|
}
|
|
610
|
+
if (childrenHtml) {
|
|
611
|
+
html += childrenHtml;
|
|
612
|
+
}
|
|
606
613
|
html += `</details>`;
|
|
607
614
|
return html;
|
|
608
615
|
}
|
|
@@ -853,31 +860,71 @@ function renderSteps(steps, stepScreenshots, nativeSteps, outputDir) {
|
|
|
853
860
|
return '';
|
|
854
861
|
}
|
|
855
862
|
if (hasScreenshots || hasNative) {
|
|
856
|
-
const
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
863
|
+
const buildNativeTree = (nss) => nss.map((ns) => ({
|
|
864
|
+
kind: 'native',
|
|
865
|
+
ns,
|
|
866
|
+
t: ns.startWallTime,
|
|
867
|
+
tEnd: ns.endWallTime,
|
|
868
|
+
children: buildNativeTree(ns.children),
|
|
869
|
+
}));
|
|
870
|
+
const roots = buildNativeTree(nativeSteps);
|
|
871
|
+
// Place each Donobu screenshot under the deepest native step whose
|
|
872
|
+
// [start, end] window contains it. Falls back to top level if none.
|
|
873
|
+
const placeDonobu = (nodes, d) => {
|
|
874
|
+
for (const n of nodes) {
|
|
875
|
+
if (n.kind !== 'native') {
|
|
876
|
+
continue;
|
|
877
|
+
}
|
|
878
|
+
if (d.ss.startedAt >= n.t && d.ss.completedAt <= n.tEnd) {
|
|
879
|
+
if (!placeDonobu(n.children, d)) {
|
|
880
|
+
n.children.push(d);
|
|
881
|
+
}
|
|
882
|
+
return true;
|
|
883
|
+
}
|
|
884
|
+
}
|
|
885
|
+
return false;
|
|
886
|
+
};
|
|
887
|
+
for (const ss of stepScreenshots) {
|
|
888
|
+
const d = { kind: 'donobu', ss, t: ss.startedAt };
|
|
889
|
+
if (!placeDonobu(roots, d)) {
|
|
890
|
+
roots.push(d);
|
|
891
|
+
}
|
|
892
|
+
}
|
|
893
|
+
const sortTree = (nodes) => {
|
|
894
|
+
nodes.sort((a, b) => a.t - b.t);
|
|
895
|
+
for (const n of nodes) {
|
|
896
|
+
if (n.kind === 'native') {
|
|
897
|
+
sortTree(n.children);
|
|
898
|
+
}
|
|
899
|
+
}
|
|
900
|
+
};
|
|
901
|
+
sortTree(roots);
|
|
902
|
+
const countNodes = (nodes) => {
|
|
903
|
+
let c = 0;
|
|
904
|
+
for (const n of nodes) {
|
|
905
|
+
c += 1;
|
|
906
|
+
if (n.kind === 'native') {
|
|
907
|
+
c += countNodes(n.children);
|
|
908
|
+
}
|
|
909
|
+
}
|
|
910
|
+
return c;
|
|
911
|
+
};
|
|
912
|
+
const renderNode = (node) => {
|
|
913
|
+
if (node.kind === 'donobu') {
|
|
914
|
+
return renderFilmstripStep(node.ss, outputDir);
|
|
915
|
+
}
|
|
916
|
+
const childrenHtml = node.children.length > 0
|
|
917
|
+
? `<div class="native-step-children">${node.children.map(renderNode).join('')}</div>`
|
|
918
|
+
: '';
|
|
919
|
+
return renderNativeStep(node.ns, childrenHtml);
|
|
920
|
+
};
|
|
921
|
+
const stepCount = countNodes(roots);
|
|
870
922
|
let html = '<details class="steps-section"><summary>Steps (' +
|
|
871
923
|
stepCount +
|
|
872
924
|
')</summary>';
|
|
873
925
|
html += '<div class="step-filmstrip">';
|
|
874
|
-
for (const
|
|
875
|
-
|
|
876
|
-
html += renderFilmstripStep(entry.ss, outputDir);
|
|
877
|
-
}
|
|
878
|
-
else {
|
|
879
|
-
html += renderNativeStep(entry.ns);
|
|
880
|
-
}
|
|
926
|
+
for (const node of roots) {
|
|
927
|
+
html += renderNode(node);
|
|
881
928
|
}
|
|
882
929
|
html += '</div>';
|
|
883
930
|
html += '</details>';
|
|
@@ -1610,6 +1657,8 @@ details.native-step>summary::-webkit-details-marker{display:none}
|
|
|
1610
1657
|
details.native-step[open]>summary .native-step-chevron{transform:rotate(90deg)}
|
|
1611
1658
|
.native-step-error{font-size:11px;font-family:var(--mono);padding:4px 0 2px 22px;margin:0;white-space:pre-wrap;word-break:break-word;color:var(--text-muted)}
|
|
1612
1659
|
.native-step-snippet{font-size:11px;font-family:var(--mono);margin:4px 0 2px 22px;overflow:hidden}
|
|
1660
|
+
.native-step-children{display:flex;flex-direction:column;margin:4px 0 0 10px;border-left:1px solid var(--border-subtle);padding-left:8px}
|
|
1661
|
+
.native-step-children>.filmstrip-step{padding-left:8px}
|
|
1613
1662
|
.snippet-line{display:flex;padding:1px 8px;white-space:pre}
|
|
1614
1663
|
.snippet-line--target{background:rgba(239,68,68,.10)}
|
|
1615
1664
|
.snippet-linenum{color:var(--text-dim);min-width:40px;user-select:none}
|
|
@@ -124,6 +124,36 @@ It will use a screenshot of the current viewport of the webpage, the webpage's t
|
|
|
124
124
|
Logger_1.appLogger.warn(msg);
|
|
125
125
|
return msg;
|
|
126
126
|
});
|
|
127
|
+
const rawAssertion = typeof context.rawParameters?.assertionToTestFor === 'string'
|
|
128
|
+
? context.rawParameters.assertionToTestFor
|
|
129
|
+
: parameters.assertionToTestFor;
|
|
130
|
+
const envEntries = Object.entries(context.envData ?? {});
|
|
131
|
+
// Only treat env vars as "in play" when the raw assertion actually
|
|
132
|
+
// references one — keeps the prompt small for the common case.
|
|
133
|
+
const referencedEnvEntries = envEntries.filter(([name]) => rawAssertion.includes(`{{$.env.${name}}}`));
|
|
134
|
+
const hasEnvRefs = referencedEnvEntries.length > 0;
|
|
135
|
+
const envBlock = hasEnvRefs
|
|
136
|
+
? `
|
|
137
|
+
|
|
138
|
+
The user's original assertion contains environment variable references using the
|
|
139
|
+
syntax \`{{$.env.NAME}}\`. To keep cached Playwright steps valid across runs with
|
|
140
|
+
different env values, you MUST emit those same placeholders in any
|
|
141
|
+
playwrightAssertionStep \`value\`/\`attributeValue\` field whose contents come from
|
|
142
|
+
an env var. Do NOT bake the literal current value into the step.
|
|
143
|
+
|
|
144
|
+
Original (uninterpolated) assertion: ${rawAssertion}
|
|
145
|
+
|
|
146
|
+
Current env mapping (use these to identify which substrings on the page came
|
|
147
|
+
from which env var, then emit the placeholder rather than the literal):
|
|
148
|
+
${referencedEnvEntries.map(([name, value]) => ` - {{$.env.${name}}} = ${JSON.stringify(value)}`).join('\n')}
|
|
149
|
+
|
|
150
|
+
Examples:
|
|
151
|
+
- Raw assertion "Welcome banner says hello {{$.env.USERNAME}}", USERNAME="alice", page shows "Welcome alice" →
|
|
152
|
+
[{ locator: "text", role: null, value: "{{$.env.USERNAME}}", valueIsRegex: false, assertion: "toBeVisible", attributeValue: null }]
|
|
153
|
+
- Raw assertion "The username field shows {{$.env.USERNAME}}", USERNAME="alice", page input value is "alice" →
|
|
154
|
+
[{ locator: "label", role: null, value: "Username", valueIsRegex: false, assertion: "toHaveValue", attributeValue: "{{$.env.USERNAME}}" }]
|
|
155
|
+
- For literal page text unrelated to env vars, keep the literal value as usual.`
|
|
156
|
+
: '';
|
|
127
157
|
const promptMessages = [
|
|
128
158
|
{
|
|
129
159
|
type: 'system',
|
|
@@ -142,7 +172,7 @@ CRITICAL RULES for generating structured steps — follow these precisely:
|
|
|
142
172
|
- Text input / textarea content: use 'toHaveValue' with locator='label' and set attributeValue to the expected text. Do NOT use 'toBeVisible' on the textbox.
|
|
143
173
|
- Selected tabs, pills, or items with aria-selected: use 'toHaveAttribute' with value='aria-selected' and attributeValue='true', NOT 'toBeVisible' on the text.
|
|
144
174
|
- Text content within an element: use 'toContainText' with attributeValue set to the substring, NOT 'toBeVisible'.
|
|
145
|
-
- Only use 'toBeVisible' when the assertion is genuinely about whether something is visible — not as a fallback for state or value checks
|
|
175
|
+
- Only use 'toBeVisible' when the assertion is genuinely about whether something is visible — not as a fallback for state or value checks.${envBlock}`,
|
|
146
176
|
},
|
|
147
177
|
{
|
|
148
178
|
type: 'user',
|
|
@@ -184,7 +214,7 @@ careful positioning lost, etc. A screenshot of the webpage has also been provide
|
|
|
184
214
|
verifiedSteps.length > 0) {
|
|
185
215
|
try {
|
|
186
216
|
const executor = (0, assertCache_1.buildAssertExecutor)(verifiedSteps);
|
|
187
|
-
await executor({ page: page });
|
|
217
|
+
await executor({ page: page, envData: context.envData });
|
|
188
218
|
}
|
|
189
219
|
catch (error) {
|
|
190
220
|
Logger_1.appLogger.debug(`Structured assertion steps failed verification for: "${parameters.assertionToTestFor}" — discarding steps. Error: ${error.message}`);
|
|
@@ -33,6 +33,13 @@ export declare const PlaywrightAssertionStepSchema: z.ZodObject<{
|
|
|
33
33
|
export type PlaywrightAssertionStep = z.infer<typeof PlaywrightAssertionStepSchema>;
|
|
34
34
|
export type AssertCacheExecutor = (context: {
|
|
35
35
|
page: Page;
|
|
36
|
+
/**
|
|
37
|
+
* Optional env mapping used to interpolate `{{$.env.X}}` placeholders that
|
|
38
|
+
* the AI may have embedded into step `value`/`attributeValue` fields. When
|
|
39
|
+
* absent, steps run unchanged (backwards compatible with cache entries
|
|
40
|
+
* recorded before env-aware caching).
|
|
41
|
+
*/
|
|
42
|
+
envData?: Record<string, string>;
|
|
36
43
|
}) => Promise<void>;
|
|
37
44
|
/**
|
|
38
45
|
* Builds an executor function from structured assertion steps.
|
|
@@ -5,6 +5,7 @@ exports.buildAssertExecutor = buildAssertExecutor;
|
|
|
5
5
|
exports.buildLocateExecutor = buildLocateExecutor;
|
|
6
6
|
const test_1 = require("@playwright/test");
|
|
7
7
|
const v4_1 = require("zod/v4");
|
|
8
|
+
const TemplateInterpolator_1 = require("../../../utils/TemplateInterpolator");
|
|
8
9
|
const buildLocator_1 = require("../locate/buildLocator");
|
|
9
10
|
// ---------------------------------------------------------------------------
|
|
10
11
|
// Structured assertion step schema
|
|
@@ -84,17 +85,33 @@ Common roles: 'heading', 'button', 'link', 'tab', 'tabpanel', 'dialog', 'navigat
|
|
|
84
85
|
- toContainText: set to the text substring to match within the element
|
|
85
86
|
- All other assertions: set to null`),
|
|
86
87
|
});
|
|
88
|
+
/**
|
|
89
|
+
* Resolves any `{{$.env.X}}` placeholders in a step field against the
|
|
90
|
+
* supplied env data. Returns the input verbatim when no env data is given,
|
|
91
|
+
* preserving backwards compatibility with cached entries that contain
|
|
92
|
+
* literal values only.
|
|
93
|
+
*/
|
|
94
|
+
function resolveStepField(value, envData) {
|
|
95
|
+
if (!envData || !value.includes('{{')) {
|
|
96
|
+
return value;
|
|
97
|
+
}
|
|
98
|
+
return (0, TemplateInterpolator_1.interpolateString)(value, { env: envData, calls: [] });
|
|
99
|
+
}
|
|
87
100
|
/**
|
|
88
101
|
* Builds an executor function from structured assertion steps.
|
|
89
102
|
* Each step maps to exactly one Playwright `expect` call — no string
|
|
90
103
|
* evaluation, no VM contexts.
|
|
91
104
|
*/
|
|
92
105
|
function buildAssertExecutor(steps) {
|
|
93
|
-
return async ({ page }) => {
|
|
106
|
+
return async ({ page, envData }) => {
|
|
94
107
|
for (const step of steps) {
|
|
108
|
+
const resolvedValue = resolveStepField(step.value, envData);
|
|
109
|
+
const resolvedAttrValue = step.attributeValue === null
|
|
110
|
+
? null
|
|
111
|
+
: resolveStepField(step.attributeValue, envData);
|
|
95
112
|
const matcher = step.valueIsRegex
|
|
96
|
-
? new RegExp(
|
|
97
|
-
:
|
|
113
|
+
? new RegExp(resolvedValue)
|
|
114
|
+
: resolvedValue;
|
|
98
115
|
// Page-level assertions (no element locator needed)
|
|
99
116
|
if (step.assertion === 'toHaveTitle') {
|
|
100
117
|
await (0, test_1.expect)(page).toHaveTitle(matcher);
|
|
@@ -138,13 +155,13 @@ function buildAssertExecutor(steps) {
|
|
|
138
155
|
await (0, test_1.expect)(locator).toBeChecked();
|
|
139
156
|
break;
|
|
140
157
|
case 'toHaveValue':
|
|
141
|
-
await (0, test_1.expect)(locator).toHaveValue(
|
|
158
|
+
await (0, test_1.expect)(locator).toHaveValue(resolvedAttrValue ?? '');
|
|
142
159
|
break;
|
|
143
160
|
case 'toContainText':
|
|
144
|
-
await (0, test_1.expect)(locator).toContainText(
|
|
161
|
+
await (0, test_1.expect)(locator).toContainText(resolvedAttrValue ?? '');
|
|
145
162
|
break;
|
|
146
163
|
case 'toHaveAttribute':
|
|
147
|
-
await (0, test_1.expect)(locator).toHaveAttribute(
|
|
164
|
+
await (0, test_1.expect)(locator).toHaveAttribute(resolvedValue, resolvedAttrValue ?? '');
|
|
148
165
|
break;
|
|
149
166
|
}
|
|
150
167
|
}
|
|
@@ -40,8 +40,32 @@ export type AssertOptions = {
|
|
|
40
40
|
* and generates equivalent Playwright code which is cached. Subsequent
|
|
41
41
|
* runs execute the cached code directly, skipping the AI call entirely.
|
|
42
42
|
* Defaults to `true`.
|
|
43
|
+
*
|
|
44
|
+
* Cached steps preserve `{{$.env.*}}` placeholders for any value that came
|
|
45
|
+
* from an env var, so changing an env value between runs replays the same
|
|
46
|
+
* cached steps with the new value rather than re-invoking the AI.
|
|
43
47
|
*/
|
|
44
48
|
cache?: boolean;
|
|
49
|
+
/**
|
|
50
|
+
* Explicit environment variable names (in addition to the heuristically
|
|
51
|
+
* derived ones) that the assertion may read via `{{$.env.*}}` interpolations.
|
|
52
|
+
*/
|
|
53
|
+
envVars?: string[];
|
|
54
|
+
/**
|
|
55
|
+
* Explicitly supply environment variable values that amend (or override)
|
|
56
|
+
* the environment observed by this `page.ai.assert` call. Keys are merged
|
|
57
|
+
* with any names derived from {@link AssertOptions.envVars} and from
|
|
58
|
+
* `{{$.env.*}}` interpolations in the assertion text.
|
|
59
|
+
*
|
|
60
|
+
* - A `string` value sets or overrides the variable for this invocation.
|
|
61
|
+
* - An `undefined` value *removes* the variable, even if it would
|
|
62
|
+
* otherwise be resolved from persistence.
|
|
63
|
+
*
|
|
64
|
+
* Only the **names** (keys) influence cache lookup; changing a value
|
|
65
|
+
* replays the cached steps with the new value via `{{$.env.*}}` placeholder
|
|
66
|
+
* substitution rather than busting the cache.
|
|
67
|
+
*/
|
|
68
|
+
envVals?: Record<string, string | undefined>;
|
|
45
69
|
};
|
|
46
70
|
type PageAiAct = {
|
|
47
71
|
<Schema extends z.ZodObject>(instruction: string, options?: PageAiActWithSchemaOptions<Schema>): Promise<z.infer<Schema>>;
|
|
@@ -10,6 +10,7 @@ const GptApiKeysNotSetupException_1 = require("../../exceptions/GptApiKeysNotSet
|
|
|
10
10
|
const TestNotFoundException_1 = require("../../exceptions/TestNotFoundException");
|
|
11
11
|
const ToolCallFailedException_1 = require("../../exceptions/ToolCallFailedException");
|
|
12
12
|
const ToolRequiresGptException_1 = require("../../exceptions/ToolRequiresGptException");
|
|
13
|
+
const DonobuFlowsManager_1 = require("../../managers/DonobuFlowsManager");
|
|
13
14
|
const InteractionVisualizer_1 = require("../../managers/InteractionVisualizer");
|
|
14
15
|
const ToolManager_1 = require("../../managers/ToolManager");
|
|
15
16
|
const WebTargetInspector_1 = require("../../managers/WebTargetInspector");
|
|
@@ -220,6 +221,33 @@ Valid options:
|
|
|
220
221
|
const clearCache = sharedState.runtimeDirectives?.clearPageAiCache ?? false;
|
|
221
222
|
const retries = options?.retries ?? 0;
|
|
222
223
|
const retryDelaySeconds = options?.retryDelaySeconds ?? 3;
|
|
224
|
+
// Distill env var names from `{{$.env.*}}` interpolations in the
|
|
225
|
+
// assertion plus any explicitly provided names/overrides. Cached
|
|
226
|
+
// Playwright steps may carry the same `{{$.env.X}}` placeholders in
|
|
227
|
+
// their `value`/`attributeValue` fields, so we resolve env data at
|
|
228
|
+
// replay time and let the executor interpolate before applying.
|
|
229
|
+
const envVarNames = (0, DonobuFlowsManager_1.distillAllowedEnvVariableNames)(assertion, [
|
|
230
|
+
...(options?.envVars ?? []),
|
|
231
|
+
...Object.keys(options?.envVals ?? {}),
|
|
232
|
+
]);
|
|
233
|
+
const hasEnvRefs = envVarNames.length > 0;
|
|
234
|
+
const resolveEnvData = async () => {
|
|
235
|
+
if (!hasEnvRefs) {
|
|
236
|
+
return undefined;
|
|
237
|
+
}
|
|
238
|
+
const envData = await sharedState.donobuStack.envDataManager.getByNames(envVarNames);
|
|
239
|
+
if (options?.envVals) {
|
|
240
|
+
for (const [k, v] of Object.entries(options.envVals)) {
|
|
241
|
+
if (v === undefined) {
|
|
242
|
+
delete envData[k];
|
|
243
|
+
}
|
|
244
|
+
else {
|
|
245
|
+
envData[k] = v;
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
return envData;
|
|
250
|
+
};
|
|
223
251
|
// --- Cache lookup (when enabled and not clearing) ---
|
|
224
252
|
if (useCache && !clearCache) {
|
|
225
253
|
const cache = getOrInitPageAiCache();
|
|
@@ -227,6 +255,7 @@ Valid options:
|
|
|
227
255
|
const cached = await cache.getAssert({ pageUrl, assertion });
|
|
228
256
|
if (cached) {
|
|
229
257
|
Logger_1.appLogger.debug(`Assert cache HIT for: "${assertion}" - running cached Playwright assertion`);
|
|
258
|
+
const envData = await resolveEnvData();
|
|
230
259
|
let lastError = null;
|
|
231
260
|
for (let attempt = 0; attempt <= retries; attempt++) {
|
|
232
261
|
if (attempt > 0) {
|
|
@@ -234,7 +263,7 @@ Valid options:
|
|
|
234
263
|
await page.waitForTimeout(retryDelaySeconds * 1000);
|
|
235
264
|
}
|
|
236
265
|
try {
|
|
237
|
-
await cached.run({ page });
|
|
266
|
+
await cached.run({ page, envData });
|
|
238
267
|
return; // Assertion passed
|
|
239
268
|
}
|
|
240
269
|
catch (error) {
|
|
@@ -263,12 +292,28 @@ Valid options:
|
|
|
263
292
|
await cache.deleteAssert({ pageUrl, assertion });
|
|
264
293
|
Logger_1.appLogger.debug(`Assert cache invalidated for: "${assertion}"`);
|
|
265
294
|
}
|
|
266
|
-
//
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
295
|
+
// Make env vars available to runTool's envData for `{{$.env.*}}`
|
|
296
|
+
// interpolation inside `assertionToTestFor` and so AssertTool can
|
|
297
|
+
// instruct the AI to emit placeholders in cached step values. Mirrors
|
|
298
|
+
// PageAi.ai for `act`: metadata.envVars is set (overwriting), envVals
|
|
299
|
+
// is restored.
|
|
300
|
+
if (hasEnvRefs) {
|
|
301
|
+
sharedState.donobuFlowMetadata.envVars = envVarNames;
|
|
302
|
+
}
|
|
303
|
+
const previousEnvVals = sharedState.envVals;
|
|
304
|
+
sharedState.envVals = options?.envVals;
|
|
305
|
+
let result;
|
|
306
|
+
try {
|
|
307
|
+
// --- Cache miss or cache disabled: run AI assertion ---
|
|
308
|
+
result = await runTool(page, AssertTool_1.AssertTool.NAME, {
|
|
309
|
+
assertionToTestFor: assertion,
|
|
310
|
+
retries: options?.retries,
|
|
311
|
+
retryWaitSeconds: options?.retryDelaySeconds,
|
|
312
|
+
}, options?.gptClient);
|
|
313
|
+
}
|
|
314
|
+
finally {
|
|
315
|
+
sharedState.envVals = previousEnvVals;
|
|
316
|
+
}
|
|
272
317
|
if (!result.outcome.isSuccessful) {
|
|
273
318
|
throw new ToolCallFailedException_1.ToolCallFailedException(AssertTool_1.AssertTool.NAME, result.outcome);
|
|
274
319
|
}
|
|
@@ -451,15 +451,25 @@ exports.test = test_1.test.extend({
|
|
|
451
451
|
* 3. `_steps` is fully populated by the time `finalizeTest()` runs (the
|
|
452
452
|
* test body has already completed).
|
|
453
453
|
*/
|
|
454
|
-
function collectNativeSteps(rawSteps) {
|
|
454
|
+
function collectNativeSteps(rawSteps, startTimes) {
|
|
455
455
|
const result = [];
|
|
456
456
|
for (const step of rawSteps) {
|
|
457
457
|
const cat = step.category ?? '';
|
|
458
|
+
const childRaw = Array.isArray(step.steps) ? step.steps : [];
|
|
459
|
+
const collectedChildren = collectNativeSteps(childRaw, startTimes);
|
|
458
460
|
if (cat === 'expect' || cat === 'test.step') {
|
|
461
|
+
// Start time comes from the onStepBegin payload (captured by the
|
|
462
|
+
// pwApiStepLogger fixture into startTimes). If unavailable, fall
|
|
463
|
+
// back to the step's endWallTime so the window is zero-width and
|
|
464
|
+
// sibling steps don't get falsely nested inside.
|
|
465
|
+
const endWallTime = step.endWallTime ?? Date.now();
|
|
466
|
+
const startWallTime = (typeof step.stepId === 'string' && startTimes.get(step.stepId)) ||
|
|
467
|
+
endWallTime;
|
|
459
468
|
result.push({
|
|
460
469
|
title: step.title ?? '',
|
|
461
470
|
category: cat,
|
|
462
|
-
|
|
471
|
+
startWallTime,
|
|
472
|
+
endWallTime,
|
|
463
473
|
passed: !step.error,
|
|
464
474
|
error: step.error
|
|
465
475
|
? { message: step.error.message, stack: step.error.stack }
|
|
@@ -471,15 +481,16 @@ function collectNativeSteps(rawSteps) {
|
|
|
471
481
|
column: step.location.column ?? 0,
|
|
472
482
|
}
|
|
473
483
|
: undefined,
|
|
484
|
+
children: collectedChildren,
|
|
474
485
|
});
|
|
475
486
|
}
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
result.push(...
|
|
487
|
+
else {
|
|
488
|
+
// Parent is not a kept category (e.g. pw:api) — promote any qualifying
|
|
489
|
+
// descendants so a nested expect() still appears in the report.
|
|
490
|
+
result.push(...collectedChildren);
|
|
480
491
|
}
|
|
481
492
|
}
|
|
482
|
-
result.sort((a, b) => a.
|
|
493
|
+
result.sort((a, b) => a.startWallTime - b.startWallTime);
|
|
483
494
|
return result;
|
|
484
495
|
}
|
|
485
496
|
// ---------------------------------------------------------------------------
|
|
@@ -689,9 +700,20 @@ function installPlaywrightStepLogger(testInfo) {
|
|
|
689
700
|
const originalOnStepEnd = callbacks && typeof callbacks.onStepEnd === 'function'
|
|
690
701
|
? callbacks.onStepEnd
|
|
691
702
|
: null;
|
|
703
|
+
const originalOnStepBegin = callbacks && typeof callbacks.onStepBegin === 'function'
|
|
704
|
+
? callbacks.onStepBegin
|
|
705
|
+
: null;
|
|
692
706
|
if (!callbacks || !originalOnStepEnd || !stepMap) {
|
|
693
707
|
return () => { };
|
|
694
708
|
}
|
|
709
|
+
// Stash a stepId -> wallTime map on testInfo so collectNativeSteps can
|
|
710
|
+
// look up start times when building the report hierarchy. Playwright
|
|
711
|
+
// doesn't store wallTime on the step object itself — it's only emitted
|
|
712
|
+
// via the onStepBegin payload — so we capture it here. Without this,
|
|
713
|
+
// `test.step` blocks have no recoverable start time and any preceding
|
|
714
|
+
// tool-call step gets falsely nested inside them.
|
|
715
|
+
const startTimes = new Map();
|
|
716
|
+
ti.__donobuStepStartTimes = startTimes;
|
|
695
717
|
let installed = false;
|
|
696
718
|
try {
|
|
697
719
|
callbacks.onStepEnd = function patchedOnStepEnd(payload) {
|
|
@@ -710,6 +732,21 @@ function installPlaywrightStepLogger(testInfo) {
|
|
|
710
732
|
}
|
|
711
733
|
return ret;
|
|
712
734
|
};
|
|
735
|
+
callbacks.onStepBegin = function patchedOnStepBegin(payload) {
|
|
736
|
+
const ret = originalOnStepBegin
|
|
737
|
+
? originalOnStepBegin.call(this, payload)
|
|
738
|
+
: undefined;
|
|
739
|
+
try {
|
|
740
|
+
if (typeof payload?.stepId === 'string' &&
|
|
741
|
+
typeof payload?.wallTime === 'number') {
|
|
742
|
+
startTimes.set(payload.stepId, payload.wallTime);
|
|
743
|
+
}
|
|
744
|
+
}
|
|
745
|
+
catch (err) {
|
|
746
|
+
Logger_1.appLogger.debug('Failed to record Playwright step start time', err);
|
|
747
|
+
}
|
|
748
|
+
return ret;
|
|
749
|
+
};
|
|
713
750
|
installed = true;
|
|
714
751
|
}
|
|
715
752
|
catch (err) {
|
|
@@ -718,6 +755,9 @@ function installPlaywrightStepLogger(testInfo) {
|
|
|
718
755
|
return () => {
|
|
719
756
|
if (installed) {
|
|
720
757
|
callbacks.onStepEnd = originalOnStepEnd;
|
|
758
|
+
if (originalOnStepBegin) {
|
|
759
|
+
callbacks.onStepBegin = originalOnStepBegin;
|
|
760
|
+
}
|
|
721
761
|
}
|
|
722
762
|
};
|
|
723
763
|
}
|
|
@@ -825,7 +865,8 @@ async function finalizeTest(page, testInfo, logBuffer, videoOption) {
|
|
|
825
865
|
// Attach native Playwright steps (expect assertions, test.step blocks)
|
|
826
866
|
// so the HTML report can show a unified timeline alongside AI tool calls.
|
|
827
867
|
try {
|
|
828
|
-
const
|
|
868
|
+
const startTimes = testInfo.__donobuStepStartTimes ?? new Map();
|
|
869
|
+
const nativeSteps = collectNativeSteps(testInfo._steps ?? [], startTimes);
|
|
829
870
|
if (nativeSteps.length > 0) {
|
|
830
871
|
await testInfo.attach('donobu-native-steps', {
|
|
831
872
|
body: JSON.stringify(nativeSteps),
|
|
@@ -87,6 +87,9 @@ class ToolManager {
|
|
|
87
87
|
startedAt: startedAt,
|
|
88
88
|
completedAt: null,
|
|
89
89
|
});
|
|
90
|
+
// Expose the un-interpolated parameters so tools that need to
|
|
91
|
+
// preserve `{{...}}` references (e.g. AssertTool) can read them.
|
|
92
|
+
context.rawParameters = toolParameters;
|
|
90
93
|
// Use the interpolated parameters when calling the tool.
|
|
91
94
|
toolCallResult = isFromGpt
|
|
92
95
|
? await tool.callFromGpt(context, tool.inputSchemaForGpt.parse(interpolatedParameters))
|
|
@@ -22,5 +22,16 @@ export type ToolCallContext = {
|
|
|
22
22
|
readonly invokedToolCalls: ToolCall[];
|
|
23
23
|
readonly metadata: FlowMetadata;
|
|
24
24
|
readonly toolCallId: string;
|
|
25
|
+
/**
|
|
26
|
+
* Original (un-interpolated) parameters supplied to the current tool call.
|
|
27
|
+
*
|
|
28
|
+
* `ToolManager.invokeTool` interpolates `{{...}}` expressions in the tool's
|
|
29
|
+
* parameters before invoking the tool, but a tool may need the raw text to
|
|
30
|
+
* preserve env var references in its output (e.g. `AssertTool` emits
|
|
31
|
+
* Playwright steps that retain `{{$.env.X}}` so cached replays stay correct
|
|
32
|
+
* across env value changes). Set by `ToolManager` immediately before the
|
|
33
|
+
* tool runs; absent for direct/legacy invocation paths.
|
|
34
|
+
*/
|
|
35
|
+
rawParameters?: Record<string, any>;
|
|
25
36
|
};
|
|
26
37
|
//# sourceMappingURL=ToolCallContext.d.ts.map
|
package/dist/reporter/render.js
CHANGED
|
@@ -562,7 +562,7 @@ function renderErrors(errors) {
|
|
|
562
562
|
}
|
|
563
563
|
return html;
|
|
564
564
|
}
|
|
565
|
-
function renderNativeStep(ns) {
|
|
565
|
+
function renderNativeStep(ns, childrenHtml) {
|
|
566
566
|
const statusIcon = ns.passed
|
|
567
567
|
? '<span class="step-status-ok">✓</span>'
|
|
568
568
|
: '<span class="step-status-fail">✗</span>';
|
|
@@ -573,7 +573,8 @@ function renderNativeStep(ns) {
|
|
|
573
573
|
const snippet = ns.location?.file
|
|
574
574
|
? readSourceSnippet(ns.location.file, ns.location.line)
|
|
575
575
|
: null;
|
|
576
|
-
const
|
|
576
|
+
const hasError = !ns.passed && !!ns.error?.message;
|
|
577
|
+
const hasBody = !!snippet || hasError || !!childrenHtml;
|
|
577
578
|
const renderHeader = (tag) => {
|
|
578
579
|
let header = `<${tag} class="filmstrip-header">`;
|
|
579
580
|
header += statusIcon;
|
|
@@ -592,17 +593,23 @@ function renderNativeStep(ns) {
|
|
|
592
593
|
if (!hasBody) {
|
|
593
594
|
return `<div class="filmstrip-step native-step">${renderHeader('div')}</div>`;
|
|
594
595
|
}
|
|
595
|
-
//
|
|
596
|
-
//
|
|
596
|
+
// Failures always render expanded so the error is immediately visible.
|
|
597
|
+
// test.step blocks with nested content also default open so users see
|
|
598
|
+
// what's inside; bare passing expects with just a snippet collapse to
|
|
599
|
+
// keep tests with many assertions scannable.
|
|
600
|
+
const defaultOpen = !ns.passed || (ns.category === 'test.step' && !!childrenHtml);
|
|
597
601
|
const passClass = ns.passed ? 'native-step--passed' : 'native-step--failed';
|
|
598
|
-
let html = `<details class="filmstrip-step native-step ${passClass}"${
|
|
602
|
+
let html = `<details class="filmstrip-step native-step ${passClass}"${defaultOpen ? ' open' : ''}>`;
|
|
599
603
|
html += renderHeader('summary');
|
|
600
|
-
if (
|
|
604
|
+
if (hasError) {
|
|
601
605
|
html += `<pre class="native-step-error">${ansiToHtml(ns.error.message)}</pre>`;
|
|
602
606
|
}
|
|
603
607
|
if (snippet) {
|
|
604
608
|
html += snippet;
|
|
605
609
|
}
|
|
610
|
+
if (childrenHtml) {
|
|
611
|
+
html += childrenHtml;
|
|
612
|
+
}
|
|
606
613
|
html += `</details>`;
|
|
607
614
|
return html;
|
|
608
615
|
}
|
|
@@ -853,31 +860,71 @@ function renderSteps(steps, stepScreenshots, nativeSteps, outputDir) {
|
|
|
853
860
|
return '';
|
|
854
861
|
}
|
|
855
862
|
if (hasScreenshots || hasNative) {
|
|
856
|
-
const
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
863
|
+
const buildNativeTree = (nss) => nss.map((ns) => ({
|
|
864
|
+
kind: 'native',
|
|
865
|
+
ns,
|
|
866
|
+
t: ns.startWallTime,
|
|
867
|
+
tEnd: ns.endWallTime,
|
|
868
|
+
children: buildNativeTree(ns.children),
|
|
869
|
+
}));
|
|
870
|
+
const roots = buildNativeTree(nativeSteps);
|
|
871
|
+
// Place each Donobu screenshot under the deepest native step whose
|
|
872
|
+
// [start, end] window contains it. Falls back to top level if none.
|
|
873
|
+
const placeDonobu = (nodes, d) => {
|
|
874
|
+
for (const n of nodes) {
|
|
875
|
+
if (n.kind !== 'native') {
|
|
876
|
+
continue;
|
|
877
|
+
}
|
|
878
|
+
if (d.ss.startedAt >= n.t && d.ss.completedAt <= n.tEnd) {
|
|
879
|
+
if (!placeDonobu(n.children, d)) {
|
|
880
|
+
n.children.push(d);
|
|
881
|
+
}
|
|
882
|
+
return true;
|
|
883
|
+
}
|
|
884
|
+
}
|
|
885
|
+
return false;
|
|
886
|
+
};
|
|
887
|
+
for (const ss of stepScreenshots) {
|
|
888
|
+
const d = { kind: 'donobu', ss, t: ss.startedAt };
|
|
889
|
+
if (!placeDonobu(roots, d)) {
|
|
890
|
+
roots.push(d);
|
|
891
|
+
}
|
|
892
|
+
}
|
|
893
|
+
const sortTree = (nodes) => {
|
|
894
|
+
nodes.sort((a, b) => a.t - b.t);
|
|
895
|
+
for (const n of nodes) {
|
|
896
|
+
if (n.kind === 'native') {
|
|
897
|
+
sortTree(n.children);
|
|
898
|
+
}
|
|
899
|
+
}
|
|
900
|
+
};
|
|
901
|
+
sortTree(roots);
|
|
902
|
+
const countNodes = (nodes) => {
|
|
903
|
+
let c = 0;
|
|
904
|
+
for (const n of nodes) {
|
|
905
|
+
c += 1;
|
|
906
|
+
if (n.kind === 'native') {
|
|
907
|
+
c += countNodes(n.children);
|
|
908
|
+
}
|
|
909
|
+
}
|
|
910
|
+
return c;
|
|
911
|
+
};
|
|
912
|
+
const renderNode = (node) => {
|
|
913
|
+
if (node.kind === 'donobu') {
|
|
914
|
+
return renderFilmstripStep(node.ss, outputDir);
|
|
915
|
+
}
|
|
916
|
+
const childrenHtml = node.children.length > 0
|
|
917
|
+
? `<div class="native-step-children">${node.children.map(renderNode).join('')}</div>`
|
|
918
|
+
: '';
|
|
919
|
+
return renderNativeStep(node.ns, childrenHtml);
|
|
920
|
+
};
|
|
921
|
+
const stepCount = countNodes(roots);
|
|
870
922
|
let html = '<details class="steps-section"><summary>Steps (' +
|
|
871
923
|
stepCount +
|
|
872
924
|
')</summary>';
|
|
873
925
|
html += '<div class="step-filmstrip">';
|
|
874
|
-
for (const
|
|
875
|
-
|
|
876
|
-
html += renderFilmstripStep(entry.ss, outputDir);
|
|
877
|
-
}
|
|
878
|
-
else {
|
|
879
|
-
html += renderNativeStep(entry.ns);
|
|
880
|
-
}
|
|
926
|
+
for (const node of roots) {
|
|
927
|
+
html += renderNode(node);
|
|
881
928
|
}
|
|
882
929
|
html += '</div>';
|
|
883
930
|
html += '</details>';
|
|
@@ -1610,6 +1657,8 @@ details.native-step>summary::-webkit-details-marker{display:none}
|
|
|
1610
1657
|
details.native-step[open]>summary .native-step-chevron{transform:rotate(90deg)}
|
|
1611
1658
|
.native-step-error{font-size:11px;font-family:var(--mono);padding:4px 0 2px 22px;margin:0;white-space:pre-wrap;word-break:break-word;color:var(--text-muted)}
|
|
1612
1659
|
.native-step-snippet{font-size:11px;font-family:var(--mono);margin:4px 0 2px 22px;overflow:hidden}
|
|
1660
|
+
.native-step-children{display:flex;flex-direction:column;margin:4px 0 0 10px;border-left:1px solid var(--border-subtle);padding-left:8px}
|
|
1661
|
+
.native-step-children>.filmstrip-step{padding-left:8px}
|
|
1613
1662
|
.snippet-line{display:flex;padding:1px 8px;white-space:pre}
|
|
1614
1663
|
.snippet-line--target{background:rgba(239,68,68,.10)}
|
|
1615
1664
|
.snippet-linenum{color:var(--text-dim);min-width:40px;user-select:none}
|
package/dist/tools/AssertTool.js
CHANGED
|
@@ -124,6 +124,36 @@ It will use a screenshot of the current viewport of the webpage, the webpage's t
|
|
|
124
124
|
Logger_1.appLogger.warn(msg);
|
|
125
125
|
return msg;
|
|
126
126
|
});
|
|
127
|
+
const rawAssertion = typeof context.rawParameters?.assertionToTestFor === 'string'
|
|
128
|
+
? context.rawParameters.assertionToTestFor
|
|
129
|
+
: parameters.assertionToTestFor;
|
|
130
|
+
const envEntries = Object.entries(context.envData ?? {});
|
|
131
|
+
// Only treat env vars as "in play" when the raw assertion actually
|
|
132
|
+
// references one — keeps the prompt small for the common case.
|
|
133
|
+
const referencedEnvEntries = envEntries.filter(([name]) => rawAssertion.includes(`{{$.env.${name}}}`));
|
|
134
|
+
const hasEnvRefs = referencedEnvEntries.length > 0;
|
|
135
|
+
const envBlock = hasEnvRefs
|
|
136
|
+
? `
|
|
137
|
+
|
|
138
|
+
The user's original assertion contains environment variable references using the
|
|
139
|
+
syntax \`{{$.env.NAME}}\`. To keep cached Playwright steps valid across runs with
|
|
140
|
+
different env values, you MUST emit those same placeholders in any
|
|
141
|
+
playwrightAssertionStep \`value\`/\`attributeValue\` field whose contents come from
|
|
142
|
+
an env var. Do NOT bake the literal current value into the step.
|
|
143
|
+
|
|
144
|
+
Original (uninterpolated) assertion: ${rawAssertion}
|
|
145
|
+
|
|
146
|
+
Current env mapping (use these to identify which substrings on the page came
|
|
147
|
+
from which env var, then emit the placeholder rather than the literal):
|
|
148
|
+
${referencedEnvEntries.map(([name, value]) => ` - {{$.env.${name}}} = ${JSON.stringify(value)}`).join('\n')}
|
|
149
|
+
|
|
150
|
+
Examples:
|
|
151
|
+
- Raw assertion "Welcome banner says hello {{$.env.USERNAME}}", USERNAME="alice", page shows "Welcome alice" →
|
|
152
|
+
[{ locator: "text", role: null, value: "{{$.env.USERNAME}}", valueIsRegex: false, assertion: "toBeVisible", attributeValue: null }]
|
|
153
|
+
- Raw assertion "The username field shows {{$.env.USERNAME}}", USERNAME="alice", page input value is "alice" →
|
|
154
|
+
[{ locator: "label", role: null, value: "Username", valueIsRegex: false, assertion: "toHaveValue", attributeValue: "{{$.env.USERNAME}}" }]
|
|
155
|
+
- For literal page text unrelated to env vars, keep the literal value as usual.`
|
|
156
|
+
: '';
|
|
127
157
|
const promptMessages = [
|
|
128
158
|
{
|
|
129
159
|
type: 'system',
|
|
@@ -142,7 +172,7 @@ CRITICAL RULES for generating structured steps — follow these precisely:
|
|
|
142
172
|
- Text input / textarea content: use 'toHaveValue' with locator='label' and set attributeValue to the expected text. Do NOT use 'toBeVisible' on the textbox.
|
|
143
173
|
- Selected tabs, pills, or items with aria-selected: use 'toHaveAttribute' with value='aria-selected' and attributeValue='true', NOT 'toBeVisible' on the text.
|
|
144
174
|
- Text content within an element: use 'toContainText' with attributeValue set to the substring, NOT 'toBeVisible'.
|
|
145
|
-
- Only use 'toBeVisible' when the assertion is genuinely about whether something is visible — not as a fallback for state or value checks
|
|
175
|
+
- Only use 'toBeVisible' when the assertion is genuinely about whether something is visible — not as a fallback for state or value checks.${envBlock}`,
|
|
146
176
|
},
|
|
147
177
|
{
|
|
148
178
|
type: 'user',
|
|
@@ -184,7 +214,7 @@ careful positioning lost, etc. A screenshot of the webpage has also been provide
|
|
|
184
214
|
verifiedSteps.length > 0) {
|
|
185
215
|
try {
|
|
186
216
|
const executor = (0, assertCache_1.buildAssertExecutor)(verifiedSteps);
|
|
187
|
-
await executor({ page: page });
|
|
217
|
+
await executor({ page: page, envData: context.envData });
|
|
188
218
|
}
|
|
189
219
|
catch (error) {
|
|
190
220
|
Logger_1.appLogger.debug(`Structured assertion steps failed verification for: "${parameters.assertionToTestFor}" — discarding steps. Error: ${error.message}`);
|