explorbot 0.4.9 → 0.4.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/package.json +1 -1
- package/dist/src/action.js +4 -2
- package/dist/src/ai/navigator.d.ts +0 -1
- package/dist/src/ai/navigator.js +11 -24
- package/dist/src/ai/pilot.js +10 -4
- package/dist/src/ai/rerunner.js +7 -0
- package/dist/src/api/xhr-capture.js +2 -1
- package/dist/src/explorer.js +2 -3
- package/dist/src/reporter.js +8 -4
- package/dist/src/utils/logger.js +1 -1
- package/dist/src/utils/step-analyzer.d.ts +3 -0
- package/dist/src/utils/step-analyzer.js +7 -0
- package/dist/src/utils/url-matcher.d.ts +1 -0
- package/dist/src/utils/url-matcher.js +7 -0
- package/package.json +1 -1
- package/src/action.ts +4 -2
- package/src/ai/navigator.ts +9 -23
- package/src/ai/pilot.ts +10 -4
- package/src/ai/rerunner.ts +4 -0
- package/src/api/xhr-capture.ts +2 -1
- package/src/explorer.ts +2 -2
- package/src/reporter.ts +8 -4
- package/src/utils/logger.ts +1 -1
- package/src/utils/step-analyzer.ts +8 -0
- package/src/utils/url-matcher.ts +7 -0
package/dist/package.json
CHANGED
package/dist/src/action.js
CHANGED
|
@@ -12,7 +12,9 @@ import { captureHtmlForSnapshot, htmlCombinedSnapshot, minifyHtml } from './util
|
|
|
12
12
|
import { createDebug, setStepSpanParent, tag } from './utils/logger.js';
|
|
13
13
|
import { Overlay, OverlayPage } from './utils/overlay.js';
|
|
14
14
|
import { sleep, waitForPageReadiness } from "./utils/page-readiness.js";
|
|
15
|
+
import { isInternalStep } from "./utils/step-analyzer.js";
|
|
15
16
|
import { safeFilename } from "./utils/strings.js";
|
|
17
|
+
import { isSameHostFamily } from './utils/url-matcher.js';
|
|
16
18
|
import { codeceptJSSandbox, hasPlaywrightCommands, playwrightSandbox, sanitizeCodeBlock } from "./utils/web-sandbox.js";
|
|
17
19
|
const debugLog = createDebug('explorbot:action');
|
|
18
20
|
const CAPTURE_NAVIGATION_TRANSITION_ATTEMPTS = 3;
|
|
@@ -303,7 +305,7 @@ class Action {
|
|
|
303
305
|
const url = URL.parse(request.url());
|
|
304
306
|
if (!url)
|
|
305
307
|
return;
|
|
306
|
-
if (url.
|
|
308
|
+
if (!isSameHostFamily(url.href, this.baseOrigin))
|
|
307
309
|
return;
|
|
308
310
|
const call = { method: request.method(), path: url.pathname, status };
|
|
309
311
|
if (this.networkRequests.some((r) => r.method === call.method && r.path === call.path && r.status === call.status))
|
|
@@ -523,7 +525,7 @@ export const attachStepLogger = (target, assertionsTarget) => {
|
|
|
523
525
|
const listener = (step, error) => {
|
|
524
526
|
if (!step?.toCode)
|
|
525
527
|
return;
|
|
526
|
-
if (step
|
|
528
|
+
if (isInternalStep(step))
|
|
527
529
|
return;
|
|
528
530
|
const existing = recorded.get(step);
|
|
529
531
|
if (existing) {
|
|
@@ -26,7 +26,6 @@ declare class Navigator implements Agent {
|
|
|
26
26
|
constructor(deps: AgentDeps);
|
|
27
27
|
get verifyAttempts(): number;
|
|
28
28
|
get verifyTimeout(): number;
|
|
29
|
-
getBaseOrigin(): string | null;
|
|
30
29
|
getComparableCurrentUrl(stateManager: any, expectedUrl: string): string;
|
|
31
30
|
comparableUrl(state: {
|
|
32
31
|
url?: string;
|
package/dist/src/ai/navigator.js
CHANGED
|
@@ -12,7 +12,7 @@ import { createDebug, pluralize, tag } from '../utils/logger.js';
|
|
|
12
12
|
import { loop, pause } from '../utils/loop.js';
|
|
13
13
|
import { RulesLoader } from "../utils/rules-loader.js";
|
|
14
14
|
import { normalizeInlineText } from "../utils/strings.js";
|
|
15
|
-
import { extractStatePath, matchesNavigationUrl } from '../utils/url-matcher.js';
|
|
15
|
+
import { extractStatePath, isSameHostFamily, matchesNavigationUrl } from '../utils/url-matcher.js';
|
|
16
16
|
import { Researcher } from "./researcher.js";
|
|
17
17
|
import { actionRule, locatorRule, unexpectedPopupRule } from './rules.js';
|
|
18
18
|
import { isInteractive } from './task-agent.js';
|
|
@@ -82,15 +82,6 @@ class Navigator {
|
|
|
82
82
|
get verifyTimeout() {
|
|
83
83
|
return this.config.ai?.agents?.navigator?.verifyTimeout ?? 1500;
|
|
84
84
|
}
|
|
85
|
-
getBaseOrigin() {
|
|
86
|
-
const baseUrl = this.config.playwright.url;
|
|
87
|
-
try {
|
|
88
|
-
return new URL(baseUrl).origin;
|
|
89
|
-
}
|
|
90
|
-
catch {
|
|
91
|
-
return null;
|
|
92
|
-
}
|
|
93
|
-
}
|
|
94
85
|
getComparableCurrentUrl(stateManager, expectedUrl) {
|
|
95
86
|
const currentState = stateManager.getCurrentState();
|
|
96
87
|
if (!currentState)
|
|
@@ -109,19 +100,14 @@ class Navigator {
|
|
|
109
100
|
const currentFullUrl = currentState.fullUrl || currentState.url || '';
|
|
110
101
|
if (!currentFullUrl)
|
|
111
102
|
return false;
|
|
112
|
-
|
|
113
|
-
const currentOrigin = new URL(currentFullUrl).origin;
|
|
114
|
-
if (/^https?:\/\//i.test(expectedUrl)) {
|
|
115
|
-
return currentOrigin === new URL(expectedUrl).origin;
|
|
116
|
-
}
|
|
117
|
-
const baseOrigin = this.getBaseOrigin();
|
|
118
|
-
if (!baseOrigin)
|
|
119
|
-
return true;
|
|
120
|
-
return currentOrigin === baseOrigin;
|
|
121
|
-
}
|
|
122
|
-
catch {
|
|
103
|
+
if (!/^https?:\/\//i.test(currentFullUrl))
|
|
123
104
|
return !/^https?:\/\//i.test(expectedUrl);
|
|
124
|
-
|
|
105
|
+
if (/^https?:\/\//i.test(expectedUrl))
|
|
106
|
+
return isSameHostFamily(currentFullUrl, expectedUrl);
|
|
107
|
+
const baseUrl = this.config.playwright.url;
|
|
108
|
+
if (!baseUrl)
|
|
109
|
+
return true;
|
|
110
|
+
return isSameHostFamily(currentFullUrl, baseUrl);
|
|
125
111
|
}
|
|
126
112
|
isOnExpectedPage(expectedUrl, stateManager) {
|
|
127
113
|
if (!this.isSameExpectedOrigin(expectedUrl, stateManager)) {
|
|
@@ -289,8 +275,9 @@ class Navigator {
|
|
|
289
275
|
tag('warning').log(`Page state did not change at ${check.freshState.url}`);
|
|
290
276
|
}
|
|
291
277
|
else {
|
|
292
|
-
|
|
293
|
-
|
|
278
|
+
const reachedUrl = check.freshState.fullUrl || check.freshState.url;
|
|
279
|
+
lastFailure = `Reached ${reachedUrl}, expected ${expectedUrl}`;
|
|
280
|
+
tag('warning').log(`URL verification failed: expected ${expectedUrl}, got ${reachedUrl}`);
|
|
294
281
|
}
|
|
295
282
|
batchFailures.push({
|
|
296
283
|
code: codeBlock,
|
package/dist/src/ai/pilot.js
CHANGED
|
@@ -95,7 +95,9 @@ export class Pilot {
|
|
|
95
95
|
}
|
|
96
96
|
}
|
|
97
97
|
const schema = z.object({
|
|
98
|
-
decision: z
|
|
98
|
+
decision: z
|
|
99
|
+
.enum(['pass', 'fail', 'continue', 'skipped'])
|
|
100
|
+
.describe('pass = scenario goal accomplished, fail = the app misbehaved, continue = tester should keep going, skipped = the scenario cannot be judged against this app (its premise does not hold, it is irrelevant, or systematic execution failures prevented testing)'),
|
|
99
101
|
reason: z.string().describe('Concise user-facing reason, maximum 1 short sentence and 120 characters. Do NOT repeat the decision status; explain only the evidence. For continue: explain why rejected and suggest alternatives.'),
|
|
100
102
|
guidance: z.string().nullable().describe('Required for "continue": specific actionable instruction for the tester — what exactly to verify, retry differently, or complete next. Be concrete.'),
|
|
101
103
|
requestVerification: z
|
|
@@ -358,9 +360,13 @@ export class Pilot {
|
|
|
358
360
|
DOM assertion can't be made.
|
|
359
361
|
Do not pass when Tester achieved only a related navigation/filter/tab/status outcome instead of the
|
|
360
362
|
requested action, workflow, or entity detail goal.
|
|
361
|
-
- "fail":
|
|
362
|
-
|
|
363
|
-
|
|
363
|
+
- "fail": the app misbehaved — the scenario's action ran against the right target and the app
|
|
364
|
+
produced a wrong, broken, or missing outcome. Not reaching the goal is not by itself a fail.
|
|
365
|
+
- "skipped": the scenario cannot be judged against this app — the page shows its premise does not
|
|
366
|
+
hold (the assumed constraint, field, or behaviour is designed differently), the target entity or
|
|
367
|
+
feature is not the one here, the scenario is irrelevant, OR systematic infrastructure failures
|
|
368
|
+
(LLM errors, crashes) prevented testing. NOT for "test failed to interact" — that's "fail" or
|
|
369
|
+
"continue".
|
|
364
370
|
- "continue": goal incomplete but the control for the NEXT step is present on the current page, or a
|
|
365
371
|
concrete missing check would change your verdict. Guidance must name that step.
|
|
366
372
|
If a verify() asserted a state that was ALREADY TRUE before the test, it proves nothing — reject.
|
package/dist/src/ai/rerunner.js
CHANGED
|
@@ -18,6 +18,7 @@ import { formatHeadings } from "../utils/context-formatter.js";
|
|
|
18
18
|
import { createDebug, tag } from "../utils/logger.js";
|
|
19
19
|
import { loop } from "../utils/loop.js";
|
|
20
20
|
import { RulesLoader } from "../utils/rules-loader.js";
|
|
21
|
+
import { isInternalStep } from "../utils/step-analyzer.js";
|
|
21
22
|
import { toolExecutionLabel } from "./conversation.js";
|
|
22
23
|
import { actionRule, locatorRule, sectionContextRule } from "./rules.js";
|
|
23
24
|
import { TaskAgent } from "./task-agent.js";
|
|
@@ -70,6 +71,8 @@ export class Rerunner extends TaskAgent {
|
|
|
70
71
|
const onStepStarted = (step) => {
|
|
71
72
|
if (!step.toCode)
|
|
72
73
|
return;
|
|
74
|
+
if (isInternalStep(step))
|
|
75
|
+
return;
|
|
73
76
|
const code = highlight(step.toCode(), { language: 'javascript' });
|
|
74
77
|
console.log(chalk.dim(` ${code}`));
|
|
75
78
|
};
|
|
@@ -77,12 +80,16 @@ export class Rerunner extends TaskAgent {
|
|
|
77
80
|
const task = this.getCurrentTask(testMap);
|
|
78
81
|
if (!task || !step.toCode)
|
|
79
82
|
return;
|
|
83
|
+
if (isInternalStep(step))
|
|
84
|
+
return;
|
|
80
85
|
task.addStep(step.toCode(), step.duration, 'passed');
|
|
81
86
|
};
|
|
82
87
|
const onStepFailed = (step, error) => {
|
|
83
88
|
const task = this.getCurrentTask(testMap);
|
|
84
89
|
if (!task || !step.toCode)
|
|
85
90
|
return;
|
|
91
|
+
if (isInternalStep(step))
|
|
92
|
+
return;
|
|
86
93
|
task.addStep(step.toCode(), step.duration, 'failed', error?.message);
|
|
87
94
|
console.log(chalk.red(` ${figureSet.cross} ${step.toCode()} — ${error?.message || 'failed'}`));
|
|
88
95
|
};
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { isSameHostFamily } from '../utils/url-matcher.js';
|
|
1
2
|
import { RequestResult, generateRequestId } from "./request-result.js";
|
|
2
3
|
const WRITE_METHODS = new Set(['POST', 'PUT', 'PATCH', 'DELETE']);
|
|
3
4
|
const JSON_CONTENT_TYPES = /application\/json|application\/.*\+json/i;
|
|
@@ -33,7 +34,7 @@ export class XhrCapture {
|
|
|
33
34
|
return;
|
|
34
35
|
const method = request.method();
|
|
35
36
|
const url = request.url();
|
|
36
|
-
if (!url
|
|
37
|
+
if (!isSameHostFamily(url, this.baseOrigin))
|
|
37
38
|
return;
|
|
38
39
|
const status = response.status();
|
|
39
40
|
if (status >= 400) {
|
package/dist/src/explorer.js
CHANGED
|
@@ -13,6 +13,7 @@ import { ConfigParser } from './config.js';
|
|
|
13
13
|
import { BrowserRecoveryError, browserErrorMessage, isFatalBrowserError, isNavigationTransitionError } from "./utils/browser-errors.js";
|
|
14
14
|
import { createDebug, log, tag } from './utils/logger.js';
|
|
15
15
|
import { sleep, waitForPageReadiness } from "./utils/page-readiness.js";
|
|
16
|
+
import { isInternalStep } from "./utils/step-analyzer.js";
|
|
16
17
|
const debugLog = createDebug('explorbot:explorer');
|
|
17
18
|
const RECOVERABLE_NAVIGATION_ERRORS = /net::ERR_ABORTED|page\.screenshot.*Timeout|waiting for fonts to load/i;
|
|
18
19
|
const RECOVERY_NAVIGATION = { waitUntil: 'domcontentloaded', timeout: 10000 };
|
|
@@ -161,9 +162,7 @@ class Explorer {
|
|
|
161
162
|
const stepHandler = (step, status, error, log) => {
|
|
162
163
|
if (!step.toCode)
|
|
163
164
|
return;
|
|
164
|
-
if (step
|
|
165
|
-
return;
|
|
166
|
-
if (step?.name?.startsWith('save'))
|
|
165
|
+
if (isInternalStep(step))
|
|
167
166
|
return;
|
|
168
167
|
test.addStep(step.toCode(), step.duration, status, error, log);
|
|
169
168
|
if (!this.stateManager.getCurrentState())
|
package/dist/src/reporter.js
CHANGED
|
@@ -111,6 +111,7 @@ export class Reporter {
|
|
|
111
111
|
}
|
|
112
112
|
combineStepsAndNotes(test, lastScreenshotFile) {
|
|
113
113
|
const noteEntries = Object.entries(test.notes)
|
|
114
|
+
.filter(([, note]) => !note.observation)
|
|
114
115
|
.map(([timestampKey, note]) => ({
|
|
115
116
|
startTime: note.startTime,
|
|
116
117
|
endTime: note.endTime,
|
|
@@ -233,9 +234,12 @@ export class Reporter {
|
|
|
233
234
|
description: test.description,
|
|
234
235
|
code: test.generatedCode || '',
|
|
235
236
|
steps,
|
|
236
|
-
logs:
|
|
237
|
-
.map((stepData) => stepData.text)
|
|
238
|
-
.
|
|
237
|
+
logs: [
|
|
238
|
+
...Object.values(test.steps).map((stepData) => stepData.text),
|
|
239
|
+
...Object.values(test.notes)
|
|
240
|
+
.filter((note) => note.observation)
|
|
241
|
+
.map((note) => note.message),
|
|
242
|
+
].join('\n'),
|
|
239
243
|
files: Object.values(test.artifacts) || [],
|
|
240
244
|
message: test.summary || this.extractLastNoteMessage(test) || '',
|
|
241
245
|
meta,
|
|
@@ -300,7 +304,7 @@ export class Reporter {
|
|
|
300
304
|
}
|
|
301
305
|
}
|
|
302
306
|
extractLastNoteMessage(test) {
|
|
303
|
-
const notes = Object.values(test.notes);
|
|
307
|
+
const notes = Object.values(test.notes).filter((note) => !note.observation);
|
|
304
308
|
if (notes.length === 0)
|
|
305
309
|
return '';
|
|
306
310
|
return notes[notes.length - 1].message;
|
package/dist/src/utils/logger.js
CHANGED
|
@@ -189,7 +189,7 @@ class SpanDestination {
|
|
|
189
189
|
if (!step?.toCode) {
|
|
190
190
|
return;
|
|
191
191
|
}
|
|
192
|
-
const stepName = step?.
|
|
192
|
+
const stepName = step?.title ? `I.${step.title}` : 'I.step';
|
|
193
193
|
const stepInput = typeof step?.toCode === 'function' ? step.toCode() : entry.content;
|
|
194
194
|
const errorFromStep = step?.error;
|
|
195
195
|
const errorMessage = stepError && typeof stepError === 'object' && 'message' in stepError && typeof stepError.message === 'string'
|
|
@@ -9,3 +9,6 @@ export declare function stripComments(code: string): string;
|
|
|
9
9
|
export declare function isNonReusableCode(code: string): boolean;
|
|
10
10
|
export declare function toReusableSessionStep(step: StepData): SessionStep | null;
|
|
11
11
|
export declare function mergeUniqueStepsByCode(primary: SessionStep[], secondary: SessionStep[]): SessionStep[];
|
|
12
|
+
export declare function isInternalStep(step: {
|
|
13
|
+
title?: string;
|
|
14
|
+
}): boolean;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { isDynamicId } from "./xpath.js";
|
|
2
2
|
export const CODECEPT_TOOLS = ['click', 'hover', 'pressKey', 'form'];
|
|
3
|
+
const INTERNAL_STEP_PREFIXES = ['grab', 'save'];
|
|
3
4
|
const CODECEPT_FORM_COMMANDS = ['I.fillField', 'I.type', 'I.selectOption', 'I.attachFile', 'I.checkOption', 'I.uncheckOption'];
|
|
4
5
|
export function isCodeceptToolName(toolName) {
|
|
5
6
|
return CODECEPT_TOOLS.includes(toolName);
|
|
@@ -66,3 +67,9 @@ export function mergeUniqueStepsByCode(primary, secondary) {
|
|
|
66
67
|
}
|
|
67
68
|
return merged;
|
|
68
69
|
}
|
|
70
|
+
export function isInternalStep(step) {
|
|
71
|
+
const title = step?.title;
|
|
72
|
+
if (!title)
|
|
73
|
+
return false;
|
|
74
|
+
return INTERNAL_STEP_PREFIXES.some((prefix) => title.startsWith(prefix));
|
|
75
|
+
}
|
|
@@ -6,3 +6,4 @@ export declare function generalizeUrl(url: string, replaceSegment?: (segment: st
|
|
|
6
6
|
export declare function matchesUrl(pattern: string, path: string): boolean;
|
|
7
7
|
export declare function extractStatePath(url: string): string;
|
|
8
8
|
export declare function matchesNavigationUrl(expected: string, current: string): boolean;
|
|
9
|
+
export declare function isSameHostFamily(urlA: string, urlB: string): boolean;
|
|
@@ -145,3 +145,10 @@ export function matchesNavigationUrl(expected, current) {
|
|
|
145
145
|
.filter(Boolean);
|
|
146
146
|
return recordSegments.length > 0 && recordSegments.every(isDynamicSegment);
|
|
147
147
|
}
|
|
148
|
+
export function isSameHostFamily(urlA, urlB) {
|
|
149
|
+
const hostA = URL.parse(urlA)?.host.toLowerCase();
|
|
150
|
+
const hostB = URL.parse(urlB)?.host.toLowerCase();
|
|
151
|
+
if (!hostA || !hostB)
|
|
152
|
+
return false;
|
|
153
|
+
return hostA === hostB || hostA.endsWith(`.${hostB}`) || hostB.endsWith(`.${hostA}`);
|
|
154
|
+
}
|
package/package.json
CHANGED
package/src/action.ts
CHANGED
|
@@ -16,7 +16,9 @@ import { createDebug, setStepSpanParent, tag } from './utils/logger.js';
|
|
|
16
16
|
import { Overlay, OverlayPage } from './utils/overlay.js';
|
|
17
17
|
import { sleep, waitForPageReadiness } from './utils/page-readiness.ts';
|
|
18
18
|
import type { Region } from './utils/region.js';
|
|
19
|
+
import { isInternalStep } from './utils/step-analyzer.ts';
|
|
19
20
|
import { safeFilename } from './utils/strings.ts';
|
|
21
|
+
import { isSameHostFamily } from './utils/url-matcher.js';
|
|
20
22
|
import { codeceptJSSandbox, hasPlaywrightCommands, playwrightSandbox, sanitizeCodeBlock } from './utils/web-sandbox.ts';
|
|
21
23
|
|
|
22
24
|
const debugLog = createDebug('explorbot:action');
|
|
@@ -317,7 +319,7 @@ class Action {
|
|
|
317
319
|
|
|
318
320
|
const url = URL.parse(request.url());
|
|
319
321
|
if (!url) return;
|
|
320
|
-
if (url.
|
|
322
|
+
if (!isSameHostFamily(url.href, this.baseOrigin)) return;
|
|
321
323
|
|
|
322
324
|
const call: NetworkCall = { method: request.method(), path: url.pathname, status };
|
|
323
325
|
if (this.networkRequests.some((r) => r.method === call.method && r.path === call.path && r.status === call.status)) return;
|
|
@@ -563,7 +565,7 @@ export const attachStepLogger = (target: ExecutedStep[], assertionsTarget?: Arra
|
|
|
563
565
|
let batchFailed = false;
|
|
564
566
|
const listener: StepListener = (step, error) => {
|
|
565
567
|
if (!step?.toCode) return;
|
|
566
|
-
if (step
|
|
568
|
+
if (isInternalStep(step)) return;
|
|
567
569
|
|
|
568
570
|
const existing = recorded.get(step);
|
|
569
571
|
if (existing) {
|
package/src/ai/navigator.ts
CHANGED
|
@@ -18,7 +18,7 @@ import { createDebug, pluralize, tag } from '../utils/logger.js';
|
|
|
18
18
|
import { loop, pause } from '../utils/loop.js';
|
|
19
19
|
import { RulesLoader } from '../utils/rules-loader.ts';
|
|
20
20
|
import { normalizeInlineText } from '../utils/strings.ts';
|
|
21
|
-
import { extractStatePath, matchesNavigationUrl } from '../utils/url-matcher.js';
|
|
21
|
+
import { extractStatePath, isSameHostFamily, matchesNavigationUrl } from '../utils/url-matcher.js';
|
|
22
22
|
import type { Agent, AgentDeps } from './agent.js';
|
|
23
23
|
import type { Conversation } from './conversation.js';
|
|
24
24
|
import type { Provider } from './provider.js';
|
|
@@ -99,15 +99,6 @@ class Navigator implements Agent {
|
|
|
99
99
|
return this.config.ai?.agents?.navigator?.verifyTimeout ?? 1500;
|
|
100
100
|
}
|
|
101
101
|
|
|
102
|
-
private getBaseOrigin(): string | null {
|
|
103
|
-
const baseUrl = this.config.playwright.url;
|
|
104
|
-
try {
|
|
105
|
-
return new URL(baseUrl).origin;
|
|
106
|
-
} catch {
|
|
107
|
-
return null;
|
|
108
|
-
}
|
|
109
|
-
}
|
|
110
|
-
|
|
111
102
|
private getComparableCurrentUrl(stateManager: any, expectedUrl: string): string {
|
|
112
103
|
const currentState = stateManager.getCurrentState();
|
|
113
104
|
if (!currentState) return '';
|
|
@@ -126,18 +117,12 @@ class Navigator implements Agent {
|
|
|
126
117
|
const currentFullUrl = currentState.fullUrl || currentState.url || '';
|
|
127
118
|
if (!currentFullUrl) return false;
|
|
128
119
|
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
if (/^https?:\/\//i.test(expectedUrl)) {
|
|
132
|
-
return currentOrigin === new URL(expectedUrl).origin;
|
|
133
|
-
}
|
|
120
|
+
if (!/^https?:\/\//i.test(currentFullUrl)) return !/^https?:\/\//i.test(expectedUrl);
|
|
121
|
+
if (/^https?:\/\//i.test(expectedUrl)) return isSameHostFamily(currentFullUrl, expectedUrl);
|
|
134
122
|
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
} catch {
|
|
139
|
-
return !/^https?:\/\//i.test(expectedUrl);
|
|
140
|
-
}
|
|
123
|
+
const baseUrl = this.config.playwright.url;
|
|
124
|
+
if (!baseUrl) return true;
|
|
125
|
+
return isSameHostFamily(currentFullUrl, baseUrl);
|
|
141
126
|
}
|
|
142
127
|
|
|
143
128
|
private isOnExpectedPage(expectedUrl: string, stateManager: any): boolean {
|
|
@@ -325,8 +310,9 @@ class Navigator implements Agent {
|
|
|
325
310
|
lastFailure = `Reached ${check.freshState.url} but the page state did not change`;
|
|
326
311
|
tag('warning').log(`Page state did not change at ${check.freshState.url}`);
|
|
327
312
|
} else {
|
|
328
|
-
|
|
329
|
-
|
|
313
|
+
const reachedUrl = check.freshState.fullUrl || check.freshState.url;
|
|
314
|
+
lastFailure = `Reached ${reachedUrl}, expected ${expectedUrl}`;
|
|
315
|
+
tag('warning').log(`URL verification failed: expected ${expectedUrl}, got ${reachedUrl}`);
|
|
330
316
|
}
|
|
331
317
|
batchFailures.push({
|
|
332
318
|
code: codeBlock,
|
package/src/ai/pilot.ts
CHANGED
|
@@ -118,7 +118,9 @@ export class Pilot implements Agent {
|
|
|
118
118
|
}
|
|
119
119
|
|
|
120
120
|
const schema = z.object({
|
|
121
|
-
decision: z
|
|
121
|
+
decision: z
|
|
122
|
+
.enum(['pass', 'fail', 'continue', 'skipped'])
|
|
123
|
+
.describe('pass = scenario goal accomplished, fail = the app misbehaved, continue = tester should keep going, skipped = the scenario cannot be judged against this app (its premise does not hold, it is irrelevant, or systematic execution failures prevented testing)'),
|
|
122
124
|
reason: z.string().describe('Concise user-facing reason, maximum 1 short sentence and 120 characters. Do NOT repeat the decision status; explain only the evidence. For continue: explain why rejected and suggest alternatives.'),
|
|
123
125
|
guidance: z.string().nullable().describe('Required for "continue": specific actionable instruction for the tester — what exactly to verify, retry differently, or complete next. Be concrete.'),
|
|
124
126
|
requestVerification: z
|
|
@@ -407,9 +409,13 @@ export class Pilot implements Agent {
|
|
|
407
409
|
DOM assertion can't be made.
|
|
408
410
|
Do not pass when Tester achieved only a related navigation/filter/tab/status outcome instead of the
|
|
409
411
|
requested action, workflow, or entity detail goal.
|
|
410
|
-
- "fail":
|
|
411
|
-
|
|
412
|
-
|
|
412
|
+
- "fail": the app misbehaved — the scenario's action ran against the right target and the app
|
|
413
|
+
produced a wrong, broken, or missing outcome. Not reaching the goal is not by itself a fail.
|
|
414
|
+
- "skipped": the scenario cannot be judged against this app — the page shows its premise does not
|
|
415
|
+
hold (the assumed constraint, field, or behaviour is designed differently), the target entity or
|
|
416
|
+
feature is not the one here, the scenario is irrelevant, OR systematic infrastructure failures
|
|
417
|
+
(LLM errors, crashes) prevented testing. NOT for "test failed to interact" — that's "fail" or
|
|
418
|
+
"continue".
|
|
413
419
|
- "continue": goal incomplete but the control for the NEXT step is present on the current page, or a
|
|
414
420
|
concrete missing check would change your verdict. Guidance must name that step.
|
|
415
421
|
If a verify() asserted a state that was ALREADY TRUE before the test, it proves nothing — reject.
|
package/src/ai/rerunner.ts
CHANGED
|
@@ -18,6 +18,7 @@ import { formatHeadings } from '../utils/context-formatter.ts';
|
|
|
18
18
|
import { createDebug, tag } from '../utils/logger.ts';
|
|
19
19
|
import { loop } from '../utils/loop.ts';
|
|
20
20
|
import { RulesLoader } from '../utils/rules-loader.ts';
|
|
21
|
+
import { isInternalStep } from '../utils/step-analyzer.ts';
|
|
21
22
|
import type { Agent, AgentDeps } from './agent.ts';
|
|
22
23
|
import { toolExecutionLabel } from './conversation.ts';
|
|
23
24
|
import type { Navigator } from './navigator.ts';
|
|
@@ -85,6 +86,7 @@ export class Rerunner extends TaskAgent implements Agent {
|
|
|
85
86
|
|
|
86
87
|
const onStepStarted = (step: any) => {
|
|
87
88
|
if (!step.toCode) return;
|
|
89
|
+
if (isInternalStep(step)) return;
|
|
88
90
|
const code = highlight(step.toCode(), { language: 'javascript' });
|
|
89
91
|
console.log(chalk.dim(` ${code}`));
|
|
90
92
|
};
|
|
@@ -92,12 +94,14 @@ export class Rerunner extends TaskAgent implements Agent {
|
|
|
92
94
|
const onStepPassed = (step: any) => {
|
|
93
95
|
const task = this.getCurrentTask(testMap);
|
|
94
96
|
if (!task || !step.toCode) return;
|
|
97
|
+
if (isInternalStep(step)) return;
|
|
95
98
|
task.addStep(step.toCode(), step.duration, 'passed');
|
|
96
99
|
};
|
|
97
100
|
|
|
98
101
|
const onStepFailed = (step: any, error: any) => {
|
|
99
102
|
const task = this.getCurrentTask(testMap);
|
|
100
103
|
if (!task || !step.toCode) return;
|
|
104
|
+
if (isInternalStep(step)) return;
|
|
101
105
|
task.addStep(step.toCode(), step.duration, 'failed', error?.message);
|
|
102
106
|
console.log(chalk.red(` ${figureSet.cross} ${step.toCode()} — ${error?.message || 'failed'}`));
|
|
103
107
|
};
|
package/src/api/xhr-capture.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { isSameHostFamily } from '../utils/url-matcher.js';
|
|
1
2
|
import { RequestResult, generateRequestId } from './request-result.ts';
|
|
2
3
|
import type { RequestStore } from './request-store.ts';
|
|
3
4
|
|
|
@@ -39,7 +40,7 @@ export class XhrCapture {
|
|
|
39
40
|
|
|
40
41
|
const method = request.method();
|
|
41
42
|
const url = request.url();
|
|
42
|
-
if (!url
|
|
43
|
+
if (!isSameHostFamily(url, this.baseOrigin)) return;
|
|
43
44
|
|
|
44
45
|
const status = response.status();
|
|
45
46
|
|
package/src/explorer.ts
CHANGED
|
@@ -21,6 +21,7 @@ import { Test, TestResult } from './test-plan.ts';
|
|
|
21
21
|
import { BrowserRecoveryError, browserErrorMessage, isFatalBrowserError, isNavigationTransitionError } from './utils/browser-errors.ts';
|
|
22
22
|
import { createDebug, log, tag } from './utils/logger.js';
|
|
23
23
|
import { sleep, waitForPageReadiness } from './utils/page-readiness.ts';
|
|
24
|
+
import { isInternalStep } from './utils/step-analyzer.ts';
|
|
24
25
|
|
|
25
26
|
declare global {
|
|
26
27
|
namespace NodeJS {
|
|
@@ -206,8 +207,7 @@ class Explorer {
|
|
|
206
207
|
|
|
207
208
|
const stepHandler = (step: any, status?: string, error?: string, log?: string) => {
|
|
208
209
|
if (!step.toCode) return;
|
|
209
|
-
if (step
|
|
210
|
-
if (step?.name?.startsWith('save')) return;
|
|
210
|
+
if (isInternalStep(step)) return;
|
|
211
211
|
|
|
212
212
|
test.addStep(step.toCode(), step.duration, status, error, log);
|
|
213
213
|
|
package/src/reporter.ts
CHANGED
|
@@ -138,6 +138,7 @@ export class Reporter {
|
|
|
138
138
|
|
|
139
139
|
protected combineStepsAndNotes(test: Test, lastScreenshotFile?: string): Step[] {
|
|
140
140
|
const noteEntries = Object.entries(test.notes)
|
|
141
|
+
.filter(([, note]) => !note.observation)
|
|
141
142
|
.map(([timestampKey, note]) => ({
|
|
142
143
|
startTime: note.startTime,
|
|
143
144
|
endTime: note.endTime,
|
|
@@ -272,9 +273,12 @@ export class Reporter {
|
|
|
272
273
|
description: test.description,
|
|
273
274
|
code: test.generatedCode || '',
|
|
274
275
|
steps,
|
|
275
|
-
logs:
|
|
276
|
-
.map((stepData) => stepData.text)
|
|
277
|
-
.
|
|
276
|
+
logs: [
|
|
277
|
+
...Object.values(test.steps).map((stepData) => stepData.text),
|
|
278
|
+
...Object.values(test.notes)
|
|
279
|
+
.filter((note) => note.observation)
|
|
280
|
+
.map((note) => note.message),
|
|
281
|
+
].join('\n'),
|
|
278
282
|
files: Object.values(test.artifacts) || [],
|
|
279
283
|
message: test.summary || this.extractLastNoteMessage(test) || '',
|
|
280
284
|
meta,
|
|
@@ -341,7 +345,7 @@ export class Reporter {
|
|
|
341
345
|
}
|
|
342
346
|
|
|
343
347
|
private extractLastNoteMessage(test: Test): string {
|
|
344
|
-
const notes = Object.values(test.notes);
|
|
348
|
+
const notes = Object.values(test.notes).filter((note) => !note.observation);
|
|
345
349
|
if (notes.length === 0) return '';
|
|
346
350
|
return notes[notes.length - 1].message;
|
|
347
351
|
}
|
package/src/utils/logger.ts
CHANGED
|
@@ -215,7 +215,7 @@ class SpanDestination implements LogDestination {
|
|
|
215
215
|
if (!step?.toCode) {
|
|
216
216
|
return;
|
|
217
217
|
}
|
|
218
|
-
const stepName = step?.
|
|
218
|
+
const stepName = step?.title ? `I.${step.title}` : 'I.step';
|
|
219
219
|
const stepInput = typeof step?.toCode === 'function' ? step.toCode() : entry.content;
|
|
220
220
|
const errorFromStep = step?.error;
|
|
221
221
|
const errorMessage =
|
|
@@ -5,6 +5,8 @@ import { isDynamicId } from './xpath.ts';
|
|
|
5
5
|
export const CODECEPT_TOOLS = ['click', 'hover', 'pressKey', 'form'] as const;
|
|
6
6
|
export type CodeceptToolName = (typeof CODECEPT_TOOLS)[number];
|
|
7
7
|
|
|
8
|
+
const INTERNAL_STEP_PREFIXES = ['grab', 'save'];
|
|
9
|
+
|
|
8
10
|
const CODECEPT_FORM_COMMANDS: readonly string[] = ['I.fillField', 'I.type', 'I.selectOption', 'I.attachFile', 'I.checkOption', 'I.uncheckOption'];
|
|
9
11
|
|
|
10
12
|
export function isCodeceptToolName(toolName: string): toolName is CodeceptToolName {
|
|
@@ -71,3 +73,9 @@ export function mergeUniqueStepsByCode(primary: SessionStep[], secondary: Sessio
|
|
|
71
73
|
}
|
|
72
74
|
return merged;
|
|
73
75
|
}
|
|
76
|
+
|
|
77
|
+
export function isInternalStep(step: { title?: string }): boolean {
|
|
78
|
+
const title = step?.title;
|
|
79
|
+
if (!title) return false;
|
|
80
|
+
return INTERNAL_STEP_PREFIXES.some((prefix) => title.startsWith(prefix));
|
|
81
|
+
}
|
package/src/utils/url-matcher.ts
CHANGED
|
@@ -132,3 +132,10 @@ export function matchesNavigationUrl(expected: string, current: string): boolean
|
|
|
132
132
|
.filter(Boolean);
|
|
133
133
|
return recordSegments.length > 0 && recordSegments.every(isDynamicSegment);
|
|
134
134
|
}
|
|
135
|
+
|
|
136
|
+
export function isSameHostFamily(urlA: string, urlB: string): boolean {
|
|
137
|
+
const hostA = URL.parse(urlA)?.host.toLowerCase();
|
|
138
|
+
const hostB = URL.parse(urlB)?.host.toLowerCase();
|
|
139
|
+
if (!hostA || !hostB) return false;
|
|
140
|
+
return hostA === hostB || hostA.endsWith(`.${hostB}`) || hostB.endsWith(`.${hostA}`);
|
|
141
|
+
}
|