@kkeve/suite 0.0.19 → 0.0.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dsl/index.d.ts +1 -1
- package/dist/dsl/index.js +2 -1
- package/dist/dsl/toDecorator.d.ts +17 -0
- package/dist/dsl/toDecorator.js +128 -35
- package/dist/engine-cypress/bridge.js +91 -2
- package/dist/engine-cypress/keveGoal.d.ts +0 -10
- package/dist/engine-cypress/keveGoal.js +5 -25
- package/dist/engine-playwright/keve-test.d.ts +0 -7
- package/dist/engine-playwright/keve-test.js +0 -22
- package/package.json +1 -1
- package/scripts/run-dsl-task.mjs +25 -13
package/dist/dsl/index.d.ts
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
* 分层:DB 装载 / 内嵌场景展开 / 元素库 source 跳解析 / 变量表查询全部留在
|
|
10
10
|
* 调用方(见 eve-backend 的 dslLoader)。这里只做纯函数投影。
|
|
11
11
|
*/
|
|
12
|
-
export { toDecorator, type DslHop, type DslValue, type DslExpectation, type DslStep, type DslVariableRow, type ToDecoratorOptions, } from './toDecorator.js';
|
|
12
|
+
export { toDecorator, stepExpectedText, type DslHop, type DslValue, type DslExpectation, type DslStep, type DslVariableRow, type ExpectedRenderContext, type ToDecoratorOptions, } from './toDecorator.js';
|
|
13
13
|
/** 语义别名:调用方按「生成一份装饰器 spec」的语义引用时更直观 */
|
|
14
14
|
export { toDecorator as generateDecoratorSpec } from './toDecorator.js';
|
|
15
15
|
export { STEP_OPERATIONS, ASSERT_OPERATIONS, TEXT_ONLY_ASSERT_OPERATIONS, EXPECTED_VALUE_CTYPES, COMPARISON_EXPRESSIONS, isKnownOperation, isKnownAssert, isTextOnlyAssert, parseComparison, type StepOperation, } from './ops.js';
|
package/dist/dsl/index.js
CHANGED
|
@@ -11,9 +11,10 @@
|
|
|
11
11
|
* 调用方(见 eve-backend 的 dslLoader)。这里只做纯函数投影。
|
|
12
12
|
*/
|
|
13
13
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
14
|
-
exports.parseComparison = exports.isTextOnlyAssert = exports.isKnownAssert = exports.isKnownOperation = exports.COMPARISON_EXPRESSIONS = exports.EXPECTED_VALUE_CTYPES = exports.TEXT_ONLY_ASSERT_OPERATIONS = exports.ASSERT_OPERATIONS = exports.STEP_OPERATIONS = exports.generateDecoratorSpec = exports.toDecorator = void 0;
|
|
14
|
+
exports.parseComparison = exports.isTextOnlyAssert = exports.isKnownAssert = exports.isKnownOperation = exports.COMPARISON_EXPRESSIONS = exports.EXPECTED_VALUE_CTYPES = exports.TEXT_ONLY_ASSERT_OPERATIONS = exports.ASSERT_OPERATIONS = exports.STEP_OPERATIONS = exports.generateDecoratorSpec = exports.stepExpectedText = exports.toDecorator = void 0;
|
|
15
15
|
var toDecorator_js_1 = require("./toDecorator.js");
|
|
16
16
|
Object.defineProperty(exports, "toDecorator", { enumerable: true, get: function () { return toDecorator_js_1.toDecorator; } });
|
|
17
|
+
Object.defineProperty(exports, "stepExpectedText", { enumerable: true, get: function () { return toDecorator_js_1.stepExpectedText; } });
|
|
17
18
|
/** 语义别名:调用方按「生成一份装饰器 spec」的语义引用时更直观 */
|
|
18
19
|
var toDecorator_js_2 = require("./toDecorator.js");
|
|
19
20
|
Object.defineProperty(exports, "generateDecoratorSpec", { enumerable: true, get: function () { return toDecorator_js_2.toDecorator; } });
|
|
@@ -96,3 +96,20 @@ export interface ToDecoratorOptions {
|
|
|
96
96
|
}
|
|
97
97
|
/** 生成装饰器脚本源码(UTF-8 文本) */
|
|
98
98
|
export declare function toDecorator(options: ToDecoratorOptions): string;
|
|
99
|
+
/** 计算 expected 文案所需的最小上下文(与脚本投影共用,保证报告与脚本同文案) */
|
|
100
|
+
export interface ExpectedRenderContext {
|
|
101
|
+
env: string;
|
|
102
|
+
variableList?: DslVariableRow[];
|
|
103
|
+
account?: {
|
|
104
|
+
name: string;
|
|
105
|
+
password: string;
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* 步骤 expected 文案的**公开**入口。
|
|
110
|
+
*
|
|
111
|
+
* 投影脚本(toDecorator)与报告 outline(run-dsl-task / report-data)必须产出
|
|
112
|
+
* 同一份文案,否则会出现「报告的预期」与「评测 Agent 看到的预期」不一致 ——
|
|
113
|
+
* 那正是「报错4」这类标签被当成字面量去页面搜索的根因。调用方统一走这里。
|
|
114
|
+
*/
|
|
115
|
+
export declare function stepExpectedText(step: DslStep, ctx: ExpectedRenderContext): string;
|
package/dist/dsl/toDecorator.js
CHANGED
|
@@ -21,6 +21,7 @@
|
|
|
21
21
|
*/
|
|
22
22
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
23
23
|
exports.toDecorator = toDecorator;
|
|
24
|
+
exports.stepExpectedText = stepExpectedText;
|
|
24
25
|
const ops_js_1 = require("./ops.js");
|
|
25
26
|
// ─── 生成入口 ────────────────────────────────────────────────────────
|
|
26
27
|
/** 生成装饰器脚本源码(UTF-8 文本) */
|
|
@@ -96,8 +97,7 @@ function renderStep(d, ctx) {
|
|
|
96
97
|
}
|
|
97
98
|
const out = [];
|
|
98
99
|
const stepText = String(d.text || op);
|
|
99
|
-
const
|
|
100
|
-
const expected = expTexts.length ? expTexts.join(';') : '操作完成,页面正常响应';
|
|
100
|
+
const expected = expectedTextOf(d.expectation || [], (e) => resolveExpectationValue(e, ctx));
|
|
101
101
|
const chain = chainExpr(d.context);
|
|
102
102
|
const expLines = [];
|
|
103
103
|
for (const e of d.expectation || []) {
|
|
@@ -106,7 +106,7 @@ function renderStep(d, ctx) {
|
|
|
106
106
|
expLines.push(a);
|
|
107
107
|
}
|
|
108
108
|
const emit = (body) => {
|
|
109
|
-
out.push(`await keveGoal({ step: ${q(stepText)}, expected: ${q(expected)}
|
|
109
|
+
out.push(`await keveGoal({ step: ${q(stepText)}, expected: ${q(expected)} }, async () => {`);
|
|
110
110
|
for (const b of [...body, ...expLines])
|
|
111
111
|
out.push(` ${b}`);
|
|
112
112
|
out.push(`});`);
|
|
@@ -232,7 +232,7 @@ function renderStep(d, ctx) {
|
|
|
232
232
|
// 区域截图容错:元素不在当前页面(权限阻断等)时降级为整页截图,
|
|
233
233
|
// 是否一致由基线比对决定,截图本身不做权限硬门。
|
|
234
234
|
if (shotChain) {
|
|
235
|
-
out.push(`await keveGoal({ step: ${q(stepText)}, expected: ${q(expected)}
|
|
235
|
+
out.push(`await keveGoal({ step: ${q(stepText)}, expected: ${q(expected)} }, async () => {`);
|
|
236
236
|
out.push(` try {`);
|
|
237
237
|
out.push(` await engine.screenshot(${q(shotName)}, ${shotChain});`);
|
|
238
238
|
out.push(` } catch (_shotErr: any) {`);
|
|
@@ -275,7 +275,8 @@ function renderOpenPageWithGates(d, gates, ctx) {
|
|
|
275
275
|
const stepText = String(d.text || 'OPEN_PAGE');
|
|
276
276
|
const gateTexts = gates.map((g) => String(g.text || '')).filter(Boolean);
|
|
277
277
|
const gateExpects = gates
|
|
278
|
-
.
|
|
278
|
+
.filter((g) => (g.expectation || []).length > 0)
|
|
279
|
+
.map((g) => expectedTextOf(g.expectation || [], (e) => resolveExpectationValue(e, ctx)))
|
|
279
280
|
.filter(Boolean);
|
|
280
281
|
const expected = [...gateTexts, ...gateExpects].filter(Boolean).join(';') || '打开目标页面并等待就绪响应';
|
|
281
282
|
const resolved = resolveOpenPage(d, ctx);
|
|
@@ -299,51 +300,143 @@ function renderOpenPageWithGates(d, gates, ctx) {
|
|
|
299
300
|
body.push(`await __gate${gi}; // 等 ${String(g.text || '就绪响应')}`);
|
|
300
301
|
});
|
|
301
302
|
}
|
|
302
|
-
out.push(`await keveGoal({ step: ${q(stepText)}, expected: ${q(expected)}
|
|
303
|
+
out.push(`await keveGoal({ step: ${q(stepText)}, expected: ${q(expected)} }, async () => {`);
|
|
303
304
|
for (const b of body)
|
|
304
305
|
out.push(` ${b}`);
|
|
305
306
|
out.push(`});`);
|
|
306
307
|
return out;
|
|
307
308
|
}
|
|
308
|
-
// ─── 断言转译 ────────────────────────────────────────────────────────
|
|
309
309
|
/**
|
|
310
|
-
*
|
|
310
|
+
* 步骤 expected 文案的**公开**入口。
|
|
311
311
|
*
|
|
312
|
-
*
|
|
313
|
-
*
|
|
314
|
-
*
|
|
315
|
-
|
|
316
|
-
|
|
312
|
+
* 投影脚本(toDecorator)与报告 outline(run-dsl-task / report-data)必须产出
|
|
313
|
+
* 同一份文案,否则会出现「报告的预期」与「评测 Agent 看到的预期」不一致 ——
|
|
314
|
+
* 那正是「报错4」这类标签被当成字面量去页面搜索的根因。调用方统一走这里。
|
|
315
|
+
*/
|
|
316
|
+
function stepExpectedText(step, ctx) {
|
|
317
|
+
const renderCtx = {
|
|
318
|
+
env: ctx.env,
|
|
319
|
+
variableList: ctx.variableList || [],
|
|
320
|
+
account: ctx.account,
|
|
321
|
+
};
|
|
322
|
+
return expectedTextOf(step.expectation || [], (e) => resolveExpectationValue(e, renderCtx));
|
|
323
|
+
}
|
|
324
|
+
/**
|
|
325
|
+
* 步骤的 expected 文案 —— 喂给评测 Agent 判断「预期是否真的达成」。
|
|
326
|
+
*
|
|
327
|
+
* 平台 expectation[].text 常常只是录制期标签(「报错1」「报错4」),不携带
|
|
328
|
+
* 任何可判定语义;真实预期藏在同一条 expectation 的 type + value + context 里
|
|
329
|
+
* (如 ELEMENT_NOT_EXIST + includes「业务逻辑错误」= 页面不应出现该错误)。
|
|
330
|
+
* 只把标签交给评测 Agent,它会去页面上找字面量「报错4」并据此误判失败。
|
|
317
331
|
*
|
|
318
|
-
*
|
|
332
|
+
* 因此按「可执行断言语义优先、标签作为备注」组装:
|
|
333
|
+
* - 有可执行断言 → 渲染成人话(页面不存在包含「业务逻辑错误」的内容);
|
|
334
|
+
* - 纯文案载体(TEXT / 无 type)→ 标签本身就是人话,直接采用;
|
|
335
|
+
* - 一个断言都渲染不出来时,退回平台标签,最后兜底为通用文案。
|
|
319
336
|
*/
|
|
320
|
-
function
|
|
321
|
-
const
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
337
|
+
function expectedTextOf(expectations, resolveValue) {
|
|
338
|
+
const semantic = [];
|
|
339
|
+
const labelOnly = [];
|
|
340
|
+
for (const e of expectations) {
|
|
341
|
+
const label = String(e.text || '').trim();
|
|
325
342
|
const type = String(e.type || '');
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
const
|
|
332
|
-
|
|
343
|
+
if (!type || (0, ops_js_1.isTextOnlyAssert)(type)) {
|
|
344
|
+
if (label)
|
|
345
|
+
labelOnly.push(label);
|
|
346
|
+
continue;
|
|
347
|
+
}
|
|
348
|
+
const phrase = describeExpectation(e, resolveValue(e));
|
|
349
|
+
// 未知断言在 renderExpectation 里会确定性报错;这里不能让 expected 抢先崩,
|
|
350
|
+
// 退化成标签即可,真正的失败由断言渲染抛出。
|
|
351
|
+
if (phrase)
|
|
352
|
+
semantic.push(label && !phrase.includes(label) ? `${phrase}(${label})` : phrase);
|
|
353
|
+
else if (label)
|
|
354
|
+
labelOnly.push(label);
|
|
355
|
+
}
|
|
356
|
+
const parts = [...semantic, ...labelOnly].filter(Boolean);
|
|
357
|
+
return parts.length ? parts.join(';') : '操作完成,页面正常响应';
|
|
358
|
+
}
|
|
359
|
+
/** 单条可执行断言 → 人类可读预期;返回 null 表示无法给出语义(由调用方兜底) */
|
|
360
|
+
function describeExpectation(e, val) {
|
|
361
|
+
const type = String(e.type || '');
|
|
362
|
+
const target = describeChain(e.context);
|
|
363
|
+
const value = String(val ?? '');
|
|
364
|
+
switch (type) {
|
|
365
|
+
case 'ELEMENT_EXIST':
|
|
366
|
+
return `页面存在${target}`;
|
|
367
|
+
case 'ELEMENT_NOT_EXIST':
|
|
368
|
+
return `页面不存在${target}`;
|
|
369
|
+
case 'ELEMENT_VISIBLE':
|
|
370
|
+
return `${target}可见`;
|
|
371
|
+
case 'ELEMENT_NOT_VISIBLE':
|
|
372
|
+
return `${target}不可见`;
|
|
373
|
+
case 'TEXT_EXIST':
|
|
374
|
+
if (isPageTitleAssertion(e))
|
|
375
|
+
return `页面标题包含「${value}」`;
|
|
376
|
+
return `页面存在包含「${value}」的内容${target}`;
|
|
377
|
+
case 'TEXT_NOT_EXIST':
|
|
378
|
+
if (!value.trim())
|
|
379
|
+
return null;
|
|
380
|
+
return `页面不存在包含「${value}」的内容${target}`;
|
|
381
|
+
case 'TEXT_EQUAL':
|
|
382
|
+
return `元素${target}文本等于「${value}」`;
|
|
383
|
+
case 'EXPECTED_VALUE': {
|
|
384
|
+
const ctype = String(e.ctype || 'text').toLowerCase();
|
|
385
|
+
const { mode, negated } = (0, ops_js_1.parseComparison)(String(e.cexpression || 'include'));
|
|
386
|
+
const dim = { text: '文本', class: '类名', placeholder: '占位提示', style: '样式', index: '元素个数' }[ctype] || ctype;
|
|
387
|
+
if (ctype === 'index')
|
|
388
|
+
return `元素${target}个数等于 ${value}`;
|
|
389
|
+
const op = mode === 'eq' ? (negated ? '不等于' : '等于') : (negated ? '不包含' : '包含');
|
|
390
|
+
return `元素${target}的${dim}${op}「${value}」`;
|
|
391
|
+
}
|
|
392
|
+
case 'EXPECTED__PAGET_VALUE': {
|
|
393
|
+
const ctype = String(e.ctype || 'url').toLowerCase();
|
|
394
|
+
const { mode, negated } = (0, ops_js_1.parseComparison)(String(e.cexpression || 'eq'));
|
|
395
|
+
const dim = ctype === 'title' ? '标题' : '地址';
|
|
396
|
+
const op = mode === 'eq' ? (negated ? '不等于' : '等于') : (negated ? '不包含' : '包含');
|
|
397
|
+
return `页面${dim}${op}「${value}」`;
|
|
398
|
+
}
|
|
399
|
+
case 'SCREEN_SHOT_COMPARE':
|
|
400
|
+
case 'SCREENSHOT_COMPARE':
|
|
401
|
+
case 'IMAGE_COMPARE':
|
|
402
|
+
return `页面截图与基线「${value || e.text || ''}」一致`;
|
|
403
|
+
default:
|
|
404
|
+
return null;
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
/** 定位链 → 人类可读描述(评测 Agent 据此知道「在哪个区域」观察) */
|
|
408
|
+
function describeChain(context) {
|
|
409
|
+
let hops;
|
|
410
|
+
try {
|
|
411
|
+
hops = collectHops(context);
|
|
412
|
+
}
|
|
413
|
+
catch {
|
|
414
|
+
return '';
|
|
415
|
+
}
|
|
416
|
+
if (!hops.length)
|
|
417
|
+
return '';
|
|
418
|
+
// 单跳文本包含:平台最常见的「页面不应出现某文案」录制形态
|
|
419
|
+
if (hops.length === 1 && (hops[0].type === 'includes' || hops[0].type === 'text')) {
|
|
420
|
+
return `包含「${hops[0].value.trim()}」的元素`;
|
|
421
|
+
}
|
|
422
|
+
const parts = hops.map((h) => {
|
|
423
|
+
const text = h.value.trim();
|
|
424
|
+
if (h.type === 'includes' || h.type === 'text')
|
|
425
|
+
return `包含「${text}」`;
|
|
426
|
+
if (h.type === 'selector')
|
|
427
|
+
return `选择器「${text}」`;
|
|
428
|
+
return `${h.type}=「${text}」`;
|
|
333
429
|
});
|
|
430
|
+
return `(${parts.join(' → ')})`;
|
|
334
431
|
}
|
|
335
432
|
/**
|
|
336
|
-
*
|
|
433
|
+
* 渲染一条 expectation 为 engine 断言;返回 null 表示「纯文案载体,不产生断言」。
|
|
337
434
|
*
|
|
338
|
-
*
|
|
339
|
-
*
|
|
340
|
-
*
|
|
341
|
-
*
|
|
435
|
+
* 注意投影脚本**不输出 `react: false`**:可执行断言只证明预期的一个子集,
|
|
436
|
+
* fn 执行成功不等于 expected 已达成,所以每个 goal 都保留评测阶段 —— Agent
|
|
437
|
+
* 先观察真实页面评测,达成即放行,未达成再继续探索;断言失败由引擎侧统一
|
|
438
|
+
* 参与终局覆盖,不允许被评审判成 pass。
|
|
342
439
|
*/
|
|
343
|
-
function reactOption(d) {
|
|
344
|
-
return needsAgentJudge(d) ? '' : ', react: false';
|
|
345
|
-
}
|
|
346
|
-
/** 渲染一条 expectation;返回 null 表示「纯文案载体,不产生断言」 */
|
|
347
440
|
function renderExpectation(e, ctx) {
|
|
348
441
|
const type = String(e.type || '');
|
|
349
442
|
// 无 type / TEXT:纯「期望文案」载体,只喂 keveGoal.expected,不是断言
|
|
@@ -55,6 +55,82 @@ const path = __importStar(require("node:path"));
|
|
|
55
55
|
const screenshotDiff_js_1 = require("../core/judge/screenshotDiff.js");
|
|
56
56
|
const confidence_js_1 = require("../core/judge/confidence.js");
|
|
57
57
|
const reportData_js_1 = require("../core/generateReport/reportData.js");
|
|
58
|
+
/** Cypress 最终状态 → 报告状态;pending 明确是跳过,其余非 passed 均按失败处理 */
|
|
59
|
+
function mapCypressState(rawState) {
|
|
60
|
+
const state = String(rawState || '').toLowerCase();
|
|
61
|
+
if (state === 'passed')
|
|
62
|
+
return 'passed';
|
|
63
|
+
if (state === 'pending' || state === 'skipped')
|
|
64
|
+
return 'skipped';
|
|
65
|
+
if (state === 'timedout' || state === 'timed_out')
|
|
66
|
+
return 'timedOut';
|
|
67
|
+
return 'failed';
|
|
68
|
+
}
|
|
69
|
+
/** 从 Cypress after:spec / after:run 的 TestResult 提取开始时间与耗时 */
|
|
70
|
+
function timingOfCypressTest(test) {
|
|
71
|
+
const attempts = Array.isArray(test?.attempts) ? test.attempts : [];
|
|
72
|
+
const last = attempts[attempts.length - 1] || {};
|
|
73
|
+
const duration = Number(last.wallClockDuration ?? test?.duration ?? 0);
|
|
74
|
+
const startedAt = last.wallClockStartedAt || test?.wallClockStartedAt;
|
|
75
|
+
if (startedAt) {
|
|
76
|
+
const parsed = new Date(startedAt);
|
|
77
|
+
if (!Number.isNaN(parsed.getTime()))
|
|
78
|
+
return { duration, startTime: parsed.toISOString() };
|
|
79
|
+
}
|
|
80
|
+
return { duration, startTime: new Date(Date.now() - (Number.isFinite(duration) ? duration : 0)).toISOString() };
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* 用 Cypress 引擎掌握的最终测试结果补录 / 校正状态。
|
|
84
|
+
*
|
|
85
|
+
* 浏览器侧 `/report/test` 依赖全局 afterEach 与场景状态;beforeEach 等钩子
|
|
86
|
+
* 直接失败时状态可能拿不到,导致用例没有任何上报,最终被 reportData 补成
|
|
87
|
+
* `missing`。Cypress 的 after:spec / after:run 仍能提供权威结果,这里是唯一
|
|
88
|
+
* 不依赖用例内钩子的兜底入口。
|
|
89
|
+
*/
|
|
90
|
+
function mergeCypressRunTests(state, tests, specFile = '') {
|
|
91
|
+
let merged = 0;
|
|
92
|
+
for (const test of Array.isArray(tests) ? tests : []) {
|
|
93
|
+
const titlePath = Array.isArray(test?.title) ? test.title.map((part) => String(part)) : [];
|
|
94
|
+
const title = (titlePath[titlePath.length - 1] || String(test?.title || '')).trim();
|
|
95
|
+
if (!title)
|
|
96
|
+
continue;
|
|
97
|
+
const suiteTitle = titlePath.length >= 2 ? String(titlePath[titlePath.length - 2] || '').trim() : '';
|
|
98
|
+
const status = mapCypressState(test?.state);
|
|
99
|
+
const { duration, startTime } = timingOfCypressTest(test);
|
|
100
|
+
const error = status === 'passed' || status === 'skipped'
|
|
101
|
+
? undefined
|
|
102
|
+
: String(test?.displayError || test?.attempts?.[test.attempts.length - 1]?.error?.message || '');
|
|
103
|
+
const existing = state.results.find((r) => r.title === title && (!specFile || !r.file || r.file === specFile));
|
|
104
|
+
if (existing) {
|
|
105
|
+
// Cypress 的最终状态优先于浏览器侧可能拿到的中间状态;日志与附件保留。
|
|
106
|
+
existing.status = status;
|
|
107
|
+
existing.duration = duration || existing.duration;
|
|
108
|
+
existing.startTime = startTime || existing.startTime;
|
|
109
|
+
// Cypress 最终结论权威:失败才写错误,最终 passed/skipped 必须清掉
|
|
110
|
+
// 浏览器侧中间状态可能留下的误报错误,避免报告出现「通过但带错误」。
|
|
111
|
+
existing.error = error;
|
|
112
|
+
if (!existing.suiteTitle && suiteTitle)
|
|
113
|
+
existing.suiteTitle = suiteTitle;
|
|
114
|
+
if (!existing.file && specFile)
|
|
115
|
+
existing.file = specFile;
|
|
116
|
+
}
|
|
117
|
+
else {
|
|
118
|
+
state.results.push({
|
|
119
|
+
title,
|
|
120
|
+
suiteTitle,
|
|
121
|
+
file: specFile,
|
|
122
|
+
status,
|
|
123
|
+
duration,
|
|
124
|
+
startTime,
|
|
125
|
+
error,
|
|
126
|
+
logs: [],
|
|
127
|
+
attachments: [],
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
merged++;
|
|
131
|
+
}
|
|
132
|
+
return merged;
|
|
133
|
+
}
|
|
58
134
|
function resultDirOf() {
|
|
59
135
|
// 优先使用 runner 注入的报告目录;缺失时回退旧布局,兼容历史执行环境。
|
|
60
136
|
if (process.env.KEVE_RESULT_DIR)
|
|
@@ -300,7 +376,10 @@ async function setupKeveBridge(on, config) {
|
|
|
300
376
|
}
|
|
301
377
|
catch { /* 软链失败不影响报告生成 */ }
|
|
302
378
|
}
|
|
303
|
-
|
|
379
|
+
// reportData.summary 是 summaryJson 本体,total 在其嵌套的 summary 字段下。
|
|
380
|
+
// 直接读 summary.total 恒为 undefined,日志会误报「用例 0」,掩盖真实用例数。
|
|
381
|
+
const cases = reportData?.summary?.summary?.total ?? reportData?.summary?.total;
|
|
382
|
+
return { ok: true, resultsPath, reportDataPath, cases: cases ?? undefined };
|
|
304
383
|
};
|
|
305
384
|
const handle = async (route, payload, signal) => {
|
|
306
385
|
switch (route) {
|
|
@@ -551,8 +630,15 @@ async function setupKeveBridge(on, config) {
|
|
|
551
630
|
console.log(`[keve-bridge] listening on 127.0.0.1:${port} → ${state.resultDir}`);
|
|
552
631
|
// 报告收口放在 after:run:此时所有 spec 的 /report/test 都已落盘,
|
|
553
632
|
// 且不依赖任何场景级的 afterAll(硬失败/中断也能出报告)。
|
|
554
|
-
on('after:run', async () => {
|
|
633
|
+
on('after:run', async (runResults) => {
|
|
555
634
|
try {
|
|
635
|
+
// 最后一次用 Cypress run 全量结果校正,防止个别 spec 的 after:spec 未触发。
|
|
636
|
+
let merged = 0;
|
|
637
|
+
for (const run of Array.isArray(runResults?.runs) ? runResults.runs : []) {
|
|
638
|
+
merged += mergeCypressRunTests(state, run?.tests || [], String(run?.spec?.relative || ''));
|
|
639
|
+
}
|
|
640
|
+
if (merged > 0)
|
|
641
|
+
console.log(`[keve-bridge] Cypress 最终结果校正 ${merged} 条`);
|
|
556
642
|
const out = await finalize();
|
|
557
643
|
console.log(`[keve-bridge] report-data.json → ${out.reportDataPath}(用例 ${out.cases ?? 0})`);
|
|
558
644
|
}
|
|
@@ -572,6 +658,9 @@ async function setupKeveBridge(on, config) {
|
|
|
572
658
|
* after:spec 早于 after:run 触发,此时 results 已由浏览器侧 /report/test 落盘。
|
|
573
659
|
*/
|
|
574
660
|
on('after:spec', async (_spec, results) => {
|
|
661
|
+
const merged = mergeCypressRunTests(state, results?.tests || [], String(results?.spec?.relative || ''));
|
|
662
|
+
if (merged > 0)
|
|
663
|
+
console.log(`[keve-bridge] Cypress 用例结果校正 ${merged} 条`);
|
|
575
664
|
const video = results?.video;
|
|
576
665
|
if (video && fs.existsSync(video)) {
|
|
577
666
|
// 相对 taskDir,与 Playwright 侧 videoPath 的语义一致
|
|
@@ -17,16 +17,6 @@ export interface KeveGoalCallOptions {
|
|
|
17
17
|
precondition?: string;
|
|
18
18
|
step: string;
|
|
19
19
|
expected: string;
|
|
20
|
-
/**
|
|
21
|
-
* 是否在 fn 之后追加一轮 Agent Re-Act 做最终判定。
|
|
22
|
-
*
|
|
23
|
-
* 默认 true(AI 场景 / 手写用例):fn 只是前置动作与确定性校验,预期是否真正
|
|
24
|
-
* 达成由 Agent 观察真实页面后判定。
|
|
25
|
-
*
|
|
26
|
-
* DSL 投影脚本会显式传 false:整条链路的判定已由确定性断言给出,Re-Act 只会
|
|
27
|
-
* 额外消耗 LLM 预算,并让「报错1」这类平台标签参与结论。
|
|
28
|
-
*/
|
|
29
|
-
react?: boolean;
|
|
30
20
|
}
|
|
31
21
|
/** agent Re-Act 的结果(P0-3 接入;未接入时由 fn 结果直接判定) */
|
|
32
22
|
export interface CyGoalExploreResult {
|
|
@@ -75,7 +75,6 @@ function createCyKeveGoal(engine) {
|
|
|
75
75
|
let fnErrorRaw;
|
|
76
76
|
let fnResult;
|
|
77
77
|
let fnBlocked = false;
|
|
78
|
-
const reactEnabled = options.react !== false;
|
|
79
78
|
const fnSource = fn?.toString() || undefined;
|
|
80
79
|
const owned = !(0, shotBuffer_js_1.isGoalShotDiffCaptureActive)();
|
|
81
80
|
if (owned)
|
|
@@ -140,31 +139,12 @@ function createCyKeveGoal(engine) {
|
|
|
140
139
|
};
|
|
141
140
|
console.log(`[keveGoal] 📊 "${options.step}" 截图差异超阈值 — 确定性失败`);
|
|
142
141
|
}
|
|
143
|
-
else if (!reactEnabled) {
|
|
144
|
-
// 确定性模式:断言是否通过就是结论,不再发起 LLM 探索。
|
|
145
|
-
reactResult = {
|
|
146
|
-
expectedMet: !fnError,
|
|
147
|
-
conclusion: fnError ? 'fail' : 'pass',
|
|
148
|
-
actions: [{
|
|
149
|
-
action: {
|
|
150
|
-
tool: 'done',
|
|
151
|
-
verdict: fnError ? 'fail' : 'pass',
|
|
152
|
-
text: fnError ? `Assertion failed: ${fnError}` : '确定性断言全部通过',
|
|
153
|
-
},
|
|
154
|
-
toolOutput: '',
|
|
155
|
-
result: 'ok',
|
|
156
|
-
}],
|
|
157
|
-
agentScreenshots: [],
|
|
158
|
-
};
|
|
159
|
-
logLine(fnError
|
|
160
|
-
? `[keveGoal] ⚙️ "${options.step}" 确定性模式:fn 失败,跳过 Agent Re-Act`
|
|
161
|
-
: `[keveGoal] ⚙️ "${options.step}" 确定性模式:fn 通过,跳过 Agent Re-Act`);
|
|
162
|
-
}
|
|
163
142
|
else {
|
|
164
|
-
// fn 成功 /
|
|
165
|
-
// fn
|
|
166
|
-
//
|
|
167
|
-
//
|
|
143
|
+
// fn 成功 / 失败都交给评测器。fn 只是前置动作与确定性校验:
|
|
144
|
+
// - fn 成功 ≠ expected 达成(fn 可能只覆盖预期的一个子集);
|
|
145
|
+
// - fn 失败 ≠ 不可恢复(普通运行时错误下页面可能仍可探索自愈)。
|
|
146
|
+
// 「先评测、再决定放行还是探索」由这一轮承担:Agent 先观察真实页面评测
|
|
147
|
+
// expected,达成即 done(pass) 放行,未达成再继续探索。
|
|
168
148
|
let explored;
|
|
169
149
|
if (explorer) {
|
|
170
150
|
try {
|
|
@@ -32,13 +32,6 @@ export interface KeveGoalCallOptions {
|
|
|
32
32
|
precondition?: string;
|
|
33
33
|
step: string;
|
|
34
34
|
expected: string;
|
|
35
|
-
/**
|
|
36
|
-
* 是否在 fn 之后追加一轮 Agent Re-Act 做最终判定。
|
|
37
|
-
*
|
|
38
|
-
* 默认 true(AI 场景 / 手写用例);DSL 投影脚本传 false,判定完全由确定性
|
|
39
|
-
* 断言给出,避免为每个步骤额外消耗一轮 LLM。
|
|
40
|
-
*/
|
|
41
|
-
react?: boolean;
|
|
42
35
|
}
|
|
43
36
|
type KeveFixture = {
|
|
44
37
|
keveGoal: (options: KeveGoalCallOptions, fn?: () => Promise<void>) => Promise<void>;
|
|
@@ -189,7 +189,6 @@ exports.test = test_1.test.extend({
|
|
|
189
189
|
let fnErrorRaw; // 原始错误对象(ScreenshotDiffError 等带结构的错误需要保留结构)
|
|
190
190
|
let fnResult; // fn 返回值透传(flow 产物,如 createDashboard 的看板名)
|
|
191
191
|
let fnBlocked = false; // true when fn error is environment-blocked (skip agent Re-Act)
|
|
192
|
-
const reactEnabled = options.react !== false;
|
|
193
192
|
// 截图对比:fn 前开缓冲、fn 后收集 —— engine.expectScreenshot 的结果经此进 attachment
|
|
194
193
|
(0, screenshotDiff_js_1.beginGoalShotDiffCapture)();
|
|
195
194
|
if (fn) {
|
|
@@ -298,27 +297,6 @@ exports.test = test_1.test.extend({
|
|
|
298
297
|
conclusion: 'fail',
|
|
299
298
|
};
|
|
300
299
|
}
|
|
301
|
-
else if (!reactEnabled) {
|
|
302
|
-
// 确定性模式(DSL 投影脚本):断言结果即结论,不再发起 LLM 探索。
|
|
303
|
-
console.log(fnError
|
|
304
|
-
? `[keveGoal] ⚙️ "${options.step}" deterministic mode: fn failed — skipping agent Re-Act`
|
|
305
|
-
: `[keveGoal] ⚙️ "${options.step}" deterministic mode: fn passed — skipping agent Re-Act`);
|
|
306
|
-
reactResult = {
|
|
307
|
-
expectedMet: !fnError,
|
|
308
|
-
actions: [{
|
|
309
|
-
action: {
|
|
310
|
-
tool: 'done',
|
|
311
|
-
verdict: fnError ? 'fail' : 'pass',
|
|
312
|
-
text: fnError ? `Assertion failed: ${fnError}` : '确定性断言全部通过',
|
|
313
|
-
},
|
|
314
|
-
toolOutput: '',
|
|
315
|
-
result: 'ok',
|
|
316
|
-
}],
|
|
317
|
-
finalSnapshot: '',
|
|
318
|
-
agentScreenshots: [],
|
|
319
|
-
conclusion: fnError ? 'fail' : 'pass',
|
|
320
|
-
};
|
|
321
|
-
}
|
|
322
300
|
else {
|
|
323
301
|
// ── Per-goal timeout: prevent one slow goal from exhausting the test timeout ──
|
|
324
302
|
// Default 300s per goal; total test timeout should be ≥ (goals × 300s + overhead)
|
package/package.json
CHANGED
package/scripts/run-dsl-task.mjs
CHANGED
|
@@ -134,8 +134,12 @@ function normalizeStep(step) {
|
|
|
134
134
|
* 投影 DSL 步骤为报告 outline,语义与 toDecorator 的 keveGoal 参数保持一致。
|
|
135
135
|
* OPEN_PAGE 后紧跟的 WAIT_RESPONSE 会合并进同一个导航步骤,避免报告步骤数
|
|
136
136
|
* 与脚本实际 keveGoal 数不一致。
|
|
137
|
+
*
|
|
138
|
+
* expected 文案统一走 stepExpectedText(与脚本投影同一个函数):报告里展示的
|
|
139
|
+
* 预期与评测 Agent 读到的预期必须逐字一致,否则平台标签(「报错4」)会在报告
|
|
140
|
+
* 与评测两侧产生不同解读。
|
|
137
141
|
*/
|
|
138
|
-
function buildCaseSteps(dslList) {
|
|
142
|
+
function buildCaseSteps(dslList, ctx, stepExpectedText) {
|
|
139
143
|
const steps = [];
|
|
140
144
|
for (let i = 0; i < dslList.length; i++) {
|
|
141
145
|
const d = dslList[i];
|
|
@@ -149,7 +153,8 @@ function buildCaseSteps(dslList) {
|
|
|
149
153
|
if (gates.length) {
|
|
150
154
|
const gateTexts = gates.map((g) => String(g?.text || '')).filter(Boolean);
|
|
151
155
|
const gateExpects = gates
|
|
152
|
-
.
|
|
156
|
+
.filter((g) => (g?.expectation || []).length > 0)
|
|
157
|
+
.map((g) => stepExpectedText(g, ctx))
|
|
153
158
|
.filter(Boolean);
|
|
154
159
|
steps.push({
|
|
155
160
|
step: String(d?.text || 'OPEN_PAGE'),
|
|
@@ -160,8 +165,7 @@ function buildCaseSteps(dslList) {
|
|
|
160
165
|
}
|
|
161
166
|
}
|
|
162
167
|
const stepText = String(d?.text || op || '');
|
|
163
|
-
const
|
|
164
|
-
const expected = expTexts.length ? expTexts.join(';') : '操作完成,页面正常响应';
|
|
168
|
+
const expected = stepExpectedText(d, ctx);
|
|
165
169
|
if (!stepText) continue;
|
|
166
170
|
steps.push({ step: stepText, expected });
|
|
167
171
|
}
|
|
@@ -217,7 +221,7 @@ async function main() {
|
|
|
217
221
|
fs.rmSync(resultFile, { force: true });
|
|
218
222
|
fs.rmSync(manifestFile, { force: true });
|
|
219
223
|
|
|
220
|
-
const { toDecorator } = await import(
|
|
224
|
+
const { toDecorator, stepExpectedText } = await import(
|
|
221
225
|
url.pathToFileURL(path.join(suiteRoot, 'dist/dsl/index.js')).href
|
|
222
226
|
);
|
|
223
227
|
|
|
@@ -233,20 +237,26 @@ async function main() {
|
|
|
233
237
|
const importFrom = engine === 'cypress'
|
|
234
238
|
? '@kkeve/suite/engine-cypress'
|
|
235
239
|
: '@kkeve/suite/keve-test';
|
|
240
|
+
const account = resolveAccount(item.accountInfo || input.accountInfo);
|
|
241
|
+
const renderCtx = {
|
|
242
|
+
env: input.env || 'test',
|
|
243
|
+
variableList: Array.isArray(item.variableList) ? item.variableList : [],
|
|
244
|
+
account,
|
|
245
|
+
};
|
|
236
246
|
const code = toDecorator({
|
|
237
247
|
stepGroupId,
|
|
238
248
|
caseName: item.caseName || `用例 ${stepGroupId}`,
|
|
239
249
|
dslList,
|
|
240
|
-
env:
|
|
241
|
-
variableList:
|
|
242
|
-
account:
|
|
250
|
+
env: renderCtx.env,
|
|
251
|
+
variableList: renderCtx.variableList,
|
|
252
|
+
account: renderCtx.account,
|
|
243
253
|
ssoInject: input.ssoInject !== false,
|
|
244
254
|
importFrom,
|
|
245
255
|
});
|
|
246
256
|
|
|
247
257
|
const fileName = `sg${stepGroupId}.spec.ts`;
|
|
248
258
|
fs.writeFileSync(path.join(specDir, fileName), code, 'utf-8');
|
|
249
|
-
const dslSteps = buildCaseSteps(dslList);
|
|
259
|
+
const dslSteps = buildCaseSteps(dslList, renderCtx, stepExpectedText);
|
|
250
260
|
caseEntries.push({
|
|
251
261
|
caseId: `S${stepGroupId}`,
|
|
252
262
|
stepGroupId,
|
|
@@ -540,15 +550,17 @@ function rewriteOpenPageTargets(input, cases) {
|
|
|
540
550
|
|
|
541
551
|
/**
|
|
542
552
|
* 逐条统计 caseResults 状态。
|
|
543
|
-
*
|
|
544
|
-
*
|
|
553
|
+
*
|
|
554
|
+
* 只有显式 `skipped` 才算跳过;`missing` / `timedOut` / `unknown` / `error`
|
|
555
|
+
* 都代表用例没有得到有效执行结果或执行异常,必须计为失败。历史实现把它们
|
|
556
|
+
* 统一算成 skipped,导致 Cypress 已报 `1 failed` 时平台仍展示 0 失败。
|
|
545
557
|
*/
|
|
546
558
|
function summarizeReport(reportData) {
|
|
547
559
|
const cases = Array.isArray(reportData?.caseResults) ? reportData.caseResults : [];
|
|
548
560
|
if (cases.length) {
|
|
549
561
|
const passed = cases.filter((item) => item?.status === 'passed').length;
|
|
550
|
-
const
|
|
551
|
-
const
|
|
562
|
+
const skipped = cases.filter((item) => item?.status === 'skipped').length;
|
|
563
|
+
const failed = cases.length - passed - skipped;
|
|
552
564
|
return { passed, failed, skipped, total: cases.length };
|
|
553
565
|
}
|
|
554
566
|
const summary = reportData?.summary?.summary || reportData?.summary || {};
|