@taodigital/agents 0.0.0-stage → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +99 -2
- package/dist/api-test-agent.d.ts +126 -0
- package/dist/api-test-agent.js +267 -0
- package/dist/artifact-intel-agent.d.ts +11 -0
- package/dist/artifact-intel-agent.js +152 -0
- package/dist/coverage-gap-agent.d.ts +25 -0
- package/dist/coverage-gap-agent.js +269 -0
- package/dist/coverage-map-agent.d.ts +15 -0
- package/dist/coverage-map-agent.js +255 -0
- package/dist/crawl-diff-agent.d.ts +27 -0
- package/dist/crawl-diff-agent.js +412 -0
- package/dist/data-agent.d.ts +101 -0
- package/dist/data-agent.js +2284 -0
- package/dist/document-agent.d.ts +10 -0
- package/dist/document-agent.js +32 -0
- package/dist/enhancement-agent.d.ts +48 -0
- package/dist/enhancement-agent.js +342 -0
- package/dist/field-detector-agent.d.ts +35 -0
- package/dist/field-detector-agent.js +220 -0
- package/dist/flow-agent.d.ts +130 -0
- package/dist/flow-agent.js +169 -0
- package/dist/github-agent.d.ts +18 -0
- package/dist/github-agent.js +39 -0
- package/dist/github-write-agent.d.ts +67 -0
- package/dist/github-write-agent.js +249 -0
- package/dist/grouping-agent.d.ts +18 -0
- package/dist/grouping-agent.js +338 -0
- package/dist/healing-agent.d.ts +73 -0
- package/dist/healing-agent.js +137 -0
- package/dist/index.d.ts +36 -0
- package/dist/index.js +67 -0
- package/dist/integration-test-agent.d.ts +118 -0
- package/dist/integration-test-agent.js +165 -0
- package/dist/label-assignment-agent.d.ts +15 -0
- package/dist/label-assignment-agent.js +97 -0
- package/dist/llm-config.d.ts +38 -0
- package/dist/llm-config.js +287 -0
- package/dist/llm-mock.d.ts +77 -0
- package/dist/llm-mock.js +192 -0
- package/dist/llm.d.ts +58 -0
- package/dist/llm.js +438 -0
- package/dist/nlp-agent.d.ts +208 -0
- package/dist/nlp-agent.js +340 -0
- package/dist/orchestrator.d.ts +285 -0
- package/dist/orchestrator.js +141 -0
- package/dist/plan-agent.d.ts +4 -0
- package/dist/plan-agent.js +200 -0
- package/dist/repo-scan-agent.d.ts +189 -0
- package/dist/repo-scan-agent.js +2032 -0
- package/dist/requirements-agent.d.ts +59 -0
- package/dist/requirements-agent.js +641 -0
- package/dist/scenario-agent.d.ts +20 -0
- package/dist/scenario-agent.js +123 -0
- package/dist/script-agent.d.ts +447 -0
- package/dist/script-agent.js +11813 -0
- package/dist/security-test-agent.d.ts +111 -0
- package/dist/security-test-agent.js +141 -0
- package/dist/skill-file-agent.d.ts +264 -0
- package/dist/skill-file-agent.js +3470 -0
- package/dist/step-coverage.d.ts +46 -0
- package/dist/step-coverage.js +288 -0
- package/dist/step-utils.d.ts +2 -0
- package/dist/step-utils.js +62 -0
- package/dist/test-case-agent.d.ts +206 -0
- package/dist/test-case-agent.js +2141 -0
- package/dist/test-import-agent.d.ts +148 -0
- package/dist/test-import-agent.js +3235 -0
- package/dist/test-script-agent.d.ts +16 -0
- package/dist/test-script-agent.js +48 -0
- package/dist/token-pricing.d.ts +14 -0
- package/dist/token-pricing.js +62 -0
- package/dist/types.d.ts +232 -0
- package/dist/types.js +7 -0
- package/dist/visual-agent.d.ts +114 -0
- package/dist/visual-agent.js +261 -0
- package/package.json +26 -4
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
export interface StepCoverageItem {
|
|
2
|
+
index: number;
|
|
3
|
+
step: string;
|
|
4
|
+
kind: 'navigation' | 'action' | 'assertion';
|
|
5
|
+
covered: boolean;
|
|
6
|
+
/** 0..1 weighted token overlap with the best matching statement. */
|
|
7
|
+
confidence: number;
|
|
8
|
+
/** 1-based line in the supplied code of the best matching statement. */
|
|
9
|
+
matchedLine?: number;
|
|
10
|
+
matchedStatement?: string;
|
|
11
|
+
/** Present when the step carries its own expected result. */
|
|
12
|
+
expected?: {
|
|
13
|
+
text: string;
|
|
14
|
+
covered: boolean;
|
|
15
|
+
matchedLine?: number;
|
|
16
|
+
matchedStatement?: string;
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
export interface StepCoverageResult {
|
|
20
|
+
method: 'token-overlap-v1';
|
|
21
|
+
totalSteps: number;
|
|
22
|
+
coveredSteps: number;
|
|
23
|
+
/** covered ÷ total × 100, rounded to 1 decimal. 0 when the test case has no steps. */
|
|
24
|
+
coveragePct: number;
|
|
25
|
+
/** Expected results (test-level + per-step) matched by an assertion; null when none are defined. */
|
|
26
|
+
assertionCoverage: {
|
|
27
|
+
total: number;
|
|
28
|
+
covered: number;
|
|
29
|
+
pct: number;
|
|
30
|
+
} | null;
|
|
31
|
+
expectedResult?: {
|
|
32
|
+
text: string;
|
|
33
|
+
covered: boolean;
|
|
34
|
+
matchedLine?: number;
|
|
35
|
+
matchedStatement?: string;
|
|
36
|
+
};
|
|
37
|
+
uncoveredSteps: number[];
|
|
38
|
+
perStep: StepCoverageItem[];
|
|
39
|
+
}
|
|
40
|
+
export declare function computeStepCoverage(opts: {
|
|
41
|
+
steps: unknown[];
|
|
42
|
+
expectedResult?: string | null;
|
|
43
|
+
code: string;
|
|
44
|
+
}): StepCoverageResult;
|
|
45
|
+
/** Aggregate helper used by reports: average of non-null percentages. */
|
|
46
|
+
export declare function averageStepCoverage(values: Array<number | null | undefined>): number | null;
|
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.computeStepCoverage = computeStepCoverage;
|
|
4
|
+
exports.averageStepCoverage = averageStepCoverage;
|
|
5
|
+
/**
|
|
6
|
+
* Deterministic step-coverage metric for generated automation scripts.
|
|
7
|
+
*
|
|
8
|
+
* stepCoveragePct = covered test-case steps ÷ total test-case steps × 100
|
|
9
|
+
*
|
|
10
|
+
* Unlike the LLM-judged alignment score (verifyScriptAlignment), this is a pure function:
|
|
11
|
+
* the same steps + code always produce the same result, and every decision is explainable
|
|
12
|
+
* (each step is mapped to the script statement that implements it, or reported as uncovered).
|
|
13
|
+
*
|
|
14
|
+
* Method (token-overlap-v1):
|
|
15
|
+
* 1. Extract executable statements from the script (actions/assertions/page calls, Gherkin
|
|
16
|
+
* steps, Java/Python calls). Comments, imports, test titles and declarations are skipped.
|
|
17
|
+
* 2. Tokenise each step and statement: split camelCase / snake_case / string literals,
|
|
18
|
+
* lowercase, drop stop words and code-structure noise, canonicalise verb synonyms
|
|
19
|
+
* (type/enter/fill → fill, press/tap/click → click, open/go/visit → navigate, …).
|
|
20
|
+
* 3. A step is covered when a statement (or two adjacent statements) matches at least one
|
|
21
|
+
* content token AND a weighted share (content=1, intent verb=0.5) of the step's tokens.
|
|
22
|
+
* 4. Expected results (test-level and per-step) are checked against assertion statements only.
|
|
23
|
+
*/
|
|
24
|
+
const step_utils_js_1 = require("./step-utils.js");
|
|
25
|
+
const STOP = new Set([
|
|
26
|
+
'the', 'a', 'an', 'to', 'of', 'on', 'in', 'into', 'and', 'or', 'with', 'for', 'from', 'then', 'that', 'is', 'are',
|
|
27
|
+
'be', 'been', 'should', 'will', 'shall', 'can', 'it', 'its', 'this', 'these', 'those', 'at', 'by', 'as', 'user',
|
|
28
|
+
'page', 'screen', 'application', 'app', 'button', 'field', 'box', 'element', 'textbox', 'input', 'link', 'tab',
|
|
29
|
+
'given', 'when', 'and', 'but', 'so', 'now', 'has', 'have', 'had', 'does', 'do', 'not', 'no', 'all', 'any',
|
|
30
|
+
// code-structure noise
|
|
31
|
+
'await', 'async', 'const', 'let', 'var', 'new', 'return', 'this', 'self', 'actions', 'assertions', 'action',
|
|
32
|
+
'assertion', 'locator', 'locators', 'getby', 'role', 'name', 'exact', 'first', 'last', 'nth', 'driver', 'find',
|
|
33
|
+
'by', 'id', 'css', 'xpath', 'void', 'public', 'private', 'string', 'null', 'true', 'false', 'timeout', 'env',
|
|
34
|
+
'process', 'td', 'testdata', 'data', 'wait', 'text', 'label', 'placeholder', 'testid', 'title',
|
|
35
|
+
]);
|
|
36
|
+
// Canonical verb / noun synonyms. Keys are post-lowercase words; values are canonical tokens.
|
|
37
|
+
const SYNONYMS = {
|
|
38
|
+
click: 'click', press: 'click', tap: 'click', hit: 'click', push: 'click', choose: 'select', pick: 'select',
|
|
39
|
+
select: 'select', enter: 'fill', type: 'fill', fill: 'fill', input: 'fill', provide: 'fill', write: 'fill',
|
|
40
|
+
set: 'fill', populate: 'fill', navigate: 'navigate', open: 'navigate', go: 'navigate', goto: 'navigate',
|
|
41
|
+
visit: 'navigate', launch: 'navigate', load: 'navigate', browse: 'navigate', verify: 'assert', assert: 'assert',
|
|
42
|
+
ensure: 'assert', confirm: 'assert', validate: 'assert', expect: 'assert', see: 'assert', displayed: 'assert',
|
|
43
|
+
display: 'assert', visible: 'assert', shown: 'assert', show: 'assert', appears: 'assert', appear: 'assert',
|
|
44
|
+
present: 'assert', contain: 'assert', contains: 'assert', equal: 'assert', equals: 'assert', have: 'assert',
|
|
45
|
+
upload: 'upload', attach: 'upload', hover: 'hover', mouseover: 'hover', signin: 'login', login: 'login',
|
|
46
|
+
logon: 'login', signout: 'logout', logout: 'logout', logoff: 'logout', submit: 'submit', save: 'submit',
|
|
47
|
+
checkbox: 'check', tick: 'check', untick: 'uncheck', uncheck: 'uncheck', dropdown: 'select', drop: 'select',
|
|
48
|
+
password: 'password', passwd: 'password', pwd: 'password', username: 'username', user_name: 'username',
|
|
49
|
+
email: 'email', mail: 'email', error: 'error', errors: 'error', message: 'message', msg: 'message',
|
|
50
|
+
redirect: 'url', redirected: 'url', redirects: 'url', url: 'url', dashboard: 'dashboard', home: 'home',
|
|
51
|
+
};
|
|
52
|
+
/** Verbs that describe intent but not subject matter. Weighted lower than content tokens. */
|
|
53
|
+
const INTENT = new Set(['click', 'fill', 'navigate', 'assert', 'select', 'upload', 'hover', 'check', 'uncheck', 'submit']);
|
|
54
|
+
const ASSERT_WORDS = new Set(['assert', 'verify', 'ensure', 'confirm', 'validate', 'expect', 'see', 'displayed',
|
|
55
|
+
'visible', 'shown', 'appear', 'appears', 'present', 'contains', 'contain', 'equals', 'redirected', 'redirects']);
|
|
56
|
+
function stem(w) {
|
|
57
|
+
if (w.length > 5 && w.endsWith('ing'))
|
|
58
|
+
return w.slice(0, -3);
|
|
59
|
+
if (w.length > 5 && w.endsWith('ies'))
|
|
60
|
+
return w.slice(0, -3) + 'y';
|
|
61
|
+
if (w.length > 4 && w.endsWith('ed'))
|
|
62
|
+
return w.slice(0, -2);
|
|
63
|
+
if (w.length > 4 && w.endsWith('es'))
|
|
64
|
+
return w.slice(0, -2);
|
|
65
|
+
if (w.length > 3 && w.endsWith('s') && !w.endsWith('ss'))
|
|
66
|
+
return w.slice(0, -1);
|
|
67
|
+
return w;
|
|
68
|
+
}
|
|
69
|
+
function splitWords(text) {
|
|
70
|
+
return text
|
|
71
|
+
.replace(/([a-z0-9])([A-Z])/g, '$1 $2') // camelCase
|
|
72
|
+
.replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2') // HTMLParser → HTML Parser
|
|
73
|
+
.replace(/[^A-Za-z0-9]+/g, ' ')
|
|
74
|
+
.toLowerCase()
|
|
75
|
+
.split(/\s+/)
|
|
76
|
+
.filter(Boolean);
|
|
77
|
+
}
|
|
78
|
+
function canon(word) {
|
|
79
|
+
if (!word || word.length < 2)
|
|
80
|
+
return null;
|
|
81
|
+
if (STOP.has(word))
|
|
82
|
+
return null;
|
|
83
|
+
const direct = SYNONYMS[word];
|
|
84
|
+
if (direct)
|
|
85
|
+
return direct;
|
|
86
|
+
const st = stem(word);
|
|
87
|
+
if (STOP.has(st))
|
|
88
|
+
return null;
|
|
89
|
+
return SYNONYMS[st] ?? st;
|
|
90
|
+
}
|
|
91
|
+
function tokenize(text) {
|
|
92
|
+
const out = new Set();
|
|
93
|
+
for (const w of splitWords(text)) {
|
|
94
|
+
const c = canon(w);
|
|
95
|
+
if (c)
|
|
96
|
+
out.add(c);
|
|
97
|
+
}
|
|
98
|
+
return out;
|
|
99
|
+
}
|
|
100
|
+
const ASSERTION_RE = /\bexpect\s*\(|\.should\b|\bassert\w*|\bverify\w*|\bensure\w*|\btoBe\w*|\btoHave\w*|\btoContain\w*|\btoEqual\w*|\bassertions?\./i;
|
|
101
|
+
const NAVIGATION_RE = /\bgoto\s*\(|\.goto\b|\bnavigate\w*\s*\(|\bvisit\s*\(|\bopen(Page|Url|App)?\s*\(|driver\.get\s*\(|\bcy\.visit\b|\bpage\.open\b/i;
|
|
102
|
+
const SKIP_LINE_RE = /^(\s*$|\s*\/\/|\s*#|\s*\*|\s*\/\*|import\s|from\s+\S+\s+import|require\(|export\s+(type|interface)|package\s|@Test|@BeforeMethod|@AfterMethod|@pytest|@Display|test\.(describe|beforeEach|afterEach|use)\b|describe\s*\(|\btest\s*\(\s*['"`]|\bit\s*\(\s*['"`]|def\s+test_|\}\)?;?\s*$|\{\s*$|Feature:|Scenario|Background:|@\w+\s*$)/;
|
|
103
|
+
const DECL_RE = /^\s*(public|private|protected|async|static|function|def|class|interface|type|export\s+(default\s+)?(class|function|const\s+\w+\s*=\s*(async\s*)?\())\b[^=;]*\{?\s*$/;
|
|
104
|
+
function extractStatements(code) {
|
|
105
|
+
const out = [];
|
|
106
|
+
const lines = code.split('\n');
|
|
107
|
+
for (let i = 0; i < lines.length; i++) {
|
|
108
|
+
const raw = lines[i];
|
|
109
|
+
const line = raw.trim();
|
|
110
|
+
if (!line)
|
|
111
|
+
continue;
|
|
112
|
+
const isGherkinStep = /^(Given|When|Then|And|But)\s+\S/.test(line);
|
|
113
|
+
if (!isGherkinStep && (SKIP_LINE_RE.test(line) || DECL_RE.test(line)))
|
|
114
|
+
continue;
|
|
115
|
+
// A statement has to *do* something: a call, an assignment from a call, or a gherkin step.
|
|
116
|
+
if (!isGherkinStep && !/[A-Za-z_$][\w$]*\s*\(/.test(line))
|
|
117
|
+
continue;
|
|
118
|
+
const kind = ASSERTION_RE.test(line) || /^(Then)\b/.test(line)
|
|
119
|
+
? 'assertion'
|
|
120
|
+
: NAVIGATION_RE.test(line) ? 'navigation' : 'action';
|
|
121
|
+
const tokens = tokenize(line);
|
|
122
|
+
if (tokens.size === 0)
|
|
123
|
+
continue;
|
|
124
|
+
out.push({ line: i + 1, text: line, kind, tokens });
|
|
125
|
+
}
|
|
126
|
+
return out;
|
|
127
|
+
}
|
|
128
|
+
function stepInfo(raw) {
|
|
129
|
+
let expected;
|
|
130
|
+
if (raw && typeof raw === 'object' && !Array.isArray(raw)) {
|
|
131
|
+
const o = raw;
|
|
132
|
+
const e = o.expected ?? o.expectedResult;
|
|
133
|
+
if (e != null && String(e).trim())
|
|
134
|
+
expected = (0, step_utils_js_1.stepToText)(e);
|
|
135
|
+
// Include the step's data so literals ("admin@x.com") count as content tokens.
|
|
136
|
+
const action = (0, step_utils_js_1.stepToText)({ action: o.action, text: o.text });
|
|
137
|
+
const data = o.data != null && typeof o.data !== 'object' ? String(o.data) : '';
|
|
138
|
+
const text = [action, data].filter(Boolean).join(' ').trim() || (0, step_utils_js_1.stepToText)(raw);
|
|
139
|
+
return { text, expected };
|
|
140
|
+
}
|
|
141
|
+
return { text: (0, step_utils_js_1.stepToText)(raw) };
|
|
142
|
+
}
|
|
143
|
+
function stepKind(text, tokens) {
|
|
144
|
+
const words = splitWords(text);
|
|
145
|
+
if (words.some((w) => ASSERT_WORDS.has(w)))
|
|
146
|
+
return 'assertion';
|
|
147
|
+
if (tokens.has('navigate') && /\b(navigate|open|go to|visit|launch|browse|load)\b/i.test(text))
|
|
148
|
+
return 'navigation';
|
|
149
|
+
return 'action';
|
|
150
|
+
}
|
|
151
|
+
function score(stepTokens, stmtTokens) {
|
|
152
|
+
let contentCount = 0, intentCount = 0, contentHits = 0, intentHits = 0;
|
|
153
|
+
for (const t of stepTokens) {
|
|
154
|
+
const isIntent = INTENT.has(t);
|
|
155
|
+
if (isIntent)
|
|
156
|
+
intentCount++;
|
|
157
|
+
else
|
|
158
|
+
contentCount++;
|
|
159
|
+
if (stmtTokens.has(t)) {
|
|
160
|
+
if (isIntent)
|
|
161
|
+
intentHits++;
|
|
162
|
+
else
|
|
163
|
+
contentHits++;
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
const denom = contentCount + 0.5 * intentCount;
|
|
167
|
+
const num = contentHits + 0.5 * intentHits;
|
|
168
|
+
return { score: denom === 0 ? 0 : num / denom, contentHits, intentHits, contentCount, intentCount };
|
|
169
|
+
}
|
|
170
|
+
function isMatch(s, minScore) {
|
|
171
|
+
if (s.contentCount > 0)
|
|
172
|
+
return s.contentHits >= 1 && s.score >= minScore;
|
|
173
|
+
return s.intentHits >= 1 && s.score >= Math.max(minScore, 0.5);
|
|
174
|
+
}
|
|
175
|
+
function windows(statements) {
|
|
176
|
+
const out = [];
|
|
177
|
+
for (let i = 0; i < statements.length; i++) {
|
|
178
|
+
out.push({ statement: statements[i], tokens: statements[i].tokens });
|
|
179
|
+
const next = statements[i + 1];
|
|
180
|
+
// Merge adjacent statements (e.g. locator declaration + .fill()) — only when contiguous lines.
|
|
181
|
+
if (next && next.line - statements[i].line <= 2) {
|
|
182
|
+
out.push({ statement: statements[i], tokens: new Set([...statements[i].tokens, ...next.tokens]) });
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
return out;
|
|
186
|
+
}
|
|
187
|
+
function best(stepTokens, kind, pool, minScore) {
|
|
188
|
+
let top = null;
|
|
189
|
+
for (const cand of pool) {
|
|
190
|
+
const s = score(stepTokens, cand.tokens);
|
|
191
|
+
// Kind compatibility: an assertion step must be satisfied by an assertion statement; action and
|
|
192
|
+
// navigation steps by non-assertion statements. A cross-kind match is allowed but penalised (0.85).
|
|
193
|
+
const compatible = kind === 'assertion'
|
|
194
|
+
? cand.statement.kind === 'assertion'
|
|
195
|
+
: cand.statement.kind !== 'assertion';
|
|
196
|
+
const adjusted = compatible ? s.score : s.score * 0.85;
|
|
197
|
+
const ok = isMatch({ ...s, score: adjusted }, minScore);
|
|
198
|
+
// Prefer any matching candidate over a non-matching one, then the higher score.
|
|
199
|
+
if (!top || (ok && !top.ok) || (ok === top.ok && adjusted > top.score)) {
|
|
200
|
+
top = { score: adjusted, ok, statement: cand.statement };
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
if (!top)
|
|
204
|
+
return { covered: false, confidence: 0 };
|
|
205
|
+
return { covered: top.ok, confidence: Math.round(Math.min(1, top.score) * 100) / 100, statement: top.statement };
|
|
206
|
+
}
|
|
207
|
+
function trim(s, n = 180) { return s.length > n ? s.slice(0, n - 1) + '…' : s; }
|
|
208
|
+
function computeStepCoverage(opts) {
|
|
209
|
+
const statements = extractStatements(opts.code ?? '');
|
|
210
|
+
const pool = windows(statements);
|
|
211
|
+
const assertionPool = windows(statements.filter((s) => s.kind === 'assertion'));
|
|
212
|
+
const infos = (opts.steps ?? []).map(stepInfo).filter((i) => i.text.trim().length > 0);
|
|
213
|
+
const perStep = infos.map((info, idx) => {
|
|
214
|
+
const tokens = tokenize(info.text);
|
|
215
|
+
const kind = stepKind(info.text, tokens);
|
|
216
|
+
// Navigation steps naming a URL are satisfied by any navigation statement mentioning the host/path
|
|
217
|
+
// or by the project's base URL helper.
|
|
218
|
+
const m = best(tokens, kind, pool, 0.45);
|
|
219
|
+
let covered = m.covered;
|
|
220
|
+
let statement = m.statement;
|
|
221
|
+
let confidence = m.confidence;
|
|
222
|
+
if (!covered && kind === 'navigation') {
|
|
223
|
+
const nav = statements.find((s) => s.kind === 'navigation');
|
|
224
|
+
if (nav && tokens.size <= 3) {
|
|
225
|
+
covered = true;
|
|
226
|
+
statement = nav;
|
|
227
|
+
confidence = Math.max(confidence, 0.6);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
const item = {
|
|
231
|
+
index: idx + 1,
|
|
232
|
+
step: trim(info.text, 240),
|
|
233
|
+
kind,
|
|
234
|
+
covered,
|
|
235
|
+
confidence,
|
|
236
|
+
...(covered && statement ? { matchedLine: statement.line, matchedStatement: trim(statement.text) } : {}),
|
|
237
|
+
};
|
|
238
|
+
if (info.expected) {
|
|
239
|
+
const em = best(tokenize(info.expected), 'assertion', assertionPool, 0.4);
|
|
240
|
+
item.expected = {
|
|
241
|
+
text: trim(info.expected, 240),
|
|
242
|
+
covered: em.covered,
|
|
243
|
+
...(em.covered && em.statement ? { matchedLine: em.statement.line, matchedStatement: trim(em.statement.text) } : {}),
|
|
244
|
+
};
|
|
245
|
+
}
|
|
246
|
+
return item;
|
|
247
|
+
});
|
|
248
|
+
const totalSteps = perStep.length;
|
|
249
|
+
const coveredSteps = perStep.filter((p) => p.covered).length;
|
|
250
|
+
let expectedResult;
|
|
251
|
+
const er = opts.expectedResult?.trim();
|
|
252
|
+
if (er) {
|
|
253
|
+
const em = best(tokenize(er), 'assertion', assertionPool, 0.4);
|
|
254
|
+
expectedResult = {
|
|
255
|
+
text: trim(er, 240),
|
|
256
|
+
covered: em.covered,
|
|
257
|
+
...(em.covered && em.statement ? { matchedLine: em.statement.line, matchedStatement: trim(em.statement.text) } : {}),
|
|
258
|
+
};
|
|
259
|
+
}
|
|
260
|
+
const expectations = [
|
|
261
|
+
...(expectedResult ? [expectedResult.covered] : []),
|
|
262
|
+
...perStep.filter((p) => p.expected).map((p) => p.expected.covered),
|
|
263
|
+
];
|
|
264
|
+
const assertionCoverage = expectations.length
|
|
265
|
+
? {
|
|
266
|
+
total: expectations.length,
|
|
267
|
+
covered: expectations.filter(Boolean).length,
|
|
268
|
+
pct: Math.round((expectations.filter(Boolean).length / expectations.length) * 1000) / 10,
|
|
269
|
+
}
|
|
270
|
+
: null;
|
|
271
|
+
return {
|
|
272
|
+
method: 'token-overlap-v1',
|
|
273
|
+
totalSteps,
|
|
274
|
+
coveredSteps,
|
|
275
|
+
coveragePct: totalSteps === 0 ? 0 : Math.round((coveredSteps / totalSteps) * 1000) / 10,
|
|
276
|
+
assertionCoverage,
|
|
277
|
+
...(expectedResult ? { expectedResult } : {}),
|
|
278
|
+
uncoveredSteps: perStep.filter((p) => !p.covered).map((p) => p.index),
|
|
279
|
+
perStep,
|
|
280
|
+
};
|
|
281
|
+
}
|
|
282
|
+
/** Aggregate helper used by reports: average of non-null percentages. */
|
|
283
|
+
function averageStepCoverage(values) {
|
|
284
|
+
const nums = values.filter((v) => typeof v === 'number' && Number.isFinite(v));
|
|
285
|
+
if (!nums.length)
|
|
286
|
+
return null;
|
|
287
|
+
return Math.round((nums.reduce((a, b) => a + b, 0) / nums.length) * 10) / 10;
|
|
288
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.stepToText = stepToText;
|
|
4
|
+
exports.normalizeSteps = normalizeSteps;
|
|
5
|
+
// Decode FZE multi-select encoding ("OR|FUNCOPR|Val1|FUNCVAL|Val2" → "Val1, Val2")
|
|
6
|
+
// and strip element-type prefixes ("Label text:", "Span text:", "I N P U T:").
|
|
7
|
+
function decodeFzeText(s) {
|
|
8
|
+
return s
|
|
9
|
+
.replace(/OR\|FUNCOPR\|/g, '')
|
|
10
|
+
.replace(/\|FUNCVAL\|/g, ', ')
|
|
11
|
+
.replace(/\|FUNCOPR\|/g, ', ')
|
|
12
|
+
.replace(/,\s*\|?\s*$/, '')
|
|
13
|
+
.replace(/^(?:Label|Span|Div|Button|Input)\s+text:\s*/i, '')
|
|
14
|
+
.replace(/^I(?:\s+N\s+P\s+U\s+T|NPUT):\s*/i, '')
|
|
15
|
+
// Preserve fze.env.* as [env:NAME] so URL/field extraction can read the variable name;
|
|
16
|
+
// strip fze.random/local (truly dynamic — no stable value) as [DYNAMIC]
|
|
17
|
+
.replace(/\{\{\s*fze\.env\.([^}]+)\}\}/gi, '[env:$1]')
|
|
18
|
+
.replace(/\{\{\s*fze\.(random|local)\.[^}]+\}\}/gi, '[DYNAMIC]')
|
|
19
|
+
.replace(/'?\[DYNAMIC\]'?/g, '[DYNAMIC]')
|
|
20
|
+
.replace(/[\r\n]+/g, ' ')
|
|
21
|
+
.replace(/\s{2,}/g, ' ')
|
|
22
|
+
.trim();
|
|
23
|
+
}
|
|
24
|
+
// Normalize any step shape to a readable string for LLM prompt injection
|
|
25
|
+
function stepToText(step) {
|
|
26
|
+
if (typeof step === 'string')
|
|
27
|
+
return decodeFzeText(step.trim());
|
|
28
|
+
if (step == null)
|
|
29
|
+
return '';
|
|
30
|
+
if (Array.isArray(step))
|
|
31
|
+
return step.map(stepToText).filter(Boolean).join(' ');
|
|
32
|
+
if (typeof step === 'object') {
|
|
33
|
+
const obj = step;
|
|
34
|
+
// When a decoded action is present, skip raw data/expected fields to avoid
|
|
35
|
+
// embedding FZE-encoded values or "expect X" text into the action string.
|
|
36
|
+
const action = obj.action != null ? decodeFzeText(String(obj.action)) : null;
|
|
37
|
+
const text = obj.text != null ? decodeFzeText(String(obj.text)) : null;
|
|
38
|
+
if (action || text) {
|
|
39
|
+
const parts = [action, text].filter((v) => Boolean(v && v.trim()));
|
|
40
|
+
return parts.join(' ').trim();
|
|
41
|
+
}
|
|
42
|
+
// Fallback: include data/expected when no action/text
|
|
43
|
+
const parts = [
|
|
44
|
+
obj.data ? stepToText(obj.data) : null,
|
|
45
|
+
obj.expected ? `expect ${stepToText(obj.expected)}` : null,
|
|
46
|
+
obj.expectedResult ? `expect ${stepToText(obj.expectedResult)}` : null,
|
|
47
|
+
].filter((v) => v != null && String(v).trim().length > 0);
|
|
48
|
+
if (parts.length > 0)
|
|
49
|
+
return parts.map((v) => stepToText(v)).filter(Boolean).join(' ');
|
|
50
|
+
try {
|
|
51
|
+
return JSON.stringify(step);
|
|
52
|
+
}
|
|
53
|
+
catch {
|
|
54
|
+
return String(step);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
return decodeFzeText(String(step).trim());
|
|
58
|
+
}
|
|
59
|
+
function normalizeSteps(steps) {
|
|
60
|
+
const raw = Array.isArray(steps) ? steps : steps == null ? [] : [steps];
|
|
61
|
+
return raw.map(stepToText).map((s) => s.trim()).filter(Boolean);
|
|
62
|
+
}
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
import type { ElementFact, TestKind } from '@taodigital/core';
|
|
2
|
+
import type { LLMConfig } from './llm.js';
|
|
3
|
+
export declare const GLOBAL_RULES_FALLBACK: string;
|
|
4
|
+
/** @implements {TestGenerationAgent.generateTestCases} */
|
|
5
|
+
export declare function generateTestCases(opts: {
|
|
6
|
+
tenantId: string;
|
|
7
|
+
projectId: string;
|
|
8
|
+
screenId: string;
|
|
9
|
+
jobId?: string;
|
|
10
|
+
oversized?: boolean;
|
|
11
|
+
viewport?: string;
|
|
12
|
+
apiEndpoints?: Array<{
|
|
13
|
+
method: string;
|
|
14
|
+
urlPattern: string;
|
|
15
|
+
}>;
|
|
16
|
+
suiteNameOverride?: string;
|
|
17
|
+
force?: boolean;
|
|
18
|
+
fullCoverage?: boolean;
|
|
19
|
+
suiteLastGenAt?: Date;
|
|
20
|
+
/**
|
|
21
|
+
* 'append' (default) — deduplicates by content hash + grounding hash + title Jaccard.
|
|
22
|
+
* 'replace' — deletes existing AI-generated cases for this screen before inserting.
|
|
23
|
+
* 'edit' — upserts: updates steps/gherkin/expectedResult on matched cases, snapshots old version.
|
|
24
|
+
*/
|
|
25
|
+
mode?: 'replace' | 'append' | 'edit';
|
|
26
|
+
repository?: {
|
|
27
|
+
getChangedScreenIds: (tenantId: string, projectId: string, sinceDate: Date) => Promise<string[]>;
|
|
28
|
+
};
|
|
29
|
+
llmCfg?: LLMConfig;
|
|
30
|
+
fastPassLlmCfg?: LLMConfig;
|
|
31
|
+
/** Override hardcoded GLOBAL_RULES_FALLBACK — load via agent-prompts.ts before calling. */
|
|
32
|
+
globalRules?: string;
|
|
33
|
+
/**
|
|
34
|
+
* When provided, only runs passes that produce these test kinds.
|
|
35
|
+
* Overrides the always-on behavior — only the requested kinds are generated.
|
|
36
|
+
* When omitted, all applicable passes run (existing behavior, unchanged).
|
|
37
|
+
*/
|
|
38
|
+
kinds?: readonly string[];
|
|
39
|
+
/**
|
|
40
|
+
* When true, skips injecting project KB (skill files, requirements, knowledge chunks, cross-project patterns).
|
|
41
|
+
* Use for BRD-only generation where the document itself is the sole source of truth.
|
|
42
|
+
*/
|
|
43
|
+
ignoreProjectContext?: boolean;
|
|
44
|
+
}): Promise<{
|
|
45
|
+
skipped: boolean;
|
|
46
|
+
reason: string;
|
|
47
|
+
existingCaseCount: number;
|
|
48
|
+
status?: undefined;
|
|
49
|
+
raw?: undefined;
|
|
50
|
+
cases?: undefined;
|
|
51
|
+
groundedFacts?: undefined;
|
|
52
|
+
saved?: undefined;
|
|
53
|
+
passBreakdown?: undefined;
|
|
54
|
+
coverageReport?: undefined;
|
|
55
|
+
} | {
|
|
56
|
+
status: string;
|
|
57
|
+
raw: string;
|
|
58
|
+
cases: {
|
|
59
|
+
title: string;
|
|
60
|
+
ref: string;
|
|
61
|
+
viewportConstraint: "ALL" | "MOBILE_ONLY" | "TABLET_ONLY";
|
|
62
|
+
techniqueOfRecord: string | undefined;
|
|
63
|
+
traceTo: string | undefined;
|
|
64
|
+
testDataInline: Record<string, unknown> | undefined;
|
|
65
|
+
category: string[] | undefined;
|
|
66
|
+
dataNote: string | undefined;
|
|
67
|
+
scriptNote: string | undefined;
|
|
68
|
+
structuredSteps: {
|
|
69
|
+
action: string;
|
|
70
|
+
target?: string;
|
|
71
|
+
data?: string;
|
|
72
|
+
}[] | undefined;
|
|
73
|
+
kind: TestKind;
|
|
74
|
+
priority: "CRITICAL" | "HIGH" | "MEDIUM" | "LOW";
|
|
75
|
+
preconditions: string;
|
|
76
|
+
expectedResult: string;
|
|
77
|
+
steps: string[];
|
|
78
|
+
groundedOn: string[];
|
|
79
|
+
gherkin: string;
|
|
80
|
+
tags: string[];
|
|
81
|
+
}[];
|
|
82
|
+
groundedFacts: ElementFact[];
|
|
83
|
+
saved: {
|
|
84
|
+
suite: {
|
|
85
|
+
tenantId: string;
|
|
86
|
+
id: string;
|
|
87
|
+
projectId: string;
|
|
88
|
+
name: string;
|
|
89
|
+
createdAt: Date;
|
|
90
|
+
genModel: string | null;
|
|
91
|
+
category: import(".prisma/client").$Enums.SuiteCategory | null;
|
|
92
|
+
parentId: string | null;
|
|
93
|
+
ownerUserId: string | null;
|
|
94
|
+
genTotalTokens: number | null;
|
|
95
|
+
genTotalCostUsd: number | null;
|
|
96
|
+
genAt: Date | null;
|
|
97
|
+
genScreenCount: number | null;
|
|
98
|
+
coverageReport: import("@prisma/client/runtime/library").JsonValue | null;
|
|
99
|
+
};
|
|
100
|
+
testCases: {
|
|
101
|
+
tenantId: string;
|
|
102
|
+
screenId: string | null;
|
|
103
|
+
id: string;
|
|
104
|
+
projectId: string;
|
|
105
|
+
source: string;
|
|
106
|
+
status: string;
|
|
107
|
+
createdAt: Date;
|
|
108
|
+
updatedAt: Date;
|
|
109
|
+
steps: import("@prisma/client/runtime/library").JsonValue;
|
|
110
|
+
kind: string;
|
|
111
|
+
tags: import("@prisma/client/runtime/library").JsonValue;
|
|
112
|
+
approvedBy: string | null;
|
|
113
|
+
suiteId: string;
|
|
114
|
+
ref: string | null;
|
|
115
|
+
title: string;
|
|
116
|
+
priority: string;
|
|
117
|
+
preconditions: string | null;
|
|
118
|
+
expectedResult: string | null;
|
|
119
|
+
gherkin: string | null;
|
|
120
|
+
approvedAt: Date | null;
|
|
121
|
+
groundedOn: import("@prisma/client/runtime/library").JsonValue;
|
|
122
|
+
caseRefs: import("@prisma/client/runtime/library").JsonValue;
|
|
123
|
+
viewports: import("@prisma/client/runtime/library").JsonValue;
|
|
124
|
+
viewportConstraint: string;
|
|
125
|
+
techniqueOfRecord: string | null;
|
|
126
|
+
traceTo: string | null;
|
|
127
|
+
traceToExtra: import("@prisma/client/runtime/library").JsonValue;
|
|
128
|
+
testDataInline: import("@prisma/client/runtime/library").JsonValue | null;
|
|
129
|
+
dataNote: string | null;
|
|
130
|
+
scriptNote: string | null;
|
|
131
|
+
structuredSteps: import("@prisma/client/runtime/library").JsonValue | null;
|
|
132
|
+
artifactId: string | null;
|
|
133
|
+
scenarioId: string | null;
|
|
134
|
+
suspect: boolean;
|
|
135
|
+
suspectReason: string | null;
|
|
136
|
+
flakyScore: number;
|
|
137
|
+
consecutiveFlakes: number;
|
|
138
|
+
lastFlakyAt: Date | null;
|
|
139
|
+
quarantined: boolean;
|
|
140
|
+
quarantinedAt: Date | null;
|
|
141
|
+
quarantineReason: string | null;
|
|
142
|
+
genPromptTokens: number | null;
|
|
143
|
+
genCompletionTokens: number | null;
|
|
144
|
+
genCostUsd: number | null;
|
|
145
|
+
genModel: string | null;
|
|
146
|
+
dataSetId: string | null;
|
|
147
|
+
repoTargetPath: string | null;
|
|
148
|
+
stepsKey: string | null;
|
|
149
|
+
gherkinKey: string | null;
|
|
150
|
+
contentHash: string | null;
|
|
151
|
+
groundingHash: string | null;
|
|
152
|
+
locatorsJson: import("@prisma/client/runtime/library").JsonValue | null;
|
|
153
|
+
locatorsExtractedAt: Date | null;
|
|
154
|
+
captureStepScreenshots: import("@prisma/client/runtime/library").JsonValue | null;
|
|
155
|
+
}[];
|
|
156
|
+
};
|
|
157
|
+
passBreakdown: {
|
|
158
|
+
[k: string]: number;
|
|
159
|
+
};
|
|
160
|
+
coverageReport: {
|
|
161
|
+
byKind: Record<string, number>;
|
|
162
|
+
byPriority: Record<string, number>;
|
|
163
|
+
byTechnique: Record<string, number>;
|
|
164
|
+
targetsDiscovered: number;
|
|
165
|
+
targetsCovered: number;
|
|
166
|
+
gaps: string[];
|
|
167
|
+
passBreakdown: {
|
|
168
|
+
[k: string]: number;
|
|
169
|
+
};
|
|
170
|
+
};
|
|
171
|
+
skipped?: undefined;
|
|
172
|
+
reason?: undefined;
|
|
173
|
+
existingCaseCount?: undefined;
|
|
174
|
+
}>;
|
|
175
|
+
/**
|
|
176
|
+
* Regenerate a test case title using AI.
|
|
177
|
+
* Deterministic: seed = testCaseId + currentTitle + attempt, so each
|
|
178
|
+
* user click on "Regenerate" after saving produces a different result.
|
|
179
|
+
*/
|
|
180
|
+
export declare function regenerateTestCaseTitle(opts: {
|
|
181
|
+
tenantId: string;
|
|
182
|
+
testCaseId: string;
|
|
183
|
+
currentTitle: string;
|
|
184
|
+
screenName?: string;
|
|
185
|
+
steps?: string[];
|
|
186
|
+
expectedResult?: string;
|
|
187
|
+
kind?: string;
|
|
188
|
+
attempt?: number;
|
|
189
|
+
}): Promise<{
|
|
190
|
+
title: string;
|
|
191
|
+
}>;
|
|
192
|
+
/**
|
|
193
|
+
* H2: Generate AI output validation test cases for testing LLM/AI feature outputs.
|
|
194
|
+
* Produces cases with AI-specific assertions: similarity, tone, safety, structured output.
|
|
195
|
+
*/
|
|
196
|
+
export declare function generateAiOutputValidationCases(opts: {
|
|
197
|
+
tenantId: string;
|
|
198
|
+
projectId: string;
|
|
199
|
+
suiteId: string;
|
|
200
|
+
featureDescription: string;
|
|
201
|
+
aiModelName?: string;
|
|
202
|
+
assertionTypes?: Array<'similarity' | 'tone' | 'safety' | 'structured-output' | 'hallucination-check'>;
|
|
203
|
+
}): Promise<{
|
|
204
|
+
cases: any[];
|
|
205
|
+
passedLint: number;
|
|
206
|
+
}>;
|