@midscene/test 1.17.8-beta-20261106101231.0 → 1.17.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/154.mjs +2 -0
- package/dist/es/164.mjs +1469 -0
- package/dist/es/164.mjs.map +1 -0
- package/dist/es/183.mjs +295 -0
- package/dist/es/183.mjs.map +1 -0
- package/dist/es/1~112.mjs +372 -0
- package/dist/es/1~112.mjs.map +1 -0
- package/dist/es/209.mjs +1 -0
- package/dist/es/299.mjs +1886 -0
- package/dist/es/299.mjs.map +1 -0
- package/dist/es/398.mjs +5 -0
- package/dist/es/398.mjs.map +1 -0
- package/dist/es/431.mjs +375 -0
- package/dist/es/431.mjs.map +1 -0
- package/dist/es/470.mjs +1 -0
- package/dist/es/487.mjs +1 -0
- package/dist/es/629.mjs +1 -0
- package/dist/es/793.mjs +1 -0
- package/dist/es/822.mjs +1 -0
- package/dist/es/946.mjs +1 -0
- package/dist/es/cli/index.mjs +3 -2196
- package/dist/es/cli/index.mjs.map +1 -1
- package/dist/es/cli/model-command.mjs +1 -296
- package/dist/es/index.mjs +10 -409
- package/dist/es/index.mjs.map +1 -1
- package/dist/es/midscene/index.mjs +9 -9
- package/dist/es/midscene/index.mjs.map +1 -1
- package/dist/es/runtime/index.mjs +4 -5
- package/dist/es/runtime/index.mjs.map +1 -1
- package/dist/es/test-cli.mjs +1 -3786
- package/dist/lib/0~703.js.map +1 -1
- package/dist/lib/cli/index.js +45 -58
- package/dist/lib/cli/index.js.map +1 -1
- package/dist/lib/cli/model-command.js +1 -1
- package/dist/lib/cli/model-command.js.map +1 -1
- package/dist/lib/index.js +8 -37
- package/dist/lib/index.js.map +1 -1
- package/dist/lib/midscene/index.js +1 -1
- package/dist/lib/midscene/index.js.map +1 -1
- package/dist/lib/runtime/index.js +1 -1
- package/dist/lib/runtime/index.js.map +1 -1
- package/dist/lib/test-cli.js +884 -877
- package/dist/lib/test-cli.js.map +1 -1
- package/dist/types/cli/index.d.ts +4 -0
- package/package.json +20 -21
- package/dist/es/1~4.mjs +0 -383
- package/dist/es/1~4.mjs.map +0 -1
- package/dist/es/1~731.mjs +0 -306
- package/dist/es/1~731.mjs.map +0 -1
- package/dist/es/cli/model-command.mjs.map +0 -1
- package/dist/es/test-cli.mjs.map +0 -1
package/dist/es/cli/index.mjs
CHANGED
|
@@ -1,2201 +1,8 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { pathToFileURL } from "node:url";
|
|
3
|
-
import { require as api_require } from "tsx/cjs/api";
|
|
4
|
-
import { NodeRegistry, WorkflowError, WorkflowExecutionFailure, WorkflowParseError, WorkflowPublicationError, asExecutionError, buildTestRunReportDump, buildTestRunReportPlan, classifyWorkflowDocumentFormat, collectTestRunReportSources, createProjectRuntime, decideRetry, getPendingScopeOperations, getResourceCleanupCompletion, isFatalDeviceError, isReportSecretKey, isResourceRecoveryError, normalizeSteps, observeResourceOperations, runDocumentAttempts, runWorkflowDocument, sanitizeReportError, sanitizeReportText, sumTestRunReportMetrics as test_runner_sumTestRunReportMetrics, trackResourceOperation, waitForScopeOperations } from "@midscene/core/internal/test-runner";
|
|
5
|
-
import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
|
|
6
|
-
import { globSync } from "tinyglobby";
|
|
7
|
-
import { createHash, randomUUID } from "node:crypto";
|
|
8
|
-
import { JSON_SCHEMA, load } from "js-yaml";
|
|
9
|
-
import { mkdir, writeFile } from "node:fs/promises";
|
|
10
|
-
import { resolveMidsceneRunDir, runWithMidsceneRunDir } from "@midscene/shared/common";
|
|
11
|
-
import { setMaxListeners } from "node:events";
|
|
12
|
-
import { ReportSession, TestRunReportAssembler, calculateTestRunHealth } from "@midscene/core/report";
|
|
13
|
-
import { getDebug, runWithConsoleLogScope } from "@midscene/shared/logger";
|
|
14
|
-
import "node:util";
|
|
15
|
-
import { globalConfigManager } from "@midscene/shared/env";
|
|
16
|
-
const registry_NodeRegistry = NodeRegistry;
|
|
17
|
-
const projectIdFromIndex = (index)=>`project-${index}`;
|
|
18
|
-
const defineTestProject = (definition)=>definition;
|
|
19
|
-
const defineProjectSetup = (definition)=>definition;
|
|
20
|
-
const defineFileSetup = (definition)=>definition;
|
|
21
|
-
const getDefaultExport = (loaded, absolutePath)=>{
|
|
22
|
-
if ('object' == typeof loaded && null !== loaded && 'default' in loaded) {
|
|
23
|
-
const defaultExport = loaded.default;
|
|
24
|
-
if ('object' == typeof defaultExport && null !== defaultExport && '__esModule' in defaultExport) {
|
|
25
|
-
if ('default' in defaultExport) return defaultExport.default;
|
|
26
|
-
throw new TypeError(`Midscene config "${absolutePath}" must have a default export.`);
|
|
27
|
-
}
|
|
28
|
-
return defaultExport;
|
|
29
|
-
}
|
|
30
|
-
throw new TypeError(`Midscene config "${absolutePath}" must have a default export.`);
|
|
31
|
-
};
|
|
32
|
-
const isRecord = (value)=>'object' == typeof value && null !== value && !Array.isArray(value);
|
|
33
|
-
const rejectUnknownKeys = (value, allowedKeys, label)=>{
|
|
34
|
-
const allowed = new Set(allowedKeys);
|
|
35
|
-
const unknown = Object.keys(value).find((key)=>!allowed.has(key));
|
|
36
|
-
if (unknown) throw new TypeError(`Midscene config ${label}.${unknown} is not supported.`);
|
|
37
|
-
};
|
|
38
|
-
const isAbsolutePattern = (pattern)=>isAbsolute(pattern) || /^[A-Za-z]:[\\/]/.test(pattern) || pattern.startsWith('\\\\');
|
|
39
|
-
const validatePatterns = (value, field, label = 'files', allowEmptyInclude = false)=>{
|
|
40
|
-
if (!Array.isArray(value)) throw new TypeError(`Midscene config ${label}.${field} must be an array.`);
|
|
41
|
-
if ('include' === field && 0 === value.length && !allowEmptyInclude) throw new TypeError(`Midscene config ${label}.include must be a non-empty array.`);
|
|
42
|
-
return value.map((pattern, index)=>{
|
|
43
|
-
if ('string' != typeof pattern || 0 === pattern.trim().length) throw new TypeError(`Midscene config ${label}.${field}[${index}] must be a non-empty string.`);
|
|
44
|
-
if (isAbsolutePattern(pattern)) throw new TypeError(`Midscene config ${label}.${field}[${index}] must be relative to the project root.`);
|
|
45
|
-
if (pattern.split(/[\\/]/).includes('..')) throw new TypeError(`Midscene config ${label}.${field}[${index}] must not contain a ".." path segment.`);
|
|
46
|
-
if (pattern.includes('\\')) throw new TypeError(`Midscene config ${label}.${field}[${index}] must use POSIX path separators (/).`);
|
|
47
|
-
if (pattern.startsWith('!')) {
|
|
48
|
-
if ('include' === field) throw new TypeError(`Midscene config ${label}.include[${index}] must not be a negated pattern. Use files.exclude instead.`);
|
|
49
|
-
throw new TypeError(`Midscene config ${label}.exclude[${index}] must not be a negated pattern.`);
|
|
50
|
-
}
|
|
51
|
-
return pattern;
|
|
52
|
-
});
|
|
53
|
-
};
|
|
54
|
-
const validateTestFileSelection = (value, label = 'files')=>{
|
|
55
|
-
if (void 0 === value) return;
|
|
56
|
-
if (!isRecord(value)) throw new TypeError(`Midscene config ${label} must be an object.`);
|
|
57
|
-
rejectUnknownKeys(value, [
|
|
58
|
-
'include',
|
|
59
|
-
'exclude'
|
|
60
|
-
], label);
|
|
61
|
-
const include = validatePatterns(value.include, 'include', label);
|
|
62
|
-
const exclude = void 0 === value.exclude ? void 0 : validatePatterns(value.exclude, 'exclude', label);
|
|
63
|
-
return Object.freeze({
|
|
64
|
-
include: Object.freeze(include),
|
|
65
|
-
...exclude ? {
|
|
66
|
-
exclude: Object.freeze(exclude)
|
|
67
|
-
} : {}
|
|
68
|
-
});
|
|
69
|
-
};
|
|
70
|
-
const validateTagList = (value, field, label)=>{
|
|
71
|
-
if (void 0 === value) return [];
|
|
72
|
-
if (!Array.isArray(value)) throw new TypeError(`Midscene config ${label}.${field} must be an array.`);
|
|
73
|
-
return value.map((tag, index)=>{
|
|
74
|
-
if ('string' != typeof tag || 0 === tag.trim().length) throw new TypeError(`Midscene config ${label}.${field}[${index}] must be a non-empty string.`);
|
|
75
|
-
return tag;
|
|
76
|
-
});
|
|
77
|
-
};
|
|
78
|
-
const validateTagSelection = (value, label)=>{
|
|
79
|
-
if (void 0 === value) return Object.freeze({
|
|
80
|
-
include: [],
|
|
81
|
-
exclude: []
|
|
82
|
-
});
|
|
83
|
-
if (!isRecord(value)) throw new TypeError(`Midscene config ${label} must be an object.`);
|
|
84
|
-
return Object.freeze({
|
|
85
|
-
include: Object.freeze(validateTagList(value.include, 'include', label)),
|
|
86
|
-
exclude: Object.freeze(validateTagList(value.exclude, 'exclude', label))
|
|
87
|
-
});
|
|
88
|
-
};
|
|
89
|
-
function assertJsonValue(value, path, seen) {
|
|
90
|
-
if (null === value || 'string' == typeof value || 'boolean' == typeof value) return;
|
|
91
|
-
if ('number' == typeof value) {
|
|
92
|
-
if (Number.isFinite(value)) return;
|
|
93
|
-
throw new TypeError(`Midscene config ${path} must be JSON-compatible.`);
|
|
94
|
-
}
|
|
95
|
-
if ('object' != typeof value || void 0 === value) throw new TypeError(`Midscene config ${path} must be JSON-compatible.`);
|
|
96
|
-
if (!Array.isArray(value) && Object.getPrototypeOf(value) !== Object.prototype && null !== Object.getPrototypeOf(value)) throw new TypeError(`Midscene config ${path} must be JSON-compatible.`);
|
|
97
|
-
if (seen.has(value)) throw new TypeError(`Midscene config ${path} must not contain cycles.`);
|
|
98
|
-
seen.add(value);
|
|
99
|
-
if (Array.isArray(value)) value.forEach((item, index)=>assertJsonValue(item, `${path}[${index}]`, seen));
|
|
100
|
-
else for (const [key, child] of Object.entries(value))assertJsonValue(child, `${path}.${key}`, seen);
|
|
101
|
-
seen.delete(value);
|
|
102
|
-
}
|
|
103
|
-
const deepFreezeJson = (value)=>{
|
|
104
|
-
if ('object' == typeof value && null !== value) {
|
|
105
|
-
for (const child of Object.values(value))deepFreezeJson(child);
|
|
106
|
-
Object.freeze(value);
|
|
107
|
-
}
|
|
108
|
-
return value;
|
|
109
|
-
};
|
|
110
|
-
const validateVariables = (value, label)=>{
|
|
111
|
-
if (void 0 === value) return Object.freeze({});
|
|
112
|
-
if (!isRecord(value)) throw new TypeError(`Midscene config ${label} must be an object.`);
|
|
113
|
-
assertJsonValue(value, label, new Set());
|
|
114
|
-
return deepFreezeJson(value);
|
|
115
|
-
};
|
|
116
|
-
const validateSetup = (value, label)=>{
|
|
117
|
-
if (void 0 === value) return;
|
|
118
|
-
if (!isRecord(value)) throw new TypeError(`Midscene config ${label} must be an object.`);
|
|
119
|
-
rejectUnknownKeys(value, [
|
|
120
|
-
'name',
|
|
121
|
-
'setup'
|
|
122
|
-
], label);
|
|
123
|
-
if ('string' != typeof value.name || 0 === value.name.trim().length) throw new TypeError(`Midscene config ${label}.name must be non-empty.`);
|
|
124
|
-
if ('function' != typeof value.setup) throw new TypeError(`Midscene config ${label}.setup must be a function.`);
|
|
125
|
-
return value;
|
|
126
|
-
};
|
|
127
|
-
const validatePositiveInteger = (value, fallback, label)=>{
|
|
128
|
-
if (void 0 === value) return fallback;
|
|
129
|
-
if (!Number.isInteger(value) || value <= 0) throw new TypeError(`Midscene config ${label} must be a positive integer.`);
|
|
130
|
-
return value;
|
|
131
|
-
};
|
|
132
|
-
const validateNonNegativeInteger = (value, fallback, label)=>{
|
|
133
|
-
if (void 0 === value) return fallback;
|
|
134
|
-
if (!Number.isInteger(value) || value < 0) throw new TypeError(`Midscene config ${label} must be a non-negative integer.`);
|
|
135
|
-
return value;
|
|
136
|
-
};
|
|
137
|
-
const validateConcurrency = (value, label)=>{
|
|
138
|
-
if (null === value) return null;
|
|
139
|
-
return validatePositiveInteger(value, 1, label);
|
|
140
|
-
};
|
|
141
|
-
const validateFileConcurrency = (maxConcurrentFiles, fileSetup, label)=>{
|
|
142
|
-
if (maxConcurrentFiles > 1 && !fileSetup) throw new TypeError(`Midscene config ${label}.fileSetup is required when the effective maxConcurrentFiles is greater than 1.`);
|
|
143
|
-
};
|
|
144
|
-
const validateExecutionProjects = (value, defaultSetup, defaultFileSetup, globalNodes, test)=>{
|
|
145
|
-
if (void 0 === value) {
|
|
146
|
-
const setup = validateSetup(defaultSetup, 'setup');
|
|
147
|
-
const fileSetup = validateSetup(defaultFileSetup, 'fileSetup');
|
|
148
|
-
const maxConcurrentFiles = test.maxConcurrentFiles ?? 1 / 0;
|
|
149
|
-
validateFileConcurrency(maxConcurrentFiles, fileSetup, 'default');
|
|
150
|
-
return {
|
|
151
|
-
hasExplicitProjects: false,
|
|
152
|
-
projects: Object.freeze([
|
|
153
|
-
Object.freeze({
|
|
154
|
-
projectId: projectIdFromIndex(0),
|
|
155
|
-
name: 'default',
|
|
156
|
-
...setup ? {
|
|
157
|
-
setup
|
|
158
|
-
} : {},
|
|
159
|
-
...fileSetup ? {
|
|
160
|
-
fileSetup
|
|
161
|
-
} : {},
|
|
162
|
-
maxConcurrentFiles,
|
|
163
|
-
tags: Object.freeze({
|
|
164
|
-
include: [],
|
|
165
|
-
exclude: []
|
|
166
|
-
}),
|
|
167
|
-
retry: 0,
|
|
168
|
-
variables: Object.freeze({}),
|
|
169
|
-
nodes: new registry_NodeRegistry(globalNodes.definitions())
|
|
170
|
-
})
|
|
171
|
-
])
|
|
172
|
-
};
|
|
173
|
-
}
|
|
174
|
-
if (void 0 !== defaultSetup) throw new TypeError('Midscene config setup cannot be used together with projects. Move setup to projects[].setup.');
|
|
175
|
-
if (void 0 !== defaultFileSetup) throw new TypeError('Midscene config fileSetup cannot be used together with projects. Move fileSetup to projects[].fileSetup.');
|
|
176
|
-
if (!Array.isArray(value) || 0 === value.length) throw new TypeError('Midscene config projects must be a non-empty array when declared.');
|
|
177
|
-
const names = new Set();
|
|
178
|
-
const projects = value.map((candidate, index)=>{
|
|
179
|
-
const label = `projects[${index}]`;
|
|
180
|
-
if (!isRecord(candidate)) throw new TypeError(`Midscene config ${label} must be an object.`);
|
|
181
|
-
rejectUnknownKeys(candidate, [
|
|
182
|
-
'name',
|
|
183
|
-
'setup',
|
|
184
|
-
'fileSetup',
|
|
185
|
-
'maxConcurrentFiles',
|
|
186
|
-
'nodes',
|
|
187
|
-
'files',
|
|
188
|
-
'tags',
|
|
189
|
-
'retry',
|
|
190
|
-
'variables'
|
|
191
|
-
], label);
|
|
192
|
-
if ('string' != typeof candidate.name || 0 === candidate.name.trim().length) throw new TypeError(`Midscene config ${label}.name must be non-empty.`);
|
|
193
|
-
if (names.has(candidate.name)) throw new TypeError(`Midscene config project name "${candidate.name}" must be unique.`);
|
|
194
|
-
names.add(candidate.name);
|
|
195
|
-
const files = validateTestFileSelection(candidate.files, `${label}.files`);
|
|
196
|
-
if (void 0 !== candidate.nodes && !Array.isArray(candidate.nodes)) throw new TypeError(`Midscene config ${label}.nodes must be an array.`);
|
|
197
|
-
const localNodes = new registry_NodeRegistry(candidate.nodes);
|
|
198
|
-
const nodes = new registry_NodeRegistry([
|
|
199
|
-
...globalNodes.definitions().filter((node)=>!localNodes.has(node.name)),
|
|
200
|
-
...localNodes.definitions()
|
|
201
|
-
]);
|
|
202
|
-
const fileSetup = validateSetup(candidate.fileSetup, `${label}.fileSetup`);
|
|
203
|
-
const globalFileLimit = test.maxConcurrentFiles ?? 1 / 0;
|
|
204
|
-
const maxConcurrentFiles = Math.min(globalFileLimit, validatePositiveInteger(candidate.maxConcurrentFiles, globalFileLimit, `${label}.maxConcurrentFiles`));
|
|
205
|
-
validateFileConcurrency(maxConcurrentFiles, fileSetup, label);
|
|
206
|
-
return Object.freeze({
|
|
207
|
-
projectId: projectIdFromIndex(index),
|
|
208
|
-
name: candidate.name,
|
|
209
|
-
nodes,
|
|
210
|
-
...fileSetup ? {
|
|
211
|
-
fileSetup
|
|
212
|
-
} : {},
|
|
213
|
-
maxConcurrentFiles,
|
|
214
|
-
...files ? {
|
|
215
|
-
files
|
|
216
|
-
} : {},
|
|
217
|
-
tags: validateTagSelection(candidate.tags, `${label}.tags`),
|
|
218
|
-
retry: validateNonNegativeInteger(candidate.retry, 0, `${label}.retry`),
|
|
219
|
-
variables: validateVariables(candidate.variables, `${label}.variables`),
|
|
220
|
-
...void 0 === candidate.setup ? {} : {
|
|
221
|
-
setup: validateSetup(candidate.setup, `${label}.setup`)
|
|
222
|
-
}
|
|
223
|
-
});
|
|
224
|
-
});
|
|
225
|
-
return {
|
|
226
|
-
projects: Object.freeze(projects),
|
|
227
|
-
hasExplicitProjects: true
|
|
228
|
-
};
|
|
229
|
-
};
|
|
230
|
-
const validateTestOptions = (value, hasExplicitProjects)=>{
|
|
231
|
-
if (void 0 !== value && !isRecord(value)) throw new TypeError('Midscene config test must be an object.');
|
|
232
|
-
const candidate = value ?? {};
|
|
233
|
-
if (void 0 !== candidate.maxConcurrency) {
|
|
234
|
-
const limit = 'number' == typeof candidate.maxConcurrency && Number.isInteger(candidate.maxConcurrency) && candidate.maxConcurrency > 0 ? candidate.maxConcurrency : 1;
|
|
235
|
-
throw new TypeError([
|
|
236
|
-
'Midscene config test.maxConcurrency is no longer supported.',
|
|
237
|
-
'It controlled project concurrency, not file concurrency. Remove it and use:',
|
|
238
|
-
` test: { maxConcurrentProjects: ${limit}, maxConcurrentFiles: ${hasExplicitProjects ? limit : 1} }`,
|
|
239
|
-
hasExplicitProjects ? ' Set projects[].maxConcurrentFiles to 1 to keep files sequential within each project.' : ' The implicit default project keeps running files sequentially.',
|
|
240
|
-
hasExplicitProjects ? 'For parallel files, configure fileSetup and raise both test.maxConcurrentFiles and projects[].maxConcurrentFiles.' : 'For parallel files, configure fileSetup and raise test.maxConcurrentFiles.',
|
|
241
|
-
'Migration guide: https://midscenejs.com/midscene-test/configuration.html#existing-configurations'
|
|
242
|
-
].join('\n'));
|
|
243
|
-
}
|
|
244
|
-
return Object.freeze({
|
|
245
|
-
maxConcurrentFiles: validateConcurrency(candidate.maxConcurrentFiles, 'test.maxConcurrentFiles'),
|
|
246
|
-
maxConcurrentProjects: validateConcurrency(candidate.maxConcurrentProjects, 'test.maxConcurrentProjects'),
|
|
247
|
-
setupTimeout: validatePositiveInteger(candidate.setupTimeout, 120000, 'test.setupTimeout'),
|
|
248
|
-
teardownTimeout: validatePositiveInteger(candidate.teardownTimeout, 30000, 'test.teardownTimeout'),
|
|
249
|
-
bail: validateNonNegativeInteger(candidate.bail, 0, 'test.bail'),
|
|
250
|
-
testTimeout: validatePositiveInteger(candidate.testTimeout, 120000, 'test.testTimeout')
|
|
251
|
-
});
|
|
252
|
-
};
|
|
253
|
-
const validateOutputPath = (value, label)=>{
|
|
254
|
-
if ('string' != typeof value || 0 === value.trim().length) throw new TypeError(`Midscene config ${label} must be non-empty.`);
|
|
255
|
-
return value;
|
|
256
|
-
};
|
|
257
|
-
const validateOutput = (value)=>{
|
|
258
|
-
if (void 0 !== value && !isRecord(value)) throw new TypeError('Midscene config output must be an object.');
|
|
259
|
-
const candidate = value ?? {};
|
|
260
|
-
rejectUnknownKeys(candidate, [
|
|
261
|
-
'reportDir',
|
|
262
|
-
'reportFormat'
|
|
263
|
-
], 'output');
|
|
264
|
-
if (void 0 !== candidate.reportFormat && 'single-html' !== candidate.reportFormat && 'html-and-external-assets' !== candidate.reportFormat) throw new TypeError('Midscene config output.reportFormat must be single-html or html-and-external-assets.');
|
|
265
|
-
return Object.freeze({
|
|
266
|
-
reportFormat: candidate.reportFormat ?? 'single-html',
|
|
267
|
-
...void 0 === candidate.reportDir ? {} : {
|
|
268
|
-
reportDir: validateOutputPath(candidate.reportDir, 'output.reportDir')
|
|
269
|
-
}
|
|
270
|
-
});
|
|
271
|
-
};
|
|
272
|
-
const validateTestProjectDefinition = (definition)=>{
|
|
273
|
-
if (!isRecord(definition)) throw new TypeError('Midscene config must default export an object.');
|
|
274
|
-
if (void 0 !== definition.nodes && !Array.isArray(definition.nodes)) throw new TypeError('Midscene config nodes must be an array.');
|
|
275
|
-
if ('setupWorkflow' in definition) throw new TypeError('Midscene config setupWorkflow is no longer supported. Use setup or projects[].setup instead.');
|
|
276
|
-
if ('setupDocument' in definition) throw new TypeError('Midscene config setupDocument is not supported. Use setup or projects[].setup for resources, and beforeAll/afterAll for document lifecycle steps.');
|
|
277
|
-
if ('root' in definition) throw new TypeError('Midscene config root is not supported. Pass the Test Project directory to midscene-test instead.');
|
|
278
|
-
if ('files' in definition) throw new TypeError('Midscene config files is not supported at the root. Move it to projects[].files.');
|
|
279
|
-
if ('testRunner' in definition) throw new TypeError('Midscene config testRunner is not supported. Rename it to test.');
|
|
280
|
-
rejectUnknownKeys(definition, [
|
|
281
|
-
'setup',
|
|
282
|
-
'fileSetup',
|
|
283
|
-
'projects',
|
|
284
|
-
'test',
|
|
285
|
-
'output',
|
|
286
|
-
'nodes'
|
|
287
|
-
], 'root');
|
|
288
|
-
const nodes = new registry_NodeRegistry(definition.nodes);
|
|
289
|
-
const test = validateTestOptions(definition.test, void 0 !== definition.projects);
|
|
290
|
-
const resolvedProjects = validateExecutionProjects(definition.projects, definition.setup, definition.fileSetup, nodes, test);
|
|
291
|
-
return {
|
|
292
|
-
...resolvedProjects,
|
|
293
|
-
test,
|
|
294
|
-
output: validateOutput(definition.output),
|
|
295
|
-
nodes,
|
|
296
|
-
resolveNode: (name)=>nodes.get(name)
|
|
297
|
-
};
|
|
298
|
-
};
|
|
299
|
-
const assertSupportedConfig = (absolutePath)=>{
|
|
300
|
-
if (!absolutePath.endsWith('.ts') && !absolutePath.endsWith('.mjs')) {
|
|
301
|
-
const extension = absolutePath.match(/(\.[^./\\]+)$/)?.[1] ?? '(none)';
|
|
302
|
-
throw new TypeError(`Unsupported Midscene config extension: ${extension}. Supported extensions: .ts, .mjs.`);
|
|
303
|
-
}
|
|
304
|
-
return absolutePath.endsWith('.mjs') ? '.mjs' : '.ts';
|
|
305
|
-
};
|
|
306
|
-
const canRetryWithCjsLoader = (error)=>{
|
|
307
|
-
if (!(error instanceof Error)) return false;
|
|
308
|
-
const errorCode = error.code;
|
|
309
|
-
return error instanceof SyntaxError && /^Unexpected (?:identifier|reserved word|token)/.test(error.message) || 'ERR_UNSUPPORTED_TYPESCRIPT_SYNTAX' === errorCode || 'ERR_UNKNOWN_FILE_EXTENSION' === errorCode && /^Unknown file extension\b/.test(error.message);
|
|
310
|
-
};
|
|
311
|
-
async function loadTestProject(configPath) {
|
|
312
|
-
if (!configPath) return validateTestProjectDefinition({
|
|
313
|
-
nodes: []
|
|
314
|
-
});
|
|
315
|
-
const absolutePath = external_node_path_resolve(configPath);
|
|
316
|
-
const extension = assertSupportedConfig(absolutePath);
|
|
317
|
-
let loaded;
|
|
318
|
-
try {
|
|
319
|
-
if ('.mjs' === extension) loaded = await import(pathToFileURL(absolutePath).href);
|
|
320
|
-
else {
|
|
321
|
-
const { tsImport } = await import("tsx/esm/api");
|
|
322
|
-
loaded = await tsImport(pathToFileURL(absolutePath).href, {
|
|
323
|
-
parentURL: pathToFileURL(`${dirname(absolutePath)}${sep}`).href,
|
|
324
|
-
tsconfig: false
|
|
325
|
-
});
|
|
326
|
-
}
|
|
327
|
-
} catch (error) {
|
|
328
|
-
try {
|
|
329
|
-
if ('.mjs' === extension || !canRetryWithCjsLoader(error)) throw error;
|
|
330
|
-
loaded = api_require(absolutePath, pathToFileURL(absolutePath));
|
|
331
|
-
} catch (fallbackError) {
|
|
332
|
-
const message = fallbackError instanceof Error ? `: ${fallbackError.message}` : '';
|
|
333
|
-
throw new Error(`Failed to load Midscene config "${absolutePath}"${message}`, {
|
|
334
|
-
cause: fallbackError
|
|
335
|
-
});
|
|
336
|
-
}
|
|
337
|
-
}
|
|
338
|
-
return validateTestProjectDefinition(getDefaultExport(loaded, absolutePath));
|
|
339
|
-
}
|
|
340
|
-
const createCaseInvocationId = (projectId, sourcePath, caseIndex, invocationIndex = 0)=>createHash('sha256').update(JSON.stringify([
|
|
341
|
-
projectId,
|
|
342
|
-
sourcePath,
|
|
343
|
-
caseIndex,
|
|
344
|
-
...invocationIndex ? [
|
|
345
|
-
invocationIndex
|
|
346
|
-
] : []
|
|
347
|
-
])).digest('hex');
|
|
348
|
-
const createDocumentInvocationId = (projectId, sourcePath, invocationIndex = 0)=>createHash('sha256').update(JSON.stringify([
|
|
349
|
-
projectId,
|
|
350
|
-
sourcePath,
|
|
351
|
-
...invocationIndex ? [
|
|
352
|
-
invocationIndex
|
|
353
|
-
] : []
|
|
354
|
-
])).digest('hex');
|
|
355
|
-
const normalize_normalizeSteps = normalizeSteps;
|
|
356
|
-
function parseWorkflowSource(source) {
|
|
357
|
-
const stack = [];
|
|
358
|
-
const nodes = [];
|
|
359
|
-
const sensitiveNodes = new Set();
|
|
360
|
-
const sensitiveValues = new Set();
|
|
361
|
-
const redactedPaths = new Set();
|
|
362
|
-
const sourceOffset = 0xfeff === source.charCodeAt(0) ? 1 : 0;
|
|
363
|
-
const parsed = load(source, {
|
|
364
|
-
schema: JSON_SCHEMA,
|
|
365
|
-
listener (event, state) {
|
|
366
|
-
if ('open' === event) return void stack.push({
|
|
367
|
-
start: state.position + sourceOffset,
|
|
368
|
-
end: 0,
|
|
369
|
-
value: null,
|
|
370
|
-
children: [],
|
|
371
|
-
kind: null
|
|
372
|
-
});
|
|
373
|
-
const node = stack.pop();
|
|
374
|
-
node.end = state.position + sourceOffset;
|
|
375
|
-
node.value = state.result;
|
|
376
|
-
node.kind = state.kind;
|
|
377
|
-
stack.at(-1)?.children.push(node);
|
|
378
|
-
nodes.push(node);
|
|
379
|
-
if ('mapping' !== state.kind) return;
|
|
380
|
-
for(let index = 0; index < node.children.length; index += 2){
|
|
381
|
-
const key = node.children[index].value;
|
|
382
|
-
const value = node.children[index + 1];
|
|
383
|
-
if ('string' == typeof key && isReportSecretKey(key) && value) {
|
|
384
|
-
sensitiveNodes.add(value);
|
|
385
|
-
if (null !== value.value) sensitiveValues.add(value.value);
|
|
386
|
-
redactedPaths.add(`$..[${JSON.stringify(key)}]`);
|
|
387
|
-
}
|
|
388
|
-
}
|
|
389
|
-
}
|
|
390
|
-
});
|
|
391
|
-
const ranges = nodes.filter((node)=>sensitiveNodes.has(node) || sensitiveValues.has(node.value)).sort((left, right)=>left.start - right.start || right.end - left.end);
|
|
392
|
-
let offset = 0;
|
|
393
|
-
let redacted = '';
|
|
394
|
-
for (const range of ranges){
|
|
395
|
-
if (range.start < offset) continue;
|
|
396
|
-
const text = source.slice(range.start, range.end);
|
|
397
|
-
const leading = text.match(/^\s*/)?.[0] ?? '';
|
|
398
|
-
const trailing = text.slice(leading.length).match(/\s*$/)?.[0] ?? '';
|
|
399
|
-
redacted += `${source.slice(offset, range.start)}${leading}"[REDACTED]"${trailing}`;
|
|
400
|
-
offset = range.end;
|
|
401
|
-
}
|
|
402
|
-
redacted += source.slice(offset);
|
|
403
|
-
const value = sanitizeReportText(redacted, 65536);
|
|
404
|
-
return {
|
|
405
|
-
parsed,
|
|
406
|
-
locate (path) {
|
|
407
|
-
let node = nodes.at(-1);
|
|
408
|
-
for (const part of path){
|
|
409
|
-
if (!node) return;
|
|
410
|
-
while(1 === node.children.length && node.children[0].value === node.value)node = node.children[0];
|
|
411
|
-
if ('number' == typeof part && 'sequence' === node.kind) node = node.children[part];
|
|
412
|
-
else {
|
|
413
|
-
if ('string' != typeof part || 'mapping' !== node.kind) return;
|
|
414
|
-
const children = node.children;
|
|
415
|
-
node = void 0;
|
|
416
|
-
for(let index = 0; index < children.length; index += 2)if (children[index].value === part) {
|
|
417
|
-
node = children[index + 1];
|
|
418
|
-
break;
|
|
419
|
-
}
|
|
420
|
-
}
|
|
421
|
-
}
|
|
422
|
-
if (!node) return;
|
|
423
|
-
let start = node.start;
|
|
424
|
-
while(start < node.end)if (/\s/.test(source[start])) start++;
|
|
425
|
-
else if ('#' === source[start]) {
|
|
426
|
-
const end = source.indexOf('\n', start);
|
|
427
|
-
if (end < 0) return;
|
|
428
|
-
start = end + 1;
|
|
429
|
-
} else break;
|
|
430
|
-
const prefix = source.slice(0, start).split(/\r\n|\r|\n/);
|
|
431
|
-
return {
|
|
432
|
-
line: prefix.length,
|
|
433
|
-
column: prefix.at(-1).length + 1
|
|
434
|
-
};
|
|
435
|
-
},
|
|
436
|
-
sourceYaml: {
|
|
437
|
-
value,
|
|
438
|
-
...redactedPaths.size ? {
|
|
439
|
-
redactedPaths: [
|
|
440
|
-
...redactedPaths
|
|
441
|
-
]
|
|
442
|
-
} : {},
|
|
443
|
-
...value !== redacted ? {
|
|
444
|
-
truncatedPaths: [
|
|
445
|
-
'$'
|
|
446
|
-
]
|
|
447
|
-
} : {}
|
|
448
|
-
}
|
|
449
|
-
};
|
|
450
|
-
}
|
|
451
|
-
const interpolationExpression = /\$\{\{([A-Za-z_][A-Za-z0-9_]*)\}\}|\$\{([^{}]+)\}/g;
|
|
452
|
-
const exactEnvExpression = /^\$\{\{([A-Za-z_][A-Za-z0-9_]*)\}\}$/;
|
|
453
|
-
const exactProjectExpression = /^\$\{([^{}]+)\}$/;
|
|
454
|
-
const missingVariable = (kind, name, options)=>{
|
|
455
|
-
throw new WorkflowParseError(`Undefined ${kind} variable "${name}" in ${options.location.sourcePath}.`, {
|
|
456
|
-
...options.location,
|
|
457
|
-
variable: name,
|
|
458
|
-
variableKind: kind
|
|
459
|
-
});
|
|
460
|
-
};
|
|
461
|
-
const projectVariable = (name, options)=>{
|
|
462
|
-
if (!Object.hasOwn(options.variables ?? {}, name)) return missingVariable('project', name, options);
|
|
463
|
-
return options.variables[name];
|
|
464
|
-
};
|
|
465
|
-
const environmentVariable = (name, options)=>{
|
|
466
|
-
const value = options.env?.[name];
|
|
467
|
-
if (void 0 === value) return missingVariable('environment', name, options);
|
|
468
|
-
return value;
|
|
469
|
-
};
|
|
470
|
-
const interpolationValue = (name, value, options)=>{
|
|
471
|
-
if ('object' == typeof value && null !== value) throw new WorkflowParseError(`Project variable "${name}" must be a primitive when embedded in a string.`, {
|
|
472
|
-
...options.location,
|
|
473
|
-
variable: name,
|
|
474
|
-
variableKind: 'project'
|
|
475
|
-
});
|
|
476
|
-
return String(value);
|
|
477
|
-
};
|
|
478
|
-
const resolveString = (value, options)=>{
|
|
479
|
-
const exactEnv = exactEnvExpression.exec(value);
|
|
480
|
-
if (exactEnv) return environmentVariable(exactEnv[1], options);
|
|
481
|
-
const exactProject = exactProjectExpression.exec(value);
|
|
482
|
-
if (exactProject) return projectVariable(exactProject[1], options);
|
|
483
|
-
return value.replace(interpolationExpression, (_match, environmentName, projectName)=>environmentName ? environmentVariable(environmentName, options) : interpolationValue(projectName, projectVariable(projectName, options), options));
|
|
484
|
-
};
|
|
485
|
-
const resolveWorkflowVariables = (value, options)=>{
|
|
486
|
-
if ('string' == typeof value) return resolveString(value, options);
|
|
487
|
-
if (Array.isArray(value)) return value.map((child)=>resolveWorkflowVariables(child, options));
|
|
488
|
-
if ('object' == typeof value && null !== value) return Object.fromEntries(Object.entries(value).map(([key, child])=>[
|
|
489
|
-
key,
|
|
490
|
-
resolveWorkflowVariables(child, options)
|
|
491
|
-
]));
|
|
492
|
-
return value;
|
|
493
|
-
};
|
|
494
|
-
function _define_property(obj, key, value) {
|
|
495
|
-
if (key in obj) Object.defineProperty(obj, key, {
|
|
496
|
-
value: value,
|
|
497
|
-
enumerable: true,
|
|
498
|
-
configurable: true,
|
|
499
|
-
writable: true
|
|
500
|
-
});
|
|
501
|
-
else obj[key] = value;
|
|
502
|
-
return obj;
|
|
503
|
-
}
|
|
504
|
-
const isMapping = (value)=>'object' == typeof value && null !== value && !Array.isArray(value);
|
|
505
|
-
class NativeWorkflowFormatError extends WorkflowParseError {
|
|
506
|
-
constructor(actualFormat, sourcePath){
|
|
507
|
-
super('legacy' === actualFormat ? `Legacy tasks/flow YAML is not supported by midscene-test: ${sourcePath}. Run it with \`midscene ${sourcePath}\`, or migrate the file to cases/steps.` : 'legacy-batch' === actualFormat ? `Legacy YAML batch configuration is not a midscene-test workflow document: ${sourcePath}. Run it with \`midscene --config ${sourcePath}\`, or migrate it to a native project config.` : `A workflow file cannot mix legacy tasks/flow with native cases/steps or lifecycle hooks: ${sourcePath}. Keep the formats in separate files and run each file with its matching command.`, {
|
|
508
|
-
sourcePath,
|
|
509
|
-
actualFormat,
|
|
510
|
-
expectedFormat: 'native'
|
|
511
|
-
}), _define_property(this, "actualFormat", void 0);
|
|
512
|
-
this.actualFormat = actualFormat;
|
|
513
|
-
}
|
|
514
|
-
}
|
|
515
|
-
const collect_rejectUnknownKeys = (value, allowed, label)=>{
|
|
516
|
-
const allowedSet = new Set(allowed);
|
|
517
|
-
const unknown = Object.keys(value).filter((key)=>!allowedSet.has(key));
|
|
518
|
-
if (unknown.length) throw new WorkflowParseError(`${label} has unsupported field "${unknown[0]}".`, {
|
|
519
|
-
field: unknown[0]
|
|
520
|
-
});
|
|
521
|
-
};
|
|
522
|
-
const createCaseId = (projectId, sourcePath, caseIndex)=>createCaseInvocationId(projectId, sourcePath, caseIndex);
|
|
523
|
-
const createWorkflowDocumentId = (projectId, sourcePath)=>createDocumentInvocationId(projectId, sourcePath);
|
|
524
|
-
function collectWorkflowDocument(source, options) {
|
|
525
|
-
let snapshot;
|
|
526
|
-
try {
|
|
527
|
-
snapshot = parseWorkflowSource(readFileSync(source.absolutePath, 'utf8'));
|
|
528
|
-
} catch (error) {
|
|
529
|
-
throw new WorkflowParseError(`Failed to parse workflow document "${source.sourcePath}".`, {
|
|
530
|
-
sourcePath: source.sourcePath
|
|
531
|
-
}, error);
|
|
532
|
-
}
|
|
533
|
-
const { parsed, sourceYaml } = snapshot;
|
|
534
|
-
if (!isMapping(parsed)) throw new WorkflowParseError('Workflow document must be a mapping.');
|
|
535
|
-
const format = classifyWorkflowDocumentFormat(parsed);
|
|
536
|
-
if ('legacy' === format || 'legacy-batch' === format || 'mixed' === format) throw new NativeWorkflowFormatError(format, source.sourcePath);
|
|
537
|
-
collect_rejectUnknownKeys(parsed, [
|
|
538
|
-
'beforeAll',
|
|
539
|
-
'beforeEach',
|
|
540
|
-
'cases',
|
|
541
|
-
'afterEach',
|
|
542
|
-
'afterAll'
|
|
543
|
-
], 'Workflow document');
|
|
544
|
-
if (!Array.isArray(parsed.cases) || 0 === parsed.cases.length) throw new WorkflowParseError('Workflow document must contain a non-empty cases array.');
|
|
545
|
-
const sourcePath = source.sourcePath.split(sep).join('/');
|
|
546
|
-
const resolveStepVariables = (step, phase, stepIndex, caseIndex)=>({
|
|
547
|
-
...step,
|
|
548
|
-
meta: {
|
|
549
|
-
...step.meta,
|
|
550
|
-
sourceLocation: snapshot.locate('steps' === phase ? [
|
|
551
|
-
'cases',
|
|
552
|
-
caseIndex,
|
|
553
|
-
'steps',
|
|
554
|
-
stepIndex
|
|
555
|
-
] : [
|
|
556
|
-
phase,
|
|
557
|
-
stepIndex
|
|
558
|
-
])
|
|
559
|
-
},
|
|
560
|
-
templateInput: step.input,
|
|
561
|
-
input: resolveWorkflowVariables(step.input, {
|
|
562
|
-
variables: options.variables,
|
|
563
|
-
env: options.env ?? process.env,
|
|
564
|
-
location: {
|
|
565
|
-
projectName: source.projectName ?? source.projectId,
|
|
566
|
-
sourcePath,
|
|
567
|
-
phase,
|
|
568
|
-
stepIndex,
|
|
569
|
-
...void 0 === caseIndex ? {} : {
|
|
570
|
-
caseIndex
|
|
571
|
-
}
|
|
572
|
-
}
|
|
573
|
-
})
|
|
574
|
-
});
|
|
575
|
-
const normalizeLifecycle = (phase)=>{
|
|
576
|
-
const value = parsed[phase];
|
|
577
|
-
if (void 0 === value) return [];
|
|
578
|
-
if (!Array.isArray(value)) throw new WorkflowParseError(`Workflow document ${phase} must be an array.`, {
|
|
579
|
-
phase
|
|
580
|
-
});
|
|
581
|
-
return normalize_normalizeSteps(value, options.resolveNode).map((normalized, stepIndex)=>{
|
|
582
|
-
const resolved = options.resolveNode(normalized.node);
|
|
583
|
-
if (!resolved) throw new WorkflowParseError(`Workflow document ${phase} step ${stepIndex + 1} references unknown node "${normalized.node}".`, {
|
|
584
|
-
phase,
|
|
585
|
-
stepIndex,
|
|
586
|
-
node: normalized.node
|
|
587
|
-
});
|
|
588
|
-
return resolveStepVariables(normalized, phase, stepIndex);
|
|
589
|
-
});
|
|
590
|
-
};
|
|
591
|
-
const lifecycle = {
|
|
592
|
-
beforeAll: normalizeLifecycle('beforeAll'),
|
|
593
|
-
beforeEach: normalizeLifecycle('beforeEach'),
|
|
594
|
-
afterEach: normalizeLifecycle('afterEach'),
|
|
595
|
-
afterAll: normalizeLifecycle('afterAll')
|
|
596
|
-
};
|
|
597
|
-
const ids = new Set();
|
|
598
|
-
const cases = parsed.cases.map((definition, caseIndex)=>{
|
|
599
|
-
if (!isMapping(definition)) throw new WorkflowParseError(`Case ${caseIndex + 1} must be a mapping.`, {
|
|
600
|
-
caseIndex
|
|
601
|
-
});
|
|
602
|
-
collect_rejectUnknownKeys(definition, [
|
|
603
|
-
'name',
|
|
604
|
-
'tags',
|
|
605
|
-
'steps'
|
|
606
|
-
], `Case ${caseIndex + 1}`);
|
|
607
|
-
if ('string' != typeof definition.name || 0 === definition.name.trim().length) throw new WorkflowParseError(`Case ${caseIndex + 1} name must be a non-empty string.`, {
|
|
608
|
-
caseIndex
|
|
609
|
-
});
|
|
610
|
-
if (!Array.isArray(definition.steps) || 0 === definition.steps.length) throw new WorkflowParseError(`Case ${caseIndex + 1} steps must be a non-empty array.`, {
|
|
611
|
-
caseIndex
|
|
612
|
-
});
|
|
613
|
-
const tags = definition.tags ?? [];
|
|
614
|
-
if (!Array.isArray(tags) || tags.some((tag)=>'string' != typeof tag || 0 === tag.trim().length)) throw new WorkflowParseError(`Case ${caseIndex + 1} tags must be an array of non-empty strings.`, {
|
|
615
|
-
caseIndex
|
|
616
|
-
});
|
|
617
|
-
const steps = normalize_normalizeSteps(definition.steps, options.resolveNode).map((normalized, stepIndex)=>{
|
|
618
|
-
if (!options.resolveNode(normalized.node)) throw new WorkflowParseError(`Case ${caseIndex + 1} step ${stepIndex + 1} references unknown node "${normalized.node}".`, {
|
|
619
|
-
caseIndex,
|
|
620
|
-
stepIndex,
|
|
621
|
-
node: normalized.node
|
|
622
|
-
});
|
|
623
|
-
return resolveStepVariables(normalized, 'steps', stepIndex, caseIndex);
|
|
624
|
-
});
|
|
625
|
-
const caseId = createCaseId(source.projectId, sourcePath, caseIndex);
|
|
626
|
-
if (ids.has(caseId)) throw new WorkflowParseError(`Case id collision: ${caseId}.`, {
|
|
627
|
-
caseId
|
|
628
|
-
});
|
|
629
|
-
ids.add(caseId);
|
|
630
|
-
return {
|
|
631
|
-
caseId,
|
|
632
|
-
projectId: source.projectId,
|
|
633
|
-
sourcePath,
|
|
634
|
-
caseIndex,
|
|
635
|
-
definition: {
|
|
636
|
-
name: definition.name,
|
|
637
|
-
tags: tags,
|
|
638
|
-
steps
|
|
639
|
-
}
|
|
640
|
-
};
|
|
641
|
-
});
|
|
642
|
-
return {
|
|
643
|
-
documentId: createWorkflowDocumentId(source.projectId, sourcePath),
|
|
644
|
-
projectId: source.projectId,
|
|
645
|
-
sourcePath,
|
|
646
|
-
sourceYaml,
|
|
647
|
-
lifecycle,
|
|
648
|
-
cases
|
|
649
|
-
};
|
|
650
|
-
}
|
|
651
|
-
const writeJson = async (path, value)=>{
|
|
652
|
-
try {
|
|
653
|
-
await mkdir(dirname(path), {
|
|
654
|
-
recursive: true
|
|
655
|
-
});
|
|
656
|
-
await writeFile(path, JSON.stringify(value, null, 2));
|
|
657
|
-
} catch (error) {
|
|
658
|
-
throw new WorkflowPublicationError('write-result', path, error);
|
|
659
|
-
}
|
|
660
|
-
};
|
|
661
|
-
const writeTestRunPlan = (runDir, projects)=>writeJson(join(runDir, 'plan.json'), {
|
|
662
|
-
schemaVersion: 1,
|
|
663
|
-
createdAt: new Date().toISOString(),
|
|
664
|
-
projects: projects.map((item)=>({
|
|
665
|
-
projectId: item.project.projectId,
|
|
666
|
-
name: item.project.name,
|
|
667
|
-
documents: item.plans ?? []
|
|
668
|
-
}))
|
|
669
|
-
});
|
|
670
|
-
const toPosix = (value)=>value.split(sep).join('/');
|
|
671
|
-
const caseAttemptResultPath = (projectId, documentId, result)=>toPosix(join(projectId, 'documents', documentId, 'cases', result.caseId, `${result.runId}.json`));
|
|
672
|
-
const workflowDocumentResultPath = (result)=>toPosix(join(result.projectId, 'documents', result.documentId, void 0 === result.attemptIndex ? 'document.json' : `${result.documentRunId}.json`));
|
|
673
|
-
const collectionErrorPath = (projectId, sourcePath)=>{
|
|
674
|
-
const id = createHash('sha256').update(JSON.stringify([
|
|
675
|
-
projectId,
|
|
676
|
-
sourcePath
|
|
677
|
-
])).digest('hex');
|
|
678
|
-
return toPosix(join(projectId, 'collection-errors', `${id}.json`));
|
|
679
|
-
};
|
|
680
|
-
const writeWorkflowDocumentResult = (runDir, result)=>{
|
|
681
|
-
const { documentRunId: _documentRunId, ...persistedResult } = result;
|
|
682
|
-
return writeJson(join(runDir, workflowDocumentResultPath(result)), persistedResult);
|
|
683
|
-
};
|
|
684
|
-
const writeCaseAttemptResult = (runDir, projectId, documentId, result)=>{
|
|
685
|
-
const { runId: attemptId, ...persistedResult } = result;
|
|
686
|
-
return writeJson(join(runDir, caseAttemptResultPath(projectId, documentId, result)), {
|
|
687
|
-
...persistedResult,
|
|
688
|
-
attemptId
|
|
689
|
-
});
|
|
690
|
-
};
|
|
691
|
-
const writeCollectionError = (runDir, collectionError)=>writeJson(join(runDir, collectionErrorPath(collectionError.projectId, collectionError.sourcePath)), {
|
|
692
|
-
kind: 'collection-error',
|
|
693
|
-
...collectionError
|
|
694
|
-
});
|
|
695
|
-
const relativeToSummary = (summaryPath, target)=>{
|
|
696
|
-
const value = relative(dirname(summaryPath), target);
|
|
697
|
-
return toPosix(value || '.');
|
|
698
|
-
};
|
|
699
|
-
const errorJson = sanitizeReportError;
|
|
700
|
-
const writeTestProjectRunResult = (options)=>{
|
|
701
|
-
const { result } = options;
|
|
702
|
-
const fact = (path)=>toPosix(path);
|
|
703
|
-
return writeJson(result.summaryPath, {
|
|
704
|
-
schemaVersion: result.schemaVersion,
|
|
705
|
-
runId: result.runId,
|
|
706
|
-
startedAt: result.startedAt,
|
|
707
|
-
endedAt: result.endedAt,
|
|
708
|
-
durationMs: result.durationMs,
|
|
709
|
-
status: result.status,
|
|
710
|
-
exitCode: result.exitCode,
|
|
711
|
-
...result.errors?.length ? {
|
|
712
|
-
errors: result.errors.map(errorJson)
|
|
713
|
-
} : {},
|
|
714
|
-
projectRoot: options.projectRoot,
|
|
715
|
-
...options.configPath ? {
|
|
716
|
-
configPath: options.configPath
|
|
717
|
-
} : {},
|
|
718
|
-
factsRoot: '.',
|
|
719
|
-
plan: 'plan.json',
|
|
720
|
-
...result.operationLog ? {
|
|
721
|
-
operationLog: {
|
|
722
|
-
...result.operationLog,
|
|
723
|
-
path: relativeToSummary(result.summaryPath, result.operationLog.path)
|
|
724
|
-
}
|
|
725
|
-
} : {},
|
|
726
|
-
reportDir: relativeToSummary(result.summaryPath, result.reportDir),
|
|
727
|
-
...result.reportPath ? {
|
|
728
|
-
report: relativeToSummary(result.summaryPath, result.reportPath)
|
|
729
|
-
} : {},
|
|
730
|
-
...result.markdownPath ? {
|
|
731
|
-
markdown: relativeToSummary(result.summaryPath, result.markdownPath)
|
|
732
|
-
} : {},
|
|
733
|
-
summary: result.summary,
|
|
734
|
-
...result.metrics ? {
|
|
735
|
-
metrics: result.metrics
|
|
736
|
-
} : {},
|
|
737
|
-
projects: result.projects.map((project)=>({
|
|
738
|
-
projectId: project.projectId,
|
|
739
|
-
name: project.name,
|
|
740
|
-
status: project.status,
|
|
741
|
-
retry: project.retry,
|
|
742
|
-
fileSelection: project.fileSelection,
|
|
743
|
-
tagSelection: project.tagSelection,
|
|
744
|
-
sourceCount: project.sourceCount,
|
|
745
|
-
selectedCaseCount: project.selectedCaseCount,
|
|
746
|
-
filteredCaseCount: project.filteredCaseCount,
|
|
747
|
-
...project.lifecycle ? {
|
|
748
|
-
lifecycle: {
|
|
749
|
-
status: project.lifecycle.status,
|
|
750
|
-
startedAt: project.lifecycle.startedAt,
|
|
751
|
-
endedAt: project.lifecycle.endedAt,
|
|
752
|
-
durationMs: project.lifecycle.durationMs,
|
|
753
|
-
...project.lifecycle.cleanup ? {
|
|
754
|
-
cleanup: project.lifecycle.cleanup
|
|
755
|
-
} : {},
|
|
756
|
-
...project.lifecycle.pendingOperations ? {
|
|
757
|
-
pendingOperations: project.lifecycle.pendingOperations
|
|
758
|
-
} : {},
|
|
759
|
-
...project.lifecycle.setupError ? {
|
|
760
|
-
setupError: errorJson(project.lifecycle.setupError)
|
|
761
|
-
} : {},
|
|
762
|
-
...project.lifecycle.teardownErrors ? {
|
|
763
|
-
teardownErrors: project.lifecycle.teardownErrors.map(errorJson)
|
|
764
|
-
} : {}
|
|
765
|
-
}
|
|
766
|
-
} : {},
|
|
767
|
-
cases: project.cases.map((outcome)=>({
|
|
768
|
-
...outcome.documentRunId ? {
|
|
769
|
-
documentRunId: outcome.documentRunId
|
|
770
|
-
} : {},
|
|
771
|
-
documentId: outcome.documentId,
|
|
772
|
-
caseId: outcome.caseId,
|
|
773
|
-
sourcePath: outcome.sourcePath,
|
|
774
|
-
caseIndex: outcome.caseIndex,
|
|
775
|
-
name: outcome.name,
|
|
776
|
-
status: outcome.status,
|
|
777
|
-
...outcome.notRunReason ? {
|
|
778
|
-
notRunReason: outcome.notRunReason
|
|
779
|
-
} : {},
|
|
780
|
-
...outcome.attempts ? {
|
|
781
|
-
attempts: outcome.attempts.map((attempt)=>({
|
|
782
|
-
attemptId: attempt.runId,
|
|
783
|
-
attemptIndex: attempt.attemptIndex,
|
|
784
|
-
status: attempt.status,
|
|
785
|
-
...attempt.cleanup ? {
|
|
786
|
-
cleanup: attempt.cleanup
|
|
787
|
-
} : {},
|
|
788
|
-
...attempt.pendingOperations ? {
|
|
789
|
-
pendingOperations: attempt.pendingOperations
|
|
790
|
-
} : {},
|
|
791
|
-
...attempt.executionStatus ? {
|
|
792
|
-
executionStatus: attempt.executionStatus
|
|
793
|
-
} : {},
|
|
794
|
-
...attempt.retryDecision ? {
|
|
795
|
-
retryDecision: attempt.retryDecision
|
|
796
|
-
} : {},
|
|
797
|
-
resultFile: fact(caseAttemptResultPath(project.projectId, outcome.documentId, attempt)),
|
|
798
|
-
...attempt.reportPaths?.length ? {
|
|
799
|
-
reports: attempt.reportPaths.map((path)=>relativeToSummary(result.summaryPath, path))
|
|
800
|
-
} : {}
|
|
801
|
-
}))
|
|
802
|
-
} : {}
|
|
803
|
-
})),
|
|
804
|
-
documents: project.documents.map((document)=>({
|
|
805
|
-
documentId: document.documentId,
|
|
806
|
-
...void 0 === document.attemptIndex ? {} : {
|
|
807
|
-
attemptIndex: document.attemptIndex,
|
|
808
|
-
documentRunId: document.documentRunId
|
|
809
|
-
},
|
|
810
|
-
sourcePath: document.sourcePath,
|
|
811
|
-
status: document.status,
|
|
812
|
-
...document.cleanup ? {
|
|
813
|
-
cleanup: document.cleanup
|
|
814
|
-
} : {},
|
|
815
|
-
...document.pendingOperations ? {
|
|
816
|
-
pendingOperations: document.pendingOperations
|
|
817
|
-
} : {},
|
|
818
|
-
...document.executionStatus ? {
|
|
819
|
-
executionStatus: document.executionStatus
|
|
820
|
-
} : {},
|
|
821
|
-
...document.retryDecision ? {
|
|
822
|
-
retryDecision: document.retryDecision
|
|
823
|
-
} : {},
|
|
824
|
-
resultFile: fact(workflowDocumentResultPath(document)),
|
|
825
|
-
...document.reportPaths?.length ? {
|
|
826
|
-
reports: document.reportPaths.map((path)=>relativeToSummary(result.summaryPath, path))
|
|
827
|
-
} : {}
|
|
828
|
-
})),
|
|
829
|
-
collectionErrors: project.collectionErrors.map((error)=>({
|
|
830
|
-
sourcePath: error.sourcePath,
|
|
831
|
-
errorFile: fact(collectionErrorPath(error.projectId, error.sourcePath))
|
|
832
|
-
}))
|
|
833
|
-
}))
|
|
834
|
-
});
|
|
835
|
-
};
|
|
836
|
-
const CONFIG_NAMES = [
|
|
837
|
-
'midscene.config.ts',
|
|
838
|
-
'midscene.config.mjs'
|
|
839
|
-
];
|
|
840
|
-
const CONFIG_PREFIX = 'midscene.config.';
|
|
841
|
-
const ALWAYS_IGNORED_PATTERNS = [
|
|
842
|
-
'.git/**',
|
|
843
|
-
'.midscene/**',
|
|
844
|
-
'midscene_run/**',
|
|
845
|
-
'node_modules/**',
|
|
846
|
-
'**/.git/**',
|
|
847
|
-
'**/.midscene/**',
|
|
848
|
-
'**/midscene_run/**',
|
|
849
|
-
'**/node_modules/**'
|
|
850
|
-
];
|
|
851
|
-
const DEFAULT_TEST_FILE_SELECTION = {
|
|
852
|
-
include: [
|
|
853
|
-
'**/*.{yaml,yml}'
|
|
854
|
-
]
|
|
855
|
-
};
|
|
856
|
-
const project_preparation_toPosix = (value)=>value.split(sep).join('/');
|
|
857
|
-
const discoverResolvedTestFiles = (projectRoot, selection, excludedDirectories = [])=>{
|
|
858
|
-
const root = external_node_path_resolve(projectRoot);
|
|
859
|
-
const outputPatterns = excludedDirectories.flatMap((directory)=>{
|
|
860
|
-
const withinRoot = relative(root, directory);
|
|
861
|
-
if (!withinRoot || isAbsolute(withinRoot) || '..' === withinRoot || withinRoot.startsWith(`..${sep}`)) return [];
|
|
862
|
-
const literal = project_preparation_toPosix(withinRoot).replace(/([*?{}[\]()!])/g, '\\$1');
|
|
863
|
-
return [
|
|
864
|
-
`${literal}/**`
|
|
865
|
-
];
|
|
866
|
-
});
|
|
867
|
-
const match = (patterns)=>globSync(patterns, {
|
|
868
|
-
absolute: true,
|
|
869
|
-
caseSensitiveMatch: false,
|
|
870
|
-
cwd: root,
|
|
871
|
-
dot: true,
|
|
872
|
-
expandDirectories: false,
|
|
873
|
-
followSymbolicLinks: false,
|
|
874
|
-
ignore: [
|
|
875
|
-
...ALWAYS_IGNORED_PATTERNS,
|
|
876
|
-
...outputPatterns,
|
|
877
|
-
...selection.exclude ?? []
|
|
878
|
-
],
|
|
879
|
-
onlyFiles: true
|
|
880
|
-
}).filter((file)=>/\.ya?ml$/i.test(file));
|
|
881
|
-
const files = match(selection.include);
|
|
882
|
-
return [
|
|
883
|
-
...new Set(files.map((file)=>external_node_path_resolve(file)))
|
|
884
|
-
].sort((a, b)=>{
|
|
885
|
-
const relativeA = project_preparation_toPosix(relative(root, a));
|
|
886
|
-
const relativeB = project_preparation_toPosix(relative(root, b));
|
|
887
|
-
return relativeA < relativeB ? -1 : relativeA > relativeB ? 1 : 0;
|
|
888
|
-
});
|
|
889
|
-
};
|
|
890
|
-
const discoverTestFiles = (projectRoot, selection = DEFAULT_TEST_FILE_SELECTION)=>{
|
|
891
|
-
const normalized = validateTestFileSelection(selection);
|
|
892
|
-
if (!normalized) throw new TypeError('Test file selection is required.');
|
|
893
|
-
return discoverResolvedTestFiles(projectRoot, normalized);
|
|
894
|
-
};
|
|
895
|
-
const discoverTestConfig = (projectRoot)=>{
|
|
896
|
-
const root = external_node_path_resolve(projectRoot);
|
|
897
|
-
const candidates = readdirSync(root).filter((name)=>name.startsWith(CONFIG_PREFIX)).sort();
|
|
898
|
-
const supported = candidates.filter((name)=>CONFIG_NAMES.includes(name));
|
|
899
|
-
const unsupported = candidates.filter((name)=>!CONFIG_NAMES.includes(name));
|
|
900
|
-
if (unsupported.length > 0) throw new Error([
|
|
901
|
-
`Unsupported or conflicting Midscene configs found in ${root}:`,
|
|
902
|
-
...candidates.map((name)=>`- ${name}`),
|
|
903
|
-
`Only ${CONFIG_NAMES.join(' and ')} are supported.`
|
|
904
|
-
].join('\n'));
|
|
905
|
-
if (supported.length > 1) throw new Error([
|
|
906
|
-
`Multiple Midscene configs found in ${root}:`,
|
|
907
|
-
...supported.map((name)=>`- ${name}`),
|
|
908
|
-
'Pass --config <path> to select one explicitly.'
|
|
909
|
-
].join('\n'));
|
|
910
|
-
return supported[0] ? join(root, supported[0]) : void 0;
|
|
911
|
-
};
|
|
912
|
-
const asCollectionError = (projectId, projectName, sourcePath, error)=>({
|
|
913
|
-
projectId,
|
|
914
|
-
projectName,
|
|
915
|
-
sourcePath,
|
|
916
|
-
error: error instanceof WorkflowError ? error : new WorkflowParseError(`Failed to collect workflow document "${sourcePath}": ${error instanceof Error ? error.message : String(error)}`, {
|
|
917
|
-
projectName,
|
|
918
|
-
sourcePath
|
|
919
|
-
}, error)
|
|
920
|
-
});
|
|
921
|
-
const matchesTags = (tags, selection)=>{
|
|
922
|
-
if (selection.exclude.some((tag)=>tags.includes(tag))) return false;
|
|
923
|
-
return 0 === selection.include.length || selection.include.some((tag)=>tags.includes(tag));
|
|
924
|
-
};
|
|
925
|
-
const filterDocumentCases = (document, project)=>{
|
|
926
|
-
const cases = document.cases.filter((item)=>matchesTags(item.definition.tags ?? [], project.tags));
|
|
927
|
-
const filtered = document.cases.length - cases.length;
|
|
928
|
-
return 0 === cases.length ? {
|
|
929
|
-
filtered
|
|
930
|
-
} : {
|
|
931
|
-
document: {
|
|
932
|
-
...document,
|
|
933
|
-
cases
|
|
934
|
-
},
|
|
935
|
-
filtered
|
|
936
|
-
};
|
|
937
|
-
};
|
|
938
|
-
const selectProjects = (projects, names)=>{
|
|
939
|
-
if (!names || 0 === names.length) return projects;
|
|
940
|
-
const requested = new Set(names);
|
|
941
|
-
if (requested.size !== names.length) throw new Error('Each --project name may only be specified once.');
|
|
942
|
-
const known = new Set(projects.map((project)=>project.name));
|
|
943
|
-
const unknown = names.find((name)=>!known.has(name));
|
|
944
|
-
if (unknown) throw new Error(`Unknown Midscene project: ${unknown}`);
|
|
945
|
-
return projects.filter((project)=>requested.has(project.name));
|
|
946
|
-
};
|
|
947
|
-
async function prepareProject(project, projectRoot, runDir, defaultTimeoutMs, options = {}) {
|
|
948
|
-
const platform = options.platform?.trim();
|
|
949
|
-
const fileSelection = project.files ?? DEFAULT_TEST_FILE_SELECTION;
|
|
950
|
-
const prerequisiteFile = options.prerequisiteFile ? external_node_path_resolve(options.prerequisiteFile) : void 0;
|
|
951
|
-
const mainFiles = options.files ?? discoverResolvedTestFiles(projectRoot, fileSelection, options.excludedDirectories);
|
|
952
|
-
const files = [
|
|
953
|
-
...prerequisiteFile ? [
|
|
954
|
-
prerequisiteFile
|
|
955
|
-
] : [],
|
|
956
|
-
...mainFiles.filter((file)=>file !== prerequisiteFile)
|
|
957
|
-
];
|
|
958
|
-
const occurrences = new Map();
|
|
959
|
-
const sources = files.map((absolutePath)=>{
|
|
960
|
-
const invocationIndex = occurrences.get(absolutePath) ?? 0;
|
|
961
|
-
occurrences.set(absolutePath, invocationIndex + 1);
|
|
962
|
-
return {
|
|
963
|
-
projectId: project.projectId,
|
|
964
|
-
projectName: project.name,
|
|
965
|
-
sourcePath: project_preparation_toPosix(relative(projectRoot, absolutePath)),
|
|
966
|
-
absolutePath,
|
|
967
|
-
invocationIndex
|
|
968
|
-
};
|
|
969
|
-
});
|
|
970
|
-
const collectionErrors = [];
|
|
971
|
-
const invocations = [];
|
|
972
|
-
const plans = [];
|
|
973
|
-
let filteredCaseCount = 0;
|
|
974
|
-
if (0 === sources.length) {
|
|
975
|
-
const error = asCollectionError(project.projectId, project.name, '<project>', new WorkflowParseError(`No workflow YAML files found for project "${project.name}" in ${projectRoot}.`, {
|
|
976
|
-
projectName: project.name,
|
|
977
|
-
projectRoot
|
|
978
|
-
}));
|
|
979
|
-
collectionErrors.push(error);
|
|
980
|
-
await writeCollectionError(runDir, error);
|
|
981
|
-
}
|
|
982
|
-
for (const source of sources)try {
|
|
983
|
-
const collected = await options.collectDocument?.(source, project);
|
|
984
|
-
if (collected) {
|
|
985
|
-
filteredCaseCount += collected.filteredCaseCount;
|
|
986
|
-
if (collected.invocation) {
|
|
987
|
-
const plan = buildTestRunReportPlan(collected.invocation.document, {
|
|
988
|
-
defaultTimeoutMs: collected.invocation.defaultTimeoutMs,
|
|
989
|
-
retry: collected.invocation.retry
|
|
990
|
-
});
|
|
991
|
-
invocations.push({
|
|
992
|
-
...collected.invocation,
|
|
993
|
-
plan
|
|
994
|
-
});
|
|
995
|
-
plans.push(plan);
|
|
996
|
-
}
|
|
997
|
-
continue;
|
|
998
|
-
}
|
|
999
|
-
const document = collectWorkflowDocument(source, {
|
|
1000
|
-
resolveNode: project.nodes.get.bind(project.nodes),
|
|
1001
|
-
variables: project.variables,
|
|
1002
|
-
env: process.env
|
|
1003
|
-
});
|
|
1004
|
-
const filtered = source.absolutePath === prerequisiteFile ? {
|
|
1005
|
-
document,
|
|
1006
|
-
filtered: 0
|
|
1007
|
-
} : filterDocumentCases(document, project);
|
|
1008
|
-
const plan = buildTestRunReportPlan(document, {
|
|
1009
|
-
selectedCaseIds: filtered.document?.cases.map((item)=>item.caseId) ?? [],
|
|
1010
|
-
defaultTimeoutMs,
|
|
1011
|
-
retry: {
|
|
1012
|
-
scope: 'case',
|
|
1013
|
-
count: project.retry
|
|
1014
|
-
}
|
|
1015
|
-
});
|
|
1016
|
-
plans.push(plan);
|
|
1017
|
-
filteredCaseCount += filtered.filtered;
|
|
1018
|
-
if (filtered.document) invocations.push({
|
|
1019
|
-
plan,
|
|
1020
|
-
document: filtered.document,
|
|
1021
|
-
retry: {
|
|
1022
|
-
scope: 'case',
|
|
1023
|
-
count: project.retry
|
|
1024
|
-
},
|
|
1025
|
-
defaultTimeoutMs,
|
|
1026
|
-
reportEnabled: true,
|
|
1027
|
-
bindings: {
|
|
1028
|
-
resolveNode: project.nodes.require.bind(project.nodes),
|
|
1029
|
-
...project.fileSetup ? {
|
|
1030
|
-
documentSetup: {
|
|
1031
|
-
name: project.fileSetup.name,
|
|
1032
|
-
setup ({ projectContext, document, signal, onTeardown }) {
|
|
1033
|
-
return project.fileSetup.setup({
|
|
1034
|
-
project,
|
|
1035
|
-
projectContext,
|
|
1036
|
-
file: document,
|
|
1037
|
-
env: Object.freeze({
|
|
1038
|
-
...process.env
|
|
1039
|
-
}),
|
|
1040
|
-
signal,
|
|
1041
|
-
onTeardown
|
|
1042
|
-
});
|
|
1043
|
-
}
|
|
1044
|
-
}
|
|
1045
|
-
} : {}
|
|
1046
|
-
}
|
|
1047
|
-
});
|
|
1048
|
-
} catch (error) {
|
|
1049
|
-
const collectionError = asCollectionError(project.projectId, project.name, source.sourcePath, error);
|
|
1050
|
-
collectionErrors.push(collectionError);
|
|
1051
|
-
await writeCollectionError(runDir, collectionError);
|
|
1052
|
-
}
|
|
1053
|
-
const documents = invocations.map((invocation)=>invocation.document);
|
|
1054
|
-
const prerequisiteSource = sources.find((source)=>source.absolutePath === prerequisiteFile);
|
|
1055
|
-
const prerequisiteDocument = documents.find((document)=>document.sourcePath === prerequisiteSource?.sourcePath);
|
|
1056
|
-
return {
|
|
1057
|
-
project,
|
|
1058
|
-
...platform ? {
|
|
1059
|
-
platform
|
|
1060
|
-
} : {},
|
|
1061
|
-
documentConcurrency: options.documentConcurrency ?? 1,
|
|
1062
|
-
...prerequisiteDocument ? {
|
|
1063
|
-
prerequisiteDocumentId: prerequisiteDocument.documentId
|
|
1064
|
-
} : {},
|
|
1065
|
-
fileSelection,
|
|
1066
|
-
sources,
|
|
1067
|
-
documents,
|
|
1068
|
-
invocations,
|
|
1069
|
-
plans,
|
|
1070
|
-
collectionErrors,
|
|
1071
|
-
selectedCaseCount: documents.reduce((total, document)=>total + document.cases.length, 0),
|
|
1072
|
-
filteredCaseCount
|
|
1073
|
-
};
|
|
1074
|
-
}
|
|
1075
|
-
const padDatePart = (value)=>String(value).padStart(2, '0');
|
|
1076
|
-
const createTestRunId = (date = new Date(), uuid = randomUUID())=>{
|
|
1077
|
-
const timestamp = [
|
|
1078
|
-
date.getFullYear(),
|
|
1079
|
-
padDatePart(date.getMonth() + 1),
|
|
1080
|
-
padDatePart(date.getDate()),
|
|
1081
|
-
padDatePart(date.getHours()),
|
|
1082
|
-
padDatePart(date.getMinutes()),
|
|
1083
|
-
padDatePart(date.getSeconds())
|
|
1084
|
-
].join('');
|
|
1085
|
-
return `${timestamp}-${uuid.slice(0, 8)}`;
|
|
1086
|
-
};
|
|
1087
|
-
const assertDirectory = (path, label)=>{
|
|
1088
|
-
if (!existsSync(path) || !statSync(path).isDirectory()) throw new Error(`${label} does not exist or is not a directory: ${path}`);
|
|
1089
|
-
};
|
|
1090
|
-
const resolveRunInput = (options)=>{
|
|
1091
|
-
const cwd = external_node_path_resolve(options.cwd ?? process.cwd());
|
|
1092
|
-
assertDirectory(cwd, 'Test working directory');
|
|
1093
|
-
const inputPath = options.projectRoot ? external_node_path_resolve(cwd, options.projectRoot) : void 0;
|
|
1094
|
-
const singleFile = inputPath && existsSync(inputPath) && statSync(inputPath).isFile() ? inputPath : void 0;
|
|
1095
|
-
if (singleFile && !/\.ya?ml$/i.test(singleFile)) throw new Error(`Test input must be a YAML file or project directory: ${singleFile}`);
|
|
1096
|
-
const cliProjectRoot = singleFile ? dirname(singleFile) : inputPath;
|
|
1097
|
-
if (cliProjectRoot) assertDirectory(cliProjectRoot, 'Test project directory');
|
|
1098
|
-
const projectRoot = cliProjectRoot ?? cwd;
|
|
1099
|
-
return {
|
|
1100
|
-
cwd,
|
|
1101
|
-
projectRoot,
|
|
1102
|
-
configSearchRoot: projectRoot,
|
|
1103
|
-
...singleFile ? {
|
|
1104
|
-
singleFile
|
|
1105
|
-
} : {}
|
|
1106
|
-
};
|
|
1107
|
-
};
|
|
1108
|
-
async function prepareTestRun(options) {
|
|
1109
|
-
const startedAt = new Date();
|
|
1110
|
-
const runId = createTestRunId(startedAt);
|
|
1111
|
-
const input = resolveRunInput(options);
|
|
1112
|
-
const configPath = options.configPath ? external_node_path_resolve(input.configSearchRoot, options.configPath) : discoverTestConfig(input.configSearchRoot);
|
|
1113
|
-
if (options.configPath && /\.ya?ml$/i.test(configPath)) throw new Error(`midscene-test --config only accepts a native TypeScript or JavaScript project config: ${configPath}. Run legacy batch YAML with \`midscene --config ${options.configPath}\`.`);
|
|
1114
|
-
if (options.configPath && !existsSync(configPath)) throw new Error(`Midscene config does not exist: ${configPath}`);
|
|
1115
|
-
const definition = await loadTestProject(configPath);
|
|
1116
|
-
const artifactDir = resolveMidsceneRunDir(input.projectRoot);
|
|
1117
|
-
return runWithMidsceneRunDir(artifactDir, async ()=>{
|
|
1118
|
-
const resultDir = options.resultDir ? external_node_path_resolve(input.cwd, options.resultDir) : join(artifactDir, 'test-results');
|
|
1119
|
-
const runDir = join(resultDir, runId);
|
|
1120
|
-
const summaryPath = join(runDir, 'summary.json');
|
|
1121
|
-
const reportDir = definition.output.reportDir ? external_node_path_resolve(input.projectRoot, definition.output.reportDir) : join(artifactDir, 'report');
|
|
1122
|
-
mkdirSync(resultDir, {
|
|
1123
|
-
recursive: true
|
|
1124
|
-
});
|
|
1125
|
-
mkdirSync(runDir);
|
|
1126
|
-
const projects = [];
|
|
1127
|
-
for (const project of selectProjects(definition.projects, options.projectNames))projects.push(await prepareProject(project, input.projectRoot, runDir, definition.test.testTimeout, {
|
|
1128
|
-
...input.singleFile ? {
|
|
1129
|
-
files: [
|
|
1130
|
-
input.singleFile
|
|
1131
|
-
]
|
|
1132
|
-
} : {},
|
|
1133
|
-
excludedDirectories: [
|
|
1134
|
-
artifactDir,
|
|
1135
|
-
resultDir,
|
|
1136
|
-
reportDir,
|
|
1137
|
-
...[
|
|
1138
|
-
'cache',
|
|
1139
|
-
'log',
|
|
1140
|
-
'output',
|
|
1141
|
-
'tmp',
|
|
1142
|
-
'dump'
|
|
1143
|
-
].map((name)=>join(artifactDir, name))
|
|
1144
|
-
]
|
|
1145
|
-
}));
|
|
1146
|
-
const hasFormatMismatch = projects.some((project)=>project.collectionErrors.some(({ error })=>error instanceof NativeWorkflowFormatError));
|
|
1147
|
-
return {
|
|
1148
|
-
startedAt,
|
|
1149
|
-
runId,
|
|
1150
|
-
projectRoot: input.projectRoot,
|
|
1151
|
-
artifactDir,
|
|
1152
|
-
...configPath ? {
|
|
1153
|
-
configPath
|
|
1154
|
-
} : {},
|
|
1155
|
-
resultDir,
|
|
1156
|
-
runDir,
|
|
1157
|
-
summaryPath,
|
|
1158
|
-
reportDir,
|
|
1159
|
-
definition,
|
|
1160
|
-
projects,
|
|
1161
|
-
preflightScope: hasFormatMismatch ? 'run' : 'project',
|
|
1162
|
-
reportEnabled: true,
|
|
1163
|
-
publications: [],
|
|
1164
|
-
onProgress: options.onProgress ?? (()=>{}),
|
|
1165
|
-
onEvent: options.onEvent
|
|
1166
|
-
};
|
|
1167
|
-
});
|
|
1168
|
-
}
|
|
1169
|
-
function renderTestBadgeLines({ status, summary }) {
|
|
1170
|
-
if ('success' !== status && 0 === summary.failed) return [];
|
|
1171
|
-
return [
|
|
1172
|
-
``,
|
|
1173
|
-
''
|
|
1174
|
-
];
|
|
1175
|
-
}
|
|
1176
|
-
const result_markdown_text = (value)=>value.replaceAll('&', '&').replaceAll('<', '<').replaceAll('>', '>').replaceAll('"', '"').replaceAll('|', '|').replaceAll('[', '[').replaceAll(']', ']').replaceAll('*', '*').replaceAll('_', '_').replaceAll('`', '`').replaceAll('\\', '\').replace(/[\r\n]+/g, ' ');
|
|
1177
|
-
const duration = (ms)=>{
|
|
1178
|
-
const seconds = Math.round(ms / 1000);
|
|
1179
|
-
return seconds < 60 ? `${seconds}s` : `${Math.floor(seconds / 60)}m ${seconds % 60}s`;
|
|
1180
|
-
};
|
|
1181
|
-
const href = (from, target)=>relative(dirname(from), target).split(sep).map(encodeURIComponent).join('/');
|
|
1182
|
-
function renderTestResultMarkdown(dump, options = {}) {
|
|
1183
|
-
const { summary } = dump;
|
|
1184
|
-
const outcome = 'success' === dump.status ? `✅ **${summary.passed}/${summary.total} passed**` : `❌ **${summary.failed} failed** · ${summary.passed} passed · ${summary.notRun} not run`;
|
|
1185
|
-
const lines = [
|
|
1186
|
-
'## Midscene Test',
|
|
1187
|
-
'',
|
|
1188
|
-
...renderTestBadgeLines(dump),
|
|
1189
|
-
`${outcome} · ${duration(dump.durationMs)}`,
|
|
1190
|
-
''
|
|
1191
|
-
];
|
|
1192
|
-
if (summary.filtered) lines.push(`**Filtered:** ${summary.filtered}`, '');
|
|
1193
|
-
const infrastructure = [
|
|
1194
|
-
summary.collectionErrors ? `${summary.collectionErrors} collection errors` : '',
|
|
1195
|
-
summary.documentFailures ? `${summary.documentFailures} document failures` : '',
|
|
1196
|
-
summary.projectFailures ? `${summary.projectFailures} project failures` : ''
|
|
1197
|
-
].filter(Boolean);
|
|
1198
|
-
if (infrastructure.length) lines.push(`**Infrastructure:** ${infrastructure.join(' · ')}`, '');
|
|
1199
|
-
const groups = [];
|
|
1200
|
-
const errors = [];
|
|
1201
|
-
const recordSteps = (label, steps)=>{
|
|
1202
|
-
for (const step of steps)if ('failed' === step.status) errors.push(`${label} / ${step.node}: ${step.error?.message ?? 'Step failed'}`);
|
|
1203
|
-
};
|
|
1204
|
-
for (const project of dump.projects){
|
|
1205
|
-
if (project.lifecycle?.setupError) errors.push(`${project.name}: ${project.lifecycle.setupError.message}`);
|
|
1206
|
-
for (const error of project.lifecycle?.teardownErrors ?? [])errors.push(`${project.name}: ${error.message}`);
|
|
1207
|
-
for (const error of project.collectionErrors)errors.push(`${project.name} / ${error.sourcePath}: ${error.error.message}`);
|
|
1208
|
-
for (const document of project.documents){
|
|
1209
|
-
const name = document.sourcePath.replace(/\\/g, '/').replace(/\.(?:yaml|yml)$/, '');
|
|
1210
|
-
const group = {
|
|
1211
|
-
name: dump.projects.length > 1 ? `${project.name} / ${name}` : name,
|
|
1212
|
-
rows: []
|
|
1213
|
-
};
|
|
1214
|
-
groups.push(group);
|
|
1215
|
-
recordSteps(`${project.name} / ${document.sourcePath}`, [
|
|
1216
|
-
...document.beforeAll,
|
|
1217
|
-
...document.afterAll
|
|
1218
|
-
]);
|
|
1219
|
-
for (const error of document.teardownErrors ?? [])errors.push(`${document.sourcePath}: ${error.message}`);
|
|
1220
|
-
for (const failure of document.hostErrors ?? [])errors.push(`${document.sourcePath} / ${failure.phase}: ${failure.error.message}`);
|
|
1221
|
-
for (const testCase of document.cases)group.rows.push(` - ${'success' === testCase.status ? '✅' : 'not-run' === testCase.status ? '⏭️' : '❌'} ${result_markdown_text(testCase.name)}`);
|
|
1222
|
-
}
|
|
1223
|
-
}
|
|
1224
|
-
for (const diagnostic of dump.diagnostics ?? [])if ('error' === diagnostic.level) errors.push(diagnostic.message);
|
|
1225
|
-
if (errors.length) lines.push('### Execution errors', '', ...errors.map((error)=>`- ${result_markdown_text(error)}`), '');
|
|
1226
|
-
for (const group of groups)if (group.rows.length) lines.push(`- ${result_markdown_text(group.name)}`, ...group.rows);
|
|
1227
|
-
if (!groups.some((group)=>group.rows.length)) lines.push('No cases were reported.');
|
|
1228
|
-
if (options.reportHref) lines.push('', `[Open the Midscene Test report ↗](<${options.reportHref}>)`);
|
|
1229
|
-
return `${lines.join('\n')}\n`;
|
|
1230
|
-
}
|
|
1231
|
-
const emptyIndex = ()=>({
|
|
1232
|
-
sources: [],
|
|
1233
|
-
metrics: {
|
|
1234
|
-
modelCallCount: 0,
|
|
1235
|
-
modelTimeMs: 0,
|
|
1236
|
-
promptTokens: 0,
|
|
1237
|
-
cachedInputTokens: 0,
|
|
1238
|
-
completionTokens: 0,
|
|
1239
|
-
totalTokens: 0
|
|
1240
|
-
}
|
|
1241
|
-
});
|
|
1242
|
-
async function writeTestResultMarkdown(result, index = emptyIndex()) {
|
|
1243
|
-
const output = join(dirname(result.summaryPath), 'result.md');
|
|
1244
|
-
const dump = buildTestRunReportDump(result, index);
|
|
1245
|
-
await writeFile(output, renderTestResultMarkdown(dump, {
|
|
1246
|
-
...result.reportPath ? {
|
|
1247
|
-
reportHref: href(output, result.reportPath)
|
|
1248
|
-
} : {}
|
|
1249
|
-
}));
|
|
1250
|
-
return output;
|
|
1251
|
-
}
|
|
1252
|
-
function scopeNodeConsole(node) {
|
|
1253
|
-
return {
|
|
1254
|
-
...node,
|
|
1255
|
-
execute (ctx) {
|
|
1256
|
-
const position = 'case' === ctx.scope ? ctx.case : ctx.document;
|
|
1257
|
-
const scope = {
|
|
1258
|
-
projectName: position.projectName,
|
|
1259
|
-
sourcePath: position.sourcePath,
|
|
1260
|
-
...'case' === ctx.scope ? {
|
|
1261
|
-
caseId: ctx.case.caseId,
|
|
1262
|
-
caseName: ctx.case.name,
|
|
1263
|
-
runId: ctx.case.runId
|
|
1264
|
-
} : {
|
|
1265
|
-
runId: ctx.document.documentRunId
|
|
1266
|
-
},
|
|
1267
|
-
phase: position.phase,
|
|
1268
|
-
stepIndex: position.stepIndex,
|
|
1269
|
-
node: node.name
|
|
1270
|
-
};
|
|
1271
|
-
return runWithConsoleLogScope(scope, ()=>node.execute({
|
|
1272
|
-
...ctx,
|
|
1273
|
-
onTeardown (teardown) {
|
|
1274
|
-
ctx.onTeardown(()=>runWithConsoleLogScope({
|
|
1275
|
-
...scope,
|
|
1276
|
-
phase: 'teardown',
|
|
1277
|
-
stepIndex: void 0
|
|
1278
|
-
}, teardown));
|
|
1279
|
-
}
|
|
1280
|
-
}));
|
|
1281
|
-
}
|
|
1282
|
-
};
|
|
1283
|
-
}
|
|
1284
|
-
const stepPosition = (info)=>'case' === info.scope ? info.case : info.document;
|
|
1285
|
-
const formatStep = (info)=>{
|
|
1286
|
-
const position = stepPosition(info);
|
|
1287
|
-
const phase = 'steps' === position.phase ? 'step' : position.phase;
|
|
1288
|
-
return `${phase} ${position.stepIndex + 1}/${info.stepCount}: ${info.node}`;
|
|
1289
|
-
};
|
|
1290
|
-
const formatStepResult = (info, result)=>{
|
|
1291
|
-
const indent = 'case' === info.scope ? ' ' : ' ';
|
|
1292
|
-
if ('not-run' === result.status) return `${indent}– ${formatStep(info)} (not run: ${result.notRunReason})`;
|
|
1293
|
-
const symbol = 'success' === result.status ? '✓' : '✗';
|
|
1294
|
-
const error = result.error ? ` — ${result.error.message}` : '';
|
|
1295
|
-
const continuation = result.continuedAfterError ? '; continuing' : '';
|
|
1296
|
-
return `${indent}${symbol} ${formatStep(info)} (${result.durationMs} ms)${error}${continuation}`;
|
|
1297
|
-
};
|
|
1298
|
-
const caseHasFatalError = (outcome)=>(outcome.attempts ?? []).some((attempt)=>[
|
|
1299
|
-
...attempt.beforeEach,
|
|
1300
|
-
...attempt.steps,
|
|
1301
|
-
...attempt.afterEach
|
|
1302
|
-
].some((step)=>step.error && isFatalDeviceError(step.error)) || (attempt.teardownErrors ?? []).some(isFatalDeviceError) || !!attempt.executionErrors?.some(isResourceRecoveryError));
|
|
1303
|
-
const documentHasFatalError = (result)=>[
|
|
1304
|
-
...result.beforeAll,
|
|
1305
|
-
...result.afterAll
|
|
1306
|
-
].some((step)=>step.error && isFatalDeviceError(step.error)) || (result.teardownErrors ?? []).some(isFatalDeviceError) || !!result.executionErrors?.some(isResourceRecoveryError);
|
|
1307
|
-
const isSharedResourceFailure = (error)=>{
|
|
1308
|
-
const seen = new Set();
|
|
1309
|
-
const pending = [
|
|
1310
|
-
error
|
|
1311
|
-
];
|
|
1312
|
-
while(pending.length && seen.size < 50){
|
|
1313
|
-
const item = pending.pop();
|
|
1314
|
-
if ('object' != typeof item || null === item || seen.has(item)) continue;
|
|
1315
|
-
seen.add(item);
|
|
1316
|
-
const failure = item;
|
|
1317
|
-
if ('PROJECT_RESOURCE_ERROR' === failure.code) return true;
|
|
1318
|
-
if (failure.cause) pending.push(failure.cause);
|
|
1319
|
-
if (Array.isArray(failure.errors)) pending.push(...failure.errors.slice(0, 50));
|
|
1320
|
-
}
|
|
1321
|
-
return false;
|
|
1322
|
-
};
|
|
1323
|
-
async function executeDocumentInvocation(options) {
|
|
1324
|
-
const { invocation, project, sinks } = options;
|
|
1325
|
-
const isProjectResourceFailure = options.isProjectResourceFailure ?? (()=>false);
|
|
1326
|
-
const { document } = invocation;
|
|
1327
|
-
const documentRetry = 'document' === invocation.retry.scope;
|
|
1328
|
-
const attempts = [];
|
|
1329
|
-
const fileSetup = invocation.bindings.documentSetup;
|
|
1330
|
-
const documentSetup = fileSetup ? {
|
|
1331
|
-
...fileSetup,
|
|
1332
|
-
setup (ctx) {
|
|
1333
|
-
const scope = {
|
|
1334
|
-
projectName: project.name,
|
|
1335
|
-
sourcePath: document.sourcePath,
|
|
1336
|
-
runId: ctx.document.documentRunId,
|
|
1337
|
-
phase: 'fileSetup'
|
|
1338
|
-
};
|
|
1339
|
-
return runWithConsoleLogScope(scope, async ()=>{
|
|
1340
|
-
try {
|
|
1341
|
-
return await fileSetup.setup({
|
|
1342
|
-
...ctx,
|
|
1343
|
-
onTeardown (teardown) {
|
|
1344
|
-
ctx.onTeardown(()=>runWithConsoleLogScope({
|
|
1345
|
-
...scope,
|
|
1346
|
-
phase: 'teardown'
|
|
1347
|
-
}, teardown));
|
|
1348
|
-
}
|
|
1349
|
-
});
|
|
1350
|
-
} catch (error) {
|
|
1351
|
-
if (isProjectResourceFailure(error)) options.markProjectFatal();
|
|
1352
|
-
throw error;
|
|
1353
|
-
}
|
|
1354
|
-
});
|
|
1355
|
-
}
|
|
1356
|
-
} : void 0;
|
|
1357
|
-
options.onEvent?.({
|
|
1358
|
-
type: 'document-start',
|
|
1359
|
-
documentId: document.documentId,
|
|
1360
|
-
projectName: project.name,
|
|
1361
|
-
sourcePath: document.sourcePath,
|
|
1362
|
-
cases: document.cases.length
|
|
1363
|
-
});
|
|
1364
|
-
await runDocumentAttempts({
|
|
1365
|
-
retry: documentRetry ? invocation.retry.count : 0,
|
|
1366
|
-
signal: options.signal,
|
|
1367
|
-
shouldStop: ()=>options.shouldBail() || options.isProjectFatal()
|
|
1368
|
-
}, async (documentAttemptIndex)=>{
|
|
1369
|
-
if (documentRetry) options.onProgress(` file attempt ${documentAttemptIndex + 1}/${invocation.retry.count + 1}: ${document.sourcePath}`);
|
|
1370
|
-
const execution = await runWithConsoleLogScope({
|
|
1371
|
-
projectName: project.name,
|
|
1372
|
-
sourcePath: document.sourcePath
|
|
1373
|
-
}, ()=>runWorkflowDocument(document, {
|
|
1374
|
-
plan: invocation.plan,
|
|
1375
|
-
documentAttemptIndex: documentRetry ? documentAttemptIndex : void 0,
|
|
1376
|
-
...invocation.bindings,
|
|
1377
|
-
resolveNode: (name)=>scopeNodeConsole(invocation.bindings.resolveNode(name)),
|
|
1378
|
-
documentSetup,
|
|
1379
|
-
setupTimeoutMs: options.setupTimeoutMs,
|
|
1380
|
-
teardownTimeoutMs: options.teardownTimeoutMs,
|
|
1381
|
-
project,
|
|
1382
|
-
projectContext: options.projectContext,
|
|
1383
|
-
retry: documentRetry ? 0 : invocation.retry.count,
|
|
1384
|
-
signal: options.signal,
|
|
1385
|
-
defaultTimeoutMs: invocation.defaultTimeoutMs,
|
|
1386
|
-
shouldStop: ()=>options.rootSignal.aborted || options.shouldBail() || options.isProjectFatal(),
|
|
1387
|
-
stopReason: ()=>options.rootSignal.aborted ? 'interrupted' : options.isProjectFatal() ? 'fatal-error' : 'bail',
|
|
1388
|
-
isFatalError: (run)=>[
|
|
1389
|
-
...run.beforeEach,
|
|
1390
|
-
...run.steps,
|
|
1391
|
-
...run.afterEach
|
|
1392
|
-
].some((step)=>step.error && (isFatalDeviceError(step.error) || isProjectResourceFailure(step.error))) || (run.teardownErrors ?? []).some((error)=>isFatalDeviceError(error) || isProjectResourceFailure(error)),
|
|
1393
|
-
onCaseStart: (collectedCase)=>{
|
|
1394
|
-
options.onProgress(` [case ${collectedCase.caseIndex + 1}/${document.cases.length}] ${collectedCase.definition.name}`);
|
|
1395
|
-
},
|
|
1396
|
-
onStepStart: (info)=>{
|
|
1397
|
-
options.onEvent?.({
|
|
1398
|
-
type: 'step-start',
|
|
1399
|
-
projectName: project.name,
|
|
1400
|
-
info
|
|
1401
|
-
});
|
|
1402
|
-
const indent = 'case' === info.scope ? ' ' : ' ';
|
|
1403
|
-
options.onProgress(`${indent}→ ${formatStep(info)}`);
|
|
1404
|
-
},
|
|
1405
|
-
onStepResult: async (info, result)=>{
|
|
1406
|
-
if (isProjectResourceFailure(result.error)) options.markProjectFatal();
|
|
1407
|
-
await invocation.bindings.onStepResult?.(info, result);
|
|
1408
|
-
options.onProgress(formatStepResult(info, result));
|
|
1409
|
-
options.onEvent?.({
|
|
1410
|
-
type: 'step-end',
|
|
1411
|
-
projectName: project.name,
|
|
1412
|
-
info,
|
|
1413
|
-
result
|
|
1414
|
-
});
|
|
1415
|
-
},
|
|
1416
|
-
onCaseResult: async (attempt)=>{
|
|
1417
|
-
await writeCaseAttemptResult(options.runDir, project.projectId, document.documentId, attempt);
|
|
1418
|
-
options.onProgress(` ${'success' === attempt.status ? '✓' : '✗'} case attempt ${attempt.attemptIndex + 1}/${documentRetry ? 1 : invocation.retry.count + 1}: ${attempt.name} (${attempt.durationMs} ms)`);
|
|
1419
|
-
if (!documentRetry) options.onEvent?.({
|
|
1420
|
-
type: 'case-attempt-end',
|
|
1421
|
-
attempt,
|
|
1422
|
-
maxAttempts: invocation.retry.count + 1
|
|
1423
|
-
});
|
|
1424
|
-
},
|
|
1425
|
-
onCaseOutcome: (outcome)=>{
|
|
1426
|
-
if (documentRetry) options.onEvent?.({
|
|
1427
|
-
type: 'case-progress',
|
|
1428
|
-
outcome,
|
|
1429
|
-
documentAttemptIndex
|
|
1430
|
-
});
|
|
1431
|
-
else options.onEvent?.({
|
|
1432
|
-
type: 'case-end',
|
|
1433
|
-
outcome
|
|
1434
|
-
});
|
|
1435
|
-
if (outcome.attempts?.some((attempt)=>[
|
|
1436
|
-
...attempt.beforeEach,
|
|
1437
|
-
...attempt.steps,
|
|
1438
|
-
...attempt.afterEach
|
|
1439
|
-
].some((step)=>isProjectResourceFailure(step.error)) || attempt.teardownErrors?.some(isProjectResourceFailure))) options.markProjectFatal();
|
|
1440
|
-
if (!options.isolatedContext && caseHasFatalError(outcome)) options.markProjectFatal();
|
|
1441
|
-
if (!documentRetry && 'failed' === outcome.status) options.addFailedCases(1);
|
|
1442
|
-
},
|
|
1443
|
-
onDocumentResult: options.onDocumentResult
|
|
1444
|
-
})).catch(async (error)=>{
|
|
1445
|
-
if (error instanceof WorkflowExecutionFailure) {
|
|
1446
|
-
const partial = error.result;
|
|
1447
|
-
const resourceOnly = error.errors.length > 0 && error.errors.every((failure)=>isResourceRecoveryError(failure) || partial.document.hostErrors?.some((host)=>'cleanup' === host.phase && (host.error === failure || host.error instanceof Error && host.error.cause === failure)));
|
|
1448
|
-
if (resourceOnly) {
|
|
1449
|
-
if (!options.isolatedContext && error.errors.some(isResourceRecoveryError)) options.markProjectFatal();
|
|
1450
|
-
return partial;
|
|
1451
|
-
}
|
|
1452
|
-
if (documentRetry) partial.document.retryDecision = decideRetry({
|
|
1453
|
-
scope: 'document',
|
|
1454
|
-
retryCount: invocation.retry.count,
|
|
1455
|
-
attemptIndex: documentAttemptIndex,
|
|
1456
|
-
status: 'failed',
|
|
1457
|
-
infrastructureError: true
|
|
1458
|
-
});
|
|
1459
|
-
attempts.push(partial);
|
|
1460
|
-
sinks.documents.push(partial.document);
|
|
1461
|
-
sinks.cases.push(...partial.cases.map((outcome)=>({
|
|
1462
|
-
...outcome,
|
|
1463
|
-
documentId: document.documentId,
|
|
1464
|
-
documentRunId: partial.document.documentRunId
|
|
1465
|
-
})));
|
|
1466
|
-
try {
|
|
1467
|
-
for (const outcome of partial.cases)for (const attempt of outcome.attempts ?? [])await writeCaseAttemptResult(options.runDir, project.projectId, document.documentId, attempt);
|
|
1468
|
-
await writeWorkflowDocumentResult(options.runDir, partial.document);
|
|
1469
|
-
} catch (publicationError) {
|
|
1470
|
-
throw new WorkflowExecutionFailure(partial, [
|
|
1471
|
-
...error.errors,
|
|
1472
|
-
publicationError
|
|
1473
|
-
]);
|
|
1474
|
-
}
|
|
1475
|
-
}
|
|
1476
|
-
throw error;
|
|
1477
|
-
});
|
|
1478
|
-
if (!options.isolatedContext && documentHasFatalError(execution.document)) options.markProjectFatal();
|
|
1479
|
-
if ([
|
|
1480
|
-
...execution.document.beforeAll,
|
|
1481
|
-
...execution.document.afterAll,
|
|
1482
|
-
...(execution.document.hostErrors ?? []).map(({ error })=>({
|
|
1483
|
-
error
|
|
1484
|
-
})),
|
|
1485
|
-
...(execution.document.teardownErrors ?? []).map((error)=>({
|
|
1486
|
-
error
|
|
1487
|
-
}))
|
|
1488
|
-
].some(({ error })=>isProjectResourceFailure(error))) options.markProjectFatal();
|
|
1489
|
-
const owners = [
|
|
1490
|
-
execution.document,
|
|
1491
|
-
...execution.cases.flatMap((outcome)=>outcome.attempts ?? [])
|
|
1492
|
-
];
|
|
1493
|
-
const cleanupPending = owners.some((owner)=>owner.cleanup?.status === 'pending');
|
|
1494
|
-
const operationPending = owners.some((owner)=>!!owner.pendingOperations?.length);
|
|
1495
|
-
const cleanupFailed = owners.some((owner)=>owner.cleanup?.status === 'failed' || !!owner.teardownErrors?.length || 'hostErrors' in owner && owner.hostErrors?.some(({ phase })=>'cleanup' === phase));
|
|
1496
|
-
if (cleanupPending || operationPending) {
|
|
1497
|
-
options.markProjectFatal();
|
|
1498
|
-
options.onUnresolvedResources?.();
|
|
1499
|
-
} else if (cleanupFailed && !options.isolatedContext) options.markProjectFatal();
|
|
1500
|
-
const failed = 'failed' === execution.document.status || execution.cases.some((item)=>'success' !== item.status);
|
|
1501
|
-
const decision = decideRetry({
|
|
1502
|
-
scope: 'document',
|
|
1503
|
-
retryCount: invocation.retry.count,
|
|
1504
|
-
attemptIndex: documentAttemptIndex,
|
|
1505
|
-
status: failed ? 'failed' : 'success',
|
|
1506
|
-
cleanupPending,
|
|
1507
|
-
cleanupFailed,
|
|
1508
|
-
operationPending,
|
|
1509
|
-
fatalError: options.isProjectFatal(),
|
|
1510
|
-
interrupted: options.signal.aborted || options.rootSignal.aborted,
|
|
1511
|
-
shouldStop: options.shouldBail()
|
|
1512
|
-
});
|
|
1513
|
-
if (documentRetry) execution.document.retryDecision = decision;
|
|
1514
|
-
if (documentRetry && 'retry' === decision.decision) options.onEvent?.({
|
|
1515
|
-
type: 'document-retry',
|
|
1516
|
-
document: execution.document,
|
|
1517
|
-
maxAttempts: invocation.retry.count + 1
|
|
1518
|
-
});
|
|
1519
|
-
attempts.push(execution);
|
|
1520
|
-
sinks.cases.push(...execution.cases.map((outcome)=>({
|
|
1521
|
-
...outcome,
|
|
1522
|
-
documentId: document.documentId,
|
|
1523
|
-
documentRunId: execution.document.documentRunId
|
|
1524
|
-
})));
|
|
1525
|
-
sinks.documents.push(execution.document);
|
|
1526
|
-
try {
|
|
1527
|
-
for (const outcome of execution.cases)for (const attempt of outcome.attempts ?? [])await writeCaseAttemptResult(options.runDir, project.projectId, document.documentId, attempt);
|
|
1528
|
-
await writeWorkflowDocumentResult(options.runDir, execution.document);
|
|
1529
|
-
} catch (error) {
|
|
1530
|
-
execution.document.status = 'failed';
|
|
1531
|
-
execution.document.hostErrors = [
|
|
1532
|
-
...execution.document.hostErrors ?? [],
|
|
1533
|
-
{
|
|
1534
|
-
phase: 'publication',
|
|
1535
|
-
error
|
|
1536
|
-
}
|
|
1537
|
-
];
|
|
1538
|
-
if (documentRetry) execution.document.retryDecision = decideRetry({
|
|
1539
|
-
scope: 'document',
|
|
1540
|
-
retryCount: invocation.retry.count,
|
|
1541
|
-
attemptIndex: documentAttemptIndex,
|
|
1542
|
-
status: 'failed',
|
|
1543
|
-
infrastructureError: true
|
|
1544
|
-
});
|
|
1545
|
-
throw new WorkflowExecutionFailure(execution, [
|
|
1546
|
-
error
|
|
1547
|
-
]);
|
|
1548
|
-
}
|
|
1549
|
-
return 'retry' === decision.decision ? 'failed' : void 0;
|
|
1550
|
-
});
|
|
1551
|
-
const final = attempts.at(-1);
|
|
1552
|
-
if (!final) return;
|
|
1553
|
-
if (documentRetry) for (const outcome of final.cases)options.onEvent?.({
|
|
1554
|
-
type: 'case-end',
|
|
1555
|
-
outcome,
|
|
1556
|
-
documentRetries: attempts.length - 1
|
|
1557
|
-
});
|
|
1558
|
-
const failedCases = final.cases.filter((item)=>'failed' === item.status).length;
|
|
1559
|
-
options.onEvent?.({
|
|
1560
|
-
type: 'document-end',
|
|
1561
|
-
document: final.document,
|
|
1562
|
-
cases: final.cases,
|
|
1563
|
-
durationMs: attempts.reduce((total, attempt)=>total + attempt.document.durationMs, 0)
|
|
1564
|
-
});
|
|
1565
|
-
if (documentRetry && failedCases) options.addFailedCases(failedCases);
|
|
1566
|
-
else if ('failed' === final.document.status && 0 === failedCases) options.addFailedCases(Math.max(1, failedCases));
|
|
1567
|
-
}
|
|
1568
|
-
async function runPreparedFiles(options) {
|
|
1569
|
-
const states = options.projects.map((prepared)=>({
|
|
1570
|
-
prepared,
|
|
1571
|
-
cases: [],
|
|
1572
|
-
documents: [],
|
|
1573
|
-
cursor: 0,
|
|
1574
|
-
active: 0,
|
|
1575
|
-
initializing: false,
|
|
1576
|
-
finished: false,
|
|
1577
|
-
fatal: false,
|
|
1578
|
-
setupFailed: false
|
|
1579
|
-
}));
|
|
1580
|
-
let nextProject = 0;
|
|
1581
|
-
let resourcesUnresolved = false;
|
|
1582
|
-
let occupied = 0;
|
|
1583
|
-
let occupiedProjects = 0;
|
|
1584
|
-
const cleanupQueue = [];
|
|
1585
|
-
const release = ()=>{
|
|
1586
|
-
occupied--;
|
|
1587
|
-
while(occupied < options.maxConcurrentFiles && cleanupQueue.length){
|
|
1588
|
-
occupied++;
|
|
1589
|
-
cleanupQueue.shift()();
|
|
1590
|
-
}
|
|
1591
|
-
wake?.();
|
|
1592
|
-
};
|
|
1593
|
-
let wake;
|
|
1594
|
-
const running = new Set();
|
|
1595
|
-
const stopped = (state)=>options.signal.aborted || options.shouldBail() || state.fatal;
|
|
1596
|
-
const invalid = (state)=>options.preflightFailed || state.prepared.collectionErrors.length > 0;
|
|
1597
|
-
const unresolved = ()=>{
|
|
1598
|
-
if (resourcesUnresolved) return;
|
|
1599
|
-
resourcesUnresolved = true;
|
|
1600
|
-
options.onInfrastructureError(new WorkflowError('Resources are still running or cleanup is pending. No new test files will start.', {
|
|
1601
|
-
code: 'UNRESOLVED_FILE_RESOURCES'
|
|
1602
|
-
}));
|
|
1603
|
-
};
|
|
1604
|
-
const finalizeProject = (state)=>{
|
|
1605
|
-
const { prepared } = state;
|
|
1606
|
-
const completed = new Set(state.cases.map((item)=>item.caseId));
|
|
1607
|
-
const reason = invalid(state) ? 'project-preflight-failed' : state.setupFailed ? 'project-setup-failed' : options.signal.aborted ? 'interrupted' : state.fatal ? 'fatal-error' : 'bail';
|
|
1608
|
-
for (const document of prepared.documents)for (const item of document.cases)if (!completed.has(item.caseId)) state.cases.push({
|
|
1609
|
-
documentId: document.documentId,
|
|
1610
|
-
caseId: item.caseId,
|
|
1611
|
-
projectName: prepared.project.name,
|
|
1612
|
-
name: item.definition.name,
|
|
1613
|
-
sourcePath: item.sourcePath,
|
|
1614
|
-
caseIndex: item.caseIndex,
|
|
1615
|
-
status: 'not-run',
|
|
1616
|
-
notRunReason: reason
|
|
1617
|
-
});
|
|
1618
|
-
const order = new Map(prepared.documents.map((document, i)=>[
|
|
1619
|
-
document.documentId,
|
|
1620
|
-
i
|
|
1621
|
-
]));
|
|
1622
|
-
const position = (id)=>order.get(id) ?? Number.MAX_SAFE_INTEGER;
|
|
1623
|
-
state.cases.sort((a, b)=>position(a.documentId) - position(b.documentId) || a.caseIndex - b.caseIndex);
|
|
1624
|
-
state.documents.sort((a, b)=>position(a.documentId) - position(b.documentId));
|
|
1625
|
-
};
|
|
1626
|
-
const finishProject = async (state)=>{
|
|
1627
|
-
if (!state.runtime || state.finished) return;
|
|
1628
|
-
state.finished = true;
|
|
1629
|
-
try {
|
|
1630
|
-
options.onEvent?.({
|
|
1631
|
-
type: 'project-phase',
|
|
1632
|
-
projectName: state.prepared.project.name,
|
|
1633
|
-
phase: 'teardown'
|
|
1634
|
-
});
|
|
1635
|
-
} catch (error) {
|
|
1636
|
-
options.onInfrastructureError(error);
|
|
1637
|
-
}
|
|
1638
|
-
const failed = state.fatal || options.signal.aborted || state.cases.some((item)=>'success' !== item.status) || state.documents.some((item)=>'success' !== item.status);
|
|
1639
|
-
state.lifecycle = await runWithConsoleLogScope({
|
|
1640
|
-
projectName: state.prepared.project.name,
|
|
1641
|
-
phase: 'teardown'
|
|
1642
|
-
}, ()=>state.runtime.finish(failed ? 'failed' : 'success'));
|
|
1643
|
-
const releaseProject = ()=>{
|
|
1644
|
-
occupiedProjects--;
|
|
1645
|
-
wake?.();
|
|
1646
|
-
};
|
|
1647
|
-
if (state.lifecycle.cleanup?.status === 'pending' || state.lifecycle.pendingOperations?.length) {
|
|
1648
|
-
unresolved();
|
|
1649
|
-
Promise.all([
|
|
1650
|
-
waitForScopeOperations(state.runtime.signal),
|
|
1651
|
-
getResourceCleanupCompletion(state.runtime.signal)
|
|
1652
|
-
]).then(releaseProject, (error)=>{
|
|
1653
|
-
options.onInfrastructureError(error);
|
|
1654
|
-
releaseProject();
|
|
1655
|
-
});
|
|
1656
|
-
} else {
|
|
1657
|
-
releaseProject();
|
|
1658
|
-
finalizeProject(state);
|
|
1659
|
-
options.onProjectEnd?.(state);
|
|
1660
|
-
}
|
|
1661
|
-
};
|
|
1662
|
-
const admit = ()=>{
|
|
1663
|
-
for(let offset = 0; offset < states.length; offset++){
|
|
1664
|
-
const index = (nextProject + offset) % states.length;
|
|
1665
|
-
const state = states[index];
|
|
1666
|
-
if (invalid(state) || state.finished || state.initializing) continue;
|
|
1667
|
-
if (!state.runtime && occupiedProjects >= options.maxConcurrentProjects) continue;
|
|
1668
|
-
const cleanupOnly = !!state.runtime && 0 === state.active && (stopped(state) || state.cursor === state.prepared.invocations.length);
|
|
1669
|
-
if (!cleanupOnly && (stopped(state) || state.cursor >= state.prepared.invocations.length || state.active >= (state.prepared.project.maxConcurrentFiles ?? options.maxConcurrentFiles))) continue;
|
|
1670
|
-
nextProject = (index + 1) % states.length;
|
|
1671
|
-
occupied++;
|
|
1672
|
-
state.active++;
|
|
1673
|
-
if (!state.runtime && !cleanupOnly) {
|
|
1674
|
-
occupiedProjects++;
|
|
1675
|
-
state.initializing = true;
|
|
1676
|
-
}
|
|
1677
|
-
const invocationIndex = cleanupOnly ? -1 : state.cursor++;
|
|
1678
|
-
return {
|
|
1679
|
-
state,
|
|
1680
|
-
index,
|
|
1681
|
-
invocationIndex
|
|
1682
|
-
};
|
|
1683
|
-
}
|
|
1684
|
-
};
|
|
1685
|
-
const run = async ({ state, index, invocationIndex })=>{
|
|
1686
|
-
const { prepared } = state;
|
|
1687
|
-
const { project } = prepared;
|
|
1688
|
-
let fileResourcesPending = false;
|
|
1689
|
-
try {
|
|
1690
|
-
if (invocationIndex < 0) return void await finishProject(state);
|
|
1691
|
-
if (!state.runtime) {
|
|
1692
|
-
state.runtime = createProjectRuntime({
|
|
1693
|
-
project,
|
|
1694
|
-
setup: project.setup,
|
|
1695
|
-
signal: options.signal,
|
|
1696
|
-
setupTimeoutMs: options.setupTimeoutMs,
|
|
1697
|
-
teardownTimeoutMs: options.teardownTimeoutMs
|
|
1698
|
-
});
|
|
1699
|
-
setMaxListeners(Math.max(10, Math.min(prepared.invocations.length, project.maxConcurrentFiles ?? options.maxConcurrentFiles) + 1), state.runtime.signal);
|
|
1700
|
-
options.announceProject(prepared, index);
|
|
1701
|
-
options.onEvent?.({
|
|
1702
|
-
type: 'project-phase',
|
|
1703
|
-
projectName: project.name,
|
|
1704
|
-
phase: 'setup'
|
|
1705
|
-
});
|
|
1706
|
-
await runWithConsoleLogScope({
|
|
1707
|
-
projectName: project.name,
|
|
1708
|
-
phase: 'setup'
|
|
1709
|
-
}, ()=>state.runtime.start());
|
|
1710
|
-
options.onEvent?.({
|
|
1711
|
-
type: 'project-phase',
|
|
1712
|
-
projectName: project.name,
|
|
1713
|
-
phase: 'running'
|
|
1714
|
-
});
|
|
1715
|
-
state.initializing = false;
|
|
1716
|
-
if (!state.runtime.canRun) {
|
|
1717
|
-
state.fatal = true;
|
|
1718
|
-
state.setupFailed = true;
|
|
1719
|
-
}
|
|
1720
|
-
wake?.();
|
|
1721
|
-
}
|
|
1722
|
-
if (stopped(state)) return;
|
|
1723
|
-
const invocation = prepared.invocations[invocationIndex];
|
|
1724
|
-
options.onProgress(prepared, ` [document ${invocationIndex + 1}/${prepared.invocations.length}] ${invocation.document.sourcePath}`);
|
|
1725
|
-
await executeDocumentInvocation({
|
|
1726
|
-
invocation,
|
|
1727
|
-
project,
|
|
1728
|
-
projectContext: state.runtime.context,
|
|
1729
|
-
runDir: options.runDir,
|
|
1730
|
-
signal: state.runtime.signal,
|
|
1731
|
-
rootSignal: options.signal,
|
|
1732
|
-
shouldBail: options.shouldBail,
|
|
1733
|
-
isProjectFatal: ()=>state.fatal,
|
|
1734
|
-
markProjectFatal: ()=>{
|
|
1735
|
-
if (state.fatal) return;
|
|
1736
|
-
state.fatal = true;
|
|
1737
|
-
state.runtime?.abort(new WorkflowError(`Project "${project.name}" shared execution environment is unavailable.`, {
|
|
1738
|
-
code: 'PROJECT_RESOURCE_ERROR'
|
|
1739
|
-
}));
|
|
1740
|
-
},
|
|
1741
|
-
addFailedCases: options.addFailedCases,
|
|
1742
|
-
onProgress: (message)=>options.onProgress(prepared, options.maxConcurrentFiles > 1 ? ` [${invocation.document.sourcePath}]${message}` : message),
|
|
1743
|
-
sinks: state,
|
|
1744
|
-
onEvent: options.onEvent,
|
|
1745
|
-
isolatedContext: void 0 !== project.fileSetup,
|
|
1746
|
-
isProjectResourceFailure: isSharedResourceFailure,
|
|
1747
|
-
setupTimeoutMs: options.setupTimeoutMs,
|
|
1748
|
-
teardownTimeoutMs: options.teardownTimeoutMs,
|
|
1749
|
-
onUnresolvedResources: ()=>{
|
|
1750
|
-
fileResourcesPending = true;
|
|
1751
|
-
unresolved();
|
|
1752
|
-
}
|
|
1753
|
-
});
|
|
1754
|
-
} catch (error) {
|
|
1755
|
-
state.fatal = true;
|
|
1756
|
-
fileResourcesPending = !!state.runtime && getPendingScopeOperations(state.runtime.signal).length > 0;
|
|
1757
|
-
options.onInfrastructureError(error);
|
|
1758
|
-
} finally{
|
|
1759
|
-
state.initializing = false;
|
|
1760
|
-
try {
|
|
1761
|
-
if (1 === state.active && (stopped(state) || state.cursor === prepared.invocations.length)) await finishProject(state);
|
|
1762
|
-
} catch (error) {
|
|
1763
|
-
options.onInfrastructureError(error);
|
|
1764
|
-
}
|
|
1765
|
-
state.active--;
|
|
1766
|
-
if (state.runtime && (fileResourcesPending || state.lifecycle?.cleanup?.status === 'pending')) waitForScopeOperations(state.runtime.signal).then(release, (error)=>{
|
|
1767
|
-
options.onInfrastructureError(error);
|
|
1768
|
-
release();
|
|
1769
|
-
});
|
|
1770
|
-
else release();
|
|
1771
|
-
wake?.();
|
|
1772
|
-
}
|
|
1773
|
-
};
|
|
1774
|
-
while(true){
|
|
1775
|
-
const changed = new Promise((resolve)=>{
|
|
1776
|
-
wake = resolve;
|
|
1777
|
-
});
|
|
1778
|
-
while(occupied < options.maxConcurrentFiles){
|
|
1779
|
-
const job = admit();
|
|
1780
|
-
if (!job) break;
|
|
1781
|
-
const task = run(job);
|
|
1782
|
-
running.add(task);
|
|
1783
|
-
const settled = ()=>{
|
|
1784
|
-
running.delete(task);
|
|
1785
|
-
wake?.();
|
|
1786
|
-
};
|
|
1787
|
-
task.then(settled, (error)=>{
|
|
1788
|
-
options.onInfrastructureError(error);
|
|
1789
|
-
settled();
|
|
1790
|
-
});
|
|
1791
|
-
}
|
|
1792
|
-
if (!running.size) break;
|
|
1793
|
-
await changed;
|
|
1794
|
-
}
|
|
1795
|
-
wake = void 0;
|
|
1796
|
-
await Promise.all(states.map(async (state)=>{
|
|
1797
|
-
if (!state.runtime || state.finished) return;
|
|
1798
|
-
const admission = new Promise((resolve)=>{
|
|
1799
|
-
cleanupQueue.push(resolve);
|
|
1800
|
-
});
|
|
1801
|
-
trackResourceOperation({}, admission, state.runtime.signal);
|
|
1802
|
-
while(occupied < options.maxConcurrentFiles && cleanupQueue.length){
|
|
1803
|
-
occupied++;
|
|
1804
|
-
cleanupQueue.shift()();
|
|
1805
|
-
}
|
|
1806
|
-
try {
|
|
1807
|
-
await finishProject(state);
|
|
1808
|
-
} catch (error) {
|
|
1809
|
-
options.onInfrastructureError(error);
|
|
1810
|
-
} finally{
|
|
1811
|
-
Promise.all([
|
|
1812
|
-
admission,
|
|
1813
|
-
getResourceCleanupCompletion(state.runtime.signal)
|
|
1814
|
-
]).then(release, (error)=>{
|
|
1815
|
-
options.onInfrastructureError(error);
|
|
1816
|
-
release();
|
|
1817
|
-
});
|
|
1818
|
-
}
|
|
1819
|
-
}));
|
|
1820
|
-
for (const state of states){
|
|
1821
|
-
const { prepared } = state;
|
|
1822
|
-
if (!state.runtime) options.announceProject(prepared, states.indexOf(state));
|
|
1823
|
-
finalizeProject(state);
|
|
1824
|
-
}
|
|
1825
|
-
return states;
|
|
1826
|
-
}
|
|
1827
|
-
function configuredModelNames() {
|
|
1828
|
-
const env = globalConfigManager.getAllEnvConfig();
|
|
1829
|
-
const defaultName = env.MIDSCENE_MODEL_NAME;
|
|
1830
|
-
const models = new Map();
|
|
1831
|
-
for (const [intent, name] of [
|
|
1832
|
-
[
|
|
1833
|
-
'default',
|
|
1834
|
-
defaultName
|
|
1835
|
-
],
|
|
1836
|
-
[
|
|
1837
|
-
'planning',
|
|
1838
|
-
env.MIDSCENE_PLANNING_MODEL_NAME || defaultName
|
|
1839
|
-
],
|
|
1840
|
-
[
|
|
1841
|
-
'insight',
|
|
1842
|
-
env.MIDSCENE_INSIGHT_MODEL_NAME || defaultName
|
|
1843
|
-
]
|
|
1844
|
-
]){
|
|
1845
|
-
if (!name) continue;
|
|
1846
|
-
const intents = models.get(name) ?? [];
|
|
1847
|
-
intents.push(intent);
|
|
1848
|
-
models.set(name, intents);
|
|
1849
|
-
}
|
|
1850
|
-
return [
|
|
1851
|
-
...models
|
|
1852
|
-
].map(([name, intents])=>({
|
|
1853
|
-
name,
|
|
1854
|
-
intents
|
|
1855
|
-
}));
|
|
1856
|
-
}
|
|
1857
|
-
function createOperationJournal(runDir) {
|
|
1858
|
-
const path = join(runDir, 'operations.jsonl');
|
|
1859
|
-
writeFileSync(path, '', {
|
|
1860
|
-
flag: 'wx'
|
|
1861
|
-
});
|
|
1862
|
-
return {
|
|
1863
|
-
path,
|
|
1864
|
-
append (event) {
|
|
1865
|
-
appendFileSync(path, `${JSON.stringify(event)}\n`);
|
|
1866
|
-
}
|
|
1867
|
-
};
|
|
1868
|
-
}
|
|
1869
|
-
const asNotRun = (documentId, collectedCase, projectName, reason)=>({
|
|
1870
|
-
documentId,
|
|
1871
|
-
caseId: collectedCase.caseId,
|
|
1872
|
-
projectName,
|
|
1873
|
-
name: collectedCase.definition.name,
|
|
1874
|
-
sourcePath: collectedCase.sourcePath,
|
|
1875
|
-
caseIndex: collectedCase.caseIndex,
|
|
1876
|
-
status: 'not-run',
|
|
1877
|
-
notRunReason: reason
|
|
1878
|
-
});
|
|
1879
|
-
const summarize = (projects)=>{
|
|
1880
|
-
const healthCases = projects.flatMap((project)=>{
|
|
1881
|
-
const documentAttemptByRunId = new Map(project.documents.map((document)=>[
|
|
1882
|
-
document.documentRunId,
|
|
1883
|
-
document.attemptIndex ?? 0
|
|
1884
|
-
]));
|
|
1885
|
-
return latestById(project.cases, (item)=>item.caseId).map((item)=>({
|
|
1886
|
-
status: item.status,
|
|
1887
|
-
...item.run ? {
|
|
1888
|
-
run: item.run
|
|
1889
|
-
} : {},
|
|
1890
|
-
...item.attempts ? {
|
|
1891
|
-
attempts: item.attempts
|
|
1892
|
-
} : {},
|
|
1893
|
-
retryCount: item.documentRunId ? documentAttemptByRunId.get(item.documentRunId) ?? 0 : 0
|
|
1894
|
-
}));
|
|
1895
|
-
});
|
|
1896
|
-
const documents = projects.flatMap((project)=>latestById(project.documents, (item)=>item.documentId));
|
|
1897
|
-
const health = calculateTestRunHealth(healthCases);
|
|
1898
|
-
return {
|
|
1899
|
-
total: health.total,
|
|
1900
|
-
passed: health.passed,
|
|
1901
|
-
failed: health.failed,
|
|
1902
|
-
notRun: health.notRun,
|
|
1903
|
-
passedAfterRetry: health.passedAfterRetry,
|
|
1904
|
-
finalPassRate: health.finalPassRate,
|
|
1905
|
-
firstPassRate: health.firstPassRate,
|
|
1906
|
-
filtered: projects.reduce((total, project)=>total + project.filteredCaseCount, 0),
|
|
1907
|
-
collectionErrors: projects.reduce((total, project)=>total + project.collectionErrors.length, 0),
|
|
1908
|
-
documentFailures: documents.filter((document)=>'failed' === document.status).length,
|
|
1909
|
-
projectFailures: projects.filter((project)=>'failed' === project.status).length
|
|
1910
|
-
};
|
|
1911
|
-
};
|
|
1912
|
-
const latestById = (items, id)=>[
|
|
1913
|
-
...new Map(items.map((item)=>[
|
|
1914
|
-
id(item),
|
|
1915
|
-
item
|
|
1916
|
-
])).values()
|
|
1917
|
-
];
|
|
1918
|
-
const notRunSuite = (prepared, reason)=>prepared.documents.flatMap((document)=>document.cases.map((item)=>asNotRun(document.documentId, item, prepared.project.name, reason)));
|
|
1919
|
-
async function runPreparedTestProject(plan) {
|
|
1920
|
-
const reports = new ReportSession(join(plan.runDir, 'records'));
|
|
1921
|
-
return runWithMidsceneRunDir(plan.artifactDir, ()=>reports.run(()=>runWithReportSession(plan, reports)));
|
|
1922
|
-
}
|
|
1923
|
-
async function runWithReportSession(plan, reports) {
|
|
1924
|
-
const { startedAt, runId, projectRoot, configPath, resultDir, runDir, summaryPath, reportDir, definition, projects: preparedProjects } = plan;
|
|
1925
|
-
const progress = plan.onProgress;
|
|
1926
|
-
const totalDocuments = preparedProjects.reduce((total, prepared)=>total + prepared.documents.length, 0);
|
|
1927
|
-
const totalCases = preparedProjects.reduce((total, prepared)=>total + prepared.selectedCaseCount, 0);
|
|
1928
|
-
const totalErrors = preparedProjects.reduce((total, prepared)=>total + prepared.collectionErrors.length, 0);
|
|
1929
|
-
plan.onEvent?.({
|
|
1930
|
-
type: 'run-start',
|
|
1931
|
-
configuredModels: configuredModelNames(),
|
|
1932
|
-
projects: preparedProjects.length,
|
|
1933
|
-
documents: totalDocuments + totalErrors,
|
|
1934
|
-
cases: totalCases,
|
|
1935
|
-
projectRoot,
|
|
1936
|
-
configPath,
|
|
1937
|
-
startedAt: startedAt.toISOString(),
|
|
1938
|
-
maxConcurrentProjects: definition.test.maxConcurrentProjects,
|
|
1939
|
-
maxConcurrentFiles: definition.test.maxConcurrentFiles,
|
|
1940
|
-
files: preparedProjects.flatMap((prepared)=>[
|
|
1941
|
-
...prepared.documents.map((document)=>({
|
|
1942
|
-
documentId: document.documentId,
|
|
1943
|
-
projectName: prepared.project.name,
|
|
1944
|
-
sourcePath: document.sourcePath,
|
|
1945
|
-
cases: document.cases.length
|
|
1946
|
-
})),
|
|
1947
|
-
...prepared.collectionErrors.map((error)=>({
|
|
1948
|
-
documentId: error.sourcePath,
|
|
1949
|
-
projectName: prepared.project.name,
|
|
1950
|
-
sourcePath: error.sourcePath,
|
|
1951
|
-
cases: 0
|
|
1952
|
-
}))
|
|
1953
|
-
])
|
|
1954
|
-
});
|
|
1955
|
-
progress(`midscene-test: preflighted ${preparedProjects.length} projects, ${totalDocuments} documents, ${totalCases} cases, ${totalErrors} collection errors`);
|
|
1956
|
-
const effectiveConcurrency = Math.min(definition.test.maxConcurrentFiles ?? 1 / 0, Math.max(1, totalDocuments));
|
|
1957
|
-
await writeTestRunPlan(runDir, preparedProjects);
|
|
1958
|
-
const journal = createOperationJournal(runDir);
|
|
1959
|
-
const rootController = new AbortController();
|
|
1960
|
-
setMaxListeners(Math.max(10, effectiveConcurrency + 1), rootController.signal);
|
|
1961
|
-
const handleSignal = (signal)=>{
|
|
1962
|
-
rootController.abort(new Error(`Workflow interrupted by ${signal}.`));
|
|
1963
|
-
};
|
|
1964
|
-
const sigint = ()=>handleSignal('SIGINT');
|
|
1965
|
-
const sigterm = ()=>handleSignal('SIGTERM');
|
|
1966
|
-
process.on('SIGINT', sigint);
|
|
1967
|
-
process.on('SIGTERM', sigterm);
|
|
1968
|
-
let failedCaseCount = 0;
|
|
1969
|
-
const bailReached = ()=>definition.test.bail > 0 && failedCaseCount >= definition.test.bail;
|
|
1970
|
-
let shutdownDeadline;
|
|
1971
|
-
const beginShutdown = ()=>{
|
|
1972
|
-
shutdownDeadline ??= Date.now() + definition.test.teardownTimeout;
|
|
1973
|
-
};
|
|
1974
|
-
rootController.signal.addEventListener('abort', beginShutdown, {
|
|
1975
|
-
once: true
|
|
1976
|
-
});
|
|
1977
|
-
const addFailedCases = (count)=>{
|
|
1978
|
-
failedCaseCount += count;
|
|
1979
|
-
if (bailReached()) beginShutdown();
|
|
1980
|
-
};
|
|
1981
|
-
const teardownTimeoutMs = ()=>Math.max(0, Math.min(definition.test.teardownTimeout, void 0 === shutdownDeadline ? 1 / 0 : shutdownDeadline - Date.now()));
|
|
1982
|
-
let hasInfrastructureError = false;
|
|
1983
|
-
const infrastructureErrors = [];
|
|
1984
|
-
const recordInfrastructureError = (error)=>{
|
|
1985
|
-
const errors = error instanceof WorkflowExecutionFailure ? error.errors : [
|
|
1986
|
-
error
|
|
1987
|
-
];
|
|
1988
|
-
for (const item of errors)if (!infrastructureErrors.includes(item)) infrastructureErrors.push(item);
|
|
1989
|
-
hasInfrastructureError = true;
|
|
1990
|
-
rootController.abort(error);
|
|
1991
|
-
};
|
|
1992
|
-
const runInfrastructureCallback = (callback)=>{
|
|
1993
|
-
try {
|
|
1994
|
-
return callback();
|
|
1995
|
-
} catch (error) {
|
|
1996
|
-
recordInfrastructureError(error);
|
|
1997
|
-
throw error;
|
|
1998
|
-
}
|
|
1999
|
-
};
|
|
2000
|
-
const stopObservingOperations = observeResourceOperations(rootController.signal, (event)=>{
|
|
2001
|
-
try {
|
|
2002
|
-
journal.append(event);
|
|
2003
|
-
} catch (error) {
|
|
2004
|
-
recordInfrastructureError(new WorkflowPublicationError('write-result', journal.path, error));
|
|
2005
|
-
getDebug('test-runner:operations', {
|
|
2006
|
-
console: true
|
|
2007
|
-
})(`Could not persist operation event: ${String(error)}`);
|
|
2008
|
-
}
|
|
2009
|
-
});
|
|
2010
|
-
const announceProject = (prepared, projectIndex)=>{
|
|
2011
|
-
const { project } = prepared;
|
|
2012
|
-
runInfrastructureCallback(()=>progress(`[project ${projectIndex + 1}/${preparedProjects.length}] ${project.name}`));
|
|
2013
|
-
};
|
|
2014
|
-
const buildProjectResult = (prepared, cases, documents, lifecycle)=>{
|
|
2015
|
-
const { project } = prepared;
|
|
2016
|
-
const projectFailed = prepared.collectionErrors.length > 0 || latestById(cases, (item)=>item.caseId).some((item)=>'success' !== item.status) || latestById(documents, (item)=>item.documentId).some((item)=>'failed' === item.status) || lifecycle?.status === 'failed';
|
|
2017
|
-
return {
|
|
2018
|
-
projectId: project.projectId,
|
|
2019
|
-
name: project.name,
|
|
2020
|
-
...prepared.platform ? {
|
|
2021
|
-
platform: prepared.platform
|
|
2022
|
-
} : {},
|
|
2023
|
-
status: projectFailed ? 'failed' : 'success',
|
|
2024
|
-
retry: project.retry,
|
|
2025
|
-
fileSelection: prepared.fileSelection,
|
|
2026
|
-
tagSelection: project.tags,
|
|
2027
|
-
sourceCount: prepared.sources.length,
|
|
2028
|
-
selectedCaseCount: prepared.selectedCaseCount,
|
|
2029
|
-
filteredCaseCount: prepared.filteredCaseCount,
|
|
2030
|
-
...lifecycle ? {
|
|
2031
|
-
lifecycle
|
|
2032
|
-
} : {},
|
|
2033
|
-
cases,
|
|
2034
|
-
plans: prepared.plans,
|
|
2035
|
-
documents,
|
|
2036
|
-
collectionErrors: prepared.collectionErrors
|
|
2037
|
-
};
|
|
2038
|
-
};
|
|
2039
|
-
const buildSkippedProjectResult = (prepared, projectIndex, reason)=>{
|
|
2040
|
-
announceProject(prepared, projectIndex);
|
|
2041
|
-
return buildProjectResult(prepared, notRunSuite(prepared, reason), []);
|
|
2042
|
-
};
|
|
2043
|
-
const projectResults = new Array(preparedProjects.length);
|
|
2044
|
-
try {
|
|
2045
|
-
const states = await runPreparedFiles({
|
|
2046
|
-
projects: preparedProjects,
|
|
2047
|
-
maxConcurrentFiles: effectiveConcurrency,
|
|
2048
|
-
maxConcurrentProjects: definition.test.maxConcurrentProjects ?? 1 / 0,
|
|
2049
|
-
setupTimeoutMs: definition.test.setupTimeout,
|
|
2050
|
-
teardownTimeoutMs,
|
|
2051
|
-
signal: rootController.signal,
|
|
2052
|
-
runDir,
|
|
2053
|
-
preflightFailed: 'run' === plan.preflightScope && totalErrors > 0,
|
|
2054
|
-
shouldBail: bailReached,
|
|
2055
|
-
addFailedCases,
|
|
2056
|
-
announceProject,
|
|
2057
|
-
onProgress: (prepared, message)=>runInfrastructureCallback(()=>progress(preparedProjects.length > 1 || effectiveConcurrency > 1 ? `[${prepared.project.name}]${message}` : message)),
|
|
2058
|
-
onEvent: (event)=>runInfrastructureCallback(()=>plan.onEvent?.(event)),
|
|
2059
|
-
onProjectEnd: (state)=>runInfrastructureCallback(()=>plan.onEvent?.({
|
|
2060
|
-
type: 'project-end',
|
|
2061
|
-
project: buildProjectResult(state.prepared, state.cases, state.documents, state.lifecycle)
|
|
2062
|
-
})),
|
|
2063
|
-
onInfrastructureError: recordInfrastructureError
|
|
2064
|
-
});
|
|
2065
|
-
states.forEach((state, index)=>{
|
|
2066
|
-
projectResults[index] = buildProjectResult(state.prepared, state.cases, state.documents, state.lifecycle);
|
|
2067
|
-
});
|
|
2068
|
-
} catch (error) {
|
|
2069
|
-
recordInfrastructureError(error);
|
|
2070
|
-
} finally{
|
|
2071
|
-
stopObservingOperations();
|
|
2072
|
-
process.off('SIGINT', sigint);
|
|
2073
|
-
process.off('SIGTERM', sigterm);
|
|
2074
|
-
}
|
|
2075
|
-
const completedProjectResults = preparedProjects.map((prepared, projectIndex)=>{
|
|
2076
|
-
const result = projectResults[projectIndex];
|
|
2077
|
-
if (result) return result;
|
|
2078
|
-
const reason = prepared.collectionErrors.length ? 'project-preflight-failed' : rootController.signal.aborted ? 'interrupted' : bailReached() ? 'bail' : void 0;
|
|
2079
|
-
if (!reason) throw new Error(`Project scheduler did not produce a result for "${prepared.project.name}".`);
|
|
2080
|
-
return buildSkippedProjectResult(prepared, projectIndex, reason);
|
|
2081
|
-
});
|
|
2082
|
-
const summary = summarize(completedProjectResults);
|
|
2083
|
-
const failed = hasInfrastructureError || rootController.signal.aborted || summary.failed > 0 || summary.notRun > 0 || summary.collectionErrors > 0 || summary.documentFailures > 0 || summary.projectFailures > 0;
|
|
2084
|
-
const endedAt = new Date();
|
|
2085
|
-
const result = {
|
|
2086
|
-
schemaVersion: 3,
|
|
2087
|
-
operationLog: {
|
|
2088
|
-
path: journal.path,
|
|
2089
|
-
snapshotAt: endedAt.toISOString()
|
|
2090
|
-
},
|
|
2091
|
-
runId,
|
|
2092
|
-
startedAt: startedAt.toISOString(),
|
|
2093
|
-
endedAt: endedAt.toISOString(),
|
|
2094
|
-
durationMs: Math.max(0, endedAt.getTime() - startedAt.getTime()),
|
|
2095
|
-
status: failed ? 'failed' : 'success',
|
|
2096
|
-
exitCode: failed ? 1 : 0,
|
|
2097
|
-
resultDir,
|
|
2098
|
-
summaryPath,
|
|
2099
|
-
reportDir,
|
|
2100
|
-
summary,
|
|
2101
|
-
projects: completedProjectResults,
|
|
2102
|
-
cases: completedProjectResults.flatMap((project)=>project.cases),
|
|
2103
|
-
documents: completedProjectResults.flatMap((project)=>project.documents),
|
|
2104
|
-
collectionErrors: completedProjectResults.flatMap((project)=>project.collectionErrors),
|
|
2105
|
-
...hasInfrastructureError ? {
|
|
2106
|
-
errors: infrastructureErrors.map(asExecutionError)
|
|
2107
|
-
} : {}
|
|
2108
|
-
};
|
|
2109
|
-
let completedResult = result;
|
|
2110
|
-
let reportIndex;
|
|
2111
|
-
const publish = async (operation, path, callback)=>{
|
|
2112
|
-
try {
|
|
2113
|
-
await callback();
|
|
2114
|
-
return true;
|
|
2115
|
-
} catch (error) {
|
|
2116
|
-
recordInfrastructureError(error instanceof WorkflowPublicationError || error instanceof WorkflowExecutionFailure ? error : new WorkflowPublicationError(operation, path, error));
|
|
2117
|
-
completedResult = {
|
|
2118
|
-
...completedResult,
|
|
2119
|
-
status: 'failed',
|
|
2120
|
-
exitCode: 1,
|
|
2121
|
-
errors: infrastructureErrors.map(asExecutionError)
|
|
2122
|
-
};
|
|
2123
|
-
return false;
|
|
2124
|
-
}
|
|
2125
|
-
};
|
|
2126
|
-
const writeSummary = ()=>publish('write-result', summaryPath, ()=>writeTestProjectRunResult({
|
|
2127
|
-
projectRoot,
|
|
2128
|
-
...configPath ? {
|
|
2129
|
-
configPath
|
|
2130
|
-
} : {},
|
|
2131
|
-
result: completedResult
|
|
2132
|
-
}));
|
|
2133
|
-
let extraSources = [];
|
|
2134
|
-
await publish('write-report', reports.directory, async ()=>{
|
|
2135
|
-
extraSources = await reports.finish();
|
|
2136
|
-
});
|
|
2137
|
-
const writeReport = ()=>publish('write-report', reportDir, async ()=>{
|
|
2138
|
-
if (!plan.reportEnabled) return;
|
|
2139
|
-
const reportPath = await new TestRunReportAssembler().assembleAsync({
|
|
2140
|
-
outputDir: reportDir,
|
|
2141
|
-
outputFormat: plan.definition.output.reportFormat,
|
|
2142
|
-
reportFileName: `midscene-e2e-${runId}`,
|
|
2143
|
-
overwrite: false,
|
|
2144
|
-
sources: [
|
|
2145
|
-
...collectTestRunReportSources(completedResult),
|
|
2146
|
-
...extraSources.map((source)=>({
|
|
2147
|
-
...source,
|
|
2148
|
-
scopeId: runId
|
|
2149
|
-
}))
|
|
2150
|
-
],
|
|
2151
|
-
buildRunnerDump: (index)=>{
|
|
2152
|
-
reportIndex = index;
|
|
2153
|
-
const dump = buildTestRunReportDump(completedResult, index);
|
|
2154
|
-
completedResult = {
|
|
2155
|
-
...completedResult,
|
|
2156
|
-
metrics: dump.metrics
|
|
2157
|
-
};
|
|
2158
|
-
const warn = getDebug('test-runner:report-assembler', {
|
|
2159
|
-
console: true
|
|
2160
|
-
});
|
|
2161
|
-
for (const diagnostic of dump.diagnostics ?? [])warn(diagnostic.message);
|
|
2162
|
-
return dump;
|
|
2163
|
-
}
|
|
2164
|
-
});
|
|
2165
|
-
completedResult = {
|
|
2166
|
-
...completedResult,
|
|
2167
|
-
reportPath
|
|
2168
|
-
};
|
|
2169
|
-
});
|
|
2170
|
-
await writeReport();
|
|
2171
|
-
for (const publication of plan.publications)await publish(publication.operation, publication.path, async ()=>{
|
|
2172
|
-
await publication.publish(completedResult);
|
|
2173
|
-
});
|
|
2174
|
-
await publish('write-result', join(dirname(summaryPath), 'result.md'), async ()=>{
|
|
2175
|
-
if (!completedResult.metrics) {
|
|
2176
|
-
const dump = buildTestRunReportDump(completedResult, reportIndex ?? {
|
|
2177
|
-
sources: [],
|
|
2178
|
-
metrics: test_runner_sumTestRunReportMetrics([])
|
|
2179
|
-
});
|
|
2180
|
-
completedResult = {
|
|
2181
|
-
...completedResult,
|
|
2182
|
-
metrics: dump.metrics
|
|
2183
|
-
};
|
|
2184
|
-
}
|
|
2185
|
-
const markdownPath = await writeTestResultMarkdown(completedResult, reportIndex);
|
|
2186
|
-
completedResult = {
|
|
2187
|
-
...completedResult,
|
|
2188
|
-
markdownPath
|
|
2189
|
-
};
|
|
2190
|
-
});
|
|
2191
|
-
if (completedResult.reportPath && !hasInfrastructureError) await publish('write-result', reports.directory, ()=>reports.prune());
|
|
2192
|
-
await writeSummary();
|
|
2193
|
-
if (hasInfrastructureError) throw new WorkflowExecutionFailure(completedResult, infrastructureErrors);
|
|
2194
|
-
return completedResult;
|
|
2195
|
-
}
|
|
1
|
+
import { prepareTestRun, runPreparedTestProject } from "../299.mjs";
|
|
2196
2
|
async function runTestProject(options = {}) {
|
|
2197
3
|
return runPreparedTestProject(await prepareTestRun(options));
|
|
2198
4
|
}
|
|
2199
|
-
export { DEFAULT_TEST_FILE_SELECTION, createTestRunId, defineFileSetup, defineProjectSetup, defineTestProject, discoverTestConfig, discoverTestFiles, loadTestProject,
|
|
5
|
+
export { DEFAULT_TEST_FILE_SELECTION, createTestRunId, defineFileSetup, defineProjectSetup, defineTestProject, discoverTestConfig, discoverTestFiles, loadTestProject, validateTestFileSelection } from "../299.mjs";
|
|
6
|
+
export { runTestProject };
|
|
2200
7
|
|
|
2201
8
|
//# sourceMappingURL=index.mjs.map
|