@midscene/test 1.17.8-beta-20261106101231.0 → 1.17.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/154.mjs +2 -0
- package/dist/es/164.mjs +1469 -0
- package/dist/es/164.mjs.map +1 -0
- package/dist/es/183.mjs +295 -0
- package/dist/es/183.mjs.map +1 -0
- package/dist/es/1~112.mjs +372 -0
- package/dist/es/1~112.mjs.map +1 -0
- package/dist/es/209.mjs +1 -0
- package/dist/es/299.mjs +1886 -0
- package/dist/es/299.mjs.map +1 -0
- package/dist/es/398.mjs +5 -0
- package/dist/es/398.mjs.map +1 -0
- package/dist/es/431.mjs +375 -0
- package/dist/es/431.mjs.map +1 -0
- package/dist/es/470.mjs +1 -0
- package/dist/es/487.mjs +1 -0
- package/dist/es/629.mjs +1 -0
- package/dist/es/793.mjs +1 -0
- package/dist/es/822.mjs +1 -0
- package/dist/es/946.mjs +1 -0
- package/dist/es/cli/index.mjs +3 -2196
- package/dist/es/cli/index.mjs.map +1 -1
- package/dist/es/cli/model-command.mjs +1 -296
- package/dist/es/index.mjs +10 -409
- package/dist/es/index.mjs.map +1 -1
- package/dist/es/midscene/index.mjs +9 -9
- package/dist/es/midscene/index.mjs.map +1 -1
- package/dist/es/runtime/index.mjs +4 -5
- package/dist/es/runtime/index.mjs.map +1 -1
- package/dist/es/test-cli.mjs +1 -3786
- package/dist/lib/0~703.js.map +1 -1
- package/dist/lib/cli/index.js +45 -58
- package/dist/lib/cli/index.js.map +1 -1
- package/dist/lib/cli/model-command.js +1 -1
- package/dist/lib/cli/model-command.js.map +1 -1
- package/dist/lib/index.js +8 -37
- package/dist/lib/index.js.map +1 -1
- package/dist/lib/midscene/index.js +1 -1
- package/dist/lib/midscene/index.js.map +1 -1
- package/dist/lib/runtime/index.js +1 -1
- package/dist/lib/runtime/index.js.map +1 -1
- package/dist/lib/test-cli.js +884 -877
- package/dist/lib/test-cli.js.map +1 -1
- package/dist/types/cli/index.d.ts +4 -0
- package/package.json +20 -21
- package/dist/es/1~4.mjs +0 -383
- package/dist/es/1~4.mjs.map +0 -1
- package/dist/es/1~731.mjs +0 -306
- package/dist/es/1~731.mjs.map +0 -1
- package/dist/es/cli/model-command.mjs.map +0 -1
- package/dist/es/test-cli.mjs.map +0 -1
package/dist/es/299.mjs
ADDED
|
@@ -0,0 +1,1886 @@
|
|
|
1
|
+
import { inspect, stripVTControlCharacters } from "node:util";
|
|
2
|
+
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
3
|
+
import { resolveMidsceneRunDir, runWithMidsceneRunDir } from "@midscene/shared/common";
|
|
4
|
+
import { globSync } from "tinyglobby";
|
|
5
|
+
import { require as api_require } from "tsx/cjs/api";
|
|
6
|
+
import { setMaxListeners } from "node:events";
|
|
7
|
+
import { ReportSession, TestRunReportAssembler, calculateTestRunHealth } from "@midscene/core/report";
|
|
8
|
+
import { WorkflowExecutionFailure, WorkflowError, isFatalDeviceError, decideRetry, WorkflowPublicationError, runDocumentAttempts, getPendingScopeOperations, runWorkflowDocument, asExecutionError, WorkflowParseError, waitForScopeOperations, createProjectRuntime, sanitizeReportError, buildTestRunReportDump, collectTestRunReportSources, buildTestRunReportPlan, trackResourceOperation, isResourceRecoveryError, observeResourceOperations, getResourceCleanupCompletion, sumTestRunReportMetrics } from "./793.mjs";
|
|
9
|
+
import { globalConfigManager } from "./209.mjs";
|
|
10
|
+
import { createHash, writeFile, NodeRegistry, randomUUID, NativeWorkflowFormatError, mkdir, collectWorkflowDocument, writeTestResultMarkdown } from "./431.mjs";
|
|
11
|
+
import { sep, dirname, readdirSync, resolve as external_node_path_resolve, realpathSync, existsSync, statSync, relative, isAbsolute, mkdirSync, writeFileSync, join, appendFileSync } from "./154.mjs";
|
|
12
|
+
import { runWithConsoleLogScope, getDebug } from "./470.mjs";
|
|
13
|
+
function _define_property(obj, key, value) {
|
|
14
|
+
if (key in obj) Object.defineProperty(obj, key, {
|
|
15
|
+
value: value,
|
|
16
|
+
enumerable: true,
|
|
17
|
+
configurable: true,
|
|
18
|
+
writable: true
|
|
19
|
+
});
|
|
20
|
+
else obj[key] = value;
|
|
21
|
+
return obj;
|
|
22
|
+
}
|
|
23
|
+
function configuredModelNames() {
|
|
24
|
+
const env = globalConfigManager.getAllEnvConfig();
|
|
25
|
+
const defaultName = env.MIDSCENE_MODEL_NAME;
|
|
26
|
+
const models = new Map();
|
|
27
|
+
for (const [intent, name] of [
|
|
28
|
+
[
|
|
29
|
+
'default',
|
|
30
|
+
defaultName
|
|
31
|
+
],
|
|
32
|
+
[
|
|
33
|
+
'planning',
|
|
34
|
+
env.MIDSCENE_PLANNING_MODEL_NAME || defaultName
|
|
35
|
+
],
|
|
36
|
+
[
|
|
37
|
+
'insight',
|
|
38
|
+
env.MIDSCENE_INSIGHT_MODEL_NAME || defaultName
|
|
39
|
+
]
|
|
40
|
+
]){
|
|
41
|
+
if (!name) continue;
|
|
42
|
+
const intents = models.get(name) ?? [];
|
|
43
|
+
intents.push(intent);
|
|
44
|
+
models.set(name, intents);
|
|
45
|
+
}
|
|
46
|
+
return [
|
|
47
|
+
...models
|
|
48
|
+
].map(([name, intents])=>({
|
|
49
|
+
name,
|
|
50
|
+
intents
|
|
51
|
+
}));
|
|
52
|
+
}
|
|
53
|
+
class CliModelUsage {
|
|
54
|
+
add(step) {
|
|
55
|
+
for (const trace of step.report?.traces ?? [])if (trace.metrics) {
|
|
56
|
+
this.executions.set(trace.executionId, trace.metrics);
|
|
57
|
+
this.missing.delete(trace.executionId);
|
|
58
|
+
} else if (!this.executions.has(trace.executionId)) this.missing.add(trace.executionId);
|
|
59
|
+
}
|
|
60
|
+
snapshot() {
|
|
61
|
+
if (!this.executions.size && !this.missing.size) return;
|
|
62
|
+
return sumTestRunReportMetrics([
|
|
63
|
+
...this.executions.values()
|
|
64
|
+
], this.missing.size ? [
|
|
65
|
+
'Some execution usage has not been recorded.'
|
|
66
|
+
] : []);
|
|
67
|
+
}
|
|
68
|
+
constructor(){
|
|
69
|
+
_define_property(this, "executions", new Map());
|
|
70
|
+
_define_property(this, "missing", new Set());
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
function formatTokenCount(value) {
|
|
74
|
+
if (value < 1000) return String(Math.round(value));
|
|
75
|
+
if (value < 1000000) return `${(value / 1000).toFixed(1)}k`;
|
|
76
|
+
return `${(value / 1000000).toFixed(1)}m`;
|
|
77
|
+
}
|
|
78
|
+
function formatModelUsage(metrics) {
|
|
79
|
+
if (!metrics) return [
|
|
80
|
+
'—'
|
|
81
|
+
];
|
|
82
|
+
if (!metrics.models?.length) {
|
|
83
|
+
const name = metrics.modelCallCount || metrics.collection?.status === 'unknown' ? 'unknown' : '—';
|
|
84
|
+
return [
|
|
85
|
+
`${name} · ${formatTokenUsage(metrics)}`
|
|
86
|
+
];
|
|
87
|
+
}
|
|
88
|
+
return metrics.models.map((model)=>{
|
|
89
|
+
const name = stripVTControlCharacters(model.modelName).replace(/\s+/g, ' ').trim() || 'unknown';
|
|
90
|
+
const usage = formatTokenUsage({
|
|
91
|
+
...metrics,
|
|
92
|
+
modelCallCount: model.callCount,
|
|
93
|
+
promptTokens: model.promptTokens,
|
|
94
|
+
cachedInputTokens: model.cachedInputTokens,
|
|
95
|
+
completionTokens: model.completionTokens,
|
|
96
|
+
totalTokens: model.totalTokens
|
|
97
|
+
});
|
|
98
|
+
return `${name} · ${usage}`;
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
function formatTokenUsage(metrics, includeCached = true) {
|
|
102
|
+
if (!metrics) return '—';
|
|
103
|
+
const state = (field)=>metrics.collection?.fields?.[field] ?? metrics.collection?.status ?? 'complete';
|
|
104
|
+
if ('unknown' === state('totalTokens')) return '— (usage unavailable)';
|
|
105
|
+
if (!metrics.modelCallCount && !metrics.totalTokens && 'complete' === state('totalTokens')) return '0 (no model calls)';
|
|
106
|
+
const count = (field)=>'unknown' === state(field) ? '—' : `${'partial' === state(field) ? '≥' : ''}${formatTokenCount(metrics[field])}`;
|
|
107
|
+
return [
|
|
108
|
+
`${count('totalTokens')} total${'partial' === state('totalTokens') ? ' (partial)' : ''}`,
|
|
109
|
+
`${count('promptTokens')} input${includeCached && metrics.cachedInputTokens ? ` (${formatTokenCount(metrics.cachedInputTokens)} cached)` : ''}`,
|
|
110
|
+
`${count('completionTokens')} output`
|
|
111
|
+
].join(' · ');
|
|
112
|
+
}
|
|
113
|
+
const writeJson = async (path, value)=>{
|
|
114
|
+
try {
|
|
115
|
+
await mkdir(dirname(path), {
|
|
116
|
+
recursive: true
|
|
117
|
+
});
|
|
118
|
+
await writeFile(path, JSON.stringify(value, null, 2));
|
|
119
|
+
} catch (error) {
|
|
120
|
+
throw new WorkflowPublicationError('write-result', path, error);
|
|
121
|
+
}
|
|
122
|
+
};
|
|
123
|
+
const writeTestRunPlan = (runDir, projects)=>writeJson(join(runDir, 'plan.json'), {
|
|
124
|
+
schemaVersion: 1,
|
|
125
|
+
createdAt: new Date().toISOString(),
|
|
126
|
+
projects: projects.map((item)=>({
|
|
127
|
+
projectId: item.project.projectId,
|
|
128
|
+
name: item.project.name,
|
|
129
|
+
documents: item.plans ?? []
|
|
130
|
+
}))
|
|
131
|
+
});
|
|
132
|
+
const toPosix = (value)=>value.split(sep).join('/');
|
|
133
|
+
const caseAttemptResultPath = (projectId, documentId, result)=>toPosix(join(projectId, 'documents', documentId, 'cases', result.caseId, `${result.runId}.json`));
|
|
134
|
+
const workflowDocumentResultPath = (result)=>toPosix(join(result.projectId, 'documents', result.documentId, void 0 === result.attemptIndex ? 'document.json' : `${result.documentRunId}.json`));
|
|
135
|
+
const collectionErrorPath = (projectId, sourcePath)=>{
|
|
136
|
+
const id = createHash('sha256').update(JSON.stringify([
|
|
137
|
+
projectId,
|
|
138
|
+
sourcePath
|
|
139
|
+
])).digest('hex');
|
|
140
|
+
return toPosix(join(projectId, 'collection-errors', `${id}.json`));
|
|
141
|
+
};
|
|
142
|
+
const writeWorkflowDocumentResult = (runDir, result)=>{
|
|
143
|
+
const { documentRunId: _documentRunId, ...persistedResult } = result;
|
|
144
|
+
return writeJson(join(runDir, workflowDocumentResultPath(result)), persistedResult);
|
|
145
|
+
};
|
|
146
|
+
const writeCaseAttemptResult = (runDir, projectId, documentId, result)=>{
|
|
147
|
+
const { runId: attemptId, ...persistedResult } = result;
|
|
148
|
+
return writeJson(join(runDir, caseAttemptResultPath(projectId, documentId, result)), {
|
|
149
|
+
...persistedResult,
|
|
150
|
+
attemptId
|
|
151
|
+
});
|
|
152
|
+
};
|
|
153
|
+
const writeCollectionError = (runDir, collectionError)=>writeJson(join(runDir, collectionErrorPath(collectionError.projectId, collectionError.sourcePath)), {
|
|
154
|
+
kind: 'collection-error',
|
|
155
|
+
...collectionError
|
|
156
|
+
});
|
|
157
|
+
const relativeToSummary = (summaryPath, target)=>{
|
|
158
|
+
const value = relative(dirname(summaryPath), target);
|
|
159
|
+
return toPosix(value || '.');
|
|
160
|
+
};
|
|
161
|
+
const errorJson = sanitizeReportError;
|
|
162
|
+
const writeTestProjectRunResult = (options)=>{
|
|
163
|
+
const { result } = options;
|
|
164
|
+
const fact = (path)=>toPosix(path);
|
|
165
|
+
return writeJson(result.summaryPath, {
|
|
166
|
+
schemaVersion: result.schemaVersion,
|
|
167
|
+
runId: result.runId,
|
|
168
|
+
startedAt: result.startedAt,
|
|
169
|
+
endedAt: result.endedAt,
|
|
170
|
+
durationMs: result.durationMs,
|
|
171
|
+
status: result.status,
|
|
172
|
+
exitCode: result.exitCode,
|
|
173
|
+
...result.errors?.length ? {
|
|
174
|
+
errors: result.errors.map(errorJson)
|
|
175
|
+
} : {},
|
|
176
|
+
projectRoot: options.projectRoot,
|
|
177
|
+
...options.configPath ? {
|
|
178
|
+
configPath: options.configPath
|
|
179
|
+
} : {},
|
|
180
|
+
factsRoot: '.',
|
|
181
|
+
plan: 'plan.json',
|
|
182
|
+
...result.operationLog ? {
|
|
183
|
+
operationLog: {
|
|
184
|
+
...result.operationLog,
|
|
185
|
+
path: relativeToSummary(result.summaryPath, result.operationLog.path)
|
|
186
|
+
}
|
|
187
|
+
} : {},
|
|
188
|
+
reportDir: relativeToSummary(result.summaryPath, result.reportDir),
|
|
189
|
+
...result.reportPath ? {
|
|
190
|
+
report: relativeToSummary(result.summaryPath, result.reportPath)
|
|
191
|
+
} : {},
|
|
192
|
+
...result.markdownPath ? {
|
|
193
|
+
markdown: relativeToSummary(result.summaryPath, result.markdownPath)
|
|
194
|
+
} : {},
|
|
195
|
+
summary: result.summary,
|
|
196
|
+
...result.metrics ? {
|
|
197
|
+
metrics: result.metrics
|
|
198
|
+
} : {},
|
|
199
|
+
projects: result.projects.map((project)=>({
|
|
200
|
+
projectId: project.projectId,
|
|
201
|
+
name: project.name,
|
|
202
|
+
status: project.status,
|
|
203
|
+
retry: project.retry,
|
|
204
|
+
fileSelection: project.fileSelection,
|
|
205
|
+
tagSelection: project.tagSelection,
|
|
206
|
+
sourceCount: project.sourceCount,
|
|
207
|
+
selectedCaseCount: project.selectedCaseCount,
|
|
208
|
+
filteredCaseCount: project.filteredCaseCount,
|
|
209
|
+
...project.lifecycle ? {
|
|
210
|
+
lifecycle: {
|
|
211
|
+
status: project.lifecycle.status,
|
|
212
|
+
startedAt: project.lifecycle.startedAt,
|
|
213
|
+
endedAt: project.lifecycle.endedAt,
|
|
214
|
+
durationMs: project.lifecycle.durationMs,
|
|
215
|
+
...project.lifecycle.cleanup ? {
|
|
216
|
+
cleanup: project.lifecycle.cleanup
|
|
217
|
+
} : {},
|
|
218
|
+
...project.lifecycle.pendingOperations ? {
|
|
219
|
+
pendingOperations: project.lifecycle.pendingOperations
|
|
220
|
+
} : {},
|
|
221
|
+
...project.lifecycle.setupError ? {
|
|
222
|
+
setupError: errorJson(project.lifecycle.setupError)
|
|
223
|
+
} : {},
|
|
224
|
+
...project.lifecycle.teardownErrors ? {
|
|
225
|
+
teardownErrors: project.lifecycle.teardownErrors.map(errorJson)
|
|
226
|
+
} : {}
|
|
227
|
+
}
|
|
228
|
+
} : {},
|
|
229
|
+
cases: project.cases.map((outcome)=>({
|
|
230
|
+
...outcome.documentRunId ? {
|
|
231
|
+
documentRunId: outcome.documentRunId
|
|
232
|
+
} : {},
|
|
233
|
+
documentId: outcome.documentId,
|
|
234
|
+
caseId: outcome.caseId,
|
|
235
|
+
sourcePath: outcome.sourcePath,
|
|
236
|
+
caseIndex: outcome.caseIndex,
|
|
237
|
+
name: outcome.name,
|
|
238
|
+
status: outcome.status,
|
|
239
|
+
...outcome.notRunReason ? {
|
|
240
|
+
notRunReason: outcome.notRunReason
|
|
241
|
+
} : {},
|
|
242
|
+
...outcome.attempts ? {
|
|
243
|
+
attempts: outcome.attempts.map((attempt)=>({
|
|
244
|
+
attemptId: attempt.runId,
|
|
245
|
+
attemptIndex: attempt.attemptIndex,
|
|
246
|
+
status: attempt.status,
|
|
247
|
+
...attempt.cleanup ? {
|
|
248
|
+
cleanup: attempt.cleanup
|
|
249
|
+
} : {},
|
|
250
|
+
...attempt.pendingOperations ? {
|
|
251
|
+
pendingOperations: attempt.pendingOperations
|
|
252
|
+
} : {},
|
|
253
|
+
...attempt.executionStatus ? {
|
|
254
|
+
executionStatus: attempt.executionStatus
|
|
255
|
+
} : {},
|
|
256
|
+
...attempt.retryDecision ? {
|
|
257
|
+
retryDecision: attempt.retryDecision
|
|
258
|
+
} : {},
|
|
259
|
+
resultFile: fact(caseAttemptResultPath(project.projectId, outcome.documentId, attempt)),
|
|
260
|
+
...attempt.reportPaths?.length ? {
|
|
261
|
+
reports: attempt.reportPaths.map((path)=>relativeToSummary(result.summaryPath, path))
|
|
262
|
+
} : {}
|
|
263
|
+
}))
|
|
264
|
+
} : {}
|
|
265
|
+
})),
|
|
266
|
+
documents: project.documents.map((document)=>({
|
|
267
|
+
documentId: document.documentId,
|
|
268
|
+
...void 0 === document.attemptIndex ? {} : {
|
|
269
|
+
attemptIndex: document.attemptIndex,
|
|
270
|
+
documentRunId: document.documentRunId
|
|
271
|
+
},
|
|
272
|
+
sourcePath: document.sourcePath,
|
|
273
|
+
status: document.status,
|
|
274
|
+
...document.cleanup ? {
|
|
275
|
+
cleanup: document.cleanup
|
|
276
|
+
} : {},
|
|
277
|
+
...document.pendingOperations ? {
|
|
278
|
+
pendingOperations: document.pendingOperations
|
|
279
|
+
} : {},
|
|
280
|
+
...document.executionStatus ? {
|
|
281
|
+
executionStatus: document.executionStatus
|
|
282
|
+
} : {},
|
|
283
|
+
...document.retryDecision ? {
|
|
284
|
+
retryDecision: document.retryDecision
|
|
285
|
+
} : {},
|
|
286
|
+
resultFile: fact(workflowDocumentResultPath(document)),
|
|
287
|
+
...document.reportPaths?.length ? {
|
|
288
|
+
reports: document.reportPaths.map((path)=>relativeToSummary(result.summaryPath, path))
|
|
289
|
+
} : {}
|
|
290
|
+
})),
|
|
291
|
+
collectionErrors: project.collectionErrors.map((error)=>({
|
|
292
|
+
sourcePath: error.sourcePath,
|
|
293
|
+
errorFile: fact(collectionErrorPath(error.projectId, error.sourcePath))
|
|
294
|
+
}))
|
|
295
|
+
}))
|
|
296
|
+
});
|
|
297
|
+
};
|
|
298
|
+
const projectIdFromIndex = (index)=>`project-${index}`;
|
|
299
|
+
const defineTestProject = (definition)=>definition;
|
|
300
|
+
const defineProjectSetup = (definition)=>definition;
|
|
301
|
+
const defineFileSetup = (definition)=>definition;
|
|
302
|
+
const getDefaultExport = (loaded, absolutePath)=>{
|
|
303
|
+
if ('object' == typeof loaded && null !== loaded && 'default' in loaded) {
|
|
304
|
+
const defaultExport = loaded.default;
|
|
305
|
+
if ('object' == typeof defaultExport && null !== defaultExport && '__esModule' in defaultExport) {
|
|
306
|
+
if ('default' in defaultExport) return defaultExport.default;
|
|
307
|
+
throw new TypeError(`Midscene config "${absolutePath}" must have a default export.`);
|
|
308
|
+
}
|
|
309
|
+
return defaultExport;
|
|
310
|
+
}
|
|
311
|
+
throw new TypeError(`Midscene config "${absolutePath}" must have a default export.`);
|
|
312
|
+
};
|
|
313
|
+
const isRecord = (value)=>'object' == typeof value && null !== value && !Array.isArray(value);
|
|
314
|
+
const rejectUnknownKeys = (value, allowedKeys, label)=>{
|
|
315
|
+
const allowed = new Set(allowedKeys);
|
|
316
|
+
const unknown = Object.keys(value).find((key)=>!allowed.has(key));
|
|
317
|
+
if (unknown) throw new TypeError(`Midscene config ${label}.${unknown} is not supported.`);
|
|
318
|
+
};
|
|
319
|
+
const isAbsolutePattern = (pattern)=>isAbsolute(pattern) || /^[A-Za-z]:[\\/]/.test(pattern) || pattern.startsWith('\\\\');
|
|
320
|
+
const validatePatterns = (value, field, label = 'files', allowEmptyInclude = false)=>{
|
|
321
|
+
if (!Array.isArray(value)) throw new TypeError(`Midscene config ${label}.${field} must be an array.`);
|
|
322
|
+
if ('include' === field && 0 === value.length && !allowEmptyInclude) throw new TypeError(`Midscene config ${label}.include must be a non-empty array.`);
|
|
323
|
+
return value.map((pattern, index)=>{
|
|
324
|
+
if ('string' != typeof pattern || 0 === pattern.trim().length) throw new TypeError(`Midscene config ${label}.${field}[${index}] must be a non-empty string.`);
|
|
325
|
+
if (isAbsolutePattern(pattern)) throw new TypeError(`Midscene config ${label}.${field}[${index}] must be relative to the project root.`);
|
|
326
|
+
if (pattern.split(/[\\/]/).includes('..')) throw new TypeError(`Midscene config ${label}.${field}[${index}] must not contain a ".." path segment.`);
|
|
327
|
+
if (pattern.includes('\\')) throw new TypeError(`Midscene config ${label}.${field}[${index}] must use POSIX path separators (/).`);
|
|
328
|
+
if (pattern.startsWith('!')) {
|
|
329
|
+
if ('include' === field) throw new TypeError(`Midscene config ${label}.include[${index}] must not be a negated pattern. Use files.exclude instead.`);
|
|
330
|
+
throw new TypeError(`Midscene config ${label}.exclude[${index}] must not be a negated pattern.`);
|
|
331
|
+
}
|
|
332
|
+
return pattern;
|
|
333
|
+
});
|
|
334
|
+
};
|
|
335
|
+
const validateTestFileSelection = (value, label = 'files')=>{
|
|
336
|
+
if (void 0 === value) return;
|
|
337
|
+
if (!isRecord(value)) throw new TypeError(`Midscene config ${label} must be an object.`);
|
|
338
|
+
rejectUnknownKeys(value, [
|
|
339
|
+
'include',
|
|
340
|
+
'exclude'
|
|
341
|
+
], label);
|
|
342
|
+
const include = validatePatterns(value.include, 'include', label);
|
|
343
|
+
const exclude = void 0 === value.exclude ? void 0 : validatePatterns(value.exclude, 'exclude', label);
|
|
344
|
+
return Object.freeze({
|
|
345
|
+
include: Object.freeze(include),
|
|
346
|
+
...exclude ? {
|
|
347
|
+
exclude: Object.freeze(exclude)
|
|
348
|
+
} : {}
|
|
349
|
+
});
|
|
350
|
+
};
|
|
351
|
+
const validateTagList = (value, field, label)=>{
|
|
352
|
+
if (void 0 === value) return [];
|
|
353
|
+
if (!Array.isArray(value)) throw new TypeError(`Midscene config ${label}.${field} must be an array.`);
|
|
354
|
+
return value.map((tag, index)=>{
|
|
355
|
+
if ('string' != typeof tag || 0 === tag.trim().length) throw new TypeError(`Midscene config ${label}.${field}[${index}] must be a non-empty string.`);
|
|
356
|
+
return tag;
|
|
357
|
+
});
|
|
358
|
+
};
|
|
359
|
+
const validateTagSelection = (value, label)=>{
|
|
360
|
+
if (void 0 === value) return Object.freeze({
|
|
361
|
+
include: [],
|
|
362
|
+
exclude: []
|
|
363
|
+
});
|
|
364
|
+
if (!isRecord(value)) throw new TypeError(`Midscene config ${label} must be an object.`);
|
|
365
|
+
return Object.freeze({
|
|
366
|
+
include: Object.freeze(validateTagList(value.include, 'include', label)),
|
|
367
|
+
exclude: Object.freeze(validateTagList(value.exclude, 'exclude', label))
|
|
368
|
+
});
|
|
369
|
+
};
|
|
370
|
+
function assertJsonValue(value, path, seen) {
|
|
371
|
+
if (null === value || 'string' == typeof value || 'boolean' == typeof value) return;
|
|
372
|
+
if ('number' == typeof value) {
|
|
373
|
+
if (Number.isFinite(value)) return;
|
|
374
|
+
throw new TypeError(`Midscene config ${path} must be JSON-compatible.`);
|
|
375
|
+
}
|
|
376
|
+
if ('object' != typeof value || void 0 === value) throw new TypeError(`Midscene config ${path} must be JSON-compatible.`);
|
|
377
|
+
if (!Array.isArray(value) && Object.getPrototypeOf(value) !== Object.prototype && null !== Object.getPrototypeOf(value)) throw new TypeError(`Midscene config ${path} must be JSON-compatible.`);
|
|
378
|
+
if (seen.has(value)) throw new TypeError(`Midscene config ${path} must not contain cycles.`);
|
|
379
|
+
seen.add(value);
|
|
380
|
+
if (Array.isArray(value)) value.forEach((item, index)=>assertJsonValue(item, `${path}[${index}]`, seen));
|
|
381
|
+
else for (const [key, child] of Object.entries(value))assertJsonValue(child, `${path}.${key}`, seen);
|
|
382
|
+
seen.delete(value);
|
|
383
|
+
}
|
|
384
|
+
const deepFreezeJson = (value)=>{
|
|
385
|
+
if ('object' == typeof value && null !== value) {
|
|
386
|
+
for (const child of Object.values(value))deepFreezeJson(child);
|
|
387
|
+
Object.freeze(value);
|
|
388
|
+
}
|
|
389
|
+
return value;
|
|
390
|
+
};
|
|
391
|
+
const validateVariables = (value, label)=>{
|
|
392
|
+
if (void 0 === value) return Object.freeze({});
|
|
393
|
+
if (!isRecord(value)) throw new TypeError(`Midscene config ${label} must be an object.`);
|
|
394
|
+
assertJsonValue(value, label, new Set());
|
|
395
|
+
return deepFreezeJson(value);
|
|
396
|
+
};
|
|
397
|
+
const validateSetup = (value, label)=>{
|
|
398
|
+
if (void 0 === value) return;
|
|
399
|
+
if (!isRecord(value)) throw new TypeError(`Midscene config ${label} must be an object.`);
|
|
400
|
+
rejectUnknownKeys(value, [
|
|
401
|
+
'name',
|
|
402
|
+
'setup'
|
|
403
|
+
], label);
|
|
404
|
+
if ('string' != typeof value.name || 0 === value.name.trim().length) throw new TypeError(`Midscene config ${label}.name must be non-empty.`);
|
|
405
|
+
if ('function' != typeof value.setup) throw new TypeError(`Midscene config ${label}.setup must be a function.`);
|
|
406
|
+
return value;
|
|
407
|
+
};
|
|
408
|
+
const validatePositiveInteger = (value, fallback, label)=>{
|
|
409
|
+
if (void 0 === value) return fallback;
|
|
410
|
+
if (!Number.isInteger(value) || value <= 0) throw new TypeError(`Midscene config ${label} must be a positive integer.`);
|
|
411
|
+
return value;
|
|
412
|
+
};
|
|
413
|
+
const validateNonNegativeInteger = (value, fallback, label)=>{
|
|
414
|
+
if (void 0 === value) return fallback;
|
|
415
|
+
if (!Number.isInteger(value) || value < 0) throw new TypeError(`Midscene config ${label} must be a non-negative integer.`);
|
|
416
|
+
return value;
|
|
417
|
+
};
|
|
418
|
+
const validateConcurrency = (value, label)=>{
|
|
419
|
+
if (null === value) return null;
|
|
420
|
+
return validatePositiveInteger(value, 1, label);
|
|
421
|
+
};
|
|
422
|
+
const validateFileConcurrency = (maxConcurrentFiles, fileSetup, label)=>{
|
|
423
|
+
if (maxConcurrentFiles > 1 && !fileSetup) throw new TypeError(`Midscene config ${label}.fileSetup is required when the effective maxConcurrentFiles is greater than 1.`);
|
|
424
|
+
};
|
|
425
|
+
const validateExecutionProjects = (value, defaultSetup, defaultFileSetup, globalNodes, test)=>{
|
|
426
|
+
if (void 0 === value) {
|
|
427
|
+
const setup = validateSetup(defaultSetup, 'setup');
|
|
428
|
+
const fileSetup = validateSetup(defaultFileSetup, 'fileSetup');
|
|
429
|
+
const maxConcurrentFiles = test.maxConcurrentFiles ?? 1 / 0;
|
|
430
|
+
validateFileConcurrency(maxConcurrentFiles, fileSetup, 'default');
|
|
431
|
+
return {
|
|
432
|
+
hasExplicitProjects: false,
|
|
433
|
+
projects: Object.freeze([
|
|
434
|
+
Object.freeze({
|
|
435
|
+
projectId: projectIdFromIndex(0),
|
|
436
|
+
name: 'default',
|
|
437
|
+
...setup ? {
|
|
438
|
+
setup
|
|
439
|
+
} : {},
|
|
440
|
+
...fileSetup ? {
|
|
441
|
+
fileSetup
|
|
442
|
+
} : {},
|
|
443
|
+
maxConcurrentFiles,
|
|
444
|
+
tags: Object.freeze({
|
|
445
|
+
include: [],
|
|
446
|
+
exclude: []
|
|
447
|
+
}),
|
|
448
|
+
retry: 0,
|
|
449
|
+
variables: Object.freeze({}),
|
|
450
|
+
nodes: new NodeRegistry(globalNodes.definitions())
|
|
451
|
+
})
|
|
452
|
+
])
|
|
453
|
+
};
|
|
454
|
+
}
|
|
455
|
+
if (void 0 !== defaultSetup) throw new TypeError('Midscene config setup cannot be used together with projects. Move setup to projects[].setup.');
|
|
456
|
+
if (void 0 !== defaultFileSetup) throw new TypeError('Midscene config fileSetup cannot be used together with projects. Move fileSetup to projects[].fileSetup.');
|
|
457
|
+
if (!Array.isArray(value) || 0 === value.length) throw new TypeError('Midscene config projects must be a non-empty array when declared.');
|
|
458
|
+
const names = new Set();
|
|
459
|
+
const projects = value.map((candidate, index)=>{
|
|
460
|
+
const label = `projects[${index}]`;
|
|
461
|
+
if (!isRecord(candidate)) throw new TypeError(`Midscene config ${label} must be an object.`);
|
|
462
|
+
rejectUnknownKeys(candidate, [
|
|
463
|
+
'name',
|
|
464
|
+
'setup',
|
|
465
|
+
'fileSetup',
|
|
466
|
+
'maxConcurrentFiles',
|
|
467
|
+
'nodes',
|
|
468
|
+
'files',
|
|
469
|
+
'tags',
|
|
470
|
+
'retry',
|
|
471
|
+
'variables'
|
|
472
|
+
], label);
|
|
473
|
+
if ('string' != typeof candidate.name || 0 === candidate.name.trim().length) throw new TypeError(`Midscene config ${label}.name must be non-empty.`);
|
|
474
|
+
if (names.has(candidate.name)) throw new TypeError(`Midscene config project name "${candidate.name}" must be unique.`);
|
|
475
|
+
names.add(candidate.name);
|
|
476
|
+
const files = validateTestFileSelection(candidate.files, `${label}.files`);
|
|
477
|
+
if (void 0 !== candidate.nodes && !Array.isArray(candidate.nodes)) throw new TypeError(`Midscene config ${label}.nodes must be an array.`);
|
|
478
|
+
const localNodes = new NodeRegistry(candidate.nodes);
|
|
479
|
+
const nodes = new NodeRegistry([
|
|
480
|
+
...globalNodes.definitions().filter((node)=>!localNodes.has(node.name)),
|
|
481
|
+
...localNodes.definitions()
|
|
482
|
+
]);
|
|
483
|
+
const fileSetup = validateSetup(candidate.fileSetup, `${label}.fileSetup`);
|
|
484
|
+
const globalFileLimit = test.maxConcurrentFiles ?? 1 / 0;
|
|
485
|
+
const maxConcurrentFiles = Math.min(globalFileLimit, validatePositiveInteger(candidate.maxConcurrentFiles, globalFileLimit, `${label}.maxConcurrentFiles`));
|
|
486
|
+
validateFileConcurrency(maxConcurrentFiles, fileSetup, label);
|
|
487
|
+
return Object.freeze({
|
|
488
|
+
projectId: projectIdFromIndex(index),
|
|
489
|
+
name: candidate.name,
|
|
490
|
+
nodes,
|
|
491
|
+
...fileSetup ? {
|
|
492
|
+
fileSetup
|
|
493
|
+
} : {},
|
|
494
|
+
maxConcurrentFiles,
|
|
495
|
+
...files ? {
|
|
496
|
+
files
|
|
497
|
+
} : {},
|
|
498
|
+
tags: validateTagSelection(candidate.tags, `${label}.tags`),
|
|
499
|
+
retry: validateNonNegativeInteger(candidate.retry, 0, `${label}.retry`),
|
|
500
|
+
variables: validateVariables(candidate.variables, `${label}.variables`),
|
|
501
|
+
...void 0 === candidate.setup ? {} : {
|
|
502
|
+
setup: validateSetup(candidate.setup, `${label}.setup`)
|
|
503
|
+
}
|
|
504
|
+
});
|
|
505
|
+
});
|
|
506
|
+
return {
|
|
507
|
+
projects: Object.freeze(projects),
|
|
508
|
+
hasExplicitProjects: true
|
|
509
|
+
};
|
|
510
|
+
};
|
|
511
|
+
const validateTestOptions = (value, hasExplicitProjects)=>{
|
|
512
|
+
if (void 0 !== value && !isRecord(value)) throw new TypeError('Midscene config test must be an object.');
|
|
513
|
+
const candidate = value ?? {};
|
|
514
|
+
if (void 0 !== candidate.maxConcurrency) {
|
|
515
|
+
const limit = 'number' == typeof candidate.maxConcurrency && Number.isInteger(candidate.maxConcurrency) && candidate.maxConcurrency > 0 ? candidate.maxConcurrency : 1;
|
|
516
|
+
throw new TypeError([
|
|
517
|
+
'Midscene config test.maxConcurrency is no longer supported.',
|
|
518
|
+
'It controlled project concurrency, not file concurrency. Remove it and use:',
|
|
519
|
+
` test: { maxConcurrentProjects: ${limit}, maxConcurrentFiles: ${hasExplicitProjects ? limit : 1} }`,
|
|
520
|
+
hasExplicitProjects ? ' Set projects[].maxConcurrentFiles to 1 to keep files sequential within each project.' : ' The implicit default project keeps running files sequentially.',
|
|
521
|
+
hasExplicitProjects ? 'For parallel files, configure fileSetup and raise both test.maxConcurrentFiles and projects[].maxConcurrentFiles.' : 'For parallel files, configure fileSetup and raise test.maxConcurrentFiles.',
|
|
522
|
+
'Migration guide: https://midscenejs.com/midscene-test/configuration.html#existing-configurations'
|
|
523
|
+
].join('\n'));
|
|
524
|
+
}
|
|
525
|
+
return Object.freeze({
|
|
526
|
+
maxConcurrentFiles: validateConcurrency(candidate.maxConcurrentFiles, 'test.maxConcurrentFiles'),
|
|
527
|
+
maxConcurrentProjects: validateConcurrency(candidate.maxConcurrentProjects, 'test.maxConcurrentProjects'),
|
|
528
|
+
setupTimeout: validatePositiveInteger(candidate.setupTimeout, 120000, 'test.setupTimeout'),
|
|
529
|
+
teardownTimeout: validatePositiveInteger(candidate.teardownTimeout, 30000, 'test.teardownTimeout'),
|
|
530
|
+
bail: validateNonNegativeInteger(candidate.bail, 0, 'test.bail'),
|
|
531
|
+
testTimeout: validatePositiveInteger(candidate.testTimeout, 120000, 'test.testTimeout')
|
|
532
|
+
});
|
|
533
|
+
};
|
|
534
|
+
const validateOutputPath = (value, label)=>{
|
|
535
|
+
if ('string' != typeof value || 0 === value.trim().length) throw new TypeError(`Midscene config ${label} must be non-empty.`);
|
|
536
|
+
return value;
|
|
537
|
+
};
|
|
538
|
+
const validateOutput = (value)=>{
|
|
539
|
+
if (void 0 !== value && !isRecord(value)) throw new TypeError('Midscene config output must be an object.');
|
|
540
|
+
const candidate = value ?? {};
|
|
541
|
+
rejectUnknownKeys(candidate, [
|
|
542
|
+
'reportDir',
|
|
543
|
+
'reportFormat'
|
|
544
|
+
], 'output');
|
|
545
|
+
if (void 0 !== candidate.reportFormat && 'single-html' !== candidate.reportFormat && 'html-and-external-assets' !== candidate.reportFormat) throw new TypeError('Midscene config output.reportFormat must be single-html or html-and-external-assets.');
|
|
546
|
+
return Object.freeze({
|
|
547
|
+
reportFormat: candidate.reportFormat ?? 'single-html',
|
|
548
|
+
...void 0 === candidate.reportDir ? {} : {
|
|
549
|
+
reportDir: validateOutputPath(candidate.reportDir, 'output.reportDir')
|
|
550
|
+
}
|
|
551
|
+
});
|
|
552
|
+
};
|
|
553
|
+
const validateTestProjectDefinition = (definition)=>{
|
|
554
|
+
if (!isRecord(definition)) throw new TypeError('Midscene config must default export an object.');
|
|
555
|
+
if (void 0 !== definition.nodes && !Array.isArray(definition.nodes)) throw new TypeError('Midscene config nodes must be an array.');
|
|
556
|
+
if ('setupWorkflow' in definition) throw new TypeError('Midscene config setupWorkflow is no longer supported. Use setup or projects[].setup instead.');
|
|
557
|
+
if ('setupDocument' in definition) throw new TypeError('Midscene config setupDocument is not supported. Use setup or projects[].setup for resources, and beforeAll/afterAll for document lifecycle steps.');
|
|
558
|
+
if ('root' in definition) throw new TypeError('Midscene config root is not supported. Run midscene-test from the project directory or select its config with --config instead.');
|
|
559
|
+
if ('files' in definition) throw new TypeError('Midscene config files is not supported at the root. Move it to projects[].files.');
|
|
560
|
+
if ('testRunner' in definition) throw new TypeError('Midscene config testRunner is not supported. Rename it to test.');
|
|
561
|
+
rejectUnknownKeys(definition, [
|
|
562
|
+
'setup',
|
|
563
|
+
'fileSetup',
|
|
564
|
+
'projects',
|
|
565
|
+
'test',
|
|
566
|
+
'output',
|
|
567
|
+
'nodes'
|
|
568
|
+
], 'root');
|
|
569
|
+
const nodes = new NodeRegistry(definition.nodes);
|
|
570
|
+
const test = validateTestOptions(definition.test, void 0 !== definition.projects);
|
|
571
|
+
const resolvedProjects = validateExecutionProjects(definition.projects, definition.setup, definition.fileSetup, nodes, test);
|
|
572
|
+
return {
|
|
573
|
+
...resolvedProjects,
|
|
574
|
+
test,
|
|
575
|
+
output: validateOutput(definition.output),
|
|
576
|
+
nodes,
|
|
577
|
+
resolveNode: (name)=>nodes.get(name)
|
|
578
|
+
};
|
|
579
|
+
};
|
|
580
|
+
const assertSupportedConfig = (absolutePath)=>{
|
|
581
|
+
if (!absolutePath.endsWith('.ts') && !absolutePath.endsWith('.mjs')) {
|
|
582
|
+
const extension = absolutePath.match(/(\.[^./\\]+)$/)?.[1] ?? '(none)';
|
|
583
|
+
throw new TypeError(`Unsupported Midscene config extension: ${extension}. Supported extensions: .ts, .mjs.`);
|
|
584
|
+
}
|
|
585
|
+
return absolutePath.endsWith('.mjs') ? '.mjs' : '.ts';
|
|
586
|
+
};
|
|
587
|
+
const canRetryWithCjsLoader = (error)=>{
|
|
588
|
+
if (!(error instanceof Error)) return false;
|
|
589
|
+
const errorCode = error.code;
|
|
590
|
+
return error instanceof SyntaxError && /^Unexpected (?:identifier|reserved word|token)/.test(error.message) || 'ERR_UNSUPPORTED_TYPESCRIPT_SYNTAX' === errorCode || 'ERR_UNKNOWN_FILE_EXTENSION' === errorCode && /^Unknown file extension\b/.test(error.message);
|
|
591
|
+
};
|
|
592
|
+
async function loadTestProject(configPath) {
|
|
593
|
+
if (!configPath) return validateTestProjectDefinition({
|
|
594
|
+
nodes: []
|
|
595
|
+
});
|
|
596
|
+
const absolutePath = external_node_path_resolve(configPath);
|
|
597
|
+
const extension = assertSupportedConfig(absolutePath);
|
|
598
|
+
let loaded;
|
|
599
|
+
try {
|
|
600
|
+
if ('.mjs' === extension) loaded = await import(pathToFileURL(absolutePath).href);
|
|
601
|
+
else {
|
|
602
|
+
const { tsImport } = await import("tsx/esm/api");
|
|
603
|
+
loaded = await tsImport(pathToFileURL(absolutePath).href, {
|
|
604
|
+
parentURL: pathToFileURL(`${dirname(absolutePath)}${sep}`).href,
|
|
605
|
+
tsconfig: false
|
|
606
|
+
});
|
|
607
|
+
}
|
|
608
|
+
} catch (error) {
|
|
609
|
+
try {
|
|
610
|
+
if ('.mjs' === extension || !canRetryWithCjsLoader(error)) throw error;
|
|
611
|
+
loaded = api_require(absolutePath, pathToFileURL(absolutePath));
|
|
612
|
+
} catch (fallbackError) {
|
|
613
|
+
const message = fallbackError instanceof Error ? `: ${fallbackError.message}` : '';
|
|
614
|
+
throw new Error(`Failed to load Midscene config "${absolutePath}"${message}`, {
|
|
615
|
+
cause: fallbackError
|
|
616
|
+
});
|
|
617
|
+
}
|
|
618
|
+
}
|
|
619
|
+
return validateTestProjectDefinition(getDefaultExport(loaded, absolutePath));
|
|
620
|
+
}
|
|
621
|
+
const CONFIG_NAMES = [
|
|
622
|
+
'midscene.config.ts',
|
|
623
|
+
'midscene.config.mjs'
|
|
624
|
+
];
|
|
625
|
+
const CONFIG_PREFIX = 'midscene.config.';
|
|
626
|
+
const ALWAYS_IGNORED_PATTERNS = [
|
|
627
|
+
'.git/**',
|
|
628
|
+
'.midscene/**',
|
|
629
|
+
'midscene_run/**',
|
|
630
|
+
'node_modules/**',
|
|
631
|
+
'**/.git/**',
|
|
632
|
+
'**/.midscene/**',
|
|
633
|
+
'**/midscene_run/**',
|
|
634
|
+
'**/node_modules/**'
|
|
635
|
+
];
|
|
636
|
+
const DEFAULT_TEST_FILE_SELECTION = {
|
|
637
|
+
include: [
|
|
638
|
+
'**/*.{yaml,yml}'
|
|
639
|
+
]
|
|
640
|
+
};
|
|
641
|
+
const project_preparation_toPosix = (value)=>value.split(sep).join('/');
|
|
642
|
+
const discoverResolvedTestFiles = (projectRoot, selection, excludedDirectories = [])=>{
|
|
643
|
+
const root = external_node_path_resolve(projectRoot);
|
|
644
|
+
const outputPatterns = excludedDirectories.flatMap((directory)=>{
|
|
645
|
+
const withinRoot = relative(root, directory);
|
|
646
|
+
if (!withinRoot || isAbsolute(withinRoot) || '..' === withinRoot || withinRoot.startsWith(`..${sep}`)) return [];
|
|
647
|
+
const literal = project_preparation_toPosix(withinRoot).replace(/([*?{}[\]()!])/g, '\\$1');
|
|
648
|
+
return [
|
|
649
|
+
`${literal}/**`
|
|
650
|
+
];
|
|
651
|
+
});
|
|
652
|
+
const match = (patterns)=>globSync(patterns, {
|
|
653
|
+
absolute: true,
|
|
654
|
+
caseSensitiveMatch: false,
|
|
655
|
+
cwd: root,
|
|
656
|
+
dot: true,
|
|
657
|
+
expandDirectories: false,
|
|
658
|
+
followSymbolicLinks: false,
|
|
659
|
+
ignore: [
|
|
660
|
+
...ALWAYS_IGNORED_PATTERNS,
|
|
661
|
+
...outputPatterns,
|
|
662
|
+
...selection.exclude ?? []
|
|
663
|
+
],
|
|
664
|
+
onlyFiles: true
|
|
665
|
+
}).filter((file)=>/\.ya?ml$/i.test(file));
|
|
666
|
+
const files = match(selection.include);
|
|
667
|
+
return [
|
|
668
|
+
...new Set(files.map((file)=>external_node_path_resolve(file)))
|
|
669
|
+
].sort((a, b)=>{
|
|
670
|
+
const relativeA = project_preparation_toPosix(relative(root, a));
|
|
671
|
+
const relativeB = project_preparation_toPosix(relative(root, b));
|
|
672
|
+
return relativeA < relativeB ? -1 : relativeA > relativeB ? 1 : 0;
|
|
673
|
+
});
|
|
674
|
+
};
|
|
675
|
+
const discoverTestFiles = (projectRoot, selection = DEFAULT_TEST_FILE_SELECTION)=>{
|
|
676
|
+
const normalized = validateTestFileSelection(selection);
|
|
677
|
+
if (!normalized) throw new TypeError('Test file selection is required.');
|
|
678
|
+
return discoverResolvedTestFiles(projectRoot, normalized);
|
|
679
|
+
};
|
|
680
|
+
const discoverTestConfig = (projectRoot)=>{
|
|
681
|
+
const root = external_node_path_resolve(projectRoot);
|
|
682
|
+
const candidates = readdirSync(root).filter((name)=>name.startsWith(CONFIG_PREFIX)).sort();
|
|
683
|
+
const supported = candidates.filter((name)=>CONFIG_NAMES.includes(name));
|
|
684
|
+
const unsupported = candidates.filter((name)=>!CONFIG_NAMES.includes(name));
|
|
685
|
+
if (unsupported.length > 0) throw new Error([
|
|
686
|
+
`Unsupported or conflicting Midscene configs found in ${root}:`,
|
|
687
|
+
...candidates.map((name)=>`- ${name}`),
|
|
688
|
+
`Only ${CONFIG_NAMES.join(' and ')} are supported.`
|
|
689
|
+
].join('\n'));
|
|
690
|
+
if (supported.length > 1) throw new Error([
|
|
691
|
+
`Multiple Midscene configs found in ${root}:`,
|
|
692
|
+
...supported.map((name)=>`- ${name}`),
|
|
693
|
+
'Pass --config <path> to select one explicitly.'
|
|
694
|
+
].join('\n'));
|
|
695
|
+
return supported[0] ? join(root, supported[0]) : void 0;
|
|
696
|
+
};
|
|
697
|
+
const asCollectionError = (projectId, projectName, sourcePath, error)=>({
|
|
698
|
+
projectId,
|
|
699
|
+
projectName,
|
|
700
|
+
sourcePath,
|
|
701
|
+
error: error instanceof WorkflowError ? error : new WorkflowParseError(`Failed to collect workflow document "${sourcePath}": ${error instanceof Error ? error.message : String(error)}`, {
|
|
702
|
+
projectName,
|
|
703
|
+
sourcePath
|
|
704
|
+
}, error)
|
|
705
|
+
});
|
|
706
|
+
const matchesTags = (tags, selection)=>{
|
|
707
|
+
if (selection.exclude.some((tag)=>tags.includes(tag))) return false;
|
|
708
|
+
return 0 === selection.include.length || selection.include.some((tag)=>tags.includes(tag));
|
|
709
|
+
};
|
|
710
|
+
const filterDocumentCases = (document, project)=>{
|
|
711
|
+
const cases = document.cases.filter((item)=>matchesTags(item.definition.tags ?? [], project.tags));
|
|
712
|
+
const filtered = document.cases.length - cases.length;
|
|
713
|
+
return 0 === cases.length ? {
|
|
714
|
+
filtered
|
|
715
|
+
} : {
|
|
716
|
+
document: {
|
|
717
|
+
...document,
|
|
718
|
+
cases
|
|
719
|
+
},
|
|
720
|
+
filtered
|
|
721
|
+
};
|
|
722
|
+
};
|
|
723
|
+
const selectProjects = (projects, names)=>{
|
|
724
|
+
if (!names || 0 === names.length) return projects;
|
|
725
|
+
const requested = new Set(names);
|
|
726
|
+
if (requested.size !== names.length) throw new Error('Each --project name may only be specified once.');
|
|
727
|
+
const known = new Set(projects.map((project)=>project.name));
|
|
728
|
+
const unknown = names.find((name)=>!known.has(name));
|
|
729
|
+
if (unknown) throw new Error(`Unknown Midscene project: ${unknown}`);
|
|
730
|
+
return projects.filter((project)=>requested.has(project.name));
|
|
731
|
+
};
|
|
732
|
+
async function prepareProject(project, projectRoot, runDir, defaultTimeoutMs, options = {}) {
|
|
733
|
+
const platform = options.platform?.trim();
|
|
734
|
+
const fileSelection = project.files ?? DEFAULT_TEST_FILE_SELECTION;
|
|
735
|
+
const prerequisiteFile = options.prerequisiteFile ? external_node_path_resolve(options.prerequisiteFile) : void 0;
|
|
736
|
+
const discoveredFiles = options.files ?? discoverResolvedTestFiles(options.discoveryRoot ?? projectRoot, fileSelection, options.excludedDirectories);
|
|
737
|
+
const selectedPath = options.testPath ? realpathSync(options.testPath) : void 0;
|
|
738
|
+
const mainFiles = selectedPath ? discoveredFiles.filter((file)=>{
|
|
739
|
+
const within = relative(selectedPath, realpathSync(file));
|
|
740
|
+
return '' === within || !isAbsolute(within) && '..' !== within && !within.startsWith(`..${sep}`);
|
|
741
|
+
}) : discoveredFiles;
|
|
742
|
+
const files = [
|
|
743
|
+
...prerequisiteFile ? [
|
|
744
|
+
prerequisiteFile
|
|
745
|
+
] : [],
|
|
746
|
+
...mainFiles.filter((file)=>file !== prerequisiteFile)
|
|
747
|
+
];
|
|
748
|
+
const occurrences = new Map();
|
|
749
|
+
const sources = files.map((absolutePath)=>{
|
|
750
|
+
const invocationIndex = occurrences.get(absolutePath) ?? 0;
|
|
751
|
+
occurrences.set(absolutePath, invocationIndex + 1);
|
|
752
|
+
return {
|
|
753
|
+
projectId: project.projectId,
|
|
754
|
+
projectName: project.name,
|
|
755
|
+
sourcePath: project_preparation_toPosix(relative(projectRoot, absolutePath)),
|
|
756
|
+
absolutePath,
|
|
757
|
+
invocationIndex
|
|
758
|
+
};
|
|
759
|
+
});
|
|
760
|
+
const collectionErrors = [];
|
|
761
|
+
const invocations = [];
|
|
762
|
+
const plans = [];
|
|
763
|
+
let filteredCaseCount = 0;
|
|
764
|
+
if (0 === sources.length && !options.testPath) {
|
|
765
|
+
const error = asCollectionError(project.projectId, project.name, '<project>', new WorkflowParseError(`No workflow YAML files found for project "${project.name}" in ${projectRoot}.`, {
|
|
766
|
+
projectName: project.name,
|
|
767
|
+
projectRoot
|
|
768
|
+
}));
|
|
769
|
+
collectionErrors.push(error);
|
|
770
|
+
await writeCollectionError(runDir, error);
|
|
771
|
+
}
|
|
772
|
+
for (const source of sources)try {
|
|
773
|
+
const collected = await options.collectDocument?.(source, project);
|
|
774
|
+
if (collected) {
|
|
775
|
+
filteredCaseCount += collected.filteredCaseCount;
|
|
776
|
+
if (collected.invocation) {
|
|
777
|
+
const plan = buildTestRunReportPlan(collected.invocation.document, {
|
|
778
|
+
defaultTimeoutMs: collected.invocation.defaultTimeoutMs,
|
|
779
|
+
retry: collected.invocation.retry
|
|
780
|
+
});
|
|
781
|
+
invocations.push({
|
|
782
|
+
...collected.invocation,
|
|
783
|
+
plan
|
|
784
|
+
});
|
|
785
|
+
plans.push(plan);
|
|
786
|
+
}
|
|
787
|
+
continue;
|
|
788
|
+
}
|
|
789
|
+
const document = collectWorkflowDocument(source, {
|
|
790
|
+
resolveNode: project.nodes.get.bind(project.nodes),
|
|
791
|
+
variables: project.variables,
|
|
792
|
+
env: process.env
|
|
793
|
+
});
|
|
794
|
+
const filtered = source.absolutePath === prerequisiteFile ? {
|
|
795
|
+
document,
|
|
796
|
+
filtered: 0
|
|
797
|
+
} : filterDocumentCases(document, project);
|
|
798
|
+
const plan = buildTestRunReportPlan(document, {
|
|
799
|
+
selectedCaseIds: filtered.document?.cases.map((item)=>item.caseId) ?? [],
|
|
800
|
+
defaultTimeoutMs,
|
|
801
|
+
retry: {
|
|
802
|
+
scope: 'case',
|
|
803
|
+
count: project.retry
|
|
804
|
+
}
|
|
805
|
+
});
|
|
806
|
+
plans.push(plan);
|
|
807
|
+
filteredCaseCount += filtered.filtered;
|
|
808
|
+
if (filtered.document) invocations.push({
|
|
809
|
+
plan,
|
|
810
|
+
document: filtered.document,
|
|
811
|
+
retry: {
|
|
812
|
+
scope: 'case',
|
|
813
|
+
count: project.retry
|
|
814
|
+
},
|
|
815
|
+
defaultTimeoutMs,
|
|
816
|
+
reportEnabled: true,
|
|
817
|
+
bindings: {
|
|
818
|
+
resolveNode: project.nodes.require.bind(project.nodes),
|
|
819
|
+
...project.fileSetup ? {
|
|
820
|
+
documentSetup: {
|
|
821
|
+
name: project.fileSetup.name,
|
|
822
|
+
setup ({ projectContext, document, signal, onTeardown }) {
|
|
823
|
+
return project.fileSetup.setup({
|
|
824
|
+
project,
|
|
825
|
+
projectContext,
|
|
826
|
+
file: document,
|
|
827
|
+
env: Object.freeze({
|
|
828
|
+
...process.env
|
|
829
|
+
}),
|
|
830
|
+
signal,
|
|
831
|
+
onTeardown
|
|
832
|
+
});
|
|
833
|
+
}
|
|
834
|
+
}
|
|
835
|
+
} : {}
|
|
836
|
+
}
|
|
837
|
+
});
|
|
838
|
+
} catch (error) {
|
|
839
|
+
const collectionError = asCollectionError(project.projectId, project.name, source.sourcePath, error);
|
|
840
|
+
collectionErrors.push(collectionError);
|
|
841
|
+
await writeCollectionError(runDir, collectionError);
|
|
842
|
+
}
|
|
843
|
+
const documents = invocations.map((invocation)=>invocation.document);
|
|
844
|
+
const prerequisiteSource = sources.find((source)=>source.absolutePath === prerequisiteFile);
|
|
845
|
+
const prerequisiteDocument = documents.find((document)=>document.sourcePath === prerequisiteSource?.sourcePath);
|
|
846
|
+
return {
|
|
847
|
+
project,
|
|
848
|
+
...platform ? {
|
|
849
|
+
platform
|
|
850
|
+
} : {},
|
|
851
|
+
documentConcurrency: options.documentConcurrency ?? 1,
|
|
852
|
+
...prerequisiteDocument ? {
|
|
853
|
+
prerequisiteDocumentId: prerequisiteDocument.documentId
|
|
854
|
+
} : {},
|
|
855
|
+
fileSelection,
|
|
856
|
+
sources,
|
|
857
|
+
documents,
|
|
858
|
+
invocations,
|
|
859
|
+
plans,
|
|
860
|
+
collectionErrors,
|
|
861
|
+
selectedCaseCount: documents.reduce((total, document)=>total + document.cases.length, 0),
|
|
862
|
+
filteredCaseCount
|
|
863
|
+
};
|
|
864
|
+
}
|
|
865
|
+
const padDatePart = (value)=>String(value).padStart(2, '0');
|
|
866
|
+
const createTestRunId = (date = new Date(), uuid = randomUUID())=>{
|
|
867
|
+
const timestamp = [
|
|
868
|
+
date.getFullYear(),
|
|
869
|
+
padDatePart(date.getMonth() + 1),
|
|
870
|
+
padDatePart(date.getDate()),
|
|
871
|
+
padDatePart(date.getHours()),
|
|
872
|
+
padDatePart(date.getMinutes()),
|
|
873
|
+
padDatePart(date.getSeconds())
|
|
874
|
+
].join('');
|
|
875
|
+
return `${timestamp}-${uuid.slice(0, 8)}`;
|
|
876
|
+
};
|
|
877
|
+
const assertDirectory = (path, label)=>{
|
|
878
|
+
if (!existsSync(path) || !statSync(path).isDirectory()) throw new Error(`${label} does not exist or is not a directory: ${path}`);
|
|
879
|
+
};
|
|
880
|
+
const resolveRunInput = (options)=>{
|
|
881
|
+
const cwd = external_node_path_resolve(options.cwd ?? process.cwd());
|
|
882
|
+
assertDirectory(cwd, 'Test working directory');
|
|
883
|
+
const searchRoot = external_node_path_resolve(cwd, options.projectRoot ?? '.');
|
|
884
|
+
assertDirectory(searchRoot, 'Test project directory');
|
|
885
|
+
const configPath = options.configPath ? external_node_path_resolve(cwd, options.configPath) : discoverTestConfig(searchRoot);
|
|
886
|
+
if (options.configPath && /\.ya?ml$/i.test(configPath)) throw new Error(`midscene-test --config only accepts a native TypeScript or JavaScript project config: ${configPath}. Run legacy batch YAML with \`midscene --config ${options.configPath}\`.`);
|
|
887
|
+
if (configPath && (!existsSync(configPath) || !statSync(configPath).isFile())) throw new Error(`Midscene config does not exist or is not a file: ${configPath}`);
|
|
888
|
+
const testPath = options.testPath ? external_node_path_resolve(cwd, options.testPath) : void 0;
|
|
889
|
+
if (testPath) {
|
|
890
|
+
if (!existsSync(testPath)) throw new Error(`Test path does not exist: ${testPath}`);
|
|
891
|
+
const stat = statSync(testPath);
|
|
892
|
+
if (!stat.isDirectory() && !(stat.isFile() && /\.ya?ml$/i.test(testPath))) throw new Error(`Test input must be a YAML file or directory: ${testPath}`);
|
|
893
|
+
}
|
|
894
|
+
return {
|
|
895
|
+
cwd,
|
|
896
|
+
projectRoot: configPath ? dirname(configPath) : searchRoot,
|
|
897
|
+
...configPath ? {
|
|
898
|
+
configPath
|
|
899
|
+
} : {},
|
|
900
|
+
...testPath ? {
|
|
901
|
+
testPath
|
|
902
|
+
} : {}
|
|
903
|
+
};
|
|
904
|
+
};
|
|
905
|
+
async function prepareTestRun(options) {
|
|
906
|
+
const startedAt = new Date();
|
|
907
|
+
const runId = createTestRunId(startedAt);
|
|
908
|
+
const input = resolveRunInput(options);
|
|
909
|
+
const { configPath } = input;
|
|
910
|
+
const standalonePath = configPath ? void 0 : input.testPath;
|
|
911
|
+
const discovery = standalonePath && statSync(standalonePath).isFile() ? {
|
|
912
|
+
files: [
|
|
913
|
+
standalonePath
|
|
914
|
+
]
|
|
915
|
+
} : {
|
|
916
|
+
discoveryRoot: standalonePath
|
|
917
|
+
};
|
|
918
|
+
const definition = await loadTestProject(configPath);
|
|
919
|
+
const artifactDir = resolveMidsceneRunDir(input.projectRoot);
|
|
920
|
+
return runWithMidsceneRunDir(artifactDir, async ()=>{
|
|
921
|
+
const resultDir = options.resultDir ? external_node_path_resolve(input.cwd, options.resultDir) : join(artifactDir, 'test-results');
|
|
922
|
+
const runDir = join(resultDir, runId);
|
|
923
|
+
const summaryPath = join(runDir, 'summary.json');
|
|
924
|
+
const reportDir = definition.output.reportDir ? external_node_path_resolve(input.projectRoot, definition.output.reportDir) : join(artifactDir, 'report');
|
|
925
|
+
mkdirSync(resultDir, {
|
|
926
|
+
recursive: true
|
|
927
|
+
});
|
|
928
|
+
mkdirSync(runDir);
|
|
929
|
+
const projects = [];
|
|
930
|
+
for (const project of selectProjects(definition.projects, options.projectNames))projects.push(await prepareProject(project, input.projectRoot, runDir, definition.test.testTimeout, {
|
|
931
|
+
testPath: input.testPath,
|
|
932
|
+
...discovery,
|
|
933
|
+
excludedDirectories: [
|
|
934
|
+
artifactDir,
|
|
935
|
+
resultDir,
|
|
936
|
+
reportDir,
|
|
937
|
+
...[
|
|
938
|
+
'cache',
|
|
939
|
+
'log',
|
|
940
|
+
'output',
|
|
941
|
+
'tmp',
|
|
942
|
+
'dump'
|
|
943
|
+
].map((name)=>join(artifactDir, name))
|
|
944
|
+
]
|
|
945
|
+
}));
|
|
946
|
+
if (input.testPath && projects.every((project)=>0 === project.selectedCaseCount && 0 === project.collectionErrors.length)) throw new Error(`No test cases match "${input.testPath}" in ${input.projectRoot}. Check --project, files.include/files.exclude, tags, and ignored output directories.`);
|
|
947
|
+
const hasFormatMismatch = projects.some((project)=>project.collectionErrors.some(({ error })=>error instanceof NativeWorkflowFormatError));
|
|
948
|
+
return {
|
|
949
|
+
startedAt,
|
|
950
|
+
runId,
|
|
951
|
+
projectRoot: input.projectRoot,
|
|
952
|
+
artifactDir,
|
|
953
|
+
...configPath ? {
|
|
954
|
+
configPath
|
|
955
|
+
} : {},
|
|
956
|
+
resultDir,
|
|
957
|
+
runDir,
|
|
958
|
+
summaryPath,
|
|
959
|
+
reportDir,
|
|
960
|
+
definition,
|
|
961
|
+
projects,
|
|
962
|
+
preflightScope: hasFormatMismatch ? 'run' : 'project',
|
|
963
|
+
reportEnabled: true,
|
|
964
|
+
publications: [],
|
|
965
|
+
onProgress: options.onProgress ?? (()=>{}),
|
|
966
|
+
onEvent: options.onEvent
|
|
967
|
+
};
|
|
968
|
+
});
|
|
969
|
+
}
|
|
970
|
+
function scopeNodeConsole(node) {
|
|
971
|
+
return {
|
|
972
|
+
...node,
|
|
973
|
+
execute (ctx) {
|
|
974
|
+
const position = 'case' === ctx.scope ? ctx.case : ctx.document;
|
|
975
|
+
const scope = {
|
|
976
|
+
projectName: position.projectName,
|
|
977
|
+
sourcePath: position.sourcePath,
|
|
978
|
+
...'case' === ctx.scope ? {
|
|
979
|
+
caseId: ctx.case.caseId,
|
|
980
|
+
caseName: ctx.case.name,
|
|
981
|
+
runId: ctx.case.runId
|
|
982
|
+
} : {
|
|
983
|
+
runId: ctx.document.documentRunId
|
|
984
|
+
},
|
|
985
|
+
phase: position.phase,
|
|
986
|
+
stepIndex: position.stepIndex,
|
|
987
|
+
node: node.name
|
|
988
|
+
};
|
|
989
|
+
return runWithConsoleLogScope(scope, ()=>node.execute({
|
|
990
|
+
...ctx,
|
|
991
|
+
onTeardown (teardown) {
|
|
992
|
+
ctx.onTeardown(()=>runWithConsoleLogScope({
|
|
993
|
+
...scope,
|
|
994
|
+
phase: 'teardown',
|
|
995
|
+
stepIndex: void 0
|
|
996
|
+
}, teardown));
|
|
997
|
+
}
|
|
998
|
+
}));
|
|
999
|
+
}
|
|
1000
|
+
};
|
|
1001
|
+
}
|
|
1002
|
+
const stepPosition = (info)=>'case' === info.scope ? info.case : info.document;
|
|
1003
|
+
const formatStep = (info)=>{
|
|
1004
|
+
const position = stepPosition(info);
|
|
1005
|
+
const phase = 'steps' === position.phase ? 'step' : position.phase;
|
|
1006
|
+
return `${phase} ${position.stepIndex + 1}/${info.stepCount}: ${info.node}`;
|
|
1007
|
+
};
|
|
1008
|
+
const formatStepResult = (info, result)=>{
|
|
1009
|
+
const indent = 'case' === info.scope ? ' ' : ' ';
|
|
1010
|
+
if ('not-run' === result.status) return `${indent}– ${formatStep(info)} (not run: ${result.notRunReason})`;
|
|
1011
|
+
const symbol = 'success' === result.status ? '✓' : '✗';
|
|
1012
|
+
const error = result.error ? ` — ${result.error.message}` : '';
|
|
1013
|
+
const continuation = result.continuedAfterError ? '; continuing' : '';
|
|
1014
|
+
return `${indent}${symbol} ${formatStep(info)} (${result.durationMs} ms)${error}${continuation}`;
|
|
1015
|
+
};
|
|
1016
|
+
const caseHasFatalError = (outcome)=>(outcome.attempts ?? []).some((attempt)=>[
|
|
1017
|
+
...attempt.beforeEach,
|
|
1018
|
+
...attempt.steps,
|
|
1019
|
+
...attempt.afterEach
|
|
1020
|
+
].some((step)=>step.error && isFatalDeviceError(step.error)) || (attempt.teardownErrors ?? []).some(isFatalDeviceError) || !!attempt.executionErrors?.some(isResourceRecoveryError));
|
|
1021
|
+
const documentHasFatalError = (result)=>[
|
|
1022
|
+
...result.beforeAll,
|
|
1023
|
+
...result.afterAll
|
|
1024
|
+
].some((step)=>step.error && isFatalDeviceError(step.error)) || (result.teardownErrors ?? []).some(isFatalDeviceError) || !!result.executionErrors?.some(isResourceRecoveryError);
|
|
1025
|
+
const isSharedResourceFailure = (error)=>{
|
|
1026
|
+
const seen = new Set();
|
|
1027
|
+
const pending = [
|
|
1028
|
+
error
|
|
1029
|
+
];
|
|
1030
|
+
while(pending.length && seen.size < 50){
|
|
1031
|
+
const item = pending.pop();
|
|
1032
|
+
if ('object' != typeof item || null === item || seen.has(item)) continue;
|
|
1033
|
+
seen.add(item);
|
|
1034
|
+
const failure = item;
|
|
1035
|
+
if ('PROJECT_RESOURCE_ERROR' === failure.code) return true;
|
|
1036
|
+
if (failure.cause) pending.push(failure.cause);
|
|
1037
|
+
if (Array.isArray(failure.errors)) pending.push(...failure.errors.slice(0, 50));
|
|
1038
|
+
}
|
|
1039
|
+
return false;
|
|
1040
|
+
};
|
|
1041
|
+
async function executeDocumentInvocation(options) {
|
|
1042
|
+
const { invocation, project, sinks } = options;
|
|
1043
|
+
const isProjectResourceFailure = options.isProjectResourceFailure ?? (()=>false);
|
|
1044
|
+
const { document } = invocation;
|
|
1045
|
+
const documentRetry = 'document' === invocation.retry.scope;
|
|
1046
|
+
const attempts = [];
|
|
1047
|
+
const fileSetup = invocation.bindings.documentSetup;
|
|
1048
|
+
const documentSetup = fileSetup ? {
|
|
1049
|
+
...fileSetup,
|
|
1050
|
+
setup (ctx) {
|
|
1051
|
+
const scope = {
|
|
1052
|
+
projectName: project.name,
|
|
1053
|
+
sourcePath: document.sourcePath,
|
|
1054
|
+
runId: ctx.document.documentRunId,
|
|
1055
|
+
phase: 'fileSetup'
|
|
1056
|
+
};
|
|
1057
|
+
return runWithConsoleLogScope(scope, async ()=>{
|
|
1058
|
+
try {
|
|
1059
|
+
return await fileSetup.setup({
|
|
1060
|
+
...ctx,
|
|
1061
|
+
onTeardown (teardown) {
|
|
1062
|
+
ctx.onTeardown(()=>runWithConsoleLogScope({
|
|
1063
|
+
...scope,
|
|
1064
|
+
phase: 'teardown'
|
|
1065
|
+
}, teardown));
|
|
1066
|
+
}
|
|
1067
|
+
});
|
|
1068
|
+
} catch (error) {
|
|
1069
|
+
if (isProjectResourceFailure(error)) options.markProjectFatal();
|
|
1070
|
+
throw error;
|
|
1071
|
+
}
|
|
1072
|
+
});
|
|
1073
|
+
}
|
|
1074
|
+
} : void 0;
|
|
1075
|
+
options.onEvent?.({
|
|
1076
|
+
type: 'document-start',
|
|
1077
|
+
documentId: document.documentId,
|
|
1078
|
+
projectName: project.name,
|
|
1079
|
+
sourcePath: document.sourcePath,
|
|
1080
|
+
cases: document.cases.length
|
|
1081
|
+
});
|
|
1082
|
+
await runDocumentAttempts({
|
|
1083
|
+
retry: documentRetry ? invocation.retry.count : 0,
|
|
1084
|
+
signal: options.signal,
|
|
1085
|
+
shouldStop: ()=>options.shouldBail() || options.isProjectFatal()
|
|
1086
|
+
}, async (documentAttemptIndex)=>{
|
|
1087
|
+
if (documentRetry) options.onProgress(` file attempt ${documentAttemptIndex + 1}/${invocation.retry.count + 1}: ${document.sourcePath}`);
|
|
1088
|
+
const execution = await runWithConsoleLogScope({
|
|
1089
|
+
projectName: project.name,
|
|
1090
|
+
sourcePath: document.sourcePath
|
|
1091
|
+
}, ()=>runWorkflowDocument(document, {
|
|
1092
|
+
plan: invocation.plan,
|
|
1093
|
+
documentAttemptIndex: documentRetry ? documentAttemptIndex : void 0,
|
|
1094
|
+
...invocation.bindings,
|
|
1095
|
+
resolveNode: (name)=>scopeNodeConsole(invocation.bindings.resolveNode(name)),
|
|
1096
|
+
documentSetup,
|
|
1097
|
+
setupTimeoutMs: options.setupTimeoutMs,
|
|
1098
|
+
teardownTimeoutMs: options.teardownTimeoutMs,
|
|
1099
|
+
project,
|
|
1100
|
+
projectContext: options.projectContext,
|
|
1101
|
+
retry: documentRetry ? 0 : invocation.retry.count,
|
|
1102
|
+
signal: options.signal,
|
|
1103
|
+
defaultTimeoutMs: invocation.defaultTimeoutMs,
|
|
1104
|
+
shouldStop: ()=>options.rootSignal.aborted || options.shouldBail() || options.isProjectFatal(),
|
|
1105
|
+
stopReason: ()=>options.rootSignal.aborted ? 'interrupted' : options.isProjectFatal() ? 'fatal-error' : 'bail',
|
|
1106
|
+
isFatalError: (run)=>[
|
|
1107
|
+
...run.beforeEach,
|
|
1108
|
+
...run.steps,
|
|
1109
|
+
...run.afterEach
|
|
1110
|
+
].some((step)=>step.error && (isFatalDeviceError(step.error) || isProjectResourceFailure(step.error))) || (run.teardownErrors ?? []).some((error)=>isFatalDeviceError(error) || isProjectResourceFailure(error)),
|
|
1111
|
+
onCaseStart: (collectedCase)=>{
|
|
1112
|
+
options.onProgress(` [case ${collectedCase.caseIndex + 1}/${document.cases.length}] ${collectedCase.definition.name}`);
|
|
1113
|
+
},
|
|
1114
|
+
onStepStart: (info)=>{
|
|
1115
|
+
options.onEvent?.({
|
|
1116
|
+
type: 'step-start',
|
|
1117
|
+
projectName: project.name,
|
|
1118
|
+
info
|
|
1119
|
+
});
|
|
1120
|
+
const indent = 'case' === info.scope ? ' ' : ' ';
|
|
1121
|
+
options.onProgress(`${indent}→ ${formatStep(info)}`);
|
|
1122
|
+
},
|
|
1123
|
+
onStepResult: async (info, result)=>{
|
|
1124
|
+
if (isProjectResourceFailure(result.error)) options.markProjectFatal();
|
|
1125
|
+
await invocation.bindings.onStepResult?.(info, result);
|
|
1126
|
+
options.onProgress(formatStepResult(info, result));
|
|
1127
|
+
options.onEvent?.({
|
|
1128
|
+
type: 'step-end',
|
|
1129
|
+
projectName: project.name,
|
|
1130
|
+
info,
|
|
1131
|
+
result
|
|
1132
|
+
});
|
|
1133
|
+
},
|
|
1134
|
+
onCaseResult: async (attempt)=>{
|
|
1135
|
+
await writeCaseAttemptResult(options.runDir, project.projectId, document.documentId, attempt);
|
|
1136
|
+
options.onProgress(` ${'success' === attempt.status ? '✓' : '✗'} case attempt ${attempt.attemptIndex + 1}/${documentRetry ? 1 : invocation.retry.count + 1}: ${attempt.name} (${attempt.durationMs} ms)`);
|
|
1137
|
+
if (!documentRetry) options.onEvent?.({
|
|
1138
|
+
type: 'case-attempt-end',
|
|
1139
|
+
attempt,
|
|
1140
|
+
maxAttempts: invocation.retry.count + 1
|
|
1141
|
+
});
|
|
1142
|
+
},
|
|
1143
|
+
onCaseOutcome: (outcome)=>{
|
|
1144
|
+
if (documentRetry) options.onEvent?.({
|
|
1145
|
+
type: 'case-progress',
|
|
1146
|
+
outcome,
|
|
1147
|
+
documentAttemptIndex
|
|
1148
|
+
});
|
|
1149
|
+
else options.onEvent?.({
|
|
1150
|
+
type: 'case-end',
|
|
1151
|
+
outcome
|
|
1152
|
+
});
|
|
1153
|
+
if (outcome.attempts?.some((attempt)=>[
|
|
1154
|
+
...attempt.beforeEach,
|
|
1155
|
+
...attempt.steps,
|
|
1156
|
+
...attempt.afterEach
|
|
1157
|
+
].some((step)=>isProjectResourceFailure(step.error)) || attempt.teardownErrors?.some(isProjectResourceFailure))) options.markProjectFatal();
|
|
1158
|
+
if (!options.isolatedContext && caseHasFatalError(outcome)) options.markProjectFatal();
|
|
1159
|
+
if (!documentRetry && 'failed' === outcome.status) options.addFailedCases(1);
|
|
1160
|
+
},
|
|
1161
|
+
onDocumentResult: options.onDocumentResult
|
|
1162
|
+
})).catch(async (error)=>{
|
|
1163
|
+
if (error instanceof WorkflowExecutionFailure) {
|
|
1164
|
+
const partial = error.result;
|
|
1165
|
+
const resourceOnly = error.errors.length > 0 && error.errors.every((failure)=>isResourceRecoveryError(failure) || partial.document.hostErrors?.some((host)=>'cleanup' === host.phase && (host.error === failure || host.error instanceof Error && host.error.cause === failure)));
|
|
1166
|
+
if (resourceOnly) {
|
|
1167
|
+
if (!options.isolatedContext && error.errors.some(isResourceRecoveryError)) options.markProjectFatal();
|
|
1168
|
+
return partial;
|
|
1169
|
+
}
|
|
1170
|
+
if (documentRetry) partial.document.retryDecision = decideRetry({
|
|
1171
|
+
scope: 'document',
|
|
1172
|
+
retryCount: invocation.retry.count,
|
|
1173
|
+
attemptIndex: documentAttemptIndex,
|
|
1174
|
+
status: 'failed',
|
|
1175
|
+
infrastructureError: true
|
|
1176
|
+
});
|
|
1177
|
+
attempts.push(partial);
|
|
1178
|
+
sinks.documents.push(partial.document);
|
|
1179
|
+
sinks.cases.push(...partial.cases.map((outcome)=>({
|
|
1180
|
+
...outcome,
|
|
1181
|
+
documentId: document.documentId,
|
|
1182
|
+
documentRunId: partial.document.documentRunId
|
|
1183
|
+
})));
|
|
1184
|
+
try {
|
|
1185
|
+
for (const outcome of partial.cases)for (const attempt of outcome.attempts ?? [])await writeCaseAttemptResult(options.runDir, project.projectId, document.documentId, attempt);
|
|
1186
|
+
await writeWorkflowDocumentResult(options.runDir, partial.document);
|
|
1187
|
+
} catch (publicationError) {
|
|
1188
|
+
throw new WorkflowExecutionFailure(partial, [
|
|
1189
|
+
...error.errors,
|
|
1190
|
+
publicationError
|
|
1191
|
+
]);
|
|
1192
|
+
}
|
|
1193
|
+
}
|
|
1194
|
+
throw error;
|
|
1195
|
+
});
|
|
1196
|
+
if (!options.isolatedContext && documentHasFatalError(execution.document)) options.markProjectFatal();
|
|
1197
|
+
if ([
|
|
1198
|
+
...execution.document.beforeAll,
|
|
1199
|
+
...execution.document.afterAll,
|
|
1200
|
+
...(execution.document.hostErrors ?? []).map(({ error })=>({
|
|
1201
|
+
error
|
|
1202
|
+
})),
|
|
1203
|
+
...(execution.document.teardownErrors ?? []).map((error)=>({
|
|
1204
|
+
error
|
|
1205
|
+
}))
|
|
1206
|
+
].some(({ error })=>isProjectResourceFailure(error))) options.markProjectFatal();
|
|
1207
|
+
const owners = [
|
|
1208
|
+
execution.document,
|
|
1209
|
+
...execution.cases.flatMap((outcome)=>outcome.attempts ?? [])
|
|
1210
|
+
];
|
|
1211
|
+
const cleanupPending = owners.some((owner)=>owner.cleanup?.status === 'pending');
|
|
1212
|
+
const operationPending = owners.some((owner)=>!!owner.pendingOperations?.length);
|
|
1213
|
+
const cleanupFailed = owners.some((owner)=>owner.cleanup?.status === 'failed' || !!owner.teardownErrors?.length || 'hostErrors' in owner && owner.hostErrors?.some(({ phase })=>'cleanup' === phase));
|
|
1214
|
+
if (cleanupPending || operationPending) {
|
|
1215
|
+
options.markProjectFatal();
|
|
1216
|
+
options.onUnresolvedResources?.();
|
|
1217
|
+
} else if (cleanupFailed && !options.isolatedContext) options.markProjectFatal();
|
|
1218
|
+
const failed = 'failed' === execution.document.status || execution.cases.some((item)=>'success' !== item.status);
|
|
1219
|
+
const decision = decideRetry({
|
|
1220
|
+
scope: 'document',
|
|
1221
|
+
retryCount: invocation.retry.count,
|
|
1222
|
+
attemptIndex: documentAttemptIndex,
|
|
1223
|
+
status: failed ? 'failed' : 'success',
|
|
1224
|
+
cleanupPending,
|
|
1225
|
+
cleanupFailed,
|
|
1226
|
+
operationPending,
|
|
1227
|
+
fatalError: options.isProjectFatal(),
|
|
1228
|
+
interrupted: options.signal.aborted || options.rootSignal.aborted,
|
|
1229
|
+
shouldStop: options.shouldBail()
|
|
1230
|
+
});
|
|
1231
|
+
if (documentRetry) execution.document.retryDecision = decision;
|
|
1232
|
+
if (documentRetry && 'retry' === decision.decision) options.onEvent?.({
|
|
1233
|
+
type: 'document-retry',
|
|
1234
|
+
document: execution.document,
|
|
1235
|
+
maxAttempts: invocation.retry.count + 1
|
|
1236
|
+
});
|
|
1237
|
+
attempts.push(execution);
|
|
1238
|
+
sinks.cases.push(...execution.cases.map((outcome)=>({
|
|
1239
|
+
...outcome,
|
|
1240
|
+
documentId: document.documentId,
|
|
1241
|
+
documentRunId: execution.document.documentRunId
|
|
1242
|
+
})));
|
|
1243
|
+
sinks.documents.push(execution.document);
|
|
1244
|
+
try {
|
|
1245
|
+
for (const outcome of execution.cases)for (const attempt of outcome.attempts ?? [])await writeCaseAttemptResult(options.runDir, project.projectId, document.documentId, attempt);
|
|
1246
|
+
await writeWorkflowDocumentResult(options.runDir, execution.document);
|
|
1247
|
+
} catch (error) {
|
|
1248
|
+
execution.document.status = 'failed';
|
|
1249
|
+
execution.document.hostErrors = [
|
|
1250
|
+
...execution.document.hostErrors ?? [],
|
|
1251
|
+
{
|
|
1252
|
+
phase: 'publication',
|
|
1253
|
+
error
|
|
1254
|
+
}
|
|
1255
|
+
];
|
|
1256
|
+
if (documentRetry) execution.document.retryDecision = decideRetry({
|
|
1257
|
+
scope: 'document',
|
|
1258
|
+
retryCount: invocation.retry.count,
|
|
1259
|
+
attemptIndex: documentAttemptIndex,
|
|
1260
|
+
status: 'failed',
|
|
1261
|
+
infrastructureError: true
|
|
1262
|
+
});
|
|
1263
|
+
throw new WorkflowExecutionFailure(execution, [
|
|
1264
|
+
error
|
|
1265
|
+
]);
|
|
1266
|
+
}
|
|
1267
|
+
return 'retry' === decision.decision ? 'failed' : void 0;
|
|
1268
|
+
});
|
|
1269
|
+
const final = attempts.at(-1);
|
|
1270
|
+
if (!final) return;
|
|
1271
|
+
if (documentRetry) for (const outcome of final.cases)options.onEvent?.({
|
|
1272
|
+
type: 'case-end',
|
|
1273
|
+
outcome,
|
|
1274
|
+
documentRetries: attempts.length - 1
|
|
1275
|
+
});
|
|
1276
|
+
const failedCases = final.cases.filter((item)=>'failed' === item.status).length;
|
|
1277
|
+
options.onEvent?.({
|
|
1278
|
+
type: 'document-end',
|
|
1279
|
+
document: final.document,
|
|
1280
|
+
cases: final.cases,
|
|
1281
|
+
durationMs: attempts.reduce((total, attempt)=>total + attempt.document.durationMs, 0)
|
|
1282
|
+
});
|
|
1283
|
+
if (documentRetry && failedCases) options.addFailedCases(failedCases);
|
|
1284
|
+
else if ('failed' === final.document.status && 0 === failedCases) options.addFailedCases(Math.max(1, failedCases));
|
|
1285
|
+
}
|
|
1286
|
+
async function runPreparedFiles(options) {
|
|
1287
|
+
const states = options.projects.map((prepared)=>({
|
|
1288
|
+
prepared,
|
|
1289
|
+
cases: [],
|
|
1290
|
+
documents: [],
|
|
1291
|
+
cursor: 0,
|
|
1292
|
+
active: 0,
|
|
1293
|
+
initializing: false,
|
|
1294
|
+
finished: false,
|
|
1295
|
+
fatal: false,
|
|
1296
|
+
setupFailed: false
|
|
1297
|
+
}));
|
|
1298
|
+
let nextProject = 0;
|
|
1299
|
+
let resourcesUnresolved = false;
|
|
1300
|
+
let occupied = 0;
|
|
1301
|
+
let occupiedProjects = 0;
|
|
1302
|
+
const cleanupQueue = [];
|
|
1303
|
+
const release = ()=>{
|
|
1304
|
+
occupied--;
|
|
1305
|
+
while(occupied < options.maxConcurrentFiles && cleanupQueue.length){
|
|
1306
|
+
occupied++;
|
|
1307
|
+
cleanupQueue.shift()();
|
|
1308
|
+
}
|
|
1309
|
+
wake?.();
|
|
1310
|
+
};
|
|
1311
|
+
let wake;
|
|
1312
|
+
const running = new Set();
|
|
1313
|
+
const stopped = (state)=>options.signal.aborted || options.shouldBail() || state.fatal;
|
|
1314
|
+
const invalid = (state)=>options.preflightFailed || state.prepared.collectionErrors.length > 0;
|
|
1315
|
+
const unresolved = ()=>{
|
|
1316
|
+
if (resourcesUnresolved) return;
|
|
1317
|
+
resourcesUnresolved = true;
|
|
1318
|
+
options.onInfrastructureError(new WorkflowError('Resources are still running or cleanup is pending. No new test files will start.', {
|
|
1319
|
+
code: 'UNRESOLVED_FILE_RESOURCES'
|
|
1320
|
+
}));
|
|
1321
|
+
};
|
|
1322
|
+
const finalizeProject = (state)=>{
|
|
1323
|
+
const { prepared } = state;
|
|
1324
|
+
const completed = new Set(state.cases.map((item)=>item.caseId));
|
|
1325
|
+
const reason = invalid(state) ? 'project-preflight-failed' : state.setupFailed ? 'project-setup-failed' : options.signal.aborted ? 'interrupted' : state.fatal ? 'fatal-error' : 'bail';
|
|
1326
|
+
for (const document of prepared.documents)for (const item of document.cases)if (!completed.has(item.caseId)) state.cases.push({
|
|
1327
|
+
documentId: document.documentId,
|
|
1328
|
+
caseId: item.caseId,
|
|
1329
|
+
projectName: prepared.project.name,
|
|
1330
|
+
name: item.definition.name,
|
|
1331
|
+
sourcePath: item.sourcePath,
|
|
1332
|
+
caseIndex: item.caseIndex,
|
|
1333
|
+
status: 'not-run',
|
|
1334
|
+
notRunReason: reason
|
|
1335
|
+
});
|
|
1336
|
+
const order = new Map(prepared.documents.map((document, i)=>[
|
|
1337
|
+
document.documentId,
|
|
1338
|
+
i
|
|
1339
|
+
]));
|
|
1340
|
+
const position = (id)=>order.get(id) ?? Number.MAX_SAFE_INTEGER;
|
|
1341
|
+
state.cases.sort((a, b)=>position(a.documentId) - position(b.documentId) || a.caseIndex - b.caseIndex);
|
|
1342
|
+
state.documents.sort((a, b)=>position(a.documentId) - position(b.documentId));
|
|
1343
|
+
};
|
|
1344
|
+
const finishProject = async (state)=>{
|
|
1345
|
+
if (!state.runtime || state.finished) return;
|
|
1346
|
+
state.finished = true;
|
|
1347
|
+
try {
|
|
1348
|
+
options.onEvent?.({
|
|
1349
|
+
type: 'project-phase',
|
|
1350
|
+
projectName: state.prepared.project.name,
|
|
1351
|
+
phase: 'teardown'
|
|
1352
|
+
});
|
|
1353
|
+
} catch (error) {
|
|
1354
|
+
options.onInfrastructureError(error);
|
|
1355
|
+
}
|
|
1356
|
+
const failed = state.fatal || options.signal.aborted || state.cases.some((item)=>'success' !== item.status) || state.documents.some((item)=>'success' !== item.status);
|
|
1357
|
+
state.lifecycle = await runWithConsoleLogScope({
|
|
1358
|
+
projectName: state.prepared.project.name,
|
|
1359
|
+
phase: 'teardown'
|
|
1360
|
+
}, ()=>state.runtime.finish(failed ? 'failed' : 'success'));
|
|
1361
|
+
const releaseProject = ()=>{
|
|
1362
|
+
occupiedProjects--;
|
|
1363
|
+
wake?.();
|
|
1364
|
+
};
|
|
1365
|
+
if (state.lifecycle.cleanup?.status === 'pending' || state.lifecycle.pendingOperations?.length) {
|
|
1366
|
+
unresolved();
|
|
1367
|
+
Promise.all([
|
|
1368
|
+
waitForScopeOperations(state.runtime.signal),
|
|
1369
|
+
getResourceCleanupCompletion(state.runtime.signal)
|
|
1370
|
+
]).then(releaseProject, (error)=>{
|
|
1371
|
+
options.onInfrastructureError(error);
|
|
1372
|
+
releaseProject();
|
|
1373
|
+
});
|
|
1374
|
+
} else {
|
|
1375
|
+
releaseProject();
|
|
1376
|
+
finalizeProject(state);
|
|
1377
|
+
options.onProjectEnd?.(state);
|
|
1378
|
+
}
|
|
1379
|
+
};
|
|
1380
|
+
const admit = ()=>{
|
|
1381
|
+
for(let offset = 0; offset < states.length; offset++){
|
|
1382
|
+
const index = (nextProject + offset) % states.length;
|
|
1383
|
+
const state = states[index];
|
|
1384
|
+
if (invalid(state) || state.finished || state.initializing) continue;
|
|
1385
|
+
if (!state.runtime && occupiedProjects >= options.maxConcurrentProjects) continue;
|
|
1386
|
+
const cleanupOnly = !!state.runtime && 0 === state.active && (stopped(state) || state.cursor === state.prepared.invocations.length);
|
|
1387
|
+
if (!cleanupOnly && (stopped(state) || state.cursor >= state.prepared.invocations.length || state.active >= (state.prepared.project.maxConcurrentFiles ?? options.maxConcurrentFiles))) continue;
|
|
1388
|
+
nextProject = (index + 1) % states.length;
|
|
1389
|
+
occupied++;
|
|
1390
|
+
state.active++;
|
|
1391
|
+
if (!state.runtime && !cleanupOnly) {
|
|
1392
|
+
occupiedProjects++;
|
|
1393
|
+
state.initializing = true;
|
|
1394
|
+
}
|
|
1395
|
+
const invocationIndex = cleanupOnly ? -1 : state.cursor++;
|
|
1396
|
+
return {
|
|
1397
|
+
state,
|
|
1398
|
+
index,
|
|
1399
|
+
invocationIndex
|
|
1400
|
+
};
|
|
1401
|
+
}
|
|
1402
|
+
};
|
|
1403
|
+
const run = async ({ state, index, invocationIndex })=>{
|
|
1404
|
+
const { prepared } = state;
|
|
1405
|
+
const { project } = prepared;
|
|
1406
|
+
let fileResourcesPending = false;
|
|
1407
|
+
try {
|
|
1408
|
+
if (invocationIndex < 0) return void await finishProject(state);
|
|
1409
|
+
if (!state.runtime) {
|
|
1410
|
+
state.runtime = createProjectRuntime({
|
|
1411
|
+
project,
|
|
1412
|
+
setup: project.setup,
|
|
1413
|
+
signal: options.signal,
|
|
1414
|
+
setupTimeoutMs: options.setupTimeoutMs,
|
|
1415
|
+
teardownTimeoutMs: options.teardownTimeoutMs
|
|
1416
|
+
});
|
|
1417
|
+
setMaxListeners(Math.max(10, Math.min(prepared.invocations.length, project.maxConcurrentFiles ?? options.maxConcurrentFiles) + 1), state.runtime.signal);
|
|
1418
|
+
options.announceProject(prepared, index);
|
|
1419
|
+
options.onEvent?.({
|
|
1420
|
+
type: 'project-phase',
|
|
1421
|
+
projectName: project.name,
|
|
1422
|
+
phase: 'setup'
|
|
1423
|
+
});
|
|
1424
|
+
await runWithConsoleLogScope({
|
|
1425
|
+
projectName: project.name,
|
|
1426
|
+
phase: 'setup'
|
|
1427
|
+
}, ()=>state.runtime.start());
|
|
1428
|
+
options.onEvent?.({
|
|
1429
|
+
type: 'project-phase',
|
|
1430
|
+
projectName: project.name,
|
|
1431
|
+
phase: 'running'
|
|
1432
|
+
});
|
|
1433
|
+
state.initializing = false;
|
|
1434
|
+
if (!state.runtime.canRun) {
|
|
1435
|
+
state.fatal = true;
|
|
1436
|
+
state.setupFailed = true;
|
|
1437
|
+
}
|
|
1438
|
+
wake?.();
|
|
1439
|
+
}
|
|
1440
|
+
if (stopped(state)) return;
|
|
1441
|
+
const invocation = prepared.invocations[invocationIndex];
|
|
1442
|
+
options.onProgress(prepared, ` [document ${invocationIndex + 1}/${prepared.invocations.length}] ${invocation.document.sourcePath}`);
|
|
1443
|
+
await executeDocumentInvocation({
|
|
1444
|
+
invocation,
|
|
1445
|
+
project,
|
|
1446
|
+
projectContext: state.runtime.context,
|
|
1447
|
+
runDir: options.runDir,
|
|
1448
|
+
signal: state.runtime.signal,
|
|
1449
|
+
rootSignal: options.signal,
|
|
1450
|
+
shouldBail: options.shouldBail,
|
|
1451
|
+
isProjectFatal: ()=>state.fatal,
|
|
1452
|
+
markProjectFatal: ()=>{
|
|
1453
|
+
if (state.fatal) return;
|
|
1454
|
+
state.fatal = true;
|
|
1455
|
+
state.runtime?.abort(new WorkflowError(`Project "${project.name}" shared execution environment is unavailable.`, {
|
|
1456
|
+
code: 'PROJECT_RESOURCE_ERROR'
|
|
1457
|
+
}));
|
|
1458
|
+
},
|
|
1459
|
+
addFailedCases: options.addFailedCases,
|
|
1460
|
+
onProgress: (message)=>options.onProgress(prepared, options.maxConcurrentFiles > 1 ? ` [${invocation.document.sourcePath}]${message}` : message),
|
|
1461
|
+
sinks: state,
|
|
1462
|
+
onEvent: options.onEvent,
|
|
1463
|
+
isolatedContext: void 0 !== project.fileSetup,
|
|
1464
|
+
isProjectResourceFailure: isSharedResourceFailure,
|
|
1465
|
+
setupTimeoutMs: options.setupTimeoutMs,
|
|
1466
|
+
teardownTimeoutMs: options.teardownTimeoutMs,
|
|
1467
|
+
onUnresolvedResources: ()=>{
|
|
1468
|
+
fileResourcesPending = true;
|
|
1469
|
+
unresolved();
|
|
1470
|
+
}
|
|
1471
|
+
});
|
|
1472
|
+
} catch (error) {
|
|
1473
|
+
state.fatal = true;
|
|
1474
|
+
fileResourcesPending = !!state.runtime && getPendingScopeOperations(state.runtime.signal).length > 0;
|
|
1475
|
+
options.onInfrastructureError(error);
|
|
1476
|
+
} finally{
|
|
1477
|
+
state.initializing = false;
|
|
1478
|
+
try {
|
|
1479
|
+
if (1 === state.active && (stopped(state) || state.cursor === prepared.invocations.length)) await finishProject(state);
|
|
1480
|
+
} catch (error) {
|
|
1481
|
+
options.onInfrastructureError(error);
|
|
1482
|
+
}
|
|
1483
|
+
state.active--;
|
|
1484
|
+
if (state.runtime && (fileResourcesPending || state.lifecycle?.cleanup?.status === 'pending')) waitForScopeOperations(state.runtime.signal).then(release, (error)=>{
|
|
1485
|
+
options.onInfrastructureError(error);
|
|
1486
|
+
release();
|
|
1487
|
+
});
|
|
1488
|
+
else release();
|
|
1489
|
+
wake?.();
|
|
1490
|
+
}
|
|
1491
|
+
};
|
|
1492
|
+
while(true){
|
|
1493
|
+
const changed = new Promise((resolve)=>{
|
|
1494
|
+
wake = resolve;
|
|
1495
|
+
});
|
|
1496
|
+
while(occupied < options.maxConcurrentFiles){
|
|
1497
|
+
const job = admit();
|
|
1498
|
+
if (!job) break;
|
|
1499
|
+
const task = run(job);
|
|
1500
|
+
running.add(task);
|
|
1501
|
+
const settled = ()=>{
|
|
1502
|
+
running.delete(task);
|
|
1503
|
+
wake?.();
|
|
1504
|
+
};
|
|
1505
|
+
task.then(settled, (error)=>{
|
|
1506
|
+
options.onInfrastructureError(error);
|
|
1507
|
+
settled();
|
|
1508
|
+
});
|
|
1509
|
+
}
|
|
1510
|
+
if (!running.size) break;
|
|
1511
|
+
await changed;
|
|
1512
|
+
}
|
|
1513
|
+
wake = void 0;
|
|
1514
|
+
await Promise.all(states.map(async (state)=>{
|
|
1515
|
+
if (!state.runtime || state.finished) return;
|
|
1516
|
+
const admission = new Promise((resolve)=>{
|
|
1517
|
+
cleanupQueue.push(resolve);
|
|
1518
|
+
});
|
|
1519
|
+
trackResourceOperation({}, admission, state.runtime.signal);
|
|
1520
|
+
while(occupied < options.maxConcurrentFiles && cleanupQueue.length){
|
|
1521
|
+
occupied++;
|
|
1522
|
+
cleanupQueue.shift()();
|
|
1523
|
+
}
|
|
1524
|
+
try {
|
|
1525
|
+
await finishProject(state);
|
|
1526
|
+
} catch (error) {
|
|
1527
|
+
options.onInfrastructureError(error);
|
|
1528
|
+
} finally{
|
|
1529
|
+
Promise.all([
|
|
1530
|
+
admission,
|
|
1531
|
+
getResourceCleanupCompletion(state.runtime.signal)
|
|
1532
|
+
]).then(release, (error)=>{
|
|
1533
|
+
options.onInfrastructureError(error);
|
|
1534
|
+
release();
|
|
1535
|
+
});
|
|
1536
|
+
}
|
|
1537
|
+
}));
|
|
1538
|
+
for (const state of states){
|
|
1539
|
+
const { prepared } = state;
|
|
1540
|
+
if (!state.runtime) options.announceProject(prepared, states.indexOf(state));
|
|
1541
|
+
finalizeProject(state);
|
|
1542
|
+
}
|
|
1543
|
+
return states;
|
|
1544
|
+
}
|
|
1545
|
+
function createOperationJournal(runDir) {
|
|
1546
|
+
const path = join(runDir, 'operations.jsonl');
|
|
1547
|
+
writeFileSync(path, '', {
|
|
1548
|
+
flag: 'wx'
|
|
1549
|
+
});
|
|
1550
|
+
return {
|
|
1551
|
+
path,
|
|
1552
|
+
append (event) {
|
|
1553
|
+
appendFileSync(path, `${JSON.stringify(event)}\n`);
|
|
1554
|
+
}
|
|
1555
|
+
};
|
|
1556
|
+
}
|
|
1557
|
+
const asNotRun = (documentId, collectedCase, projectName, reason)=>({
|
|
1558
|
+
documentId,
|
|
1559
|
+
caseId: collectedCase.caseId,
|
|
1560
|
+
projectName,
|
|
1561
|
+
name: collectedCase.definition.name,
|
|
1562
|
+
sourcePath: collectedCase.sourcePath,
|
|
1563
|
+
caseIndex: collectedCase.caseIndex,
|
|
1564
|
+
status: 'not-run',
|
|
1565
|
+
notRunReason: reason
|
|
1566
|
+
});
|
|
1567
|
+
const summarize = (projects)=>{
|
|
1568
|
+
const healthCases = projects.flatMap((project)=>{
|
|
1569
|
+
const documentAttemptByRunId = new Map(project.documents.map((document)=>[
|
|
1570
|
+
document.documentRunId,
|
|
1571
|
+
document.attemptIndex ?? 0
|
|
1572
|
+
]));
|
|
1573
|
+
return latestById(project.cases, (item)=>item.caseId).map((item)=>({
|
|
1574
|
+
status: item.status,
|
|
1575
|
+
...item.run ? {
|
|
1576
|
+
run: item.run
|
|
1577
|
+
} : {},
|
|
1578
|
+
...item.attempts ? {
|
|
1579
|
+
attempts: item.attempts
|
|
1580
|
+
} : {},
|
|
1581
|
+
retryCount: item.documentRunId ? documentAttemptByRunId.get(item.documentRunId) ?? 0 : 0
|
|
1582
|
+
}));
|
|
1583
|
+
});
|
|
1584
|
+
const documents = projects.flatMap((project)=>latestById(project.documents, (item)=>item.documentId));
|
|
1585
|
+
const health = calculateTestRunHealth(healthCases);
|
|
1586
|
+
return {
|
|
1587
|
+
total: health.total,
|
|
1588
|
+
passed: health.passed,
|
|
1589
|
+
failed: health.failed,
|
|
1590
|
+
notRun: health.notRun,
|
|
1591
|
+
passedAfterRetry: health.passedAfterRetry,
|
|
1592
|
+
finalPassRate: health.finalPassRate,
|
|
1593
|
+
firstPassRate: health.firstPassRate,
|
|
1594
|
+
filtered: projects.reduce((total, project)=>total + project.filteredCaseCount, 0),
|
|
1595
|
+
collectionErrors: projects.reduce((total, project)=>total + project.collectionErrors.length, 0),
|
|
1596
|
+
documentFailures: documents.filter((document)=>'failed' === document.status).length,
|
|
1597
|
+
projectFailures: projects.filter((project)=>'failed' === project.status).length
|
|
1598
|
+
};
|
|
1599
|
+
};
|
|
1600
|
+
const latestById = (items, id)=>[
|
|
1601
|
+
...new Map(items.map((item)=>[
|
|
1602
|
+
id(item),
|
|
1603
|
+
item
|
|
1604
|
+
])).values()
|
|
1605
|
+
];
|
|
1606
|
+
const notRunSuite = (prepared, reason)=>prepared.documents.flatMap((document)=>document.cases.map((item)=>asNotRun(document.documentId, item, prepared.project.name, reason)));
|
|
1607
|
+
async function runPreparedTestProject(plan) {
|
|
1608
|
+
const reports = new ReportSession(join(plan.runDir, 'records'));
|
|
1609
|
+
return runWithMidsceneRunDir(plan.artifactDir, ()=>reports.run(()=>runWithReportSession(plan, reports)));
|
|
1610
|
+
}
|
|
1611
|
+
async function runWithReportSession(plan, reports) {
|
|
1612
|
+
const { startedAt, runId, projectRoot, configPath, resultDir, runDir, summaryPath, reportDir, definition, projects: preparedProjects } = plan;
|
|
1613
|
+
const progress = plan.onProgress;
|
|
1614
|
+
const totalDocuments = preparedProjects.reduce((total, prepared)=>total + prepared.documents.length, 0);
|
|
1615
|
+
const totalCases = preparedProjects.reduce((total, prepared)=>total + prepared.selectedCaseCount, 0);
|
|
1616
|
+
const totalErrors = preparedProjects.reduce((total, prepared)=>total + prepared.collectionErrors.length, 0);
|
|
1617
|
+
plan.onEvent?.({
|
|
1618
|
+
type: 'run-start',
|
|
1619
|
+
configuredModels: configuredModelNames(),
|
|
1620
|
+
projects: preparedProjects.length,
|
|
1621
|
+
documents: totalDocuments + totalErrors,
|
|
1622
|
+
cases: totalCases,
|
|
1623
|
+
projectRoot,
|
|
1624
|
+
configPath,
|
|
1625
|
+
startedAt: startedAt.toISOString(),
|
|
1626
|
+
maxConcurrentProjects: definition.test.maxConcurrentProjects,
|
|
1627
|
+
maxConcurrentFiles: definition.test.maxConcurrentFiles,
|
|
1628
|
+
files: preparedProjects.flatMap((prepared)=>[
|
|
1629
|
+
...prepared.documents.map((document)=>({
|
|
1630
|
+
documentId: document.documentId,
|
|
1631
|
+
projectName: prepared.project.name,
|
|
1632
|
+
sourcePath: document.sourcePath,
|
|
1633
|
+
cases: document.cases.length
|
|
1634
|
+
})),
|
|
1635
|
+
...prepared.collectionErrors.map((error)=>({
|
|
1636
|
+
documentId: error.sourcePath,
|
|
1637
|
+
projectName: prepared.project.name,
|
|
1638
|
+
sourcePath: error.sourcePath,
|
|
1639
|
+
cases: 0
|
|
1640
|
+
}))
|
|
1641
|
+
])
|
|
1642
|
+
});
|
|
1643
|
+
progress(`midscene-test: preflighted ${preparedProjects.length} projects, ${totalDocuments} documents, ${totalCases} cases, ${totalErrors} collection errors`);
|
|
1644
|
+
const effectiveConcurrency = Math.min(definition.test.maxConcurrentFiles ?? 1 / 0, Math.max(1, totalDocuments));
|
|
1645
|
+
await writeTestRunPlan(runDir, preparedProjects);
|
|
1646
|
+
const journal = createOperationJournal(runDir);
|
|
1647
|
+
const rootController = new AbortController();
|
|
1648
|
+
setMaxListeners(Math.max(10, effectiveConcurrency + 1), rootController.signal);
|
|
1649
|
+
const handleSignal = (signal)=>{
|
|
1650
|
+
rootController.abort(new Error(`Workflow interrupted by ${signal}.`));
|
|
1651
|
+
};
|
|
1652
|
+
const sigint = ()=>handleSignal('SIGINT');
|
|
1653
|
+
const sigterm = ()=>handleSignal('SIGTERM');
|
|
1654
|
+
process.on('SIGINT', sigint);
|
|
1655
|
+
process.on('SIGTERM', sigterm);
|
|
1656
|
+
let failedCaseCount = 0;
|
|
1657
|
+
const bailReached = ()=>definition.test.bail > 0 && failedCaseCount >= definition.test.bail;
|
|
1658
|
+
let shutdownDeadline;
|
|
1659
|
+
const beginShutdown = ()=>{
|
|
1660
|
+
shutdownDeadline ??= Date.now() + definition.test.teardownTimeout;
|
|
1661
|
+
};
|
|
1662
|
+
rootController.signal.addEventListener('abort', beginShutdown, {
|
|
1663
|
+
once: true
|
|
1664
|
+
});
|
|
1665
|
+
const addFailedCases = (count)=>{
|
|
1666
|
+
failedCaseCount += count;
|
|
1667
|
+
if (bailReached()) beginShutdown();
|
|
1668
|
+
};
|
|
1669
|
+
const teardownTimeoutMs = ()=>Math.max(0, Math.min(definition.test.teardownTimeout, void 0 === shutdownDeadline ? 1 / 0 : shutdownDeadline - Date.now()));
|
|
1670
|
+
let hasInfrastructureError = false;
|
|
1671
|
+
const infrastructureErrors = [];
|
|
1672
|
+
const recordInfrastructureError = (error)=>{
|
|
1673
|
+
const errors = error instanceof WorkflowExecutionFailure ? error.errors : [
|
|
1674
|
+
error
|
|
1675
|
+
];
|
|
1676
|
+
for (const item of errors)if (!infrastructureErrors.includes(item)) infrastructureErrors.push(item);
|
|
1677
|
+
hasInfrastructureError = true;
|
|
1678
|
+
rootController.abort(error);
|
|
1679
|
+
};
|
|
1680
|
+
const runInfrastructureCallback = (callback)=>{
|
|
1681
|
+
try {
|
|
1682
|
+
return callback();
|
|
1683
|
+
} catch (error) {
|
|
1684
|
+
recordInfrastructureError(error);
|
|
1685
|
+
throw error;
|
|
1686
|
+
}
|
|
1687
|
+
};
|
|
1688
|
+
const stopObservingOperations = observeResourceOperations(rootController.signal, (event)=>{
|
|
1689
|
+
try {
|
|
1690
|
+
journal.append(event);
|
|
1691
|
+
} catch (error) {
|
|
1692
|
+
recordInfrastructureError(new WorkflowPublicationError('write-result', journal.path, error));
|
|
1693
|
+
getDebug('test-runner:operations', {
|
|
1694
|
+
console: true
|
|
1695
|
+
})(`Could not persist operation event: ${String(error)}`);
|
|
1696
|
+
}
|
|
1697
|
+
});
|
|
1698
|
+
const announceProject = (prepared, projectIndex)=>{
|
|
1699
|
+
const { project } = prepared;
|
|
1700
|
+
runInfrastructureCallback(()=>progress(`[project ${projectIndex + 1}/${preparedProjects.length}] ${project.name}`));
|
|
1701
|
+
};
|
|
1702
|
+
const buildProjectResult = (prepared, cases, documents, lifecycle)=>{
|
|
1703
|
+
const { project } = prepared;
|
|
1704
|
+
const projectFailed = prepared.collectionErrors.length > 0 || latestById(cases, (item)=>item.caseId).some((item)=>'success' !== item.status) || latestById(documents, (item)=>item.documentId).some((item)=>'failed' === item.status) || lifecycle?.status === 'failed';
|
|
1705
|
+
return {
|
|
1706
|
+
projectId: project.projectId,
|
|
1707
|
+
name: project.name,
|
|
1708
|
+
...prepared.platform ? {
|
|
1709
|
+
platform: prepared.platform
|
|
1710
|
+
} : {},
|
|
1711
|
+
status: projectFailed ? 'failed' : 'success',
|
|
1712
|
+
retry: project.retry,
|
|
1713
|
+
fileSelection: prepared.fileSelection,
|
|
1714
|
+
tagSelection: project.tags,
|
|
1715
|
+
sourceCount: prepared.sources.length,
|
|
1716
|
+
selectedCaseCount: prepared.selectedCaseCount,
|
|
1717
|
+
filteredCaseCount: prepared.filteredCaseCount,
|
|
1718
|
+
...lifecycle ? {
|
|
1719
|
+
lifecycle
|
|
1720
|
+
} : {},
|
|
1721
|
+
cases,
|
|
1722
|
+
plans: prepared.plans,
|
|
1723
|
+
documents,
|
|
1724
|
+
collectionErrors: prepared.collectionErrors
|
|
1725
|
+
};
|
|
1726
|
+
};
|
|
1727
|
+
const buildSkippedProjectResult = (prepared, projectIndex, reason)=>{
|
|
1728
|
+
announceProject(prepared, projectIndex);
|
|
1729
|
+
return buildProjectResult(prepared, notRunSuite(prepared, reason), []);
|
|
1730
|
+
};
|
|
1731
|
+
const projectResults = new Array(preparedProjects.length);
|
|
1732
|
+
try {
|
|
1733
|
+
const states = await runPreparedFiles({
|
|
1734
|
+
projects: preparedProjects,
|
|
1735
|
+
maxConcurrentFiles: effectiveConcurrency,
|
|
1736
|
+
maxConcurrentProjects: definition.test.maxConcurrentProjects ?? 1 / 0,
|
|
1737
|
+
setupTimeoutMs: definition.test.setupTimeout,
|
|
1738
|
+
teardownTimeoutMs,
|
|
1739
|
+
signal: rootController.signal,
|
|
1740
|
+
runDir,
|
|
1741
|
+
preflightFailed: 'run' === plan.preflightScope && totalErrors > 0,
|
|
1742
|
+
shouldBail: bailReached,
|
|
1743
|
+
addFailedCases,
|
|
1744
|
+
announceProject,
|
|
1745
|
+
onProgress: (prepared, message)=>runInfrastructureCallback(()=>progress(preparedProjects.length > 1 || effectiveConcurrency > 1 ? `[${prepared.project.name}]${message}` : message)),
|
|
1746
|
+
onEvent: (event)=>runInfrastructureCallback(()=>plan.onEvent?.(event)),
|
|
1747
|
+
onProjectEnd: (state)=>runInfrastructureCallback(()=>plan.onEvent?.({
|
|
1748
|
+
type: 'project-end',
|
|
1749
|
+
project: buildProjectResult(state.prepared, state.cases, state.documents, state.lifecycle)
|
|
1750
|
+
})),
|
|
1751
|
+
onInfrastructureError: recordInfrastructureError
|
|
1752
|
+
});
|
|
1753
|
+
states.forEach((state, index)=>{
|
|
1754
|
+
projectResults[index] = buildProjectResult(state.prepared, state.cases, state.documents, state.lifecycle);
|
|
1755
|
+
});
|
|
1756
|
+
} catch (error) {
|
|
1757
|
+
recordInfrastructureError(error);
|
|
1758
|
+
} finally{
|
|
1759
|
+
stopObservingOperations();
|
|
1760
|
+
process.off('SIGINT', sigint);
|
|
1761
|
+
process.off('SIGTERM', sigterm);
|
|
1762
|
+
}
|
|
1763
|
+
const completedProjectResults = preparedProjects.map((prepared, projectIndex)=>{
|
|
1764
|
+
const result = projectResults[projectIndex];
|
|
1765
|
+
if (result) return result;
|
|
1766
|
+
const reason = prepared.collectionErrors.length ? 'project-preflight-failed' : rootController.signal.aborted ? 'interrupted' : bailReached() ? 'bail' : void 0;
|
|
1767
|
+
if (!reason) throw new Error(`Project scheduler did not produce a result for "${prepared.project.name}".`);
|
|
1768
|
+
return buildSkippedProjectResult(prepared, projectIndex, reason);
|
|
1769
|
+
});
|
|
1770
|
+
const summary = summarize(completedProjectResults);
|
|
1771
|
+
const failed = hasInfrastructureError || rootController.signal.aborted || summary.failed > 0 || summary.notRun > 0 || summary.collectionErrors > 0 || summary.documentFailures > 0 || summary.projectFailures > 0;
|
|
1772
|
+
const endedAt = new Date();
|
|
1773
|
+
const result = {
|
|
1774
|
+
schemaVersion: 3,
|
|
1775
|
+
operationLog: {
|
|
1776
|
+
path: journal.path,
|
|
1777
|
+
snapshotAt: endedAt.toISOString()
|
|
1778
|
+
},
|
|
1779
|
+
runId,
|
|
1780
|
+
startedAt: startedAt.toISOString(),
|
|
1781
|
+
endedAt: endedAt.toISOString(),
|
|
1782
|
+
durationMs: Math.max(0, endedAt.getTime() - startedAt.getTime()),
|
|
1783
|
+
status: failed ? 'failed' : 'success',
|
|
1784
|
+
exitCode: failed ? 1 : 0,
|
|
1785
|
+
resultDir,
|
|
1786
|
+
summaryPath,
|
|
1787
|
+
reportDir,
|
|
1788
|
+
summary,
|
|
1789
|
+
projects: completedProjectResults,
|
|
1790
|
+
cases: completedProjectResults.flatMap((project)=>project.cases),
|
|
1791
|
+
documents: completedProjectResults.flatMap((project)=>project.documents),
|
|
1792
|
+
collectionErrors: completedProjectResults.flatMap((project)=>project.collectionErrors),
|
|
1793
|
+
...hasInfrastructureError ? {
|
|
1794
|
+
errors: infrastructureErrors.map(asExecutionError)
|
|
1795
|
+
} : {}
|
|
1796
|
+
};
|
|
1797
|
+
let completedResult = result;
|
|
1798
|
+
let reportIndex;
|
|
1799
|
+
const publish = async (operation, path, callback)=>{
|
|
1800
|
+
try {
|
|
1801
|
+
await callback();
|
|
1802
|
+
return true;
|
|
1803
|
+
} catch (error) {
|
|
1804
|
+
recordInfrastructureError(error instanceof WorkflowPublicationError || error instanceof WorkflowExecutionFailure ? error : new WorkflowPublicationError(operation, path, error));
|
|
1805
|
+
completedResult = {
|
|
1806
|
+
...completedResult,
|
|
1807
|
+
status: 'failed',
|
|
1808
|
+
exitCode: 1,
|
|
1809
|
+
errors: infrastructureErrors.map(asExecutionError)
|
|
1810
|
+
};
|
|
1811
|
+
return false;
|
|
1812
|
+
}
|
|
1813
|
+
};
|
|
1814
|
+
const writeSummary = ()=>publish('write-result', summaryPath, ()=>writeTestProjectRunResult({
|
|
1815
|
+
projectRoot,
|
|
1816
|
+
...configPath ? {
|
|
1817
|
+
configPath
|
|
1818
|
+
} : {},
|
|
1819
|
+
result: completedResult
|
|
1820
|
+
}));
|
|
1821
|
+
let extraSources = [];
|
|
1822
|
+
await publish('write-report', reports.directory, async ()=>{
|
|
1823
|
+
extraSources = await reports.finish();
|
|
1824
|
+
});
|
|
1825
|
+
const writeReport = ()=>publish('write-report', reportDir, async ()=>{
|
|
1826
|
+
if (!plan.reportEnabled) return;
|
|
1827
|
+
const reportPath = await new TestRunReportAssembler().assembleAsync({
|
|
1828
|
+
outputDir: reportDir,
|
|
1829
|
+
outputFormat: plan.definition.output.reportFormat,
|
|
1830
|
+
reportFileName: `midscene-e2e-${runId}`,
|
|
1831
|
+
overwrite: false,
|
|
1832
|
+
sources: [
|
|
1833
|
+
...collectTestRunReportSources(completedResult),
|
|
1834
|
+
...extraSources.map((source)=>({
|
|
1835
|
+
...source,
|
|
1836
|
+
scopeId: runId
|
|
1837
|
+
}))
|
|
1838
|
+
],
|
|
1839
|
+
buildRunnerDump: (index)=>{
|
|
1840
|
+
reportIndex = index;
|
|
1841
|
+
const dump = buildTestRunReportDump(completedResult, index);
|
|
1842
|
+
completedResult = {
|
|
1843
|
+
...completedResult,
|
|
1844
|
+
metrics: dump.metrics
|
|
1845
|
+
};
|
|
1846
|
+
const warn = getDebug('test-runner:report-assembler', {
|
|
1847
|
+
console: true
|
|
1848
|
+
});
|
|
1849
|
+
for (const diagnostic of dump.diagnostics ?? [])warn(diagnostic.message);
|
|
1850
|
+
return dump;
|
|
1851
|
+
}
|
|
1852
|
+
});
|
|
1853
|
+
completedResult = {
|
|
1854
|
+
...completedResult,
|
|
1855
|
+
reportPath
|
|
1856
|
+
};
|
|
1857
|
+
});
|
|
1858
|
+
await writeReport();
|
|
1859
|
+
for (const publication of plan.publications)await publish(publication.operation, publication.path, async ()=>{
|
|
1860
|
+
await publication.publish(completedResult);
|
|
1861
|
+
});
|
|
1862
|
+
await publish('write-result', join(dirname(summaryPath), 'result.md'), async ()=>{
|
|
1863
|
+
if (!completedResult.metrics) {
|
|
1864
|
+
const dump = buildTestRunReportDump(completedResult, reportIndex ?? {
|
|
1865
|
+
sources: [],
|
|
1866
|
+
metrics: sumTestRunReportMetrics([])
|
|
1867
|
+
});
|
|
1868
|
+
completedResult = {
|
|
1869
|
+
...completedResult,
|
|
1870
|
+
metrics: dump.metrics
|
|
1871
|
+
};
|
|
1872
|
+
}
|
|
1873
|
+
const markdownPath = await writeTestResultMarkdown(completedResult, reportIndex);
|
|
1874
|
+
completedResult = {
|
|
1875
|
+
...completedResult,
|
|
1876
|
+
markdownPath
|
|
1877
|
+
};
|
|
1878
|
+
});
|
|
1879
|
+
if (completedResult.reportPath && !hasInfrastructureError) await publish('write-result', reports.directory, ()=>reports.prune());
|
|
1880
|
+
await writeSummary();
|
|
1881
|
+
if (hasInfrastructureError) throw new WorkflowExecutionFailure(completedResult, infrastructureErrors);
|
|
1882
|
+
return completedResult;
|
|
1883
|
+
}
|
|
1884
|
+
export { CliModelUsage, DEFAULT_TEST_FILE_SELECTION, createTestRunId, defineFileSetup, defineProjectSetup, defineTestProject, discoverTestConfig, discoverTestFiles, fileURLToPath, formatModelUsage, inspect, loadTestProject, prepareTestRun, runPreparedTestProject, stripVTControlCharacters, validateTestFileSelection };
|
|
1885
|
+
|
|
1886
|
+
//# sourceMappingURL=299.mjs.map
|