@midscene/test 1.12.10-beta-20260920031616.0 → 1.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -4,11 +4,13 @@ import { z } from "zod/v4";
4
4
  import { pathToFileURL } from "node:url";
5
5
  import { require as api_require } from "tsx/cjs/api";
6
6
  import { tsImport } from "tsx/esm/api";
7
+ import { NodeRegistry, WorkflowError, WorkflowExecutionFailure, WorkflowParseError, WorkflowPublicationError, asExecutionError, buildTestRunReportDump, classifyWorkflowDocumentFormat, collectTestRunReportSources, createProjectRuntime, isFatalDeviceError, normalizeSteps, runConcurrentJobs, runDocumentAttempts, runWorkflowDocument } from "@midscene/core/internal/test-runner";
8
+ import { globSync } from "tinyglobby";
9
+ import { JSON_SCHEMA, load } from "js-yaml";
7
10
  import { createHash, randomUUID } from "node:crypto";
11
+ import { mkdir, writeFile } from "node:fs/promises";
8
12
  import { setMaxListeners } from "node:events";
9
13
  import { TestRunReportAssembler } from "@midscene/core/report";
10
- import { globSync } from "tinyglobby";
11
- import { JSON_SCHEMA, load } from "js-yaml";
12
14
  import { getDebug } from "@midscene/shared/logger";
13
15
  var __webpack_modules__ = {
14
16
  "node:fs" (module) {
@@ -16,6 +18,11 @@ var __webpack_modules__ = {
16
18
  },
17
19
  "node:path" (module) {
18
20
  module.exports = __rspack_external_node_path_c5b9b54f;
21
+ },
22
+ "./package.json" (module) {
23
+ module.exports = {
24
+ rE: "1.13.0"
25
+ };
19
26
  }
20
27
  };
21
28
  var __webpack_module_cache__ = {};
@@ -89,6 +96,7 @@ __webpack_require__.m = __webpack_modules__;
89
96
  })();
90
97
  var external_node_fs_ = __webpack_require__("node:fs");
91
98
  var external_node_path_ = __webpack_require__("node:path");
99
+ var package_0 = __webpack_require__("./package.json");
92
100
  const GENERATED_COMMENT = '<!-- Generated by `midscene-test nodes`. Do not edit directly. -->';
93
101
  const sortNodesForReference = (nodes)=>{
94
102
  const priority = (name)=>'aiAct' === name ? 0 : 'aiAssert' === name ? 1 : 2;
@@ -140,7 +148,9 @@ cases:
140
148
  continue-on-error: false
141
149
  \`\`\`
142
150
 
143
- A Node may map a string step value to one declared input field. Each Node section states whether it supports this shorthand and names the target field.`;
151
+ A Node may map a string step value to one declared input field. Each Node section states whether it supports this shorthand and names the target field.
152
+
153
+ Each Node's returned \`data\` is saved on that Step's result. Results are not named or automatically passed to later Steps. A failed Case does not stop later Cases by default; \`test.bail\` controls early stopping across the run.`;
144
154
  const sortJsonValue = (value)=>{
145
155
  if (Array.isArray(value)) return value.map(sortJsonValue);
146
156
  if ('object' != typeof value || null === value) return value;
@@ -218,275 +228,7 @@ const renderNodeReference = (nodes, location)=>{
218
228
  warnings
219
229
  };
220
230
  };
221
- function _define_property(obj, key, value) {
222
- if (key in obj) Object.defineProperty(obj, key, {
223
- value: value,
224
- enumerable: true,
225
- configurable: true,
226
- writable: true
227
- });
228
- else obj[key] = value;
229
- return obj;
230
- }
231
- class WorkflowError extends Error {
232
- toJSON() {
233
- return {
234
- name: this.name,
235
- message: this.message,
236
- code: this.code,
237
- ...void 0 === this.details ? {} : {
238
- details: this.details
239
- }
240
- };
241
- }
242
- constructor(message, options = {}){
243
- super(message, {
244
- cause: options.cause
245
- }), _define_property(this, "code", void 0), _define_property(this, "details", void 0);
246
- this.name = new.target.name;
247
- this.code = options.code ?? 'WORKFLOW_ERROR';
248
- this.details = options.details;
249
- }
250
- }
251
- class WorkflowParseError extends WorkflowError {
252
- constructor(message, details, cause){
253
- super(message, {
254
- code: 'WORKFLOW_PARSE_ERROR',
255
- details,
256
- cause
257
- });
258
- }
259
- }
260
- class NodeDefinitionError extends WorkflowError {
261
- constructor(message, details){
262
- super(message, {
263
- code: 'NODE_DEFINITION_ERROR',
264
- details
265
- });
266
- }
267
- }
268
- class DuplicateNodeError extends WorkflowError {
269
- constructor(node){
270
- super(`Node "${node}" is already registered.`, {
271
- code: 'DUPLICATE_NODE',
272
- details: {
273
- node
274
- }
275
- }), _define_property(this, "node", void 0);
276
- this.node = node;
277
- }
278
- }
279
- class NodeNotFoundError extends WorkflowError {
280
- constructor(node){
281
- super(`Node "${node}" is not registered.`, {
282
- code: 'NODE_NOT_FOUND',
283
- details: {
284
- node
285
- }
286
- }), _define_property(this, "node", void 0);
287
- this.node = node;
288
- }
289
- }
290
- class NodeInputValidationError extends WorkflowError {
291
- static fromZod(node, error) {
292
- const issues = error.issues.map((issue)=>({
293
- code: issue.code,
294
- path: issue.path.map(String).join('.'),
295
- message: issue.message
296
- }));
297
- const firstIssue = issues[0];
298
- const path = firstIssue?.path || '<root>';
299
- const message = firstIssue?.message ?? 'invalid input';
300
- return new NodeInputValidationError(`Node "${node}" input validation failed at "${path}": ${message}`, {
301
- node,
302
- issues
303
- });
304
- }
305
- constructor(message, details){
306
- super(message, {
307
- code: 'NODE_INPUT_VALIDATION_ERROR',
308
- details
309
- });
310
- }
311
- }
312
- class StepTimeoutError extends WorkflowError {
313
- constructor(timeoutMs, node){
314
- super(node ? `Node "${node}" timed out after ${timeoutMs}ms.` : `Step timed out after ${timeoutMs}ms.`, {
315
- code: 'STEP_TIMEOUT',
316
- details: {
317
- timeoutMs,
318
- ...void 0 === node ? {} : {
319
- node
320
- }
321
- }
322
- }), _define_property(this, "timeoutMs", void 0), _define_property(this, "node", void 0);
323
- this.timeoutMs = timeoutMs;
324
- this.node = node;
325
- }
326
- }
327
- class NodeExecutionError extends WorkflowError {
328
- constructor(node, cause){
329
- const causeMessage = cause instanceof Error ? cause.message : String(cause ?? 'Unknown error');
330
- super(`Node "${node}" failed: ${causeMessage}`, {
331
- code: 'NODE_EXECUTION_ERROR',
332
- details: {
333
- node
334
- },
335
- cause
336
- }), _define_property(this, "node", void 0);
337
- this.node = node;
338
- }
339
- }
340
- class WorkflowLifecycleError extends WorkflowError {
341
- constructor(message, details){
342
- super(message, {
343
- code: 'WORKFLOW_LIFECYCLE_ERROR',
344
- details
345
- });
346
- }
347
- }
348
- class ProjectSetupError extends WorkflowError {
349
- constructor(cause, details){
350
- const causeMessage = cause instanceof Error ? cause.message : String(cause ?? 'Unknown error');
351
- super(`Project "${details.projectName}" setup failed: ${causeMessage}`, {
352
- code: 'PROJECT_SETUP_ERROR',
353
- details,
354
- cause
355
- });
356
- }
357
- }
358
- class ProjectTeardownError extends WorkflowError {
359
- constructor(cause, details){
360
- const causeMessage = cause instanceof Error ? cause.message : String(cause ?? 'Unknown error');
361
- super(`Project "${details.projectName}" teardown failed: ${causeMessage}`, {
362
- code: 'PROJECT_TEARDOWN_ERROR',
363
- details,
364
- cause
365
- });
366
- }
367
- }
368
- class NodeScopeTeardownError extends WorkflowError {
369
- constructor(cause, details){
370
- const causeMessage = cause instanceof Error ? cause.message : String(cause ?? 'Unknown error');
371
- super(`${'case' === details.scope ? 'Case attempt' : 'Workflow document'} node teardown failed for "${details.node}": ${causeMessage}`, {
372
- code: 'NODE_SCOPE_TEARDOWN_ERROR',
373
- details,
374
- cause
375
- });
376
- }
377
- }
378
- class FatalDeviceError extends WorkflowError {
379
- constructor(message, cause){
380
- super(message, {
381
- code: 'FATAL_DEVICE_ERROR',
382
- cause
383
- });
384
- }
385
- }
386
- const isFatalDeviceError = (error)=>{
387
- if (error instanceof FatalDeviceError) return true;
388
- if (error instanceof WorkflowError && 'FATAL_DEVICE_ERROR' === error.code) return true;
389
- if (error instanceof Error && /device offline|device not found|(?:adb|bdc|device) connection (?:was )?closed/i.test(error.message)) return true;
390
- return error instanceof Error && void 0 !== error.cause ? isFatalDeviceError(error.cause) : false;
391
- };
392
- function normalizeNodeExecutionError(error, node) {
393
- return error instanceof WorkflowError ? error : new NodeExecutionError(node, error);
394
- }
395
- const validateOptionalText = (value, field, node)=>{
396
- if (void 0 !== value && ('string' != typeof value || 0 === value.trim().length)) throw new NodeDefinitionError(`Node "${node}" ${field} must be a non-empty string.`, {
397
- node,
398
- field
399
- });
400
- };
401
- const validateInputSchema = (schema, node)=>{
402
- if (void 0 === schema) return;
403
- if (!(schema instanceof z.ZodObject)) throw new NodeDefinitionError(`Node "${node}" inputSchema must be a Zod object schema.`, {
404
- node,
405
- field: 'inputSchema'
406
- });
407
- if ('$' in schema.shape) throw new NodeDefinitionError(`Node "${node}" inputSchema must not declare "$" as an input property.`, {
408
- node,
409
- field: 'inputSchema.$'
410
- });
411
- };
412
- const validateDefinition = (options)=>{
413
- if (!options || 'object' != typeof options) throw new NodeDefinitionError('Node definition must be an object.');
414
- if ('string' != typeof options.name || 0 === options.name.trim().length) throw new NodeDefinitionError('Node name must be a non-empty string.');
415
- validateOptionalText(options.title, 'title', options.name);
416
- validateOptionalText(options.description, "description", options.name);
417
- if (void 0 !== options.stringInputKey && false !== options.stringInputKey && ('string' != typeof options.stringInputKey || 0 === options.stringInputKey.trim().length)) throw new NodeDefinitionError(`Node "${options.name}" stringInputKey must be a non-empty string or false.`, {
418
- node: options.name,
419
- field: 'stringInputKey'
420
- });
421
- validateInputSchema(options.inputSchema, options.name);
422
- if ('string' == typeof options.stringInputKey && options.inputSchema instanceof z.ZodObject && !(options.stringInputKey in options.inputSchema.shape)) throw new NodeDefinitionError(`Node "${options.name}" stringInputKey must reference a field in inputSchema.`, {
423
- node: options.name,
424
- field: 'stringInputKey'
425
- });
426
- if ('function' != typeof options.execute) throw new NodeDefinitionError(`Node "${options.name}" must provide an execute function.`, {
427
- node: options.name
428
- });
429
- };
430
- function defineNode(options) {
431
- validateDefinition(options);
432
- return options;
433
- }
434
- function _check_private_redeclaration(obj, privateCollection) {
435
- if (privateCollection.has(obj)) throw new TypeError("Cannot initialize the same private elements twice on an object");
436
- }
437
- function _class_apply_descriptor_get(receiver, descriptor) {
438
- if (descriptor.get) return descriptor.get.call(receiver);
439
- return descriptor.value;
440
- }
441
- function _class_extract_field_descriptor(receiver, privateMap, action) {
442
- if (!privateMap.has(receiver)) throw new TypeError("attempted to " + action + " private field on non-instance");
443
- return privateMap.get(receiver);
444
- }
445
- function _class_private_field_get(receiver, privateMap) {
446
- var descriptor = _class_extract_field_descriptor(receiver, privateMap, "get");
447
- return _class_apply_descriptor_get(receiver, descriptor);
448
- }
449
- function _class_private_field_init(obj, privateMap, value) {
450
- _check_private_redeclaration(obj, privateMap);
451
- privateMap.set(obj, value);
452
- }
453
- var _nodes = /*#__PURE__*/ new WeakMap();
454
- class NodeRegistry {
455
- register(node) {
456
- defineNode(node);
457
- if (_class_private_field_get(this, _nodes).has(node.name)) throw new DuplicateNodeError(node.name);
458
- _class_private_field_get(this, _nodes).set(node.name, node);
459
- return this;
460
- }
461
- get(name) {
462
- return _class_private_field_get(this, _nodes).get(name);
463
- }
464
- require(name) {
465
- const node = this.get(name);
466
- if (!node) throw new NodeNotFoundError(name);
467
- return node;
468
- }
469
- has(name) {
470
- return _class_private_field_get(this, _nodes).has(name);
471
- }
472
- names() {
473
- return [
474
- ..._class_private_field_get(this, _nodes).keys()
475
- ];
476
- }
477
- definitions() {
478
- return [
479
- ..._class_private_field_get(this, _nodes).values()
480
- ];
481
- }
482
- constructor(nodes = []){
483
- _class_private_field_init(this, _nodes, {
484
- writable: true,
485
- value: new Map()
486
- });
487
- for (const node of nodes)this.register(node);
488
- }
489
- }
231
+ const registry_NodeRegistry = NodeRegistry;
490
232
  const projectIdFromIndex = (index)=>`project-${index}`;
491
233
  const getDefaultExport = (loaded, absolutePath)=>{
492
234
  if ('object' == typeof loaded && null !== loaded && 'default' in loaded) {
@@ -521,9 +263,13 @@ const validatePatterns = (value, field, label = 'files', allowEmptyInclude = fal
521
263
  return pattern;
522
264
  });
523
265
  };
524
- const validateTestFileSelection = (value, label = 'files')=>{
266
+ const test_project_validateTestFileSelection = (value, label = 'files')=>{
525
267
  if (void 0 === value) return;
526
268
  if (!isRecord(value)) throw new TypeError(`Midscene config ${label} must be an object.`);
269
+ rejectUnknownKeys(value, [
270
+ 'include',
271
+ 'exclude'
272
+ ], label);
527
273
  const include = validatePatterns(value.include, 'include', label);
528
274
  const exclude = void 0 === value.exclude ? void 0 : validatePatterns(value.exclude, 'exclude', label);
529
275
  return Object.freeze({
@@ -579,7 +325,7 @@ const validateVariables = (value, label)=>{
579
325
  assertJsonValue(value, label, new Set());
580
326
  return deepFreezeJson(value);
581
327
  };
582
- const validateProjectSetup = (value, label)=>{
328
+ const validateSetup = (value, label)=>{
583
329
  if (void 0 === value) return;
584
330
  if (!isRecord(value)) throw new TypeError(`Midscene config ${label} must be an object.`);
585
331
  rejectUnknownKeys(value, [
@@ -602,7 +348,7 @@ const validateNonNegativeInteger = (value, fallback, label)=>{
602
348
  };
603
349
  const validateExecutionProjects = (value, defaultSetup, globalNodes)=>{
604
350
  if (void 0 === value) {
605
- const setup = validateProjectSetup(defaultSetup, 'setup');
351
+ const setup = validateSetup(defaultSetup, 'setup');
606
352
  return {
607
353
  hasExplicitProjects: false,
608
354
  projects: Object.freeze([
@@ -618,7 +364,7 @@ const validateExecutionProjects = (value, defaultSetup, globalNodes)=>{
618
364
  }),
619
365
  retry: 0,
620
366
  variables: Object.freeze({}),
621
- nodes: new NodeRegistry(globalNodes.definitions())
367
+ nodes: new registry_NodeRegistry(globalNodes.definitions())
622
368
  })
623
369
  ])
624
370
  };
@@ -641,10 +387,10 @@ const validateExecutionProjects = (value, defaultSetup, globalNodes)=>{
641
387
  if ('string' != typeof candidate.name || 0 === candidate.name.trim().length) throw new TypeError(`Midscene config ${label}.name must be non-empty.`);
642
388
  if (names.has(candidate.name)) throw new TypeError(`Midscene config project name "${candidate.name}" must be unique.`);
643
389
  names.add(candidate.name);
644
- const files = validateTestFileSelection(candidate.files, `${label}.files`);
390
+ const files = test_project_validateTestFileSelection(candidate.files, `${label}.files`);
645
391
  if (void 0 !== candidate.nodes && !Array.isArray(candidate.nodes)) throw new TypeError(`Midscene config ${label}.nodes must be an array.`);
646
- const localNodes = new NodeRegistry(candidate.nodes);
647
- const nodes = new NodeRegistry([
392
+ const localNodes = new registry_NodeRegistry(candidate.nodes);
393
+ const nodes = new registry_NodeRegistry([
648
394
  ...globalNodes.definitions().filter((node)=>!localNodes.has(node.name)),
649
395
  ...localNodes.definitions()
650
396
  ]);
@@ -659,7 +405,7 @@ const validateExecutionProjects = (value, defaultSetup, globalNodes)=>{
659
405
  retry: validateNonNegativeInteger(candidate.retry, 0, `${label}.retry`),
660
406
  variables: validateVariables(candidate.variables, `${label}.variables`),
661
407
  ...void 0 === candidate.setup ? {} : {
662
- setup: validateProjectSetup(candidate.setup, `${label}.setup`)
408
+ setup: validateSetup(candidate.setup, `${label}.setup`)
663
409
  }
664
410
  });
665
411
  });
@@ -707,7 +453,7 @@ const validateTestProjectDefinition = (definition)=>{
707
453
  'output',
708
454
  'nodes'
709
455
  ], 'root');
710
- const nodes = new NodeRegistry(definition.nodes);
456
+ const nodes = new registry_NodeRegistry(definition.nodes);
711
457
  const resolvedProjects = validateExecutionProjects(definition.projects, definition.setup, nodes);
712
458
  return {
713
459
  ...resolvedProjects,
@@ -754,857 +500,133 @@ async function loadTestProject(configPath) {
754
500
  }
755
501
  return validateTestProjectDefinition(getDefaultExport(loaded, absolutePath));
756
502
  }
757
- const createProjectRuntime = (options)=>{
758
- const startedAt = new Date();
759
- const controller = new AbortController();
760
- const parentSignal = options.signal;
761
- const abortFromParent = ()=>controller.abort(parentSignal?.reason);
762
- if (parentSignal?.aborted) abortFromParent();
763
- else parentSignal?.addEventListener('abort', abortFromParent, {
764
- once: true
503
+ const createCaseInvocationId = (projectId, sourcePath, caseIndex, invocationIndex = 0)=>createHash('sha256').update(JSON.stringify([
504
+ projectId,
505
+ sourcePath,
506
+ caseIndex,
507
+ ...invocationIndex ? [
508
+ invocationIndex
509
+ ] : []
510
+ ])).digest('hex');
511
+ const createDocumentInvocationId = (projectId, sourcePath, invocationIndex = 0)=>createHash('sha256').update(JSON.stringify([
512
+ projectId,
513
+ sourcePath,
514
+ ...invocationIndex ? [
515
+ invocationIndex
516
+ ] : []
517
+ ])).digest('hex');
518
+ const normalize_normalizeSteps = normalizeSteps;
519
+ const interpolationExpression = /\$\{\{([A-Za-z_][A-Za-z0-9_]*)\}\}|\$\{([^{}]+)\}/g;
520
+ const exactEnvExpression = /^\$\{\{([A-Za-z_][A-Za-z0-9_]*)\}\}$/;
521
+ const exactProjectExpression = /^\$\{([^{}]+)\}$/;
522
+ const missingVariable = (kind, name, options)=>{
523
+ throw new WorkflowParseError(`Undefined ${kind} variable "${name}" in ${options.location.sourcePath}.`, {
524
+ ...options.location,
525
+ variable: name,
526
+ variableKind: kind
765
527
  });
766
- const teardownStack = [];
767
- let context;
768
- let setupError;
769
- let started = false;
770
- let acceptingTeardowns = void 0 !== options.setup;
771
- let finishedResult;
772
- const createResult = (requestedStatus, teardownErrors = [])=>{
773
- const endedAt = new Date();
774
- return {
775
- projectName: options.project.name,
776
- status: 'failed' === requestedStatus || setupError || teardownErrors.length > 0 ? 'failed' : 'success',
777
- ...setupError ? {
778
- setupError
779
- } : {},
780
- ...teardownErrors.length > 0 ? {
781
- teardownErrors
782
- } : {},
783
- startedAt: startedAt.toISOString(),
784
- endedAt: endedAt.toISOString(),
785
- durationMs: Math.max(0, endedAt.getTime() - startedAt.getTime())
786
- };
787
- };
788
- const onTeardown = (teardown)=>{
789
- if (!acceptingTeardowns) throw new WorkflowLifecycleError('onTeardown() can only be called while Project setup is running.', {
790
- projectName: options.project.name
791
- });
792
- if ('function' != typeof teardown) throw new WorkflowLifecycleError('Project onTeardown() requires a teardown function.', {
793
- projectName: options.project.name
794
- });
795
- teardownStack.push({
796
- registrationIndex: teardownStack.length,
797
- teardown
798
- });
799
- };
800
- return {
801
- get context () {
802
- return context;
803
- },
804
- get signal () {
805
- return controller.signal;
806
- },
807
- get canRun () {
808
- return started && !setupError && !controller.signal.aborted;
809
- },
810
- async start () {
811
- if (started) throw new WorkflowLifecycleError('Project runtime has already started.', {
812
- projectName: options.project.name
813
- });
814
- started = true;
815
- if (controller.signal.aborted) {
816
- acceptingTeardowns = false;
817
- return createResult('failed');
818
- }
819
- if (options.setup) try {
820
- context = await options.setup.setup({
821
- project: options.project,
822
- env: Object.freeze({
823
- ...process.env
824
- }),
825
- signal: controller.signal,
826
- onTeardown
827
- });
828
- } catch (error) {
829
- setupError = new ProjectSetupError(error, {
830
- projectName: options.project.name
831
- });
832
- } finally{
833
- acceptingTeardowns = false;
834
- }
835
- return createResult(setupError ? 'failed' : 'success');
836
- },
837
- async finish (requestedStatus = 'success') {
838
- if (finishedResult) return finishedResult;
839
- if (!started) throw new WorkflowLifecycleError('Project runtime must start before it can finish.', {
840
- projectName: options.project.name
841
- });
842
- const statusBeforeTeardown = createResult(requestedStatus).status;
843
- const teardownErrors = [];
844
- for (const { registrationIndex, teardown } of teardownStack.reverse())try {
845
- await teardown({
846
- project: options.project,
847
- context,
848
- status: statusBeforeTeardown,
849
- ...setupError ? {
850
- setupError
851
- } : {}
852
- });
853
- } catch (error) {
854
- teardownErrors.push(new ProjectTeardownError(error, {
855
- projectName: options.project.name,
856
- registrationIndex
857
- }));
858
- }
859
- parentSignal?.removeEventListener('abort', abortFromParent);
860
- finishedResult = createResult(requestedStatus, teardownErrors);
861
- return finishedResult;
862
- },
863
- abort (reason) {
864
- controller.abort(reason);
865
- }
866
- };
867
528
  };
868
- const assertJsonSerializable = (value, label, node)=>{
869
- const visit = (candidate, path, seen)=>{
870
- if (null === candidate || 'string' == typeof candidate || 'boolean' == typeof candidate) return;
871
- if ('number' == typeof candidate) {
872
- if (Number.isFinite(candidate)) return;
873
- throw new TypeError(`${path} contains a non-finite number.`);
874
- }
875
- if (void 0 === candidate) throw new TypeError(`${path} contains undefined.`);
876
- if ('object' != typeof candidate) throw new TypeError(`${path} is not JSON-serializable.`);
877
- if (!Array.isArray(candidate) && Object.getPrototypeOf(candidate) !== Object.prototype && null !== Object.getPrototypeOf(candidate)) throw new TypeError(`${path} must contain only plain objects.`);
878
- if (seen.has(candidate)) throw new TypeError(`${path} contains a cycle.`);
879
- seen.add(candidate);
880
- if (Array.isArray(candidate)) candidate.forEach((child, index)=>visit(child, `${path}[${index}]`, seen));
881
- else for (const [key, child] of Object.entries(candidate))visit(child, `${path}.${key}`, seen);
882
- seen.delete(candidate);
883
- };
884
- try {
885
- visit(value, label, new Set());
886
- } catch (error) {
887
- throw new NodeExecutionError(node, error);
888
- }
529
+ const projectVariable = (name, options)=>{
530
+ if (!Object.hasOwn(options.variables ?? {}, name)) return missingVariable('project', name, options);
531
+ return options.variables[name];
889
532
  };
890
- function validateNodeOutput(output, node) {
891
- if (void 0 === output) return;
892
- if ('object' != typeof output || null === output || Array.isArray(output)) throw new NodeExecutionError(node, new TypeError('Node output must be an object or undefined.'));
893
- if ('summary' in output && void 0 !== output.summary && 'string' != typeof output.summary) throw new NodeExecutionError(node, new TypeError('Node output summary must be a string.'));
894
- if ('data' in output && void 0 !== output.data) assertJsonSerializable(output.data, 'output.data', node);
895
- return output;
896
- }
897
- function createStepResultBase(step, startedAt, phase, stepIndex) {
898
- const endedAt = new Date();
899
- return {
900
- node: step.node,
901
- phase,
902
- stepIndex,
903
- input: step.input,
904
- meta: step.meta,
905
- startedAt: startedAt.toISOString(),
906
- endedAt: endedAt.toISOString(),
907
- durationMs: Math.max(0, endedAt.getTime() - startedAt.getTime())
908
- };
909
- }
910
- async function executeNode(step, execute, phase, stepIndex, options) {
911
- const startedAt = new Date();
912
- const abortController = new AbortController();
913
- const parentSignal = options.parentSignal;
914
- const abortFromParent = ()=>abortController.abort(parentSignal?.reason);
915
- if (parentSignal?.aborted) abortFromParent();
916
- else parentSignal?.addEventListener('abort', abortFromParent, {
917
- once: true
918
- });
919
- const timeoutMs = step.meta.timeoutMs ?? options.defaultTimeoutMs;
920
- let timeout;
921
- let abortGraceTimeout;
922
- const execution = Promise.resolve().then(()=>{
923
- abortController.signal.throwIfAborted();
924
- return execute(abortController.signal);
533
+ const environmentVariable = (name, options)=>{
534
+ const value = options.env?.[name];
535
+ if (void 0 === value) return missingVariable('environment', name, options);
536
+ return value;
537
+ };
538
+ const interpolationValue = (name, value, options)=>{
539
+ if ('object' == typeof value && null !== value) throw new WorkflowParseError(`Project variable "${name}" must be a primitive when embedded in a string.`, {
540
+ ...options.location,
541
+ variable: name,
542
+ variableKind: 'project'
925
543
  });
926
- let settleOnAbort;
927
- const abortedExecution = new Promise((_, reject)=>{
928
- settleOnAbort = ()=>{
929
- abortGraceTimeout = setTimeout(()=>{
930
- reject(abortController.signal.reason);
931
- }, 0);
932
- };
933
- if (abortController.signal.aborted) settleOnAbort();
934
- else abortController.signal.addEventListener('abort', settleOnAbort, {
935
- once: true
936
- });
544
+ return String(value);
545
+ };
546
+ const resolveString = (value, options)=>{
547
+ const exactEnv = exactEnvExpression.exec(value);
548
+ if (exactEnv) return environmentVariable(exactEnv[1], options);
549
+ const exactProject = exactProjectExpression.exec(value);
550
+ if (exactProject) return projectVariable(exactProject[1], options);
551
+ return value.replace(interpolationExpression, (_match, environmentName, projectName)=>environmentName ? environmentVariable(environmentName, options) : interpolationValue(projectName, projectVariable(projectName, options), options));
552
+ };
553
+ const resolveWorkflowVariables = (value, options)=>{
554
+ if ('string' == typeof value) return resolveString(value, options);
555
+ if (Array.isArray(value)) return value.map((child)=>resolveWorkflowVariables(child, options));
556
+ if ('object' == typeof value && null !== value) return Object.fromEntries(Object.entries(value).map(([key, child])=>[
557
+ key,
558
+ resolveWorkflowVariables(child, options)
559
+ ]));
560
+ return value;
561
+ };
562
+ function _define_property(obj, key, value) {
563
+ if (key in obj) Object.defineProperty(obj, key, {
564
+ value: value,
565
+ enumerable: true,
566
+ configurable: true,
567
+ writable: true
937
568
  });
938
- const timeoutExecution = void 0 === timeoutMs ? void 0 : new Promise((_, reject)=>{
939
- timeout = setTimeout(()=>{
940
- if (abortController.signal.aborted) return void reject(abortController.signal.reason);
941
- const timeoutError = new StepTimeoutError(timeoutMs, step.node);
942
- abortController.abort(timeoutError);
943
- reject(timeoutError);
944
- }, timeoutMs);
569
+ else obj[key] = value;
570
+ return obj;
571
+ }
572
+ const isMapping = (value)=>'object' == typeof value && null !== value && !Array.isArray(value);
573
+ class NativeWorkflowFormatError extends WorkflowParseError {
574
+ constructor(actualFormat, sourcePath){
575
+ super('legacy' === actualFormat ? `Legacy tasks/flow YAML is not supported by midscene-test: ${sourcePath}. Run it with \`midscene ${sourcePath}\`, or migrate the file to cases/steps.` : 'legacy-batch' === actualFormat ? `Legacy YAML batch configuration is not a midscene-test workflow document: ${sourcePath}. Run it with \`midscene --config ${sourcePath}\`, or migrate it to a native project config.` : `A workflow file cannot mix legacy tasks/flow with native cases/steps or lifecycle hooks: ${sourcePath}. Keep the formats in separate files and run each file with its matching command.`, {
576
+ sourcePath,
577
+ actualFormat,
578
+ expectedFormat: 'native'
579
+ }), _define_property(this, "actualFormat", void 0);
580
+ this.actualFormat = actualFormat;
581
+ }
582
+ }
583
+ const collect_rejectUnknownKeys = (value, allowed, label)=>{
584
+ const allowedSet = new Set(allowed);
585
+ const unknown = Object.keys(value).filter((key)=>!allowedSet.has(key));
586
+ if (unknown.length) throw new WorkflowParseError(`${label} has unsupported field "${unknown[0]}".`, {
587
+ field: unknown[0]
945
588
  });
946
- const settledExecution = Promise.race([
947
- execution,
948
- abortedExecution,
949
- ...void 0 === timeoutExecution ? [] : [
950
- timeoutExecution
951
- ]
952
- ]);
589
+ };
590
+ const createCaseId = (projectId, sourcePath, caseIndex)=>createCaseInvocationId(projectId, sourcePath, caseIndex);
591
+ const createWorkflowDocumentId = (projectId, sourcePath)=>createDocumentInvocationId(projectId, sourcePath);
592
+ function collectWorkflowDocument(source, options) {
593
+ let parsed;
953
594
  try {
954
- const output = validateNodeOutput(await settledExecution, step.node);
955
- return {
956
- ...createStepResultBase(step, startedAt, phase, stepIndex),
957
- status: 'success',
958
- continuedAfterError: false,
959
- ...void 0 === output ? {} : {
960
- output
961
- }
962
- };
595
+ parsed = load((0, external_node_fs_.readFileSync)(source.absolutePath, 'utf8'), {
596
+ schema: JSON_SCHEMA
597
+ });
963
598
  } catch (error) {
964
- return {
965
- ...createStepResultBase(step, startedAt, phase, stepIndex),
966
- status: 'failed',
967
- continuedAfterError: step.meta.continueOnError,
968
- error: normalizeNodeExecutionError(error, step.node)
969
- };
970
- } finally{
971
- if (void 0 !== timeout) clearTimeout(timeout);
972
- if (void 0 !== abortGraceTimeout) clearTimeout(abortGraceTimeout);
973
- if (void 0 !== settleOnAbort) abortController.signal.removeEventListener('abort', settleOnAbort);
974
- parentSignal?.removeEventListener('abort', abortFromParent);
599
+ throw new WorkflowParseError(`Failed to parse workflow document "${source.sourcePath}".`, {
600
+ sourcePath: source.sourcePath
601
+ }, error);
975
602
  }
976
- }
977
- function createNodeReportCollector(node) {
978
- const traces = new Map();
979
- let accepting = true;
980
- return {
981
- collector: Object.freeze({
982
- addTrace (trace) {
983
- if (!accepting) throw new NodeExecutionError(node, new Error('Report traces can only be added while the current Step is running.'));
984
- if (!trace || 'object' != typeof trace || 'midscene-execution' !== trace.type) throw new NodeExecutionError(node, new TypeError('Report trace type must be "midscene-execution".'));
985
- if ('string' != typeof trace.executionId || 0 === trace.executionId.trim().length) throw new NodeExecutionError(node, new TypeError('Report trace executionId must be non-empty.'));
986
- const normalized = Object.freeze({
987
- type: 'midscene-execution',
988
- executionId: trace.executionId
989
- });
990
- traces.set(`${normalized.type}:${normalized.executionId}`, normalized);
991
- }
992
- }),
993
- close () {
994
- accepting = false;
995
- return Object.freeze([
996
- ...traces.values()
997
- ]);
998
- }
999
- };
1000
- }
1001
- async function parseNodeInput(node, input) {
1002
- if (!node.inputSchema) return input;
1003
- const parsed = await node.inputSchema.safeParseAsync(input);
1004
- if (!parsed.success) throw NodeInputValidationError.fromZod(node.name, parsed.error);
1005
- return parsed.data;
1006
- }
1007
- async function executeStep(step, node, target, context, execution = {}) {
1008
- const phase = 'case' === target.scope ? target.case.phase : target.document.phase;
1009
- const stepIndex = 'case' === target.scope ? target.case.stepIndex : target.document.stepIndex;
1010
- const reportCollector = createNodeReportCollector(step.node);
1011
- let result;
1012
- try {
1013
- result = await executeNode(step, async (signal)=>{
1014
- const input = await parseNodeInput(node, step.input);
1015
- signal.throwIfAborted();
1016
- const common = {
1017
- input,
1018
- $: step.meta,
1019
- signal,
1020
- context,
1021
- report: reportCollector.collector,
1022
- onTeardown: (teardown)=>{
1023
- if (!execution.onTeardown) throw new NodeExecutionError(step.node, new Error('The current execution scope cannot register teardown.'));
1024
- execution.onTeardown(step.node, teardown);
603
+ if (!isMapping(parsed)) throw new WorkflowParseError('Workflow document must be a mapping.');
604
+ const format = classifyWorkflowDocumentFormat(parsed);
605
+ if ('legacy' === format || 'legacy-batch' === format || 'mixed' === format) throw new NativeWorkflowFormatError(format, source.sourcePath);
606
+ collect_rejectUnknownKeys(parsed, [
607
+ 'beforeAll',
608
+ 'beforeEach',
609
+ 'cases',
610
+ 'afterEach',
611
+ 'afterAll'
612
+ ], 'Workflow document');
613
+ if (!Array.isArray(parsed.cases) || 0 === parsed.cases.length) throw new WorkflowParseError('Workflow document must contain a non-empty cases array.');
614
+ const sourcePath = source.sourcePath.split(external_node_path_.sep).join('/');
615
+ const resolveStepVariables = (step, phase, stepIndex, caseIndex)=>({
616
+ ...step,
617
+ input: resolveWorkflowVariables(step.input, {
618
+ variables: options.variables,
619
+ env: options.env ?? process.env,
620
+ location: {
621
+ projectName: source.projectName ?? source.projectId,
622
+ sourcePath,
623
+ phase,
624
+ stepIndex,
625
+ ...void 0 === caseIndex ? {} : {
626
+ caseIndex
627
+ }
1025
628
  }
1026
- };
1027
- return 'case' === target.scope ? node.execute({
1028
- ...common,
1029
- scope: 'case',
1030
- case: target.case
1031
- }) : node.execute({
1032
- ...common,
1033
- scope: 'document',
1034
- document: target.document
1035
- });
1036
- }, phase, stepIndex, {
1037
- parentSignal: execution.signal,
1038
- defaultTimeoutMs: execution.defaultTimeoutMs
1039
- });
1040
- } finally{
1041
- const traces = reportCollector.close();
1042
- if (traces.length > 0 && result) result = {
1043
- ...result,
1044
- report: {
1045
- traces
1046
- }
1047
- };
1048
- }
1049
- if (!result) throw new NodeExecutionError(step.node, new Error('Step execution completed without a result.'));
1050
- return result;
1051
- }
1052
- const reportPathsFromTeardown = (result)=>{
1053
- if (void 0 === result) return [];
1054
- if ('object' != typeof result || null === result || Array.isArray(result)) throw new TypeError('Node teardown result must be an object or undefined.');
1055
- const paths = result.reportPaths;
1056
- if (void 0 === paths) return [];
1057
- if (!Array.isArray(paths)) throw new TypeError('Node teardown reportPaths must be an array.');
1058
- return paths.map((path, index)=>{
1059
- if ('string' != typeof path || 0 === path.trim().length) throw new TypeError(`Node teardown reportPaths[${index}] must be a non-empty string.`);
1060
- if (!(0, external_node_path_.isAbsolute)(path)) throw new TypeError(`Node teardown reportPaths[${index}] must be an absolute path.`);
1061
- return path;
1062
- });
1063
- };
1064
- function createDocumentRuntime(document, options) {
1065
- const documentRunId = options.createDocumentRunId?.() ?? randomUUID();
1066
- const startedAt = new Date();
1067
- const beforeAll = [];
1068
- const afterAll = [];
1069
- const nodes = {
1070
- beforeAll: document.lifecycle.beforeAll.map((step)=>options.resolveNode(step.node)),
1071
- afterAll: document.lifecycle.afterAll.map((step)=>options.resolveNode(step.node))
1072
- };
1073
- const project = options.project ?? Object.freeze({
1074
- projectId: document.projectId,
1075
- name: document.projectId,
1076
- tags: Object.freeze({
1077
- include: [],
1078
- exclude: []
1079
- }),
1080
- retry: 0,
1081
- variables: Object.freeze({})
1082
- });
1083
- const context = options.projectContext;
1084
- const signal = options.signal ?? new AbortController().signal;
1085
- const nodeTeardownStack = [];
1086
- let acceptingNodeTeardowns = true;
1087
- let started = false;
1088
- let finishedResult;
1089
- const reportPaths = new Set();
1090
- const createResult = (teardownErrors = [])=>{
1091
- const endedAt = new Date();
1092
- return {
1093
- documentId: document.documentId,
1094
- documentRunId,
1095
- projectId: document.projectId,
1096
- projectName: project.name,
1097
- sourcePath: document.sourcePath,
1098
- status: beforeAll.some((step)=>'failed' === step.status) || afterAll.some((step)=>'failed' === step.status) || teardownErrors.length > 0 ? 'failed' : 'success',
1099
- startedAt: startedAt.toISOString(),
1100
- endedAt: endedAt.toISOString(),
1101
- durationMs: Math.max(0, endedAt.getTime() - startedAt.getTime()),
1102
- beforeAll: [
1103
- ...beforeAll
1104
- ],
1105
- afterAll: [
1106
- ...afterAll
1107
- ],
1108
- ...teardownErrors.length > 0 ? {
1109
- teardownErrors
1110
- } : {},
1111
- ...reportPaths.size > 0 ? {
1112
- reportPaths: [
1113
- ...reportPaths
1114
- ]
1115
- } : {}
1116
- };
1117
- };
1118
- const runPhase = async (phase, results, executionSignal = signal)=>{
1119
- for (const [stepIndex, step] of document.lifecycle[phase].entries()){
1120
- const documentContext = {
1121
- documentId: document.documentId,
1122
- documentRunId,
1123
- projectId: document.projectId,
1124
- projectName: project.name,
1125
- sourcePath: document.sourcePath,
1126
- phase,
1127
- stepIndex
1128
- };
1129
- const stepInfo = {
1130
- scope: 'document',
1131
- node: step.node,
1132
- stepCount: document.lifecycle[phase].length,
1133
- document: documentContext
1134
- };
1135
- await options.onStepStart?.(stepInfo);
1136
- const result = await executeStep(step, nodes[phase][stepIndex], {
1137
- scope: 'document',
1138
- document: documentContext
1139
- }, context, {
1140
- signal: executionSignal,
1141
- defaultTimeoutMs: options.defaultTimeoutMs,
1142
- onTeardown (node, teardown) {
1143
- if (!acceptingNodeTeardowns) throw new WorkflowLifecycleError('Node teardown can only be registered while a workflow document is running.', {
1144
- documentId: document.documentId,
1145
- documentRunId,
1146
- node
1147
- });
1148
- if ('function' != typeof teardown) throw new WorkflowLifecycleError('Node onTeardown() requires a teardown function.', {
1149
- documentId: document.documentId,
1150
- documentRunId,
1151
- node
1152
- });
1153
- nodeTeardownStack.push({
1154
- registrationIndex: nodeTeardownStack.length,
1155
- node,
1156
- teardown
1157
- });
1158
- }
1159
- });
1160
- await options.onStepResult?.(stepInfo, result);
1161
- results.push(result);
1162
- if ('failed' === result.status && !result.continuedAfterError) break;
1163
- }
1164
- };
1165
- return {
1166
- get context () {
1167
- return context;
1168
- },
1169
- get canRunCases () {
1170
- return started && !signal.aborted && !beforeAll.some((step)=>'failed' === step.status);
1171
- },
1172
- async start () {
1173
- if (started) throw new WorkflowLifecycleError('Workflow document runtime has already started.', {
1174
- documentId: document.documentId,
1175
- documentRunId
1176
- });
1177
- started = true;
1178
- if (!signal.aborted) await runPhase('beforeAll', beforeAll);
1179
- return createResult();
1180
- },
1181
- async finish () {
1182
- if (finishedResult) return finishedResult;
1183
- if (!started) throw new WorkflowLifecycleError('Workflow document runtime must start before it can finish.', {
1184
- documentId: document.documentId,
1185
- documentRunId
1186
- });
1187
- let finishError;
1188
- try {
1189
- await runPhase('afterAll', afterAll, signal.aborted ? new AbortController().signal : signal);
1190
- } catch (error) {
1191
- finishError = error;
1192
- }
1193
- acceptingNodeTeardowns = false;
1194
- const teardownErrors = [];
1195
- for (const { registrationIndex, node, teardown } of nodeTeardownStack.reverse())try {
1196
- for (const path of reportPathsFromTeardown(await teardown()))reportPaths.add(path);
1197
- } catch (error) {
1198
- teardownErrors.push(new NodeScopeTeardownError(error, {
1199
- scope: 'document',
1200
- scopeId: documentRunId,
1201
- node,
1202
- registrationIndex
1203
- }));
1204
- }
1205
- finishedResult = createResult(teardownErrors);
1206
- if (finishError) throw finishError;
1207
- await options.onResult?.(finishedResult);
1208
- return finishedResult;
1209
- }
1210
- };
1211
- }
1212
- async function runCollectedCase(collectedCase, options) {
1213
- const phases = {
1214
- beforeEach: options.beforeEach ?? [],
1215
- steps: collectedCase.definition.steps,
1216
- afterEach: options.afterEach ?? []
1217
- };
1218
- const nodes = {
1219
- beforeEach: phases.beforeEach.map((step)=>options.resolveNode(step.node)),
1220
- steps: phases.steps.map((step)=>options.resolveNode(step.node)),
1221
- afterEach: phases.afterEach.map((step)=>options.resolveNode(step.node))
1222
- };
1223
- const runId = options.createRunId?.() ?? randomUUID();
1224
- const startedAt = new Date();
1225
- const beforeEach = [];
1226
- const steps = [];
1227
- const afterEach = [];
1228
- const teardownStack = [];
1229
- let acceptingTeardowns = true;
1230
- const onTeardown = (node, teardown)=>{
1231
- if (!acceptingTeardowns) throw new WorkflowLifecycleError('Node teardown can only be registered while a case attempt is running.', {
1232
- runId,
1233
- node
1234
- });
1235
- if ('function' != typeof teardown) throw new WorkflowLifecycleError('Node onTeardown() requires a teardown function.', {
1236
- runId,
1237
- node
1238
- });
1239
- teardownStack.push({
1240
- registrationIndex: teardownStack.length,
1241
- node,
1242
- teardown
1243
- });
1244
- };
1245
- const runPhase = async (phase, results, signal = options.signal)=>{
1246
- for (const [stepIndex, step] of phases[phase].entries()){
1247
- const caseContext = {
1248
- caseId: collectedCase.caseId,
1249
- runId,
1250
- projectName: options.projectName ?? collectedCase.projectId,
1251
- attemptIndex: options.attemptIndex ?? 0,
1252
- name: collectedCase.definition.name,
1253
- sourcePath: collectedCase.sourcePath,
1254
- caseIndex: collectedCase.caseIndex,
1255
- phase,
1256
- stepIndex
1257
- };
1258
- const stepInfo = {
1259
- scope: 'case',
1260
- node: step.node,
1261
- stepCount: phases[phase].length,
1262
- case: caseContext
1263
- };
1264
- await options.onStepStart?.(stepInfo);
1265
- const result = await executeStep(step, nodes[phase][stepIndex], {
1266
- scope: 'case',
1267
- case: caseContext
1268
- }, options.context, {
1269
- signal,
1270
- defaultTimeoutMs: options.defaultTimeoutMs,
1271
- onTeardown
1272
- });
1273
- await options.onStepResult?.(stepInfo, result);
1274
- results.push(result);
1275
- if ('failed' === result.status && !result.continuedAfterError) break;
1276
- }
1277
- };
1278
- let executionError;
1279
- try {
1280
- await runPhase('beforeEach', beforeEach);
1281
- if (!beforeEach.some((step)=>'failed' === step.status)) await runPhase('steps', steps);
1282
- } catch (error) {
1283
- executionError = error;
1284
- }
1285
- try {
1286
- const cleanupSignal = options.signal?.aborted ? new AbortController().signal : options.signal;
1287
- await runPhase('afterEach', afterEach, cleanupSignal);
1288
- } catch (error) {
1289
- executionError ??= error;
1290
- }
1291
- acceptingTeardowns = false;
1292
- const teardownErrors = [];
1293
- const reportPaths = new Set();
1294
- for (const { registrationIndex, node, teardown } of teardownStack.reverse())try {
1295
- for (const path of reportPathsFromTeardown(await teardown()))reportPaths.add(path);
1296
- } catch (error) {
1297
- teardownErrors.push(new NodeScopeTeardownError(error, {
1298
- scope: 'case',
1299
- scopeId: runId,
1300
- node,
1301
- registrationIndex
1302
- }));
1303
- }
1304
- if (executionError) throw executionError;
1305
- const endedAt = new Date();
1306
- const result = {
1307
- caseId: collectedCase.caseId,
1308
- runId,
1309
- projectName: options.projectName ?? collectedCase.projectId,
1310
- attemptIndex: options.attemptIndex ?? 0,
1311
- name: collectedCase.definition.name,
1312
- sourcePath: collectedCase.sourcePath,
1313
- caseIndex: collectedCase.caseIndex,
1314
- status: beforeEach.some((step)=>'failed' === step.status) || steps.some((step)=>'failed' === step.status) || afterEach.some((step)=>'failed' === step.status) || teardownErrors.length > 0 ? 'failed' : 'success',
1315
- beforeEach,
1316
- steps,
1317
- afterEach,
1318
- ...teardownErrors.length > 0 ? {
1319
- teardownErrors
1320
- } : {},
1321
- ...reportPaths.size > 0 ? {
1322
- reportPaths: [
1323
- ...reportPaths
1324
- ]
1325
- } : {},
1326
- startedAt: startedAt.toISOString(),
1327
- endedAt: endedAt.toISOString(),
1328
- durationMs: Math.max(0, endedAt.getTime() - startedAt.getTime())
1329
- };
1330
- await options.onResult?.(result);
1331
- return result;
1332
- }
1333
- const asNotRun = (collectedCase, projectName, reason)=>({
1334
- caseId: collectedCase.caseId,
1335
- projectName,
1336
- name: collectedCase.definition.name,
1337
- sourcePath: collectedCase.sourcePath,
1338
- caseIndex: collectedCase.caseIndex,
1339
- status: 'not-run',
1340
- notRunReason: reason
1341
- });
1342
- async function runWorkflowDocument(document, options) {
1343
- const createDocumentRunId = options.createDocumentRunId;
1344
- const createCaseRunId = options.createCaseRunId;
1345
- const projectName = options.project?.name ?? document.projectId;
1346
- const retry = options.retry ?? options.project?.retry ?? 0;
1347
- if (!Number.isInteger(retry) || retry < 0) throw new TypeError('Workflow document retry must be a non-negative integer.');
1348
- const runtime = createDocumentRuntime(document, {
1349
- resolveNode: options.resolveNode,
1350
- project: options.project,
1351
- projectContext: options.projectContext,
1352
- signal: options.signal,
1353
- defaultTimeoutMs: options.defaultTimeoutMs,
1354
- onStepStart: options.onStepStart,
1355
- onStepResult: options.onStepResult,
1356
- onResult: options.onDocumentResult,
1357
- createDocumentRunId: createDocumentRunId ? ()=>createDocumentRunId(document) : void 0
1358
- });
1359
- const cases = [];
1360
- let runtimeStarted = false;
1361
- let documentResult;
1362
- let executionError;
1363
- let fatalError = false;
1364
- try {
1365
- runtimeStarted = true;
1366
- await runtime.start();
1367
- if (runtime.canRunCases) for (const collectedCase of document.cases){
1368
- if (fatalError || options.shouldStop?.()) {
1369
- const outcome = asNotRun(collectedCase, projectName, fatalError ? 'fatal-error' : options.stopReason?.() ?? 'interrupted');
1370
- cases.push(outcome);
1371
- await options.onCaseOutcome?.(outcome);
1372
- continue;
1373
- }
1374
- await options.onCaseStart?.(collectedCase);
1375
- const attempts = [];
1376
- for(let attemptIndex = 0; attemptIndex <= retry; attemptIndex += 1){
1377
- const run = await runCollectedCase(collectedCase, {
1378
- resolveNode: options.resolveNode,
1379
- beforeEach: document.lifecycle.beforeEach,
1380
- afterEach: document.lifecycle.afterEach,
1381
- context: runtime.context,
1382
- projectName,
1383
- attemptIndex,
1384
- signal: options.signal,
1385
- defaultTimeoutMs: options.defaultTimeoutMs,
1386
- onStepStart: options.onStepStart,
1387
- onStepResult: options.onStepResult,
1388
- onResult: options.onCaseResult,
1389
- createRunId: createCaseRunId ? ()=>createCaseRunId(collectedCase, attemptIndex) : void 0
1390
- });
1391
- attempts.push(run);
1392
- fatalError = options.isFatalError?.(run) ?? false;
1393
- if ('success' === run.status || fatalError) break;
1394
- }
1395
- const run = attempts.at(-1);
1396
- const outcome = {
1397
- caseId: collectedCase.caseId,
1398
- projectName,
1399
- name: collectedCase.definition.name,
1400
- sourcePath: collectedCase.sourcePath,
1401
- caseIndex: collectedCase.caseIndex,
1402
- status: run.status,
1403
- run,
1404
- attempts
1405
- };
1406
- cases.push(outcome);
1407
- await options.onCaseOutcome?.(outcome);
1408
- }
1409
- else {
1410
- const reason = options.signal?.aborted ? options.stopReason?.() ?? 'interrupted' : 'document-start-failed';
1411
- cases.push(...document.cases.map((collectedCase)=>asNotRun(collectedCase, projectName, reason)));
1412
- }
1413
- } catch (error) {
1414
- executionError = error;
1415
- } finally{
1416
- if (runtimeStarted) try {
1417
- documentResult = await runtime.finish();
1418
- } catch (error) {
1419
- executionError ??= error;
1420
- }
1421
- }
1422
- if (executionError) throw executionError;
1423
- if (!documentResult) throw new Error(`Workflow document "${document.sourcePath}" did not produce a result.`);
1424
- return {
1425
- document: documentResult,
1426
- cases
1427
- };
1428
- }
1429
- const supportedMetaKeys = new Set([
1430
- 'timeout',
1431
- 'continue-on-error'
1432
- ]);
1433
- function isMapping(value) {
1434
- return 'object' == typeof value && null !== value && !Array.isArray(value) && (Object.getPrototypeOf(value) === Object.prototype || null === Object.getPrototypeOf(value));
1435
- }
1436
- function formatStep(index) {
1437
- return `step ${index + 1}`;
1438
- }
1439
- function normalizeMeta(value, index) {
1440
- if (void 0 === value) return {
1441
- continueOnError: false
1442
- };
1443
- if (!isMapping(value)) throw new WorkflowParseError(`${formatStep(index)} "$" must be a mapping.`, {
1444
- index
1445
- });
1446
- for (const key of Object.keys(value))if (!supportedMetaKeys.has(key)) throw new WorkflowParseError(`${formatStep(index)} has unsupported engine metadata "${key}".`, {
1447
- index,
1448
- key
1449
- });
1450
- const timeout = value.timeout;
1451
- if (void 0 !== timeout && ('number' != typeof timeout || !Number.isFinite(timeout) || timeout <= 0)) throw new WorkflowParseError(`${formatStep(index)} "$.timeout" must be a positive number of milliseconds.`, {
1452
- index,
1453
- timeout
1454
- });
1455
- const continueOnError = value['continue-on-error'];
1456
- if (void 0 !== continueOnError && 'boolean' != typeof continueOnError) throw new WorkflowParseError(`${formatStep(index)} "$.continue-on-error" must be a boolean.`, {
1457
- index,
1458
- continueOnError
1459
- });
1460
- return {
1461
- ...void 0 === timeout ? {} : {
1462
- timeoutMs: timeout
1463
- },
1464
- continueOnError: continueOnError ?? false
1465
- };
1466
- }
1467
- function normalizeStep(value, index = 0, resolveNode) {
1468
- if (!isMapping(value)) throw new WorkflowParseError(`${formatStep(index)} must be a mapping.`, {
1469
- index
1470
- });
1471
- const entries = Object.entries(value);
1472
- if (1 !== entries.length) throw new WorkflowParseError(`${formatStep(index)} must contain exactly one node.`, {
1473
- index,
1474
- nodes: Object.keys(value)
1475
- });
1476
- const [node, rawValue] = entries[0];
1477
- if (0 === node.trim().length) throw new WorkflowParseError(`${formatStep(index)} node name must not be empty.`, {
1478
- index
1479
- });
1480
- if ('string' == typeof rawValue) {
1481
- const resolvedNode = resolveNode?.(node);
1482
- const stringInputKey = resolvedNode ? resolvedNode.stringInputKey : 'prompt';
1483
- if (false === stringInputKey || void 0 === stringInputKey) throw new WorkflowParseError(`${formatStep(index)} node "${node}" does not accept string shorthand.`, {
1484
- index,
1485
- node
1486
- });
1487
- return {
1488
- node,
1489
- input: {
1490
- [stringInputKey]: rawValue
1491
- },
1492
- meta: {
1493
- continueOnError: false
1494
- }
1495
- };
1496
- }
1497
- if (!isMapping(rawValue)) throw new WorkflowParseError(`${formatStep(index)} value must be a string shorthand or a mapping.`, {
1498
- index,
1499
- node
1500
- });
1501
- const { $: rawMeta, ...input } = rawValue;
1502
- return {
1503
- node,
1504
- input,
1505
- meta: normalizeMeta(rawMeta, index)
1506
- };
1507
- }
1508
- function normalizeSteps(steps, resolveNode) {
1509
- if (!Array.isArray(steps)) throw new WorkflowParseError('Steps must be an array.');
1510
- return steps.map((step, index)=>normalizeStep(step, index, resolveNode));
1511
- }
1512
- const interpolationExpression = /\$\{\{([A-Za-z_][A-Za-z0-9_]*)\}\}|\$\{([^{}]+)\}/g;
1513
- const exactEnvExpression = /^\$\{\{([A-Za-z_][A-Za-z0-9_]*)\}\}$/;
1514
- const exactProjectExpression = /^\$\{([^{}]+)\}$/;
1515
- const missingVariable = (kind, name, options)=>{
1516
- throw new WorkflowParseError(`Undefined ${kind} variable "${name}" in ${options.location.sourcePath}.`, {
1517
- ...options.location,
1518
- variable: name,
1519
- variableKind: kind
1520
- });
1521
- };
1522
- const projectVariable = (name, options)=>{
1523
- if (!Object.hasOwn(options.variables ?? {}, name)) return missingVariable('project', name, options);
1524
- return options.variables[name];
1525
- };
1526
- const environmentVariable = (name, options)=>{
1527
- const value = options.env?.[name];
1528
- if (void 0 === value) return missingVariable('environment', name, options);
1529
- return value;
1530
- };
1531
- const interpolationValue = (name, value, options)=>{
1532
- if ('object' == typeof value && null !== value) throw new WorkflowParseError(`Project variable "${name}" must be a primitive when embedded in a string.`, {
1533
- ...options.location,
1534
- variable: name,
1535
- variableKind: 'project'
1536
- });
1537
- return String(value);
1538
- };
1539
- const resolveString = (value, options)=>{
1540
- const exactEnv = exactEnvExpression.exec(value);
1541
- if (exactEnv) return environmentVariable(exactEnv[1], options);
1542
- const exactProject = exactProjectExpression.exec(value);
1543
- if (exactProject) return projectVariable(exactProject[1], options);
1544
- return value.replace(interpolationExpression, (_match, environmentName, projectName)=>environmentName ? environmentVariable(environmentName, options) : interpolationValue(projectName, projectVariable(projectName, options), options));
1545
- };
1546
- const resolveWorkflowVariables = (value, options)=>{
1547
- if ('string' == typeof value) return resolveString(value, options);
1548
- if (Array.isArray(value)) return value.map((child)=>resolveWorkflowVariables(child, options));
1549
- if ('object' == typeof value && null !== value) return Object.fromEntries(Object.entries(value).map(([key, child])=>[
1550
- key,
1551
- resolveWorkflowVariables(child, options)
1552
- ]));
1553
- return value;
1554
- };
1555
- const collect_isMapping = (value)=>'object' == typeof value && null !== value && !Array.isArray(value);
1556
- const collect_rejectUnknownKeys = (value, allowed, label)=>{
1557
- const allowedSet = new Set(allowed);
1558
- const unknown = Object.keys(value).filter((key)=>!allowedSet.has(key));
1559
- if (unknown.length) throw new WorkflowParseError(`${label} has unsupported field "${unknown[0]}".`, {
1560
- field: unknown[0]
1561
- });
1562
- };
1563
- const createCaseId = (projectId, sourcePath, caseIndex)=>createHash('sha256').update(JSON.stringify([
1564
- projectId,
1565
- sourcePath,
1566
- caseIndex
1567
- ])).digest('hex');
1568
- const createWorkflowDocumentId = (projectId, sourcePath)=>createHash('sha256').update(JSON.stringify([
1569
- projectId,
1570
- sourcePath
1571
- ])).digest('hex');
1572
- function collectWorkflowDocument(source, options) {
1573
- let parsed;
1574
- try {
1575
- parsed = load((0, external_node_fs_.readFileSync)(source.absolutePath, 'utf8'), {
1576
- schema: JSON_SCHEMA
1577
- });
1578
- } catch (error) {
1579
- throw new WorkflowParseError(`Failed to parse workflow document "${source.sourcePath}".`, {
1580
- sourcePath: source.sourcePath
1581
- }, error);
1582
- }
1583
- if (!collect_isMapping(parsed)) throw new WorkflowParseError('Workflow document must be a mapping.');
1584
- collect_rejectUnknownKeys(parsed, [
1585
- 'beforeAll',
1586
- 'beforeEach',
1587
- 'cases',
1588
- 'afterEach',
1589
- 'afterAll'
1590
- ], 'Workflow document');
1591
- if (!Array.isArray(parsed.cases) || 0 === parsed.cases.length) throw new WorkflowParseError('Workflow document must contain a non-empty cases array.');
1592
- const sourcePath = source.sourcePath.split(external_node_path_.sep).join('/');
1593
- const resolveStepVariables = (step, phase, stepIndex, caseIndex)=>({
1594
- ...step,
1595
- input: resolveWorkflowVariables(step.input, {
1596
- variables: options.variables,
1597
- env: options.env ?? process.env,
1598
- location: {
1599
- projectName: source.projectName ?? source.projectId,
1600
- sourcePath,
1601
- phase,
1602
- stepIndex,
1603
- ...void 0 === caseIndex ? {} : {
1604
- caseIndex
1605
- }
1606
- }
1607
- })
629
+ })
1608
630
  });
1609
631
  const normalizeLifecycle = (phase)=>{
1610
632
  const value = parsed[phase];
@@ -1612,7 +634,7 @@ function collectWorkflowDocument(source, options) {
1612
634
  if (!Array.isArray(value)) throw new WorkflowParseError(`Workflow document ${phase} must be an array.`, {
1613
635
  phase
1614
636
  });
1615
- return normalizeSteps(value, options.resolveNode).map((normalized, stepIndex)=>{
637
+ return normalize_normalizeSteps(value, options.resolveNode).map((normalized, stepIndex)=>{
1616
638
  const resolved = options.resolveNode(normalized.node);
1617
639
  if (!resolved) throw new WorkflowParseError(`Workflow document ${phase} step ${stepIndex + 1} references unknown node "${normalized.node}".`, {
1618
640
  phase,
@@ -1630,7 +652,7 @@ function collectWorkflowDocument(source, options) {
1630
652
  };
1631
653
  const ids = new Set();
1632
654
  const cases = parsed.cases.map((definition, caseIndex)=>{
1633
- if (!collect_isMapping(definition)) throw new WorkflowParseError(`Case ${caseIndex + 1} must be a mapping.`, {
655
+ if (!isMapping(definition)) throw new WorkflowParseError(`Case ${caseIndex + 1} must be a mapping.`, {
1634
656
  caseIndex
1635
657
  });
1636
658
  collect_rejectUnknownKeys(definition, [
@@ -1648,7 +670,7 @@ function collectWorkflowDocument(source, options) {
1648
670
  if (!Array.isArray(tags) || tags.some((tag)=>'string' != typeof tag || 0 === tag.trim().length)) throw new WorkflowParseError(`Case ${caseIndex + 1} tags must be an array of non-empty strings.`, {
1649
671
  caseIndex
1650
672
  });
1651
- const steps = normalizeSteps(definition.steps, options.resolveNode).map((normalized, stepIndex)=>{
673
+ const steps = normalize_normalizeSteps(definition.steps, options.resolveNode).map((normalized, stepIndex)=>{
1652
674
  if (!options.resolveNode(normalized.node)) throw new WorkflowParseError(`Case ${caseIndex + 1} step ${stepIndex + 1} references unknown node "${normalized.node}".`, {
1653
675
  caseIndex,
1654
676
  stepIndex,
@@ -1659,449 +681,41 @@ function collectWorkflowDocument(source, options) {
1659
681
  const caseId = createCaseId(source.projectId, sourcePath, caseIndex);
1660
682
  if (ids.has(caseId)) throw new WorkflowParseError(`Case id collision: ${caseId}.`, {
1661
683
  caseId
1662
- });
1663
- ids.add(caseId);
1664
- return {
1665
- caseId,
1666
- projectId: source.projectId,
1667
- sourcePath,
1668
- caseIndex,
1669
- definition: {
1670
- name: definition.name,
1671
- tags: tags,
1672
- steps
1673
- }
1674
- };
1675
- });
1676
- return {
1677
- documentId: createWorkflowDocumentId(source.projectId, sourcePath),
1678
- projectId: source.projectId,
1679
- sourcePath,
1680
- lifecycle,
1681
- cases
1682
- };
1683
- }
1684
- const MAX_DEPTH = 6;
1685
- const MAX_STRING_LENGTH = 2000;
1686
- const MAX_ARRAY_ITEMS = 50;
1687
- const MAX_OBJECT_KEYS = 50;
1688
- const MAX_SERIALIZED_BYTES = 65536;
1689
- const TRUNCATED_VALUE = '[Truncated: display value exceeded 64 KiB]';
1690
- const normalizedSecretKeys = [
1691
- 'password',
1692
- 'passwd',
1693
- 'secret',
1694
- 'token',
1695
- 'authorization',
1696
- 'cookie',
1697
- 'apikey'
1698
- ];
1699
- const normalizedKey = (key)=>key.toLowerCase().replace(/[^a-z0-9]/g, '');
1700
- const isSecretKey = (key)=>{
1701
- const normalized = normalizedKey(key);
1702
- return normalizedSecretKeys.some((secret)=>normalized.includes(secret));
1703
- };
1704
- const objectPath = (parent, key)=>/^[A-Za-z_$][\w$]*$/.test(key) ? `${parent}.${key}` : `${parent}[${JSON.stringify(key)}]`;
1705
- const stringByteLength = (value)=>Buffer.byteLength(value, 'utf8');
1706
- function sanitizeReportText(value, maximumLength = MAX_STRING_LENGTH) {
1707
- return value.length <= maximumLength ? value : `${value.slice(0, maximumLength)}… [truncated]`;
1708
- }
1709
- function sanitizeReportValue(value) {
1710
- const truncatedPaths = new Set();
1711
- const redactedPaths = new Set();
1712
- const ancestors = new WeakSet();
1713
- let remainingBytes = MAX_SERIALIZED_BYTES;
1714
- const consume = (path, text)=>{
1715
- remainingBytes -= stringByteLength(text);
1716
- if (remainingBytes >= 0) return true;
1717
- truncatedPaths.add(path);
1718
- return false;
1719
- };
1720
- const visit = (current, path, depth)=>{
1721
- if (remainingBytes <= 0) {
1722
- truncatedPaths.add(path);
1723
- return '[Truncated: size limit]';
1724
- }
1725
- if (null === current || 'boolean' == typeof current) {
1726
- consume(path, String(current));
1727
- return current;
1728
- }
1729
- if ('string' == typeof current) {
1730
- if (current.length > MAX_STRING_LENGTH) {
1731
- truncatedPaths.add(path);
1732
- const shortened = `${current.slice(0, MAX_STRING_LENGTH)}… [truncated]`;
1733
- consume(path, shortened);
1734
- return shortened;
1735
- }
1736
- consume(path, current);
1737
- return current;
1738
- }
1739
- if ('number' == typeof current) {
1740
- if (Number.isFinite(current)) {
1741
- consume(path, String(current));
1742
- return current;
1743
- }
1744
- truncatedPaths.add(path);
1745
- return `[Unsupported number: ${String(current)}]`;
1746
- }
1747
- if ('bigint' == typeof current) {
1748
- truncatedPaths.add(path);
1749
- return `[Unsupported bigint: ${String(current)}]`;
1750
- }
1751
- if (void 0 === current || 'function' == typeof current || 'symbol' == typeof current) {
1752
- truncatedPaths.add(path);
1753
- return `[Unsupported ${typeof current}]`;
1754
- }
1755
- if (depth >= MAX_DEPTH) {
1756
- truncatedPaths.add(path);
1757
- return '[Truncated: maximum depth]';
1758
- }
1759
- if (current instanceof Date) {
1760
- const date = Number.isNaN(current.getTime()) ? '[Invalid Date]' : current.toISOString();
1761
- consume(path, date);
1762
- return date;
1763
- }
1764
- if (ancestors.has(current)) {
1765
- truncatedPaths.add(path);
1766
- return '[Unsupported circular value]';
1767
- }
1768
- if (Array.isArray(current)) {
1769
- ancestors.add(current);
1770
- const output = [];
1771
- const itemCount = Math.min(current.length, MAX_ARRAY_ITEMS);
1772
- for(let index = 0; index < itemCount; index += 1){
1773
- if (remainingBytes <= 0) break;
1774
- output.push(visit(current[index], `${path}[${index}]`, depth + 1));
1775
- }
1776
- if (current.length > output.length) truncatedPaths.add(path);
1777
- ancestors.delete(current);
1778
- return output;
1779
- }
1780
- const prototype = Object.getPrototypeOf(current);
1781
- if (prototype !== Object.prototype && null !== prototype) {
1782
- truncatedPaths.add(path);
1783
- const constructorName = current.constructor?.name ?? 'object';
1784
- return `[Unsupported object: ${constructorName}]`;
1785
- }
1786
- ancestors.add(current);
1787
- const output = {};
1788
- const keys = Object.keys(current);
1789
- for (const key of keys.slice(0, MAX_OBJECT_KEYS)){
1790
- const childPath = objectPath(path, key);
1791
- if (!consume(childPath, key)) break;
1792
- if (isSecretKey(key)) {
1793
- output[key] = '[REDACTED]';
1794
- redactedPaths.add(childPath);
1795
- } else output[key] = visit(current[key], childPath, depth + 1);
1796
- if (remainingBytes <= 0) break;
1797
- }
1798
- if (keys.length > Object.keys(output).length) truncatedPaths.add(path);
1799
- ancestors.delete(current);
1800
- return output;
1801
- };
1802
- const sanitized = visit(value, '$', 0);
1803
- const serialized = JSON.stringify(sanitized);
1804
- if (void 0 === serialized || stringByteLength(serialized) > MAX_SERIALIZED_BYTES) return {
1805
- value: TRUNCATED_VALUE,
1806
- truncatedPaths: [
1807
- '$'
1808
- ]
1809
- };
1810
- return {
1811
- value: sanitized,
1812
- ...truncatedPaths.size > 0 ? {
1813
- truncatedPaths: [
1814
- ...truncatedPaths
1815
- ]
1816
- } : {},
1817
- ...redactedPaths.size > 0 ? {
1818
- redactedPaths: [
1819
- ...redactedPaths
1820
- ]
1821
- } : {}
1822
- };
1823
- }
1824
- function sanitizeReportError(error) {
1825
- if (error instanceof Error) {
1826
- const workflowLike = error;
1827
- return {
1828
- name: sanitizeReportText(error.name || 'Error', 200),
1829
- message: sanitizeReportText(error.message || String(error)),
1830
- ...'string' == typeof workflowLike.code ? {
1831
- code: sanitizeReportText(workflowLike.code, 200)
1832
- } : {},
1833
- ...void 0 === workflowLike.details ? {} : {
1834
- details: sanitizeReportValue(workflowLike.details)
1835
- }
1836
- };
1837
- }
1838
- return {
1839
- name: 'Error',
1840
- message: sanitizeReportText(String(error))
1841
- };
1842
- }
1843
- const warnReport = getDebug('test-runner:report-assembler', {
1844
- console: true
1845
- });
1846
- const exactMidsceneDetailNodes = new Set([
1847
- 'aiAct',
1848
- 'aiAssert',
1849
- 'recordToReport'
1850
- ]);
1851
- const scopedExecutionKey = (scopeId, executionId)=>`${scopeId}\u0000${executionId}`;
1852
- const reportSourcesForScope = (context, scopeId)=>context.sourcesByScope.get(scopeId) ?? [];
1853
- const addDiagnostic = (context, diagnostic)=>{
1854
- context.diagnostics.push(diagnostic);
1855
- const key = JSON.stringify([
1856
- diagnostic.code,
1857
- diagnostic.scopeId,
1858
- diagnostic.executionId
1859
- ]);
1860
- if (!context.warnedDiagnostics.has(key)) {
1861
- context.warnedDiagnostics.add(key);
1862
- warnReport(diagnostic.message);
1863
- }
1864
- };
1865
- const buildStep = (step, scopeId, context)=>{
1866
- const details = [];
1867
- const unresolvedExecutionIds = [];
1868
- for (const trace of step.report?.traces ?? []){
1869
- const previousScope = context.tracedScopeByExecution.get(trace.executionId);
1870
- if (previousScope && previousScope !== scopeId) throw new Error(`Agent execution ${trace.executionId} was traced by multiple test scopes (${previousScope}, ${scopeId}).`);
1871
- context.tracedScopeByExecution.set(trace.executionId, scopeId);
1872
- const detail = context.detailsByScopeAndExecution.get(scopedExecutionKey(scopeId, trace.executionId));
1873
- if (detail) details.push(detail);
1874
- else unresolvedExecutionIds.push(trace.executionId);
1875
- }
1876
- let agentDetailDiagnostic;
1877
- if (unresolvedExecutionIds.length > 0) {
1878
- agentDetailDiagnostic = `Agent detail could not be resolved for ${unresolvedExecutionIds.join(', ')}.`;
1879
- const sources = reportSourcesForScope(context, scopeId);
1880
- for (const executionId of unresolvedExecutionIds)addDiagnostic(context, {
1881
- level: 'warning',
1882
- code: 'agent-detail-unresolved',
1883
- message: `Agent detail could not be resolved for execution ${executionId} in test scope ${scopeId}.`,
1884
- scopeId,
1885
- executionId,
1886
- ...sources.length > 0 ? {
1887
- sourcePaths: sources.map((source)=>source.sourcePath)
1888
- } : {}
1889
- });
1890
- } else if (0 === details.length && exactMidsceneDetailNodes.has(step.node)) {
1891
- agentDetailDiagnostic = 'This Agent did not expose a stable execution reference for the Step.';
1892
- addDiagnostic(context, {
1893
- level: 'warning',
1894
- code: 'agent-detail-unavailable',
1895
- message: `No stable Agent execution reference was captured for ${step.node} in test scope ${scopeId}.`,
1896
- scopeId,
1897
- ...reportSourcesForScope(context, scopeId).length > 0 ? {
1898
- sourcePaths: reportSourcesForScope(context, scopeId).map((source)=>source.sourcePath)
1899
- } : {}
1900
- });
1901
- }
1902
- const prompt = 'string' == typeof step.input?.prompt ? step.input.prompt : void 0;
1903
- return {
1904
- id: `${scopeId}:${step.phase}:${step.stepIndex}`,
1905
- phase: step.phase,
1906
- stepIndex: step.stepIndex,
1907
- node: step.node,
1908
- ...prompt ? {
1909
- title: sanitizeReportText(prompt, 240)
1910
- } : {},
1911
- status: step.status,
1912
- continuedAfterError: step.continuedAfterError,
1913
- startedAt: step.startedAt,
1914
- endedAt: step.endedAt,
1915
- durationMs: step.durationMs,
1916
- input: sanitizeReportValue(step.input),
1917
- ...step.output ? {
1918
- output: {
1919
- ...void 0 === step.output.summary ? {} : {
1920
- summary: sanitizeReportText(step.output.summary)
1921
- },
1922
- ...void 0 === step.output.data ? {} : {
1923
- data: sanitizeReportValue(step.output.data)
1924
- }
1925
- }
1926
- } : {},
1927
- ...step.error ? {
1928
- error: sanitizeReportError(step.error)
1929
- } : {},
1930
- ...details.length > 0 ? {
1931
- agentDetails: details
1932
- } : {},
1933
- ...agentDetailDiagnostic ? {
1934
- agentDetailDiagnostic
1935
- } : {}
1936
- };
1937
- };
1938
- const buildAttempt = (attempt, context)=>({
1939
- attemptId: attempt.runId,
1940
- attemptIndex: attempt.attemptIndex,
1941
- status: attempt.status,
1942
- startedAt: attempt.startedAt,
1943
- endedAt: attempt.endedAt,
1944
- durationMs: attempt.durationMs,
1945
- beforeEach: attempt.beforeEach.map((step)=>buildStep(step, attempt.runId, context)),
1946
- steps: attempt.steps.map((step)=>buildStep(step, attempt.runId, context)),
1947
- afterEach: attempt.afterEach.map((step)=>buildStep(step, attempt.runId, context)),
1948
- ...attempt.teardownErrors?.length ? {
1949
- teardownErrors: attempt.teardownErrors.map(sanitizeReportError)
1950
- } : {},
1951
- ...reportSourcesForScope(context, attempt.runId).length > 0 ? {
1952
- scopeReportIds: reportSourcesForScope(context, attempt.runId).map((source)=>source.reportId)
1953
- } : {}
1954
- });
1955
- const buildCase = (outcome, context)=>({
1956
- caseId: outcome.caseId,
1957
- name: outcome.name,
1958
- caseIndex: outcome.caseIndex,
1959
- status: outcome.status,
1960
- ...outcome.notRunReason ? {
1961
- notRunReason: outcome.notRunReason
1962
- } : {},
1963
- attempts: (outcome.attempts ?? (outcome.run ? [
1964
- outcome.run
1965
- ] : [])).map((attempt)=>buildAttempt(attempt, context))
1966
- });
1967
- const buildDocument = (documentId, document, cases, context)=>{
1968
- const sourcePath = document?.sourcePath ?? cases[0]?.sourcePath ?? '<unknown>';
1969
- const status = document?.status ?? (cases.some((item)=>'success' !== item.status) ? 'failed' : 'success');
1970
- return {
1971
- documentId,
1972
- sourcePath,
1973
- status,
1974
- ...document ? {
1975
- startedAt: document.startedAt,
1976
- endedAt: document.endedAt,
1977
- durationMs: document.durationMs
1978
- } : {},
1979
- beforeAll: (document?.beforeAll ?? []).map((step)=>buildStep(step, document.documentRunId, context)),
1980
- cases: cases.map((outcome)=>buildCase(outcome, context)),
1981
- afterAll: (document?.afterAll ?? []).map((step)=>buildStep(step, document.documentRunId, context)),
1982
- ...document?.teardownErrors?.length ? {
1983
- teardownErrors: document.teardownErrors.map(sanitizeReportError)
1984
- } : {},
1985
- ...document && reportSourcesForScope(context, document.documentRunId).length > 0 ? {
1986
- scopeReportIds: reportSourcesForScope(context, document.documentRunId).map((source)=>source.reportId)
1987
- } : {}
1988
- };
1989
- };
1990
- const buildProject = (project, context)=>{
1991
- const documentById = new Map(project.documents.map((document)=>[
1992
- document.documentId,
1993
- document
1994
- ]));
1995
- const casesByDocumentId = new Map();
1996
- for (const outcome of project.cases){
1997
- const current = casesByDocumentId.get(outcome.documentId) ?? [];
1998
- current.push(outcome);
1999
- casesByDocumentId.set(outcome.documentId, current);
2000
- }
2001
- const documentIds = new Set([
2002
- ...project.documents.map((document)=>document.documentId),
2003
- ...project.cases.map((outcome)=>outcome.documentId)
2004
- ]);
2005
- return {
2006
- projectId: project.projectId,
2007
- name: project.name,
2008
- status: project.status,
2009
- retry: project.retry,
2010
- ...project.lifecycle ? {
2011
- lifecycle: {
2012
- status: project.lifecycle.status,
2013
- startedAt: project.lifecycle.startedAt,
2014
- endedAt: project.lifecycle.endedAt,
2015
- durationMs: project.lifecycle.durationMs,
2016
- ...project.lifecycle.setupError ? {
2017
- setupError: sanitizeReportError(project.lifecycle.setupError)
2018
- } : {},
2019
- ...project.lifecycle.teardownErrors?.length ? {
2020
- teardownErrors: project.lifecycle.teardownErrors.map(sanitizeReportError)
2021
- } : {}
2022
- }
2023
- } : {},
2024
- documents: [
2025
- ...documentIds
2026
- ].map((documentId)=>buildDocument(documentId, documentById.get(documentId), casesByDocumentId.get(documentId) ?? [], context)),
2027
- collectionErrors: project.collectionErrors.map((collectionError)=>({
2028
- sourcePath: collectionError.sourcePath,
2029
- error: sanitizeReportError(collectionError.error)
2030
- }))
2031
- };
2032
- };
2033
- function collectTestRunReportSources(result) {
2034
- const sources = [];
2035
- for (const project of result.projects){
2036
- for (const document of project.documents)for (const sourcePath of document.reportPaths ?? [])sources.push({
2037
- scopeId: document.documentRunId,
2038
- sourcePath
2039
- });
2040
- for (const outcome of project.cases)for (const attempt of outcome.attempts ?? (outcome.run ? [
2041
- outcome.run
2042
- ] : []))for (const sourcePath of attempt.reportPaths ?? [])sources.push({
2043
- scopeId: attempt.runId,
2044
- sourcePath
2045
- });
2046
- }
2047
- return sources;
2048
- }
2049
- function buildTestRunReportDump(result, index) {
2050
- const sourcesByScope = new Map();
2051
- const detailsByScopeAndExecution = new Map();
2052
- for (const source of index.sources){
2053
- const scopeSources = sourcesByScope.get(source.scopeId) ?? [];
2054
- scopeSources.push(source);
2055
- sourcesByScope.set(source.scopeId, scopeSources);
2056
- for (const executionId of source.executionIds)detailsByScopeAndExecution.set(scopedExecutionKey(source.scopeId, executionId), {
2057
- reportId: source.reportId,
2058
- executionId
2059
- });
2060
- }
2061
- const context = {
2062
- diagnostics: [],
2063
- sourcesByScope,
2064
- detailsByScopeAndExecution,
2065
- tracedScopeByExecution: new Map(),
2066
- warnedDiagnostics: new Set()
2067
- };
2068
- for (const source of index.sources)if (0 === source.executionIds.length) addDiagnostic(context, {
2069
- level: 'warning',
2070
- code: 'source-without-executions',
2071
- message: `Agent report contains no executions: ${source.sourcePath}`,
2072
- scopeId: source.scopeId,
2073
- sourcePaths: [
2074
- source.sourcePath
2075
- ]
684
+ });
685
+ ids.add(caseId);
686
+ return {
687
+ caseId,
688
+ projectId: source.projectId,
689
+ sourcePath,
690
+ caseIndex,
691
+ definition: {
692
+ name: definition.name,
693
+ tags: tags,
694
+ steps
695
+ }
696
+ };
2076
697
  });
2077
- const projects = result.projects.map((project)=>buildProject(project, context));
2078
698
  return {
2079
- schemaVersion: 1,
2080
- kind: 'test-runner',
2081
- runId: result.runId,
2082
- status: result.status,
2083
- startedAt: result.startedAt,
2084
- endedAt: result.endedAt,
2085
- durationMs: result.durationMs,
2086
- summary: {
2087
- ...result.summary
2088
- },
2089
- metrics: index.metrics,
2090
- projects,
2091
- ...context.diagnostics.length > 0 ? {
2092
- diagnostics: context.diagnostics
2093
- } : {}
699
+ documentId: createWorkflowDocumentId(source.projectId, sourcePath),
700
+ projectId: source.projectId,
701
+ sourcePath,
702
+ lifecycle,
703
+ cases
2094
704
  };
2095
705
  }
2096
- const writeJson = (path, value)=>{
2097
- (0, external_node_fs_.mkdirSync)((0, external_node_path_.dirname)(path), {
2098
- recursive: true
2099
- });
2100
- (0, external_node_fs_.writeFileSync)(path, JSON.stringify(value, null, 2));
706
+ const writeJson = async (path, value)=>{
707
+ try {
708
+ await mkdir((0, external_node_path_.dirname)(path), {
709
+ recursive: true
710
+ });
711
+ await writeFile(path, JSON.stringify(value, null, 2));
712
+ } catch (error) {
713
+ throw new WorkflowPublicationError('write-result', path, error);
714
+ }
2101
715
  };
2102
716
  const toPosix = (value)=>value.split(external_node_path_.sep).join('/');
2103
717
  const caseAttemptResultPath = (projectId, documentId, result)=>toPosix((0, external_node_path_.join)(projectId, 'documents', documentId, 'cases', result.caseId, `${result.runId}.json`));
2104
- const workflowDocumentResultPath = (result)=>toPosix((0, external_node_path_.join)(result.projectId, 'documents', result.documentId, 'document.json'));
718
+ const workflowDocumentResultPath = (result)=>toPosix((0, external_node_path_.join)(result.projectId, 'documents', result.documentId, void 0 === result.attemptIndex ? 'document.json' : `${result.documentRunId}.json`));
2105
719
  const collectionErrorPath = (projectId, sourcePath)=>{
2106
720
  const id = createHash('sha256').update(JSON.stringify([
2107
721
  projectId,
@@ -2111,21 +725,19 @@ const collectionErrorPath = (projectId, sourcePath)=>{
2111
725
  };
2112
726
  const writeWorkflowDocumentResult = (runDir, result)=>{
2113
727
  const { documentRunId: _documentRunId, ...persistedResult } = result;
2114
- writeJson((0, external_node_path_.join)(runDir, workflowDocumentResultPath(result)), persistedResult);
728
+ return writeJson((0, external_node_path_.join)(runDir, workflowDocumentResultPath(result)), persistedResult);
2115
729
  };
2116
730
  const writeCaseAttemptResult = (runDir, projectId, documentId, result)=>{
2117
731
  const { runId: attemptId, ...persistedResult } = result;
2118
- writeJson((0, external_node_path_.join)(runDir, caseAttemptResultPath(projectId, documentId, result)), {
732
+ return writeJson((0, external_node_path_.join)(runDir, caseAttemptResultPath(projectId, documentId, result)), {
2119
733
  ...persistedResult,
2120
734
  attemptId
2121
735
  });
2122
736
  };
2123
- const writeCollectionError = (runDir, collectionError)=>{
2124
- writeJson((0, external_node_path_.join)(runDir, collectionErrorPath(collectionError.projectId, collectionError.sourcePath)), {
737
+ const writeCollectionError = (runDir, collectionError)=>writeJson((0, external_node_path_.join)(runDir, collectionErrorPath(collectionError.projectId, collectionError.sourcePath)), {
2125
738
  kind: 'collection-error',
2126
739
  ...collectionError
2127
740
  });
2128
- };
2129
741
  const relativeToSummary = (summaryPath, target)=>{
2130
742
  const value = (0, external_node_path_.relative)((0, external_node_path_.dirname)(summaryPath), target);
2131
743
  return toPosix(value || '.');
@@ -2134,7 +746,7 @@ const errorJson = (error)=>error instanceof Error && 'toJSON' in error ? error.t
2134
746
  const writeTestProjectRunResult = (options)=>{
2135
747
  const { result } = options;
2136
748
  const fact = (path)=>toPosix(path);
2137
- writeJson(result.summaryPath, {
749
+ return writeJson(result.summaryPath, {
2138
750
  schemaVersion: result.schemaVersion,
2139
751
  runId: result.runId,
2140
752
  startedAt: result.startedAt,
@@ -2142,6 +754,9 @@ const writeTestProjectRunResult = (options)=>{
2142
754
  durationMs: result.durationMs,
2143
755
  status: result.status,
2144
756
  exitCode: result.exitCode,
757
+ ...result.errors?.length ? {
758
+ errors: result.errors.map(errorJson)
759
+ } : {},
2145
760
  projectRoot: options.projectRoot,
2146
761
  ...options.configPath ? {
2147
762
  configPath: options.configPath
@@ -2177,6 +792,9 @@ const writeTestProjectRunResult = (options)=>{
2177
792
  }
2178
793
  } : {},
2179
794
  cases: project.cases.map((outcome)=>({
795
+ ...outcome.documentRunId ? {
796
+ documentRunId: outcome.documentRunId
797
+ } : {},
2180
798
  documentId: outcome.documentId,
2181
799
  caseId: outcome.caseId,
2182
800
  sourcePath: outcome.sourcePath,
@@ -2200,6 +818,10 @@ const writeTestProjectRunResult = (options)=>{
2200
818
  })),
2201
819
  documents: project.documents.map((document)=>({
2202
820
  documentId: document.documentId,
821
+ ...void 0 === document.attemptIndex ? {} : {
822
+ attemptIndex: document.attemptIndex,
823
+ documentRunId: document.documentRunId
824
+ },
2203
825
  sourcePath: document.sourcePath,
2204
826
  status: document.status,
2205
827
  resultFile: fact(workflowDocumentResultPath(document)),
@@ -2234,29 +856,28 @@ const DEFAULT_TEST_FILE_SELECTION = {
2234
856
  '**/*.{yaml,yml}'
2235
857
  ]
2236
858
  };
2237
- const test_project_runner_toPosix = (value)=>value.split(external_node_path_.sep).join('/');
2238
- const discoverTestFiles = (projectRoot, selection = DEFAULT_TEST_FILE_SELECTION)=>{
859
+ const project_preparation_toPosix = (value)=>value.split(external_node_path_.sep).join('/');
860
+ const discoverResolvedTestFiles = (projectRoot, selection)=>{
2239
861
  const root = (0, external_node_path_.resolve)(projectRoot);
2240
- const normalized = validateTestFileSelection(selection);
2241
- if (!normalized) throw new TypeError('Test file selection is required.');
2242
- const files = globSync(normalized.include, {
2243
- absolute: true,
2244
- caseSensitiveMatch: false,
2245
- cwd: root,
2246
- dot: true,
2247
- expandDirectories: false,
2248
- followSymbolicLinks: false,
2249
- ignore: [
2250
- ...ALWAYS_IGNORED_PATTERNS,
2251
- ...normalized.exclude ?? []
2252
- ],
2253
- onlyFiles: true
2254
- }).filter((file)=>/\.ya?ml$/i.test(file));
862
+ const match = (patterns)=>globSync(patterns, {
863
+ absolute: true,
864
+ caseSensitiveMatch: false,
865
+ cwd: root,
866
+ dot: true,
867
+ expandDirectories: false,
868
+ followSymbolicLinks: false,
869
+ ignore: [
870
+ ...ALWAYS_IGNORED_PATTERNS,
871
+ ...selection.exclude ?? []
872
+ ],
873
+ onlyFiles: true
874
+ }).filter((file)=>/\.ya?ml$/i.test(file));
875
+ const files = match(selection.include);
2255
876
  return [
2256
877
  ...new Set(files.map((file)=>(0, external_node_path_.resolve)(file)))
2257
878
  ].sort((a, b)=>{
2258
- const relativeA = test_project_runner_toPosix((0, external_node_path_.relative)(root, a));
2259
- const relativeB = test_project_runner_toPosix((0, external_node_path_.relative)(root, b));
879
+ const relativeA = project_preparation_toPosix((0, external_node_path_.relative)(root, a));
880
+ const relativeB = project_preparation_toPosix((0, external_node_path_.relative)(root, b));
2260
881
  return relativeA < relativeB ? -1 : relativeA > relativeB ? 1 : 0;
2261
882
  });
2262
883
  };
@@ -2277,22 +898,6 @@ const discoverTestConfig = (projectRoot)=>{
2277
898
  ].join('\n'));
2278
899
  return supported[0] ? (0, external_node_path_.join)(root, supported[0]) : void 0;
2279
900
  };
2280
- const defaultResultDir = (projectRoot)=>(0, external_node_path_.join)(projectRoot, '.midscene', 'test-results');
2281
- const padDatePart = (value)=>String(value).padStart(2, '0');
2282
- const createTestRunId = (date = new Date(), uuid = randomUUID())=>{
2283
- const timestamp = [
2284
- date.getFullYear(),
2285
- padDatePart(date.getMonth() + 1),
2286
- padDatePart(date.getDate()),
2287
- padDatePart(date.getHours()),
2288
- padDatePart(date.getMinutes()),
2289
- padDatePart(date.getSeconds())
2290
- ].join('');
2291
- return `${timestamp}-${uuid.slice(0, 8)}`;
2292
- };
2293
- const assertDirectory = (path, label)=>{
2294
- if (!(0, external_node_fs_.existsSync)(path) || !(0, external_node_fs_.statSync)(path).isDirectory()) throw new Error(`${label} does not exist or is not a directory: ${path}`);
2295
- };
2296
901
  const asCollectionError = (projectId, projectName, sourcePath, error)=>({
2297
902
  projectId,
2298
903
  projectName,
@@ -2319,52 +924,6 @@ const filterDocumentCases = (document, project)=>{
2319
924
  filtered
2320
925
  };
2321
926
  };
2322
- const test_project_runner_asNotRun = (documentId, collectedCase, projectName, reason)=>({
2323
- documentId,
2324
- caseId: collectedCase.caseId,
2325
- projectName,
2326
- name: collectedCase.definition.name,
2327
- sourcePath: collectedCase.sourcePath,
2328
- caseIndex: collectedCase.caseIndex,
2329
- status: 'not-run',
2330
- notRunReason: reason
2331
- });
2332
- const summarize = (projects)=>{
2333
- const cases = projects.flatMap((project)=>project.cases);
2334
- const documents = projects.flatMap((project)=>project.documents);
2335
- return {
2336
- total: cases.length,
2337
- passed: cases.filter((item)=>'success' === item.status).length,
2338
- failed: cases.filter((item)=>'failed' === item.status).length,
2339
- notRun: cases.filter((item)=>'not-run' === item.status).length,
2340
- filtered: projects.reduce((total, project)=>total + project.filteredCaseCount, 0),
2341
- collectionErrors: projects.reduce((total, project)=>total + project.collectionErrors.length, 0),
2342
- documentFailures: documents.filter((document)=>'failed' === document.status).length,
2343
- projectFailures: projects.filter((project)=>'failed' === project.status).length
2344
- };
2345
- };
2346
- const stepPosition = (info)=>'case' === info.scope ? info.case : info.document;
2347
- const test_project_runner_formatStep = (info)=>{
2348
- const position = stepPosition(info);
2349
- const phase = 'steps' === position.phase ? 'step' : position.phase;
2350
- return `${phase} ${position.stepIndex + 1}/${info.stepCount}: ${info.node}`;
2351
- };
2352
- const formatStepResult = (info, result)=>{
2353
- const indent = 'case' === info.scope ? ' ' : ' ';
2354
- const symbol = 'success' === result.status ? '✓' : '✗';
2355
- const error = result.error ? ` — ${result.error.message}` : '';
2356
- const continuation = result.continuedAfterError ? '; continuing' : '';
2357
- return `${indent}${symbol} ${test_project_runner_formatStep(info)} (${result.durationMs} ms)${error}${continuation}`;
2358
- };
2359
- const caseHasFatalError = (outcome)=>(outcome.attempts ?? []).some((attempt)=>[
2360
- ...attempt.beforeEach,
2361
- ...attempt.steps,
2362
- ...attempt.afterEach
2363
- ].some((step)=>step.error && isFatalDeviceError(step.error)) || (attempt.teardownErrors ?? []).some(isFatalDeviceError));
2364
- const documentHasFatalError = (result)=>[
2365
- ...result.beforeAll,
2366
- ...result.afterAll
2367
- ].some((step)=>step.error && isFatalDeviceError(step.error)) || (result.teardownErrors ?? []).some(isFatalDeviceError);
2368
927
  const selectProjects = (projects, names)=>{
2369
928
  if (!names || 0 === names.length) return projects;
2370
929
  const requested = new Set(names);
@@ -2374,17 +933,30 @@ const selectProjects = (projects, names)=>{
2374
933
  if (unknown) throw new Error(`Unknown Midscene project: ${unknown}`);
2375
934
  return projects.filter((project)=>requested.has(project.name));
2376
935
  };
2377
- const prepareProject = (project, projectRoot, runDir)=>{
936
+ async function prepareProject(project, projectRoot, runDir, defaultTimeoutMs, options = {}) {
2378
937
  const fileSelection = project.files ?? DEFAULT_TEST_FILE_SELECTION;
2379
- const files = discoverTestFiles(projectRoot, fileSelection);
2380
- const sources = files.map((absolutePath)=>({
938
+ const prerequisiteFile = options.prerequisiteFile ? (0, external_node_path_.resolve)(options.prerequisiteFile) : void 0;
939
+ const mainFiles = options.files ?? discoverResolvedTestFiles(projectRoot, fileSelection);
940
+ const files = [
941
+ ...prerequisiteFile ? [
942
+ prerequisiteFile
943
+ ] : [],
944
+ ...mainFiles.filter((file)=>file !== prerequisiteFile)
945
+ ];
946
+ const occurrences = new Map();
947
+ const sources = files.map((absolutePath)=>{
948
+ const invocationIndex = occurrences.get(absolutePath) ?? 0;
949
+ occurrences.set(absolutePath, invocationIndex + 1);
950
+ return {
2381
951
  projectId: project.projectId,
2382
952
  projectName: project.name,
2383
- sourcePath: test_project_runner_toPosix((0, external_node_path_.relative)(projectRoot, absolutePath)),
2384
- absolutePath
2385
- }));
953
+ sourcePath: project_preparation_toPosix((0, external_node_path_.relative)(projectRoot, absolutePath)),
954
+ absolutePath,
955
+ invocationIndex
956
+ };
957
+ });
2386
958
  const collectionErrors = [];
2387
- const documents = [];
959
+ const invocations = [];
2388
960
  let filteredCaseCount = 0;
2389
961
  if (0 === sources.length) {
2390
962
  const error = asCollectionError(project.projectId, project.name, '<project>', new WorkflowParseError(`No workflow YAML files found for project "${project.name}" in ${projectRoot}.`, {
@@ -2392,56 +964,282 @@ const prepareProject = (project, projectRoot, runDir)=>{
2392
964
  projectRoot
2393
965
  }));
2394
966
  collectionErrors.push(error);
2395
- writeCollectionError(runDir, error);
967
+ await writeCollectionError(runDir, error);
2396
968
  }
2397
969
  for (const source of sources)try {
2398
- const collected = collectWorkflowDocument(source, {
970
+ const collected = await options.collectDocument?.(source, project);
971
+ if (collected) {
972
+ filteredCaseCount += collected.filteredCaseCount;
973
+ if (collected.invocation) invocations.push(collected.invocation);
974
+ continue;
975
+ }
976
+ const document = collectWorkflowDocument(source, {
2399
977
  resolveNode: project.nodes.get.bind(project.nodes),
2400
978
  variables: project.variables,
2401
979
  env: process.env
2402
980
  });
2403
- const filtered = filterDocumentCases(collected, project);
981
+ const filtered = source.absolutePath === prerequisiteFile ? {
982
+ document,
983
+ filtered: 0
984
+ } : filterDocumentCases(document, project);
2404
985
  filteredCaseCount += filtered.filtered;
2405
- if (filtered.document) documents.push(filtered.document);
986
+ if (filtered.document) invocations.push({
987
+ document: filtered.document,
988
+ retry: {
989
+ scope: 'case',
990
+ count: project.retry
991
+ },
992
+ defaultTimeoutMs,
993
+ reportEnabled: true,
994
+ bindings: {
995
+ resolveNode: project.nodes.require.bind(project.nodes)
996
+ }
997
+ });
2406
998
  } catch (error) {
2407
999
  const collectionError = asCollectionError(project.projectId, project.name, source.sourcePath, error);
2408
1000
  collectionErrors.push(collectionError);
2409
- writeCollectionError(runDir, collectionError);
1001
+ await writeCollectionError(runDir, collectionError);
2410
1002
  }
1003
+ const documents = invocations.map((invocation)=>invocation.document);
1004
+ const prerequisiteSource = sources.find((source)=>source.absolutePath === prerequisiteFile);
1005
+ const prerequisiteDocument = documents.find((document)=>document.sourcePath === prerequisiteSource?.sourcePath);
2411
1006
  return {
2412
1007
  project,
1008
+ platform: options.platform ?? 'test',
1009
+ documentConcurrency: options.documentConcurrency ?? 1,
1010
+ ...prerequisiteDocument ? {
1011
+ prerequisiteDocumentId: prerequisiteDocument.documentId
1012
+ } : {},
2413
1013
  fileSelection,
2414
1014
  sources,
2415
1015
  documents,
1016
+ invocations,
2416
1017
  collectionErrors,
2417
1018
  selectedCaseCount: documents.reduce((total, document)=>total + document.cases.length, 0),
2418
1019
  filteredCaseCount
2419
1020
  };
1021
+ }
1022
+ const padDatePart = (value)=>String(value).padStart(2, '0');
1023
+ const createTestRunId = (date = new Date(), uuid = randomUUID())=>{
1024
+ const timestamp = [
1025
+ date.getFullYear(),
1026
+ padDatePart(date.getMonth() + 1),
1027
+ padDatePart(date.getDate()),
1028
+ padDatePart(date.getHours()),
1029
+ padDatePart(date.getMinutes()),
1030
+ padDatePart(date.getSeconds())
1031
+ ].join('');
1032
+ return `${timestamp}-${uuid.slice(0, 8)}`;
2420
1033
  };
2421
- const notRunSuite = (prepared, reason)=>prepared.documents.flatMap((document)=>document.cases.map((item)=>test_project_runner_asNotRun(document.documentId, item, prepared.project.name, reason)));
2422
- async function runTestProject(options = {}) {
2423
- const startedAt = new Date();
2424
- const runId = createTestRunId(startedAt);
1034
+ const assertDirectory = (path, label)=>{
1035
+ if (!(0, external_node_fs_.existsSync)(path) || !(0, external_node_fs_.statSync)(path).isDirectory()) throw new Error(`${label} does not exist or is not a directory: ${path}`);
1036
+ };
1037
+ const resolveRunInput = (options)=>{
2425
1038
  const cwd = (0, external_node_path_.resolve)(options.cwd ?? process.cwd());
2426
1039
  assertDirectory(cwd, 'Test working directory');
2427
- const cliProjectRoot = options.projectRoot ? (0, external_node_path_.resolve)(cwd, options.projectRoot) : void 0;
1040
+ const inputPath = options.projectRoot ? (0, external_node_path_.resolve)(cwd, options.projectRoot) : void 0;
1041
+ const singleFile = inputPath && (0, external_node_fs_.existsSync)(inputPath) && (0, external_node_fs_.statSync)(inputPath).isFile() ? inputPath : void 0;
1042
+ if (singleFile && !/\.ya?ml$/i.test(singleFile)) throw new Error(`Test input must be a YAML file or project directory: ${singleFile}`);
1043
+ const cliProjectRoot = singleFile ? (0, external_node_path_.dirname)(singleFile) : inputPath;
2428
1044
  if (cliProjectRoot) assertDirectory(cliProjectRoot, 'Test project directory');
2429
- const configSearchRoot = cliProjectRoot ?? cwd;
2430
- const configPath = options.configPath ? (0, external_node_path_.resolve)(configSearchRoot, options.configPath) : discoverTestConfig(configSearchRoot);
2431
- if (options.configPath && (!configPath || !(0, external_node_fs_.existsSync)(configPath))) throw new Error(`Midscene config does not exist: ${configPath}`);
2432
- const definition = await loadTestProject(configPath);
2433
1045
  const projectRoot = cliProjectRoot ?? cwd;
2434
- const resultDir = options.resultDir ? (0, external_node_path_.resolve)(cwd, options.resultDir) : defaultResultDir(projectRoot);
1046
+ return {
1047
+ cwd,
1048
+ projectRoot,
1049
+ configSearchRoot: projectRoot,
1050
+ ...singleFile ? {
1051
+ singleFile
1052
+ } : {}
1053
+ };
1054
+ };
1055
+ async function prepareTestRun(options) {
1056
+ const startedAt = new Date();
1057
+ const runId = createTestRunId(startedAt);
1058
+ const input = resolveRunInput(options);
1059
+ const configPath = options.configPath ? (0, external_node_path_.resolve)(input.configSearchRoot, options.configPath) : discoverTestConfig(input.configSearchRoot);
1060
+ if (options.configPath && /\.ya?ml$/i.test(configPath)) throw new Error(`midscene-test --config only accepts a native TypeScript or JavaScript project config: ${configPath}. Run legacy batch YAML with \`midscene --config ${options.configPath}\`.`);
1061
+ if (options.configPath && !(0, external_node_fs_.existsSync)(configPath)) throw new Error(`Midscene config does not exist: ${configPath}`);
1062
+ const definition = await loadTestProject(configPath);
1063
+ const resultDir = options.resultDir ? (0, external_node_path_.resolve)(input.cwd, options.resultDir) : (0, external_node_path_.join)(input.projectRoot, '.midscene', 'test-results');
2435
1064
  const runDir = (0, external_node_path_.join)(resultDir, runId);
2436
1065
  const summaryPath = (0, external_node_path_.join)(runDir, 'summary.json');
2437
- const reportDir = (0, external_node_path_.resolve)(projectRoot, definition.output.reportDir);
1066
+ const reportDir = (0, external_node_path_.resolve)(input.projectRoot, definition.output.reportDir);
2438
1067
  (0, external_node_fs_.mkdirSync)(resultDir, {
2439
1068
  recursive: true
2440
1069
  });
2441
1070
  (0, external_node_fs_.mkdirSync)(runDir);
2442
- const selectedProjects = selectProjects(definition.projects, options.projectNames);
2443
- const preparedProjects = selectedProjects.map((project)=>prepareProject(project, projectRoot, runDir));
2444
- const progress = options.onProgress ?? (()=>{});
1071
+ const projects = [];
1072
+ for (const project of selectProjects(definition.projects, options.projectNames))projects.push(await prepareProject(project, input.projectRoot, runDir, definition.test.testTimeout, input.singleFile ? {
1073
+ files: [
1074
+ input.singleFile
1075
+ ]
1076
+ } : void 0));
1077
+ const hasFormatMismatch = projects.some((project)=>project.collectionErrors.some(({ error })=>error instanceof NativeWorkflowFormatError));
1078
+ return {
1079
+ startedAt,
1080
+ runId,
1081
+ projectRoot: input.projectRoot,
1082
+ ...configPath ? {
1083
+ configPath
1084
+ } : {},
1085
+ resultDir,
1086
+ runDir,
1087
+ summaryPath,
1088
+ reportDir,
1089
+ definition,
1090
+ projects,
1091
+ preflightScope: hasFormatMismatch ? 'run' : 'project',
1092
+ reportEnabled: true,
1093
+ publications: [],
1094
+ onProgress: options.onProgress ?? (()=>{})
1095
+ };
1096
+ }
1097
+ const stepPosition = (info)=>'case' === info.scope ? info.case : info.document;
1098
+ const formatStep = (info)=>{
1099
+ const position = stepPosition(info);
1100
+ const phase = 'steps' === position.phase ? 'step' : position.phase;
1101
+ return `${phase} ${position.stepIndex + 1}/${info.stepCount}: ${info.node}`;
1102
+ };
1103
+ const formatStepResult = (info, result)=>{
1104
+ const indent = 'case' === info.scope ? ' ' : ' ';
1105
+ const symbol = 'success' === result.status ? '✓' : '✗';
1106
+ const error = result.error ? ` — ${result.error.message}` : '';
1107
+ const continuation = result.continuedAfterError ? '; continuing' : '';
1108
+ return `${indent}${symbol} ${formatStep(info)} (${result.durationMs} ms)${error}${continuation}`;
1109
+ };
1110
+ const caseHasFatalError = (outcome)=>(outcome.attempts ?? []).some((attempt)=>[
1111
+ ...attempt.beforeEach,
1112
+ ...attempt.steps,
1113
+ ...attempt.afterEach
1114
+ ].some((step)=>step.error && isFatalDeviceError(step.error)) || (attempt.teardownErrors ?? []).some(isFatalDeviceError));
1115
+ const documentHasFatalError = (result)=>[
1116
+ ...result.beforeAll,
1117
+ ...result.afterAll
1118
+ ].some((step)=>step.error && isFatalDeviceError(step.error)) || (result.teardownErrors ?? []).some(isFatalDeviceError);
1119
+ async function executeDocumentInvocation(options) {
1120
+ const { invocation, project, sinks } = options;
1121
+ const { document } = invocation;
1122
+ const documentRetry = 'document' === invocation.retry.scope;
1123
+ const attempts = [];
1124
+ await runDocumentAttempts({
1125
+ retry: documentRetry ? invocation.retry.count : 0,
1126
+ signal: options.signal,
1127
+ shouldStop: ()=>options.shouldBail() || options.isProjectFatal()
1128
+ }, async (documentAttemptIndex)=>{
1129
+ if (documentRetry) options.onProgress(` file attempt ${documentAttemptIndex + 1}/${invocation.retry.count + 1}: ${document.sourcePath}`);
1130
+ const execution = await runWorkflowDocument(document, {
1131
+ documentAttemptIndex: documentRetry ? documentAttemptIndex : void 0,
1132
+ ...invocation.bindings,
1133
+ project,
1134
+ projectContext: options.projectContext,
1135
+ retry: documentRetry ? 0 : invocation.retry.count,
1136
+ signal: options.signal,
1137
+ defaultTimeoutMs: invocation.defaultTimeoutMs,
1138
+ shouldStop: ()=>options.rootSignal.aborted || options.shouldBail() || options.isProjectFatal(),
1139
+ stopReason: ()=>options.rootSignal.aborted ? 'interrupted' : options.isProjectFatal() ? 'fatal-error' : 'bail',
1140
+ isFatalError: (run)=>[
1141
+ ...run.beforeEach,
1142
+ ...run.steps,
1143
+ ...run.afterEach
1144
+ ].some((step)=>step.error && isFatalDeviceError(step.error)) || (run.teardownErrors ?? []).some(isFatalDeviceError),
1145
+ onCaseStart: (collectedCase)=>{
1146
+ options.onProgress(` [case ${collectedCase.caseIndex + 1}/${document.cases.length}] ${collectedCase.definition.name}`);
1147
+ },
1148
+ onStepStart: (info)=>{
1149
+ const indent = 'case' === info.scope ? ' ' : ' ';
1150
+ options.onProgress(`${indent}→ ${formatStep(info)}`);
1151
+ },
1152
+ onStepResult: async (info, result)=>{
1153
+ await invocation.bindings.onStepResult?.(info, result);
1154
+ options.onProgress(formatStepResult(info, result));
1155
+ },
1156
+ onCaseResult: async (attempt)=>{
1157
+ await writeCaseAttemptResult(options.runDir, project.projectId, document.documentId, attempt);
1158
+ options.onProgress(` ${'success' === attempt.status ? '✓' : '✗'} attempt ${attempt.attemptIndex + 1}/${documentRetry ? 1 : invocation.retry.count + 1}: ${attempt.name} (${attempt.durationMs} ms)`);
1159
+ },
1160
+ onCaseOutcome: (outcome)=>{
1161
+ if (caseHasFatalError(outcome)) options.markProjectFatal();
1162
+ if (!documentRetry && 'failed' === outcome.status) options.addFailedCases(1);
1163
+ },
1164
+ onDocumentResult: options.onDocumentResult
1165
+ }).catch(async (error)=>{
1166
+ if (error instanceof WorkflowExecutionFailure) {
1167
+ const partial = error.result;
1168
+ attempts.push(partial);
1169
+ sinks.documents.push(partial.document);
1170
+ sinks.cases.push(...partial.cases.map((outcome)=>({
1171
+ ...outcome,
1172
+ documentId: document.documentId,
1173
+ documentRunId: partial.document.documentRunId
1174
+ })));
1175
+ try {
1176
+ await writeWorkflowDocumentResult(options.runDir, partial.document);
1177
+ } catch (publicationError) {
1178
+ throw new WorkflowExecutionFailure(partial, [
1179
+ ...error.errors,
1180
+ publicationError
1181
+ ]);
1182
+ }
1183
+ }
1184
+ throw error;
1185
+ });
1186
+ attempts.push(execution);
1187
+ sinks.cases.push(...execution.cases.map((outcome)=>({
1188
+ ...outcome,
1189
+ documentId: document.documentId,
1190
+ documentRunId: execution.document.documentRunId
1191
+ })));
1192
+ sinks.documents.push(execution.document);
1193
+ await writeWorkflowDocumentResult(options.runDir, execution.document);
1194
+ if (documentHasFatalError(execution.document)) options.markProjectFatal();
1195
+ const cleanupErrors = [
1196
+ ...execution.document.teardownErrors ?? [],
1197
+ ...execution.cases.flatMap((outcome)=>(outcome.attempts ?? []).flatMap((attempt)=>attempt.teardownErrors ?? []))
1198
+ ];
1199
+ if (cleanupErrors.length) throw new WorkflowExecutionFailure(execution, cleanupErrors);
1200
+ const failed = 'failed' === execution.document.status || execution.cases.some((item)=>'success' !== item.status);
1201
+ return failed ? 'failed' : 'success';
1202
+ });
1203
+ const final = attempts.at(-1);
1204
+ if (!final) return;
1205
+ const failedCases = final.cases.filter((item)=>'failed' === item.status).length;
1206
+ if (documentRetry && failedCases) options.addFailedCases(failedCases);
1207
+ else if ('failed' === final.document.status && 0 === failedCases) options.addFailedCases(Math.max(1, failedCases));
1208
+ }
1209
+ const asNotRun = (documentId, collectedCase, projectName, reason)=>({
1210
+ documentId,
1211
+ caseId: collectedCase.caseId,
1212
+ projectName,
1213
+ name: collectedCase.definition.name,
1214
+ sourcePath: collectedCase.sourcePath,
1215
+ caseIndex: collectedCase.caseIndex,
1216
+ status: 'not-run',
1217
+ notRunReason: reason
1218
+ });
1219
+ const summarize = (projects)=>{
1220
+ const cases = projects.flatMap((project)=>latestById(project.cases, (item)=>item.caseId));
1221
+ const documents = projects.flatMap((project)=>latestById(project.documents, (item)=>item.documentId));
1222
+ return {
1223
+ total: cases.length,
1224
+ passed: cases.filter((item)=>'success' === item.status).length,
1225
+ failed: cases.filter((item)=>'failed' === item.status).length,
1226
+ notRun: cases.filter((item)=>'not-run' === item.status).length,
1227
+ filtered: projects.reduce((total, project)=>total + project.filteredCaseCount, 0),
1228
+ collectionErrors: projects.reduce((total, project)=>total + project.collectionErrors.length, 0),
1229
+ documentFailures: documents.filter((document)=>'failed' === document.status).length,
1230
+ projectFailures: projects.filter((project)=>'failed' === project.status).length
1231
+ };
1232
+ };
1233
+ const latestById = (items, id)=>[
1234
+ ...new Map(items.map((item)=>[
1235
+ id(item),
1236
+ item
1237
+ ])).values()
1238
+ ];
1239
+ const notRunSuite = (prepared, reason)=>prepared.documents.flatMap((document)=>document.cases.map((item)=>asNotRun(document.documentId, item, prepared.project.name, reason)));
1240
+ async function runPreparedTestProject(plan) {
1241
+ const { startedAt, runId, projectRoot, configPath, resultDir, runDir, summaryPath, reportDir, definition, projects: preparedProjects } = plan;
1242
+ const progress = plan.onProgress;
2445
1243
  const totalDocuments = preparedProjects.reduce((total, prepared)=>total + prepared.documents.length, 0);
2446
1244
  const totalCases = preparedProjects.reduce((total, prepared)=>total + prepared.selectedCaseCount, 0);
2447
1245
  const totalErrors = preparedProjects.reduce((total, prepared)=>total + prepared.collectionErrors.length, 0);
@@ -2459,11 +1257,13 @@ async function runTestProject(options = {}) {
2459
1257
  let failedCaseCount = 0;
2460
1258
  const bailReached = ()=>definition.test.bail > 0 && failedCaseCount >= definition.test.bail;
2461
1259
  let hasInfrastructureError = false;
2462
- let firstInfrastructureError;
1260
+ const infrastructureErrors = [];
2463
1261
  const recordInfrastructureError = (error)=>{
2464
- if (hasInfrastructureError) return;
1262
+ const errors = error instanceof WorkflowExecutionFailure ? error.errors : [
1263
+ error
1264
+ ];
1265
+ for (const item of errors)if (!infrastructureErrors.includes(item)) infrastructureErrors.push(item);
2465
1266
  hasInfrastructureError = true;
2466
- firstInfrastructureError = error;
2467
1267
  rootController.abort(error);
2468
1268
  };
2469
1269
  const runInfrastructureCallback = (callback)=>{
@@ -2480,10 +1280,11 @@ async function runTestProject(options = {}) {
2480
1280
  };
2481
1281
  const buildProjectResult = (prepared, cases, documents, lifecycle)=>{
2482
1282
  const { project } = prepared;
2483
- const projectFailed = prepared.collectionErrors.length > 0 || cases.some((item)=>'success' !== item.status) || documents.some((item)=>'failed' === item.status) || lifecycle?.status === 'failed';
1283
+ const projectFailed = prepared.collectionErrors.length > 0 || latestById(cases, (item)=>item.caseId).some((item)=>'success' !== item.status) || latestById(documents, (item)=>item.documentId).some((item)=>'failed' === item.status) || lifecycle?.status === 'failed';
2484
1284
  return {
2485
1285
  projectId: project.projectId,
2486
1286
  name: project.name,
1287
+ platform: prepared.platform,
2487
1288
  status: projectFailed ? 'failed' : 'success',
2488
1289
  retry: project.retry,
2489
1290
  fileSelection: prepared.fileSelection,
@@ -2511,7 +1312,7 @@ async function runTestProject(options = {}) {
2511
1312
  const documents = [];
2512
1313
  let lifecycle;
2513
1314
  let projectFatal = false;
2514
- if (prepared.collectionErrors.length > 0) cases.push(...notRunSuite(prepared, 'project-preflight-failed'));
1315
+ if (prepared.collectionErrors.length > 0 || 'run' === plan.preflightScope && totalErrors > 0) cases.push(...notRunSuite(prepared, 'project-preflight-failed'));
2515
1316
  else if (rootController.signal.aborted) cases.push(...notRunSuite(prepared, 'interrupted'));
2516
1317
  else if (bailReached()) cases.push(...notRunSuite(prepared, 'bail'));
2517
1318
  else {
@@ -2521,96 +1322,101 @@ async function runTestProject(options = {}) {
2521
1322
  signal: rootController.signal
2522
1323
  });
2523
1324
  let hasProjectExecutionError = false;
2524
- let projectExecutionError;
2525
1325
  try {
2526
1326
  await runtime.start();
2527
- if (runtime.canRun) for (const [documentIndex, document] of prepared.documents.entries()){
2528
- if (rootController.signal.aborted || bailReached() || projectFatal) {
2529
- const reason = rootController.signal.aborted ? 'interrupted' : projectFatal ? 'fatal-error' : 'bail';
2530
- cases.push(...document.cases.map((item)=>test_project_runner_asNotRun(document.documentId, item, project.name, reason)));
2531
- continue;
1327
+ if (runtime.canRun) {
1328
+ const runDocument = async (invocation, documentIndex)=>{
1329
+ const { document } = invocation;
1330
+ if (rootController.signal.aborted || bailReached() || projectFatal) {
1331
+ const reason = rootController.signal.aborted ? 'interrupted' : projectFatal ? 'fatal-error' : 'bail';
1332
+ cases.push(...document.cases.map((item)=>asNotRun(document.documentId, item, project.name, reason)));
1333
+ return;
1334
+ }
1335
+ projectProgress(` [document ${documentIndex + 1}/${prepared.documents.length}] ${document.sourcePath}`);
1336
+ await executeDocumentInvocation({
1337
+ invocation,
1338
+ project,
1339
+ projectContext: runtime.context,
1340
+ runDir,
1341
+ signal: runtime.signal,
1342
+ rootSignal: rootController.signal,
1343
+ shouldBail: bailReached,
1344
+ isProjectFatal: ()=>projectFatal,
1345
+ markProjectFatal: ()=>{
1346
+ projectFatal = true;
1347
+ },
1348
+ addFailedCases: (count)=>{
1349
+ failedCaseCount += count;
1350
+ },
1351
+ onProgress: projectProgress,
1352
+ sinks: {
1353
+ cases,
1354
+ documents
1355
+ }
1356
+ });
1357
+ };
1358
+ let prerequisiteFailed = false;
1359
+ const prerequisite = prepared.invocations.find((invocation)=>invocation.document.documentId === prepared.prerequisiteDocumentId);
1360
+ const prerequisiteCount = prerequisite ? 1 : 0;
1361
+ if (prerequisite) {
1362
+ await runDocument(prerequisite, 0);
1363
+ prerequisiteFailed = documents.at(-1)?.status === 'failed' || latestById(cases, (item)=>item.caseId).some((item)=>'success' !== item.status);
2532
1364
  }
2533
- projectProgress(` [document ${documentIndex + 1}/${prepared.documents.length}] ${document.sourcePath}`);
2534
- const execution = await runWorkflowDocument(document, {
2535
- resolveNode: project.nodes.require.bind(project.nodes),
2536
- project,
2537
- projectContext: runtime.context,
2538
- retry: project.retry,
2539
- signal: runtime.signal,
2540
- defaultTimeoutMs: definition.test.testTimeout,
2541
- shouldStop: ()=>rootController.signal.aborted || bailReached() || projectFatal,
2542
- stopReason: ()=>rootController.signal.aborted ? 'interrupted' : projectFatal ? 'fatal-error' : 'bail',
2543
- isFatalError: (run)=>[
2544
- ...run.beforeEach,
2545
- ...run.steps,
2546
- ...run.afterEach
2547
- ].some((step)=>step.error && isFatalDeviceError(step.error)) || (run.teardownErrors ?? []).some(isFatalDeviceError),
2548
- onCaseStart: (collectedCase)=>{
2549
- projectProgress(` [case ${collectedCase.caseIndex + 1}/${document.cases.length}] ${collectedCase.definition.name}`);
2550
- },
2551
- onStepStart: (info)=>{
2552
- const indent = 'case' === info.scope ? ' ' : ' ';
2553
- projectProgress(`${indent}→ ${test_project_runner_formatStep(info)}`);
2554
- },
2555
- onStepResult: (info, result)=>projectProgress(formatStepResult(info, result)),
2556
- onCaseResult: (attempt)=>{
2557
- runInfrastructureCallback(()=>writeCaseAttemptResult(runDir, project.projectId, document.documentId, attempt));
2558
- projectProgress(` ${'success' === attempt.status ? '✓' : '✗'} attempt ${attempt.attemptIndex + 1}/${project.retry + 1}: ${attempt.name} (${attempt.durationMs} ms)`);
2559
- },
2560
- onCaseOutcome: (outcome)=>{
2561
- if ('failed' === outcome.status) failedCaseCount += 1;
2562
- if (caseHasFatalError(outcome)) projectFatal = true;
2563
- },
2564
- onDocumentResult: (documentResult)=>runInfrastructureCallback(()=>writeWorkflowDocumentResult(runDir, documentResult))
1365
+ if (!prerequisiteFailed) await runConcurrentJobs(prepared.invocations.filter((invocation)=>invocation !== prerequisite), {
1366
+ concurrency: prepared.documentConcurrency,
1367
+ shouldStop: ()=>rootController.signal.aborted || bailReached() || projectFatal
1368
+ }, async (invocation, index)=>{
1369
+ try {
1370
+ await runDocument(invocation, index + prerequisiteCount);
1371
+ } catch (error) {
1372
+ recordInfrastructureError(error);
1373
+ throw error;
1374
+ }
2565
1375
  });
2566
- cases.push(...execution.cases.map((outcome)=>({
2567
- ...outcome,
2568
- documentId: document.documentId
2569
- })));
2570
- if (documentHasFatalError(execution.document)) projectFatal = true;
2571
- documents.push(execution.document);
2572
- }
2573
- else cases.push(...notRunSuite(prepared, rootController.signal.aborted ? 'interrupted' : 'project-setup-failed'));
1376
+ const completed = new Set(cases.map((item)=>item.caseId));
1377
+ const reason = prerequisiteFailed ? 'project-setup-failed' : rootController.signal.aborted ? 'interrupted' : projectFatal ? 'fatal-error' : 'bail';
1378
+ cases.push(...notRunSuite(prepared, reason).filter((item)=>!completed.has(item.caseId)));
1379
+ } else cases.push(...notRunSuite(prepared, rootController.signal.aborted ? 'interrupted' : 'project-setup-failed'));
2574
1380
  } catch (error) {
2575
1381
  hasProjectExecutionError = true;
2576
- projectExecutionError = error;
2577
1382
  recordInfrastructureError(error);
2578
1383
  } finally{
2579
- const hasFailure = hasProjectExecutionError || cases.some((item)=>'success' !== item.status) || documents.some((item)=>'failed' === item.status) || projectFatal || rootController.signal.aborted;
1384
+ const hasFailure = hasProjectExecutionError || latestById(cases, (item)=>item.caseId).some((item)=>'success' !== item.status) || latestById(documents, (item)=>item.documentId).some((item)=>'failed' === item.status) || projectFatal || rootController.signal.aborted;
2580
1385
  lifecycle = await runtime.finish(hasFailure ? 'failed' : 'success');
2581
1386
  }
2582
- if (hasProjectExecutionError) throw projectExecutionError;
1387
+ if (hasProjectExecutionError) {
1388
+ const completed = new Set(cases.map((item)=>item.caseId));
1389
+ cases.push(...notRunSuite(prepared, 'interrupted').filter((item)=>!completed.has(item.caseId)));
1390
+ }
2583
1391
  }
1392
+ const order = new Map(prepared.documents.map((document, index)=>[
1393
+ document.documentId,
1394
+ index
1395
+ ]));
1396
+ const documentOrder = (id)=>order.get(id) ?? Number.MAX_SAFE_INTEGER;
1397
+ cases.sort((a, b)=>documentOrder(a.documentId) - documentOrder(b.documentId) || a.caseIndex - b.caseIndex);
1398
+ documents.sort((a, b)=>documentOrder(a.documentId) - documentOrder(b.documentId) || (a.attemptIndex ?? 0) - (b.attemptIndex ?? 0));
2584
1399
  return buildProjectResult(prepared, cases, documents, lifecycle);
2585
1400
  };
2586
1401
  const projectResults = new Array(preparedProjects.length);
2587
- let nextProjectIndex = 0;
2588
- const claimNextProject = ()=>{
2589
- if (hasInfrastructureError || rootController.signal.aborted || bailReached() || nextProjectIndex >= preparedProjects.length) return;
2590
- const projectIndex = nextProjectIndex;
2591
- nextProjectIndex += 1;
2592
- return projectIndex;
2593
- };
2594
- const worker = async ()=>{
2595
- try {
2596
- while(true){
2597
- const projectIndex = claimNextProject();
2598
- if (void 0 === projectIndex) return;
2599
- projectResults[projectIndex] = await runPreparedProject(preparedProjects[projectIndex], projectIndex);
2600
- }
2601
- } catch (error) {
2602
- recordInfrastructureError(error);
2603
- }
2604
- };
2605
1402
  try {
2606
- await Promise.allSettled(Array.from({
2607
- length: effectiveConcurrency
2608
- }, ()=>worker()));
1403
+ await runConcurrentJobs(preparedProjects, {
1404
+ concurrency: effectiveConcurrency,
1405
+ shouldStop: ()=>hasInfrastructureError || rootController.signal.aborted || bailReached()
1406
+ }, async (prepared, index)=>{
1407
+ try {
1408
+ projectResults[index] = await runPreparedProject(prepared, index);
1409
+ } catch (error) {
1410
+ recordInfrastructureError(error);
1411
+ throw error;
1412
+ }
1413
+ });
1414
+ } catch (error) {
1415
+ recordInfrastructureError(error);
2609
1416
  } finally{
2610
1417
  process.off('SIGINT', sigint);
2611
1418
  process.off('SIGTERM', sigterm);
2612
1419
  }
2613
- if (hasInfrastructureError) throw firstInfrastructureError;
2614
1420
  const completedProjectResults = preparedProjects.map((prepared, projectIndex)=>{
2615
1421
  const result = projectResults[projectIndex];
2616
1422
  if (result) return result;
@@ -2619,7 +1425,7 @@ async function runTestProject(options = {}) {
2619
1425
  return buildSkippedProjectResult(prepared, projectIndex, reason);
2620
1426
  });
2621
1427
  const summary = summarize(completedProjectResults);
2622
- const failed = rootController.signal.aborted || summary.failed > 0 || summary.notRun > 0 || summary.collectionErrors > 0 || summary.documentFailures > 0 || summary.projectFailures > 0;
1428
+ const failed = hasInfrastructureError || rootController.signal.aborted || summary.failed > 0 || summary.notRun > 0 || summary.collectionErrors > 0 || summary.documentFailures > 0 || summary.projectFailures > 0;
2623
1429
  const endedAt = new Date();
2624
1430
  const result = {
2625
1431
  schemaVersion: 3,
@@ -2636,34 +1442,66 @@ async function runTestProject(options = {}) {
2636
1442
  projects: completedProjectResults,
2637
1443
  cases: completedProjectResults.flatMap((project)=>project.cases),
2638
1444
  documents: completedProjectResults.flatMap((project)=>project.documents),
2639
- collectionErrors: completedProjectResults.flatMap((project)=>project.collectionErrors)
1445
+ collectionErrors: completedProjectResults.flatMap((project)=>project.collectionErrors),
1446
+ ...hasInfrastructureError ? {
1447
+ errors: infrastructureErrors.map(asExecutionError)
1448
+ } : {}
2640
1449
  };
2641
- writeTestProjectRunResult({
2642
- projectRoot,
2643
- ...configPath ? {
2644
- configPath
2645
- } : {},
2646
- result
2647
- });
2648
- const reportPath = new TestRunReportAssembler().assemble({
2649
- outputDir: reportDir,
2650
- reportFileName: `test-run-${runId}`,
2651
- sources: collectTestRunReportSources(result),
2652
- buildRunnerDump: (index)=>buildTestRunReportDump(result, index)
2653
- });
2654
- const completedResult = {
2655
- ...result,
2656
- reportPath
1450
+ let completedResult = result;
1451
+ const publish = async (operation, path, callback)=>{
1452
+ try {
1453
+ await callback();
1454
+ return true;
1455
+ } catch (error) {
1456
+ recordInfrastructureError(error instanceof WorkflowPublicationError || error instanceof WorkflowExecutionFailure ? error : new WorkflowPublicationError(operation, path, error));
1457
+ completedResult = {
1458
+ ...completedResult,
1459
+ status: 'failed',
1460
+ exitCode: 1,
1461
+ errors: infrastructureErrors.map(asExecutionError)
1462
+ };
1463
+ return false;
1464
+ }
2657
1465
  };
2658
- writeTestProjectRunResult({
2659
- projectRoot,
2660
- ...configPath ? {
2661
- configPath
2662
- } : {},
2663
- result: completedResult
1466
+ const writeSummary = ()=>publish('write-result', summaryPath, ()=>writeTestProjectRunResult({
1467
+ projectRoot,
1468
+ ...configPath ? {
1469
+ configPath
1470
+ } : {},
1471
+ result: completedResult
1472
+ }));
1473
+ const writeReport = ()=>publish('write-report', reportDir, async ()=>{
1474
+ if (!plan.reportEnabled) return;
1475
+ const reportPath = await new TestRunReportAssembler().assembleAsync({
1476
+ outputDir: reportDir,
1477
+ reportFileName: `midscene-e2e-${runId}`,
1478
+ overwrite: false,
1479
+ sources: collectTestRunReportSources(completedResult),
1480
+ buildRunnerDump: (index)=>{
1481
+ const dump = buildTestRunReportDump(completedResult, index);
1482
+ const warn = getDebug('test-runner:report-assembler', {
1483
+ console: true
1484
+ });
1485
+ for (const diagnostic of dump.diagnostics ?? [])warn(diagnostic.message);
1486
+ return dump;
1487
+ }
1488
+ });
1489
+ completedResult = {
1490
+ ...completedResult,
1491
+ reportPath
1492
+ };
1493
+ });
1494
+ await writeReport();
1495
+ for (const publication of plan.publications)await publish(publication.operation, publication.path, async ()=>{
1496
+ await publication.publish(completedResult);
2664
1497
  });
1498
+ await writeSummary();
1499
+ if (hasInfrastructureError) throw new WorkflowExecutionFailure(completedResult, infrastructureErrors);
2665
1500
  return completedResult;
2666
1501
  }
1502
+ async function runTestProject(options = {}) {
1503
+ return runPreparedTestProject(await prepareTestRun(options));
1504
+ }
2667
1505
  const parseTestCliArgs = (args, cwd = process.cwd())=>{
2668
1506
  const command = 'nodes' === args[0] ? args[0] : void 0;
2669
1507
  const commandOffset = command ? 1 : 0;
@@ -2672,20 +1510,23 @@ const parseTestCliArgs = (args, cwd = process.cwd())=>{
2672
1510
  let resultDir;
2673
1511
  const projectNames = [];
2674
1512
  for(let index = commandOffset; index < args.length; index += 1){
2675
- const arg = args[index];
1513
+ const token = args[index];
1514
+ const equal = token.startsWith('-') ? token.indexOf('=') : -1;
1515
+ const arg = equal < 0 ? token : token.slice(0, equal);
1516
+ const inlineValue = equal < 0 ? void 0 : token.slice(equal + 1);
2676
1517
  if (!arg.startsWith('-')) {
2677
1518
  if (projectRoot) throw new Error('Only one test project directory is allowed.');
2678
1519
  projectRoot = (0, external_node_path_.resolve)(cwd, arg);
2679
1520
  continue;
2680
1521
  }
2681
1522
  if ('--config' === arg || '--result-dir' === arg || '--project' === arg) {
2682
- const value = args[index + 1];
2683
- if (!value) throw new Error(`${arg} requires a value.`);
1523
+ const value = inlineValue ?? args[index + 1];
1524
+ if (!value || value.startsWith('--')) throw new Error(`${arg} requires a value.`);
2684
1525
  if ('--config' === arg) configPath = value;
2685
1526
  else if ('--result-dir' === arg) resultDir = (0, external_node_path_.resolve)(cwd, value);
2686
1527
  else projectNames.push(value);
2687
- index += 1;
2688
- } else throw new Error(`Unknown option: ${arg}`);
1528
+ if (void 0 === inlineValue) index += 1;
1529
+ } else throw new Error(`Unknown option: ${token}`);
2689
1530
  }
2690
1531
  if ('nodes' === command && resultDir) throw new Error('--result-dir is not supported by nodes.');
2691
1532
  if ('nodes' === command && projectNames.length > 1) throw new Error('nodes accepts only one --project name.');
@@ -2707,6 +1548,23 @@ const defaultCliIO = {
2707
1548
  error: console.error,
2708
1549
  write: (message)=>process.stdout.write(message)
2709
1550
  };
1551
+ const testCliHelp = `Midscene Test: run native cases/steps Test projects.
1552
+
1553
+ Usage:
1554
+ midscene-test [file.yaml | directory] [options]
1555
+ midscene-test --config <midscene.config.ts> [options]
1556
+ midscene-test nodes [directory] [--config midscene.config.ts]
1557
+ midscene-test create --help
1558
+
1559
+ Options:
1560
+ --config <path> Native TypeScript or JavaScript Test config
1561
+ --project <name> Select a native execution Project (repeatable)
1562
+ --result-dir <path> Store Test result files in this directory
1563
+ --help, -h Show this help
1564
+ --version Show the package version
1565
+
1566
+ Legacy tasks/flow YAML and its CLI options belong to the midscene command.
1567
+ `;
2710
1568
  const test_command_assertDirectory = (path, label)=>{
2711
1569
  if (!(0, external_node_fs_.existsSync)(path) || !(0, external_node_fs_.statSync)(path).isDirectory()) throw new Error(`${label} does not exist or is not a directory: ${path}`);
2712
1570
  };
@@ -2717,6 +1575,7 @@ const runNodesCommand = async (options, io)=>{
2717
1575
  if (projectRoot) test_command_assertDirectory(projectRoot, 'Test project directory');
2718
1576
  const configSearchRoot = projectRoot ?? cwd;
2719
1577
  const configPath = options.configPath ? (0, external_node_path_.resolve)(configSearchRoot, options.configPath) : discoverTestConfig(configSearchRoot);
1578
+ if (options.configPath && /\.ya?ml$/i.test(configPath)) throw new Error(`midscene-test nodes --config only accepts a native TypeScript or JavaScript project config: ${configPath}.`);
2720
1579
  if (options.configPath && (!configPath || !(0, external_node_fs_.existsSync)(configPath))) throw new Error(`Midscene config does not exist: ${configPath}`);
2721
1580
  const project = await loadTestProject(configPath);
2722
1581
  const projectName = options.projectNames?.[0];
@@ -2744,10 +1603,18 @@ const runNodesCommand = async (options, io)=>{
2744
1603
  async function runTestCli(args, io = defaultCliIO) {
2745
1604
  try {
2746
1605
  if ('create' === args[0]) {
2747
- const { runCreateCommand } = await __webpack_require__.e("891").then(__webpack_require__.bind(__webpack_require__, "./src/cli/create-command.ts"));
1606
+ const { runCreateCommand } = await __webpack_require__.e("214").then(__webpack_require__.bind(__webpack_require__, "./src/cli/create-command.ts"));
2748
1607
  await runCreateCommand(args.slice(1), io);
2749
1608
  return 0;
2750
1609
  }
1610
+ if (args.includes('--help') || args.includes('-h')) {
1611
+ io.log(testCliHelp);
1612
+ return 0;
1613
+ }
1614
+ if (args.includes('--version')) {
1615
+ io.log(package_0.rE);
1616
+ return 0;
1617
+ }
2751
1618
  const options = parseTestCliArgs(args);
2752
1619
  if ('nodes' === options.command) {
2753
1620
  await runNodesCommand(options, io);
@@ -2757,6 +1624,25 @@ async function runTestCli(args, io = defaultCliIO) {
2757
1624
  ...options,
2758
1625
  onProgress: (message)=>io.log(message)
2759
1626
  });
1627
+ for (const failure of result.collectionErrors)io.error(`midscene-test: ${failure.projectName}/${failure.sourcePath}: ${failure.error.message}`);
1628
+ const finalCases = new Map(result.cases.map((outcome)=>[
1629
+ outcome.caseId,
1630
+ outcome
1631
+ ]));
1632
+ for (const outcome of finalCases.values()){
1633
+ if ('failed' !== outcome.status || !outcome.run) continue;
1634
+ const run = outcome.run;
1635
+ const source = `${run.projectName}/${run.sourcePath} / ${run.name}`;
1636
+ for (const step of [
1637
+ ...run.beforeEach,
1638
+ ...run.steps,
1639
+ ...run.afterEach
1640
+ ])if (step.error) io.error(`midscene-test: ${source} / ${step.phase}[${step.stepIndex + 1}] ${step.node}: ${step.error.message}`);
1641
+ for (const error of [
1642
+ ...run.executionErrors ?? [],
1643
+ ...run.teardownErrors ?? []
1644
+ ])io.error(`midscene-test: ${source}: ${error.message}`);
1645
+ }
2760
1646
  io.log(`midscene-test: ${result.summary.passed}/${result.summary.total} cases passed, ${result.summary.failed} failed, ${result.summary.notRun} not run`);
2761
1647
  io.log(`Results: ${result.resultDir}`);
2762
1648
  io.log(`Summary: ${result.summaryPath}`);