codeceptjs 4.0.0-rc.21 → 4.0.0-rc.23

package/lib/plugin/junitReporter.js

@@ -0,0 +1,303 @@
+import fs from 'fs'
+import path from 'path'
+import os from 'os'
+import { mkdirp } from 'mkdirp'
+import { DOMImplementation, XMLSerializer } from '@xmldom/xmldom'
+
+import event from '../event.js'
+import store from '../store.js'
+import output from '../output.js'
+
+const defaultConfig = {
+  outputName: 'report.xml',
+  output: null,
+  testGroupName: 'CodeceptJS',
+  attachSteps: true,
+  attachMeta: true,
+  stepsInFailure: true,
+}
+
+const INVALID_XML_CHARS = new RegExp('[\\u0000-\\u0008\\u000B\\u000C\\u000E-\\u001F\\uFFFE\\uFFFF]', 'g')
+
+/**
+ *
+ * Generates a JUnit-compatible XML report after a test run.
+ *
+ * Unlike Mocha's `mocha-junit-reporter`, this plugin understands CodeceptJS steps and substeps.
+ * For every `<testcase>` it includes:
+ *
+ * * `<properties>` — the test's meta information: every `meta` key from `Scenario('...', { meta })`, plus its `tags` and `retries`
+ * * `<system-out>` — an indented step/substep log (substeps are nested under their meta step); only failed steps are marked
+ * * `<failure>` — for failed tests: the error message, type, stack trace and (optionally) the step trace
+ *
+ * The produced file is consumable by Jenkins, GitLab CI, CircleCI, GitHub Actions test reporters, etc.
+ *
+ * #### Configuration
+ *
+ * ```js
+ * "plugins": {
+ *    "junitReporter": {
+ *      "enabled": true
+ *    }
+ *  }
+ * ```
+ *
+ * Possible config options:
+ *
+ * * `outputName`: file name for the report. Default: `report.xml`.
+ * * `output`: directory where the report is stored, relative to the project root. Default: the `output` directory.
+ * * `testGroupName`: value of the `name` attribute on the root `<testsuites>` element. Default: `CodeceptJS`.
+ * * `attachMeta`: add the test's meta information (`meta` keys, `tags`, `retries`) as `<properties>`. Default: true.
+ * * `attachSteps`: add the step/substep log as `<system-out>`. Default: true.
+ * * `stepsInFailure`: append the step trace to the `<failure>` body. Default: true.
+ *
+ * CLI examples:
+ *
+ * ```
+ * npx codeceptjs run -p junitReporter
+ * npx codeceptjs run -p junitReporter:outputName=junit.xml
+ * ```
+ *
+ * > ℹ When running with `run-workers`, steps are serialized between processes and substep nesting is flattened.
+ *
+ * @param {*} config
+ */
+export default function (config = {}) {
+  config = Object.assign({}, defaultConfig, config)
+
+  let written = false
+
+  const writeReport = result => {
+    if (written) return
+    if (!result || !Array.isArray(result.tests)) return
+    written = true
+
+    const dir = config.output ? path.resolve(store.codeceptDir || process.cwd(), config.output) : store.outputDir || process.cwd()
+    mkdirp.sync(dir)
+    const file = path.join(dir, config.outputName)
+
+    fs.writeFileSync(file, buildXml(result, config))
+    output.plugin('junitReporter', `JUnit report saved to ${file}`)
+  }
+
+  event.dispatcher.on(event.all.result, writeReport)
+  event.dispatcher.on(event.workers.result, writeReport)
+}
+
+function buildXml(result, config) {
+  const doc = new DOMImplementation().createDocument(null, null, null)
+  const suites = groupBySuite(result.tests)
+
+  const root = doc.createElement('testsuites')
+  setAttr(root, 'name', config.testGroupName)
+  setAttr(root, 'tests', result.tests.length)
+  setAttr(root, 'failures', countState(result.tests, 'failed'))
+  setAttr(root, 'skipped', countSkipped(result.tests))
+  setAttr(root, 'errors', 0)
+  setAttr(root, 'time', toSeconds(sumDuration(result.tests)))
+  setAttr(root, 'timestamp', toIso(result.stats && result.stats.start))
+  doc.appendChild(root)
+
+  suites.forEach((tests, index) => {
+    const suite = tests[0] && tests[0].parent
+    const suiteName = (suite && suite.title) || 'Tests'
+    const suiteFile = (suite && suite.file) || (tests[0] && tests[0].file) || ''
+
+    const suiteEl = doc.createElement('testsuite')
+    setAttr(suiteEl, 'name', suiteName)
+    setAttr(suiteEl, 'id', index)
+    setAttr(suiteEl, 'tests', tests.length)
+    setAttr(suiteEl, 'failures', countState(tests, 'failed'))
+    setAttr(suiteEl, 'skipped', countSkipped(tests))
+    setAttr(suiteEl, 'errors', 0)
+    setAttr(suiteEl, 'time', toSeconds(sumDuration(tests)))
+    setAttr(suiteEl, 'timestamp', toIso(suite && suite.startedAt))
+    setAttr(suiteEl, 'hostname', os.hostname())
+    if (suiteFile) setAttr(suiteEl, 'file', suiteFile)
+    root.appendChild(suiteEl)
+
+    for (const test of tests) {
+      suiteEl.appendChild(buildTestCase(doc, test, suiteName, config))
+    }
+  })
+
+  return '<?xml version="1.0" encoding="UTF-8"?>\n' + new XMLSerializer().serializeToString(doc) + '\n'
+}
+
+function buildTestCase(doc, test, suiteName, config) {
+  const testEl = doc.createElement('testcase')
+  setAttr(testEl, 'name', test.title || '(no title)')
+  setAttr(testEl, 'classname', suiteName)
+  setAttr(testEl, 'time', toSeconds(test.duration || 0))
+  const file = test.file || (test.parent && test.parent.file)
+  if (file) setAttr(testEl, 'file', file)
+
+  if (config.attachMeta) {
+    const properties = metaProperties(test)
+    if (properties.length) {
+      const propertiesEl = doc.createElement('properties')
+      for (const [name, value] of properties) {
+        const prop = doc.createElement('property')
+        setAttr(prop, 'name', name)
+        setAttr(prop, 'value', value)
+        propertiesEl.appendChild(prop)
+      }
+      testEl.appendChild(propertiesEl)
+    }
+  }
+
+  const flat = flattenSteps(Array.isArray(test.steps) ? test.steps : [])
+
+  if (test.state === 'skipped' || test.state === 'pending') {
+    const skipped = doc.createElement('skipped')
+    const reason = skipReason(test)
+    if (reason) setAttr(skipped, 'message', reason)
+    testEl.appendChild(skipped)
+  } else if (test.state === 'failed') {
+    const err = test.err || {}
+    const failure = doc.createElement('failure')
+    setAttr(failure, 'message', err.message || 'Test failed')
+    setAttr(failure, 'type', err.name || 'Error')
+    let body = err.stack || err.message || 'Test failed'
+    if (config.stepsInFailure && flat.length) {
+      body += '\n\nSteps:\n' + flat.map(stepLogLine).join('\n')
+    }
+    failure.appendChild(doc.createTextNode(cleanText(body)))
+    testEl.appendChild(failure)
+  }
+
+  if (config.attachSteps && flat.length) {
+    const out = doc.createElement('system-out')
+    out.appendChild(doc.createTextNode(cleanText(flat.map(stepLogLine).join('\n'))))
+    testEl.appendChild(out)
+  }
+
+  return testEl
+}
+
+function metaProperties(test) {
+  const props = []
+  const meta = test.meta || {}
+  for (const key of Object.keys(meta)) {
+    if (meta[key] === undefined || meta[key] === null) continue
+    props.push([key, stringifyMeta(meta[key])])
+  }
+  if (Array.isArray(test.tags) && test.tags.length) {
+    props.push(['tags', test.tags.join(' ')])
+  }
+  if (test.retries > 0 || test.retryNum > 0) {
+    props.push(['retries', String(test.retryNum || test.retries)])
+  }
+  return props
+}
+
+function stringifyMeta(value) {
+  if (typeof value === 'string') return value
+  if (typeof value === 'number' || typeof value === 'boolean') return String(value)
+  try {
+    return JSON.stringify(value)
+  } catch (err) {
+    return String(value)
+  }
+}
+
+function flattenSteps(steps) {
+  const out = []
+  let prevChain = []
+
+  for (const step of steps) {
+    const chain = metaChain(step)
+
+    let common = 0
+    while (common < chain.length && common < prevChain.length && chain[common].key === prevChain[common].key) common++
+
+    for (let d = common; d < chain.length; d++) {
+      out.push({ depth: d, step: chain[d].step })
+    }
+    out.push({ depth: chain.length, step })
+    prevChain = chain
+  }
+
+  return out
+}
+
+function metaChain(step) {
+  const chain = []
+  let meta = step && step.metaStep
+  while (meta) {
+    chain.unshift({ step: meta, key: meta })
+    meta = meta.metaStep
+  }
+  if (!chain.length && step && step.parent && step.parent.title) {
+    chain.push({ step: { title: step.parent.title, status: step.status }, key: `meta:${step.parent.title}` })
+  }
+  return chain
+}
+
+function stepLogLine(entry) {
+  const indent = '  '.repeat(entry.depth)
+  const mark = entry.step && entry.step.status === 'failed' ? '[FAILED] ' : ''
+  return `${indent}${mark}${stepText(entry.step)} (${stepDuration(entry.step)}ms)`
+}
+
+function stepText(step) {
+  if (step && typeof step.toString === 'function' && step.toString !== Object.prototype.toString) return step.toString()
+  return (step && step.title) || 'step'
+}
+
+function stepDuration(step) {
+  if (!step) return 0
+  if (typeof step.duration === 'number' && step.duration >= 0) return step.duration
+  if (step.startTime && step.endTime) return Math.max(0, step.endTime - step.startTime)
+  return 0
+}
+
+function groupBySuite(tests) {
+  const groups = []
+  const byKey = new Map()
+  for (const test of tests) {
+    const key = test.parent || test
+    if (!byKey.has(key)) {
+      const list = []
+      byKey.set(key, list)
+      groups.push(list)
+    }
+    byKey.get(key).push(test)
+  }
+  return groups
+}
+
+function skipReason(test) {
+  if (test.opts && test.opts.skipInfo && test.opts.skipInfo.message) return test.opts.skipInfo.message
+  if (test.meta && test.meta.skipReason) return test.meta.skipReason
+  return ''
+}
+
+function countState(tests, state) {
+  return tests.filter(t => t.state === state).length
+}
+
+function countSkipped(tests) {
+  return tests.filter(t => t.state === 'skipped' || t.state === 'pending').length
+}
+
+function sumDuration(tests) {
+  return tests.reduce((sum, t) => sum + (t.duration || 0), 0)
+}
+
+function toSeconds(ms) {
+  return (Math.max(0, ms) / 1000).toFixed(3)
+}
+
+function toIso(value) {
+  const date = value ? new Date(value) : new Date()
+  return Number.isNaN(date.getTime()) ? new Date().toISOString() : date.toISOString()
+}
+
+function cleanText(text) {
+  return String(text == null ? '' : text).replace(INVALID_XML_CHARS, '')
+}
+
+function setAttr(el, name, value) {
+  el.setAttribute(name, cleanText(value))
+}

package/lib/plugin/retryFailedStep.js

@@ -111,11 +111,12 @@ export default function (config) {
   }
 
   event.dispatcher.on(event.step.started, step => {
+    if (!step.title) return
     for (const ignored of config.ignoredSteps) {
-      if (step.name === ignored) return
+      if (step.title === ignored) return
       if (ignored instanceof RegExp) {
-        if (step.name.match(ignored)) return
-      } else if (ignored.indexOf('*') && step.name.startsWith(ignored.slice(0, -1))) return
+        if (step.title.match(ignored)) return
+      } else if (ignored.indexOf('*') && step.title.startsWith(ignored.slice(0, -1))) return
     }
     enableRetry = true
   })

package/lib/plugin/screencast.js

@@ -258,7 +258,7 @@ function formatTimestamp(timestampInMs) {
 }
 
 function stepTitle(step) {
-  let title = `${step.actor}.${step.name}(${step.args ? step.args.join(',') : ''})`
+  let title = `${step.actor}.${step.title}(${step.args ? step.args.join(',') : ''})`
   if (title.length > 100) title = `${title.substring(0, 100)}...`
   return title
 }

package/lib/plugin/screenshot.js

@@ -325,7 +325,7 @@ function wireSlides(options, trigger) {
     if (stepNum === -1) return
     if (savedStep === step) return
     if (scenarioFailed) return
-    if (step.metaStep && step.metaStep.name === 'BeforeSuite') return
+    if (step.metaStep && step.metaStep.title === 'BeforeSuite') return
     if (!currentTest) return
     if (!stepFilter(step)) return
     if (isStepIgnored(step, options.ignoreSteps)) return
@@ -404,7 +404,7 @@ function makeStepFilter(trigger, options) {
 function isStepIgnored(step, patterns) {
   if (!patterns || !patterns.length) return false
   for (const pattern of patterns) {
-    if (step.name && step.name.match(pattern)) return true
+    if (step.title && step.title.match(pattern)) return true
   }
   return false
 }

package/lib/plugin/stepTimeout.js

@@ -68,6 +68,7 @@ export default function(config) {
   config.customTimeoutSteps = config.customTimeoutSteps.concat(config.noTimeoutSteps).concat(config.customTimeoutSteps)
 
   event.dispatcher.on(event.step.before, step => {
+    if (!step.title) return
     let stepTimeout
     for (let stepRule of config.customTimeoutSteps) {
       let customTimeout = 0
@@ -75,7 +76,7 @@ export default function(config) {
         if (stepRule.length > 1) customTimeout = stepRule[1]
         stepRule = stepRule[0]
       }
-      if (stepRule instanceof RegExp ? step.name.match(stepRule) : step.name === stepRule || (stepRule.indexOf('*') && step.name.startsWith(stepRule.slice(0, -1)))) {
+      if (stepRule instanceof RegExp ? step.title.match(stepRule) : step.title === stepRule || (stepRule.indexOf('*') && step.title.startsWith(stepRule.slice(0, -1)))) {
         stepTimeout = customTimeout
         break
       }

package/lib/step/base.js

@@ -10,12 +10,12 @@ const STACK_LINE = 5
 /**
  * Each command in test executed through `I.` object is wrapped in Step.
  * Step allows logging executed commands and triggers hook before and after step execution.
- * @param {string} name
+ * @param {string} title
  */
 class Step {
-  constructor(name) {
+  constructor(title) {
     /** @member {string} */
-    this.name = name
+    this.title = title
     /** @member {Map<number, number>} */
     this.timeouts = new Map()
 
@@ -43,7 +43,7 @@ class Step {
     /** @member {any} */
     this.helper = null
     /** @member {string} */
-    this.helperMethod = name
+    this.helperMethod = title
 
     this.startTime = 0
     this.endTime = 0
@@ -103,7 +103,7 @@ class Step {
 
   /** @return {string} */
   humanize() {
-    return humanizeString(this.name)
+    return humanizeString(this.title)
   }
 
   /** @return {string} */
@@ -180,7 +180,7 @@ class Step {
 
   /** @return {string} */
   toCode() {
-    return `${this.prefix}${this.actor}.${this.name}(${this.humanizeArgs()})${this.suffix}`
+    return `${this.prefix}${this.actor}.${this.title}(${this.humanizeArgs()})${this.suffix}`
   }
 
   isMetaStep() {
@@ -223,7 +223,7 @@ class Step {
 
     return {
       opts: step.opts || {},
-      title: step.name,
+      title: step.title,
       args: args,
       status: step.status,
       startTime: step.startTime,

package/lib/step/comment.js

@@ -1,8 +1,8 @@
 import FuncStep from './func.js'
 
 class CommentStep extends FuncStep {
-  constructor(name, comment) {
-    super(name)
+  constructor(title, comment) {
+    super(title)
     this.fn = () => {}
   }
 }

package/lib/step/helper.js

@@ -2,12 +2,12 @@ import Step from './base.js'
 import store from '../store.js'
 
 class HelperStep extends Step {
-  constructor(helper, name) {
-    super(name)
+  constructor(helper, title) {
+    super(title)
     /** @member {CodeceptJS.Helper} helper corresponding helper */
     this.helper = helper
-    /** @member {string} helperMethod name of method to be executed */
-    this.helperMethod = name
+    /** @member {string} helperMethod title of method to be executed */
+    this.helperMethod = title
   }
 
   /**

package/lib/step/meta.js

@@ -29,7 +29,7 @@ class MetaStep extends Step {
     const actorText = this.actor
 
     if (this.isBDD()) {
-      return `${this.prefix}${actorText} ${this.name} "${this.humanizeArgs()}${this.suffix}"`
+      return `${this.prefix}${actorText} ${this.title} "${this.humanizeArgs()}${this.suffix}"`
     }
 
     if (actorText === 'I') {
@@ -37,14 +37,14 @@ class MetaStep extends Step {
     }
 
     if (!this.actor) {
-      return `${this.name} ${this.humanizeArgs()}${this.suffix}`.trim()
+      return `${this.title} ${this.humanizeArgs()}${this.suffix}`.trim()
     }
 
     return `On ${this.prefix}${actorText}: ${this.humanize()} ${this.humanizeArgs()}${this.suffix}`.trim()
   }
 
   humanize() {
-    return humanizeString(this.name)
+    return humanizeString(this.title)
   }
 
   setTrace() {}

package/lib/step/record.js

@@ -16,12 +16,12 @@ function recordStep(step, args) {
     const { opts, timeout, retry } = stepConfig.getConfig()
 
     if (opts) {
-      output.debug(`Step ${step.name}: options applied ${JSON.stringify(opts)}`)
+      output.debug(`Step ${step.title}: options applied ${JSON.stringify(opts)}`)
       store.stepOptions = opts
       step.opts = opts
     }
     if (timeout) {
-      output.debug(`Step ${step.name} timeout ${timeout}s`)
+      output.debug(`Step ${step.title} timeout ${timeout}s`)
       step.setTimeout(timeout * 1000, TIMEOUT_ORDER.codeLimitTime)
     }
     if (retry) retryStep(retry)
@@ -31,7 +31,7 @@ function recordStep(step, args) {
   // run async before step hooks
   event.emit(event.step.before, step)
 
-  const task = `${step.name}: ${step.humanizeArgs()}`
+  const task = `${step.title}: ${step.humanizeArgs()}`
   let val
 
   // run step inside promise

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "codeceptjs",
-  "version": "4.0.0-rc.21",
+  "version": "4.0.0-rc.23",
   "type": "module",
   "description": "Supercharged End 2 End Testing Framework for NodeJS",
   "keywords": [