codeceptjs 4.0.0-rc.21 → 4.0.0-rc.22

package/lib/plugin/junitReporter.js

@@ -0,0 +1,303 @@
+import fs from 'fs'
+import path from 'path'
+import os from 'os'
+import { mkdirp } from 'mkdirp'
+import { DOMImplementation, XMLSerializer } from '@xmldom/xmldom'
+
+import event from '../event.js'
+import store from '../store.js'
+import output from '../output.js'
+
+const defaultConfig = {
+  outputName: 'report.xml',
+  output: null,
+  testGroupName: 'CodeceptJS',
+  attachSteps: true,
+  attachMeta: true,
+  stepsInFailure: true,
+}
+
+const INVALID_XML_CHARS = new RegExp('[\\u0000-\\u0008\\u000B\\u000C\\u000E-\\u001F\\uFFFE\\uFFFF]', 'g')
+
+/**
+ *
+ * Generates a JUnit-compatible XML report after a test run.
+ *
+ * Unlike Mocha's `mocha-junit-reporter`, this plugin understands CodeceptJS steps and substeps.
+ * For every `<testcase>` it includes:
+ *
+ * * `<properties>` — the test's meta information: every `meta` key from `Scenario('...', { meta })`, plus its `tags` and `retries`
+ * * `<system-out>` — an indented step/substep log (substeps are nested under their meta step); only failed steps are marked
+ * * `<failure>` — for failed tests: the error message, type, stack trace and (optionally) the step trace
+ *
+ * The produced file is consumable by Jenkins, GitLab CI, CircleCI, GitHub Actions test reporters, etc.
+ *
+ * #### Configuration
+ *
+ * ```js
+ * "plugins": {
+ *    "junitReporter": {
+ *      "enabled": true
+ *    }
+ *  }
+ * ```
+ *
+ * Possible config options:
+ *
+ * * `outputName`: file name for the report. Default: `report.xml`.
+ * * `output`: directory where the report is stored, relative to the project root. Default: the `output` directory.
+ * * `testGroupName`: value of the `name` attribute on the root `<testsuites>` element. Default: `CodeceptJS`.
+ * * `attachMeta`: add the test's meta information (`meta` keys, `tags`, `retries`) as `<properties>`. Default: true.
+ * * `attachSteps`: add the step/substep log as `<system-out>`. Default: true.
+ * * `stepsInFailure`: append the step trace to the `<failure>` body. Default: true.
+ *
+ * CLI examples:
+ *
+ * ```
+ * npx codeceptjs run -p junitReporter
+ * npx codeceptjs run -p junitReporter:outputName=junit.xml
+ * ```
+ *
+ * > ℹ When running with `run-workers`, steps are serialized between processes and substep nesting is flattened.
+ *
+ * @param {*} config
+ */
+export default function (config = {}) {
+  config = Object.assign({}, defaultConfig, config)
+
+  let written = false
+
+  const writeReport = result => {
+    if (written) return
+    if (!result || !Array.isArray(result.tests)) return
+    written = true
+
+    const dir = config.output ? path.resolve(store.codeceptDir || process.cwd(), config.output) : store.outputDir || process.cwd()
+    mkdirp.sync(dir)
+    const file = path.join(dir, config.outputName)
+
+    fs.writeFileSync(file, buildXml(result, config))
+    output.plugin('junitReporter', `JUnit report saved to ${file}`)
+  }
+
+  event.dispatcher.on(event.all.result, writeReport)
+  event.dispatcher.on(event.workers.result, writeReport)
+}
+
+function buildXml(result, config) {
+  const doc = new DOMImplementation().createDocument(null, null, null)
+  const suites = groupBySuite(result.tests)
+
+  const root = doc.createElement('testsuites')
+  setAttr(root, 'name', config.testGroupName)
+  setAttr(root, 'tests', result.tests.length)
+  setAttr(root, 'failures', countState(result.tests, 'failed'))
+  setAttr(root, 'skipped', countSkipped(result.tests))
+  setAttr(root, 'errors', 0)
+  setAttr(root, 'time', toSeconds(sumDuration(result.tests)))
+  setAttr(root, 'timestamp', toIso(result.stats && result.stats.start))
+  doc.appendChild(root)
+
+  suites.forEach((tests, index) => {
+    const suite = tests[0] && tests[0].parent
+    const suiteName = (suite && suite.title) || 'Tests'
+    const suiteFile = (suite && suite.file) || (tests[0] && tests[0].file) || ''
+
+    const suiteEl = doc.createElement('testsuite')
+    setAttr(suiteEl, 'name', suiteName)
+    setAttr(suiteEl, 'id', index)
+    setAttr(suiteEl, 'tests', tests.length)
+    setAttr(suiteEl, 'failures', countState(tests, 'failed'))
+    setAttr(suiteEl, 'skipped', countSkipped(tests))
+    setAttr(suiteEl, 'errors', 0)
+    setAttr(suiteEl, 'time', toSeconds(sumDuration(tests)))
+    setAttr(suiteEl, 'timestamp', toIso(suite && suite.startedAt))
+    setAttr(suiteEl, 'hostname', os.hostname())
+    if (suiteFile) setAttr(suiteEl, 'file', suiteFile)
+    root.appendChild(suiteEl)
+
+    for (const test of tests) {
+      suiteEl.appendChild(buildTestCase(doc, test, suiteName, config))
+    }
+  })
+
+  return '<?xml version="1.0" encoding="UTF-8"?>\n' + new XMLSerializer().serializeToString(doc) + '\n'
+}
+
+function buildTestCase(doc, test, suiteName, config) {
+  const testEl = doc.createElement('testcase')
+  setAttr(testEl, 'name', test.title || '(no title)')
+  setAttr(testEl, 'classname', suiteName)
+  setAttr(testEl, 'time', toSeconds(test.duration || 0))
+  const file = test.file || (test.parent && test.parent.file)
+  if (file) setAttr(testEl, 'file', file)
+
+  if (config.attachMeta) {
+    const properties = metaProperties(test)
+    if (properties.length) {
+      const propertiesEl = doc.createElement('properties')
+      for (const [name, value] of properties) {
+        const prop = doc.createElement('property')
+        setAttr(prop, 'name', name)
+        setAttr(prop, 'value', value)
+        propertiesEl.appendChild(prop)
+      }
+      testEl.appendChild(propertiesEl)
+    }
+  }
+
+  const flat = flattenSteps(Array.isArray(test.steps) ? test.steps : [])
+
+  if (test.state === 'skipped' || test.state === 'pending') {
+    const skipped = doc.createElement('skipped')
+    const reason = skipReason(test)
+    if (reason) setAttr(skipped, 'message', reason)
+    testEl.appendChild(skipped)
+  } else if (test.state === 'failed') {
+    const err = test.err || {}
+    const failure = doc.createElement('failure')
+    setAttr(failure, 'message', err.message || 'Test failed')
+    setAttr(failure, 'type', err.name || 'Error')
+    let body = err.stack || err.message || 'Test failed'
+    if (config.stepsInFailure && flat.length) {
+      body += '\n\nSteps:\n' + flat.map(stepLogLine).join('\n')
+    }
+    failure.appendChild(doc.createTextNode(cleanText(body)))
+    testEl.appendChild(failure)
+  }
+
+  if (config.attachSteps && flat.length) {
+    const out = doc.createElement('system-out')
+    out.appendChild(doc.createTextNode(cleanText(flat.map(stepLogLine).join('\n'))))
+    testEl.appendChild(out)
+  }
+
+  return testEl
+}
+
+function metaProperties(test) {
+  const props = []
+  const meta = test.meta || {}
+  for (const key of Object.keys(meta)) {
+    if (meta[key] === undefined || meta[key] === null) continue
+    props.push([key, stringifyMeta(meta[key])])
+  }
+  if (Array.isArray(test.tags) && test.tags.length) {
+    props.push(['tags', test.tags.join(' ')])
+  }
+  if (test.retries > 0 || test.retryNum > 0) {
+    props.push(['retries', String(test.retryNum || test.retries)])
+  }
+  return props
+}
+
+function stringifyMeta(value) {
+  if (typeof value === 'string') return value
+  if (typeof value === 'number' || typeof value === 'boolean') return String(value)
+  try {
+    return JSON.stringify(value)
+  } catch (err) {
+    return String(value)
+  }
+}
+
+function flattenSteps(steps) {
+  const out = []
+  let prevChain = []
+
+  for (const step of steps) {
+    const chain = metaChain(step)
+
+    let common = 0
+    while (common < chain.length && common < prevChain.length && chain[common].key === prevChain[common].key) common++
+
+    for (let d = common; d < chain.length; d++) {
+      out.push({ depth: d, step: chain[d].step })
+    }
+    out.push({ depth: chain.length, step })
+    prevChain = chain
+  }
+
+  return out
+}
+
+function metaChain(step) {
+  const chain = []
+  let meta = step && step.metaStep
+  while (meta) {
+    chain.unshift({ step: meta, key: meta })
+    meta = meta.metaStep
+  }
+  if (!chain.length && step && step.parent && step.parent.title) {
+    chain.push({ step: { title: step.parent.title, status: step.status }, key: `meta:${step.parent.title}` })
+  }
+  return chain
+}
+
+function stepLogLine(entry) {
+  const indent = '  '.repeat(entry.depth)
+  const mark = entry.step && entry.step.status === 'failed' ? '[FAILED] ' : ''
+  return `${indent}${mark}${stepText(entry.step)} (${stepDuration(entry.step)}ms)`
+}
+
+function stepText(step) {
+  if (step && typeof step.toString === 'function' && step.toString !== Object.prototype.toString) return step.toString()
+  return (step && (step.title || step.name)) || 'step'
+}
+
+function stepDuration(step) {
+  if (!step) return 0
+  if (typeof step.duration === 'number' && step.duration >= 0) return step.duration
+  if (step.startTime && step.endTime) return Math.max(0, step.endTime - step.startTime)
+  return 0
+}
+
+function groupBySuite(tests) {
+  const groups = []
+  const byKey = new Map()
+  for (const test of tests) {
+    const key = test.parent || test
+    if (!byKey.has(key)) {
+      const list = []
+      byKey.set(key, list)
+      groups.push(list)
+    }
+    byKey.get(key).push(test)
+  }
+  return groups
+}
+
+function skipReason(test) {
+  if (test.opts && test.opts.skipInfo && test.opts.skipInfo.message) return test.opts.skipInfo.message
+  if (test.meta && test.meta.skipReason) return test.meta.skipReason
+  return ''
+}
+
+function countState(tests, state) {
+  return tests.filter(t => t.state === state).length
+}
+
+function countSkipped(tests) {
+  return tests.filter(t => t.state === 'skipped' || t.state === 'pending').length
+}
+
+function sumDuration(tests) {
+  return tests.reduce((sum, t) => sum + (t.duration || 0), 0)
+}
+
+function toSeconds(ms) {
+  return (Math.max(0, ms) / 1000).toFixed(3)
+}
+
+function toIso(value) {
+  const date = value ? new Date(value) : new Date()
+  return Number.isNaN(date.getTime()) ? new Date().toISOString() : date.toISOString()
+}
+
+function cleanText(text) {
+  return String(text == null ? '' : text).replace(INVALID_XML_CHARS, '')
+}
+
+function setAttr(el, name, value) {
+  el.setAttribute(name, cleanText(value))
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "codeceptjs",
-  "version": "4.0.0-rc.21",
+  "version": "4.0.0-rc.22",
   "type": "module",
   "description": "Supercharged End 2 End Testing Framework for NodeJS",
   "keywords": [

package/docs/internal-api.md

@@ -1,265 +0,0 @@
----
-permalink: /internal-api
-title: Internal API
----
-
-## Concepts
-
-In this guide we will overview the internal API of CodeceptJS.
-This knowledge is required for customization, writing plugins, etc.
-
-CodeceptJS exposes its internal API as named exports of the `codeceptjs` package. Import only what you need:
-
-```js
-import { recorder, event, output, container, config } from 'codeceptjs'
-```
-
-> Older code may have relied on a global `codeceptjs` object (`const { recorder } = codeceptjs`). That global only exists under `noGlobals: false` (the deprecated 3.x default) — prefer named imports.
-
-These internal objects are available:
-
-* [`codecept`](https://github.com/Codeception/CodeceptJS/blob/master/lib/codecept.js): test runner class
-* [`config`](https://github.com/Codeception/CodeceptJS/blob/master/lib/config.js): current codecept config
-* [`event`](https://github.com/Codeception/CodeceptJS/blob/master/lib/event.js): event listener
-* [`recorder`](https://github.com/Codeception/CodeceptJS/blob/master/lib/recorder.js): global promise chain
-* [`output`](https://github.com/Codeception/CodeceptJS/blob/master/lib/output.js): internal printer
-* [`container`](https://github.com/Codeception/CodeceptJS/blob/master/lib/container.js): dependency injection container for tests, includes current helpers and support objects
-* [`helper`](https://github.com/Codeception/CodeceptJS/blob/master/lib/helper.js): basic helper class
-* [`actor`](https://github.com/Codeception/CodeceptJS/blob/master/lib/actor.js): basic actor (I) class
-
-[API reference](https://github.com/Codeception/CodeceptJS/tree/master/docs/api) is available on GitHub.
-Also please check the source code of corresponding modules.
-
-### Container
-
-CodeceptJS has a dependency injection container with helpers and support objects.
-They can be retrieved from the container:
-
-```js
-import { container } from 'codeceptjs';
-
-// get object with all helpers
-const helpers = container.helpers();
-
-// get helper by name
-const { WebDriver } = container.helpers();
-
-// get support objects
-const supportObjects = container.support();
-
-// get support object by name
-const { UserPage } = container.support();
-
-// get all registered plugins
-const plugins = container.plugins();
-```
-
-New objects can also be added to container in runtime:
-
-```js
-import { container } from 'codeceptjs';
-import UserPage from './pages/user.js';
-
-container.append({
-  helpers: { // add helper
-    MyHelper: new MyHelper({ config1: 'val1' });
-  },
-  support: { // add page object
-    UserPage,
-  }
-})
-```
-
-> Use this trick to define custom objects inside `boostrap` script
-
-The container also contains the current Mocha instance:
-
-```js
-const mocha = container.mocha();
-```
-
-### Event Listeners
-
-CodeceptJS provides a module with an [event dispatcher and set of predefined events](https://github.com/Codeception/CodeceptJS/blob/master/lib/event.js).
-
-It can be required from codeceptjs package if it is installed locally.
-
-```js
-import { event } from 'codeceptjs';
-
-export default function() {
-
-  event.dispatcher.on(event.test.before, function (test) {
-
-    console.log('--- I am before test --');
-
-  });
-}
-```
-
-Available events:
-
-* `event.test.before(test)` - *async* when `Before` hooks from helpers and from test is executed
-* `event.test.after(test)` - *async* after each test
-* `event.test.started(test)` - *sync* at the very beginning of a test.
-* `event.test.passed(test)` - *sync* when test passed
-* `event.test.failed(test, error)` - *sync* when test failed
-* `event.test.finished(test)` - *sync* when test finished
-* `event.suite.before(suite)` - *async* before a suite
-* `event.suite.after(suite)` - *async* after a suite
-* `event.step.before(step)` - *async* when the step is scheduled for execution
-* `event.step.after(step)`- *async* after a step
-* `event.step.started(step)` - *sync* when step starts.
-* `event.step.passed(step)` - *sync* when step passed.
-* `event.step.failed(step, err)` - *sync* when step failed.
-* `event.step.finished(step)` - *sync* when step finishes.
-* `event.step.comment(step)` - *sync* fired for comments like `I.say`.
-* `event.all.before` - before running tests
-* `event.all.after` - after running tests
-* `event.all.result` - when results are printed
-* `event.workers.before` - before spawning workers in parallel run
-* `event.workers.after` - after workers finished in parallel run
-* `event.workers.result` - test results after workers finished in parallel run
-
-
-> *sync* - means that event is fired in the moment of the action happening.
- *async* - means that event is fired when an action is scheduled. Use `recorder` to schedule your actions.
-
-For further reference look for [currently available listeners](https://github.com/Codeception/CodeceptJS/tree/master/lib/listener) using the event system.
-
-
-### Recorder
-
-To inject asynchronous functions in a test or before/after a test you can subscribe to corresponding event and register a function inside a recorder object. [Recorder](https://github.com/Codeception/CodeceptJS/blob/master/lib/recorder.js) represents a global promises chain.
-
-Provide a function in the first parameter, a function must be async or must return a promise:
-
-```js
-import { event, recorder } from 'codeceptjs';
-import request from 'request';
-
-export default function() {
-
-  event.dispatcher.on(event.test.before, function (test) {
-
-    recorder.add('create fixture data via API', function() {
-      return new Promise((doneFn, errFn) => {
-        request({
-          baseUrl: 'http://api.site.com/',
-          method: 'POST',
-          url: '/users',
-          json: { name: 'john', email: 'john@john.com' }
-        }), (err, httpResponse, body) => {
-          if (err) return errFn(err);
-          doneFn();
-        }
-      });
-    }
-  });
-}
-```
-
-### Config
-
-CodeceptJS config can be accessed from `import { config } from 'codeceptjs'` then `config.get()`:
-
-```js
-import { config } from 'codeceptjs';
-
-// config object has access to all values of the current config file
-
-if (config.get().myKey == 'value') {
-  // run something
-}
-```
-
-
-### Output
-
-Output module provides four verbosity levels. Depending on the mode you can have different information printed using corresponding functions.
-
-* `default`: prints basic information using `output.print`
-* `steps`: toggled by `--steps` option, prints step execution
-* `debug`: toggled by `--debug` option, prints steps, and debug information with `output.debug`
-* `verbose`: toggled by `--verbose` prints debug information and internal logs with `output.log`
-
-It is recommended to avoid `console.log` and use output.* methods for printing.
-
-```js
-import { output } from 'codeceptjs';
-
-output.print('This is basic information');
-output.debug('This is debug information');
-output.log('This is verbose logging information');
-```
-
-#### Test Object
-
-The test events are providing a test object with following properties:
-
-* `title` title of the test
-* `body` test function as a string
-* `opts` additional test options like retries, and others
-* `pending` true if test is scheduled for execution and false if a test has finished
-* `tags` array of tags for this test
-* `artifacts` list of files attached to this test. Screenshots, videos and other files can be saved here and shared accross different reporters
-* `file` path to a file with a test
-* `steps` array of executed steps (available only in `test.passed`, `test.failed`, `test.finished` event)
-* `skipInfo` additional test options when test skipped 
-* * `message` string with reason for skip
-* * `description` string with test body
-and others
-
-#### Step Object
-
-Step events provide step objects with following fields:
-
-* `name` name of a step, like 'see', 'click', and others
-* `actor` current actor, in most cases it is `I`
-* `helper` current helper instance used to execute this step
-* `helperMethod` corresponding helper method, in most cases is the same as `name`
-* `status` status of a step (passed or failed)
-* `prefix` if a step is executed inside `within` block contain within text, like: 'Within .js-signup-form'.
-* `args` passed arguments
-
-Whenever you execute tests with `--verbose` option you will see registered events and promises executed by a recorder.
-
-## Custom Runner
-
-You can run CodeceptJS tests from your script.
-
-```js
-import { codecept as Codecept } from 'codeceptjs';
-
-// define main config
-const config = { 
-  helpers: { 
-    WebDriver: { 
-      browser: 'chrome', 
-      url: 'http://localhost' 
-    }
-  }
-};
-
-const opts = { steps: true };
-
-// run CodeceptJS inside async function
-(async () => {
-  const codecept = new Codecept(config, options);
-  codecept.init(__dirname);
-
-  try {
-    await codecept.bootstrap();
-    codecept.loadTests('**_test.js');
-    // run tests
-    await codecept.run(test);
-  } catch (err) {
-    printError(err);
-    process.exitCode = 1;
-  } finally {
-    await codecept.teardown();
-  }    
-})();
-```
-
-> Also, you can run tests inside workers in a custom scripts. Please refer to the [parallel execution](/parallel) guide for more details.