@lumjs/tests 1.5.0 → 1.7.0

package/lib/harness/browser.js

@@ -0,0 +1,18 @@
+// Browser harness plugin
+const Plugin = require('./plugin');
+
+/**
+ * Browser Harness plugin
+ * 
+ * @exports module:@lumjs/tests/harness~BrowserPlugin
+ * @extends module:@lumjs/tetss/harness~Plugin
+ */
+class BrowserPlugin extends Plugin
+{
+  run(queued)
+  {
+    throw new Error("Browser tests not supported yet");
+  }
+} // Node plugin class
+
+module.exports = BrowserPlugin;

package/lib/harness/errors.js

@@ -0,0 +1,44 @@
+
+/**
+ * An error for when a test set had failures
+ * 
+ * @prop {number} failed - The number of tests that failed
+ * 
+ * @exports module:@lumjs/tests/harness~TestFailure
+ * @extends Error
+ */
+class TestFailure extends Error 
+{
+  constructor(failed)
+  {
+    super(`${failed} tests failed`);
+    this.name = 'TestFailure';
+    this.failed = failed;
+  }
+}
+
+/**
+ * An error for when the test plan was broken
+ * 
+ * @prop {number} planned - The number of tests planned
+ * @prop {number} ran - The number of tests actually ran
+ * 
+ * @exports module:@lumjs/tests/harness~PlanFailure
+ * @extends Error
+ */
+class PlanFailure extends Error
+{
+  constructor(planned, ran)
+  {
+    super(`${planned} tests planned; ${ran} tests ran`);
+    this.name = 'PlanFailure';
+    this.planned = planned;
+    this.ran = ran;
+  }
+}
+
+// Export all of our custom errors.
+module.exports =
+{
+  TestFailure, PlanFailure,
+}

package/lib/harness/index.js

@@ -0,0 +1,260 @@
+const core = require('@lumjs/core');
+const {def,lazy,context:ctx} = core;
+const {B,F} = core.types;
+const QueuedTest = require('./queuedtest');
+const Plugin = require('./plugin');
+
+/**
+ * A class that acts as a *test harness* for running other tests.
+ * 
+ * Loosely based off of my PHP `Lum\Test\Harness` class,
+ * but with a much more modular design.
+ * 
+ * @prop {object} opts - Options passed to constructor
+ * 
+ * @prop {object} testSuite - A `Stats` (or `Test`) instance
+ * 
+ * This meta-test will keep track of the success or failure of
+ * all the other tests. So its `plan` will be equal to the number of
+ * tests we're running, and so forth.
+ * 
+ * @prop {object} plugin - A platform-specific `Plugin`
+ * 
+ * This may be provided as a parameter to the constructor,
+ * or auto-selected based on our Javascript environment.
+ * 
+ * @prop {Array} queue - An array of `QueuedTest` instances
+ * representing tests that we want to run.
+ * 
+ * @prop {Array} stats - An array of `Stats` or `Error` instances
+ * representing tests that have been ran.
+ * 
+ * @prop {object} parser - A lazy-loaded `Parser` instance
+ * 
+ * @exports module:@lumjs/tests/harness
+ */
+class Harness
+{
+  /**
+   * Build a new Harness
+   * 
+   * @param {object} [opts] Options
+   * @param {object} [opts.testSuite] A `Test` or `Stats` instance
+   * 
+   * This is probably not needed. We'll generate a new `Stats` instance.
+   * 
+   * @param {object} [opts.plugin] A `Plugin` instance.
+   * 
+   * This is probably not needed. We'll automatically determine a default
+   * plugin to use based on the JS environment.
+   * 
+   * @param {boolean} [opts.plannedFailure=true] Is a broken plan a failure?
+   * 
+   * If this is `true` and a test set has a non-zero plan, then the
+   * number of tests ran must match the plan or it will be a failure.
+   * 
+   * If this is `false`, then see `opts.plannedWarning` for how we'll
+   * report broken plans.
+   * 
+   * @param {boolean} [opts.plannedWarning=true] Show a broken plan warning?
+   * 
+   * If this is `true` and `opts.plannedFailure` is `false` then when
+   * a test set has a broken plan we'll send a warning to the JS console.
+   * 
+   * If this is `false` then no warning will be shown.
+   * 
+   */
+  constructor(opts={})
+  {
+    // Save a reference to the options.
+    this.opts = opts;
+
+    if (opts.testSuite instanceof Stats)
+    { // Set the test suite instance.
+      this.testSuite = opts.testSuite;
+    }
+    else 
+    { // Build a new test suite instance.
+      this.testSuite = new Stats(opts.testSuite);
+    }
+
+    if (opts.plugin instanceof Plugin)
+    {
+      this.plugin = opts.plugin;
+    }
+    else if (ctx.isBrowser)
+    {
+      this.plugin = require('./browser').new(this);
+    }
+    else if (ctx.isNode)
+    {
+      this.plugin = require('./node').new(this);
+    }
+    else
+    {
+      throw new Error('Could not determine plugin');
+    }
+
+    //console.debug("# ~ plugin", this.plugin, this);
+
+    this.queue = []; // Tests to be ran.
+    this.stats = []; // Results from tests that have been ran
+
+    lazy(this, 'parser', () => require('./parser').new(this));
+
+    this.plannedFailure = opts.plannedFailure ?? true;
+    this.plannedWarning = opts.plannedWarning ?? true;
+
+  } // constructor()
+
+  /**
+   * Add `Test` or `Stats` instances that have been run
+   * to our list of test stats. 
+   * 
+   * @protected
+   * 
+   * @param {object} info - The `QueuedTest` instance
+   * @param {?object} result - The returned result
+   * 
+   * Will usually be a `Test` or `Stats` instance, but may
+   * be an `Error` if an exception was thrown trying to run the test.
+   * 
+   * @returns {object} `this`
+   */
+  addStats(info, stats)
+  {
+    if (info instanceof QueuedTest 
+      && (stats instanceof Stats 
+      || stats instanceof Error))
+    {
+      this.stats[info.name] = {info, stats};
+    }
+    else
+    {
+      throw new TypeError("Invalid object type");
+    }
+    return this;
+  }
+  
+  /**
+   * Add a test to our testing queue.
+   * 
+   * @param {string} filename - A filename for the test.
+   * @param {object} [opts] Options for the `QueuedTest` instance.
+   * @returns {object} `this`
+   */
+  addTest(filename, opts={})
+  {
+    const queued = new QueuedTest(this, filename, opts);
+    this.queue.push(queued);
+    return this;
+  }
+
+  /**
+   * Add an entire folder of tests.
+   * 
+   * This method may not be implemented by all
+   * Harness Plugins. Your mileage may vary.
+   * 
+   * @param {string} dir - The directory name/path
+   * @param {object} [opts] Options 
+   * @returns {object} `this`
+   */
+  addDir(dir, opts={})
+  {
+    if (typeof this.plugin.addDir === F)
+    {
+      this.plugin.addDir(dir, opts);
+    }
+    else 
+    {
+      console.warn("The plugin does not support addDir()");
+    }
+    return this;
+  }
+
+  /**
+   * Run all the queued tests
+   * 
+   * @param {boolean} [plan=true] Set a test plan for the entire suite 
+   * @returns {object} `this`
+   */
+  run(plan=true)
+  {
+    if (plan)
+    {
+      this.testSuite.plan(this.queue.length);
+    }
+
+    for (const queued of this.queue)
+    {
+      this.runTest(queued);
+    }
+
+    this.testSuite.done();
+
+    return this;
+  }
+
+  runTest(queued)
+  {
+    const name = queued.filename;
+    let test;
+
+    try 
+    { 
+      test = this.plugin.run(queued);
+    }
+    catch (err)
+    {
+      test = err;
+    }
+
+    this.addStats(queued, test);
+
+    if (typeof test === Error)
+    { // Something went wrong.
+      this.testSuite.fail(name, test);
+    }
+    else
+    { // Evaluate if the test is successful or not.
+      const ok = this.plugin.ok(test);
+      if (typeof ok === B)
+      { // A simple boolean response
+        this.testSuite.ok(ok, name);
+      }
+      else if (ok instanceof Error)
+      { // An error was returned
+        this.testSuite.fail(name, ok);
+      }
+      else 
+      { // This should never happen...
+        throw new Error("Invalid result from plugin.ok() !!");
+      }
+    }
+
+    return this;
+  }
+
+  tap()
+  {
+    return this.testSuite.tap();
+  }
+
+  get TAP()
+  {
+    return this.tap();
+  }
+
+} // Harness class
+
+def(Harness, 'Plugin', Plugin);
+def(Harness, 'QueuedTest', QueuedTest);
+def(Harness, 'Errors', require('./errors'));
+
+// Export it.
+module.exports = Harness;
+
+// Some classes required at end for recursive sanity reasons.
+
+const Stats = require('../test/stats');

package/lib/harness/node.js

@@ -0,0 +1,71 @@
+// Node.js harness plugin
+const core = require('@lumjs/core');
+const {N} = core.types;
+const Plugin = require('./plugin');
+const cp = require('node:child_process');
+const getCwd = require('node:process').cwd;
+const fs = require('node:fs');
+const path = require('node:path');
+
+/**
+ * Node.js Harness plugin
+ * 
+ * @exports module:@lumjs/tests/harness~NodePlugin
+ * @extends module:@lumjs/tetss/harness~Plugin
+ */
+class NodePlugin extends Plugin
+{
+  runInternal(queued)
+  {
+    let name = path.join(getCwd(), queued.filename);
+    return require(name);
+  }
+
+  runExternal(queued)
+  {
+    const name = queued.filename;
+    const args = queued.options.args ?? [];
+    const proc = cp.spawnSync(name, args, {encoding: 'utf8'});
+    return this.harness.parser.parse(proc.stdout);
+  }
+
+  run(queued)
+  {
+    if (queued.options.external)
+    {
+      return this.runExternal(queued);
+    }
+    else 
+    {
+      return this.runInternal(queued);
+    }
+  }
+
+  addDir(dir, opts={}, recurse)
+  {
+    if (typeof recurse !== N && typeof opts.recurse === N)
+    {
+      recurse = opts.recurse;
+    }
+
+    const ext = opts.ext ?? '.js';
+    const testOpts = opts.test ?? opts;
+    const files = fs.readdirSync(dir, {encoding: 'utf8', withFileTypes: true});
+
+    for (const file of files)
+    {
+      if (file.name === '.' || file.name === '..') continue;
+      if (typeof recurse === N && recurse > 0 && file.isDirectory())
+      { // Recurse to a nested directory.
+        this.addDir(path.join(dir, file.name), opts, recurse-1);
+      }
+      else if (file.isFile() && file.name.endsWith(ext))
+      { // It would seem to be a valid test.
+        this.harness.addTest(path.join(dir, file.name), testOpts);
+      }
+    }
+  }
+
+} // Node plugin class
+
+module.exports = NodePlugin;

package/lib/harness/parser.js

@@ -0,0 +1,153 @@
+const {S,N,needType,isArray,isObj} = require('@lumjs/core/types');
+const Stats = require('../test/stats');
+const Tap = require('../grammar/tap');
+
+const T = 'test',
+      I = 'info',
+      M = 'meta',
+      A = 'item',
+      C = 'comment',
+      O = 'other';
+
+/**
+ * A simplistic TAP parser.
+ * 
+ * Currently only handles *very basic* TAP v12 output.
+ * This is loosely based on the TAP parser in my PHP
+ * `Lum\Test\Harness` library, but is split off into
+ * its own class, and how it handles the parsed data
+ * is quite different. I think I may eventually update
+ * the PHP version to be more like this one.
+ * 
+ * @exports module:@lumjs/tests/harness/parser
+ */
+class Parser
+{
+  /**
+   * Build a TAP Parser.
+   * @param {object} harness - The parent Harness
+   */
+  constructor(harness)
+  {
+    this.harness = harness;
+  }
+
+  /**
+   * Parse a multiline TAP string
+   * 
+   * @param {string} tapSource - The full TAP string to parse.
+   * 
+   * @returns {object} A simple `Test`-like object with the parsed results.
+   */
+  parse(tapSource)
+  {
+    needType(S, tapSource);
+
+    // Our base Test for populating values into.
+    const test = new Stats();
+
+    // An AST that we'll use to populate the stats.
+    const tap = Tap.parse(tapSource);
+    
+    // Test the first line for a plan.
+    if (typeof tap.plan === N)
+    {
+      test.plan(tap.plan);
+    }
+
+    if (tap.items.length === 0) return test; // No lines, cannot continue.
+
+    // A structure to store our parsing state.
+    const cur = 
+    {
+      log: null,
+      pos: -1,
+      cmd: null,
+    }
+
+    const unhandled = line => console.log("unhandled log line", {line, cur});
+
+    for (const line of tap.lines)
+    {
+      if (line.type === T)
+      { // A regular test line; will increment the counter
+        if (line.skip)
+        {
+          cur.log = test.skip(line.comment, line.desc);
+        }
+        else if (line.todo)
+        {
+          cur.log = test.todo(line.comment, line.desc);
+        }
+        else 
+        {
+          cur.log = test.ok(line.ok, line.desc, line.comment);
+        }
+        cur.pos++;
+      }
+      else if (line.type === I && cur.log)
+      { // Log detail fields
+        cur.cmd = line.info;
+        cur.log.details[line.info] = line.comment;
+      }
+      else if (line.type === M)
+      { // We skip meta info.
+        cur.cmd = null;
+        cur.log = null;
+        continue;
+      }
+      else if (line.type === A && cur.log)
+      { // An array item
+        cur.cmd = null;
+        if (cur.log.details.info === undefined)
+        {
+          cur.log.details.info = [];
+        }
+        cur.log.details.info.push(line.text);
+      }
+      else if (line.type === C)
+      { // A regular comment
+        test.diag(line.text);
+        cur.cmd = null;
+        cur.log = null;
+        cur.pos++;
+      }
+      else if (line.type === O)
+      { // A line that doesn't match anything else.
+        if (cur.log && cur.cmd)
+        { // There's a comment command statement in use.
+          const ld = cur.log.details;
+          ld[cur.cmd] += "\n" + line.text;
+        }
+        else if (cur.log && isArray(cur.log.details.info))
+        { // The current log has info, let's add to it.
+          const ldi = cur.log.details.info;
+          const last = ldi.length - 1;
+          ldi[last] += "\n" + line.text;
+        }
+        else if (cur.pos > -1 && typeof test.log[cur.pos] === S)
+        { // The current entry is a string, let's add to it.
+          test.log[cur.pos] += "\n" + line.text;
+        }
+        else 
+        { // Don't know what to do with this.
+          unhandled(line);
+        }
+      }
+      else 
+      { // Can't do anything, so warn about it and move on.
+        unhandled(line);
+      }
+    }
+    
+    return test;
+  }
+
+  static new(opts)
+  {
+    return new Parser(opts);
+  }
+}
+
+// Export the class as the parser.
+module.exports = Parser;

package/lib/harness/plugin.js

@@ -0,0 +1,77 @@
+const core = require('@lumjs/core');
+const {AbstractClass} = core;
+const {TestFailure,PlanFailure} = require('./errors');
+
+/**
+ * An abstract base class for Harness plugins
+ * 
+ * @exports module:@lumjs/tests/harness~Plugin
+ */
+class Plugin extends AbstractClass
+{
+  /**
+   * (Internal class constructor)
+   * @param {module:@lumjs/tests/harness} harness - Parent `Harness` instance
+   */
+  constructor(harness)
+  {
+    super();
+    this.harness = harness;
+    this.$needs('run');
+  }
+
+  /**
+   * (ABSTRACT METHOD) Run a test
+   * 
+   * This method must be implemented by every plugin.
+   * 
+   * @name module:@lumjs/tests/harness~Plugin#run
+   * @function
+   * @param {object} queued - A queued test object
+   */
+
+  /**
+   * See if a test had failures
+   * 
+   * @param {object} test - The `Test` or `Stats` object to check
+   * @returns {true|Error} 
+   * 
+   * - Will return `true` if no failures reported.
+   * - Will return a `TestFailure` if the test had failures reported.
+   * - May return a `PlanFailure` if the test plan was broken.
+   *   This only applies if `this.harness.plannedFailure` is `true`.
+   * 
+   */
+  ok(test)
+  {
+    if (test.failed !== 0) 
+    { // The test had failed units.
+      return new TestFailure(test.failed);
+    }
+
+    const planned = test.planned,
+          ran     = test.ran;
+
+    const planOk = (planned === 0 || planned === ran);
+
+    if (this.harness.plannedFailure && !planOk)
+    { // The number of tests ran was not the number of tests planned.
+      return new PlanFailure(planned, ran);
+    }
+    else if (this.harness.plannedWarning && !planOk)
+    { // Report the plan failure to the console log.
+      console.warn(new PlanFailure(planned, ran));
+    }
+
+    // If we reached here, everything looks good.
+    return true;
+  }
+
+  static new(harness)
+  {
+    return new this(harness);
+  }
+
+}
+
+module.exports = Plugin;

package/lib/harness/queuedtest.js

@@ -0,0 +1,14 @@
+/**
+ * A class representing a queued test.
+ * @exports module:@lumjs/tests/harness~QueuedTest
+ */
+class QueuedTest 
+{
+  constructor(harness, filename, opts)
+  {
+    this.harness  = harness;
+    this.filename = filename;
+    this.options  = opts;
+  }
+}
+module.exports = QueuedTest;

package/lib/index.js

@@ -2,28 +2,34 @@
  * Several test related classes.
  * @module @lumjs/tests
  */
+const {lazy} = require('@lumjs/core/types');
 
 /**
- * Test class.
- * @alias module:@lumjs/tests.Test
+ * The main Test class
+ * 
  * @see module:@lumjs/tests/test
  */
-const Test = require('./test');
-module.exports.Test = Test;
+exports.Test = require('./test');
+
+const lz = {enumerable: true};
 
 /**
- * Test Harness class.
- * @alias module:@lumjs/tests.Harness
+ * Test Harness class (lazy loaded)
+ * 
+ * @name module:@lumjs/tests.Harness
+ * @class
  * @see module:@lumjs/tests/harness
  */
-module.exports.Harness = require('./harness');
+lazy(exports, 'Harness', () => require('./harness'), lz);
 
 /**
- * Functional API registration.
- * @alias module:@lumjs/tests.functional
- * @see module:@lumjs/tests/functional
+ * Functional API registration (lazy loaded)
+ * 
+ * @name module:@lumjs/tests.functional
+ * @function
+ * @see module:@lumjs/tests/test/functional
  */
-module.exports.functional = require('./functional');
+lazy(exports, 'functional', () => require('./test/functional'), lz);
 
 /**
  * Create a new Test instance.
@@ -31,8 +37,7 @@ module.exports.functional = require('./functional');
  * @param {object} [opts] Options to pass to the `Test` constructor.
  * @returns {Test} A new test instance.
  */
-module.exports.new = function(opts={})
+exports.new = function(opts={})
 {
-  return new Test(opts);
+  return new exports.Test(opts);
 }
-