kryptheon 0.1.10 → 0.1.12

package/kryptheon-project.js

@@ -0,0 +1,199 @@
+// Working out whether this folder is a project at all.
+//
+// `kryptheon check` reads recordings out of ./tests and baselines out of the
+// folder it was run in. Run it somewhere that is not a project - a home folder,
+// the place a global install happened to leave you - and it will happily find
+// whatever tests/ happens to be lying around and run those instead, reporting
+// results for a project nobody asked about. That has already happened to
+// somebody who installed into their home folder.
+//
+// Quietly testing the wrong project is the worst thing this tool can do. A
+// green that belongs to something else is not a result, it is a lie with a tick
+// next to it. So the answer to "is this a project?" has to be settled before
+// anything runs, and the answer has to be a stop rather than a guess.
+//
+// Pure path work: nothing here runs a test or reads a recording.
+
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+
+// For someone who genuinely keeps recordings outside a package. Deliberately
+// long-winded: it should be something you decide to type, not something you
+// stumble into.
+const ESCAPE_HATCH = 'KRYPTHEON_ALLOW_NO_PACKAGE_JSON';
+
+function hasPackageJson(dir) {
+  try {
+    return fs.statSync(path.join(dir, 'package.json')).isFile();
+  } catch (e) {
+    return false;
+  }
+}
+
+/**
+ * One comparable form of a path.
+ *
+ * The trailing separator has to go or C:\Users and C:\Users\ never match, but
+ * it must not be taken off before the root test below. "C:" without a separator
+ * is not the drive - Node resolves it to the current directory - so a root
+ * compared in this form would quietly become "wherever I happen to be".
+ */
+function tidy(dir) {
+  return path.resolve(String(dir || '')).replace(/[\\/]+$/, '').toLowerCase();
+}
+
+/** The folders that are somebody's whole account or the machine itself. */
+function isHomeOrSystem(dir) {
+  // A drive or filesystem root: C:\, D:\, /. Done on the resolved path, with
+  // its separator still on, for the reason above.
+  const resolved = path.resolve(String(dir || ''));
+  if (resolved === path.parse(resolved).root) return true;
+
+  const here = tidy(dir);
+  if (here === tidy(os.homedir())) return true;
+
+  for (const candidate of systemFolders()) {
+    if (here === tidy(candidate)) return true;
+  }
+  return false;
+}
+
+/**
+ * Every folder that belongs to the machine or to an account rather than to a
+ * project, resolved.
+ *
+ * The first two entries are derived from the running system; the literals after
+ * them are fallbacks for the usual layouts. They overlap - on Windows
+ * USERPROFILE is the home folder and "/Users" resolves to C:\Users - and the
+ * overlap is deliberate: a machine whose home is somewhere unusual is covered
+ * by the derived pair, and a machine whose environment is missing entries is
+ * covered by the literals.
+ */
+function systemFolders() {
+  return [
+    // The folder every account lives in - C:\Users, /home, /Users. On its own
+    // it has no package.json so it would be refused anyway, but one stray npm
+    // install there and it would look exactly like a project.
+    path.dirname(os.homedir()),
+    process.env.SystemRoot,
+    process.env.ProgramFiles,
+    process.env['ProgramFiles(x86)'],
+    process.env.ProgramData,
+    process.env.APPDATA,
+    process.env.LOCALAPPDATA,
+    process.env.USERPROFILE,
+    '/root',
+    '/home',
+    '/Users',
+    '/usr',
+    '/etc',
+    '/bin',
+    '/var',
+    '/opt',
+  ].filter(Boolean);
+}
+
+/**
+ * Whether this folder can be checked, and if not, why not.
+ *
+ * `allowed` is the escape hatch having been set. It is reported rather than
+ * hidden, so a run that only worked because of it says so.
+ */
+function inspectProject(dir, env) {
+  const environment = env || process.env;
+  const allowed = String(environment[ESCAPE_HATCH] || '') !== '';
+  const packaged = hasPackageJson(dir);
+  const personal = isHomeOrSystem(dir);
+  const base = { dir: dir, packaged: packaged, personal: personal, allowed: allowed };
+
+  if (allowed) return Object.assign({ ok: true, reason: null }, base);
+
+  // A home folder is refused even when it has a package.json, and this is not
+  // belt and braces - it is the case that actually happened. Someone installed
+  // the tool into their home folder, so npm left a package.json and a
+  // node_modules there, and a tests folder alongside them. Every sign this
+  // could look for says "project"; none of them is true. The measured state of
+  // one real machine: C:\Users\<name> holding package.json, node_modules and
+  // tests, none of which belong to anything.
+  if (personal) return Object.assign({ ok: false, reason: 'personal' }, base);
+  if (!packaged) return Object.assign({ ok: false, reason: 'no-package' }, base);
+  return Object.assign({ ok: true, reason: null }, base);
+}
+
+/**
+ * Why this folder is not a project, and what to do instead.
+ *
+ * The folder is printed back because the commonest version of this is not
+ * knowing where you are - a new terminal, a global install, a shell that opened
+ * somewhere else.
+ */
+function noProjectLines(inspection) {
+  const lines = [''];
+
+  if (inspection.personal) {
+    lines.push('  This is your home or a system folder, not a project.');
+    lines.push('');
+    lines.push('  You are in:  ' + inspection.dir);
+    lines.push('');
+    // Almost never deliberate, and the one place where running anyway does the
+    // most damage. Said explicitly because a home folder that has had anything
+    // installed into it looks exactly like a project from the outside.
+    lines.push('  Recordings and results are kept beside the app they belong to,');
+    lines.push('  so running here would check whichever tests folder happens to be');
+    lines.push('  sitting in this one - almost certainly not the one you meant.');
+    if (inspection.packaged) {
+      lines.push('');
+      // Said plainly, because otherwise the package.json sitting right there
+      // makes this message look like the bug rather than the diagnosis.
+      lines.push('  There is a package.json here, but that is not what makes a project.');
+      lines.push('  It looks like npm install was run in this folder by mistake, which');
+      lines.push('  leaves a package.json and a node_modules behind.');
+    }
+    lines.push('');
+  } else {
+    lines.push('  There is no project here - this folder has no package.json.');
+    lines.push('');
+    lines.push('  You are in:  ' + inspection.dir);
+    lines.push('');
+  }
+
+  lines.push('  Change into your project folder and run this again:');
+  lines.push('');
+  lines.push('    cd path\\to\\your\\project');
+  lines.push('    kryptheon check');
+  lines.push('');
+  lines.push('  If you really do keep recordings outside a package, set');
+  lines.push('  ' + ESCAPE_HATCH + '=1 and this will run anyway.');
+  lines.push('');
+  return lines;
+}
+
+/** A real project, with nothing recorded in it yet. */
+function noRecordingsLines() {
+  return [
+    '',
+    '  No recordings were found in this project.',
+    '',
+    '  This is not a failure, and there is nothing in the code to fix.',
+    '',
+    '  Record a flow through your app first:',
+    '',
+    '    kryptheon record http://localhost:3000',
+    '',
+    '  That opens a browser window, so it has to be run by a person in a',
+    '  normal terminal - it cannot be done from an AI assistant.',
+    '',
+  ];
+}
+
+module.exports = {
+  ESCAPE_HATCH: ESCAPE_HATCH,
+  hasPackageJson: hasPackageJson,
+  systemFolders: systemFolders,
+  tidy: tidy,
+  isHomeOrSystem: isHomeOrSystem,
+  inspectProject: inspectProject,
+  noProjectLines: noProjectLines,
+  noRecordingsLines: noRecordingsLines,
+};

package/kryptheon-reporter.js

@@ -170,6 +170,94 @@ function firstMeaningfulLine(message) {
   return line ? line.replace(/^(TimeoutError|Error):\s*/, '').trim() : 'The test failed.';
 }
 
+// ---------------------------------------------------------------------------
+// Errors that belong to no test.
+//
+// A spec file with a syntax error, or one importing something that is not
+// there, never becomes a test at all - so nothing calls onTestEnd and the run
+// ends with no results. Playwright reports it through onError. A reporter that
+// does not implement onError prints a summary of nothing and exits 1, and the
+// person is left with a silent failure and no idea which file is at fault.
+// ---------------------------------------------------------------------------
+
+// Playwright says this once the load errors have already been reported. On its
+// own it means an empty tests folder; after a real error it is a consequence,
+// not a cause, and repeating it just buries the thing that has to be fixed.
+function isNoTestsFound(error) {
+  return /No tests found/i.test(String((error && error.message) || ''));
+}
+
+/**
+ * A load error, reduced to the file, the line and one sentence.
+ *
+ * The message carries a code frame in ANSI colour and repeats the absolute
+ * path inside its own first line. Neither helps here: the path is printed
+ * separately and relative, and the frame belongs in an editor.
+ */
+function readLoadError(error, userDir) {
+  const raw = stripAnsi((error && error.message) || (error && error.value) || String(error || ""));
+  const location = (error && error.location) || {};
+
+  let file = location.file || null;
+  if (file) {
+    try {
+      file = path.relative(userDir || process.cwd(), file);
+    } catch (e) {
+      /* an absolute path is still better than nothing */
+    }
+  }
+
+  // First line only, with the absolute path taken back out of it.
+  let message = raw.split("\n").map(function (l) { return l.trim(); }).filter(Boolean)[0] || "The file could not be read.";
+  if (location.file) message = message.split(location.file + ": ").join("");
+  message = message.replace(/\s*\(\d+:\d+\)\s*$/, "");
+
+  return {
+    file: file,
+    line: typeof location.line === "number" ? location.line : null,
+    message: message,
+  };
+}
+
+/**
+ * What to print when nothing could be loaded.
+ *
+ * Said as "this file", not "the tests failed", because nothing was tested. The
+ * app may be perfectly fine; a file in the tests folder is not.
+ */
+function loadErrorLines(errors, userDir) {
+  const real = (errors || []).filter(function (e) { return !isNoTestsFound(e); });
+  const lines = [];
+
+  if (!real.length) {
+    // Nothing ran and nothing explained why. Saying so is still better than a
+    // summary of zero, which reads as "everything is fine".
+    lines.push("Nothing ran.");
+    lines.push("");
+    lines.push("No recording produced a result, and the test runner gave no reason.");
+    lines.push("Check that the files in tests/ contain a test.");
+    lines.push("");
+    return lines;
+  }
+
+  const many = real.length > 1;
+  lines.push(many ? real.length + " of your recordings could not be read:" : "One of your recordings could not be read:");
+  lines.push("");
+  for (const error of real) {
+    const read = readLoadError(error, userDir);
+    const where = read.file ? read.file + (read.line ? "  line " + read.line : "") : "(the file was not named)";
+    lines.push("  " + where);
+    lines.push("  " + read.message);
+    lines.push("");
+  }
+  lines.push(many
+    ? "Nothing else ran, because those files could not be loaded at all."
+    : "Nothing else ran, because that file could not be loaded at all.");
+  lines.push("Fix it and run this again.");
+  lines.push("");
+  return lines;
+}
+
 // ---------------------------------------------------------------------------
 // Translation: Playwright error -> { reason, advice }
 // ---------------------------------------------------------------------------
@@ -575,10 +663,19 @@ class KryptheonReporter {
     this.previousRuns = [];
     this.records = [];
     this.startedAt = new Date();
+    // Errors that belong to no test: a spec file that would not load.
+    this.loadErrors = [];
     // Set by any test whose page never said what encoding it was in.
     this.sawGuessedEncoding = false;
   }
 
+  // Playwright calls this for anything that happened outside a test: a spec
+  // file that would not load, most often. Kept rather than printed here, so
+  // the reason lands beside the summary instead of ahead of the run.
+  onError(error) {
+    this.loadErrors.push(error);
+  }
+
   onBegin() {
     this.startedAt = new Date();
     // Read history before this run is appended, so "was working on" looks
@@ -726,7 +823,14 @@ class KryptheonReporter {
     out.push('X  ' + test.title);
     out.push('   ' + translated.reason);
 
-    for (const detail of translated.details || []) out.push('   ' + detail);
+    // A difference against a baseline only means something if there is a
+    // baseline of this recording's own to differ from. Saying "this has not
+    // passed before" and then listing headings that are gone says two things
+    // that cannot both be true, and the reader spends their time trying to
+    // reconcile them instead of reading the actual failure.
+    const aboutBaseline = /Baseline changed|ended up somewhere different/.test(translated.reason || '');
+    const details = lastPass || !aboutBaseline ? translated.details || [] : [];
+    for (const detail of details) out.push('   ' + detail);
 
     out.push(lastPass ? '   This was working on ' + formatWhen(lastPass) + '.' : '   This has not passed before.');
 
@@ -800,10 +904,35 @@ class KryptheonReporter {
     // has ever had a baseline - says the app is at fault when nothing is.
     const counts = recordings.summarise(this.records);
 
-    // The one line stands in for the whole report only when there is no report
-    // to make. A recording that has never passed is not counted as broken, but
-    // it did fail and its reasons were printed above, so the run still owes the
-    // reader its closing count.
+    // A file that would not load is said first, and always - before the quiet
+    // shortcut, and whatever else the run managed to do.
+    //
+    // In practice a file that will not load stops the whole run, so nothing
+    // else has anything to report and the block below returns. This is not
+    // written as though that were guaranteed: onError also carries errors that
+    // arrive outside any test, and one of those turning up after results would
+    // otherwise be dropped in silence - which is the bug this whole thing is
+    // here to fix.
+    const unreadable = this.loadErrors.some(function (e) { return !isNoTestsFound(e); });
+    if (unreadable) {
+      process.stdout.write(
+        String.fromCharCode(10) + loadErrorLines(this.loadErrors, USER_DIR).join(String.fromCharCode(10)),
+      );
+    }
+
+    // Nothing ran at all. A summary of zero reads as good news, and in quiet
+    // mode it would say "0 recordings still working" over the top of a file
+    // that does not compile, so neither is printed. If the reason has not
+    // already been given above, it is given now.
+    if (!this.records.length) {
+      if (!unreadable) {
+        process.stdout.write(
+          String.fromCharCode(10) + loadErrorLines(this.loadErrors, USER_DIR).join(String.fromCharCode(10)),
+        );
+      }
+      return;
+    }
+
     if (QUIET && !counts.broken && !counts.unproven) {
       const s = counts.working === 1 ? '' : 's';
       process.stdout.write('OK  ' + counts.working + ' recording' + s + ' still working.\n');
@@ -831,3 +960,7 @@ module.exports = KryptheonReporter;
 // Shared with the fixture so baselines strip hosts exactly the same way.
 // Attached to the exported class, so `new (require(...))()` keeps working.
 module.exports.requestPath = requestPath;
+
+// Exported for the checks: what a file that would not load turns into.
+module.exports.readLoadError = readLoadError;
+module.exports.loadErrorLines = loadErrorLines;