kryptheon 0.1.9 → 0.1.11

package/bin/kryptheon.js

@@ -1,4 +1,4 @@
-#!/usr/bin/env node
+#!/usr/bin/env node
 // The kryptheon CLI. A thin wrapper so nobody has to remember Playwright's
 // command line or edit a config file. Everything here shells out to the
 // Playwright binary that ships with this package.
@@ -14,6 +14,11 @@ const { spawn, spawnSync } = require('child_process');
 const os = require('os');
 const secrets = require(require('path').join(__dirname, '..', 'kryptheon-secrets.js'));
 const replay = require('../kryptheon-replay.js');
+const selectors = require('../kryptheon-selectors.js');
+const recordings = require('../kryptheon-recordings.js');
+const cleanup = require('../kryptheon-cleanup.js');
+const baselines = require('../kryptheon-baselines.js');
+const project = require('../kryptheon-project.js');
 
 const PACKAGE_DIR = path.join(__dirname, '..');
 const USER_DIR = process.cwd();
@@ -66,6 +71,7 @@ function usage() {
   console.log('  kryptheon record <url>   open your app and record what you do as a test');
   console.log('  kryptheon check          run every recorded test and report in plain language');
   console.log('  kryptheon accept <name>  agree that one test\'s new result is the correct one');
+  console.log('  kryptheon remove [name]  list the recordings, or remove one you no longer want');
   console.log('  kryptheon setup-ai       tell your AI assistant to check its work');
   console.log('');
   console.log('  add --quiet to check for one line when everything passes');
@@ -853,6 +859,89 @@ function reportSecrets(replacements) {
   console.log('');
 }
 
+// One keypress, no Enter, because the safe answer is the default and getting
+// out of the way should cost nothing.
+//
+// Falls back to the default where there is no terminal to read keys from - a
+// CI job, a pipe, an assistant running the command - rather than waiting for
+// an answer that will never arrive.
+function askKey(question, keys, fallback) {
+  return new Promise((resolve) => {
+    const allowed = keys.map((k) => k.toLowerCase());
+    if (!process.stdin.isTTY || !process.stdin.setRawMode) return resolve(fallback);
+    process.stdout.write(question);
+
+    const finish = (answer) => {
+      process.stdin.removeListener("data", onKey);
+      try {
+        process.stdin.setRawMode(false);
+        process.stdin.pause();
+      } catch (err) {
+        /* nothing more to do */
+      }
+      process.stdout.write(answer + String.fromCharCode(10));
+      resolve(answer);
+    };
+
+    const onKey = (chunk) => {
+      const key = String(chunk);
+      // In raw mode Ctrl-C arrives as a keystroke rather than a signal, so it
+      // has to be turned back into one. Sent as a signal rather than exiting
+      // here, because exiting would cut off whatever is still buffered - and
+      // it must not be read as an answer either: stopping is not "keep going".
+      if (key === String.fromCharCode(3)) {
+        process.stdin.removeListener('data', onKey);
+        try {
+          process.stdin.setRawMode(false);
+          process.stdin.pause();
+        } catch (err) {
+          /* nothing more to do */
+        }
+        process.stdout.write(String.fromCharCode(10));
+        process.kill(process.pid, 'SIGINT');
+        return;
+      }
+      if (key === String.fromCharCode(13) || key === String.fromCharCode(10)) return finish(fallback);
+      const pressed = key.toLowerCase();
+      if (allowed.indexOf(pressed) === -1) return; // an unknown key: keep waiting
+      finish(pressed);
+    };
+
+    try {
+      process.stdin.setRawMode(true);
+    } catch (err) {
+      return resolve(fallback);
+    }
+    process.stdin.resume();
+    process.stdin.on("data", onKey);
+  });
+}
+
+// Removing a recording means removing what was remembered about it too, or
+// the saved results of a file that no longer exists stay behind for ever.
+function deleteRecording(name) {
+  const relative = path.join("tests", name).split(path.sep).join("/");
+  try {
+    fs.unlinkSync(path.join(TESTS_DIR, name));
+  } catch (err) {
+    return { ok: false, name: name, why: err.message };
+  }
+
+  try {
+    const api = require(path.join(PACKAGE_DIR, "kryptheon-fixture.js"));
+    const all = api.readBaselines(api.BASELINE_FILE);
+    const keys = recordings.baselineKeysForSpec(all, relative);
+    if (keys.length) {
+      for (const key of keys) delete all[key];
+      fs.writeFileSync(api.BASELINE_FILE, JSON.stringify(all, null, 2) + String.fromCharCode(10), "utf8");
+    }
+  } catch (err) {
+    // The recording is gone either way, and a leftover baseline is not worth
+    // failing the command over.
+  }
+  return { ok: true, name: name, file: relative };
+}
+
 function askYesNo(question) {
   return new Promise((resolve) => {
     if (!process.stdin.isTTY) return resolve(false);
@@ -871,6 +960,24 @@ function askYesNo(question) {
   });
 }
 
+// A typed answer, for questions whose answers are not single letters.
+function askLine(question) {
+  return new Promise((resolve) => {
+    if (!process.stdin.isTTY) return resolve("");
+    const readline = require("readline");
+    const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
+    rl.question(question, (answer) => {
+      rl.close();
+      try {
+        process.stdin.pause();
+      } catch (err) {
+        /* nothing to do */
+      }
+      resolve(String(answer));
+    });
+  });
+}
+
 // Ending on a logout means the next run starts somewhere else. Almost nobody
 // wants that, so ask.
 async function offerToDropLogout(relativeFile) {
@@ -897,6 +1004,70 @@ async function offerToDropLogout(relativeFile) {
   return true;
 }
 
+// Codegen writes whatever identified the element at the moment it was clicked,
+// which on a page of live figures means the figures end up in the selector. Say
+// so, and say nothing when there is nothing to say.
+// Codegen records the wrong keystrokes along with the right ones, so a typo
+// corrected on the second attempt becomes a wrong login performed on every
+// run. This takes those out and says what it took, because a tool that edits
+// your recording without telling you is worse than one that leaves the mess.
+// A saved result whose recording is gone, or has been recorded again since,
+// describes a page nothing here visits any more. Left in place it is handed
+// to whichever new recording lands on the same name, which then fails on its
+// first run against a baseline it never created.
+function dropDeadBaselines() {
+  const api = (function () {
+    try {
+      return require(path.join(PACKAGE_DIR, 'kryptheon-fixture.js'));
+    } catch (err) {
+      return null;
+    }
+  })();
+  if (!api) return [];
+
+  let all;
+  try {
+    all = api.readBaselines(api.BASELINE_FILE);
+  } catch (err) {
+    return [];
+  }
+  if (!all || !Object.keys(all).length) return [];
+
+  const result = baselines.pruneBaselines(all, USER_DIR);
+  if (!result.changed) return [];
+  try {
+    fs.writeFileSync(api.BASELINE_FILE, JSON.stringify(result.baselines, null, 2) + String.fromCharCode(10), "utf8");
+  } catch (err) {
+    return [];
+  }
+  for (const line of baselines.pruneLines(result.dropped)) console.log(line);
+  return result.dropped;
+}
+
+function tidyRecording(relativeFile) {
+  const source = readSpec(relativeFile);
+  if (source === null) return [];
+  const result = cleanup.cleanRecording(source);
+  if (!result.removed.length) return [];
+  if (!writeSpec(relativeFile, result.source)) return [];
+  return result.removed;
+}
+
+// Said after the file has its name, with the other closing notes, so the
+// order on screen matches the order things happened in.
+function reportTidying(removed) {
+  for (const line of cleanup.describeCleanup(removed)) console.log(line);
+  return removed || [];
+}
+
+function reportFragileSelectors(relativeFile) {
+  const source = readSpec(relativeFile);
+  if (source === null) return [];
+  const findings = selectors.findFragileSelectors(source);
+  for (const line of selectors.describeFragileSelectors(findings)) console.log(line);
+  return findings;
+}
+
 function reportReplayRisks(relativeFile) {
   const source = readSpec(relativeFile);
   if (source === null) return [];
@@ -937,16 +1108,23 @@ async function finaliseRecording(outFile, context) {
     // this point. Take it out before doing anything else with the file.
     const secrets = takeOutSecrets(outFile);
 
+    // After the secrets, deliberately. By this point a password is an
+    // environment variable rather than a literal, so nothing that is said
+    // about a field that was typed twice can put the value back on screen.
+    const tidied = tidyRecording(outFile);
+
     const named = await nameRecording(outFile);
     console.log('');
     console.log('  Saved to ' + named);
     console.log('  Run it any time with:  kryptheon check');
     console.log('');
 
+    reportTidying(tidied);
     reportSecrets(secrets);
     keepArtefactsOutOfGit();
     await offerToDropLogout(named);
     reportReplayRisks(named);
+    reportFragileSelectors(named);
 
     return { code: 0, savedAs: named };
   }
@@ -1003,6 +1181,125 @@ async function finaliseRecording(outFile, context) {
   return { code: 1, savedAs: null };
 }
 
+// Every record writes another file and check runs all of them, so a first day
+// spent learning the tool leaves a pile of recordings that never worked and a
+// check that reads like a wall of failures. This is the moment asking costs
+// nothing.
+//
+// Keeping is the default because it is the answer that cannot lose work, and
+// the other one is a single keypress away. Where there is no terminal to
+// answer with - a CI job, an assistant running the command - nothing is asked
+// and nothing is deleted: a question nobody can see is not a question.
+async function askAboutExisting(names) {
+  if (!names.length || !process.stdin.isTTY) return false;
+
+  for (const line of recordings.existingRecordingsQuestion(names.length)) console.log(line);
+  const answer = await askKey("  Keep or replace? [K/r] ", ["k", "r"], "k");
+  if (answer !== "r") return false;
+
+  console.log("");
+  console.log("  They will be deleted once this new recording is saved.");
+  return true;
+}
+
+// The old recordings, once the new one is safely on disk. The new file is
+// never in this list, but it is checked by name anyway - deleting the thing
+// just recorded is the one mistake here that cannot be undone.
+function dropOldRecordings(names, savedAs) {
+  const keep = path.basename(String(savedAs || ""));
+  const gone = [];
+  for (const name of names) {
+    if (path.basename(name) === keep) continue;
+    const result = deleteRecording(name);
+    if (result.ok) gone.push(name);
+  }
+  if (!gone.length) return gone;
+  console.log("  Removed " + recordings.plural(gone.length, "older recording") + ":");
+  for (const name of gone) console.log("    " + path.join("tests", name));
+  console.log("");
+  return gone;
+}
+
+// --- remove -----------------------------------------------------------------
+//
+// Recordings pile up. Without a way to take one out, the only way to stop a
+// half-finished experiment being run for ever is to know where the files live
+// and delete one by hand.
+
+function listRecordings(names) {
+  console.log("");
+  console.log("  " + recordings.plural(names.length, "recording") + " in this project:");
+  console.log("");
+  names.forEach((name, i) => {
+    console.log("    " + String(i + 1).padStart(2) + "  " + path.join("tests", name));
+  });
+  console.log("");
+}
+
+async function remove(name) {
+  const names = listSpecFiles();
+  if (!names.length) {
+    console.log("");
+    console.log("  There are no recordings to remove.");
+    console.log("");
+    return 0;
+  }
+
+  let choice;
+  if (name) {
+    choice = recordings.chooseFromList(names, name);
+  } else {
+    listRecordings(names);
+    if (!process.stdin.isTTY) {
+      console.log("  To remove one, run:  kryptheon remove <name>");
+      console.log("");
+      return 0;
+    }
+    const answer = await askLine("  Which one? [number, a for all, Enter to cancel] ");
+    choice = recordings.chooseFromList(names, answer);
+  }
+
+  if (choice.action === "cancel") {
+    console.log("");
+    console.log("  Nothing was removed.");
+    console.log("");
+    return 0;
+  }
+  if (choice.action === "unclear") {
+    console.error("");
+    console.error("  " + choice.why + ": " + JSON.stringify(choice.said));
+    console.error("  Run \"kryptheon remove\" on its own to see the list.");
+    console.error("");
+    return 1;
+  }
+
+  const gone = [];
+  const failed = [];
+  for (const target of choice.names) {
+    const result = deleteRecording(target);
+    if (result.ok) gone.push(target);
+    else failed.push(result);
+  }
+
+  console.log("");
+  if (gone.length) {
+    console.log("  Removed " + recordings.plural(gone.length, "recording") + ":");
+    for (const g of gone) console.log("    " + path.join("tests", g));
+    console.log("");
+    console.log(
+      gone.length === 1
+        ? "  Its saved result was removed too, so nothing is left behind."
+        : "  Their saved results were removed too, so nothing is left behind.",
+    );
+    console.log("");
+  }
+  for (const f of failed) {
+    console.error("  Could not remove " + path.join("tests", f.name) + ": " + f.why);
+    console.error("");
+  }
+  return failed.length ? 1 : 0;
+}
+
 async function record(url) {
   if (!url) {
     console.error('');
@@ -1025,6 +1322,9 @@ async function record(url) {
 
   if (!ensureBrowser()) return 1;
 
+  const existingBefore = listSpecFiles();
+  const replaceExisting = await askAboutExisting(existingBefore);
+
   const hadTestsDir = fs.existsSync(TESTS_DIR);
   try {
     fs.mkdirSync(TESTS_DIR, { recursive: true });
@@ -1064,6 +1364,9 @@ async function record(url) {
       stderr: stderr,
       reason: reason,
     });
+    // Only now, and only if something was actually saved. Deleting first
+    // would mean a recording that went wrong takes the old ones with it.
+    if (replaceExisting && outcome.savedAs) dropOldRecordings(existingBefore, outcome.savedAs);
     return outcome.code;
   };
 
@@ -1222,22 +1525,23 @@ function triageSpecs() {
 
 function check(options) {
   const quiet = !!(options && options.quiet);
+
+  // Before anything else. Recordings and results are read out of whatever
+  // folder this was run in, so being in the wrong one does not fail - it
+  // succeeds, about the wrong project. That is the one outcome worth exiting
+  // non-zero over even though nothing was tested.
+  const here = project.inspectProject(USER_DIR);
+  if (!here.ok) {
+    for (const line of project.noProjectLines(here)) console.error(line);
+    return 1;
+  }
+
   // Deliberately not a failure. An assistant told to run this after every
   // change will read a non-zero exit as "something broke" and start fixing
   // code that is perfectly fine. Nothing has been recorded, so there is
   // nothing to verify and nothing to repair.
   if (!listSpecFiles().length) {
-    console.log('');
-    console.log('  Nothing has been recorded yet, so there is nothing to check.');
-    console.log('');
-    console.log('  This is not a failure, and there is nothing in the code to fix.');
-    console.log('');
-    console.log('  To start, record a flow through the app:');
-    console.log('    npx kryptheon record <url>');
-    console.log('');
-    console.log('  That opens a browser window, so it has to be run by a person in');
-    console.log('  a normal terminal - it cannot be done from an AI assistant.');
-    console.log('');
+    for (const line of project.noRecordingsLines()) console.log(line);
     return 0;
   }
   if (!fixtureResolvesForUser() && specsNeedThePackage()) {
@@ -1254,6 +1558,7 @@ function check(options) {
     return 1;
   }
   keepArtefactsOutOfGit();
+  dropDeadBaselines();
 
   const triage = triageSpecs();
 
@@ -1606,6 +1911,9 @@ async function main() {
     case 'check':
       return finishWith(check({ quiet: rest.indexOf('--quiet') !== -1 || rest.indexOf('-q') !== -1 }));
       break;
+    case 'remove':
+    case 'rm':
+      return finishWith(await remove(rest.join(" ").trim()));
     case 'setup-ai':
       return finishWith(setupAi());
     case 'accept':
@@ -1660,6 +1968,15 @@ module.exports = {
   inspectBrowserWindows: inspectBrowserWindows,
   parseWindowProbe: parseWindowProbe,
   codegenComplaints: codegenComplaints,
+  reportFragileSelectors: reportFragileSelectors,
+  tidyRecording: tidyRecording,
+  dropDeadBaselines: dropDeadBaselines,
+  reportTidying: reportTidying,
+  remove: remove,
+  deleteRecording: deleteRecording,
+  dropOldRecordings: dropOldRecordings,
+  askAboutExisting: askAboutExisting,
+  askKey: askKey,
   WINDOW_TIMEOUT_MS: WINDOW_TIMEOUT_MS,
   WINDOW_HARD_LIMIT_MS: WINDOW_HARD_LIMIT_MS,
   WINDOW_POLL_MS: WINDOW_POLL_MS,

package/kryptheon-baselines.js

@@ -0,0 +1,176 @@
+// Keeping a saved result tied to the recording it came from.
+//
+// A baseline used to be identified by where the recording lived and what it was
+// called. That is not enough. Delete the spec files with `del tests\*.spec.js` -
+// which is what anyone does before the tool has taught them `kryptheon remove` -
+// record the flow again, and the new recording is handed the same name. Same
+// path, same title, same key. It inherits the old recording's saved page and
+// fails on its very first run, against a baseline it never created and could
+// never have matched.
+//
+// The report that came out of that said both of these at once:
+//
+//   This has not passed before.
+//   Headings that are gone: "Transactions"
+//
+// Which cannot both be true. If it has never passed there is no baseline of its
+// own to have changed, so there is nothing to show a difference against.
+//
+// So a saved result now also records a fingerprint of the recording that made
+// it. When the recording is gone, or its steps are not the steps the baseline
+// was taken from, the saved result describes something that no longer exists
+// and is thrown away rather than compared against.
+//
+// Pure data work: no Playwright, no test runner.
+
+const fs = require('fs');
+const path = require('path');
+const replay = require('./kryptheon-replay.js');
+
+/** The spec file half of a baseline key, in one consistent shape. */
+function specOf(key) {
+  return String(key || '').split(' :: ')[0].split('\\').join('/');
+}
+
+function titleOf(key) {
+  const parts = String(key || '').split(' :: ');
+  return parts.length > 1 ? parts.slice(1).join(' :: ') : '';
+}
+
+/** The fingerprint of the recording in a file, or null if it cannot be read. */
+function fingerprintOf(userDir, specPath) {
+  let source;
+  try {
+    source = fs.readFileSync(path.join(userDir, specPath), 'utf8');
+  } catch (e) {
+    return null;
+  }
+  return replay.recordingId(source);
+}
+
+/**
+ * What each saved result is: still current, orphaned, or stale.
+ *
+ * `orphan`  - the recording it belongs to is not there any more.
+ * `stale`   - a recording of that name exists, but it is not the one this was
+ *             taken from. This is the case that cost twenty minutes: the file
+ *             is present, so nothing looks wrong.
+ * `legacy`  - written before fingerprints existed. There is no way to tell
+ *             whether it matches, so it is kept, and a fingerprint is written
+ *             in so the question can be answered next time. Guessing here would
+ *             mean either throwing away a good baseline or, far worse, quietly
+ *             accepting a real regression as the new normal.
+ */
+function classifyBaselines(baselines, userDir) {
+  const all = baselines || {};
+  const verdicts = [];
+
+  for (const key of Object.keys(all)) {
+    const entry = all[key];
+    const spec = specOf(key);
+    if (!entry || typeof entry !== 'object') {
+      verdicts.push({ key: key, state: 'orphan', spec: spec, title: titleOf(key) });
+      continue;
+    }
+
+    let exists = false;
+    try {
+      exists = fs.statSync(path.join(userDir, spec)).isFile();
+    } catch (e) {
+      exists = false;
+    }
+    if (!exists) {
+      verdicts.push({ key: key, state: 'orphan', spec: spec, title: titleOf(key) });
+      continue;
+    }
+
+    const now = fingerprintOf(userDir, spec);
+    if (!entry.recordingId) {
+      verdicts.push({ key: key, state: 'legacy', spec: spec, title: titleOf(key), fingerprint: now });
+      continue;
+    }
+    if (now && entry.recordingId !== now) {
+      verdicts.push({ key: key, state: 'stale', spec: spec, title: titleOf(key), fingerprint: now });
+      continue;
+    }
+    verdicts.push({ key: key, state: 'current', spec: spec, title: titleOf(key) });
+  }
+
+  return verdicts;
+}
+
+/**
+ * The saved results with the dead ones taken out.
+ *
+ * Returns a new object; nothing is written here. The caller decides whether the
+ * file is worth rewriting, so a check that changes nothing never touches it.
+ */
+function pruneBaselines(baselines, userDir) {
+  const all = baselines || {};
+  const verdicts = classifyBaselines(all, userDir);
+  const kept = {};
+  const dropped = [];
+  let stamped = 0;
+
+  for (const verdict of verdicts) {
+    if (verdict.state === 'orphan' || verdict.state === 'stale') {
+      dropped.push(verdict);
+      continue;
+    }
+    const entry = all[verdict.key];
+    if (verdict.state === 'legacy' && verdict.fingerprint) {
+      // Kept as it is, but tied to this recording from now on.
+      kept[verdict.key] = Object.assign({}, entry, { recordingId: verdict.fingerprint });
+      stamped++;
+      continue;
+    }
+    kept[verdict.key] = entry;
+  }
+
+  return { baselines: kept, dropped: dropped, stamped: stamped, changed: dropped.length > 0 || stamped > 0 };
+}
+
+function plural(n, word) {
+  return n + ' ' + word + (n === 1 ? '' : 's');
+}
+
+/**
+ * One line about what was thrown away, said as a fact rather than a warning.
+ *
+ * Nothing has gone wrong when this happens - a recording was deleted or redone,
+ * which is a normal thing to do. It is said out loud only so that a flow which
+ * suddenly starts from scratch is not a mystery.
+ */
+function pruneLines(dropped) {
+  const list = dropped || [];
+  if (!list.length) return [];
+
+  const orphans = list.filter((d) => d.state === 'orphan');
+  const stale = list.filter((d) => d.state === 'stale');
+  const lines = [];
+
+  if (orphans.length) {
+    lines.push(
+      '  ' + plural(orphans.length, 'old baseline') + ' removed - ' +
+        (orphans.length === 1 ? 'its recording is' : 'their recordings are') + ' no longer here.',
+    );
+  }
+  if (stale.length) {
+    lines.push(
+      '  ' + plural(stale.length, 'old baseline') + ' removed - ' +
+        (stale.length === 1 ? 'its recording has' : 'those recordings have') +
+        ' been recorded again since.',
+    );
+  }
+  lines.push('');
+  return lines;
+}
+
+module.exports = {
+  specOf: specOf,
+  titleOf: titleOf,
+  fingerprintOf: fingerprintOf,
+  classifyBaselines: classifyBaselines,
+  pruneBaselines: pruneBaselines,
+  pruneLines: pruneLines,
+};