@antelopejs/dms 0.11.1 → 0.12.0

package/bin/control/journey.mjs

@@ -0,0 +1,380 @@
+import { randomBytes } from "node:crypto";
+import {
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  rmSync,
+  writeFileSync,
+} from "node:fs";
+import { join, relative } from "node:path";
+import { pathToFileURL } from "node:url";
+import { withBaseBackend } from "./base.mjs";
+import { launchBrowser, newContext } from "./browser.mjs";
+import { capturedMail } from "./mail.mjs";
+import { saveRun } from "./run-state.mjs";
+import {
+  adminCredentials,
+  apiRequest,
+  newPassword,
+  onboard,
+  recordedRequest,
+} from "./session.mjs";
+import {
+  ensureMail,
+  freshDatabase,
+  resolutionFolders,
+  restartBackend,
+} from "./stack.mjs";
+import { Tab } from "./tab.mjs";
+
+const EXPECT_TIMEOUT_MS = 15_000;
+const POLL_MS = 250;
+const DETAIL_LENGTH = 300;
+const UNIQUE_BYTES = 3;
+
+class JourneyAbort extends Error {}
+
+function slug(text) {
+  return text
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-|-$/g, "")
+    .slice(0, 60);
+}
+
+function writeJson(file, value) {
+  writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`);
+}
+
+/** An equality check over JSON-serialisable values. */
+function equal(name, actual, expected) {
+  const [left, right] = [JSON.stringify(actual), JSON.stringify(expected)];
+  return {
+    name,
+    pass: left === right,
+    detail: `expected ${right}, got ${left}`,
+  };
+}
+
+/** A check that an API answer has `expected` as its status. */
+function status(answer, expected) {
+  const body = JSON.stringify(answer.body) ?? "";
+  return {
+    name: `HTTP ${expected}`,
+    pass: answer.status === expected,
+    detail: `status ${answer.status}: ${body.slice(0, DETAIL_LENGTH)}`,
+  };
+}
+
+/** A check that a condition holds, with what was observed. */
+function holds(name, pass, detail) {
+  return { name, pass: Boolean(pass), detail: String(detail) };
+}
+
+/**
+ * What a journey drives the run through: tabs per account, the API as an
+ * account, the captured mail, and checks recorded into numbered step
+ * folders under `<run>/evidence/<feature>/<journey>/`.
+ */
+export class JourneyContext {
+  constructor(state, feature, journey, browser, skill) {
+    this.state = state;
+    this.skill = skill;
+    this.feature = feature;
+    this.journey = journey;
+    this.browser = browser;
+    this.folder = join(state.evidence, feature, journey);
+    this.steps = [];
+    this.contexts = new Map();
+    this.index = 0;
+    this.startedAt = new Date().toISOString();
+    this.equal = equal;
+    this.status = status;
+    this.holds = holds;
+    rmSync(this.folder, { recursive: true, force: true });
+    mkdirSync(this.folder, { recursive: true });
+  }
+
+  /** A new numbered step folder. */
+  step(label) {
+    this.index += 1;
+    const name = `${String(this.index).padStart(2, "0")}-${slug(label)}`;
+    const folder = join(this.folder, name);
+    mkdirSync(folder, { recursive: true });
+    return folder;
+  }
+
+  /** Records the checks of a step folder; a failed critical step ends the journey. */
+  record(folder, checks, options = {}) {
+    this.steps.push({
+      name: relative(this.folder, folder),
+      folder: relative(this.state.evidence, folder),
+      checks,
+    });
+    writeJson(join(folder, "checks.json"), checks);
+    const passed = checks.every((check) => check.pass);
+    if (!passed && options.critical) {
+      throw new JourneyAbort(
+        `critical step ${relative(this.folder, folder)} failed`,
+      );
+    }
+    return passed;
+  }
+
+  /**
+   * Records a step of checks, which may be promises (`tab.visible(...)`).
+   * With `{ critical: true }` a failure ends the journey.
+   */
+  async verify(label, checks, options = {}) {
+    return this.record(this.step(label), await Promise.all(checks), options);
+  }
+
+  /** A new tab as `account` (`anon` for nobody), in that account's own browser context. */
+  async tab(account = "admin", label = account) {
+    if (!this.contexts.has(account)) {
+      this.contexts.set(account, await newContext(this.browser));
+    }
+    return new Tab(this, await this.contexts.get(account).newPage(), label);
+  }
+
+  /**
+   * Calls the backend as `as` (an account, or `anon`; `admin` by default),
+   * or with an explicit `token` and no retry, and records the exchange.
+   */
+  api(label, request) {
+    if (request.token) {
+      return recordedRequest(this.step(`api ${label}`), {
+        ...request,
+        url: `${this.state.apiUrl}${request.path}`,
+      });
+    }
+    return apiRequest(this.state, {
+      ...request,
+      as: request.as ?? "admin",
+      folder: () => this.step(`api ${label}`),
+    });
+  }
+
+  /** Sends anonymous requests at once, for races, each recorded in its own step folder. */
+  concurrent(label, requests) {
+    return Promise.all(
+      requests.map((request, index) =>
+        recordedRequest(this.step(`${label} ${index + 1}`), {
+          ...request,
+          url: `${this.state.apiUrl}${request.path}`,
+        }),
+      ),
+    );
+  }
+
+  /** Adds an account the journey created (an invited member) so tabs and the API can act as it. */
+  addAccount(name, credentials) {
+    this.state.accounts = { ...this.state.accounts, [name]: credentials };
+    saveRun(this.state);
+  }
+
+  /** An address unique to this run and journey, which the mail sink receives. */
+  email(name) {
+    return `${name}+${this.state.id}-${slug(this.journey)}@verify.local`.toLowerCase();
+  }
+
+  /** A string unique to this journey, for names a journey creates. */
+  unique(prefix) {
+    return `${prefix} ${randomBytes(UNIQUE_BYTES).toString("hex")}`;
+  }
+
+  /** A password the DMS policy accepts. */
+  password() {
+    return newPassword();
+  }
+
+  /** The credentials `onboard` creates for the run's administrator. */
+  adminCredentials() {
+    return adminCredentials(this.state);
+  }
+
+  /** Creates the administrator through `POST /api/onboarding/register`, then signs it in. */
+  onboard(options = {}) {
+    return onboard(this.state, options);
+  }
+
+  /**
+   * Runs `action` against the base version's backend on the same database,
+   * then the run's own backend again: data written inside `action` is data
+   * the previous release wrote. Records the base commit as a step.
+   */
+  async withBaseBackend(action) {
+    const sha = await withBaseBackend(this.state, this.skill, action);
+    writeJson(join(this.step("base version"), "base.json"), {
+      baseRef: this.state.baseRef,
+      sha,
+    });
+    return sha;
+  }
+
+  /** Restarts the backend on the same database, to prove what persists. */
+  restartBackend() {
+    return restartBackend(this.state);
+  }
+
+  /**
+   * The mail the sink captured since the journey started, waiting until
+   * `accept` matches one or the timeout passes; `match` is the latest match.
+   */
+  async mail(accept, timeoutMs = EXPECT_TIMEOUT_MS) {
+    const deadline = Date.now() + timeoutMs;
+    const since = (message) => message.receivedAt >= this.startedAt;
+    let messages = capturedMail(this.state.mail).filter(since);
+    while (!messages.some(accept) && Date.now() < deadline) {
+      await new Promise((resolve) => setTimeout(resolve, POLL_MS));
+      messages = capturedMail(this.state.mail).filter(since);
+    }
+    const folder = this.step("mail");
+    writeJson(join(folder, "mail.json"), messages);
+    return { folder, messages, match: messages.findLast(accept) };
+  }
+
+  async close() {
+    for (const context of this.contexts.values()) {
+      await context.close().catch(() => undefined);
+    }
+  }
+}
+
+async function prepare(state, feature, journey, context) {
+  await ensureMail(state);
+  if (journey.database !== "shared") {
+    const index = String((state.journeyCount ?? 0) + 1).padStart(3, "0");
+    state.journeyCount = Number(index);
+    await freshDatabase(state, `j${index}`);
+    writeJson(join(context.step("fresh database"), "database.json"), {
+      database: state.database,
+    });
+  }
+  if (journey.onboard === false || state.accounts?.admin) return;
+  const folder = context.step("onboard the administrator");
+  const answer = await onboard(state, {});
+  context.record(
+    folder,
+    [
+      holds(
+        "POST /api/onboarding/register answers 2xx",
+        answer.register.status === 200,
+        answer.register.status,
+      ),
+      holds(
+        "the administrator signs in",
+        answer.login?.status === 200,
+        answer.login?.status,
+      ),
+    ],
+    { critical: true },
+  );
+}
+
+/**
+ * Runs one journey: by default on a fresh database with an onboarded
+ * administrator (`database: "shared"` keeps the current one, `onboard:
+ * false` skips the administrator), with a fresh browser, and writes its
+ * result.json.
+ *
+ * @param {Record<string, any>} state The run
+ * @param {string} feature The feature it belongs to
+ * @param {{ id: string, title: string, entryPoint: string, database?: string, onboard?: boolean, run: (context: JourneyContext) => Promise<void> }} journey The journey
+ * @returns {Promise<Record<string, any>>} The result
+ */
+export async function runJourney(state, feature, journey, skill) {
+  const browser = await launchBrowser(resolutionFolders(state));
+  const context = new JourneyContext(
+    state,
+    feature,
+    journey.id,
+    browser,
+    skill,
+  );
+  let error = null;
+  try {
+    await prepare(state, feature, journey, context);
+    await journey.run(context);
+  } catch (failure) {
+    error =
+      failure instanceof JourneyAbort
+        ? failure.message
+        : String(failure?.stack ?? failure);
+  } finally {
+    await context.close();
+    await browser.close();
+  }
+  const failed =
+    error !== null ||
+    context.steps.some((step) => step.checks.some((check) => !check.pass));
+  const result = {
+    feature,
+    journey: journey.id,
+    title: journey.title,
+    entryPoint: journey.entryPoint,
+    status: failed ? "FAIL" : "PASS",
+    database: state.database,
+    steps: context.steps,
+    error,
+    folder: relative(state.evidence, context.folder),
+  };
+  writeJson(join(context.folder, "result.json"), result);
+  return result;
+}
+
+/**
+ * Loads the journeys a feature's file exports (`export const journeys`).
+ *
+ * @param {string} file The journeys module
+ * @returns {Promise<Record<string, any>[]>} The journeys
+ */
+export async function loadJourneys(file) {
+  if (!existsSync(file)) throw new Error(`${file} does not exist`);
+  const { journeys } = await import(pathToFileURL(file).href);
+  if (!Array.isArray(journeys))
+    throw new Error(`${file} exports no journeys array`);
+  return journeys;
+}
+
+/**
+ * Runs a feature's journeys, all or those named, and merges their results
+ * into `<feature>/feature.json` so a partial run never drops earlier ones.
+ *
+ * @param {Record<string, any>} state The run
+ * @param {{ file: string, feature: string, ids: string[], skill: Record<string, any> }} request Which journeys, and the skill they belong to
+ * @returns {Promise<Record<string, any>[]>} The results of this run
+ */
+export async function runJourneyFile(state, request) {
+  const journeys = await loadJourneys(request.file);
+  const unknown = request.ids.filter(
+    (id) => !journeys.some((journey) => journey.id === id),
+  );
+  if (unknown.length > 0) {
+    throw new Error(
+      `Unknown journeys of ${request.feature}: ${unknown.join(", ")}`,
+    );
+  }
+  const selected =
+    request.ids.length > 0
+      ? journeys.filter((journey) => request.ids.includes(journey.id))
+      : journeys;
+  const results = [];
+  for (const journey of selected)
+    results.push(
+      await runJourney(state, request.feature, journey, request.skill),
+    );
+  const file = join(state.evidence, request.feature, "feature.json");
+  const earlier = existsSync(file)
+    ? JSON.parse(readFileSync(file, "utf8"))
+    : [];
+  const merged = journeys.flatMap((journey) => {
+    const fresh = results.filter((result) => result.journey === journey.id);
+    return fresh.length > 0
+      ? fresh
+      : earlier.filter((result) => result.journey === journey.id);
+  });
+  writeJson(file, merged);
+  saveRun(state);
+  return results;
+}

package/bin/control/mail.mjs

@@ -0,0 +1,163 @@
+import {
+  existsSync,
+  mkdirSync,
+  readdirSync,
+  readFileSync,
+  writeFileSync,
+} from "node:fs";
+import { createServer } from "node:net";
+import { join } from "node:path";
+
+const END_OF_DATA = "\r\n.\r\n";
+const QUOTED_SOFT_BREAK = /=\r?\n/g;
+const QUOTED_BYTE = /=([0-9A-F]{2})/g;
+const HEX = 16;
+
+function reply(socket, line) {
+  socket.write(`${line}\r\n`);
+}
+
+function decodeQuotedPrintable(text) {
+  return text
+    .replace(QUOTED_SOFT_BREAK, "")
+    .replace(QUOTED_BYTE, (_match, byte) =>
+      String.fromCharCode(Number.parseInt(byte, HEX)),
+    );
+}
+
+function headerOf(message, name) {
+  const match = new RegExp(`^${name}:\\s*(.+(?:\\r?\\n[ \\t].+)*)`, "im").exec(
+    message,
+  );
+  return match ? match[1].replace(/\r?\n[ \t]+/g, " ").trim() : "";
+}
+
+const ENTITIES = {
+  amp: "&",
+  lt: "<",
+  gt: ">",
+  quot: '"',
+  "#39": "'",
+  nbsp: " ",
+};
+
+function readableText(html) {
+  const body = html.slice(html.search(/\r?\n\r?\n/));
+  return body
+    .replace(/<(style|script)[\s\S]*?<\/\1>/gi, " ")
+    .replace(/<[^>]+>/g, " ")
+    .replace(/&(amp|lt|gt|quot|#39|nbsp);/g, (_match, name) => ENTITIES[name])
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+function saveMessage(folder, envelope, raw) {
+  mkdirSync(folder, { recursive: true });
+  const index =
+    readdirSync(folder).filter((name) => name.endsWith(".eml")).length + 1;
+  const base = String(index).padStart(3, "0");
+  const decoded = Buffer.from(decodeQuotedPrintable(raw), "latin1").toString(
+    "utf8",
+  );
+  const links =
+    decoded.replaceAll("&amp;", "&").match(/https?:\/\/[^\s"'<>]+/g) ?? [];
+  const summary = {
+    from: envelope.from,
+    to: envelope.to,
+    subject: headerOf(raw, "Subject"),
+    links: [...new Set(links)],
+    text: readableText(decoded),
+    receivedAt: new Date().toISOString(),
+  };
+  writeFileSync(join(folder, `${base}.eml`), raw);
+  writeFileSync(
+    join(folder, `${base}.json`),
+    `${JSON.stringify(summary, null, 2)}\n`,
+  );
+}
+
+function command(state, socket, line) {
+  const verb = line.slice(0, 4).toUpperCase();
+  if (verb === "EHLO" || verb === "HELO")
+    return reply(socket, "250 dms-control");
+  if (verb === "MAIL") {
+    state.envelope = { from: line.slice(line.indexOf(":") + 1).trim(), to: [] };
+    return reply(socket, "250 OK");
+  }
+  if (verb === "RCPT") {
+    state.envelope.to.push(line.slice(line.indexOf(":") + 1).trim());
+    return reply(socket, "250 OK");
+  }
+  if (verb === "DATA") {
+    state.inData = true;
+    return reply(socket, "354 End data with <CR><LF>.<CR><LF>");
+  }
+  if (verb === "QUIT") {
+    reply(socket, "221 Bye");
+    return socket.end();
+  }
+  return reply(socket, "250 OK");
+}
+
+function session(folder) {
+  return (socket) => {
+    const state = { buffer: "", inData: false, envelope: { from: "", to: [] } };
+    reply(socket, "220 dms-control mail sink");
+    socket.on("data", (chunk) => {
+      state.buffer += chunk.toString("latin1");
+      while (state.buffer.length > 0) {
+        if (state.inData) {
+          const end = state.buffer.indexOf(END_OF_DATA);
+          if (end === -1) return;
+          saveMessage(
+            folder,
+            state.envelope,
+            state.buffer.slice(0, end).replace(/^\.\./gm, "."),
+          );
+          state.buffer = state.buffer.slice(end + END_OF_DATA.length);
+          state.inData = false;
+          reply(socket, "250 OK: queued");
+          continue;
+        }
+        const end = state.buffer.indexOf("\r\n");
+        if (end === -1) return;
+        const line = state.buffer.slice(0, end);
+        state.buffer = state.buffer.slice(end + 2);
+        command(state, socket, line);
+      }
+    });
+    socket.on("error", () => socket.destroy());
+  };
+}
+
+/**
+ * Serves a plain SMTP sink on `port` that writes each message it receives
+ * to `<folder>/NNN.eml`, with its envelope, subject, links and readable
+ * text in `NNN.json`. It never relays anything.
+ *
+ * @param {string} folder Where the messages go
+ * @param {number} port The loopback port to listen on
+ * @returns {Promise<void>} Resolves once listening
+ */
+export function serveMail(folder, port) {
+  mkdirSync(folder, { recursive: true });
+  const server = createServer(session(folder));
+  return new Promise((resolve) => server.listen(port, "127.0.0.1", resolve));
+}
+
+/**
+ * The messages the sink captured so far, oldest first.
+ *
+ * @param {string} folder The sink's folder
+ * @returns {{ file: string, from: string, to: string[], subject: string, links: string[], text: string, receivedAt: string }[]} The captured messages
+ */
+export function capturedMail(folder) {
+  if (!existsSync(folder)) return [];
+  return readdirSync(folder)
+    .filter((name) => name.endsWith(".json"))
+    .sort()
+    .map((name) => ({
+      file: join(folder, name.replace(/\.json$/, ".eml")),
+      ...JSON.parse(readFileSync(join(folder, name), "utf8")),
+    }));
+}

package/bin/control/mongo.mjs

@@ -0,0 +1,156 @@
+import { spawnSync } from "node:child_process";
+import { mkdirSync } from "node:fs";
+import { createRequire } from "node:module";
+import { join } from "node:path";
+import { pathToFileURL } from "node:url";
+import { freePort, startDetached, waitForPort } from "./processes.mjs";
+
+/** The server version the backend test harness pins, already cached by it. */
+const MONGO_VERSION = "8.0.8";
+const READY_TIMEOUT_MS = 30_000;
+
+function withDefaultExports(namespace) {
+  return { ...namespace.default, ...namespace };
+}
+
+/**
+ * Imports a package the way the project under test resolves it: from the
+ * playground first, then from this package.
+ *
+ * @param {string} name The package name
+ * @param {string[]} from Folders to resolve from, in order
+ * @returns {Promise<any>} The module namespace
+ * @throws When no folder resolves it, saying which devDependency to add
+ */
+export async function importFrom(name, from) {
+  for (const folder of from) {
+    try {
+      const resolved = createRequire(join(folder, "package.json")).resolve(
+        name,
+      );
+      return withDefaultExports(await import(pathToFileURL(resolved).href));
+    } catch {
+      continue;
+    }
+  }
+  throw new Error(
+    `dms-control needs ${name}: add it to the playground's devDependencies.`,
+  );
+}
+
+function mongodOnPath() {
+  return spawnSync("mongod", ["--version"], { stdio: "ignore" }).status === 0
+    ? "mongod"
+    : undefined;
+}
+
+/**
+ * The mongod binary: `DMS_CONTROL_MONGOD`, else `mongod` on PATH, else the
+ * one mongodb-memory-server-core caches (and downloads once).
+ *
+ * @param {string[]} from Folders to resolve mongodb-memory-server-core from
+ * @returns {Promise<string>} The binary path
+ */
+async function mongodBinary(from) {
+  const configured = process.env.DMS_CONTROL_MONGOD ?? mongodOnPath();
+  if (configured) return configured;
+  const { MongoBinary } = await importFrom("mongodb-memory-server-core", from);
+  return MongoBinary.getPath({ version: MONGO_VERSION });
+}
+
+/**
+ * Starts a mongod owned by the run, on a free port, with its data under the
+ * run folder.
+ *
+ * @param {{ mongo: string, logs: string }} paths The run's folders
+ * @param {string[]} from Folders to resolve mongodb-memory-server-core from
+ * @returns {Promise<{ url: string, record: ReturnType<typeof startDetached> }>} Its URL and process record
+ */
+export async function startMongod(paths, from) {
+  const binary = await mongodBinary(from);
+  const port = await freePort();
+  mkdirSync(paths.mongo, { recursive: true });
+  const record = startDetached({
+    name: "mongod",
+    program: binary,
+    args: [
+      "--dbpath",
+      paths.mongo,
+      "--port",
+      String(port),
+      "--bind_ip",
+      "127.0.0.1",
+    ],
+    cwd: paths.mongo,
+    env: process.env,
+    logs: paths.logs,
+  });
+  await waitForPort(port, record, READY_TIMEOUT_MS);
+  return { url: `mongodb://127.0.0.1:${port}`, record: { ...record, port } };
+}
+
+function driverBesideMemoryServer(from) {
+  for (const folder of from) {
+    try {
+      const server = createRequire(join(folder, "package.json")).resolve(
+        "mongodb-memory-server-core",
+      );
+      return createRequire(server).resolve("mongodb");
+    } catch {
+      continue;
+    }
+  }
+  return undefined;
+}
+
+async function mongoDriver(from) {
+  const beside = driverBesideMemoryServer(from);
+  if (beside)
+    return withDefaultExports(await import(pathToFileURL(beside).href));
+  return importFrom("mongodb", from);
+}
+
+/**
+ * Drops every database whose name starts with the run's prefix, through the
+ * MongoDB driver that mongodb-memory-server-core depends on, or the
+ * project's own.
+ *
+ * @param {string} url The server URL
+ * @param {string} prefix The run's database prefix, `verify_<run id>`
+ * @param {string[]} from Folders to resolve the driver from
+ * @returns {Promise<string[]>} One line per dropped database
+ */
+export async function dropRunDatabases(url, prefix, from) {
+  const { MongoClient } = await mongoDriver(from);
+  const client = new MongoClient(url, { serverSelectionTimeoutMS: 5_000 });
+  try {
+    await client.connect();
+    const { databases } = await client.db("admin").admin().listDatabases();
+    const owned = databases
+      .map((database) => database.name)
+      .filter((name) => name.startsWith(prefix));
+    for (const name of owned) await client.db(name).dropDatabase();
+    return owned.map((name) => `dropped database ${name}`);
+  } finally {
+    await client.close();
+  }
+}
+
+/**
+ * Drops one database of the run.
+ *
+ * @param {string} url The server URL
+ * @param {string} name The database
+ * @param {string[]} from Folders to resolve the driver from
+ * @returns {Promise<void>} Resolves once dropped
+ */
+export async function dropDatabase(url, name, from) {
+  const { MongoClient } = await mongoDriver(from);
+  const client = new MongoClient(url, { serverSelectionTimeoutMS: 5_000 });
+  try {
+    await client.connect();
+    await client.db(name).dropDatabase();
+  } finally {
+    await client.close();
+  }
+}