@pixel-point/toolcraft 0.0.17 → 0.0.19

package/templates/starter/scripts/toolcraft-motion-reference-publication-isolation.test.mjs

@@ -0,0 +1,234 @@
+import assert from "node:assert/strict";
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import test from "node:test";
+
+import {
+  writeCanonicalToolcraftMotionReferenceEvidence,
+} from "../src/app/acceptance/motion-reference-evidence.mjs";
+import {
+  createToolcraftMotionPublicationEvidenceGroup,
+} from "./toolcraft-motion-reference-publication.fixtures.mjs";
+import {
+  publishToolcraftMotionReferenceStudies,
+} from "./toolcraft-motion-reference-publication.mjs";
+
+async function exists(target) {
+  return fs.stat(target).then(() => true, (error) => {
+    if (error.code === "ENOENT") return false;
+    throw error;
+  });
+}
+
+async function writeStudy(directory, evidence, sheetBytes) {
+  await fs.mkdir(directory, { recursive: true });
+  await Promise.all([
+    fs.writeFile(path.join(directory, "contact-sheet.png"), sheetBytes),
+    fs.writeFile(path.join(directory, "evidence.json"),
+      writeCanonicalToolcraftMotionReferenceEvidence(evidence)),
+  ]);
+}
+
+async function fixture(context) {
+  const rootDir = await fs.mkdtemp(path.join(os.tmpdir(), "motion-isolation-"));
+  context.after(() => fs.rm(rootDir, { force: true, recursive: true }));
+  const finalRoot = path.join(rootDir, "src", "app", "reference-studies");
+  const temporaryRoot = path.join(rootDir, ".toolcraft", "tmp");
+  await Promise.all([
+    fs.mkdir(finalRoot, { recursive: true }),
+    fs.mkdir(temporaryRoot, { recursive: true }),
+  ]);
+  const createSource = async (sourceCharacter, count) => {
+    const group = createToolcraftMotionPublicationEvidenceGroup({
+      count, sourceCharacter,
+    });
+    const finalDirectories = group.evidences.map(({ studyId }) =>
+      path.join(finalRoot, studyId));
+    await Promise.all(group.evidences.map((evidence, index) =>
+      writeStudy(finalDirectories[index], evidence, group.sheetBytes[index])));
+    return {
+      ...group,
+      finalDirectories,
+      finalRoot,
+      lockDirectory: path.join(temporaryRoot, `${group.referenceId}.lock`),
+      rootDir,
+      temporaryRoot,
+    };
+  };
+  return {
+    a: await createSource("a", 2),
+    b: await createSource("b", 1),
+    finalRoot,
+    rootDir,
+  };
+}
+
+async function stage(source, label) {
+  const stagingRoot = path.join(
+    source.temporaryRoot, `work-${label}`, `${source.referenceId}.staging`,
+  );
+  const stagingDirectories = [];
+  for (const [index, evidence] of source.evidences.entries()) {
+    const directory = path.join(stagingRoot, evidence.studyId);
+    await writeStudy(directory, evidence, source.sheetBytes[index]);
+    stagingDirectories.push(directory);
+  }
+  return { ...source, stagingDirectories, stagingRoot };
+}
+
+function settings(source, fileSystem = fs) {
+  return {
+    fileSystem,
+    lockLeaseMilliseconds: 500,
+    lockDirectory: source.lockDirectory,
+    now: () => 1_000,
+    pid: 444,
+    publicationGroup: {
+      evidences: source.evidences,
+      referenceStudiesRoot: source.finalRoot,
+      rootDir: source.rootDir,
+      stagingRoot: source.stagingRoot,
+    },
+    sourceIdentity: source.referenceId,
+    tokenFactory: () => `token-${source.referenceId}`,
+  };
+}
+
+function publish(source, fileSystem) {
+  return publishToolcraftMotionReferenceStudies(
+    source.finalDirectories.map((finalDirectory, index) => ({
+      finalDirectory,
+      stagingDirectory: source.stagingDirectories[index],
+    })),
+    settings(source, fileSystem),
+  );
+}
+
+test("transaction backups stay outside the committed artifact root",
+  async (context) => {
+    const value = await fixture(context);
+    const a = await stage(value.a, "a");
+    const renames = [];
+    await publish(a, {
+      ...fs,
+      rename: async (from, to) => {
+        renames.push([from, to]);
+        return fs.rename(from, to);
+      },
+    });
+    const transactionRoot = path.join(
+      a.temporaryRoot, `${a.referenceId}.transaction`,
+    );
+    assert.deepEqual(renames.filter(([from]) =>
+      a.finalDirectories.includes(from)), a.finalDirectories.map((finalDirectory) => [
+      finalDirectory, path.join(transactionRoot, path.basename(finalDirectory)),
+    ]));
+    assert.deepEqual((await fs.readdir(value.finalRoot)).sort(), [
+      ...a.finalDirectories, ...value.b.finalDirectories,
+    ].map((directory) => path.basename(directory)).sort());
+  });
+
+test("a partial real source transaction is invisible to another source",
+  async (context) => {
+    const value = await fixture(context);
+    const a = await stage(value.a, "a");
+    const b = await stage(value.b, "b");
+    let releaseA;
+    let reportPaused;
+    const paused = new Promise((resolve) => { reportPaused = resolve; });
+    const resume = new Promise((resolve) => { releaseA = resolve; });
+    let held = false;
+    const aFileSystem = {
+      ...fs,
+      rename: async (from, to) => {
+        await fs.rename(from, to);
+        if (!held && from === a.stagingDirectories[0] &&
+            to === a.finalDirectories[0]) {
+          held = true;
+          reportPaused();
+          await resume;
+        }
+      },
+    };
+    const aPublication = publish(a, aFileSystem);
+    await paused;
+
+    assert.equal(await exists(a.finalDirectories[0]), true);
+    assert.equal(await exists(a.finalDirectories[1]), false);
+    const bMutations = [];
+    const bFileSystem = {
+      ...fs,
+      rename: async (from, to) => {
+        bMutations.push([from, to]);
+        return fs.rename(from, to);
+      },
+      rm: async (target, options) => {
+        bMutations.push([target]);
+        return fs.rm(target, options);
+      },
+    };
+    await publish(b, bFileSystem);
+    assert.equal(await exists(b.finalDirectories[0]), true);
+    assert.equal(bMutations.some((paths) => paths.some((target) =>
+      a.finalDirectories.includes(target) ||
+      target.includes(`${a.referenceId}.transaction`))), false);
+
+    releaseA();
+    await aPublication;
+    assert.deepEqual((await fs.readdir(value.finalRoot)).sort(), [
+      ...a.finalDirectories, ...b.finalDirectories,
+    ].map((directory) => path.basename(directory)).sort());
+  });
+
+test("retained cleanup residue is source-owned and cleaned by that source later",
+  async (context) => {
+    const value = await fixture(context);
+    let a = await stage(value.a, "a-first");
+    const b = await stage(value.b, "b");
+    let retainedBackup;
+    const cleanupFailure = {
+      ...fs,
+      rm: async (target, options) => {
+        if (target.includes(`${a.referenceId}.transaction`) &&
+            retainedBackup === undefined) {
+          retainedBackup = target;
+          throw Object.assign(new Error("retained A cleanup"), { code: "EBUSY" });
+        }
+        return fs.rm(target, options);
+      },
+    };
+    const aResult = await publish(a, cleanupFailure);
+    assert.equal(aResult.committed, true);
+    assert.equal(await exists(retainedBackup), true);
+
+    await publish(b);
+    assert.equal(await exists(retainedBackup), true);
+
+    a = await stage(value.a, "a-later");
+    await publish(a);
+    assert.equal(await exists(retainedBackup), false);
+    assert.equal(await exists(value.b.finalDirectories[0]), true);
+  });
+
+test("same-source residue without its journal fails closed", async (context) => {
+  const value = await fixture(context);
+  let a = await stage(value.a, "a-first");
+  let retainedBackup;
+  await publish(a, {
+    ...fs,
+    rm: async (target, options) => {
+      if (target.includes(`${a.referenceId}.transaction`) &&
+          retainedBackup === undefined) {
+        retainedBackup = target;
+        throw Object.assign(new Error("retain backup"), { code: "EBUSY" });
+      }
+      return fs.rm(target, options);
+    },
+  });
+  await fs.rm(`${a.lockDirectory}.publication.json`);
+  a = await stage(value.a, "a-unexpected");
+
+  await assert.rejects(() => publish(a), /unexplained transaction residue/iu);
+  assert.equal(await exists(retainedBackup), true);
+});

package/templates/starter/scripts/toolcraft-motion-reference-publication-journal-schema.mjs

@@ -0,0 +1,122 @@
+import path from "node:path";
+
+import {
+  defineToolcraftMotionReferenceStagingScope,
+} from "./toolcraft-motion-reference-publication-staging.mjs";
+
+export const TOOLCRAFT_MOTION_PUBLICATION_JOURNAL_VERSION = 4;
+
+function fail(message, cause) {
+  throw new Error(`Toolcraft motion reference publication journal failed: ${message}`, {
+    cause,
+  });
+}
+
+function exactKeys(value, keys) {
+  return value && typeof value === "object" && !Array.isArray(value) &&
+    Object.keys(value).sort().join("\0") === [...keys].sort().join("\0");
+}
+
+function sortJson(value) {
+  if (Array.isArray(value)) return value.map(sortJson);
+  if (value && typeof value === "object") return Object.fromEntries(
+    Object.keys(value).sort().map((key) => [key, sortJson(value[key])]),
+  );
+  return value;
+}
+
+export function serializeToolcraftMotionPublicationJournal(journal) {
+  return `${JSON.stringify(sortJson(journal), null, 2)}\n`;
+}
+
+export function getToolcraftMotionReferenceBackupPath(
+  transactionRoot,
+  finalDirectory,
+) {
+  return path.join(transactionRoot, path.basename(finalDirectory));
+}
+
+export function getToolcraftMotionReferencePublicationJournalPath(lockDirectory) {
+  return `${lockDirectory}.publication.json`;
+}
+
+function validateRecord(record, seen, transactionRoot) {
+  const keys = ["backedUp", "backupDirectory", "finalDirectory", "hadOld",
+    "kind", "published", "stagingDirectory"];
+  if (!exactKeys(record, keys) ||
+      !path.isAbsolute(record.backupDirectory ?? "") ||
+      !path.isAbsolute(record.finalDirectory ?? "") ||
+      !["publish", "retire"].includes(record.kind) ||
+      (record.kind === "publish" && !path.isAbsolute(record.stagingDirectory ?? "")) ||
+      (record.kind === "retire" && record.stagingDirectory !== null) ||
+      record.backupDirectory !==
+        getToolcraftMotionReferenceBackupPath(transactionRoot, record.finalDirectory) ||
+      typeof record.backedUp !== "boolean" || typeof record.hadOld !== "boolean" ||
+      typeof record.published !== "boolean" || seen.has(record.finalDirectory)) {
+    fail("journal records must be canonical unique absolute publication records");
+  }
+  if (record.backedUp && !record.hadOld) {
+    fail("a journal record cannot back up a final that did not previously exist");
+  }
+  if (record.kind === "retire" && (!record.hadOld || record.published)) {
+    fail("a retirement record must describe an unpublished previous final");
+  }
+  seen.add(record.finalDirectory);
+}
+
+export function validateToolcraftMotionPublicationJournal(
+  journal,
+  sourceIdentity,
+  transactionRoot,
+) {
+  if (!exactKeys(journal,
+    ["operation", "phase", "records", "sourceIdentity", "version"]) ||
+      journal.version !== TOOLCRAFT_MOTION_PUBLICATION_JOURNAL_VERSION ||
+      !["preparing", "rolling-back", "committed"].includes(journal.phase) ||
+      journal.sourceIdentity !== sourceIdentity ||
+      !Array.isArray(journal.records) || journal.records.length === 0) {
+    fail("journal shape, version, phase, or source identity is invalid");
+  }
+  if (journal.operation !== null &&
+      (!exactKeys(journal.operation, ["index", "kind"]) ||
+        !Number.isSafeInteger(journal.operation.index) ||
+        journal.operation.index < 0 || journal.operation.index >= journal.records.length ||
+        !["backup", "publish", "remove-published", "restore-backup"]
+          .includes(journal.operation.kind))) {
+    fail("journal operation must be null or a canonical record mutation");
+  }
+  if (journal.phase === "committed" && journal.operation !== null) {
+    fail("a committed journal cannot retain an active mutation");
+  }
+  if (journal.operation &&
+      ((journal.phase === "preparing" &&
+        !["backup", "publish"].includes(journal.operation.kind)) ||
+       (journal.phase === "rolling-back" &&
+        !["remove-published", "restore-backup"].includes(journal.operation.kind)))) {
+    fail("journal operation does not match its publication phase");
+  }
+  if (journal.operation?.kind === "publish" &&
+      journal.records[journal.operation.index]?.kind !== "publish") {
+    fail("only a staged publication record may own a publish mutation");
+  }
+  const seen = new Set();
+  journal.records.forEach((record) =>
+    validateRecord(record, seen, transactionRoot));
+  return journal;
+}
+
+export function validateToolcraftMotionPublicationJournalScope(
+  journal,
+  { artifactRoot, temporaryRoot, transactionRoot },
+) {
+  defineToolcraftMotionReferenceStagingScope(journal.records, {
+    sourceIdentity: journal.sourceIdentity,
+    temporaryRoot,
+  });
+  for (const record of journal.records) {
+    if (path.dirname(record.finalDirectory) !== artifactRoot ||
+        path.dirname(record.backupDirectory) !== transactionRoot) {
+      fail("journal records escape the checked artifact or transaction roots");
+    }
+  }
+}

package/templates/starter/scripts/toolcraft-motion-reference-publication-journal-store.mjs

@@ -0,0 +1,89 @@
+import {
+  serializeToolcraftMotionPublicationJournal,
+  validateToolcraftMotionPublicationJournal,
+} from "./toolcraft-motion-reference-publication-journal-schema.mjs";
+
+function fail(message, cause) {
+  throw new Error(`Toolcraft motion reference publication journal failed: ${message}`, {
+    cause,
+  });
+}
+
+export function createToolcraftMotionPublicationJournalStore({
+  fileSystem,
+  heartbeat,
+  journalPath,
+  sourceIdentity,
+  token,
+  transactionRoot,
+}) {
+  async function checked(operation) {
+    await heartbeat();
+    const result = await operation();
+    await heartbeat();
+    return result;
+  }
+
+  async function status(target) {
+    try {
+      return await checked(() => fileSystem.stat(target));
+    } catch (error) {
+      if (error?.code === "ENOENT") return undefined;
+      throw error;
+    }
+  }
+
+  async function read() {
+    let raw;
+    try {
+      raw = await checked(() => fileSystem.readFile(journalPath, "utf8"));
+    } catch (error) {
+      if (error?.code === "ENOENT") return undefined;
+      fail("journal could not be read", error);
+    }
+    let journal;
+    try {
+      journal = JSON.parse(raw);
+    } catch (error) {
+      fail("journal must contain strict JSON", error);
+    }
+    validateToolcraftMotionPublicationJournal(
+      journal, sourceIdentity, transactionRoot,
+    );
+    if (raw !== serializeToolcraftMotionPublicationJournal(journal)) {
+      fail("journal must use canonical serialization");
+    }
+    return journal;
+  }
+
+  async function write(journal) {
+    validateToolcraftMotionPublicationJournal(
+      journal, sourceIdentity, transactionRoot,
+    );
+    const temporaryPath = `${journalPath}.tmp-${token}`;
+    try {
+      await checked(() => fileSystem.writeFile(
+        temporaryPath, serializeToolcraftMotionPublicationJournal(journal),
+        { flag: "wx" },
+      ));
+      await checked(() => fileSystem.rename(temporaryPath, journalPath));
+    } catch (error) {
+      try {
+        await checked(() => fileSystem.rm(temporaryPath, { force: true }));
+      } catch {}
+      fail("atomic journal write failed", error);
+    }
+  }
+
+  return {
+    checked,
+    fileSystem,
+    heartbeat,
+    journalPath,
+    read,
+    remove: () => checked(() => fileSystem.rm(journalPath, { force: true })),
+    status,
+    transactionRoot,
+    write,
+  };
+}

package/templates/starter/scripts/toolcraft-motion-reference-publication-journal-store.test.mjs

@@ -0,0 +1,74 @@
+import assert from "node:assert/strict";
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import test from "node:test";
+
+import {
+  createToolcraftMotionPublicationJournalStore,
+} from "./toolcraft-motion-reference-publication-journal-store.mjs";
+import {
+  getToolcraftMotionReferenceBackupPath,
+} from "./toolcraft-motion-reference-publication-journal-schema.mjs";
+
+const sourceA = `motion-reference-v1-${"a".repeat(64)}`;
+const sourceB = `motion-reference-v1-${"b".repeat(64)}`;
+
+async function exists(target) {
+  return fs.stat(target).then(() => true, (error) => {
+    if (error.code === "ENOENT") return false;
+    throw error;
+  });
+}
+
+function journal(sourceIdentity, transactionRoot, root) {
+  const finalDirectory = path.join(root, "final");
+  return {
+    operation: null,
+    phase: "preparing",
+    records: [{
+      backedUp: false,
+      backupDirectory: getToolcraftMotionReferenceBackupPath(
+        transactionRoot, finalDirectory,
+      ),
+      finalDirectory,
+      hadOld: false,
+      kind: "publish",
+      published: false,
+      stagingDirectory: path.join(
+        root, `${sourceIdentity}.staging`, path.basename(finalDirectory),
+      ),
+    }],
+    sourceIdentity,
+    version: 4,
+  };
+}
+
+test("binds journal writes to the store source identity before filesystem mutation",
+  async (context) => {
+    const root = await fs.mkdtemp(path.join(os.tmpdir(), "motion-journal-store-"));
+    context.after(() => fs.rm(root, { force: true, recursive: true }));
+    const journalPath = path.join(root, "source.lock.publication.json");
+    const temporaryPath = `${journalPath}.tmp-token`;
+    const transactionRoot = path.join(root, "transaction");
+    await fs.mkdir(transactionRoot);
+    const store = createToolcraftMotionPublicationJournalStore({
+      fileSystem: fs,
+      heartbeat: async () => {},
+      journalPath,
+      sourceIdentity: sourceA,
+      token: "token",
+      transactionRoot,
+    });
+
+    await assert.rejects(() => store.write(journal(sourceB, transactionRoot, root)),
+      /source identity.*invalid/iu);
+    assert.equal(await exists(temporaryPath), false);
+    assert.equal(await exists(journalPath), false);
+
+    const matching = journal(sourceA, transactionRoot, root);
+    await store.write(matching);
+    assert.deepEqual(await store.read(), matching);
+    assert.equal(await exists(temporaryPath), false);
+    assert.equal(await exists(journalPath), true);
+  });