@pixel-point/toolcraft 0.0.17 → 0.0.19

package/templates/starter/scripts/toolcraft-motion-reference-publication-staging.test.mjs

@@ -0,0 +1,209 @@
+import assert from "node:assert/strict";
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import test from "node:test";
+
+import {
+  createToolcraftMotionPublicationEvidenceGroup,
+} from "./toolcraft-motion-reference-publication.fixtures.mjs";
+import {
+  publishToolcraftMotionReferenceStudies,
+} from "./toolcraft-motion-reference-publication.mjs";
+
+async function exists(target) {
+  return fs.lstat(target).then(() => true, (error) => {
+    if (error.code === "ENOENT") return false;
+    throw error;
+  });
+}
+
+async function fixture(context) {
+  const rootDir = await fs.mkdtemp(path.join(os.tmpdir(), "motion-staging-scope-"));
+  context.after(() => fs.rm(rootDir, { force: true, recursive: true }));
+  const group = createToolcraftMotionPublicationEvidenceGroup();
+  const temporaryRoot = path.join(rootDir, ".toolcraft", "tmp");
+  const finalRoot = path.join(rootDir, "src", "app", "reference-studies");
+  const finalDirectory = path.join(finalRoot, group.evidences[0].studyId);
+  const lockDirectory = path.join(temporaryRoot, `${group.referenceId}.lock`);
+  const transactionRoot = path.join(
+    temporaryRoot, `${group.referenceId}.transaction`,
+  );
+  await Promise.all([
+    fs.mkdir(temporaryRoot, { recursive: true }),
+    fs.mkdir(finalRoot, { recursive: true }),
+  ]);
+  return {
+    finalDirectory,
+    group,
+    lockDirectory,
+    rootDir,
+    temporaryRoot,
+    transactionRoot,
+  };
+}
+
+function settings(value, stagingRoot, fileSystem = fs) {
+  return {
+    discoverPreviousGroup: async () => ({ obsoleteStudyIds: [] }),
+    fileSystem,
+    lockLeaseMilliseconds: 500,
+    lockDirectory: value.lockDirectory,
+    now: () => 1_000,
+    pid: 444,
+    publicationGroup: {
+      evidences: value.group.evidences,
+      referenceStudiesRoot: path.dirname(value.finalDirectory),
+      rootDir: value.rootDir,
+      stagingRoot,
+    },
+    sourceIdentity: value.group.referenceId,
+    tokenFactory: () => "staging-token",
+  };
+}
+
+function mutationGuard(value) {
+  const mutations = [];
+  const protectedTargets = [
+    value.finalDirectory,
+    value.lockDirectory,
+    `${value.lockDirectory}.publication.json`,
+    value.transactionRoot,
+  ];
+  const record = (...targets) => {
+    if (targets.some((target) => typeof target === "string" &&
+        protectedTargets.some((protectedTarget) =>
+          target === protectedTarget || target.startsWith(`${protectedTarget}${path.sep}`)))) {
+      mutations.push(targets);
+    }
+  };
+  return {
+    fileSystem: {
+      ...fs,
+      mkdir: async (target, options) => {
+        record(target);
+        return fs.mkdir(target, options);
+      },
+      rename: async (from, to) => {
+        record(from, to);
+        return fs.rename(from, to);
+      },
+      rm: async (target, options) => {
+        record(target);
+        return fs.rm(target, options);
+      },
+      writeFile: async (target, ...args) => {
+        record(target);
+        return fs.writeFile(target, ...args);
+      },
+    },
+    mutations,
+  };
+}
+
+async function rejectsWithoutPublicationMutation(
+  value,
+  { stagingDirectory, stagingRoot },
+) {
+  const guard = mutationGuard(value);
+  await assert.rejects(() => publishToolcraftMotionReferenceStudies([{
+    finalDirectory: value.finalDirectory,
+    stagingDirectory,
+  }], settings(value, stagingRoot, guard.fileSystem)),
+  /canonical.*staging|staging.*scope|staging.*source|staging.*link/iu);
+  assert.deepEqual(guard.mutations, []);
+  assert.equal(await exists(value.finalDirectory), false);
+  assert.equal(await exists(value.lockDirectory), false);
+  assert.equal(await exists(`${value.lockDirectory}.publication.json`), false);
+}
+
+test("publishes a direct study from its declared source-owned staging root",
+  async (context) => {
+    const value = await fixture(context);
+    const stagingRoot = path.join(
+      value.temporaryRoot, "work", `${value.group.referenceId}.staging`,
+    );
+    const stagingDirectory = path.join(
+      stagingRoot, value.group.evidences[0].studyId,
+    );
+    await fs.mkdir(stagingDirectory, { recursive: true });
+    await fs.writeFile(path.join(stagingDirectory, "version"), "canonical");
+
+    await publishToolcraftMotionReferenceStudies([{
+      finalDirectory: value.finalDirectory,
+      stagingDirectory,
+    }], settings(value, stagingRoot));
+
+    assert.equal(await fs.readFile(
+      path.join(value.finalDirectory, "version"), "utf8",
+    ), "canonical");
+  });
+
+test("rejects staging outside its canonical source scope before publication mutation",
+  async (context) => {
+    await context.test("external absolute directory", async (subtest) => {
+      const value = await fixture(subtest);
+      const stagingRoot = path.join(value.rootDir, "external-staging");
+      const stagingDirectory = path.join(stagingRoot, value.group.evidences[0].studyId);
+      await fs.mkdir(stagingDirectory, { recursive: true });
+      await rejectsWithoutPublicationMutation(value, { stagingDirectory, stagingRoot });
+    });
+
+    await context.test("symbolic-link leaf", async (subtest) => {
+      const value = await fixture(subtest);
+      const outside = path.join(value.rootDir, "outside-leaf");
+      const stagingRoot = path.join(
+        value.temporaryRoot, "work", `${value.group.referenceId}.staging`,
+      );
+      const stagingDirectory = path.join(stagingRoot, value.group.evidences[0].studyId);
+      await Promise.all([
+        fs.mkdir(outside),
+        fs.mkdir(stagingRoot, { recursive: true }),
+      ]);
+      await fs.symlink(outside, stagingDirectory, "dir");
+      await rejectsWithoutPublicationMutation(value, { stagingDirectory, stagingRoot });
+    });
+
+    await context.test("symbolic-link ancestor", async (subtest) => {
+      const value = await fixture(subtest);
+      const outsideParent = path.join(value.rootDir, "outside-ancestor");
+      const outsideRoot = path.join(
+        outsideParent, `${value.group.referenceId}.staging`,
+      );
+      const stagingDirectory = path.join(outsideRoot, value.group.evidences[0].studyId);
+      await fs.mkdir(stagingDirectory, { recursive: true });
+      const linkedParent = path.join(value.temporaryRoot, "linked-work");
+      await fs.symlink(outsideParent, linkedParent, "dir");
+      await rejectsWithoutPublicationMutation(value, {
+        stagingDirectory: path.join(
+          linkedParent, `${value.group.referenceId}.staging`,
+          value.group.evidences[0].studyId,
+        ),
+        stagingRoot: path.join(linkedParent, `${value.group.referenceId}.staging`),
+      });
+    });
+
+    await context.test("parent traversal", async (subtest) => {
+      const value = await fixture(subtest);
+      const workRoot = path.join(value.temporaryRoot, "work");
+      const stagingRoot = path.join(workRoot, `${value.group.referenceId}.staging`);
+      const escaped = path.join(workRoot, "escaped");
+      await Promise.all([
+        fs.mkdir(stagingRoot, { recursive: true }),
+        fs.mkdir(escaped, { recursive: true }),
+      ]);
+      await rejectsWithoutPublicationMutation(value, {
+        stagingDirectory: `${stagingRoot}${path.sep}..${path.sep}escaped`,
+        stagingRoot,
+      });
+    });
+
+    await context.test("another source transaction root", async (subtest) => {
+      const value = await fixture(subtest);
+      const otherSource = `motion-reference-v1-${"b".repeat(64)}`;
+      const stagingRoot = path.join(value.temporaryRoot, `${otherSource}.transaction`);
+      const stagingDirectory = path.join(stagingRoot, value.group.evidences[0].studyId);
+      await fs.mkdir(stagingDirectory, { recursive: true });
+      await rejectsWithoutPublicationMutation(value, { stagingDirectory, stagingRoot });
+    });
+  });

package/templates/starter/scripts/toolcraft-motion-reference-publication.fixtures.mjs

@@ -0,0 +1,88 @@
+import {
+  createToolcraftMotionReferenceGroupId,
+  createToolcraftMotionReferenceId,
+  createToolcraftMotionReferencePlanSha256,
+  createToolcraftMotionReferenceStudyId,
+  createToolcraftSha256,
+} from "./toolcraft-motion-reference-artifacts.mjs";
+
+const encoder = new TextEncoder();
+
+export function createToolcraftMotionPublicationEvidenceGroup({
+  count = 1,
+  sourceCharacter = "a",
+} = {}) {
+  const sourceSha256 = sourceCharacter.repeat(64);
+  const scope = { endSeconds: count * 2, startSeconds: 0 };
+  const evidences = Array.from({ length: count }, (_, index) => {
+    const segment = count === 1 ? undefined : {
+      endSeconds: (index + 1) * 2,
+      startSeconds: index * 2,
+    };
+    const sourceIndices = Array.from({ length: 3 }, (__, offset) =>
+      index * 3 + offset);
+    const reviewedFrames = sourceIndices.map((sourceFrameIndex, position) => ({
+      cell: { column: position, row: 0 },
+      frameId: `source-frame:${sourceFrameIndex}`,
+      sourceFrameIndex,
+      sourceLocator: `reference-${sourceCharacter}.mp4#frame=${sourceFrameIndex}`,
+      sourceSha256,
+      timeSeconds: (segment?.startSeconds ?? 0) + position,
+    }));
+    const evidence = {
+      changeMetric: "max-yuva-diff-v1",
+      contactSheet: {
+        columns: 3,
+        height: 100,
+        path: "",
+        rows: 1,
+        sha256: createToolcraftSha256(encoder.encode(`sheet-${sourceCharacter}-${index}`)),
+        width: 300,
+      },
+      decodedFrameCount: count * 3,
+      detectedEvents: [],
+      durationSeconds: scope.endSeconds,
+      evidenceVersion: 1,
+      ffmpegVersion: "ffmpeg 8.0",
+      ffprobeVersion: "ffprobe 8.0",
+      focusWindows: [],
+      height: 180,
+      maximumSourceFrameGapSeconds: 0,
+      overviewFrameIds: reviewedFrames.map(({ frameId }) => frameId),
+      overviewTargetFps: 1,
+      reviewedFrames,
+      scannedFrameCount: count * 3,
+      ...(segment ? { segment } : {}),
+      sourceBasename: `reference-${sourceCharacter}.mp4`,
+      sourceKind: "video",
+      sourceSha256,
+      studyId: "",
+      timing: { authority: "decoded-timestamps" },
+      timingMode: "seconds",
+      width: 320,
+    };
+    evidence.studyId = createToolcraftMotionReferenceStudyId(evidence);
+    evidence.contactSheet.path =
+      `src/app/reference-studies/${evidence.studyId}/contact-sheet.png`;
+    return evidence;
+  });
+  if (count > 1) {
+    const groupId = createToolcraftMotionReferenceGroupId(evidences[0], scope);
+    const planSha256 = createToolcraftMotionReferencePlanSha256(evidences);
+    evidences.forEach((evidence, index) => {
+      evidence.reviewPartition = {
+        count,
+        groupId,
+        index,
+        planSha256,
+        scope,
+      };
+    });
+  }
+  return {
+    evidences,
+    referenceId: createToolcraftMotionReferenceId(evidences[0]),
+    sheetBytes: evidences.map((__, index) =>
+      encoder.encode(`sheet-${sourceCharacter}-${index}`)),
+  };
+}

package/templates/starter/scripts/toolcraft-motion-reference-publication.mjs

@@ -0,0 +1,249 @@
+import { randomUUID } from "node:crypto";
+import fs from "node:fs/promises";
+import path from "node:path";
+
+import {
+  acquireToolcraftMotionReferenceLease,
+} from "./toolcraft-motion-reference-publication-lease.mjs";
+import {
+  defineToolcraftMotionReferencePublicationGroup,
+  discoverToolcraftMotionReferencePreviousGroup,
+} from "./toolcraft-motion-reference-group-inventory.mjs";
+import {
+  isStudyId,
+} from "../src/app/acceptance/motion-reference-evidence-validation.mjs";
+import {
+  publishToolcraftMotionReferenceJournaledGroup,
+  recoverToolcraftMotionReferencePublication,
+} from "./toolcraft-motion-reference-publication-journal.mjs";
+import {
+  getToolcraftMotionReferencePublicationJournalPath,
+} from "./toolcraft-motion-reference-publication-journal-schema.mjs";
+import {
+  requireToolcraftMotionReferenceStagingScope,
+} from "./toolcraft-motion-reference-publication-staging.mjs";
+import {
+  defineToolcraftMotionReferenceTransactionPaths,
+  requireToolcraftMotionReferenceStudiesRoot,
+  requireToolcraftMotionReferenceTemporaryRoot,
+} from "./toolcraft-motion-reference-roots.mjs";
+
+const DEFAULT_LOCK_LEASE_MILLISECONDS = 300_000;
+const DEFAULT_MAXIMUM_UNINTERRUPTIBLE_OPERATION_MILLISECONDS = 60_000;
+
+function fail(message, cause) {
+  throw new Error(`Toolcraft motion reference publication failed: ${message}`, {
+    cause,
+  });
+}
+
+async function status(fileSystem, target) {
+  try {
+    return await fileSystem.stat(target);
+  } catch (error) {
+    if (error?.code === "ENOENT") return undefined;
+    throw error;
+  }
+}
+
+function validatePairs(pairs) {
+  if (!Array.isArray(pairs) || pairs.length === 0) {
+    fail("at least one staging/final pair is required");
+  }
+  const staging = new Set();
+  const finals = new Set();
+  for (const pair of pairs) {
+    if (!pair || !path.isAbsolute(pair.stagingDirectory ?? "") ||
+        !path.isAbsolute(pair.finalDirectory ?? "") ||
+        staging.has(pair.stagingDirectory) || finals.has(pair.finalDirectory)) {
+      fail("staging and final directories must be absolute and unique");
+    }
+    staging.add(pair.stagingDirectory);
+    finals.add(pair.finalDirectory);
+  }
+}
+
+async function validateSameFileSystem(pairs, fileSystem) {
+  let device;
+  for (const pair of pairs) {
+    const staging = pair.stagingDirectory === null ? undefined :
+      await status(fileSystem, pair.stagingDirectory);
+    const finalParent = await status(fileSystem, path.dirname(pair.finalDirectory));
+    if ((pair.stagingDirectory !== null && !staging?.isDirectory()) ||
+        !finalParent?.isDirectory()) {
+      fail("every staging directory and final parent must already exist");
+    }
+    device ??= finalParent.dev;
+    if ((staging && staging.dev !== device) || finalParent.dev !== device) {
+      fail("staging and final studies must use the same filesystem");
+    }
+  }
+}
+
+async function prepareTransactionRoot(
+  fileSystem, artifactRoot, transactionRoot,
+) {
+  await fileSystem.mkdir(transactionRoot, { mode: 0o700, recursive: true });
+  const [artifactStatus, transactionStatus] = await Promise.all([
+    fileSystem.stat(artifactRoot),
+    fileSystem.lstat(transactionRoot),
+  ]);
+  if (!transactionStatus.isDirectory() || transactionStatus.isSymbolicLink()) {
+    fail("transaction root must be a direct directory, not a link");
+  }
+  if (artifactStatus.dev !== transactionStatus.dev) {
+    fail("transaction and committed studies must use the same filesystem");
+  }
+}
+
+function obsoleteFinalDirectories(discovery, checkedRoot, newStudyIds) {
+  if (!discovery || typeof discovery !== "object" || Array.isArray(discovery) ||
+      Object.keys(discovery).join("\0") !== "obsoleteStudyIds" ||
+      !Array.isArray(discovery.obsoleteStudyIds)) {
+    fail("discovery must return exactly canonical obsolete study ids");
+  }
+  const seen = new Set();
+  return discovery.obsoleteStudyIds.map((studyId) => {
+    if (!isStudyId(studyId) || seen.has(studyId) || newStudyIds.has(studyId)) {
+      fail("obsolete study ids must be canonical, unique, and distinct from new studies");
+    }
+    seen.add(studyId);
+    const finalDirectory = path.join(checkedRoot, studyId);
+    if (path.dirname(finalDirectory) !== checkedRoot) {
+      fail("obsolete study ids must resolve to direct canonical reference children");
+    }
+    return finalDirectory;
+  });
+}
+
+function combineTransactionPairs(pairs, obsoleteDirectories) {
+  const combined = [...pairs];
+  for (const finalDirectory of obsoleteDirectories) {
+    combined.push({ finalDirectory, stagingDirectory: null });
+  }
+  return combined;
+}
+
+function getBoundPublicationStudyIds(pairs, expectedRoot) {
+  return pairs.map(({ finalDirectory }) => {
+    if (path.dirname(finalDirectory) !== expectedRoot) {
+      fail("new study finals must be direct children of the canonical reference root");
+    }
+    return path.basename(finalDirectory);
+  }).sort();
+}
+
+function validatePublicationGroupBindings(actualIds, authority) {
+  if (actualIds.join("\0") !== authority.studyIds.join("\0")) {
+    fail("new study finals must match the complete canonical evidence group");
+  }
+}
+
+export async function publishToolcraftMotionReferenceStudies(pairs, settings = {}) {
+  const lockLeaseMilliseconds = settings.lockLeaseMilliseconds ??
+    DEFAULT_LOCK_LEASE_MILLISECONDS;
+  const options = {
+    fileSystem: fs,
+    discoverPreviousGroup: discoverToolcraftMotionReferencePreviousGroup,
+    lockLeaseMilliseconds,
+    maximumUninterruptibleOperationMilliseconds:
+      Math.min(DEFAULT_MAXIMUM_UNINTERRUPTIBLE_OPERATION_MILLISECONDS,
+        lockLeaseMilliseconds / 2),
+    now: Date.now,
+    pid: process.pid,
+    tokenFactory: randomUUID,
+    ...settings,
+  };
+  validatePairs(pairs);
+  if (!options.publicationGroup) {
+    fail("a canonical publication group is required");
+  }
+  const checkedRoot = await requireToolcraftMotionReferenceStudiesRoot({
+    fileSystem: options.fileSystem,
+    referenceStudiesRoot: options.publicationGroup.referenceStudiesRoot,
+    rootDir: options.publicationGroup.rootDir,
+  });
+  const temporaryRoot = await requireToolcraftMotionReferenceTemporaryRoot({
+    fileSystem: options.fileSystem,
+    rootDir: options.publicationGroup.rootDir,
+    temporaryRoot: path.dirname(options.lockDirectory),
+  });
+  const transactionPaths = defineToolcraftMotionReferenceTransactionPaths(
+    temporaryRoot, options.sourceIdentity,
+  );
+  if (options.lockDirectory !== transactionPaths.lockDirectory) {
+    fail("publication lock must be the canonical source-scoped lock");
+  }
+  if (typeof options.publicationGroup.stagingRoot !== "string") {
+    fail("a declared canonical staging root is required");
+  }
+  const boundStudyIds = getBoundPublicationStudyIds(pairs, checkedRoot);
+  const authority = defineToolcraftMotionReferencePublicationGroup(
+    options.publicationGroup.evidences,
+  );
+  if (authority.referenceId !== options.sourceIdentity) {
+    fail("publication source identity does not match the canonical evidence group");
+  }
+  validatePublicationGroupBindings(boundStudyIds, authority);
+  await requireToolcraftMotionReferenceStagingScope(pairs, {
+    fileSystem: options.fileSystem,
+    sourceIdentity: options.sourceIdentity,
+    stagingRoot: options.publicationGroup.stagingRoot,
+    temporaryRoot,
+  });
+  await validateSameFileSystem(pairs, options.fileSystem);
+  const lease = await acquireToolcraftMotionReferenceLease(options);
+  const journalPath = getToolcraftMotionReferencePublicationJournalPath(
+    options.lockDirectory,
+  );
+  let result;
+  let operationError;
+  try {
+    await prepareTransactionRoot(
+      options.fileSystem, checkedRoot, transactionPaths.transactionRoot,
+    );
+    await recoverToolcraftMotionReferencePublication({
+      artifactRoot: checkedRoot,
+      fileSystem: options.fileSystem,
+      heartbeat: lease.heartbeat,
+      journalPath,
+      sourceIdentity: options.sourceIdentity,
+      temporaryRoot,
+      token: lease.token,
+      transactionRoot: transactionPaths.transactionRoot,
+    });
+    const discovery = await options.discoverPreviousGroup({
+      evidences: options.publicationGroup.evidences,
+      fileSystem: options.fileSystem,
+      referenceStudiesRoot: options.publicationGroup.referenceStudiesRoot,
+      rootDir: options.publicationGroup.rootDir,
+    });
+    const obsoleteDirectories = obsoleteFinalDirectories(
+      discovery, checkedRoot, new Set(boundStudyIds),
+    );
+    const transactionPairs = combineTransactionPairs(
+      pairs, obsoleteDirectories,
+    );
+    await validateSameFileSystem(transactionPairs, options.fileSystem);
+    result = await publishToolcraftMotionReferenceJournaledGroup({
+      fileSystem: options.fileSystem,
+      heartbeat: lease.heartbeat,
+      journalPath,
+      pairs: transactionPairs,
+      sourceIdentity: options.sourceIdentity,
+      token: lease.token,
+      transactionRoot: transactionPaths.transactionRoot,
+    });
+  } catch (error) {
+    operationError = error;
+  }
+  const unlockWarning = await lease.release();
+  if (unlockWarning && result) result.warnings.push(unlockWarning);
+  if (operationError) {
+    if (operationError.message?.startsWith("Toolcraft motion reference publication")) {
+      throw operationError;
+    }
+    fail(operationError.message, operationError);
+  }
+  return result;
+}