@taprootio/site-authoring 0.14.1 → 0.16.0

package/src/video-prepare.js

@@ -62,19 +62,10 @@ export async function prepareVideoFile({ filePath, byteLength, maxRemuxBytes = D
   const input = new Input({ source: new FilePathSource(filePath), formats: ALL_FORMATS });
   try {
     const format = await input.getFormat();
-    const video = await input.getPrimaryVideoTrack();
-    const audio = await input.getPrimaryAudioTrack();
-    // The server refuses a file if any track is not H.264 or AAC, so the first track that is
-    // not is the one declared and named. A track whose codec mediabunny cannot name is
-    // "unknown", not absent: an absent audio track is fine, a silent copy of one that exists is not.
-    const videoCodec = await declaredCodec(await input.getVideoTracks(), isH264);
-    const audioCodec = await declaredCodec(await input.getAudioTracks(), isAac);
+    const tracks = await inspectTracks(input);
+    const { video, audio, videoCodec, audioCodec, primaryVideo, primaryAudio } = tracks;
     const described = { videoCodec, audioCodec, remuxed: null };
-
-    // A remux keeps only the primary tracks, so they are what must be accepted for it to help.
-    const primaryVideo = video ? await codecOf(video) : "";
-    const primaryAudio = audio ? await codecOf(audio) : "";
-    if (!isH264(primaryVideo) || !isAac(primaryAudio)) return described;
+    if (!primariesAccepted(tracks)) return described;
 
     const everyTrackAccepted = isH264(videoCodec) && isAac(audioCodec);
     if (everyTrackAccepted && format.mimeType === "video/mp4" && await indexComesFirst(filePath)) return described;
@@ -97,6 +88,51 @@ export async function prepareVideoFile({ filePath, byteLength, maxRemuxBytes = D
   }
 }
 
+/**
+ * Whether the upload would send this file in a form the server accepts (TR00823): H.264
+ * and AAC as it is, or primaries a remux can keep within the ceiling. Read only; `plan`
+ * uses it so an unsupported video blocks before anything is uploaded. A remux that fails
+ * at upload time is still refused then.
+ */
+export async function videoAcceptance({ filePath, byteLength, maxRemuxBytes = DEFAULT_MAX_REMUX_BYTES }) {
+  const input = new Input({ source: new FilePathSource(filePath), formats: ALL_FORMATS });
+  try {
+    const tracks = await inspectTracks(input);
+    const { videoCodec, audioCodec } = tracks;
+    const asItIs = isH264(videoCodec) && isAac(audioCodec);
+    const remuxable = primariesAccepted(tracks) && byteLength <= maxRemuxBytes;
+    return { accepted: asItIs || remuxable, videoCodec, audioCodec };
+  } catch {
+    return { accepted: false, videoCodec: "", audioCodec: "" };
+  } finally {
+    input.dispose();
+  }
+}
+
+/**
+ * The server refuses a file if any track is not H.264 or AAC, so the first track that is
+ * not is the one declared and named. A track whose codec mediabunny cannot name is
+ * "unknown", not absent: an absent audio track is fine, a silent copy of one that exists is
+ * not. A remux keeps only the primary tracks, so they are what must be accepted for it to help.
+ */
+async function inspectTracks(input) {
+  const video = await input.getPrimaryVideoTrack();
+  const audio = await input.getPrimaryAudioTrack();
+  return {
+    video,
+    audio,
+    videoCodec: await declaredCodec(await input.getVideoTracks(), isH264),
+    audioCodec: await declaredCodec(await input.getAudioTracks(), isAac),
+    primaryVideo: video ? await codecOf(video) : "",
+    primaryAudio: audio ? await codecOf(audio) : "",
+  };
+}
+
+/** Whether a remux, which keeps only the primary tracks, would produce a file the server accepts. */
+function primariesAccepted({ primaryVideo, primaryAudio }) {
+  return isH264(primaryVideo) && isAac(primaryAudio);
+}
+
 async function codecOf(track) {
   return (await track.getCodecParameterString()) ?? "unknown";
 }

package/src/workspace-check.js

@@ -0,0 +1,199 @@
+import { sharedThemeContextNames } from "./content/free-form-sections.js";
+import { readAppearanceWorkspaceContext, FOOTER_SETTINGS_FILE } from "./footer-workspace.js";
+import { readWorkspaceForms } from "./forms-workspace.js";
+import { CHECK_AREA, collectProblems, problemsOf } from "./problems.js";
+import { normalizeRedirectPath, REDIRECTS_FILE_NAME, validateRedirectsDocument } from "./redirects-contract.js";
+import { SETTINGS_TYPE_SITE_PUBLISHING_PREFERENCES } from "./settings-catalog.js";
+import { validateFooterWorkspaceDocument } from "./verbs/footer-push.js";
+import { validateNavigationWorkspaceDocument } from "./verbs/nav-push.js";
+import { planPages } from "./verbs/pages-push.js";
+import { validateThemeWorkspace } from "./verbs/theme-push.js";
+import {
+  NAVIGATION_FILE_NAME,
+  normalizePagePath,
+  PAGE_SOURCE_EXTENSIONS,
+  PAGES_DIRECTORY,
+  readWorkspaceJson,
+  walkWorkspaceFiles,
+  WORKSPACE_LIMITS,
+  workspaceFileExists,
+} from "./workspace.js";
+import { SiteAuthoringError } from "./errors.js";
+
+const ALL_AREAS = new Set(Object.values(CHECK_AREA));
+
+/**
+ * The one validator for a whole authoring workspace (TR01002, TR00823).
+ *
+ * `validate` runs it over a pulled workspace or an offline fixture, and `plan`
+ * runs it before reading the site. Each area is checked by the function its
+ * push uses — `planPages` for pages, `validateThemeWorkspace` for the theme —
+ * so a workspace that passes here is not refused for its shape by a push.
+ *
+ * Every area is checked, and every page, whatever an earlier one found: the
+ * result lists all the problems, and no area's refusal hides another's.
+ *
+ * `binding` says what the workspace's references may point at. A pulled
+ * workspace binds the images and pages its manifests record (or, for `plan`,
+ * the site's live pages); a fixture binds the identities its own manifest
+ * declares.
+ */
+export async function checkWorkspace({
+  workspaceDir,
+  siteId,
+  manifest,
+  mediaManifest,
+  binding = {},
+  content,
+  areas = ALL_AREAS,
+  onProgress = () => {},
+}) {
+  const problems = [];
+  // A managed Docs site's workspace holds its settings only (TR00790).
+  const settingsOnly = manifest.authoringSurface !== undefined;
+  const pageIds = binding.pageIds
+    ?? new Set(manifest.pages.map((entry) => entry?.pageId).filter((value) => typeof value === "string"));
+  const resourceIds = binding.resourceIds
+    ?? new Set(manifest.pages.map((entry) => entry?.resourceId).filter((value) => typeof value === "string"));
+  // Navigation may be judged against other pages than the footer: `plan`
+  // checks it against the site's live pages, which is what nav push does,
+  // while footer push resolves its targets from the pull manifest.
+  const navigationPages = binding.navigationPages ?? { pageIds, resourceIds };
+  const knownImageIds = binding.knownImageIds
+    ?? await collectProblems(problems, { area: CHECK_AREA.presentation }, async () =>
+      (await readAppearanceWorkspaceContext(workspaceDir, siteId)).knownImageIds);
+
+  onProgress("Validating the complete light/dark theme, appearance, header, brand, and footer colors.");
+  const presentation = knownImageIds === undefined
+    ? undefined
+    : await collectProblems(problems, { area: CHECK_AREA.presentation }, async () =>
+      await validateThemeWorkspace(workspaceDir, siteId, knownImageIds, binding.settingsEntityIds));
+
+  onProgress("Validating the closed footer document and its page and image targets.");
+  const footer = knownImageIds === undefined
+    ? undefined
+    : await collectProblems(problems, { area: CHECK_AREA.footer, file: FOOTER_SETTINGS_FILE }, async () =>
+      validateFooterWorkspaceDocument(
+        await readWorkspaceJson(workspaceDir, FOOTER_SETTINGS_FILE, WORKSPACE_LIMITS.settingsBytes),
+        siteId,
+        {
+          expectedEntityId: binding.settingsEntityIds?.get(SETTINGS_TYPE_SITE_PUBLISHING_PREFERENCES),
+          knownPageResourceIds: resourceIds,
+          knownImageIds,
+        },
+      ));
+
+  if (settingsOnly) return { problems, settingsOnly, presentation, footer };
+
+  onProgress("Validating navigation shape and its page targets.");
+  const navigation = await collectProblems(problems, { area: CHECK_AREA.navigation, file: NAVIGATION_FILE_NAME }, async () =>
+    validateNavigationWorkspaceDocument(
+      await readWorkspaceJson(workspaceDir, NAVIGATION_FILE_NAME, WORKSPACE_LIMITS.navigationBytes),
+      siteId,
+      navigationPages,
+    ));
+
+  // `plan` checks pages against the live site instead (TR00823).
+  const checkPages = areas.has(CHECK_AREA.pages);
+  if (checkPages) onProgress("Validating every page source.");
+  const pages = !checkPages ? undefined : await collectProblems(problems, { area: CHECK_AREA.pages }, async () =>
+    await planPages({
+      workspaceDir,
+      siteId,
+      manifest,
+      mediaManifest,
+      content,
+      files: await walkWorkspaceFiles(workspaceDir, PAGES_DIRECTORY, PAGE_SOURCE_EXTENSIONS),
+      // The pages the manifest records are what the site held when it was
+      // pulled, which is what an offline check can know.
+      livePages: manifest.pages.filter((entry) => typeof entry?.pageId === "string"),
+      online: false,
+      // The theme was validated above, so its contexts come from it; reading
+      // the styles file again would re-check a binding a fixture spells
+      // differently, and repeat a theme problem once per page.
+      getSharedThemeContexts: async () =>
+        presentation === undefined
+          ? undefined
+          : sharedThemeContextNames(presentation.style.lightTheme, presentation.style.darkTheme),
+      onProgress,
+    }));
+  if (pages !== undefined) problems.push(...pages.problems);
+
+  // `plan` leaves redirects and forms to their own pushes.
+  if (areas.has(CHECK_AREA.redirects)) {
+    onProgress("Validating the redirect map: paths, targets, statuses, chains, loops, and page paths.");
+  }
+  const redirects = !areas.has(CHECK_AREA.redirects)
+    ? undefined
+    : await collectProblems(problems, { area: CHECK_AREA.redirects, file: REDIRECTS_FILE_NAME }, async () =>
+      await checkRedirects(workspaceDir, siteId, manifest, pages?.sources));
+
+  const forms = !areas.has(CHECK_AREA.forms)
+    ? undefined
+    : await collectProblems(problems, { area: CHECK_AREA.forms }, async () => await readWorkspaceForms(workspaceDir));
+  if (forms?.length > 0) {
+    onProgress(`Validated ${forms.length} form file${forms.length === 1 ? "" : "s"} against the shared field schema.`);
+  }
+
+  return { problems, settingsOnly, presentation, footer, navigation, pages, redirects, forms };
+}
+
+/**
+ * The redirect map's own rules, and the one rule the workspace can check
+ * against itself: a source a workspace page occupies is refused by the site.
+ * Paths compare case-insensitively, as the site's citext path column does,
+ * and a refusal names the entry where the author wrote it: the validated map
+ * is sorted by path, so its index is not the document's.
+ */
+async function checkRedirects(workspaceDir, siteId, manifest, sources) {
+  if (!await workspaceFileExists(workspaceDir, REDIRECTS_FILE_NAME)) {
+    // A workspace pulled before redirects existed holds no map, and nothing in
+    // it asks for one.
+    if (manifest.redirects === undefined) return undefined;
+    throw new SiteAuthoringError(
+      "redirects.file_missing",
+      `No ${REDIRECTS_FILE_NAME} was found, but the pull manifest records a redirect baseline. Run 'taproot-site `
+        + "redirects pull' again.",
+      { field: REDIRECTS_FILE_NAME },
+    );
+  }
+  const document_ = await readWorkspaceJson(workspaceDir, REDIRECTS_FILE_NAME, WORKSPACE_LIMITS.redirectsBytes);
+  const redirects = validateRedirectsDocument(document_, siteId);
+  // The pages the next push leaves on the site: each workspace source at the
+  // path it declares, and every recorded page no source here claims. A page a
+  // source renames frees its old path for a redirect.
+  const claimed = new Set((sources ?? []).map((source) => source.file));
+  const occupied = [
+    ...(sources ?? []).map((source) => source.pagePath),
+    ...manifest.pages
+      .filter((entry) => typeof entry?.path === "string" && !claimed.has(entry.file))
+      .map((entry) => normalizePagePath(entry.path)),
+  ];
+  const pagePaths = new Set(
+    occupied.filter((pagePath) => pagePath !== undefined)
+      .map((pagePath) => (`/${pagePath}`.replace(/\/+$/u, "") || "/").toLowerCase()),
+  );
+  const authoredIndexByPath = new Map();
+  for (const [index, entry] of document_.entries.entries()) {
+    const path = normalizeRedirectPath(entry?.path);
+    if (path !== undefined && !authoredIndexByPath.has(path.toLowerCase())) {
+      authoredIndexByPath.set(path.toLowerCase(), index);
+    }
+  }
+  const problems = [];
+  for (const entry of redirects.entries) {
+    const key = entry.path.toLowerCase();
+    if (!pagePaths.has(key)) continue;
+    problems.push(...problemsOf(new SiteAuthoringError(
+      "redirects.path_occupied",
+      `Redirect source '${entry.path}' is a page in this workspace, so it cannot also be a redirect source.`,
+      { field: `entries[${authoredIndexByPath.get(key) ?? 0}].path` },
+    ), { area: CHECK_AREA.redirects, file: REDIRECTS_FILE_NAME }));
+  }
+  if (problems.length > 0) {
+    const [first] = problems;
+    const error = new SiteAuthoringError(first.code, first.message, { field: first.field });
+    throw problems.length === 1 ? error : error.withProblems(problems);
+  }
+  return redirects;
+}

package/src/workspace-ledger.js

@@ -0,0 +1,173 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+import { createHash } from "node:crypto";
+
+import { SiteAuthoringError } from "./errors.js";
+import { planStale } from "./problems.js";
+
+/**
+ * What `plan` read from the workspace, so `apply` can send nothing else
+ * (TR00823).
+ *
+ * Every workspace read goes through `workspace.js`, which reports it here.
+ * While `plan` (or `apply`'s re-plan) runs, each read is recorded: a file's
+ * digest, a stream's size and modification time, whether a path exists, what a
+ * directory walk listed. While `apply`'s steps run, each read is compared with
+ * that record. A read the plan never made, or one whose answer changed, stops
+ * the step before it writes, because whatever it was about to send was never
+ * reviewed. A file that `apply` itself writes, such as the media manifest the
+ * upload step records, replaces its own entry, so the next step reads it as
+ * expected.
+ *
+ * The ledger guards the workspace only. What lives on the site (page
+ * revisions, the navigation tree) is checked by the steps against the plan.
+ */
+
+const scope = new AsyncLocalStorage();
+
+const MISSING = "missing";
+
+export function digest(bytes) {
+  return createHash("sha256").update(bytes).digest("hex");
+}
+
+/** Runs `action` while recording every workspace read; returns its result and the ledger. */
+export async function recordWorkspaceReads(action) {
+  const ledger = { mode: "record", entries: new Map() };
+  const result = await scope.run(ledger, action);
+  return { result, ledger: Object.freeze({ entries: ledger.entries }) };
+}
+
+/** Runs `action` refusing any workspace read the ledger does not hold. */
+export async function enforceWorkspaceReads(recorded, action) {
+  const ledger = { mode: "enforce", entries: new Map(recorded.entries), written: new Set(), deleted: new Set() };
+  return await scope.run(ledger, action);
+}
+
+/** A stable digest of everything the ledger recorded, for the plan hash. */
+export function ledgerDigest(recorded) {
+  const entries = [...recorded.entries.entries()].sort(([left], [right]) => (left < right ? -1 : 1));
+  return digest(JSON.stringify(entries));
+}
+
+function stale(path, mode) {
+  // While planning, the plan itself is what moved; while applying, the step.
+  return mode === "record"
+    ? new SiteAuthoringError(
+      "plan.workspace_changed",
+      `'${path}' changed while the plan was reading the workspace. Wait for whatever is writing it to finish, then `
+        + "run 'taproot-site plan' again.",
+      { field: path },
+    )
+    : planStale(`'${path}'`, path);
+}
+
+function key(kind, path) {
+  return `${kind}:${path}`;
+}
+
+/**
+ * One read's answer. While recording, the first answer for a key is kept; a
+ * different later answer means the workspace moved while it was being planned.
+ * While enforcing, the answer must match what was recorded.
+ */
+function noteRead(kind, path, value) {
+  const ledger = scope.getStore();
+  if (ledger === undefined) return;
+  const entry = key(kind, path);
+  const recorded = ledger.entries.get(entry);
+  if (ledger.mode === "record") {
+    if (recorded === undefined) ledger.entries.set(entry, value);
+    else if (recorded !== value) throw stale(path, "record");
+    return;
+  }
+  if (recorded === value) return;
+  if (recorded === undefined && impliedByRecordedRead(ledger, kind, path, value)) return;
+  throw stale(path, "enforce");
+}
+
+/**
+ * Whether the plan already answered this question another way. A file the plan
+ * read was a regular file, and one it found missing did not exist; a step that
+ * asks only whether it exists, or what kind of entry it is, learns nothing the
+ * plan did not see. The bytes themselves are still compared when the step reads
+ * them.
+ */
+function impliedByRecordedRead(ledger, kind, path, value) {
+  if (kind !== "exists" && kind !== "entry") return false;
+  const read = ledger.entries.get(key("file", path)) ?? ledger.entries.get(key("stream", path));
+  if (read === undefined) return false;
+  const present = read !== MISSING;
+  return kind === "exists" ? value === present : value === (present ? "file" : MISSING);
+}
+
+// The digests are computed only inside a ledger: every other verb reads
+// without paying for them.
+export function noteFileRead(path, bytes) {
+  if (scope.getStore() !== undefined) noteRead("file", path, digest(bytes));
+}
+
+export function noteFileMissing(path) {
+  noteRead("file", path, MISSING);
+}
+
+export function noteStreamOpened(path, byteLength, modifiedMilliseconds, header) {
+  if (scope.getStore() !== undefined) noteRead("stream", path, `${byteLength}:${modifiedMilliseconds}:${digest(header)}`);
+}
+
+export function noteStreamMissing(path) {
+  noteRead("stream", path, MISSING);
+}
+
+export function noteExists(path, exists) {
+  noteRead("exists", path, exists);
+}
+
+export function noteEntry(path, kind) {
+  noteRead("entry", path, kind);
+}
+
+/**
+ * A directory walk. While enforcing, the listing must be the recorded one,
+ * plus the files this apply wrote under that root and minus those it removed.
+ */
+export function noteWalk(root, filter, files) {
+  const ledger = scope.getStore();
+  if (ledger === undefined) return;
+  const entry = key("walk", `${root}|${filter}`);
+  if (ledger.mode === "record") {
+    noteRead("walk", `${root}|${filter}`, JSON.stringify(files));
+    return;
+  }
+  const recorded = ledger.entries.get(entry);
+  if (recorded === undefined) throw stale(root, "enforce");
+  const expected = new Set(JSON.parse(recorded));
+  const prefix = `${root}/`;
+  for (const path of ledger.written) if (path.startsWith(prefix) && files.includes(path)) expected.add(path);
+  for (const path of ledger.deleted) if (path.startsWith(prefix)) expected.delete(path);
+  const listed = new Set(files);
+  // Name the first file that appeared or went missing, not just the directory.
+  const changed = files.find((path) => !expected.has(path)) ?? [...expected].sort().find((path) => !listed.has(path));
+  if (changed !== undefined) throw stale(changed, "enforce");
+}
+
+/** A file this apply wrote: later reads of it must find exactly these bytes. */
+export function noteFileWritten(path, contents) {
+  const ledger = scope.getStore();
+  if (ledger === undefined || ledger.mode !== "enforce") return;
+  ledger.entries.set(key("file", path), digest(contents));
+  ledger.entries.set(key("exists", path), true);
+  ledger.entries.set(key("entry", path), "file");
+  ledger.written.add(path);
+  ledger.deleted.delete(path);
+}
+
+/** A file this apply removed. */
+export function noteFileDeleted(path) {
+  const ledger = scope.getStore();
+  if (ledger === undefined || ledger.mode !== "enforce") return;
+  ledger.entries.set(key("file", path), MISSING);
+  ledger.entries.set(key("exists", path), false);
+  ledger.entries.set(key("entry", path), MISSING);
+  ledger.deleted.add(path);
+  ledger.written.delete(path);
+}