@taprootio/site-authoring 0.12.6 → 0.13.0

package/src/verbs/validate.js

@@ -15,6 +15,8 @@ import {
 } from "../fixture-contract.js";
 import { initializeFixture } from "../fixture-init.js";
 import { appearanceManifestEntry, FOOTER_SETTINGS_FILE, footerManifestEntry } from "../footer-workspace.js";
+import { readWorkspaceForms } from "../forms-workspace.js";
+import { FORMS_DIRECTORY } from "../forms-contract.js";
 import {
   isRedirectMapRevision,
   normalizeRedirectPath,
@@ -41,7 +43,13 @@ import { validateFooterWorkspaceDocument } from "./footer-push.js";
 import { validateNavigationWorkspaceDocument } from "./nav-push.js";
 import { validateWorkspacePageDocument, validateWorkspacePageSource } from "./pages-push.js";
 import { validateThemeWorkspace } from "./theme-push.js";
-import { documentImageIds, documentTemplate, isAuthorableTemplateType, PAGE_TEMPLATES } from "../typed-pages.js";
+import {
+  documentImageIds,
+  documentTemplate,
+  isAuthorableTemplateType,
+  PAGE_TEMPLATES,
+  placedVideoIds,
+} from "../typed-pages.js";
 
 const MAXIMUM_REPORTED = 200;
 const FIXTURE_ROOT_KEYS = new Set(FIXTURE_ROOT_FIELDS);
@@ -192,6 +200,25 @@ function validateFixtureManifest(manifest) {
   if (imageIds.size !== manifest.fixture.imageIds.length) {
     fail("fixture.image_ids_invalid", "fixture.imageIds must not contain duplicates.", "fixture.imageIds");
   }
+  // Optional: a fixture that places no video omits the list. Whatever is listed
+  // is a `ready` video a page may place, the offline stand-in for the library
+  // read a live push makes.
+  const declaredVideoIds = manifest.fixture.videoIds ?? [];
+  if (
+    !Array.isArray(declaredVideoIds)
+    || declaredVideoIds.length > WORKSPACE_LIMITS.files
+    || declaredVideoIds.some((value) => !isCanonicalUuid(value))
+  ) {
+    fail(
+      "fixture.video_ids_invalid",
+      "fixture.videoIds must be a bounded list of deterministic canonical UUIDs.",
+      "fixture.videoIds",
+    );
+  }
+  const videoIds = new Set(declaredVideoIds);
+  if (videoIds.size !== declaredVideoIds.length) {
+    fail("fixture.video_ids_invalid", "fixture.videoIds must not contain duplicates.", "fixture.videoIds");
+  }
   if (
     !Array.isArray(manifest.fixture.deliveryOrigins)
     || manifest.fixture.deliveryOrigins.length > FIXTURE_MAXIMUM_DELIVERY_ORIGINS
@@ -315,7 +342,7 @@ function validateFixtureManifest(manifest) {
     files.add(entry.file);
     paths.add(pagePath);
   }
-  return { manifest, imageIds, deliveryOrigins, pageIds, resourceIds, files, settingsEntityIds };
+  return { manifest, imageIds, videoIds, deliveryOrigins, pageIds, resourceIds, files, settingsEntityIds };
 }
 
 function validateDeliveryUrl(value, field, deliveryOrigins) {
@@ -405,6 +432,57 @@ function headerWidthHints(headerWidth, rootBandPages) {
   }];
 }
 
+/**
+ * The fixture's forms, when it carries any. The manifest binding and the files
+ * must name the same keys: a form file the manifest does not mention would be
+ * validated by no one, and a key with no file would be a baseline for nothing.
+ * Returns undefined for a fixture with no forms.
+ */
+async function validateFixtureForms(fixtureRoot, bound, onProgress) {
+  const files = await readWorkspaceForms(fixtureRoot);
+  if (bound === undefined) {
+    if (files.length > 0) {
+      fail(
+        "fixture.forms_unbound",
+        "The fixture carries form files but its manifest does not bind them in 'forms'.",
+        "forms",
+      );
+    }
+    return undefined;
+  }
+  const items = isPlainObject(bound) && isPlainObject(bound.items) ? bound.items : undefined;
+  const keys = items === undefined ? [] : Object.keys(items).sort();
+  if (
+    items === undefined
+    || Object.keys(bound).length !== 1
+    || keys.length === 0
+    || keys.some((key) =>
+      !isPlainObject(items[key])
+      || Object.keys(items[key]).sort().join() !== "id,version"
+      || !isCanonicalUuid(items[key].id)
+      || !Number.isSafeInteger(items[key].version)
+      || items[key].version < 1
+    )
+  ) {
+    fail(
+      "fixture.forms_invalid",
+      "The fixture manifest's 'forms' must be { items: { <key>: { id, version } } } with at least one form, "
+        + "a canonical UUID id, and a positive version.",
+      "forms",
+    );
+  }
+  const fileKeys = files.map((form) => form.key);
+  if (fileKeys.join() !== keys.join()) {
+    fail(
+      "fixture.forms_mismatch",
+      `The fixture manifest binds forms [${keys.join(", ")}] but ${FORMS_DIRECTORY}/ holds [${fileKeys.join(", ")}].`,
+      "forms",
+    );
+  }
+  onProgress(`Validated ${files.length} form file${files.length === 1 ? "" : "s"} against the shared field schema.`);
+  return files;
+}
+
 export async function validateFixture(invocation = {}) {
   if (invocation.init) return await initializeFixture(invocation, validateFixture);
   const onProgress = typeof invocation.onProgress === "function" ? invocation.onProgress : () => {};
@@ -416,6 +494,7 @@ export async function validateFixture(invocation = {}) {
   const {
     manifest,
     imageIds,
+    videoIds,
     deliveryOrigins,
     pageIds,
     resourceIds,
@@ -521,6 +600,14 @@ export async function validateFixture(invocation = {}) {
         file,
       );
     }
+    const unknownVideo = [...placedVideoIds(page.document)].find((videoId) => !videoIds.has(videoId));
+    if (unknownVideo !== undefined) {
+      fail(
+        "fixture.video_reference_unknown",
+        `Fixture video '${unknownVideo}' is not declared in fixture.videoIds, so it is not a ready video of this site.`,
+        file,
+      );
+    }
     validatedPages.push({ file, path: page.pagePath });
     if (pageUsesRootBand(page.document)) rootBandPages.push(file);
   }
@@ -597,6 +684,8 @@ export async function validateFixture(invocation = {}) {
     },
   );
 
+  const forms = await validateFixtureForms(fixtureRoot, manifest.forms, onProgress);
+
   const reportedPages = boundedList(validatedPages, MAXIMUM_REPORTED);
   onProgress(
     `Validated ${validatedPages.length} page(s), ${navigation.items} navigation item(s), two themes, appearance, and footer without credentials or mutation.`,
@@ -607,6 +696,7 @@ export async function validateFixture(invocation = {}) {
       contractVersion: FIXTURE_CONTRACT_VERSION,
       manifest: FIXTURE_MANIFEST_FILE_NAME,
       imageIds: imageIds.size,
+      videoIds: videoIds.size,
       deliveryOrigins: deliveryOrigins.size,
     },
     validated: {
@@ -620,6 +710,7 @@ export async function validateFixture(invocation = {}) {
         entries: redirects.entries.length,
         gone: redirects.entries.filter((entry) => entry.kind === REDIRECT_KIND_GONE).length,
       },
+      ...(forms === undefined ? {} : { forms: { count: forms.length } }),
       themes: 2,
       appearanceSettings: presentation.scalarOperations.length,
       footer: true,
@@ -636,6 +727,7 @@ export async function validateFixture(invocation = {}) {
       "navigation shape and local page-resource references",
       "redirect-map shape, normalization, chains, loops, and fixture-local path occupancy",
       "complete theme, appearance, header, brand, and footer semantics",
+      "form files, when the manifest binds any, against the shared field schema",
       "fixture-local image identities and reserved delivery origins",
     ],
     doesNotProve: [

package/src/video-metadata.js

@@ -0,0 +1,90 @@
+import { SiteAuthoringError } from "./errors.js";
+
+/**
+ * Container sniffing for video uploads in `media upload`.
+ *
+ * Like the image sniffer this reads a fixed, bounded prefix and nothing else:
+ * it names the container (MP4, MOV or WebM) so `video-prepare.js` can read the
+ * file, and leaves the real validation to that read and to the API, which
+ * accepts only an H.264 and AAC MP4. A file that is not one of the three is not
+ * claimed here, so the caller falls through to the image path and its own
+ * refusal.
+ */
+
+export const VIDEO_CONTENT_TYPES = Object.freeze({
+  mp4: "video/mp4",
+  quicktime: "video/quicktime",
+  webm: "video/webm",
+});
+
+/** The extensions a directory walk treats as video candidates; the container decides. */
+export const VIDEO_EXTENSIONS = Object.freeze([".mp4", ".mov", ".webm"]);
+
+/**
+ * The major brands of an ISO base media file that are an MP4 video. The `ftyp`
+ * box is shared with HEIC, HEIF, AVIF, M4A and 3GP, none of which are videos
+ * this upload accepts, so the brand decides rather than the box alone.
+ */
+const MP4_BRANDS = Object.freeze([
+  "isom",
+  "iso2",
+  "iso3",
+  "iso4",
+  "iso5",
+  "iso6",
+  "mp41",
+  "mp42",
+  "avc1",
+  "dash",
+  "M4V ",
+  "M4VH",
+  "M4VP",
+  "MSNV",
+  "f4v ",
+]);
+const QUICKTIME_BRAND = "qt  ";
+
+const EBML_MAGIC = [0x1a, 0x45, 0xdf, 0xa3];
+// The EBML header carries its DocType within the first few dozen bytes.
+const EBML_DOCTYPE_WINDOW = 64;
+
+function startsWithAscii(bytes, text, offset) {
+  if (bytes.byteLength < offset + text.length) return false;
+  for (let index = 0; index < text.length; index += 1) {
+    if (bytes[offset + index] !== text.charCodeAt(index)) return false;
+  }
+  return true;
+}
+
+function containsAscii(bytes, text, limit) {
+  const end = Math.min(bytes.byteLength, limit) - text.length;
+  for (let offset = 0; offset <= end; offset += 1) {
+    if (startsWithAscii(bytes, text, offset)) return true;
+  }
+  return false;
+}
+
+/**
+ * The video content type for a file's leading bytes, or `undefined` when they
+ * are not an MP4, MOV or WebM container (an ISO base media file with another
+ * brand, such as HEIC or M4A, is not one). A Matroska file shares WebM's EBML
+ * header but is not accepted, so the DocType decides.
+ */
+export function sniffVideoContentType(header) {
+  if (header.byteLength >= 12 && startsWithAscii(header, "ftyp", 4)) {
+    if (startsWithAscii(header, QUICKTIME_BRAND, 8)) return VIDEO_CONTENT_TYPES.quicktime;
+    return MP4_BRANDS.some((brand) => startsWithAscii(header, brand, 8)) ? VIDEO_CONTENT_TYPES.mp4 : undefined;
+  }
+  if (header.byteLength >= 4 && EBML_MAGIC.every((byte, index) => header[index] === byte)) {
+    return containsAscii(header, "webm", EBML_DOCTYPE_WINDOW) ? VIDEO_CONTENT_TYPES.webm : undefined;
+  }
+  return undefined;
+}
+
+export function videoTooLarge(fileName, maximumBytes) {
+  return new SiteAuthoringError(
+    "media.video_too_large",
+    `'${fileName}' is larger than the ${Math.round(maximumBytes / (1024 * 1024 * 1024))} GiB a single video upload accepts.`,
+    { field: fileName },
+  );
+}

package/src/video-prepare.js

@@ -0,0 +1,136 @@
+import { open } from "node:fs/promises";
+
+import { ALL_FORMATS, BufferTarget, Conversion, FilePathSource, Input, Mp4OutputFormat, Output } from "mediabunny";
+
+/**
+ * Prepares a video for `media upload` the way the Videos page does: read the container
+ * and codecs, rewrite an H.264 and AAC file that is not an MP4 with its index first as
+ * one (copying the tracks, never re-encoding), and declare the real codec strings so
+ * the server decides what is accepted. Anything that is not H.264 and AAC is left
+ * alone and declared as it is, so the server's refusal names what the file actually is.
+ */
+
+/** The most any licence's upload cap allows; a larger file is described, not remuxed. */
+export const DEFAULT_MAX_REMUX_BYTES = 250 * 1024 * 1024;
+
+const MAX_TOP_LEVEL_BOXES = 64;
+
+export function isH264(codec) {
+  return /^avc[13]/iu.test(codec);
+}
+
+/** No audio track (an empty string) is allowed. */
+export function isAac(codec) {
+  return codec === "" || /^mp4a\.(40\.(1|2|3|4|5|29)|6[678])$/iu.test(codec);
+}
+
+export function mp4Name(name) {
+  return /\.mp4$/iu.test(name) ? name : `${name.replace(/\.[^./\\]+$/u, "")}.mp4`;
+}
+
+/** Whether `moov` precedes `mdat` among a file's top-level boxes, read with a few small reads. */
+export async function indexComesFirst(filePath) {
+  const handle = await open(filePath, "r");
+  try {
+    const { size: fileSize } = await handle.stat();
+    const header = Buffer.alloc(16);
+    let offset = 0;
+    for (let boxes = 0; boxes < MAX_TOP_LEVEL_BOXES && offset < fileSize; boxes += 1) {
+      const { bytesRead } = await handle.read(header, 0, 16, offset);
+      if (bytesRead < 8) return false;
+      const type = header.toString("latin1", 4, 8);
+      if (type === "moov") return true;
+      if (type === "mdat") return false;
+      let size = header.readUInt32BE(0);
+      if (size === 1 && bytesRead >= 16) size = Number(header.readBigUInt64BE(8));
+      else if (size === 0) return false;
+      if (!Number.isFinite(size) || size < 8) return false;
+      offset += size;
+    }
+    return false;
+  } finally {
+    await handle.close();
+  }
+}
+
+/**
+ * @param {{ filePath: string, byteLength: number, maxRemuxBytes?: number }} file
+ * @returns {Promise<{ videoCodec: string, audioCodec: string, remuxed: Buffer | null }>}
+ *   `remuxed` is the rewritten MP4 when one was needed and possible, else null (upload the file as it is).
+ */
+export async function prepareVideoFile({ filePath, byteLength, maxRemuxBytes = DEFAULT_MAX_REMUX_BYTES }) {
+  const input = new Input({ source: new FilePathSource(filePath), formats: ALL_FORMATS });
+  try {
+    const format = await input.getFormat();
+    const video = await input.getPrimaryVideoTrack();
+    const audio = await input.getPrimaryAudioTrack();
+    // The server refuses a file if any track is not H.264 or AAC, so the first track that is
+    // not is the one declared and named. A track whose codec mediabunny cannot name is
+    // "unknown", not absent: an absent audio track is fine, a silent copy of one that exists is not.
+    const videoCodec = await declaredCodec(await input.getVideoTracks(), isH264);
+    const audioCodec = await declaredCodec(await input.getAudioTracks(), isAac);
+    const described = { videoCodec, audioCodec, remuxed: null };
+
+    // A remux keeps only the primary tracks, so they are what must be accepted for it to help.
+    const primaryVideo = video ? await codecOf(video) : "";
+    const primaryAudio = audio ? await codecOf(audio) : "";
+    if (!isH264(primaryVideo) || !isAac(primaryAudio)) return described;
+
+    const everyTrackAccepted = isH264(videoCodec) && isAac(audioCodec);
+    if (everyTrackAccepted && format.mimeType === "video/mp4" && await indexComesFirst(filePath)) return described;
+    if (byteLength > maxRemuxBytes) return described;
+
+    try {
+      const remuxed = await remux(input, { video: video?.id, audio: audio?.id });
+      // The rewrite holds only the primaries, so those are the codecs the server will see.
+      return remuxed ? { videoCodec: primaryVideo, audioCodec: primaryAudio, remuxed } : described;
+    } catch {
+      // A remux that fails leaves the real codecs on the original, so the server's
+      // structure refusal names the real problem.
+      return described;
+    }
+  } catch {
+    // An unreadable file is declared with no codecs; the server answers in its own words.
+    return { videoCodec: "", audioCodec: "", remuxed: null };
+  } finally {
+    input.dispose();
+  }
+}
+
+async function codecOf(track) {
+  return (await track.getCodecParameterString()) ?? "unknown";
+}
+
+/** The first codec that is not accepted, else the first track's, else none. */
+async function declaredCodec(tracks, accepts) {
+  let first = "";
+  for (const track of tracks) {
+    const codec = await codecOf(track);
+    if (!accepts(codec)) return codec;
+    first ||= codec;
+  }
+  return first;
+}
+
+async function remux(input, primary) {
+  const target = new BufferTarget();
+  const output = new Output({ format: new Mp4OutputFormat({ fastStart: "in-memory" }), target });
+  // Copy only: a track that cannot be copied is dropped, never re-encoded.
+  const conversion = await Conversion.init({
+    input,
+    output,
+    copy: { mode: "forced", shiftTolerance: Number.POSITIVE_INFINITY },
+    // Only the primary picture and sound go in; an extra track is dropped, not copied.
+    video: (track) => (track.id === primary.video ? {} : { discard: true }),
+    audio: (track) => (track.id === primary.audio ? {} : { discard: true }),
+  });
+  const kept = conversion.utilizedTracks;
+  if (
+    !conversion.isValid
+    || !kept.some((track) => track.isVideoTrack())
+    || (primary.audio !== undefined && !kept.some((track) => track.isAudioTrack()))
+  ) return null;
+
+  await conversion.execute();
+  return target.buffer ? Buffer.from(target.buffer) : null;
+}

package/src/workspace.js

@@ -270,6 +270,10 @@ export const WORKSPACE_LIMITS = Object.freeze({
   settingsBytes: 256 * 1024,
   manifestBytes: 8 * 1024 * 1024,
   mediaBytes: 32 * 1024 * 1024,
+  // A video is streamed unless it must be remuxed: this mirrors the API's own
+  // single-PUT ceiling (`VideoUploadLimits.MaxSingleUploadMb`, 5 GiB). The licence's
+  // upload cap, which the server enforces, is far lower.
+  videoBytes: 5 * 1024 * 1024 * 1024,
   // Pull holds the bodies of the pages this workspace already tracks until it
   // knows whether any of them conflict, because a refusal has to leave `pages/`
   // untouched. Generous for any real authoring workspace, and an explicit
@@ -760,6 +764,106 @@ export async function readWorkspaceFile(workspaceDir, relativePath, maximumBytes
   }
 }
 
+/** How much of a file's start `openWorkspaceFileStream` hands back for container sniffing. */
+const STREAM_HEADER_BYTES = 4096;
+
+/**
+ * Opens one workspace file for streaming under the same discipline as
+ * `readWorkspaceFile` — no link anywhere on the way, a regular file, a byte
+ * bound — without reading it into memory. Returns its size, a bounded prefix
+ * for container sniffing, and a factory for a fresh read stream, because an
+ * upload that is retried reads the file again from the start. The caller must
+ * `close()`.
+ */
+export async function openWorkspaceFileStream(workspaceDir, relativePath, maximumBytes) {
+  const filePath = resolveWorkspacePath(workspaceDir, relativePath);
+  if (!await requireRealDirectoryChain(workspaceDir, relativePath)) {
+    throw new SiteAuthoringError(
+      "workspace.file_missing",
+      `Workspace file '${relativePath}' does not exist.`,
+      { field: relativePath },
+    );
+  }
+  let handle;
+  try {
+    handle = await open(filePath, fsConstants.O_RDONLY | (fsConstants.O_NOFOLLOW ?? 0));
+    const stats = await handle.stat({ bigint: true });
+    if (!stats.isFile()) {
+      throw new SiteAuthoringError(
+        "workspace.not_regular",
+        `Workspace file '${relativePath}' is not a regular file.`,
+        { field: relativePath },
+      );
+    }
+    if (stats.size > BigInt(maximumBytes)) {
+      throw new SiteAuthoringError(
+        "workspace.file_too_large",
+        `Workspace file '${relativePath}' exceeds ${maximumBytes} bytes.`,
+        { field: relativePath },
+      );
+    }
+    const byteLength = Number(stats.size);
+    const modifiedMilliseconds = Math.floor(Number(stats.mtimeMs));
+    const header = Buffer.allocUnsafe(Math.min(byteLength, STREAM_HEADER_BYTES));
+    let total = 0;
+    while (total < header.byteLength) {
+      const { bytesRead } = await handle.read(header, total, header.byteLength - total, total);
+      if (bytesRead === 0) break;
+      total += bytesRead;
+    }
+    const opened = handle;
+    return {
+      byteLength,
+      modifiedMilliseconds,
+      header: header.subarray(0, total),
+      // Exactly the size that was inspected. A file that shrinks under the
+      // upload fails the signed length check; one that grows would not, since
+      // only its first `byteLength` bytes are read, so `verifyUnchanged` is how
+      // the caller proves what it sent is still the file on disk.
+      stream: () => opened.createReadStream({ start: 0, end: byteLength - 1, autoClose: false }),
+      verifyUnchanged: async () => {
+        // The held handle catches an in-place append or rewrite; the path is
+        // checked too, because an editor that saves by renaming a new file over
+        // the old one leaves the handle looking untouched.
+        const current = await opened.stat({ bigint: true });
+        const onDisk = await lstat(filePath, { bigint: true }).catch(() => undefined);
+        if (
+          current.size !== stats.size
+          || current.mtimeMs !== stats.mtimeMs
+          || onDisk === undefined
+          || onDisk.ino !== stats.ino
+          || onDisk.dev !== stats.dev
+        ) {
+          throw new SiteAuthoringError(
+            "workspace.file_changed",
+            `Workspace file '${relativePath}' changed while it was being read, so what was sent is not the file on disk. `
+              + "Wait for whatever is writing it to finish and run the command again.",
+            { field: relativePath },
+          );
+        }
+      },
+      close: async () => {
+        await opened.close().catch(() => {});
+      },
+    };
+  } catch (error) {
+    await handle?.close().catch(() => {});
+    if (error instanceof SiteAuthoringError) throw error;
+    if (error && typeof error === "object" && error.code === "ENOENT") {
+      throw new SiteAuthoringError(
+        "workspace.file_missing",
+        `Workspace file '${relativePath}' does not exist.`,
+        { field: relativePath },
+      );
+    }
+    throw new SiteAuthoringError(
+      "workspace.unreadable",
+      `Could not read workspace file '${relativePath}'.`,
+      { field: relativePath },
+    );
+  }
+}
+
 export async function readWorkspaceJson(workspaceDir, relativePath, maximumBytes) {
   const bytes = await readWorkspaceFile(workspaceDir, relativePath, maximumBytes);
   try {
@@ -1180,7 +1284,7 @@ export async function readMediaManifest(workspaceDir, expectedSiteId) {
         { field: MEDIA_MANIFEST_FILE_NAME },
       );
     }
-    return { mediaManifestVersion: MEDIA_MANIFEST_VERSION, siteId: expectedSiteId, media: {} };
+    return { mediaManifestVersion: MEDIA_MANIFEST_VERSION, siteId: expectedSiteId, media: {}, videos: {} };
   }
   const parsed = requireManifestObject(
     await readWorkspaceJson(workspaceDir, MEDIA_MANIFEST_FILE_NAME, WORKSPACE_LIMITS.manifestBytes),
@@ -1209,6 +1313,14 @@ export async function readMediaManifest(workspaceDir, expectedSiteId) {
       "media map",
       MEDIA_MANIFEST_FILE_NAME,
     ),
+    // Videos uploaded from this workspace, by file: absent in a manifest
+    // written before video upload, which reads as none.
+    videos: requireManifestObject(
+      parsed.videos ?? {},
+      "workspace.media_manifest_invalid",
+      "video map",
+      MEDIA_MANIFEST_FILE_NAME,
+    ),
   };
 }