@taprootio/site-authoring 0.14.0 → 0.15.0

package/src/workspace-check.js

@@ -0,0 +1,199 @@
+import { sharedThemeContextNames } from "./content/free-form-sections.js";
+import { readAppearanceWorkspaceContext, FOOTER_SETTINGS_FILE } from "./footer-workspace.js";
+import { readWorkspaceForms } from "./forms-workspace.js";
+import { CHECK_AREA, collectProblems, problemsOf } from "./problems.js";
+import { normalizeRedirectPath, REDIRECTS_FILE_NAME, validateRedirectsDocument } from "./redirects-contract.js";
+import { SETTINGS_TYPE_SITE_PUBLISHING_PREFERENCES } from "./settings-catalog.js";
+import { validateFooterWorkspaceDocument } from "./verbs/footer-push.js";
+import { validateNavigationWorkspaceDocument } from "./verbs/nav-push.js";
+import { planPages } from "./verbs/pages-push.js";
+import { validateThemeWorkspace } from "./verbs/theme-push.js";
+import {
+  NAVIGATION_FILE_NAME,
+  normalizePagePath,
+  PAGE_SOURCE_EXTENSIONS,
+  PAGES_DIRECTORY,
+  readWorkspaceJson,
+  walkWorkspaceFiles,
+  WORKSPACE_LIMITS,
+  workspaceFileExists,
+} from "./workspace.js";
+import { SiteAuthoringError } from "./errors.js";
+
+const ALL_AREAS = new Set(Object.values(CHECK_AREA));
+
+/**
+ * The one validator for a whole authoring workspace (TR01002, TR00823).
+ *
+ * `validate` runs it over a pulled workspace or an offline fixture, and `plan`
+ * runs it before reading the site. Each area is checked by the function its
+ * push uses — `planPages` for pages, `validateThemeWorkspace` for the theme —
+ * so a workspace that passes here is not refused for its shape by a push.
+ *
+ * Every area is checked, and every page, whatever an earlier one found: the
+ * result lists all the problems, and no area's refusal hides another's.
+ *
+ * `binding` says what the workspace's references may point at. A pulled
+ * workspace binds the images and pages its manifests record (or, for `plan`,
+ * the site's live pages); a fixture binds the identities its own manifest
+ * declares.
+ */
+export async function checkWorkspace({
+  workspaceDir,
+  siteId,
+  manifest,
+  mediaManifest,
+  binding = {},
+  content,
+  areas = ALL_AREAS,
+  onProgress = () => {},
+}) {
+  const problems = [];
+  // A managed Docs site's workspace holds its settings only (TR00790).
+  const settingsOnly = manifest.authoringSurface !== undefined;
+  const pageIds = binding.pageIds
+    ?? new Set(manifest.pages.map((entry) => entry?.pageId).filter((value) => typeof value === "string"));
+  const resourceIds = binding.resourceIds
+    ?? new Set(manifest.pages.map((entry) => entry?.resourceId).filter((value) => typeof value === "string"));
+  // Navigation may be judged against other pages than the footer: `plan`
+  // checks it against the site's live pages, which is what nav push does,
+  // while footer push resolves its targets from the pull manifest.
+  const navigationPages = binding.navigationPages ?? { pageIds, resourceIds };
+  const knownImageIds = binding.knownImageIds
+    ?? await collectProblems(problems, { area: CHECK_AREA.presentation }, async () =>
+      (await readAppearanceWorkspaceContext(workspaceDir, siteId)).knownImageIds);
+
+  onProgress("Validating the complete light/dark theme, appearance, header, brand, and footer colors.");
+  const presentation = knownImageIds === undefined
+    ? undefined
+    : await collectProblems(problems, { area: CHECK_AREA.presentation }, async () =>
+      await validateThemeWorkspace(workspaceDir, siteId, knownImageIds, binding.settingsEntityIds));
+
+  onProgress("Validating the closed footer document and its page and image targets.");
+  const footer = knownImageIds === undefined
+    ? undefined
+    : await collectProblems(problems, { area: CHECK_AREA.footer, file: FOOTER_SETTINGS_FILE }, async () =>
+      validateFooterWorkspaceDocument(
+        await readWorkspaceJson(workspaceDir, FOOTER_SETTINGS_FILE, WORKSPACE_LIMITS.settingsBytes),
+        siteId,
+        {
+          expectedEntityId: binding.settingsEntityIds?.get(SETTINGS_TYPE_SITE_PUBLISHING_PREFERENCES),
+          knownPageResourceIds: resourceIds,
+          knownImageIds,
+        },
+      ));
+
+  if (settingsOnly) return { problems, settingsOnly, presentation, footer };
+
+  onProgress("Validating navigation shape and its page targets.");
+  const navigation = await collectProblems(problems, { area: CHECK_AREA.navigation, file: NAVIGATION_FILE_NAME }, async () =>
+    validateNavigationWorkspaceDocument(
+      await readWorkspaceJson(workspaceDir, NAVIGATION_FILE_NAME, WORKSPACE_LIMITS.navigationBytes),
+      siteId,
+      navigationPages,
+    ));
+
+  // `plan` checks pages against the live site instead (TR00823).
+  const checkPages = areas.has(CHECK_AREA.pages);
+  if (checkPages) onProgress("Validating every page source.");
+  const pages = !checkPages ? undefined : await collectProblems(problems, { area: CHECK_AREA.pages }, async () =>
+    await planPages({
+      workspaceDir,
+      siteId,
+      manifest,
+      mediaManifest,
+      content,
+      files: await walkWorkspaceFiles(workspaceDir, PAGES_DIRECTORY, PAGE_SOURCE_EXTENSIONS),
+      // The pages the manifest records are what the site held when it was
+      // pulled, which is what an offline check can know.
+      livePages: manifest.pages.filter((entry) => typeof entry?.pageId === "string"),
+      online: false,
+      // The theme was validated above, so its contexts come from it; reading
+      // the styles file again would re-check a binding a fixture spells
+      // differently, and repeat a theme problem once per page.
+      getSharedThemeContexts: async () =>
+        presentation === undefined
+          ? undefined
+          : sharedThemeContextNames(presentation.style.lightTheme, presentation.style.darkTheme),
+      onProgress,
+    }));
+  if (pages !== undefined) problems.push(...pages.problems);
+
+  // `plan` leaves redirects and forms to their own pushes.
+  if (areas.has(CHECK_AREA.redirects)) {
+    onProgress("Validating the redirect map: paths, targets, statuses, chains, loops, and page paths.");
+  }
+  const redirects = !areas.has(CHECK_AREA.redirects)
+    ? undefined
+    : await collectProblems(problems, { area: CHECK_AREA.redirects, file: REDIRECTS_FILE_NAME }, async () =>
+      await checkRedirects(workspaceDir, siteId, manifest, pages?.sources));
+
+  const forms = !areas.has(CHECK_AREA.forms)
+    ? undefined
+    : await collectProblems(problems, { area: CHECK_AREA.forms }, async () => await readWorkspaceForms(workspaceDir));
+  if (forms?.length > 0) {
+    onProgress(`Validated ${forms.length} form file${forms.length === 1 ? "" : "s"} against the shared field schema.`);
+  }
+
+  return { problems, settingsOnly, presentation, footer, navigation, pages, redirects, forms };
+}
+
+/**
+ * The redirect map's own rules, and the one rule the workspace can check
+ * against itself: a source a workspace page occupies is refused by the site.
+ * Paths compare case-insensitively, as the site's citext path column does,
+ * and a refusal names the entry where the author wrote it: the validated map
+ * is sorted by path, so its index is not the document's.
+ */
+async function checkRedirects(workspaceDir, siteId, manifest, sources) {
+  if (!await workspaceFileExists(workspaceDir, REDIRECTS_FILE_NAME)) {
+    // A workspace pulled before redirects existed holds no map, and nothing in
+    // it asks for one.
+    if (manifest.redirects === undefined) return undefined;
+    throw new SiteAuthoringError(
+      "redirects.file_missing",
+      `No ${REDIRECTS_FILE_NAME} was found, but the pull manifest records a redirect baseline. Run 'taproot-site `
+        + "redirects pull' again.",
+      { field: REDIRECTS_FILE_NAME },
+    );
+  }
+  const document_ = await readWorkspaceJson(workspaceDir, REDIRECTS_FILE_NAME, WORKSPACE_LIMITS.redirectsBytes);
+  const redirects = validateRedirectsDocument(document_, siteId);
+  // The pages the next push leaves on the site: each workspace source at the
+  // path it declares, and every recorded page no source here claims. A page a
+  // source renames frees its old path for a redirect.
+  const claimed = new Set((sources ?? []).map((source) => source.file));
+  const occupied = [
+    ...(sources ?? []).map((source) => source.pagePath),
+    ...manifest.pages
+      .filter((entry) => typeof entry?.path === "string" && !claimed.has(entry.file))
+      .map((entry) => normalizePagePath(entry.path)),
+  ];
+  const pagePaths = new Set(
+    occupied.filter((pagePath) => pagePath !== undefined)
+      .map((pagePath) => (`/${pagePath}`.replace(/\/+$/u, "") || "/").toLowerCase()),
+  );
+  const authoredIndexByPath = new Map();
+  for (const [index, entry] of document_.entries.entries()) {
+    const path = normalizeRedirectPath(entry?.path);
+    if (path !== undefined && !authoredIndexByPath.has(path.toLowerCase())) {
+      authoredIndexByPath.set(path.toLowerCase(), index);
+    }
+  }
+  const problems = [];
+  for (const entry of redirects.entries) {
+    const key = entry.path.toLowerCase();
+    if (!pagePaths.has(key)) continue;
+    problems.push(...problemsOf(new SiteAuthoringError(
+      "redirects.path_occupied",
+      `Redirect source '${entry.path}' is a page in this workspace, so it cannot also be a redirect source.`,
+      { field: `entries[${authoredIndexByPath.get(key) ?? 0}].path` },
+    ), { area: CHECK_AREA.redirects, file: REDIRECTS_FILE_NAME }));
+  }
+  if (problems.length > 0) {
+    const [first] = problems;
+    const error = new SiteAuthoringError(first.code, first.message, { field: first.field });
+    throw problems.length === 1 ? error : error.withProblems(problems);
+  }
+  return redirects;
+}

package/src/workspace-ledger.js

@@ -0,0 +1,173 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+import { createHash } from "node:crypto";
+
+import { SiteAuthoringError } from "./errors.js";
+import { planStale } from "./problems.js";
+
+/**
+ * What `plan` read from the workspace, so `apply` can send nothing else
+ * (TR00823).
+ *
+ * Every workspace read goes through `workspace.js`, which reports it here.
+ * While `plan` (or `apply`'s re-plan) runs, each read is recorded: a file's
+ * digest, a stream's size and modification time, whether a path exists, what a
+ * directory walk listed. While `apply`'s steps run, each read is compared with
+ * that record. A read the plan never made, or one whose answer changed, stops
+ * the step before it writes, because whatever it was about to send was never
+ * reviewed. A file that `apply` itself writes, such as the media manifest the
+ * upload step records, replaces its own entry, so the next step reads it as
+ * expected.
+ *
+ * The ledger guards the workspace only. What lives on the site (page
+ * revisions, the navigation tree) is checked by the steps against the plan.
+ */
+
+const scope = new AsyncLocalStorage();
+
+const MISSING = "missing";
+
+export function digest(bytes) {
+  return createHash("sha256").update(bytes).digest("hex");
+}
+
+/** Runs `action` while recording every workspace read; returns its result and the ledger. */
+export async function recordWorkspaceReads(action) {
+  const ledger = { mode: "record", entries: new Map() };
+  const result = await scope.run(ledger, action);
+  return { result, ledger: Object.freeze({ entries: ledger.entries }) };
+}
+
+/** Runs `action` refusing any workspace read the ledger does not hold. */
+export async function enforceWorkspaceReads(recorded, action) {
+  const ledger = { mode: "enforce", entries: new Map(recorded.entries), written: new Set(), deleted: new Set() };
+  return await scope.run(ledger, action);
+}
+
+/** A stable digest of everything the ledger recorded, for the plan hash. */
+export function ledgerDigest(recorded) {
+  const entries = [...recorded.entries.entries()].sort(([left], [right]) => (left < right ? -1 : 1));
+  return digest(JSON.stringify(entries));
+}
+
+function stale(path, mode) {
+  // While planning, the plan itself is what moved; while applying, the step.
+  return mode === "record"
+    ? new SiteAuthoringError(
+      "plan.workspace_changed",
+      `'${path}' changed while the plan was reading the workspace. Wait for whatever is writing it to finish, then `
+        + "run 'taproot-site plan' again.",
+      { field: path },
+    )
+    : planStale(`'${path}'`, path);
+}
+
+function key(kind, path) {
+  return `${kind}:${path}`;
+}
+
+/**
+ * One read's answer. While recording, the first answer for a key is kept; a
+ * different later answer means the workspace moved while it was being planned.
+ * While enforcing, the answer must match what was recorded.
+ */
+function noteRead(kind, path, value) {
+  const ledger = scope.getStore();
+  if (ledger === undefined) return;
+  const entry = key(kind, path);
+  const recorded = ledger.entries.get(entry);
+  if (ledger.mode === "record") {
+    if (recorded === undefined) ledger.entries.set(entry, value);
+    else if (recorded !== value) throw stale(path, "record");
+    return;
+  }
+  if (recorded === value) return;
+  if (recorded === undefined && impliedByRecordedRead(ledger, kind, path, value)) return;
+  throw stale(path, "enforce");
+}
+
+/**
+ * Whether the plan already answered this question another way. A file the plan
+ * read was a regular file, and one it found missing did not exist; a step that
+ * asks only whether it exists, or what kind of entry it is, learns nothing the
+ * plan did not see. The bytes themselves are still compared when the step reads
+ * them.
+ */
+function impliedByRecordedRead(ledger, kind, path, value) {
+  if (kind !== "exists" && kind !== "entry") return false;
+  const read = ledger.entries.get(key("file", path)) ?? ledger.entries.get(key("stream", path));
+  if (read === undefined) return false;
+  const present = read !== MISSING;
+  return kind === "exists" ? value === present : value === (present ? "file" : MISSING);
+}
+
+// The digests are computed only inside a ledger: every other verb reads
+// without paying for them.
+export function noteFileRead(path, bytes) {
+  if (scope.getStore() !== undefined) noteRead("file", path, digest(bytes));
+}
+
+export function noteFileMissing(path) {
+  noteRead("file", path, MISSING);
+}
+
+export function noteStreamOpened(path, byteLength, modifiedMilliseconds, header) {
+  if (scope.getStore() !== undefined) noteRead("stream", path, `${byteLength}:${modifiedMilliseconds}:${digest(header)}`);
+}
+
+export function noteStreamMissing(path) {
+  noteRead("stream", path, MISSING);
+}
+
+export function noteExists(path, exists) {
+  noteRead("exists", path, exists);
+}
+
+export function noteEntry(path, kind) {
+  noteRead("entry", path, kind);
+}
+
+/**
+ * A directory walk. While enforcing, the listing must be the recorded one,
+ * plus the files this apply wrote under that root and minus those it removed.
+ */
+export function noteWalk(root, filter, files) {
+  const ledger = scope.getStore();
+  if (ledger === undefined) return;
+  const entry = key("walk", `${root}|${filter}`);
+  if (ledger.mode === "record") {
+    noteRead("walk", `${root}|${filter}`, JSON.stringify(files));
+    return;
+  }
+  const recorded = ledger.entries.get(entry);
+  if (recorded === undefined) throw stale(root, "enforce");
+  const expected = new Set(JSON.parse(recorded));
+  const prefix = `${root}/`;
+  for (const path of ledger.written) if (path.startsWith(prefix) && files.includes(path)) expected.add(path);
+  for (const path of ledger.deleted) if (path.startsWith(prefix)) expected.delete(path);
+  const listed = new Set(files);
+  // Name the first file that appeared or went missing, not just the directory.
+  const changed = files.find((path) => !expected.has(path)) ?? [...expected].sort().find((path) => !listed.has(path));
+  if (changed !== undefined) throw stale(changed, "enforce");
+}
+
+/** A file this apply wrote: later reads of it must find exactly these bytes. */
+export function noteFileWritten(path, contents) {
+  const ledger = scope.getStore();
+  if (ledger === undefined || ledger.mode !== "enforce") return;
+  ledger.entries.set(key("file", path), digest(contents));
+  ledger.entries.set(key("exists", path), true);
+  ledger.entries.set(key("entry", path), "file");
+  ledger.written.add(path);
+  ledger.deleted.delete(path);
+}
+
+/** A file this apply removed. */
+export function noteFileDeleted(path) {
+  const ledger = scope.getStore();
+  if (ledger === undefined || ledger.mode !== "enforce") return;
+  ledger.entries.set(key("file", path), MISSING);
+  ledger.entries.set(key("exists", path), false);
+  ledger.entries.set(key("entry", path), MISSING);
+  ledger.deleted.add(path);
+  ledger.written.delete(path);
+}

package/src/workspace.js

@@ -5,6 +5,17 @@ import path from "node:path";
 
 import { atomicWriteFile } from "./atomic-file.js";
 import { hasAsciiControl, sanitizeDiagnostic, SiteAuthoringError } from "./errors.js";
+import {
+  noteEntry,
+  noteExists,
+  noteFileDeleted,
+  noteFileMissing,
+  noteFileRead,
+  noteFileWritten,
+  noteStreamMissing,
+  noteStreamOpened,
+  noteWalk,
+} from "./workspace-ledger.js";
 
 /**
  * The local workspace: what `pull` writes, what the push verbs read back, and
@@ -713,6 +724,7 @@ export async function ensureWorkspaceDirectory(workspaceDir, relativeDirectory =
 export async function readWorkspaceFile(workspaceDir, relativePath, maximumBytes) {
   const filePath = resolveWorkspacePath(workspaceDir, relativePath);
   if (!await requireRealDirectoryChain(workspaceDir, relativePath)) {
+    noteFileMissing(relativePath);
     throw new SiteAuthoringError(
       "workspace.file_missing",
       `Workspace file '${relativePath}' does not exist.`,
@@ -744,10 +756,13 @@ export async function readWorkspaceFile(workspaceDir, relativePath, maximumBytes
       if (bytesRead === 0) break;
       total += bytesRead;
     }
-    return buffer.subarray(0, total);
+    const contents = buffer.subarray(0, total);
+    noteFileRead(relativePath, contents);
+    return contents;
   } catch (error) {
     if (error instanceof SiteAuthoringError) throw error;
     if (error && typeof error === "object" && error.code === "ENOENT") {
+      noteFileMissing(relativePath);
       throw new SiteAuthoringError(
         "workspace.file_missing",
         `Workspace file '${relativePath}' does not exist.`,
@@ -778,6 +793,7 @@ const STREAM_HEADER_BYTES = 4096;
 export async function openWorkspaceFileStream(workspaceDir, relativePath, maximumBytes) {
   const filePath = resolveWorkspacePath(workspaceDir, relativePath);
   if (!await requireRealDirectoryChain(workspaceDir, relativePath)) {
+    noteStreamMissing(relativePath);
     throw new SiteAuthoringError(
       "workspace.file_missing",
       `Workspace file '${relativePath}' does not exist.`,
@@ -812,6 +828,7 @@ export async function openWorkspaceFileStream(workspaceDir, relativePath, maximu
       total += bytesRead;
     }
     const opened = handle;
+    noteStreamOpened(relativePath, byteLength, modifiedMilliseconds, header.subarray(0, total));
     return {
       byteLength,
       modifiedMilliseconds,
@@ -850,6 +867,7 @@ export async function openWorkspaceFileStream(workspaceDir, relativePath, maximu
     await handle?.close().catch(() => {});
     if (error instanceof SiteAuthoringError) throw error;
     if (error && typeof error === "object" && error.code === "ENOENT") {
+      noteStreamMissing(relativePath);
       throw new SiteAuthoringError(
         "workspace.file_missing",
         `Workspace file '${relativePath}' does not exist.`,
@@ -885,6 +903,12 @@ export async function readWorkspaceJson(workspaceDir, relativePath, maximumBytes
  * read outside the workspace.
  */
 export async function inspectWorkspaceEntry(workspaceDir, relativePath) {
+  const kind = await classifyWorkspaceEntry(workspaceDir, relativePath);
+  noteEntry(relativePath, kind);
+  return kind;
+}
+
+async function classifyWorkspaceEntry(workspaceDir, relativePath) {
   const target = resolveWorkspacePath(workspaceDir, relativePath);
   // The leaf `lstat` below refuses a linked final component; this refuses a
   // linked one anywhere above it, which is how `media/link/sub` would otherwise
@@ -929,6 +953,12 @@ export async function inspectWorkspaceEntry(workspaceDir, relativePath) {
  * missing.
  */
 export async function workspaceFileExists(workspaceDir, relativePath) {
+  const exists = await regularFileExists(workspaceDir, relativePath);
+  noteExists(relativePath, exists);
+  return exists;
+}
+
+async function regularFileExists(workspaceDir, relativePath) {
   const filePath = resolveWorkspacePath(workspaceDir, relativePath);
   if (!await requireRealDirectoryChain(workspaceDir, relativePath)) return false;
   let stats;
@@ -982,6 +1012,7 @@ export async function writeWorkspaceFile(workspaceDir, relativePath, contents, {
         ),
     },
   });
+  noteFileWritten(relativePath, contents);
   return relativePath;
 }
 
@@ -998,7 +1029,7 @@ export async function writeWorkspaceJson(workspaceDir, relativePath, value) {
  * exists to prevent.
  */
 export async function deleteWorkspaceFile(workspaceDir, relativePath) {
-  if (!await workspaceFileExists(workspaceDir, relativePath)) return false;
+  if (!await regularFileExists(workspaceDir, relativePath)) return false;
   try {
     await unlink(resolveWorkspacePath(workspaceDir, relativePath));
   } catch (error) {
@@ -1009,6 +1040,7 @@ export async function deleteWorkspaceFile(workspaceDir, relativePath) {
       { field: relativePath },
     );
   }
+  noteFileDeleted(relativePath);
   return true;
 }
 
@@ -1040,8 +1072,12 @@ export async function walkWorkspaceFiles(
   extensions,
   { segmentPattern = SAFE_SEGMENT, segmentDescription = "letters, digits, '.', '_', and '-'" } = {},
 ) {
+  const filter = `${extensions.join(",")}|${segmentPattern}`;
   const rootPath = resolveWorkspacePath(workspaceDir, relativeRoot);
-  if (!await requireRealDirectoryChain(workspaceDir, relativeRoot, { includeLeaf: true })) return [];
+  if (!await requireRealDirectoryChain(workspaceDir, relativeRoot, { includeLeaf: true })) {
+    noteWalk(relativeRoot, filter, []);
+    return [];
+  }
   const collected = [];
   const queue = [{ directory: rootPath, relative: relativeRoot, depth: 0 }];
   while (queue.length > 0) {
@@ -1088,7 +1124,9 @@ export async function walkWorkspaceFiles(
       }
     }
   }
-  return collected.sort();
+  collected.sort();
+  noteWalk(relativeRoot, filter, collected);
+  return collected;
 }
 
 /**
@@ -1324,6 +1362,11 @@ export async function readMediaManifest(workspaceDir, expectedSiteId) {
   };
 }
 
+/** The alt text a media record keeps when its file is uploaded again: the author's own, or none. */
+export function recordedAlt(entry) {
+  return typeof entry?.alt === "string" ? entry.alt : "";
+}
+
 export async function writeMediaManifest(workspaceDir, mediaManifest) {
   return await writeWorkspaceJson(workspaceDir, MEDIA_MANIFEST_FILE_NAME, mediaManifest);
 }