@micacms/runtime 0.5.0 → 0.6.0-rc.1

package/src/services/media.ts

@@ -0,0 +1,229 @@
+/**
+ * Media uploads. Every upload is checked here for size and for its real content
+ * type, read from the bytes. An upload is a file in the content repository; its
+ * name is slugified and unique in its own directory.
+ */
+
+import path from "node:path";
+
+import { contentPath, ContentPathError, MEDIA_DIR, slugify } from "@micacms/core";
+import type { ContentFiles } from "../workspace/files.js";
+import { imageSize } from "image-size";
+
+import { ApiError, badRequest } from "../lib/errors.js";
+import { detectFileType, isAccepted, isImageType, SVG_TYPE } from "../lib/file-type.js";
+
+/** Where uploads go when they are not attached to a particular entry. */
+export const SHARED_MEDIA_DIR = "media";
+
+export interface MediaServiceDeps {
+  files: ContentFiles;
+  maxBytes: number;
+}
+
+export interface MediaFile {
+  /** Path relative to the content root: the value a `media()` field holds. */
+  path: string;
+  filename: string;
+  mimeType: string;
+  size: number;
+  width: number | null;
+  height: number | null;
+}
+
+export function createMediaService(deps: MediaServiceDeps) {
+  const { files, maxBytes } = deps;
+
+  /** A name that is unique in `directory`. Only there can a collision overwrite a file. */
+  const uniqueName = async (directory: string, filename: string): Promise<string> => {
+    const taken = new Set(
+      (await files.list(directory)).filter((item) => item.type === "file").map((item) => item.name),
+    );
+
+    const extension = path.extname(filename);
+    const stem = filename.slice(0, filename.length - extension.length);
+
+    let candidate = filename;
+    for (let n = 2; taken.has(candidate); n += 1) candidate = `${stem}-${n}${extension}`;
+    return candidate;
+  };
+
+  return {
+    /** Reads and checks an upload, without writing it. */
+    async prepare(input: {
+      file: File;
+      /** Directory to place the file in, relative to the content root. */
+      directory?: string;
+      accept?: "image" | "document" | "any";
+    }): Promise<{ file: MediaFile; contents: Uint8Array }> {
+      const { file } = input;
+      const directory = uploadDirectory(input.directory ?? SHARED_MEDIA_DIR);
+
+      if (file.size === 0) throw badRequest("The file is empty.");
+      if (file.size > maxBytes) {
+        throw new ApiError(
+          "payload_too_large",
+          `The file is ${formatBytes(file.size)}. The limit is ${formatBytes(maxBytes)}.`,
+        );
+      }
+
+      const contents = new Uint8Array(await file.arrayBuffer());
+      // From the bytes, never from the browser's claim.
+      const detected = detectFileType(contents);
+
+      if (!detected) {
+        throw new ApiError("unsupported_media_type", "This file type is not supported.");
+      }
+      if (detected.mimeType === SVG_TYPE) throw refuseSvg();
+      if (!isAccepted(detected.mimeType, input.accept ?? "any")) {
+        throw new ApiError(
+          "unsupported_media_type",
+          `A ${input.accept} was expected, but this is a ${detected.mimeType}.`,
+        );
+      }
+
+      const filename = await uniqueName(directory, safeName(file.name, detected.extension));
+
+      return {
+        contents,
+        file: {
+          path: `${directory}/${filename}`,
+          filename,
+          mimeType: detected.mimeType,
+          size: contents.byteLength,
+          ...dimensions(contents, detected.mimeType),
+        },
+      };
+    },
+
+    /**
+     * Every media file in the repository. A tree walk is sufficient because
+     * content trees are small.
+     */
+    async list(): Promise<MediaFile[]> {
+      const found: MediaFile[] = [];
+
+      const walk = async (directory: string): Promise<void> => {
+        for (const item of await files.list(directory)) {
+          const child = directory === "" ? item.name : `${directory}/${item.name}`;
+
+          if (item.type === "directory") {
+            await walk(child);
+            continue;
+          }
+
+          if (!isMediaName(item.name)) continue;
+
+          const bytes = await files.readBinary(child);
+          if (!bytes) continue;
+
+          const mimeType = typeOf(bytes, item.name);
+          if (mimeType === SVG_TYPE) continue;
+
+          found.push({
+            path: child,
+            filename: item.name,
+            mimeType,
+            size: bytes.byteLength,
+            ...dimensions(bytes, mimeType),
+          });
+        }
+      };
+
+      await walk("");
+      return found.sort((a, b) => a.path.localeCompare(b.path));
+    },
+
+    async read(mediaPath: string): Promise<{ bytes: Uint8Array; mimeType: string } | null> {
+      const bytes = await files.readBinary(contentPath(mediaPath));
+      if (!bytes) return null;
+
+      const mimeType = typeOf(bytes, mediaPath);
+      if (mimeType === SVG_TYPE) throw refuseSvg();
+      return { bytes, mimeType };
+    },
+  };
+}
+
+export type MediaService = ReturnType<typeof createMediaService>;
+
+/** An upload goes into a `media/` folder inside the content root, and nowhere else. */
+function uploadDirectory(directory: string): string {
+  let clean: string;
+  try {
+    clean = contentPath(directory);
+  } catch (error) {
+    if (error instanceof ContentPathError) throw badRequest(error.message);
+    throw error;
+  }
+  if (clean.split("/").at(-1) !== MEDIA_DIR) {
+    throw badRequest(`Uploads go into a "${MEDIA_DIR}" folder, not "${directory}".`);
+  }
+  return clean;
+}
+
+/** A slugified filename: safe as a path segment and as part of a URL. */
+function safeName(original: string, extension: string): string {
+  const stem = slugify(path.basename(original).replace(/\.[^.]*$/, "")).slice(0, 80);
+  return `${stem || "upload"}.${extension}`;
+}
+
+/** Extension to media type, only for formats that byte-sniffing does not identify. */
+const BY_EXTENSION: Record<string, string> = {
+  svg: "image/svg+xml",
+  pdf: "application/pdf",
+  txt: "text/plain",
+  csv: "text/csv",
+  doc: "application/msword",
+  docx: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+  xls: "application/vnd.ms-excel",
+  xlsx: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+};
+
+/**
+ * The bytes win over the extension, except that an SVG name always counts, so a
+ * leading comment cannot hide an SVG.
+ */
+function typeOf(bytes: Uint8Array, name: string): string {
+  const byName = mimeFromName(name);
+  if (byName === SVG_TYPE) return SVG_TYPE;
+  return detectFileType(bytes)?.mimeType ?? byName;
+}
+
+const refuseSvg = () =>
+  new ApiError(
+    "unsupported_media_type",
+    "Mica does not accept SVG files, because they can contain scripts. Use PNG or WebP instead.",
+  );
+
+function mimeFromName(name: string): string {
+  const extension = path.extname(name).slice(1).toLowerCase();
+  return BY_EXTENSION[extension] ?? "application/octet-stream";
+}
+
+/** Files a `media()` field could plausibly point at. */
+function isMediaName(name: string): boolean {
+  return /\.(jpe?g|png|webp|avif|gif|tiff?|pdf|docx?|xlsx?|txt|csv)$/i.test(name);
+}
+
+function dimensions(
+  bytes: Uint8Array,
+  mimeType: string,
+): { width: number | null; height: number | null } {
+  if (bytes.byteLength === 0 || !isImageType(mimeType)) {
+    return { width: null, height: null };
+  }
+  try {
+    const size = imageSize(Buffer.from(bytes));
+    return { width: size.width ?? null, height: size.height ?? null };
+  } catch {
+    // A format image-size cannot read is still a valid upload.
+    return { width: null, height: null };
+  }
+}
+
+export function formatBytes(bytes: number): string {
+  if (bytes < 1024) return `${bytes} B`;
+  if (bytes < 1024 * 1024) return `${Math.round(bytes / 1024)} kB`;
+  return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
+}

package/src/services/staging.ts

@@ -0,0 +1,330 @@
+/**
+ * The staging service on a Git working tree. Autosave has already written the
+ * files; Publish commits the reviewed paths.
+ *
+ *   - Every Git operation is limited to an explicit path set, so other work in
+ *     the same checkout is never swept in.
+ *   - Only the reviewed files are published, as the editor saw them. Drafts of
+ *     other entries stay drafts.
+ *   - When the push fails, the commit stays as `committed-pending-push`; a
+ *     retry pushes and never commits again.
+ */
+
+import { MANIFEST_PATH, memorySource, type ContentSource, type PublishResult, type Site } from "@micacms/core";
+
+import { ApiError, badRequest, notFound } from "../lib/errors.js";
+import type { Logger } from "../lib/logger.js";
+import { developerPolicy } from "../workspace/developer.js";
+import type { ContentFiles } from "../workspace/files.js";
+import { GitError } from "../workspace/git.js";
+import { createWorkspaceLocks, type WorkspaceLocks } from "../workspace/locks.js";
+import type { OwnerPolicy } from "../workspace/policy.js";
+import {
+  aheadOfUpstream,
+  commitPaths,
+  currentBranch,
+  discardChange,
+  headRevision,
+  headSource,
+  overlaySource,
+  listTreeChanges,
+  locateTree,
+  type GitTree,
+  type TreeChange,
+} from "../workspace/tree.js";
+import {
+  analyseChanges,
+  compareSources,
+  describeProblems,
+  flattenProblems,
+  type AnnotatedChange,
+  type CompareTarget,
+  type Comparison,
+  type ContentStagingService,
+  type PublishInput,
+  type RepositoryInfo,
+  type StagingStatus,
+  type StagingValidation,
+} from "./changes.js";
+
+export interface WorkingTreeStagingDeps {
+  model: Site;
+  /**
+   * The working tree's files without the lock wrapper: this service holds the
+   * locks around whole operations.
+   */
+  files: ContentFiles;
+  contentRoot: string;
+  schemaRevision: string | null;
+  /** From `readOnlyParts`: a change to one of these is never published by this build. */
+  readOnly?: ReadonlyMap<string, string>;
+  /** `MICA_LOCAL_PUBLISH`, for the default developer policy. */
+  publishEnabled: boolean;
+  logger: Logger;
+  /** Defaults to the developer. */
+  policy?: OwnerPolicy;
+  /** Shared with the services that write content. */
+  locks?: WorkspaceLocks;
+  /**
+   * Set by a managed clone, whose Git runs in an isolated environment. Without
+   * it, the developer's Git finds the tree.
+   */
+  tree?: GitTree;
+}
+
+export function createWorkingTreeStagingService(
+  deps: WorkingTreeStagingDeps,
+): ContentStagingService {
+  const { model, files, contentRoot, logger } = deps;
+  const policy = deps.policy ?? developerPolicy({ publishEnabled: deps.publishEnabled, logger });
+  const locks = deps.locks ?? createWorkspaceLocks();
+
+  /** `source` is what the result holds: the whole working tree, or HEAD with a selection. */
+  const analyse = async (tree: GitTree, treeChanges: TreeChange[], source: ContentSource = files) => {
+    const { changes, validation } = await analyseChanges(
+      model,
+      source,
+      treeChanges,
+      headSource(tree),
+    );
+
+    // The field validator cannot see a merge conflict: the file holds Git's conflict format.
+    for (const change of treeChanges) {
+      if (!change.conflicted) continue;
+      validation.problems.push({
+        path: change.path,
+        label: change.path,
+        issues: [
+          { path: "", message: "This file has a Git merge conflict. Resolve it with your Git tools." },
+        ],
+      });
+    }
+    // By the first path segment, so that a stray file in a read-only collection is caught too.
+    for (const change of changes) {
+      const head = change.path.split("/")[0] ?? "";
+      const reason = deps.readOnly?.get(`collections.${head}`) ?? deps.readOnly?.get(`singletons.${head.replace(/\.json$/, "")}`);
+      if (reason) {
+        validation.problems.push({ path: change.path, label: change.owner.label, issues: [{ path: "", message: reason }] });
+      }
+    }
+    validation.ok = validation.problems.length === 0;
+
+    return { changes, validation };
+  };
+
+  const locate = async (): Promise<GitTree | { reason: string }> => deps.tree ?? locateTree(contentRoot);
+
+  const located = async (): Promise<GitTree> => {
+    const tree = await locate();
+    if ("reason" in tree) throw badRequest(tree.reason);
+    return tree;
+  };
+
+  const publishable = async (): Promise<GitTree> => {
+    const availability = policy.publishAvailability();
+    if (!availability.enabled) throw badRequest(availability.reason);
+    return located();
+  };
+
+  /** The key that keeps an entry's files together: a publication never splits one. */
+  const groupOf = (change: AnnotatedChange): string => {
+    const owner = change.owner;
+    if (owner.kind === "entry") return `entry:${owner.collection}/${owner.slug}`;
+    if (owner.kind === "singleton") return `singleton:${owner.name}`;
+    if (owner.kind === "order") return `order:${owner.collection}`;
+    return `file:${change.path}`;
+  };
+
+  const withRevisions = async (changes: AnnotatedChange[]): Promise<AnnotatedChange[]> =>
+    Promise.all(
+      changes.map(async (change) => ({
+        ...change,
+        revision: change.kind === "delete" ? null : await files.revision(change.path),
+      })),
+    );
+
+  /** Refused for the developer's checkout: Mica never fetches or merges there. */
+  const managedOnly = <T>(action: T | undefined): T => {
+    if (action === undefined) throw notFound("This is the developer's checkout; Git there is theirs.");
+    return action;
+  };
+
+  return {
+    async status(): Promise<StagingStatus> {
+      const tree = await locate();
+      const sync = policy.syncStatus?.() ?? null;
+
+      if ("reason" in tree) {
+        return {
+          owner: policy.owner,
+          publish: { enabled: false, reason: tree.reason },
+          head: null,
+          branch: null,
+          ahead: null,
+          changes: [],
+          validation: emptyValidation(),
+          sync,
+        };
+      }
+
+      const { changes, validation, head, branch } = await locks.tree.read(async () => {
+        const analysed = await analyse(tree, await listTreeChanges(tree));
+        return {
+          ...analysed,
+          changes: await withRevisions(analysed.changes),
+          head: await headRevision(tree),
+          branch: await currentBranch(tree),
+        };
+      });
+
+      return {
+        owner: policy.owner,
+        publish: policy.publishAvailability(),
+        head,
+        branch,
+        ahead: await aheadOfUpstream(tree),
+        changes,
+        validation,
+        sync,
+      };
+    },
+
+    async publish(input: PublishInput): Promise<PublishResult> {
+      const message = input.message.trim();
+      if (!message) throw badRequest("A publication needs a commit message.");
+      const reviewed = Object.keys(input.paths);
+      if (reviewed.length === 0) throw badRequest("Choose at least one change to publish.");
+
+      const tree = await publishable();
+
+      // One publication at a time from sync to push: no sync can run in
+      // between, and no second commit can stack up.
+      return locks.remote.run(async () => {
+        await policy.prepare(tree);
+
+        const revision = await locks.tree.write(async () => {
+          const pending = await withRevisions((await analyse(tree, await listTreeChanges(tree))).changes);
+          const byPath = new Map(pending.map((change) => [change.path, change]));
+
+          // The review gate refuses a file saved again since the review, a
+          // reviewed file that is no longer pending, or another file of a
+          // reviewed entry. Drafts of other entries never block a publication.
+          const reviewedGroups = new Set(
+            reviewed.map((path) => byPath.get(path)).flatMap((change) => (change ? [groupOf(change)] : [])),
+          );
+          const stale = [
+            ...reviewed.filter((path) => byPath.get(path)?.revision !== input.paths[path] || !byPath.has(path)),
+            ...pending
+              .filter((change) => reviewedGroups.has(groupOf(change)) && !(change.path in input.paths))
+              .map((change) => change.path),
+          ];
+          if (stale.length > 0) {
+            throw new ApiError(
+              "conflict",
+              "Some of these changes were edited after you reviewed them. Review them again, then publish.",
+            );
+          }
+
+          // The model drift gate: validation used the model that this process started with.
+          if ((await files.revision(MANIFEST_PATH)) !== deps.schemaRevision) {
+            throw new ApiError(
+              "conflict",
+              "The content model changed after this server started. Restart Mica to load " +
+                "the new model, then review the pending changes and publish again.",
+            );
+          }
+
+          const selected = (await listTreeChanges(tree)).filter((change) => change.path in input.paths);
+          // Checked against what the commit will hold: drafts left behind are not on the site.
+          const result = overlaySource(headSource(tree), files, selected.map((change) => change.path));
+          const { validation } = await analyse(tree, selected, result);
+          if (!validation.ok) {
+            throw new ApiError("validation_failed", describeProblems(validation.problems), {
+              issues: flattenProblems(validation.problems),
+            });
+          }
+
+          try {
+            const revision = await commitPaths(tree, reviewed, message, policy.commitAuthor(input.author));
+            logger.info("Published", { revision, files: reviewed.length, owner: policy.owner });
+            return revision;
+          } catch (error) {
+            if (error instanceof GitError) {
+              // Git's message is the actionable part: no identity, a hook that
+              // refused, a rebase in progress.
+              throw new ApiError("conflict", `Git refused the commit: ${error.message}`);
+            }
+            throw error;
+          }
+        });
+
+        return policy.push(tree, revision);
+      });
+    },
+
+    async push(): Promise<PublishResult> {
+      const tree = await publishable();
+
+      return locks.remote.run(async () => {
+        const revision = await headRevision(tree);
+        if (revision === null) throw badRequest("There is no commit to push yet.");
+        return policy.push(tree, revision);
+      });
+    },
+
+    async discard(requested: string[]): Promise<{ discarded: string[] }> {
+      const tree = await located();
+
+      return locks.tree.write(async () => {
+        // Discard is destructive, so it reaches only paths that the change list names.
+        const current = new Map((await listTreeChanges(tree)).map((change) => [change.path, change]));
+
+        const discarded: string[] = [];
+        for (const path of requested) {
+          const change = current.get(path);
+          if (!change) continue;
+          await discardChange(tree, change);
+          discarded.push(path);
+        }
+
+        return { discarded };
+      });
+    },
+
+    async compare(target: CompareTarget, against: "head" | "origin" = "head"): Promise<Comparison> {
+      const tree = await locate();
+      if (against === "origin" && !policy.upstreamRef) throw badRequest("There is no website version to compare with here.");
+      // Without a repository nothing is published, so "before" is an empty source.
+      const published =
+        "reason" in tree
+          ? memorySource({}, "unpublished")
+          : headSource(tree, against === "origin" ? policy.upstreamRef : "HEAD");
+      return locks.tree.read(() => compareSources(model, published, files, target));
+    },
+
+    async sync(): Promise<void> {
+      await managedOnly(policy.sync)();
+    },
+
+    async resolve(group: string, choice: "mine" | "website"): Promise<void> {
+      await managedOnly(policy.resolve)(group, choice);
+    },
+
+    async repository(): Promise<RepositoryInfo> {
+      const tree = await locate();
+      const described = (await policy.describe?.()) ?? null;
+      return {
+        owner: policy.owner,
+        branch: "reason" in tree ? null : await currentBranch(tree),
+        head: "reason" in tree ? null : await headRevision(tree),
+        url: described?.url ?? null,
+        cloneBytes: described?.cloneBytes ?? null,
+        sync: policy.syncStatus?.() ?? null,
+      };
+    },
+  };
+}
+
+function emptyValidation(): StagingValidation {
+  return { ok: true, problems: [] };
+}

package/src/workspace/developer.ts

@@ -0,0 +1,46 @@
+/**
+ * The policy for the developer's own checkout. Mica never fetches, merges or
+ * resets here. Publish pushes with the developer's `git`, credentials and
+ * upstream; a failed push keeps the commit.
+ */
+
+import type { PublishResult } from "@micacms/core";
+
+import type { Logger } from "../lib/logger.js";
+import { GitError } from "./git.js";
+import type { OwnerPolicy, PublishAvailability } from "./policy.js";
+import type { GitTree } from "./tree.js";
+
+export interface DeveloperPolicyOptions {
+  /** `MICA_LOCAL_PUBLISH`. */
+  publishEnabled: boolean;
+  logger: Logger;
+}
+
+export function developerPolicy(options: DeveloperPolicyOptions): OwnerPolicy {
+  return {
+    owner: "developer",
+
+    publishAvailability(): PublishAvailability {
+      return options.publishEnabled
+        ? { enabled: true }
+        : { enabled: false, reason: "Publishing from local mode is disabled (MICA_LOCAL_PUBLISH=false)." };
+    },
+
+    async prepare() {},
+
+    // The commit keeps the developer's own Git identity.
+    commitAuthor: () => undefined,
+
+    async push(tree: GitTree, revision: string): Promise<PublishResult> {
+      try {
+        await tree.git.run(["push"]);
+        return { state: "published", revision };
+      } catch (error) {
+        const pushError = error instanceof GitError ? error.message : String(error);
+        options.logger.warn("Publish committed but the push failed", { revision, pushError });
+        return { state: "committed-pending-push", revision, pushError };
+      }
+    },
+  };
+}