filegrc 0.3.4 → 0.4.0

package/src/files.js

@@ -2,9 +2,10 @@ import { createHash, randomUUID } from "node:crypto";
 import { constants, link, lstat, mkdir, open, readFile, rename, rm, stat } from "node:fs/promises";
 import { basename, dirname, join, resolve } from "node:path";
 import { getResourceDefinition } from "../model/index.js";
-import { serializeWorkspaceMutation } from "./mutation.js";
+import { serializeWorkspaceMutation, workspaceValidationDeferred } from "./mutation.js";
 import { isCanonicalDataPath, resolveDataPath, resolveWorkspaceRoot } from "./paths.js";
 import { markdownEntries } from "./resource-markdown.js";
+import { measureTiming } from "./timing.js";
 import { loadWorkspace } from "./workspace.js";
 import { validateWorkspace } from "./validate.js";
 
@@ -198,6 +199,95 @@ export async function createResources(input, records) {
   return serializeWorkspaceMutation(input, (root) => createResourcesUnlocked(root, records));
 }
 
+export async function applyResourceBatch(input, changes) {
+  return serializeWorkspaceMutation(input, (root) => applyResourceBatchUnlocked(root, changes));
+}
+
+async function applyResourceBatchUnlocked(input, changes = {}) {
+  const creates = changes.create || [];
+  const updates = changes.update || [];
+  const expectedRevisions = changes.expectedRevisions || {};
+  if (!Array.isArray(creates) || !Array.isArray(updates) || (!creates.length && !updates.length)) {
+    throw new Error("A resource batch needs at least one create or update.");
+  }
+  if (Array.isArray(expectedRevisions) || typeof expectedRevisions !== "object") {
+    throw new Error("Batch expected revisions must be keyed by resource ID.");
+  }
+  const loaded = await loadWorkspace(input);
+  const deferValidation = workspaceValidationDeferred();
+  const before = deferValidation || changes.validateWholeWorkspace
+    ? null
+    : await validateWorkspace(loaded);
+  const existingById = new Map(loaded.entries.map((entry) => [entry.record.id, entry]));
+  const ids = new Set();
+  const writes = [];
+  for (const record of creates) {
+    validateBatchRecord(record, ids);
+    if (existingById.has(record.id)) throw new Error(`Resource "${record.id}" already exists.`);
+    const path = resourcePath(loaded.root, loaded.model, record);
+    writes.push({ operation: "create", path, record, previous: null, fileMode: 0o666 });
+  }
+  for (const record of updates) {
+    validateBatchRecord(record, ids);
+    const existing = existingById.get(record.id);
+    if (!existing) throw new Error(`Resource "${record.id}" was not found.`);
+    if (existing.record.type !== record.type) {
+      throw new Error(`Resource "${record.id}" cannot change type.`);
+    }
+    const path = resourcePath(loaded.root, loaded.model, record);
+    const previous = await readFile(path, "utf8");
+    const mode = (await stat(path)).mode & 0o777;
+    assertRevision(
+      previous,
+      expectedRevisions[record.id] || existing.revision,
+      `Resource "${record.id}"`
+    );
+    writes.push({ operation: "update", path, record, previous, fileMode: mode });
+  }
+  const written = [];
+  try {
+    for (const item of writes) {
+      await writeAtomic(item.path, item.record, { exclusive: item.operation === "create" });
+      written.push(item);
+    }
+    let validation = null;
+    if (!deferValidation) {
+      validation = await validateWorkspace(loaded.root);
+      const errors = changes.validateWholeWorkspace
+        ? validation.diagnostics.filter(({ severity }) => severity === "error")
+        : newErrors(validation, before);
+      if (errors.length) throw new Error(formatWriteFailure(errors, "resource batch"));
+    }
+    return {
+      created: creates,
+      updated: updates,
+      validation
+    };
+  } catch (error) {
+    const rollbackErrors = [];
+    for (const item of written.reverse()) {
+      try {
+        if (item.operation === "create") await rm(item.path, { force: true });
+        else await writeTextAtomic(item.path, item.previous, { mode: item.fileMode });
+      } catch (rollbackError) {
+        rollbackErrors.push(rollbackError.message);
+      }
+    }
+    if (rollbackErrors.length) {
+      throw new Error(`${error.message} FileGRC could not restore every file in the resource batch: ${rollbackErrors.join(" ")}`);
+    }
+    throw error;
+  }
+}
+
+function validateBatchRecord(record, ids) {
+  if (!record || Array.isArray(record) || typeof record !== "object") {
+    throw new Error("Every resource in a batch must be a JSON object.");
+  }
+  if (ids.has(record.id)) throw new Error(`Resource "${record.id}" appears more than once in the batch.`);
+  ids.add(record.id);
+}
+
 export async function createResourceAndLink(input, record, linkTarget, options = {}) {
   return serializeWorkspaceMutation(input, (root) => createResourceAndLinkUnlocked(root, record, linkTarget, options));
 }
@@ -254,7 +344,8 @@ async function createResourceAndLinkUnlocked(input, record, linkTarget, options)
 async function createResourcesUnlocked(input, records) {
   if (!Array.isArray(records) || records.length === 0) throw new Error("At least one resource is required.");
   const loaded = await loadWorkspace(input);
-  const before = await validateWorkspace(loaded);
+  const deferValidation = workspaceValidationDeferred();
+  const before = deferValidation ? null : await validateWorkspace(loaded);
   const ids = new Set();
   const writes = [];
   for (const record of records) {
@@ -276,9 +367,11 @@ async function createResourcesUnlocked(input, records) {
       await writeAtomic(item.path, item.record, { exclusive: true });
       written.push(item);
     }
-    const result = await validateWorkspace(loaded.root);
-    const introduced = newErrors(result, before);
-    if (introduced.length) throw new Error(formatWriteFailure(introduced, "resource batch"));
+    if (!deferValidation) {
+      const result = await validateWorkspace(loaded.root);
+      const introduced = newErrors(result, before);
+      if (introduced.length) throw new Error(formatWriteFailure(introduced, "resource batch"));
+    }
   } catch (error) {
     for (const item of written.reverse()) await rm(item.path, { force: true });
     throw error;
@@ -288,7 +381,8 @@ async function createResourcesUnlocked(input, records) {
 
 async function createResourceUnlocked(input, record, options) {
   const loaded = await loadWorkspace(input);
-  const before = await validateWorkspace(loaded);
+  const deferValidation = workspaceValidationDeferred();
+  const before = deferValidation ? null : await validateWorkspace(loaded);
   const path = resourcePath(loaded.root, loaded.model, record);
   try {
     await stat(path);
@@ -297,6 +391,7 @@ async function createResourceUnlocked(input, record, options) {
     if (error.code !== "ENOENT") throw error;
   }
   const contentWrites = await prepareContentWrites(loaded, record, options.content, { exclusive: true });
+  const nextRecord = await prepareApprovalBinding(loaded, record, contentWrites);
   const written = [];
   let recordWritten = false;
   try {
@@ -304,17 +399,19 @@ async function createResourceUnlocked(input, record, options) {
       await writeTextAtomic(item.path, item.source, { exclusive: true });
       written.push(item);
     }
-    await writeAtomic(path, record, { exclusive: true });
+    await writeAtomic(path, nextRecord, { exclusive: true });
     recordWritten = true;
-    const result = await validateWorkspace(loaded.root);
-    const introduced = newErrors(result, before);
-    if (introduced.length) throw new Error(formatWriteFailure(introduced, record.id));
+    if (!deferValidation) {
+      const result = await validateWorkspace(loaded.root);
+      const introduced = newErrors(result, before);
+      if (introduced.length) throw new Error(formatWriteFailure(introduced, record.id));
+    }
   } catch (error) {
     if (recordWritten) await rm(path, { force: true });
     for (const item of written) await rm(item.path, { force: true });
     throw error;
   }
-  return { record, path };
+  return { record: nextRecord, path };
 }
 
 export async function updateResource(input, type, id, record, options = {}) {
@@ -326,19 +423,25 @@ async function updateResourceUnlocked(input, type, id, record, options) {
     throw new Error("The type and ID in the record must match the resource being updated.");
   }
   const loaded = await loadWorkspace(input);
-  const before = await validateWorkspace(loaded);
+  const deferValidation = workspaceValidationDeferred();
+  const before = deferValidation ? null : await validateWorkspace(loaded);
   const path = resourcePath(loaded.root, loaded.model, record);
   const previous = await readFile(path, "utf8");
   assertRevision(previous, options.expectedRevision, "The record");
   const contentWrites = await prepareContentWrites(loaded, record, options.content, {
-    expectedRevisions: options.expectedContentRevisions
+    expectedRevisions: options.expectedContentRevisions,
+    requireExpectedRevisions: options.requireExpectedContentRevisions
   });
+  const existing = loaded.entries.find(({ record: candidate }) => candidate.id === id)?.record;
+  const nextRecord = await prepareApprovalBinding(loaded, record, contentWrites, existing);
   try {
     for (const item of contentWrites) await writeTextAtomic(item.path, item.source);
-    await writeAtomic(path, record);
-    const result = await validateWorkspace(loaded.root);
-    const introduced = newErrors(result, before);
-    if (introduced.length) throw new Error(formatWriteFailure(introduced, id));
+    await writeAtomic(path, nextRecord);
+    if (!deferValidation) {
+      const result = await validateWorkspace(loaded.root);
+      const introduced = newErrors(result, before);
+      if (introduced.length) throw new Error(formatWriteFailure(introduced, id));
+    }
   } catch (error) {
     await writeTextAtomic(path, previous);
     for (const item of contentWrites) {
@@ -347,7 +450,7 @@ async function updateResourceUnlocked(input, type, id, record, options) {
     }
     throw error;
   }
-  return { record, path };
+  return { record: nextRecord, path };
 }
 
 export async function updateContent(input, dataRelativePath, source, options = {}) {
@@ -368,8 +471,18 @@ async function updateContentUnlocked(input, dataRelativePath, source, options) {
   const path = resolveDataPath(loaded.root, dataRelativePath);
   const previous = await readFile(path, "utf8");
   assertRevision(previous, options.expectedRevision, "The Markdown file");
-  await writeTextAtomic(path, source.endsWith("\n") ? source : `${source}\n`);
-  return { path, dataRelativePath };
+  const before = await validateWorkspace(loaded);
+  const nextSource = source.endsWith("\n") ? source : `${source}\n`;
+  await writeTextAtomic(path, nextSource);
+  try {
+    const result = await validateWorkspace(loaded.root);
+    const introduced = newErrors(result, before);
+    if (introduced.length) throw new Error(formatWriteFailure(introduced, dataRelativePath));
+    return { path, dataRelativePath };
+  } catch (error) {
+    await writeTextAtomic(path, previous);
+    throw error;
+  }
 }
 
 export async function deleteResource(input, type, id, options = {}) {
@@ -378,7 +491,8 @@ export async function deleteResource(input, type, id, options = {}) {
 
 async function deleteResourceUnlocked(input, type, id, options) {
   const loaded = await loadWorkspace(input);
-  const before = await validateWorkspace(loaded);
+  const deferValidation = workspaceValidationDeferred();
+  const before = deferValidation ? null : await validateWorkspace(loaded);
   const definition = getResourceDefinition(loaded.model, type);
   if (definition.singleton) throw new Error("Singleton records cannot be deleted.");
   const path = resourcePath(loaded.root, loaded.model, { type, id });
@@ -393,9 +507,11 @@ async function deleteResourceUnlocked(input, type, id, options) {
   try {
     await rm(path);
     for (const item of contentFiles) await rm(item.path, { force: true });
-    const result = await validateWorkspace(loaded.root);
-    const introduced = newErrors(result, before);
-    if (introduced.length) throw new Error(formatWriteFailure(introduced, id));
+    if (!deferValidation) {
+      const result = await validateWorkspace(loaded.root);
+      const introduced = newErrors(result, before);
+      if (introduced.length) throw new Error(formatWriteFailure(introduced, id));
+    }
   } catch (error) {
     await writeTextAtomic(path, source, { mode });
     for (const item of contentFiles) {
@@ -418,11 +534,17 @@ export function resourcePath(input, model, record) {
 }
 
 async function writeAtomic(path, value, options = {}) {
-  const source = `${JSON.stringify(value, null, 2)}\n`;
-  await writeTextAtomic(path, source, options);
+  return measureTiming("writes", async () => {
+    const source = `${JSON.stringify(value, null, 2)}\n`;
+    await writeTextAtomicUnmeasured(path, source, options);
+  });
 }
 
 async function writeTextAtomic(path, source, options = {}) {
+  return measureTiming("writes", () => writeTextAtomicUnmeasured(path, source, options));
+}
+
+async function writeTextAtomicUnmeasured(path, source, options = {}) {
   await mkdir(dirname(path), { recursive: true });
   const temp = join(dirname(path), `.${randomUUID()}.tmp`);
   let mode = options.mode ?? 0o666;
@@ -454,7 +576,9 @@ async function writeTextAtomic(path, source, options = {}) {
 
 function newErrors(after, before) {
   const existing = new Set(before.diagnostics.filter(({ severity }) => severity === "error").map(diagnosticKey));
-  return after.diagnostics.filter(({ severity }) => severity === "error").filter((item) => !existing.has(diagnosticKey(item)));
+  return after.diagnostics
+    .filter(({ severity }) => severity === "error")
+    .filter((item) => item.code === "unsupported-model" || !existing.has(diagnosticKey(item)));
 }
 
 function diagnosticKey(item) {
@@ -490,6 +614,12 @@ async function prepareContentWrites(loaded, record, content, options = {}) {
     try {
       previous = await readFile(path, "utf8");
       if (options.exclusive) throw new Error(`Content already exists at data/${dataRelativePath}.`);
+      if (
+        options.requireExpectedRevisions
+        && !Object.hasOwn(options.expectedRevisions ?? {}, dataRelativePath)
+      ) {
+        throw new Error(`A content revision is required for existing content at data/${dataRelativePath}.`);
+      }
       assertRevision(previous, options.expectedRevisions?.[dataRelativePath], `Content at data/${dataRelativePath}`);
     } catch (error) {
       if (error.code !== "ENOENT") throw error;
@@ -505,10 +635,82 @@ function assertRevision(source, expected, label) {
   }
 }
 
-function contentRevision(source) {
+export function contentRevision(source) {
   return createHash("sha256").update(source).digest("hex");
 }
 
+async function prepareApprovalBinding(loaded, record, contentWrites, previousRecord = null) {
+  if (record.type === "attestation") {
+    return prepareAttestationBinding(loaded, record, previousRecord);
+  }
+  if (!["policy", "document"].includes(record.type)) return record;
+  const nextRecord = structuredClone(record);
+  if (!approvalBound(record)) {
+    delete nextRecord.approvedContentRevisions;
+    return nextRecord;
+  }
+  if (approvalBound(previousRecord) && previousRecord.approvedContentRevisions) {
+    nextRecord.approvedContentRevisions = structuredClone(previousRecord.approvedContentRevisions);
+    return nextRecord;
+  }
+  const proposed = new Map(contentWrites.map((item) => [item.dataRelativePath, item.source]));
+  const revisions = {};
+  for (const item of markdownEntries(loaded.model, nextRecord)) {
+    let source = proposed.get(item.path);
+    if (source === undefined) {
+      try {
+        source = await readFile(resolveDataPath(loaded.root, item.path), "utf8");
+      } catch (error) {
+        if (error.code === "ENOENT") continue;
+        throw error;
+      }
+    }
+    revisions[item.path] = contentRevision(source);
+  }
+  nextRecord.approvedContentRevisions = revisions;
+  return nextRecord;
+}
+
+async function prepareAttestationBinding(loaded, record, previousRecord = null) {
+  const nextRecord = structuredClone(record);
+  const bound = record.status === "completed" && record.attestationMethod === "git-approval";
+  if (!bound) {
+    delete nextRecord.contentRevisions;
+    return nextRecord;
+  }
+  if (
+    previousRecord?.status === "completed"
+    && previousRecord.attestationMethod === "git-approval"
+    && previousRecord.contentRevisions
+  ) {
+    nextRecord.contentRevisions = structuredClone(previousRecord.contentRevisions);
+    return nextRecord;
+  }
+  const revisions = {};
+  for (const id of record.subjectResourceIds || []) {
+    const subject = loaded.resources.find((candidate) => candidate.id === id);
+    if (!subject) continue;
+    for (const item of markdownEntries(loaded.model, subject)) {
+      try {
+        const source = await readFile(resolveDataPath(loaded.root, item.path), "utf8");
+        revisions[item.path] = contentRevision(source);
+      } catch (error) {
+        if (error.code !== "ENOENT") throw error;
+      }
+    }
+  }
+  nextRecord.contentRevisions = revisions;
+  return nextRecord;
+}
+
+function approvalBound(record) {
+  if (!record || !["policy", "document"].includes(record.type)) return false;
+  const statuses = record.type === "policy"
+    ? ["approved", "active", "superseded", "retired"]
+    : ["active", "superseded", "retired"];
+  return statuses.includes(record.status);
+}
+
 async function exclusiveContentFiles(loaded, record) {
   const candidates = markdownEntries(loaded.model, record).map(({ path }) => path);
   const files = [];