filegrc 0.3.4 → 0.5.0

package/src/server.js

@@ -1,25 +1,56 @@
 import { createServer as createHttpServer } from "node:http";
-import { readFile, realpath } from "node:fs/promises";
-import { extname, resolve } from "node:path";
+import { mkdtemp, readFile, realpath, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { extname, join, resolve } from "node:path";
 import { getResourceDefinition } from "../model/index.js";
 import { prepareAuditWorkspace } from "./audit-preparation.js";
+import { createNextAuditCycle, planNextAuditCycle } from "./audit-transition.js";
+import { applyApplicabilityReview, planApplicabilityReview } from "./batch-review.js";
+import { applyCollectionReview, planCollectionReview } from "./collection-review.js";
 import { generateEvidencePacket, prepareEvidencePacket } from "./evidence-packet.js";
-import { ensureEvidenceTestDrafts } from "./evidence-tests.js";
 import { FAVICON_PNG, LOGO_MARK_PNG } from "./favicon.js";
-import { createResource, deleteResource, updateContent, updateResource } from "./files.js";
 import {
+  addEvidenceAttachment,
+  createResource,
+  deleteResource,
+  removeEvidenceAttachment,
+  updateContent,
+  updateResource
+} from "./files.js";
+import {
+  BROWSER_VALIDATION,
   commitAndPushWorkspace,
   getBrowserRepositoryState,
   getFileHistory,
+  getGitSummary,
   pullWorkspace,
   pushWorkspace,
   retryBrowserSync,
   runBrowserMutation
 } from "./git.js";
-import { completeObligationOccurrence, createObligationEvent, planObligations } from "./obligations.js";
+import { normalizeResourceMutation, serializeWorkspaceMutation } from "./mutation.js";
+import {
+  completeObligationAction,
+  completeObligationEvent,
+  completeObligationOccurrence,
+  createObligationEvent,
+  planObligations
+} from "./obligations.js";
+import {
+  planExternalReviewerGovernance,
+  setupExternalReviewerGovernance
+} from "./external-reviewer.js";
 import { isWithin, relativeToWorkspace, resolveWorkspacePath } from "./paths.js";
-import { createAppState } from "./state.js";
+import { applyReconciliation, planReconciliation } from "./reconciliation.js";
+import { createAppState, createResourceDetail } from "./state.js";
 import { setupWorkspace } from "./setup.js";
+import { collectTimings, measureTiming, timingEnabled } from "./timing.js";
+import {
+  assessWorkflow,
+  buildWorkflowDelta,
+  previewWorkflowMutation,
+  workflowForResource
+} from "./workflow.js";
 import { loadWorkspace } from "./workspace.js";
 import { APP_SCRIPT, APP_STYLES, renderIndex } from "./web.js";
 
@@ -34,8 +65,17 @@ export function createFilegrcServer(input = process.cwd(), options = {}) {
         return json(response, 403, { error: "Cross-origin writes are not allowed." });
       }
       if (request.method === "GET" && url.pathname === "/api/state") {
+        if (timingEnabled()) {
+          const { result, timings } = await collectTimings(() => measureTiming("state", () => createAppState(input, {
+            allowNonAuthoritativeWrites: options.allowNonAuthoritativeWrites,
+            includeDetails: false
+          })));
+          console.error(`[filegrc timing] ${JSON.stringify({ operation: "state", ...timings })}`);
+          return json(response, 200, result);
+        }
         return json(response, 200, await createAppState(input, {
-          allowNonAuthoritativeWrites: options.allowNonAuthoritativeWrites
+          allowNonAuthoritativeWrites: options.allowNonAuthoritativeWrites,
+          includeDetails: false
         }));
       }
       if (request.method === "GET" && url.pathname === "/api/history") {
@@ -50,9 +90,84 @@ export function createFilegrcServer(input = process.cwd(), options = {}) {
           from: url.searchParams.get("from") || undefined,
           through: url.searchParams.get("through") || undefined,
           now: url.searchParams.get("now") || undefined,
+          includeComplete: url.searchParams.get("includeComplete") === "true",
+          model: loaded.model
+        }));
+      }
+      if (request.method === "GET" && url.pathname === "/api/workflow") {
+        const start = url.searchParams.get("start");
+        const end = url.searchParams.get("end");
+        return json(response, 200, await assessWorkflow(input, {
+          auditId: url.searchParams.get("auditId") || undefined,
+          asOf: url.searchParams.get("asOf") || undefined,
+          through: url.searchParams.get("through") || undefined,
+          coverage: start || end ? { kind: "range", startsOn: start, endsOn: end } : undefined,
           includeComplete: url.searchParams.get("includeComplete") === "true"
         }));
       }
+      if (request.method === "POST" && url.pathname === "/api/workflow/preview") {
+        return json(response, 200, await previewWorkflowMutation(input, await readJson(request)));
+      }
+      if (request.method === "GET" && url.pathname === "/api/reconciliation") {
+        return json(response, 200, await planReconciliation(input));
+      }
+      if (request.method === "POST" && url.pathname === "/api/reconciliation") {
+        const payload = await readJson(request);
+        return json(response, 201, await browserMutation(input, options, {
+          message: (result) => `Reconcile policy event: ${result.event?.title || payload.candidateId}`
+        }, () => applyReconciliation(input, {
+          ...payload,
+          confirmed: payload.confirmed === true
+        })));
+      }
+      if (request.method === "POST" && url.pathname === "/api/external-reviewer-governance/preview") {
+        return json(response, 200, await planExternalReviewerGovernance(input, await readJson(request)));
+      }
+      if (request.method === "POST" && url.pathname === "/api/external-reviewer-governance") {
+        const payload = await readJson(request);
+        return json(response, 201, await browserMutation(input, options, {
+          message: (result) => `Assign external independent reviewer: ${result.reviewerId}`
+        }, () => setupExternalReviewerGovernance(input, {
+          ...payload,
+          confirmed: payload.confirmed === true
+        })));
+      }
+      if (request.method === "POST" && url.pathname === "/api/audit-cycle/preview") {
+        return json(response, 200, await planNextAuditCycle(input, await readJson(request)));
+      }
+      if (request.method === "POST" && url.pathname === "/api/audit-cycle") {
+        const payload = await readJson(request);
+        return json(response, 201, await browserMutation(input, options, {
+          message: (result) => `Create next audit cycle: ${result.audit?.title || payload.priorAuditId}`
+        }, () => createNextAuditCycle(input, {
+          ...payload,
+          confirmed: payload.confirmed === true
+        })));
+      }
+      if (request.method === "POST" && url.pathname === "/api/applicability-review/preview") {
+        return json(response, 200, await planApplicabilityReview(input, await readJson(request)));
+      }
+      if (request.method === "POST" && url.pathname === "/api/applicability-review") {
+        const payload = await readJson(request);
+        return json(response, 201, await browserMutation(input, options, {
+          message: (result) => `Record ${result.reviewedIds?.length || 0} applicability decisions`
+        }, () => applyApplicabilityReview(input, {
+          ...payload,
+          confirmed: payload.confirmed === true
+        })));
+      }
+      if (request.method === "POST" && url.pathname === "/api/collection-review/preview") {
+        return json(response, 200, await planCollectionReview(input, await readJson(request)));
+      }
+      if (request.method === "POST" && url.pathname === "/api/collection-review") {
+        const payload = await readJson(request);
+        return json(response, 201, await browserMutation(input, options, {
+          message: (result) => `Confirm ${result.assessment?.configuration?.title || payload.resourceType}`
+        }, () => applyCollectionReview(input, {
+          ...payload,
+          confirmed: payload.confirmed === true
+        })));
+      }
       if (request.method === "POST" && url.pathname === "/api/obligation-events") {
         const payload = await readJson(request);
         return json(response, 201, await browserMutation(input, options, {
@@ -68,10 +183,77 @@ export function createFilegrcServer(input = process.cwd(), options = {}) {
           obligationId: payload.obligationId,
           record: payload.record,
           content: payload.content,
-          expectedRevision: payload.revision
+          expectedRevision: requireRevision(payload.revision, `obligation/${payload.obligationId}`)
         }));
         return json(response, 201, result);
       }
+      if (request.method === "POST" && url.pathname === "/api/action-completions") {
+        const payload = await readJson(request);
+        if (!safeSegment(payload.actionItemId)) {
+          return json(response, 400, { error: "A safe Action Item ID is required." });
+        }
+        const result = await browserMutation(input, options, {
+          message: () => `Complete action item: ${payload.actionItemId}`
+        }, () => completeObligationAction(input, {
+          actionItemId: payload.actionItemId,
+          completedOn: payload.completedOn,
+          record: payload.record,
+          content: payload.content,
+          expectedRevision: requireRevision(payload.revision, `action-item/${payload.actionItemId}`)
+        }));
+        return json(response, 201, result);
+      }
+      if (request.method === "POST" && url.pathname === "/api/obligation-event-completions") {
+        const payload = await readJson(request);
+        if (!safeSegment(payload.eventId)) {
+          return json(response, 400, { error: "A safe Policy Event ID is required." });
+        }
+        const result = await browserMutation(input, options, {
+          message: () => `Complete policy event: ${payload.eventId}`
+        }, () => completeObligationEvent(input, {
+          eventId: payload.eventId,
+          completedOn: payload.completedOn,
+          expectedRevision: requireRevision(payload.revision, `obligation-event/${payload.eventId}`)
+        }));
+        return json(response, 200, result);
+      }
+      const attachmentMatch = /^\/api\/evidence-attachments\/([^/]+)\/([^/]+)$/.exec(url.pathname);
+      if (attachmentMatch && request.method === "POST") {
+        const evidenceId = decodeURIComponent(attachmentMatch[1]);
+        const attachment = decodeURIComponent(attachmentMatch[2]);
+        if (!safeSegment(evidenceId) || !safeAttachmentName(attachment)) {
+          return json(response, 400, { error: "Safe Evidence and attachment identifiers are required." });
+        }
+        const revision = requireRevision(url.searchParams.get("revision"), `evidence/${evidenceId}`);
+        const temporaryDirectory = await mkdtemp(join(tmpdir(), "filegrc-evidence-upload-"));
+        try {
+          const temporaryPath = join(temporaryDirectory, attachment);
+          await writeFile(temporaryPath, await readBytes(request));
+          const result = await browserMutation(input, options, {
+            message: () => `Attach evidence file: ${attachment}`
+          }, () => addEvidenceAttachment(input, evidenceId, temporaryPath, {
+            name: attachment,
+            expectedRevision: revision
+          }));
+          return json(response, 201, result);
+        } finally {
+          await rm(temporaryDirectory, { recursive: true, force: true });
+        }
+      }
+      if (attachmentMatch && request.method === "DELETE") {
+        const evidenceId = decodeURIComponent(attachmentMatch[1]);
+        const attachment = decodeURIComponent(attachmentMatch[2]);
+        if (!safeSegment(evidenceId) || !safeAttachmentName(attachment)) {
+          return json(response, 400, { error: "Safe Evidence and attachment identifiers are required." });
+        }
+        const revision = requireRevision(url.searchParams.get("revision"), `evidence/${evidenceId}`);
+        const result = await browserMutation(input, options, {
+          message: () => `Remove evidence file: ${attachment}`
+        }, () => removeEvidenceAttachment(input, evidenceId, attachment, {
+          expectedRevision: revision
+        }));
+        return json(response, 200, result);
+      }
       if (request.method === "GET" && url.pathname === "/api/evidence-packet") {
         return json(response, 200, await prepareEvidencePacket(input, {
           start: url.searchParams.get("start"),
@@ -102,49 +284,66 @@ export function createFilegrcServer(input = process.cwd(), options = {}) {
           message: () => `Prepare audit: ${payload.auditId || "engagement"}`
         }, () => prepareAuditWorkspace(input, payload)));
       }
-      if (request.method === "POST" && url.pathname === "/api/evidence-test-drafts") {
-        return json(response, 201, await browserMutation(input, options, {
-          message: "Create evidence collection test drafts"
-        }, () => ensureEvidenceTestDrafts(input)));
-      }
       if (request.method === "POST" && url.pathname === "/api/setup") {
         const payload = await readJson(request);
-        return json(response, 200, await browserMutation(input, options, {
-          message: (result) => `${payload.draft === true ? "Save onboarding draft" : "Complete onboarding"} for ${result.workspace.organizationName}`
-        }, () => setupWorkspace(input, payload)));
+        const completeSetup = async () => {
+          return browserMutation(input, options, {
+            message: (setupResult) => `${payload.draft === true ? "Save onboarding draft" : "Complete onboarding"} for ${setupResult.workspace.organizationName}`
+          }, () => setupWorkspace(input, payload));
+        };
+        return json(response, 200, await completeSetup());
       }
       if (request.method === "POST" && url.pathname === "/api/resources") {
-        const payload = await readJson(request);
-        const record = payload.record ?? payload;
+        const payload = normalizeResourceMutation(await readJson(request));
+        const { record } = payload;
         const result = await browserMutation(input, options, {
           message: () => `Create ${resourceTypeLabel(record.type)}: ${record.title || record.id}`
-        }, () => createResource(input, record, { content: payload.record ? payload.content : undefined }));
-        return json(response, 201, { record: result.record, synchronization: result.synchronization });
+        }, () => createResource(input, record, { content: payload.content }));
+        return json(response, 201, result);
       }
       if (request.method === "POST" && url.pathname === "/api/commit") {
         await requireManualBrowserGit(input, options);
         const payload = await readJson(request);
-        return json(response, 201, await commitAndPushWorkspace(input, payload.message));
+        return json(response, 201, await manualGitResultWithState(input, options, () => commitAndPushWorkspace(input, payload.message)));
       }
       if (request.method === "POST" && url.pathname === "/api/git/pull") {
         await requireManualBrowserGit(input, options);
-        return json(response, 200, await pullWorkspace(input));
+        return json(response, 200, await manualGitResultWithState(input, options, () => pullWorkspace(input)));
       }
       if (request.method === "POST" && url.pathname === "/api/git/push") {
         await requireManualBrowserGit(input, options);
-        return json(response, 200, await pushWorkspace(input));
+        return json(response, 200, await manualGitResultWithState(input, options, () => pushWorkspace(input)));
       }
       if (request.method === "POST" && url.pathname === "/api/git/retry-sync") {
-        return json(response, 200, await retryBrowserSync(input, {
+        const result = await retryBrowserSync(input, {
           allowNonAuthoritativeWrites: options.allowNonAuthoritativeWrites
-        }));
+        });
+        const state = await createAppState(input, {
+          allowNonAuthoritativeWrites: options.allowNonAuthoritativeWrites,
+          includeDetails: false
+        });
+        return json(response, 200, { ...result, state });
+      }
+      if (request.method === "GET" && url.pathname === "/api/git/sync-status") {
+        const repository = await getBrowserRepositoryState(input, {
+          allowNonAuthoritativeWrites: options.allowNonAuthoritativeWrites
+        });
+        const git = getGitSummary(input);
+        delete git.root;
+        return json(response, 200, {
+          repository,
+          git,
+          readOnly: repository.mode === "trunk" && !repository.writesAllowed
+        });
       }
       if (request.method === "PUT" && url.pathname === "/api/content") {
         const payload = await readJson(request);
         const result = await browserMutation(input, options, {
           message: () => `Update content: ${payload.path}`
-        }, () => updateContent(input, payload.path, payload.source, { expectedRevision: payload.revision }));
-        return json(response, 200, { path: result.dataRelativePath, synchronization: result.synchronization });
+        }, () => updateContent(input, payload.path, payload.source, {
+          expectedRevision: requireRevision(payload.revision, `content/${payload.path}`)
+        }));
+        return json(response, 200, result);
       }
       const match = /^\/api\/resource\/([^/]+)\/([^/]+)$/.exec(url.pathname);
       if (match) {
@@ -152,32 +351,40 @@ export function createFilegrcServer(input = process.cwd(), options = {}) {
         const id = decodeURIComponent(match[2]);
         if (!safeSegment(type) || !safeSegment(id)) return json(response, 400, { error: "Unsafe resource identifier." });
         if (request.method === "GET") {
-          const state = await createAppState(input);
-          const entry = state.resources.find(({ record }) => record.type === type && record.id === id);
-          return entry ? json(response, 200, entry) : json(response, 404, { error: "Resource not found." });
+          const entry = await createResourceDetail(input, type, id);
+          if (!entry) return json(response, 404, { error: "Resource not found." });
+          if (url.searchParams.get("workflow") === "true") {
+            const workflow = await assessWorkflow(input);
+            entry.workflow = workflowForResource(workflow, type, id);
+          }
+          return json(response, 200, entry);
         }
         if (request.method === "PUT") {
-          const payload = await readJson(request);
-          const record = payload.record ?? payload;
+          const payload = normalizeResourceMutation(await readJson(request), { requireRevision: true });
+          const { record } = payload;
           const result = await browserMutation(input, options, {
             message: () => `Update ${resourceTypeLabel(type)}: ${record.title || id}`
           }, () => updateResource(input, type, id, record, {
-            content: payload.record ? payload.content : undefined,
+            content: payload.content,
             expectedRevision: payload.revision,
-            expectedContentRevisions: payload.contentRevisions
+            expectedContentRevisions: payload.contentRevisions,
+            requireExpectedContentRevisions: true
           }));
-          return json(response, 200, { record: result.record, synchronization: result.synchronization });
+          return json(response, 200, result);
         }
         if (request.method === "DELETE") {
+          const revision = requireRevision(url.searchParams.get("revision"), `${type}/${id}`);
           const result = await browserMutation(input, options, {
             message: () => `Delete ${resourceTypeLabel(type)}: ${id}`
-          }, () => deleteResource(input, type, id, { expectedRevision: url.searchParams.get("revision") }));
+          }, () => deleteResource(input, type, id, { expectedRevision: revision }));
           return json(response, 200, {
             deleted: true,
             type,
             id,
             deletedContent: result.deletedContent,
-            synchronization: result.synchronization
+            synchronization: result.synchronization,
+            workflowDelta: result.workflowDelta,
+            state: result.state
           });
         }
       }
@@ -232,25 +439,81 @@ export async function serveWorkspace(input = process.cwd(), options = {}) {
   getResourceDefinition(loaded.model, "workspace");
   const server = createFilegrcServer(loaded.root, {
     allowedHosts: [host],
-    allowNonAuthoritativeWrites: options.allowNonAuthoritativeWrites === true
-  });
-  await new Promise((resolve, reject) => {
-    server.once("error", reject);
-    server.listen(port, host, resolve);
+    allowNonAuthoritativeWrites: options.allowNonAuthoritativeWrites === true,
+    backgroundPushDelayMs: options.backgroundPushDelayMs
   });
+  let usedFallbackPort = false;
+  try {
+    await listen(server, port, host);
+  } catch (error) {
+    if (
+      error.code !== "EADDRINUSE"
+      || port === 0
+      || options.fallbackToAvailablePort !== true
+    ) {
+      throw error;
+    }
+    await listen(server, 0, host);
+    usedFallbackPort = true;
+  }
   return {
     server,
     root: loaded.root,
     address: server.address(),
-    url: `http://${urlHost(host)}:${server.address().port}`
+    url: `http://${urlHost(host)}:${server.address().port}`,
+    requestedPort: port,
+    usedFallbackPort
   };
 }
 
+function listen(server, port, host) {
+  return new Promise((resolve, reject) => {
+    const onError = (error) => {
+      server.off("listening", onListening);
+      reject(error);
+    };
+    const onListening = () => {
+      server.off("error", onError);
+      resolve();
+    };
+    server.once("error", onError);
+    server.once("listening", onListening);
+    server.listen(port, host);
+  });
+}
+
 function browserMutation(input, options, mutationOptions, task) {
-  return runBrowserMutation(input, {
-    ...mutationOptions,
-    allowNonAuthoritativeWrites: options.allowNonAuthoritativeWrites === true
-  }, task);
+  const run = () => serializeWorkspaceMutation(input, async (root) => {
+    const workflowBefore = await assessWorkflow(root);
+    const result = await runBrowserMutation(root, {
+      ...mutationOptions,
+      allowNonAuthoritativeWrites: options.allowNonAuthoritativeWrites === true,
+      backgroundPushDelayMs: options.backgroundPushDelayMs
+    }, task);
+    const state = await measureTiming("state", () => createAppState(root, {
+      allowNonAuthoritativeWrites: options.allowNonAuthoritativeWrites,
+      includeDetails: false,
+      validationProof: result?.[BROWSER_VALIDATION]
+    }));
+    if (result?.synchronization?.status === "syncing" && state.repository.status !== "syncing") {
+      result.synchronization = {
+        ...result.synchronization,
+        status: state.repository.status === "synced" ? "synced" : "not-synced",
+        synchronizedAt: state.repository.lastSuccessfulSynchronization ?? null,
+        pushError: state.repository.backgroundSyncError ?? null
+      };
+    }
+    return {
+      ...result,
+      workflowDelta: buildWorkflowDelta(workflowBefore, state.workflow),
+      state
+    };
+  });
+  if (!timingEnabled()) return run();
+  return collectTimings(run).then(({ result, timings }) => {
+    console.error(`[filegrc timing] ${JSON.stringify({ operation: "browser-mutation", ...timings })}`);
+    return result;
+  });
 }
 
 async function requireManualBrowserGit(input, options) {
@@ -262,6 +525,15 @@ async function requireManualBrowserGit(input, options) {
   }
 }
 
+async function manualGitResultWithState(input, options, task) {
+  const result = await task();
+  const state = await createAppState(input, {
+    allowNonAuthoritativeWrites: options.allowNonAuthoritativeWrites,
+    includeDetails: false
+  });
+  return { ...result, state };
+}
+
 function resourceTypeLabel(value) {
   return String(value || "record").replaceAll("-", " ");
 }
@@ -283,6 +555,18 @@ async function readJson(request) {
   return value;
 }
 
+async function readBytes(request) {
+  const chunks = [];
+  let size = 0;
+  for await (const chunk of request) {
+    size += chunk.length;
+    if (size > 25_000_000) throw new Error("Attachment exceeds 25 MB.");
+    chunks.push(chunk);
+  }
+  if (size === 0) throw new Error("An attachment body is required.");
+  return Buffer.concat(chunks);
+}
+
 function json(response, status, value) {
   text(response, status, `${JSON.stringify(value, null, 2)}\n`, "application/json; charset=utf-8");
 }
@@ -319,6 +603,20 @@ function safeSegment(value) {
   return /^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(value);
 }
 
+function safeAttachmentName(value) {
+  return typeof value === "string"
+    && value.length > 0
+    && value === value.split(/[\\/]/).at(-1)
+    && !value.startsWith(".");
+}
+
+function requireRevision(value, target) {
+  if (typeof value !== "string" || value.length === 0) {
+    throw new Error(`A revision is required when changing ${target}. Reload the resource and try again.`);
+  }
+  return value;
+}
+
 function sameOrigin(request) {
   const origin = request.headers.origin;
   if (!origin) return true;
@@ -362,9 +660,10 @@ function urlHost(host) {
 function statusFor(error) {
   if (error instanceof SyntaxError || error instanceof URIError) return 400;
   if (/exceeds 2 MB/i.test(error.message)) return 413;
+  if (/exceeds 25 MB/i.test(error.message)) return 413;
   if (/changed after you opened|source changed|revision changed/i.test(error.message)) return 409;
   if (/already exists|target file already exists/i.test(error.message)) return 409;
-  if (/Git could not|upstream branch|multiple remotes|no Git remote|configured repository remote|safe Git name|check out a branch|before trying to (?:pull|push)|authoritative branch|not synchronized|not synced|diverged|waiting to be pushed|Retry sync|outside this FileGRC workspace|worktree has uncommitted changes|development write override|browser commit, pull, and push/i.test(error.message)) return 409;
+  if (/Git could not|upstream branch|multiple remotes|no Git remote|configured repository remote|safe Git name|check out a branch|before trying to (?:pull|push)|authoritative branch|not synchronized|not synced|diverged|waiting to be pushed|Retry sync|background push|outside this FileGRC workspace|worktree has uncommitted changes|development write override|browser commit, pull, and push/i.test(error.message)) return 409;
   if (/not found|ENOENT/i.test(error.message)) return 404;
   if (/invalid|required|unsafe|match|workspace|singleton|commit message|no changes|git history|git user|unknown resource type|must use|must be|content path|data path|path leaves|valid .*date|not found|no active obligations|end date|through date|already exists|EEXIST/i.test(error.message)) return 400;
   return 500;