filegrc 0.12.2 → 0.12.4

package/src/validate.js

@@ -1,4 +1,5 @@
 import { createHash } from "node:crypto";
+import { createReadStream } from "node:fs";
 import { readFile, stat } from "node:fs/promises";
 import { performance } from "node:perf_hooks";
 import { getResourceDefinition, modelSupports } from "../model/index.js";
@@ -36,6 +37,17 @@ import {
 } from "./reporting-route-integrity.js";
 import { validateWorkflowHistoryIntegrity } from "./workflow-history-integrity.js";
 
+let fingerprintFileReadObserver = null;
+
+export function setFingerprintFileReadObserverForTests(observer) {
+  if (observer !== null && typeof observer !== "function") {
+    throw new TypeError("The fingerprint file-read observer must be a function or null.");
+  }
+  const previous = fingerprintFileReadObserver;
+  fingerprintFileReadObserver = observer;
+  return () => { fingerprintFileReadObserver = previous; };
+}
+
 const ID_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
 const NAMESPACE_PATTERN = /^[a-z][a-z0-9]*(?:[.-][a-z0-9]+)*$/;
 const DATE_PATTERN = /^\d{4}-\d{2}-\d{2}$/;
@@ -218,14 +230,30 @@ async function validateWorkspaceUnmeasured(input) {
   }
   if (modelSupports(loaded.model, "guided-workflow")) {
     const { planReconciliation, validateReconciliationDismissals } = await import("./reconciliation.js");
-    const rawReconciliation = await validateReconciliationDismissals(loaded, diagnostics);
-    reconciliation = rawReconciliation?.filteredPlan ?? await planReconciliation(loaded);
-    for (const candidate of reconciliation.candidates.filter(({ committedRevision }) => committedRevision)) {
-      diagnostics.push(warning(
-        "unreconciled-committed-transition",
-        candidate.sourcePath,
-        `${candidate.eventType} transition in Git commit ${candidate.committedRevision.slice(0, 8)} still needs confirmation or dismissal.`
+    try {
+      const rawReconciliation = await validateReconciliationDismissals(loaded, diagnostics);
+      reconciliation = rawReconciliation?.filteredPlan ?? await planReconciliation(loaded);
+      for (const candidate of reconciliation.candidates.filter(({ committedRevision }) => committedRevision)) {
+        diagnostics.push(warning(
+          "unreconciled-committed-transition",
+          candidate.sourcePath,
+          `${candidate.eventType} transition in Git commit ${candidate.committedRevision.slice(0, 8)} still needs confirmation or dismissal.`
+        ));
+      }
+    } catch (cause) {
+      if (cause?.code === "FILEGRC_GIT_DEADLINE") throw cause;
+      diagnostics.push(error(
+        "reconciliation-history-unavailable",
+        "data/workspace.json",
+        cause instanceof Error ? cause.message : "Git history is unavailable for reconciliation."
       ));
+      reconciliation = {
+        contractVersion: 1,
+        gitRevision: null,
+        changedPaths: [],
+        candidates: [],
+        unavailable: true
+      };
     }
   }
 
@@ -1209,19 +1237,27 @@ function validateCollectionReviewSet(resources, pathById, diagnostics) {
   }
 }
 
-export async function fingerprintWorkspace(input = process.cwd()) {
+export async function fingerprintWorkspace(input = process.cwd(), options = {}) {
   const loaded = typeof input === "object" && input.entries ? input : await loadWorkspace(input);
   const hash = createHash("sha256");
   hash.update(`model\0${loaded.model.modelVersion}\0`);
-  for (const entry of [...loaded.entries].sort((a, b) => a.relativePath.localeCompare(b.relativePath))) {
+  const sourceEntries = loaded.sourceEntries || loaded.entries;
+  const fingerprintBudget = { uncachedBytes: 0 };
+  for (const entry of [...sourceEntries].sort((a, b) => a.relativePath.localeCompare(b.relativePath))) {
     hash.update(`record\0${entry.relativePath}\0${entry.source.length}\0${entry.source}`);
+  }
+  for (const entry of [...loaded.entries].sort((a, b) => a.relativePath.localeCompare(b.relativePath))) {
     const definition = loaded.model.resources[entry.record?.type];
     if (!definition) continue;
     for (const item of markdownEntries(loaded.model, entry.record).sort((a, b) => a.path.localeCompare(b.path))) {
       try {
-        const source = await readFile(resolveDataPath(loaded.root, item.path), "utf8");
-        hash.update(`markdown\0${item.path}\0${source.length}\0${source}`);
-      } catch {
+        const resolvedPath = resolveDataPath(loaded.root, item.path);
+        const file = await stat(resolvedPath);
+        if (!file.isFile()) throw new Error("Markdown path is not a file.");
+        const content = await fingerprintFileContent(resolvedPath, file, options, fingerprintBudget);
+        hash.update(`markdown\0${item.path}\0${content.size}\0${content.digest}\0`);
+      } catch (error) {
+        if (error?.code === "FILEGRC_FINGERPRINT_BUDGET") throw error;
         hash.update(`markdown-missing\0${item.path}\0`);
       }
     }
@@ -1232,9 +1268,18 @@ export async function fingerprintWorkspace(input = process.cwd()) {
       const paths = Array.isArray(value) ? value : value === undefined || value === null ? [] : [value];
       for (const path of [...paths].sort()) {
         try {
-          const file = await stat(resolveDataPath(loaded.root, path));
-          hash.update(`data-path\0${path}\0${file.isFile() ? "file" : "other"}\0`);
-        } catch {
+          const resolvedPath = resolveDataPath(loaded.root, path);
+          const file = await stat(resolvedPath);
+          if (file.isFile()) {
+            const identity = `${file.dev}:${file.ino}:${file.mode}:${file.size}:${file.mtimeMs}:${file.ctimeMs}`;
+            let content = options.fileDigestCache?.get(resolvedPath);
+            if (!content || content.identity !== identity) {
+              content = await fingerprintFileContent(resolvedPath, file, options, fingerprintBudget);
+            }
+            hash.update(`data-path\0${path}\0file\0${content.size}\0${content.digest}\0`);
+          } else hash.update(`data-path\0${path}\0other\0`);
+        } catch (error) {
+          if (error?.code === "FILEGRC_FINGERPRINT_BUDGET") throw error;
           hash.update(`data-path-missing\0${path}\0`);
         }
       }
@@ -1243,6 +1288,41 @@ export async function fingerprintWorkspace(input = process.cwd()) {
   return { fingerprint: hash.digest("hex"), loaded };
 }
 
+async function fingerprintFileContent(path, file, options, budget) {
+  const identity = `${file.dev}:${file.ino}:${file.mode}:${file.size}:${file.mtimeMs}:${file.ctimeMs}`;
+  const cached = options.fileDigestCache?.get(path);
+  if (cached?.identity === identity) return cached;
+  assertFingerprintBudget(options, budget.uncachedBytes + file.size);
+  budget.uncachedBytes += file.size;
+  const contentHash = createHash("sha256");
+  let size = 0;
+  await fingerprintFileReadObserver?.(path);
+  for await (const chunk of createReadStream(path)) {
+    assertFingerprintBudget(options, budget.uncachedBytes - file.size + size + chunk.length);
+    size += chunk.length;
+    contentHash.update(chunk);
+  }
+  const content = { identity, size, digest: contentHash.digest("hex") };
+  if (options.fileDigestCache?.size < 20_000 || options.fileDigestCache?.has(path)) {
+    options.fileDigestCache.set(path, content);
+  }
+  return content;
+}
+
+function assertFingerprintBudget(options, uncachedBytes) {
+  if (options.deadlineAt !== undefined && performance.now() >= options.deadlineAt) {
+    const error = new Error("Workspace attachment verification exceeded its time limit. Reload and try again.");
+    error.code = "FILEGRC_FINGERPRINT_BUDGET";
+    throw error;
+  }
+  if (options.maxUncachedFileBytes !== undefined
+    && uncachedBytes > options.maxUncachedFileBytes) {
+    const error = new Error("Workspace file verification exceeded its byte limit. Reload and try again.");
+    error.code = "FILEGRC_FINGERPRINT_BUDGET";
+    throw error;
+  }
+}
+
 function validateImplementedControlSchedules(record, obligationsByControl, path, diagnostics) {
   if (record.type !== "control" || record.status !== "implemented") return;
   const schedules = obligationsByControl.get(record.id) || [];
@@ -2134,6 +2214,23 @@ function validateLocation(record, definition, relativePath, diagnostics) {
   }
 }
 
+export function assertHistoricalRecordValid(record, model, relativePath) {
+  const displayPath = `data/${relativePath}`;
+  const diagnostics = [];
+  let definition;
+  try {
+    definition = getResourceDefinition(model, record?.type);
+  } catch {
+    throw new Error(`Unknown historical resource type "${record?.type ?? ""}" at ${displayPath}.`);
+  }
+  validateLocation(record, definition, relativePath, diagnostics);
+  validateRecord(record, definition, model, displayPath, diagnostics);
+  validateDateRanges(record, displayPath, diagnostics);
+  if (diagnostics.some(({ severity }) => severity === "error")) {
+    throw new Error(`Invalid historical resource at ${displayPath}: ${diagnostics.map(({ message }) => message).join(" ")}`);
+  }
+}
+
 function validateRecord(record, definition, model, path, diagnostics) {
   const fields = { ...model.commonFields, ...definition.fields };
   const required = new Set([

package/src/web.js

@@ -95,6 +95,7 @@ let repositorySyncPollInFlight = false;
 let mutationStateRefreshInFlight = false;
 let mutationStateRefreshTimer = null;
 let programSelectionGeneration = 0;
+let expiredStateRefresh = null;
 
 start().catch((error) => {
   root.innerHTML = '<main class="fatal"><h1>Could Not Load the Workspace</h1><pre></pre></main>';
@@ -193,14 +194,22 @@ function loadStateForRoute() {
   for (const section of desiredStateSections(route)) loadStateSection(section);
 }
 
-function loadStateSection(section) {
+function loadStateSection(section, refreshExpired = true) {
   if (!state.stateToken || state.sections?.[section] === "complete") return Promise.resolve();
   if (stateSectionRequests.has(section)) return stateSectionRequests.get(section);
   state.sections[section] = "loading";
   const token = state.stateToken;
   const programQuery = state.selectedProgramId ? "&programId=" + encodeURIComponent(state.selectedProgramId) : "";
-  const request = fetchJson("/api/state/" + encodeURIComponent(section) + "?token=" + encodeURIComponent(token) + programQuery)
-    .then((result) => {
+  const request = localFetch("/api/state/" + encodeURIComponent(section) + "?token=" + encodeURIComponent(token) + programQuery)
+    .then(async (response) => {
+      if (response.status === 409 && refreshExpired && state.stateToken === token) {
+        await refreshExpiredAppState(token);
+        render();
+        if (stateSectionRequests.get(section) === request) stateSectionRequests.delete(section);
+        return loadStateSection(section, false);
+      }
+      if (!response.ok) throw new Error(await responseMessage(response));
+      const result = await response.json();
       if (state.stateToken !== result.stateToken) return;
       Object.assign(state, result.state);
       state.sections[section] = "complete";
@@ -2909,17 +2918,35 @@ function evidenceAttachmentPanel(entry) {
 }
 
 async function loadResourceDetail(type, id) {
-  const key = type + "\0" + id;
+  const key = (state.stateToken || "live") + "\0" + type + "\0" + id;
   if (resourceDetailRequests.has(key)) return resourceDetailRequests.get(key);
   const request = (async () => {
     try {
-      const response = await localFetch("/api/resource/" + encodeURIComponent(type) + "/" + encodeURIComponent(id));
-      if (!response.ok) throw new Error(await responseMessage(response));
-      const detail = await response.json();
+      let token = state.stateToken;
+      let detail = null;
+      for (let attempt = 0; attempt < 3; attempt += 1) {
+        const tokenQuery = token ? "?token=" + encodeURIComponent(token) : "";
+        const response = await localFetch("/api/resource/" + encodeURIComponent(type) + "/" + encodeURIComponent(id) + tokenQuery);
+        if (response.status === 409 && token) {
+          await refreshExpiredAppState(token);
+          token = state.stateToken;
+          continue;
+        }
+        if (!response.ok) throw new Error(await responseMessage(response));
+        detail = await response.json();
+        if (!token || (detail.stateToken === token && state.stateToken === token)) break;
+        token = state.stateToken;
+        detail = null;
+      }
+      if (!detail) throw new Error("The workspace state changed while loading this record. Reload and try again.");
+      delete detail.stateToken;
       const index = state.resources.findIndex(({ record }) => record.type === type && record.id === id);
       if (index >= 0) state.resources[index] = detail;
       const route = parseRoute();
-      if (route.name === "detail" && route.type === type && route.id === id) render();
+      if (route.name === "detail" && route.type === type && route.id === id) {
+        render();
+        loadStateForRoute();
+      }
     } catch (error) {
       const route = parseRoute();
       if (route.name === "detail" && route.type === type && route.id === id) {
@@ -2927,13 +2954,52 @@ async function loadResourceDetail(type, id) {
         if (main) main.innerHTML = '<div class="page"><section class="panel"><div class="dialog-error" role="alert">' + esc(error.message) + '</div></section></div>';
       }
     } finally {
-      resourceDetailRequests.delete(key);
+      for (const [requestKey, pending] of resourceDetailRequests) {
+        if (pending === request) resourceDetailRequests.delete(requestKey);
+      }
     }
   })();
   resourceDetailRequests.set(key, request);
   return request;
 }
 
+function refreshExpiredAppState(expectedToken) {
+  if (expiredStateRefresh) {
+    if (expiredStateRefresh.token === expectedToken) return expiredStateRefresh.promise;
+    return expiredStateRefresh.promise
+      .catch(() => {})
+      .then(() => refreshExpiredAppState(expectedToken));
+  }
+  if (expectedToken && state.stateToken !== expectedToken) return Promise.resolve();
+  const programQuery = state.selectedProgramId ? "?programId=" + encodeURIComponent(state.selectedProgramId) : "";
+  const holder = { token: expectedToken, promise: null };
+  const refresh = (async () => {
+    const response = await fetch("/api/state/bootstrap" + programQuery);
+    if (!response.ok) throw new Error(await responseMessage(response));
+    const next = normalizeAppState(await response.json());
+    if (expectedToken && state.stateToken !== expectedToken) return;
+    rekeyResourceDetailRequests(state.stateToken, next.stateToken);
+    state = next;
+    stateSectionRequests.clear();
+  })().finally(() => {
+    if (expiredStateRefresh === holder) expiredStateRefresh = null;
+  });
+  holder.promise = refresh;
+  expiredStateRefresh = holder;
+  return refresh;
+}
+
+function rekeyResourceDetailRequests(previousToken, nextToken) {
+  if (!previousToken || !nextToken || previousToken === nextToken) return;
+  const prefix = previousToken + "\0";
+  for (const [key, request] of resourceDetailRequests) {
+    if (!key.startsWith(prefix)) continue;
+    const nextKey = nextToken + key.slice(previousToken.length);
+    resourceDetailRequests.delete(key);
+    if (!resourceDetailRequests.has(nextKey)) resourceDetailRequests.set(nextKey, request);
+  }
+}
+
 function issueSeed(type, source) {
   const owners = source.ownerIds || source.reviewerIds || source.assessorIds || source.testerIds || [];
   if (type === "finding") {

package/src/workflow-history-integrity.js

@@ -116,7 +116,7 @@ export async function validateWorkflowHistoryIntegrity(loaded, diagnostics) {
       entry.record.type === "exception"
       && historical.record.reportingRouteSetId
       && historical.commit !== commits.find((commit) => {
-        try { return Number(JSON.parse(getFileAtRevision(loaded.root, commit, "data/workspace.json"))?.dataModelVersion) >= 10; } catch { return false; }
+        try { return Number(JSON.parse(getFileAtRevision(loaded.root, commit, "data/workspace.json"))?.dataModelVersion) >= 10; } catch (error) { rethrowGitDeadline(error); return false; }
       })
     ) {
       const records = recordsAtRevision(loaded, historical.commit);
@@ -146,7 +146,7 @@ export async function validateWorkflowHistoryIntegrity(loaded, diagnostics) {
   }
 
   const firstV10Commit = commits.find((commit) => {
-    try { return Number(JSON.parse(getFileAtRevision(loaded.root, commit, "data/workspace.json"))?.dataModelVersion) >= 10; } catch { return false; }
+    try { return Number(JSON.parse(getFileAtRevision(loaded.root, commit, "data/workspace.json"))?.dataModelVersion) >= 10; } catch (error) { rethrowGitDeadline(error); return false; }
   });
   const protectedRouteSources = new Map();
   for (const [commit, records] of revisionRecords) {
@@ -307,7 +307,8 @@ function legacyCollectionReviewTransition(loaded, previous, current) {
   try {
     const workspace = JSON.parse(getFileAtRevision(loaded.root, current.commit, "data/workspace.json"));
     return Number(workspace?.dataModelVersion) < 9;
-  } catch {
+  } catch (error) {
+    rethrowGitDeadline(error);
     return false;
   }
 }
@@ -319,7 +320,8 @@ function historicalState(loaded) {
     try {
       const workspace = JSON.parse(getFileAtRevision(loaded.root, commit, "data/workspace.json"));
       return Number(workspace?.dataModelVersion) >= 9;
-    } catch {
+    } catch (error) {
+      rethrowGitDeadline(error);
       return false;
     }
   });
@@ -392,7 +394,8 @@ function legacyCollectionReview(loaded, historical) {
   let workspace;
   try {
     workspace = JSON.parse(getFileAtRevision(loaded.root, historical.commit, "data/workspace.json"));
-  } catch {
+  } catch (error) {
+    rethrowGitDeadline(error);
     return false;
   }
   return Number(workspace?.dataModelVersion) < 9
@@ -408,7 +411,8 @@ function recordAtRevision(loaded, commit, id, identityHistory) {
   try {
     const record = JSON.parse(source);
     return record?.id === id ? { record, path: item.path } : null;
-  } catch {
+  } catch (error) {
+    rethrowGitDeadline(error);
     return null;
   }
 }
@@ -454,7 +458,7 @@ async function validateReportingRouteAuthorityHistory(loaded, commits, revisionR
   ));
   if (!routeSets.length) return;
   const firstV10Commit = commits.find((commit) => {
-    try { return Number(JSON.parse(getFileAtRevision(loaded.root, commit, "data/workspace.json"))?.dataModelVersion) >= 10; } catch { return false; }
+    try { return Number(JSON.parse(getFileAtRevision(loaded.root, commit, "data/workspace.json"))?.dataModelVersion) >= 10; } catch (error) { rethrowGitDeadline(error); return false; }
   });
   for (const route of routeSets) {
     const versions = changedVersions(route.id, commits, revisionRecords, historiesById);
@@ -733,7 +737,8 @@ function authorityAppointmentOverlapsRoute(appointment, route, workspaceId, time
     routeEnd = route.cancellation?.canceledAt
       ? currentCalendarDate(timezone, new Date(route.cancellation.canceledAt))
       : null;
-  } catch {
+  } catch (error) {
+    rethrowGitDeadline(error);
     return false;
   }
   return (!routeEnd || appointment.startsOn <= routeEnd)
@@ -786,7 +791,8 @@ function historicalWorkspaceTimezoneAtRevision(loaded, commit) {
     if (typeof workspace?.timezone !== "string" || !workspace.timezone.trim()) return null;
     new Intl.DateTimeFormat("en", { timeZone: workspace.timezone }).format();
     return workspace.timezone;
-  } catch {
+  } catch (error) {
+    rethrowGitDeadline(error);
     return null;
   }
 }
@@ -901,3 +907,7 @@ function integrityError(entry, message) {
     message
   };
 }
+
+function rethrowGitDeadline(error) {
+  if (error?.code === "FILEGRC_GIT_DEADLINE") throw error;
+}

package/src/workflow.js

@@ -141,7 +141,9 @@ async function assessWorkflowUnmeasured(input, options = {}) {
     asOf,
     evaluatedAt,
     programId: programRecord.type === "program" ? programRecord.id : undefined,
-    controlIds: options.auditId ? audits[0]?.controlIds : programRecord?.controlIds
+    controlIds: options.auditId ? audits[0]?.controlIds : programRecord?.controlIds,
+    historyDeadlineAt: options.historyDeadlineAt,
+    strictHistory: options.strictHistory
   });
   const guidedFindings = [
     ...programFindings(program),
@@ -732,7 +734,10 @@ async function assessPeriodHealth(loaded, options) {
     "component"
   ].includes(record.type));
   const paths = relevantEntries.map(({ relativePath }) => `data/${relativePath}`);
-  const histories = getWorkspaceHistories(loaded.root, paths, 50);
+  const histories = getWorkspaceHistories(loaded.root, paths, 50, {
+    deadlineAt: options.historyDeadlineAt,
+    strict: options.strictHistory === true
+  });
   const allHistory = [...histories.values()].flat().filter(Boolean);
   const earliestCommitDate = allHistory
     .map(({ timestamp }) => timestamp?.slice(0, 10))

package/src/workspace.js

@@ -14,11 +14,13 @@ async function loadWorkspaceUnmeasured(input) {
   const diagnostics = [];
   const files = await collectJsonFiles(dataRoot);
   const resources = [];
+  const sourceEntries = [];
 
   for (const path of files) {
     const relativePath = relative(dataRoot, path).split(sep).join("/");
     try {
       const source = await readFile(path, "utf8");
+      sourceEntries.push({ path, relativePath, source });
       const record = JSON.parse(source);
       resources.push({ record, path, relativePath, source });
     } catch (error) {
@@ -68,6 +70,7 @@ async function loadWorkspaceUnmeasured(input) {
     model,
     workspace: workspaceEntry?.record ?? null,
     entries: resources,
+    sourceEntries,
     resources: resources.map(({ record }) => record),
     diagnostics
   };