filegrc 0.15.1 → 0.15.3

package/src/retention-schedule-approval.js

@@ -3,7 +3,7 @@ import { readFileSync } from "node:fs";
 import { openPlaceholderCount, substantiveMarkdown } from "./content-readiness.js";
 import { markdownEntries } from "./resource-markdown.js";
 import { resolveDataPath } from "./paths.js";
-import { currentPartyPeople, partiesIndependent } from "./parties.js";
+import { currentPartyPeople } from "./parties.js";
 import {
   nearDuplicateInformationTypes,
   resourceReviewRevisionsSync,
@@ -26,7 +26,7 @@ export function retentionScheduleApprovalIssues(loaded, program, rows, options =
     && !["superseded", "retired"].includes(record.status)
   ));
   if (!schedules.length) {
-    issues.push(issue("missing-retention-schedule-document", "Add and approve the Data Retention Schedule document before approving the schedule revision."));
+    issues.push(issue("missing-retention-schedule-document", "Add the Data Retention Schedule document before requesting approval of the complete schedule."));
   }
   if (schedules.length !== 1) {
     if (schedules.length) {
@@ -38,17 +38,22 @@ export function retentionScheduleApprovalIssues(loaded, program, rows, options =
   const linkedControl = schedule && (schedule.controlIds || []).some((id) => (program.controlIds || []).includes(id));
   if (
     schedule
-    && (!["approved", "active"].includes(schedule.status)
-    || !schedule.approvedOn
-    || !schedule.approvedContentRevisions
-    || currentPartyPeople(schedule.ownerIds || [], byId).size === 0
-    || !partiesIndependent(schedule.ownerIds, schedule.approverIds, byId)
+    && (currentPartyPeople(schedule.ownerIds || [], byId).size === 0
     || !linkedControl
     || !substantiveMarkdown(scheduleContent)
-    || openPlaceholderCount(scheduleContent) > 0
+    || openPlaceholderCount(scheduleContent) > 0)
+  ) {
+    issues.push(issue("incomplete-retention-schedule-document", "Complete the Data Retention Schedule document, its owner, linked Control, and Markdown before requesting approval of the complete schedule."));
+  }
+  if (
+    schedule
+    && options.approvalReview
+    && (!["approved", "active"].includes(schedule.status)
+    || schedule.approvedOn !== options.approvalReview.reviewedOn
+    || !sameIds(schedule.approverIds, options.approvalReview.reviewedByIds)
     || !approvalBindingsMatch(loaded, schedule))
   ) {
-    issues.push(issue("unapproved-retention-schedule-document", "Complete and independently approve the Data Retention Schedule document before approving the complete schedule."));
+    issues.push(issue("invalid-retention-schedule-approval-binding", "The Data Retention Schedule document no longer matches the recorded whole-schedule approval. Review and approve the current complete schedule again."));
   }
   const activeRows = rows.filter(({ status }) => status === "active");
   const mismatched = schedule
@@ -65,15 +70,20 @@ export function retentionScheduleApprovalIssues(loaded, program, rows, options =
   if (incomplete.length) {
     issues.push(issue("incomplete-retention-schedule-row", `Complete the retention decisions before approving the schedule revision: ${incomplete[0].title}.`));
   }
-  const usableRows = activeRows.filter((row) => !incomplete.includes(row));
-  const uncovered = retentionUses(loaded, program).find((use) => !usableRows.some((row) => (
+  const proposedRows = rows.filter(rowProposesCoverage);
+  const scopedPrograms = program.programIds
+    ? program.programIds.map((id) => byId.get(id)).filter((record) => record?.type === "program")
+    : [program];
+  const uncovered = scopedPrograms.flatMap((candidate) => (
+    retentionUses(loaded, candidate).map((use) => ({ ...use, programId: candidate.id }))
+  )).find((use) => !proposedRows.some((row) => (
     (row.informationTypeIds || []).includes(use.informationTypeId)
-    && ((row.scopeResourceIds || []).includes(use.resource.id) || (row.scopeResourceIds || []).includes(program.id))
+    && ((row.scopeResourceIds || []).includes(use.resource.id) || (row.scopeResourceIds || []).includes(use.programId))
   )));
   if (uncovered) {
     issues.push(issue(
       "uncovered-retention-information-use",
-      `${uncovered.resource.title} uses ${byId.get(uncovered.informationTypeId)?.title || uncovered.informationTypeId}, but no active, current retention schedule row covers both the type and scope.`
+      `${uncovered.resource.title} uses ${byId.get(uncovered.informationTypeId)?.title || uncovered.informationTypeId}, but no proposed retention schedule row covers both the type and scope.`
     ));
   }
   if (nearDuplicateInformationTypes(loaded.resources).length && !options.informationTypesReviewed) {
@@ -82,6 +92,10 @@ export function retentionScheduleApprovalIssues(loaded, program, rows, options =
   return issues;
 }
 
+function sameIds(left = [], right = []) {
+  return JSON.stringify([...left].sort()) === JSON.stringify([...right].sort());
+}
+
 function issue(code, message) {
   return { code, message };
 }
@@ -99,6 +113,14 @@ function primaryMarkdown(loaded, record) {
   }
 }
 
+function rowProposesCoverage(row) {
+  return Boolean(
+    (row.informationTypeIds || []).length
+    && (row.scopeResourceIds || []).length
+    && row.scheduleDocumentId
+  );
+}
+
 function approvalBindingsMatch(loaded, record) {
   const actual = {};
   for (const entry of markdownEntries(loaded.model, record)) {
@@ -114,8 +136,5 @@ function approvalBindingsMatch(loaded, record) {
   const actualKeys = Object.keys(actual).sort();
   const expectedKeys = Object.keys(expected).sort();
   return actualKeys.length === expectedKeys.length
-    && actualKeys.every((key, index) => (
-      key === expectedKeys[index]
-      && actual[key] === expected[key]
-    ));
+    && actualKeys.every((key, index) => key === expectedKeys[index] && actual[key] === expected[key]);
 }

package/src/retention.js

@@ -10,18 +10,26 @@ export async function assessRetentionReadiness(loaded, program, options = {}) {
   if (!loaded.model.resources["retention-schedule-item"]) return [];
   const byId = new Map(loaded.resources.map((record) => [record.id, record]));
   const rules = loaded.resources.filter((record) => (
-    record.type === "retention-schedule-item" && record.status === "active"
+    record.type === "retention-schedule-item"
+    && !["retired", "superseded"].includes(record.status)
   ));
-  const revisions = await resourceReviewRevisions(loaded, rules.flatMap((rule) => retentionReviewResourceIds(rule, loaded)));
-  const usableRules = rules.filter((rule) => retentionRuleIsCurrent(rule, revisions, byId, loaded));
+  const activeRules = rules.filter(({ status }) => status === "active");
+  const revisions = await resourceReviewRevisions(loaded, activeRules.flatMap((rule) => retentionReviewResourceIds(rule, loaded)));
+  const usableRules = activeRules.filter((rule) => retentionRuleIsCurrent(rule, revisions, byId, loaded));
+  const proposedRules = rules.filter(retentionRuleHasCoverageProposal);
   const uses = retentionUses(loaded, program);
-  const items = rules.filter((rule) => !usableRules.includes(rule)).map((rule) => readinessItem(
+  const items = rules.map((rule) => readinessItem(
     `retention-rule-${rule.id}`,
-    "action",
-    `Review ${rule.title}`,
-    "This active retention schedule item is incomplete or is not bound to every current source revision. Review it before relying on its period or disposition behavior.",
+    usableRules.includes(rule) ? "complete" : "action",
+    usableRules.includes(rule) ? `${rule.title} proposal is ready` : `Finish ${rule.title}`,
+    usableRules.includes(rule)
+      ? "The owner completed this row and bound it to its current source revisions. It will become part of the authoritative schedule only when the complete schedule is approved."
+      : rule.status === "planned"
+        ? "Finish the type, scope, cutoff, period, disposition, sources, and owner review for this proposed row before requesting schedule approval."
+        : "This row is incomplete or is not bound to every current source revision. Review it before requesting schedule approval.",
     rule,
     {
+      coverageState: usableRules.includes(rule) ? "proposed" : "incomplete",
       sourceResourceIds: retentionReviewResourceIds(rule, loaded),
       commands: [
         `npx filegrc get ${rule.id} --mutation`,
@@ -39,31 +47,41 @@ export async function assessRetentionReadiness(loaded, program, options = {}) {
     usesByInformationType.set(use.informationTypeId, grouped);
   }
   items.push(...[...usesByInformationType].map(([informationTypeId, informationTypeUses]) => {
-    const uncoveredUses = informationTypeUses.filter((use) => (
+    const absentUses = informationTypeUses.filter((use) => (
+      !proposedRules.some((rule) => ruleCoversUse(rule, use, program))
+    ));
+    const pendingUses = informationTypeUses.filter((use) => (
       !usableRules.some((rule) => ruleCoversUse(rule, use, program))
     ));
+    const approvalPending = !options.scheduleApproved;
     const matches = usableRules.filter((rule) => (
       informationTypeUses.some((use) => ruleCoversUse(rule, use, program))
     ));
     const informationTypeTitle = byId.get(informationTypeId)?.title || informationTypeId;
-    const affectedTitles = uncoveredUses.map(({ resource }) => resource.title);
+    const affectedTitles = absentUses.map(({ resource }) => resource.title);
     const affectedSummary = affectedTitles.length > 3
       ? `${affectedTitles.slice(0, 3).join(", ")}, and ${affectedTitles.length - 3} more`
       : affectedTitles.join(", ");
     return readinessItem(
       `retention-use-${informationTypeId}`,
-      uncoveredUses.length ? "action" : "complete",
+      absentUses.length ? "action" : pendingUses.length || approvalPending ? "info" : "complete",
       `Decide retention for ${informationTypeTitle}`,
-      uncoveredUses.length
+      absentUses.length
         ? `${affectedTitles.length} in-scope ${affectedTitles.length === 1 ? "resource uses" : "resources use"} this Information Type without an active, current retention decision, including ${affectedSummary}. Management must choose the cutoff, period, disposition, and whether one program-wide rule or separate scoped rules apply.`
-        : `All ${informationTypeUses.length} in-scope ${informationTypeUses.length === 1 ? "use is" : "uses are"} covered by ${matches.map(({ title }) => title).join(", ")}.`,
+        : pendingUses.length
+          ? `Proposed rows cover every in-scope type-and-scope pair. Finish the row review, then approve the complete schedule once instead of approving these coverage checks separately.`
+          : approvalPending
+            ? "Completed row proposals cover every in-scope type-and-scope pair. Approve the complete schedule once; these derived checks are not separate approval work."
+          : `The approved schedule covers all ${informationTypeUses.length} in-scope ${informationTypeUses.length === 1 ? "use" : "uses"}.`,
       { type: "retention-schedule-item" },
       {
+        progressUnit: false,
+        coverageState: absentUses.length ? "absent" : pendingUses.length || approvalPending ? "proposed" : "approved",
         informationTypeId,
-        affectedResourceIds: uncoveredUses.map(({ resource }) => resource.id),
-        retentionScopeResourceIds: uncoveredUses.length === informationTypeUses.length
+        affectedResourceIds: absentUses.map(({ resource }) => resource.id),
+        retentionScopeResourceIds: absentUses.length === informationTypeUses.length
           ? [program.id]
-          : uncoveredUses.map(({ resource }) => resource.id),
+          : absentUses.map(({ resource }) => resource.id),
         retentionScheduleItemIds: matches.map(({ id }) => id),
         commands: [
           "npx filegrc guide retention-schedule-item --json",
@@ -75,6 +93,11 @@ export async function assessRetentionReadiness(loaded, program, options = {}) {
 
   for (const coverage of loaded.resources.filter((record) => record.type === "source-coverage" && record.status === "active")) {
     const linked = (coverage.retentionScheduleItemIds || []).map((id) => byId.get(id)).filter(Boolean);
+    const proposed = linked.filter((rule) => (
+      rule.type === "retention-schedule-item"
+      && proposedRules.includes(rule)
+      && (rule.scopeResourceIds || []).includes(coverage.id)
+    ));
     const matching = linked.filter((rule) => (
       rule.type === "retention-schedule-item"
       && rule.status === "active"
@@ -83,13 +106,17 @@ export async function assessRetentionReadiness(loaded, program, options = {}) {
     ));
     items.push(readinessItem(
       `retention-source-coverage-${coverage.id}`,
-      matching.length ? "complete" : "action",
+      matching.length && options.scheduleApproved ? "complete" : proposed.length ? "info" : "action",
       `Confirm retained evidence for ${coverage.title}`,
-      matching.length
+      matching.length && options.scheduleApproved
         ? `The source-coverage record references a current schedule item scoped to this population.`
-        : `The source-coverage record must reference an active, current schedule item whose scope includes ${coverage.id}. A draft schedule or unrelated rule does not satisfy this check.`,
+        : proposed.length
+          ? "A proposed schedule row covers this evidence source. Finish that row and approve the complete schedule before relying on it."
+          : `No proposed schedule row covers ${coverage.id}. Add the genuine type-and-scope decision before schedule approval.`,
       coverage,
       {
+        progressUnit: false,
+        coverageState: matching.length && options.scheduleApproved ? "approved" : proposed.length ? "proposed" : "absent",
         retentionScheduleItemIds: matching.map(({ id }) => id),
         commands: [
           `npx filegrc get ${coverage.id} --mutation`,
@@ -110,6 +137,7 @@ export async function assessRetentionReadiness(loaded, program, options = {}) {
         : `${duplicates.length} similar pair${duplicates.length === 1 ? " needs" : "s need"} management review. FileGRC will not merge records or rewrite relationships automatically.`,
       { type: "information-type" },
       {
+        progressUnit: false,
         candidates: duplicates,
         commands: [
           "npx filegrc list information-type --workflow --json",
@@ -218,8 +246,11 @@ function reviewSource(loaded, entry) {
     && entry.record.workflowScope === "program"
   ) {
     const approved = structuredClone(entry.record);
-    if (approved.status === "active") approved.status = "approved";
     for (const field of [
+      "status",
+      "approverIds",
+      "approvedOn",
+      "approvedContentRevisions",
       "activationBasis",
       "activatedByIds",
       "activatedOn",
@@ -252,6 +283,14 @@ function ruleCoversUse(rule, use, program) {
   return scope.has(use.resource.id) || scope.has(program.id);
 }
 
+function retentionRuleHasCoverageProposal(rule) {
+  return Boolean(
+    (rule.informationTypeIds || []).length
+    && (rule.scopeResourceIds || []).length
+    && rule.scheduleDocumentId
+  );
+}
+
 export function retentionReviewResourceIds(rule, loaded) {
   const programUseIds = [];
   if (loaded) {
@@ -279,7 +318,7 @@ export function retentionRuleIsCurrent(rule, revisions, byId = new Map(), loaded
   if (!(rule.informationTypeIds || []).length || !(rule.scopeResourceIds || []).length || !rule.scheduleDocumentId) return false;
   const schedule = byId.get(rule.scheduleDocumentId);
   if (schedule?.type !== "document" || schedule.documentKind !== "schedule" || schedule.workflowScope !== "program" || ["superseded", "retired"].includes(schedule.status)) return false;
-  if (!(rule.ownerIds || []).length || !(rule.approvedByIds || []).length || !rule.approvedOn) return false;
+  if (!(rule.ownerIds || []).length) return false;
   if (!rule.reviewedSourceRevisions || typeof rule.reviewedSourceRevisions !== "object") return false;
   if (!retentionCutoffIsComplete(rule.cutoff) || !retentionPeriodIsComplete(rule.retentionPeriod)) return false;
   if (!["delete", "destroy", "erase", "anonymize", "transfer", "retain-permanently"].includes(rule.dispositionAction)) return false;

package/src/server.js

@@ -33,6 +33,7 @@ import {
   pushWorkspace,
   retryBrowserSync,
   runBrowserMutation,
+  synchronizeBrowserStartup,
   withGitCommandCache,
   withGitCommandDeadline
 } from "./git.js";
@@ -777,6 +778,18 @@ export async function serveWorkspace(input = process.cwd(), options = {}) {
   if (!Number.isInteger(port) || port < 0 || port > 65_535) {
     throw new Error("The server port must be an integer from 0 through 65535.");
   }
+  let startupSynchronization;
+  try {
+    startupSynchronization = await synchronizeBrowserStartup(input, {
+      allowNonAuthoritativeWrites: options.allowNonAuthoritativeWrites === true
+    });
+  } catch (error) {
+    startupSynchronization = {
+      status: "failed",
+      updated: false,
+      message: error.message
+    };
+  }
   const loaded = await loadWorkspace(input);
   getResourceDefinition(loaded.model, "workspace");
   const server = createFilegrcServer(loaded.root, {
@@ -806,7 +819,8 @@ export async function serveWorkspace(input = process.cwd(), options = {}) {
     address: server.address(),
     url: `http://${urlHost(host)}:${server.address().port}`,
     requestedPort: port,
-    usedFallbackPort
+    usedFallbackPort,
+    startupSynchronization
   };
 }
 

package/src/source-coverage.js

@@ -1,4 +1,7 @@
 import { resourceReviewRevisions, retentionReviewResourceIds, retentionRuleIsCurrent } from "./retention.js";
+import { retentionScheduleIsAuthoritative } from "./collection-review.js";
+import { modelSupports } from "../model/index.js";
+import { resolveProgram } from "./program.js";
 
 export async function sourceCoverageComplete(record, loaded, program = loaded.workspace) {
   const structuredRetention = loaded.model.resources["retention-schedule-item"];
@@ -6,6 +9,10 @@ export async function sourceCoverageComplete(record, loaded, program = loaded.wo
     return false;
   }
   if (structuredRetention) {
+    const retentionProgram = modelSupports(loaded.model, "program-scope")
+      ? resolveProgram(loaded, program?.type === "program" ? program.id : undefined)
+      : program;
+    if (!retentionScheduleIsAuthoritative(loaded, retentionProgram)) return false;
     const linkedRules = (record.retentionScheduleItemIds || []).map((id) => (
       loaded.resources.find((resource) => resource.id === id && resource.type === "retention-schedule-item")
     ));

package/src/state.js

@@ -347,6 +347,8 @@ function serializeCollectionReviewAssessment(assessment) {
     status: assessment.status,
     complete: assessment.complete,
     message: assessment.message,
+    ...(assessment.recordProposals ? { recordProposals: assessment.recordProposals } : {}),
+    ...(assessment.incompleteRecordProposals ? { incompleteRecordProposals: assessment.incompleteRecordProposals } : {}),
     ...(assessment.eligibleReviewerIds ? { eligibleReviewerIds: assessment.eligibleReviewerIds } : {}),
     ...(assessment.reviewerConflictIds ? { reviewerConflictIds: assessment.reviewerConflictIds } : {}),
     ...(assessment.approvalIssues ? { approvalIssues: assessment.approvalIssues } : {})

package/src/validate.js

@@ -3,7 +3,7 @@ import { createReadStream } from "node:fs";
 import { readFile, stat } from "node:fs/promises";
 import { performance } from "node:perf_hooks";
 import { getResourceDefinition, modelSupports } from "../model/index.js";
-import { scopedCollectionRecords, selectScopedCollectionRecords } from "./collection-scope.js";
+import { collectionRecordProposals, retentionScheduleReviewScope, scopedCollectionRecords, selectScopedCollectionRecords } from "./collection-scope.js";
 import {
   collectionRevision,
   collectionRevisionMatches
@@ -44,6 +44,7 @@ import {
   recordsAtRevision
 } from "./reporting-route-integrity.js";
 import { validateWorkflowHistoryIntegrity } from "./workflow-history-integrity.js";
+import { occurrenceMemberExceptionIsValid } from "./obligation-members.js";
 
 let fingerprintFileReadObserver = null;
 
@@ -667,18 +668,7 @@ function validateObligationOccurrence(record, model, resources, entries, root, w
       ));
     }
     if (member.disposition === "exception") {
-      const exception = byId.get(member.exceptionId);
-      const occurrenceStart = record.coverage?.kind === "as-of" ? record.coverage.on : record.coverage?.startsOn;
-      const occurrenceEnd = record.coverage?.kind === "as-of" ? record.coverage.on : record.coverage?.endsOn;
-      const exceptionCoversMember = exception?.scopeResourceIds?.includes(member.resourceId)
-        || exception?.scopeResourceIds?.includes(record.obligationId);
-      if (
-        exception?.type !== "exception"
-        || exception.status !== "approved"
-        || !exceptionCoversMember
-        || exception.approval?.approvedOn > occurrenceStart
-        || exception.approval?.expiresOn < occurrenceEnd
-      ) {
+      if (!occurrenceMemberExceptionIsValid(member, record, byId)) {
         diagnostics.push(error(
           "invalid-member-exception",
           path,
@@ -1224,6 +1214,10 @@ function sameRouteProposal(proposal, finalized) {
   return [...keys].every((key) => allowed.has(key) || JSON.stringify(proposal[key]) === JSON.stringify(finalized[key]));
 }
 
+function sameStringIds(left = [], right = []) {
+  return JSON.stringify([...left].sort()) === JSON.stringify([...right].sort());
+}
+
 function validateTimestampTimezone(value, timezone, path, diagnostics, label) {
   if (!value || !timezone) return;
   const source = String(value);
@@ -1252,10 +1246,19 @@ function validateCollectionReview(record, loaded, byId, path, diagnostics) {
   const program = modelSupports(model, "program-scope")
     ? (record.scopeResourceIds || []).map((id) => byId.get(id)).find(({ type } = {}) => type === "program")
     : null;
+  const workspaceWideRetentionReview = record.resourceType === "retention-schedule-item"
+    && modelSupports(model, "retention-schedule-approval");
+  const retentionScope = workspaceWideRetentionReview ? retentionScheduleReviewScope(loaded) : null;
   const recordCount = scopedCollectionRecords(loaded, record.resourceType, program).length;
   const currentPopulationIds = scopedCollectionRecords(loaded, record.resourceType, program)
     .map(({ id }) => id)
     .sort();
+  const incompleteProposals = collectionRecordProposals(
+    loaded,
+    record.resourceType,
+    scopedCollectionRecords(loaded, record.resourceType, program),
+    program
+  ).filter(({ complete }) => !complete);
   const currentRevision = collectionRevision(loaded, record.resourceType, {
     program,
     authoritativeSourceId: record.decision === "externally-managed"
@@ -1274,6 +1277,20 @@ function validateCollectionReview(record, loaded, byId, path, diagnostics) {
       currentRevision
     }
   );
+  if (current && workspaceWideRetentionReview && !sameStringIds(record.scopeResourceIds, retentionScope.programIds)) {
+    diagnostics.push(error(
+      "invalid-retention-schedule-review-scope",
+      path,
+      "The current Data Retention Schedule review must cover every active Program as one workspace-wide approval."
+    ));
+  }
+  if (current && incompleteProposals.length) {
+    diagnostics.push(error(
+      "incomplete-collection-review-proposals",
+      path,
+      `Complete every ${configuration.title.toLowerCase()} record proposal before the collection review. Remaining: ${incompleteProposals.map(({ title }) => title).join(", ")}.`
+    ));
+  }
   if (current && record.resourceType === "control") {
     const controls = scopedCollectionRecords(loaded, "control", program);
     const controlIds = new Set(controls.map(({ id }) => id));
@@ -1299,8 +1316,11 @@ function validateCollectionReview(record, loaded, byId, path, diagnostics) {
   }
   if (current && record.resourceType === "retention-schedule-item" && modelSupports(model, "retention-schedule-approval")) {
     const rows = scopedCollectionRecords(loaded, record.resourceType, program);
-    for (const issue of retentionScheduleApprovalIssues(loaded, program || {}, rows, {
-      informationTypesReviewed: collectionReviewIsCurrent(loaded, "information-type", program)
+    for (const issue of retentionScheduleApprovalIssues(loaded, retentionScope, rows, {
+      informationTypesReviewed: retentionScope.programIds.every((id) => (
+        collectionReviewIsCurrent(loaded, "information-type", byId.get(id))
+      )),
+      approvalReview: record
     })) {
       diagnostics.push(error(issue.code, path, issue.message));
     }