filegrc 0.16.10 → 0.16.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "filegrc",
3
- "version": "0.16.10",
3
+ "version": "0.16.12",
4
4
  "description": "Zero-dependency Git-native GRC engine",
5
5
  "license": "MIT",
6
6
  "repository": {
@@ -1,4 +1,10 @@
1
1
  import { CALCULATED_REVISION_FIELDS, CALCULATED_REVISION_MAP_FIELDS, calculateRevision, canonicalCalculatedRevision, revisionsMatch } from "./revisions.js";
2
+ import { reviewCommitsChangingTypes, reviewHistoricalWorkspace, reviewHistoryCommits, reviewHistoryContext } from "./historical-workspace.js";
3
+
4
+ const scopeSourceTypes = new Set([
5
+ "workspace", "program", "commitment", "control", "complementary-control",
6
+ "system", "framework", "component", "vendor", "policy", "requirement"
7
+ ]);
2
8
 
3
9
  const excludedResourceFields = new Set([
4
10
  "applicabilityReview",
@@ -22,13 +28,30 @@ const unorderedStringArrayFields = new Set([
22
28
  ]);
23
29
 
24
30
  export function applicabilityScopeRevision(record, program, resources, model, options = {}) {
25
- const selectedSystemIds = new Set(program?.systemIds || []);
26
- const selectedFrameworkIds = new Set(program?.frameworkIds || []);
27
- const selectedControlIds = new Set(program?.controlIds || []);
28
- const selectedComponents = resources.filter(({ type, status, systemUses }) => (
31
+ const scopedResource = ["commitment", "control", "complementary-control"].includes(record?.type)
32
+ && !options.legacyBroadScope && !options.legacyBroadCommitment;
33
+ const linkedRequirementIds = new Set(record?.requirementIds || []);
34
+ const linkedControlIds = record?.type === "control"
35
+ ? [record.id]
36
+ : record?.type === "complementary-control"
37
+ ? record.relatedControlIds || []
38
+ : record?.controlIds || [];
39
+ const selectedSystemIds = new Set((program?.systemIds || []).filter((id) => (
40
+ !scopedResource || !(record.systemIds || []).length || record.systemIds.includes(id)
41
+ )));
42
+ const selectedFrameworkIds = new Set((program?.frameworkIds || []).filter((id) => (
43
+ !scopedResource || !linkedRequirementIds.size || resources.some((candidate) => (
44
+ candidate.type === "requirement" && candidate.frameworkId === id && linkedRequirementIds.has(candidate.id)
45
+ ))
46
+ )));
47
+ const selectedControlIds = new Set((program?.controlIds || []).filter((id) => (
48
+ !scopedResource || !linkedControlIds.length || linkedControlIds.includes(id)
49
+ )));
50
+ const selectedComponents = resources.filter(({ type, status, systemUses, id }) => (
29
51
  type === "component"
30
52
  && status !== "retired"
31
53
  && (systemUses || []).some(({ systemId }) => selectedSystemIds.has(systemId))
54
+ && (!scopedResource || !(record.componentIds || []).length || record.componentIds.includes(id))
32
55
  ));
33
56
  const selectedVendorIds = new Set(selectedComponents.map(({ vendorId }) => vendorId).filter(Boolean));
34
57
  const selectedPolicyIds = new Set(resources
@@ -38,17 +61,24 @@ export function applicabilityScopeRevision(record, program, resources, model, op
38
61
  const facts = {
39
62
  modelVersion: model.modelVersion,
40
63
  program: {
41
- ...pick(program || {}, [
64
+ ...pick(scopedResource ? {
65
+ ...program,
66
+ systemIds: [...selectedSystemIds],
67
+ frameworkIds: [...selectedFrameworkIds],
68
+ requirementIds: (program?.requirementIds || []).filter((id) => !linkedRequirementIds.size || linkedRequirementIds.has(id)),
69
+ controlIds: [...selectedControlIds]
70
+ } : program || {}, [
42
71
  "id",
43
72
  "assuranceGoal",
44
73
  "systemIds",
45
74
  "frameworkIds",
46
75
  "requirementIds",
47
76
  "controlIds",
48
- "riskMethodology"
77
+ ...(record?.type === "commitment" && scopedResource ? [] : ["riskMethodology"])
49
78
  ], model),
50
79
  ...(record.type === "requirement" ? {} : {
51
80
  requirementApplicability: (program?.requirementApplicability || [])
81
+ .filter((review) => !scopedResource || !linkedRequirementIds.size || linkedRequirementIds.has(review.requirementId))
52
82
  .map((review) => pick(review, ["requirementId", "decision"], model, "program-applicability"))
53
83
  .sort((left, right) => left.requirementId.localeCompare(right.requirementId))
54
84
  })
@@ -134,14 +164,19 @@ export function applicabilityScopeRevision(record, program, resources, model, op
134
164
  ], model)
135
165
  : pick(policy, ["id", "requirementIds"], model)),
136
166
  requirements: resources
137
- .filter(({ type, frameworkId }) => type === "requirement" && selectedFrameworkIds.has(frameworkId))
167
+ .filter(({ type, frameworkId, id }) => type === "requirement" && selectedFrameworkIds.has(frameworkId)
168
+ && (!scopedResource || !linkedRequirementIds.size || linkedRequirementIds.has(id)))
138
169
  .sort(compareRecordIds)
139
170
  .map((requirement) => pick(requirement, ["id", "frameworkId", "reference", "description", "parentRequirementId"], model)),
140
171
  commitments: resources
141
- .filter(({ type, status, systemIds }) => (
172
+ .filter(({ type, status, systemIds, controlIds, id }) => (
142
173
  type === "commitment"
143
174
  && !["retired", "superseded"].includes(status)
144
175
  && (systemIds || []).some((id) => selectedSystemIds.has(id))
176
+ && (!scopedResource
177
+ || record.type === "commitment" && id === record.id
178
+ || record.type === "control" && (controlIds || []).includes(record.id)
179
+ || record.type === "complementary-control" && (controlIds || []).some((controlId) => selectedControlIds.has(controlId)))
145
180
  ))
146
181
  .sort(compareRecordIds)
147
182
  .map((commitment) => pick(commitment, [
@@ -161,7 +196,7 @@ function compareRecordIds(left, right) {
161
196
  return left.id.localeCompare(right.id);
162
197
  }
163
198
 
164
- export function applicabilityReviewIsCurrent(review, record, program, resources, model) {
199
+ export function applicabilityReviewIsCurrent(review, record, program, resources, model, root = null) {
165
200
  if (
166
201
  review?.scopeRevision
167
202
  && !review.scopeRevision.startsWith("scope:")
@@ -171,6 +206,12 @@ export function applicabilityReviewIsCurrent(review, record, program, resources,
171
206
  if (!review?.scopeRevision) return false;
172
207
  const matches = (revision) => revisionsMatch("applicability-scope", review.scopeRevision, revision);
173
208
  if (matches(applicabilityScopeRevision(record, program, resources, model))) return true;
209
+ if (["commitment", "control", "complementary-control"].includes(record?.type) && matches(applicabilityScopeRevision(
210
+ record, program, resources, model, { legacyBroadScope: true }
211
+ ))) return true;
212
+ if (["commitment", "control", "complementary-control"].includes(record?.type) && root && historicalScopedReviewMatches(
213
+ review.scopeRevision, record, program, resources, model, root
214
+ )) return true;
174
215
  // Older reviews hashed Policy approval metadata alongside scope facts.
175
216
  // Reconstruct each Policy independently because other Policies may already
176
217
  // have been approved when one Policy changes. Material fields stay current.
@@ -204,6 +245,31 @@ export function applicabilityReviewIsCurrent(review, record, program, resources,
204
245
  return false;
205
246
  }
206
247
 
248
+ function historicalScopedReviewMatches(stored, record, program, resources, model, root) {
249
+ const current = applicabilityScopeRevision(record, program, resources, model);
250
+ const context = reviewHistoryContext(root, resources);
251
+ const { index } = context;
252
+ const commits = index?.available
253
+ ? reviewCommitsChangingTypes(index, scopeSourceTypes)
254
+ : reviewHistoryCommits(context);
255
+ for (const commit of [...commits].reverse()) {
256
+ const snapshot = reviewHistoricalWorkspace(context, commit);
257
+ if (!snapshot) continue;
258
+ const historicalResources = snapshot.resources;
259
+ const historicalRecord = historicalResources.find(({ id }) => id === record.id);
260
+ const historicalProgram = historicalResources.find(({ id }) => id === program?.id);
261
+ if (!historicalRecord || !historicalProgram) continue;
262
+ const historicalModel = snapshot.model;
263
+ if (!revisionsMatch("applicability-scope", current, applicabilityScopeRevision(
264
+ historicalRecord, historicalProgram, historicalResources, historicalModel
265
+ ))) continue;
266
+ if (revisionsMatch("applicability-scope", stored, applicabilityScopeRevision(
267
+ historicalRecord, historicalProgram, historicalResources, historicalModel, { legacyBroadScope: true }
268
+ ))) return true;
269
+ }
270
+ return false;
271
+ }
272
+
207
273
  function legacyPolicyAlternatives(policy) {
208
274
  const prior = {
209
275
  active: ["approved", "in-review", "draft"],
@@ -373,7 +373,7 @@ function scopeStage(loaded, audit, records, byId, programReadiness) {
373
373
  const v4Decisions = new Map((program?.requirementApplicability || [])
374
374
  .filter((decision) => (
375
375
  requirementById.has(decision.requirementId)
376
- && applicabilityReviewIsCurrent(decision, requirementById.get(decision.requirementId), program, records, loaded.model)
376
+ && applicabilityReviewIsCurrent(decision, requirementById.get(decision.requirementId), program, records, loaded.model, loaded.root)
377
377
  ))
378
378
  .map((decision) => [decision.requirementId, decision]));
379
379
  const unresolvedRequirements = frameworkRequirementIds
@@ -37,7 +37,7 @@ export async function scaffoldApplicabilityReview(input = process.cwd(), options
37
37
  .filter((review) => (
38
38
  ["applicable", "not-applicable"].includes(review.decision)
39
39
  && requirementById.has(review.requirementId)
40
- && applicabilityReviewIsCurrent(review, requirementById.get(review.requirementId), program, loaded.resources, loaded.model)
40
+ && applicabilityReviewIsCurrent(review, requirementById.get(review.requirementId), program, loaded.resources, loaded.model, loaded.root)
41
41
  ))
42
42
  .map(({ requirementId }) => requirementId));
43
43
  const requirementReviews = new Map((program.requirementApplicability || [])
@@ -47,7 +47,7 @@ export async function scaffoldApplicabilityReview(input = process.cwd(), options
47
47
  && (!requestedType || record.type === requestedType)
48
48
  && (record.type === "requirement" && modelSupports(loaded.model, "program-scope")
49
49
  ? !reviewedRequirementIds.has(record.id)
50
- : !applicabilityReviewIsCurrent(record.applicabilityReview, record, program, loaded.resources, loaded.model))
50
+ : !applicabilityReviewIsCurrent(record.applicabilityReview, record, program, loaded.resources, loaded.model, loaded.root))
51
51
  && !["retired", "superseded"].includes(record.status)
52
52
  ));
53
53
  return {
@@ -7,6 +7,8 @@ import {
7
7
  collectionScopeRevisionFacts
8
8
  } from "./collection-scope.js";
9
9
  import { resolveDataPath } from "./paths.js";
10
+ import { getFileAtRevision } from "./git.js";
11
+ import { reviewHistoricalWorkspace, reviewHistoryCommits, reviewHistoryContext } from "./historical-workspace.js";
10
12
  import { resolveProgram } from "./program.js";
11
13
  import { markdownEntries } from "./resource-markdown.js";
12
14
  import { CALCULATED_REVISION_FIELDS, CALCULATED_REVISION_MAP_FIELDS, calculateRevision, canonicalCalculatedRevision, revisionsMatch } from "./revisions.js";
@@ -39,6 +41,36 @@ export function collectionRevisionMatches(loaded, resourceType, storedRevision,
39
41
  }
40
42
  }
41
43
  }
44
+ return historicallyEquivalentCollection(loaded, resourceType, storedRevision, options, currentRevision);
45
+ }
46
+
47
+ function historicallyEquivalentCollection(loaded, resourceType, stored, options, current) {
48
+ if (!loaded.root) return false;
49
+ const context = reviewHistoryContext(loaded.root, loaded);
50
+ for (const commit of [...reviewHistoryCommits(context)].reverse()) {
51
+ const snapshot = reviewHistoricalWorkspace(context, commit);
52
+ if (!snapshot) continue;
53
+ const historicalOptions = {
54
+ ...options,
55
+ programId: options.programId || options.program?.id,
56
+ historicalCommit: commit
57
+ };
58
+ delete historicalOptions.program;
59
+ delete historicalOptions.currentRevision;
60
+ // Require the complete reviewed basis to be unchanged under the current
61
+ // semantic calculation before accepting an older byte-sensitive digest.
62
+ if (!revisionsMatch("collection", current, collectionRevision(snapshot, resourceType, historicalOptions))) continue;
63
+ for (const legacy of [false, true]) {
64
+ for (const scopeHashInput of ["legacy", "digest"]) {
65
+ for (const scopeFactsInput of ["legacy", "source"]) {
66
+ if (revisionsMatch("collection", stored, calculateCollectionRevision(
67
+ snapshot, resourceType, { ...historicalOptions, historicalReviewMetadata: true },
68
+ legacy, scopeHashInput, scopeFactsInput
69
+ ))) return true;
70
+ }
71
+ }
72
+ }
73
+ }
42
74
  return false;
43
75
  }
44
76
 
@@ -51,7 +83,7 @@ function calculateCollectionRevision(loaded, resourceType, options, legacy, scop
51
83
  : Object.hasOwn(options, "program")
52
84
  ? options.program
53
85
  : resolveProgram(loaded, options.programId);
54
- const inputs = new Map(collectionRevisionInputs(loaded, resourceType, program, { legacy })
86
+ const inputs = new Map(collectionRevisionInputs(loaded, resourceType, program, { legacy, historicalReviewMetadata: options.historicalReviewMetadata })
55
87
  .map((input) => [input.record.id, input]));
56
88
  const authoritativeSource = loaded.resources.find(({ id }) => id === options.authoritativeSourceId);
57
89
  if (authoritativeSource) {
@@ -71,7 +103,12 @@ function calculateCollectionRevision(loaded, resourceType, options, legacy, scop
71
103
  .digest("hex"),
72
104
  contentRevisions: (legacy || includeContent ? markdownEntries(loaded.model, record) : []).flatMap(({ path }) => {
73
105
  try {
74
- const content = readFileSync(resolveDataPath(loaded.root, path), "utf8");
106
+ const content = options.historicalCommit
107
+ ? loaded.historicalFiles
108
+ ? loaded.historicalFiles.get(`data/${path}`) ?? null
109
+ : getFileAtRevision(loaded.root, options.historicalCommit, `data/${path}`)
110
+ : readFileSync(resolveDataPath(loaded.root, path), "utf8");
111
+ if (content === null) return [];
75
112
  return [{ path, revision: createHash("sha256").update(content).digest("hex") }];
76
113
  } catch (error) {
77
114
  if (error.code === "ENOENT") return [];
@@ -80,7 +117,9 @@ function calculateCollectionRevision(loaded, resourceType, options, legacy, scop
80
117
  })
81
118
  }))
82
119
  .sort((left, right) => left.id.localeCompare(right.id));
83
- const scopeFacts = collectionScopeRevisionFacts(loaded, resourceType, program);
120
+ const scopeFacts = collectionScopeRevisionFacts(loaded, resourceType, program, {
121
+ historicalReviewMetadata: options.historicalReviewMetadata
122
+ });
84
123
  const workspaceScope = scopeFactsInput === "source" ? scopeFacts : canonicalScopeFacts(scopeFacts);
85
124
  const source = JSON.stringify({ resourceType, records, workspaceScope });
86
125
  return legacy
@@ -360,23 +360,26 @@ export function collectionRevisionInputs(loaded, resourceType, program, options
360
360
  record,
361
361
  value: reviewedIds.has(record.id)
362
362
  ? resourceType === "retention-schedule-item" && modelSupports(loaded.model, "retention-schedule-approval")
363
- ? retentionScheduleRowProposalValue(record)
364
- : record
363
+ ? retentionScheduleRowProposalValue(record, options.historicalReviewMetadata)
364
+ : options.historicalReviewMetadata ? record : substantiveRevisionValue(record)
365
365
  : resourceType === "retention-schedule-item"
366
366
  && modelSupports(loaded.model, "retention-schedule-approval")
367
367
  && record.type === "document"
368
368
  ? retentionScheduleDocumentProposalValue(record)
369
- : dependencyRevisionValue(resourceType, record, legacy),
369
+ : options.historicalReviewMetadata
370
+ ? dependencyRevisionValue(resourceType, record, legacy)
371
+ : substantiveRevisionValue(dependencyRevisionValue(resourceType, record, legacy)),
370
372
  includeContent: legacy
371
373
  || reviewedIds.has(record.id)
372
374
  || dependencyContentAffectsRevision(resourceType, record.type)
373
375
  }));
374
376
  }
375
377
 
376
- function retentionScheduleRowProposalValue(record) {
378
+ function retentionScheduleRowProposalValue(record, historicalReviewMetadata = false) {
377
379
  return Object.fromEntries(Object.entries(record).filter(([field]) => ![
378
380
  "approvedByIds",
379
- "approvedOn"
381
+ "approvedOn",
382
+ ...(historicalReviewMetadata ? [] : ["reviewedSourceRevisions"])
380
383
  ].includes(field)));
381
384
  }
382
385
 
@@ -396,7 +399,7 @@ function retentionScheduleDocumentProposalValue(record) {
396
399
  ].includes(field)));
397
400
  }
398
401
 
399
- export function collectionScopeRevisionFacts(loaded, resourceType, program) {
402
+ export function collectionScopeRevisionFacts(loaded, resourceType, program, options = {}) {
400
403
  if (!modelSupports(loaded.model, "program-scope")) {
401
404
  const common = { programId: program?.id ?? null };
402
405
  if (resourceType === "framework") {
@@ -426,7 +429,7 @@ export function collectionScopeRevisionFacts(loaded, resourceType, program) {
426
429
  programs: programs.map((record) => ({
427
430
  id: record.id,
428
431
  systemIds: sorted(record.systemIds),
429
- controlIds: sorted(record.controlIds)
432
+ ...(options.historicalReviewMetadata ? { controlIds: sorted(record.controlIds) } : {})
430
433
  }))
431
434
  };
432
435
  }
@@ -454,7 +457,11 @@ export function collectionScopeRevisionFacts(loaded, resourceType, program) {
454
457
  assuranceGoal: program?.assuranceGoal ?? null,
455
458
  frameworkIds: sorted(program?.frameworkIds),
456
459
  requirementIds: sorted(selectedRequirementIds(program || {}, loaded.model)),
457
- requirementApplicability: program?.requirementApplicability || [],
460
+ requirementApplicability: options.historicalReviewMetadata
461
+ ? program?.requirementApplicability || []
462
+ : (program?.requirementApplicability || [])
463
+ .map(({ requirementId, decision }) => ({ requirementId, decision }))
464
+ .sort((left, right) => left.requirementId.localeCompare(right.requirementId)),
458
465
  riskMethodology: program?.riskMethodology || null
459
466
  } : scope;
460
467
  }
@@ -462,9 +469,33 @@ export function collectionScopeRevisionFacts(loaded, resourceType, program) {
462
469
  }
463
470
 
464
471
  export function authoritativeSourceRevisionValue(record) {
465
- return Object.fromEntries(Object.entries(record).filter(([field]) => (
472
+ return substantiveRevisionValue(Object.fromEntries(Object.entries(record).filter(([field]) => (
466
473
  !authoritativeSourceBookkeepingFields.has(field)
467
- )));
474
+ ))));
475
+ }
476
+
477
+ function substantiveRevisionValue(record) {
478
+ if (!record) return record;
479
+ const value = { ...record };
480
+ if (value.type === "retention-schedule-item") {
481
+ delete value.approvedByIds;
482
+ delete value.approvedOn;
483
+ delete value.reviewedSourceRevisions;
484
+ }
485
+ if (value.type === "program" && value.requirementApplicability) {
486
+ value.requirementApplicability = value.requirementApplicability
487
+ .map(({ requirementId, decision }) => ({ requirementId, decision }))
488
+ .sort((left, right) => left.requirementId.localeCompare(right.requirementId));
489
+ }
490
+ if (value.applicabilityReview) value.applicabilityReview = { decision: value.applicabilityReview.decision };
491
+ if (["policy", "document", "training"].includes(value.type)) {
492
+ for (const field of [
493
+ "approverIds", "approvedOn", "activationBasis", "activatedByIds",
494
+ "activatedOn", "activatedContentRevisions", "statusTransition"
495
+ ]) delete value[field];
496
+ if (["approved", "active"].includes(value.status)) value.status = "approved";
497
+ }
498
+ return value;
468
499
  }
469
500
 
470
501
  const authoritativeSourceBookkeepingFields = new Set([
@@ -0,0 +1,131 @@
1
+ import { loadModel } from "../model/index.js";
2
+ import { getDataCommitHistory, getDataFilesAtRevision, getDataRecordHistoryIndex, getFileAtRevision } from "./git.js";
3
+ import { performance } from "node:perf_hooks";
4
+
5
+ const reviewContexts = new WeakMap();
6
+
7
+ export function reviewHistoryContext(root, owner) {
8
+ const key = owner.resources || owner;
9
+ let context = reviewContexts.get(key);
10
+ if (context) return context;
11
+ let index = null;
12
+ try {
13
+ // Leave time for the original Git recovery path if this larger index
14
+ // cannot be built within a browser request.
15
+ index = getDataRecordHistoryIndex(root, { deadline: performance.now() + 3_000 });
16
+ } catch (error) {
17
+ if (error.code !== "FILEGRC_HISTORY_DEADLINE") throw error;
18
+ }
19
+ context = { root, index, statesByCommit: new Map(), workspacesByCommit: new Map() };
20
+ reviewContexts.set(key, context);
21
+ return context;
22
+ }
23
+
24
+ export function reviewHistoryCommits(context) {
25
+ if (context.index?.available) return context.index.commits;
26
+ context.fallbackCommits ||= getDataCommitHistory(context.root);
27
+ return context.fallbackCommits;
28
+ }
29
+
30
+ export function reviewCommitsChangingTypes(index, types) {
31
+ const paths = new Set();
32
+ for (const records of index.recordsByCommit.values()) {
33
+ for (const { record, path } of records.values()) {
34
+ if (types.has(record.type)) paths.add(path);
35
+ }
36
+ }
37
+ return index.commits.filter((commit) => [...(index.fileChangesByCommit.get(commit)?.keys() || [])]
38
+ .some((path) => paths.has(path)));
39
+ }
40
+
41
+ export function reviewHistoricalWorkspace(context, commit) {
42
+ if (context.index?.available) return indexedHistoricalWorkspace(context, commit);
43
+ if (context.workspacesByCommit.has(commit)) return context.workspacesByCommit.get(commit);
44
+ const loaded = historicalWorkspace(context.root, commit);
45
+ context.workspacesByCommit.set(commit, loaded);
46
+ if (context.workspacesByCommit.size > 128) context.workspacesByCommit.delete(context.workspacesByCommit.keys().next().value);
47
+ return loaded;
48
+ }
49
+
50
+ // Preserve recovery beyond the reconciliation index's size and build limits.
51
+ // It is slower, but still requires the same substantive comparison.
52
+ function historicalWorkspace(root, commit) {
53
+ const entries = [];
54
+ for (const path of getDataFilesAtRevision(root, commit).filter(authoritativeJsonPath)) {
55
+ const source = getFileAtRevision(root, commit, path);
56
+ if (!source) return null;
57
+ try {
58
+ entries.push({ record: JSON.parse(source), source, relativePath: path.slice("data/".length) });
59
+ } catch {
60
+ return null;
61
+ }
62
+ }
63
+ const resources = entries.map(({ record }) => record);
64
+ const workspace = resources.find(({ type }) => type === "workspace");
65
+ if (!workspace) return null;
66
+ try {
67
+ return { root, entries, resources, workspace, model: loadModel(workspace.dataModelVersion) };
68
+ } catch {
69
+ return null;
70
+ }
71
+ }
72
+
73
+ // The history index already contains every committed JSON and Markdown source.
74
+ // Materialize its first-parent states once instead of asking Git to list and
75
+ // read the entire workspace for every legacy review binding.
76
+ export function indexedHistoricalWorkspace(context, commit) {
77
+ const { root, index } = context;
78
+ if (!index?.available || !index.parentsByCommit.has(commit)) return null;
79
+ // Reuse a full request-sized history for ordinary workspaces while bounding
80
+ // retained maps when a repository contains many thousands of data files.
81
+ const cacheLimit = Math.max(1, Math.min(512, Math.floor(200_000 / Math.max(1, index.historicalRecordPaths.size))));
82
+ if (context.workspacesByCommit.has(commit)) return context.workspacesByCommit.get(commit);
83
+ let state = context.statesByCommit.get(commit);
84
+ if (!state) {
85
+ const lineage = [];
86
+ let cursor = commit;
87
+ while (cursor && !context.statesByCommit.has(cursor)) {
88
+ lineage.push(cursor);
89
+ cursor = index.parentsByCommit.get(cursor)?.[0] || null;
90
+ }
91
+ state = cursor ? context.statesByCommit.get(cursor) : new Map();
92
+ for (let position = lineage.length - 1; position >= 0; position -= 1) {
93
+ const changedAt = lineage[position];
94
+ const next = new Map(state);
95
+ for (const [path, source] of index.fileChangesByCommit.get(changedAt) || []) {
96
+ if (source === null) next.delete(path);
97
+ else next.set(path, source);
98
+ }
99
+ context.statesByCommit.set(changedAt, next);
100
+ if (context.statesByCommit.size > cacheLimit) context.statesByCommit.delete(context.statesByCommit.keys().next().value);
101
+ state = next;
102
+ }
103
+ }
104
+ const entries = [];
105
+ for (const [path, source] of state) {
106
+ if (!authoritativeJsonPath(path)) continue;
107
+ try {
108
+ entries.push({ record: JSON.parse(source), source, relativePath: path.slice("data/".length) });
109
+ } catch {
110
+ return null;
111
+ }
112
+ }
113
+ const resources = entries.map(({ record }) => record);
114
+ const workspace = resources.find(({ type }) => type === "workspace");
115
+ if (!workspace) return null;
116
+ try {
117
+ const loaded = { root, entries, resources, workspace, model: loadModel(workspace.dataModelVersion), historicalFiles: state };
118
+ context.workspacesByCommit.set(commit, loaded);
119
+ if (context.workspacesByCommit.size > cacheLimit) context.workspacesByCommit.delete(context.workspacesByCommit.keys().next().value);
120
+ return loaded;
121
+ } catch {
122
+ return null;
123
+ }
124
+ }
125
+
126
+ function authoritativeJsonPath(path) {
127
+ const parts = path.split("/");
128
+ if (parts[0] !== "data" || !path.endsWith(".json")) return false;
129
+ if (parts.slice(1).some((part) => part.startsWith(".") || part === "content")) return false;
130
+ return !(parts[1] === "evidence" && parts.length > 3 && parts.at(-1) !== "evidence.json");
131
+ }
@@ -155,7 +155,7 @@ export async function assessProgramReadiness(input, options = {}) {
155
155
  };
156
156
 
157
157
  const policyStage = await policiesStage(scope, records, byId, readMarkdown, loaded.model);
158
- const controlStage = await controlsStage(scope, byId, readMarkdown, asOf, loaded.model);
158
+ const controlStage = await controlsStage(scope, byId, readMarkdown, asOf, loaded.model, loaded.root);
159
159
  controlStage.items.unshift(...collectionReviews
160
160
  .filter(({ resourceType }) => resourceType === "complementary-control")
161
161
  .map(collectionReviewReadinessItem));
@@ -218,7 +218,8 @@ export async function assessProgramReadiness(input, options = {}) {
218
218
  byId,
219
219
  loaded.model,
220
220
  collectionReviews.filter(({ resourceType }) => ["person", "framework", "system", "component", "vendor", "classification", "information-type"].includes(resourceType)),
221
- workspace?.timezone || "UTC"
221
+ workspace?.timezone || "UTC",
222
+ loaded.root
222
223
  );
223
224
  if (modelSupports(loaded.model, "reporting-route-sets")) {
224
225
  const routeSets = await assessReportingRouteSets(loaded, {
@@ -373,7 +374,7 @@ function programScope(program, records, byId, model, loaded) {
373
374
  };
374
375
  }
375
376
 
376
- function scopeStage(workspace, scope, records, byId, model, collectionReviews = [], timezone = "UTC") {
377
+ function scopeStage(workspace, scope, records, byId, model, collectionReviews = [], timezone = "UTC", root = null) {
377
378
  const items = [];
378
379
  const goal = workspace?.assuranceGoal || "none";
379
380
  items.push(item(
@@ -459,7 +460,7 @@ function scopeStage(workspace, scope, records, byId, model, collectionReviews =
459
460
  && record.effectiveOn
460
461
  && (!model.resources.commitment?.fields?.applicabilityReview || (
461
462
  record.applicabilityReview?.decision === "applicable"
462
- && applicabilityReviewIsCurrent(record.applicabilityReview, record, workspace, records, model)
463
+ && applicabilityReviewIsCurrent(record.applicabilityReview, record, workspace, records, model, root)
463
464
  ))
464
465
  && currentPartyPeople(record.ownerIds, byId).size > 0
465
466
  && (record.requirementIds || []).length > 0
@@ -496,7 +497,7 @@ function scopeStage(workspace, scope, records, byId, model, collectionReviews =
496
497
  const reviewedRequirements = (workspace?.requirementApplicability || [])
497
498
  .filter((decision) => (
498
499
  requirementById.has(decision.requirementId)
499
- && applicabilityReviewIsCurrent(decision, requirementById.get(decision.requirementId), workspace, records, model)
500
+ && applicabilityReviewIsCurrent(decision, requirementById.get(decision.requirementId), workspace, records, model, root)
500
501
  ));
501
502
  const v4Decisions = new Map(reviewedRequirements.map((decision) => [decision.requirementId, decision.decision]));
502
503
  const staleRequirementIds = new Set((workspace?.requirementApplicability || [])
@@ -1617,7 +1618,7 @@ function legacyPolicyLibraryProposals(records) {
1617
1618
  }];
1618
1619
  }
1619
1620
 
1620
- async function controlsStage(scope, byId, readMarkdown, asOf, model) {
1621
+ async function controlsStage(scope, byId, readMarkdown, asOf, model, root = null) {
1621
1622
  const items = [];
1622
1623
  const families = selectedControlFamilies(scope.controls, model);
1623
1624
  if (!scope.controls.length) {
@@ -1651,7 +1652,7 @@ async function controlsStage(scope, byId, readMarkdown, asOf, model) {
1651
1652
  const checks = {
1652
1653
  ...(model.resources.control?.fields?.applicabilityReview ? {
1653
1654
  applicability: control.applicabilityReview?.decision === "applicable"
1654
- && applicabilityReviewIsCurrent(control.applicabilityReview, control, scope.program, [...byId.values()], model)
1655
+ && applicabilityReviewIsCurrent(control.applicabilityReview, control, scope.program, [...byId.values()], model, root)
1655
1656
  } : {}),
1656
1657
  implemented: control.status === "implemented",
1657
1658
  owner: (control.ownerIds || []).length > 0,
package/src/retention.js CHANGED
@@ -6,6 +6,8 @@ import { programComponents } from "./program.js";
6
6
  import { markdownEntries } from "./resource-markdown.js";
7
7
  import { modelSupports } from "../model/index.js";
8
8
  import { canonicalCalculatedRevisionJson, revisionDigest, revisionsMatch } from "./revisions.js";
9
+ import { getFileAtRevision } from "./git.js";
10
+ import { reviewHistoricalWorkspace, reviewHistoryCommits, reviewHistoryContext } from "./historical-workspace.js";
9
11
 
10
12
  export async function assessRetentionReadiness(loaded, program, options = {}) {
11
13
  if (!loaded.model.resources["retention-schedule-item"]) return [];
@@ -187,7 +189,7 @@ export async function resourceReviewRevisions(loaded, ids, scopeHashInput = "leg
187
189
  return new Map([...revisions].filter(([id]) => wanted.has(id)));
188
190
  }
189
191
 
190
- export function resourceReviewRevisionsSync(loaded, ids, scopeHashInput = "legacy") {
192
+ export function resourceReviewRevisionsSync(loaded, ids, scopeHashInput = "legacy", legacySource = false, historicalCommit = null) {
191
193
  const wanted = new Set(ids);
192
194
  const entries = new Map(loaded.entries.map((entry) => [entry.record.id, entry]));
193
195
  const revisions = new Map();
@@ -197,10 +199,15 @@ export function resourceReviewRevisionsSync(loaded, ids, scopeHashInput = "legac
197
199
  const entry = entries.get(id);
198
200
  if (!entry || reviewing.has(id)) return null;
199
201
  reviewing.add(id);
200
- const parts = [reviewSource(loaded, entry, scopeHashInput)];
202
+ const parts = [reviewSource(loaded, entry, scopeHashInput, legacySource)];
201
203
  for (const markdown of markdownEntries(loaded.model, entry.record)) {
202
204
  try {
203
- parts.push(readFileSync(resolveDataPath(loaded.root, markdown.path), "utf8"));
205
+ const source = historicalCommit
206
+ ? loaded.historicalFiles
207
+ ? loaded.historicalFiles.get(`data/${markdown.path}`) ?? null
208
+ : getFileAtRevision(loaded.root, historicalCommit, `data/${markdown.path}`)
209
+ : readFileSync(resolveDataPath(loaded.root, markdown.path), "utf8");
210
+ if (source !== null) parts.push(source);
204
211
  } catch (error) {
205
212
  if (error.code !== "ENOENT") throw error;
206
213
  }
@@ -222,10 +229,47 @@ export function resourceReviewRevisionMatches(loaded, revisions, id, stored) {
222
229
  const current = revisions.get(id);
223
230
  if (!current || !revisionDigest("content", stored)) return false;
224
231
  if (revisionsMatch("content", stored, current)) return true;
232
+ const legacy = resourceReviewRevisionsSync(loaded, [id], "legacy", true).get(id);
233
+ if (revisionsMatch("content", stored, legacy)) return true;
225
234
  // Version 0.16.0 used a bare digest as the hash input for scopeRevision.
226
235
  // Read that binding without changing the legacy scope: basis restored here.
227
- const compatible = resourceReviewRevisionsSync(loaded, [id], "digest").get(id);
228
- return revisionsMatch("content", stored, compatible);
236
+ const compatible = resourceReviewRevisionsSync(loaded, [id], "digest", true).get(id);
237
+ if (revisionsMatch("content", stored, compatible)) return true;
238
+ return historicallyEquivalentReviewSource(loaded, id, stored, current);
239
+ }
240
+
241
+ // Recompute the old byte-sensitive binding at each committed state. The
242
+ // semantic source graph must still equal today's graph, including linked
243
+ // records and Markdown, before the old approval is treated as current.
244
+ function historicallyEquivalentReviewSource(loaded, id, stored, current) {
245
+ if (!loaded.root) return false;
246
+ const context = reviewHistoryContext(loaded.root, loaded);
247
+ const { index } = context;
248
+ const relevantPaths = index?.available ? new Set(["data/workspace.json"]) : null;
249
+ const graphIds = new Set();
250
+ const visit = (resourceId) => {
251
+ if (graphIds.has(resourceId)) return;
252
+ graphIds.add(resourceId);
253
+ const entry = loaded.entries.find(({ record }) => record.id === resourceId);
254
+ if (!entry) return;
255
+ for (const path of (index.historiesById.get(resourceId) || []).map(({ path }) => path)) relevantPaths.add(path);
256
+ for (const markdown of markdownEntries(loaded.model, entry.record)) relevantPaths.add(`data/${markdown.path}`);
257
+ for (const sourceId of entry.record.sourceResourceIds || []) visit(sourceId);
258
+ };
259
+ if (relevantPaths) visit(id);
260
+ for (const commit of [...reviewHistoryCommits(context)].reverse()) {
261
+ const changed = index?.available ? index.fileChangesByCommit.get(commit) : null;
262
+ if (relevantPaths && ![...changed.keys()].some((path) => relevantPaths.has(path) || path.endsWith(".md"))) continue;
263
+ const snapshot = reviewHistoricalWorkspace(context, commit);
264
+ if (!snapshot?.entries.some(({ record }) => record.id === id)) continue;
265
+ const semantic = resourceReviewRevisionsSync(snapshot, [id], "legacy", false, commit).get(id);
266
+ if (!revisionsMatch("content", semantic, current)) continue;
267
+ for (const scopeHashInput of ["legacy", "digest"]) {
268
+ const legacy = resourceReviewRevisionsSync(snapshot, [id], scopeHashInput, true, commit).get(id);
269
+ if (revisionsMatch("content", stored, legacy)) return true;
270
+ }
271
+ }
272
+ return false;
229
273
  }
230
274
 
231
275
  export function retentionUses(loaded, program) {
@@ -249,7 +293,7 @@ export function retentionUses(loaded, program) {
249
293
  return [...new Map(uses.map((use) => [`${use.resource.id}:${use.informationTypeId}`, use])).values()];
250
294
  }
251
295
 
252
- function reviewSource(loaded, entry, scopeHashInput = "legacy") {
296
+ function reviewSource(loaded, entry, scopeHashInput = "legacy", legacySource = false) {
253
297
  if (
254
298
  modelSupports(loaded.model, "retention-schedule-approval")
255
299
  && entry.record.type === "document"
@@ -271,7 +315,34 @@ function reviewSource(loaded, entry, scopeHashInput = "legacy") {
271
315
  ]) delete approved[field];
272
316
  return JSON.stringify(approved);
273
317
  }
274
- return canonicalCalculatedRevisionJson(entry.source, scopeHashInput);
318
+ return legacySource
319
+ ? canonicalCalculatedRevisionJson(entry.source, scopeHashInput)
320
+ : canonicalCalculatedRevisionJson(JSON.stringify(substantiveReviewSource(JSON.parse(entry.source))), scopeHashInput);
321
+ }
322
+
323
+ function substantiveReviewSource(record) {
324
+ const source = record.type === "program" ? {
325
+ id: record.id,
326
+ type: record.type,
327
+ status: record.status,
328
+ assuranceGoal: record.assuranceGoal,
329
+ systemIds: [...(record.systemIds || [])].sort(),
330
+ extensions: record.extensions,
331
+ requirementApplicability: (record.requirementApplicability || [])
332
+ .map(({ requirementId, decision }) => ({ requirementId, decision }))
333
+ .sort((left, right) => left.requirementId.localeCompare(right.requirementId))
334
+ } : { ...record };
335
+ if (source.applicabilityReview) {
336
+ source.applicabilityReview = { decision: source.applicabilityReview.decision };
337
+ }
338
+ if (["policy", "document", "training"].includes(source.type)) {
339
+ for (const field of [
340
+ "approverIds", "approvedOn", "activationBasis", "activatedByIds",
341
+ "activatedOn", "activatedContentRevisions", "statusTransition"
342
+ ]) delete source[field];
343
+ if (["approved", "active"].includes(source.status)) source.status = "approved";
344
+ }
345
+ return source;
275
346
  }
276
347
 
277
348
  export function nearDuplicateInformationTypes(records) {
package/src/state.js CHANGED
@@ -351,7 +351,7 @@ function assessApplicabilityReviewStatuses(loaded, program) {
351
351
  : record.applicabilityReview;
352
352
  return [record.id, !review || !["applicable", "not-applicable", "externally-managed", "zero-population"].includes(review.decision)
353
353
  ? "missing" : applicabilityReviewIsCurrent(
354
- review, record, program, loaded.resources, loaded.model
354
+ review, record, program, loaded.resources, loaded.model, loaded.root
355
355
  ) ? "current" : "stale"];
356
356
  }));
357
357
  }
package/src/workflow.js CHANGED
@@ -397,7 +397,7 @@ function recordFinalizationFindings(loaded, program) {
397
397
  if (incomplete) {
398
398
  findings.push(finalizationFinding(record, incomplete, loaded.model));
399
399
  }
400
- for (const missing of finalizationFields(record, loaded.model, loaded.resources, program)) {
400
+ for (const missing of finalizationFields(record, loaded.model, loaded.resources, program, loaded.root)) {
401
401
  findings.push(fieldFinding(record, missing, loaded.model));
402
402
  }
403
403
  }
@@ -532,11 +532,11 @@ function recordFinalizationRequiredness(record, loaded, program) {
532
532
  return "conditional";
533
533
  }
534
534
 
535
- function finalizationFields(record, model, resources, program) {
535
+ function finalizationFields(record, model, resources, program, root) {
536
536
  const fields = [];
537
537
  const needsReview = ["requirement", "commitment", "complementary-control", "control"].includes(record.type)
538
538
  && model.resources[record.type]?.fields?.applicabilityReview
539
- && !applicabilityReviewIsCurrent(record.applicabilityReview, record, program, resources, model);
539
+ && !applicabilityReviewIsCurrent(record.applicabilityReview, record, program, resources, model, root);
540
540
  if (needsReview) {
541
541
  const currentRevision = applicabilityScopeRevision(record, program, resources, model);
542
542
  fields.push({