filegrc 0.16.11 → 0.16.12

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "filegrc",
-  "version": "0.16.11",
+  "version": "0.16.12",
   "description": "Zero-dependency Git-native GRC engine",
   "license": "MIT",
   "repository": {

package/src/applicability-scope.js

@@ -1,6 +1,10 @@
 import { CALCULATED_REVISION_FIELDS, CALCULATED_REVISION_MAP_FIELDS, calculateRevision, canonicalCalculatedRevision, revisionsMatch } from "./revisions.js";
-import { getDataCommitHistory } from "./git.js";
-import { historicalWorkspace } from "./historical-workspace.js";
+import { reviewCommitsChangingTypes, reviewHistoricalWorkspace, reviewHistoryCommits, reviewHistoryContext } from "./historical-workspace.js";
+
+const scopeSourceTypes = new Set([
+  "workspace", "program", "commitment", "control", "complementary-control",
+  "system", "framework", "component", "vendor", "policy", "requirement"
+]);
 
 const excludedResourceFields = new Set([
   "applicabilityReview",
@@ -243,8 +247,13 @@ export function applicabilityReviewIsCurrent(review, record, program, resources,
 
 function historicalScopedReviewMatches(stored, record, program, resources, model, root) {
   const current = applicabilityScopeRevision(record, program, resources, model);
-  for (const commit of getDataCommitHistory(root).reverse()) {
-    const snapshot = historicalWorkspace(root, commit);
+  const context = reviewHistoryContext(root, resources);
+  const { index } = context;
+  const commits = index?.available
+    ? reviewCommitsChangingTypes(index, scopeSourceTypes)
+    : reviewHistoryCommits(context);
+  for (const commit of [...commits].reverse()) {
+    const snapshot = reviewHistoricalWorkspace(context, commit);
     if (!snapshot) continue;
     const historicalResources = snapshot.resources;
     const historicalRecord = historicalResources.find(({ id }) => id === record.id);

package/src/collection-revision.js

@@ -7,8 +7,8 @@ import {
   collectionScopeRevisionFacts
 } from "./collection-scope.js";
 import { resolveDataPath } from "./paths.js";
-import { getDataCommitHistory, getFileAtRevision } from "./git.js";
-import { historicalWorkspace } from "./historical-workspace.js";
+import { getFileAtRevision } from "./git.js";
+import { reviewHistoricalWorkspace, reviewHistoryCommits, reviewHistoryContext } from "./historical-workspace.js";
 import { resolveProgram } from "./program.js";
 import { markdownEntries } from "./resource-markdown.js";
 import { CALCULATED_REVISION_FIELDS, CALCULATED_REVISION_MAP_FIELDS, calculateRevision, canonicalCalculatedRevision, revisionsMatch } from "./revisions.js";
@@ -46,8 +46,9 @@ export function collectionRevisionMatches(loaded, resourceType, storedRevision,
 
 function historicallyEquivalentCollection(loaded, resourceType, stored, options, current) {
   if (!loaded.root) return false;
-  for (const commit of getDataCommitHistory(loaded.root).reverse()) {
-    const snapshot = historicalWorkspace(loaded.root, commit);
+  const context = reviewHistoryContext(loaded.root, loaded);
+  for (const commit of [...reviewHistoryCommits(context)].reverse()) {
+    const snapshot = reviewHistoricalWorkspace(context, commit);
     if (!snapshot) continue;
     const historicalOptions = {
       ...options,
@@ -103,7 +104,9 @@ function calculateCollectionRevision(loaded, resourceType, options, legacy, scop
       contentRevisions: (legacy || includeContent ? markdownEntries(loaded.model, record) : []).flatMap(({ path }) => {
         try {
           const content = options.historicalCommit
-            ? getFileAtRevision(loaded.root, options.historicalCommit, `data/${path}`)
+            ? loaded.historicalFiles
+              ? loaded.historicalFiles.get(`data/${path}`) ?? null
+              : getFileAtRevision(loaded.root, options.historicalCommit, `data/${path}`)
             : readFileSync(resolveDataPath(loaded.root, path), "utf8");
           if (content === null) return [];
           return [{ path, revision: createHash("sha256").update(content).digest("hex") }];

package/src/historical-workspace.js

@@ -1,10 +1,55 @@
 import { loadModel } from "../model/index.js";
-import { getDataFilesAtRevision, getFileAtRevision } from "./git.js";
+import { getDataCommitHistory, getDataFilesAtRevision, getDataRecordHistoryIndex, getFileAtRevision } from "./git.js";
+import { performance } from "node:perf_hooks";
 
-// Rebuild only the authoritative data needed for a historical revision check.
-// The caller must compare current and historical facts before accepting a
-// legacy binding; this snapshot never supplies a new approval.
-export function historicalWorkspace(root, commit) {
+const reviewContexts = new WeakMap();
+
+export function reviewHistoryContext(root, owner) {
+  const key = owner.resources || owner;
+  let context = reviewContexts.get(key);
+  if (context) return context;
+  let index = null;
+  try {
+    // Leave time for the original Git recovery path if this larger index
+    // cannot be built within a browser request.
+    index = getDataRecordHistoryIndex(root, { deadline: performance.now() + 3_000 });
+  } catch (error) {
+    if (error.code !== "FILEGRC_HISTORY_DEADLINE") throw error;
+  }
+  context = { root, index, statesByCommit: new Map(), workspacesByCommit: new Map() };
+  reviewContexts.set(key, context);
+  return context;
+}
+
+export function reviewHistoryCommits(context) {
+  if (context.index?.available) return context.index.commits;
+  context.fallbackCommits ||= getDataCommitHistory(context.root);
+  return context.fallbackCommits;
+}
+
+export function reviewCommitsChangingTypes(index, types) {
+  const paths = new Set();
+  for (const records of index.recordsByCommit.values()) {
+    for (const { record, path } of records.values()) {
+      if (types.has(record.type)) paths.add(path);
+    }
+  }
+  return index.commits.filter((commit) => [...(index.fileChangesByCommit.get(commit)?.keys() || [])]
+    .some((path) => paths.has(path)));
+}
+
+export function reviewHistoricalWorkspace(context, commit) {
+  if (context.index?.available) return indexedHistoricalWorkspace(context, commit);
+  if (context.workspacesByCommit.has(commit)) return context.workspacesByCommit.get(commit);
+  const loaded = historicalWorkspace(context.root, commit);
+  context.workspacesByCommit.set(commit, loaded);
+  if (context.workspacesByCommit.size > 128) context.workspacesByCommit.delete(context.workspacesByCommit.keys().next().value);
+  return loaded;
+}
+
+// Preserve recovery beyond the reconciliation index's size and build limits.
+// It is slower, but still requires the same substantive comparison.
+function historicalWorkspace(root, commit) {
   const entries = [];
   for (const path of getDataFilesAtRevision(root, commit).filter(authoritativeJsonPath)) {
     const source = getFileAtRevision(root, commit, path);
@@ -25,6 +70,59 @@ export function historicalWorkspace(root, commit) {
   }
 }
 
+// The history index already contains every committed JSON and Markdown source.
+// Materialize its first-parent states once instead of asking Git to list and
+// read the entire workspace for every legacy review binding.
+export function indexedHistoricalWorkspace(context, commit) {
+  const { root, index } = context;
+  if (!index?.available || !index.parentsByCommit.has(commit)) return null;
+  // Reuse a full request-sized history for ordinary workspaces while bounding
+  // retained maps when a repository contains many thousands of data files.
+  const cacheLimit = Math.max(1, Math.min(512, Math.floor(200_000 / Math.max(1, index.historicalRecordPaths.size))));
+  if (context.workspacesByCommit.has(commit)) return context.workspacesByCommit.get(commit);
+  let state = context.statesByCommit.get(commit);
+  if (!state) {
+    const lineage = [];
+    let cursor = commit;
+    while (cursor && !context.statesByCommit.has(cursor)) {
+      lineage.push(cursor);
+      cursor = index.parentsByCommit.get(cursor)?.[0] || null;
+    }
+    state = cursor ? context.statesByCommit.get(cursor) : new Map();
+    for (let position = lineage.length - 1; position >= 0; position -= 1) {
+      const changedAt = lineage[position];
+      const next = new Map(state);
+      for (const [path, source] of index.fileChangesByCommit.get(changedAt) || []) {
+        if (source === null) next.delete(path);
+        else next.set(path, source);
+      }
+      context.statesByCommit.set(changedAt, next);
+      if (context.statesByCommit.size > cacheLimit) context.statesByCommit.delete(context.statesByCommit.keys().next().value);
+      state = next;
+    }
+  }
+  const entries = [];
+  for (const [path, source] of state) {
+    if (!authoritativeJsonPath(path)) continue;
+    try {
+      entries.push({ record: JSON.parse(source), source, relativePath: path.slice("data/".length) });
+    } catch {
+      return null;
+    }
+  }
+  const resources = entries.map(({ record }) => record);
+  const workspace = resources.find(({ type }) => type === "workspace");
+  if (!workspace) return null;
+  try {
+    const loaded = { root, entries, resources, workspace, model: loadModel(workspace.dataModelVersion), historicalFiles: state };
+    context.workspacesByCommit.set(commit, loaded);
+    if (context.workspacesByCommit.size > cacheLimit) context.workspacesByCommit.delete(context.workspacesByCommit.keys().next().value);
+    return loaded;
+  } catch {
+    return null;
+  }
+}
+
 function authoritativeJsonPath(path) {
   const parts = path.split("/");
   if (parts[0] !== "data" || !path.endsWith(".json")) return false;

package/src/retention.js

@@ -6,8 +6,8 @@ import { programComponents } from "./program.js";
 import { markdownEntries } from "./resource-markdown.js";
 import { modelSupports } from "../model/index.js";
 import { canonicalCalculatedRevisionJson, revisionDigest, revisionsMatch } from "./revisions.js";
-import { getDataCommitHistory, getFileAtRevision } from "./git.js";
-import { historicalWorkspace } from "./historical-workspace.js";
+import { getFileAtRevision } from "./git.js";
+import { reviewHistoricalWorkspace, reviewHistoryCommits, reviewHistoryContext } from "./historical-workspace.js";
 
 export async function assessRetentionReadiness(loaded, program, options = {}) {
   if (!loaded.model.resources["retention-schedule-item"]) return [];
@@ -203,7 +203,9 @@ export function resourceReviewRevisionsSync(loaded, ids, scopeHashInput = "legac
     for (const markdown of markdownEntries(loaded.model, entry.record)) {
       try {
         const source = historicalCommit
-          ? getFileAtRevision(loaded.root, historicalCommit, `data/${markdown.path}`)
+          ? loaded.historicalFiles
+            ? loaded.historicalFiles.get(`data/${markdown.path}`) ?? null
+            : getFileAtRevision(loaded.root, historicalCommit, `data/${markdown.path}`)
           : readFileSync(resolveDataPath(loaded.root, markdown.path), "utf8");
         if (source !== null) parts.push(source);
       } catch (error) {
@@ -241,8 +243,24 @@ export function resourceReviewRevisionMatches(loaded, revisions, id, stored) {
 // records and Markdown, before the old approval is treated as current.
 function historicallyEquivalentReviewSource(loaded, id, stored, current) {
   if (!loaded.root) return false;
-  for (const commit of getDataCommitHistory(loaded.root).reverse()) {
-    const snapshot = historicalWorkspace(loaded.root, commit);
+  const context = reviewHistoryContext(loaded.root, loaded);
+  const { index } = context;
+  const relevantPaths = index?.available ? new Set(["data/workspace.json"]) : null;
+  const graphIds = new Set();
+  const visit = (resourceId) => {
+    if (graphIds.has(resourceId)) return;
+    graphIds.add(resourceId);
+    const entry = loaded.entries.find(({ record }) => record.id === resourceId);
+    if (!entry) return;
+    for (const path of (index.historiesById.get(resourceId) || []).map(({ path }) => path)) relevantPaths.add(path);
+    for (const markdown of markdownEntries(loaded.model, entry.record)) relevantPaths.add(`data/${markdown.path}`);
+    for (const sourceId of entry.record.sourceResourceIds || []) visit(sourceId);
+  };
+  if (relevantPaths) visit(id);
+  for (const commit of [...reviewHistoryCommits(context)].reverse()) {
+    const changed = index?.available ? index.fileChangesByCommit.get(commit) : null;
+    if (relevantPaths && ![...changed.keys()].some((path) => relevantPaths.has(path) || path.endsWith(".md"))) continue;
+    const snapshot = reviewHistoricalWorkspace(context, commit);
     if (!snapshot?.entries.some(({ record }) => record.id === id)) continue;
     const semantic = resourceReviewRevisionsSync(snapshot, [id], "legacy", false, commit).get(id);
     if (!revisionsMatch("content", semantic, current)) continue;