@joshuaswarren/openclaw-engram 9.0.51 → 9.0.53

package/README.md

@@ -32,6 +32,8 @@ AI agents forget everything between conversations. Engram fixes that.
 - **Memory OS features** — Graph recall, temporal memory tree, lifecycle policy, compounding, shared context, memory boxes, and identity continuity can be enabled progressively as your install grows.
 - **Benchmark-first roadmap** — Engram now has an evaluation harness with live shadow recall recording and a CI benchmark delta gate, so memory improvements can be measured and regression-checked instead of argued from anecdotes.
 - **Baseline snapshot discipline** — Engram can now, when `benchmarkBaselineSnapshotsEnabled` is enabled, capture typed baseline snapshots of the latest completed benchmark runs so later PR delta reporting can compare candidates against a stable stored reference instead of an ad hoc branch state.
+- **Named baseline delta reporting** — Engram can now, when `benchmarkDeltaReporterEnabled` is enabled, compare the current eval store against a stored baseline snapshot, emit a machine-readable delta report plus markdown summary, and fail fast when a candidate regresses a benchmark that previously passed.
+- **Required CI baseline gate** — Engram's `eval-benchmark-gate` workflow now reads a named stored baseline snapshot from the base branch fixture store and blocks merges when the candidate branch regresses relative to that required baseline.
 - **Objective-state recall** — Engram can now store normalized file, process, and tool outcomes and, when `objectiveStateRecallEnabled` is enabled, inject the most relevant objective-state snapshots back into recall context as a separate `Objective State` section.
 - **Causal trajectory graph foundation** — Engram can now persist typed `goal -> action -> observation -> outcome -> follow-up` chains when `causalTrajectoryMemoryEnabled` is enabled and, with `actionGraphRecallEnabled`, emit deterministic action-conditioned edges into the causal graph for later trajectory-aware retrieval.
 - **Causal trajectory recall** — Engram can now, when `causalTrajectoryRecallEnabled` is enabled, inject prompt-relevant causal chains back into recall context as a separate `Causal Trajectories` section with lightweight match explainability.
@@ -172,6 +174,7 @@ openclaw engram benchmark-status             # Benchmark/eval harness packs, run
 openclaw engram benchmark-validate <path>    # Validate a benchmark manifest or pack directory
 openclaw engram benchmark-import <path>      # Import a validated benchmark pack into the eval store
 openclaw engram benchmark-baseline-snapshot  # Capture a typed baseline snapshot of the latest completed benchmark runs
+openclaw engram benchmark-baseline-report    # Compare the current eval store against a stored baseline snapshot
 openclaw engram benchmark-ci-gate            # Compare base vs candidate eval stores and fail on regressions
 openclaw engram objective-state-status       # Objective-state snapshot counts and latest stored snapshot
 openclaw engram causal-trajectory-status    # Causal-trajectory record counts and latest stored chain
@@ -212,6 +215,13 @@ Key settings:
 | `evalHarnessEnabled` | `false` | Enable the evaluation harness for benchmark packs, run summaries, and shadow recall bookkeeping |
 | `evalShadowModeEnabled` | `false` | Record live recall decisions to the eval store without changing injected output |
 | `benchmarkBaselineSnapshotsEnabled` | `false` | Enable versioned baseline snapshot artifacts for the latest completed benchmark runs |
+| `benchmarkDeltaReporterEnabled` | `false` | Enable named-baseline delta reports against the current eval store |
+
+The repo's required benchmark check uses the committed fixture snapshot at
+`tests/fixtures/eval-ci/store/baselines/required-main.json` as the stable
+release baseline for PR gating. During the rollout PR that first introduces
+that file, the gate bootstraps from the candidate branch snapshot once; after
+that, PRs resolve the required baseline from the base branch checkout.
 | `evalStoreDir` | `{memoryDir}/state/evals` | Root directory for benchmark packs, run summaries, and shadow recall records |
 | `objectiveStateMemoryEnabled` | `false` | Enable the objective-state memory foundation for normalized world/tool state snapshots |
 | `objectiveStateSnapshotWritesEnabled` | `false` | Permit objective-state snapshot writers to persist typed state records |

package/dist/index.js

@@ -288,6 +288,7 @@ function parseConfig(raw) {
     evalHarnessEnabled: cfg.evalHarnessEnabled === true,
     evalShadowModeEnabled: cfg.evalShadowModeEnabled === true,
     benchmarkBaselineSnapshotsEnabled: cfg.benchmarkBaselineSnapshotsEnabled === true,
+    benchmarkDeltaReporterEnabled: cfg.benchmarkDeltaReporterEnabled === true,
     evalStoreDir: typeof cfg.evalStoreDir === "string" && cfg.evalStoreDir.trim().length > 0 ? cfg.evalStoreDir.trim() : path.join(memoryDir, "state", "evals"),
     objectiveStateMemoryEnabled: cfg.objectiveStateMemoryEnabled === true,
     objectiveStateSnapshotWritesEnabled: cfg.objectiveStateSnapshotWritesEnabled === true,
@@ -11908,6 +11909,48 @@ function compareMetricDeltas(baseMetrics, candidateMetrics) {
   }
   return { deltas, regressions, improvements };
 }
+function formatEvalBaselineDeltaMarkdown(report) {
+  const lines = [
+    "# Eval Baseline Delta Report",
+    "",
+    `- Passed: ${report.passed ? "yes" : "no"}`,
+    `- Baseline snapshot: ${report.baselineSnapshotId}`,
+    `- Baseline created: ${report.baselineCreatedAt}`,
+    `- Baseline source root: ${report.baselineSourceRootDir}`,
+    `- Candidate root: ${report.candidateRootDir}`,
+    `- Benchmarks compared: ${report.comparedBenchmarks}`
+  ];
+  if (report.missingCandidateBenchmarks.length > 0) {
+    lines.push(`- Missing candidate benchmarks: ${report.missingCandidateBenchmarks.join(", ")}`);
+  }
+  lines.push(
+    `- Invalid candidate artifacts: benchmarks=${report.invalidArtifacts.candidate.benchmarks}, runs=${report.invalidArtifacts.candidate.runs}, shadows=${report.invalidArtifacts.candidate.shadows}, baselines=${report.invalidArtifacts.candidate.baselines}`,
+    "",
+    "## Regressions"
+  );
+  if (report.regressions.length === 0) {
+    lines.push("- none");
+  } else {
+    for (const regression of report.regressions) lines.push(`- ${regression}`);
+  }
+  lines.push("", "## Improvements");
+  if (report.improvements.length === 0) {
+    lines.push("- none");
+  } else {
+    for (const improvement of report.improvements) lines.push(`- ${improvement}`);
+  }
+  lines.push("", "## Benchmark Deltas");
+  if (report.deltas.length === 0) {
+    lines.push("- none");
+  } else {
+    for (const delta of report.deltas) {
+      lines.push(
+        `- ${delta.benchmarkId}: passRate ${delta.basePassRate} -> ${delta.candidatePassRate} (delta ${delta.passRateDelta})`
+      );
+    }
+  }
+  return lines.join("\n");
+}
 async function collectEvalStoreSnapshot(options) {
   const rootDir = options.rootDir;
   const benchmarkDir = path15.join(rootDir, "benchmarks");
@@ -12179,6 +12222,112 @@ async function createEvalBaselineSnapshot(options) {
   await writeFile11(targetPath, JSON.stringify(snapshot, null, 2), "utf-8");
   return { targetPath, snapshot };
 }
+async function runEvalBaselineDeltaReport(options) {
+  if (options.benchmarkDeltaReporterEnabled !== true) {
+    throw new Error("benchmark delta reporter is disabled");
+  }
+  const snapshotId = assertSafePathSegment(assertString(options.snapshotId, "snapshotId"), "snapshotId");
+  const candidateRootDir = resolveEvalStoreDir(options.memoryDir, options.evalStoreDir);
+  const candidateSnapshot = await collectEvalStoreSnapshot({
+    rootDir: candidateRootDir,
+    enabled: true,
+    shadowModeEnabled: true,
+    baselineSnapshotsEnabled: true,
+    memoryRedTeamBenchEnabled: true
+  });
+  const baselineSnapshot = candidateSnapshot.baselines.find((snapshot) => snapshot.snapshotId === snapshotId);
+  if (!baselineSnapshot) {
+    throw new Error(`benchmark baseline snapshot not found: ${snapshotId}`);
+  }
+  return buildEvalBaselineDeltaReport({
+    baselineSnapshot,
+    candidateSnapshot
+  });
+}
+function buildEvalBaselineDeltaReport(options) {
+  const { baselineSnapshot, candidateSnapshot } = options;
+  const regressions = [];
+  const improvements = [];
+  if (candidateSnapshot.status.invalidBenchmarks.length > 0) {
+    regressions.push(`candidate store has ${candidateSnapshot.status.invalidBenchmarks.length} invalid benchmark manifest(s)`);
+  }
+  if (candidateSnapshot.status.invalidRuns.length > 0) {
+    regressions.push(`candidate store has ${candidateSnapshot.status.invalidRuns.length} invalid run summary file(s)`);
+  }
+  if (candidateSnapshot.status.invalidShadows.length > 0) {
+    regressions.push(`candidate store has ${candidateSnapshot.status.invalidShadows.length} invalid shadow record(s)`);
+  }
+  if (candidateSnapshot.status.invalidBaselines.length > 0) {
+    regressions.push(`candidate store has ${candidateSnapshot.status.invalidBaselines.length} invalid baseline snapshot file(s)`);
+  }
+  const candidateRuns = latestCompletedRunsByBenchmark(candidateSnapshot.runs);
+  const baselineBenchmarks = new Map(
+    baselineSnapshot.benchmarks.map((benchmark) => [benchmark.benchmarkId, benchmark])
+  );
+  const missingCandidateBenchmarks = [...baselineBenchmarks.keys()].filter((benchmarkId) => !candidateRuns.has(benchmarkId)).sort();
+  for (const benchmarkId of missingCandidateBenchmarks) {
+    regressions.push(`candidate is missing latest completed benchmark run for ${benchmarkId}`);
+  }
+  const deltas = [];
+  for (const benchmarkId of [...baselineBenchmarks.keys()].sort()) {
+    const baseBenchmark = baselineBenchmarks.get(benchmarkId);
+    const candidateRun = candidateRuns.get(benchmarkId);
+    if (!baseBenchmark || !candidateRun) continue;
+    const passRateDelta = computePassRate(candidateRun) - baseBenchmark.passRate;
+    const delta = {
+      benchmarkId,
+      baseRunId: baseBenchmark.runId,
+      candidateRunId: candidateRun.runId,
+      basePassRate: baseBenchmark.passRate,
+      candidatePassRate: computePassRate(candidateRun),
+      passRateDelta,
+      metricDeltas: {},
+      regressions: [],
+      improvements: []
+    };
+    if (passRateDelta < 0) {
+      delta.regressions.push(`passRate ${baseBenchmark.passRate} -> ${delta.candidatePassRate}`);
+      regressions.push(`${benchmarkId} pass rate regressed (${baseBenchmark.passRate} -> ${delta.candidatePassRate})`);
+    } else if (passRateDelta > 0) {
+      delta.improvements.push(`passRate ${baseBenchmark.passRate} -> ${delta.candidatePassRate}`);
+      improvements.push(`${benchmarkId} pass rate improved (${baseBenchmark.passRate} -> ${delta.candidatePassRate})`);
+    }
+    const metricDelta = compareMetricDeltas(baseBenchmark.metrics, candidateRun.metrics);
+    delta.metricDeltas = metricDelta.deltas;
+    for (const regression of metricDelta.regressions) {
+      delta.regressions.push(regression);
+      regressions.push(`${benchmarkId} ${regression}`);
+    }
+    for (const improvement of metricDelta.improvements) {
+      delta.improvements.push(improvement);
+      improvements.push(`${benchmarkId} ${improvement}`);
+    }
+    deltas.push(delta);
+  }
+  const report = {
+    passed: regressions.length === 0,
+    baselineSnapshotId: baselineSnapshot.snapshotId,
+    baselineCreatedAt: baselineSnapshot.createdAt,
+    baselineSourceRootDir: baselineSnapshot.sourceRootDir,
+    candidateRootDir: candidateSnapshot.status.rootDir,
+    comparedBenchmarks: deltas.length,
+    missingCandidateBenchmarks,
+    invalidArtifacts: {
+      candidate: {
+        benchmarks: candidateSnapshot.status.invalidBenchmarks.length,
+        runs: candidateSnapshot.status.invalidRuns.length,
+        shadows: candidateSnapshot.status.invalidShadows.length,
+        baselines: candidateSnapshot.status.invalidBaselines.length
+      }
+    },
+    regressions,
+    improvements,
+    deltas,
+    markdownReport: ""
+  };
+  report.markdownReport = formatEvalBaselineDeltaMarkdown(report);
+  return report;
+}
 function resolveRequiredEvalStoreRoot(options, label) {
   if (typeof options.evalStoreDir === "string" && options.evalStoreDir.trim().length > 0) {
     return options.evalStoreDir.trim();
@@ -28964,6 +29113,14 @@ async function runBenchmarkCiGateCliCommand(options) {
     candidateEvalStoreDir: options.candidateEvalStoreDir
   });
 }
+async function runBenchmarkBaselineReportCliCommand(options) {
+  return runEvalBaselineDeltaReport({
+    memoryDir: options.memoryDir,
+    evalStoreDir: options.evalStoreDir,
+    benchmarkDeltaReporterEnabled: options.benchmarkDeltaReporterEnabled,
+    snapshotId: options.snapshotId
+  });
+}
 async function runObjectiveStateStatusCliCommand(options) {
   return getObjectiveStateStoreStatus({
     memoryDir: options.memoryDir,
@@ -30354,6 +30511,22 @@ function registerCli(api, orchestrator) {
         }
         console.log("OK");
       });
+      cmd.command("benchmark-baseline-report").description("Compare the current eval store against a named stored benchmark baseline snapshot").requiredOption("--snapshot-id <id>", "Stable baseline snapshot identifier").action(async (...args) => {
+        const options = args[0] ?? {};
+        const summary = await runBenchmarkBaselineReportCliCommand({
+          memoryDir: orchestrator.config.memoryDir,
+          evalStoreDir: orchestrator.config.evalStoreDir,
+          benchmarkDeltaReporterEnabled: orchestrator.config.benchmarkDeltaReporterEnabled,
+          snapshotId: typeof options.snapshotId === "string" ? options.snapshotId : ""
+        });
+        const { markdownReport, ...jsonSummary } = summary;
+        console.log(JSON.stringify(jsonSummary, null, 2));
+        console.log(markdownReport);
+        if (!summary.passed) {
+          throw new Error("benchmark baseline report detected regressions");
+        }
+        console.log("OK");
+      });
       cmd.command("objective-state-status").description("Show objective-state store status, snapshot counts, and latest stored snapshot").action(async () => {
         const status = await runObjectiveStateStatusCliCommand({
           memoryDir: orchestrator.config.memoryDir,