@joshuaswarren/openclaw-engram 9.0.33 → 9.0.35

package/README.md

@@ -37,6 +37,7 @@ AI agents forget everything between conversations. Engram fixes that.
 - **Trust-zone promotion path** — Engram can now, when `trustZonesEnabled` and `quarantinePromotionEnabled` are enabled, persist typed quarantine, working, and trusted records, plan explicit promotions, block direct `quarantine -> trusted` jumps, and require anchored provenance before promoting risky working records into `trusted`.
 - **Trust-zone recall** — Engram can now, when `trustZoneRecallEnabled` is enabled, inject prompt-relevant `working` and `trusted` trust-zone records into recall context as a separate `Trust Zones` section while keeping `quarantine` material out of recall by default.
 - **Poisoning-defense corroboration** — Engram can now, when `memoryPoisoningDefenseEnabled` is enabled, score trust-zone provenance deterministically and require independent non-quarantine corroboration before risky `working -> trusted` promotions succeed.
+- **Red-team benchmark packs** — Engram's eval harness can now validate and count typed `memory-red-team` benchmark packs so poisoning-defense regression suites stay explicit and reviewable instead of hiding inside generic benchmark metadata.
 - **Zero-config start** — Install, add an API key, restart. Engram works out of the box with sensible defaults and progressively unlocks advanced features as you enable them.
 
 ## Quick Start
@@ -199,6 +200,7 @@ Key settings:
 | `trustZoneStoreDir` | `{memoryDir}/state/trust-zones` | Root directory for trust-zone records |
 | `trustZoneRecallEnabled` | `false` | Inject prompt-relevant working and trusted trust-zone records into recall context |
 | `memoryPoisoningDefenseEnabled` | `false` | Enable deterministic provenance trust scoring and corroboration requirements for risky trusted promotions |
+| `memoryRedTeamBenchEnabled` | `false` | Enable typed memory red-team benchmark pack support and status accounting for poisoning-defense suites |
 
 Full reference: [Config Reference](docs/config-reference.md)
 

package/dist/index.js

@@ -301,6 +301,7 @@ function parseConfig(raw) {
     trustZoneStoreDir: typeof cfg.trustZoneStoreDir === "string" && cfg.trustZoneStoreDir.trim().length > 0 ? cfg.trustZoneStoreDir.trim() : path.join(memoryDir, "state", "trust-zones"),
     trustZoneRecallEnabled: cfg.trustZoneRecallEnabled === true,
     memoryPoisoningDefenseEnabled: cfg.memoryPoisoningDefenseEnabled === true,
+    memoryRedTeamBenchEnabled: cfg.memoryRedTeamBenchEnabled === true,
     // Local LLM Provider (v2.1)
     localLlmEnabled: cfg.localLlmEnabled === true || cfg.localLlmEnabled === "true",
     // default: false
@@ -6333,7 +6334,7 @@ import path6 from "path";
 import { mkdir as mkdir2, readFile as readFile3, stat, writeFile as writeFile2 } from "fs/promises";
 import path5 from "path";
 function toSafeTimestamp(ts) {
-  return ts.toISOString().replace(/[:.]/g, "").replace("Z", "Z");
+  return ts.toISOString().replace(/[:.]/g, "");
 }
 async function lintWorkspaceFiles(opts) {
   const warnings = [];
@@ -11630,10 +11631,14 @@ function assertSafeBenchmarkId(benchmarkId) {
   }
   return benchmarkId;
 }
-function validateEvalBenchmarkManifest(raw) {
+function validateEvalBenchmarkManifest(raw, options) {
   if (!isRecord(raw)) throw new Error("benchmark manifest must be an object");
   if (raw.schemaVersion !== 1) throw new Error("schemaVersion must be 1");
   if (!Array.isArray(raw.cases)) throw new Error("cases must be an array");
+  const benchmarkTypeRaw = typeof raw.benchmarkType === "string" && raw.benchmarkType.trim().length > 0 ? raw.benchmarkType.trim() : "standard";
+  if (!["standard", "memory-red-team"].includes(benchmarkTypeRaw)) {
+    throw new Error("benchmarkType must be one of standard|memory-red-team");
+  }
   const cases = raw.cases.map((item, index) => {
     if (!isRecord(item)) throw new Error(`cases[${index}] must be an object`);
     return {
@@ -11643,13 +11648,28 @@ function validateEvalBenchmarkManifest(raw) {
       notes: typeof item.notes === "string" && item.notes.trim().length > 0 ? item.notes.trim() : void 0
     };
   });
+  const benchmarkType = benchmarkTypeRaw;
+  if (benchmarkType === "memory-red-team" && options?.memoryRedTeamBenchEnabled !== true) {
+    throw new Error("memory-red-team benchmark packs require memoryRedTeamBenchEnabled");
+  }
+  const attackClass = typeof raw.attackClass === "string" && raw.attackClass.trim().length > 0 ? raw.attackClass.trim() : void 0;
+  const targetSurface = typeof raw.targetSurface === "string" && raw.targetSurface.trim().length > 0 ? raw.targetSurface.trim() : void 0;
+  if (benchmarkType === "memory-red-team" && attackClass === void 0) {
+    throw new Error("attackClass must be a non-empty string");
+  }
+  if (benchmarkType === "memory-red-team" && targetSurface === void 0) {
+    throw new Error("targetSurface must be a non-empty string");
+  }
   return {
     schemaVersion: 1,
     benchmarkId: assertString(raw.benchmarkId, "benchmarkId"),
+    benchmarkType,
     title: assertString(raw.title, "title"),
     description: typeof raw.description === "string" && raw.description.trim().length > 0 ? raw.description.trim() : void 0,
     tags: optionalStringArray(raw.tags, "tags"),
     sourceLinks: optionalStringArray(raw.sourceLinks, "sourceLinks"),
+    attackClass,
+    targetSurface,
     cases
   };
 }
@@ -11815,7 +11835,11 @@ async function collectEvalStoreSnapshot(options) {
   const manifests = [];
   for (const filePath of benchmarkFiles) {
     try {
-      manifests.push(validateEvalBenchmarkManifest(await readJsonFile(filePath)));
+      manifests.push(
+        validateEvalBenchmarkManifest(await readJsonFile(filePath), {
+          memoryRedTeamBenchEnabled: options.memoryRedTeamBenchEnabled
+        })
+      );
     } catch (error) {
       invalidBenchmarks.push({
         path: filePath,
@@ -11852,10 +11876,18 @@ async function collectEvalStoreSnapshot(options) {
   });
   shadows.sort((a, b) => b.recordedAt.localeCompare(a.recordedAt));
   const tags = /* @__PURE__ */ new Set();
+  const attackClasses = /* @__PURE__ */ new Set();
   const sourceLinks = /* @__PURE__ */ new Set();
+  const targetSurfaces = /* @__PURE__ */ new Set();
   let totalCases = 0;
+  let redTeam = 0;
   for (const manifest of manifests) {
     totalCases += manifest.cases.length;
+    if (manifest.benchmarkType === "memory-red-team") {
+      redTeam += 1;
+      if (manifest.attackClass) attackClasses.add(manifest.attackClass);
+      if (manifest.targetSurface) targetSurfaces.add(manifest.targetSurface);
+    }
     for (const tag of manifest.tags ?? []) tags.add(tag);
     for (const link of manifest.sourceLinks ?? []) sourceLinks.add(link);
   }
@@ -11870,8 +11902,11 @@ async function collectEvalStoreSnapshot(options) {
         total: benchmarkFiles.length,
         valid: manifests.length,
         invalid: invalidBenchmarks.length,
+        redTeam,
         totalCases,
+        attackClasses: [...attackClasses].sort(),
         tags: [...tags].sort(),
+        targetSurfaces: [...targetSurfaces].sort(),
         sourceLinks: [...sourceLinks].sort()
       },
       runs: {
@@ -11919,25 +11954,32 @@ async function resolveBenchmarkManifestPath(sourcePath) {
   }
   throw new Error("benchmark pack source must be a file or directory");
 }
-async function validateEvalBenchmarkPack(sourcePath) {
+async function validateEvalBenchmarkPack(sourcePath, options) {
   const trimmedSourcePath = sourcePath.trim();
   if (trimmedSourcePath.length === 0) {
     throw new Error("benchmark pack path must be a non-empty string");
   }
   const { manifestPath } = await resolveBenchmarkManifestPath(trimmedSourcePath);
-  const manifest = validateEvalBenchmarkManifest(await readJsonFile(manifestPath));
+  const manifest = validateEvalBenchmarkManifest(await readJsonFile(manifestPath), {
+    memoryRedTeamBenchEnabled: options?.memoryRedTeamBenchEnabled
+  });
   return {
     sourcePath: trimmedSourcePath,
     manifestPath,
     benchmarkId: assertSafeBenchmarkId(manifest.benchmarkId),
+    benchmarkType: manifest.benchmarkType ?? "standard",
     title: manifest.title,
+    attackClass: manifest.attackClass,
+    targetSurface: manifest.targetSurface,
     totalCases: manifest.cases.length,
     tags: [...manifest.tags ?? []],
     sourceLinks: [...manifest.sourceLinks ?? []]
   };
 }
 async function importEvalBenchmarkPack(options) {
-  const summary = await validateEvalBenchmarkPack(options.sourcePath);
+  const summary = await validateEvalBenchmarkPack(options.sourcePath, {
+    memoryRedTeamBenchEnabled: options.memoryRedTeamBenchEnabled
+  });
   const rootDir = resolveEvalStoreDir(options.memoryDir, options.evalStoreDir);
   const benchmarkDir = path15.join(rootDir, "benchmarks");
   const targetDir = path15.join(benchmarkDir, summary.benchmarkId);
@@ -11982,7 +12024,8 @@ async function getEvalHarnessStatus(options) {
   return (await collectEvalStoreSnapshot({
     rootDir: resolveEvalStoreDir(options.memoryDir, options.evalStoreDir),
     enabled: options.enabled,
-    shadowModeEnabled: options.shadowModeEnabled
+    shadowModeEnabled: options.shadowModeEnabled,
+    memoryRedTeamBenchEnabled: options.memoryRedTeamBenchEnabled
   })).status;
 }
 function resolveRequiredEvalStoreRoot(options, label) {
@@ -12006,12 +12049,14 @@ async function runEvalBenchmarkCiGate(options) {
   const baseSnapshot = await collectEvalStoreSnapshot({
     rootDir: baseRootDir,
     enabled: true,
-    shadowModeEnabled: true
+    shadowModeEnabled: true,
+    memoryRedTeamBenchEnabled: true
   });
   const candidateSnapshot = await collectEvalStoreSnapshot({
     rootDir: candidateRootDir,
     enabled: true,
-    shadowModeEnabled: true
+    shadowModeEnabled: true,
+    memoryRedTeamBenchEnabled: true
   });
   const regressions = [];
   const improvements = [];
@@ -26852,18 +26897,22 @@ async function runBenchmarkStatusCliCommand(options) {
     memoryDir: options.memoryDir,
     evalStoreDir: options.evalStoreDir,
     enabled: options.evalHarnessEnabled,
-    shadowModeEnabled: options.evalShadowModeEnabled
+    shadowModeEnabled: options.evalShadowModeEnabled,
+    memoryRedTeamBenchEnabled: options.memoryRedTeamBenchEnabled
   });
 }
 async function runBenchmarkValidateCliCommand(options) {
-  return validateEvalBenchmarkPack(options.path);
+  return validateEvalBenchmarkPack(options.path, {
+    memoryRedTeamBenchEnabled: options.memoryRedTeamBenchEnabled
+  });
 }
 async function runBenchmarkImportCliCommand(options) {
   return importEvalBenchmarkPack({
     sourcePath: options.path,
     memoryDir: options.memoryDir,
     evalStoreDir: options.evalStoreDir,
-    force: options.force === true
+    force: options.force === true,
+    memoryRedTeamBenchEnabled: options.memoryRedTeamBenchEnabled
   });
 }
 async function runBenchmarkCiGateCliCommand(options) {
@@ -28017,7 +28066,8 @@ function registerCli(api, orchestrator) {
           memoryDir: orchestrator.config.memoryDir,
           evalStoreDir: orchestrator.config.evalStoreDir,
           evalHarnessEnabled: orchestrator.config.evalHarnessEnabled,
-          evalShadowModeEnabled: orchestrator.config.evalShadowModeEnabled
+          evalShadowModeEnabled: orchestrator.config.evalShadowModeEnabled,
+          memoryRedTeamBenchEnabled: orchestrator.config.memoryRedTeamBenchEnabled
         });
         console.log(JSON.stringify(status, null, 2));
         console.log("OK");
@@ -28025,7 +28075,8 @@ function registerCli(api, orchestrator) {
       cmd.command("benchmark-validate").description("Validate a benchmark manifest file or pack directory without importing it").argument("<path>", "Path to a benchmark manifest JSON file or a directory with manifest.json").action(async (...args) => {
         const inputPath = args[0];
         const summary = await runBenchmarkValidateCliCommand({
-          path: typeof inputPath === "string" ? inputPath : ""
+          path: typeof inputPath === "string" ? inputPath : "",
+          memoryRedTeamBenchEnabled: orchestrator.config.memoryRedTeamBenchEnabled
         });
         console.log(JSON.stringify(summary, null, 2));
         console.log("OK");
@@ -28037,7 +28088,8 @@ function registerCli(api, orchestrator) {
           path: typeof inputPath === "string" ? inputPath : "",
           memoryDir: orchestrator.config.memoryDir,
           evalStoreDir: orchestrator.config.evalStoreDir,
-          force: options.force === true
+          force: options.force === true,
+          memoryRedTeamBenchEnabled: orchestrator.config.memoryRedTeamBenchEnabled
         });
         console.log(JSON.stringify(summary, null, 2));
         console.log("OK");