@cohortapp/agent-sdk 2.17.0 → 2.18.5

package/lib/security/audit-engine.test.mjs

@@ -1,424 +0,0 @@
-/**
- * audit-engine.test.mjs — security audit engine + attestations (G15).
- * Run: node --test lib/security/audit-engine.test.mjs
- *
- * Uses real temp directories on disk (the engine's default fs path) plus a few
- * injected-fs/suppression cases to keep the suppression + attestation logic
- * deterministic. Every assertion is behavioural, not a smoke test.
- */
-
-"use strict";
-
-import { test } from "node:test";
-import assert from "node:assert/strict";
-import {
-  mkdtempSync, mkdirSync, writeFileSync, chmodSync, existsSync, statSync, rmSync, symlinkSync,
-} from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-
-import {
-  CHECKS,
-  SEVERITIES,
-  runAudit,
-  applyFixes,
-  loadSuppressions,
-  buildAttestation,
-} from "./audit-engine.mjs";
-
-// ---------------------------------------------------------------------------
-// Fixture: a fully-conformant agent repo on disk.
-// ---------------------------------------------------------------------------
-
-const GOOD_SETTINGS = JSON.stringify({
-  hooks: {
-    PreToolUse: [
-      { matcher: "mcp__claude_ai_Slack__slack_send_message", hooks: [{ type: "command", command: "./scripts/hooks/block-mcp-slack-send.sh" }] },
-      { matcher: "mcp__claude_ai_Gmail__gmail_create_draft", hooks: [{ type: "command", command: "./scripts/hooks/pre-send-audit.sh gmail" }] },
-    ],
-    PostToolUse: [{ matcher: "", hooks: [{ type: "command", command: "./scripts/hooks/post-action-log.sh" }] }],
-    Stop: [{ matcher: "", hooks: [{ type: "command", command: "./scripts/hooks/session-end-log.sh" }] }],
-  },
-}, null, 2);
-
-function makeGoodAgent() {
-  const root = mkdtempSync(join(tmpdir(), "maestro-audit-"));
-  mkdirSync(join(root, "config"), { recursive: true });
-  mkdirSync(join(root, ".claude"), { recursive: true });
-  mkdirSync(join(root, "scripts/hooks"), { recursive: true });
-  mkdirSync(join(root, ".maestro"), { recursive: true });
-  mkdirSync(join(root, "policies"), { recursive: true });
-
-  // .env mode 600
-  const envPath = join(root, ".env");
-  writeFileSync(envPath, "ANTHROPIC_API_KEY=sk-test\nMAESTRO_SCOPED_PERMISSIONS=1\n");
-  chmodSync(envPath, 0o600);
-
-  // send-gate + daemon hooks
-  writeFileSync(join(root, ".claude/settings.json"), GOOD_SETTINGS);
-  for (const s of ["block-mcp-slack-send.sh", "pre-send-audit.sh", "post-action-log.sh", "session-end-log.sh"]) {
-    writeFileSync(join(root, "scripts/hooks", s), "#!/bin/bash\nexit 0\n");
-  }
-
-  // backup configured
-  writeFileSync(join(root, ".maestro/backup-config.yaml"), "enabled: true\nprovider: s3\nbucket: agent-backups\n");
-
-  // disclosure policy enabled
-  writeFileSync(join(root, "config/validation-rules.yaml"),
-    "ai_disclosure:\n  enabled: true\n  severity: block\n  blocked_phrases:\n    - \"I am an AI\"\n");
-
-  // governing policies (for the attestation digest)
-  writeFileSync(join(root, "policies/action-classification.yaml"), "classes: []\n");
-  writeFileSync(join(root, "config/agent.json"), JSON.stringify({ function: "executive-operator", altitude: "c-suite" }));
-
-  return root;
-}
-
-function cleanup(root) {
-  try { rmSync(root, { recursive: true, force: true }); } catch { /* */ }
-}
-
-// ---------------------------------------------------------------------------
-// Registry shape
-// ---------------------------------------------------------------------------
-
-test("CHECKS registry: every entry has id/severity/title/run and a valid severity", () => {
-  assert.ok(CHECKS.length >= 8, "expected the confirmed-posture checks");
-  const ids = new Set();
-  for (const c of CHECKS) {
-    assert.equal(typeof c.id, "string");
-    assert.ok(!ids.has(c.id), `duplicate check id ${c.id}`);
-    ids.add(c.id);
-    assert.equal(typeof c.run, "function");
-    assert.equal(typeof c.title, "string");
-    assert.ok(SEVERITIES.includes(c.severity), `bad severity ${c.severity} on ${c.id}`);
-  }
-  for (const want of ["env-mode-600", "scoped-permissions-configured", "send-gate-present",
-    "no-tracked-pyc-or-symlinks", "backup-configured", "disclosure-policy-present", "daemon-hooks-wired"]) {
-    assert.ok(ids.has(want), `missing check ${want}`);
-  }
-});
-
-// ---------------------------------------------------------------------------
-// runAudit — structured results
-// ---------------------------------------------------------------------------
-
-test("runAudit: a fully-conformant agent passes with structured per-check rows", async () => {
-  const root = makeGoodAgent();
-  try {
-    const r = await runAudit(root);
-    assert.equal(r.ok, true, JSON.stringify(r.results.filter((x) => !x.ok), null, 2));
-    assert.equal(r.results.length, CHECKS.length);
-    for (const row of r.results) {
-      assert.equal(typeof row.id, "string");
-      assert.equal(typeof row.ok, "boolean");
-      assert.equal(typeof row.detail, "string");
-      assert.ok(SEVERITIES.includes(row.severity));
-    }
-    assert.equal(r.summary.failed, 0);
-    assert.equal(r.summary.passed, CHECKS.length);
-    assert.equal(r.summary.worstSeverity, null);
-  } finally { cleanup(root); }
-});
-
-test("runAudit: a group-readable .env fails env-mode-600 and fails the run", async () => {
-  const root = makeGoodAgent();
-  try {
-    chmodSync(join(root, ".env"), 0o644);
-    const r = await runAudit(root);
-    assert.equal(r.ok, false);
-    const env = r.results.find((x) => x.id === "env-mode-600");
-    assert.equal(env.ok, false);
-    assert.match(env.detail, /readable/i);
-    assert.match(env.fix, /chmod 600/);
-    assert.equal(env.fixable, true);
-    assert.equal(r.summary.worstSeverity, "high");
-  } finally { cleanup(root); }
-});
-
-test("runAudit: missing scoped-permissions opt-in is a CRITICAL failure", async () => {
-  const root = makeGoodAgent();
-  try {
-    // Remove the opt-in from .env and ensure env override isn't set.
-    writeFileSync(join(root, ".env"), "ANTHROPIC_API_KEY=sk-test\n");
-    chmodSync(join(root, ".env"), 0o600);
-    const r = await runAudit(root, { env: {} });
-    const sp = r.results.find((x) => x.id === "scoped-permissions-configured");
-    assert.equal(sp.ok, false);
-    assert.equal(sp.severity, "critical");
-    assert.equal(r.ok, false);
-    assert.equal(r.summary.worstSeverity, "critical");
-  } finally { cleanup(root); }
-});
-
-test("runAudit: env override MAESTRO_SCOPED_PERMISSIONS=1 satisfies the check", async () => {
-  const root = makeGoodAgent();
-  try {
-    writeFileSync(join(root, ".env"), "ANTHROPIC_API_KEY=sk-test\n");
-    chmodSync(join(root, ".env"), 0o600);
-    const r = await runAudit(root, { env: { MAESTRO_SCOPED_PERMISSIONS: "1" } });
-    const sp = r.results.find((x) => x.id === "scoped-permissions-configured");
-    assert.equal(sp.ok, true);
-  } finally { cleanup(root); }
-});
-
-test("runAudit: a tracked .pyc fails the build-artifact check", async () => {
-  const root = makeGoodAgent();
-  try {
-    mkdirSync(join(root, "scripts/__pycache__"), { recursive: true });
-    writeFileSync(join(root, "scripts/__pycache__/validate.cpython-311.pyc"), "\0\0compiled");
-    const r = await runAudit(root, { env: { MAESTRO_SCOPED_PERMISSIONS: "1" } });
-    const art = r.results.find((x) => x.id === "no-tracked-pyc-or-symlinks");
-    assert.equal(art.ok, false);
-    assert.match(art.detail, /\.pyc/);
-    assert.equal(art.fixable, true, "a pure-.pyc offence is auto-fixable");
-  } finally { cleanup(root); }
-});
-
-test("runAudit: a symlink under scripts/ fails and is NOT auto-fixable", async () => {
-  const root = makeGoodAgent();
-  try {
-    writeFileSync(join(root, "scripts/real-target.py"), "# real\n");
-    symlinkSync(join(root, "scripts/real-target.py"), join(root, "scripts/linked.py"));
-    const r = await runAudit(root, { env: { MAESTRO_SCOPED_PERMISSIONS: "1" } });
-    const art = r.results.find((x) => x.id === "no-tracked-pyc-or-symlinks");
-    assert.equal(art.ok, false);
-    assert.match(art.detail, /symlink/);
-    assert.equal(art.fixable, false, "symlinks are never blindly removed");
-  } finally { cleanup(root); }
-});
-
-test("runAudit: disclosure policy disabled is a HIGH failure", async () => {
-  const root = makeGoodAgent();
-  try {
-    writeFileSync(join(root, "config/validation-rules.yaml"),
-      "ai_disclosure:\n  enabled: false\n  severity: block\n");
-    const r = await runAudit(root, { env: { MAESTRO_SCOPED_PERMISSIONS: "1" } });
-    const d = r.results.find((x) => x.id === "disclosure-policy-present");
-    assert.equal(d.ok, false);
-    assert.match(d.detail, /Art\. 50|deny being an AI/i);
-  } finally { cleanup(root); }
-});
-
-test("runAudit: backup not configured is a failure", async () => {
-  const root = makeGoodAgent();
-  try {
-    rmSync(join(root, ".maestro/backup-config.yaml"));
-    const r = await runAudit(root, { env: { MAESTRO_SCOPED_PERMISSIONS: "1" } });
-    const b = r.results.find((x) => x.id === "backup-configured");
-    assert.equal(b.ok, false);
-    assert.match(b.detail, /backup/i);
-  } finally { cleanup(root); }
-});
-
-test("runAudit: backup — an EMPTY bucket is not an off-machine tier", async () => {
-  // The regex form matched /^\s*bucket:\s*\S/m against `bucket: ""` — `\S` hits
-  // the quote character — and that is the value THIS PROJECT SHIPS in its own
-  // default config. The audit said "off-machine tier set" while
-  // resolveBackupPlan said offsiteConfigured:false and the runner logged
-  // "offsite is NOT configured": three subsystems, three answers, one file.
-  const root = makeGoodAgent();
-  try {
-    writeFileSync(
-      join(root, ".maestro/backup-config.yaml"),
-      "enabled: true\nprefix: acme\nlocal:\n  enabled: true\noffsite:\n  provider: gcs\n  bucket: \"\"\n"
-    );
-    const r = await runAudit(root, { env: { MAESTRO_SCOPED_PERMISSIONS: "1" } });
-    const b = r.results.find((x) => x.id === "backup-configured");
-    assert.equal(b.ok, true, "local restore points are a real posture");
-    assert.match(b.detail, /LOCAL restore points only/, "but it must not claim off-machine");
-  } finally { cleanup(root); }
-});
-
-test("runAudit: backup — top-level `enabled: false` is a FAILURE even with a nested local.enabled: true", async () => {
-  // `^\s*` under /m matched the INDENTED key, so the audit passed on a machine
-  // where backup-run.mjs refuses to run and doctor FAILs.
-  const root = makeGoodAgent();
-  try {
-    writeFileSync(
-      join(root, ".maestro/backup-config.yaml"),
-      "enabled: false\nprefix: acme\nlocal:\n  enabled: true\n"
-    );
-    const r = await runAudit(root, { env: { MAESTRO_SCOPED_PERMISSIONS: "1" } });
-    const b = r.results.find((x) => x.id === "backup-configured");
-    assert.equal(b.ok, false);
-    assert.match(b.detail, /REFUSES to run/);
-  } finally { cleanup(root); }
-});
-
-test("runAudit: a check that throws becomes an ok:false posture row, not a crash", async () => {
-  const root = makeGoodAgent();
-  try {
-    const boom = { id: "boom", severity: "low", title: "throws", run() { throw new Error("kaboom"); } };
-    const r = await runAudit(root, { env: { MAESTRO_SCOPED_PERMISSIONS: "1" }, checks: [boom] });
-    assert.equal(r.results.length, 1);
-    assert.equal(r.results[0].ok, false);
-    assert.match(r.results[0].detail, /kaboom/);
-    assert.equal(r.ok, false);
-  } finally { cleanup(root); }
-});
-
-// ---------------------------------------------------------------------------
-// Suppressions-with-reasons
-// ---------------------------------------------------------------------------
-
-test("runAudit: a failing check is suppressed (not failed) when given a reason", async () => {
-  const root = makeGoodAgent();
-  try {
-    chmodSync(join(root, ".env"), 0o644); // breaks env-mode-600
-    const r = await runAudit(root, {
-      env: { MAESTRO_SCOPED_PERMISSIONS: "1" },
-      suppressions: [{ id: "env-mode-600", reason: "shared CI runner; secrets are ephemeral" }],
-    });
-    assert.equal(r.ok, true, "suppressed-with-reason should not fail the run");
-    assert.equal(r.suppressed.length, 1);
-    assert.equal(r.suppressed[0].id, "env-mode-600");
-    assert.match(r.suppressed[0].suppressionReason, /shared CI/);
-    const env = r.results.find((x) => x.id === "env-mode-600");
-    assert.equal(env.suppressed, true);
-    assert.equal(env.ok, false, "the underlying finding is still recorded as failing");
-  } finally { cleanup(root); }
-});
-
-test("loadSuppressions: an entry with no reason is NOT honoured (fail-safe)", async () => {
-  const root = makeGoodAgent();
-  try {
-    const m = await loadSuppressions(root, { suppressions: [{ id: "env-mode-600" }, { id: "x", reason: "ok" }] });
-    assert.equal(m.has("env-mode-600"), false, "reasonless suppression dropped");
-    assert.equal(m.has("x"), true);
-  } finally { cleanup(root); }
-});
-
-test("loadSuppressions: an expired `until` is not honoured", async () => {
-  const root = makeGoodAgent();
-  try {
-    const now = Date.parse("2026-06-11T00:00:00Z");
-    const m = await loadSuppressions(root, {
-      now,
-      suppressions: [
-        { id: "expired", reason: "old waiver", until: "2026-01-01" },
-        { id: "live", reason: "current waiver", until: "2026-12-31" },
-      ],
-    });
-    assert.equal(m.has("expired"), false);
-    assert.equal(m.has("live"), true);
-  } finally { cleanup(root); }
-});
-
-test("loadSuppressions: reads config/audit-suppressions.yaml from disk", async () => {
-  const root = makeGoodAgent();
-  try {
-    writeFileSync(join(root, "config/audit-suppressions.yaml"),
-      "- id: backup-configured\n  reason: dev laptop, no backup needed\n");
-    const m = await loadSuppressions(root);
-    assert.equal(m.has("backup-configured"), true);
-    assert.match(m.get("backup-configured").reason, /dev laptop/);
-  } finally { cleanup(root); }
-});
-
-test("loadSuppressions: a malformed file is ignored, never trusted", async () => {
-  const root = makeGoodAgent();
-  try {
-    writeFileSync(join(root, "config/audit-suppressions.yaml"), ":::not yaml::: [[[\n");
-    const m = await loadSuppressions(root);
-    assert.equal(m.size, 0);
-  } finally { cleanup(root); }
-});
-
-// ---------------------------------------------------------------------------
-// --fix
-// ---------------------------------------------------------------------------
-
-test("applyFixes: chmods a group-readable .env back to 600 and clears the finding", async () => {
-  const root = makeGoodAgent();
-  try {
-    chmodSync(join(root, ".env"), 0o644);
-    const { fixed, audit } = await applyFixes(root, { env: { MAESTRO_SCOPED_PERMISSIONS: "1" } });
-    assert.ok(fixed.includes("env-mode-600"));
-    const mode = statSync(join(root, ".env")).mode & 0o777;
-    assert.equal(mode, 0o600);
-    assert.equal(audit.results.find((x) => x.id === "env-mode-600").ok, true);
-  } finally { cleanup(root); }
-});
-
-test("applyFixes: removes a tracked __pycache__/.pyc but leaves symlinks alone", async () => {
-  const root = makeGoodAgent();
-  try {
-    mkdirSync(join(root, "lib/__pycache__"), { recursive: true });
-    writeFileSync(join(root, "lib/__pycache__/x.cpython-311.pyc"), "\0compiled");
-    const { fixed } = await applyFixes(root, { env: { MAESTRO_SCOPED_PERMISSIONS: "1" } });
-    assert.ok(fixed.includes("no-tracked-pyc-or-symlinks"));
-    assert.equal(existsSync(join(root, "lib/__pycache__")), false);
-  } finally { cleanup(root); }
-});
-
-test("applyFixes: does NOT touch checks that are not auto-fixable", async () => {
-  const root = makeGoodAgent();
-  try {
-    rmSync(join(root, ".maestro/backup-config.yaml")); // not auto-fixable
-    const { fixed, audit } = await applyFixes(root, { env: { MAESTRO_SCOPED_PERMISSIONS: "1" } });
-    assert.ok(!fixed.includes("backup-configured"));
-    assert.equal(audit.results.find((x) => x.id === "backup-configured").ok, false);
-  } finally { cleanup(root); }
-});
-
-// ---------------------------------------------------------------------------
-// Attestation — stable + policy-sensitive
-// ---------------------------------------------------------------------------
-
-test("buildAttestation: hash is stable across two runs of the same posture", async () => {
-  const root = makeGoodAgent();
-  try {
-    const a1 = buildAttestation(await runAudit(root), root, { now: 1000 });
-    const a2 = buildAttestation(await runAudit(root), root, { now: 9999 });
-    assert.equal(a1.hash, a2.hash, "same posture + policy ⇒ same hash regardless of timestamp");
-    assert.equal(a1.schema, "maestro.audit.attestation.v1");
-    assert.equal(a1.ok, true);
-    assert.equal(a1.identity.function, "executive-operator");
-    assert.notEqual(a1.generatedAt, a2.generatedAt, "generatedAt is carried but excluded from the hash");
-  } finally { cleanup(root); }
-});
-
-test("buildAttestation: hash CHANGES when a governing policy changes", async () => {
-  const root = makeGoodAgent();
-  try {
-    const before = buildAttestation(await runAudit(root), root);
-    writeFileSync(join(root, "policies/action-classification.yaml"), "classes:\n  - id: financial\n    risk: high\n");
-    const after = buildAttestation(await runAudit(root), root);
-    assert.notEqual(before.hash, after.hash, "policy change must move the attestation hash");
-  } finally { cleanup(root); }
-});
-
-test("buildAttestation: hash CHANGES when the archetype changes", async () => {
-  const root = makeGoodAgent();
-  try {
-    const before = buildAttestation(await runAudit(root), root);
-    writeFileSync(join(root, "config/agent.json"), JSON.stringify({ function: "operations-leader", altitude: "vp" }));
-    const after = buildAttestation(await runAudit(root), root);
-    assert.notEqual(before.hash, after.hash);
-    assert.equal(after.identity.function, "operations-leader");
-  } finally { cleanup(root); }
-});
-
-test("buildAttestation: hash CHANGES when a check outcome flips", async () => {
-  const root = makeGoodAgent();
-  try {
-    const before = buildAttestation(await runAudit(root), root);
-    chmodSync(join(root, ".env"), 0o644); // flip env-mode-600 to failing
-    const after = buildAttestation(await runAudit(root), root);
-    assert.notEqual(before.hash, after.hash);
-    assert.equal(after.ok, false);
-  } finally { cleanup(root); }
-});
-
-test("buildAttestation: posture projection is sorted by id and excludes detail text", async () => {
-  const root = makeGoodAgent();
-  try {
-    const att = buildAttestation(await runAudit(root), root);
-    const ids = att.posture.map((p) => p.id);
-    assert.deepEqual(ids, [...ids].sort());
-    for (const p of att.posture) {
-      assert.deepEqual(Object.keys(p).sort(), ["id", "ok", "severity", "suppressed"]);
-    }
-  } finally { cleanup(root); }
-});