@fluxpointstudios/orynq-sdk-process-trace 0.1.0 → 0.3.0

package/src/__tests__/hardening-round2.test.ts

@@ -0,0 +1,173 @@
+/**
+ * @summary Round-2 hardening regression tests for process-trace (#58, #59).
+ *
+ *  1. eip712 governance verifier must pin/validate the attacker-controlled
+ *     `event.eip712.{domain,primaryType,types}` against the expected
+ *     governance-attestation schema — an unexpected domain/primaryType is
+ *     rejected even when the raw signature "verifies".
+ *  2. verifyBundle must also verify the publicView model-manifest fields (the
+ *     shared artifact an external verifier reads), not just privateRun's — a
+ *     fabricated publicView.modelManifestHash must fail.
+ */
+
+import { describe, it, expect } from "vitest";
+import {
+  createTrace,
+  addSpan,
+  addEvent,
+  closeSpan,
+  finalizeTrace,
+  verifyBundle,
+  verifyGovernanceAttestations,
+  createEip712GovernanceVerifier,
+} from "../index.js";
+import type { TraceBundle, TraceRun } from "../index.js";
+
+const EXPECTED_DOMAIN = { name: "Orynq", version: "1" };
+const EXPECTED_TYPES = {
+  Attestation: [
+    { name: "role", type: "string" },
+    { name: "policyRef", type: "string" },
+    { name: "decisionRef", type: "string" },
+    { name: "runId", type: "string" },
+    { name: "signedAt", type: "string" },
+  ],
+};
+
+async function eip712Bundle(bindingOverrides: Record<string, unknown>): Promise<TraceBundle> {
+  const run = await createTrace({ agentId: "agent-1" });
+  const span = addSpan(run, { name: "approve", visibility: "public" });
+  const signedAt = new Date().toISOString();
+  await addEvent(run, span.id, {
+    kind: "governance-attestation",
+    visibility: "public",
+    role: "compliance",
+    policyRef: "sha256:policy",
+    decisionRef: "decision-1",
+    attestor: {
+      address: "0x1111111111111111111111111111111111111111",
+      signatureScheme: "eip712",
+    },
+    signature: "0x" + "ab".repeat(65),
+    signedAt,
+    eip712: {
+      domain: EXPECTED_DOMAIN,
+      types: EXPECTED_TYPES,
+      primaryType: "Attestation",
+      message: {
+        role: "compliance",
+        policyRef: "sha256:policy",
+        decisionRef: "decision-1",
+        runId: run.id,
+        signedAt,
+      },
+      ...bindingOverrides,
+    },
+  });
+  await closeSpan(run, span.id);
+  return finalizeTrace(run);
+}
+
+describe("eip712 domain/primaryType pinning (#58)", () => {
+  const ATTESTOR = "0x1111111111111111111111111111111111111111";
+
+  it("rejects an unexpected primaryType even when the raw signature verifies", async () => {
+    const bundle = await eip712Bundle({ primaryType: "EvilType" });
+    const verifier = createEip712GovernanceVerifier({
+      verifyTypedData: async () => true, // signature is 'valid' over attacker data
+      expectedDomain: EXPECTED_DOMAIN,
+      expectedPrimaryType: "Attestation",
+      expectedTypes: EXPECTED_TYPES,
+    });
+    const summaries = await verifyGovernanceAttestations(bundle, {
+      verifiers: { eip712: verifier },
+      authorizedAttestors: [ATTESTOR],
+    });
+    expect(summaries[0]!.authorized).toBe(true);
+    expect(summaries[0]!.verified).toBe(false);
+  });
+
+  it("rejects an unexpected domain (attacker swaps verifyingContract/chainId)", async () => {
+    const bundle = await eip712Bundle({
+      domain: { name: "Orynq", version: "1", chainId: 1, verifyingContract: "0xevil" },
+    });
+    const verifier = createEip712GovernanceVerifier({
+      verifyTypedData: async () => true,
+      expectedDomain: EXPECTED_DOMAIN,
+      expectedPrimaryType: "Attestation",
+      expectedTypes: EXPECTED_TYPES,
+    });
+    const summaries = await verifyGovernanceAttestations(bundle, {
+      verifiers: { eip712: verifier },
+      authorizedAttestors: [ATTESTOR],
+    });
+    expect(summaries[0]!.authorized).toBe(true);
+    expect(summaries[0]!.verified).toBe(false);
+  });
+
+  it("accepts the expected domain + primaryType", async () => {
+    const bundle = await eip712Bundle({});
+    const verifier = createEip712GovernanceVerifier({
+      verifyTypedData: async () => true,
+      expectedDomain: EXPECTED_DOMAIN,
+      expectedPrimaryType: "Attestation",
+      expectedTypes: EXPECTED_TYPES,
+    });
+    const summaries = await verifyGovernanceAttestations(bundle, {
+      verifiers: { eip712: verifier },
+      authorizedAttestors: [ATTESTOR],
+    });
+    expect(summaries[0]!.verified).toBe(true);
+  });
+});
+
+describe("publicView model-manifest verification (#59)", () => {
+  async function manifestBundle(): Promise<TraceBundle> {
+    const run: TraceRun = await createTrace({
+      agentId: "agent-m",
+      manifest: {
+        modelHash: "sha256:" + "1".repeat(64),
+        modelId: "gpt-x",
+        framework: "acme",
+      },
+    });
+    const span = addSpan(run, { name: "work", visibility: "public" });
+    await addEvent(run, span.id, { kind: "command", command: "go", visibility: "public" });
+    await closeSpan(run, span.id);
+    return finalizeTrace(run);
+  }
+
+  it("an honest bundle with a pinned manifest verifies", async () => {
+    const bundle = await manifestBundle();
+    expect(bundle.publicView.modelManifestHash).toBeDefined();
+    const result = await verifyBundle(bundle, { });
+    expect(result.checks.modelManifestValid).toBe(true);
+    expect(result.valid).toBe(true);
+  });
+
+  it("a fabricated publicView.modelManifestHash fails verification", async () => {
+    const bundle = await manifestBundle();
+    const forged = JSON.parse(JSON.stringify(bundle)) as TraceBundle;
+    // privateRun manifest is left honest; only the shared publicView field an
+    // external verifier reads is fabricated.
+    forged.publicView.modelManifestHash = "f".repeat(64);
+
+    const result = await verifyBundle(forged);
+    expect(result.checks.modelManifestValid).toBe(false);
+    expect(result.valid).toBe(false);
+  });
+
+  it("a fabricated publicView.modelManifest (recomputes to a different hash) fails", async () => {
+    const bundle = await manifestBundle();
+    const forged = JSON.parse(JSON.stringify(bundle)) as TraceBundle;
+    (forged.publicView.modelManifest as Record<string, unknown>) = {
+      modelHash: "sha256:" + "9".repeat(64),
+      modelId: "EVIL",
+      framework: "attacker",
+    };
+
+    const result = await verifyBundle(forged);
+    expect(result.checks.modelManifestValid).toBe(false);
+    expect(result.valid).toBe(false);
+  });
+});

package/src/__tests__/hardening-round3.test.ts

@@ -0,0 +1,208 @@
+/**
+ * @summary Round-3 hardening regression tests for process-trace governance (#58).
+ *
+ *  1. eip712 schema pins are MANDATORY. An attacker signs an EMPTY `Attestation`
+ *     struct (`types:{Attestation:[]}`) with their own key and carries
+ *     role/policyRef/decisionRef/runId as UNTYPED message extras — viem verifies
+ *     the empty struct and the field-binding post-check reads never-signed
+ *     fields. The verifier must reject unless the primaryType's declared type
+ *     fields actually include role/policyRef/decisionRef/runId.
+ *  2. Governance attestations require an authorized-attestor allow-list. An
+ *     attestation from an unlisted signer must NOT count toward a passing
+ *     verdict, and with no allow-list governance verification must fail closed.
+ */
+
+import { describe, it, expect } from "vitest";
+import {
+  createTrace,
+  addSpan,
+  addEvent,
+  closeSpan,
+  finalizeTrace,
+  verifyBundle,
+  addGovernanceAttestation,
+  createSr25519GovernanceSigner,
+  verifyGovernanceAttestations,
+  createEip712GovernanceVerifier,
+} from "../index.js";
+import type { TraceBundle, GovernanceAttestationEvent } from "../index.js";
+
+const SEED_A = "0x" + "11".repeat(32);
+const SEED_B = "0x" + "22".repeat(32);
+
+const EXPECTED_DOMAIN = { name: "Orynq", version: "1" };
+const EXPECTED_TYPES = {
+  Attestation: [
+    { name: "role", type: "string" },
+    { name: "policyRef", type: "string" },
+    { name: "decisionRef", type: "string" },
+    { name: "runId", type: "string" },
+    { name: "signedAt", type: "string" },
+  ],
+};
+
+async function eip712Bundle(binding: Record<string, unknown>): Promise<TraceBundle> {
+  const run = await createTrace({ agentId: "agent-1" });
+  const span = addSpan(run, { name: "approve", visibility: "public" });
+  const signedAt = new Date().toISOString();
+  await addEvent(run, span.id, {
+    kind: "governance-attestation",
+    visibility: "public",
+    role: "release-authority",
+    policyRef: "sha256:policy",
+    decisionRef: "decision-1",
+    attestor: {
+      address: "0x1111111111111111111111111111111111111111",
+      signatureScheme: "eip712",
+    },
+    signature: "0x" + "ab".repeat(65),
+    signedAt,
+    eip712: {
+      domain: EXPECTED_DOMAIN,
+      types: EXPECTED_TYPES,
+      primaryType: "Attestation",
+      message: {
+        role: "release-authority",
+        policyRef: "sha256:policy",
+        decisionRef: "decision-1",
+        runId: run.id,
+        signedAt,
+      },
+      ...binding,
+    },
+  });
+  await closeSpan(run, span.id);
+  return finalizeTrace(run);
+}
+
+describe("eip712 schema pins are mandatory (#58 round-3)", () => {
+  it("rejects an EMPTY Attestation struct that carries fields as untyped extras", async () => {
+    // The signed struct declares NO fields; role/policyRef/decisionRef/runId ride
+    // as untyped message extras the signature does not commit to.
+    const bundle = await eip712Bundle({
+      types: { Attestation: [] },
+    });
+    const verifier = createEip712GovernanceVerifier({
+      verifyTypedData: async () => true, // empty struct 'verifies' with attacker key
+      expectedDomain: EXPECTED_DOMAIN,
+      expectedPrimaryType: "Attestation",
+      expectedTypes: EXPECTED_TYPES,
+    });
+    const summaries = await verifyGovernanceAttestations(bundle, {
+      verifiers: { eip712: verifier },
+      authorizedAttestors: ["0x1111111111111111111111111111111111111111"],
+    });
+    expect(summaries[0]!.verified).toBe(false);
+  });
+
+  it("throws at construction when the schema pins are omitted", () => {
+    expect(() =>
+      // @ts-expect-error — pins are now mandatory
+      createEip712GovernanceVerifier({ verifyTypedData: async () => true })
+    ).toThrow(/expectedTypes|expectedDomain|expectedPrimaryType|required/i);
+  });
+
+  it("throws when the pinned primaryType omits a required field", () => {
+    expect(() =>
+      createEip712GovernanceVerifier({
+        verifyTypedData: async () => true,
+        expectedDomain: EXPECTED_DOMAIN,
+        expectedPrimaryType: "Attestation",
+        expectedTypes: {
+          Attestation: [
+            { name: "role", type: "string" },
+            { name: "policyRef", type: "string" },
+            // decisionRef + runId missing → signature would not commit to them
+          ],
+        },
+      })
+    ).toThrow(/decisionRef|runId|must include/i);
+  });
+
+  it("accepts a correctly-typed genuine attestation", async () => {
+    const bundle = await eip712Bundle({});
+    const verifier = createEip712GovernanceVerifier({
+      verifyTypedData: async () => true,
+      expectedDomain: EXPECTED_DOMAIN,
+      expectedPrimaryType: "Attestation",
+      expectedTypes: EXPECTED_TYPES,
+    });
+    const summaries = await verifyGovernanceAttestations(bundle, {
+      verifiers: { eip712: verifier },
+      authorizedAttestors: ["0x1111111111111111111111111111111111111111"],
+    });
+    expect(summaries[0]!.verified).toBe(true);
+  });
+});
+
+describe("governance requires an authorized-attestor allow-list (#58 round-3)", () => {
+  async function attestedBundle(): Promise<{ bundle: TraceBundle; attestor: string }> {
+    const run = await createTrace({ agentId: "agent-1" });
+    const span = addSpan(run, { name: "release", visibility: "public" });
+    const decision = await addEvent(run, span.id, {
+      kind: "decision",
+      decision: "ship model v2",
+      visibility: "public",
+    });
+    const signer = await createSr25519GovernanceSigner({ seed: SEED_A });
+    await addGovernanceAttestation(run, span.id, {
+      role: "release-authority",
+      policyRef: "sha256:policy-doc-hash",
+      decisionRef: decision.id,
+      signer,
+    });
+    await closeSpan(run, span.id);
+    return { bundle: await finalizeTrace(run), attestor: signer.address };
+  }
+
+  it("an attestation from an UNLISTED signer is not authorized", async () => {
+    const { bundle } = await attestedBundle();
+    const otherSigner = await createSr25519GovernanceSigner({ seed: SEED_B });
+    const summaries = await verifyGovernanceAttestations(bundle, {
+      authorizedAttestors: [otherSigner.address], // genuine signer NOT listed
+    });
+    expect(summaries[0]!.verified).toBe(false);
+    expect(summaries[0]!.authorized).toBe(false);
+  });
+
+  it("an attestation from an ALLOW-LISTED signer verifies", async () => {
+    const { bundle, attestor } = await attestedBundle();
+    const summaries = await verifyGovernanceAttestations(bundle, {
+      authorizedAttestors: [attestor],
+    });
+    expect(summaries[0]!.verified).toBe(true);
+    expect(summaries[0]!.authorized).toBe(true);
+  });
+
+  it("with NO allow-list, governance fails closed and does not pass the bundle", async () => {
+    const { bundle } = await attestedBundle();
+    // A genuine self-signed attestation with a valid signature — but nobody said
+    // this key is a real release-authority. It must not fold into a pass.
+    const summaries = await verifyGovernanceAttestations(bundle);
+    expect(summaries[0]!.verified).toBe(false);
+    expect(summaries[0]!.authorized).toBe(false);
+
+    const result = await verifyBundle(bundle, { governance: true });
+    expect(result.checks.governanceValid).toBe(false);
+    expect(result.valid).toBe(false);
+  });
+
+  it("verifyBundle with an allow-list folds a real sign-off into valid", async () => {
+    const { bundle, attestor } = await attestedBundle();
+    const result = await verifyBundle(bundle, {
+      governance: { authorizedAttestors: [attestor] },
+    });
+    expect(result.checks.governanceValid).toBe(true);
+    expect(result.valid).toBe(true);
+  });
+
+  it("an unlisted signer fails the bundle", async () => {
+    const { bundle } = await attestedBundle();
+    const other = await createSr25519GovernanceSigner({ seed: SEED_B });
+    const result = await verifyBundle(bundle, {
+      governance: { authorizedAttestors: [other.address] },
+    });
+    expect(result.checks.governanceValid).toBe(false);
+    expect(result.valid).toBe(false);
+  });
+});

package/src/__tests__/model-manifest.test.ts

@@ -0,0 +1,136 @@
+/**
+ * @summary Tests for pre-execution model-manifest pinning (issue #59).
+ */
+
+import { describe, it, expect, vi } from "vitest";
+import {
+  createTrace,
+  addSpan,
+  addEvent,
+  closeSpan,
+  finalizeTrace,
+  verifyBundle,
+  computeModelManifestHash,
+  validateModelManifest,
+  manifestFromHuggingFace,
+  manifestFromOpenAI,
+  manifestFromAnthropic,
+} from "../index.js";
+import type { ModelManifest } from "../index.js";
+
+async function sampleManifest(): Promise<ModelManifest> {
+  return manifestFromHuggingFace({
+    modelId: "meta-llama/Llama-3.1-8B",
+    revision: "0e9e39f249a16976918f6564b8830bc894c89659",
+    tokenizerHash: "sha256:deadbeef",
+  });
+}
+
+describe("model-manifest builders", () => {
+  it("produces a deterministic manifestHash for identical inputs", async () => {
+    const a = await manifestFromHuggingFace({ modelId: "m", revision: "abc" });
+    const b = await manifestFromHuggingFace({ modelId: "m", revision: "abc" });
+    expect(await computeModelManifestHash(a)).toBe(await computeModelManifestHash(b));
+  });
+
+  it("changes the manifestHash when the revision changes", async () => {
+    const a = await manifestFromHuggingFace({ modelId: "m", revision: "abc" });
+    const c = await manifestFromHuggingFace({ modelId: "m", revision: "def" });
+    expect(await computeModelManifestHash(a)).not.toBe(await computeModelManifestHash(c));
+  });
+
+  it("records the framework and identity for OpenAI / Anthropic", async () => {
+    const openai = await manifestFromOpenAI({ model: "gpt-4o", snapshotId: "gpt-4o-2024-08-06" });
+    expect(openai.framework).toBe("openai");
+    expect(openai.modelId).toBe("gpt-4o");
+    expect(openai.modelHash.startsWith("sha256:")).toBe(true);
+
+    const anthropic = await manifestFromAnthropic({
+      model: "claude-3-5-sonnet",
+      snapshotId: "claude-3-5-sonnet-20241022",
+    });
+    expect(anthropic.framework).toBe("anthropic");
+    expect(anthropic.revision).toBe("claude-3-5-sonnet-20241022");
+  });
+
+  it("hashes a raw system prompt into systemPromptHash", async () => {
+    const m = await manifestFromHuggingFace({
+      modelId: "m",
+      systemPrompt: "You are a helpful assistant.",
+    });
+    expect(m.systemPromptHash?.startsWith("sha256:")).toBe(true);
+  });
+
+  it("validateModelManifest requires a modelHash", () => {
+    expect(() => validateModelManifest({ modelHash: "" } as ModelManifest)).toThrow(/modelHash/);
+  });
+});
+
+describe("createTrace manifest pinning", () => {
+  it("pins the manifest hash at creation and freezes the manifest", async () => {
+    const manifest = await sampleManifest();
+    const run = await createTrace({ agentId: "agent-1", manifest });
+
+    expect(run.modelManifestHash).toBeDefined();
+    expect(run.modelManifestHash).toBe(await computeModelManifestHash(manifest));
+    expect(run.modelManifest).toBe(manifest);
+
+    // Immutability: mutating the pinned manifest throws (frozen).
+    expect(() => {
+      (run.modelManifest as unknown as Record<string, unknown>).modelHash = "tampered";
+    }).toThrow();
+  });
+
+  it("rejects strict createTrace without a manifest", async () => {
+    await expect(createTrace({ agentId: "agent-1", strict: true })).rejects.toThrow(
+      /strict mode requires/i
+    );
+  });
+});
+
+describe("finalizeTrace manifest enforcement", () => {
+  it("carries the manifest onto the bundle and public view, and verifies", async () => {
+    const manifest = await manifestFromOpenAI({
+      model: "gpt-4o",
+      snapshotId: "gpt-4o-2024-08-06",
+    });
+    const run = await createTrace({ agentId: "agent-1", manifest, strict: true });
+    const span = addSpan(run, { name: "infer", visibility: "public" });
+    await addEvent(run, span.id, {
+      kind: "observation",
+      observation: "ok",
+      visibility: "public",
+    });
+    await closeSpan(run, span.id);
+
+    const bundle = await finalizeTrace(run);
+
+    expect(bundle.modelManifestHash).toBe(run.modelManifestHash);
+    expect(bundle.modelManifest?.framework).toBe("openai");
+    expect(bundle.publicView.modelManifestHash).toBe(run.modelManifestHash);
+
+    const result = await verifyBundle(bundle);
+    expect(result.valid).toBe(true);
+    expect(result.errors).toHaveLength(0);
+  });
+
+  it("refuses to finalize a strict run with no manifest", async () => {
+    const run = await createTrace({ agentId: "agent-1" });
+    run.strict = true; // simulate a run that should have been pinned
+    await expect(finalizeTrace(run)).rejects.toThrow(/strict mode requires a model manifest/i);
+  });
+
+  it("warns (warn-only path) when finalizing an unpinned non-strict run", async () => {
+    const run = await createTrace({ agentId: "agent-1" });
+    const span = addSpan(run, { name: "s", visibility: "public" });
+    await addEvent(run, span.id, { kind: "observation", observation: "ok", visibility: "public" });
+    await closeSpan(run, span.id);
+
+    const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => undefined);
+    const bundle = await finalizeTrace(run);
+    expect(warnSpy).toHaveBeenCalled();
+    warnSpy.mockRestore();
+
+    expect(bundle.modelManifestHash).toBeUndefined();
+  });
+});

package/src/__tests__/types.test.ts

@@ -33,6 +33,8 @@ describe('HASH_DOMAIN_PREFIXES', () => {
       'node',
       'manifest',
       'root',
+      'modelManifest',
+      'governance',
     ];
 
     const actualKeys = Object.keys(HASH_DOMAIN_PREFIXES);
@@ -46,8 +48,10 @@ describe('HASH_DOMAIN_PREFIXES', () => {
   it('has correct prefix format for all domains', () => {
     for (const [domain, prefix] of Object.entries(HASH_DOMAIN_PREFIXES)) {
       // All prefixes should follow pattern "poi-trace:<domain>:v1|"
-      expect(prefix).toMatch(/^poi-trace:[a-z]+:v1\|$/);
-      expect(prefix).toContain(domain);
+      // (domain segment may be kebab-cased, e.g. "model-manifest").
+      expect(prefix).toMatch(/^poi-trace:[a-z-]+:v1\|$/);
+      // The camelCase key maps to the (possibly kebab-cased) prefix segment.
+      expect(prefix.replace(/-/g, '')).toContain(domain.toLowerCase());
     }
   });
 
@@ -101,6 +105,8 @@ describe('DEFAULT_EVENT_VISIBILITY', () => {
       'observation',
       'error',
       'custom',
+      'governance-attestation',
+      'tool-receipt',
     ];
 
     const actualKinds = Object.keys(DEFAULT_EVENT_VISIBILITY);