chainlesschain 0.166.22 → 0.166.25

package/src/lib/evolution/evolution-evidence-projector.js

@@ -10,6 +10,15 @@
  */
 
 import { createHash, randomBytes } from "node:crypto";
+import {
+  projectAgentOpaqueTransportBlock,
+  snapshotAgentModelRequest,
+} from "./agent-model-projection.js";
+import {
+  createEvidenceJsonTextBoundary,
+  EVIDENCE_JSON_POLICY,
+  isAgentToolArgumentsPath,
+} from "./evidence-json-text.js";
 
 export const EVOLUTION_SOURCE_VERIFICATION_SCHEMA =
   "chainlesschain.evolution-source-verification/v1";
@@ -448,6 +457,70 @@ const RULESET_DIGEST = `sha256:${createHash("sha256")
   )
   .digest("hex")}`;
 
+// A separate policy keeps existing v2 artifacts readable without changing
+// their meaning/digest. Agent messages are authenticated and scanned as whole
+// fields; the existing total model-content ceiling still applies. Never scan
+// independent 8 KiB slices, which would miss secrets/injections at the seams.
+const PREFIXED_CREDENTIAL_SOURCE =
+  "(?:sk|ghp|github_pat|xox[baprs])[-_][A-Za-z0-9_-]{12,}";
+const EMBEDDED_CREDENTIAL_SOURCE =
+  "(?:sk-|ghp_|github_pat_|xox[baprs]-)[A-Za-z0-9_-]{12,}";
+const AGENT_MODEL_RULESET_V1 = Object.freeze({
+  ...RULESET,
+  schema: "chainlesschain.evolution-agent-model-projection-rules/v1",
+  version: 1,
+  maxStringChars: RULESET.maxModelBytes,
+  maxInputBytes: 1024 * 1024,
+  textPolicy: "whole-field-before-budget-v1",
+  overflowPolicy: "reject",
+  embeddedCredentialPattern: EMBEDDED_CREDENTIAL_SOURCE,
+});
+const AGENT_MODEL_RULESET_V1_DIGEST = `sha256:${createHash("sha256")
+  .update(
+    `${AGENT_MODEL_RULESET_V1.schema}\0${JSON.stringify(AGENT_MODEL_RULESET_V1)}`,
+    "utf8",
+  )
+  .digest("hex")}`;
+const AGENT_MODEL_RULESET_V2 = Object.freeze({
+  ...AGENT_MODEL_RULESET_V1,
+  schema: "chainlesschain.evolution-agent-model-projection-rules/v2",
+  version: 2,
+  jsonTextPolicy: EVIDENCE_JSON_POLICY,
+});
+export const EVOLUTION_AGENT_MODEL_PROJECTION_RULESET_V2_DIGEST = `sha256:${createHash(
+  "sha256",
+)
+  .update(
+    `${AGENT_MODEL_RULESET_V2.schema}\0${JSON.stringify(AGENT_MODEL_RULESET_V2)}`,
+    "utf8",
+  )
+  .digest("hex")}`;
+const AGENT_MODEL_RULESET = Object.freeze({
+  ...AGENT_MODEL_RULESET_V2,
+  schema: "chainlesschain.evolution-agent-model-projection-rules/v3",
+  version: 3,
+  opaqueTransportPolicy:
+    "digest-bound-base64-image-and-signed-thinking-no-plaintext-projection-v1",
+});
+export const EVOLUTION_AGENT_MODEL_PROJECTION_RULESET_DIGEST = `sha256:${createHash(
+  "sha256",
+)
+  .update(
+    `${AGENT_MODEL_RULESET.schema}\0${JSON.stringify(AGENT_MODEL_RULESET)}`,
+    "utf8",
+  )
+  .digest("hex")}`;
+const SUPPORTED_RULESETS = new Map([
+  [RULESET_DIGEST, RULESET],
+  [AGENT_MODEL_RULESET_V1_DIGEST, AGENT_MODEL_RULESET_V1],
+  [EVOLUTION_AGENT_MODEL_PROJECTION_RULESET_V2_DIGEST, AGENT_MODEL_RULESET_V2],
+  [EVOLUTION_AGENT_MODEL_PROJECTION_RULESET_DIGEST, AGENT_MODEL_RULESET],
+]);
+
+const PREFIXED_CREDENTIAL_PATTERN = Object.freeze({
+  type: "credential",
+  regex: new RegExp(`\\b${PREFIXED_CREDENTIAL_SOURCE}\\b`, "gu"),
+});
 const VALUE_PATTERNS = Object.freeze([
   Object.freeze({
     type: "private-key",
@@ -463,10 +536,7 @@ const VALUE_PATTERNS = Object.freeze([
     regex:
       /\b(?:api[_-]?key|access[_-]?token|refresh[_-]?token|client[_-]?secret|password|passwd|secret)\s*[:=]\s*["']?[A-Za-z0-9._~+/=-]{8,}["']?/giu,
   }),
-  Object.freeze({
-    type: "credential",
-    regex: /\b(?:sk|ghp|github_pat|xox[baprs])[-_][A-Za-z0-9_-]{12,}\b/gu,
-  }),
+  PREFIXED_CREDENTIAL_PATTERN,
   Object.freeze({
     type: "credential",
     regex: /\bAKIA[A-Z0-9]{16}\b/gu,
@@ -506,6 +576,24 @@ const VALUE_PATTERNS = Object.freeze([
   }),
 ]);
 
+const AGENT_VALUE_PATTERNS = Object.freeze([
+  ...VALUE_PATTERNS,
+  Object.freeze({
+    type: "credential",
+    // Use each provider's real prefix/delimiter for embedded matches. Treating
+    // embedded "sk_" as a key would corrupt identifiers like ask_user_question.
+    regex: new RegExp(`${EMBEDDED_CREDENTIAL_SOURCE}\\b`, "gu"),
+  }),
+]);
+
+function valuePatterns(ruleset) {
+  return ruleset === AGENT_MODEL_RULESET ||
+    ruleset === AGENT_MODEL_RULESET_V2 ||
+    ruleset === AGENT_MODEL_RULESET_V1
+    ? AGENT_VALUE_PATTERNS
+    : VALUE_PATTERNS;
+}
+
 const REDACTION_TYPES = new Set([
   ...VALUE_PATTERNS.map(({ type }) => type),
   "pii",
@@ -689,10 +777,10 @@ function deepFreeze(value, seen = new WeakSet()) {
   return Object.freeze(value);
 }
 
-function assertNoKnownSecrets(content) {
+function assertNoKnownSecrets(content, patterns = VALUE_PATTERNS) {
   const serialized =
     typeof content === "string" ? content : canonicalJson(content);
-  for (const pattern of VALUE_PATTERNS) {
+  for (const pattern of patterns) {
     pattern.regex.lastIndex = 0;
     if (pattern.regex.test(serialized)) {
       throw projectionError(
@@ -1080,9 +1168,9 @@ function sanitizeKey(key, path, findings, injections) {
   return safeKey;
 }
 
-function sanitizeString(value, path, findings, injections) {
+function sanitizeString(value, path, findings, injections, ruleset, patterns) {
   let output = value;
-  for (const pattern of VALUE_PATTERNS) {
+  for (const pattern of patterns) {
     pattern.regex.lastIndex = 0;
     let count = 0;
     output = output.replace(pattern.regex, () => {
@@ -1094,7 +1182,7 @@ function sanitizeString(value, path, findings, injections) {
   }
   const securityNormalized = unicodeSecurityText(output);
   if (securityNormalized !== output) {
-    for (const pattern of VALUE_PATTERNS) {
+    for (const pattern of patterns) {
       pattern.regex.lastIndex = 0;
       if (pattern.regex.test(securityNormalized)) {
         findings.push({
@@ -1115,25 +1203,40 @@ function sanitizeString(value, path, findings, injections) {
     });
     output = "[QUARANTINED:POTENTIAL_PROMPT_INJECTION]";
   }
-  if (output.length > RULESET.maxStringChars) {
+  if (output.length > ruleset.maxStringChars) {
+    if (ruleset.overflowPolicy === "reject") {
+      throw projectionError(
+        EVOLUTION_PROJECTION_INVALID_CODE,
+        "Agent model projection exceeds the text budget",
+      );
+    }
     findings.push({
       type: "content-truncation",
       path: findingPath(path),
       count: 1,
     });
-    output = `${output.slice(0, RULESET.maxStringChars)}[TRUNCATED]`;
+    output = `${output.slice(0, ruleset.maxStringChars)}[TRUNCATED]`;
   }
   return output;
 }
 
-function sanitizePayload(payload) {
+function sanitizePayload(
+  payload,
+  ruleset = RULESET,
+  patterns = valuePatterns(ruleset),
+) {
   const findings = [];
   const injections = [];
   let nodes = 0;
   let truncated = false;
-  const visit = (value, path, depth) => {
+  const jsonText = ruleset.jsonTextPolicy
+    ? createEvidenceJsonTextBoundary((message) =>
+        projectionError(EVOLUTION_PROJECTION_INVALID_CODE, message),
+      )
+    : null;
+  const visit = (value, path, depth, layer = 0) => {
     nodes += 1;
-    if (nodes > RULESET.maxNodes || depth > RULESET.maxDepth) {
+    if (nodes > ruleset.maxNodes || depth > ruleset.maxDepth) {
       truncated = true;
       findings.push({
         type: "content-truncation",
@@ -1142,8 +1245,53 @@ function sanitizePayload(payload) {
       });
       return "[TRUNCATED:CONTENT_BUDGET]";
     }
-    if (typeof value === "string")
-      return sanitizeString(value, path, findings, injections);
+    const toolArguments = jsonText && isAgentToolArgumentsPath(path);
+    const opaqueTransportBlock =
+      ruleset === AGENT_MODEL_RULESET
+        ? projectAgentOpaqueTransportBlock(value, path.join("."))
+        : null;
+    if (opaqueTransportBlock !== null) return opaqueTransportBlock;
+    if (toolArguments && typeof value !== "string" && !isPlainRecord(value)) {
+      throw projectionError(
+        EVOLUTION_PROJECTION_INVALID_CODE,
+        "Agent tool arguments require a JSON object",
+      );
+    }
+    if (typeof value === "string") {
+      const plain = (text) =>
+        sanitizeString(text, path, findings, injections, ruleset, patterns);
+      if (!jsonText) return plain(value);
+      const output = jsonText(value, {
+        layer,
+        objectOnly: toolArguments,
+        visit: (decoded, nextLayer) =>
+          visit(decoded, [...path, "$json"], depth + 1, nextLayer),
+        plain,
+        equal: (left, right) => canonicalJson(left) === canonicalJson(right),
+      });
+      if (output.length > ruleset.maxStringChars) {
+        throw projectionError(
+          EVOLUTION_PROJECTION_INVALID_CODE,
+          "Agent model projection exceeds the text budget",
+        );
+      }
+      // Check instruction spans, but never regex-rewrite JSON syntax/numbers
+      // after decoding: numeric PII must become a quoted, valid JSON marker.
+      const matched = injectionMatches(output);
+      if (matched.length) {
+        injections.push({
+          path: findingPath(path),
+          patternIds: matched.map(({ id }) => id),
+        });
+        if (toolArguments)
+          throw projectionError(
+            EVOLUTION_PROJECTION_INVALID_CODE,
+            "Agent tool arguments contain unsafe cross-field text",
+          );
+        return "[QUARANTINED:POTENTIAL_PROMPT_INJECTION]";
+      }
+      return output;
+    }
     if (value === null || typeof value === "boolean") return value;
     if (typeof value === "number") {
       if (!Number.isFinite(value)) {
@@ -1152,12 +1300,30 @@ function sanitizePayload(payload) {
           "Raw payload contains a non-finite number",
         );
       }
+      if (jsonText) {
+        if (Number.isInteger(value) && !Number.isSafeInteger(value)) {
+          throw projectionError(
+            EVOLUTION_PROJECTION_INVALID_CODE,
+            "Agent model JSON number exceeds the safe integer budget",
+          );
+        }
+        const numberText = String(value);
+        const safe = sanitizeString(
+          numberText,
+          path,
+          findings,
+          injections,
+          ruleset,
+          patterns,
+        );
+        if (safe !== numberText) return safe;
+      }
       return value;
     }
     if (Array.isArray(value)) {
       const output = [];
       for (let index = 0; index < value.length; index += 1) {
-        if (nodes >= RULESET.maxNodes) {
+        if (nodes >= ruleset.maxNodes) {
           truncated = true;
           findings.push({
             type: "content-truncation",
@@ -1167,7 +1333,9 @@ function sanitizePayload(payload) {
           output.push("[TRUNCATED:CONTENT_BUDGET]");
           break;
         }
-        output.push(visit(value[index], [...path, String(index)], depth + 1));
+        output.push(
+          visit(value[index], [...path, String(index)], depth + 1, layer),
+        );
       }
       return output;
     }
@@ -1183,7 +1351,7 @@ function sanitizePayload(payload) {
     // projected content before the receipt is attested.
     const output = Object.create(null);
     for (const key of Object.keys(value).sort()) {
-      if (nodes >= RULESET.maxNodes) {
+      if (nodes >= ruleset.maxNodes) {
         truncated = true;
         findings.push({
           type: "content-truncation",
@@ -1210,7 +1378,12 @@ function sanitizePayload(payload) {
           count: 1,
         });
       } else {
-        projectedValue = visit(value[key], [...path, safeKey], depth + 1);
+        projectedValue = visit(
+          value[key],
+          [...path, safeKey],
+          depth + 1,
+          layer,
+        );
       }
       Object.defineProperty(output, safeKey, {
         value: projectedValue,
@@ -1223,7 +1396,7 @@ function sanitizePayload(payload) {
   };
   let content = visit(payload, [], 0);
   if (
-    Buffer.byteLength(canonicalJson(content), "utf8") > RULESET.maxModelBytes
+    Buffer.byteLength(canonicalJson(content), "utf8") > ruleset.maxModelBytes
   ) {
     truncated = true;
     content = {
@@ -1241,12 +1414,21 @@ function sanitizePayload(payload) {
     byType[finding.type] = (byType[finding.type] || 0) + finding.count;
     paths.add(finding.path);
   }
+  if (
+    ruleset.overflowPolicy === "reject" &&
+    (truncated || (byType["content-truncation"] ?? 0) > 0)
+  ) {
+    throw projectionError(
+      EVOLUTION_PROJECTION_INVALID_CODE,
+      "Agent model projection exceeds the content budget",
+    );
+  }
   const redactionSummary = {
     total: Object.values(byType).reduce((sum, count) => sum + count, 0),
     byType,
     paths: [...paths].sort(),
   };
-  assertNoKnownSecrets(content);
+  assertNoKnownSecrets(content, patterns);
   return deepFreeze({
     content,
     redactionSummary,
@@ -1936,7 +2118,7 @@ function buildRawRecord(source, commitment, evidenceId, storage) {
   });
 }
 
-function buildModelProjection(raw, sanitized) {
+function buildModelProjection(raw, sanitized, rulesetDigest) {
   const opaque = ["confidential", "restricted"].includes(raw.sensitivity);
   const core = {
     schema: EVOLUTION_MODEL_PROJECTION_SCHEMA,
@@ -1952,7 +2134,7 @@ function buildModelProjection(raw, sanitized) {
       : sanitized.redactionSummary,
     injectionFindings: opaque ? [] : sanitized.injectionFindings,
     truncated: opaque ? false : sanitized.truncated,
-    rulesetDigest: RULESET_DIGEST,
+    rulesetDigest,
   };
   return deepFreeze({
     ...core,
@@ -1990,7 +2172,7 @@ function buildTrustedProjection(
     status,
     reasonCodes,
     content: status === "trusted" ? trustedSanitized.content : null,
-    rulesetDigest: RULESET_DIGEST,
+    rulesetDigest: model.rulesetDigest,
   };
   return deepFreeze({
     ...core,
@@ -2013,7 +2195,7 @@ function buildReceipt(raw, model, trusted, createdAt) {
     rawRecordDigest: raw.rawRecordDigest,
     modelProjectionDigest: model.projectionDigest,
     trustedProjectionDigest: trusted.projectionDigest,
-    rulesetDigest: RULESET_DIGEST,
+    rulesetDigest: model.rulesetDigest,
     redactionSummary: model.redactionSummary,
     injectionCount: model.injectionFindings.length,
     trustedStatus: trusted.status,
@@ -2767,6 +2949,13 @@ function validateBundleIntegrity(bundle) {
     "projection receipt",
   );
   validateRawMetadata(raw, receipt);
+  const ruleset = SUPPORTED_RULESETS.get(model.rulesetDigest);
+  if (!ruleset) {
+    throw projectionError(
+      EVOLUTION_PROJECTION_INVALID_CODE,
+      "unsupported projection ruleset",
+    );
+  }
   validateRedactionSummary(model.redactionSummary);
   if (
     !Array.isArray(model.injectionFindings) ||
@@ -2864,9 +3053,8 @@ function validateBundleIntegrity(bundle) {
     canonicalJson(receipt.redactionSummary) !==
       canonicalJson(model.redactionSummary) ||
     receipt.trustedStatus !== trusted.status ||
-    model.rulesetDigest !== RULESET_DIGEST ||
-    trusted.rulesetDigest !== RULESET_DIGEST ||
-    receipt.rulesetDigest !== RULESET_DIGEST ||
+    trusted.rulesetDigest !== model.rulesetDigest ||
+    receipt.rulesetDigest !== model.rulesetDigest ||
     trusted.status !== expectedStatus ||
     canonicalJson(trusted.reasonCodes) !== canonicalJson(expectedReasons) ||
     (trusted.status === "quarantined" && trusted.content !== null) ||
@@ -2891,7 +3079,7 @@ function validateBundleIntegrity(bundle) {
     !Number.isSafeInteger(receipt.injectionCount) ||
     receipt.injectionCount < 0 ||
     Buffer.byteLength(canonicalJson(model.content), "utf8") >
-      RULESET.maxModelBytes
+      ruleset.maxModelBytes
   ) {
     throw projectionError(
       EVOLUTION_PROJECTION_INVALID_CODE,
@@ -2899,11 +3087,40 @@ function validateBundleIntegrity(bundle) {
     );
   }
   if (model.content !== null) {
-    assertNoKnownSecrets(model.content);
+    if (
+      ruleset === AGENT_MODEL_RULESET ||
+      ruleset === AGENT_MODEL_RULESET_V2 ||
+      ruleset === AGENT_MODEL_RULESET_V1
+    ) {
+      snapshotAgentModelRequest(model.content, {
+        allowStructuredArgumentText: ruleset !== AGENT_MODEL_RULESET_V1,
+        allowOpaqueTransportBlocks: ruleset === AGENT_MODEL_RULESET,
+      });
+      if (
+        model.truncated ||
+        (model.redactionSummary.byType["content-truncation"] ?? 0) > 0
+      ) {
+        throw projectionError(
+          EVOLUTION_PROJECTION_INVALID_CODE,
+          "Agent model projection cannot be truncated",
+        );
+      }
+    }
+    if (
+      ruleset.jsonTextPolicy &&
+      canonicalJson(sanitizePayload(model.content, ruleset).content) !==
+        canonicalJson(model.content)
+    ) {
+      throw projectionError(
+        EVOLUTION_PROJECTION_INVALID_CODE,
+        "Agent model JSON content is not a safe projection",
+      );
+    }
+    assertNoKnownSecrets(model.content, valuePatterns(ruleset));
     assertNoInjectionText(model.content);
   }
   if (trusted.content !== null) {
-    assertNoKnownSecrets(trusted.content);
+    assertNoKnownSecrets(trusted.content, valuePatterns(ruleset));
     assertNoInjectionText(trusted.content);
   }
   const attestation = validateAttestation(bundle.attestation, receipt);
@@ -3129,7 +3346,15 @@ export class EvolutionEvidenceProjector {
     return { iso, milliseconds: new Date(iso).getTime() };
   }
 
-  async project(input) {
+  project(input) {
+    return this.#project(input, RULESET);
+  }
+
+  projectAgentModelRequest(input) {
+    return this.#project(input, AGENT_MODEL_RULESET);
+  }
+
+  async #project(input, ruleset) {
     assertExactRecord(input, INPUT_KEYS, "evidence projection input");
     if (
       typeof input.sourceEnvelope !== "string" ||
@@ -3141,7 +3366,10 @@ export class EvolutionEvidenceProjector {
         "sourceEnvelope is invalid",
       );
     }
-    const payload = cloneCanonical(input.payload);
+    const payload =
+      ruleset === AGENT_MODEL_RULESET
+        ? snapshotAgentModelRequest(input.payload)
+        : cloneCanonical(input.payload);
     const sourceInputDigest = digest(
       payload,
       "chainlesschain.evolution-raw-plaintext/v2",
@@ -3381,10 +3609,16 @@ export class EvolutionEvidenceProjector {
           injectionFindings: [],
           truncated: false,
         })
-      : sanitizePayload(payload);
-    const modelProjection = buildModelProjection(rawRecord, sanitized);
+      : sanitizePayload(payload, ruleset);
+    const modelProjection = buildModelProjection(
+      rawRecord,
+      sanitized,
+      ruleset === AGENT_MODEL_RULESET
+        ? EVOLUTION_AGENT_MODEL_PROJECTION_RULESET_DIGEST
+        : RULESET_DIGEST,
+    );
     const trustedSanitized = source.compilable
-      ? sanitizePayload(source.trustedPayload)
+      ? sanitizePayload(source.trustedPayload, RULESET, valuePatterns(ruleset))
       : null;
     const trustedInputStructured =
       source.compilable && isStructuredTrustedPayload(source.trustedPayload);