chainlesschain 0.166.117 → 0.166.118

package/src/lib/bounded-artifact-file-read.js

@@ -2,7 +2,10 @@
 import fs from "node:fs";
 import path from "node:path";
 import { readBoundedDescriptor } from "./evolution/bounded-descriptor-read.js";
-import { withEvolutionFileIdentity } from "./evolution/evolution-file-identity.js";
+import {
+  withEvolutionDirectoryFileIdentity,
+  withEvolutionFileIdentity,
+} from "./evolution/evolution-file-identity.js";
 import { artifactPhysicalIdentity } from "./evolution/evolution-artifact-identity.js";
 
 export const ARTIFACT_BOUNDED_READ_FAILED_CODE =
@@ -67,10 +70,10 @@ function sameStat(left, right) {
   );
 }
 
-/** Read one stable window only. Callers retain hash/signature and root checks. */
-export function readBoundedArtifactFile(target, expectedSize, maximumBytes) {
+function assertReadArguments(args) {
+  const [target, expectedSize, maximumBytes] = args;
   if (
-    arguments.length !== 3 ||
+    args.length !== 3 ||
     typeof target !== "string" ||
     !path.isAbsolute(target) ||
     path.resolve(target) !== target ||
@@ -83,49 +86,128 @@ export function readBoundedArtifactFile(target, expectedSize, maximumBytes) {
     throw new TypeError(
       "bounded artifact read requires a canonical path and finite size bounds",
     );
+}
+
+function readWithIdentity(target, expectedSize, maximumBytes, samePathHandle) {
+  const before = fs.lstatSync(target, { bigint: true });
+  safeStat(before, expectedSize, maximumBytes, "artifact path");
+  if (canonical(fs.realpathSync.native(target)) !== canonical(target))
+    fail("artifact path is not physically canonical");
+  const fd = fs.openSync(
+    target,
+    fs.constants.O_RDONLY |
+      (fs.constants.O_NOFOLLOW || 0) |
+      (fs.constants.O_NONBLOCK || 0),
+  );
   try {
-    return withEvolutionFileIdentity(DIRECTORY_FS, target, (samePathHandle) => {
-      const before = fs.lstatSync(target, { bigint: true });
-      safeStat(before, expectedSize, maximumBytes, "artifact path");
-      if (canonical(fs.realpathSync.native(target)) !== canonical(target))
-        fail("artifact path is not physically canonical");
-      const fd = fs.openSync(
-        target,
-        fs.constants.O_RDONLY |
-          (fs.constants.O_NOFOLLOW || 0) |
-          (fs.constants.O_NONBLOCK || 0),
-      );
-      try {
-        const opened = fs.fstatSync(fd, { bigint: true });
-        safeStat(opened, expectedSize, maximumBytes, "artifact descriptor");
-        if (!samePathHandle(before, opened) || !sameMetadata(before, opened))
-          fail("artifact path and descriptor identities differ");
-        const bytes = readBoundedDescriptor(
-          fs,
-          fd,
-          Number(opened.size),
-          maximumBytes,
-        );
-        const closed = fs.fstatSync(fd, { bigint: true });
-        const after = fs.lstatSync(target, { bigint: true });
-        safeStat(closed, expectedSize, maximumBytes, "artifact descriptor");
-        safeStat(after, expectedSize, maximumBytes, "artifact path");
-        if (!sameStat(opened, closed))
-          fail("artifact descriptor changed during bounded read");
-        if (
-          !sameStat(before, after) ||
-          !samePathHandle(after, closed) ||
-          !sameMetadata(after, closed) ||
-          canonical(fs.realpathSync.native(target)) !== canonical(target)
-        )
-          fail("artifact pathname changed during bounded read");
-        return bytes;
-      } finally {
-        fs.closeSync(fd);
-      }
-    });
+    const opened = fs.fstatSync(fd, { bigint: true });
+    safeStat(opened, expectedSize, maximumBytes, "artifact descriptor");
+    if (!samePathHandle(before, opened) || !sameMetadata(before, opened))
+      fail("artifact path and descriptor identities differ");
+    const bytes = readBoundedDescriptor(
+      fs,
+      fd,
+      Number(opened.size),
+      maximumBytes,
+    );
+    const closed = fs.fstatSync(fd, { bigint: true });
+    const after = fs.lstatSync(target, { bigint: true });
+    safeStat(closed, expectedSize, maximumBytes, "artifact descriptor");
+    safeStat(after, expectedSize, maximumBytes, "artifact path");
+    if (!sameStat(opened, closed))
+      fail("artifact descriptor changed during bounded read");
+    if (
+      !sameStat(before, after) ||
+      !samePathHandle(after, closed) ||
+      !sameMetadata(after, closed) ||
+      canonical(fs.realpathSync.native(target)) !== canonical(target)
+    )
+      fail("artifact pathname changed during bounded read");
+    return bytes;
+  } finally {
+    fs.closeSync(fd);
+  }
+}
+
+function wrapReadFailure(cause) {
+  if (cause?.code === ARTIFACT_BOUNDED_READ_FAILED_CODE) throw cause;
+  fail("artifact bounded descriptor read failed", cause);
+}
+
+/** Read one stable window only. Callers retain hash/signature and root checks. */
+export function readBoundedArtifactFile(target, expectedSize, maximumBytes) {
+  assertReadArguments(arguments);
+  try {
+    return withEvolutionFileIdentity(DIRECTORY_FS, target, (samePathHandle) =>
+      readWithIdentity(target, expectedSize, maximumBytes, samePathHandle),
+    );
+  } catch (cause) {
+    wrapReadFailure(cause);
+  }
+}
+
+/**
+ * A synchronous batch shares one authenticated parent handle, never file
+ * contents or authority. Each child keeps the complete independent read
+ * window above. The parent is reattested before the batch can return.
+ */
+export function withBoundedArtifactDirectoryRead(directory, callback) {
+  if (
+    arguments.length !== 2 ||
+    typeof directory !== "string" ||
+    !path.isAbsolute(directory) ||
+    path.resolve(directory) !== directory ||
+    typeof callback !== "function" ||
+    callback.constructor?.name === "AsyncFunction"
+  )
+    throw new TypeError(
+      "bounded artifact directory read requires a canonical directory and synchronous callback",
+    );
+  let callbackFailure = null;
+  try {
+    return withEvolutionDirectoryFileIdentity(
+      DIRECTORY_FS,
+      directory,
+      (samePathHandle, _sameStableHandle, { parentPath }) => {
+        if (canonical(parentPath) !== canonical(directory))
+          fail("artifact batch parent is not physically canonical");
+        let active = true;
+        const read = Object.freeze((...args) => {
+          if (!active) fail("artifact batch read window is closed");
+          assertReadArguments(args);
+          const [target, expectedSize, maximumBytes] = args;
+          if (canonical(path.dirname(target)) !== canonical(parentPath))
+            fail(
+              "artifact batch read must name a direct child of its captured parent",
+            );
+          try {
+            return readWithIdentity(
+              target,
+              expectedSize,
+              maximumBytes,
+              samePathHandle,
+            );
+          } catch (cause) {
+            wrapReadFailure(cause);
+          }
+        });
+        try {
+          const result = callback(read);
+          if (result && typeof result.then === "function")
+            throw new TypeError(
+              "artifact batch callback must not return a promise",
+            );
+          return result;
+        } catch (cause) {
+          callbackFailure = cause;
+          throw cause;
+        } finally {
+          active = false;
+        }
+      },
+    );
   } catch (cause) {
-    if (cause?.code === ARTIFACT_BOUNDED_READ_FAILED_CODE) throw cause;
-    fail("artifact bounded descriptor read failed", cause);
+    if (cause === callbackFailure) throw cause;
+    wrapReadFailure(cause);
   }
 }

package/src/lib/evolution/agent-evolution-ingress.js

@@ -1,6 +1,7 @@
 import crypto from "node:crypto";
 import { types } from "node:util";
 import evolutionRun from "@chainlesschain/session-core/evolution-run";
+import { storageFailureCauseSuffix } from "../storage-failure-cause-codes.js";
 
 import { EvolutionEvidenceArtifactAdapter } from "./evolution-evidence-artifact-adapter.js";
 import {
@@ -8,7 +9,14 @@ import {
   EVOLUTION_SKILL_DECISION_PROJECTION_RULESET_DIGEST,
   EVOLUTION_PROJECTION_RULESET_DIGEST,
 } from "./evolution-evidence-projector.js";
-import { EvolutionRunLedgerAdapter } from "./evolution-run-ledger-adapter.js";
+import {
+  EvolutionRunLedgerAdapter,
+  captureEvolutionRunAppendNext,
+} from "./evolution-run-ledger-adapter.js";
+import {
+  evidenceStorageLayoutForRun,
+  validateEvidenceStorageLayout,
+} from "./evolution-evidence-storage-layout.js";
 import { captureEvolutionRunWikiMaintenanceProducer } from "./evolution-run-wiki-maintenance-source.js";
 import { WIKI_MAINTENANCE_TRIGGER_KIND } from "./wiki-maintenance-trigger-ledger-adapter.js";
 import { captureEvolutionReleaseTrain } from "./evolution-release-train.js";
@@ -152,6 +160,7 @@ function ingressFailure(cause) {
     `Agent evolution ingress failed: ${cause?.message || String(cause)}`,
     { cause },
   );
+  error.message += storageFailureCauseSuffix(cause);
   error.code = AGENT_EVOLUTION_INGRESS_FAILED_CODE;
   return error;
 }
@@ -165,6 +174,7 @@ function guardIngress(operation) {
 export function createAgentEvolutionIngress({
   evidenceAdapter,
   runAdapter,
+  evidenceStorageLayout = "legacy",
   sourceEnvelopeAuthority,
   openCacheSourceRun = null,
   wikiMaintenanceProducer = null,
@@ -173,6 +183,7 @@ export function createAgentEvolutionIngress({
   now = () => new Date(),
   idGenerator = () => crypto.randomUUID(),
 } = {}) {
+  validateEvidenceStorageLayout(evidenceStorageLayout);
   if (!(evidenceAdapter instanceof EvolutionEvidenceArtifactAdapter)) {
     throw new TypeError("an EvolutionEvidenceArtifactAdapter is required");
   }
@@ -245,10 +256,24 @@ export function createAgentEvolutionIngress({
     return timestamp(iso, "ingress timestamp");
   };
 
+  const assertLoadedLayout = (loaded) => {
+    if (
+      evidenceStorageLayoutForRun(loaded.events, evidenceStorageLayout) !==
+      evidenceStorageLayout
+    )
+      throw new Error("Authenticated Run Evidence storage layout changed");
+  };
+
   const appendStarted = () => {
     const loaded = runAdapter.load();
+    assertLoadedLayout(loaded);
     if (loaded.events.length > 0) return loaded.projection;
     const occurredAt = currentTimestamp();
+    const data = {
+      occurredAt,
+      runtime: "agent",
+      ...(evidenceStorageLayout === "legacy" ? {} : { evidenceStorageLayout }),
+    };
     return runAdapter.appendEvent({
       schema: EVOLUTION_RUN_EVENT_SCHEMA,
       tenantId: descriptor.tenantId,
@@ -257,10 +282,10 @@ export function createAgentEvolutionIngress({
       sequence: 1,
       type: EVENT_TYPES.RUN_STARTED,
       subjectId: descriptor.runId,
-      payloadDigest: digest({ occurredAt, runtime: "agent" }),
+      payloadDigest: digest(data),
       artifactRef: null,
       keyRef: null,
-      data: { occurredAt, runtime: "agent" },
+      data,
     }).projection;
   };
 
@@ -347,13 +372,14 @@ export function createAgentEvolutionIngress({
             "Agent evidence persistence returned an unbound result",
           );
         }
-        const current = runAdapter.load();
-        const appended = runAdapter.appendEvent({
+        const appendNext = captureEvolutionRunAppendNext(runAdapter);
+        // Custom ports keep their established load-before-append phase.
+        const current = appendNext === null ? runAdapter.load() : null;
+        const event = {
           schema: EVOLUTION_RUN_EVENT_SCHEMA,
           tenantId: descriptor.tenantId,
           runId: descriptor.runId,
           eventId,
-          sequence: current.events.length + 1,
           type: EVENT_TYPES.RAW_EVENT_REFERENCED,
           subjectId: persisted.evidenceId,
           payloadDigest: persisted.manifest.digest,
@@ -371,7 +397,14 @@ export function createAgentEvolutionIngress({
                   candidateSetDigest: skillDecisionRequest.candidateSetDigest,
                 }),
           },
-        });
+        };
+        const appended =
+          appendNext !== null
+            ? appendNext(event)
+            : runAdapter.appendEvent({
+                ...event,
+                sequence: current.events.length + 1,
+              });
         if (kind === "model-input") {
           lastModelResponse = null;
           modelAdmissionPending = true;
@@ -677,6 +710,7 @@ export function createAgentEvolutionIngress({
         serialize(async () => {
           if (admissionFailure !== null) throw admissionFailure;
           const loaded = runAdapter.load();
+          assertLoadedLayout(loaded);
           let projection;
           if (loaded.projection?.status === "completed") {
             projection = loaded.projection;

package/src/lib/evolution/agent-evolution-runtime-composition.js

@@ -14,6 +14,11 @@ import {
   EvolutionEvidenceReader,
 } from "./evolution-evidence-projector.js";
 import { EvolutionArtifactPorts } from "./evolution-artifact-ports.js";
+import {
+  createTypedEvidenceArtifactPorts,
+  evidenceStorageLayoutForRun,
+  validateEvidenceStorageLayout,
+} from "./evolution-evidence-storage-layout.js";
 import {
   captureEvolutionLedgerFileBackend,
   createEvolutionLedgerFileBackend,
@@ -172,16 +177,14 @@ function layout(stateRootDir, tenantId, runId, initialize = true, fsImpl = fs) {
   const result = Object.freeze({
     rootDir,
     artifactDir: path.join(scope, "artifacts"),
+    evidenceArtifactDir: path.join(scope, "evidence-artifacts"),
     rawDir: path.join(scope, "raw"),
     ledgerRootDir: path.join(scope, "ledger-events"),
     ledgerAuthorityRootDir: path.join(scope, "ledger-authority"),
     witnessFilePath: path.join(scope, "witness", "checkpoint.json"),
   });
   if (initialize) {
-    ensureDirectoryTreeDurable(
-      fsImpl,
-      path.dirname(result.witnessFilePath),
-    );
+    ensureDirectoryTreeDurable(fsImpl, path.dirname(result.witnessFilePath));
   }
   return result;
 }
@@ -224,6 +227,7 @@ export function createAgentEvolutionRuntimeComposition({
   stateRootDir,
   authorities: authorityInput,
   evidenceTtlMs,
+  evidenceStorageLayout = "legacy",
   witnessId: witnessIdInput,
   clock: clockInput = Date.now,
   evidenceIdGenerator,
@@ -240,6 +244,7 @@ export function createAgentEvolutionRuntimeComposition({
   witnessMaximumBytes,
   ledgerV2: ledgerV2Input = null,
 } = {}) {
+  validateEvidenceStorageLayout(evidenceStorageLayout);
   const ledgerV2 =
     ledgerV2Input === null
       ? null
@@ -445,14 +450,6 @@ export function createAgentEvolutionRuntimeComposition({
     attestationVerifier: ports.attestationVerifier,
     now: () => new Date(clock()),
   });
-  const evidenceAdapter = new EvolutionEvidenceArtifactAdapter({
-    tenantId,
-    audience,
-    projector,
-    bundleVerifier,
-    artifactPorts,
-    ...(evidenceTtlMs === undefined ? {} : { ttlMs: evidenceTtlMs }),
-  });
   const ledgerArtifactResolver =
     artifactPorts.createEvolutionLedgerArtifactResolver({
       purpose: "evolution-ledger",
@@ -516,6 +513,46 @@ export function createAgentEvolutionRuntimeComposition({
     ledgerArtifactResolver,
     now: clock,
   });
+  const createEvidenceAdapter = (run, runStorage, mainPorts, emptyLayout) => {
+    const loaded = run.load();
+    const selectedLayout = evidenceStorageLayoutForRun(
+      loaded.events,
+      emptyLayout,
+    );
+    return {
+      selectedLayout,
+      adapter: new EvolutionEvidenceArtifactAdapter({
+        tenantId,
+        audience,
+        projector,
+        bundleVerifier,
+        ...(selectedLayout === "legacy"
+          ? { artifactPorts: mainPorts }
+          : {
+              typedArtifactPorts: createTypedEvidenceArtifactPorts({
+                rootDir: runStorage.evidenceArtifactDir,
+                tenantId,
+                audience,
+                artifactAuthority: {
+                  envelopeSigner: ports.artifactEnvelopeSigner,
+                  envelopeVerifier: ports.artifactEnvelopeVerifier,
+                  currentAuthorityResolver:
+                    ports.artifactCurrentAuthorityResolver,
+                },
+                now: clock,
+                existingRun: loaded.events.length > 0,
+              }),
+            }),
+        ...(evidenceTtlMs === undefined ? {} : { ttlMs: evidenceTtlMs }),
+      }),
+    };
+  };
+  const { adapter: evidenceAdapter, selectedLayout } = createEvidenceAdapter(
+    runAdapter,
+    storage,
+    artifactPorts,
+    evidenceStorageLayout,
+  );
   let wikiMaintenance = null;
   let wikiMaintainer = null;
   if (wikiConfiguration !== null) {
@@ -707,6 +744,7 @@ export function createAgentEvolutionRuntimeComposition({
   const evolutionIngress = createAgentEvolutionIngress({
     evidenceAdapter,
     runAdapter,
+    evidenceStorageLayout: selectedLayout,
     openCacheSourceRun: (input) => {
       const sourceRunId = identifier(input, "cache source runId");
       const sourceStorage = layout(
@@ -758,22 +796,22 @@ export function createAgentEvolutionRuntimeComposition({
         ...(lockTimeoutMs === undefined ? {} : { lockTimeoutMs }),
         ...(witnessMaximumBytes === undefined ? {} : { witnessMaximumBytes }),
       });
+      const sourceRunAdapter = new EvolutionRunLedgerAdapter({
+        descriptor: { ...runAdapter.descriptor, runId: sourceRunId },
+        artifactPorts: sourcePorts,
+        ledger: sourceBackend.ledger,
+        ledgerArtifactResolver: sourceResolver,
+        now: clock,
+      });
+      const sourceEvidence = createEvidenceAdapter(
+        sourceRunAdapter,
+        sourceStorage,
+        sourcePorts,
+        "legacy",
+      );
       return {
-        runAdapter: new EvolutionRunLedgerAdapter({
-          descriptor: { ...runAdapter.descriptor, runId: sourceRunId },
-          artifactPorts: sourcePorts,
-          ledger: sourceBackend.ledger,
-          ledgerArtifactResolver: sourceResolver,
-          now: clock,
-        }),
-        evidenceAdapter: new EvolutionEvidenceArtifactAdapter({
-          tenantId,
-          audience,
-          projector,
-          bundleVerifier,
-          artifactPorts: sourcePorts,
-          ...(evidenceTtlMs === undefined ? {} : { ttlMs: evidenceTtlMs }),
-        }),
+        runAdapter: sourceRunAdapter,
+        evidenceAdapter: sourceEvidence.adapter,
       };
     },
     sourceEnvelopeAuthority: ports.sourceEnvelope,

package/src/lib/evolution/evidence-json-text.js

@@ -21,6 +21,43 @@ export const AGENT_CODE_TEXT_RESULT_TOOLS = Object.freeze([
   "search_files",
 ]);
 
+export const AGENT_FENCED_CODE_LANGUAGES = Object.freeze([
+  "javascript",
+  "js",
+  "typescript",
+  "ts",
+  "jsx",
+  "tsx",
+  "python",
+  "py",
+]);
+
+function fencedCodeRanges(text, languages) {
+  const ranges = [];
+  const fences = /^ {0,3}(`{3,}|~{3,})([^\r\n]*)\r?$/gmu;
+  let opening = null;
+  for (const match of text.matchAll(fences)) {
+    const marker = match[1];
+    const info = match[2].trim().toLowerCase();
+    if (opening === null) {
+      opening = {
+        marker,
+        supported: languages.includes(info),
+        start: match.index + match[0].length,
+      };
+    } else if (
+      marker[0] === opening.marker[0] &&
+      marker.length >= opening.marker.length &&
+      info === ""
+    ) {
+      if (opening.supported) ranges.push([opening.start, match.index]);
+      opening = null;
+    }
+  }
+  // Unclosed/unknown fences never acquire code semantics.
+  return ranges;
+}
+
 /* eslint-disable no-control-regex -- Quoted literals exclude unescaped control bytes. */
 const JSON_STRING_PREFIX =
   /^"(?:[^"\\\x00-\x1f]|\\(?:["\\/bfnrt]|u[0-9a-fA-F]{4}))*"/u;
@@ -271,10 +308,35 @@ export function createEvidenceJsonTextBoundary(
     };
   };
 
-  return (
+  return function scanText(
     text,
-    { layer = 0, objectOnly = false, codeText = false, visit, plain, equal },
-  ) => {
+    {
+      layer = 0,
+      objectOnly = false,
+      codeText = false,
+      fencedCodeLanguages = null,
+      visit,
+      plain,
+      equal,
+    },
+  ) {
+    if (!objectOnly && !codeText && fencedCodeLanguages) {
+      const ranges = fencedCodeRanges(text, fencedCodeLanguages);
+      if (ranges.length > 0) {
+        let output = "";
+        let offset = 0;
+        const options = { layer, visit, plain, equal };
+        for (const [start, end] of ranges) {
+          output += scanText(text.slice(offset, start), options);
+          output += scanText(text.slice(start, end), {
+            ...options,
+            codeText: true,
+          });
+          offset = end;
+        }
+        return output + scanText(text.slice(offset), options);
+      }
+    }
     bytes += Buffer.byteLength(text, "utf8");
     if (bytes > policy.maxDecodedBytes)
       reject("Agent model JSON text exceeds its budget");