chainlesschain 0.166.115 → 0.166.118

package/src/lib/artifact-store.js

@@ -28,6 +28,11 @@ import { types as utilTypes } from "node:util";
 import { withFileLock } from "./with-file-lock.js";
 import { readBoundedArtifactFile } from "./bounded-artifact-file-read.js";
 
+// Per-call, internal formatting opt-in. Public options/schema strings cannot
+// select it, and it grants no read, signature or storage authority.
+const TYPED_INDEX_ENCODERS = new WeakMap();
+const DEFAULT_FUNCTION_BIND = Function.prototype.bind;
+
 export const ARTIFACT_KINDS = Object.freeze([
   "report",
   "patch",
@@ -305,18 +310,19 @@ export class ArtifactStore {
    * caller still owns the semantic validation of the persisted lineage and
    * must verifyIntegrity() before treating the returned entry as readable.
    */
-  publishDataOnce({
-    data,
-    fileName = "artifact.bin",
-    title,
-    kind,
-    mime,
-    sessionId = null,
-    ttlDays,
-    immutable = false,
-    recordDigest,
-    lineage,
-  } = {}) {
+  publishDataOnce(options = {}) {
+    const {
+      data,
+      fileName = "artifact.bin",
+      title,
+      kind,
+      mime,
+      sessionId = null,
+      ttlDays,
+      immutable = false,
+      recordDigest,
+      lineage,
+    } = options;
     if (typeof data !== "string" && !Buffer.isBuffer(data)) {
       throw new Error("publishDataOnce requires string or Buffer data");
     }
@@ -337,7 +343,7 @@ export class ArtifactStore {
       if (matches.length === 1) {
         return Object.freeze({ entry: matches[0], published: false });
       }
-      const entry = this._publishDataUnlocked(Buffer.from(data), {
+      const metadata = {
         fileName,
         title,
         kind,
@@ -348,14 +354,22 @@ export class ArtifactStore {
         recordDigest: stableDigest,
         lineage: normalizedLineage,
         sourcePath: null,
-      });
+      };
+      const encoder = TYPED_INDEX_ENCODERS.get(options);
+      if (encoder && hasDefaultTypedIndexWriter(this))
+        TYPED_INDEX_ENCODERS.set(metadata, encoder);
+      let entry;
+      try {
+        entry = this._publishDataUnlocked(Buffer.from(data), metadata);
+      } finally {
+        TYPED_INDEX_ENCODERS.delete(metadata);
+      }
       return Object.freeze({ entry, published: true });
     });
   }
 
-  _publishDataUnlocked(
-    body,
-    {
+  _publishDataUnlocked(body, metadata) {
+    const {
       fileName,
       title,
       kind,
@@ -366,8 +380,7 @@ export class ArtifactStore {
       recordDigest = null,
       lineage = null,
       sourcePath = null,
-    },
-  ) {
+    } = metadata;
     if (body.length > MAX_ARTIFACT_BYTES) {
       throw new Error(
         `artifact too large (${body.length} bytes > ${MAX_ARTIFACT_BYTES}); artifacts are deliverables, not build output`,
@@ -428,7 +441,14 @@ export class ArtifactStore {
       if (!indexStat.isFile() || Number(indexStat.nlink) !== 1) {
         throw new Error("artifact index identity is invalid");
       }
-      fs.writeFileSync(descriptor, `${JSON.stringify(entry)}\n`, "utf-8");
+      const encoder = TYPED_INDEX_ENCODERS.get(metadata);
+      const encoded =
+        encoder && hasDefaultTypedIndexWriter(this) ? encoder(entry) : null;
+      fs.writeFileSync(
+        descriptor,
+        `${encoded ?? JSON.stringify(entry)}\n`,
+        "utf-8",
+      );
       fs.fsyncSync(descriptor);
     } catch (error) {
       try {
@@ -645,3 +665,92 @@ export class ArtifactStore {
     }
   }
 }
+
+// Capture repository method identities before any importer can replace the
+// prototype. These references grant no storage or read authority by themselves.
+export const ARTIFACT_STORE_DEFAULT_READ_METHODS = Object.freeze({
+  list: ArtifactStore.prototype.list,
+  get: ArtifactStore.prototype.get,
+  storedPath: ArtifactStore.prototype.storedPath,
+  verifyIntegrity: ArtifactStore.prototype.verifyIntegrity,
+  filesDir: ArtifactStore.prototype._filesDir,
+});
+
+export const ARTIFACT_STORE_DEFAULT_PUBLISH_DATA_ONCE =
+  ArtifactStore.prototype.publishDataOnce;
+
+const TYPED_INDEX_DEFAULT_METHODS = Object.freeze(
+  Object.fromEntries(
+    [
+      "publishDataOnce",
+      "_publishDataUnlocked",
+      "_readEntries",
+      "_withIndexLock",
+      "_ensureDirs",
+      "_indexFile",
+      "_filesDir",
+      "_deleteStoredFile",
+      "list",
+      "get",
+      "storedPath",
+      "verifyIntegrity",
+    ].map((name) => [name, ArtifactStore.prototype[name]]),
+  ),
+);
+
+function hasDefaultTypedIndexWriter(store) {
+  if (
+    utilTypes.isProxy(store) ||
+    Object.getPrototypeOf(store) !== ArtifactStore.prototype ||
+    Object.getOwnPropertyDescriptor(store, "_indexLock")?.value !==
+      withFileLock ||
+    Object.getOwnPropertyDescriptor(Object.prototype, "toJSON") !== undefined
+  )
+    return false;
+  for (const [name, expected] of Object.entries(TYPED_INDEX_DEFAULT_METHODS)) {
+    const descriptor =
+      Object.getOwnPropertyDescriptor(store, name) ||
+      Object.getOwnPropertyDescriptor(ArtifactStore.prototype, name);
+    if (
+      !descriptor ||
+      !("value" in descriptor) ||
+      descriptor.value !== expected
+    )
+      return false;
+  }
+  return true;
+}
+
+/** Internal ports adapter; callers still invoke their captured publication hook. */
+export function captureArtifactStoreTypedIndexPublisher(
+  store,
+  publish,
+  encoder,
+) {
+  // A custom bind is itself a publication hook, even on the default method.
+  // Inspect descriptors without invoking getters; the legacy bind call below
+  // still runs exactly once at the original capture point.
+  const defaultBind =
+    publish === ARTIFACT_STORE_DEFAULT_PUBLISH_DATA_ONCE &&
+    Object.getPrototypeOf(publish) === Function.prototype &&
+    Object.getOwnPropertyDescriptor(publish, "bind") === undefined &&
+    Object.getOwnPropertyDescriptor(Function.prototype, "bind")?.value ===
+      DEFAULT_FUNCTION_BIND;
+  // Preserve the original capture timing and custom bind semantics. A valid
+  // function can shadow its public .call property; invocation must not use it.
+  const boundPublish = Object.freeze(publish.bind(store));
+  return Object.freeze((options) => {
+    if (
+      defaultBind &&
+      publish === ARTIFACT_STORE_DEFAULT_PUBLISH_DATA_ONCE &&
+      hasDefaultTypedIndexWriter(store)
+    ) {
+      TYPED_INDEX_ENCODERS.set(options, encoder);
+    }
+    try {
+      return boundPublish(options);
+    } finally {
+      TYPED_INDEX_ENCODERS.delete(options);
+    }
+  });
+}

package/src/lib/bounded-artifact-file-read.js

@@ -2,7 +2,10 @@
 import fs from "node:fs";
 import path from "node:path";
 import { readBoundedDescriptor } from "./evolution/bounded-descriptor-read.js";
-import { withEvolutionFileIdentity } from "./evolution/evolution-file-identity.js";
+import {
+  withEvolutionDirectoryFileIdentity,
+  withEvolutionFileIdentity,
+} from "./evolution/evolution-file-identity.js";
 import { artifactPhysicalIdentity } from "./evolution/evolution-artifact-identity.js";
 
 export const ARTIFACT_BOUNDED_READ_FAILED_CODE =
@@ -67,10 +70,10 @@ function sameStat(left, right) {
   );
 }
 
-/** Read one stable window only. Callers retain hash/signature and root checks. */
-export function readBoundedArtifactFile(target, expectedSize, maximumBytes) {
+function assertReadArguments(args) {
+  const [target, expectedSize, maximumBytes] = args;
   if (
-    arguments.length !== 3 ||
+    args.length !== 3 ||
     typeof target !== "string" ||
     !path.isAbsolute(target) ||
     path.resolve(target) !== target ||
@@ -83,49 +86,128 @@ export function readBoundedArtifactFile(target, expectedSize, maximumBytes) {
     throw new TypeError(
       "bounded artifact read requires a canonical path and finite size bounds",
     );
+}
+
+function readWithIdentity(target, expectedSize, maximumBytes, samePathHandle) {
+  const before = fs.lstatSync(target, { bigint: true });
+  safeStat(before, expectedSize, maximumBytes, "artifact path");
+  if (canonical(fs.realpathSync.native(target)) !== canonical(target))
+    fail("artifact path is not physically canonical");
+  const fd = fs.openSync(
+    target,
+    fs.constants.O_RDONLY |
+      (fs.constants.O_NOFOLLOW || 0) |
+      (fs.constants.O_NONBLOCK || 0),
+  );
   try {
-    return withEvolutionFileIdentity(DIRECTORY_FS, target, (samePathHandle) => {
-      const before = fs.lstatSync(target, { bigint: true });
-      safeStat(before, expectedSize, maximumBytes, "artifact path");
-      if (canonical(fs.realpathSync.native(target)) !== canonical(target))
-        fail("artifact path is not physically canonical");
-      const fd = fs.openSync(
-        target,
-        fs.constants.O_RDONLY |
-          (fs.constants.O_NOFOLLOW || 0) |
-          (fs.constants.O_NONBLOCK || 0),
-      );
-      try {
-        const opened = fs.fstatSync(fd, { bigint: true });
-        safeStat(opened, expectedSize, maximumBytes, "artifact descriptor");
-        if (!samePathHandle(before, opened) || !sameMetadata(before, opened))
-          fail("artifact path and descriptor identities differ");
-        const bytes = readBoundedDescriptor(
-          fs,
-          fd,
-          Number(opened.size),
-          maximumBytes,
-        );
-        const closed = fs.fstatSync(fd, { bigint: true });
-        const after = fs.lstatSync(target, { bigint: true });
-        safeStat(closed, expectedSize, maximumBytes, "artifact descriptor");
-        safeStat(after, expectedSize, maximumBytes, "artifact path");
-        if (!sameStat(opened, closed))
-          fail("artifact descriptor changed during bounded read");
-        if (
-          !sameStat(before, after) ||
-          !samePathHandle(after, closed) ||
-          !sameMetadata(after, closed) ||
-          canonical(fs.realpathSync.native(target)) !== canonical(target)
-        )
-          fail("artifact pathname changed during bounded read");
-        return bytes;
-      } finally {
-        fs.closeSync(fd);
-      }
-    });
+    const opened = fs.fstatSync(fd, { bigint: true });
+    safeStat(opened, expectedSize, maximumBytes, "artifact descriptor");
+    if (!samePathHandle(before, opened) || !sameMetadata(before, opened))
+      fail("artifact path and descriptor identities differ");
+    const bytes = readBoundedDescriptor(
+      fs,
+      fd,
+      Number(opened.size),
+      maximumBytes,
+    );
+    const closed = fs.fstatSync(fd, { bigint: true });
+    const after = fs.lstatSync(target, { bigint: true });
+    safeStat(closed, expectedSize, maximumBytes, "artifact descriptor");
+    safeStat(after, expectedSize, maximumBytes, "artifact path");
+    if (!sameStat(opened, closed))
+      fail("artifact descriptor changed during bounded read");
+    if (
+      !sameStat(before, after) ||
+      !samePathHandle(after, closed) ||
+      !sameMetadata(after, closed) ||
+      canonical(fs.realpathSync.native(target)) !== canonical(target)
+    )
+      fail("artifact pathname changed during bounded read");
+    return bytes;
+  } finally {
+    fs.closeSync(fd);
+  }
+}
+
+function wrapReadFailure(cause) {
+  if (cause?.code === ARTIFACT_BOUNDED_READ_FAILED_CODE) throw cause;
+  fail("artifact bounded descriptor read failed", cause);
+}
+
+/** Read one stable window only. Callers retain hash/signature and root checks. */
+export function readBoundedArtifactFile(target, expectedSize, maximumBytes) {
+  assertReadArguments(arguments);
+  try {
+    return withEvolutionFileIdentity(DIRECTORY_FS, target, (samePathHandle) =>
+      readWithIdentity(target, expectedSize, maximumBytes, samePathHandle),
+    );
+  } catch (cause) {
+    wrapReadFailure(cause);
+  }
+}
+
+/**
+ * A synchronous batch shares one authenticated parent handle, never file
+ * contents or authority. Each child keeps the complete independent read
+ * window above. The parent is reattested before the batch can return.
+ */
+export function withBoundedArtifactDirectoryRead(directory, callback) {
+  if (
+    arguments.length !== 2 ||
+    typeof directory !== "string" ||
+    !path.isAbsolute(directory) ||
+    path.resolve(directory) !== directory ||
+    typeof callback !== "function" ||
+    callback.constructor?.name === "AsyncFunction"
+  )
+    throw new TypeError(
+      "bounded artifact directory read requires a canonical directory and synchronous callback",
+    );
+  let callbackFailure = null;
+  try {
+    return withEvolutionDirectoryFileIdentity(
+      DIRECTORY_FS,
+      directory,
+      (samePathHandle, _sameStableHandle, { parentPath }) => {
+        if (canonical(parentPath) !== canonical(directory))
+          fail("artifact batch parent is not physically canonical");
+        let active = true;
+        const read = Object.freeze((...args) => {
+          if (!active) fail("artifact batch read window is closed");
+          assertReadArguments(args);
+          const [target, expectedSize, maximumBytes] = args;
+          if (canonical(path.dirname(target)) !== canonical(parentPath))
+            fail(
+              "artifact batch read must name a direct child of its captured parent",
+            );
+          try {
+            return readWithIdentity(
+              target,
+              expectedSize,
+              maximumBytes,
+              samePathHandle,
+            );
+          } catch (cause) {
+            wrapReadFailure(cause);
+          }
+        });
+        try {
+          const result = callback(read);
+          if (result && typeof result.then === "function")
+            throw new TypeError(
+              "artifact batch callback must not return a promise",
+            );
+          return result;
+        } catch (cause) {
+          callbackFailure = cause;
+          throw cause;
+        } finally {
+          active = false;
+        }
+      },
+    );
   } catch (cause) {
-    if (cause?.code === ARTIFACT_BOUNDED_READ_FAILED_CODE) throw cause;
-    fail("artifact bounded descriptor read failed", cause);
+    if (cause === callbackFailure) throw cause;
+    wrapReadFailure(cause);
   }
 }

package/src/lib/context-memory-kernel/context-image-transport.js

@@ -0,0 +1,248 @@
+import { createHash, randomBytes } from "node:crypto";
+import { normalizeContextItem } from "@chainlesschain/context-memory-kernel";
+import { AGENT_MODEL_IMAGE_LIMITS } from "../evolution/agent-model-projection.js";
+import {
+  IMAGE_TOKEN_RESERVE,
+  LOW_DETAIL_IMAGE_TOKEN_RESERVE,
+  inspectCanonicalImageDataUrl,
+} from "../image-token-estimate.js";
+
+export const CONTEXT_IMAGE_TRANSPORT_PREFIX = "cc-context-image://v1/";
+const CONTEXT_IMAGE_TRANSPORT_MARKER = "cc-context-image";
+
+const PLACEHOLDER_URL =
+  /^cc-context-image:\/\/v1\/([a-f0-9]{32})\/(sha256-[a-f0-9]{64})$/u;
+
+function sha256(value) {
+  return `sha256-${createHash("sha256").update(value, "ascii").digest("hex")}`;
+}
+
+function supportedImageUrlBlock(message, block) {
+  if (
+    message?.role !== "user" ||
+    !Array.isArray(message.content) ||
+    !block ||
+    typeof block !== "object" ||
+    Array.isArray(block) ||
+    Object.keys(block).sort().join() !== "image_url,type" ||
+    block.type !== "image_url" ||
+    !block.image_url ||
+    typeof block.image_url !== "object" ||
+    Array.isArray(block.image_url) ||
+    Object.keys(block.image_url).some((key) => !["url", "detail"].includes(key))
+  )
+    return null;
+
+  const { url, detail } = block.image_url;
+  if (
+    typeof url !== "string" ||
+    (detail !== undefined && !["auto", "low", "high"].includes(detail))
+  )
+    return null;
+  if (
+    url.length >
+    Math.ceil(AGENT_MODEL_IMAGE_LIMITS.maxImageBytes / 3) * 4 + 128
+  )
+    throw codedTransportError(
+      "Image transport exceeds its per-image byte budget",
+    );
+  const image = inspectCanonicalImageDataUrl(url);
+  return image ? { url, detail, bytes: image.decodedBytes } : null;
+}
+
+function codedTransportError(message) {
+  const error = new Error(message);
+  error.code = "CC_CONTEXT_IMAGE_TRANSPORT_INVALID";
+  return error;
+}
+
+/** Replace supported user image data URLs with request-local digest commitments. */
+export function captureContextImageTransport(messagesInput) {
+  const messages = Array.isArray(messagesInput) ? messagesInput : [];
+  const captures = new Map();
+  let requestImageBytes = 0;
+  let changed = false;
+  const transportedMessages = messages.map((message) => {
+    if (message?.role !== "user" || !Array.isArray(message.content)) {
+      return message;
+    }
+
+    let contentChanged = false;
+    let messageImageBytes = 0;
+    let messageImageCount = 0;
+    const content = message.content.map((block) => {
+      const image = supportedImageUrlBlock(message, block);
+      if (!image) return block;
+
+      messageImageCount++;
+      messageImageBytes += image.bytes;
+      requestImageBytes += image.bytes;
+      if (
+        image.bytes > AGENT_MODEL_IMAGE_LIMITS.maxImageBytes ||
+        messageImageCount > AGENT_MODEL_IMAGE_LIMITS.maxMessageImages ||
+        messageImageBytes > AGENT_MODEL_IMAGE_LIMITS.maxMessageImageBytes ||
+        requestImageBytes > AGENT_MODEL_IMAGE_LIMITS.maxRequestImageBytes
+      )
+        throw codedTransportError(
+          "Images exceed the bounded message or request image budget",
+        );
+
+      const digest = sha256(image.url);
+      const nonce = randomBytes(16).toString("hex");
+      const placeholder = `${CONTEXT_IMAGE_TRANSPORT_PREFIX}${nonce}/${digest}`;
+      captures.set(placeholder, {
+        url: image.url,
+        digest,
+        detail: image.detail,
+        hasDetail: Object.hasOwn(block.image_url, "detail"),
+        reserveTokens:
+          image.detail === "low"
+            ? LOW_DETAIL_IMAGE_TOKEN_RESERVE
+            : IMAGE_TOKEN_RESERVE,
+      });
+      contentChanged = true;
+      changed = true;
+      return {
+        ...block,
+        image_url: { ...block.image_url, url: placeholder },
+      };
+    });
+    return contentChanged ? { ...message, content } : message;
+  });
+
+  return {
+    messages: changed ? transportedMessages : messages,
+    captures,
+  };
+}
+
+/** Preserve image token reserves while removing transport placeholders from item digests. */
+export function adjustContextImageTokenEstimates(itemsInput, captures) {
+  const items = Array.isArray(itemsInput) ? itemsInput : [];
+  if (!(captures instanceof Map) || captures.size === 0) return items;
+
+  return items.map((item) => {
+    if (typeof item?.content !== "string") return item;
+    let placeholderBytes = 0;
+    let reserveTokens = 0;
+    for (const [placeholder, capture] of captures) {
+      const first = item.content.indexOf(placeholder);
+      if (first < 0) continue;
+      if (item.content.indexOf(placeholder, first + placeholder.length) >= 0) {
+        throw codedTransportError(
+          "A context image transport placeholder was duplicated during planning",
+        );
+      }
+      placeholderBytes += Buffer.byteLength(placeholder, "utf8");
+      reserveTokens += capture.reserveTokens;
+    }
+    if (placeholderBytes === 0) return item;
+
+    const { digest: _previousDigest, ...withoutDigest } = item;
+    return normalizeContextItem({
+      ...withoutDigest,
+      tokenEstimate: Math.max(
+        1,
+        Math.ceil(
+          (Buffer.byteLength(item.content, "utf8") - placeholderBytes) / 4,
+        ) + reserveTokens,
+      ),
+    });
+  });
+}
+
+function containsTransportImageUrl(messages) {
+  return messages.some(
+    (message) =>
+      Array.isArray(message?.content) &&
+      message.content.some(
+        (block) =>
+          block?.type === "image_url" &&
+          typeof block.image_url?.url === "string" &&
+          block.image_url.url.includes(CONTEXT_IMAGE_TRANSPORT_MARKER),
+      ),
+  );
+}
+
+function containsCapturedPlaceholder(value, captures, seen = new Set()) {
+  if (typeof value === "string") {
+    for (const match of value.matchAll(
+      /cc-context-image:\/\/v1\/[a-f0-9]{32}\/sha256-[a-f0-9]{64}/gu,
+    )) {
+      if (captures.has(match[0])) return true;
+    }
+    return false;
+  }
+  if (!value || typeof value !== "object" || seen.has(value)) return false;
+  seen.add(value);
+  return Object.values(value).some((entry) =>
+    containsCapturedPlaceholder(entry, captures, seen),
+  );
+}
+
+/** Restore captured image data URLs and reject every unrecognized or altered placeholder. */
+export function restoreContextImageTransport(messagesInput, captures) {
+  const messages = Array.isArray(messagesInput) ? messagesInput : [];
+  const captured = captures instanceof Map ? captures : new Map();
+  const seenPlaceholders = new Set();
+  const restored = messages.map((message) => {
+    if (message?.role !== "user" || !Array.isArray(message.content)) {
+      return message;
+    }
+    let contentChanged = false;
+    const content = message.content.map((block) => {
+      if (
+        !block ||
+        typeof block !== "object" ||
+        Array.isArray(block) ||
+        block.type !== "image_url" ||
+        !block.image_url ||
+        typeof block.image_url !== "object" ||
+        Array.isArray(block.image_url) ||
+        typeof block.image_url.url !== "string" ||
+        !block.image_url.url.includes(CONTEXT_IMAGE_TRANSPORT_MARKER)
+      )
+        return block;
+
+      const placeholder = block.image_url.url;
+      const match = PLACEHOLDER_URL.exec(placeholder);
+      const capture = captured.get(placeholder);
+      if (
+        Object.keys(block).sort().join() !== "image_url,type" ||
+        Object.keys(block.image_url).some(
+          (key) => !["url", "detail"].includes(key),
+        ) ||
+        (block.image_url.detail !== undefined &&
+          !["auto", "low", "high"].includes(block.image_url.detail)) ||
+        !match ||
+        !capture ||
+        capture.hasDetail !== Object.hasOwn(block.image_url, "detail") ||
+        capture.detail !== block.image_url.detail ||
+        match[2] !== capture.digest ||
+        sha256(capture.url) !== capture.digest ||
+        seenPlaceholders.has(placeholder)
+      ) {
+        throw codedTransportError(
+          "A context image transport placeholder is unknown, altered, or duplicated",
+        );
+      }
+      seenPlaceholders.add(placeholder);
+      contentChanged = true;
+      return {
+        ...block,
+        image_url: { ...block.image_url, url: capture.url },
+      };
+    });
+    return contentChanged ? { ...message, content } : message;
+  });
+
+  if (
+    containsTransportImageUrl(restored) ||
+    containsCapturedPlaceholder(restored, captured)
+  ) {
+    throw codedTransportError(
+      "An unrecognized context image transport placeholder reached provider context",
+    );
+  }
+  return restored;
+}

package/src/lib/context-memory-kernel/jsonl-session-context-port.js

@@ -120,6 +120,7 @@ export class JsonlSessionContextPort {
           items: messagesToContextItems(authority.readMessages(), {
             sessionId,
             allowedSinks: this.allowedSinks,
+            allowExternalization: true,
           }),
         }),
       };
@@ -215,7 +216,9 @@ export class JsonlSessionContextPort {
 
   async appendCompaction(event, expectedHead) {
     this._assertSession(event.sessionId);
-    const messages = contextItemsToMessages(event.outputItems);
+    const messages = contextItemsToMessages(event.outputItems, {
+      sessionId: this.sessionId,
+    });
     const stats = compactStats(event, messages);
     const appendHead = this._appendHead(event.operationId, expectedHead);
     try {