chainlesschain 0.166.100 → 0.166.102

package/src/lib/context-memory-kernel/live-compaction.js

@@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto";
 import {
   canonicalDigest,
   InMemorySessionContextPort,
+  getCompactionPlanRejection,
 } from "@chainlesschain/context-memory-kernel";
 import { sessionExists } from "../../harness/jsonl-session-store.js";
 import {
@@ -11,6 +12,17 @@ import {
 } from "./message-adapter.js";
 import { JsonlSessionContextPort } from "./jsonl-session-context-port.js";
 import { createCliContextMemoryRuntime } from "./runtime.js";
+import { projectRuntimeTokenUsage } from "../runtime-usage-ledger.js";
+
+const rejectedCompactionUsage = new WeakMap();
+
+// Only the owned canonical adapter can associate an error with its completed
+// summarizer. A caller-supplied error property is never a settlement authority.
+export function getRejectedCompactionUsage(error) {
+  return error && typeof error === "object"
+    ? rejectedCompactionUsage.get(error) || null
+    : null;
+}
 
 function estimatedTokens(messages) {
   return Math.max(
@@ -20,7 +32,9 @@ function estimatedTokens(messages) {
         typeof message?.content === "string"
           ? message.content
           : JSON.stringify(message?.content ?? "");
-      return total + Math.max(1, Math.ceil(Buffer.byteLength(content, "utf8") / 4));
+      return (
+        total + Math.max(1, Math.ceil(Buffer.byteLength(content, "utf8") / 4))
+      );
     }, 0),
   );
 }
@@ -125,13 +139,22 @@ class CallbackSessionContextPort extends InMemorySessionContextPort {
 }
 
 function kernelBudget(compressor) {
-  const inputBudget = Math.max(256, Math.trunc(Number(compressor?.maxTokens)) || 16_384);
-  const reservedOutputTokens = Math.max(1, Math.min(2048, Math.floor(inputBudget / 10)));
+  const inputBudget = Math.max(
+    256,
+    Math.trunc(Number(compressor?.maxTokens)) || 16_384,
+  );
+  const reservedOutputTokens = Math.max(
+    1,
+    Math.min(2048, Math.floor(inputBudget / 10)),
+  );
   return {
     modelWindowTokens: inputBudget + reservedOutputTokens,
     reservedOutputTokens,
     safetyMarginTokens: 0,
-    recoveryReserveTokens: Math.min(256, Math.max(1, Math.floor(inputBudget * 0.05))),
+    recoveryReserveTokens: Math.min(
+      256,
+      Math.max(1, Math.floor(inputBudget * 0.05)),
+    ),
   };
 }
 
@@ -175,7 +198,10 @@ function usageReceipt(stats, provider, model) {
  * summarizer for the Kernel-selected dropped partition; it cannot commit or
  * choose the authoritative output on its own.
  */
-export async function compactLiveMessagesCanonical(messagesInput, options = {}) {
+export async function compactLiveMessagesCanonical(
+  messagesInput,
+  options = {},
+) {
   const messages = Array.isArray(messagesInput) ? [...messagesInput] : [];
   const sessionId = String(options.sessionId || `live-${randomUUID()}`);
   const operationId = String(options.operationId || `compact-${randomUUID()}`);
@@ -185,10 +211,13 @@ export async function compactLiveMessagesCanonical(messagesInput, options = {})
     new Date(Number((options.clock || Date.now)())).toISOString();
   const compressor = options.compressor;
   if (!compressor || typeof compressor.compress !== "function") {
-    throw new TypeError("canonical live compaction requires a compatibility summarizer");
+    throw new TypeError(
+      "canonical live compaction requires a compatibility summarizer",
+    );
   }
 
   let summaryStats = null;
+  let summaryReceipt = null;
   let runtime;
   let verifiedAuthoritySnapshot = null;
   const persistentSession =
@@ -227,9 +256,7 @@ export async function compactLiveMessagesCanonical(messagesInput, options = {})
       memoryPort: bootstrapRuntime.memoryPort,
       sessionPort,
     });
-  } else if (
-    persistentSession
-  ) {
+  } else if (persistentSession) {
     runtime = createCliContextMemoryRuntime({
       sessionId,
       scopeKey: `cli:session:${sessionId}`,
@@ -265,13 +292,13 @@ export async function compactLiveMessagesCanonical(messagesInput, options = {})
     });
   }
   if (!runtime.decision.canonical) {
-    const error = new Error("canonical live compaction is not authoritative for this cohort");
+    const error = new Error(
+      "canonical live compaction is not authoritative for this cohort",
+    );
     error.code = "legacy_writer_fenced";
     throw error;
   }
-  if (
-    persistentSession
-  ) {
+  if (persistentSession) {
     const authority =
       verifiedAuthoritySnapshot ||
       (await runtime.sessionPort.readSnapshot(sessionId));
@@ -296,52 +323,102 @@ export async function compactLiveMessagesCanonical(messagesInput, options = {})
   const provider = options.provider || "provider.local";
   const model = options.model || "default";
   const memoryRevision = await runtime.memoryPort.getRevision();
-  const receipt = await runtime.kernel.compactContext({
-    operationId,
-    sessionId,
-    ...kernelBudget(compressor),
-    sink: provider,
-    scopeAdmissions: [{ scope: "session", scopeId: sessionId }],
-    policyVersion: "cli-context-policy-v1",
-    modelProfile: model,
-    memoryRevision,
-    allowFallback: false,
-    now: operationNow,
-    metadata: {
-      inputMessageCount: messages.length,
-      originalTokens: estimatedTokens(messages),
-      trigger,
-      provider,
-      model,
-    },
-    summarizer: async (droppedItems, context) => {
-      const droppedMessages = contextItemsToMessages(droppedItems);
-      const result = await compressor.compress(droppedMessages, {
-        preserveToolPairs: true,
-        ...(options.compressOptions || {}),
-      });
-      summaryStats = result.stats || {};
-      const reduced =
-        Number(summaryStats.saved || 0) > 0 ||
-        result.messages.length < droppedMessages.length;
-      const item = reduced
-        ? createSummaryContextItem({
-            messages: result.messages,
-            parents: droppedItems,
-            operationId: context.operationId,
-            now: operationNow,
-          })
-        : null;
-      return {
-        items: item ? [item] : [],
-        usageReceipt: usageReceipt(summaryStats, provider, model),
-        degraded: summaryStats.degraded === true,
-        ...(summaryStats.degradedReason
-          ? { degradedReason: String(summaryStats.degradedReason).slice(0, 160) }
-          : {}),
-      };
-    },
-  });
+  let receipt;
+  try {
+    receipt = await runtime.kernel.compactContext({
+      operationId,
+      sessionId,
+      ...kernelBudget(compressor),
+      sink: provider,
+      scopeAdmissions: [{ scope: "session", scopeId: sessionId }],
+      policyVersion: "cli-context-policy-v1",
+      modelProfile: model,
+      memoryRevision,
+      allowFallback: false,
+      now: operationNow,
+      metadata: {
+        inputMessageCount: messages.length,
+        originalTokens: estimatedTokens(messages),
+        trigger,
+        provider,
+        model,
+      },
+      summarizer: async (droppedItems, context) => {
+        const droppedMessages = contextItemsToMessages(droppedItems);
+        const result = await compressor.compress(droppedMessages, {
+          preserveToolPairs: true,
+          ...(options.compressOptions || {}),
+        });
+        summaryStats = result.stats || {};
+        summaryReceipt = usageReceipt(summaryStats, provider, model);
+        const reduced =
+          Number(summaryStats.saved || 0) > 0 ||
+          result.messages.length < droppedMessages.length;
+        const item = reduced
+          ? createSummaryContextItem({
+              messages: result.messages,
+              parents: droppedItems,
+              operationId: context.operationId,
+              now: operationNow,
+            })
+          : null;
+        return {
+          items: item ? [item] : [],
+          usageReceipt: summaryReceipt,
+          degraded: summaryStats.degraded === true,
+          ...(summaryStats.degradedReason
+            ? {
+                degradedReason: String(summaryStats.degradedReason).slice(
+                  0,
+                  160,
+                ),
+              }
+            : {}),
+        };
+      },
+    });
+  } catch (error) {
+    const rejection = getCompactionPlanRejection(error);
+    if (
+      rejection?.operationId === operationId &&
+      rejection.sessionId === sessionId &&
+      summaryReceipt !== null &&
+      rejection.usageReceipt &&
+      canonicalDigest(rejection.usageReceipt) ===
+        canonicalDigest(summaryReceipt) &&
+      summaryStats?.summaryUsageUnknown !== true
+    ) {
+      try {
+        const usage = projectRuntimeTokenUsage({
+          ...(summaryStats.summaryCallId
+            ? { callId: summaryStats.summaryCallId }
+            : {}),
+          provider: summaryStats.summaryProvider || provider,
+          model: summaryStats.summaryModel || model,
+          source: "semantic-compaction",
+          usage: {
+            input_tokens: summaryStats.summaryUsage?.inputTokens,
+            output_tokens: summaryStats.summaryUsage?.outputTokens,
+            cache_read_input_tokens:
+              summaryStats.summaryUsage?.cacheReadTokens ?? 0,
+            cache_creation_input_tokens:
+              summaryStats.summaryUsage?.cacheCreationTokens ?? 0,
+          },
+        });
+        rejectedCompactionUsage.set(
+          error,
+          Object.freeze({
+            ...usage,
+            usage: Object.freeze(usage.usage),
+            receipt: Object.freeze({ ...summaryReceipt }),
+          }),
+        );
+      } catch {
+        // Missing or invalid usage retains the caller's unknown outcome path.
+      }
+    }
+    throw error;
+  }
   if (!["committed", "degraded"].includes(receipt.status)) {
     return {
       messages,
@@ -361,12 +438,16 @@ export async function compactLiveMessagesCanonical(messagesInput, options = {})
     messages: outputMessages,
     stats: {
       ...(summaryStats || {}),
-      strategy: receipt.status === "degraded" ? "deterministic-fallback" : "canonical",
+      strategy:
+        receipt.status === "degraded" ? "deterministic-fallback" : "canonical",
       originalMessages: messages.length,
       compressedMessages: outputMessages.length,
       originalTokens: estimatedTokens(messages),
       compressedTokens: estimatedTokens(outputMessages),
-      saved: Math.max(0, estimatedTokens(messages) - estimatedTokens(outputMessages)),
+      saved: Math.max(
+        0,
+        estimatedTokens(messages) - estimatedTokens(outputMessages),
+      ),
       canonicalAlreadySettled: true,
       canonicalReceipt: receipt,
     },

package/src/lib/context-memory-kernel/memory-query-index.js

@@ -2,6 +2,9 @@ import { createHash } from "node:crypto";
 
 export const MEMORY_QUERY_INDEX_SCHEMA = "chainlesschain.memory-query-index/v1";
 const CURSOR_SCHEMA = "chainlesschain.memory-list-cursor/v1";
+// Preserve the existing English collation while avoiding constructing locale
+// options for every comparison during shard sorts, cursor search and merging.
+const memoryRowCollator = new Intl.Collator("en");
 
 export function queryDigest(bytes) {
   return `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
@@ -10,8 +13,8 @@ export function queryDigest(bytes) {
 export function compareMemoryRows(left, right) {
   return (
     right.importance - left.importance ||
-    right.updatedAt.localeCompare(left.updatedAt, "en") ||
-    left.memoryId.localeCompare(right.memoryId, "en")
+    memoryRowCollator.compare(right.updatedAt, left.updatedAt) ||
+    memoryRowCollator.compare(left.memoryId, right.memoryId)
   );
 }
 

package/src/lib/context-memory-kernel/runtime.js

@@ -22,6 +22,10 @@ export function createCliContextMemoryRuntime(options = {}) {
   const memoryPort =
     options.memoryPort ||
     new SegmentedMemoryPort({
+      queryIndexCacheMode:
+        options.memoryQueryIndexCacheMode ||
+        (options.env || process.env).CC_CONTEXT_MEMORY_QUERY_CACHE ||
+        "validate-disk",
       readOnly: !decision.canonical,
       ...(options.memoryFilePath ? { filePath: options.memoryFilePath } : {}),
     });

package/src/lib/context-memory-kernel/segmented-memory-port.js

@@ -170,6 +170,7 @@ export class SegmentedMemoryPort extends DurableJsonMemoryPort {
     maxTotalBytes = DEFAULT_MAX_TOTAL_BYTES,
     maxEvents = DEFAULT_SEGMENTED_MAX_EVENTS,
     readOnly = false,
+    queryIndexCacheMode = "validate-disk",
     ...options
   } = {}) {
     super({ ...options, maxEvents });
@@ -177,6 +178,9 @@ export class SegmentedMemoryPort extends DurableJsonMemoryPort {
       throw new TypeError("maxTotalBytes must be a positive safe integer");
     this.maxTotalBytes = maxTotalBytes;
     this.readOnly = Boolean(readOnly);
+    if (!["validate-disk", "verified-memory"].includes(queryIndexCacheMode))
+      throw new TypeError("Unknown query index cache mode");
+    this.queryIndexCacheMode = queryIndexCacheMode;
     this.filePath = resolve(this.filePath);
     this.shardDirectory = `${this.filePath}.shards`;
     this._queryIndexes = new Map();
@@ -656,6 +660,27 @@ export class SegmentedMemoryPort extends DurableJsonMemoryPort {
 
   _readQueryIndex(manifest, bucket) {
     const descriptor = manifest.shards[bucket];
+    const binding = JSON.stringify([
+      manifest.digest,
+      manifest.storeRevision,
+      descriptor.digest,
+      descriptor.queryDigest,
+      descriptor.queryBytes,
+      descriptor.recordCount,
+    ]);
+    const cached = this._queryIndexes.get(bucket);
+    // This is an already verified, host-owned decoded snapshot, never a trust
+    // decision based on filesystem timestamps. Re-read and verify the current
+    // manifest on every operation; any generation/authority change invalidates
+    // this binding. Selected authority shards are still read and verified.
+    // Disk repair remains eager in validate-disk mode; verified-memory repairs
+    // a derived index on the next cache miss, rather than every warm query.
+    if (
+      this.queryIndexCacheMode === "verified-memory" &&
+      descriptor.queryDigest &&
+      cached?.binding === binding
+    )
+      return cached.index;
     let bytes;
     if (descriptor.queryDigest) {
       const file = this._queryIndexPath(bucket, descriptor);
@@ -705,8 +730,8 @@ export class SegmentedMemoryPort extends DurableJsonMemoryPort {
       bytes = encodeQueryIndex(this._readShard(manifest, bucket));
     }
     const digest = descriptor.queryDigest || queryDigest(bytes);
-    const cached = this._queryIndexes.get(bucket);
-    if (cached?.digest === digest) return cached.index;
+    if (cached?.digest === digest && cached.binding === binding)
+      return cached.index;
     let index;
     try {
       index = compileQueryIndex(bytes, descriptor);
@@ -717,7 +742,7 @@ export class SegmentedMemoryPort extends DurableJsonMemoryPort {
         cause,
       );
     }
-    this._queryIndexes.set(bucket, { digest, index });
+    this._queryIndexes.set(bucket, { digest, binding, index });
     return index;
   }
 

package/src/lib/context-source-ledger.js

@@ -40,7 +40,28 @@ class ContextSourceLedger {
    * @returns {string} entryId
    */
   record(entry) {
+    if (
+      entry.tokenCount != null &&
+      (!Number.isSafeInteger(entry.tokenCount) || entry.tokenCount < 0)
+    ) {
+      throw new TypeError(
+        "Context tokenCount must be a non-negative safe integer",
+      );
+    }
     const entryId = entry.entryId || crypto.randomUUID();
+    const previous = this._entries.get(entryId);
+    if (previous) {
+      for (const [index, key] of [
+        [this._byTurn, previous.turnId],
+        [this._bySession, previous.sessionId],
+      ]) {
+        const remaining = (index.get(key) || []).filter(
+          (item) => item.entryId !== entryId,
+        );
+        if (remaining.length) index.set(key, remaining);
+        else index.delete(key);
+      }
+    }
     const full = {
       entryId,
       traceId: entry.traceId || null,
@@ -53,6 +74,7 @@ class ContextSourceLedger {
       summary: entry.summary,
       metadata: entry.metadata,
       tokenCount: entry.tokenCount || 0,
+      tokenBasis: entry.tokenBasis === "observed" ? "observed" : "estimated",
       insertedAt: new Date(),
       tags: entry.tags || [],
     };
@@ -64,7 +86,8 @@ class ContextSourceLedger {
     this._byTurn.get(full.turnId).push(full);
 
     // Index by session
-    if (!this._bySession.has(full.sessionId)) this._bySession.set(full.sessionId, []);
+    if (!this._bySession.has(full.sessionId))
+      this._bySession.set(full.sessionId, []);
     this._bySession.get(full.sessionId).push(full);
 
     return entryId;
@@ -82,6 +105,7 @@ class ContextSourceLedger {
       summary: entry.summary || entry.content,
       metadata: entry.metadata,
       tokenCount: entry.tokenCount ?? entry.tokens ?? 0,
+      tokenBasis: entry.tokenBasis,
       traceId: entry.traceId,
     });
   }
@@ -95,14 +119,31 @@ class ContextSourceLedger {
   }
 
   /** Aggregate token usage by source type. */
-  getTokenBreakdown() {
-    const bySource = {};
+  getTokenBreakdown(options = {}) {
+    const sources = new Map();
     let total = 0;
-    for (const entry of this._entries.values()) {
+    for (const entry of this.getProvenance(options)) {
       total += entry.tokenCount;
-      bySource[entry.sourceType] = (bySource[entry.sourceType] || 0) + entry.tokenCount;
+      sources.set(
+        entry.sourceType,
+        (sources.get(entry.sourceType) || 0) + entry.tokenCount,
+      );
     }
-    return { total, bySource };
+    return { total, bySource: Object.fromEntries(sources) };
+  }
+
+  /** Source attribution is separate from provider request usage and billing. */
+  getTokenReport(options = {}) {
+    const entries = this.getProvenance(options);
+    const byBasis = { estimated: 0, observed: 0 };
+    for (const entry of entries) byBasis[entry.tokenBasis] += entry.tokenCount;
+    return {
+      ...this.getTokenBreakdown(options),
+      byBasis,
+      entries: entries.length,
+      currentRequestVerified: false,
+      billingVerified: false,
+    };
   }
 
   /** Clear the in-memory ledger (test/runtime lifecycle boundary). */
@@ -146,7 +187,8 @@ class ContextSourceLedger {
 
     for (const e of entries) {
       byType[e.sourceType] = (byType[e.sourceType] || 0) + 1;
-      byPermission[e.permissionMode] = (byPermission[e.permissionMode] || 0) + 1;
+      byPermission[e.permissionMode] =
+        (byPermission[e.permissionMode] || 0) + 1;
       bySource[e.sourceId] = (bySource[e.sourceId] || 0) + 1;
       totalTokens += e.tokenCount;
       avgConfidence += e.confidence;
@@ -162,7 +204,7 @@ class ContextSourceLedger {
       byType,
       byPermission,
       bySource,
-      entries: entries.map(e => ({
+      entries: entries.map((e) => ({
         sourceType: e.sourceType,
         sourceId: e.sourceId,
         permissionMode: e.permissionMode,
@@ -184,8 +226,9 @@ class ContextSourceLedger {
       this._entries.delete(e.entryId);
       const turnEntries = this._byTurn.get(e.turnId);
       if (turnEntries) {
-        const idx = turnEntries.findIndex(x => x.entryId === e.entryId);
+        const idx = turnEntries.findIndex((x) => x.entryId === e.entryId);
         if (idx >= 0) turnEntries.splice(idx, 1);
+        if (!turnEntries.length) this._byTurn.delete(e.turnId);
       }
     }
     this._bySession.delete(sessionId);