@remnic/core 9.57.5 → 9.58.0

package/src/lcm-engine.test.ts

@@ -7,6 +7,7 @@ import test from "node:test";
 import { LcmArchive } from "./lcm/archive.js";
 import { LcmEngine, extractLcmConfig } from "./lcm/engine.js";
 import { openLcmDatabase } from "./lcm/schema.js";
+import { estimateTokenCount } from "./token-estimate.js";
 import type { PluginConfig } from "./types.js";
 
 function createPluginConfig(memoryDir: string): PluginConfig {
@@ -661,20 +662,99 @@ test("expandContext preserves first, middle, and last row identity when truncate
         session_id: "session-truncated",
         turn_index: 2,
         role: "assistant",
-        content: "BBBBBB",
+        content: "BBBBBBBB",
       },
       {
         id: last.id,
         session_id: "session-truncated",
         turn_index: 4,
         role: "assistant",
-        content: "DDDDDD",
+        content: "DDDD",
       },
     ]);
   } finally {
     await rm(memoryDir, { recursive: true, force: true });
   }
 });
+test("expandContext applies token budgets to wide-script messages", async () => {
+  const memoryDir = await mkdtemp(
+    path.join(os.tmpdir(), "engram-lcm-expand-wide-script-"),
+  );
+
+  try {
+    const engine = new LcmEngine(
+      createPluginConfig(memoryDir),
+      async () => "summary",
+    );
+    await engine.ensureInitialized();
+
+    const db = openLcmDatabase(memoryDir);
+    try {
+      const archive = new LcmArchive(db);
+      archive.appendMessages("session-wide-script", [
+        { turnIndex: 1, role: "user", content: "日本語".repeat(40) },
+        { turnIndex: 2, role: "assistant", content: "日本語".repeat(40) },
+        { turnIndex: 3, role: "user", content: "日本語".repeat(40) },
+      ]);
+    } finally {
+      db.close();
+    }
+
+    const expanded = await engine.expandContext(
+      "session-wide-script",
+      1,
+      3,
+      10,
+    );
+
+    assert.ok(
+      expanded.reduce((sum, message) => sum + estimateTokenCount(message.content), 0) <= 10,
+    );
+    assert.ok(expanded.every((message) => message.content.length < 40));
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+test("expandContext preserves the final message at the smallest positive budget", async () => {
+  const memoryDir = await mkdtemp(
+    path.join(os.tmpdir(), "engram-lcm-expand-minimum-budget-"),
+  );
+
+  let engine: LcmEngine | undefined;
+  try {
+    engine = new LcmEngine(
+      createPluginConfig(memoryDir),
+      async () => "summary",
+    );
+    await engine.ensureInitialized();
+
+    const db = openLcmDatabase(memoryDir);
+    try {
+      const archive = new LcmArchive(db);
+      archive.appendMessages("session-minimum-budget", [
+        { turnIndex: 1, role: "user", content: "first message" },
+        { turnIndex: 2, role: "assistant", content: "最後のメッセージ" },
+      ]);
+    } finally {
+      db.close();
+    }
+
+    const expanded = await engine.expandContext(
+      "session-minimum-budget",
+      1,
+      2,
+      1,
+    );
+
+    assert.equal(expanded.at(-1)?.turn_index, 2);
+    assert.ok(
+      expanded.reduce((sum, message) => sum + estimateTokenCount(message.content), 0) <= 1,
+    );
+  } finally {
+    engine?.close();
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
 
 test("assembleRecallWithTrace preserves text and reports selected summary metadata", async () => {
   const memoryDir = await mkdtemp(

package/src/local-llm.ts

@@ -1,4 +1,5 @@
 import { log } from "./logger.js";
+import { estimateTokenCount } from "./token-estimate.js";
 import type { PluginConfig } from "./types.js";
 import fs from "node:fs";
 import os from "node:os";
@@ -1431,18 +1432,15 @@ export class LocalLlmClient {
 
     return await this.runChatCompletionRequest(messages, options);
   }
-
   /**
-   * Estimate tokens when local LLM doesn't return usage stats
-   * Rough estimate: 1 token ≈ 4 characters
+   * Estimate tokens when local LLM does not return usage stats.
    */
   private estimateTokens(
     messages: Array<{ role: string; content: string }>,
     response: string
   ): { promptTokens: number; completionTokens: number; totalTokens: number } {
-    const promptChars = messages.reduce((sum, m) => sum + m.content.length, 0);
-    const promptTokens = Math.ceil(promptChars / 4);
-    const completionTokens = Math.ceil(response.length / 4);
+    const promptTokens = messages.reduce((sum, message) => sum + estimateTokenCount(message.content), 0);
+    const completionTokens = estimateTokenCount(response);
 
     return {
       promptTokens,

package/src/memory-extension-host/render-extensions-block.ts

@@ -6,16 +6,10 @@
  * truncates with a footer listing omitted extensions when over budget.
  */
 
+import { estimateTokenCount } from "../token-estimate.js";
 import { REMNIC_EXTENSIONS_TOTAL_TOKEN_LIMIT } from "./host-discovery.js";
 import type { DiscoveredExtension } from "./types.js";
 
-/**
- * Approximate token count using the 4 chars per token heuristic.
- */
-function estimateTokens(text: string): number {
-  return Math.ceil(text.length / 4);
-}
-
 /**
  * Render a markdown block containing extension instructions for injection
  * into consolidation prompts.
@@ -36,14 +30,14 @@ extension produces or curates.
 `;
 
   let budget = REMNIC_EXTENSIONS_TOTAL_TOKEN_LIMIT;
-  budget -= estimateTokens(header);
+  budget -= estimateTokenCount(header);
 
   const inlined: string[] = [];
   const omitted: string[] = [];
 
   for (const ext of extensions) {
     const block = `### remnic-extension/${ext.name}\n\`\`\`\n${ext.instructions}\n\`\`\`\n\n`;
-    const cost = estimateTokens(block);
+    const cost = estimateTokenCount(block);
     if (cost <= budget) {
       inlined.push(block);
       budget -= cost;

package/src/orchestration/recall-section-coordinator.ts

@@ -17,8 +17,9 @@
  * TierMigrationCoordinator accessor pattern.
  */
 
-import type { PluginConfig, RecallSectionConfig } from "../types.js";
 import type { LastRecallBudgetSummary } from "../recall-state.js";
+import { estimateTokenCount } from "../token-estimate.js";
+import type { PluginConfig, RecallSectionConfig } from "../types.js";
 
 export interface RecallSectionAppendOptions {
   atomic?: boolean;
@@ -179,18 +180,54 @@ export class RecallSectionCoordinator {
     const end = boundary > 0 ? boundary : contentLimit;
     return `${content.slice(0, end).trimEnd()}${suffix}`;
   }
-
   truncateRecallSectionToBudget(
     content: string,
     maxChars: number,
+    maxTokens?: number,
   ): string {
-    if (maxChars <= 0) return "";
-    if (content.length <= maxChars) return content;
+    if (maxChars <= 0 || maxTokens === 0) return "";
     const suffix = "\n\n...(memory context trimmed)";
-    if (maxChars <= suffix.length) {
-      return content.slice(0, maxChars);
+    const charBounded =
+      content.length <= maxChars
+        ? content
+        : maxChars <= suffix.length
+          ? content.slice(0, maxChars)
+          : `${content.slice(0, maxChars - suffix.length)}${suffix}`;
+    if (maxTokens === undefined || estimateTokenCount(charBounded) <= maxTokens) {
+      return charBounded;
     }
-    return `${content.slice(0, maxChars - suffix.length)}${suffix}`;
+    const markerFits =
+      maxChars > suffix.length && estimateTokenCount(suffix) <= maxTokens;
+    const tokenSource = content.length > maxChars ? content.slice(0, maxChars) : content;
+    const codePoints = [...tokenSource];
+    let low = 0;
+    let high = codePoints.length;
+    while (low < high) {
+      const mid = Math.ceil((low + high) / 2);
+      const prefix = codePoints.slice(0, mid).join("");
+      const candidate = markerFits ? `${prefix}${suffix}` : prefix;
+      if (candidate.length <= maxChars && estimateTokenCount(candidate) <= maxTokens) {
+        low = mid;
+      } else {
+        high = mid - 1;
+      }
+    }
+    if (markerFits && low === 0) {
+      low = 0;
+      high = codePoints.length;
+      while (low < high) {
+        const mid = Math.ceil((low + high) / 2);
+        const prefix = codePoints.slice(0, mid).join("");
+        if (prefix.length <= maxChars && estimateTokenCount(prefix) <= maxTokens) {
+          low = mid;
+        } else {
+          high = mid - 1;
+        }
+      }
+      return codePoints.slice(0, low).join("");
+    }
+    const prefix = codePoints.slice(0, low).join("");
+    return markerFits ? `${prefix}${suffix}` : prefix;
   }
 
 
@@ -280,6 +317,7 @@ export class RecallSectionCoordinator {
     }
 
     const budget = this.getRecallBudgetChars(budgetOverride);
+    const tokenBudget = Math.max(0, Math.floor(this.getConfig().maxMemoryTokens));
     const candidateMemoryChunks =
       orderedSections
         .find((section) => section.id === "memories")
@@ -312,10 +350,19 @@ export class RecallSectionCoordinator {
     const memoryIndex = allocationOrder.indexOf("memories");
     const memoryBudget = this.getRecallSectionMaxChars("memories") ?? budget;
     const memoryAllocationBudget = Math.min(budget, memoryBudget);
-    let firstAtomicMemoryReserveChars = 0;
+    let firstAtomicMemoryReserve:
+      | { chars: number; tokens: number }
+      | undefined;
     for (const chunk of memorySection?.chunks ?? []) {
-      if (chunk.atomic && chunk.content.length <= memoryAllocationBudget) {
-        firstAtomicMemoryReserveChars = chunk.content.length;
+      if (!chunk.atomic || chunk.content.length > memoryAllocationBudget) {
+        continue;
+      }
+      const tokens = estimateTokenCount(chunk.content);
+      if (tokens <= tokenBudget) {
+        firstAtomicMemoryReserve = {
+          chars: chunk.content.length,
+          tokens,
+        };
         break;
       }
     }
@@ -323,6 +370,7 @@ export class RecallSectionCoordinator {
     const includedMemoryPaths: string[] = [];
     const includedMemoryNamespaces: Array<string | undefined> = [];
     let usedChars = 0;
+    let usedTokens = 0;
     let truncated = false;
     const selected = new Map<string, string>();
 
@@ -331,19 +379,30 @@ export class RecallSectionCoordinator {
       if (!section || selected.has(id)) continue;
       const separatorChars = selected.size > 0 ? separator.length : 0;
       const available = budget - usedChars - separatorChars;
+      const separatorTokens = selected.size > 0 ? estimateTokenCount(separator) : 0;
+      const availableTokens = tokenBudget - usedTokens - separatorTokens;
       const sectionMaxChars = this.getRecallSectionMaxChars(id);
-      const reservesFirstMemory =
-        memoryIndex > allocationOrder.indexOf(id) &&
-        firstAtomicMemoryReserveChars > 0;
-      const memoryReserve = reservesFirstMemory
-        ? firstAtomicMemoryReserveChars + separator.length
+      const reservedMemory =
+        memoryIndex > allocationOrder.indexOf(id)
+          ? firstAtomicMemoryReserve
+          : undefined;
+      const memoryReserveSeparatorChars = separator.length;
+      const memoryReserveSeparatorTokens = estimateTokenCount(separator);
+      const memoryReserve = reservedMemory
+        ? reservedMemory.chars + memoryReserveSeparatorChars
         : 0;
       const availableAfterMemoryReserve = available - memoryReserve;
       const allocatedSectionAvailable =
         typeof sectionMaxChars === "number"
           ? Math.min(availableAfterMemoryReserve, sectionMaxChars)
           : availableAfterMemoryReserve;
-      if (allocatedSectionAvailable <= 0) {
+      const availableAfterMemoryReserveTokens = reservedMemory
+        ? availableTokens -
+          reservedMemory.tokens -
+          memoryReserveSeparatorTokens
+        : availableTokens;
+      const allocatedTokenAvailable = availableAfterMemoryReserveTokens;
+      if (allocatedSectionAvailable <= 0 || allocatedTokenAvailable <= 0) {
         truncated = true;
         continue;
       }
@@ -372,13 +431,11 @@ export class RecallSectionCoordinator {
                 Math.min(profileLimit, allocatedSectionAvailable),
               )
             : content;
-        finalContent =
-          id === "profile"
-            ? boundedContent
-            : this.truncateRecallSectionToBudget(
-                boundedContent,
-                allocatedSectionAvailable,
-              );
+        finalContent = this.truncateRecallSectionToBudget(
+          boundedContent,
+          allocatedSectionAvailable,
+          allocatedTokenAvailable,
+        );
         if (!finalContent) {
           truncated = true;
           continue;
@@ -400,7 +457,10 @@ export class RecallSectionCoordinator {
             const candidate = rendered
               ? `${rendered}\n\n${chunk.content}`
               : chunk.content;
-            if (candidate.length <= allocatedSectionAvailable) {
+            if (
+              candidate.length <= allocatedSectionAvailable &&
+              estimateTokenCount(candidate) <= allocatedTokenAvailable
+            ) {
               rendered = candidate;
               includedLeadingContent = true;
             } else {
@@ -412,12 +472,16 @@ export class RecallSectionCoordinator {
           const candidate = rendered
             ? `${rendered}\n\n${chunk.content}`
             : chunk.content;
-          if (candidate.length > allocatedSectionAvailable) {
+          if (
+            candidate.length > allocatedSectionAvailable ||
+            estimateTokenCount(candidate) > allocatedTokenAvailable
+          ) {
             if (
               chunk.atomic &&
               includedAtomicCount === 0 &&
               includedLeadingContent &&
-              chunk.content.length <= allocatedSectionAvailable
+              chunk.content.length <= allocatedSectionAvailable &&
+              estimateTokenCount(chunk.content) <= allocatedTokenAvailable
             ) {
               rendered = chunk.content;
               includedLeadingContent = false;
@@ -435,7 +499,8 @@ export class RecallSectionCoordinator {
               !chunk.atomic &&
               includedAtomicCount === 0 &&
               includedLeadingContent &&
-              chunk.content.length <= allocatedSectionAvailable
+              chunk.content.length <= allocatedSectionAvailable &&
+              estimateTokenCount(chunk.content) <= allocatedTokenAvailable
             ) {
               rendered = chunk.content;
               includedLeadingContent = false;
@@ -479,6 +544,7 @@ export class RecallSectionCoordinator {
       }
       selected.set(id, finalContent);
       usedChars += separatorChars + finalContent.length;
+      usedTokens += separatorTokens + estimateTokenCount(finalContent);
     }
 
     const sections: string[] = [];

package/src/recall-token-budget.test.ts

@@ -0,0 +1,51 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { parseConfig } from "./config.js";
+import { RecallSectionCoordinator } from "./orchestration/recall-section-coordinator.js";
+import { estimateTokenCount } from "./token-estimate.js";
+
+test("recall derived budget keeps Japanese injection within maxMemoryTokens", () => {
+  const config = parseConfig({
+    maxMemoryTokens: 12,
+    recallPipeline: [{ id: "memories", enabled: true }],
+  });
+  const coordinator = new RecallSectionCoordinator({ getConfig: () => config });
+  const buckets = new Map<string, string[]>();
+
+  coordinator.appendRecallSection(buckets, "memories", "日本語の記憶。".repeat(20));
+  const assembled = coordinator.assembleRecallSections(buckets);
+  const injected = assembled.sections.join("\n\n---\n\n");
+
+  assert.equal(config.recallBudgetChars, config.maxMemoryTokens * 4);
+  assert.ok(estimateTokenCount(injected) <= config.maxMemoryTokens);
+});
+test("recall reserves one memory chunk that fits both character and token budgets", () => {
+  const config = parseConfig({
+    maxMemoryTokens: 50,
+    recallBudgetChars: 150,
+    recallPipeline: [
+      { id: "profile", enabled: true },
+      { id: "memories", enabled: true },
+    ],
+  });
+  const coordinator = new RecallSectionCoordinator({ getConfig: () => config });
+  const buckets = new Map<string, string[]>();
+
+  coordinator.appendRecallSection(buckets, "profile", "x".repeat(90));
+  coordinator.appendRecallSection(buckets, "memories", "日".repeat(51), {
+    atomic: true,
+    memoryId: "wide",
+    memoryPath: "memories/wide.md",
+  });
+  coordinator.appendRecallSection(buckets, "memories", "a".repeat(100), {
+    atomic: true,
+    memoryId: "latin",
+    memoryPath: "memories/latin.md",
+  });
+
+  const assembled = coordinator.assembleRecallSections(buckets);
+
+  assert.deepEqual(assembled.includedMemoryIds, ["latin"]);
+  assert.ok(estimateTokenCount(assembled.sections.join("\n\n---\n\n")) <= 50);
+});

package/src/recall-xray.ts

@@ -19,24 +19,17 @@
 
 import { randomUUID } from "node:crypto";
 
-import {
-  normalizeRetrievedMemoryProvenance,
-  type RetrievedMemoryProvenance,
-} from "./memory-provenance.js";
+import { normalizeRetrievedMemoryProvenance, type RetrievedMemoryProvenance } from "./memory-provenance.js";
+import { estimateTokenCount } from "./token-estimate.js";
 import type { RecallDisclosure, RecallTierExplain } from "./types.js";
 import { isRecallDisclosure } from "./types.js";
 
 /**
- * Estimate token cost of a payload at the rough ~4 chars/token English
- * heuristic.  Non-negative integer; returns 0 for empty / null input.
- * Used by recall surfaces to attach `estimatedTokens` to X-ray results
- * (issue #677 PR 3/4).  Identical to the private heuristic in
- * `chunking.ts`; kept self-contained here so X-ray callers don't pull
- * in chunking internals.
+ * Estimate token cost of a payload for X-ray disclosure accounting.
  */
 export function estimateRecallTokens(text: string | null | undefined): number {
   if (typeof text !== "string" || text.length === 0) return 0;
-  return Math.ceil(text.length / 4);
+  return estimateTokenCount(text);
 }
 
 /**

package/src/semantic-chunking.ts

@@ -6,7 +6,14 @@
  * natural topic boundaries, producing more coherent chunks.
  */
 
-import { chunkContent, type Chunk, type ChunkResult } from "./chunking.js";
+import {
+  chunkContent,
+  joinSentences,
+  splitSentences,
+  type Chunk,
+  type ChunkResult,
+} from "./chunking.js";
+import { estimateTokenCount } from "./token-estimate.js";
 
 // ---------------------------------------------------------------------------
 // Configuration
@@ -177,59 +184,7 @@ export function findLocalMinima(
 }
 
 // ---------------------------------------------------------------------------
-// Sentence tokenizer
-// ---------------------------------------------------------------------------
-
-/**
- * Split text into sentences at punctuation boundaries.
- * Preserves punctuation with the preceding sentence.
- */
-function splitSentences(text: string): string[] {
-  // Linear character scan instead of a regex. Every regex form of this split is
-  // either polynomial (CodeQL js/polynomial-redos) or — once bounded/anchored to
-  // satisfy CodeQL — mishandles long runs or interior punctuation (a global
-  // match drops a skipped prefix; a sticky match stops at the first non-boundary
-  // `.`, e.g. "v1.2.3" / "example.com", returning the whole document as one
-  // sentence and bypassing chunking). The scan is O(n), drops nothing, and
-  // handles interior punctuation correctly; normal prose splits identically to
-  // the previous /[^.!?]*[.!?]+(?:\s+|$)/g form.
-  const sentences: string[] = [];
-  let start = 0;
-  for (let i = 0; i < text.length; i++) {
-    const ch = text[i];
-    if (ch !== "." && ch !== "!" && ch !== "?") continue;
-    let end = i;
-    while (end + 1 < text.length) {
-      const n = text[end + 1];
-      if (n !== "." && n !== "!" && n !== "?") break;
-      end++;
-    }
-    const after = text[end + 1];
-    // A real boundary only if the terminator run ends the string or is followed
-    // by whitespace. Interior punctuation (no following whitespace) is left in
-    // place and the scan continues.
-    if (after === undefined || /\s/.test(after)) {
-      const sentence = text.slice(start, end + 1).trim();
-      if (sentence.length > 0) sentences.push(sentence);
-      start = end + 1;
-    }
-    i = end;
-  }
-  if (start < text.length) {
-    const remaining = text.slice(start).trim();
-    if (remaining.length > 0) sentences.push(remaining);
-  }
-  return sentences;
-}
 
-// ---------------------------------------------------------------------------
-// Token estimation
-// ---------------------------------------------------------------------------
-
-/** Rough token estimate: ~4 chars per token for English. */
-function estimateTokens(text: string): number {
-  return Math.ceil(text.length / 4);
-}
 
 // ---------------------------------------------------------------------------
 // Core semantic chunking
@@ -316,7 +271,7 @@ function mergeShortSegments(
 
   for (let i = 0; i < segments.length; i++) {
     buffer = [...buffer, ...segments[i]];
-    const tokenCount = estimateTokens(buffer.join(" "));
+    const tokenCount = estimateTokenCount(joinSentences(buffer));
 
     if (tokenCount >= minTokens || i === segments.length - 1) {
       merged.push(buffer);
@@ -344,7 +299,7 @@ function splitLongSegment(
   maxTokens: number,
   targetTokens: number,
 ): SemanticChunk[] {
-  const text = segment.join(" ");
+  const text = joinSentences(segment);
   // Cap targetTokens to maxTokens so recursive splitting never produces
   // segments larger than the configured maximum (Finding 2, PR #420).
   const cappedTarget = Math.min(targetTokens, maxTokens);
@@ -396,7 +351,7 @@ export async function semanticChunkContent(
   const sentences = splitSentences(content);
 
   if (sentences.length <= 1) {
-    const tokenCount = estimateTokens(content);
+    const tokenCount = estimateTokenCount(content);
     return {
       chunked: false,
       chunks: [
@@ -413,7 +368,7 @@ export async function semanticChunkContent(
   }
 
   // If total tokens is short enough, return as single chunk
-  const totalTokens = estimateTokens(content);
+  const totalTokens = estimateTokenCount(content);
   if (totalTokens <= cfg.minTokens) {
     return {
       chunked: false,
@@ -511,8 +466,8 @@ export async function semanticChunkContent(
 
   for (let segIdx = 0; segIdx < segments.length; segIdx++) {
     const segment = segments[segIdx];
-    const segText = segment.join(" ");
-    const segTokens = estimateTokens(segText);
+    const segText = joinSentences(segment);
+    const segTokens = estimateTokenCount(segText);
 
     if (segTokens > cfg.maxTokens) {
       // Recursive split for oversized segment

package/src/session-observer-state.ts

@@ -14,7 +14,7 @@ interface SessionObserverCursor {
 }
 
 interface SessionObserverPersistedState {
-  version: 1;
+  version: 2;
   sessions: Record<string, SessionObserverCursor>;
 }
 
@@ -134,7 +134,7 @@ export class SessionObserverState {
     try {
       const raw = await readFile(this.statePath, "utf-8");
       const parsed = JSON.parse(raw) as SessionObserverPersistedState;
-      if (parsed?.version !== 1 || !parsed.sessions || typeof parsed.sessions !== "object") {
+      if (parsed?.version !== 2 || !parsed.sessions || typeof parsed.sessions !== "object") {
         return null;
       }
       return parsed;
@@ -246,7 +246,7 @@ export class SessionObserverState {
     for (const [key, value] of sessionsMap.entries()) {
       sessions[key] = value;
     }
-    const payload: SessionObserverPersistedState = { version: 1, sessions };
+    const payload: SessionObserverPersistedState = { version: 2, sessions };
     await mkdir(path.dirname(this.statePath), { recursive: true });
     await writeFile(this.statePath, JSON.stringify(payload, null, 2), "utf-8");
   }

package/src/testing/subjects/recall-budget.test.ts

@@ -9,7 +9,7 @@
  *
  * The nine rows are realized honestly against the coordinator's REAL behavior —
  * no mocks. Config is built through the production `parseConfig` path so the
- * budget derivation (recallBudgetChars = maxMemoryTokens * 4), section
+ * budget derivation (recallBudgetChars = maxMemoryTokens), section
  * enablement filtering, per-section maxChars caps, and cap-after-filter
  * ordering are all exercised as they ship. Each row asserts one recall-budget
  * contract invariant:
@@ -28,6 +28,7 @@ import assert from "node:assert/strict";
 
 import { parseConfig } from "../../config.js";
 import { RecallSectionCoordinator } from "../../orchestration/recall-section-coordinator.js";
+import { estimateTokenCount } from "../../token-estimate.js";
 import { createRecallSectionMetricRecorder, formatRecallSectionMetric } from "../../recall-qos.js";
 import { reorderRecallResultsWithMmr } from "../../recall-mmr.js";
 import { applyRuntimeRetrievalPolicy, expandQuery } from "../../retrieval.js";
@@ -297,10 +298,10 @@ const subject: LifecycleSubject<RecallBudgetState> = {
         return;
       }
       case "sparse-metadata-with-binding": {
-        // Budget is derived from maxMemoryTokens (recallBudgetChars = tokens*4).
+        // Budget keeps Latin-script headroom and token-aware assembly caps wide scripts.
         assert.equal(config.recallBudgetChars, config.maxMemoryTokens * 4, "budget derives from maxMemoryTokens");
-        assert.equal(budget, config.maxMemoryTokens * 4, "coordinator honors the token-derived budget");
-        assert.ok(assembled.finalChars <= budget, "section sum stays within the token-derived budget");
+        assert.ok(budget <= config.maxMemoryTokens * 4, "coordinator honors the character budget");
+        assert.ok(assembled.finalChars <= budget, "section sum stays within the character budget");
         assert.ok(assembled.truncated, "oversized content forces truncation");
         return;
       }
@@ -372,7 +373,22 @@ const subject: LifecycleSubject<RecallBudgetState> = {
         // truncateRecallSectionToBudget respects a hard per-section budget too.
         const truncated = coordinator.truncateRecallSectionToBudget(body("m", 200), 50);
         assert.equal(truncated.length, 50, "budget-aware truncation lands exactly on the per-section limit");
-        // A budget-skipped section reports as a debug-level "skip" metric — the
+        const tokenTrimmed = coordinator.truncateRecallSectionToBudget("x".repeat(200), 200, 20);
+        assert.match(tokenTrimmed, /\.\.\.\(memory context trimmed\)$/);
+        assert.ok(estimateTokenCount(tokenTrimmed) <= 20, "token-aware truncation keeps its marker inside the cap");
+        const markerBudget = estimateTokenCount("\n\n...(memory context trimmed)");
+        const markerOnlyBudget = coordinator.truncateRecallSectionToBudget("日本語".repeat(100), 200, markerBudget);
+        assert.ok(markerOnlyBudget.length > 0, "a marker-only budget keeps useful content");
+        assert.doesNotMatch(markerOnlyBudget, /\.\.\.\(memory context trimmed\)$/);
+        assert.ok(estimateTokenCount(markerOnlyBudget) <= markerBudget);
+        const wideScriptTrimmed = coordinator.truncateRecallSectionToBudget("日".repeat(100), 30, markerBudget);
+        assert.equal([...wideScriptTrimmed].length, 8, "wide-script fallback uses the available character room");
+        assert.ok(estimateTokenCount(wideScriptTrimmed) <= markerBudget);
+        assert.doesNotMatch(wideScriptTrimmed, /\.\.\.\(memory context trimmed\)$/);
+        const emojiTrimmed = coordinator.truncateRecallSectionToBudget("😀".repeat(30), 60, 29);
+        assert.ok(emojiTrimmed.length <= 60, "astral text stays within the UTF-16 character cap");
+        assert.ok(estimateTokenCount(emojiTrimmed) <= 29, "astral text stays within the token cap");
+        assert.match(emojiTrimmed, /\.\.\.\(memory context trimmed\)$/);
         // QoS accounting that flags a section dropped for the budget deadline.
         const skipMetric = formatRecallSectionMetric({
           section: "memories",