@remnic/core 9.57.6 → 9.58.0

package/src/orchestration/recall-section-coordinator.ts

@@ -17,8 +17,9 @@
  * TierMigrationCoordinator accessor pattern.
  */
 
-import type { PluginConfig, RecallSectionConfig } from "../types.js";
 import type { LastRecallBudgetSummary } from "../recall-state.js";
+import { estimateTokenCount } from "../token-estimate.js";
+import type { PluginConfig, RecallSectionConfig } from "../types.js";
 
 export interface RecallSectionAppendOptions {
   atomic?: boolean;
@@ -179,18 +180,54 @@ export class RecallSectionCoordinator {
     const end = boundary > 0 ? boundary : contentLimit;
     return `${content.slice(0, end).trimEnd()}${suffix}`;
   }
-
   truncateRecallSectionToBudget(
     content: string,
     maxChars: number,
+    maxTokens?: number,
   ): string {
-    if (maxChars <= 0) return "";
-    if (content.length <= maxChars) return content;
+    if (maxChars <= 0 || maxTokens === 0) return "";
     const suffix = "\n\n...(memory context trimmed)";
-    if (maxChars <= suffix.length) {
-      return content.slice(0, maxChars);
+    const charBounded =
+      content.length <= maxChars
+        ? content
+        : maxChars <= suffix.length
+          ? content.slice(0, maxChars)
+          : `${content.slice(0, maxChars - suffix.length)}${suffix}`;
+    if (maxTokens === undefined || estimateTokenCount(charBounded) <= maxTokens) {
+      return charBounded;
     }
-    return `${content.slice(0, maxChars - suffix.length)}${suffix}`;
+    const markerFits =
+      maxChars > suffix.length && estimateTokenCount(suffix) <= maxTokens;
+    const tokenSource = content.length > maxChars ? content.slice(0, maxChars) : content;
+    const codePoints = [...tokenSource];
+    let low = 0;
+    let high = codePoints.length;
+    while (low < high) {
+      const mid = Math.ceil((low + high) / 2);
+      const prefix = codePoints.slice(0, mid).join("");
+      const candidate = markerFits ? `${prefix}${suffix}` : prefix;
+      if (candidate.length <= maxChars && estimateTokenCount(candidate) <= maxTokens) {
+        low = mid;
+      } else {
+        high = mid - 1;
+      }
+    }
+    if (markerFits && low === 0) {
+      low = 0;
+      high = codePoints.length;
+      while (low < high) {
+        const mid = Math.ceil((low + high) / 2);
+        const prefix = codePoints.slice(0, mid).join("");
+        if (prefix.length <= maxChars && estimateTokenCount(prefix) <= maxTokens) {
+          low = mid;
+        } else {
+          high = mid - 1;
+        }
+      }
+      return codePoints.slice(0, low).join("");
+    }
+    const prefix = codePoints.slice(0, low).join("");
+    return markerFits ? `${prefix}${suffix}` : prefix;
   }
 
 
@@ -280,6 +317,7 @@ export class RecallSectionCoordinator {
     }
 
     const budget = this.getRecallBudgetChars(budgetOverride);
+    const tokenBudget = Math.max(0, Math.floor(this.getConfig().maxMemoryTokens));
     const candidateMemoryChunks =
       orderedSections
         .find((section) => section.id === "memories")
@@ -312,10 +350,19 @@ export class RecallSectionCoordinator {
     const memoryIndex = allocationOrder.indexOf("memories");
     const memoryBudget = this.getRecallSectionMaxChars("memories") ?? budget;
     const memoryAllocationBudget = Math.min(budget, memoryBudget);
-    let firstAtomicMemoryReserveChars = 0;
+    let firstAtomicMemoryReserve:
+      | { chars: number; tokens: number }
+      | undefined;
     for (const chunk of memorySection?.chunks ?? []) {
-      if (chunk.atomic && chunk.content.length <= memoryAllocationBudget) {
-        firstAtomicMemoryReserveChars = chunk.content.length;
+      if (!chunk.atomic || chunk.content.length > memoryAllocationBudget) {
+        continue;
+      }
+      const tokens = estimateTokenCount(chunk.content);
+      if (tokens <= tokenBudget) {
+        firstAtomicMemoryReserve = {
+          chars: chunk.content.length,
+          tokens,
+        };
         break;
       }
     }
@@ -323,6 +370,7 @@ export class RecallSectionCoordinator {
     const includedMemoryPaths: string[] = [];
     const includedMemoryNamespaces: Array<string | undefined> = [];
     let usedChars = 0;
+    let usedTokens = 0;
     let truncated = false;
     const selected = new Map<string, string>();
 
@@ -331,19 +379,30 @@ export class RecallSectionCoordinator {
       if (!section || selected.has(id)) continue;
       const separatorChars = selected.size > 0 ? separator.length : 0;
       const available = budget - usedChars - separatorChars;
+      const separatorTokens = selected.size > 0 ? estimateTokenCount(separator) : 0;
+      const availableTokens = tokenBudget - usedTokens - separatorTokens;
       const sectionMaxChars = this.getRecallSectionMaxChars(id);
-      const reservesFirstMemory =
-        memoryIndex > allocationOrder.indexOf(id) &&
-        firstAtomicMemoryReserveChars > 0;
-      const memoryReserve = reservesFirstMemory
-        ? firstAtomicMemoryReserveChars + separator.length
+      const reservedMemory =
+        memoryIndex > allocationOrder.indexOf(id)
+          ? firstAtomicMemoryReserve
+          : undefined;
+      const memoryReserveSeparatorChars = separator.length;
+      const memoryReserveSeparatorTokens = estimateTokenCount(separator);
+      const memoryReserve = reservedMemory
+        ? reservedMemory.chars + memoryReserveSeparatorChars
         : 0;
       const availableAfterMemoryReserve = available - memoryReserve;
       const allocatedSectionAvailable =
         typeof sectionMaxChars === "number"
           ? Math.min(availableAfterMemoryReserve, sectionMaxChars)
           : availableAfterMemoryReserve;
-      if (allocatedSectionAvailable <= 0) {
+      const availableAfterMemoryReserveTokens = reservedMemory
+        ? availableTokens -
+          reservedMemory.tokens -
+          memoryReserveSeparatorTokens
+        : availableTokens;
+      const allocatedTokenAvailable = availableAfterMemoryReserveTokens;
+      if (allocatedSectionAvailable <= 0 || allocatedTokenAvailable <= 0) {
         truncated = true;
         continue;
       }
@@ -372,13 +431,11 @@ export class RecallSectionCoordinator {
                 Math.min(profileLimit, allocatedSectionAvailable),
               )
             : content;
-        finalContent =
-          id === "profile"
-            ? boundedContent
-            : this.truncateRecallSectionToBudget(
-                boundedContent,
-                allocatedSectionAvailable,
-              );
+        finalContent = this.truncateRecallSectionToBudget(
+          boundedContent,
+          allocatedSectionAvailable,
+          allocatedTokenAvailable,
+        );
         if (!finalContent) {
           truncated = true;
           continue;
@@ -400,7 +457,10 @@ export class RecallSectionCoordinator {
             const candidate = rendered
               ? `${rendered}\n\n${chunk.content}`
               : chunk.content;
-            if (candidate.length <= allocatedSectionAvailable) {
+            if (
+              candidate.length <= allocatedSectionAvailable &&
+              estimateTokenCount(candidate) <= allocatedTokenAvailable
+            ) {
               rendered = candidate;
               includedLeadingContent = true;
             } else {
@@ -412,12 +472,16 @@ export class RecallSectionCoordinator {
           const candidate = rendered
             ? `${rendered}\n\n${chunk.content}`
             : chunk.content;
-          if (candidate.length > allocatedSectionAvailable) {
+          if (
+            candidate.length > allocatedSectionAvailable ||
+            estimateTokenCount(candidate) > allocatedTokenAvailable
+          ) {
             if (
               chunk.atomic &&
               includedAtomicCount === 0 &&
               includedLeadingContent &&
-              chunk.content.length <= allocatedSectionAvailable
+              chunk.content.length <= allocatedSectionAvailable &&
+              estimateTokenCount(chunk.content) <= allocatedTokenAvailable
             ) {
               rendered = chunk.content;
               includedLeadingContent = false;
@@ -435,7 +499,8 @@ export class RecallSectionCoordinator {
               !chunk.atomic &&
               includedAtomicCount === 0 &&
               includedLeadingContent &&
-              chunk.content.length <= allocatedSectionAvailable
+              chunk.content.length <= allocatedSectionAvailable &&
+              estimateTokenCount(chunk.content) <= allocatedTokenAvailable
             ) {
               rendered = chunk.content;
               includedLeadingContent = false;
@@ -479,6 +544,7 @@ export class RecallSectionCoordinator {
       }
       selected.set(id, finalContent);
       usedChars += separatorChars + finalContent.length;
+      usedTokens += separatorTokens + estimateTokenCount(finalContent);
     }
 
     const sections: string[] = [];

package/src/recall-token-budget.test.ts

@@ -0,0 +1,51 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { parseConfig } from "./config.js";
+import { RecallSectionCoordinator } from "./orchestration/recall-section-coordinator.js";
+import { estimateTokenCount } from "./token-estimate.js";
+
+test("recall derived budget keeps Japanese injection within maxMemoryTokens", () => {
+  const config = parseConfig({
+    maxMemoryTokens: 12,
+    recallPipeline: [{ id: "memories", enabled: true }],
+  });
+  const coordinator = new RecallSectionCoordinator({ getConfig: () => config });
+  const buckets = new Map<string, string[]>();
+
+  coordinator.appendRecallSection(buckets, "memories", "日本語の記憶。".repeat(20));
+  const assembled = coordinator.assembleRecallSections(buckets);
+  const injected = assembled.sections.join("\n\n---\n\n");
+
+  assert.equal(config.recallBudgetChars, config.maxMemoryTokens * 4);
+  assert.ok(estimateTokenCount(injected) <= config.maxMemoryTokens);
+});
+test("recall reserves one memory chunk that fits both character and token budgets", () => {
+  const config = parseConfig({
+    maxMemoryTokens: 50,
+    recallBudgetChars: 150,
+    recallPipeline: [
+      { id: "profile", enabled: true },
+      { id: "memories", enabled: true },
+    ],
+  });
+  const coordinator = new RecallSectionCoordinator({ getConfig: () => config });
+  const buckets = new Map<string, string[]>();
+
+  coordinator.appendRecallSection(buckets, "profile", "x".repeat(90));
+  coordinator.appendRecallSection(buckets, "memories", "日".repeat(51), {
+    atomic: true,
+    memoryId: "wide",
+    memoryPath: "memories/wide.md",
+  });
+  coordinator.appendRecallSection(buckets, "memories", "a".repeat(100), {
+    atomic: true,
+    memoryId: "latin",
+    memoryPath: "memories/latin.md",
+  });
+
+  const assembled = coordinator.assembleRecallSections(buckets);
+
+  assert.deepEqual(assembled.includedMemoryIds, ["latin"]);
+  assert.ok(estimateTokenCount(assembled.sections.join("\n\n---\n\n")) <= 50);
+});

package/src/recall-xray.ts

@@ -19,24 +19,17 @@
 
 import { randomUUID } from "node:crypto";
 
-import {
-  normalizeRetrievedMemoryProvenance,
-  type RetrievedMemoryProvenance,
-} from "./memory-provenance.js";
+import { normalizeRetrievedMemoryProvenance, type RetrievedMemoryProvenance } from "./memory-provenance.js";
+import { estimateTokenCount } from "./token-estimate.js";
 import type { RecallDisclosure, RecallTierExplain } from "./types.js";
 import { isRecallDisclosure } from "./types.js";
 
 /**
- * Estimate token cost of a payload at the rough ~4 chars/token English
- * heuristic.  Non-negative integer; returns 0 for empty / null input.
- * Used by recall surfaces to attach `estimatedTokens` to X-ray results
- * (issue #677 PR 3/4).  Identical to the private heuristic in
- * `chunking.ts`; kept self-contained here so X-ray callers don't pull
- * in chunking internals.
+ * Estimate token cost of a payload for X-ray disclosure accounting.
  */
 export function estimateRecallTokens(text: string | null | undefined): number {
   if (typeof text !== "string" || text.length === 0) return 0;
-  return Math.ceil(text.length / 4);
+  return estimateTokenCount(text);
 }
 
 /**

package/src/semantic-chunking.ts

@@ -13,6 +13,7 @@ import {
   type Chunk,
   type ChunkResult,
 } from "./chunking.js";
+import { estimateTokenCount } from "./token-estimate.js";
 
 // ---------------------------------------------------------------------------
 // Configuration
@@ -184,14 +185,6 @@ export function findLocalMinima(
 
 // ---------------------------------------------------------------------------
 
-// ---------------------------------------------------------------------------
-// Token estimation
-// ---------------------------------------------------------------------------
-
-/** Rough token estimate: ~4 chars per token for English. */
-function estimateTokens(text: string): number {
-  return Math.ceil(text.length / 4);
-}
 
 // ---------------------------------------------------------------------------
 // Core semantic chunking
@@ -278,7 +271,7 @@ function mergeShortSegments(
 
   for (let i = 0; i < segments.length; i++) {
     buffer = [...buffer, ...segments[i]];
-    const tokenCount = estimateTokens(joinSentences(buffer));
+    const tokenCount = estimateTokenCount(joinSentences(buffer));
 
     if (tokenCount >= minTokens || i === segments.length - 1) {
       merged.push(buffer);
@@ -358,7 +351,7 @@ export async function semanticChunkContent(
   const sentences = splitSentences(content);
 
   if (sentences.length <= 1) {
-    const tokenCount = estimateTokens(content);
+    const tokenCount = estimateTokenCount(content);
     return {
       chunked: false,
       chunks: [
@@ -375,7 +368,7 @@ export async function semanticChunkContent(
   }
 
   // If total tokens is short enough, return as single chunk
-  const totalTokens = estimateTokens(content);
+  const totalTokens = estimateTokenCount(content);
   if (totalTokens <= cfg.minTokens) {
     return {
       chunked: false,
@@ -474,7 +467,7 @@ export async function semanticChunkContent(
   for (let segIdx = 0; segIdx < segments.length; segIdx++) {
     const segment = segments[segIdx];
     const segText = joinSentences(segment);
-    const segTokens = estimateTokens(segText);
+    const segTokens = estimateTokenCount(segText);
 
     if (segTokens > cfg.maxTokens) {
       // Recursive split for oversized segment

package/src/session-observer-state.ts

@@ -14,7 +14,7 @@ interface SessionObserverCursor {
 }
 
 interface SessionObserverPersistedState {
-  version: 1;
+  version: 2;
   sessions: Record<string, SessionObserverCursor>;
 }
 
@@ -134,7 +134,7 @@ export class SessionObserverState {
     try {
       const raw = await readFile(this.statePath, "utf-8");
       const parsed = JSON.parse(raw) as SessionObserverPersistedState;
-      if (parsed?.version !== 1 || !parsed.sessions || typeof parsed.sessions !== "object") {
+      if (parsed?.version !== 2 || !parsed.sessions || typeof parsed.sessions !== "object") {
         return null;
       }
       return parsed;
@@ -246,7 +246,7 @@ export class SessionObserverState {
     for (const [key, value] of sessionsMap.entries()) {
       sessions[key] = value;
     }
-    const payload: SessionObserverPersistedState = { version: 1, sessions };
+    const payload: SessionObserverPersistedState = { version: 2, sessions };
     await mkdir(path.dirname(this.statePath), { recursive: true });
     await writeFile(this.statePath, JSON.stringify(payload, null, 2), "utf-8");
   }

package/src/testing/subjects/recall-budget.test.ts

@@ -9,7 +9,7 @@
  *
  * The nine rows are realized honestly against the coordinator's REAL behavior —
  * no mocks. Config is built through the production `parseConfig` path so the
- * budget derivation (recallBudgetChars = maxMemoryTokens * 4), section
+ * budget derivation (recallBudgetChars = maxMemoryTokens), section
  * enablement filtering, per-section maxChars caps, and cap-after-filter
  * ordering are all exercised as they ship. Each row asserts one recall-budget
  * contract invariant:
@@ -28,6 +28,7 @@ import assert from "node:assert/strict";
 
 import { parseConfig } from "../../config.js";
 import { RecallSectionCoordinator } from "../../orchestration/recall-section-coordinator.js";
+import { estimateTokenCount } from "../../token-estimate.js";
 import { createRecallSectionMetricRecorder, formatRecallSectionMetric } from "../../recall-qos.js";
 import { reorderRecallResultsWithMmr } from "../../recall-mmr.js";
 import { applyRuntimeRetrievalPolicy, expandQuery } from "../../retrieval.js";
@@ -297,10 +298,10 @@ const subject: LifecycleSubject<RecallBudgetState> = {
         return;
       }
       case "sparse-metadata-with-binding": {
-        // Budget is derived from maxMemoryTokens (recallBudgetChars = tokens*4).
+        // Budget keeps Latin-script headroom and token-aware assembly caps wide scripts.
         assert.equal(config.recallBudgetChars, config.maxMemoryTokens * 4, "budget derives from maxMemoryTokens");
-        assert.equal(budget, config.maxMemoryTokens * 4, "coordinator honors the token-derived budget");
-        assert.ok(assembled.finalChars <= budget, "section sum stays within the token-derived budget");
+        assert.ok(budget <= config.maxMemoryTokens * 4, "coordinator honors the character budget");
+        assert.ok(assembled.finalChars <= budget, "section sum stays within the character budget");
         assert.ok(assembled.truncated, "oversized content forces truncation");
         return;
       }
@@ -372,7 +373,22 @@ const subject: LifecycleSubject<RecallBudgetState> = {
         // truncateRecallSectionToBudget respects a hard per-section budget too.
         const truncated = coordinator.truncateRecallSectionToBudget(body("m", 200), 50);
         assert.equal(truncated.length, 50, "budget-aware truncation lands exactly on the per-section limit");
-        // A budget-skipped section reports as a debug-level "skip" metric — the
+        const tokenTrimmed = coordinator.truncateRecallSectionToBudget("x".repeat(200), 200, 20);
+        assert.match(tokenTrimmed, /\.\.\.\(memory context trimmed\)$/);
+        assert.ok(estimateTokenCount(tokenTrimmed) <= 20, "token-aware truncation keeps its marker inside the cap");
+        const markerBudget = estimateTokenCount("\n\n...(memory context trimmed)");
+        const markerOnlyBudget = coordinator.truncateRecallSectionToBudget("日本語".repeat(100), 200, markerBudget);
+        assert.ok(markerOnlyBudget.length > 0, "a marker-only budget keeps useful content");
+        assert.doesNotMatch(markerOnlyBudget, /\.\.\.\(memory context trimmed\)$/);
+        assert.ok(estimateTokenCount(markerOnlyBudget) <= markerBudget);
+        const wideScriptTrimmed = coordinator.truncateRecallSectionToBudget("日".repeat(100), 30, markerBudget);
+        assert.equal([...wideScriptTrimmed].length, 8, "wide-script fallback uses the available character room");
+        assert.ok(estimateTokenCount(wideScriptTrimmed) <= markerBudget);
+        assert.doesNotMatch(wideScriptTrimmed, /\.\.\.\(memory context trimmed\)$/);
+        const emojiTrimmed = coordinator.truncateRecallSectionToBudget("😀".repeat(30), 60, 29);
+        assert.ok(emojiTrimmed.length <= 60, "astral text stays within the UTF-16 character cap");
+        assert.ok(estimateTokenCount(emojiTrimmed) <= 29, "astral text stays within the token cap");
+        assert.match(emojiTrimmed, /\.\.\.\(memory context trimmed\)$/);
         // QoS accounting that flags a section dropped for the budget deadline.
         const skipMetric = formatRecallSectionMetric({
           section: "memories",

package/src/token-estimate.test.ts

@@ -0,0 +1,38 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { estimateTokenCount } from "./token-estimate.js";
+
+test("estimates English text at about four code points per token", () => {
+  assert.equal(estimateTokenCount("abcdefghijklmnop"), 4);
+});
+
+test("estimates Japanese text at about one token per code point", () => {
+  assert.equal(estimateTokenCount("日本語の文章です"), 8);
+});
+
+test("combines script-aware counts for mixed text", () => {
+  assert.equal(estimateTokenCount("abcd世界"), 3);
+});
+
+test("normalizes composed characters before counting code points", () => {
+  assert.equal(estimateTokenCount("e\u0301e\u0301e\u0301e\u0301"), 1);
+});
+
+test("counts Thai combining marks in wide-script text", () => {
+  assert.equal(estimateTokenCount("ที่"), 3);
+});
+
+test("counts token-dense Indic code points and emoji", () => {
+  assert.equal(estimateTokenCount("नमस्ते"), 6);
+  assert.equal(estimateTokenCount("😀😀😀😀"), 4);
+});
+
+test("counts unlisted letters and regional indicators conservatively", () => {
+  assert.equal(estimateTokenCount("ქართული"), 7);
+  assert.equal(estimateTokenCount("🇺🇸🇬🇧"), 4);
+});
+
+test("counts non-ASCII punctuation with token-dense text", () => {
+  assert.equal(estimateTokenCount("日。日。日。日。"), 8);
+});

package/src/token-estimate.ts

@@ -0,0 +1,22 @@
+const NARROW_LETTER = /[\p{Script=Latin}\p{Script=Greek}\p{Script=Cyrillic}]/u;
+
+/**
+ * Estimate model tokens without a tokenizer.
+ *
+ * Wide-script code points approximate one token each. Latin, Greek, Cyrillic,
+ * punctuation, and symbols use the established four-code-point heuristic.
+ * Unknown letters count conservatively as wide-script code points.
+ */
+export function estimateTokenCount(text: string): number {
+  let wideTokens = 0;
+  let narrowCodePoints = 0;
+  for (const char of text.normalize("NFC")) {
+    const codePoint = char.codePointAt(0) ?? 0;
+    if (codePoint <= 0x7f || NARROW_LETTER.test(char)) {
+      narrowCodePoints++;
+    } else {
+      wideTokens++;
+    }
+  }
+  return wideTokens + Math.ceil(narrowCodePoints / 4);
+}

package/src/transcript.test.ts

@@ -1,8 +1,9 @@
-import test from "node:test";
 import assert from "node:assert/strict";
+import { mkdtemp } from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
-import { mkdtemp } from "node:fs/promises";
+import test from "node:test";
+import { estimateTokenCount } from "./token-estimate.js";
 import { TranscriptManager } from "./transcript.js";
 import type { PluginConfig, TranscriptEntry } from "./types.js";
 
@@ -24,6 +25,40 @@ function entryAt(timestamp: string, sessionKey: string, turnId: string): Transcr
   return { timestamp, role: "user", content: `turn ${turnId}`, sessionKey, turnId };
 }
 
+test("formatForRecall counts transcript separators inside its token budget", () => {
+  const manager = new TranscriptManager(makeConfig("/tmp/remnic-transcript-format-test"));
+  const entries = ["one", "two", "three"].map((turnId) =>
+    entryAt("2026-06-29T12:00:00.000Z", "agent:test-agent:main", turnId)
+  );
+
+  const formatted = manager.formatForRecall(entries, 23);
+
+  assert.ok(estimateTokenCount(formatted) <= 23);
+});
+test("formatForRecall returns no content for zero or undersized budgets", () => {
+  const manager = new TranscriptManager(makeConfig("/tmp/remnic-transcript-small-budget"));
+  const entries = [
+    {
+      ...entryAt("2026-06-29T12:00:00.000Z", "agent:test-agent:main", "oversized"),
+      content: "日本語".repeat(100),
+    },
+  ];
+
+  assert.equal(manager.formatForRecall(entries, 0), "");
+  assert.equal(manager.formatForRecall(entries, 10), "");
+});
+test("formatForRecall does not fall back to stale older entries", () => {
+  const manager = new TranscriptManager(makeConfig("/tmp/remnic-transcript-newest-budget"));
+  const entries = [
+    entryAt("2026-06-29T12:00:00.000Z", "agent:test-agent:main", "older"),
+    {
+      ...entryAt("2026-06-29T12:00:00.000Z", "agent:test-agent:main", "newest"),
+      content: "日本語".repeat(100),
+    },
+  ];
+
+  assert.equal(manager.formatForRecall(entries, 23), "");
+});
 /**
  * Freeze `new Date()` / `Date.now()` to a fixed instant for the duration of `fn`.
  *
@@ -50,7 +85,7 @@ async function withFrozenNow<T>(now: Date, fn: () => Promise<T>): Promise<T> {
           args[3] as number,
           args[4] as number,
           args[5] as number,
-          args[6] as number,
+          args[6] as number
         );
       }
     }