@remnic/core 9.57.6 → 9.58.0

package/src/chunking.ts

@@ -5,6 +5,8 @@
  * Preserves coherent thoughts by never splitting mid-sentence.
  */
 
+import { estimateTokenCount } from "./token-estimate.js";
+
 export interface ChunkingConfig {
   /** Target tokens per chunk (default 200) */
   targetTokens: number;
@@ -37,14 +39,6 @@ export const DEFAULT_CHUNKING_CONFIG: ChunkingConfig = {
   overlapSentences: 2,
 };
 
-/**
- * Estimate token count for text.
- * Rough approximation: ~4 characters per token for English.
- */
-function estimateTokens(text: string): number {
-  return Math.ceil(text.length / 4);
-}
-
 /**
  * Split text into sentences.
  *
@@ -189,7 +183,7 @@ export function chunkContent(
   content: string,
   config: ChunkingConfig = DEFAULT_CHUNKING_CONFIG,
 ): ChunkResult {
-  const totalTokens = estimateTokens(content);
+  const totalTokens = estimateTokenCount(content);
 
   // Don't chunk if below minimum threshold
   if (totalTokens < config.minTokens) {
@@ -224,7 +218,7 @@ export function chunkContent(
 
   for (let i = 0; i < sentences.length; i++) {
     const sentence = sentences[i];
-    const sentenceTokens = estimateTokens(sentence);
+    const sentenceTokens = estimateTokenCount(sentence);
 
     // Add sentence to current chunk
     currentChunkSentences.push(sentence);
@@ -241,7 +235,7 @@ export function chunkContent(
       chunks.push({
         content: chunkContent,
         index: chunkIndex,
-        tokenCount: estimateTokens(chunkContent),
+        tokenCount: estimateTokenCount(chunkContent),
       });
       chunkIndex++;
 
@@ -256,7 +250,7 @@ export function chunkContent(
           currentTokens = 0;
         } else {
           currentChunkSentences = currentChunkSentences.slice(-overlapCount);
-          currentTokens = currentChunkSentences.reduce((sum, s) => sum + estimateTokens(s), 0);
+          currentTokens = currentChunkSentences.reduce((sum, s) => sum + estimateTokenCount(s), 0);
         }
       }
     }

package/src/config.ts

@@ -4508,6 +4508,8 @@ function buildRecallPipelineConfig(cfg: Record<string, unknown>): RecallPipeline
       ? Math.max(0, Math.floor(cfg.maxMemoryTokens))
       : 2000;
   const recallBudgetCharsRaw = clampNonNegativeNumber(cfg.recallBudgetChars);
+  // Keep four characters per token as Latin-script headroom. Assemblers enforce
+  // maxMemoryTokens with the shared script-aware estimator.
   const recallBudgetChars = recallBudgetCharsRaw ?? maxMemoryTokens * 4;
 
   const rawPipeline = cfg.recallPipeline;

package/src/connectors/codex-materialize.ts

@@ -28,8 +28,8 @@
  *  - **Idempotent no-ops.** A content hash is written into the sentinel. If
  *    the re-rendered hash matches the previous run, we skip writes entirely.
  *  - **Token budget.** `memory_summary.md` is truncated to fit under the
- *    configured token budget (whitespace-tokenized approximation), leaving
- *    headroom under Codex's 5000-token summary cap.
+ *    configured script-aware token estimate, leaving headroom under Codex's
+ *    5000-token summary cap.
  *
  * Privacy
  * ───────
@@ -47,6 +47,7 @@ import path from "node:path";
 import { log } from "../logger.js";
 import { readEnvVar, resolveHomeDir } from "../runtime/env.js";
 import type { MemoryFile } from "../types.js";
+import { estimateTokenCount } from "../token-estimate.js";
 import { expandTildePath } from "../utils/path.js";
 
 // ─── Types ─────────────────────────────────────────────────────────────────
@@ -60,7 +61,7 @@ export interface MaterializeOptions {
   memories: MemoryFile[];
   /** Override `<codex_home>`. Defaults to `$CODEX_HOME` or `~/.codex`. */
   codexHome?: string;
-  /** Maximum whitespace-tokenized size of memory_summary.md. Default 4500. */
+  /** Maximum estimated token size of memory_summary.md. Default 4500. */
   maxSummaryTokens?: number;
   /** Maximum age of rollout_summaries/*.md in days. Default 30. */
   rolloutRetentionDays?: number;
@@ -990,35 +991,23 @@ function trimHyphenEdges(value: string): string {
   while (end > start && value[end - 1] === "-") end -= 1;
   return value.slice(start, end);
 }
-
-/**
- * Whitespace-tokenized approximation used by the budget check. Matches the
- * simple heuristic Codex's usage.rs reporting uses for the "5000 token"
- * memory_summary cap.
- */
+/** Shared script-aware approximation used by the budget check. */
 export function approximateTokenCount(text: string): number {
-  const trimmed = text.trim();
-  if (trimmed.length === 0) return 0;
-  return trimmed.split(/\s+/u).length;
+  return estimateTokenCount(text);
 }
 
 /**
- * Truncate `text` so it fits under `maxTokens` whitespace tokens. We drop
- * trailing lines until we're under the budget and then append an ellipsis
- * marker so downstream readers can see that truncation happened.
+ * Truncate `text` so it fits under `maxTokens`. We drop trailing lines until
+ * under budget, then append a marker so readers can see that truncation occurred.
  */
 export function truncateToTokenBudget(text: string, maxTokens: number): string {
   if (maxTokens <= 0) return "";
   if (approximateTokenCount(text) <= maxTokens) return text;
 
-  // Reserve headroom for the truncation marker so the line-preserving path
-  // can actually fit the marker without flipping to the hard-cut fallback.
-  // Both markers are counted with the same whitespace heuristic the budget
-  // check uses, so the arithmetic stays consistent.
   const lineMarker = "_[truncated for summary budget]_";
   const tailMarker = "[truncated]";
+  if (approximateTokenCount(tailMarker) > maxTokens) return "";
   const lineMarkerTokens = approximateTokenCount(lineMarker);
-  const tailMarkerTokens = approximateTokenCount(tailMarker);
 
   const lines = text.split(/\r?\n/u);
   const lineBudget = Math.max(0, maxTokens - lineMarkerTokens);
@@ -1028,13 +1017,19 @@ export function truncateToTokenBudget(text: string, maxTokens: number): string {
   lines.push(lineMarker);
   let result = lines.join("\n");
 
-  // If a single huge line still blows the budget, hard-cut tokens. Reserve
-  // space for the tail marker's own token count so the final string stays
-  // within maxTokens rather than sneaking over by a few tokens.
   if (approximateTokenCount(result) > maxTokens) {
-    const tokens = result.split(/\s+/u);
-    const keep = Math.max(0, maxTokens - tailMarkerTokens);
-    result = keep > 0 ? `${tokens.slice(0, keep).join(" ")} ${tailMarker}` : tailMarker;
+    const codePoints = [...result];
+    let low = 0;
+    let high = codePoints.length;
+    while (low < high) {
+      const mid = Math.ceil((low + high) / 2);
+      const prefix = codePoints.slice(0, mid).join("");
+      const candidate = prefix.length > 0 ? `${prefix} ${tailMarker}` : tailMarker;
+      if (approximateTokenCount(candidate) <= maxTokens) low = mid;
+      else high = mid - 1;
+    }
+    const prefix = codePoints.slice(0, low).join("");
+    result = prefix.length > 0 ? `${prefix} ${tailMarker}` : tailMarker;
   }
   return result;
 }

package/src/lcm/archive.ts

@@ -1,10 +1,11 @@
 import type Database from "better-sqlite3";
 import { log } from "../logger.js";
 import {
-  parseMessageParts,
   type LcmMessagePartInput,
   type MessagePartSourceFormat,
+  parseMessageParts,
 } from "../message-parts/index.js";
+import { estimateTokenCount } from "../token-estimate.js";
 
 export interface LcmMessage {
   id: number;
@@ -48,10 +49,8 @@ export interface LcmStructuredRecallMatch {
   score: number;
 }
 
-/** Rough token count: ~4 chars per token. */
-export function estimateTokens(text: string): number {
-  return Math.ceil(text.length / 4);
-}
+/** Shared script-aware token estimate used by LCM accounting. */
+export const estimateTokens = estimateTokenCount;
 
 const PUBLIC_LCM_SEARCH_LIMIT = 100;
 

package/src/lcm/engine.ts

@@ -10,8 +10,8 @@ import {
 } from "./recall.js";
 import { LcmWorkQueue, type LcmObserveMessage } from "./queue.js";
 import type { PluginConfig } from "../types.js";
+import { estimateTokenCount } from "../token-estimate.js";
 import { log } from "../logger.js";
-
 export interface LcmEngineConfig {
   enabled: boolean;
   leafBatchSize: number;
@@ -556,12 +556,14 @@ export class LcmEngine {
     const messages = this.archive!.getMessages(normalizedSessionId, fromTurn, toTurn);
     if (messages.length === 0) return [];
 
-    // Enforce token budget — keep first and last, truncate middle
-    const maxChars = maxTokens * 4;
-    let totalChars = 0;
-    for (const m of messages) totalChars += m.content.length;
-
-    if (totalChars <= maxChars) {
+    // Enforce the token budget while keeping the existing first/last shape.
+    const tokenBudget = Math.max(0, Math.floor(maxTokens));
+    if (tokenBudget === 0) return [];
+    const totalTokens = messages.reduce(
+      (sum, message) => sum + estimateTokenCount(message.content),
+      0,
+    );
+    if (totalTokens <= tokenBudget) {
       return messages.map((m) => ({
         id: m.id,
         session_id: m.session_id,
@@ -571,45 +573,48 @@ export class LcmEngine {
       }));
     }
 
-    // Keep first and last messages, truncate from middle
     const result: LcmExpandedMessage[] = [];
-    let budget = maxChars;
-
-    // Reserve space for the last message
     const lastMsg = messages[messages.length - 1];
-    const lastMsgChars = Math.min(
-      lastMsg.content.length,
-      Math.floor(maxChars * 0.3),
+    const reservedLastTokens = Math.min(
+      estimateTokenCount(lastMsg.content),
+      Math.max(1, Math.floor(tokenBudget * 0.3)),
     );
-    budget -= lastMsgChars;
+    let remainingTokens = tokenBudget - reservedLastTokens;
 
-    // Add messages from the beginning
     for (let i = 0; i < messages.length - 1; i++) {
-      if (budget <= 0) break;
-      const m = messages[i];
-      const truncated = m.content.slice(0, budget);
+      if (remainingTokens <= 0) break;
+      const content = truncateToTokenBudget(
+        messages[i].content,
+        remainingTokens,
+      );
+      if (!content) continue;
       result.push({
-        id: m.id,
-        session_id: m.session_id,
-        turn_index: m.turn_index,
-        role: m.role,
-        content: truncated,
+        id: messages[i].id,
+        session_id: messages[i].session_id,
+        turn_index: messages[i].turn_index,
+        role: messages[i].role,
+        content,
       });
-      budget -= truncated.length;
+      remainingTokens -= estimateTokenCount(content);
     }
 
-    // Always append the last message
-    result.push({
-      id: lastMsg.id,
-      session_id: lastMsg.session_id,
-      turn_index: lastMsg.turn_index,
-      role: lastMsg.role,
-      content: lastMsg.content.slice(0, lastMsgChars + Math.max(0, budget)),
-    });
-
+    const lastContent = truncateToTokenBudget(
+      lastMsg.content,
+      reservedLastTokens + remainingTokens,
+    );
+    if (lastContent) {
+      result.push({
+        id: lastMsg.id,
+        session_id: lastMsg.session_id,
+        turn_index: lastMsg.turn_index,
+        role: lastMsg.role,
+        content: lastContent,
+      });
+    }
     return result;
   }
 
+
   /** Get statistics about the LCM archive. */
   async getStats(sessionId?: string): Promise<{
     totalMessages: number;
@@ -691,3 +696,19 @@ export class LcmEngine {
     this.initPromise = null;
   }
 }
+function truncateToTokenBudget(text: string, maxTokens: number): string {
+  if (maxTokens <= 0) return "";
+  if (estimateTokenCount(text) <= maxTokens) return text;
+  const codePoints = [...text];
+  let low = 0;
+  let high = codePoints.length;
+  while (low < high) {
+    const mid = Math.ceil((low + high) / 2);
+    if (estimateTokenCount(codePoints.slice(0, mid).join("")) <= maxTokens) {
+      low = mid;
+    } else {
+      high = mid - 1;
+    }
+  }
+  return codePoints.slice(0, low).join("");
+}

package/src/lcm/summarizer.ts

@@ -1,8 +1,9 @@
 import { log } from "../logger.js";
+import { looksLikeMechanicalTelemetryTranscript } from "../telemetry-transcript.js";
+import { estimateTokenCount } from "../token-estimate.js";
 import type { LcmArchive, LcmMessage } from "./archive.js";
-import { LcmDag } from "./dag.js";
 import { estimateTokens } from "./archive.js";
-import { looksLikeMechanicalTelemetryTranscript } from "../telemetry-transcript.js";
+import type { LcmDag } from "./dag.js";
 
 /** Generate a ULID-like ID (timestamp + random). */
 function generateNodeId(): string {
@@ -199,32 +200,42 @@ export class LcmSummarizer {
 
 /** Deterministic truncation: first and last sentence, plus middle truncation. */
 function deterministicTruncate(text: string, maxTokens: number): string {
-  const maxChars = maxTokens * 4;
-  if (text.length <= maxChars) return text;
+  if (maxTokens <= 0 || estimateTokenCount(text) <= maxTokens) return text;
 
   const sentences = text.split(/(?<=[.!?])\s+/).filter((s) => s.length > 0);
-  if (sentences.length <= 2) {
-    return text.slice(0, maxChars);
-  }
+  if (sentences.length <= 2) return clampToTokenBudget(text, maxTokens);
 
   const first = sentences[0];
   const last = sentences[sentences.length - 1];
-  const budget = maxChars - first.length - last.length - 20;
+  const render = (middle: string[], truncated: boolean): string =>
+    `${first} ${middle.join(" ")}${truncated ? " [...] " : " "}${last}`;
 
-  if (budget <= 0) {
-    return text.slice(0, maxChars);
-  }
+  const base = render([], false);
+  if (estimateTokenCount(base) > maxTokens) return clampToTokenBudget(text, maxTokens);
+  if (estimateTokenCount(render([], true)) > maxTokens) return base;
 
-  // Fill from the beginning until budget runs out
   const middle: string[] = [];
-  let used = 0;
   for (let i = 1; i < sentences.length - 1; i++) {
-    if (used + sentences[i].length > budget) break;
+    const candidate = [...middle, sentences[i]];
+    const truncated = i < sentences.length - 2;
+    if (estimateTokenCount(render(candidate, truncated)) > maxTokens) break;
     middle.push(sentences[i]);
-    used += sentences[i].length;
   }
 
-  return `${first} ${middle.join(" ")}${middle.length < sentences.length - 2 ? " [...] " : " "}${last}`;
+  const truncated = middle.length < sentences.length - 2;
+  return clampToTokenBudget(render(middle, truncated), maxTokens);
+}
+
+function clampToTokenBudget(text: string, maxTokens: number): string {
+  const codePoints = [...text];
+  let low = 0;
+  let high = codePoints.length;
+  while (low < high) {
+    const mid = Math.ceil((low + high) / 2);
+    if (estimateTokenCount(codePoints.slice(0, mid).join("")) <= maxTokens) low = mid;
+    else high = mid - 1;
+  }
+  return codePoints.slice(0, low).join("");
 }
 
 function chunkArray<T>(arr: T[], size: number): T[][] {

package/src/lcm-engine.test.ts

@@ -7,6 +7,7 @@ import test from "node:test";
 import { LcmArchive } from "./lcm/archive.js";
 import { LcmEngine, extractLcmConfig } from "./lcm/engine.js";
 import { openLcmDatabase } from "./lcm/schema.js";
+import { estimateTokenCount } from "./token-estimate.js";
 import type { PluginConfig } from "./types.js";
 
 function createPluginConfig(memoryDir: string): PluginConfig {
@@ -661,20 +662,99 @@ test("expandContext preserves first, middle, and last row identity when truncate
         session_id: "session-truncated",
         turn_index: 2,
         role: "assistant",
-        content: "BBBBBB",
+        content: "BBBBBBBB",
       },
       {
         id: last.id,
         session_id: "session-truncated",
         turn_index: 4,
         role: "assistant",
-        content: "DDDDDD",
+        content: "DDDD",
       },
     ]);
   } finally {
     await rm(memoryDir, { recursive: true, force: true });
   }
 });
+test("expandContext applies token budgets to wide-script messages", async () => {
+  const memoryDir = await mkdtemp(
+    path.join(os.tmpdir(), "engram-lcm-expand-wide-script-"),
+  );
+
+  try {
+    const engine = new LcmEngine(
+      createPluginConfig(memoryDir),
+      async () => "summary",
+    );
+    await engine.ensureInitialized();
+
+    const db = openLcmDatabase(memoryDir);
+    try {
+      const archive = new LcmArchive(db);
+      archive.appendMessages("session-wide-script", [
+        { turnIndex: 1, role: "user", content: "日本語".repeat(40) },
+        { turnIndex: 2, role: "assistant", content: "日本語".repeat(40) },
+        { turnIndex: 3, role: "user", content: "日本語".repeat(40) },
+      ]);
+    } finally {
+      db.close();
+    }
+
+    const expanded = await engine.expandContext(
+      "session-wide-script",
+      1,
+      3,
+      10,
+    );
+
+    assert.ok(
+      expanded.reduce((sum, message) => sum + estimateTokenCount(message.content), 0) <= 10,
+    );
+    assert.ok(expanded.every((message) => message.content.length < 40));
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+test("expandContext preserves the final message at the smallest positive budget", async () => {
+  const memoryDir = await mkdtemp(
+    path.join(os.tmpdir(), "engram-lcm-expand-minimum-budget-"),
+  );
+
+  let engine: LcmEngine | undefined;
+  try {
+    engine = new LcmEngine(
+      createPluginConfig(memoryDir),
+      async () => "summary",
+    );
+    await engine.ensureInitialized();
+
+    const db = openLcmDatabase(memoryDir);
+    try {
+      const archive = new LcmArchive(db);
+      archive.appendMessages("session-minimum-budget", [
+        { turnIndex: 1, role: "user", content: "first message" },
+        { turnIndex: 2, role: "assistant", content: "最後のメッセージ" },
+      ]);
+    } finally {
+      db.close();
+    }
+
+    const expanded = await engine.expandContext(
+      "session-minimum-budget",
+      1,
+      2,
+      1,
+    );
+
+    assert.equal(expanded.at(-1)?.turn_index, 2);
+    assert.ok(
+      expanded.reduce((sum, message) => sum + estimateTokenCount(message.content), 0) <= 1,
+    );
+  } finally {
+    engine?.close();
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
 
 test("assembleRecallWithTrace preserves text and reports selected summary metadata", async () => {
   const memoryDir = await mkdtemp(

package/src/local-llm.ts

@@ -1,4 +1,5 @@
 import { log } from "./logger.js";
+import { estimateTokenCount } from "./token-estimate.js";
 import type { PluginConfig } from "./types.js";
 import fs from "node:fs";
 import os from "node:os";
@@ -1431,18 +1432,15 @@ export class LocalLlmClient {
 
     return await this.runChatCompletionRequest(messages, options);
   }
-
   /**
-   * Estimate tokens when local LLM doesn't return usage stats
-   * Rough estimate: 1 token ≈ 4 characters
+   * Estimate tokens when local LLM does not return usage stats.
    */
   private estimateTokens(
     messages: Array<{ role: string; content: string }>,
     response: string
   ): { promptTokens: number; completionTokens: number; totalTokens: number } {
-    const promptChars = messages.reduce((sum, m) => sum + m.content.length, 0);
-    const promptTokens = Math.ceil(promptChars / 4);
-    const completionTokens = Math.ceil(response.length / 4);
+    const promptTokens = messages.reduce((sum, message) => sum + estimateTokenCount(message.content), 0);
+    const completionTokens = estimateTokenCount(response);
 
     return {
       promptTokens,

package/src/memory-extension-host/render-extensions-block.ts

@@ -6,16 +6,10 @@
  * truncates with a footer listing omitted extensions when over budget.
  */
 
+import { estimateTokenCount } from "../token-estimate.js";
 import { REMNIC_EXTENSIONS_TOTAL_TOKEN_LIMIT } from "./host-discovery.js";
 import type { DiscoveredExtension } from "./types.js";
 
-/**
- * Approximate token count using the 4 chars per token heuristic.
- */
-function estimateTokens(text: string): number {
-  return Math.ceil(text.length / 4);
-}
-
 /**
  * Render a markdown block containing extension instructions for injection
  * into consolidation prompts.
@@ -36,14 +30,14 @@ extension produces or curates.
 `;
 
   let budget = REMNIC_EXTENSIONS_TOTAL_TOKEN_LIMIT;
-  budget -= estimateTokens(header);
+  budget -= estimateTokenCount(header);
 
   const inlined: string[] = [];
   const omitted: string[] = [];
 
   for (const ext of extensions) {
     const block = `### remnic-extension/${ext.name}\n\`\`\`\n${ext.instructions}\n\`\`\`\n\n`;
-    const cost = estimateTokens(block);
+    const cost = estimateTokenCount(block);
     if (cost <= budget) {
       inlined.push(block);
       budget -= cost;