@remnic/core 9.57.5 → 9.58.0

package/src/chunking.ts

@@ -5,6 +5,8 @@
  * Preserves coherent thoughts by never splitting mid-sentence.
  */
 
+import { estimateTokenCount } from "./token-estimate.js";
+
 export interface ChunkingConfig {
   /** Target tokens per chunk (default 200) */
   targetTokens: number;
@@ -38,60 +40,137 @@ export const DEFAULT_CHUNKING_CONFIG: ChunkingConfig = {
 };
 
 /**
- * Estimate token count for text.
- * Rough approximation: ~4 characters per token for English.
+ * Split text into sentences.
+ *
+ * The scan stays linear and does not use a backtracking regular expression.
+ * ASCII punctuation keeps its old whitespace rule. Unicode terminators also
+ * split when the next sentence starts immediately, as in CJK text.
  */
-function estimateTokens(text: string): number {
-  return Math.ceil(text.length / 4);
+const UNICODE_SENTENCE_TERMINATORS = "。．！？؟۔।॥｡…";
+const ASCII_SENTENCE_TERMINATORS = ".!?";
+const CJK_NO_SPACE_TERMINATORS = "。．！？｡";
+const CLOSING_PUNCTUATION = "\"'”’»」』）］】〉》)]}";
+const DIGITS = "0123456789０１２３４５６７８９";
+const SENTENCE_SEGMENTER =
+  typeof Intl.Segmenter === "function"
+    ? new Intl.Segmenter(undefined, { granularity: "sentence" })
+    : undefined;
+
+function isSentenceTerminator(character: string): boolean {
+  return (
+    ASCII_SENTENCE_TERMINATORS.includes(character) ||
+    UNICODE_SENTENCE_TERMINATORS.includes(character)
+  );
 }
 
-/**
- * Split text into sentences.
- * Handles common abbreviations and edge cases.
- */
-function splitSentences(text: string): string[] {
-  // Split on sentence-ending punctuation (. ! ?) that is followed by whitespace
-  // or end of string; the punctuation stays with the sentence.
-  //
-  // Implemented as a single linear scan rather than a regex. Every regex form of
-  // this split is either polynomial (CodeQL js/polynomial-redos) or — once
-  // bounded/anchored to satisfy CodeQL — mishandles long runs or non-boundary
-  // punctuation (a global match silently drops a skipped prefix; a sticky match
-  // stops at the first interior `.` that is not a real boundary, e.g. "v1.2.3"
-  // or "example.com", emitting the whole document as one chunk). A character
-  // scan is O(n), allocation-free, drops nothing, and treats interior
-  // punctuation correctly. Normal prose splits identically to the previous
-  // /[^.!?]*[.!?]+(?:\s+|$)/g form.
-  const sentences: string[] = [];
+function isCjk(character: string): boolean {
+  return /[\u3000-\u9fff\uf900-\ufaff\uac00-\ud7af]/u.test(character);
+}
+
+type SentenceList = string[] & { separators: string[] };
+
+function splitSentencesFallback(text: string): SentenceList {
+  const sentences = [] as unknown as SentenceList;
+  Object.defineProperty(sentences, "separators", {
+    value: [],
+    writable: true,
+  });
   let start = 0;
+  let separator = "";
   for (let i = 0; i < text.length; i++) {
     const ch = text[i];
-    if (ch !== "." && ch !== "!" && ch !== "?") continue;
-    // Consume a run of terminators (e.g. "?!", "...").
+    if (!isSentenceTerminator(ch)) continue;
+
     let end = i;
-    while (end + 1 < text.length) {
-      const n = text[end + 1];
-      if (n !== "." && n !== "!" && n !== "?") break;
+    let hasUnicodeTerminator = UNICODE_SENTENCE_TERMINATORS.includes(ch);
+    while (end + 1 < text.length && isSentenceTerminator(text[end + 1])) {
       end++;
+      hasUnicodeTerminator ||= UNICODE_SENTENCE_TERMINATORS.includes(text[end]);
     }
+
+    const isFullwidthDecimal =
+      ch === "．" &&
+      DIGITS.includes(text[i - 1] ?? "") &&
+      DIGITS.includes(text[end + 1] ?? "");
+    while (end + 1 < text.length && CLOSING_PUNCTUATION.includes(text[end + 1])) {
+      end++;
+    }
+
     const after = text[end + 1];
-    // A real boundary only if the terminator run ends the string or is followed
-    // by whitespace. Interior punctuation (no following whitespace) is left in
-    // place and the scan continues.
-    if (after === undefined || /\s/.test(after)) {
+    const boundary =
+      !isFullwidthDecimal &&
+      (hasUnicodeTerminator || after === undefined || /\s/u.test(after));
+    if (boundary) {
       const sentence = text.slice(start, end + 1).trim();
-      if (sentence.length > 0) sentences.push(sentence);
-      start = end + 1;
+      if (sentence.length > 0) {
+        sentences.push(sentence);
+        if (sentences.length > 1) sentences.separators.push(separator);
+      }
+      let nextStart = end + 1;
+      while (nextStart < text.length && /\s/u.test(text[nextStart])) nextStart++;
+      separator = text.slice(end + 1, nextStart);
+      start = nextStart;
     }
     i = end;
   }
-  // Trailing text without a closing terminator.
-  if (start < text.length) {
-    const remaining = text.slice(start).trim();
-    if (remaining.length > 0) sentences.push(remaining);
+
+  const remaining = text.slice(start).trim();
+  if (remaining.length > 0) {
+    sentences.push(remaining);
+    if (sentences.length > 1) sentences.separators.push(separator);
   }
   return sentences;
 }
+export function splitSentences(text: string): string[] {
+  const canUseSegmenter =
+    SENTENCE_SEGMENTER !== undefined &&
+    !/\s/u.test(text) &&
+    [...text].some((character) => UNICODE_SENTENCE_TERMINATORS.includes(character));
+  if (canUseSegmenter) {
+    const segments = Array.from(SENTENCE_SEGMENTER.segment(text), ({ segment }) =>
+      segment.trim(),
+    ).filter((segment) => segment.length > 0) as SentenceList;
+    if (
+      segments.length > 1 &&
+      segments.slice(0, -1).every((segment) =>
+        [...segment].some((character) =>
+          UNICODE_SENTENCE_TERMINATORS.includes(character),
+        ),
+      )
+    ) {
+      Object.defineProperty(segments, "separators", {
+        value: Array(segments.length - 1).fill(""),
+        writable: true,
+      });
+      return segments;
+    }
+  }
+  return splitSentencesFallback(text);
+}
+
+function endsWithoutSpace(text: string): boolean {
+  let index = text.length - 1;
+  while (index >= 0 && CLOSING_PUNCTUATION.includes(text[index])) index--;
+  const terminator = text[index] ?? "";
+  if (CJK_NO_SPACE_TERMINATORS.includes(terminator)) {
+    return isCjk(text[index - 1] ?? "");
+  }
+  return terminator === "…" && isCjk(text[index - 1] ?? "");
+}
+export function joinSentences(sentences: string[]): string {
+  let result = "";
+  const separators = (sentences as Partial<SentenceList>).separators;
+  for (let i = 0; i < sentences.length; i++) {
+    const sentence = sentences[i];
+    if (result.length > 0) {
+      const separator = separators?.[i - 1];
+      result += separator ?? (endsWithoutSpace(result) ? "" : " ");
+    }
+    result += sentence;
+  }
+  return result;
+}
+
 
 /**
  * Chunk content into overlapping segments at sentence boundaries.
@@ -104,7 +183,7 @@ export function chunkContent(
   content: string,
   config: ChunkingConfig = DEFAULT_CHUNKING_CONFIG,
 ): ChunkResult {
-  const totalTokens = estimateTokens(content);
+  const totalTokens = estimateTokenCount(content);
 
   // Don't chunk if below minimum threshold
   if (totalTokens < config.minTokens) {
@@ -139,7 +218,7 @@ export function chunkContent(
 
   for (let i = 0; i < sentences.length; i++) {
     const sentence = sentences[i];
-    const sentenceTokens = estimateTokens(sentence);
+    const sentenceTokens = estimateTokenCount(sentence);
 
     // Add sentence to current chunk
     currentChunkSentences.push(sentence);
@@ -152,11 +231,11 @@ export function chunkContent(
 
     if (atTarget || isLastSentence) {
       // Create chunk from accumulated sentences
-      const chunkContent = currentChunkSentences.join(" ");
+      const chunkContent = joinSentences(currentChunkSentences);
       chunks.push({
         content: chunkContent,
         index: chunkIndex,
-        tokenCount: estimateTokens(chunkContent),
+        tokenCount: estimateTokenCount(chunkContent),
       });
       chunkIndex++;
 
@@ -171,7 +250,7 @@ export function chunkContent(
           currentTokens = 0;
         } else {
           currentChunkSentences = currentChunkSentences.slice(-overlapCount);
-          currentTokens = currentChunkSentences.reduce((sum, s) => sum + estimateTokens(s), 0);
+          currentTokens = currentChunkSentences.reduce((sum, s) => sum + estimateTokenCount(s), 0);
         }
       }
     }
@@ -216,14 +295,14 @@ export function reassembleChunks(chunks: string[]): string {
       const prevEnd = prevSentences.slice(-(j + 1));
       const currStart = currSentences.slice(0, j + 1);
 
-      if (prevEnd.join(" ") === currStart.join(" ")) {
+      if (joinSentences(prevEnd) === joinSentences(currStart)) {
         overlapCount = j + 1;
       }
     }
 
     // Add non-overlapping portion
     if (overlapCount > 0 && overlapCount < currSentences.length) {
-      result.push(currSentences.slice(overlapCount).join(" "));
+      result.push(joinSentences(currSentences.slice(overlapCount)));
     } else if (overlapCount === 0) {
       // No detected overlap, add full chunk
       result.push(currChunk);
@@ -231,5 +310,5 @@ export function reassembleChunks(chunks: string[]): string {
     // If overlapCount === currSentences.length, skip (fully contained)
   }
 
-  return result.join(" ");
+  return joinSentences(result);
 }

package/src/config.ts

@@ -4508,6 +4508,8 @@ function buildRecallPipelineConfig(cfg: Record<string, unknown>): RecallPipeline
       ? Math.max(0, Math.floor(cfg.maxMemoryTokens))
       : 2000;
   const recallBudgetCharsRaw = clampNonNegativeNumber(cfg.recallBudgetChars);
+  // Keep four characters per token as Latin-script headroom. Assemblers enforce
+  // maxMemoryTokens with the shared script-aware estimator.
   const recallBudgetChars = recallBudgetCharsRaw ?? maxMemoryTokens * 4;
 
   const rawPipeline = cfg.recallPipeline;

package/src/connectors/codex-materialize.ts

@@ -28,8 +28,8 @@
  *  - **Idempotent no-ops.** A content hash is written into the sentinel. If
  *    the re-rendered hash matches the previous run, we skip writes entirely.
  *  - **Token budget.** `memory_summary.md` is truncated to fit under the
- *    configured token budget (whitespace-tokenized approximation), leaving
- *    headroom under Codex's 5000-token summary cap.
+ *    configured script-aware token estimate, leaving headroom under Codex's
+ *    5000-token summary cap.
  *
  * Privacy
  * ───────
@@ -47,6 +47,7 @@ import path from "node:path";
 import { log } from "../logger.js";
 import { readEnvVar, resolveHomeDir } from "../runtime/env.js";
 import type { MemoryFile } from "../types.js";
+import { estimateTokenCount } from "../token-estimate.js";
 import { expandTildePath } from "../utils/path.js";
 
 // ─── Types ─────────────────────────────────────────────────────────────────
@@ -60,7 +61,7 @@ export interface MaterializeOptions {
   memories: MemoryFile[];
   /** Override `<codex_home>`. Defaults to `$CODEX_HOME` or `~/.codex`. */
   codexHome?: string;
-  /** Maximum whitespace-tokenized size of memory_summary.md. Default 4500. */
+  /** Maximum estimated token size of memory_summary.md. Default 4500. */
   maxSummaryTokens?: number;
   /** Maximum age of rollout_summaries/*.md in days. Default 30. */
   rolloutRetentionDays?: number;
@@ -990,35 +991,23 @@ function trimHyphenEdges(value: string): string {
   while (end > start && value[end - 1] === "-") end -= 1;
   return value.slice(start, end);
 }
-
-/**
- * Whitespace-tokenized approximation used by the budget check. Matches the
- * simple heuristic Codex's usage.rs reporting uses for the "5000 token"
- * memory_summary cap.
- */
+/** Shared script-aware approximation used by the budget check. */
 export function approximateTokenCount(text: string): number {
-  const trimmed = text.trim();
-  if (trimmed.length === 0) return 0;
-  return trimmed.split(/\s+/u).length;
+  return estimateTokenCount(text);
 }
 
 /**
- * Truncate `text` so it fits under `maxTokens` whitespace tokens. We drop
- * trailing lines until we're under the budget and then append an ellipsis
- * marker so downstream readers can see that truncation happened.
+ * Truncate `text` so it fits under `maxTokens`. We drop trailing lines until
+ * under budget, then append a marker so readers can see that truncation occurred.
  */
 export function truncateToTokenBudget(text: string, maxTokens: number): string {
   if (maxTokens <= 0) return "";
   if (approximateTokenCount(text) <= maxTokens) return text;
 
-  // Reserve headroom for the truncation marker so the line-preserving path
-  // can actually fit the marker without flipping to the hard-cut fallback.
-  // Both markers are counted with the same whitespace heuristic the budget
-  // check uses, so the arithmetic stays consistent.
   const lineMarker = "_[truncated for summary budget]_";
   const tailMarker = "[truncated]";
+  if (approximateTokenCount(tailMarker) > maxTokens) return "";
   const lineMarkerTokens = approximateTokenCount(lineMarker);
-  const tailMarkerTokens = approximateTokenCount(tailMarker);
 
   const lines = text.split(/\r?\n/u);
   const lineBudget = Math.max(0, maxTokens - lineMarkerTokens);
@@ -1028,13 +1017,19 @@ export function truncateToTokenBudget(text: string, maxTokens: number): string {
   lines.push(lineMarker);
   let result = lines.join("\n");
 
-  // If a single huge line still blows the budget, hard-cut tokens. Reserve
-  // space for the tail marker's own token count so the final string stays
-  // within maxTokens rather than sneaking over by a few tokens.
   if (approximateTokenCount(result) > maxTokens) {
-    const tokens = result.split(/\s+/u);
-    const keep = Math.max(0, maxTokens - tailMarkerTokens);
-    result = keep > 0 ? `${tokens.slice(0, keep).join(" ")} ${tailMarker}` : tailMarker;
+    const codePoints = [...result];
+    let low = 0;
+    let high = codePoints.length;
+    while (low < high) {
+      const mid = Math.ceil((low + high) / 2);
+      const prefix = codePoints.slice(0, mid).join("");
+      const candidate = prefix.length > 0 ? `${prefix} ${tailMarker}` : tailMarker;
+      if (approximateTokenCount(candidate) <= maxTokens) low = mid;
+      else high = mid - 1;
+    }
+    const prefix = codePoints.slice(0, low).join("");
+    result = prefix.length > 0 ? `${prefix} ${tailMarker}` : tailMarker;
   }
   return result;
 }

package/src/lcm/archive.ts

@@ -1,10 +1,11 @@
 import type Database from "better-sqlite3";
 import { log } from "../logger.js";
 import {
-  parseMessageParts,
   type LcmMessagePartInput,
   type MessagePartSourceFormat,
+  parseMessageParts,
 } from "../message-parts/index.js";
+import { estimateTokenCount } from "../token-estimate.js";
 
 export interface LcmMessage {
   id: number;
@@ -48,10 +49,8 @@ export interface LcmStructuredRecallMatch {
   score: number;
 }
 
-/** Rough token count: ~4 chars per token. */
-export function estimateTokens(text: string): number {
-  return Math.ceil(text.length / 4);
-}
+/** Shared script-aware token estimate used by LCM accounting. */
+export const estimateTokens = estimateTokenCount;
 
 const PUBLIC_LCM_SEARCH_LIMIT = 100;
 

package/src/lcm/engine.ts

@@ -10,8 +10,8 @@ import {
 } from "./recall.js";
 import { LcmWorkQueue, type LcmObserveMessage } from "./queue.js";
 import type { PluginConfig } from "../types.js";
+import { estimateTokenCount } from "../token-estimate.js";
 import { log } from "../logger.js";
-
 export interface LcmEngineConfig {
   enabled: boolean;
   leafBatchSize: number;
@@ -556,12 +556,14 @@ export class LcmEngine {
     const messages = this.archive!.getMessages(normalizedSessionId, fromTurn, toTurn);
     if (messages.length === 0) return [];
 
-    // Enforce token budget — keep first and last, truncate middle
-    const maxChars = maxTokens * 4;
-    let totalChars = 0;
-    for (const m of messages) totalChars += m.content.length;
-
-    if (totalChars <= maxChars) {
+    // Enforce the token budget while keeping the existing first/last shape.
+    const tokenBudget = Math.max(0, Math.floor(maxTokens));
+    if (tokenBudget === 0) return [];
+    const totalTokens = messages.reduce(
+      (sum, message) => sum + estimateTokenCount(message.content),
+      0,
+    );
+    if (totalTokens <= tokenBudget) {
       return messages.map((m) => ({
         id: m.id,
         session_id: m.session_id,
@@ -571,45 +573,48 @@ export class LcmEngine {
       }));
     }
 
-    // Keep first and last messages, truncate from middle
     const result: LcmExpandedMessage[] = [];
-    let budget = maxChars;
-
-    // Reserve space for the last message
     const lastMsg = messages[messages.length - 1];
-    const lastMsgChars = Math.min(
-      lastMsg.content.length,
-      Math.floor(maxChars * 0.3),
+    const reservedLastTokens = Math.min(
+      estimateTokenCount(lastMsg.content),
+      Math.max(1, Math.floor(tokenBudget * 0.3)),
     );
-    budget -= lastMsgChars;
+    let remainingTokens = tokenBudget - reservedLastTokens;
 
-    // Add messages from the beginning
     for (let i = 0; i < messages.length - 1; i++) {
-      if (budget <= 0) break;
-      const m = messages[i];
-      const truncated = m.content.slice(0, budget);
+      if (remainingTokens <= 0) break;
+      const content = truncateToTokenBudget(
+        messages[i].content,
+        remainingTokens,
+      );
+      if (!content) continue;
       result.push({
-        id: m.id,
-        session_id: m.session_id,
-        turn_index: m.turn_index,
-        role: m.role,
-        content: truncated,
+        id: messages[i].id,
+        session_id: messages[i].session_id,
+        turn_index: messages[i].turn_index,
+        role: messages[i].role,
+        content,
       });
-      budget -= truncated.length;
+      remainingTokens -= estimateTokenCount(content);
     }
 
-    // Always append the last message
-    result.push({
-      id: lastMsg.id,
-      session_id: lastMsg.session_id,
-      turn_index: lastMsg.turn_index,
-      role: lastMsg.role,
-      content: lastMsg.content.slice(0, lastMsgChars + Math.max(0, budget)),
-    });
-
+    const lastContent = truncateToTokenBudget(
+      lastMsg.content,
+      reservedLastTokens + remainingTokens,
+    );
+    if (lastContent) {
+      result.push({
+        id: lastMsg.id,
+        session_id: lastMsg.session_id,
+        turn_index: lastMsg.turn_index,
+        role: lastMsg.role,
+        content: lastContent,
+      });
+    }
     return result;
   }
 
+
   /** Get statistics about the LCM archive. */
   async getStats(sessionId?: string): Promise<{
     totalMessages: number;
@@ -691,3 +696,19 @@ export class LcmEngine {
     this.initPromise = null;
   }
 }
+function truncateToTokenBudget(text: string, maxTokens: number): string {
+  if (maxTokens <= 0) return "";
+  if (estimateTokenCount(text) <= maxTokens) return text;
+  const codePoints = [...text];
+  let low = 0;
+  let high = codePoints.length;
+  while (low < high) {
+    const mid = Math.ceil((low + high) / 2);
+    if (estimateTokenCount(codePoints.slice(0, mid).join("")) <= maxTokens) {
+      low = mid;
+    } else {
+      high = mid - 1;
+    }
+  }
+  return codePoints.slice(0, low).join("");
+}

package/src/lcm/summarizer.ts

@@ -1,8 +1,9 @@
 import { log } from "../logger.js";
+import { looksLikeMechanicalTelemetryTranscript } from "../telemetry-transcript.js";
+import { estimateTokenCount } from "../token-estimate.js";
 import type { LcmArchive, LcmMessage } from "./archive.js";
-import { LcmDag } from "./dag.js";
 import { estimateTokens } from "./archive.js";
-import { looksLikeMechanicalTelemetryTranscript } from "../telemetry-transcript.js";
+import type { LcmDag } from "./dag.js";
 
 /** Generate a ULID-like ID (timestamp + random). */
 function generateNodeId(): string {
@@ -199,32 +200,42 @@ export class LcmSummarizer {
 
 /** Deterministic truncation: first and last sentence, plus middle truncation. */
 function deterministicTruncate(text: string, maxTokens: number): string {
-  const maxChars = maxTokens * 4;
-  if (text.length <= maxChars) return text;
+  if (maxTokens <= 0 || estimateTokenCount(text) <= maxTokens) return text;
 
   const sentences = text.split(/(?<=[.!?])\s+/).filter((s) => s.length > 0);
-  if (sentences.length <= 2) {
-    return text.slice(0, maxChars);
-  }
+  if (sentences.length <= 2) return clampToTokenBudget(text, maxTokens);
 
   const first = sentences[0];
   const last = sentences[sentences.length - 1];
-  const budget = maxChars - first.length - last.length - 20;
+  const render = (middle: string[], truncated: boolean): string =>
+    `${first} ${middle.join(" ")}${truncated ? " [...] " : " "}${last}`;
 
-  if (budget <= 0) {
-    return text.slice(0, maxChars);
-  }
+  const base = render([], false);
+  if (estimateTokenCount(base) > maxTokens) return clampToTokenBudget(text, maxTokens);
+  if (estimateTokenCount(render([], true)) > maxTokens) return base;
 
-  // Fill from the beginning until budget runs out
   const middle: string[] = [];
-  let used = 0;
   for (let i = 1; i < sentences.length - 1; i++) {
-    if (used + sentences[i].length > budget) break;
+    const candidate = [...middle, sentences[i]];
+    const truncated = i < sentences.length - 2;
+    if (estimateTokenCount(render(candidate, truncated)) > maxTokens) break;
     middle.push(sentences[i]);
-    used += sentences[i].length;
   }
 
-  return `${first} ${middle.join(" ")}${middle.length < sentences.length - 2 ? " [...] " : " "}${last}`;
+  const truncated = middle.length < sentences.length - 2;
+  return clampToTokenBudget(render(middle, truncated), maxTokens);
+}
+
+function clampToTokenBudget(text: string, maxTokens: number): string {
+  const codePoints = [...text];
+  let low = 0;
+  let high = codePoints.length;
+  while (low < high) {
+    const mid = Math.ceil((low + high) / 2);
+    if (estimateTokenCount(codePoints.slice(0, mid).join("")) <= maxTokens) low = mid;
+    else high = mid - 1;
+  }
+  return codePoints.slice(0, low).join("");
 }
 
 function chunkArray<T>(arr: T[], size: number): T[][] {