@martian-engineering/lossless-claw 0.7.0 → 0.8.0

package/src/engine.ts

@@ -63,6 +63,7 @@ import {
 import { SummaryStore } from "./store/summary-store.js";
 import { createLcmSummarizeFromLegacyParams, LcmProviderAuthError } from "./summarize.js";
 import type { LcmDependencies } from "./types.js";
+import { estimateTokens } from "./estimate-tokens.js";
 
 type AgentMessage = Parameters<ContextEngine["ingest"]>[0]["message"];
 type AssembleResultWithSystemPrompt = AssembleResult & { systemPromptAddition?: string };
@@ -124,11 +125,6 @@ const DYNAMIC_ACTIVITY_HIGH_DOWNSHIFT_FACTOR = 0.75;
 
 // ── Helpers ──────────────────────────────────────────────────────────────────
 
-/** Rough token estimate: ~4 chars per token. */
-function estimateTokens(text: string): number {
-  return Math.ceil(text.length / 4);
-}
-
 function toJson(value: unknown): string {
   const encoded = JSON.stringify(value);
   return typeof encoded === "string" ? encoded : "";
@@ -138,6 +134,10 @@ function safeString(value: unknown): string | undefined {
   return typeof value === "string" ? value : undefined;
 }
 
+function formatDurationMs(durationMs: number): string {
+  return `${durationMs}ms`;
+}
+
 function asRecord(value: unknown): Record<string, unknown> | undefined {
   return value && typeof value === "object" && !Array.isArray(value)
     ? (value as Record<string, unknown>)
@@ -1235,13 +1235,14 @@ export class LcmContextEngine implements ContextEngine {
     this.statelessSessionPatterns = compileSessionPatterns(this.config.statelessSessionPatterns);
     this.db = database;
 
-    this.fts5Available = getLcmDbFeatures(this.db).fts5Available;
-
     // Run migrations eagerly at construction time so the schema exists
     // before any lifecycle hook fires.
     let migrationOk = false;
+    const migrationStartedAt = Date.now();
     try {
-      runLcmMigrations(this.db, { fts5Available: this.fts5Available });
+      runLcmMigrations(this.db, {
+        log: this.deps.log,
+      });
       this.migrated = true;
 
       // Verify tables were actually created
@@ -1254,16 +1255,21 @@ export class LcmContextEngine implements ContextEngine {
         );
       } else {
         migrationOk = true;
+        this.deps.log.info(
+          `[lcm] Migration run completed during engine init: duration=${formatDurationMs(Date.now() - migrationStartedAt)} fts5=${this.fts5Available}`,
+        );
         this.deps.log.debug(
           `[lcm] Migration successful — ${tables.length} tables: ${tables.map((t) => t.name).join(", ")}`,
         );
       }
     } catch (err) {
       this.deps.log.error(
-        `[lcm] Migration failed: ${err instanceof Error ? err.message : String(err)}`,
+        `[lcm] Migration failed after ${formatDurationMs(Date.now() - migrationStartedAt)}: ${err instanceof Error ? err.message : String(err)}`,
       );
     }
 
+    this.fts5Available = getLcmDbFeatures(this.db).fts5Available;
+
     // Only claim ownership of compaction when the DB is operational.
     // Without a working schema, ownsCompaction would disable the runtime's
     // built-in compaction safeguard and inflate the context budget.
@@ -1428,17 +1434,29 @@ export class LcmContextEngine implements ContextEngine {
     if (this.migrated) {
       return;
     }
-    runLcmMigrations(this.db, { fts5Available: this.fts5Available });
+    const migrationStartedAt = Date.now();
+    this.deps.log.info("[lcm] ensureMigrated: running migrations lazily");
+    runLcmMigrations(this.db, {
+      log: this.deps.log,
+    });
     this.migrated = true;
+    this.deps.log.info(
+      `[lcm] ensureMigrated: completed in ${formatDurationMs(Date.now() - migrationStartedAt)}`,
+    );
   }
 
   /**
    * Serialize mutating operations per stable session identity to prevent
    * ingest/compaction races across runtime UUID recycling.
    */
-  private async withSessionQueue<T>(queueKey: string, operation: () => Promise<T>): Promise<T> {
+  private async withSessionQueue<T>(
+    queueKey: string,
+    operation: () => Promise<T>,
+    options?: { operationName?: string; context?: string },
+  ): Promise<T> {
     const entry = this.sessionOperationQueues.get(queueKey);
     const previous = entry?.promise ?? Promise.resolve();
+    const queuedAhead = entry?.refCount ?? 0;
     let releaseQueue: () => void = () => {};
     const current = new Promise<void>((resolve) => {
       releaseQueue = resolve;
@@ -1452,7 +1470,15 @@ export class LcmContextEngine implements ContextEngine {
       this.sessionOperationQueues.set(queueKey, { promise: next, refCount: 1 });
     }
 
+    const waitStartedAt = Date.now();
     await previous.catch(() => {});
+    const waitMs = Date.now() - waitStartedAt;
+    if (options?.operationName) {
+      const detail = options.context ? ` ${options.context}` : "";
+      this.deps.log.info(
+        `[lcm] ${options.operationName}: session queue acquired queueKey=${queueKey} queuedAhead=${queuedAhead} wait=${formatDurationMs(waitMs)}${detail}`,
+      );
+    }
     try {
       return await operation();
     } finally {
@@ -2420,12 +2446,24 @@ export class LcmContextEngine implements ContextEngine {
     hasOverlap: boolean;
   }> {
     const { sessionId, conversationId, historicalMessages } = params;
+    const startedAt = Date.now();
+    const sessionContext = this.formatSessionLogContext({
+      conversationId,
+      sessionId,
+      sessionKey: params.sessionKey,
+    });
     if (historicalMessages.length === 0) {
+      this.deps.log.info(
+        `[lcm] reconcileSessionTail: skipped for ${sessionContext} duration=${formatDurationMs(Date.now() - startedAt)} historicalMessages=0 reason=empty-history`,
+      );
       return { blockedByImportCap: false, importedMessages: 0, hasOverlap: false };
     }
 
     const latestDbMessage = await this.conversationStore.getLastMessage(conversationId);
     if (!latestDbMessage) {
+      this.deps.log.info(
+        `[lcm] reconcileSessionTail: skipped for ${sessionContext} duration=${formatDurationMs(Date.now() - startedAt)} historicalMessages=${historicalMessages.length} reason=no-db-tail`,
+      );
       return { blockedByImportCap: false, importedMessages: 0, hasOverlap: false };
     }
 
@@ -2447,6 +2485,9 @@ export class LcmContextEngine implements ContextEngine {
         }
       }
       if (dbOccurrences === historicalOccurrences) {
+        this.deps.log.info(
+          `[lcm] reconcileSessionTail: fast path for ${sessionContext} duration=${formatDurationMs(Date.now() - startedAt)} historicalMessages=${historicalMessages.length} importedMessages=0 overlap=true`,
+        );
         return { blockedByImportCap: false, importedMessages: 0, hasOverlap: true };
       }
     }
@@ -2499,9 +2540,15 @@ export class LcmContextEngine implements ContextEngine {
     }
 
     if (anchorIndex < 0) {
+      this.deps.log.info(
+        `[lcm] reconcileSessionTail: no anchor for ${sessionContext} duration=${formatDurationMs(Date.now() - startedAt)} historicalMessages=${historicalMessages.length} importedMessages=0 overlap=false`,
+      );
       return { blockedByImportCap: false, importedMessages: 0, hasOverlap: false };
     }
     if (anchorIndex >= historicalMessages.length - 1) {
+      this.deps.log.info(
+        `[lcm] reconcileSessionTail: anchor at tip for ${sessionContext} duration=${formatDurationMs(Date.now() - startedAt)} historicalMessages=${historicalMessages.length} importedMessages=0 overlap=true`,
+      );
       return { blockedByImportCap: false, importedMessages: 0, hasOverlap: true };
     }
 
@@ -2509,14 +2556,12 @@ export class LcmContextEngine implements ContextEngine {
 
     const existingDbCount = await this.conversationStore.getMessageCount(conversationId);
     if (existingDbCount > 0 && missingTail.length > Math.max(existingDbCount * 0.2, 50)) {
-      const sessionContext = this.formatSessionLogContext({
-        conversationId,
-        sessionId,
-        sessionKey: params.sessionKey,
-      });
       this.deps.log.warn(
         `[lcm] reconcileSessionTail: import cap exceeded for ${sessionContext} — would import ${missingTail.length} messages (existing: ${existingDbCount}). Aborting to prevent flood.`,
       );
+      this.deps.log.info(
+        `[lcm] reconcileSessionTail: blocked for ${sessionContext} duration=${formatDurationMs(Date.now() - startedAt)} historicalMessages=${historicalMessages.length} missingTail=${missingTail.length} existingDbCount=${existingDbCount}`,
+      );
       return { blockedByImportCap: true, importedMessages: 0, hasOverlap: true };
     }
 
@@ -2528,6 +2573,9 @@ export class LcmContextEngine implements ContextEngine {
       }
     }
 
+    this.deps.log.info(
+      `[lcm] reconcileSessionTail: slow path for ${sessionContext} duration=${formatDurationMs(Date.now() - startedAt)} historicalMessages=${historicalMessages.length} anchorIndex=${anchorIndex} missingTail=${missingTail.length} importedMessages=${importedMessages}`,
+    );
     return { blockedByImportCap: false, importedMessages, hasOverlap: true };
   }
 
@@ -2582,6 +2630,11 @@ export class LcmContextEngine implements ContextEngine {
       };
     }
     this.ensureMigrated();
+    const startedAt = Date.now();
+    const sessionLabel = [
+      `session=${params.sessionId}`,
+      ...(params.sessionKey?.trim() ? [`sessionKey=${params.sessionKey.trim()}`] : []),
+    ].join(" ");
     const sessionFileStats = statSync(params.sessionFile);
     const sessionFileSize = sessionFileStats.size;
     const sessionFileMtimeMs = Math.trunc(sessionFileStats.mtimeMs);
@@ -2624,6 +2677,9 @@ export class LcmContextEngine implements ContextEngine {
             if (!conversation.bootstrappedAt) {
               await this.conversationStore.markConversationBootstrapped(conversationId);
             }
+            this.deps.log.info(
+              `[lcm] bootstrap: checkpoint hit conversation=${conversationId} ${sessionLabel} existingCount=${existingCount} duration=${formatDurationMs(Date.now() - startedAt)}`,
+            );
             return {
               bootstrapped: false,
               importedMessages: 0,
@@ -2685,6 +2741,9 @@ export class LcmContextEngine implements ContextEngine {
                 }
 
                 await persistBootstrapState(conversationId);
+                this.deps.log.info(
+                  `[lcm] bootstrap: append-only conversation=${conversationId} ${sessionLabel} existingCount=${existingCount} appendedMessages=${appended.messages.length} importedMessages=${importedMessages} duration=${formatDurationMs(Date.now() - startedAt)}`,
+                );
 
                 if (importedMessages > 0) {
                   return {
@@ -2704,6 +2763,9 @@ export class LcmContextEngine implements ContextEngine {
           }
 
           const historicalMessages = await readLeafPathMessages(params.sessionFile);
+          this.deps.log.info(
+            `[lcm] bootstrap: full transcript read conversation=${conversationId} ${sessionLabel} existingCount=${existingCount} historicalMessages=${historicalMessages.length} duration=${formatDurationMs(Date.now() - startedAt)}`,
+          );
 
           // First-time import path: no LCM rows yet, so seed directly from the
           // active leaf context snapshot.
@@ -2753,6 +2815,9 @@ export class LcmContextEngine implements ContextEngine {
             }
 
             await persistBootstrapState(conversationId);
+            this.deps.log.info(
+              `[lcm] bootstrap: initial import conversation=${conversationId} ${sessionLabel} importedMessages=${inserted.length} sourceMessages=${historicalMessages.length} duration=${formatDurationMs(Date.now() - startedAt)}`,
+            );
 
             return {
               bootstrapped: true,
@@ -2768,6 +2833,9 @@ export class LcmContextEngine implements ContextEngine {
             conversationId,
             historicalMessages,
           });
+          this.deps.log.info(
+            `[lcm] bootstrap: reconcile finished conversation=${conversationId} ${sessionLabel} importedMessages=${reconcile.importedMessages} overlap=${reconcile.hasOverlap} blockedByImportCap=${reconcile.blockedByImportCap} duration=${formatDurationMs(Date.now() - startedAt)}`,
+          );
 
           if (reconcile.blockedByImportCap) {
             return {
@@ -2810,6 +2878,7 @@ export class LcmContextEngine implements ContextEngine {
               : "conversation already has messages",
           };
         }),
+      { operationName: "bootstrap", context: sessionLabel },
     );
 
     // Post-bootstrap pruning: clean HEARTBEAT_OK turns that were already
@@ -2839,6 +2908,9 @@ export class LcmContextEngine implements ContextEngine {
       }
     }
 
+    this.deps.log.info(
+      `[lcm] bootstrap: done ${sessionLabel} bootstrapped=${result.bootstrapped} importedMessages=${result.importedMessages} reason=${result.reason ?? "none"} duration=${formatDurationMs(Date.now() - startedAt)}`,
+    );
     return result;
   }
 
@@ -2978,6 +3050,12 @@ export class LcmContextEngine implements ContextEngine {
       };
     }
 
+    const rewriteTranscriptEntries = params.runtimeContext.rewriteTranscriptEntries;
+    const startedAt = Date.now();
+    const sessionLabel = [
+      `session=${params.sessionId}`,
+      ...(params.sessionKey?.trim() ? [`sessionKey=${params.sessionKey.trim()}`] : []),
+    ].join(" ");
     return this.withSessionQueue(
       this.resolveSessionQueueKey(params.sessionId, params.sessionKey),
       async () => {
@@ -2999,6 +3077,9 @@ export class LcmContextEngine implements ContextEngine {
           { limit: TRANSCRIPT_GC_BATCH_SIZE },
         );
         if (candidates.length === 0) {
+          this.deps.log.info(
+            `[lcm] maintain: no transcript GC candidates conversation=${conversation.conversationId} ${sessionLabel} duration=${formatDurationMs(Date.now() - startedAt)}`,
+          );
           return {
             changed: false,
             bytesFreed: 0,
@@ -3034,6 +3115,9 @@ export class LcmContextEngine implements ContextEngine {
         }
 
         if (replacements.length === 0) {
+          this.deps.log.info(
+            `[lcm] maintain: no matching transcript entries conversation=${conversation.conversationId} ${sessionLabel} candidates=${candidates.length} duration=${formatDurationMs(Date.now() - startedAt)}`,
+          );
           return {
             changed: false,
             bytesFreed: 0,
@@ -3042,7 +3126,7 @@ export class LcmContextEngine implements ContextEngine {
           };
         }
 
-        const result = await params.runtimeContext.rewriteTranscriptEntries({
+        const result = await rewriteTranscriptEntries({
           replacements,
         });
 
@@ -3059,8 +3143,12 @@ export class LcmContextEngine implements ContextEngine {
           }
         }
 
+        this.deps.log.info(
+          `[lcm] maintain: done conversation=${conversation.conversationId} ${sessionLabel} candidates=${candidates.length} replacements=${replacements.length} changed=${result.changed} rewrittenEntries=${result.rewrittenEntries} bytesFreed=${result.bytesFreed} duration=${formatDurationMs(Date.now() - startedAt)}`,
+        );
         return result;
       },
+      { operationName: "maintain", context: sessionLabel },
     );
   }
   private async ingestSingle(params: {
@@ -3073,6 +3161,34 @@ export class LcmContextEngine implements ContextEngine {
     if (isHeartbeat) {
       return { ingested: false };
     }
+
+    // Skip assistant messages that failed with an error and have no useful content.
+    // These occur when an API call returns a 500 or similar transient error.
+    // Ingesting them pollutes the LCM database: on retry, the error messages
+    // accumulate and get assembled into context, creating a positive feedback
+    // loop where each retry sends an increasingly large (and malformed) payload
+    // that continues to fail.
+    if (message.role === "assistant") {
+      const topLevel = message as unknown as Record<string, unknown>;
+      const stopReason =
+        typeof topLevel.stopReason === "string"
+          ? topLevel.stopReason
+          : typeof topLevel.stop_reason === "string"
+            ? topLevel.stop_reason
+            : undefined;
+      if (stopReason === "error" || stopReason === "aborted") {
+        const content = topLevel.content;
+        const isEmpty =
+          content === undefined ||
+          content === null ||
+          content === "" ||
+          (Array.isArray(content) && content.length === 0);
+        if (isEmpty) {
+          return { ingested: false };
+        }
+      }
+    }
+
     const stored = toStoredMessage(message);
 
     // Get or create conversation for this session
@@ -3153,6 +3269,13 @@ export class LcmContextEngine implements ContextEngine {
     return this.withSessionQueue(
       this.resolveSessionQueueKey(params.sessionId, params.sessionKey),
       () => this.ingestSingle(params),
+      {
+        operationName: "ingest",
+        context: [
+          `session=${params.sessionId}`,
+          ...(params.sessionKey?.trim() ? [`sessionKey=${params.sessionKey.trim()}`] : []),
+        ].join(" "),
+      },
     );
   }
 
@@ -3189,6 +3312,14 @@ export class LcmContextEngine implements ContextEngine {
         }
         return { ingestedCount };
       },
+      {
+        operationName: "ingestBatch",
+        context: [
+          `session=${params.sessionId}`,
+          ...(params.sessionKey?.trim() ? [`sessionKey=${params.sessionKey.trim()}`] : []),
+          `messages=${params.messages.length}`,
+        ].join(" "),
+      },
     );
   }
 
@@ -3213,6 +3344,11 @@ export class LcmContextEngine implements ContextEngine {
       return;
     }
     this.ensureMigrated();
+    const startedAt = Date.now();
+    const sessionLabel = [
+      `session=${params.sessionId}`,
+      ...(params.sessionKey?.trim() ? [`sessionKey=${params.sessionKey.trim()}`] : []),
+    ].join(" ");
 
     // Dedup guard: prevent duplicate ingestion when gateway restart replays
     // full history. Run on newMessages BEFORE prepending autoCompactionSummary
@@ -3234,6 +3370,9 @@ export class LcmContextEngine implements ContextEngine {
 
     ingestBatch.push(...dedupedNewMessages);
     if (ingestBatch.length === 0) {
+      this.deps.log.info(
+        `[lcm] afterTurn: nothing to ingest ${sessionLabel} newMessages=${newMessages.length} duration=${formatDurationMs(Date.now() - startedAt)}`,
+      );
       return;
     }
 
@@ -3309,6 +3448,9 @@ export class LcmContextEngine implements ContextEngine {
       sessionKey: params.sessionKey,
     });
     if (!conversation) {
+      this.deps.log.info(
+        `[lcm] afterTurn: conversation lookup missed ${sessionLabel} ingestBatch=${ingestBatch.length} duration=${formatDurationMs(Date.now() - startedAt)}`,
+      );
       return;
     }
 
@@ -3366,6 +3508,10 @@ export class LcmContextEngine implements ContextEngine {
     } catch {
       // Proactive compaction is best-effort in the post-turn lifecycle.
     }
+
+    this.deps.log.info(
+      `[lcm] afterTurn: done conversation=${conversation.conversationId} ${sessionLabel} newMessages=${newMessages.length} dedupedMessages=${dedupedNewMessages.length} ingestedMessages=${ingestBatch.length} duration=${formatDurationMs(Date.now() - startedAt)}`,
+    );
   }
 
   async assemble(params: {
@@ -3384,12 +3530,20 @@ export class LcmContextEngine implements ContextEngine {
     }
     try {
       this.ensureMigrated();
+      const startedAt = Date.now();
+      const sessionLabel = [
+        `session=${params.sessionId}`,
+        ...(params.sessionKey?.trim() ? [`sessionKey=${params.sessionKey.trim()}`] : []),
+      ].join(" ");
 
       const conversation = await this.conversationStore.getConversationForSession({
         sessionId: params.sessionId,
         sessionKey: params.sessionKey,
       });
       if (!conversation) {
+        this.deps.log.info(
+          `[lcm] assemble: conversation lookup missed ${sessionLabel} duration=${formatDurationMs(Date.now() - startedAt)}`,
+        );
         return {
           messages: params.messages,
           estimatedTokens: 0,
@@ -3398,6 +3552,9 @@ export class LcmContextEngine implements ContextEngine {
 
       const contextItems = await this.summaryStore.getContextItems(conversation.conversationId);
       if (contextItems.length === 0) {
+        this.deps.log.info(
+          `[lcm] assemble: no context items conversation=${conversation.conversationId} ${sessionLabel} duration=${formatDurationMs(Date.now() - startedAt)}`,
+        );
         return {
           messages: params.messages,
           estimatedTokens: 0,
@@ -3409,6 +3566,9 @@ export class LcmContextEngine implements ContextEngine {
       // the live path to avoid dropping prompt context.
       const hasSummaryItems = contextItems.some((item) => item.itemType === "summary");
       if (!hasSummaryItems && contextItems.length < params.messages.length) {
+        this.deps.log.info(
+          `[lcm] assemble: falling back to live context conversation=${conversation.conversationId} ${sessionLabel} contextItems=${contextItems.length} liveMessages=${params.messages.length} duration=${formatDurationMs(Date.now() - startedAt)}`,
+        );
         return {
           messages: params.messages,
           estimatedTokens: 0,
@@ -3433,12 +3593,19 @@ export class LcmContextEngine implements ContextEngine {
       // If assembly produced no messages for a non-empty live session,
       // fail safe to the live context.
       if (assembled.messages.length === 0 && params.messages.length > 0) {
+        this.deps.log.info(
+          `[lcm] assemble: empty assembled output, using live context conversation=${conversation.conversationId} ${sessionLabel} contextItems=${contextItems.length} tokenBudget=${tokenBudget} duration=${formatDurationMs(Date.now() - startedAt)}`,
+        );
         return {
           messages: params.messages,
           estimatedTokens: 0,
         };
       }
 
+      this.deps.log.info(
+        `[lcm] assemble: done conversation=${conversation.conversationId} ${sessionLabel} contextItems=${contextItems.length} hasSummaryItems=${hasSummaryItems} inputMessages=${params.messages.length} outputMessages=${assembled.messages.length} tokenBudget=${tokenBudget} estimatedTokens=${assembled.estimatedTokens} duration=${formatDurationMs(Date.now() - startedAt)}`,
+      );
+
       const result: AssembleResultWithSystemPrompt = {
         messages: assembled.messages,
         estimatedTokens: assembled.estimatedTokens,
@@ -3447,7 +3614,10 @@ export class LcmContextEngine implements ContextEngine {
           : {}),
       };
       return result;
-    } catch {
+    } catch (err) {
+      this.deps.log.info(
+        `[lcm] assemble: failed for session=${params.sessionId}${params.sessionKey?.trim() ? ` sessionKey=${params.sessionKey.trim()}` : ""} error=${describeLogError(err)}`,
+      );
       return {
         messages: params.messages,
         estimatedTokens: 0,
@@ -3850,11 +4020,21 @@ export class LcmContextEngine implements ContextEngine {
             ? decision.threshold
             : tokenBudget;
 
+        // When forced (overflow recovery) and the caller did not supply an
+        // observed token count, assume we are at least at the token budget so
+        // compactUntilUnder does not bail with "already under target" while the
+        // live context is actually overflowing.
+        const effectiveCurrentTokens =
+          observedTokens !== undefined
+            ? observedTokens
+            : forceCompaction
+              ? tokenBudget
+              : undefined;
         const compactResult = await this.compaction.compactUntilUnder({
           conversationId,
           tokenBudget,
           targetTokens: convergenceTargetTokens,
-          ...(observedTokens !== undefined ? { currentTokens: observedTokens } : {}),
+          ...(effectiveCurrentTokens !== undefined ? { currentTokens: effectiveCurrentTokens } : {}),
           summarize,
           summaryModel,
         });

package/src/estimate-tokens.ts

@@ -0,0 +1,80 @@
+/**
+ * Shared token estimation utility.
+ *
+ * Uses code-point-aware weighting instead of `text.length / 4`:
+ *   - CJK (Chinese/Japanese/Korean) characters: ~1.5 tokens/char
+ *   - Emoji / Supplementary Plane: ~2 tokens/char
+ *   - ASCII / Latin: ~0.25 tokens/char (≈ 4 chars/token)
+ *
+ * Why not `text.length / 4`?
+ * JavaScript `String.length` counts UTF-16 code units, not Unicode code points.
+ * CJK characters are 1 UTF-16 unit but ~1.5 tokens; emoji are 2 UTF-16 units
+ * (surrogate pairs) but ~2-4 tokens. The naive formula underestimates CJK by
+ * ~6× and emoji by ~2-4×, causing compaction to trigger far too late for
+ * non-English conversations.
+ */
+
+/** Detect CJK code points across all relevant Unicode ranges. */
+function isCjkCodePoint(cp: number): boolean {
+  return (
+    (cp >= 0x4e00 && cp <= 0x9fff) ||   // CJK Unified Ideographs
+    (cp >= 0x3400 && cp <= 0x4dbf) ||    // CJK Extension A
+    (cp >= 0x20000 && cp <= 0x2a6df) ||  // CJK Extension B
+    (cp >= 0x2a700 && cp <= 0x2b73f) ||  // CJK Extension C
+    (cp >= 0x2b740 && cp <= 0x2b81f) ||  // CJK Extension D
+    (cp >= 0x2b820 && cp <= 0x2ceaf) ||  // CJK Extension E
+    (cp >= 0x2ceb0 && cp <= 0x2ebef) ||  // CJK Extension F
+    (cp >= 0x3000 && cp <= 0x303f) ||    // CJK Symbols and Punctuation
+    (cp >= 0x3040 && cp <= 0x30ff) ||    // Hiragana + Katakana
+    (cp >= 0xac00 && cp <= 0xd7af) ||    // Hangul Syllables
+    (cp >= 0xff00 && cp <= 0xffef)       // Fullwidth Forms
+  );
+}
+
+/** Estimate token cost for a single Unicode code point. */
+function estimateCodePointTokens(cp: number): number {
+  if (isCjkCodePoint(cp)) {
+    return 1.5;
+  }
+  if (cp > 0xffff) {
+    return 2;
+  }
+  return 0.25;
+}
+
+/** Estimate text tokens using Unicode-aware character weighting. */
+export function estimateTokens(text: string): number {
+  let tokens = 0;
+  for (const char of text) {
+    const cp = char.codePointAt(0) ?? 0;
+    tokens += estimateCodePointTokens(cp);
+  }
+  return Math.ceil(tokens);
+}
+
+/**
+ * Truncate text so the estimated token count stays within `maxTokens`.
+ *
+ * Iterates by Unicode code point to avoid splitting surrogate pairs while
+ * preserving the same weighting model as `estimateTokens()`.
+ */
+export function truncateTextToEstimatedTokens(text: string, maxTokens: number): string {
+  if (maxTokens <= 0 || !text) {
+    return "";
+  }
+
+  let tokens = 0;
+  let end = 0;
+
+  for (const char of text) {
+    const cp = char.codePointAt(0) ?? 0;
+    const nextTokens = tokens + estimateCodePointTokens(cp);
+    if (Math.ceil(nextTokens) > maxTokens) {
+      break;
+    }
+    tokens = nextTokens;
+    end += char.length;
+  }
+
+  return text.slice(0, end);
+}