@remnic/core 9.63.9 → 9.63.10

package/src/chat/chat-engine.ts

@@ -85,10 +85,21 @@ export interface ChatEngineOptions {
 // Confirmation keywords — deterministic, not LLM-interpreted
 // ---------------------------------------------------------------------------
 
-const CONFIRMATION_KEYWORDS = new Set(["yes", "y", "apply", "confirm"]);
+// Exact-match confirmations. Unrecognized text keeps the conservative
+// no-apply default, so this stays a small fixed set (issue #2198).
+const CONFIRMATION_KEYWORDS: Record<string, true> = {
+  yes: true,
+  y: true,
+  apply: true,
+  confirm: true,
+  "はい": true, // Japanese
+  "sí": true, // Spanish
+  ja: true, // German/Dutch
+  "نعم": true, // Arabic
+};
 
 export function isConfirmationMessage(text: string): boolean {
-  return CONFIRMATION_KEYWORDS.has(text.trim().toLowerCase());
+  return CONFIRMATION_KEYWORDS[text.trim().toLowerCase()] === true;
 }
 
 // ---------------------------------------------------------------------------

package/src/day-summary.test.ts

@@ -0,0 +1,63 @@
+/**
+ * Day-summary composition tests — bidi isolation of user content (issue #2198).
+ * All fixtures are synthetic — no real user data.
+ */
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { formatDaySummaryMemories } from "./day-summary.js";
+import type { MemoryFile } from "./types.js";
+
+const FSI = "\u2068";
+const PDI = "\u2069";
+
+function makeMemory(content: string): MemoryFile {
+  return {
+    path: "/synthetic/mem.md",
+    frontmatter: {
+      id: "synthetic-mem",
+      category: "fact",
+      created: "2026-08-17T00:00:00.000Z",
+      updated: "2026-08-17T00:00:00.000Z",
+      source: "test",
+      confidence: 0.9,
+      confidenceTier: "explicit",
+      tags: [],
+    },
+    content,
+  };
+}
+
+test("formatDaySummaryMemories wraps RTL content in FSI/PDI isolates (issue #2198)", () => {
+  const rtl = makeMemory("اجتماع الفريق غدا في العاشرة");
+  const out = formatDaySummaryMemories([rtl]);
+
+  const expected = `[synthetic-mem] (fact, 2026-08-17T00:00:00.000Z)\n${FSI}اجتماع الفريق غدا في العاشرة${PDI}`;
+  assert.equal(out, expected);
+});
+
+test("formatDaySummaryMemories isolates LTR content too — labels stay outside the isolate", () => {
+  const out = formatDaySummaryMemories([makeMemory("standup at ten")]);
+
+  assert.ok(out.includes(`${FSI}standup at ten${PDI}`), "content wrapped in FSI/PDI");
+  assert.ok(out.startsWith("[synthetic-mem] (fact,"), "Latin label precedes the isolate");
+  assert.ok(!out.includes(`${PDI}[`), "label not inside the isolate");
+});
+
+test("formatDaySummaryMemories isolates each memory's content independently", () => {
+  const out = formatDaySummaryMemories([
+    makeMemory("first item"),
+    makeMemory("عنصر ثان"),
+  ]);
+
+  const parts = out.split("\n\n");
+  assert.equal(parts.length, 2);
+  assert.ok(parts[0]!.includes(`${FSI}first item${PDI}`));
+  assert.ok(parts[1]!.includes(`${FSI}عنصر ثان${PDI}`));
+});
+
+test("formatDaySummaryMemories raw-string passthrough is not wrapped", () => {
+  const out = formatDaySummaryMemories("  plain string input  ");
+  assert.equal(out, "plain string input");
+});

package/src/day-summary.ts

@@ -93,6 +93,18 @@ export async function loadDaySummaryPrompt(): Promise<string> {
   return EMBEDDED_DAY_SUMMARY_PROMPT;
 }
 
+const FSI = "\u2068"; // FIRST STRONG ISOLATE
+const PDI = "\u2069"; // POP DIRECTIONAL ISOLATE
+
+/**
+ * Wrap user content in Unicode bidi isolates so RTL text (Arabic, Hebrew)
+ * cannot reorder the surrounding Latin labels on the same composed line.
+ * No visible change for LTR content (issue #2198).
+ */
+function bidiIsolate(text: string): string {
+  return `${FSI}${text}${PDI}`;
+}
+
 export function formatDaySummaryMemories(memories: string | MemoryFile[]): string {
   if (typeof memories === "string") {
     return memories.trim();
@@ -102,7 +114,7 @@ export function formatDaySummaryMemories(memories: string | MemoryFile[]): strin
     .map((memory) => {
       const category = memory.frontmatter.category || "fact";
       const created = memory.frontmatter.created || "unknown";
-      return `[${memory.frontmatter.id}] (${category}, ${created})\n${memory.content}`;
+      return `[${memory.frontmatter.id}] (${category}, ${created})\n${bidiIsolate(memory.content)}`;
     })
     .join("\n\n")
     .trim();

package/src/entity-retrieval-i18n.test.ts

@@ -0,0 +1,115 @@
+import assert from "node:assert/strict";
+import os from "node:os";
+import path from "node:path";
+import { mkdtemp, rm } from "node:fs/promises";
+import test from "node:test";
+
+import { parseConfig } from "./config.js";
+import { normalizeEntityText } from "./entity-schema.js";
+import { buildEntityRecallSection } from "./entity-retrieval.js";
+import { detectNonEnglishEntityQueryMode } from "./entity-retrieval-i18n.js";
+import { StorageManager } from "./storage.js";
+import type { PluginConfig, TranscriptEntry } from "./types.js";
+
+function dialogueTurn(content: string): TranscriptEntry[] {
+  return [{
+    timestamp: new Date().toISOString(),
+    role: "user",
+    content,
+    sessionKey: "test-session",
+    turnId: "test-turn-1",
+  }];
+}
+
+function recallBase(
+  config: PluginConfig,
+  storage: StorageManager,
+  transcriptEntries: TranscriptEntry[],
+): Parameters<typeof buildEntityRecallSection>[0] {
+  return {
+    config,
+    storage,
+    query: "",
+    recentTurns: 2,
+    maxHints: 2,
+    maxSupportingFacts: 2,
+    maxRelatedEntities: 2,
+    maxChars: 10_000,
+    transcriptEntries,
+  };
+}
+
+test("non-English follow-ups resolve the recently discussed entity like the English pronoun follow-up (#2193)", async () => {
+  const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-entity-i18n-"));
+  try {
+    const storage = new StorageManager(memoryDir);
+    const entityRef = await storage.writeEntity("Alice", "person", []);
+    await storage.writeMemory("fact", "Alice leads the platform launch review.", {
+      source: "test",
+      entityRef,
+    });
+    const config = parseConfig({
+      openaiApiKey: "sk-test",
+      memoryDir,
+      workspaceDir: path.join(memoryDir, "workspace"),
+    });
+    const dialogue = dialogueTurn("I spoke with Alice about the launch review yesterday.");
+
+    // English pronoun follow-up and its Japanese/Chinese equivalents must
+    // resolve to the same recently-discussed entity.
+    for (const query of ["What about her?", "それで、どうなった？", "彼はどうなった？", "然后呢？"]) {
+      const section = await buildEntityRecallSection({
+        ...recallBase(config, storage, dialogue),
+        query,
+      });
+      assert.ok(section, `expected an entity hint section for query: ${query}`);
+      assert.match(section, /alice/i);
+    }
+
+    // Without recent dialogue both decline identically — no entity to carry.
+    for (const query of ["What about him?", "それで、どうなった？"]) {
+      const section = await buildEntityRecallSection({
+        ...recallBase(config, storage, []),
+        query,
+      });
+      assert.equal(section, null, `expected no entity section without dialogue: ${query}`);
+    }
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+
+test("zero-pronoun Japanese follow-up retrieves a CJK entity from a particle-bound mention (#2193)", async () => {
+  const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-entity-i18n-cjk-"));
+  try {
+    const storage = new StorageManager(memoryDir);
+    const entityRef = await storage.writeEntity("田中", "person", []);
+    await storage.writeMemory("fact", "田中は来週の発表を準備している。", {
+      source: "test",
+      entityRef,
+    });
+    const config = parseConfig({
+      openaiApiKey: "sk-test",
+      memoryDir,
+      workspaceDir: path.join(memoryDir, "workspace"),
+    });
+
+    for (const query of ["それで、どうなった？", "彼女はどうなった？"]) {
+      const section = await buildEntityRecallSection({
+        ...recallBase(config, storage, dialogueTurn("田中とは昨日打ち合わせをした。")),
+        query,
+      });
+      assert.ok(section, `expected an entity hint section for query: ${query}`);
+      assert.match(section, /田中/);
+    }
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+
+test("non-English query-mode cue table classification (#2193)", () => {
+  assert.equal(detectNonEnglishEntityQueryMode(normalizeEntityText("彼はどうなった")), "follow_up");
+  assert.equal(detectNonEnglishEntityQueryMode(normalizeEntityText("¿y ella?")), "follow_up");
+  assert.equal(detectNonEnglishEntityQueryMode(normalizeEntityText("田中さんの最近の状況")), "timeline");
+  assert.equal(detectNonEnglishEntityQueryMode(normalizeEntityText("了解しました")), null);
+});

package/src/entity-retrieval-i18n.ts

@@ -0,0 +1,161 @@
+import { normalizeEntityText } from "./entity-schema.js";
+
+/**
+ * Non-English query-mode cues for entity retrieval (#2193).
+ *
+ * #2161 made explicit entity mentions language-independent, but follow-up /
+ * pronoun coreference and timeline phrasing still classify through English
+ * word lists in `detectEntityQueryMode`. These cue tables plus the structural
+ * follow-up signal below close that gap without building a full coreference
+ * engine: the structural layer covers zero-pronoun scripts (Japanese,
+ * Chinese, Korean — languages that omit the subject entirely and can never
+ * trip a pronoun word list), and the tables add precision for other scripts,
+ * including Latin-script languages whose pronoun follow-ups carry their
+ * pronoun explicitly. Latin-script queries never use the structural layer:
+ * their generic technical questions ("what does this error mean?") are
+ * indistinguishable from subject-less follow-ups by shape alone.
+ */
+
+export type EntityQueryMode = "direct" | "timeline" | "follow_up";
+
+/** Shared short-question bound. Whitespace tokens are a poor length proxy
+ * for unspaced scripts, so bound graphemes too (48 graphemes ≈ 8 words). */
+const FOLLOW_UP_MAX_TOKENS = 8;
+const FOLLOW_UP_MAX_GRAPHEMES = 48;
+
+/** Latin letters mark a script with explicit pronouns; structural fallback
+ * never applies there (English "what does this error mean?" and its
+ * relatives keep the technical-question guard in detectEntityQueryMode). */
+const LATIN_LETTER_RE = /\p{Script=Latin}/u;
+
+ /** Latin / question marks: ASCII `?`, fullwidth `？` (CJK), Arabic `؟`. */
+ const QUESTION_MARK_RE = /[?？؟]/;
+
+/**
+ * CJK cues and normalized multi-word phrases match by containment; single
+ * Latin/Cyrillic/Arabic words match with a script-agnostic letter boundary
+ * so `él` never matches `el` and `il` never matches `famille`.
+ */
+const FOLLOW_UP_CONTAINS_CUES = [
+  // ja
+  "彼", "彼女", "あの人", "その人", "それで", "ちなみに",
+  // zh (simplified + traditional)
+  "他", "她", "它", "他们", "她们", "他們", "她們", "然后呢", "然後呢",
+  // ko
+  "그녀", "그 사람", "걔",
+] as const;
+
+const FOLLOW_UP_WORD_CUES = [
+  // es
+  "él", "ella", "ellos", "ellas",
+  // fr
+  "elle", "eux",
+  // de
+  "sie",
+  // pt
+  "ele", "ela",
+  // it
+  "lui", "lei",
+  // ru
+  "он", "она", "они",
+  // ar
+  "هو", "هي", "هم",
+] as const;
+
+const TIMELINE_CONTAINS_CUES = [
+  // ja
+  "最近", "その後", "どうなった", "何があった",
+  // zh (simplified + traditional)
+  "怎么样了", "怎麼樣了", "后来", "後來", "最新", "近况", "近況", "什么情况", "什麼情況",
+  // ko
+  "요즘", "요새", "최근",
+  // multi-word phrases, already in normalized form (punctuation → space)
+  "qué hay de nuevo", "qué pasó con", "quoi de neuf", "что нового",
+  "что случилось", "ما الجديد", "ماذا حدث", "was gibt s neues",
+] as const;
+
+function wordCueRegex(cues: readonly string[]): RegExp {
+  const alternation = cues
+    .map((cue) => cue.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"))
+    .join("|");
+  return new RegExp(`(?<![\\p{L}\\p{N}])(?:${alternation})(?![\\p{L}\\p{N}])`, "u");
+}
+
+const FOLLOW_UP_WORD_RE = wordCueRegex(FOLLOW_UP_WORD_CUES);
+
+function isShortFollowUpShaped(normalized: string): boolean {
+  if (!normalized) return false;
+  const tokens = normalized.split(/\s+/).filter(Boolean);
+  if (tokens.length > FOLLOW_UP_MAX_TOKENS) return false;
+  return Array.from(normalized).length <= FOLLOW_UP_MAX_GRAPHEMES;
+}
+
+/**
+ * Classify already-normalized non-English queries. Returns null when no
+ * table cue matches; callers keep their English rules untouched.
+ */
+export function detectNonEnglishEntityQueryMode(normalizedQuery: string): EntityQueryMode | null {
+  if (!normalizedQuery) return null;
+  if (
+    isShortFollowUpShaped(normalizedQuery)
+    && (
+      FOLLOW_UP_CONTAINS_CUES.some((cue) => normalizedQuery.includes(cue))
+      || FOLLOW_UP_WORD_RE.test(normalizedQuery)
+    )
+  ) {
+    return "follow_up";
+  }
+  if (TIMELINE_CONTAINS_CUES.some((cue) => normalizedQuery.includes(cue))) {
+    return "timeline";
+  }
+  return null;
+}
+
+/**
+ * Structural follow-up signal, independent of pronoun vocabulary: a short
+ * question carrying no entity name, asked while recent dialogue exists, is a
+ * coreference cue in zero-pronoun scripts (Latin-script queries are excluded
+ * above — their follow-ups carry pronouns that the cue tables and English
+ * rules classify). Recent-turn candidate resolution (and its empty result →
+ * section absent) remains the real gate, so this only routes the query into
+ * the follow-up path; it never invents an entity.
+ */
+export function isStructuralEntityFollowUpQuery(query: string, hasRecentDialogue: boolean): boolean {
+  if (!hasRecentDialogue) return false;
+  if (LATIN_LETTER_RE.test(query)) return false;
+  const normalized = normalizeEntityText(query);
+  return isShortFollowUpShaped(normalized) && QUESTION_MARK_RE.test(query);
+}
+
+const ENTITY_PRONOUN_RE = /\b(he|him|his|she|her|they|them|their|it|its)\b/i;
+
+export function detectEntityQueryMode(query: string): EntityQueryMode | null {
+  const normalized = normalizeEntityText(query);
+  if (!normalized) return null;
+  if (
+    /^(what about|and what about|how about|what happened (with|to) (he|him|his|she|her|they|them|their|it|its)|did (he|she|they|it)|is (he|she|they|it)|was (he|she|they|it))\b/.test(normalized)
+  ) {
+    return "follow_up";
+  }
+  if (
+    /^(who is|who s|what do we know about|what does|tell me about|what can you tell me about|what s new with|what happened with|what happened to|status of|where is|how is)\b/.test(normalized)
+  ) {
+    if (/^what does\b/.test(normalized)) {
+      if (/^what does (?:this|that|it|the|a|an|my|our|your|their)\b/.test(normalized)) {
+        return null;
+      }
+      if (
+        /^what does [a-z0-9-]+ (?:error|warning|exception|failure|stack|trace|code|message|log)\b/.test(normalized)
+        && /\b(mean|means|indicate|indicates|imply|implies)\b/.test(normalized)
+      ) {
+        return null;
+      }
+    }
+    return /what happened|what s new|status of|how is|where is/.test(normalized) ? "timeline" : "direct";
+  }
+  if (ENTITY_PRONOUN_RE.test(normalized) && normalized.split(/\s+/).length <= 8) {
+    return "follow_up";
+  }
+  return detectNonEnglishEntityQueryMode(normalized);
+}
+

package/src/entity-retrieval.ts

@@ -10,6 +10,11 @@ import { normalizeEntityText, resolveRequestedEntitySectionKeys } from "./entity
 import type { EntityStructuredSection, MemoryFile, PluginConfig, TranscriptEntry } from "./types.js";
 import { truncateGraphemeSafe } from "./whitespace.js";
 import { containsPhrase } from "./entity-retrieval-boundaries.js";
+import {
+  detectEntityQueryMode,
+  isStructuralEntityFollowUpQuery,
+  type EntityQueryMode,
+} from "./entity-retrieval-i18n.js";
 import { buildOriginStructuredSections, sanitizeOriginatedFacts, type EntityOriginStructuredSection } from "./entity-origin-fields.js";
 import {
   checkEntityRecallAbort,
@@ -21,10 +26,8 @@ const ENTITY_INDEX_VERSION = 3;
 const RECENT_TRANSCRIPT_LOOKBACK_HOURS = 24;
 const INSTRUCTION_LIKE_RE = /\b(always|never|must|should|remember to|do not|don't|process|workflow|template|checklist|instruction)\b/i;
 const METADATA_WRAPPER_RE = /^(source|context|metadata|notes?):/i;
-const ENTITY_PRONOUN_RE = /\b(he|him|his|she|her|they|them|their|it|its)\b/i;
 const BELIEF_LEDGER_SECTION_KEY = "belief_ledger";
 const BELIEF_LEDGER_FACT_RE = /^claim=([^;]+);\s*status=([^;]+);\s*updatedAt=([^;]+);\s*(.+)$/;
-type EntityQueryMode = "direct" | "timeline" | "follow_up";
 type EntityMentionIndexEntry = {
   canonicalId: string;
   name: string;
@@ -171,35 +174,6 @@ function relationLine(entry: EntityMentionIndexEntry, relationship: { target: st
   if (normalizedLabel.length === 0) return `${entry.name} is connected to ${relationship.target}`;
   return `${entry.name} ${normalizedLabel} ${relationship.target}`;
 }
-function detectEntityQueryMode(query: string): EntityQueryMode | null {
-  const normalized = normalizeEntityText(query);
-  if (!normalized) return null;
-  if (
-    /^(what about|and what about|how about|what happened (with|to) (he|him|his|she|her|they|them|their|it|its)|did (he|she|they|it)|is (he|she|they|it)|was (he|she|they|it))\b/.test(normalized)
-  ) {
-    return "follow_up";
-  }
-  if (
-    /^(who is|who s|what do we know about|what does|tell me about|what can you tell me about|what s new with|what happened with|what happened to|status of|where is|how is)\b/.test(normalized)
-  ) {
-    if (/^what does\b/.test(normalized)) {
-      if (/^what does (?:this|that|it|the|a|an|my|our|your|their)\b/.test(normalized)) {
-        return null;
-      }
-      if (
-        /^what does [a-z0-9-]+ (?:error|warning|exception|failure|stack|trace|code|message|log)\b/.test(normalized)
-        && /\b(mean|means|indicate|indicates|imply|implies)\b/.test(normalized)
-      ) {
-        return null;
-      }
-    }
-    return /what happened|what s new|status of|how is|where is/.test(normalized) ? "timeline" : "direct";
-  }
-  if (ENTITY_PRONOUN_RE.test(normalized) && normalized.split(/\s+/).length <= 8) {
-    return "follow_up";
-  }
-  return null;
-}
 function scoreAliasMatch(query: string, alias: string): number {
   const normalizedQuery = normalizeEntityText(query);
   const normalizedAlias = normalizeEntityText(alias);
@@ -1061,7 +1035,18 @@ export async function buildEntityRecallSection(options: BuildEntityRecallSection
   // during a scan is observed here (issue #2291).
   checkEntityRecallAbort(options.abortSignal);
   const prefixedMode = detectEntityQueryMode(options.query);
-  const persistedIndex = prefixedMode
+  // #2193: in zero-pronoun scripts a short question with no entity name
+  // never trips a pronoun word list, so the structural signal routes it
+  // into the same coreference path as an English "what about him?".
+  // Latin-script queries stay on the cue tables / English rules so the
+  // generic technical-question guard keeps working. Resolution gates.
+  const structuralFollowUp = prefixedMode === null
+    && isStructuralEntityFollowUpQuery(
+      options.query,
+      options.recentTurns > 0 && options.transcriptEntries.length > 0,
+    );
+  const earlyMode = prefixedMode ?? (structuralFollowUp ? "follow_up" : null);
+  const persistedIndex = earlyMode
     ? null
     : await readCurrentPersistedEntityIndex(
       options.storage,
@@ -1072,7 +1057,7 @@ export async function buildEntityRecallSection(options: BuildEntityRecallSection
   checkEntityRecallAbort(options.abortSignal);
   let nativeChunks: NativeKnowledgeChunk[] | undefined;
   if (
-    !prefixedMode &&
+    !earlyMode &&
     persistedIndex &&
     resolveLanguageIndependentExplicitCandidates(persistedIndex, options.query).length === 0
   ) {
@@ -1129,7 +1114,22 @@ export async function buildEntityRecallSection(options: BuildEntityRecallSection
   const queryCandidates = prefixedMode
     ? explicitCandidates
     : resolveLanguageIndependentExplicitCandidates(index, options.query);
-  const mode = prefixedMode ?? (queryCandidates.length > 0 ? "direct" : null);
+  if (
+    queryCandidates.length === 0
+    && /^what does (?:this|that|it|the|a|an|my|our|your|their)\b/i.test(normalizeEntityText(options.query))
+  ) {
+    return entityRecallSectionAbsent(options.abortSignal);
+  }
+
+  // Explicit mentions keep #2161 behavior (direct mode) even when the
+  // structural follow-up signal also fired; only name-less short questions
+  // fall through to coreference.
+  const mode = prefixedMode
+    ?? (queryCandidates.length > 0
+      ? "direct"
+      : structuralFollowUp
+        ? "follow_up"
+        : null);
   if (!mode) return entityRecallSectionAbsent(options.abortSignal);
 
   const candidates = queryCandidates.length > 0

package/src/extraction-source-grounding-helpers.ts

@@ -334,9 +334,9 @@ export interface GroundingLexeme {
 }
 
 export function groundingLexemes(text: string): GroundingLexeme[] {
-  const rawTokens = text.normalize("NFKC").match(
+  const rawTokens = (text.normalize("NFKC").match(
     /[\p{L}\p{N}]+(?:['’][\p{L}\p{N}]+)?(?:\+\+[\p{L}\p{N}]*|#[\p{L}\p{N}]*)?/gu,
-  ) ?? [];
+  ) ?? []).flatMap(splitSpacelessScriptToken);
   const tokens = rawTokens.map((rawToken) =>
     rawToken.replaceAll("’", "'").toLocaleLowerCase());
   let predicateIndex = tokens.findIndex((token, index) => {
@@ -500,6 +500,116 @@ export function tokenize(text: string): Set<string> {
   return tokens;
 }
 
+const SPACELESS_SCRIPT_CHARACTER_PATTERN = new RegExp(
+  "[\\p{Script=Han}\\p{Script=Hiragana}\\p{Script=Katakana}\\p{Script=Hangul}"
+    + "\\p{Script=Thai}\\p{Script=Lao}\\p{Script=Khmer}\\p{Script=Myanmar}]",
+  "u",
+);
+const SPACELESS_SCRIPT_LCS_SOURCE_LIMIT = 4096;
+
+function containsSpacelessScriptCharacter(text: string): boolean {
+  return SPACELESS_SCRIPT_CHARACTER_PATTERN.test(text);
+}
+
+/**
+ * Scripts without word separators (CJK, Hangul, Thai, ...) must not become one
+ * whitespace-delimited token: a paraphrase never matches a whole-run token, so
+ * grounded CJK facts and entity names were dropped. Split those runs into
+ * per-character lexemes so contiguous-token matching behaves as substring
+ * matching for these scripts.
+ */
+function splitSpacelessScriptToken(rawToken: string): string[] {
+  if (!containsSpacelessScriptCharacter(rawToken)) return [rawToken];
+  const parts: string[] = [];
+  let otherScriptRun = "";
+  for (const character of rawToken) {
+    if (containsSpacelessScriptCharacter(character)) {
+      if (otherScriptRun.length > 0) {
+        parts.push(otherScriptRun);
+        otherScriptRun = "";
+      }
+      parts.push(character);
+    } else {
+      otherScriptRun += character;
+    }
+  }
+  if (otherScriptRun.length > 0) parts.push(otherScriptRun);
+  return parts;
+}
+
+function spacelessScriptCharacterSequence(text: string): string[] {
+  const characters: string[] = [];
+  for (const character of text.normalize("NFKC")) {
+    if (containsSpacelessScriptCharacter(character)) characters.push(character);
+  }
+  return characters;
+}
+
+function spacelessScriptCharacterNgrams(text: string): Set<string> {
+  const grams = new Set<string>();
+  let previousCharacter = "";
+  for (const character of spacelessScriptCharacterSequence(text)) {
+    if (previousCharacter !== "") grams.add(previousCharacter + character);
+    previousCharacter = character;
+  }
+  return grams;
+}
+
+function longestCommonSubstringLength(
+  left: ReadonlyArray<string>,
+  right: ReadonlyArray<string>,
+): number {
+  if (left.length === 0 || right.length === 0) return 0;
+  let longest = 0;
+  let previousRow = new Array<number>(right.length).fill(0);
+  for (const leftCharacter of left) {
+    const currentRow = new Array<number>(right.length).fill(0);
+    for (let index = 0; index < right.length; index += 1) {
+      if (leftCharacter !== right[index]) continue;
+      currentRow[index] = (index > 0 ? previousRow[index - 1] : 0) + 1;
+      if (currentRow[index] > longest) longest = currentRow[index];
+    }
+    previousRow = currentRow;
+  }
+  return longest;
+}
+
+/**
+ * Script-aware grounding score for candidates written in scripts without word
+ * separators. Returns null when the candidate has no such characters, so
+ * whitespace-delimited candidates keep the default scorer. The longest-common-
+ * span gate rejects topical overlap that shares mostly function characters.
+ * ponytail: n-grams and the common-span gate ignore argument order; swap in a
+ * segmentation-aware matcher if CJK role confusion surfaces.
+ */
+export function spacelessScriptGroundedTokenScore(candidate: string, source: string): number | null {
+  const candidateGrams = spacelessScriptCharacterNgrams(candidate);
+  if (candidateGrams.size === 0) return null;
+  const sourceGrams = spacelessScriptCharacterNgrams(source);
+  let sharedGrams = 0;
+  for (const gram of candidateGrams) {
+    if (sourceGrams.has(gram)) sharedGrams += 1;
+  }
+  if (sharedGrams < GROUNDING_MIN_SHARED_TOKENS) return 0;
+  const coverage = sharedGrams / candidateGrams.size;
+  if (coverage < GROUNDING_MIN_COVERAGE) return 0;
+  const candidateCharacters = spacelessScriptCharacterSequence(candidate);
+  const sourceCharacters = spacelessScriptCharacterSequence(source);
+  if (sourceCharacters.length <= SPACELESS_SCRIPT_LCS_SOURCE_LIMIT) {
+    const anchoredFraction = longestCommonSubstringLength(candidateCharacters, sourceCharacters)
+      / candidateCharacters.length;
+    if (anchoredFraction < GROUNDING_MIN_COVERAGE) return 0;
+  }
+  const sourceTokens = tokenize(source);
+  for (const token of tokenize(candidate)) {
+    if (containsSpacelessScriptCharacter(token)) continue;
+    if (![...sourceTokens].some((sourceToken) => areGroundingTokensCompatible(token, sourceToken))) {
+      return 0;
+    }
+  }
+  return coverage;
+}
+
 export function normalizeForExactMatch(text: string): string {
   return text.normalize("NFKC").toLocaleLowerCase().replace(/\s+/gu, " ").trim();
 }

package/src/extraction-source-grounding-rules.ts

@@ -14,12 +14,12 @@ import {
   groundingTokenSequence,
   hasContradictoryPolarity,
   hasExplicitRoleSubjectToken,
-  isAttachedNegatedAuxiliary,
   isInterrogativeSourceSentence,
   normalizeForExactMatch,
   normalizedGroundingAlignmentTokenSequence,
   normalizedGroundingTokenSequence,
   sourceSentences,
+  spacelessScriptGroundedTokenScore,
   splitGroundingClauses,
   stemToken,
   tokenize,
@@ -229,6 +229,8 @@ function groundedTokenScore(
   requireAlignedArguments = true,
   requireAllCandidateTokensGrounded = true,
 ): number {
+  const scriptAwareScore = spacelessScriptGroundedTokenScore(candidate, source);
+  if (scriptAwareScore !== null) return scriptAwareScore;
   const candidateTokens = tokenize(candidate);
   if (candidateTokens.size === 0) return 0;
   const sourceTokens = tokenize(source);