@remnic/core 9.64.4 → 9.65.1

package/dist/chunk-3WU7XOFN.js

@@ -0,0 +1,102 @@
+// src/cross-lingual-recall.ts
+var SCRIPT_RANGES = [
+  ["latn", 65, 90],
+  ["latn", 97, 122],
+  ["latn", 192, 591],
+  ["grek", 880, 1023],
+  ["cyrl", 1024, 1279],
+  ["cyrl", 1280, 1327],
+  ["hebr", 1424, 1535],
+  ["arab", 1536, 1791],
+  ["arab", 1872, 1919],
+  ["deva", 2304, 2431],
+  ["thai", 3584, 3711],
+  ["kore", 4352, 4607],
+  ["latn", 7680, 7935],
+  ["kana", 12352, 12447],
+  ["kana", 12448, 12543],
+  ["kana", 12784, 12799],
+  ["hani", 13312, 19903],
+  ["hani", 19968, 40959],
+  ["kore", 44032, 55203],
+  ["kana", 65382, 65437]
+];
+function detectLanguageHint(text) {
+  const counts = /* @__PURE__ */ new Map();
+  for (let i = 0; i < text.length; i += 1) {
+    const code = text.charCodeAt(i);
+    for (const [script, low, high] of SCRIPT_RANGES) {
+      if (code < low) break;
+      if (code <= high) {
+        counts.set(script, (counts.get(script) ?? 0) + 1);
+        break;
+      }
+    }
+  }
+  if (counts.has("kana")) return "jpan";
+  let best;
+  let bestCount = 0;
+  for (const [script, count] of counts) {
+    if (script === "kana") continue;
+    if (count > bestCount) {
+      best = script;
+      bestCount = count;
+    }
+  }
+  return best;
+}
+function corpusDominantLanguage(hints) {
+  const counts = /* @__PURE__ */ new Map();
+  for (const hint of hints) {
+    if (hint === void 0 || hint === "") continue;
+    counts.set(hint, (counts.get(hint) ?? 0) + 1);
+  }
+  let best;
+  let bestCount = 0;
+  for (const [hint, count] of counts) {
+    if (count > bestCount) {
+      best = hint;
+      bestCount = count;
+    }
+  }
+  return best;
+}
+function planCrossScript(input) {
+  const crossScript = input.queryLanguage !== void 0 && input.corpusLanguage !== void 0 && input.queryLanguage !== input.corpusLanguage;
+  return {
+    queryLanguage: input.queryLanguage,
+    corpusLanguage: input.corpusLanguage,
+    crossScript,
+    degradation: crossScript && !input.vectorTierAvailable ? {
+      backend: "qmd",
+      code: "vector_tier_unavailable",
+      detail: `cross-script query (${input.queryLanguage}) over a ${input.corpusLanguage} corpus: lexical recall cannot match across scripts \u2014 enable a multilingual embedding model (embedding fallback) so the vector tier can serve it`
+    } : void 0
+  };
+}
+var CORPUS_SAMPLE_CAP = 500;
+var corpusLanguageCache = /* @__PURE__ */ new Map();
+async function dominantCorpusLanguage(storage) {
+  try {
+    const version = storage.getCorpusScanVersion();
+    const cached = corpusLanguageCache.get(storage.dir);
+    if (cached?.version === version) return cached.language;
+    const memories = await storage.readAllMemories();
+    const language = corpusDominantLanguage(
+      memories.slice(0, CORPUS_SAMPLE_CAP).map((memory) => memory.frontmatter.language)
+    );
+    if (corpusLanguageCache.size > 8) corpusLanguageCache.clear();
+    corpusLanguageCache.set(storage.dir, { version, language });
+    return language;
+  } catch {
+    return void 0;
+  }
+}
+
+export {
+  detectLanguageHint,
+  corpusDominantLanguage,
+  planCrossScript,
+  dominantCorpusLanguage
+};
+//# sourceMappingURL=chunk-3WU7XOFN.js.map

package/dist/chunk-3WU7XOFN.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/cross-lingual-recall.ts"],"sourcesContent":["/**\n * Cross-lingual recall planner (issue #2197).\n *\n * Lexical tiers (BM25/FTS) match on shared surface tokens, so a query whose\n * dominant script differs from the corpus's cannot retrieve lexically — an\n * English query never token-overlaps a Japanese fact. When the planner\n * detects that mismatch it leans on the vector tier (the embedding-fallback\n * supplement in the recall search pipeline), and when NO vector tier exists\n * it emits a {@link SearchDegradation} so \"no results\" stays distinguishable\n * from \"cross-script query, vectors unavailable\" (rule 34).\n *\n * The write-time half is a dominant-script hint stamped into frontmatter\n * `language` by `StorageManager.writeMemory` (ISO 15924 script codes);\n * legacy files without the field simply do not vote for the corpus script.\n */\n\nimport type { MemoryFile } from \"./types.js\";\nimport type { SearchDegradation } from \"./search/port.js\";\n\ntype CorpusLanguageStorage = {\n  dir: string;\n  readAllMemories: () => Promise<MemoryFile[]>;\n  getCorpusScanVersion: () => string;\n};\n\n/** Script classes the cheap detector can distinguish. ISO 15924 codes. */\nexport type LanguageHint =\n  | \"latn\"\n  | \"jpan\"\n  | \"kore\"\n  | \"hani\"\n  | \"cyrl\"\n  | \"grek\"\n  | \"arab\"\n  | \"hebr\"\n  | \"thai\"\n  | \"deva\";\n\n/** Inclusive codepoint ranges per script class. `kana` folds into `jpan` at detection. */\nconst SCRIPT_RANGES: ReadonlyArray<readonly [LanguageHint | \"kana\", number, number]> = [\n  [\"latn\", 0x41, 0x5a],\n  [\"latn\", 0x61, 0x7a],\n  [\"latn\", 0xc0, 0x24f],\n  [\"grek\", 0x370, 0x3ff],\n  [\"cyrl\", 0x400, 0x4ff],\n  [\"cyrl\", 0x500, 0x52f],\n  [\"hebr\", 0x590, 0x5ff],\n  [\"arab\", 0x600, 0x6ff],\n  [\"arab\", 0x750, 0x77f],\n  [\"deva\", 0x900, 0x97f],\n  [\"thai\", 0xe00, 0xe7f],\n  [\"kore\", 0x1100, 0x11ff],\n  [\"latn\", 0x1e00, 0x1eff],\n  [\"kana\", 0x3040, 0x309f],\n  [\"kana\", 0x30a0, 0x30ff],\n  [\"kana\", 0x31f0, 0x31ff],\n  [\"hani\", 0x3400, 0x4dbf],\n  [\"hani\", 0x4e00, 0x9fff],\n  [\"kore\", 0xac00, 0xd7a3],\n  [\"kana\", 0xff66, 0xff9d],\n];\n\n/**\n * Dominant-script hint for a text. Codepoint-range counting (no regex):\n * kana anywhere maps to `jpan` — kana is unique to Japanese and disambiguates\n * it from `hani` (Chinese, or kanji-only Japanese). Returns `undefined` when\n * the text contains no lettered script class.\n */\nexport function detectLanguageHint(text: string): LanguageHint | undefined {\n  const counts = new Map<LanguageHint | \"kana\", number>();\n  for (let i = 0; i < text.length; i += 1) {\n    const code = text.charCodeAt(i);\n    for (const [script, low, high] of SCRIPT_RANGES) {\n      if (code < low) break;\n      if (code <= high) {\n        counts.set(script, (counts.get(script) ?? 0) + 1);\n        break;\n      }\n    }\n  }\n  // SCRIPT_RANGES is sorted by start codepoint, so `break` on the first\n  // range starting past the codepoint is exact — one range test per char.\n  if (counts.has(\"kana\")) return \"jpan\";\n  let best: LanguageHint | undefined;\n  let bestCount = 0;\n  for (const [script, count] of counts) {\n    if (script === \"kana\") continue;\n    if (count > bestCount) {\n      best = script;\n      bestCount = count;\n    }\n  }\n  return best;\n}\n\n/** Dominant hint over a corpus sample; `undefined` entries (legacy files) never vote. */\nexport function corpusDominantLanguage(\n  hints: Iterable<string | undefined>,\n): LanguageHint | undefined {\n  const counts = new Map<string, number>();\n  for (const hint of hints) {\n    if (hint === undefined || hint === \"\") continue;\n    counts.set(hint, (counts.get(hint) ?? 0) + 1);\n  }\n  let best: string | undefined;\n  let bestCount = 0;\n  for (const [hint, count] of counts) {\n    if (count > bestCount) {\n      best = hint;\n      bestCount = count;\n    }\n  }\n  return best as LanguageHint | undefined;\n}\n\n/** Planner decision for one recall query. */\nexport interface CrossScriptPlan {\n  queryLanguage: LanguageHint | undefined;\n  corpusLanguage: LanguageHint | undefined;\n  /** True when both hints resolved and differ — lexical tiers cannot serve this query. */\n  crossScript: boolean;\n  /** Emitted when `crossScript` is true and no vector tier can compensate. */\n  degradation?: SearchDegradation;\n}\n\n/**\n * Pure planner core: compare query/corpus hints and, on a mismatch without a\n * vector tier, build the visible degradation signal.\n */\nexport function planCrossScript(input: {\n  queryLanguage: LanguageHint | undefined;\n  corpusLanguage: LanguageHint | undefined;\n  vectorTierAvailable: boolean;\n}): CrossScriptPlan {\n  const crossScript =\n    input.queryLanguage !== undefined &&\n    input.corpusLanguage !== undefined &&\n    input.queryLanguage !== input.corpusLanguage;\n  return {\n    queryLanguage: input.queryLanguage,\n    corpusLanguage: input.corpusLanguage,\n    crossScript,\n    degradation:\n      crossScript && !input.vectorTierAvailable\n        ? {\n            backend: \"qmd\",\n            code: \"vector_tier_unavailable\",\n            detail:\n              `cross-script query (${input.queryLanguage}) over a ${input.corpusLanguage} corpus: ` +\n              \"lexical recall cannot match across scripts — enable a multilingual embedding model \" +\n              \"(embedding fallback) so the vector tier can serve it\",\n          }\n        : undefined,\n  };\n}\n\n/** Corpus sample cap: dominant script stabilizes well below the full corpus. */\nconst CORPUS_SAMPLE_CAP = 500;\n\n/** Memoized dominant corpus language per storage dir, keyed by corpus scan version. */\nconst corpusLanguageCache = new Map<string, { version: string; language: LanguageHint | undefined }>();\n\n/**\n * Dominant corpus language for a storage manager, sampled from the newest\n * frontmatter `language` hints (capped, memoized on the corpus scan version so\n * recurring recalls pay one pass per corpus generation, not per query).\n * Read failures degrade to `undefined` — never break recall.\n */\nexport async function dominantCorpusLanguage(\n  storage: CorpusLanguageStorage,\n): Promise<LanguageHint | undefined> {\n  try {\n    const version = storage.getCorpusScanVersion();\n    const cached = corpusLanguageCache.get(storage.dir);\n    if (cached?.version === version) return cached.language;\n    const memories = await storage.readAllMemories();\n    const language = corpusDominantLanguage(\n      memories.slice(0, CORPUS_SAMPLE_CAP).map((memory: MemoryFile) => memory.frontmatter.language),\n    );\n    if (corpusLanguageCache.size > 8) corpusLanguageCache.clear();\n    corpusLanguageCache.set(storage.dir, { version, language });\n    return language;\n  } catch {\n    return undefined;\n  }\n}\n"],"mappings":";AAuCA,IAAM,gBAAiF;AAAA,EACrF,CAAC,QAAQ,IAAM,EAAI;AAAA,EACnB,CAAC,QAAQ,IAAM,GAAI;AAAA,EACnB,CAAC,QAAQ,KAAM,GAAK;AAAA,EACpB,CAAC,QAAQ,KAAO,IAAK;AAAA,EACrB,CAAC,QAAQ,MAAO,IAAK;AAAA,EACrB,CAAC,QAAQ,MAAO,IAAK;AAAA,EACrB,CAAC,QAAQ,MAAO,IAAK;AAAA,EACrB,CAAC,QAAQ,MAAO,IAAK;AAAA,EACrB,CAAC,QAAQ,MAAO,IAAK;AAAA,EACrB,CAAC,QAAQ,MAAO,IAAK;AAAA,EACrB,CAAC,QAAQ,MAAO,IAAK;AAAA,EACrB,CAAC,QAAQ,MAAQ,IAAM;AAAA,EACvB,CAAC,QAAQ,MAAQ,IAAM;AAAA,EACvB,CAAC,QAAQ,OAAQ,KAAM;AAAA,EACvB,CAAC,QAAQ,OAAQ,KAAM;AAAA,EACvB,CAAC,QAAQ,OAAQ,KAAM;AAAA,EACvB,CAAC,QAAQ,OAAQ,KAAM;AAAA,EACvB,CAAC,QAAQ,OAAQ,KAAM;AAAA,EACvB,CAAC,QAAQ,OAAQ,KAAM;AAAA,EACvB,CAAC,QAAQ,OAAQ,KAAM;AACzB;AAQO,SAAS,mBAAmB,MAAwC;AACzE,QAAM,SAAS,oBAAI,IAAmC;AACtD,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK,GAAG;AACvC,UAAM,OAAO,KAAK,WAAW,CAAC;AAC9B,eAAW,CAAC,QAAQ,KAAK,IAAI,KAAK,eAAe;AAC/C,UAAI,OAAO,IAAK;AAChB,UAAI,QAAQ,MAAM;AAChB,eAAO,IAAI,SAAS,OAAO,IAAI,MAAM,KAAK,KAAK,CAAC;AAChD;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAGA,MAAI,OAAO,IAAI,MAAM,EAAG,QAAO;AAC/B,MAAI;AACJ,MAAI,YAAY;AAChB,aAAW,CAAC,QAAQ,KAAK,KAAK,QAAQ;AACpC,QAAI,WAAW,OAAQ;AACvB,QAAI,QAAQ,WAAW;AACrB,aAAO;AACP,kBAAY;AAAA,IACd;AAAA,EACF;AACA,SAAO;AACT;AAGO,SAAS,uBACd,OAC0B;AAC1B,QAAM,SAAS,oBAAI,IAAoB;AACvC,aAAW,QAAQ,OAAO;AACxB,QAAI,SAAS,UAAa,SAAS,GAAI;AACvC,WAAO,IAAI,OAAO,OAAO,IAAI,IAAI,KAAK,KAAK,CAAC;AAAA,EAC9C;AACA,MAAI;AACJ,MAAI,YAAY;AAChB,aAAW,CAAC,MAAM,KAAK,KAAK,QAAQ;AAClC,QAAI,QAAQ,WAAW;AACrB,aAAO;AACP,kBAAY;AAAA,IACd;AAAA,EACF;AACA,SAAO;AACT;AAgBO,SAAS,gBAAgB,OAIZ;AAClB,QAAM,cACJ,MAAM,kBAAkB,UACxB,MAAM,mBAAmB,UACzB,MAAM,kBAAkB,MAAM;AAChC,SAAO;AAAA,IACL,eAAe,MAAM;AAAA,IACrB,gBAAgB,MAAM;AAAA,IACtB;AAAA,IACA,aACE,eAAe,CAAC,MAAM,sBAClB;AAAA,MACE,SAAS;AAAA,MACT,MAAM;AAAA,MACN,QACE,uBAAuB,MAAM,aAAa,YAAY,MAAM,cAAc;AAAA,IAG9E,IACA;AAAA,EACR;AACF;AAGA,IAAM,oBAAoB;AAG1B,IAAM,sBAAsB,oBAAI,IAAqE;AAQrG,eAAsB,uBACpB,SACmC;AACnC,MAAI;AACF,UAAM,UAAU,QAAQ,qBAAqB;AAC7C,UAAM,SAAS,oBAAoB,IAAI,QAAQ,GAAG;AAClD,QAAI,QAAQ,YAAY,QAAS,QAAO,OAAO;AAC/C,UAAM,WAAW,MAAM,QAAQ,gBAAgB;AAC/C,UAAM,WAAW;AAAA,MACf,SAAS,MAAM,GAAG,iBAAiB,EAAE,IAAI,CAAC,WAAuB,OAAO,YAAY,QAAQ;AAAA,IAC9F;AACA,QAAI,oBAAoB,OAAO,EAAG,qBAAoB,MAAM;AAC5D,wBAAoB,IAAI,QAAQ,KAAK,EAAE,SAAS,SAAS,CAAC;AAC1D,WAAO;AAAA,EACT,QAAQ;AACN,WAAO;AAAA,EACT;AACF;","names":[]}

package/dist/{chunk-XH4IQVTN.js → chunk-4W6INNXO.js}

@@ -1,6 +1,6 @@
 import {
   resolveCorpusNamespaceRoots
-} from "./chunk-SB6YBB2V.js";
+} from "./chunk-53ENM6F6.js";
 import {
   capabilityAllowsNamespace
 } from "./chunk-MZWSOLMT.js";
@@ -200,4 +200,4 @@ async function readAggregateExtractionWatermark(options) {
 export {
   readAggregateExtractionWatermark
 };
-//# sourceMappingURL=chunk-XH4IQVTN.js.map
+//# sourceMappingURL=chunk-4W6INNXO.js.map

package/dist/{chunk-SB6YBB2V.js → chunk-53ENM6F6.js}

@@ -1,6 +1,6 @@
 import {
   listNamespaces
-} from "./chunk-CKZHKFY3.js";
+} from "./chunk-7QFBLYFP.js";
 import {
   capabilityAllowsNamespace
 } from "./chunk-MZWSOLMT.js";
@@ -282,4 +282,4 @@ export {
   computeServiceCorpusWatermarks,
   computeServiceCorpusCensus
 };
-//# sourceMappingURL=chunk-SB6YBB2V.js.map
+//# sourceMappingURL=chunk-53ENM6F6.js.map

package/dist/{chunk-7V3AZWWZ.js → chunk-5UYSCYR7.js}

@@ -9,23 +9,25 @@ import {
 } from "./chunk-666A3MOW.js";
 import {
   rebuildMemoryLifecycleLedger
-} from "./chunk-ZSYE2QVJ.js";
+} from "./chunk-GXWDWOWP.js";
 import {
   planNamespaceMaintenance,
   runNamespaceMaintenanceBatchPlan
-} from "./chunk-PE76XIDM.js";
+} from "./chunk-YEQEGSJA.js";
 import {
   rebuildMemoryProjection
-} from "./chunk-JCSUNXYJ.js";
+} from "./chunk-ESCQ6LY7.js";
+import {
+  validateActivityBaseUrl
+} from "./chunk-XAB6STPO.js";
 import {
   activityDayWindow,
   activityDigestPath,
   composeActivityDigestBody,
   composeActivityDigestMeta,
   isValidActivityDate,
-  serializeActivityDigest,
-  validateActivityBaseUrl
-} from "./chunk-7RAJ3GXO.js";
+  serializeActivityDigest
+} from "./chunk-M6L7HI7R.js";
 import {
   STATE_FILE_MAX_DECRYPT_BYTES,
   drainPendingLifecycleLedgerIfAny,
@@ -1719,4 +1721,4 @@ export {
   ActivitySyncScheduler,
   MaintenanceScheduler
 };
-//# sourceMappingURL=chunk-7V3AZWWZ.js.map
+//# sourceMappingURL=chunk-5UYSCYR7.js.map