@remnic/core 9.64.4 → 9.65.1

package/src/cross-lingual-recall.ts

@@ -0,0 +1,186 @@
+/**
+ * Cross-lingual recall planner (issue #2197).
+ *
+ * Lexical tiers (BM25/FTS) match on shared surface tokens, so a query whose
+ * dominant script differs from the corpus's cannot retrieve lexically — an
+ * English query never token-overlaps a Japanese fact. When the planner
+ * detects that mismatch it leans on the vector tier (the embedding-fallback
+ * supplement in the recall search pipeline), and when NO vector tier exists
+ * it emits a {@link SearchDegradation} so "no results" stays distinguishable
+ * from "cross-script query, vectors unavailable" (rule 34).
+ *
+ * The write-time half is a dominant-script hint stamped into frontmatter
+ * `language` by `StorageManager.writeMemory` (ISO 15924 script codes);
+ * legacy files without the field simply do not vote for the corpus script.
+ */
+
+import type { MemoryFile } from "./types.js";
+import type { SearchDegradation } from "./search/port.js";
+
+type CorpusLanguageStorage = {
+  dir: string;
+  readAllMemories: () => Promise<MemoryFile[]>;
+  getCorpusScanVersion: () => string;
+};
+
+/** Script classes the cheap detector can distinguish. ISO 15924 codes. */
+export type LanguageHint =
+  | "latn"
+  | "jpan"
+  | "kore"
+  | "hani"
+  | "cyrl"
+  | "grek"
+  | "arab"
+  | "hebr"
+  | "thai"
+  | "deva";
+
+/** Inclusive codepoint ranges per script class. `kana` folds into `jpan` at detection. */
+const SCRIPT_RANGES: ReadonlyArray<readonly [LanguageHint | "kana", number, number]> = [
+  ["latn", 0x41, 0x5a],
+  ["latn", 0x61, 0x7a],
+  ["latn", 0xc0, 0x24f],
+  ["grek", 0x370, 0x3ff],
+  ["cyrl", 0x400, 0x4ff],
+  ["cyrl", 0x500, 0x52f],
+  ["hebr", 0x590, 0x5ff],
+  ["arab", 0x600, 0x6ff],
+  ["arab", 0x750, 0x77f],
+  ["deva", 0x900, 0x97f],
+  ["thai", 0xe00, 0xe7f],
+  ["kore", 0x1100, 0x11ff],
+  ["latn", 0x1e00, 0x1eff],
+  ["kana", 0x3040, 0x309f],
+  ["kana", 0x30a0, 0x30ff],
+  ["kana", 0x31f0, 0x31ff],
+  ["hani", 0x3400, 0x4dbf],
+  ["hani", 0x4e00, 0x9fff],
+  ["kore", 0xac00, 0xd7a3],
+  ["kana", 0xff66, 0xff9d],
+];
+
+/**
+ * Dominant-script hint for a text. Codepoint-range counting (no regex):
+ * kana anywhere maps to `jpan` — kana is unique to Japanese and disambiguates
+ * it from `hani` (Chinese, or kanji-only Japanese). Returns `undefined` when
+ * the text contains no lettered script class.
+ */
+export function detectLanguageHint(text: string): LanguageHint | undefined {
+  const counts = new Map<LanguageHint | "kana", number>();
+  for (let i = 0; i < text.length; i += 1) {
+    const code = text.charCodeAt(i);
+    for (const [script, low, high] of SCRIPT_RANGES) {
+      if (code < low) break;
+      if (code <= high) {
+        counts.set(script, (counts.get(script) ?? 0) + 1);
+        break;
+      }
+    }
+  }
+  // SCRIPT_RANGES is sorted by start codepoint, so `break` on the first
+  // range starting past the codepoint is exact — one range test per char.
+  if (counts.has("kana")) return "jpan";
+  let best: LanguageHint | undefined;
+  let bestCount = 0;
+  for (const [script, count] of counts) {
+    if (script === "kana") continue;
+    if (count > bestCount) {
+      best = script;
+      bestCount = count;
+    }
+  }
+  return best;
+}
+
+/** Dominant hint over a corpus sample; `undefined` entries (legacy files) never vote. */
+export function corpusDominantLanguage(
+  hints: Iterable<string | undefined>,
+): LanguageHint | undefined {
+  const counts = new Map<string, number>();
+  for (const hint of hints) {
+    if (hint === undefined || hint === "") continue;
+    counts.set(hint, (counts.get(hint) ?? 0) + 1);
+  }
+  let best: string | undefined;
+  let bestCount = 0;
+  for (const [hint, count] of counts) {
+    if (count > bestCount) {
+      best = hint;
+      bestCount = count;
+    }
+  }
+  return best as LanguageHint | undefined;
+}
+
+/** Planner decision for one recall query. */
+export interface CrossScriptPlan {
+  queryLanguage: LanguageHint | undefined;
+  corpusLanguage: LanguageHint | undefined;
+  /** True when both hints resolved and differ — lexical tiers cannot serve this query. */
+  crossScript: boolean;
+  /** Emitted when `crossScript` is true and no vector tier can compensate. */
+  degradation?: SearchDegradation;
+}
+
+/**
+ * Pure planner core: compare query/corpus hints and, on a mismatch without a
+ * vector tier, build the visible degradation signal.
+ */
+export function planCrossScript(input: {
+  queryLanguage: LanguageHint | undefined;
+  corpusLanguage: LanguageHint | undefined;
+  vectorTierAvailable: boolean;
+}): CrossScriptPlan {
+  const crossScript =
+    input.queryLanguage !== undefined &&
+    input.corpusLanguage !== undefined &&
+    input.queryLanguage !== input.corpusLanguage;
+  return {
+    queryLanguage: input.queryLanguage,
+    corpusLanguage: input.corpusLanguage,
+    crossScript,
+    degradation:
+      crossScript && !input.vectorTierAvailable
+        ? {
+            backend: "qmd",
+            code: "vector_tier_unavailable",
+            detail:
+              `cross-script query (${input.queryLanguage}) over a ${input.corpusLanguage} corpus: ` +
+              "lexical recall cannot match across scripts — enable a multilingual embedding model " +
+              "(embedding fallback) so the vector tier can serve it",
+          }
+        : undefined,
+  };
+}
+
+/** Corpus sample cap: dominant script stabilizes well below the full corpus. */
+const CORPUS_SAMPLE_CAP = 500;
+
+/** Memoized dominant corpus language per storage dir, keyed by corpus scan version. */
+const corpusLanguageCache = new Map<string, { version: string; language: LanguageHint | undefined }>();
+
+/**
+ * Dominant corpus language for a storage manager, sampled from the newest
+ * frontmatter `language` hints (capped, memoized on the corpus scan version so
+ * recurring recalls pay one pass per corpus generation, not per query).
+ * Read failures degrade to `undefined` — never break recall.
+ */
+export async function dominantCorpusLanguage(
+  storage: CorpusLanguageStorage,
+): Promise<LanguageHint | undefined> {
+  try {
+    const version = storage.getCorpusScanVersion();
+    const cached = corpusLanguageCache.get(storage.dir);
+    if (cached?.version === version) return cached.language;
+    const memories = await storage.readAllMemories();
+    const language = corpusDominantLanguage(
+      memories.slice(0, CORPUS_SAMPLE_CAP).map((memory: MemoryFile) => memory.frontmatter.language),
+    );
+    if (corpusLanguageCache.size > 8) corpusLanguageCache.clear();
+    corpusLanguageCache.set(storage.dir, { version, language });
+    return language;
+  } catch {
+    return undefined;
+  }
+}

package/src/location/adapter-isolation.test.ts

@@ -0,0 +1,44 @@
+import assert from "node:assert/strict";
+import { readdirSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { test } from "node:test";
+
+/**
+ * Architecture boundary (issue #2045): `@remnic/core` must never statically
+ * depend on an optional provider package. Provider implementations register
+ * through the location registry at runtime; the connector package consumes
+ * core, never the reverse. A static import here would break à-la-carte
+ * installs (`npm install @remnic/core` alone must work) and drag a network
+ * adapter into every base install.
+ */
+const FORBIDDEN_SPECIFIERS = ["@remnic/connector-reitti", "connector-reitti/src"];
+
+// Production sources only: this test file itself names the banned specifier.
+function collectSourceFiles(dir: string, files: string[] = []): string[] {
+  for (const entry of readdirSync(dir, { withFileTypes: true })) {
+    const path = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      if (entry.name === "dist" || entry.name === "node_modules") continue;
+      collectSourceFiles(path, files);
+    } else if (entry.isFile() && /\.tsx?$/.test(entry.name) && !/\.test\.tsx?$/.test(entry.name)) {
+      files.push(path);
+    }
+  }
+  return files;
+}
+
+test("core source never statically imports @remnic/connector-reitti", () => {
+  // The whole core tree, not just src/location: a static import anywhere in
+  // core would break a base install.
+  const coreSrc = join(import.meta.dirname, "..");
+  const offenders: string[] = [];
+  for (const file of collectSourceFiles(coreSrc)) {
+    const content = readFileSync(file, "utf8");
+    for (const specifier of FORBIDDEN_SPECIFIERS) {
+      if (content.includes(specifier)) {
+        offenders.push(`${file} references ${specifier}`);
+      }
+    }
+  }
+  assert.deepEqual(offenders, []);
+});

package/src/location/index.ts

@@ -0,0 +1,16 @@
+/**
+ * Location context subsystem — public surface (issues #2044, #2045).
+ *
+ * Consumed by host adapters and optional provider packages via the
+ * `@remnic/core/location` subpath export; the root index stays frozen at its
+ * ratchet ceiling, and core never statically imports a provider package
+ * (register implementations through `registerLocationProvider` instead).
+ */
+
+export * from "./types.js";
+export * from "./registry.js";
+export * from "./config.js";
+export * from "./intervals.js";
+export * from "./store.js";
+export * from "./pipeline.js";
+export * from "./cli.js";

package/src/orchestration/recall-internal-deps.ts

@@ -219,6 +219,8 @@ export interface RecallInternalDeps {
       searchOptions?: SearchQueryOptions;
       onDebugSnapshot?: (snapshot: QmdRecallSnapshot) => Promise<void>;
       /** Backend degradation observer, threaded into every QMD call (#1536). */
+      /** Cross-lingual plan flag (#2197): supplement the lexical page with vector-tier hits. */
+      crossScript?: boolean;
       onDegradation?: (degradation: SearchDegradation) => void;
       abortSignal?: AbortSignal;
     },

package/src/orchestration/recall-internal.ts

@@ -25,6 +25,7 @@ import path from "node:path";
 import { type BoxFrontmatter } from "../boxes.js";
 import { type CapabilitySet, type GraphConstructionCapabilitySet, type MemoryLifecycleCapabilitySet, type SecurityCapabilitySet, resolveCapabilities, resolveCompressionCapabilities, resolveConsolidationCapabilities, resolveConversationContextCapabilities, resolveCreationMemoryCapabilities, resolveGraphConstructionCapabilities, resolveIdentityContinuityCapabilities, resolveIndexingCapabilities, resolveMemoryLifecycleCapabilities, resolveNamespaceCapabilities, resolveObjectiveStateCapabilities, resolvePipelineProcessingCapabilities, resolvePresentationCapabilities, resolveQmdCapabilities, resolveRecallAuxiliaryCapabilities, resolveRecallEnhancementCapabilities, resolveSecurityCapabilities } from "../capabilities.js";
 import { searchCausalTrajectories } from "../causal-trajectory.js";
+import { detectLanguageHint, dominantCorpusLanguage, planCrossScript } from "../cross-lingual-recall.js";
 import { buildEntityRecallSection, entityRecentTranscriptLookbackHours, readRecentEntityTranscriptEntries } from "../entity-retrieval.js";
 import { buildEventOrderRecallSection, shouldRecallEventOrderEvidence } from "../event-order-recall.js";
 import { buildExplicitCueRecallSection } from "../explicit-cue-recall.js";
@@ -261,34 +262,22 @@ export class RecallInternalCoordinator {
     // per-result explain data (e.g. reinforcementBoost) from the result that
     // was actually served.
     let xrayRecalledResults: QmdSearchResult[] = [];
-    // Issue #1577 — per-recall trust map (admitted + quarantined items).
-    // A LOCAL, not instance state, so concurrent recalls on the same
-    // orchestrator cannot race on it (review: shared-trust-map concurrency).
-    // Populated by applyTrustScoreRerank on whichever recall path runs;
-    // consumed by formatQmdResults (epistemic hedge), publishRecallResults
-    // (quarantine filtering on ALL paths), and the X-ray capture (trust
-    // projection + quarantined-item visibility — rule 34).
+    // Issue #1577 — per-recall trust map (admitted + quarantined). LOCAL, not
+    // instance state, so concurrent recalls cannot race on it. Consumed by
+    // formatQmdResults, publishRecallResults (quarantine filtering on ALL
+    // paths), and the X-ray capture (rule 34).
     let recallTrustByPath: Map<string, TrustStageResultItem> | null = null;
-    // Issue #1577 — sink for the cold-fallback pipeline's trust map. The cold
-    // pipeline scores internally (applyColdFallbackPipeline) and writes its
-    // per-path trust map here; each cold caller reads it back into
-    // recallTrustByPath so cold/embedding/recent paths feed the same X-ray +
-    // epistemic-rendering + publisher-quarantine consumers the hot path uses
-    // (rule 41: the feature gate applies across ALL parallel recall paths).
+    // Issue #1577 — sink for the cold pipeline's trust map: cold callers read
+    // it back into recallTrustByPath so every path feeds the same consumers
+    // (rule 41 parity).
     const trustByPathSink: { trustByPath: Map<string, TrustStageResultItem> | null } = {
       trustByPath: null,
     };
     const lcmStructuredXrayResults: RecallXrayResult[] = [];
-    // Per-branch pre-limit candidate pool size for the X-ray filter
-    // trace (issue #570 PR 1).  `recalledMemoryCount` is assigned
-    // AFTER MMR + truncation so using it alone would make the
-    // `recall-result-limit` trace report `considered == admitted`
-    // even when many candidates were dropped.  Each entry captures
-    // the pool size BEFORE truncation at that branch.  The X-ray
-    // capture block picks the pool that corresponds to the branch
-    // that actually produced the admitted results (via `recallSource`)
-    // so a pool from a branch whose candidates were killed by an
-    // earlier gate cannot leak into the `considered` count.
+    // Per-branch PRE-truncation pool size for the X-ray filter trace
+    // (issue #570 PR 1): `recalledMemoryCount` is post-truncation, so the
+    // trace would otherwise report considered == admitted. The capture block
+    // picks the branch that produced the admitted results via `recallSource`.
     const xrayBranchPoolSize: Record<
       "hot_qmd" | "hot_embedding" | "cold_fallback" | "recent_scan",
       number
@@ -298,11 +287,9 @@ export class RecallInternalCoordinator {
       cold_fallback: 0,
       recent_scan: 0,
     };
-    // Shared out-parameter sink the cold-fallback pipeline writes
-    // its pre-truncation pool size into (issue #570 PR 1).  Declared
-    // once so every call to `applyColdFallbackPipeline` (four call
-    // sites) updates the same counter; the X-ray capture block
-    // reads this as the cold-fallback pool.
+    // Sink the cold-fallback pipeline (four call sites) writes its
+    // pre-truncation pool size into (issue #570 PR 1); the X-ray capture
+    // block reads it as the cold-fallback pool.
     const xrayColdPoolSink = { size: 0 };
     let selectedResultPartition: RecallResultPartition | null = null;
     let identityInjectionModeUsed: IdentityInjectionMode | "none" = "none";
@@ -1908,6 +1895,19 @@ export class RecallInternalCoordinator {
       degradations?: SearchDegradation[];
     } | null;
 
+    // Cross-lingual recall (#2197): computed beside the prefilter; never
+    // throws — a failed corpus sample only skips the plan.
+    const crossScriptPlanPromise = (async () => {
+      const queryLanguage = detectLanguageHint(retrievalQuery);
+      if (queryLanguage === undefined) return null;
+      const corpusLanguage = await dominantCorpusLanguage(this.deps.storage);
+      if (corpusLanguage === undefined) return null;
+      return planCrossScript({
+        queryLanguage,
+        corpusLanguage,
+        vectorTierAvailable: resolveMemoryLifecycleCapabilities(this.deps.config).embeddingFallback,
+      });
+    })().catch(() => null);
     const qmdEnrichmentAbort = createEnrichmentAbortHandle(options.abortSignal);
     const qmdPromise = observeEnrichmentPromise(
       (async (): Promise<QmdPhaseResult> => {
@@ -1989,7 +1989,11 @@ export class RecallInternalCoordinator {
           }
           return cachedQmd.value;
         }
-
+        const crossScriptPlan = await crossScriptPlanPromise;
+        if (crossScriptPlan?.degradation) {
+          backendDegradations.push(crossScriptPlan.degradation);
+        }
+        const crossScript = crossScriptPlan?.crossScript === true;
         if (!this.deps.qmd.isAvailable()) {
           const now = Date.now();
           const QMD_REPROBE_COOLDOWN_MS = 60_000;
@@ -2141,6 +2145,7 @@ export class RecallInternalCoordinator {
                 resolveNamespace: (p) => this.deps.namespaceFromPath(p),
                 queryAwarePrefilter,
                 searchOptions: qmdSearchOptions,
+                crossScript,
                 abortSignal: qmdEnrichmentAbort.signal,
                 onDegradation: (degradation) => {
                   backendDegradations.push(degradation);
@@ -4105,8 +4110,6 @@ export class RecallInternalCoordinator {
               undefined,
               { asOfMs, requestingConnector: options.sourceConnector },
             );
-            // MMR runs on the pre-truncation pool so diverse candidates just
-            // below the cutoff can be promoted into the injected set.
             xrayBranchPoolSize.hot_embedding = Math.max(
               xrayBranchPoolSize.hot_embedding,
               boostedScoped.length,
@@ -4319,8 +4322,7 @@ export class RecallInternalCoordinator {
               undefined,
               { asOfMs, requestingConnector: options.sourceConnector },
             );
-            // MMR runs on the pre-truncation pool so diverse candidates just
-            // below the cutoff can be promoted into the injected set.
+            // MMR runs on the pre-truncation pool (see the hot-embedding branch).
             xrayBranchPoolSize.hot_embedding = Math.max(
               xrayBranchPoolSize.hot_embedding,
               boostedScoped.length,

package/src/orchestration/recall-search-pipeline.ts

@@ -281,6 +281,8 @@ export class RecallSearchPipelineCoordinator {
       searchOptions?: SearchQueryOptions;
       onDebugSnapshot?: (snapshot: QmdRecallSnapshot) => Promise<void>;
       /** Backend degradation observer, threaded into every QMD call (#1536). */
+      /** Cross-lingual plan flag (#2197): supplement the lexical page with vector-tier hits. */
+      crossScript?: boolean;
       onDegradation?: (degradation: SearchDegradation) => void;
       abortSignal?: AbortSignal;
     },
@@ -334,6 +336,12 @@ export class RecallSearchPipelineCoordinator {
     const debugSearchOptions = backendHonorsQmdSearchSignals
       ? resolvedSearchOptions
       : undefined;
+    // Cross-lingual recall (#2197): lexical tiers cannot match a query whose
+    // dominant script differs from the corpus's, whatever the page fill.
+    // Supplement with the embedding-fallback tier ONCE, before widening.
+    const crossScriptVectorHits = options.crossScript
+      ? await this.searchEmbeddingFallback(prompt, fetchLimit)
+      : [];
     let bestFiltered = filterRecallCandidates(scopedSeedResults, {
       namespacesEnabled: options.namespacesEnabled,
       recallNamespaces: options.recallNamespaces,
@@ -475,6 +483,14 @@ export class RecallSearchPipelineCoordinator {
         }
       }
 
+      if (crossScriptVectorHits.length > 0) {
+        mergedResults = dedupeResultsByNamespace(
+          [...mergedResults, ...crossScriptVectorHits],
+          this.deps.namespaceFromPath,
+          fetchLimit,
+          { filter: recallable },
+        );
+      }
       if (scopedSeedResults.length > 0) {
         mergedResults = dedupeResultsByNamespace(
           [...scopedSeedResults, ...mergedResults],

package/src/search/port.ts

@@ -27,7 +27,8 @@ export interface SearchDegradation {
     | "daemon_loading"
     | "subprocess_error"
     | "deadline_exceeded"
-    | "remote_error";
+    | "remote_error"
+    | "vector_tier_unavailable";
   detail?: string;
 }
 

package/src/storage.ts

@@ -244,6 +244,7 @@ import { normalizeProjectionPreview, normalizeProjectionTags } from "./memory-pr
 import { isSupportPassportPrivateMemory } from "./support-passport/card-projection.js";
 import { parseFlexibleIsoTimestamp } from "./utils/iso-timestamp.js";
 import { parseOriginClass } from "./security/origin-authority.js";
+import { detectLanguageHint } from "./cross-lingual-recall.js";
 import {
   composeMemoryEnvelope,
   isSealedMemoryEnvelope,
@@ -322,6 +323,7 @@ function serializeFrontmatter(fm: MemoryFrontmatter): string {
     `tags: [${fm.tags.map((t) => `"${t}"`).join(", ")}]`,
   ];
   // OKF v0.1 interop (issue #1946): inert presentation metadata; `category` stays canonical.
+  if (fm.language) lines.push(`language: ${fm.language}`);
   if (fm.type !== undefined) lines.push(`type: ${fm.type}`);
   if (fm.origin) lines.push(`origin: ${/^[A-Za-z0-9:_.*-]+$/.test(fm.origin) ? fm.origin : JSON.stringify(fm.origin)}`);
   if (fm.entityRef) lines.push(`entityRef: ${fm.entityRef}`);
@@ -859,8 +861,9 @@ export function parseFrontmatter(raw: string): { frontmatter: MemoryFrontmatter;
     frontmatter: {
       id: fm.id ?? "",
       category: (fm.category ?? "fact") as MemoryCategory,
-      type: parseFrontmatterStringValue(fm.type),
       created: fm.created ?? new Date().toISOString(),
+      language: fm.language || undefined,
+      type: parseFrontmatterStringValue(fm.type),
       updated: fm.updated ?? new Date().toISOString(),
       source: fm.source ?? "unknown",
       confidence: conf,
@@ -3729,6 +3732,7 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
       category,
       created: now.toISOString(),
       updated: now.toISOString(),
+      language: detectLanguageHint(content),
       source: options.source ?? "extraction",
       confidence: conf,
       confidenceTier: tier,

package/src/types.ts

@@ -2807,6 +2807,8 @@ export interface MemoryFrontmatter extends SourceConnectorProvenance, OriginMeta
   category: MemoryCategory;
   /** OKF v0.1 interop metadata (issue #1946). Inert: presentation only, never overrides `category`. */
   type?: string;
+  /** Dominant-script hint (ISO 15924, e.g. `latn`/`jpan`) for cross-lingual recall (#2197). Absent on legacy files. */
+  language?: string;
   created: string;
   updated: string;
   source: string;