@remnic/core 9.27.0 → 9.29.0

package/dist/verified-recall.js

@@ -1,32 +1,34 @@
 import {
   compareVerifiedEpisodeResults,
   searchVerifiedEpisodes
-} from "./chunk-UT3OQPFA.js";
+} from "./chunk-KUJQHXPH.js";
 import "./chunk-B7RWBZXW.js";
-import "./chunk-CIQXI47W.js";
+import "./chunk-FCDWWCLX.js";
 import "./chunk-UHVXE24Y.js";
 import "./chunk-2QMKMVHH.js";
 import "./chunk-DEBHXHJR.js";
 import "./chunk-DNHVPZOD.js";
 import "./chunk-6CIAGZ6X.js";
-import "./chunk-HKF5VASQ.js";
 import "./chunk-J6A3CX5N.js";
-import "./chunk-V25OWXAC.js";
-import "./chunk-XKGPIWGI.js";
+import "./chunk-HKF5VASQ.js";
+import "./chunk-HUHX2OCX.js";
+import "./chunk-OFHFH655.js";
 import "./chunk-EGGE52UU.js";
-import "./chunk-GCASGYIO.js";
 import "./chunk-WL5EBRDH.js";
-import "./chunk-ILGC6JH6.js";
-import "./chunk-SCU65EZI.js";
+import "./chunk-WRJVB3C2.js";
+import "./chunk-GCASGYIO.js";
 import "./chunk-3CSX3ETE.js";
 import "./chunk-RL6NEVXT.js";
+import "./chunk-ILGC6JH6.js";
+import "./chunk-SCU65EZI.js";
 import "./chunk-DM2T26WE.js";
 import "./chunk-LDXUBPMO.js";
+import "./chunk-JYIJNNN3.js";
+import "./chunk-AAD2ZS6P.js";
 import "./chunk-CGASUJVP.js";
 import "./chunk-R3DJVOSM.js";
 import "./chunk-BT7NVCML.js";
-import "./chunk-JYIJNNN3.js";
-import "./chunk-AAD2ZS6P.js";
+import "./chunk-YXZG5TVN.js";
 import "./chunk-5VQJBH2G.js";
 import "./chunk-AITCS2RC.js";
 import "./chunk-XMMESCPD.js";
@@ -41,8 +43,8 @@ import "./chunk-45GF7Y3V.js";
 import "./chunk-RGMVMVMF.js";
 import "./chunk-N7RWREIQ.js";
 import "./chunk-5I4M26Q7.js";
+import "./chunk-ZIKENA2C.js";
 import "./chunk-5GKPFL3S.js";
-import "./chunk-4DJQYKMN.js";
 import "./chunk-PQVCH2AE.js";
 import "./chunk-EYIEWJNI.js";
 import "./chunk-JUC24CTX.js";
@@ -53,7 +55,7 @@ import "./chunk-BJMBJZ2Y.js";
 import "./chunk-UKJAGEXH.js";
 import "./chunk-FP2373TW.js";
 import "./chunk-P7FMDTKL.js";
-import "./chunk-RDQGLJRR.js";
+import "./chunk-2XRE6FLM.js";
 import "./chunk-FF4KLI5W.js";
 import "./chunk-AIK5W3IW.js";
 import "./chunk-A6XUJE5D.js";

package/dist/write-envelope.d.ts

@@ -1,4 +1,4 @@
-import { t as MemoryCategory } from './types-DHBxIF0s.js';
+import { v as MemoryCategory } from './types-Dg0w9ADr.js';
 import './message-parts/index.js';
 import './bounded-jsonl-state.js';
 import './operator-doctor-types.js';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remnic/core",
-  "version": "9.27.0",
+  "version": "9.29.0",
   "description": "Framework-agnostic Remnic memory engine — orchestrator, storage, extraction, search, trust zones",
   "type": "module",
   "main": "dist/index.js",
@@ -3076,7 +3076,7 @@
     "core"
   ],
   "peerDependencies": {
-    "@remnic/coding-graph": "^9.27.0"
+    "@remnic/coding-graph": "^9.29.0"
   },
   "peerDependenciesMeta": {
     "@remnic/coding-graph": {

package/src/access-health-types.ts

@@ -10,6 +10,7 @@
  */
 import type { ExtractionLivenessStatus } from "./extraction-liveness.js";
 import type { CorpusWatermark } from "./corpus-watermark.js";
+import type { ReplicaDivergenceStatus } from "./replica-divergence.js";
 
 export interface EngramAccessHealthResponse {
   ok: true;
@@ -23,6 +24,13 @@ export interface EngramAccessHealthResponse {
   projectionAvailable: boolean;
   extraction: ExtractionLivenessStatus;
   corpus: CorpusWatermark[];
+  /**
+   * Whether `corpus` above is the COMPLETE census for this responder, from the
+   * same call that produced it. A polling peer must refuse to certify
+   * convergence against a partial array (issue #2149).
+   */
+  corpusComplete: boolean;
+  replica: ReplicaDivergenceStatus;
 }
 
 export type EngramAccessQmdCollectionState =

package/src/access-http.test.ts

@@ -1870,6 +1870,8 @@ test("HTTP externalRequestHandler runs pre-auth, can end responses, and falls th
         watermarkScope: "root-store",
       },
       corpus: [],
+      corpusComplete: true,
+      replica: { enabled: false, pending: false, polledAt: null, peers: [] },
     }),
   } satisfies Pick<EngramAccessService, "health">;
   const service = healthStub as unknown as EngramAccessService;
@@ -1995,6 +1997,8 @@ test("HTTP authTokenEntriesGetter is authoritative: scope policy binds connector
         watermarkScope: "root-store",
       },
       corpus: [],
+      corpusComplete: true,
+      replica: { enabled: false, pending: false, polledAt: null, peers: [] },
     }),
   } satisfies Pick<EngramAccessService, "health">;
   const entries = [

package/src/access-service-health.test.ts

@@ -108,6 +108,29 @@ test("health reports active QMD version and collection state", async () => {
   }
 });
 
+test("health reports replica divergence disabled by default (no peers -> behaves as today)", async () => {
+  const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-health-replica-"));
+  try {
+    const config = parseConfig({ memoryDir });
+    const service = new EngramAccessService({
+      config,
+      qmd: makeQmd({}),
+      async getStorage() {
+        return { dir: memoryDir };
+      },
+    } as unknown as Orchestrator);
+
+    const health = await service.health();
+
+    assert.equal(health.replica.enabled, false);
+    assert.equal(health.replica.pending, false);
+    assert.equal(health.replica.polledAt, null);
+    assert.equal(health.replica.peers.length, 0);
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+
 test("health treats qmdEmbeddingBacklogThreshold 0 as backlog degradation disabled", async () => {
   const config = parseConfig({ qmdEnabled: true, qmdEmbeddingBacklogThreshold: 0 });
   const qmd = makeQmd({

package/src/access-service.ts

@@ -19,7 +19,9 @@ import { resolveNamespaceCapabilities,
   resolveMemoryLifecycleCapabilities,
   resolveQmdCapabilities,
   resolveSecurityCapabilities, resolveObjectiveStateCapabilities, resolveCompressionCapabilities, resolveRecallAuxiliaryCapabilities } from "./capabilities.js";
-import { CorpusWatermarkCache, computeServiceCorpusWatermarks } from "./corpus-watermark.js";
+import { CorpusWatermarkCache, computeServiceCorpusCensus } from "./corpus-watermark.js";
+import { ReplicaDivergenceMonitor } from "./replica-divergence.js";
+import type { ResolveSecretRefFn } from "./resolve-auth-token.js";
 import type {
   EngramAccessHealthResponse,
   EngramAccessQmdCollectionState,
@@ -1277,6 +1279,7 @@ export class EngramAccessService {
   private readonly budget: CrossNamespaceBudget;
   private readonly auditAdapter: AccessAuditAdapter | null;
   private readonly corpusWatermarkCache = new CorpusWatermarkCache();
+  private readonly replicaDivergenceMonitor: ReplicaDivergenceMonitor;
 
   /** AccessObserveWriteSurface (access-service decomposition). Lazy; selfDeps live wiring. */
   private _accessObserveWriteSurface: AccessObserveWriteSurface | undefined;
@@ -1339,8 +1342,13 @@ export class EngramAccessService {
   }
   private readonly extractionLivenessWarn = new ExtractionLivenessWarnThrottle();
 
-  constructor(private readonly orchestrator: Orchestrator) {
+  constructor(private readonly orchestrator: Orchestrator, options: { resolveSecretRef?: ResolveSecretRefFn | null } = {}) {
     this.idempotency = new AccessIdempotencyStore(orchestrator.config.memoryDir);
+    // Peer SecretRef tokens resolve at poll time through the host resolver, the
+    // same indirection as agentAccessHttp.authToken (review round 1). Absent a
+    // resolver, string/${ENV} tokens still work and a SecretRef degrades to a
+    // per-peer `unreachable` (never a throw).
+    this.replicaDivergenceMonitor = new ReplicaDivergenceMonitor({ resolveSecretRef: options.resolveSecretRef });
     const accessCaps = resolveAccessSetupCapabilities(orchestrator.config); // #1566 Cluster B
     this.budget = new CrossNamespaceBudget({
       enabled: accessCaps.recallCrossNamespaceBudget,
@@ -2432,6 +2440,11 @@ export class EngramAccessService {
     const searchBackend = this.orchestrator.config.searchBackend ?? "qmd";
     const qmdEnabled = resolveQmdCapabilities(this.orchestrator.config).qmd === true;
     const extraction = await computeExtractionLivenessStatus(this.orchestrator, this.extractionLivenessWarn);
+    // ONE call: the corpus array and the flag describing it must not come from
+    // two independently-cached scans (round 8, codex P1).
+    const caps = tokenCapabilityStore.getStore();
+    const corpusCensus = await computeServiceCorpusCensus(this.orchestrator, {
+      cache: this.corpusWatermarkCache, caps });
     let projectionAvailable = false;
     try {
       await stat(getMemoryProjectionPath(storage.dir));
@@ -2456,8 +2469,18 @@ export class EngramAccessService {
       nativeKnowledgeEnabled: this.orchestrator.config.nativeKnowledge?.enabled === true,
       projectionAvailable,
       extraction,
-      corpus: await computeServiceCorpusWatermarks(this.orchestrator, {
-        cache: this.corpusWatermarkCache, caps: tokenCapabilityStore.getStore() }),
+      corpus: corpusCensus.watermarks,
+      // Describes THIS response's array (round 8): a separate scan could report
+      // complete while the shipped corpus was partial, and peers trust it.
+      corpusComplete: corpusCensus.complete,
+      replica: this.replicaDivergenceMonitor.getReport({
+        config: this.orchestrator.config.replicaPeers,
+        // Capability-INDEPENDENT: this poll is cached instance-wide, so a
+        // restricted caller's filtered census must never seed it (round 9).
+        computeLocalWatermarks: () => computeServiceCorpusCensus(this.orchestrator, {}),
+        caps,
+        localCensusComplete: corpusCensus.complete,
+      }),
     };
   }
 

package/src/capabilities.test.ts

@@ -1246,6 +1246,7 @@ const PIPELINE_FIELD_TO_FLAG: Record<keyof PipelineProcessingCapabilitySet, stri
   lcm: "lcmEnabled",
   localLlmFast: "localLlmFastEnabled",
   proactiveExtraction: "proactiveExtractionEnabled",
+  sourceGrounding: "extractionSourceGroundingEnabled",
   delinearize: "delinearizeEnabled",
   slowLog: "slowLogEnabled",
   hostEmbeddingProvider: "hostEmbeddingProviderEnabled",

package/src/capabilities.ts

@@ -1132,13 +1132,10 @@ export function resolveRecallEnhancementCapabilities(config: RecallEnhancementCo
 }
 
 // ---------------------------------------------------------------------------
-// PipelineProcessing capability set (issue #1523 batch 9).
-//
-// These 22 flags gate the extraction/chunking/summarization pipeline and LLM
-// infrastructure. Read sites span orchestrator, extraction, summarizer,
-// embedding-fallback, local-llm, search, day-summary, and semantic-consolidation.
-// All flags are non-optional booleans on PluginConfig (defaults resolved at
-// the parse boundary), so the projection is a pure pass-through.
+// PipelineProcessing capability set (issue #1523 batch 9). These 23 flags gate
+// extraction/chunking/summarization and LLM infrastructure across orchestrator,
+// extraction, summarizer, embedding-fallback, local-llm, search, day-summary, and
+// semantic-consolidation. Defaults resolve at parse boundary, so pure pass-through.
 // ---------------------------------------------------------------------------
 
 /**
@@ -1161,6 +1158,7 @@ export interface PipelineProcessingCapabilitySet {
   readonly lcm: boolean;
   readonly localLlmFast: boolean;
   readonly proactiveExtraction: boolean;
+  readonly sourceGrounding: boolean;
   readonly delinearize: boolean;
   readonly slowLog: boolean;
   readonly hostEmbeddingProvider: boolean;
@@ -1189,6 +1187,7 @@ export type PipelineProcessingConfigProjection = Pick<
   | "lcmEnabled"
   | "localLlmFastEnabled"
   | "proactiveExtractionEnabled"
+  | "extractionSourceGroundingEnabled"
   | "delinearizeEnabled"
   | "slowLogEnabled"
   | "hostEmbeddingProviderEnabled"
@@ -1217,6 +1216,7 @@ export function resolvePipelineProcessingCapabilities(config: PipelineProcessing
     lcm: config.lcmEnabled,
     localLlmFast: config.localLlmFastEnabled,
     proactiveExtraction: config.proactiveExtractionEnabled,
+    sourceGrounding: config.extractionSourceGroundingEnabled,
     delinearize: config.delinearizeEnabled,
     slowLog: config.slowLogEnabled,
     hostEmbeddingProvider: config.hostEmbeddingProviderEnabled,

package/src/cli.ts

@@ -115,6 +115,8 @@ import {
   renderActionConfidenceText,
 } from "./action-confidence.js";
 import {
+  hostSecretRefResolver,
+  loadHostSecretRefResolver,
   resolveAgentAccessAuthToken,
   type ResolveSecretRefFn,
 } from "./resolve-auth-token.js";
@@ -3797,7 +3799,8 @@ export function registerCli(
         .option("--json", "Emit machine-readable JSON only")
         .action(async (...args: unknown[]) => {
           const options = (args[0] ?? {}) as Record<string, unknown>;
-          const report = await runOperatorDoctor({ orchestrator });
+          const resolveSecretRef = await loadHostSecretRefResolver(registerOptions);
+          const report = await runOperatorDoctor({ orchestrator, resolveSecretRef });
           if (reportHasMachineReadableOutput(options)) {
             console.log(JSON.stringify(report, null, 2));
           } else {
@@ -6767,7 +6770,10 @@ export function registerCli(
           console.log("OK");
         });
 
-      const accessService = new EngramAccessService(orchestrator);
+      // /health's replica monitor resolves peer-token SecretRefs at poll time (#2149).
+      const accessService = new EngramAccessService(orchestrator, {
+        resolveSecretRef: hostSecretRefResolver(registerOptions),
+      });
       const accessCmd = cmd
         .command("access")
         .description("Manage Engram HTTP and MCP access surfaces");
@@ -6796,11 +6802,7 @@ export function registerCli(
             typeof options.token === "string" && options.token.trim().length > 0
               ? options.token
               : undefined;
-          const resolveSecretRef =
-            registerOptions.resolveSecretRef ??
-            (registerOptions.loadResolveSecretRef
-              ? await registerOptions.loadResolveSecretRef()
-              : null);
+          const resolveSecretRef = await loadHostSecretRefResolver(registerOptions);
           const resolvedConfigAuthToken = cliTokenOverride
             ? undefined
             : await resolveAgentAccessAuthToken(

package/src/config.test.ts

@@ -907,6 +907,11 @@ test("parseConfig validates localLlmMaxContext as a usable context window", () =
     );
   }
 });
+test("parseConfig extractionSourceGroundingEnabled defaults on and accepts string false", () => {
+  assert.equal(parseConfig({}).extractionSourceGroundingEnabled, true);
+  assert.equal(parseConfig({ extractionSourceGroundingEnabled: "false" }).extractionSourceGroundingEnabled, false);
+});
+
 test("parseConfig extractionTelemetryPrefilterEnabled defaults on and accepts string false", () => {
   assert.equal(parseConfig({}).extractionTelemetryPrefilterEnabled, true);
   assert.equal(parseConfig({ extractionTelemetryPrefilterEnabled: "false" }).extractionTelemetryPrefilterEnabled, false);

package/src/config.ts

@@ -38,6 +38,7 @@ import { expandTildePath } from "./utils/path.js";
 import { coerceBool, coerceBooleanLike, coerceInstallExtension, coerceNumber } from "./connectors/coerce.js";
 import { parseRecallConcurrencyConfig } from "./recall-concurrency-config.js";
 import { parseExtractionLivenessConfig } from "./extraction-liveness.js";
+import { parseReplicaPeersConfig } from "./replica-peers-config.js";
 import { hasLegacyConnectorEntries } from "./connectors/paths.js";
 import {
   parseQmdChunkStrategy,
@@ -72,7 +73,6 @@ const DEFAULT_MEMORY_DIR = path.join(
   "memory",
   "local",
 );
-
 const DEFAULT_WORKSPACE_DIR = path.join(
   resolveHomeDir(),
   ".openclaw",
@@ -2335,6 +2335,7 @@ export function parseConfig(
     dreamsPhases,
     procedural,
     extractionLiveness: parseExtractionLivenessConfig(cfg),
+    replicaPeers: parseReplicaPeersConfig(cfg),
     wearables,
     meetings,
     activity,
@@ -2349,7 +2350,6 @@ export function parseConfig(
     codexCompat,
     codingKnowledge: parseCodingKnowledgeConfig(cfg.codingKnowledge),
     chat: parseChatConfig(cfg.chat),
-     // Hourly summaries
     hourlySummariesEnabled: cfg.hourlySummariesEnabled !== false, // default: true
     daySummaryEnabled: cfg.daySummaryEnabled !== false, // default: true
     hourlySummaryCronAutoRegister: cfg.hourlySummaryCronAutoRegister === true,
@@ -2734,6 +2734,7 @@ export function parseConfig(
         : 100,
     // Extraction stability guards (P0/P1)
     extractionDedupeEnabled: cfg.extractionDedupeEnabled !== false,
+    extractionSourceGroundingEnabled: coerceBool(cfg.extractionSourceGroundingEnabled) !== false,
     extractionDedupeWindowMs:
       typeof cfg.extractionDedupeWindowMs === "number" ? cfg.extractionDedupeWindowMs : 5 * 60_000,
     extractionMinChars:

package/src/corpus-watermark.test.ts

@@ -25,6 +25,7 @@ import {
   buildPartitionCensus,
   computeCorpusWatermark,
   computeCorpusWatermarks,
+  computeServiceCorpusCensus,
   computeServiceCorpusWatermarks,
   digestPartitionCensus,
   resolveCorpusNamespaceRoots,
@@ -838,3 +839,23 @@ test("finding round-10: a non-directory category root fails a strict census (nam
     await rm(memoryDir, { recursive: true, force: true });
   }
 });
+
+test("round 6 (coderabbit): a failed namespace enumeration is an INCOMPLETE census, not an empty corpus", async () => {
+  // computeServiceCorpusCensus wraps resolveCorpusNamespaceRoots: when enumeration
+  // throws, the census must report complete:false (so a peer is never certified
+  // converged against it) rather than an empty set that reads as an empty deployment.
+  const throwingConfig = new Proxy({} as PluginConfig, {
+    get() {
+      throw new Error("namespace enumeration unavailable");
+    },
+  });
+  const host = {
+    config: throwingConfig,
+    getStorage: () => {
+      throw new Error("getStorage must not be reached once enumeration fails");
+    },
+  };
+  const census = await computeServiceCorpusCensus(host);
+  assert.equal(census.complete, false, "an enumeration failure is incomplete, never a certified empty census");
+  assert.deepEqual(census.watermarks, [], "and it yields no watermarks");
+});

package/src/corpus-watermark.ts

@@ -500,6 +500,22 @@ export async function computeServiceCorpusWatermarks(
   host: CorpusWatermarkHost,
   options: ServiceCorpusWatermarkOptions = {},
 ): Promise<CorpusWatermark[]> {
+  return (await computeServiceCorpusCensus(host, options)).watermarks;
+}
+
+/**
+ * Watermarks PLUS whether every configured namespace made it into the set.
+ *
+ * Completeness is returned ATOMICALLY with the data rather than stashed in a
+ * side channel: every early exit here (enumeration threw, roots still warming,
+ * a per-namespace scan failed, a cache entry not yet populated) yields an
+ * incomplete set, and a caller that certifies convergence against a partial
+ * census would hide the divergence of whatever was dropped (issue #2149).
+ */
+export async function computeServiceCorpusCensus(
+  host: CorpusWatermarkHost,
+  options: ServiceCorpusWatermarkOptions = {},
+): Promise<{ watermarks: CorpusWatermark[]; complete: boolean }> {
   let roots: CorpusNamespaceRoot[] | undefined;
   if (options.cache) {
     roots = options.cache.getResolvedRoots(() => resolveCorpusNamespaceRoots({ config: host.config }));
@@ -507,25 +523,31 @@ export async function computeServiceCorpusWatermarks(
     try {
       roots = await resolveCorpusNamespaceRoots({ config: host.config });
     } catch {
-      return [];
+      return { watermarks: [], complete: false }; // never enumerated — not an empty corpus
     }
   }
-  if (!roots) return []; // enumeration still warming (cold) — served on a later probe
+  // Enumeration still warming (cold): served on a later probe, and NOT complete.
+  if (!roots) return { watermarks: [], complete: false };
   const visible = roots.filter((root) => root.namespaces.some((ns) => capabilityAllowsNamespace(options.caps, ns)));
   const watermarks: CorpusWatermark[] = [];
+  let complete = true;
   for (const { namespace } of visible) {
     const compute = async (): Promise<CorpusWatermark> =>
       computeNamespaceWatermark(namespace, await host.getStorage(namespace), options.now);
     if (options.cache) {
       const cached = options.cache.get(namespace, compute);
       if (cached) watermarks.push(cached);
+      else complete = false; // still warming — the set is not the full census yet
     } else {
       try {
         watermarks.push(await compute());
       } catch {
         // One tenant's storage/scan failed — omit just that namespace.
+        complete = false;
       }
     }
   }
-  return watermarks;
+  return { watermarks, complete };
 }
+
+

package/src/entity-id-normalization.ts

@@ -0,0 +1,55 @@
+const BUILTIN_ALIASES: Record<string, string> = {
+  openclaw: "openclaw",
+  "open-claw": "openclaw",
+};
+
+function normalizeEntityNameWithPattern(
+  raw: string,
+  type: string,
+  aliases: Readonly<Record<string, string>> | undefined,
+  pattern: RegExp,
+  normalizeUnicode: boolean
+): string {
+  const rawStr = typeof raw === "string" ? raw : "";
+  const typeStr = typeof type === "string" && type.trim().length > 0 ? type : "entity";
+  const typePrefix = `${typeStr.toLowerCase()}-`;
+  let name = normalizeUnicode ? rawStr.normalize("NFC").toLowerCase().trim() : rawStr.toLowerCase().trim();
+  if (name.startsWith(typePrefix)) name = name.slice(typePrefix.length);
+
+  let normalized = name.replace(pattern, "-").replace(/-+/g, "-").replace(/^-|-$/g, "");
+  let legacyNormalized: string | undefined;
+  if (normalizeUnicode) {
+    let legacyName = rawStr.toLowerCase().trim();
+    if (legacyName.startsWith(typePrefix)) legacyName = legacyName.slice(typePrefix.length);
+    legacyNormalized = legacyName.replace(/[^a-z0-9]+/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, "");
+  }
+  const unicodeAlias = aliases !== undefined && Object.hasOwn(aliases, normalized) ? aliases[normalized] : undefined;
+  const legacyAlias =
+    legacyNormalized !== undefined && legacyNormalized !== normalized && aliases !== undefined &&
+    Object.hasOwn(aliases, legacyNormalized)
+      ? aliases[legacyNormalized]
+      : undefined;
+  const userAlias =
+    typeof unicodeAlias === "string" && unicodeAlias.length > 0
+      ? unicodeAlias
+      : legacyAlias;
+  if (typeof userAlias === "string" && userAlias.length > 0) {
+    normalized = userAlias;
+  } else if (Object.hasOwn(BUILTIN_ALIASES, normalized)) {
+    normalized = BUILTIN_ALIASES[normalized];
+  }
+
+  return `${typeStr.toLowerCase()}-${normalized}`;
+}
+
+export function normalizeEntityName(raw: string, type: string, aliases?: Readonly<Record<string, string>>): string {
+  return normalizeEntityNameWithPattern(raw, type, aliases, /[^\p{L}\p{M}\p{N}]+/gu, true);
+}
+
+export function normalizeLegacyEntityName(
+  raw: string,
+  type: string,
+  aliases?: Readonly<Record<string, string>>
+): string {
+  return normalizeEntityNameWithPattern(raw, type, aliases, /[^a-z0-9]+/g, false);
+}

package/src/entity-retrieval-boundaries.ts

@@ -0,0 +1,134 @@
+const UNICODE_WORD_OR_NUMBER_RE = /[\p{L}\p{M}\p{N}]/u;
+
+function firstUnicodeCharacter(value: string): string {
+  const codePoint = value.codePointAt(0);
+  return codePoint === undefined ? "" : String.fromCodePoint(codePoint);
+}
+
+function lastUnicodeCharacter(value: string): string {
+  const codePoint = [...value].at(-1);
+  return codePoint ?? "";
+}
+
+const JAPANESE_PARTICLE_RE = /^[のはがをにへともやかでだ]$/u;
+const JAPANESE_MULTI_CHARACTER_PARTICLES = [
+  "について",
+  "にとって",
+  "によって",
+  "に関して",
+  "に対して",
+  "とは",
+  "では",
+  "には",
+  "にも",
+  "って",
+] as const;
+const KOREAN_PARTICLES = [
+  "에서", "에게", "한테", "으로", "까지", "부터", "보다", "처럼", "같이",
+  "이나", "라도", "밖에", "마다", "조차", "이랑", "하고", "은", "는",
+  "이", "가", "을", "를", "의", "께", "와", "과", "도", "만", "든",
+  "에", "로", "나", "랑",
+] as const;
+const KOREAN_UNSPACED_QUESTION_PARTICLES = ["은", "는", "이", "가"] as const;
+const KOREAN_INTERROGATIVE_PREFIXES = [
+  "어디", "무엇", "뭐", "왜", "어떻게", "언제", "누구", "어느", "몇", "얼마",
+] as const;
+
+const HANGUL_RUN_RE = /^[\p{Script=Hangul}]+/u;
+
+const UNICODE_WORD_SEGMENTER =
+  typeof Intl.Segmenter === "function" ? new Intl.Segmenter(undefined, { granularity: "word" }) : null;
+const KOREAN_WORD_SEGMENTER =
+  typeof Intl.Segmenter === "function" ? new Intl.Segmenter("ko", { granularity: "word" }) : null;
+
+const JAPANESE_WORD_SEGMENTER =
+  typeof Intl.Segmenter === "function" ? new Intl.Segmenter("ja", { granularity: "word" }) : null;
+const JAPANESE_KANA_RE = /[\p{Script=Hiragana}\p{Script=Katakana}]/u;
+
+function isJapaneseParticleBoundary(suffix: string): boolean {
+  const firstSegment = JAPANESE_WORD_SEGMENTER?.segment(suffix).containing(0)?.segment;
+  if (firstSegment !== undefined) {
+    if (JAPANESE_MULTI_CHARACTER_PARTICLES.some((particle) => particle === firstSegment)) return true;
+    return JAPANESE_PARTICLE_RE.test(firstSegment);
+  }
+  const firstCharacter = firstUnicodeCharacter(suffix);
+  return (
+    JAPANESE_MULTI_CHARACTER_PARTICLES.some((particle) => suffix.startsWith(particle)) ||
+    (JAPANESE_PARTICLE_RE.test(firstCharacter) &&
+      !UNICODE_WORD_OR_NUMBER_RE.test(firstUnicodeCharacter(suffix.slice(firstCharacter.length))))
+  );
+}
+
+function isIntlWordBoundary(source: string, index: number): boolean {
+  if (index <= 0 || index >= source.length) return true;
+  return UNICODE_WORD_SEGMENTER?.segment(source).containing(index)?.index === index;
+}
+
+function isKoreanUnspacedQuestionBoundary(suffix: string): boolean {
+  return KOREAN_UNSPACED_QUESTION_PARTICLES.some((particle) =>
+    suffix.startsWith(particle) &&
+    KOREAN_INTERROGATIVE_PREFIXES.some((prefix) => suffix.startsWith(prefix, particle.length)),
+  );
+}
+
+function isKoreanParticleBoundary(suffix: string): boolean {
+  if (isKoreanUnspacedQuestionBoundary(suffix)) return true;
+  const firstSegment = KOREAN_WORD_SEGMENTER?.segment(suffix).containing(0)?.segment;
+  if (firstSegment !== undefined) {
+    return KOREAN_PARTICLES.some((particle) => particle === firstSegment);
+  }
+  const particleRun = suffix.match(HANGUL_RUN_RE)?.[0] ?? "";
+  const nextCharacter = firstUnicodeCharacter(suffix.slice(particleRun.length));
+  return (
+    KOREAN_PARTICLES.some((particle) => particle === particleRun) &&
+    (nextCharacter.length === 0 || !UNICODE_WORD_OR_NUMBER_RE.test(nextCharacter))
+  );
+}
+
+function isUnicodePhraseBoundary(
+  character: string,
+  suffix: string = "",
+  source: string = "",
+  boundaryIndex: number = -1,
+  strictUnicode = false,
+  allowLeadingParticle = false,
+): boolean {
+  return (
+    character.length === 0
+    || !UNICODE_WORD_OR_NUMBER_RE.test(character)
+    || (allowLeadingParticle && isJapaneseParticleBoundary(character))
+    || isJapaneseParticleBoundary(suffix)
+    || (!strictUnicode &&
+      boundaryIndex >= 0 &&
+      !JAPANESE_KANA_RE.test(suffix) &&
+      isIntlWordBoundary(source, boundaryIndex))
+    || (allowLeadingParticle && isKoreanParticleBoundary(character))
+    || isKoreanParticleBoundary(suffix)
+  );
+}
+
+export function containsPhrase(haystack: string, needle: string): boolean {
+  if (!needle) return false;
+  const caseInsensitive = /^[a-z0-9 ]+$/i.test(needle);
+  const searchHaystack = caseInsensitive ? haystack.toLowerCase() : haystack;
+  const searchNeedle = caseInsensitive ? needle.toLowerCase() : needle;
+  let offset = searchHaystack.indexOf(searchNeedle);
+  while (offset >= 0) {
+    const before = lastUnicodeCharacter(searchHaystack.slice(0, offset));
+    const after = searchHaystack.slice(offset + searchNeedle.length);
+    if (
+      isUnicodePhraseBoundary(before, "", searchHaystack, offset, caseInsensitive, true)
+      && isUnicodePhraseBoundary(
+        firstUnicodeCharacter(after),
+        after,
+        searchHaystack,
+        offset + searchNeedle.length,
+        caseInsensitive,
+      )
+    ) {
+      return true;
+    }
+    offset = searchHaystack.indexOf(searchNeedle, offset + searchNeedle.length);
+  }
+  return false;
+}