@remnic/core 9.53.0 → 9.54.0

package/src/wearables/cleanup.ts

@@ -7,6 +7,7 @@
  * everything here is conservative and reversible by re-syncing.
  */
 
+import { detectScriptHints, type ScriptHint } from "./text-language.js";
 import type {
   WearableCleanupSettings,
   WearableConversation,
@@ -33,24 +34,155 @@ const MERGE_GAP_MS = 30_000;
 const GENERIC_SPEAKER_PATTERN = /^unknown$/i;
 
 /**
- * Standalone filler tokens stripped when `stripFillers` is on. Matched
- * case-insensitively on word boundaries, only as whole tokens — "um"
- * inside "umbrella" is never touched. Deliberately short, low-risk
- * list; meaning-bearing hedges ("like", "well") are NOT stripped.
+ * Standalone filler tokens stripped when `stripFillers` is on, for
+ * scripts that separate words with spaces. Matched case-insensitively
+ * as whole tokens — "um" inside "umbrella" is never touched.
+ * Deliberately short and low-risk: meaning-bearing hedges ("like",
+ * "well", Spanish "este", Arabic "يعني") are NOT stripped, and neither
+ * is "mm", which is also a unit of length.
  */
-const FILLER_TOKENS = ["um", "uh", "uhm", "umm", "uhh", "erm", "hmm", "mhm"];
+const TOKEN_FILLERS: Readonly<Partial<Record<ScriptHint, readonly string[]>>> = {
+  latin: [
+    "um",
+    "uh",
+    "uhm",
+    "umm",
+    "uhh",
+    "erm",
+    "hmm",
+    "mhm",
+    "ähm",
+    "äh",
+    "ehm",
+    "euh",
+    "mmm",
+  ],
+  korean: ["음", "으음", "엄", "어어"],
+  arabic: ["امم", "اممم", "اه"],
+  cyrillic: ["эм", "ээ", "ммм"],
+};
 
-const FILLER_PATTERN = new RegExp(
-  // Leading/trailing punctuation around the filler collapses with it so
-  // "Um, so we should" -> "so we should" rather than ", so we should".
-  `(?:^|\\s)(?:${FILLER_TOKENS.join("|")})[,.]?(?=\\s|$)`,
-  "gi",
-);
+/**
+ * Filler tokens for scripts without word spaces. These are removed
+ * inline, so each entry must be unambiguous on its own: the elongated
+ * Japanese hesitation forms and the two Chinese hesitation particles
+ * qualify, while a bare 「あの」 ("that") does not (issue #2196).
+ */
+const INLINE_FILLERS: Readonly<Partial<Record<ScriptHint, readonly string[]>>> = {
+  japanese: ["えーと", "えっと", "ええと", "あのー", "あのう", "うーん"],
+  han: ["呃", "嗯"],
+};
+
+/**
+ * Punctuation that trails a filler and collapses with it, so
+ * 「えーと、では」 becomes 「では」 rather than 「、では」. Arabic sentence
+ * punctuation is included: an Arabic filler is normally followed by
+ * `،`, and without it the token survives the pass.
+ */
+const TRAILING_FILLER_PUNCTUATION = "[,.、。،؛؟!?]?";
+
+interface FillerMatchers {
+  token: RegExp | null;
+  inline: RegExp | null;
+}
+
+/**
+ * Compiled matchers are cached per (script hints + extra tokens) key.
+ * A day of transcripts is thousands of segments over a handful of
+ * scripts, so this compiles a few regexes instead of one per segment.
+ */
+const FILLER_MATCHER_CACHE = new Map<string, FillerMatchers>();
+
+function escapeRegExp(value: string): string {
+  return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+/**
+ * Build one alternation from `tokens`, or `null` when none remain.
+ *
+ * Each token contributes BOTH its composed and its decomposed spelling,
+ * so `ähm` also matches an ASR that emits `a\u0308hm`. Matching both
+ * forms keeps the surrounding transcript byte-identical, which
+ * normalizing the text before stripping would not.
+ *
+ * JavaScript alternation is first-match, not longest-match, so the
+ * tokens are sorted longest-first. Without that a built-in prefix such
+ * as 「あのー」 would match ahead of a longer operator token that starts
+ * with it and leave the tail in the transcript. Ties break on the token
+ * itself so the pattern is stable across runs.
+ */
+function buildAlternation(
+  tokens: readonly string[],
+  shape: (pattern: string) => { source: string; flags: string },
+): RegExp | null {
+  const forms = new Set<string>();
+  for (const token of tokens) {
+    forms.add(token.normalize("NFC"));
+    forms.add(token.normalize("NFD"));
+  }
+  const unique = [...forms].sort(
+    (left, right) => right.length - left.length || (left < right ? -1 : left > right ? 1 : 0),
+  );
+  if (unique.length === 0) return null;
+  const { source, flags } = shape(unique.map((token) => escapeRegExp(token)).join("|"));
+  return new RegExp(source, flags);
+}
+
+function buildFillerMatchers(
+  hints: readonly ScriptHint[],
+  extraTokens: readonly string[],
+): FillerMatchers {
+  // Structural key: comma-joining collapses `["a,b"]` and `["a","b"]`
+  // onto one entry, and the first caller's regex would then clean the
+  // second caller's transcripts.
+  const key = JSON.stringify([hints, extraTokens]);
+  const cached = FILLER_MATCHER_CACHE.get(key);
+  if (cached) return cached;
+
+  const tokens: string[] = [];
+  const inline: string[] = [];
+  for (const hint of hints) {
+    tokens.push(...(TOKEN_FILLERS[hint] ?? []));
+    inline.push(...(INLINE_FILLERS[hint] ?? []));
+  }
+  // Operator tokens apply to every script: the operator knows their own
+  // language, and routing them by script hint would silently drop them.
+  for (const extra of extraTokens) {
+    const trimmed = extra.trim();
+    if (trimmed.length === 0) continue;
+    if (SPACE_DELIMITED_TOKEN.test(trimmed)) tokens.push(trimmed);
+    else inline.push(trimmed);
+  }
+
+  const matchers: FillerMatchers = {
+    token: buildAlternation(tokens, (pattern) => ({
+      source: `(?:^|\\s)(?:${pattern})${TRAILING_FILLER_PUNCTUATION}(?=\\s|$)`,
+      flags: "giu",
+    })),
+    inline: buildAlternation(inline, (pattern) => ({
+      source: `(?:${pattern})${TRAILING_FILLER_PUNCTUATION}`,
+      flags: "gu",
+    })),
+  };
+  FILLER_MATCHER_CACHE.set(key, matchers);
+  return matchers;
+}
+
+/**
+ * A token whose first character belongs to a space-delimited script.
+ *
+ * Anything outside this set is removed INLINE, so a script that does
+ * space its words must appear here. Hebrew `אה` routed to the inline
+ * matcher would strip the suffix of `נראה` and corrupt the transcript.
+ */
+const SPACE_DELIMITED_TOKEN =
+  /^[\p{Script=Latin}\p{Script=Cyrillic}\p{Script=Greek}\p{Script=Hangul}\p{Script=Arabic}\p{Script=Hebrew}\p{N}]/u;
 
 /** Apply configured cleanup passes to one conversation. */
 export function cleanConversation(
   conversation: WearableConversation,
   settings: WearableCleanupSettings,
+  extraFillerTokens: readonly string[] = [],
 ): CleanupResult {
   let segments = conversation.segments.map((segment) => ({ ...segment }));
   let droppedSegments = 0;
@@ -61,7 +193,7 @@ export function cleanConversation(
 
   if (settings.stripFillers) {
     for (const segment of segments) {
-      segment.text = stripFillerTokens(segment.text);
+      segment.text = stripFillerTokens(segment.text, extraFillerTokens);
     }
   }
 
@@ -130,15 +262,30 @@ function canMerge(
     const label = previous.speakerKey.trim();
     if (label.length === 0 || GENERIC_SPEAKER_PATTERN.test(label)) return false;
   }
-  const previousEnd = previous.endIso ? Date.parse(previous.endIso) : NaN;
-  const nextStart = next.startIso ? Date.parse(next.startIso) : NaN;
+  const previousEnd = previous.endIso
+    ? Date.parse(previous.endIso)
+    : Number.NaN;
+  const nextStart = next.startIso ? Date.parse(next.startIso) : Number.NaN;
   // Without timestamps, adjacency is the only signal — still merge.
   if (Number.isNaN(previousEnd) || Number.isNaN(nextStart)) return true;
   return nextStart - previousEnd <= MERGE_GAP_MS;
 }
 
-export function stripFillerTokens(text: string): string {
-  return normalizeWhitespace(text.replace(FILLER_PATTERN, " "));
+/**
+ * Remove filler tokens from `text`. The built-in token set is chosen
+ * from the scripts present in the text, so a Japanese or Korean segment
+ * is cleaned by its own list instead of silently keeping every filler
+ * (issue #2196). Operator tokens apply to every script.
+ */
+export function stripFillerTokens(
+  text: string,
+  extraTokens: readonly string[] = [],
+): string {
+  const matchers = buildFillerMatchers(detectScriptHints(text), extraTokens);
+  let result = text;
+  if (matchers.token) result = result.replace(matchers.token, " ");
+  if (matchers.inline) result = result.replace(matchers.inline, "");
+  return normalizeWhitespace(result);
 }
 
 /**

package/src/wearables/config.test.ts

@@ -277,3 +277,65 @@ test("fusion knobs reject non-positive integers", () => {
 test("fusion block must be an object when present", () => {
   assert.throws(() => parseWearablesConfig({ fusion: "on" }), /must be an object/);
 });
+
+test("off-the-record marker phrases and filler tokens parse", () => {
+  const parsed = parseWearablesConfig({
+    offTheRecordMarkers: {
+      start: ["  poza protokołem  "],
+      end: ["z powrotem do protokołu"],
+    },
+    fillerTokens: ["bueno", "那个"],
+  });
+  assert.deepEqual(parsed.offTheRecordMarkers.start, ["poza protokołem"]);
+  assert.deepEqual(parsed.offTheRecordMarkers.end, ["z powrotem do protokołu"]);
+  assert.equal(parsed.offTheRecordMarkers.useBuiltIns, true);
+  assert.deepEqual(parsed.fillerTokens, ["bueno", "那个"]);
+});
+
+test("marker and filler lists reject unusable entries loudly", () => {
+  assert.throws(
+    () => parseWearablesConfig({ offTheRecordMarkers: { start: "poza" } }),
+    /offTheRecordMarkers\.start must be an array of strings/,
+  );
+  assert.throws(
+    () => parseWearablesConfig({ offTheRecordMarkers: { end: ["  "] } }),
+    /offTheRecordMarkers\.end\[0\] must be a non-empty string/,
+  );
+  assert.throws(
+    () => parseWearablesConfig({ fillerTokens: ["a".repeat(65)] }),
+    /fillerTokens\[0\] exceeds 64 characters/,
+  );
+  assert.throws(
+    () => parseWearablesConfig({ offTheRecordMarkers: { start: ["a".repeat(129)] } }),
+    /offTheRecordMarkers\.start\[0\] exceeds 128 characters/,
+  );
+  assert.throws(
+    () => parseWearablesConfig({ offTheRecordMarkers: [] }),
+    /offTheRecordMarkers must be an object/,
+  );
+});
+
+test("disabling built-in markers without a start phrase is rejected, not silent", () => {
+  assert.throws(
+    () =>
+      parseWearablesConfig({
+        offTheRecordEnabled: true,
+        offTheRecordMarkers: { useBuiltIns: false },
+      }),
+    /useBuiltIns is false but no start phrase is configured/,
+  );
+  // Turning the whole feature off is the documented escape hatch.
+  const parsed = parseWearablesConfig({
+    offTheRecordEnabled: false,
+    offTheRecordMarkers: { useBuiltIns: false },
+  });
+  assert.equal(parsed.offTheRecordEnabled, false);
+  assert.equal(parsed.offTheRecordMarkers.useBuiltIns, false);
+});
+
+test("an unknown off-the-record marker key is rejected, not dropped", () => {
+  assert.throws(
+    () => parseWearablesConfig({ offTheRecordMarkers: { starts: ["private"] } }),
+    /offTheRecordMarkers has unknown key\(s\) starts/,
+  );
+});

package/src/wearables/config.ts

@@ -13,6 +13,7 @@ import type { ImportanceLevel } from "../types.js";
 import { compileCorrectionRules } from "./corrections.js";
 import { compileRedactionPatterns } from "./redaction.js";
 import type {
+  OffTheRecordMarkerSettings,
   WearableCleanupSettings,
   WearableCorrectionRule,
   WearableMemoryMode,
@@ -111,6 +112,8 @@ export function defaultWearablesConfig(): WearablesConfig {
     redactionEnabled: true,
     redactionPatterns: [],
     offTheRecordEnabled: true,
+    offTheRecordMarkers: { start: [], end: [], useBuiltIns: true },
+    fillerTokens: [],
     digestEnabled: true,
     autoSyncEnabled: DEFAULT_AUTO_SYNC_ENABLED,
     autoSyncIntervalMinutes: DEFAULT_AUTO_SYNC_INTERVAL_MINUTES,
@@ -134,6 +137,37 @@ function requireObject(
   return value as Record<string, unknown>;
 }
 
+/** Longest accepted off-the-record marker phrase. */
+const MAX_MARKER_PHRASE_LENGTH = 128;
+/** Longest accepted filler token. */
+const MAX_FILLER_TOKEN_LENGTH = 64;
+
+/**
+ * Parse an optional array of non-empty phrases. Rejects a non-array, a
+ * non-string entry, a blank entry, and an over-long entry — an operator
+ * privacy phrase that is silently discarded is the failure this parser
+ * exists to prevent (issue #2196).
+ */
+function parsePhraseList(
+  value: unknown,
+  keyPath: string,
+  maxLength: number,
+): string[] {
+  if (value === undefined) return [];
+  if (!Array.isArray(value)) {
+    throw new Error(`${keyPath} must be an array of strings`);
+  }
+  return value.map((entry, index) => {
+    if (typeof entry !== "string" || entry.trim().length === 0) {
+      throw new Error(`${keyPath}[${index}] must be a non-empty string`);
+    }
+    if (entry.length > maxLength) {
+      throw new Error(`${keyPath}[${index}] exceeds ${maxLength} characters`);
+    }
+    return entry.trim();
+  });
+}
+
 function parseBool(
   value: unknown,
   keyPath: string,
@@ -411,6 +445,68 @@ export function parseWearablesConfig(value: unknown): WearablesConfig {
     compileRedactionPatterns(redactionPatterns);
   }
 
+  const offTheRecordEnabled = parseBool(
+    raw.offTheRecordEnabled,
+    "wearables.offTheRecordEnabled",
+    defaults.offTheRecordEnabled,
+  );
+  const rawMarkers =
+    raw.offTheRecordMarkers === undefined
+      ? {}
+      : requireObject(raw.offTheRecordMarkers, "wearables.offTheRecordMarkers");
+  // A typo such as `starts:` would otherwise be dropped without a word,
+  // and the operator would believe a private span is protected. Core and
+  // standalone callers never see OpenClaw's JSON schema, so the check
+  // belongs here. Keys are compared literally so the config-contract
+  // extractor can still enumerate this parser's surface.
+  const {
+    start: rawMarkerStart,
+    end: rawMarkerEnd,
+    useBuiltIns: rawMarkerUseBuiltIns,
+    ...unknownMarkerKeys
+  } = rawMarkers;
+  const unknownMarkerNames = Object.keys(unknownMarkerKeys);
+  if (unknownMarkerNames.length > 0) {
+    throw new Error(
+      `wearables.offTheRecordMarkers has unknown key(s) ${unknownMarkerNames.sort().join(", ")} — valid keys are start, end, useBuiltIns`,
+    );
+  }
+  const offTheRecordMarkers: OffTheRecordMarkerSettings = {
+    start: parsePhraseList(
+      rawMarkerStart,
+      "wearables.offTheRecordMarkers.start",
+      MAX_MARKER_PHRASE_LENGTH,
+    ),
+    end: parsePhraseList(
+      rawMarkerEnd,
+      "wearables.offTheRecordMarkers.end",
+      MAX_MARKER_PHRASE_LENGTH,
+    ),
+    useBuiltIns: parseBool(
+      rawMarkerUseBuiltIns,
+      "wearables.offTheRecordMarkers.useBuiltIns",
+      defaults.offTheRecordMarkers.useBuiltIns,
+    ),
+  };
+  // A silent no-op is the exact failure this feature exists to remove
+  // (issue #2196): turning the built-ins off without supplying a start
+  // phrase would leave the gate on and matching nothing.
+  if (
+    offTheRecordEnabled &&
+    !offTheRecordMarkers.useBuiltIns &&
+    offTheRecordMarkers.start.length === 0
+  ) {
+    throw new Error(
+      "wearables.offTheRecordMarkers.useBuiltIns is false but no start phrase is configured — add wearables.offTheRecordMarkers.start, or set wearables.offTheRecordEnabled to false",
+    );
+  }
+
+  const fillerTokens = parsePhraseList(
+    raw.fillerTokens,
+    "wearables.fillerTokens",
+    MAX_FILLER_TOKEN_LENGTH,
+  );
+
   const parseBoundedInt = (
     value: unknown,
     name: string,
@@ -488,11 +584,9 @@ export function parseWearablesConfig(value: unknown): WearablesConfig {
       defaults.redactionEnabled,
     ),
     redactionPatterns,
-    offTheRecordEnabled: parseBool(
-      raw.offTheRecordEnabled,
-      "wearables.offTheRecordEnabled",
-      defaults.offTheRecordEnabled,
-    ),
+    offTheRecordEnabled,
+    offTheRecordMarkers,
+    fillerTokens,
     digestEnabled: parseBool(
       raw.digestEnabled,
       "wearables.digestEnabled",

package/src/wearables/index.ts

@@ -35,10 +35,20 @@ export {
 } from "./cleanup.js";
 export {
   applyOffTheRecord,
+  BUILT_IN_OFF_THE_RECORD_END,
+  BUILT_IN_OFF_THE_RECORD_START,
+  compileOffTheRecordMarkers,
   compileRedactionPatterns,
   redactText,
   REDACTION_PLACEHOLDER,
+  type CompiledOffTheRecordMarkers,
+  type OffTheRecordMarkerInput,
 } from "./redaction.js";
+export {
+  buildPhraseMatcher,
+  detectScriptHints,
+  type ScriptHint,
+} from "./text-language.js";
 export {
   applyCorrections,
   compileCorrectionRule,

package/src/wearables/pipeline.test.ts

@@ -1039,3 +1039,39 @@ test("a transcript write failure prevents the sync watermark from advancing", as
     rmSync(memoryDir, { recursive: true, force: true });
   }
 });
+
+test("configured non-English off-the-record markers elide a span end to end", async () => {
+  const memoryDir = mkdtempSync(path.join(tmpdir(), "remnic-pipeline-otr-"));
+  try {
+    const byDate = {
+      "2026-06-11": [
+        makeConversation("c1", "2026-06-11", [
+          { speaker: "user", isWearer: true, text: "えーと、ここからはオフレコでお願いします。" },
+          { speaker: "Speaker 2", text: "買収の金額は非公開です。" },
+          { speaker: "user", isWearer: true, text: "オンレコに戻ります。" },
+          { speaker: "Speaker 2", text: "bueno, 天気の話をしましょう。" },
+        ]),
+      ],
+    };
+    const { deps, written } = makeDeps(memoryDir);
+    const summary = await syncWearableSource(
+      fakeConnector(byDate),
+      settings({ memoryMode: "off" }),
+      config({ offTheRecordEnabled: true, fillerTokens: ["bueno"] }),
+      { days: 1 },
+      deps,
+    );
+
+    assert.equal(summary.transcriptsWritten.length, 1);
+    const body = written[0]?.serialized ?? "";
+    assert.ok(!body.includes("買収の金額は非公開です"), "off-record segment must not be stored");
+    assert.ok(body.includes("[off the record"), "elision placeholder is visible");
+    assert.ok(body.includes("[back on the record]"), "span closes on the Japanese end marker");
+    assert.ok(body.includes("天気の話をしましょう"), "on-record speech survives");
+    assert.ok(!body.includes("えーと"), "Japanese filler is stripped");
+    assert.ok(!body.includes("bueno"), "configured filler token is stripped");
+    assert.equal(summary.segmentsDropped, 1);
+  } finally {
+    rmSync(memoryDir, { recursive: true, force: true });
+  }
+});

package/src/wearables/pipeline.ts

@@ -37,7 +37,13 @@ import {
   type WearableMemoryGenDeps,
 } from "./memory-gen.js";
 import { tokenizeDayBody, type CorroborationContext } from "./trust.js";
-import { applyOffTheRecord, compileRedactionPatterns, redactText } from "./redaction.js";
+import {
+  applyOffTheRecord,
+  compileOffTheRecordMarkers,
+  compileRedactionPatterns,
+  redactText,
+  type CompiledOffTheRecordMarkers,
+} from "./redaction.js";
 import { loadSpeakerRegistry } from "./speakers.js";
 import {
   loadSyncState,
@@ -259,6 +265,7 @@ function cleanDay(
   config: WearablesConfig,
   userRedaction: RegExp[],
   correctionRules: CompiledCorrectionRule[],
+  offTheRecordMarkers: CompiledOffTheRecordMarkers,
 ): CleanedDay {
   const { offTheRecordEnabled, redactionEnabled } = config;
   const out: CleanedDay = {
@@ -271,11 +278,15 @@ function cleanDay(
   for (const conversation of raw) {
     let current = conversation;
     if (offTheRecordEnabled) {
-      const otr = applyOffTheRecord(current);
+      const otr = applyOffTheRecord(current, offTheRecordMarkers);
       current = otr.conversation;
       out.segmentsDropped += otr.droppedSegments;
     }
-    const cleaned = cleanConversation(current, settings.cleanup);
+    const cleaned = cleanConversation(
+      current,
+      settings.cleanup,
+      config.fillerTokens,
+    );
     current = cleaned.conversation;
     out.segmentsDropped += cleaned.droppedSegments;
 
@@ -337,6 +348,9 @@ export async function syncWearableSource(
     ...compileCorrectionRules(stateRules, "state corrections"),
   ];
   const userRedaction = compileRedactionPatterns(config.redactionPatterns);
+  const offTheRecordMarkers = compileOffTheRecordMarkers(
+    config.offTheRecordMarkers,
+  );
 
   let syncState = await loadSyncState(deps.memoryDir);
   const previousState = syncState.sources[connector.id];
@@ -360,6 +374,7 @@ export async function syncWearableSource(
       config,
       userRedaction,
       correctionRules,
+      offTheRecordMarkers,
     );
     summary.conversations += cleaned.conversations.length;
     summary.segmentsKept += cleaned.segmentsKept;