@remnic/core 9.5.0 → 9.6.0

package/src/wearables/day-store.ts

@@ -34,6 +34,127 @@ export const WEARABLES_DIR_NAME = "wearables";
 
 const DATE_PATTERN = /^\d{4}-\d{2}-\d{2}$/;
 
+/**
+ * Day-transcript body serialization format version. Folded into
+ * `hashTranscriptBody` so a version bump invalidates files written by an
+ * older serializer and forces an idempotent rewrite. Decoders gate on this:
+ * only bodies whose parsed meta carries >= this version have escape
+ * sequences decoded; legacy bodies (absent / older) are left byte-for-byte
+ * unchanged so a literal two-character `\n`/`\r` in a pre-escaper
+ * transcript is never altered (issue #1849).
+ */
+export const TRANSCRIPT_FORMAT_VERSION = 2;
+
+/**
+ * Result of parsing a rendered transcript segment line.
+ */
+export interface TranscriptSegmentMatch {
+  /** Speaker label text between the `**` delimiters. */
+  label: string;
+  /** Clock text between the `[` `]` delimiters. */
+  clock: string;
+  /** Segment text after the colon. */
+  text: string;
+}
+
+/**
+ * Test a single character code point against the JavaScript regex `\s`
+ * whitespace class without invoking a regex. Used by the linear segment
+ * line parser so no quantified pattern is ever evaluated on untrusted
+ * transcript content (CodeQL polynomial-redos, issue #1849).
+ */
+function isRegexWhitespace(code: number): boolean {
+  return (
+    (code >= 0x09 && code <= 0x0d) ||
+    code === 0x20 ||
+    code === 0xa0 ||
+    code === 0x1680 ||
+    (code >= 0x2000 && code <= 0x200a) ||
+    code === 0x2028 ||
+    code === 0x2029 ||
+    code === 0x202f ||
+    code === 0x205f ||
+    code === 0x3000 ||
+    code === 0xfeff
+  );
+}
+
+/**
+ * Parse a rendered transcript segment line (`**label** [clock]: text`)
+ * into its three components using a linear, bounded scan — a CodeQL-safe
+ * replacement for the polynomial-risk regex that previously matched this
+ * format. Returns null when the line is not a segment line.
+ *
+ * The scan finds the FIRST occurrence of each structural delimiter
+ * (`**` after the label, `]` after the clock) exactly as the original
+ * non-greedy regex did, and verifies the fixed separator characters
+ * (`\s`, `[`, `:`, `\s`) by single-character code-point comparison.
+ * No quantified regex is evaluated on the line (#1849).
+ *
+ * Escaped labels (formatVersion >= 2) never contain an unescaped `**`,
+ * `[`, or `]`, and rendered clocks are always `HH:MM` / `--:--`, so the
+ * first delimiter is always the correct one for every well-formed
+ * transcript. Legacy unescaped labels are used verbatim by the caller
+ * and parsed identically here.
+ */
+export function parseTranscriptSegmentLine(
+  line: string,
+): TranscriptSegmentMatch | null {
+  // Must start with '**'.
+  if (!line.startsWith("**")) return null;
+
+  // Find the closing '**' of the label: the first '**' at index >= 3
+  // (label is at least 1 char) that is immediately followed by \s and
+  // then '['. This mirrors the non-greedy regex: the engine tries the
+  // shortest label first and extends only when the subsequent fixed
+  // delimiters do not line up. An escaped label ending with '\*' is
+  // correctly handled because the spurious '**' (escape-star + close-
+  // star) is not followed by \s and the scan continues.
+  let labelEnd = -1;
+  for (let i = 3; i < line.length; i++) {
+    if (
+      line.charCodeAt(i) !== 0x2a /* '*' */ ||
+      i + 3 >= line.length ||
+      line.charCodeAt(i + 1) !== 0x2a /* '*' */ ||
+      !isRegexWhitespace(line.charCodeAt(i + 2)) ||
+      line.charCodeAt(i + 3) !== 0x5b /* '[' */
+    ) {
+      continue;
+    }
+    labelEnd = i;
+    break;
+  }
+  if (labelEnd === -1) return null;
+
+  const label = line.slice(2, labelEnd);
+  // Clock content starts after the closing '**', the \s, and the '['.
+  const clockStart = labelEnd + 4;
+
+  // Find the closing ']' of the clock: the first ']' at index >=
+  // clockStart + 1 (clock is at least 1 char) that is immediately
+  // followed by ':' and \s — matching the non-greedy regex.
+  let clockEnd = -1;
+  for (let i = clockStart + 1; i < line.length; i++) {
+    if (
+      line.charCodeAt(i) !== 0x5d /* ']' */ ||
+      i + 2 >= line.length ||
+      line.charCodeAt(i + 1) !== 0x3a /* ':' */ ||
+      !isRegexWhitespace(line.charCodeAt(i + 2))
+    ) {
+      continue;
+    }
+    clockEnd = i;
+    break;
+  }
+  if (clockEnd === -1) return null;
+
+  return {
+    label,
+    clock: line.slice(clockStart, clockEnd),
+    text: line.slice(clockEnd + 3),
+  };
+}
+
 export function isValidTranscriptDate(date: string): boolean {
   if (!DATE_PATTERN.test(date)) return false;
   const parsed = new Date(`${date}T00:00:00Z`);
@@ -41,7 +162,14 @@ export function isValidTranscriptDate(date: string): boolean {
 }
 
 export function hashTranscriptBody(body: string): string {
-  return createHash("sha256").update(body, "utf-8").digest("hex");
+  // The version prefix makes the hash an idempotency key for the body AS
+  // SERIALIZED under the current format: when the escape encoding changes
+  // (version bump) every existing file's stored hash no longer matches and
+  // the pipeline rewrites it with the new marker — no separate migration
+  // pass needed (issue #1849).
+  return createHash("sha256")
+    .update(`v${TRANSCRIPT_FORMAT_VERSION}\n${body}`, "utf-8")
+    .digest("hex");
 }
 
 function formatClockTime(iso: string | undefined, timezone: string): string {
@@ -69,6 +197,135 @@ function conversationDurationMinutes(conversation: WearableConversation): number
   return (end - start) / 60_000;
 }
 
+/**
+ * Escape segment text for the line-based markdown format so that embedded
+ * newlines, carriage returns, and backslashes survive the serialize →
+ * reconstruct round-trip losslessly.  Reversed by `unescapeSegmentText`
+ * (this module — the single decode primitive shared with the fusion
+ * reconstruct path and every view/search/index surface).
+ *
+ * Without this, a segment whose text contains a newline is split across
+ * multiple physical lines; the reconstruct path treats each line
+ * independently and the continuation is silently dropped (or worse, a
+ * continuation line that looks like a heading/clock is mis-parsed).
+ */
+export function escapeSegmentText(text: string): string {
+  return text
+    .replace(/\\/g, "\\\\")
+    .replace(/\n/g, "\\n")
+    .replace(/\r/g, "\\r");
+}
+
+/**
+ * Reverse `escapeSegmentText`. Unknown escape sequences (a lone
+ * backslash followed by a character the escaper never emits) are passed
+ * through literally so legacy transcripts that never went through the
+ * escaper still round-trip their original text. This is the SINGLE
+ * decode primitive: the fusion reconstruct path and every user-facing
+ * view/search/index surface call it so escaped storage never leaks to
+ * display (#1849).
+ */
+export function unescapeSegmentText(text: string): string {
+  return text.replace(/\\(.)/g, (_match, ch: string) => {
+    if (ch === "n") return "\n";
+    if (ch === "r") return "\r";
+    if (ch === "\\") return "\\";
+    return "\\" + ch;
+  });
+}
+
+/**
+ * Escape a speaker label for the line-based markdown format so that
+ * markdown-delimiter characters in an arbitrary user/provider label
+ * (`**`, `[`, `]`), the escape character (`\`), and embedded newlines/
+ * carriage returns cannot break `parseTranscriptSegmentLine` parsing.
+ * Without this, a label containing `**` (or `** [clock]:`) can make the
+ * non-greedy delimiter match land on the wrong `**` and mis-parse the
+ * clock/text — or fail to parse the line at all. Reversed by
+ * `unescapeSpeakerLabel` (this module). Legacy transcripts whose labels
+ * were never escaped still parse: `unescapeSpeakerLabel` is a no-op on
+ * any label without a backslash escape sequence (#1849).
+ */
+export function escapeSpeakerLabel(label: string): string {
+  return label
+    .replace(/\\/g, "\\\\")
+    .replace(/\*/g, "\\*")
+    .replace(/\[/g, "\\[")
+    .replace(/\]/g, "\\]")
+    .replace(/\n/g, "\\n")
+    .replace(/\r/g, "\\r");
+}
+
+/**
+ * Reverse `escapeSpeakerLabel`. Unknown escape sequences (a backslash
+ * followed by a character the escaper never emits) are passed through
+ * literally so legacy transcripts with raw labels still round-trip
+ * their original text, mirroring `unescapeSegmentText` for segment text.
+ */
+export function unescapeSpeakerLabel(label: string): string {
+  return label.replace(/\\(.)/g, (_match, ch: string) => {
+    if (ch === "\\") return "\\";
+    if (ch === "*") return "*";
+    if (ch === "[") return "[";
+    if (ch === "]") return "]";
+    if (ch === "n") return "\n";
+    if (ch === "r") return "\r";
+    return "\\" + ch;
+  });
+}
+
+/**
+ * Whether a parsed day-transcript's body was written by the escape-aware
+ * serializer (meta carries `formatVersion` >= `TRANSCRIPT_FORMAT_VERSION`).
+ * Legacy bodies (no marker, or an older version) must NOT be decoded: their
+ * literal two-character `\n`, `\r`, and lone backslashes are original
+ * content, not escape sequences (issue #1849).
+ */
+export function bodyIsEscaped(
+  meta: { formatVersion?: number } | null | undefined,
+): boolean {
+  return (
+    meta !== null &&
+    meta !== undefined &&
+    typeof meta.formatVersion === "number" &&
+    meta.formatVersion >= TRANSCRIPT_FORMAT_VERSION
+  );
+}
+
+/**
+ * Decode the escaped segment text AND speaker label of a stored
+ * transcript body into the ORIGINAL forms for user-facing
+ * view/search/index surfaces. Only segment lines are touched: the text
+ * is decoded via `unescapeSegmentText` and the label via
+ * `unescapeSpeakerLabel`; headings, locations, and the H1 header are
+ * NOT escaped on write and pass through unchanged, so a title or
+ * location containing a literal backslash is never altered. The fusion
+ * reconstruct path does NOT use this — it decodes each segment once
+ * during parse — so callers feeding bodies back into reconstruct must
+ * pass the RAW stored body, not a decoded one, to avoid double-decoding.
+ *
+ * FORMAT-AWARE (issue #1849): when `escaped` is falsy (the default) the
+ * body is returned byte-for-byte unchanged. Only bodies written by the
+ * escape-aware serializer (`escaped = true`, derived from
+ * `bodyIsEscaped(meta)`) are decoded, so a legacy transcript's literal
+ * two-character `\n`/`\r` or lone backslash is never altered. Callers
+ * that have the parsed meta should pass `bodyIsEscaped(meta)`.
+ */
+export function decodeTranscriptBody(body: string, escaped = false): string {
+  if (!escaped) return body;
+  const lines = body.split("\n");
+  for (let i = 0; i < lines.length; i++) {
+    const match = parseTranscriptSegmentLine(lines[i]);
+    if (match === null) continue;
+    const { label, clock, text: rawText } = match;
+    const decodedText = unescapeSegmentText(rawText);
+    const decodedLabel = unescapeSpeakerLabel(label);
+    if (decodedText === rawText && decodedLabel === label) continue;
+    lines[i] = `**${decodedLabel}** [${clock}]: ${decodedText}`;
+  }
+  return lines.join("\n");
+}
+
 /** Compose the markdown body (no frontmatter) for one source/day. */
 export function composeDayTranscriptBody(
   sourceId: string,
@@ -101,7 +358,9 @@ export function composeDayTranscriptBody(
     for (const segment of conversation.segments) {
       const { label } = resolveSpeaker(sourceId, segment, registry);
       const at = formatClockTime(segment.startIso, timezone);
-      lines.push(`**${label}** [${at}]: ${segment.text}`);
+      lines.push(
+        `**${escapeSpeakerLabel(label)}** [${at}]: ${escapeSegmentText(segment.text)}`,
+      );
     }
     lines.push("");
   }
@@ -132,6 +391,7 @@ export function composeDayTranscriptMeta(
     durationMinutes,
     contentHash: hashTranscriptBody(body),
     syncedAt,
+    formatVersion: TRANSCRIPT_FORMAT_VERSION,
   };
 }
 
@@ -142,6 +402,9 @@ export function serializeDayTranscript(
 ): string {
   const lines: string[] = ["---"];
   lines.push(`kind: ${meta.kind}`);
+  if (typeof meta.formatVersion === "number") {
+    lines.push(`formatVersion: ${meta.formatVersion}`);
+  }
   lines.push(`source: ${JSON.stringify(meta.source)}`);
   lines.push(`date: ${JSON.stringify(meta.date)}`);
   lines.push(`timezone: ${JSON.stringify(meta.timezone)}`);
@@ -204,9 +467,15 @@ export function parseDayTranscript(raw: string): WearableDayTranscript | null {
 
   const meta: WearableDayTranscriptMeta = {
     kind: "wearable-transcript",
+    ...(scalars.has("formatVersion")
+      ? { formatVersion: parseNonNegativeInt(scalars.get("formatVersion")) }
+      : {}),
     source,
     date,
-    timezone: scalars.get("timezone") ?? "UTC",
+    // Preserve a missing timezone field as "" rather than coercing to a
+    // default: the fusion identity guard must see "no resolvable tz id"
+    // and skip, never silently match a known zone.
+    timezone: scalars.get("timezone") ?? "",
     conversationCount: parseNonNegativeInt(scalars.get("conversationCount")),
     segmentCount: parseNonNegativeInt(scalars.get("segmentCount")),
     speakers,

package/src/wearables/fusion/cluster.ts

@@ -0,0 +1,230 @@
+/**
+ * Wearable cross-source fusion — deterministic conversation clustering.
+ *
+ * Conversations from multiple sources recording the same real-world event
+ * overlap or sit within a proximity gap. This module groups them into
+ * clusters that are fully deterministic: the only inputs are start/end
+ * times, source ids, and conversation ids (no randomness, no LLM).
+ *
+ * Cross-source time proximity BRIDGES the same real-world conversation
+ * recorded by DIFFERENT sources. A single source's own distinct
+ * conversations are never merged by time proximity alone — that would
+ * collapse the source's conversation boundaries. Two same-source
+ * conversations may end up in one cluster only when a different-source
+ * conversation within the gap corroborates that they are part of the same
+ * real-world event (a cross-source chain/bridge).
+ */
+
+import type { FusionConversationInput } from "./types.js";
+
+/** Default max gap (ms) between two conversations to merge into one cluster. */
+export const DEFAULT_PROXIMITY_GAP_MS = 5 * 60_000;
+
+interface Interval {
+  start: number;
+  end: number;
+}
+
+/** The latest extent of a set of segments as an epoch-ms value + its ISO.
+ * Each segment contributes its OWN extent — its `endIso` when known,
+ * otherwise its `startIso` — and the result is the maximum. Reconstructed
+ * `--:--` transcripts rebuild segments with only a start, so this reaches
+ * the last segment start; segment ISOs are already cross-midnight-rolled by
+ * the reconstruct layer, so the derived end is consistent with the segment
+ * timeline. Returns undefined when no segment carries a parseable extent.
+ *
+ * This is the SINGLE shared primitive behind every end/interval derivation
+ * in the fusion pipeline (cluster interval, fused conversation end): when a
+ * conversation-level end is missing, the window falls back to the latest
+ * segment extent instead of collapsing to a zero-length point at the start.
+ */
+export interface SegmentExtent {
+  /** Epoch ms of the latest extent. */
+  ms: number;
+  /** ISO string that produced `ms` (a segment endIso, else its startIso). */
+  iso: string;
+}
+
+export function maxSegmentExtent(
+  segments: readonly { endIso?: string; startIso?: string }[],
+): SegmentExtent | undefined {
+  let best: SegmentExtent | undefined;
+  for (const seg of segments) {
+    const iso = seg.endIso ?? seg.startIso;
+    if (iso === undefined) continue;
+    const ms = Date.parse(iso);
+    if (!Number.isFinite(ms)) continue;
+    if (best === undefined || ms > best.ms) {
+      best = { ms, iso };
+    }
+  }
+  return best;
+}
+
+/**
+ * Resolve a conversation's effective [start, end] window in epoch ms.
+ *
+ * When the conversation carries no explicit end (`endIso` undefined — the
+ * reconstructed form of a stored transcript whose heading end renders as
+ * `--:--`), the window is derived from the segments so it spans the actual
+ * utterances instead of collapsing to a zero-length point at the start:
+ *
+ *   1. conversation `endIso`, when known;
+ *   2. the latest segment END (`segment.endIso`);
+ *   3. the latest segment START (`segment.startIso`) — reconstructed
+ *      transcripts rebuild segments with only a start, so the window must
+ *      reach the last segment start;
+ *   4. the conversation start itself (no segments at all).
+ *
+ * Each segment contributes its own extent (end when known, otherwise its
+ * start); the window end is the maximum extent. Segment ISOs are already
+ * rolled for cross-midnight by the reconstruct layer, so the derived end is
+ * consistent with the segment timeline. The end is clamped to `>= start` so
+ * the window is always valid regardless of input anomalies.
+ */
+function effectiveInterval(conversation: FusionConversationInput): Interval {
+  const start = Date.parse(conversation.startIso);
+  const endRaw =
+    conversation.endIso !== undefined ? Date.parse(conversation.endIso) : NaN;
+  // No conversation-level end ("--:--"): derive the window from the latest
+  // SEGMENT extent via the shared primitive, so every end/interval derivation
+  // in the pipeline uses ONE coherent implementation. Falling back only to
+  // `start` would produce a zero/negative-length window that drops or
+  // mis-clusters the conversation (issue #1810).
+  const end = Number.isFinite(endRaw)
+    ? (endRaw as number)
+    : (maxSegmentExtent(conversation.segments)?.ms ?? start);
+  // Clamp universally so the window is always non-negative. An explicit
+  // conversation end that precedes the start (malformed/cross-day input)
+  // would otherwise yield a negative-length window that shrinks the merge
+  // horizon and can split a within-gap cluster.
+  return { start, end: Math.max(end, start) };
+}
+
+function compareConversation(
+  a: FusionConversationInput,
+  b: FusionConversationInput,
+): number {
+  if (a.source !== b.source) return a.source < b.source ? -1 : 1;
+  if (a.conversationId !== b.conversationId) {
+    return a.conversationId < b.conversationId ? -1 : 1;
+  }
+  return 0;
+}
+
+/**
+ * Cluster conversations across sources for one day. Returns clusters in
+ * chronological order; each cluster is sorted by (start, source, id).
+ * Conversations whose start time is unparseable are emitted as
+ * single-element clusters at the tail (sorted by source then id) so no
+ * data is lost.
+ *
+ * ## Cross-source bridging rule (issue #1849)
+ *
+ * Time proximity is meant to bridge the SAME real-world conversation
+ * recorded by DIFFERENT sources — not to merge one source's own distinct
+ * conversations. Two conversations from the SAME source are therefore
+ * never directly joined by time proximity alone; they may end up in the
+ * same cluster only through a cross-source chain where a different-source
+ * conversation within the gap corroborates the bridge. This preserves the
+ * source's own conversation boundaries when no other source corroborates
+ * a merge.
+ *
+ * Concretely, a union-find pass links every pair of conversations that
+ * (a) come from DIFFERENT sources and (b) sit within the proximity gap
+ * (the later start ≤ the earlier end + gap). The transitive closure then
+ * groups same-source conversations that are bridged by at least one
+ * intervening different-source conversation. A single source with two
+ * near-back-to-back conversations and no other source recording the same
+ * window stays in two separate clusters.
+ */
+export function clusterConversations(
+  inputs: readonly FusionConversationInput[],
+  proximityGapMs: number = DEFAULT_PROXIMITY_GAP_MS,
+): FusionConversationInput[][] {
+  const finite: Array<{ conv: FusionConversationInput; interval: Interval }> = [];
+  const noStart: FusionConversationInput[] = [];
+  for (const conv of inputs) {
+    const interval = effectiveInterval(conv);
+    if (Number.isFinite(interval.start)) {
+      finite.push({ conv, interval });
+    } else {
+      noStart.push(conv);
+    }
+  }
+
+  // Deterministic processing + output ordering: (start, source, id).
+  finite.sort((a, b) => {
+    if (a.interval.start !== b.interval.start) {
+      return a.interval.start - b.interval.start;
+    }
+    return compareConversation(a.conv, b.conv);
+  });
+
+  const n = finite.length;
+
+  // Union-find so that same-source pairs are never directly joined by time
+  // proximity alone; they may share a cluster only through a cross-source
+  // bridge (transitive closure over different-source proximity edges).
+  const parent = new Array<number>(n);
+  for (let i = 0; i < n; i++) parent[i] = i;
+
+  const find = (x: number): number => {
+    let root = x;
+    while (parent[root] !== root) root = parent[root];
+    // Path compression.
+    let cur = x;
+    while (parent[cur] !== root) {
+      const next = parent[cur]!;
+      parent[cur] = root;
+      cur = next;
+    }
+    return root;
+  };
+
+  const union = (a: number, b: number): void => {
+    const ra = find(a);
+    const rb = find(b);
+    if (ra !== rb) parent[ra] = rb;
+  };
+
+  // Link every cross-source pair within the proximity gap. `finite` is
+  // sorted by start, so once j's start exceeds i's end + gap, every later
+  // j is also out of reach — break early.
+  for (let i = 0; i < n; i++) {
+    const endGap = finite[i]!.interval.end + proximityGapMs;
+    for (let j = i + 1; j < n; j++) {
+      if (finite[j]!.interval.start > endGap) break;
+      // Only DIFFERENT sources may be directly joined by proximity.
+      if (finite[i]!.conv.source === finite[j]!.conv.source) continue;
+      union(i, j);
+    }
+  }
+
+  // Group conversations by union-find root. Because `finite` is sorted and
+  // we iterate in index order, each group's conversations stay in (start,
+  // source, id) order, and groups are discovered in chronological order
+  // (the first root seen belongs to the earliest-starting conversation).
+  const clusters: FusionConversationInput[][] = [];
+  const rootToCluster = new Map<number, number>();
+  for (let i = 0; i < n; i++) {
+    const root = find(i);
+    let idx = rootToCluster.get(root);
+    if (idx === undefined) {
+      idx = clusters.length;
+      rootToCluster.set(root, idx);
+      clusters.push([]);
+    }
+    clusters[idx]!.push(finite[i]!.conv);
+  }
+
+  // Conversations with no parseable start can't be ordered in time, so
+  // each becomes its own cluster, appended after the time-ordered ones.
+  // Sort for determinism (source, id).
+  noStart.sort(compareConversation);
+  for (const conv of noStart) {
+    clusters.push([conv]);
+  }
+
+  return clusters;
+}