@remnic/core 9.66.12 → 9.68.0

package/src/activity/journal-strip.test.ts

@@ -0,0 +1,126 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { stripRemnicOwnedRegions } from "./journal-strip.js";
+
+test("stripRemnicOwnedRegions removes a marker pair and keeps user text", () => {
+  const text = [
+    "Morning standup.",
+    "<!-- remnic:timeline:start -->",
+    "9:00 deep work",
+    "<!-- remnic:timeline:end -->",
+    "Wrote the parser.",
+  ].join("\n");
+  assert.equal(stripRemnicOwnedRegions(text, []), "Morning standup.\nWrote the parser.");
+});
+
+test("stripRemnicOwnedRegions keeps one blank line where a removal doubled it", () => {
+  const text = [
+    "Before.",
+    "",
+    "<!-- remnic:timeline:start -->",
+    "generated",
+    "<!-- remnic:timeline:end -->",
+    "",
+    "After.",
+  ].join("\n");
+  assert.equal(stripRemnicOwnedRegions(text, []), "Before.\n\nAfter.");
+});
+
+test("stripRemnicOwnedRegions removes two distinct marker pairs", () => {
+  const text = [
+    "head",
+    "<!-- remnic:timeline:start -->",
+    "timeline body",
+    "<!-- remnic:timeline:end -->",
+    "middle",
+    "<!-- remnic:digest:start -->",
+    "digest body",
+    "<!-- remnic:digest:end -->",
+    "tail",
+  ].join("\n");
+  assert.equal(stripRemnicOwnedRegions(text, []), "head\nmiddle\ntail");
+});
+
+test("stripRemnicOwnedRegions strips an unterminated start marker to the end of input", () => {
+  const text = ["kept", "<!-- remnic:timeline:start -->", "generated", "later user text"].join("\n");
+  assert.equal(stripRemnicOwnedRegions(text, []), "kept");
+});
+
+test("stripRemnicOwnedRegions removes an orphan end marker and keeps surrounding text", () => {
+  const text = ["Before.", "<!-- remnic:timeline:end -->", "After."].join("\n");
+  assert.equal(stripRemnicOwnedRegions(text, []), "Before.\nAfter.");
+});
+
+test("stripRemnicOwnedRegions removes an owned heading section up to the next same-level heading", () => {
+  const text = ["## Journal", "intro", "## Timeline", "card one", "## Notes", "kept"].join("\n");
+  assert.equal(stripRemnicOwnedRegions(text, ["Timeline"]), "## Journal\nintro\n## Notes\nkept");
+});
+
+test("stripRemnicOwnedRegions does not end an owned section at a deeper heading", () => {
+  const text = [
+    "## Journal",
+    "## Timeline",
+    "card one",
+    "### Deep",
+    "deep card",
+    "## Notes",
+    "kept",
+  ].join("\n");
+  assert.equal(stripRemnicOwnedRegions(text, ["Timeline"]), "## Journal\n## Notes\nkept");
+});
+
+test("stripRemnicOwnedRegions preserves a non-owned heading section", () => {
+  const text = ["## Journal", "## Timeline", "card one", "## Notes", "kept"].join("\n");
+  assert.equal(stripRemnicOwnedRegions(text, ["Digest"]), text);
+});
+
+test("stripRemnicOwnedRegions ignores blank and whitespace owned heading entries", () => {
+  const text = ["## Timeline", "card one", ""].join("\n");
+  assert.equal(stripRemnicOwnedRegions(text, ["", "   ", "\t"]), text);
+});
+
+test("stripRemnicOwnedRegions trims configured owned heading entries", () => {
+  const text = ["## Timeline", "card", "## Notes", "kept"].join("\n");
+  assert.equal(stripRemnicOwnedRegions(text, ["  Timeline  "]), "## Notes\nkept");
+});
+
+test("stripRemnicOwnedRegions returns empty string for empty input", () => {
+  assert.equal(stripRemnicOwnedRegions("", ["Timeline"]), "");
+});
+
+test("stripRemnicOwnedRegions returns unchanged text when nothing is owned", () => {
+  const text = ["first", "", "second", "", "third"].join("\n");
+  assert.equal(stripRemnicOwnedRegions(text, ["Timeline"]), text);
+});
+
+// Regression: the journal-section reader normalizes tab separators, extra
+// spaces, trailing whitespace, and closing # sequences. The stripper must
+// recognize exactly the same set, or an owned heading the reader treats as
+// the journal section would keep its generated body in the journal text.
+test("owned heading matching accepts every ATX form the section reader accepts", () => {
+  const body = ["generated line", "", "## User Notes", "kept"].join("\n");
+  for (const headingLine of [
+    "## Timeline",
+    "##  Timeline",
+    "##\tTimeline",
+    "## Timeline   ",
+    "## Timeline ##",
+    "## Timeline ### ",
+  ]) {
+    const stripped = stripRemnicOwnedRegions(
+      [headingLine, body].join("\n"),
+      ["Timeline"],
+    );
+    assert.equal(
+      stripped,
+      ["## User Notes", "kept"].join("\n"),
+      `heading line ${JSON.stringify(headingLine)} must strip as owned`,
+    );
+  }
+});
+
+test("an indented heading is not owned, matching the section reader", () => {
+  const text = ["  ## Timeline", "not owned body"].join("\n");
+  assert.equal(stripRemnicOwnedRegions(text, ["Timeline"]), text);
+});

package/src/activity/journal-strip.ts

@@ -0,0 +1,115 @@
+/**
+ * Strip Remnic-owned regions from a vault journal section (issue #1987).
+ *
+ * Loop prevention for vault journals: before daily-note text is treated
+ * as journal text, every managed marker region and every configured
+ * owned heading section is removed, so published output can never
+ * re-enter the journal. An unterminated start marker fails closed and
+ * strips to the end of the section. Pure string-in/string-out. No
+ * filesystem. Heading lines parse through the SAME exported ATX
+ * parser the journal-section reader uses, so a heading the reader
+ * recognizes is always a heading this stripper strips.
+ */
+
+import { parseAtxHeading } from "./journal-section.js";
+
+const MARKER_OPEN = "<!-- remnic:";
+const MARKER_CLOSE = "-->";
+const START_SUFFIX = ":start";
+const END_SUFFIX = ":end";
+
+type RegionMarker = { kind: "start" | "end"; name: string };
+
+export function stripRemnicOwnedRegions(
+  sectionText: string,
+  ownedHeadings: readonly string[],
+): string {
+  return stripOwnedHeadings(stripMarkerRegions(sectionText), ownedHeadings);
+}
+
+function stripMarkerRegions(text: string): string {
+  const lines = text.split("\n");
+  const keep = lines.map(() => true);
+  let openName: string | null = null;
+  for (let i = 0; i < lines.length; i++) {
+    const marker = parseRegionMarker(lines[i]!);
+    if (openName !== null) {
+      keep[i] = false;
+      if (marker && marker.kind === "end" && marker.name === openName) openName = null;
+      continue;
+    }
+    if (marker) {
+      keep[i] = false;
+      if (marker.kind === "start") openName = marker.name;
+    }
+  }
+  return emitKeptLines(lines, keep);
+}
+
+function stripOwnedHeadings(text: string, ownedHeadings: readonly string[]): string {
+  const wanted = ownedHeadingSet(ownedHeadings);
+  if (wanted.size === 0 || text.length === 0) return text;
+  const lines = text.split("\n");
+  const keep = lines.map(() => true);
+  for (let i = 0; i < lines.length; i++) {
+    if (!keep[i]) continue;
+    const heading = parseAtxHeading(lines[i]!);
+    if (!heading || !wanted.has(heading.text)) continue;
+    keep[i] = false;
+    for (let j = i + 1; j < lines.length; j++) {
+      const next = parseAtxHeading(lines[j]!);
+      if (next && next.level <= heading.level) break;
+      keep[j] = false;
+    }
+  }
+  return emitKeptLines(lines, keep);
+}
+
+/**
+ * Emit kept lines, collapsing the one blank line each removal would
+ * otherwise double. Text outside removals is preserved byte-for-byte.
+ */
+function emitKeptLines(lines: readonly string[], keep: readonly boolean[]): string {
+  const out: string[] = [];
+  let dropped = false;
+  for (let i = 0; i < lines.length; i++) {
+    if (!keep[i]) {
+      dropped = true;
+      continue;
+    }
+    const line = lines[i]!;
+    const previous = out.length > 0 ? out[out.length - 1]! : null;
+    if (dropped && previous !== null && previous.trim().length === 0 && line.trim().length === 0) {
+      dropped = false;
+      continue;
+    }
+    out.push(line);
+    dropped = false;
+  }
+  return out.join("\n");
+}
+
+function parseRegionMarker(line: string): RegionMarker | null {
+  const trimmed = line.trim();
+  if (!trimmed.startsWith(MARKER_OPEN) || !trimmed.endsWith(MARKER_CLOSE)) return null;
+  const inner = trimmed.slice(MARKER_OPEN.length, trimmed.length - MARKER_CLOSE.length).trim();
+  if (inner.endsWith(START_SUFFIX)) {
+    const name = inner.slice(0, -START_SUFFIX.length);
+    return name.length > 0 ? { kind: "start", name } : null;
+  }
+  if (inner.endsWith(END_SUFFIX)) {
+    const name = inner.slice(0, -END_SUFFIX.length);
+    return name.length > 0 ? { kind: "end", name } : null;
+  }
+  return null;
+}
+
+function ownedHeadingSet(ownedHeadings: readonly string[]): Set<string> {
+  const wanted = new Set<string>();
+  for (const entry of ownedHeadings) {
+    if (typeof entry !== "string") continue;
+    const trimmed = entry.trim();
+    if (trimmed.length > 0) wanted.add(trimmed);
+  }
+  return wanted;
+}

package/src/activity/privacy-delete-plan.test.ts

@@ -0,0 +1,215 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { MS_PER_DAY } from "./privacy.js";
+import {
+  ACTIVITY_DELETE_SCOPES,
+  planActivityDeletion,
+  type ActivityDeleteCandidate,
+} from "./privacy-delete-plan.js";
+
+const NOW = 100 * MS_PER_DAY;
+
+function cand(
+  scope: string,
+  relPath: string,
+  capturedAtMs: number,
+): ActivityDeleteCandidate {
+  return { scope, relPath, capturedAtMs };
+}
+
+function plan(
+  candidates: readonly ActivityDeleteCandidate[],
+  overrides: Partial<Parameters<typeof planActivityDeletion>[0]> = {},
+) {
+  return planActivityDeletion({
+    candidates,
+    scopes: ACTIVITY_DELETE_SCOPES,
+    retentionDays: 30,
+    nowMs: NOW,
+    ...overrides,
+  });
+}
+
+test("expired in-scope activity paths are planned across every owned root", () => {
+  const expired = NOW - 40 * MS_PER_DAY;
+  const result = plan([
+    cand("observations", "activity/2026-01-01.md", expired),
+    cand("cards", "wearables/limitless/2026-01-01.md", expired),
+    cand("journal", "meetings/2026-01-01/mtg-x.md", expired),
+  ]);
+  assert.deepEqual(result.deletePaths, [
+    "activity/2026-01-01.md",
+    "meetings/2026-01-01/mtg-x.md",
+    "wearables/limitless/2026-01-01.md",
+  ]);
+  assert.equal(result.keptCount, 0);
+  assert.deepEqual(result.refused, []);
+});
+
+test("fresh in-scope candidates are kept, not deleted", () => {
+  const result = plan([cand("observations", "activity/2026-08-19.md", NOW - MS_PER_DAY)]);
+  assert.deepEqual(result.deletePaths, []);
+  assert.equal(result.keptCount, 1);
+  assert.deepEqual(result.refused, []);
+});
+
+test("retentionDays 0 deletes nothing but counts kept", () => {
+  const ancient = NOW - 4000 * MS_PER_DAY;
+  const result = plan([cand("weekly", "activity/2015-01-01.md", ancient)], {
+    retentionDays: 0,
+  });
+  assert.deepEqual(result.deletePaths, []);
+  assert.equal(result.keptCount, 1);
+});
+
+test("facts paths are refused even when expired and in scope", () => {
+  const expired = NOW - 40 * MS_PER_DAY;
+  const result = plan([cand("observations", "facts/2026-01-01/fact-1.md", expired)]);
+  assert.deepEqual(result.deletePaths, []);
+  assert.deepEqual(result.refused, ["facts/2026-01-01/fact-1.md"]);
+  assert.equal(result.keptCount, 0);
+});
+
+test("profile.md, entities, and state files are refused", () => {
+  const expired = NOW - 40 * MS_PER_DAY;
+  const result = plan([
+    cand("journal", "profile.md", expired),
+    cand("journal", "entities/person-a.md", expired),
+    cand("providerCache", "state/buffer.json", expired),
+  ]);
+  assert.deepEqual(result.deletePaths, []);
+  assert.deepEqual(result.refused, ["entities/person-a.md", "profile.md", "state/buffer.json"]);
+});
+
+test("absolute, traversal, and blank paths are refused", () => {
+  const expired = NOW - 40 * MS_PER_DAY;
+  const result = plan([
+    cand("observations", "/etc/passwd", expired),
+    cand("observations", "activity/../../facts/fact-1.md", expired),
+    cand("observations", "   ", expired),
+  ]);
+  assert.deepEqual(result.deletePaths, []);
+  assert.deepEqual(result.refused, [
+    "   ",
+    "/etc/passwd",
+    "activity/../../facts/fact-1.md",
+  ]);
+});
+
+test("unknown candidate scope is refused", () => {
+  const expired = NOW - 40 * MS_PER_DAY;
+  const result = plan([cand("bogus", "activity/2026-01-01.md", expired)]);
+  assert.deepEqual(result.deletePaths, []);
+  assert.deepEqual(result.refused, ["activity/2026-01-01.md"]);
+  assert.equal(result.keptCount, 0);
+});
+
+test("scope absent from scopes is skipped entirely", () => {
+  const expired = NOW - 40 * MS_PER_DAY;
+  const result = plan([cand("cards", "activity/2026-01-01.md", expired)], {
+    scopes: ["observations"],
+  });
+  assert.deepEqual(result.deletePaths, []);
+  assert.deepEqual(result.refused, []);
+  assert.equal(result.keptCount, 0);
+});
+
+test("unknown value in scopes throws RangeError", () => {
+  assert.throws(
+    () => plan([], { scopes: ["observations", "everything"] }),
+    (error: unknown) => error instanceof RangeError && /scope/.test(error.message),
+  );
+});
+
+test("negative or float retentionDays throws RangeError", () => {
+  assert.throws(
+    () => plan([], { retentionDays: -1 }),
+    (error: unknown) => error instanceof RangeError && /retentionDays/.test(error.message),
+  );
+  assert.throws(
+    () => plan([], { retentionDays: 2.5 }),
+    (error: unknown) => error instanceof RangeError && /retentionDays/.test(error.message),
+  );
+});
+
+test("non-finite nowMs throws RangeError", () => {
+  assert.throws(
+    () => plan([], { nowMs: Number.POSITIVE_INFINITY }),
+    (error: unknown) => error instanceof RangeError && /nowMs/.test(error.message),
+  );
+  assert.throws(
+    () => plan([], { nowMs: Number.NaN }),
+    (error: unknown) => error instanceof RangeError && /nowMs/.test(error.message),
+  );
+});
+
+test("deletePaths and refused are sorted and deduplicated", () => {
+  const expired = NOW - 40 * MS_PER_DAY;
+  const result = plan([
+    cand("observations", "activity/b.md", expired),
+    cand("cards", "activity/a.md", expired),
+    cand("journal", "activity/b.md", expired),
+    cand("observations", "facts/z.md", expired),
+    cand("cards", "facts/z.md", expired),
+  ]);
+  assert.deepEqual(result.deletePaths, ["activity/a.md", "activity/b.md"]);
+  assert.deepEqual(result.refused, ["facts/z.md"]);
+});
+
+test("half-open boundary: age exactly retentionDays * MS_PER_DAY is deleted", () => {
+  const boundary = NOW - 30 * MS_PER_DAY;
+  const result = plan([
+    cand("observations", "activity/boundary.md", boundary),
+    cand("observations", "activity/inside.md", boundary + 1),
+  ]);
+  assert.deepEqual(result.deletePaths, ["activity/boundary.md"]);
+  assert.equal(result.keptCount, 1);
+});
+
+test("input candidates are not mutated", () => {
+  const candidates = [
+    cand("observations", "activity/2026-01-01.md", NOW - 40 * MS_PER_DAY),
+    cand("observations", "facts/2026-01-01/fact-1.md", NOW - 40 * MS_PER_DAY),
+  ];
+  const snapshot = structuredClone(candidates);
+  plan(candidates);
+  assert.deepEqual(candidates, snapshot);
+});
+
+// Review round 1: the planner trimmed relPath before validating it, so
+// " activity/x.md" validated as owned and the plan then targeted
+// "activity/x.md" — a path the caller never supplied.
+test("a path with surrounding whitespace is refused, not normalized into a delete", () => {
+  const expired = 0;
+  for (const relPath of [" activity/day.md", "activity/day.md ", "\tactivity/day.md"]) {
+    const plan = planActivityDeletion({
+      candidates: [{ scope: "cards", relPath, capturedAtMs: expired }],
+      scopes: ["cards"],
+      retentionDays: 1,
+      nowMs: 10 * 86_400_000,
+    });
+    assert.deepEqual(plan.deletePaths, [], `${JSON.stringify(relPath)} must not be planned`);
+    assert.deepEqual(plan.refused, [relPath], "the exact candidate path is reported as refused");
+  }
+});
+
+// Review round 1: every comparison against NaN is false, so shouldRetain
+// reported "not retained" and a corrupt timestamp resolved toward deletion.
+test("a non-finite capturedAtMs is refused instead of deleted", () => {
+  for (const capturedAtMs of [Number.NaN, Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY]) {
+    const plan = planActivityDeletion({
+      candidates: [{ scope: "cards", relPath: "activity/corrupt.md", capturedAtMs }],
+      scopes: ["cards"],
+      retentionDays: 1,
+      nowMs: 10 * 86_400_000,
+    });
+    assert.deepEqual(
+      plan.deletePaths,
+      [],
+      `capturedAtMs ${String(capturedAtMs)} must never resolve toward deletion`,
+    );
+    assert.deepEqual(plan.refused, ["activity/corrupt.md"]);
+    assert.equal(plan.keptCount, 0);
+  }
+});

package/src/activity/privacy-delete-plan.ts

@@ -0,0 +1,116 @@
+/**
+ * Activity retention deletion planner (issue #2053).
+ *
+ * Pure: decides what a retention pass may delete. Never performs I/O.
+ * Safety rule: a path outside the activity-owned roots is refused, never
+ * planned for deletion, so unrelated Remnic memories survive.
+ */
+import { shouldRetain } from "./privacy.js";
+
+export const ACTIVITY_DELETE_SCOPES = [
+  "observations",
+  "cards",
+  "journal",
+  "weekly",
+  "providerCache",
+] as const;
+export type ActivityDeleteScope = (typeof ACTIVITY_DELETE_SCOPES)[number];
+
+const KNOWN_SCOPES: readonly string[] = ACTIVITY_DELETE_SCOPES;
+
+/** Deletion only ever touches these memory-dir-relative roots. */
+const OWNED_ROOTS: readonly string[] = ["activity", "wearables", "meetings"];
+
+export interface ActivityDeleteCandidate {
+  /** Which activity artifact family this item belongs to. */
+  scope: string;
+  /** Memory-dir-relative path, POSIX separators. */
+  relPath: string;
+  capturedAtMs: number;
+}
+
+export interface ActivityDeletePlan {
+  deletePaths: string[];
+  keptCount: number;
+  /** Candidates refused because they are not activity-owned paths. */
+  refused: string[];
+}
+
+function isRefusablePath(relPath: unknown): boolean {
+  if (typeof relPath !== "string") return true;
+  if (relPath.trim() === "") return true;
+  // Validate the EXACT candidate path. Trimming first would let
+  // " activity/x.md" validate as owned and then plan a delete for
+  // "activity/x.md" — a path the caller never supplied. Surrounding
+  // whitespace means this is not a clean owned path: refuse it.
+  if (relPath !== relPath.trim()) return true;
+  if (relPath.startsWith("/")) return true;
+  const segments = relPath.split("/");
+  if (segments.includes("..")) return true;
+  const [root = "", ...rest] = segments;
+  return (
+    rest.length === 0 ||
+    !OWNED_ROOTS.includes(root) ||
+    rest.some((segment) => segment.length === 0)
+  );
+}
+
+function sortedUnique(paths: Iterable<string>): string[] {
+  return [...new Set(paths)].sort();
+}
+
+export function planActivityDeletion(input: {
+  candidates: readonly ActivityDeleteCandidate[];
+  scopes: readonly string[];
+  retentionDays: number;
+  nowMs: number;
+}): ActivityDeletePlan {
+  const { candidates, scopes, retentionDays, nowMs } = input;
+
+  for (const scope of scopes) {
+    if (!KNOWN_SCOPES.includes(scope)) {
+      throw new RangeError(`unknown activity delete scope: ${scope}`);
+    }
+  }
+  if (!Number.isInteger(retentionDays) || retentionDays < 0) {
+    throw new RangeError("retentionDays must be a non-negative integer");
+  }
+  if (!Number.isFinite(nowMs)) {
+    throw new RangeError("nowMs must be a finite number");
+  }
+
+  const selectedScopes = new Set(scopes);
+  const deletePaths: string[] = [];
+  const refused: string[] = [];
+  let keptCount = 0;
+
+  for (const candidate of candidates) {
+    if (!KNOWN_SCOPES.includes(candidate.scope)) {
+      refused.push(String(candidate.relPath));
+      continue;
+    }
+    if (!selectedScopes.has(candidate.scope)) continue;
+    if (isRefusablePath(candidate.relPath)) {
+      refused.push(String(candidate.relPath));
+      continue;
+    }
+    // A corrupt timestamp must never resolve toward deletion: every
+    // comparison against NaN is false, which shouldRetain reads as
+    // "not retained". Refuse the candidate instead.
+    if (typeof candidate.capturedAtMs !== "number" || !Number.isFinite(candidate.capturedAtMs)) {
+      refused.push(candidate.relPath);
+      continue;
+    }
+    if (shouldRetain(candidate.capturedAtMs, nowMs, retentionDays, true)) {
+      keptCount += 1;
+      continue;
+    }
+    deletePaths.push(candidate.relPath);
+  }
+
+  return {
+    deletePaths: sortedUnique(deletePaths),
+    keptCount,
+    refused: sortedUnique(refused),
+  };
+}

package/src/activity/timeline/index.ts

@@ -22,3 +22,4 @@ export * from "./analysis-claim.js";
 export * from "./analysis-order.js";
 export * from "./recap-window.js";
 export * from "./analysis-failure.js";
+export * from "./recap-merge-edits.js";