@remnic/core 9.69.33 → 9.69.34

package/src/activity/journal-state.test.ts

@@ -0,0 +1,144 @@
+import assert from "node:assert/strict";
+import { spawn } from "node:child_process";
+import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { createRequire } from "node:module";
+import path from "node:path";
+import test from "node:test";
+import { fileURLToPath } from "node:url";
+
+import { withTempDir, withTempDirSync } from "../testing/tmp-dir.js";
+import {
+  hashJournalText,
+  journalUnchanged,
+  readTimelineState,
+  setJournalHash,
+  timelineStatePath,
+  writeTimelineState,
+} from "./journal-state.js";
+
+const require = createRequire(import.meta.url);
+
+test("missing state file reads as empty", () => {
+  withTempDirSync((dir) => {
+    assert.deepEqual(readTimelineState(dir), { version: 1, journal: {} });
+  });
+});
+
+test("corrupt state file reads as empty and never blocks extraction", () => {
+  withTempDirSync((dir) => {
+    mkdirSync(path.join(dir, "state"), { recursive: true });
+    writeFileSync(timelineStatePath(dir), "{not json");
+    assert.deepEqual(readTimelineState(dir), { version: 1, journal: {} });
+  });
+});
+
+test("non-string or empty hash entries are dropped on read", () => {
+  withTempDirSync((dir) => {
+    writeTimelineState(dir, {
+      version: 1,
+      journal: { "2026-08-20": "abc", "2026-08-21": "", bad: 5 } as unknown as Record<string, string>,
+    });
+    const state = readTimelineState(dir);
+    assert.deepEqual(state.journal, { "2026-08-20": "abc" });
+  });
+});
+
+test("write is a durable round-trip through JSON on disk", () => {
+  withTempDirSync((dir) => {
+    const state = setJournalHash({ version: 1, journal: {} }, "2026-08-20", hashJournalText("day one"));
+    writeTimelineState(dir, state);
+    assert.equal(readFileSync(timelineStatePath(dir), "utf8"), `${JSON.stringify(state, null, 2)}\n`);
+    assert.deepEqual(readTimelineState(dir), state);
+  });
+});
+
+test("journalUnchanged is true only for identical post-strip text", () => {
+  const state = setJournalHash({ version: 1, journal: {} }, "2026-08-20", hashJournalText("text"));
+  assert.equal(journalUnchanged(state, "2026-08-20", "text"), true);
+  assert.equal(journalUnchanged(state, "2026-08-20", "text "), false);
+  assert.equal(journalUnchanged(state, "2026-08-20", "edited"), false);
+  assert.equal(journalUnchanged(state, "2026-08-21", "text"), false);
+  assert.equal(journalUnchanged({ version: 1, journal: {} }, "2026-08-20", "text"), false);
+});
+
+test("setJournalHash is pure — the input state object is not mutated", () => {
+  const original = { version: 1, journal: {} } as const;
+  const next = setJournalHash(original, "2026-08-20", "h");
+  assert.deepEqual(original.journal, {});
+  assert.deepEqual(next.journal, { "2026-08-20": "h" });
+});
+
+test("hashJournalText is deterministic and content-sensitive", () => {
+  assert.equal(hashJournalText("a"), hashJournalText("a"));
+  assert.notEqual(hashJournalText("a"), hashJournalText("b"));
+  assert.match(hashJournalText("a"), /^[0-9a-f]{64}$/);
+});
+
+test("writeTimelineState uses a unique temp path then rename", () => {
+  withTempDirSync((dir) => {
+    const first = setJournalHash({ version: 1, journal: {} }, "2026-08-20", hashJournalText("a"));
+    writeTimelineState(dir, first);
+    const second = setJournalHash(first, "2026-08-21", hashJournalText("b"));
+    writeTimelineState(dir, second);
+    assert.deepEqual(readTimelineState(dir), second);
+    assert.match(readFileSync(timelineStatePath(dir), "utf8"), /2026-08-21/);
+  });
+});
+
+function runLockWorker(env: Record<string, string>): Promise<void> {
+  const tsxCli = require.resolve("tsx/cli");
+  const worker = fileURLToPath(new URL("./journal-state.lock-worker.ts", import.meta.url));
+  return new Promise((resolve, reject) => {
+    const child = spawn(process.execPath, [tsxCli, worker], {
+      env: { ...process.env, ...env },
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+    let stderr = "";
+    child.stderr.setEncoding("utf8");
+    child.stderr.on("data", (chunk) => {
+      stderr += chunk;
+    });
+    child.on("error", reject);
+    child.on("close", (code) => {
+      if (code === 0) resolve();
+      else reject(new Error(`lock worker exited ${code}: ${stderr}`));
+    });
+  });
+}
+
+test("two processes extracting the same date produce one candidate", async () => {
+  await withTempDir(async (dir) => {
+    const marker = path.join(dir, "candidates");
+    writeFileSync(marker, "");
+    const env = {
+      JOURNAL_LOCK_DIR: dir,
+      JOURNAL_LOCK_DATE: "2026-08-20",
+      JOURNAL_LOCK_TEXT: "same day body",
+      JOURNAL_LOCK_MARKER: marker,
+    };
+    await Promise.all([runLockWorker(env), runLockWorker(env)]);
+    assert.equal(readFileSync(marker, "utf8"), "1");
+  });
+});
+
+test("different dates extract concurrently without sharing a lock", async () => {
+  await withTempDir(async (dir) => {
+    const marker = path.join(dir, "candidates");
+    writeFileSync(marker, "");
+    await Promise.all([
+      runLockWorker({
+        JOURNAL_LOCK_DIR: dir,
+        JOURNAL_LOCK_DATE: "2026-08-20",
+        JOURNAL_LOCK_TEXT: "day a",
+        JOURNAL_LOCK_MARKER: marker,
+      }),
+      runLockWorker({
+        JOURNAL_LOCK_DIR: dir,
+        JOURNAL_LOCK_DATE: "2026-08-21",
+        JOURNAL_LOCK_TEXT: "day b",
+        JOURNAL_LOCK_MARKER: marker,
+      }),
+    ]);
+    assert.equal(readFileSync(marker, "utf8"), "11");
+  });
+});

package/src/activity/journal-state.ts

@@ -0,0 +1,107 @@
+/**
+ * Per-day change detection for journal extraction (issue #1987).
+ *
+ * `<memoryDir>/state/timeline.json` stores a content hash of the
+ * POST-STRIP journal text per day. An unchanged day is never re-extracted
+ * (same pattern as the wearables day-skip); a day edited weeks later
+ * re-extracts exactly once per content change.
+ */
+import { createHash, randomBytes } from "node:crypto";
+import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
+import path from "node:path";
+
+import { withHeldFileLock } from "../utils/serialize-mutations.js";
+
+export interface TimelineState {
+  version: 1;
+  journal: Record<string, string>;
+}
+
+const STATE_DIR_NAME = "state";
+const STATE_FILE_NAME = "timeline.json";
+
+export function timelineStatePath(memoryDir: string): string {
+  return path.join(memoryDir, STATE_DIR_NAME, STATE_FILE_NAME);
+}
+
+/** Hash the post-strip section text — Remnic-stripped noise must not re-trigger extraction. */
+export function hashJournalText(text: string): string {
+  return createHash("sha256").update(text, "utf8").digest("hex");
+}
+
+export function readTimelineState(memoryDir: string): TimelineState {
+  const filePath = timelineStatePath(memoryDir);
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(readFileSync(filePath, "utf8"));
+  } catch {
+    // A corrupt state file must never block extraction: treat as empty.
+    return { version: 1, journal: {} };
+  }
+  if (typeof parsed !== "object" || parsed === null || !("journal" in parsed)) {
+    return { version: 1, journal: {} };
+  }
+  const journal: unknown = parsed.journal;
+  if (typeof journal !== "object" || journal === null) return { version: 1, journal: {} };
+  const hashes: Record<string, string> = {};
+  for (const [date, value] of Object.entries(journal)) {
+    if (typeof value === "string" && value.length > 0) hashes[date] = value;
+  }
+  return { version: 1, journal: hashes };
+}
+
+/** Write new before deleting old (§42): unique temp file first, then an atomic rename. */
+export function writeTimelineState(memoryDir: string, state: TimelineState): void {
+  const filePath = timelineStatePath(memoryDir);
+  mkdirSync(path.dirname(filePath), { recursive: true });
+  const tempPath = `${filePath}.${process.pid}.${randomBytes(8).toString("hex")}.tmp`;
+  writeFileSync(tempPath, `${JSON.stringify(state, null, 2)}\n`);
+  renameSync(tempPath, filePath);
+}
+
+export function journalDateLockPath(memoryDir: string, date: string): string {
+  return path.join(memoryDir, STATE_DIR_NAME, `journal-extract-${date}.lock`);
+}
+
+export function timelineStateLockPath(memoryDir: string): string {
+  return `${timelineStatePath(memoryDir)}.lock`;
+}
+
+const DATE_LOCK_OPTS = { staleMs: 120_000, maxWaitMs: 60_000 } as const;
+const STATE_LOCK_OPTS = { staleMs: 30_000, maxWaitMs: 15_000 } as const;
+
+/** Hold the per-date extract lock for hash reread → extract → commit. Fail closed. */
+export async function withJournalDateLock<T>(
+  memoryDir: string,
+  date: string,
+  task: () => Promise<T>,
+): Promise<T> {
+  return withHeldFileLock(journalDateLockPath(memoryDir, date), DATE_LOCK_OPTS, async (acquired) => {
+    if (!acquired) {
+      throw new Error(`journal: could not acquire the extract lock for ${date}`);
+    }
+    return task();
+  });
+}
+
+/** Reread-merge one day's hash under the state-file lock, unique tmp + rename. */
+export async function commitJournalHash(memoryDir: string, date: string, hash: string): Promise<void> {
+  await withHeldFileLock(timelineStateLockPath(memoryDir), STATE_LOCK_OPTS, async (acquired) => {
+    if (!acquired) {
+      throw new Error("journal: could not acquire the timeline state lock");
+    }
+    const latest = readTimelineState(memoryDir);
+    writeTimelineState(memoryDir, setJournalHash(latest, date, hash));
+  });
+}
+
+/** Pure update: set one day's hash. Returns a new state object. */
+export function setJournalHash(state: TimelineState, date: string, hash: string): TimelineState {
+  return { ...state, journal: { ...state.journal, [date]: hash } };
+}
+
+/** True when the day's stored hash matches — unchanged days are skipped. */
+export function journalUnchanged(state: TimelineState, date: string, text: string): boolean {
+  const stored = state.journal[date];
+  return typeof stored === "string" && stored === hashJournalText(text);
+}

package/src/activity/journal-strip.test.ts

@@ -3,61 +3,68 @@ import test from "node:test";
 
 import { stripRemnicOwnedRegions } from "./journal-strip.js";
 
-test("stripRemnicOwnedRegions removes a marker pair and keeps user text", () => {
-  const text = [
-    "Morning standup.",
-    "<!-- remnic:timeline:start -->",
-    "9:00 deep work",
-    "<!-- remnic:timeline:end -->",
-    "Wrote the parser.",
-  ].join("\n");
-  assert.equal(stripRemnicOwnedRegions(text, []), "Morning standup.\nWrote the parser.");
+const START = (name = "timeline") => `<!-- remnic:${name}:start -->`;
+const END = (name = "timeline") => `<!-- remnic:${name}:end -->`;
+
+test("removes a marker pair and keeps user text", () => {
+  const text = ["Morning standup.", START(), "9:00 deep work", END(), "Wrote the parser."].join("\n");
+  const result = stripRemnicOwnedRegions(text, []);
+  assert.equal(result.text, "Morning standup.\nWrote the parser.");
+  assert.deepEqual(result.warnings, []);
 });
 
-test("stripRemnicOwnedRegions keeps one blank line where a removal doubled it", () => {
-  const text = [
-    "Before.",
-    "",
-    "<!-- remnic:timeline:start -->",
-    "generated",
-    "<!-- remnic:timeline:end -->",
-    "",
-    "After.",
-  ].join("\n");
-  assert.equal(stripRemnicOwnedRegions(text, []), "Before.\n\nAfter.");
+test("keeps one blank line where a removal doubled it", () => {
+  const text = ["Before.", "", START(), "generated", END(), "", "After."].join("\n");
+  assert.equal(stripRemnicOwnedRegions(text, []).text, "Before.\n\nAfter.");
 });
 
-test("stripRemnicOwnedRegions removes two distinct marker pairs", () => {
+test("removes two distinct marker pairs", () => {
   const text = [
     "head",
-    "<!-- remnic:timeline:start -->",
+    START("timeline"),
     "timeline body",
-    "<!-- remnic:timeline:end -->",
+    END("timeline"),
     "middle",
-    "<!-- remnic:digest:start -->",
+    START("digest"),
     "digest body",
-    "<!-- remnic:digest:end -->",
+    END("digest"),
     "tail",
   ].join("\n");
-  assert.equal(stripRemnicOwnedRegions(text, []), "head\nmiddle\ntail");
+  const result = stripRemnicOwnedRegions(text, []);
+  assert.equal(result.text, "head\nmiddle\ntail");
+  assert.deepEqual(result.warnings, []);
+});
+
+test("strips an unterminated start marker to the end and warns", () => {
+  const text = ["kept", START(), "generated", "later user text"].join("\n");
+  const result = stripRemnicOwnedRegions(text, []);
+  assert.equal(result.text, "kept");
+  assert.equal(result.warnings.length, 1);
+  assert.match(result.warnings[0]!, /unclosed remnic region "timeline"/);
 });
 
-test("stripRemnicOwnedRegions strips an unterminated start marker to the end of input", () => {
-  const text = ["kept", "<!-- remnic:timeline:start -->", "generated", "later user text"].join("\n");
-  assert.equal(stripRemnicOwnedRegions(text, []), "kept");
+test("an unmatched end marker fails closed: everything before it is stripped, with a warning", () => {
+  // A pair split across the section boundary leaves a bare END inside the
+  // journal section; text before it is presumed published output.
+  const text = ["Before.", END(), "After."].join("\n");
+  const result = stripRemnicOwnedRegions(text, []);
+  assert.equal(result.text, "After.");
+  assert.equal(result.warnings.length, 1);
+  assert.match(result.warnings[0]!, /unmatched remnic region end "timeline"/);
 });
 
-test("stripRemnicOwnedRegions removes an orphan end marker and keeps surrounding text", () => {
-  const text = ["Before.", "<!-- remnic:timeline:end -->", "After."].join("\n");
-  assert.equal(stripRemnicOwnedRegions(text, []), "Before.\nAfter.");
+test("matched pairs after an unmatched end are still removed", () => {
+  const text = ["leaked", END(), "kept", START(), "owned", END(), "also kept"].join("\n");
+  const result = stripRemnicOwnedRegions(text, []);
+  assert.equal(result.text, "kept\nalso kept");
 });
 
-test("stripRemnicOwnedRegions removes an owned heading section up to the next same-level heading", () => {
+test("removes an owned heading section up to the next same-level heading", () => {
   const text = ["## Journal", "intro", "## Timeline", "card one", "## Notes", "kept"].join("\n");
-  assert.equal(stripRemnicOwnedRegions(text, ["Timeline"]), "## Journal\nintro\n## Notes\nkept");
+  assert.equal(stripRemnicOwnedRegions(text, ["Timeline"]).text, "## Journal\nintro\n## Notes\nkept");
 });
 
-test("stripRemnicOwnedRegions does not end an owned section at a deeper heading", () => {
+test("does not end an owned section at a deeper heading", () => {
   const text = [
     "## Journal",
     "## Timeline",
@@ -67,31 +74,35 @@ test("stripRemnicOwnedRegions does not end an owned section at a deeper heading"
     "## Notes",
     "kept",
   ].join("\n");
-  assert.equal(stripRemnicOwnedRegions(text, ["Timeline"]), "## Journal\n## Notes\nkept");
+  assert.equal(stripRemnicOwnedRegions(text, ["Timeline"]).text, "## Journal\n## Notes\nkept");
 });
 
-test("stripRemnicOwnedRegions preserves a non-owned heading section", () => {
+test("preserves a non-owned heading section", () => {
   const text = ["## Journal", "## Timeline", "card one", "## Notes", "kept"].join("\n");
-  assert.equal(stripRemnicOwnedRegions(text, ["Digest"]), text);
+  assert.equal(stripRemnicOwnedRegions(text, ["Digest"]).text, text);
 });
 
-test("stripRemnicOwnedRegions ignores blank and whitespace owned heading entries", () => {
+test("ignores blank and whitespace owned heading entries", () => {
   const text = ["## Timeline", "card one", ""].join("\n");
-  assert.equal(stripRemnicOwnedRegions(text, ["", "   ", "\t"]), text);
+  assert.equal(stripRemnicOwnedRegions(text, ["", "   ", "\t"]).text, text);
 });
 
-test("stripRemnicOwnedRegions trims configured owned heading entries", () => {
+test("trims configured owned heading entries", () => {
   const text = ["## Timeline", "card", "## Notes", "kept"].join("\n");
-  assert.equal(stripRemnicOwnedRegions(text, ["  Timeline  "]), "## Notes\nkept");
+  assert.equal(stripRemnicOwnedRegions(text, ["  Timeline  "]).text, "## Notes\nkept");
 });
 
-test("stripRemnicOwnedRegions returns empty string for empty input", () => {
-  assert.equal(stripRemnicOwnedRegions("", ["Timeline"]), "");
+test("returns empty string for empty input", () => {
+  const result = stripRemnicOwnedRegions("", ["Timeline"]);
+  assert.equal(result.text, "");
+  assert.deepEqual(result.warnings, []);
 });
 
-test("stripRemnicOwnedRegions returns unchanged text when nothing is owned", () => {
+test("returns unchanged text when nothing is owned", () => {
   const text = ["first", "", "second", "", "third"].join("\n");
-  assert.equal(stripRemnicOwnedRegions(text, ["Timeline"]), text);
+  const result = stripRemnicOwnedRegions(text, ["Timeline"]);
+  assert.equal(result.text, text);
+  assert.deepEqual(result.warnings, []);
 });
 
 // Regression: the journal-section reader normalizes tab separators, extra
@@ -108,12 +119,9 @@ test("owned heading matching accepts every ATX form the section reader accepts",
     "## Timeline ##",
     "## Timeline ### ",
   ]) {
-    const stripped = stripRemnicOwnedRegions(
-      [headingLine, body].join("\n"),
-      ["Timeline"],
-    );
+    const result = stripRemnicOwnedRegions([headingLine, body].join("\n"), ["Timeline"]);
     assert.equal(
-      stripped,
+      result.text,
       ["## User Notes", "kept"].join("\n"),
       `heading line ${JSON.stringify(headingLine)} must strip as owned`,
     );
@@ -122,5 +130,51 @@ test("owned heading matching accepts every ATX form the section reader accepts",
 
 test("an indented heading is not owned, matching the section reader", () => {
   const text = ["  ## Timeline", "not owned body"].join("\n");
-  assert.equal(stripRemnicOwnedRegions(text, ["Timeline"]), text);
+  assert.equal(stripRemnicOwnedRegions(text, ["Timeline"]).text, text);
+});
+
+// Generative property test (issue #1987): for random user text with random
+// Remnic regions injected at random positions, strip returns EXACTLY the
+// user text — no user line lost, no owned line kept. Seeded PRNG for
+// deterministic CI.
+test("property: injected regions vanish and user text survives byte-for-byte", () => {
+  let seed = 0x2a1987;
+  const rand = (max: number): number => {
+    seed = (seed * 1103515245 + 12345) & 0x7fffffff;
+    return seed % max;
+  };
+  const USER_LINES = [
+    "Walked the dog.",
+    "Tagebuch 📝 Eintrag.",
+    "  indented thought",
+    "I decided to ship the parser.",
+    "We committed to the review pass.",
+  ];
+  const REGION_NAMES = ["timeline", "weekly", "digest", "standup"];
+
+  for (let iteration = 0; iteration < 200; iteration += 1) {
+    const userLineCount = 1 + rand(USER_LINES.length);
+    const userLines: string[] = [];
+    for (let i = 0; i < userLineCount; i += 1) userLines.push(USER_LINES[rand(USER_LINES.length)]!);
+    const regionCount = rand(4);
+    const injections: Array<{ at: number; lines: string[] }> = [];
+    for (let i = 0; i < regionCount; i += 1) {
+      const name = REGION_NAMES[rand(REGION_NAMES.length)]!;
+      injections.push({
+        at: rand(userLines.length + 1),
+        lines: [START(name), `OWNED ${name} ${i}`, END(name)],
+      });
+    }
+    let mixed: string[] = [];
+    let userCursor = 0;
+    for (const injection of injections.sort((a, b) => a.at - b.at)) {
+      mixed = mixed.concat(userLines.slice(userCursor, injection.at), injection.lines);
+      userCursor = injection.at;
+    }
+    mixed = mixed.concat(userLines.slice(userCursor));
+
+    const result = stripRemnicOwnedRegions(mixed.join("\n"), ["Timeline"]);
+    assert.equal(result.text, userLines.join("\n"), `iteration ${iteration}: ${JSON.stringify(mixed)}`);
+    assert.deepEqual(result.warnings, [], `iteration ${iteration} must be warning-free`);
+  }
 });

package/src/activity/journal-strip.ts

@@ -4,13 +4,19 @@
  * Loop prevention for vault journals: before daily-note text is treated
  * as journal text, every managed marker region and every configured
  * owned heading section is removed, so published output can never
- * re-enter the journal. An unterminated start marker fails closed and
- * strips to the end of the section. Pure string-in/string-out. No
- * filesystem. Heading lines parse through the SAME exported ATX
- * parser the journal-section reader uses, so a heading the reader
- * recognizes is always a heading this stripper strips.
+ * re-enter the journal. Pure string-in/string-out. No filesystem.
+ *
+ * Fail-closed marker semantics — a partial Remnic region is NEVER kept:
+ *  - an unterminated start marker strips to the end of the section and
+ *    records a warning;
+ *  - an end marker with no matching start (a pair split across the
+ *    journal-section boundary) strips everything before it in the
+ *    section, so pre-section published output cannot leak in, and
+ *    records a warning.
+ * Heading lines parse through the SAME exported ATX parser the
+ * journal-section reader uses, so a heading the reader recognizes is
+ * always a heading this stripper strips.
  */
-
 import { parseAtxHeading } from "./journal-section.js";
 
 const MARKER_OPEN = "<!-- remnic:";
@@ -18,16 +24,23 @@ const MARKER_CLOSE = "-->";
 const START_SUFFIX = ":start";
 const END_SUFFIX = ":end";
 
+export interface StripResult {
+  text: string;
+  warnings: readonly string[];
+}
+
 type RegionMarker = { kind: "start" | "end"; name: string };
 
 export function stripRemnicOwnedRegions(
   sectionText: string,
   ownedHeadings: readonly string[],
-): string {
-  return stripOwnedHeadings(stripMarkerRegions(sectionText), ownedHeadings);
+): StripResult {
+  const warnings: string[] = [];
+  const marked = stripMarkerRegions(sectionText, warnings);
+  return { text: stripOwnedHeadings(marked, ownedHeadings), warnings };
 }
 
-function stripMarkerRegions(text: string): string {
+function stripMarkerRegions(text: string, warnings: string[]): string {
   const lines = text.split("\n");
   const keep = lines.map(() => true);
   let openName: string | null = null;
@@ -40,9 +53,21 @@ function stripMarkerRegions(text: string): string {
     }
     if (marker) {
       keep[i] = false;
-      if (marker.kind === "start") openName = marker.name;
+      if (marker.kind === "start") {
+        openName = marker.name;
+      } else {
+        for (let j = 0; j < i; j++) keep[j] = false;
+        // Stray end marker: the matching start lives outside this section
+        // (a pair split across the section boundary). Everything before it
+        // in the section is presumed published output — fail closed.
+        warnings.push(`unmatched remnic region end "${marker.name}"`);
+
+      }
     }
   }
+  if (openName !== null) {
+    warnings.push(`unclosed remnic region "${openName}"`);
+  }
   return emitKeptLines(lines, keep);
 }