@remnic/core 9.69.33 → 9.69.35

package/src/activity/journal-read.ts

@@ -0,0 +1,183 @@
+/**
+ * Filesystem readJournal(D) for the vault daily-note journal (issue #1987).
+ *
+ * Reuses the #1985 template resolver (every layout #1985 can resolve works
+ * here — no separate path configuration) and the #1985 containment
+ * discipline: symlinked vault roots and notes that escape the vault are
+ * refused before any byte is read. Exactly ONE readFile per call; a
+ * missing note is a legitimate no-journal day, not an error (§22).
+ */
+import { closeSync, constants as fsConstants, fstatSync, lstatSync, openSync, readFileSync, realpathSync, statSync } from "node:fs";
+import path from "node:path";
+
+import { expandTildePath } from "../utils/path.js";
+import { readVaultJournal } from "./journal-vault-read.js";
+import type { ActivityTimelineVaultConfig } from "./types.js";
+import { resolveVaultNotePath } from "./vault-path.js";
+
+export type ReadJournalResult =
+  | { ok: true; exists: false; reason: "missing_file" | "missing_heading"; filePath: string }
+  | {
+      ok: true;
+      exists: true;
+      text: string;
+      heading: string;
+      warnings: readonly string[];
+      filePath: string;
+    }
+  | { ok: false; reason: "duplicate_heading"; lines: readonly number[]; filePath: string }
+  | { ok: false; reason: "not_directory" | "path_escape" | "symlink_escape"; filePath: string };
+
+/** Publisher-owned daily-note heading sections that must never count as journal text. */
+export function publisherOwnedSectionNames(vault: ActivityTimelineVaultConfig): string[] {
+  if (vault.sectionStrategy !== "heading") return [];
+  const names: string[] = [];
+  for (const target of Object.values(vault.publish)) {
+    if (target.enabled && target.target === "daily" && target.section.trim().length > 0) {
+      names.push(target.section);
+    }
+  }
+  return names;
+}
+
+export function readJournalForDate(input: {
+  vault: ActivityTimelineVaultConfig;
+  date: string;
+  timezone?: string;
+}): ReadJournalResult {
+  const journalSection = input.vault.readback.journalSection;
+  const relative = resolveVaultNotePath(input.vault.dailyNotePath, input.date, {
+    timezone: input.timezone,
+  });
+  const root = path.resolve(expandTildePath(input.vault.vaultPath));
+  const filePath = path.join(root, relative);
+
+  try {
+    const rootStat = lstatSync(root);
+    if (rootStat.isSymbolicLink() || !rootStat.isDirectory()) {
+      return { ok: false, reason: "not_directory", filePath };
+    }
+  } catch {
+    // The vault root itself is missing: no note can exist, so this is a
+    // no-journal day, not a configuration error.
+    return { ok: true, exists: false, reason: "missing_file", filePath };
+  }
+
+  const lexical = path.relative(root, filePath);
+  if (
+    lexical.length === 0 ||
+    lexical === ".." ||
+    lexical.startsWith(`..${path.sep}`) ||
+    path.isAbsolute(lexical)
+  ) {
+    return { ok: false, reason: "path_escape", filePath };
+  }
+  if (!containedBySymlinks(root, filePath)) {
+    return { ok: false, reason: "symlink_escape", filePath };
+  }
+
+  // One read of a no-follow fd. Ancestor/note symlink swaps cannot redirect
+  // the bytes: the path is verified, then the same fd is read once.
+  let fileText: string;
+  try {
+    fileText = readVerifiedDailyNote(root, filePath);
+  } catch (err) {
+    const code = (err as NodeJS.ErrnoException).code;
+    if (code === "ENOENT") {
+      return { ok: true, exists: false, reason: "missing_file", filePath };
+    }
+    if (code === "ELOOP" || code === "EISDIR") {
+      return { ok: false, reason: "symlink_escape", filePath };
+    }
+    throw err;
+  }
+
+  const result = readVaultJournal({
+    fileText,
+    journalSection,
+    publishSectionNames: publisherOwnedSectionNames(input.vault),
+  });
+  return { ...result, filePath };
+}
+
+/**
+ * Containment under symlinks (mirrors the #1985 publisher helper): the
+ * deepest existing ancestor of the note must still resolve inside the
+ * real vault root. A symlink chain that leaves the vault is refused
+ * before any note byte is read.
+ */
+function containedBySymlinks(root: string, dest: string): boolean {
+  const realRoot = realpathSync(root);
+  let probe = dest;
+  for (;;) {
+    try {
+      if (statSync(probe).isFile() || statSync(probe).isDirectory()) {
+        const real = realpathSync(probe);
+        return real === realRoot || real.startsWith(`${realRoot}${path.sep}`);
+      }
+    } catch {
+      // ENOENT: walk up to the parent.
+    }
+    const parent = path.dirname(probe);
+    if (parent === probe) return false;
+    probe = parent;
+  }
+}
+
+function symlinkRejected(): NodeJS.ErrnoException {
+  const err = new Error("symlink refused") as NodeJS.ErrnoException;
+  err.code = "ELOOP";
+  return err;
+}
+
+/**
+ * Open the verified daily note with O_NOFOLLOW, fstat a regular file whose
+ * identity still matches the path, then read that same fd once. Any symlink
+ * in the root→note chain is refused.
+ */
+function readVerifiedDailyNote(root: string, filePath: string): string {
+  const relative = path.relative(root, filePath);
+  const parts = relative.split(path.sep).filter((part) => part.length > 0 && part !== ".");
+  let cursor = root;
+  for (const part of parts) {
+    cursor = path.join(cursor, part);
+    const st = lstatSync(cursor);
+    if (st.isSymbolicLink()) throw symlinkRejected();
+  }
+
+  const parent = path.dirname(filePath);
+  const parentLstat = lstatSync(parent);
+  if (parentLstat.isSymbolicLink() || !parentLstat.isDirectory()) throw symlinkRejected();
+  const nofollow = fsConstants.O_NOFOLLOW ?? 0;
+  const parentFd = openSync(parent, fsConstants.O_RDONLY | fsConstants.O_DIRECTORY | nofollow);
+  try {
+    const parentFstat = fstatSync(parentFd);
+    if (parentFstat.ino !== parentLstat.ino || parentFstat.dev !== parentLstat.dev) {
+      throw symlinkRejected();
+    }
+  } finally {
+    closeSync(parentFd);
+  }
+
+  const fd = openSync(filePath, fsConstants.O_RDONLY | nofollow);
+  try {
+    const opened = fstatSync(fd);
+    if (!opened.isFile()) throw symlinkRejected();
+    const pathLstat = lstatSync(filePath);
+    if (pathLstat.isSymbolicLink() || pathLstat.ino !== opened.ino || pathLstat.dev !== opened.dev) {
+      throw symlinkRejected();
+    }
+    try {
+      const fdPath = realpathSync(`/proc/self/fd/${fd}`);
+      const realRoot = realpathSync(root);
+      if (fdPath !== realRoot && !fdPath.startsWith(`${realRoot}${path.sep}`)) {
+        throw symlinkRejected();
+      }
+    } catch (err) {
+      if ((err as NodeJS.ErrnoException).code === "ELOOP") throw err;
+    }
+    return readFileSync(fd, "utf8");
+  } finally {
+    closeSync(fd);
+  }
+}

package/src/activity/journal-source.test.ts

@@ -3,34 +3,34 @@ import test from "node:test";
 
 import { resolveJournalSource } from "./journal-source.js";
 
-test("source file resolves to file mode", () => {
-  assert.deepEqual(resolveJournalSource({ source: "file", heading: "" }), {
+test("source memoryDir resolves to memoryDir mode", () => {
+  assert.deepEqual(resolveJournalSource({ source: "memoryDir" }), {
     ok: true,
-    mode: "file",
+    mode: "memoryDir",
   });
 });
 
-test("source vault with a heading resolves to vault mode", () => {
-  assert.deepEqual(resolveJournalSource({ source: "vault", heading: "Journal" }), {
+test("source vault resolves to vault mode", () => {
+  assert.deepEqual(resolveJournalSource({ source: "vault" }), {
     ok: true,
     mode: "vault",
-    heading: "Journal",
   });
 });
 
-test("source vault with an empty heading is missing_heading", () => {
-  assert.deepEqual(resolveJournalSource({ source: "vault", heading: "" }), {
-    ok: false,
-    error: "missing_heading",
-  });
-  assert.deepEqual(resolveJournalSource({ source: "vault", heading: "   " }), {
-    ok: false,
-    error: "missing_heading",
+test("legacy source file aliases memoryDir", () => {
+  assert.deepEqual(resolveJournalSource({ source: "file" }), {
+    ok: true,
+    mode: "memoryDir",
+    deprecatedAlias: "file",
   });
 });
 
 test("unknown source is unknown_source", () => {
-  assert.deepEqual(resolveJournalSource({ source: "memoryDir", heading: "Journal" }), {
+  assert.deepEqual(resolveJournalSource({ source: "" }), {
+    ok: false,
+    error: "unknown_source",
+  });
+  assert.deepEqual(resolveJournalSource({ source: "disk" }), {
     ok: false,
     error: "unknown_source",
   });

package/src/activity/journal-source.ts

@@ -1,23 +1,30 @@
 /**
- * Resolve journal source mode (issue #1987 leftover).
+ * Resolve journal source mode (issue #1987).
  *
- * Pure. File mode ignores heading. Vault mode needs a non-empty heading.
+ * Pure. `memoryDir` mode reads `<memoryDir>/journal/<date>.md` (#1984
+ * behavior, the default); `vault` mode reads the configured section of
+ * the vault daily note. The heading is NOT resolved here — vault
+ * prerequisites (vault.enabled, dailyNotePath, readback.journalSection)
+ * are checked by journal-vault-prereq.ts at config parse time.
+ *
+ * Legacy `source: "file"` is accepted as an alias of `memoryDir` and
+ * flagged so config parse can emit a deprecation warning.
  */
 
+export type JournalSource = "memoryDir" | "vault";
+
+const JOURNAL_SOURCES: readonly JournalSource[] = ["memoryDir", "vault"];
+
 export type JournalSourceResult =
-  | { ok: true; mode: "file" }
-  | { ok: true; mode: "vault"; heading: string }
-  | { ok: false; error: "missing_heading" | "unknown_source" };
+  | { ok: true; mode: JournalSource; deprecatedAlias?: "file" }
+  | { ok: false; error: "unknown_source" };
 
-export function resolveJournalSource(input: {
-  source: string;
-  heading: string;
-}): JournalSourceResult {
-  if (input.source === "file") return { ok: true, mode: "file" };
-  if (input.source === "vault") {
-    const heading = input.heading.trim();
-    if (heading.length === 0) return { ok: false, error: "missing_heading" };
-    return { ok: true, mode: "vault", heading };
+export function resolveJournalSource(input: { source: string }): JournalSourceResult {
+  if (input.source === "file") {
+    return { ok: true, mode: "memoryDir", deprecatedAlias: "file" };
+  }
+  if ((JOURNAL_SOURCES as readonly string[]).includes(input.source)) {
+    return { ok: true, mode: input.source as JournalSource };
   }
   return { ok: false, error: "unknown_source" };
 }

package/src/activity/journal-state.lock-worker.ts

@@ -0,0 +1,30 @@
+/**
+ * Cross-process helper for journal date-lock tests. Spawned as a child:
+ * reread hash → optional candidate marker → commit, all under withJournalDateLock.
+ */
+import { appendFileSync } from "node:fs";
+
+import {
+  commitJournalHash,
+  hashJournalText,
+  journalUnchanged,
+  readTimelineState,
+  withJournalDateLock,
+} from "./journal-state.js";
+
+const memoryDir = process.env.JOURNAL_LOCK_DIR;
+const date = process.env.JOURNAL_LOCK_DATE;
+const text = process.env.JOURNAL_LOCK_TEXT;
+const marker = process.env.JOURNAL_LOCK_MARKER;
+
+if (!memoryDir || !date || !text || !marker) {
+  throw new Error("journal lock worker missing JOURNAL_LOCK_* env");
+}
+
+await withJournalDateLock(memoryDir, date, async () => {
+  const state = readTimelineState(memoryDir);
+  if (journalUnchanged(state, date, text)) return;
+  await new Promise((resolve) => setTimeout(resolve, 80));
+  appendFileSync(marker, "1");
+  await commitJournalHash(memoryDir, date, hashJournalText(text));
+});

package/src/activity/journal-state.test.ts

@@ -0,0 +1,144 @@
+import assert from "node:assert/strict";
+import { spawn } from "node:child_process";
+import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { createRequire } from "node:module";
+import path from "node:path";
+import test from "node:test";
+import { fileURLToPath } from "node:url";
+
+import { withTempDir, withTempDirSync } from "../testing/tmp-dir.js";
+import {
+  hashJournalText,
+  journalUnchanged,
+  readTimelineState,
+  setJournalHash,
+  timelineStatePath,
+  writeTimelineState,
+} from "./journal-state.js";
+
+const require = createRequire(import.meta.url);
+
+test("missing state file reads as empty", () => {
+  withTempDirSync((dir) => {
+    assert.deepEqual(readTimelineState(dir), { version: 1, journal: {} });
+  });
+});
+
+test("corrupt state file reads as empty and never blocks extraction", () => {
+  withTempDirSync((dir) => {
+    mkdirSync(path.join(dir, "state"), { recursive: true });
+    writeFileSync(timelineStatePath(dir), "{not json");
+    assert.deepEqual(readTimelineState(dir), { version: 1, journal: {} });
+  });
+});
+
+test("non-string or empty hash entries are dropped on read", () => {
+  withTempDirSync((dir) => {
+    writeTimelineState(dir, {
+      version: 1,
+      journal: { "2026-08-20": "abc", "2026-08-21": "", bad: 5 } as unknown as Record<string, string>,
+    });
+    const state = readTimelineState(dir);
+    assert.deepEqual(state.journal, { "2026-08-20": "abc" });
+  });
+});
+
+test("write is a durable round-trip through JSON on disk", () => {
+  withTempDirSync((dir) => {
+    const state = setJournalHash({ version: 1, journal: {} }, "2026-08-20", hashJournalText("day one"));
+    writeTimelineState(dir, state);
+    assert.equal(readFileSync(timelineStatePath(dir), "utf8"), `${JSON.stringify(state, null, 2)}\n`);
+    assert.deepEqual(readTimelineState(dir), state);
+  });
+});
+
+test("journalUnchanged is true only for identical post-strip text", () => {
+  const state = setJournalHash({ version: 1, journal: {} }, "2026-08-20", hashJournalText("text"));
+  assert.equal(journalUnchanged(state, "2026-08-20", "text"), true);
+  assert.equal(journalUnchanged(state, "2026-08-20", "text "), false);
+  assert.equal(journalUnchanged(state, "2026-08-20", "edited"), false);
+  assert.equal(journalUnchanged(state, "2026-08-21", "text"), false);
+  assert.equal(journalUnchanged({ version: 1, journal: {} }, "2026-08-20", "text"), false);
+});
+
+test("setJournalHash is pure — the input state object is not mutated", () => {
+  const original = { version: 1, journal: {} } as const;
+  const next = setJournalHash(original, "2026-08-20", "h");
+  assert.deepEqual(original.journal, {});
+  assert.deepEqual(next.journal, { "2026-08-20": "h" });
+});
+
+test("hashJournalText is deterministic and content-sensitive", () => {
+  assert.equal(hashJournalText("a"), hashJournalText("a"));
+  assert.notEqual(hashJournalText("a"), hashJournalText("b"));
+  assert.match(hashJournalText("a"), /^[0-9a-f]{64}$/);
+});
+
+test("writeTimelineState uses a unique temp path then rename", () => {
+  withTempDirSync((dir) => {
+    const first = setJournalHash({ version: 1, journal: {} }, "2026-08-20", hashJournalText("a"));
+    writeTimelineState(dir, first);
+    const second = setJournalHash(first, "2026-08-21", hashJournalText("b"));
+    writeTimelineState(dir, second);
+    assert.deepEqual(readTimelineState(dir), second);
+    assert.match(readFileSync(timelineStatePath(dir), "utf8"), /2026-08-21/);
+  });
+});
+
+function runLockWorker(env: Record<string, string>): Promise<void> {
+  const tsxCli = require.resolve("tsx/cli");
+  const worker = fileURLToPath(new URL("./journal-state.lock-worker.ts", import.meta.url));
+  return new Promise((resolve, reject) => {
+    const child = spawn(process.execPath, [tsxCli, worker], {
+      env: { ...process.env, ...env },
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+    let stderr = "";
+    child.stderr.setEncoding("utf8");
+    child.stderr.on("data", (chunk) => {
+      stderr += chunk;
+    });
+    child.on("error", reject);
+    child.on("close", (code) => {
+      if (code === 0) resolve();
+      else reject(new Error(`lock worker exited ${code}: ${stderr}`));
+    });
+  });
+}
+
+test("two processes extracting the same date produce one candidate", async () => {
+  await withTempDir(async (dir) => {
+    const marker = path.join(dir, "candidates");
+    writeFileSync(marker, "");
+    const env = {
+      JOURNAL_LOCK_DIR: dir,
+      JOURNAL_LOCK_DATE: "2026-08-20",
+      JOURNAL_LOCK_TEXT: "same day body",
+      JOURNAL_LOCK_MARKER: marker,
+    };
+    await Promise.all([runLockWorker(env), runLockWorker(env)]);
+    assert.equal(readFileSync(marker, "utf8"), "1");
+  });
+});
+
+test("different dates extract concurrently without sharing a lock", async () => {
+  await withTempDir(async (dir) => {
+    const marker = path.join(dir, "candidates");
+    writeFileSync(marker, "");
+    await Promise.all([
+      runLockWorker({
+        JOURNAL_LOCK_DIR: dir,
+        JOURNAL_LOCK_DATE: "2026-08-20",
+        JOURNAL_LOCK_TEXT: "day a",
+        JOURNAL_LOCK_MARKER: marker,
+      }),
+      runLockWorker({
+        JOURNAL_LOCK_DIR: dir,
+        JOURNAL_LOCK_DATE: "2026-08-21",
+        JOURNAL_LOCK_TEXT: "day b",
+        JOURNAL_LOCK_MARKER: marker,
+      }),
+    ]);
+    assert.equal(readFileSync(marker, "utf8"), "11");
+  });
+});

package/src/activity/journal-state.ts

@@ -0,0 +1,107 @@
+/**
+ * Per-day change detection for journal extraction (issue #1987).
+ *
+ * `<memoryDir>/state/timeline.json` stores a content hash of the
+ * POST-STRIP journal text per day. An unchanged day is never re-extracted
+ * (same pattern as the wearables day-skip); a day edited weeks later
+ * re-extracts exactly once per content change.
+ */
+import { createHash, randomBytes } from "node:crypto";
+import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
+import path from "node:path";
+
+import { withHeldFileLock } from "../utils/serialize-mutations.js";
+
+export interface TimelineState {
+  version: 1;
+  journal: Record<string, string>;
+}
+
+const STATE_DIR_NAME = "state";
+const STATE_FILE_NAME = "timeline.json";
+
+export function timelineStatePath(memoryDir: string): string {
+  return path.join(memoryDir, STATE_DIR_NAME, STATE_FILE_NAME);
+}
+
+/** Hash the post-strip section text — Remnic-stripped noise must not re-trigger extraction. */
+export function hashJournalText(text: string): string {
+  return createHash("sha256").update(text, "utf8").digest("hex");
+}
+
+export function readTimelineState(memoryDir: string): TimelineState {
+  const filePath = timelineStatePath(memoryDir);
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(readFileSync(filePath, "utf8"));
+  } catch {
+    // A corrupt state file must never block extraction: treat as empty.
+    return { version: 1, journal: {} };
+  }
+  if (typeof parsed !== "object" || parsed === null || !("journal" in parsed)) {
+    return { version: 1, journal: {} };
+  }
+  const journal: unknown = parsed.journal;
+  if (typeof journal !== "object" || journal === null) return { version: 1, journal: {} };
+  const hashes: Record<string, string> = {};
+  for (const [date, value] of Object.entries(journal)) {
+    if (typeof value === "string" && value.length > 0) hashes[date] = value;
+  }
+  return { version: 1, journal: hashes };
+}
+
+/** Write new before deleting old (§42): unique temp file first, then an atomic rename. */
+export function writeTimelineState(memoryDir: string, state: TimelineState): void {
+  const filePath = timelineStatePath(memoryDir);
+  mkdirSync(path.dirname(filePath), { recursive: true });
+  const tempPath = `${filePath}.${process.pid}.${randomBytes(8).toString("hex")}.tmp`;
+  writeFileSync(tempPath, `${JSON.stringify(state, null, 2)}\n`);
+  renameSync(tempPath, filePath);
+}
+
+export function journalDateLockPath(memoryDir: string, date: string): string {
+  return path.join(memoryDir, STATE_DIR_NAME, `journal-extract-${date}.lock`);
+}
+
+export function timelineStateLockPath(memoryDir: string): string {
+  return `${timelineStatePath(memoryDir)}.lock`;
+}
+
+const DATE_LOCK_OPTS = { staleMs: 120_000, maxWaitMs: 60_000 } as const;
+const STATE_LOCK_OPTS = { staleMs: 30_000, maxWaitMs: 15_000 } as const;
+
+/** Hold the per-date extract lock for hash reread → extract → commit. Fail closed. */
+export async function withJournalDateLock<T>(
+  memoryDir: string,
+  date: string,
+  task: () => Promise<T>,
+): Promise<T> {
+  return withHeldFileLock(journalDateLockPath(memoryDir, date), DATE_LOCK_OPTS, async (acquired) => {
+    if (!acquired) {
+      throw new Error(`journal: could not acquire the extract lock for ${date}`);
+    }
+    return task();
+  });
+}
+
+/** Reread-merge one day's hash under the state-file lock, unique tmp + rename. */
+export async function commitJournalHash(memoryDir: string, date: string, hash: string): Promise<void> {
+  await withHeldFileLock(timelineStateLockPath(memoryDir), STATE_LOCK_OPTS, async (acquired) => {
+    if (!acquired) {
+      throw new Error("journal: could not acquire the timeline state lock");
+    }
+    const latest = readTimelineState(memoryDir);
+    writeTimelineState(memoryDir, setJournalHash(latest, date, hash));
+  });
+}
+
+/** Pure update: set one day's hash. Returns a new state object. */
+export function setJournalHash(state: TimelineState, date: string, hash: string): TimelineState {
+  return { ...state, journal: { ...state.journal, [date]: hash } };
+}
+
+/** True when the day's stored hash matches — unchanged days are skipped. */
+export function journalUnchanged(state: TimelineState, date: string, text: string): boolean {
+  const stored = state.journal[date];
+  return typeof stored === "string" && stored === hashJournalText(text);
+}