@remnic/core 9.69.33 → 9.69.35

package/src/activity/journal-extract.ts

@@ -0,0 +1,247 @@
+/**
+ * Review-only journal extraction pass (issue #1987, reusing #1984's design).
+ *
+ * Vault journal text is user-authored but unverified: candidates land
+ * `pending_review` ONLY — there is no auto-approve path by design, so no
+ * journal-derived memory ever reaches `active` without explicit review
+ * approval. A judge reject drops the candidate even in review mode.
+ * Standard sanitization applies before extraction; unsafe text extracts
+ * nothing (the placeholder is never written as memory content).
+ *
+ * Per-day change detection lives in journal-state.ts (timeline.json);
+ * this module is the pure pass over one day's POST-STRIP text.
+ */
+import { scoreImportance } from "../importance.js";
+import { getVerdictKind, type JudgeCandidate, type JudgeVerdict } from "../extraction-judge.js";
+import { log } from "../logger.js";
+import { sanitizeMemoryContent } from "../sanitize.js";
+import { stripAttributesSuffix } from "../structured-attributes.js";
+import type {
+  BufferTurn,
+  ExtractedFact,
+  ExtractionResult,
+  ImportanceScore,
+  MemoryStatus,
+} from "../types.js";
+import { composeMemoryEnvelope, type SealedMemoryEnvelope, type WriteContext } from "../write-envelope.js";
+import { buildJournalMemoryProvenance } from "./journal-vault-provenance.js";
+import type { ActivityTimelineJournalConfig } from "./types.js";
+
+/** The subset of StorageManager the journal pass writes through. */
+export interface JournalStorageIo {
+  writeSealedMemory(
+    envelope: SealedMemoryEnvelope,
+    extras: { status: MemoryStatus; contentHashSource: string; importance?: ImportanceScore },
+  ): Promise<{ tombstoneBlocked?: boolean }>;
+  hasFactContentHash(content: string): Promise<boolean>;
+  readAllMemories(): Promise<Array<{ path: string; frontmatter: { tags?: string[]; source?: unknown }; content: string }>>;
+}
+
+export interface JournalMemoryWriter {
+  writeSealedMemory: JournalStorageIo["writeSealedMemory"];
+  /** Dedup beyond the fact-only hash index: scans journal-tagged memories. */
+  hasJournalMemoryContent(content: string): Promise<boolean>;
+}
+
+/**
+ * The storage fact-hash index only covers category "fact"; journal
+ * candidates span every category, so dedup additionally scans existing
+ * journal-tagged memories for an exact content match (wearables parity,
+ * Codex P2 on PR #1458).
+ */
+export function createJournalMemoryWriter(storage: JournalStorageIo): JournalMemoryWriter {
+  return {
+    writeSealedMemory: storage.writeSealedMemory.bind(storage),
+    hasJournalMemoryContent: async (content: string) => {
+      if (await storage.hasFactContentHash(content)) return true;
+      const needle = stripAttributesSuffix(content);
+      const memories = await storage.readAllMemories();
+      return memories.some(
+        (memory) =>
+          Array.isArray(memory.frontmatter.tags) &&
+          memory.frontmatter.tags.includes("journal") &&
+          stripAttributesSuffix(memory.content) === needle,
+      );
+    },
+  };
+}
+
+/** Result of one day's journal extraction run, including the skip reason. */
+export type JournalExtractionRunResult = JournalExtractionResult & {
+  skippedReason?: string;
+  filePath?: string;
+};
+export interface JournalExtractionDeps {
+  extract(turns: BufferTurn[]): Promise<ExtractionResult>;
+  writer: JournalMemoryWriter;
+  /** LLM-as-judge batch evaluation; absent degrades to review-only writes. */
+  judge?(candidates: JudgeCandidate[]): Promise<Map<number, JudgeVerdict>>;
+  now?(): Date;
+  /** Fires once after any candidate write — the §31 reindex hook. */
+  afterWrites?(): Promise<void>;
+}
+
+export interface JournalExtractionResult {
+  /** pending_review candidates written this run. */
+  pendingReview: number;
+  /** Candidates the judge rejected — dropped, never queued. */
+  rejectedByJudge: number;
+  /** Candidates skipped by in-run or persisted dedup. */
+  skipped: number;
+  /** True when the pass extracted the day (caller advances the state hash). */
+  completed: boolean;
+  warnings: string[];
+}
+
+export async function runJournalReviewExtraction(input: {
+  date: string;
+  journalText: string;
+  source: "memoryDir" | "vault";
+  journalConfig: Pick<ActivityTimelineJournalConfig, "extractionMode">;
+  deps: JournalExtractionDeps;
+}): Promise<JournalExtractionResult> {
+  const result: JournalExtractionResult = {
+    pendingReview: 0,
+    rejectedByJudge: 0,
+    skipped: 0,
+    completed: false,
+    warnings: [],
+  };
+  if (input.journalConfig.extractionMode !== "review") return result;
+  if (input.journalText.trim().length === 0) {
+    result.completed = true;
+    return result;
+  }
+
+  // Standard sanitization like any other source. A violation extracts
+  // NOTHING — the sanitized placeholder must never become memory content.
+  const sanitized = sanitizeMemoryContent(input.journalText);
+  if (!sanitized.clean) {
+    result.warnings.push(
+      `journal text for ${input.date} failed sanitization (${sanitized.violations.length} violation(s)); extraction skipped`,
+    );
+    result.completed = true;
+    return result;
+  }
+
+  const provenance = buildJournalMemoryProvenance({ source: input.source, date: input.date });
+
+  let extracted: ExtractionResult;
+  try {
+    extracted = await input.deps.extract([
+      {
+        role: "user",
+        content: sanitized.text,
+        timestamp: (input.deps.now ?? ((): Date => new Date()))().toISOString(),
+      },
+    ]);
+  } catch (err) {
+    // Provider/parse failure must not throw out of the pass; the caller
+    // does NOT advance the state hash, so the next run retries this day.
+    log.warn(
+      `journal extraction failed for ${input.date}: ${err instanceof Error ? err.message : String(err)}`,
+    );
+    return result;
+  }
+  if (extracted.extractionFailure !== undefined) {
+    log.warn(
+      `journal extraction reported a failure for ${input.date}: ${extracted.extractionFailure}`,
+    );
+    return result;
+  }
+  result.completed = true;
+
+  let verdicts = new Map<number, JudgeVerdict>();
+  const candidates = dedupeInRun(extracted.facts, result);
+  if (input.deps.judge !== undefined && candidates.length > 0) {
+    try {
+      verdicts = await input.deps.judge(
+        candidates.map((fact) => ({
+          text: fact.content,
+          category: fact.category,
+          confidence: fact.confidence,
+          tags: fact.tags,
+          importanceLevel: scoreImportance(fact.content, fact.category, fact.tags).level,
+        })),
+      );
+    } catch (err) {
+      log.warn(
+        `journal extraction judge unavailable for ${input.date}: ${err instanceof Error ? err.message : String(err)} — review-only writes continue without verdicts`,
+      );
+    }
+  }
+
+  let wrote = false;
+  const ctx: WriteContext = { source: "journal", now: input.deps.now };
+  for (let index = 0; index < candidates.length; index += 1) {
+    const fact = candidates[index]!;
+    const verdict = verdicts.get(index);
+    const verdictKind = verdict === undefined ? undefined : getVerdictKind(verdict);
+    if (verdictKind === "reject") {
+      result.rejectedByJudge += 1;
+      continue;
+    }
+    const normalizedKey = fact.content.trim().toLowerCase();
+    if (await input.deps.writer.hasJournalMemoryContent(fact.content)) {
+      result.skipped += 1;
+      continue;
+    }
+    // Journal text is machine-extracted: salvage mode, warn-logged drops.
+    const envelope = composeMemoryEnvelope(
+      {
+        content: fact.content,
+        category: fact.category,
+        tags: [...fact.tags, ...provenance.tags],
+        entityRef: fact.entityRef,
+        confidence: fact.confidence,
+        validAt: provenance.validAt,
+        structuredAttributes: { ...provenance.structuredAttributes },
+        sourceConnector: "journal",
+        sourceReason: input.source === "vault" ? "vault daily-note journal" : "memoryDir journal",
+      },
+      ctx,
+      { salvage: true },
+    );
+    if (envelope.salvageNotes.length > 0) {
+      log.warn(`journal write salvaged invalid fields: ${envelope.salvageNotes.join("; ")}`);
+    }
+    const write = await input.deps.writer.writeSealedMemory(envelope, {
+      status: "pending_review",
+      contentHashSource: normalizedKey,
+      importance: scoreImportance(fact.content, fact.category, fact.tags),
+    });
+    if (write.tombstoneBlocked) {
+      result.skipped += 1;
+      continue;
+    }
+    result.pendingReview += 1;
+    wrote = true;
+  }
+
+  if (wrote && input.deps.afterWrites !== undefined) {
+    try {
+      await input.deps.afterWrites();
+    } catch (err) {
+      // The writes are durable; a reindex failure is reported, not thrown
+      // (§31 — the data indexes on the next update).
+      result.warnings.push(
+        `reindex after journal writes failed: ${err instanceof Error ? err.message : String(err)}`,
+      );
+    }
+  }
+  return result;
+}
+
+/** Keep the strongest copy per normalized content so one day cannot duplicate itself. */
+function dedupeInRun(
+  facts: ExtractedFact[],
+  result: JournalExtractionResult,
+): ExtractedFact[] {
+  const seen = new Map<string, ExtractedFact>();
+  for (const fact of facts) {
+    const key = fact.content.trim().toLowerCase();
+    if (!seen.has(key)) seen.set(key, fact);
+    else result.skipped += 1;
+  }
+  return [...seen.values()];
+}

package/src/activity/journal-heading.test.ts

@@ -1,7 +1,7 @@
 import assert from "node:assert/strict";
 import test from "node:test";
 
-import { normalizeJournalHeading } from "./journal-heading.js";
+import { applyLegacyJournalHeading, normalizeJournalHeading } from "./journal-heading.js";
 
 test("normalizeJournalHeading accepts a heading", () => {
   assert.deepEqual(normalizeJournalHeading("Journal"), { ok: true, heading: "Journal" });
@@ -20,3 +20,26 @@ test("normalizeJournalHeading rejects a heading with a newline", () => {
 test("normalizeJournalHeading trims surrounding whitespace", () => {
   assert.deepEqual(normalizeJournalHeading("  Journal  "), { ok: true, heading: "Journal" });
 });
+
+test("applyLegacyJournalHeading copies heading when journalSection is empty", () => {
+  assert.deepEqual(applyLegacyJournalHeading({ journalSection: "", heading: "  Diary  " }), {
+    journalSection: "Diary",
+    usedLegacyHeading: true,
+    ignoredLegacyHeading: false,
+  });
+});
+
+test("applyLegacyJournalHeading keeps journalSection when both keys are set", () => {
+  assert.deepEqual(applyLegacyJournalHeading({ journalSection: "Journal", heading: "Diary" }), {
+    journalSection: "Journal",
+    usedLegacyHeading: false,
+    ignoredLegacyHeading: true,
+  });
+});
+
+test("applyLegacyJournalHeading rejects an empty legacy heading", () => {
+  assert.throws(
+    () => applyLegacyJournalHeading({ journalSection: "", heading: "   " }),
+    /activity\.timeline\.journal\.heading must be a non-empty string/,
+  );
+});

package/src/activity/journal-heading.ts

@@ -16,3 +16,39 @@ export function normalizeJournalHeading(value: string): NormalizeJournalHeadingR
   }
   return { ok: true, heading };
 }
+
+/**
+ * Legacy `activity.timeline.journal.heading` → `vault.readback.journalSection`.
+ * The new key wins when both are present; the legacy key is still validated.
+ */
+export function applyLegacyJournalHeading(input: {
+  journalSection: string;
+  heading: unknown;
+}): { journalSection: string; usedLegacyHeading: boolean; ignoredLegacyHeading: boolean } {
+  if (input.heading === undefined) {
+    return { journalSection: input.journalSection, usedLegacyHeading: false, ignoredLegacyHeading: false };
+  }
+  if (typeof input.heading !== "string") {
+    throw new TypeError("activity.timeline.journal.heading must be a string");
+  }
+  const normalized = normalizeJournalHeading(input.heading);
+  if (!normalized.ok) {
+    throw new RangeError(
+      normalized.error === "empty_heading"
+        ? "activity.timeline.journal.heading must be a non-empty string"
+        : "activity.timeline.journal.heading must not contain a newline",
+    );
+  }
+  if (input.journalSection.trim().length > 0) {
+    return {
+      journalSection: input.journalSection,
+      usedLegacyHeading: false,
+      ignoredLegacyHeading: true,
+    };
+  }
+  return {
+    journalSection: normalized.heading,
+    usedLegacyHeading: true,
+    ignoredLegacyHeading: false,
+  };
+}

package/src/activity/journal-read.test.ts

@@ -0,0 +1,242 @@
+import assert from "node:assert/strict";
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import test from "node:test";
+
+import { readJournalForDate } from "./journal-read.js";
+import type { ActivityTimelineVaultConfig } from "./types.js";
+
+const START = "<!-- remnic:timeline:start -->";
+const END = "<!-- remnic:timeline:end -->";
+
+function makeVault(): string {
+  return mkdtempSync(path.join(tmpdir(), "remnic-journal-read-"));
+}
+
+function vaultConfig(vaultPath: string, overrides: Partial<ActivityTimelineVaultConfig> = {}): ActivityTimelineVaultConfig {
+  return {
+    enabled: true,
+    vaultPath,
+    dailyNotePath: "{yyyy}-{MM}-{dd}.md",
+    weeklyNotePath: "",
+    createMissingNotes: false,
+    noteTemplate: "",
+    sectionStrategy: "markers",
+    publish: {
+      timeline: { enabled: true, target: "daily", section: "Timeline" },
+      standup: { enabled: false, target: "daily", section: "Standup" },
+      weekly: { enabled: false, target: "weekly", section: "Weekly Review" },
+      locations: { enabled: false, target: "daily", section: "Locations" },
+    },
+    insertUnderHeading: "",
+    readback: { journalSection: "Journal" },
+    wikilinks: { places: false, placesFolder: "Places" },
+    properties: { mode: "off", prefix: "remnic_" },
+    autoPublish: true,
+    ...overrides,
+  };
+}
+
+test("resolves the note through the #1985 template resolver and returns provenance", () => {
+  const vault = makeVault();
+  try {
+    writeFileSync(
+      path.join(vault, "2026-08-20.md"),
+      ["## Journal", "user text", "## Other", ""].join("\n"),
+    );
+    const result = readJournalForDate({ vault: vaultConfig(vault), date: "2026-08-20" });
+    assert.equal(result.ok, true);
+    assert.ok(result.ok && result.exists);
+    assert.equal(result.filePath, path.join(vault, "2026-08-20.md"));
+    assert.equal(result.heading, "Journal");
+    assert.equal(result.text, "user text");
+  } finally {
+    rmSync(vault, { recursive: true, force: true });
+  }
+});
+
+test("every template layout the resolver supports works unchanged", () => {
+  const vault = makeVault();
+  try {
+    mkdirSync(path.join(vault, "days"), { recursive: true });
+    writeFileSync(path.join(vault, "days", "20.md"), ["## Journal", "nested", ""].join("\n"));
+    const result = readJournalForDate({
+      vault: vaultConfig(vault, { dailyNotePath: "days/{dd}.md" }),
+      date: "2026-08-20",
+    });
+    assert.ok(result.ok && result.exists);
+    assert.equal(result.text, "nested");
+  } finally {
+    rmSync(vault, { recursive: true, force: true });
+  }
+});
+
+test("missing note is a no-journal day, not an error, with the resolved path", () => {
+  const vault = makeVault();
+  try {
+    const result = readJournalForDate({ vault: vaultConfig(vault), date: "2026-08-20" });
+    assert.deepEqual(result, {
+      ok: true,
+      exists: false,
+      reason: "missing_file",
+      filePath: path.join(vault, "2026-08-20.md"),
+    });
+  } finally {
+    rmSync(vault, { recursive: true, force: true });
+  }
+});
+
+test("missing vault root is a no-journal day", () => {
+  const result = readJournalForDate({
+    vault: vaultConfig(path.join(tmpdir(), "remnic-journal-read-absent-root")),
+    date: "2026-08-20",
+  });
+  assert.ok(result.ok && !result.exists);
+});
+
+test("missing section is distinct from a missing note", () => {
+  const vault = makeVault();
+  try {
+    writeFileSync(path.join(vault, "2026-08-20.md"), "## Notes\nonly notes\n");
+    const result = readJournalForDate({ vault: vaultConfig(vault), date: "2026-08-20" });
+    assert.deepEqual(result, {
+      ok: true,
+      exists: false,
+      reason: "missing_heading",
+      filePath: path.join(vault, "2026-08-20.md"),
+    });
+  } finally {
+    rmSync(vault, { recursive: true, force: true });
+  }
+});
+
+test("duplicate headings refuse with line numbers", () => {
+  const vault = makeVault();
+  try {
+    writeFileSync(
+      path.join(vault, "2026-08-20.md"),
+      ["## Journal", "one", "## Notes", "## Journal", "two", ""].join("\n"),
+    );
+    const result = readJournalForDate({ vault: vaultConfig(vault), date: "2026-08-20" });
+    assert.ok(!result.ok);
+    assert.ok(result.reason === "duplicate_heading");
+    assert.deepEqual(result.lines, [1, 4]);
+  } finally {
+    rmSync(vault, { recursive: true, force: true });
+  }
+});
+
+test("non-ASCII heading resolves and extracts identically to ASCII", () => {
+  const vault = makeVault();
+  try {
+    writeFileSync(
+      path.join(vault, "2026-08-20.md"),
+      ["## Tagebuch 📝", "Heute ruhig.", "## Andere", ""].join("\n"),
+    );
+    const result = readJournalForDate({
+      vault: vaultConfig(vault, { readback: { journalSection: "Tagebuch 📝" } }),
+      date: "2026-08-20",
+    });
+    assert.ok(result.ok && result.exists);
+    assert.equal(result.heading, "Tagebuch 📝");
+    assert.equal(result.text, "Heute ruhig.");
+  } finally {
+    rmSync(vault, { recursive: true, force: true });
+  }
+});
+
+test("embedded remnic regions are stripped before the text is returned", () => {
+  const vault = makeVault();
+  try {
+    writeFileSync(
+      path.join(vault, "2026-08-20.md"),
+      ["## Journal", "I walked.", START, "card", END, "Still thinking.", ""].join("\n"),
+    );
+    const result = readJournalForDate({ vault: vaultConfig(vault), date: "2026-08-20" });
+    assert.ok(result.ok && result.exists);
+    assert.match(result.text, /I walked\./);
+    assert.match(result.text, /Still thinking\./);
+    assert.doesNotMatch(result.text, /card/);
+    assert.doesNotMatch(result.text, /remnic:/);
+  } finally {
+    rmSync(vault, { recursive: true, force: true });
+  }
+});
+
+test("heading-strategy publisher sections nested inside the journal section are stripped", () => {
+  const vault = makeVault();
+  try {
+    writeFileSync(
+      path.join(vault, "2026-08-20.md"),
+      ["## Journal", "mine", "### Timeline", "published", "## After", ""].join("\n"),
+    );
+    const markers = readJournalForDate({
+      vault: vaultConfig(vault),
+      date: "2026-08-20",
+    });
+    assert.ok(markers.ok && markers.exists);
+    assert.equal(markers.text, "mine\n### Timeline\npublished");
+
+    const heading = readJournalForDate({
+      vault: vaultConfig(vault, { sectionStrategy: "heading" }),
+      date: "2026-08-20",
+    });
+    assert.ok(heading.ok && heading.exists);
+    assert.equal(heading.text, "mine");
+  } finally {
+    rmSync(vault, { recursive: true, force: true });
+  }
+});
+
+test("reads exactly one file per call — the note is read once", () => {
+  const vault = makeVault();
+  try {
+    const notePath = path.join(vault, "2026-08-20.md");
+    writeFileSync(notePath, "## Journal\nuser\n");
+    const before = readFileSync(notePath, "utf8");
+    const result = readJournalForDate({ vault: vaultConfig(vault), date: "2026-08-20" });
+    assert.ok(result.ok && result.exists);
+    // Snapshot semantics: the note is untouched by reads.
+    assert.equal(readFileSync(notePath, "utf8"), before);
+  } finally {
+    rmSync(vault, { recursive: true, force: true });
+  }
+});
+
+test("refuses a daily note that is a symlink", () => {
+  const vault = makeVault();
+  const outside = makeVault();
+  try {
+    writeFileSync(path.join(outside, "escaped.md"), "## Journal\nstolen\n");
+    symlinkSync(path.join(outside, "escaped.md"), path.join(vault, "2026-08-20.md"));
+    const result = readJournalForDate({ vault: vaultConfig(vault), date: "2026-08-20" });
+    assert.equal(result.ok, false);
+    assert.equal(result.ok === false ? result.reason : "", "symlink_escape");
+  } finally {
+    rmSync(vault, { recursive: true, force: true });
+    rmSync(outside, { recursive: true, force: true });
+  }
+});
+
+test("refuses an ancestor directory swapped for a symlink out of the vault", () => {
+  const vault = makeVault();
+  const outside = makeVault();
+  try {
+    const nested = path.join(vault, "days");
+    mkdirSync(nested);
+    writeFileSync(path.join(nested, "2026-08-20.md"), "## Journal\nmine\n");
+    writeFileSync(path.join(outside, "2026-08-20.md"), "## Journal\nstolen\n");
+    rmSync(nested, { recursive: true, force: true });
+    symlinkSync(outside, nested);
+    const result = readJournalForDate({
+      vault: vaultConfig(vault, { dailyNotePath: "days/{yyyy}-{MM}-{dd}.md" }),
+      date: "2026-08-20",
+    });
+    assert.equal(result.ok, false);
+    assert.equal(result.ok === false ? result.reason : "", "symlink_escape");
+  } finally {
+    rmSync(vault, { recursive: true, force: true });
+    rmSync(outside, { recursive: true, force: true });
+  }
+});