@remnic/core 1.1.11 → 1.1.13

package/src/surfaces/heartbeat.ts

@@ -0,0 +1,325 @@
+import { createHash } from "node:crypto";
+import { statSync, watch, type FSWatcher } from "node:fs";
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+
+export interface HeartbeatEntry {
+  id: string;
+  slug: string;
+  title: string;
+  body: string;
+  schedule: string | null;
+  tags: string[];
+  sourceOffset: number;
+}
+
+export interface HeartbeatSurface {
+  read(path: string): Promise<HeartbeatEntry[]>;
+  watch(path: string, onChange: (entries: HeartbeatEntry[]) => void): () => void;
+  findBySlug(entries: HeartbeatEntry[], slug: string): HeartbeatEntry | null;
+}
+
+function stableHeartbeatId(params: {
+  sourceOffset: number;
+}): string {
+  const digest = createHash("sha1")
+    .update(
+      JSON.stringify({
+        sourceOffset: params.sourceOffset,
+      }),
+    )
+    .digest("hex")
+    .slice(0, 12);
+  return `heartbeat-${digest}`;
+}
+
+type ParsedHeartbeatEntry = Omit<HeartbeatEntry, "id">;
+
+function slugify(value: string): string {
+  const normalized = value
+    .trim()
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-+|-+$/g, "");
+  if (normalized.length > 0) return normalized;
+  const trimmed = value.trim();
+  if (trimmed.length === 0) return "heartbeat-untitled";
+  return `heartbeat-${createHash("sha1").update(trimmed).digest("hex").slice(0, 8)}`;
+}
+
+function parseTags(line: string): string[] {
+  const match = /^Tags:\s*(.*)$/i.exec(line.trim());
+  if (!match) return [];
+  return match[1]
+    .split(/\s+/)
+    .map((token) => token.trim())
+    .filter(Boolean)
+    .map((token) => token.replace(/^#/, ""));
+}
+
+function buildEntry(params: {
+  slug: string;
+  title: string;
+  body: string;
+  schedule: string | null;
+  tags: string[];
+  sourceOffset: number;
+}): ParsedHeartbeatEntry {
+  return {
+    slug: params.slug,
+    title: params.title.trim(),
+    body: params.body.trim(),
+    schedule: params.schedule?.trim() || null,
+    tags: params.tags,
+    sourceOffset: params.sourceOffset,
+  };
+}
+
+function finalizeHeartbeatEntries(entries: ParsedHeartbeatEntry[]): HeartbeatEntry[] {
+  return entries.map((entry) => {
+    return {
+      ...entry,
+      id: stableHeartbeatId({
+        sourceOffset: entry.sourceOffset,
+      }),
+    };
+  });
+}
+
+function parseSectionEntries(content: string): ParsedHeartbeatEntry[] {
+  const entries: ParsedHeartbeatEntry[] = [];
+  const regex = /^##\s+(.+)$/gm;
+  const matches = [...content.matchAll(regex)];
+  for (let index = 0; index < matches.length; index += 1) {
+    const match = matches[index];
+    const start = match.index ?? 0;
+    const end = index + 1 < matches.length ? (matches[index + 1]?.index ?? content.length) : content.length;
+    const rawTitle = match[1]?.trim() ?? "";
+    const chunk = content.slice(start + match[0].length, end).replace(/^\s+/, "");
+    const trimmed = chunk.replace(/\n---\s*$/, "").trim();
+    const lines = trimmed.split("\n");
+    const tags = lines.length > 0 ? parseTags(lines[lines.length - 1] ?? "") : [];
+    const withoutTags =
+      tags.length > 0
+        ? lines.slice(0, Math.max(0, lines.length - 1))
+        : lines;
+    let schedule: string | null = null;
+    const bodyLines: string[] = [];
+    for (const line of withoutTags) {
+      const scheduleMatch = /^Schedule:\s*(.+)$/i.exec(line.trim());
+      if (scheduleMatch) {
+        schedule = scheduleMatch[1]?.trim() || null;
+        continue;
+      }
+      bodyLines.push(line);
+    }
+    entries.push(
+      buildEntry({
+        slug: slugify(rawTitle),
+        title: rawTitle,
+        body: bodyLines.join("\n"),
+        schedule,
+        tags,
+        sourceOffset: start,
+      }),
+    );
+  }
+  return entries;
+}
+
+function parseTaskBlock(content: string): ParsedHeartbeatEntry[] {
+  const lines = content.split("\n");
+  const entries: ParsedHeartbeatEntry[] = [];
+  let inTasks = false;
+  let offset = 0;
+  for (let index = 0; index < lines.length; index += 1) {
+    const line = lines[index] ?? "";
+    const lineOffset = offset;
+    offset += line.length + 1;
+    const trimmed = line.trim();
+    if (trimmed === "tasks:") {
+      inTasks = true;
+      continue;
+    }
+    if (!inTasks) continue;
+    if (trimmed.startsWith("- name:")) {
+      const title = trimmed.replace(/^- name:\s*/, "").replace(/^["']|["']$/g, "").trim();
+      let schedule: string | null = null;
+      let body = "";
+      let cursor = index + 1;
+      for (; cursor < lines.length; cursor += 1) {
+        const next = lines[cursor] ?? "";
+        const nextTrimmed = next.trim();
+        if (nextTrimmed.startsWith("- name:")) break;
+        if (!next.startsWith(" ") && nextTrimmed) break;
+        if (nextTrimmed.startsWith("interval:")) {
+          schedule = nextTrimmed.replace(/^interval:\s*/, "").replace(/^["']|["']$/g, "").trim();
+          continue;
+        }
+        if (nextTrimmed.startsWith("prompt:")) {
+          const promptValue = nextTrimmed.replace(/^prompt:\s*/, "").trim();
+          if (promptValue === "|" || promptValue === ">") {
+            const scalarIndent = next.match(/^\s*/)?.[0].length ?? 0;
+            const collectedLines: string[] = [];
+            let scalarCursor = cursor + 1;
+            for (; scalarCursor < lines.length; scalarCursor += 1) {
+              const scalarLine = lines[scalarCursor] ?? "";
+              const scalarTrimmed = scalarLine.trim();
+              const scalarIndentation = scalarLine.match(/^\s*/)?.[0].length ?? 0;
+              if (scalarTrimmed.length > 0 && scalarIndentation <= scalarIndent) break;
+              if (scalarTrimmed.length === 0) {
+                collectedLines.push("");
+                continue;
+              }
+              collectedLines.push(scalarLine);
+            }
+            const meaningful = collectedLines.filter((line) => line.trim().length > 0);
+            const commonIndent =
+              meaningful.length > 0
+                ? Math.min(
+                    ...meaningful.map((line) => line.match(/^\s*/)?.[0].length ?? 0),
+                  )
+                : scalarIndent + 2;
+            const normalizedPromptLines = collectedLines.map((line) =>
+              line.trim().length > 0 ? line.slice(commonIndent) : "",
+            );
+            body =
+              promptValue === ">"
+                ? normalizedPromptLines
+                    .map((line) => line.trim())
+                    .filter((line) => line.length > 0)
+                    .join(" ")
+                : normalizedPromptLines.join("\n").trim();
+            cursor = Math.max(cursor, scalarCursor - 1);
+            continue;
+          }
+          body = promptValue.replace(/^["']|["']$/g, "").trim();
+        }
+      }
+      entries.push(
+        buildEntry({
+          slug: slugify(title),
+          title,
+          body,
+          schedule,
+          tags: [],
+          sourceOffset: lineOffset,
+        }),
+      );
+      index = Math.max(index, cursor - 1);
+    }
+  }
+  return entries;
+}
+
+function parseHeartbeatEntries(content: string): HeartbeatEntry[] {
+  const normalized = content.replace(/\r\n/g, "\n");
+  const sectionEntries = parseSectionEntries(normalized);
+  if (sectionEntries.length > 0) return finalizeHeartbeatEntries(sectionEntries);
+  return finalizeHeartbeatEntries(parseTaskBlock(normalized));
+}
+
+export function createHeartbeatSurface(): HeartbeatSurface {
+  return {
+    async read(filePath: string): Promise<HeartbeatEntry[]> {
+      try {
+        const content = await readFile(filePath, "utf8");
+        return parseHeartbeatEntries(content);
+      } catch (error) {
+        if ((error as NodeJS.ErrnoException).code === "ENOENT") {
+          return [];
+        }
+        throw error;
+      }
+    },
+
+    watch(filePath: string, onChange: (entries: HeartbeatEntry[]) => void): () => void {
+      let fileWatcher: FSWatcher | null = null;
+      let parentWatcher: FSWatcher | null = null;
+      let timer: NodeJS.Timeout | null = null;
+      const watchedName = path.basename(filePath);
+      const watchedDir = path.dirname(filePath);
+
+      const resolveParentWatchTarget = (): { dir: string; expectedName: string } | null => {
+        let candidateDir = watchedDir;
+        while (true) {
+          try {
+            if (statSync(candidateDir).isDirectory()) {
+              const relative = path.relative(candidateDir, watchedDir);
+              return {
+                dir: candidateDir,
+                expectedName:
+                  relative.length === 0
+                    ? watchedName
+                    : (relative.split(path.sep)[0] ?? watchedName),
+              };
+            }
+          } catch {}
+          const parentDir = path.dirname(candidateDir);
+          if (parentDir === candidateDir) {
+            return null;
+          }
+          candidateDir = parentDir;
+        }
+      };
+
+      const rearmFileWatcher = () => {
+        fileWatcher?.close();
+        fileWatcher = null;
+        try {
+          fileWatcher = watch(filePath, { persistent: false }, emit);
+        } catch {
+          fileWatcher = null;
+        }
+      };
+
+      const ensureParentWatcher = () => {
+        if (parentWatcher) return;
+        const target = resolveParentWatchTarget();
+        if (!target) return;
+        try {
+          parentWatcher = watch(
+            target.dir,
+            { persistent: false },
+            (_eventType, changed) => {
+              if (changed && String(changed) !== target.expectedName) return;
+              parentWatcher?.close();
+              parentWatcher = null;
+              ensureParentWatcher();
+              rearmFileWatcher();
+              if (target.expectedName === watchedName || fileWatcher) {
+                emit();
+              }
+            },
+          );
+        } catch {
+          parentWatcher = null;
+        }
+      };
+
+      const emit = () => {
+        if (timer) clearTimeout(timer);
+        timer = setTimeout(async () => {
+          timer = null;
+          try {
+            rearmFileWatcher();
+            onChange(await this.read(filePath));
+          } catch (error) {
+            console.warn("[remnic] heartbeat surface watch update failed", error);
+          }
+        }, 25);
+      };
+      rearmFileWatcher();
+      ensureParentWatcher();
+      return () => {
+        if (timer) clearTimeout(timer);
+        fileWatcher?.close();
+        parentWatcher?.close();
+      };
+    },
+
+    findBySlug(entries: HeartbeatEntry[], slug: string): HeartbeatEntry | null {
+      return entries.find((entry) => entry.slug === slug) ?? null;
+    },
+  };
+}

package/src/sync/index.ts

@@ -0,0 +1,308 @@
+/**
+ * @remnic/core — Diff-Aware Sync
+ *
+ * Watches source files for changes and triggers re-ingestion
+ * only for changed content. Uses file hashing to detect changes.
+ */
+
+import fs from "node:fs";
+import path from "node:path";
+import crypto from "node:crypto";
+
+// ── Types ────────────────────────────────────────────────────────────────────
+
+export interface SyncOptions {
+  /** Source directory to watch */
+  sourceDir: string;
+  /** Memory root directory */
+  memoryDir: string;
+  /** State file path (stores hashes). Default: memoryDir/.sync-state.json */
+  stateFile?: string;
+  /** File extensions to watch (default: .md, .txt, .mdx) */
+  extensions?: string[];
+  /** Directories to exclude */
+  excludeDirs?: string[];
+  /** Whether to actually write changes (default: true) */
+  dryRun?: boolean;
+}
+
+export interface SyncResult {
+  /** Files scanned */
+  scanned: number;
+  /** Files changed since last sync */
+  changed: FileChange[];
+  /** Files unchanged */
+  unchanged: number;
+  /** Files deleted since last sync */
+  deleted: string[];
+  /** Files newly added */
+  added: string[];
+  /** Duration in ms */
+  durationMs: number;
+  /** State file path */
+  stateFile: string;
+}
+
+export interface FileChange {
+  /** Absolute file path */
+  filePath: string;
+  /** Relative path from source root */
+  relativePath: string;
+  /** Change type */
+  type: "added" | "modified" | "deleted";
+  /** Current content hash */
+  currentHash: string;
+  /** Previous content hash (if modified) */
+  previousHash?: string;
+  /** File size in bytes */
+  size: number;
+}
+
+export interface SyncState {
+  /** Map of relative path → content hash */
+  fileHashes: Record<string, string>;
+  /** Last sync timestamp */
+  lastSyncAt: string;
+  /** Version of state format */
+  version: number;
+}
+
+// ── Constants ────────────────────────────────────────────────────────────────
+
+const DEFAULT_EXTENSIONS = new Set([".md", ".txt", ".mdx", ".rst"]);
+const DEFAULT_EXCLUDE = new Set([
+  "node_modules",
+  ".git",
+  "dist",
+  "build",
+  ".engram",
+  "coverage",
+]);
+
+// ── Main function ────────────────────────────────────────────────────────────
+
+export function syncChanges(options: SyncOptions): SyncResult {
+  const startTime = Date.now();
+  const {
+    sourceDir,
+    memoryDir,
+    extensions = [...DEFAULT_EXTENSIONS],
+    excludeDirs = [],
+    dryRun = false,
+  } = options;
+
+  const extSet = new Set(extensions);
+  const excludeSet = new Set([...DEFAULT_EXCLUDE, ...excludeDirs]);
+  const stateFilePath = options.stateFile ?? path.join(memoryDir, ".sync-state.json");
+
+  // Load previous state
+  const prevState = loadState(stateFilePath);
+
+  // Scan current files
+  const currentFiles = scanFiles(sourceDir, extSet, excludeSet);
+
+  // Compute diffs
+  const changes = computeDiff(currentFiles, prevState.fileHashes, sourceDir);
+
+  const added = changes.filter((c) => c.type === "added").map((c) => c.relativePath);
+  const modified = changes.filter((c) => c.type === "modified");
+  const deleted = changes
+    .filter((c) => c.type === "deleted")
+    .map((c) => c.relativePath);
+
+  // Update state (even in dry run, we want to show what would change)
+  if (!dryRun) {
+    const newState: SyncState = {
+      fileHashes: {},
+      lastSyncAt: new Date().toISOString(),
+      version: 1,
+    };
+
+    // Build new state from current files
+    for (const [relPath, hash] of Object.entries(currentFiles)) {
+      newState.fileHashes[relPath] = hash;
+    }
+
+    // Write state
+    fs.mkdirSync(path.dirname(stateFilePath), { recursive: true });
+    fs.writeFileSync(stateFilePath, JSON.stringify(newState, null, 2));
+  }
+
+  return {
+    scanned: Object.keys(currentFiles).length,
+    changed: changes,
+    unchanged:
+      Object.keys(currentFiles).length - changes.filter((c) => c.type !== "deleted").length,
+    deleted,
+    added,
+    durationMs: Date.now() - startTime,
+    stateFile: stateFilePath,
+  };
+}
+
+/**
+ * Watch for changes and call callback on file changes.
+ * Returns a stop function.
+ */
+export function watchForChanges(
+  options: SyncOptions,
+  onChange: (changes: FileChange[]) => void,
+): { stop: () => void } {
+  const { sourceDir, extensions, excludeDirs } = options;
+  const extSet = new Set(extensions ?? DEFAULT_EXTENSIONS);
+  const excludeSet = new Set([...DEFAULT_EXCLUDE, ...(excludeDirs ?? [])]);
+
+  let lastHashes: Record<string, string> = {};
+
+  // Initial scan
+  const currentFiles = scanFiles(sourceDir, extSet, excludeSet);
+  lastHashes = { ...currentFiles };
+
+  // Poll interval (FSWatcher doesn't reliably work for all platforms)
+  const interval = setInterval(() => {
+    const nowFiles = scanFiles(sourceDir, extSet, excludeSet);
+    const changes = computeDiff(nowFiles, lastHashes, sourceDir);
+
+    if (changes.length > 0) {
+      // Update hashes
+      for (const change of changes) {
+        if (change.type === "deleted") {
+          delete lastHashes[change.relativePath];
+        } else {
+          lastHashes[change.relativePath] = change.currentHash;
+        }
+      }
+      onChange(changes);
+    }
+  }, 5000);
+
+  return {
+    stop: () => clearInterval(interval),
+  };
+}
+
+// ── File scanning ────────────────────────────────────────────────────────────
+
+function scanFiles(
+  root: string,
+  extensions: Set<string>,
+  exclude: Set<string>,
+): Record<string, string> {
+  const result: Record<string, string> = {};
+
+  function walk(dir: string): void {
+    let entries: fs.Dirent[];
+    try {
+      entries = fs.readdirSync(dir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+
+    for (const entry of entries) {
+      if (exclude.has(entry.name)) continue;
+
+      const fullPath = path.join(dir, entry.name);
+      if (entry.isDirectory()) {
+        walk(fullPath);
+      } else if (entry.isFile()) {
+        const ext = path.extname(entry.name).toLowerCase();
+        if (!extensions.has(ext)) continue;
+
+        const relPath = path.relative(root, fullPath);
+        try {
+          const content = fs.readFileSync(fullPath, "utf8");
+          result[relPath] = hashContent(content);
+        } catch {
+          // Can't read — skip
+        }
+      }
+    }
+  }
+
+  walk(root);
+  return result;
+}
+
+// ── Diff computation ─────────────────────────────────────────────────────────
+
+function computeDiff(
+  current: Record<string, string>,
+  previous: Record<string, string>,
+  sourceDir: string,
+): FileChange[] {
+  const changes: FileChange[] = [];
+
+  // Find added and modified
+  for (const [relPath, hash] of Object.entries(current)) {
+    const fullPath = path.join(sourceDir, relPath);
+
+    if (!(relPath in previous)) {
+      // Added
+      let size = 0;
+      try {
+        size = fs.statSync(fullPath).size;
+      } catch {
+        // ignore
+      }
+      changes.push({
+        filePath: fullPath,
+        relativePath: relPath,
+        type: "added",
+        currentHash: hash,
+        size,
+      });
+    } else if (previous[relPath] !== hash) {
+      // Modified
+      let size = 0;
+      try {
+        size = fs.statSync(fullPath).size;
+      } catch {
+        // ignore
+      }
+      changes.push({
+        filePath: fullPath,
+        relativePath: relPath,
+        type: "modified",
+        currentHash: hash,
+        previousHash: previous[relPath],
+        size,
+      });
+    }
+  }
+
+  // Find deleted
+  for (const relPath of Object.keys(previous)) {
+    if (!(relPath in current)) {
+      changes.push({
+        filePath: path.join(sourceDir, relPath),
+        relativePath: relPath,
+        type: "deleted",
+        currentHash: "",
+        size: 0,
+      });
+    }
+  }
+
+  return changes;
+}
+
+// ── State management ─────────────────────────────────────────────────────────
+
+function loadState(stateFilePath: string): SyncState {
+  try {
+    const raw = fs.readFileSync(stateFilePath, "utf8");
+    return JSON.parse(raw);
+  } catch {
+    return {
+      fileHashes: {},
+      lastSyncAt: new Date(0).toISOString(),
+      version: 1,
+    };
+  }
+}
+
+// ── Helpers ───────────────────────────────────────────────────────────────────
+
+function hashContent(content: string): string {
+  return crypto.createHash("sha256").update(content).digest("hex").slice(0, 16);
+}