@remnic/core 1.1.11 → 1.1.13

package/src/binary-lifecycle/pipeline.ts

@@ -0,0 +1,352 @@
+/**
+ * Binary lifecycle pipeline — mirror, redirect, clean.
+ *
+ * Three-stage pipeline:
+ * 1. Mirror:   upload binary to backend, record in manifest
+ * 2. Redirect: scan markdown for inline refs, replace with redirect path
+ * 3. Clean:    after grace period, delete local copy
+ */
+
+import fsp from "node:fs/promises";
+import path from "node:path";
+import crypto from "node:crypto";
+import type {
+  BinaryAssetRecord,
+  BinaryLifecycleConfig,
+  PipelineResult,
+} from "./types.js";
+import type { BinaryStorageBackend } from "./backend.js";
+import { readManifest, writeManifest } from "./manifest.js";
+import { scanForBinaries } from "./scanner.js";
+
+/** Minimal logger interface so we don't depend on the full logger module. */
+interface PipelineLogger {
+  info(msg: string): void;
+  warn(msg: string): void;
+  error(msg: string): void;
+}
+
+interface PipelineOptions {
+  dryRun?: boolean;
+  /** Force-clean all files past grace period, ignoring redirect status. */
+  forceClean?: boolean;
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+async function hashFile(filePath: string): Promise<string> {
+  const content = await fsp.readFile(filePath);
+  return crypto.createHash("sha256").update(content).digest("hex");
+}
+
+function guessMimeType(ext: string): string {
+  const map: Record<string, string> = {
+    ".png": "image/png",
+    ".jpg": "image/jpeg",
+    ".jpeg": "image/jpeg",
+    ".gif": "image/gif",
+    ".pdf": "application/pdf",
+    ".mp3": "audio/mpeg",
+    ".mp4": "video/mp4",
+    ".wav": "audio/wav",
+  };
+  return map[ext.toLowerCase()] ?? "application/octet-stream";
+}
+
+/**
+ * Escape special regex characters in a string.
+ * CLAUDE.md #46: always escapeRegex on user-derived parts.
+ */
+function escapeRegex(s: string): string {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+// ---------------------------------------------------------------------------
+// Pipeline stages
+// ---------------------------------------------------------------------------
+
+async function stageMirror(
+  memoryDir: string,
+  newPaths: string[],
+  backend: BinaryStorageBackend,
+  assets: BinaryAssetRecord[],
+  log: PipelineLogger,
+  dryRun: boolean,
+): Promise<{ mirrored: number; errors: string[] }> {
+  let mirrored = 0;
+  const errors: string[] = [];
+
+  for (const relPath of newPaths) {
+    const fullPath = path.join(memoryDir, relPath);
+    try {
+      const stat = await fsp.stat(fullPath);
+      const contentHash = await hashFile(fullPath);
+      const ext = path.extname(relPath);
+      const mimeType = guessMimeType(ext);
+      const remotePath = relPath;
+
+      let backendLocation = remotePath;
+      if (!dryRun) {
+        backendLocation = await backend.upload(fullPath, remotePath);
+      }
+
+      const record: BinaryAssetRecord = {
+        originalPath: relPath,
+        mirroredPath: backendLocation,
+        contentHash,
+        sizeBytes: stat.size,
+        mimeType,
+        mirroredAt: new Date().toISOString(),
+        status: "mirrored",
+      };
+
+      assets.push(record);
+      mirrored++;
+      log.info(`[binary-lifecycle] mirrored: ${relPath} (${stat.size} bytes)${dryRun ? " [dry-run]" : ""}`);
+    } catch (err) {
+      const msg = `mirror failed for ${relPath}: ${err instanceof Error ? err.message : String(err)}`;
+      log.error(`[binary-lifecycle] ${msg}`);
+      errors.push(msg);
+    }
+  }
+
+  return { mirrored, errors };
+}
+
+async function stageRedirect(
+  memoryDir: string,
+  assets: BinaryAssetRecord[],
+  log: PipelineLogger,
+  dryRun: boolean,
+): Promise<{ redirected: number; errors: string[] }> {
+  let redirected = 0;
+  const errors: string[] = [];
+
+  // Only redirect assets that are mirrored but not yet redirected.
+  const candidates = assets.filter((a) => a.status === "mirrored");
+  if (candidates.length === 0) return { redirected, errors };
+
+  // Find all markdown files in memoryDir (recursive).
+  const mdFiles = await findMarkdownFiles(memoryDir);
+
+  for (const asset of candidates) {
+    let matchCount = 0;
+    let writeFailCount = 0;
+    for (const mdPath of mdFiles) {
+      try {
+        const content = await fsp.readFile(mdPath, "utf-8");
+
+        // Build the match path relative to this markdown file's directory.
+        // Markdown links like `![img](./image.png)` are file-relative, but
+        // asset.originalPath is memory-root relative (e.g. `sub/image.png`).
+        // Resolve the asset path relative to the markdown file's directory
+        // so both forms match correctly.
+        const mdDir = path.dirname(mdPath);
+        const assetAbsolute = path.join(memoryDir, asset.originalPath);
+        const relativeToMd = path.relative(mdDir, assetAbsolute);
+        // Normalise to forward slashes for regex matching (markdown uses /).
+        const relativeForward = relativeToMd.split(path.sep).join("/");
+        const escaped = escapeRegex(relativeForward);
+
+        // Build a regex that matches markdown image/link references to the file.
+        // Handles: ![alt](./path) , ![alt](path) , [text](./path)
+        const pattern = new RegExp(
+          `(!?\\[[^\\]]*\\]\\()(\\.\\/)?(${escaped})(\\))`,
+          "g",
+        );
+
+        if (!pattern.test(content)) continue;
+        matchCount++;
+
+        if (!dryRun) {
+          // Reset lastIndex after test().
+          pattern.lastIndex = 0;
+          const updated = content.replace(pattern, (_match, open, _dotSlash, _file, close) => {
+            return `${open as string}${asset.mirroredPath}${close as string}`;
+          });
+          await fsp.writeFile(mdPath, updated, "utf-8");
+        }
+      } catch (err) {
+        // Track write failures separately so we don't transition status
+        // when some markdown rewrites failed (P1: block redirect on failure).
+        writeFailCount++;
+        const msg = `redirect scan failed for ${mdPath}: ${err instanceof Error ? err.message : String(err)}`;
+        log.error(`[binary-lifecycle] ${msg}`);
+        errors.push(msg);
+      }
+    }
+
+    // Only transition to "redirected" when at least one reference was found
+    // AND all matched files were rewritten successfully.
+    if (matchCount > 0 && writeFailCount === 0) {
+      asset.status = "redirected";
+      asset.redirectedAt = new Date().toISOString();
+      redirected++;
+      log.info(`[binary-lifecycle] redirected: ${asset.originalPath}${dryRun ? " [dry-run]" : ""}`);
+    } else if (matchCount > 0 && writeFailCount > 0) {
+      // Some rewrites failed — set error status so the asset is not cleaned
+      // prematurely. It can be retried on the next pipeline run.
+      asset.status = "error";
+      log.warn(
+        `[binary-lifecycle] redirect partial failure for ${asset.originalPath}: ` +
+          `${matchCount} match(es), ${writeFailCount} write failure(s) — status set to error`,
+      );
+    }
+  }
+
+  return { redirected, errors };
+}
+
+async function stageClean(
+  memoryDir: string,
+  assets: BinaryAssetRecord[],
+  gracePeriodDays: number,
+  log: PipelineLogger,
+  dryRun: boolean,
+  forceClean: boolean,
+): Promise<{ cleaned: number; errors: string[] }> {
+  let cleaned = 0;
+  const errors: string[] = [];
+  const now = Date.now();
+  const graceMs = gracePeriodDays * 24 * 60 * 60 * 1000;
+
+  // Clean only assets that have been redirected (markdown refs already rewritten).
+  // Mirrored-only assets must NOT be cleaned — their markdown refs still point
+  // to the local file, so deletion would break links.
+  const candidates = assets.filter(
+    (a) => a.status === "redirected",
+  );
+
+  for (const asset of candidates) {
+    const mirroredMs = new Date(asset.mirroredAt).getTime();
+    const ageMs = now - mirroredMs;
+
+    if (!forceClean && ageMs < graceMs) {
+      // Not yet past grace period.
+      continue;
+    }
+
+    const fullPath = path.join(memoryDir, asset.originalPath);
+    try {
+      if (!dryRun) {
+        await fsp.unlink(fullPath);
+      }
+      asset.status = "cleaned";
+      asset.cleanedAt = new Date().toISOString();
+      cleaned++;
+      log.info(`[binary-lifecycle] cleaned: ${asset.originalPath}${dryRun ? " [dry-run]" : ""}`);
+    } catch (err) {
+      if ((err as NodeJS.ErrnoException).code === "ENOENT") {
+        // Already gone — mark as cleaned.
+        asset.status = "cleaned";
+        asset.cleanedAt = new Date().toISOString();
+        cleaned++;
+      } else {
+        const msg = `clean failed for ${asset.originalPath}: ${err instanceof Error ? err.message : String(err)}`;
+        log.error(`[binary-lifecycle] ${msg}`);
+        errors.push(msg);
+      }
+    }
+  }
+
+  return { cleaned, errors };
+}
+
+// ---------------------------------------------------------------------------
+// Markdown file discovery
+// ---------------------------------------------------------------------------
+
+async function findMarkdownFiles(dir: string): Promise<string[]> {
+  const results: string[] = [];
+
+  async function walk(current: string): Promise<void> {
+    let entries: import("node:fs").Dirent[];
+    try {
+      entries = await fsp.readdir(current, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    for (const entry of entries) {
+      const full = path.join(current, entry.name);
+      if (entry.isDirectory()) {
+        if (entry.name === ".binary-lifecycle") continue;
+        await walk(full);
+      } else if (entry.isFile() && entry.name.endsWith(".md")) {
+        results.push(full);
+      }
+    }
+  }
+
+  await walk(dir);
+  return results;
+}
+
+// ---------------------------------------------------------------------------
+// Main pipeline entry point
+// ---------------------------------------------------------------------------
+
+/**
+ * Run the binary lifecycle pipeline: scan, mirror, redirect, clean.
+ */
+export async function runBinaryLifecyclePipeline(
+  memoryDir: string,
+  config: BinaryLifecycleConfig,
+  backend: BinaryStorageBackend,
+  log: PipelineLogger,
+  opts?: PipelineOptions,
+): Promise<PipelineResult> {
+  const dryRun = opts?.dryRun ?? false;
+  const forceClean = opts?.forceClean ?? false;
+
+  const manifest = await readManifest(memoryDir);
+
+  // Stage 0: Scan
+  const newPaths = await scanForBinaries(memoryDir, config, manifest);
+  const scanned = newPaths.length;
+
+  // Stage 1: Mirror
+  const mirrorResult = await stageMirror(
+    memoryDir,
+    newPaths,
+    backend,
+    manifest.assets,
+    log,
+    dryRun,
+  );
+
+  // Stage 2: Redirect
+  const redirectResult = await stageRedirect(memoryDir, manifest.assets, log, dryRun);
+
+  // Stage 3: Clean
+  const cleanResult = await stageClean(
+    memoryDir,
+    manifest.assets,
+    config.gracePeriodDays,
+    log,
+    dryRun,
+    forceClean,
+  );
+
+  // Persist manifest (unless dry-run).
+  manifest.lastScanAt = new Date().toISOString();
+  if (!dryRun) {
+    await writeManifest(memoryDir, manifest);
+  }
+
+  const allErrors = [
+    ...mirrorResult.errors,
+    ...redirectResult.errors,
+    ...cleanResult.errors,
+  ];
+
+  return {
+    scanned,
+    mirrored: mirrorResult.mirrored,
+    redirected: redirectResult.redirected,
+    cleaned: cleanResult.cleaned,
+    errors: allErrors,
+    dryRun,
+  };
+}

package/src/binary-lifecycle/scanner.ts

@@ -0,0 +1,89 @@
+/**
+ * Binary file scanner.
+ *
+ * Recursively walks the memory directory, matches files against configured
+ * glob patterns, skips files already tracked in the manifest, and respects
+ * the max-size limit.
+ */
+
+import fsp from "node:fs/promises";
+import path from "node:path";
+import type { BinaryLifecycleConfig, BinaryLifecycleManifest } from "./types.js";
+
+/**
+ * Test whether a filename matches any of the provided glob patterns.
+ * Supports simple `*.ext` patterns (the default scan patterns).
+ * For more complex globs a proper library should be used; this covers
+ * the 95% case without adding a dependency.
+ */
+export function matchesPatterns(filename: string, patterns: string[]): boolean {
+  const lower = filename.toLowerCase();
+  for (const pattern of patterns) {
+    // Simple *.ext matching
+    if (pattern.startsWith("*.")) {
+      const ext = pattern.slice(1).toLowerCase(); // e.g. ".png"
+      if (lower.endsWith(ext)) return true;
+    } else if (lower === pattern.toLowerCase()) {
+      return true;
+    }
+  }
+  return false;
+}
+
+/**
+ * Scan memoryDir recursively for binary files matching the configured patterns.
+ * Returns relative paths (relative to memoryDir) for files not yet tracked.
+ */
+export async function scanForBinaries(
+  memoryDir: string,
+  config: BinaryLifecycleConfig,
+  manifest: BinaryLifecycleManifest,
+): Promise<string[]> {
+  const tracked = new Set(manifest.assets.map((a) => a.originalPath));
+  const results: string[] = [];
+
+  async function walk(dir: string): Promise<void> {
+    let entries: import("node:fs").Dirent[];
+    try {
+      entries = await fsp.readdir(dir, { withFileTypes: true });
+    } catch {
+      return; // skip unreadable directories
+    }
+
+    for (const entry of entries) {
+      const fullPath = path.join(dir, entry.name);
+      // Normalize to POSIX separators so redirect matching works on Windows
+      // (markdown links always use forward slashes).
+      const relativePath = path.relative(memoryDir, fullPath).split(path.sep).join("/");
+
+      if (entry.isDirectory()) {
+        // Skip the manifest directory itself and hidden dirs starting with .
+        if (entry.name === ".binary-lifecycle") continue;
+        await walk(fullPath);
+        continue;
+      }
+
+      if (!entry.isFile()) continue;
+
+      // Check pattern match
+      if (!matchesPatterns(entry.name, config.scanPatterns)) continue;
+
+      // Skip already tracked
+      if (tracked.has(relativePath)) continue;
+
+      // Check file size
+      try {
+        const stat = await fsp.stat(fullPath);
+        if (stat.size > config.maxBinarySizeBytes) continue;
+        if (stat.size === 0) continue; // skip empty files
+      } catch {
+        continue; // stat failure, skip
+      }
+
+      results.push(relativePath);
+    }
+  }
+
+  await walk(memoryDir);
+  return results;
+}

package/src/binary-lifecycle/types.ts

@@ -0,0 +1,89 @@
+/**
+ * Binary file lifecycle management types.
+ *
+ * Defines the configuration, manifest, and record structures for the
+ * three-stage binary lifecycle pipeline: mirror, redirect, clean.
+ */
+
+export interface BinaryLifecycleConfig {
+  /** Master toggle. Default: false. */
+  enabled: boolean;
+  /** Days after mirror before local copy is eligible for cleanup. Default: 7. */
+  gracePeriodDays: number;
+  /** Files larger than this are skipped during scan. Default: 50 MB. */
+  maxBinarySizeBytes: number;
+  /** Glob patterns for binary file types to manage. */
+  scanPatterns: string[];
+  /** Backend configuration for binary storage. */
+  backend: BinaryStorageBackendConfig;
+}
+
+export interface BinaryStorageBackendConfig {
+  /** Backend type. "filesystem" copies to a local directory. "none" is a no-op (dry-run/testing). */
+  type: "filesystem" | "s3" | "none";
+  /** Destination directory for the filesystem backend. */
+  basePath?: string;
+  /** S3 bucket name (future). */
+  s3Bucket?: string;
+  /** S3 region (future). */
+  s3Region?: string;
+  /** S3 key prefix (future). */
+  s3Prefix?: string;
+}
+
+export type BinaryAssetStatus =
+  | "pending"
+  | "mirrored"
+  | "redirected"
+  | "cleaned"
+  | "error";
+
+export interface BinaryAssetRecord {
+  /** Relative path from memoryDir to the original file. */
+  originalPath: string;
+  /** Path (or URL) in the backend storage. */
+  mirroredPath: string;
+  /** SHA-256 hex digest of file content. */
+  contentHash: string;
+  /** File size in bytes. */
+  sizeBytes: number;
+  /** MIME type (e.g. "image/png"). */
+  mimeType: string;
+  /** ISO 8601 timestamp when the file was mirrored. */
+  mirroredAt: string;
+  /** ISO 8601 timestamp when markdown references were rewritten. */
+  redirectedAt?: string;
+  /** ISO 8601 timestamp when the local copy was deleted. */
+  cleanedAt?: string;
+  /** Current lifecycle status. */
+  status: BinaryAssetStatus;
+}
+
+export interface BinaryLifecycleManifest {
+  version: 1;
+  assets: BinaryAssetRecord[];
+  lastScanAt?: string;
+}
+
+export interface PipelineResult {
+  scanned: number;
+  mirrored: number;
+  redirected: number;
+  cleaned: number;
+  errors: string[];
+  dryRun: boolean;
+}
+
+export const DEFAULT_SCAN_PATTERNS = [
+  "*.png",
+  "*.jpg",
+  "*.jpeg",
+  "*.gif",
+  "*.pdf",
+  "*.mp3",
+  "*.mp4",
+  "*.wav",
+];
+
+export const DEFAULT_MAX_BINARY_SIZE_BYTES = 50 * 1024 * 1024; // 50 MB
+export const DEFAULT_GRACE_PERIOD_DAYS = 7;

package/src/bootstrap.ts

@@ -0,0 +1,178 @@
+import path from "node:path";
+import { readdir, readFile } from "node:fs/promises";
+import { scanSignals } from "./signal.js";
+import { log } from "./logger.js";
+import type { PluginConfig } from "./types.js";
+import type { Orchestrator } from "./orchestrator.js";
+
+export type BootstrapResult = {
+  sessionsScanned: number;
+  turnsProcessed: number;
+  highSignalTurns: number;
+  memoriesCreated: number;
+  skipped: number;
+};
+
+export type BootstrapOptions = {
+  dryRun?: boolean;
+  sessionsDir?: string;
+  limit?: number;
+  since?: Date;
+};
+
+type BootstrapEntry = {
+  role: string;
+  content: string;
+  timestamp: string;
+  sessionKey?: string;
+};
+
+export class BootstrapEngine {
+  constructor(private config: PluginConfig, private orchestrator: Orchestrator) {}
+
+  async run(options: BootstrapOptions): Promise<BootstrapResult> {
+    const dryRun = options.dryRun === true;
+    const since = options.since ?? new Date(0);
+    const end = new Date();
+    const limit = Math.max(0, options.limit ?? Number.POSITIVE_INFINITY);
+
+    const sessions = await this.resolveSessions(options.sessionsDir, since, end);
+    const selected = sessions.slice(0, Number.isFinite(limit) ? limit : sessions.length);
+
+    const beforeCount = dryRun ? 0 : (await this.orchestrator.storage.readAllMemories()).length;
+    let turnsProcessed = 0;
+    let highSignalTurns = 0;
+    let skipped = 0;
+
+    for (const session of selected) {
+      for (const entry of session.entries) {
+        if (entry.role !== "user") continue;
+        turnsProcessed += 1;
+
+        const content = typeof entry.content === "string" ? entry.content.trim() : "";
+        if (!content) {
+          skipped += 1;
+          continue;
+        }
+
+        const signal = scanSignals(content, this.config.highSignalPatterns);
+        if (signal.level !== "high") {
+          skipped += 1;
+          continue;
+        }
+
+        highSignalTurns += 1;
+        if (!dryRun) {
+          await this.orchestrator.processTurn("user", content, session.sessionKey);
+        }
+      }
+    }
+
+    let memoriesCreated = 0;
+    if (!dryRun && highSignalTurns > 0) {
+      await this.orchestrator.waitForExtractionIdle();
+      const afterCount = (await this.orchestrator.storage.readAllMemories()).length;
+      memoriesCreated = Math.max(0, afterCount - beforeCount);
+    }
+
+    const result: BootstrapResult = {
+      sessionsScanned: selected.length,
+      turnsProcessed,
+      highSignalTurns,
+      memoriesCreated,
+      skipped,
+    };
+    log.info(
+      `bootstrap complete: sessions=${result.sessionsScanned}, turns=${result.turnsProcessed}, high=${result.highSignalTurns}, created=${result.memoriesCreated}, skipped=${result.skipped}, dryRun=${dryRun}`,
+    );
+    return result;
+  }
+
+  private async resolveSessions(
+    sessionsDir: string | undefined,
+    since: Date,
+    until: Date,
+  ): Promise<Array<{ sessionKey: string; entries: BootstrapEntry[] }>> {
+    if (sessionsDir && sessionsDir.trim().length > 0) {
+      return this.readSessionsFromDir(sessionsDir, since, until);
+    }
+
+    const keys = await this.orchestrator.transcript.listSessionKeys();
+    const sessions: Array<{ sessionKey: string; entries: BootstrapEntry[] }> = [];
+    for (const key of keys) {
+      const entries = await this.orchestrator.transcript.readRange(
+        since.toISOString(),
+        until.toISOString(),
+        key,
+      );
+      if (entries.length > 0) {
+        sessions.push({ sessionKey: key, entries });
+      }
+    }
+    return sessions;
+  }
+
+  private async readSessionsFromDir(
+    baseDir: string,
+    since: Date,
+    until: Date,
+  ): Promise<Array<{ sessionKey: string; entries: BootstrapEntry[] }>> {
+    const files = await this.listJsonlFiles(baseDir);
+    const bySession = new Map<string, BootstrapEntry[]>();
+
+    for (const filePath of files) {
+      let raw = "";
+      try {
+        raw = await readFile(filePath, "utf-8");
+      } catch {
+        continue;
+      }
+      const lines = raw.split("\n").filter((line) => line.trim().length > 0);
+      for (const line of lines) {
+        let parsed: any;
+        try {
+          parsed = JSON.parse(line);
+        } catch {
+          continue;
+        }
+        const ts = new Date(String(parsed?.timestamp ?? ""));
+        if (Number.isNaN(ts.getTime())) continue;
+        if (ts < since || ts > until) continue;
+        const role = String(parsed?.role ?? "");
+        const content = typeof parsed?.content === "string" ? parsed.content : "";
+        if (!role || !content) continue;
+        const sessionKey = typeof parsed?.sessionKey === "string" && parsed.sessionKey.length > 0
+          ? parsed.sessionKey
+          : path.relative(baseDir, filePath);
+        const list = bySession.get(sessionKey) ?? [];
+        list.push({
+          role,
+          content,
+          timestamp: ts.toISOString(),
+          sessionKey,
+        });
+        bySession.set(sessionKey, list);
+      }
+    }
+
+    return Array.from(bySession.entries()).map(([sessionKey, entries]) => ({
+      sessionKey,
+      entries: entries.sort((a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()),
+    }));
+  }
+
+  private async listJsonlFiles(dir: string): Promise<string[]> {
+    const out: string[] = [];
+    const entries = await readdir(dir, { withFileTypes: true }).catch(() => []);
+    for (const entry of entries) {
+      const full = path.join(dir, entry.name);
+      if (entry.isDirectory()) {
+        out.push(...(await this.listJsonlFiles(full)));
+      } else if (entry.isFile() && full.endsWith(".jsonl")) {
+        out.push(full);
+      }
+    }
+    return out;
+  }
+}
+