@chendpoc/pi-memory 0.1.11 → 0.1.13

package/src/consolidation/memoryIndex.ts

@@ -0,0 +1,127 @@
+import fs from "node:fs";
+import fsPromises from "node:fs/promises";
+
+import {
+  lineMatchesScope,
+  type MemoryScope,
+} from "./scope.js";
+
+export interface MemoryIndexCapOptions {
+  maxLines?: number;
+  maxBytes?: number;
+  scopes?: readonly MemoryScope[];
+}
+
+export interface MemoryIndexStats {
+  path: string;
+  exists: boolean;
+  lines: number;
+  bytes: number;
+  cappedLines: number;
+  cappedBytes: number;
+}
+
+const DEFAULT_MAX_LINES = 200;
+const DEFAULT_MAX_BYTES = 25_600;
+
+function capLines(
+  text: string,
+  opts: Required<Pick<MemoryIndexCapOptions, "maxLines" | "maxBytes">> & {
+    scopes?: readonly MemoryScope[];
+  },
+): string {
+  const out: string[] = [];
+  let bytes = 0;
+  for (const line of text.split("\n")) {
+    if (out.length >= opts.maxLines) break;
+    if (!lineMatchesScope(line, opts.scopes)) continue;
+    const nextBytes = Buffer.byteLength(line + "\n", "utf8");
+    if (bytes + nextBytes > opts.maxBytes) break;
+    out.push(line);
+    bytes += nextBytes;
+  }
+  return out.join("\n").trim();
+}
+
+function normalizedOptions(opts: MemoryIndexCapOptions = {}) {
+  return {
+    maxLines: opts.maxLines ?? DEFAULT_MAX_LINES,
+    maxBytes: opts.maxBytes ?? DEFAULT_MAX_BYTES,
+    scopes: opts.scopes,
+  };
+}
+
+export function readMemoryIndexCap(
+  paths: string[],
+  opts: MemoryIndexCapOptions = {},
+): string {
+  const normalized = normalizedOptions(opts);
+  for (const p of paths) {
+    let text: string;
+    try {
+      text = fs.readFileSync(p, "utf8");
+    } catch {
+      continue;
+    }
+    const capped = capLines(text, normalized);
+    if (capped) return capped;
+  }
+  return "";
+}
+
+export async function memoryIndexSnippet(
+  paths: string[],
+  query: string,
+  opts: MemoryIndexCapOptions = {},
+): Promise<string> {
+  const q = query.trim().toLowerCase();
+  if (!q) return "";
+  const normalized = normalizedOptions(opts);
+
+  for (const p of paths) {
+    let text: string;
+    try {
+      text = await fsPromises.readFile(p, "utf8");
+    } catch {
+      continue;
+    }
+    const matches: string[] = [];
+    let bytes = 0;
+    for (const line of text.split("\n")) {
+      if (!lineMatchesScope(line, normalized.scopes)) continue;
+      if (!line.toLowerCase().includes(q)) continue;
+      const nextBytes = Buffer.byteLength(line + "\n", "utf8");
+      if (bytes + nextBytes > normalized.maxBytes) break;
+      matches.push(line);
+      bytes += nextBytes;
+      if (matches.length >= normalized.maxLines) break;
+    }
+    if (matches.length > 0) return matches.join("\n");
+  }
+  return "";
+}
+
+export function getMemoryIndexStats(
+  paths: string[],
+  opts: MemoryIndexCapOptions = {},
+): MemoryIndexStats[] {
+  const normalized = normalizedOptions(opts);
+  return paths.map((p) => {
+    let text = "";
+    let exists = true;
+    try {
+      text = fs.readFileSync(p, "utf8");
+    } catch {
+      exists = false;
+    }
+    const capped = exists ? capLines(text, normalized) : "";
+    return {
+      path: p,
+      exists,
+      lines: exists ? text.split("\n").length : 0,
+      bytes: exists ? Buffer.byteLength(text, "utf8") : 0,
+      cappedLines: capped ? capped.split("\n").length : 0,
+      cappedBytes: Buffer.byteLength(capped, "utf8"),
+    };
+  });
+}

package/src/consolidation/phase2/runPhase2.ts

@@ -0,0 +1,255 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+
+import type { ConsolidationStore } from "../stage1/store.js";
+import type { Stage1Output } from "../types.js";
+
+export interface Phase2Options {
+  memoryMdPath: string;
+  bundleRoot: string;
+  topN: number;
+  dryRun?: boolean;
+  maxLines?: number;
+  maxBytes?: number;
+  maxUnusedDays?: number;
+}
+
+export interface Phase2Report {
+  stage1Selected: number;
+  appended: number;
+  skippedDedup: number;
+  backupsCreated: number;
+  topicsUpdated: number;
+  migratedProjectLines: number;
+  prunedLines: number;
+  dryRun: boolean;
+  targets: string[];
+}
+
+async function readFileOptional(filePath: string): Promise<string> {
+  try {
+    return await fs.readFile(filePath, "utf8");
+  } catch {
+    return "";
+  }
+}
+
+function stripScopeComment(line: string): string {
+  return line.replace(/<!--\s*scope:[^>]+-->/g, "").trim();
+}
+
+function normalizedLine(line: string): string {
+  return stripScopeComment(line).replace(/\s+/g, " ").toLowerCase();
+}
+
+function targetForRow(row: Stage1Output, opts: Phase2Options): string {
+  if (row.scope.startsWith("project:")) {
+    const hash = row.scope.slice("project:".length);
+    return path.join(opts.bundleRoot, "projects", hash, "MEMORY.md");
+  }
+  return opts.memoryMdPath;
+}
+
+function targetForScope(scope: string, opts: Phase2Options): string {
+  if (scope.startsWith("project:")) {
+    const hash = scope.slice("project:".length);
+    return path.join(opts.bundleRoot, "projects", hash, "MEMORY.md");
+  }
+  return opts.memoryMdPath;
+}
+
+function rowsToMemoryLines(row: Stage1Output): string[] {
+  const body = row.raw_memory.trim() || row.rollout_summary.trim();
+  if (!body) return [];
+  return body
+    .split("\n")
+    .map((line) => line.trim())
+    .filter(Boolean)
+    .map((line) => {
+      const bullet = line.startsWith("- ") ? line : `- ${line}`;
+      return `${bullet} <!-- scope:${row.scope} -->`;
+    });
+}
+
+async function backupMemoryFile(filePath: string): Promise<boolean> {
+  const existing = await readFileOptional(filePath);
+  if (!existing) return false;
+  const stamp = new Date().toISOString().replace(/[:.]/g, "-");
+  await fs.copyFile(filePath, `${filePath}.${stamp}.bak`);
+  return true;
+}
+
+async function appendDedup(
+  filePath: string,
+  lines: string[],
+  dryRun: boolean,
+): Promise<{ appended: number; skipped: number; backup: boolean }> {
+  if (lines.length === 0) return { appended: 0, skipped: 0, backup: false };
+  const existing = await readFileOptional(filePath);
+  const seen = new Set(
+    existing
+      .split("\n")
+      .map(normalizedLine)
+      .filter(Boolean),
+  );
+  const nextLines: string[] = [];
+  let skipped = 0;
+  for (const line of lines) {
+    const key = normalizedLine(line);
+    if (seen.has(key)) {
+      skipped++;
+      continue;
+    }
+    seen.add(key);
+    nextLines.push(line);
+  }
+  if (dryRun || nextLines.length === 0) {
+    return { appended: nextLines.length, skipped, backup: false };
+  }
+  await fs.mkdir(path.dirname(filePath), { recursive: true, mode: 0o700 });
+  const backup = await backupMemoryFile(filePath);
+  const prefix = existing.trim() ? "\n" : "# Memory\n\n";
+  await fs.appendFile(filePath, `${prefix}${nextLines.join("\n")}\n`, {
+    encoding: "utf8",
+    mode: 0o600,
+  });
+  return { appended: nextLines.length, skipped, backup };
+}
+
+async function migrateProjectScopedLines(
+  opts: Phase2Options,
+): Promise<{ migrated: number; targets: string[] }> {
+  const globalText = await readFileOptional(opts.memoryMdPath);
+  if (!globalText.trim()) return { migrated: 0, targets: [] };
+
+  const keep: string[] = [];
+  const byTarget = new Map<string, string[]>();
+  for (const line of globalText.split("\n")) {
+    const scope = line.match(/<!--\s*scope:(project:[a-f0-9]{12})\s*-->/i)?.[1];
+    if (!scope) {
+      keep.push(line);
+      continue;
+    }
+    const target = targetForScope(scope, opts);
+    byTarget.set(target, [...(byTarget.get(target) ?? []), line]);
+  }
+  const migrated = Array.from(byTarget.values()).reduce((sum, lines) => sum + lines.length, 0);
+  if (opts.dryRun || migrated === 0) {
+    return { migrated, targets: Array.from(byTarget.keys()) };
+  }
+
+  await backupMemoryFile(opts.memoryMdPath);
+  await fs.writeFile(opts.memoryMdPath, `${keep.join("\n").trim()}\n`, "utf8");
+  for (const [target, lines] of byTarget) {
+    await appendDedup(target, lines, false);
+  }
+  return { migrated, targets: Array.from(byTarget.keys()) };
+}
+
+async function pruneLinesFromTargets(
+  rows: Stage1Output[],
+  opts: Phase2Options,
+): Promise<number> {
+  if (rows.length === 0) return 0;
+  const staleKeys = new Set<string>();
+  for (const row of rows) {
+    for (const line of rowsToMemoryLines(row)) staleKeys.add(normalizedLine(line));
+  }
+  if (staleKeys.size === 0) return 0;
+
+  const targets = new Set<string>([
+    opts.memoryMdPath,
+    ...rows.map((row) => targetForRow(row, opts)),
+  ]);
+  let pruned = 0;
+  for (const target of targets) {
+    const existing = await readFileOptional(target);
+    if (!existing.trim()) continue;
+    const kept: string[] = [];
+    let targetPruned = 0;
+    for (const line of existing.split("\n")) {
+      if (staleKeys.has(normalizedLine(line))) {
+        pruned++;
+        targetPruned++;
+        continue;
+      }
+      kept.push(line);
+    }
+    if (!opts.dryRun && targetPruned > 0) {
+      await backupMemoryFile(target);
+      await fs.writeFile(target, `${kept.join("\n").trim()}\n`, "utf8");
+    }
+  }
+  return pruned;
+}
+
+async function syncWorkspace(
+  rows: Stage1Output[],
+  bundleRoot: string,
+  dryRun: boolean,
+): Promise<void> {
+  if (dryRun) return;
+  const workspace = path.join(bundleRoot, "workspace");
+  const summaries = path.join(workspace, "rollout_summaries");
+  await fs.mkdir(summaries, { recursive: true, mode: 0o700 });
+  const raw = rows
+    .map((row) => `## ${row.session_id}\n\n${row.raw_memory || row.rollout_summary}`)
+    .join("\n\n");
+  await fs.writeFile(path.join(workspace, "raw_memories.md"), `${raw}\n`, "utf8");
+  for (const row of rows) {
+    await fs.writeFile(
+      path.join(summaries, `${row.session_id}.md`),
+      `${row.rollout_summary || row.raw_memory}\n`,
+      "utf8",
+    );
+  }
+}
+
+export async function runPhase2(
+  store: ConsolidationStore,
+  opts: Phase2Options,
+): Promise<Phase2Report> {
+  const dryRun = opts.dryRun ?? false;
+  const rows = store.listUnselectedStage1(opts.topN);
+  const migration = await migrateProjectScopedLines(opts);
+  const grouped = new Map<string, string[]>();
+
+  for (const row of rows) {
+    const target = targetForRow(row, opts);
+    const lines = rowsToMemoryLines(row);
+    grouped.set(target, [...(grouped.get(target) ?? []), ...lines]);
+  }
+
+  const report: Phase2Report = {
+    stage1Selected: rows.length,
+    appended: 0,
+    skippedDedup: 0,
+    backupsCreated: 0,
+    topicsUpdated: 0,
+    migratedProjectLines: migration.migrated,
+    prunedLines: 0,
+    dryRun,
+    targets: Array.from(new Set([...grouped.keys(), ...migration.targets])),
+  };
+
+  for (const [target, lines] of grouped) {
+    const result = await appendDedup(target, lines, dryRun);
+    report.appended += result.appended;
+    report.skippedDedup += result.skipped;
+    if (result.backup) report.backupsCreated++;
+  }
+
+  await syncWorkspace(rows, opts.bundleRoot, dryRun);
+  if (opts.maxUnusedDays && opts.maxUnusedDays > 0) {
+    const cutoff = new Date(
+      Date.now() - opts.maxUnusedDays * 24 * 60 * 60 * 1000,
+    ).toISOString();
+    report.prunedLines = await pruneLinesFromTargets(
+      store.listStage1OlderThan(cutoff),
+      opts,
+    );
+  }
+  if (!dryRun) store.markStage1Selected(rows.map((row) => row.session_id));
+
+  return report;
+}

package/src/consolidation/scheduler/launchd.ts

@@ -0,0 +1,62 @@
+import os from "node:os";
+import path from "node:path";
+
+import type { ScheduleOptions } from "./types.js";
+
+export const LAUNCHD_LABEL = "dev.pi.memory-consolidate";
+const PLIST_FILE = `${LAUNCHD_LABEL}.plist`;
+
+function commandArgs(opts: ScheduleOptions): string[] {
+  if (opts.commandPath) {
+    return [opts.commandPath, "consolidate"];
+  }
+  if (opts.npxPath) {
+    return [opts.npxPath, "pi-memory", "consolidate"];
+  }
+  return ["pi-memory", "consolidate"];
+}
+
+function escapeXml(value: string): string {
+  return value
+    .replaceAll("&", "&")
+    .replaceAll("<", "&lt;")
+    .replaceAll(">", "&gt;");
+}
+
+export function launchdPlistPath(): string {
+  return path.join(os.homedir(), "Library", "LaunchAgents", PLIST_FILE);
+}
+
+export function buildLaunchdPlist(opts: ScheduleOptions): string {
+  const args = commandArgs(opts);
+
+  const argumentsXml = args
+    .map((arg) => `    <string>${escapeXml(arg)}</string>`)
+    .join("\n");
+
+  return [
+    `<?xml version="1.0" encoding="UTF-8"?>`,
+    `<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">`,
+    `<plist version="1.0">`,
+    `<dict>`,
+    `  <key>Label</key>`,
+    `  <string>${LAUNCHD_LABEL}</string>`,
+    `  <key>ProgramArguments</key>`,
+    `  <array>`,
+    `${argumentsXml}`,
+    `  </array>`,
+    `  <key>StandardOutPath</key>`,
+    `  <string>${escapeXml(opts.logPath)}</string>`,
+    `  <key>StandardErrorPath</key>`,
+    `  <string>${escapeXml(opts.logPath)}</string>`,
+    `  <key>StartCalendarInterval</key>`,
+    `  <dict>`,
+    `    <key>Hour</key>`,
+    `    <integer>${opts.hour}</integer>`,
+    `    <key>Minute</key>`,
+    `    <integer>${opts.minute}</integer>`,
+    `  </dict>`,
+    `</dict>`,
+    `</plist>`,
+  ].join("\n");
+}

package/src/consolidation/scheduler/runConsolidate.ts

@@ -0,0 +1,115 @@
+import path from "node:path";
+
+import type { MemoryConfig } from "../../config.js";
+import { trainBundle } from "../../trainer/index.js";
+import { withFileLock } from "../lock.js";
+import { appendConsolidationLog } from "../log.js";
+import { runPhase2, type Phase2Report } from "../phase2/runPhase2.js";
+import { drainQueue, type DrainQueueReport } from "../stage1/drainQueue.js";
+import { openConsolidationStore } from "../stage1/store.js";
+
+export interface ConsolidateOptions {
+  config: MemoryConfig;
+  dbPath?: string;
+  dryRun?: boolean;
+  phase1Only?: boolean;
+  phase2Only?: boolean;
+}
+
+export interface ConsolidateReport {
+  dryRun: boolean;
+  dbPath: string;
+  drain?: DrainQueueReport;
+  train?: {
+    sessionsProcessed: number;
+    entityCount: number;
+    relationCount: number;
+    eventCount: number;
+    dryRun: boolean;
+  };
+  phase2?: Phase2Report;
+}
+
+export function defaultConsolidationDbPath(cfg: MemoryConfig): string {
+  return path.join(cfg.bundleRoot, "memories.sqlite");
+}
+
+export async function runConsolidate(
+  opts: ConsolidateOptions,
+): Promise<ConsolidateReport> {
+  const cfg = opts.config;
+  const dbPath = opts.dbPath ?? defaultConsolidationDbPath(cfg);
+  const dryRun = opts.dryRun ?? false;
+  const lockPath = path.join(cfg.bundleRoot, ".consolidation.lock");
+  const logPath = cfg.consolidation.schedule.log_path;
+
+  return withFileLock(lockPath, async () => {
+    const store = openConsolidationStore(dbPath);
+    if (!store) {
+      throw new Error("better-sqlite3 unavailable; cannot open consolidation store");
+    }
+
+    try {
+      const report: ConsolidateReport = { dryRun, dbPath };
+
+      if (!opts.phase2Only) {
+        report.drain = await drainQueue(store, { dryRun });
+        await appendConsolidationLog(logPath, {
+          phase: "phase1",
+          sessions_processed: report.drain.sessionsProcessed,
+          sessions_skipped: report.drain.sessionsSkipped,
+          sessions_failed: report.drain.sessionsFailed,
+          stage1_new_rows: report.drain.stage1NewRows,
+          dry_run: dryRun,
+        });
+
+        const trainResult = await trainBundle({
+          sessionsDir: cfg.sessionsDir,
+          bundleRoot: cfg.bundleRoot,
+          dryRun,
+          full: false,
+        });
+        report.train = {
+          sessionsProcessed: trainResult.sessionsProcessed,
+          entityCount: trainResult.entityCount,
+          relationCount: trainResult.relationCount,
+          eventCount: trainResult.eventCount,
+          dryRun: trainResult.dryRun,
+        };
+        await appendConsolidationLog(logPath, {
+          phase: "graph_train",
+          sessions_processed: trainResult.sessionsProcessed,
+          entities: trainResult.entityCount,
+          relations: trainResult.relationCount,
+          events: trainResult.eventCount,
+          dry_run: dryRun,
+        });
+      }
+
+      if (!opts.phase1Only) {
+        const memoryMdPath = cfg.memoryMdPaths[0] ?? path.join(cfg.bundleRoot, "MEMORY.md");
+        report.phase2 = await runPhase2(store, {
+          memoryMdPath,
+          bundleRoot: cfg.bundleRoot,
+          topN: cfg.consolidation.phase2_top_n,
+          dryRun,
+          maxLines: cfg.consolidation.memory_index_max_lines,
+          maxBytes: cfg.consolidation.memory_index_max_bytes,
+          maxUnusedDays: cfg.consolidation.max_unused_days,
+        });
+        await appendConsolidationLog(logPath, {
+          phase: "phase2",
+          stage1_selected: report.phase2.stage1Selected,
+          appended: report.phase2.appended,
+          skipped_dedup: report.phase2.skippedDedup,
+          backups_created: report.phase2.backupsCreated,
+          dry_run: dryRun,
+        });
+      }
+
+      return report;
+    } finally {
+      store.close();
+    }
+  });
+}

package/src/consolidation/scheduler/setupSchedule.ts

@@ -0,0 +1,106 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+
+import { buildLaunchdPlist, launchdPlistPath } from "./launchd.js";
+import {
+  buildSystemdService,
+  buildSystemdTimer,
+  systemdServicePath,
+  systemdTimerPath,
+} from "./systemd.js";
+import {
+  type SetupScheduleResult,
+  type ScheduledFile,
+  type ScheduleAction,
+  type ScheduleOptions,
+  type SchedulePlatform,
+} from "./types.js";
+
+async function exists(filePath: string): Promise<boolean> {
+  try {
+    await fs.access(filePath);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+async function ensureDir(filePath: string): Promise<void> {
+  await fs.mkdir(path.dirname(filePath), { recursive: true });
+}
+
+async function writeIfNeeded(
+  filePath: string,
+  content: string,
+  dryRun: boolean,
+): Promise<void> {
+  if (dryRun) return;
+  await ensureDir(filePath);
+  await fs.writeFile(filePath, content, "utf8");
+}
+
+async function removeFileIfExists(filePath: string, dryRun: boolean): Promise<void> {
+  if (dryRun) return;
+  try {
+    await fs.rm(filePath, { force: true });
+  } catch {
+    // Ignore cleanup failures for idempotency.
+  }
+}
+
+function setupPlan(
+  platform: SchedulePlatform,
+  opts: ScheduleOptions,
+): Array<{ path: string; content: string }> {
+  if (platform === "darwin") {
+    return [{ path: launchdPlistPath(), content: buildLaunchdPlist(opts) }];
+  }
+  return [
+    { path: systemdServicePath(), content: buildSystemdService(opts) },
+    { path: systemdTimerPath(), content: buildSystemdTimer(opts) },
+  ];
+}
+
+function resolveAction(opts: ScheduleOptions): ScheduleAction {
+  if (opts.status) return "status";
+  if (opts.remove) return "remove";
+  return "write";
+}
+
+export async function setupSchedule(
+  opts: ScheduleOptions,
+  platform: SchedulePlatform,
+): Promise<SetupScheduleResult> {
+  const dryRun = Boolean(opts.dryRun);
+  const action = resolveAction(opts);
+  const filesToProcess = setupPlan(platform, opts);
+
+  const files: ScheduledFile[] = [];
+
+  if (action === "status") {
+    for (const file of filesToProcess) {
+      files.push({ path: file.path, exists: await exists(file.path) });
+    }
+    return { platform, action, files, dryRun };
+  }
+
+  if (action === "remove") {
+    for (const file of filesToProcess) {
+      await removeFileIfExists(file.path, dryRun);
+    }
+    for (const file of filesToProcess) {
+      files.push({ path: file.path, exists: await exists(file.path) });
+    }
+    return { platform, action, files, dryRun };
+  }
+
+  for (const file of filesToProcess) {
+    await writeIfNeeded(file.path, file.content, dryRun);
+    files.push({
+      path: file.path,
+      exists: await exists(file.path),
+      content: file.content,
+    });
+  }
+  return { platform, action, files, dryRun };
+}