@chendpoc/pi-memory 0.1.11 → 0.1.13

package/src/cli.ts

@@ -5,6 +5,15 @@ import path from "node:path";
 import { createStandaloneLLMClient } from "./adapters/piComplete.js";
 import { installBundle } from "./bundle/install.js";
 import type { MemoryConfig } from "./config.js";
+import { getMemoryIndexStats } from "./consolidation/memoryIndex.js";
+import { readRecentConsolidationLogs } from "./consolidation/log.js";
+import { getConsolidationStatus } from "./consolidation/enqueue.js";
+import {
+  defaultConsolidationDbPath,
+  runConsolidate,
+} from "./consolidation/scheduler/runConsolidate.js";
+import { setupSchedule } from "./consolidation/scheduler/setupSchedule.js";
+import type { SchedulePlatform } from "./consolidation/scheduler/types.js";
 import { loadMemoryConfig } from "./settings.js";
 import { openSessionIndex } from "./fallback/sessionIndex.js";
 import { SidecarClient } from "./sidecar/client.js";
@@ -73,6 +82,55 @@ async function main(): Promise<void> {
     return;
   }
 
+  if (cmd === "memory-status") {
+    const dbPath = defaultConsolidationDbPath(cfg);
+    const status = getConsolidationStatus(dbPath);
+    const memoryIndex = getMemoryIndexStats(cfg.memoryMdPaths, {
+      maxLines: cfg.consolidation.memory_index_max_lines,
+      maxBytes: cfg.consolidation.memory_index_max_bytes,
+    });
+    const recentLogs = await readRecentConsolidationLogs(
+      cfg.consolidation.schedule.log_path,
+      5,
+    );
+    const schedule = await readScheduleStatus(cfg);
+    console.log(JSON.stringify({
+      db_path: dbPath,
+      queue: status,
+      memory_index: memoryIndex,
+      schedule,
+      recent_logs: recentLogs,
+    }, null, 2));
+    return;
+  }
+
+  if (cmd === "consolidate") {
+    const flags = parseConsolidateFlags(rest);
+    const result = await runConsolidate({
+      config: cfg,
+      dryRun: flags.dryRun,
+      phase1Only: flags.phase1Only,
+      phase2Only: flags.phase2Only,
+    });
+    console.log(JSON.stringify(result, null, 2));
+    return;
+  }
+
+  if (cmd === "setup-schedule") {
+    const flags = parseScheduleFlags(rest);
+    const platform = resolveSchedulePlatform();
+    const result = await setupSchedule({
+      hour: flags.hour ?? cfg.consolidation.schedule.hour,
+      minute: flags.minute ?? cfg.consolidation.schedule.minute,
+      logPath: cfg.consolidation.schedule.log_path,
+      dryRun: flags.dryRun,
+      remove: flags.remove,
+      status: flags.status,
+    }, platform);
+    console.log(JSON.stringify(result, null, 2));
+    return;
+  }
+
   if (cmd === "install-bundle") {
     const bundlePath = rest[0];
     if (!bundlePath) {
@@ -214,12 +272,98 @@ function parseTrainFlags(args: string[]): {
   return { sessionsDir, full, dryRun, noMerge, extractor, watch, model };
 }
 
+function parseConsolidateFlags(args: string[]): {
+  dryRun: boolean;
+  phase1Only: boolean;
+  phase2Only: boolean;
+} {
+  return {
+    dryRun: args.includes("--dry-run"),
+    phase1Only: args.includes("--phase1-only"),
+    phase2Only: args.includes("--phase2-only"),
+  };
+}
+
+function parseScheduleFlags(args: string[]): {
+  hour?: number;
+  minute?: number;
+  dryRun: boolean;
+  remove: boolean;
+  status: boolean;
+} {
+  let hour: number | undefined;
+  let minute: number | undefined;
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i]!;
+    if (arg === "--hour" && i + 1 < args.length) {
+      hour = Number(args[++i]);
+      continue;
+    }
+    if (arg === "--minute" && i + 1 < args.length) {
+      minute = Number(args[++i]);
+      continue;
+    }
+  }
+  return {
+    hour,
+    minute,
+    dryRun: args.includes("--dry-run"),
+    remove: args.includes("--remove"),
+    status: args.includes("--status"),
+  };
+}
+
+function resolveSchedulePlatform(): SchedulePlatform {
+  if (process.platform === "darwin") return "darwin";
+  if (process.platform === "linux") return "linux";
+  throw new Error(`setup-schedule is not supported on ${process.platform}`);
+}
+
+function tryResolveSchedulePlatform(): SchedulePlatform | null {
+  if (process.platform === "darwin") return "darwin";
+  if (process.platform === "linux") return "linux";
+  return null;
+}
+
+async function readScheduleStatus(cfg: MemoryConfig) {
+  const platform = tryResolveSchedulePlatform();
+  if (!platform) {
+    return {
+      supported: false,
+      platform: process.platform,
+      files: [],
+    };
+  }
+  const result = await setupSchedule({
+    hour: cfg.consolidation.schedule.hour,
+    minute: cfg.consolidation.schedule.minute,
+    logPath: cfg.consolidation.schedule.log_path,
+    status: true,
+  }, platform);
+  return {
+    supported: true,
+    platform,
+    files: result.files,
+  };
+}
+
 function printHelp(): void {
   console.log(`pi-memory — local TLM episodic memory (mode B)
 
 Commands:
   health              Start sidecar (if bundle present) and print /health
   status              Print MemoryService status snapshot
+  memory-status       Print consolidation queue, MEMORY.md cap, schedule status, and recent job logs
+  consolidate         Run offline memory consolidation
+    --dry-run         Report planned work without writing MEMORY.md/stage1
+    --phase1-only     Drain queue and train graph only
+    --phase2-only     Consume existing stage1 rows only
+  setup-schedule      Install/update the OS user scheduler
+    --hour            Local hour (default from memory.json consolidation.schedule.hour)
+    --minute          Local minute (default from memory.json consolidation.schedule.minute)
+    --dry-run         Print files that would be written
+    --remove          Remove scheduler files
+    --status          Report scheduler file existence
   query               POST /query with JSON QueryIntent
   install-bundle      Copy a local bundle dir into ~/.pi/memory/current
   train               Build a bundle from session history (delta merge by default)

package/src/config.ts

@@ -21,6 +21,32 @@ export interface TrainerConfig {
   auto_interval: string | null;
 }
 
+export interface ConsolidationScheduleConfig {
+  /** Local hour for the background consolidation job (default 3). */
+  hour: number;
+  /** Local minute for the background consolidation job (default 0). */
+  minute: number;
+  /** Structured JSONL log path (default ~/.pi/memory/consolidation.log). */
+  log_path: string;
+}
+
+export interface ConsolidationConfig {
+  /** Whether session_shutdown enqueues sessions for offline consolidation. */
+  enabled: boolean;
+  /** Minimum non-empty user turns before enqueue (default 3). */
+  min_user_turns: number;
+  /** MEMORY.md session cap by line count (default 200). */
+  memory_index_max_lines: number;
+  /** MEMORY.md session cap by UTF-8 bytes (default 25KB). */
+  memory_index_max_bytes: number;
+  /** Maximum stage1 rows selected per Phase2 run (default 20). */
+  phase2_top_n: number;
+  /** M4 prune threshold by last usage (default 90 days). */
+  max_unused_days: number;
+  /** Daily OS scheduler defaults. */
+  schedule: ConsolidationScheduleConfig;
+}
+
 export interface MemoryConfig {
   provider: MemoryProvider;
   tlmPath: string;
@@ -32,6 +58,7 @@ export interface MemoryConfig {
   sessionsDir: string;
   memoryMdPaths: string[];
   trainer: TrainerConfig;
+  consolidation: ConsolidationConfig;
 }
 
 export const defaultTrainerConfig: TrainerConfig = {
@@ -40,10 +67,37 @@ export const defaultTrainerConfig: TrainerConfig = {
   auto_interval: null,
 };
 
+export function defaultConsolidationConfig(
+  overrides: Partial<ConsolidationConfig> & {
+    schedule?: Partial<ConsolidationScheduleConfig>;
+  } = {},
+): ConsolidationConfig {
+  const defaultSchedule: ConsolidationScheduleConfig = {
+    hour: 3,
+    minute: 0,
+    log_path: path.join(defaultBundleRoot(), "consolidation.log"),
+  };
+  const { schedule, ...rest } = overrides;
+  return {
+    enabled: true,
+    min_user_turns: 3,
+    memory_index_max_lines: 200,
+    memory_index_max_bytes: 25_600,
+    phase2_top_n: 20,
+    max_unused_days: 90,
+    schedule: { ...defaultSchedule, ...schedule },
+    ...rest,
+  };
+}
+
 export function defaultMemoryConfig(
   overrides: Partial<MemoryConfig> = {},
 ): MemoryConfig {
-  const { trainer: trainerOverrides, ...rest } = overrides;
+  const {
+    trainer: trainerOverrides,
+    consolidation: consolidationOverrides,
+    ...rest
+  } = overrides;
   return {
     provider: "local",
     tlmPath: "tlm",
@@ -55,6 +109,7 @@ export function defaultMemoryConfig(
     sessionsDir: defaultSessionsDir(),
     memoryMdPaths: [path.join(defaultPiHome(), "MEMORY.md")],
     trainer: { ...defaultTrainerConfig, ...trainerOverrides },
+    consolidation: defaultConsolidationConfig(consolidationOverrides),
     ...rest,
   };
 }
@@ -65,6 +120,12 @@ export function normalizeMemoryConfig(
 ): MemoryConfig {
   const base = defaultMemoryConfig();
   const rawTrainer = (raw.trainer ?? {}) as Partial<TrainerConfig>;
+  const rawConsolidation = (raw.consolidation ?? {}) as
+    Partial<ConsolidationConfig> & {
+      schedule?: Partial<ConsolidationScheduleConfig>;
+    };
+  const rawSchedule = (rawConsolidation.schedule ?? {}) as
+    Partial<ConsolidationScheduleConfig>;
   return {
     provider: (raw.provider as MemoryProvider) ?? base.provider,
     tlmPath: expandPath(String(raw.tlmPath ?? base.tlmPath)),
@@ -88,5 +149,32 @@ export function normalizeMemoryConfig(
         ? rawTrainer.auto_interval
         : base.trainer.auto_interval,
     },
+    consolidation: {
+      enabled: rawConsolidation.enabled ?? base.consolidation.enabled,
+      min_user_turns: Number(
+        rawConsolidation.min_user_turns ?? base.consolidation.min_user_turns,
+      ),
+      memory_index_max_lines: Number(
+        rawConsolidation.memory_index_max_lines ??
+          base.consolidation.memory_index_max_lines,
+      ),
+      memory_index_max_bytes: Number(
+        rawConsolidation.memory_index_max_bytes ??
+          base.consolidation.memory_index_max_bytes,
+      ),
+      phase2_top_n: Number(
+        rawConsolidation.phase2_top_n ?? base.consolidation.phase2_top_n,
+      ),
+      max_unused_days: Number(
+        rawConsolidation.max_unused_days ?? base.consolidation.max_unused_days,
+      ),
+      schedule: {
+        hour: Number(rawSchedule.hour ?? base.consolidation.schedule.hour),
+        minute: Number(rawSchedule.minute ?? base.consolidation.schedule.minute),
+        log_path: expandPath(
+          String(rawSchedule.log_path ?? base.consolidation.schedule.log_path),
+        ),
+      },
+    },
   };
 }

package/src/consolidation/enqueue.ts

@@ -0,0 +1,205 @@
+import fs from "node:fs";
+
+import type {
+  ConsolidationStatus,
+  JobReport,
+  PendingSession,
+  ConsolidationStatusReport,
+} from "./types.js";
+import { openConsolidationStore, type SqliteDatabase } from "./stage1/store.js";
+
+export interface EnqueueSessionInput {
+  session_id: string;
+  session_file: string;
+  cwd: string;
+  git_root: string | null;
+  project_hash: string | null;
+  parent_session_id: string | null;
+  parent_session_file: string | null;
+  user_turn_count: number;
+  ended_at: string;
+}
+
+export interface EnqueueOptions {
+  enabled?: boolean;
+  minUserTurns?: number;
+  /** Injected sqlite handle for tests. */
+  db?: SqliteDatabase;
+  /** Force replacing existing done/skipped sessions. */
+  force?: boolean;
+  /** Override status for the newly enqueued job (default: pending). */
+  status?: ConsolidationStatus;
+  /** Optional now source for created_at / updated_at. */
+  now?: Date | string;
+  /** Test seam: production defaults to checking session file existence. */
+  checkFileExists?: boolean;
+}
+
+function toIso(value: Date | string | undefined): string {
+  const dt = value ? new Date(value) : new Date();
+  return dt.toISOString();
+}
+
+function isMissingFile(file: string): boolean {
+  return typeof file !== "string" || !file.trim();
+}
+
+function isAllowedStatus(status: string): status is ConsolidationStatus {
+  return status === "pending" || status === "processing" || status === "done" || status === "skipped" ||
+    status === "failed";
+}
+
+export function enqueueSession(
+  dbPath: string,
+  meta: EnqueueSessionInput,
+  opts: EnqueueOptions = {},
+): JobReport {
+  const enabled = opts.enabled !== false;
+  const minUserTurns = opts.minUserTurns ?? 3;
+  const now = toIso(opts.now);
+
+  if (!enabled) {
+    return {
+      session_id: meta.session_id,
+      enqueued: false,
+      action: "skipped",
+      status: "skipped",
+      reason: "consolidation_disabled",
+      now,
+    };
+  }
+
+  if (isMissingFile(meta.session_file)) {
+    return {
+      session_id: meta.session_id,
+      enqueued: false,
+      action: "skipped",
+      status: "skipped",
+      reason: "missing_session_file",
+      now,
+    };
+  }
+
+  if (opts.checkFileExists !== false && !fs.existsSync(meta.session_file)) {
+    return {
+      session_id: meta.session_id,
+      enqueued: false,
+      action: "skipped",
+      status: "skipped",
+      reason: "session_file_not_found",
+      now,
+    };
+  }
+
+  if (meta.user_turn_count < minUserTurns) {
+    return {
+      session_id: meta.session_id,
+      enqueued: false,
+      action: "skipped",
+      status: "skipped",
+      reason: "insufficient_user_turns",
+      now,
+    };
+  }
+
+  const store = openConsolidationStore(dbPath, opts.db);
+  if (!store) {
+    return {
+      session_id: meta.session_id,
+      enqueued: false,
+      action: "skipped",
+      status: "skipped",
+      reason: "missing_sqlite",
+      now,
+    };
+  }
+
+  const incomingStatus: ConsolidationStatus = isAllowedStatus(opts.status ?? "") ? opts.status! : "pending";
+  const existing = store.getPendingSession(meta.session_id);
+  const isFinal = existing?.status === "done" || existing?.status === "skipped";
+
+  if (existing && isFinal && !opts.force) {
+    store.close();
+    return {
+      session_id: meta.session_id,
+      enqueued: false,
+      action: "skipped",
+      status: existing.status,
+      reason: "already_finalized",
+      now,
+    };
+  }
+
+  const existingCreatedAt = existing?.created_at ?? now;
+  const payload: Omit<PendingSession, "created_at" | "updated_at"> & {
+    created_at: string;
+    updated_at: string;
+  } = {
+    session_id: meta.session_id,
+    session_file: meta.session_file,
+    cwd: meta.cwd,
+    git_root: meta.git_root,
+    project_hash: meta.project_hash,
+    parent_session_id: meta.parent_session_id,
+    parent_session_file: meta.parent_session_file,
+    user_turn_count: meta.user_turn_count,
+    ended_at: meta.ended_at,
+    status: incomingStatus,
+    error_message: null,
+    created_at: existingCreatedAt,
+    updated_at: now,
+  };
+
+  try {
+    store.upsertPendingSession(payload);
+    const action = existing ? "updated" : "created";
+    return {
+      session_id: meta.session_id,
+      enqueued: true,
+      action,
+      status: incomingStatus,
+      now,
+    };
+  } finally {
+    store.close();
+  }
+}
+
+export interface ConsolidationStatusQueryOptions {
+  db?: SqliteDatabase;
+}
+
+export function getConsolidationStatus(
+  dbPath: string,
+  opts: ConsolidationStatusQueryOptions = {},
+): ConsolidationStatusReport {
+  const store = openConsolidationStore(dbPath, opts.db);
+  if (!store) {
+    return {
+      pending: 0,
+      processing: 0,
+      done: 0,
+      skipped: 0,
+      failed: 0,
+      stage1Count: 0,
+      lastGeneratedAt: null,
+      lastUpdatedAt: null,
+    };
+  }
+
+  try {
+    const byStatus = store.getConsolidationStatusCounts();
+    return {
+      pending: byStatus.pending,
+      processing: byStatus.processing,
+      done: byStatus.done,
+      skipped: byStatus.skipped,
+      failed: byStatus.failed,
+      stage1Count: store.getStage1Count(),
+      lastGeneratedAt: store.getLastGeneratedAt(),
+      lastUpdatedAt: store.getLastUpdatedAt(),
+    };
+  } finally {
+    store.close();
+  }
+}

package/src/consolidation/index.ts

@@ -0,0 +1,90 @@
+export type {
+  ConsolidationStatus,
+  ConsolidationStatusReport,
+  JobReport,
+  PendingSession,
+  Stage1Output,
+} from "./types.js";
+
+export {
+  getMemoryIndexStats,
+  memoryIndexSnippet,
+  readMemoryIndexCap,
+  type MemoryIndexCapOptions,
+  type MemoryIndexStats,
+} from "./memoryIndex.js";
+
+export {
+  findGitRoot,
+  lineMatchesScope,
+  parseScopeComment,
+  projectHash,
+  projectScope,
+  scopeForCwd,
+  type MemoryScope,
+} from "./scope.js";
+
+export {
+  appendConsolidationLog,
+  readRecentConsolidationLogs,
+  type ConsolidationLogEntry,
+} from "./log.js";
+
+export { withFileLock } from "./lock.js";
+
+export { enqueueSession, getConsolidationStatus } from "./enqueue.js";
+export type {
+  ConsolidationStatusQueryOptions,
+  EnqueueOptions,
+  EnqueueSessionInput,
+} from "./enqueue.js";
+
+export { openConsolidationStore } from "./stage1/store.js";
+export type { SqliteDatabase, ConsolidationStore } from "./stage1/store.js";
+
+export {
+  computeDeltaTurns,
+} from "./stage1/deltaExtract.js";
+
+export {
+  drainQueue,
+  type DrainQueueOptions,
+  type DrainQueueReport,
+} from "./stage1/drainQueue.js";
+
+export { extractSessionToStage1 } from "./stage1/extractSession.js";
+
+export {
+  runPhase2,
+  type Phase2Options,
+  type Phase2Report,
+} from "./phase2/runPhase2.js";
+
+export {
+  defaultConsolidationDbPath,
+  runConsolidate,
+  type ConsolidateOptions,
+  type ConsolidateReport,
+} from "./scheduler/runConsolidate.js";
+
+export {
+  buildLaunchdPlist,
+  launchdPlistPath,
+} from "./scheduler/launchd.js";
+
+export {
+  buildSystemdService,
+  buildSystemdTimer,
+  systemdServicePath,
+  systemdTimerPath,
+} from "./scheduler/systemd.js";
+
+export { setupSchedule } from "./scheduler/setupSchedule.js";
+
+export type {
+  ScheduleAction,
+  ScheduledFile,
+  ScheduleOptions,
+  SchedulePlatform,
+  SetupScheduleResult,
+} from "./scheduler/types.js";

package/src/consolidation/lock.ts

@@ -0,0 +1,41 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+
+const LOCK_RETRIES = 50;
+const LOCK_DELAY_MS = 100;
+
+async function sleep(ms: number): Promise<void> {
+  await new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+export async function withFileLock<T>(
+  lockPath: string,
+  fn: () => Promise<T>,
+  opts: { retries?: number; delayMs?: number } = {},
+): Promise<T> {
+  const retries = opts.retries ?? LOCK_RETRIES;
+  const delayMs = opts.delayMs ?? LOCK_DELAY_MS;
+  let handle: fs.FileHandle | null = null;
+
+  await fs.mkdir(path.dirname(lockPath), { recursive: true, mode: 0o700 });
+
+  for (let i = 0; i < retries; i++) {
+    try {
+      handle = await fs.open(lockPath, "wx");
+      break;
+    } catch {
+      await sleep(delayMs);
+    }
+  }
+
+  if (!handle) {
+    throw new Error(`could not acquire lock: ${lockPath}`);
+  }
+
+  try {
+    return await fn();
+  } finally {
+    await handle.close().catch(() => {});
+    await fs.unlink(lockPath).catch(() => {});
+  }
+}

package/src/consolidation/log.ts

@@ -0,0 +1,43 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+
+export interface ConsolidationLogEntry {
+  ts?: string;
+  phase: string;
+  [key: string]: unknown;
+}
+
+export async function appendConsolidationLog(
+  logPath: string,
+  entry: ConsolidationLogEntry,
+): Promise<void> {
+  await fs.mkdir(path.dirname(logPath), { recursive: true, mode: 0o700 });
+  const line = JSON.stringify({
+    ts: entry.ts ?? new Date().toISOString(),
+    ...entry,
+  });
+  await fs.appendFile(logPath, `${line}\n`, { encoding: "utf8", mode: 0o600 });
+}
+
+export async function readRecentConsolidationLogs(
+  logPath: string,
+  limit = 5,
+): Promise<ConsolidationLogEntry[]> {
+  let text: string;
+  try {
+    text = await fs.readFile(logPath, "utf8");
+  } catch {
+    return [];
+  }
+  return text
+    .split("\n")
+    .filter((line) => line.trim())
+    .slice(-limit)
+    .map((line) => {
+      try {
+        return JSON.parse(line) as ConsolidationLogEntry;
+      } catch {
+        return { phase: "invalid_log_line", raw: line };
+      }
+    });
+}