@chendpoc/pi-memory 0.1.12 → 0.1.13

package/src/config.ts

@@ -21,6 +21,32 @@ export interface TrainerConfig {
   auto_interval: string | null;
 }
 
+export interface ConsolidationScheduleConfig {
+  /** Local hour for the background consolidation job (default 3). */
+  hour: number;
+  /** Local minute for the background consolidation job (default 0). */
+  minute: number;
+  /** Structured JSONL log path (default ~/.pi/memory/consolidation.log). */
+  log_path: string;
+}
+
+export interface ConsolidationConfig {
+  /** Whether session_shutdown enqueues sessions for offline consolidation. */
+  enabled: boolean;
+  /** Minimum non-empty user turns before enqueue (default 3). */
+  min_user_turns: number;
+  /** MEMORY.md session cap by line count (default 200). */
+  memory_index_max_lines: number;
+  /** MEMORY.md session cap by UTF-8 bytes (default 25KB). */
+  memory_index_max_bytes: number;
+  /** Maximum stage1 rows selected per Phase2 run (default 20). */
+  phase2_top_n: number;
+  /** M4 prune threshold by last usage (default 90 days). */
+  max_unused_days: number;
+  /** Daily OS scheduler defaults. */
+  schedule: ConsolidationScheduleConfig;
+}
+
 export interface MemoryConfig {
   provider: MemoryProvider;
   tlmPath: string;
@@ -32,6 +58,7 @@ export interface MemoryConfig {
   sessionsDir: string;
   memoryMdPaths: string[];
   trainer: TrainerConfig;
+  consolidation: ConsolidationConfig;
 }
 
 export const defaultTrainerConfig: TrainerConfig = {
@@ -40,10 +67,37 @@ export const defaultTrainerConfig: TrainerConfig = {
   auto_interval: null,
 };
 
+export function defaultConsolidationConfig(
+  overrides: Partial<ConsolidationConfig> & {
+    schedule?: Partial<ConsolidationScheduleConfig>;
+  } = {},
+): ConsolidationConfig {
+  const defaultSchedule: ConsolidationScheduleConfig = {
+    hour: 3,
+    minute: 0,
+    log_path: path.join(defaultBundleRoot(), "consolidation.log"),
+  };
+  const { schedule, ...rest } = overrides;
+  return {
+    enabled: true,
+    min_user_turns: 3,
+    memory_index_max_lines: 200,
+    memory_index_max_bytes: 25_600,
+    phase2_top_n: 20,
+    max_unused_days: 90,
+    schedule: { ...defaultSchedule, ...schedule },
+    ...rest,
+  };
+}
+
 export function defaultMemoryConfig(
   overrides: Partial<MemoryConfig> = {},
 ): MemoryConfig {
-  const { trainer: trainerOverrides, ...rest } = overrides;
+  const {
+    trainer: trainerOverrides,
+    consolidation: consolidationOverrides,
+    ...rest
+  } = overrides;
   return {
     provider: "local",
     tlmPath: "tlm",
@@ -55,6 +109,7 @@ export function defaultMemoryConfig(
     sessionsDir: defaultSessionsDir(),
     memoryMdPaths: [path.join(defaultPiHome(), "MEMORY.md")],
     trainer: { ...defaultTrainerConfig, ...trainerOverrides },
+    consolidation: defaultConsolidationConfig(consolidationOverrides),
     ...rest,
   };
 }
@@ -65,6 +120,12 @@ export function normalizeMemoryConfig(
 ): MemoryConfig {
   const base = defaultMemoryConfig();
   const rawTrainer = (raw.trainer ?? {}) as Partial<TrainerConfig>;
+  const rawConsolidation = (raw.consolidation ?? {}) as
+    Partial<ConsolidationConfig> & {
+      schedule?: Partial<ConsolidationScheduleConfig>;
+    };
+  const rawSchedule = (rawConsolidation.schedule ?? {}) as
+    Partial<ConsolidationScheduleConfig>;
   return {
     provider: (raw.provider as MemoryProvider) ?? base.provider,
     tlmPath: expandPath(String(raw.tlmPath ?? base.tlmPath)),
@@ -88,5 +149,32 @@ export function normalizeMemoryConfig(
         ? rawTrainer.auto_interval
         : base.trainer.auto_interval,
     },
+    consolidation: {
+      enabled: rawConsolidation.enabled ?? base.consolidation.enabled,
+      min_user_turns: Number(
+        rawConsolidation.min_user_turns ?? base.consolidation.min_user_turns,
+      ),
+      memory_index_max_lines: Number(
+        rawConsolidation.memory_index_max_lines ??
+          base.consolidation.memory_index_max_lines,
+      ),
+      memory_index_max_bytes: Number(
+        rawConsolidation.memory_index_max_bytes ??
+          base.consolidation.memory_index_max_bytes,
+      ),
+      phase2_top_n: Number(
+        rawConsolidation.phase2_top_n ?? base.consolidation.phase2_top_n,
+      ),
+      max_unused_days: Number(
+        rawConsolidation.max_unused_days ?? base.consolidation.max_unused_days,
+      ),
+      schedule: {
+        hour: Number(rawSchedule.hour ?? base.consolidation.schedule.hour),
+        minute: Number(rawSchedule.minute ?? base.consolidation.schedule.minute),
+        log_path: expandPath(
+          String(rawSchedule.log_path ?? base.consolidation.schedule.log_path),
+        ),
+      },
+    },
   };
 }

package/src/consolidation/enqueue.ts

@@ -0,0 +1,205 @@
+import fs from "node:fs";
+
+import type {
+  ConsolidationStatus,
+  JobReport,
+  PendingSession,
+  ConsolidationStatusReport,
+} from "./types.js";
+import { openConsolidationStore, type SqliteDatabase } from "./stage1/store.js";
+
+export interface EnqueueSessionInput {
+  session_id: string;
+  session_file: string;
+  cwd: string;
+  git_root: string | null;
+  project_hash: string | null;
+  parent_session_id: string | null;
+  parent_session_file: string | null;
+  user_turn_count: number;
+  ended_at: string;
+}
+
+export interface EnqueueOptions {
+  enabled?: boolean;
+  minUserTurns?: number;
+  /** Injected sqlite handle for tests. */
+  db?: SqliteDatabase;
+  /** Force replacing existing done/skipped sessions. */
+  force?: boolean;
+  /** Override status for the newly enqueued job (default: pending). */
+  status?: ConsolidationStatus;
+  /** Optional now source for created_at / updated_at. */
+  now?: Date | string;
+  /** Test seam: production defaults to checking session file existence. */
+  checkFileExists?: boolean;
+}
+
+function toIso(value: Date | string | undefined): string {
+  const dt = value ? new Date(value) : new Date();
+  return dt.toISOString();
+}
+
+function isMissingFile(file: string): boolean {
+  return typeof file !== "string" || !file.trim();
+}
+
+function isAllowedStatus(status: string): status is ConsolidationStatus {
+  return status === "pending" || status === "processing" || status === "done" || status === "skipped" ||
+    status === "failed";
+}
+
+export function enqueueSession(
+  dbPath: string,
+  meta: EnqueueSessionInput,
+  opts: EnqueueOptions = {},
+): JobReport {
+  const enabled = opts.enabled !== false;
+  const minUserTurns = opts.minUserTurns ?? 3;
+  const now = toIso(opts.now);
+
+  if (!enabled) {
+    return {
+      session_id: meta.session_id,
+      enqueued: false,
+      action: "skipped",
+      status: "skipped",
+      reason: "consolidation_disabled",
+      now,
+    };
+  }
+
+  if (isMissingFile(meta.session_file)) {
+    return {
+      session_id: meta.session_id,
+      enqueued: false,
+      action: "skipped",
+      status: "skipped",
+      reason: "missing_session_file",
+      now,
+    };
+  }
+
+  if (opts.checkFileExists !== false && !fs.existsSync(meta.session_file)) {
+    return {
+      session_id: meta.session_id,
+      enqueued: false,
+      action: "skipped",
+      status: "skipped",
+      reason: "session_file_not_found",
+      now,
+    };
+  }
+
+  if (meta.user_turn_count < minUserTurns) {
+    return {
+      session_id: meta.session_id,
+      enqueued: false,
+      action: "skipped",
+      status: "skipped",
+      reason: "insufficient_user_turns",
+      now,
+    };
+  }
+
+  const store = openConsolidationStore(dbPath, opts.db);
+  if (!store) {
+    return {
+      session_id: meta.session_id,
+      enqueued: false,
+      action: "skipped",
+      status: "skipped",
+      reason: "missing_sqlite",
+      now,
+    };
+  }
+
+  const incomingStatus: ConsolidationStatus = isAllowedStatus(opts.status ?? "") ? opts.status! : "pending";
+  const existing = store.getPendingSession(meta.session_id);
+  const isFinal = existing?.status === "done" || existing?.status === "skipped";
+
+  if (existing && isFinal && !opts.force) {
+    store.close();
+    return {
+      session_id: meta.session_id,
+      enqueued: false,
+      action: "skipped",
+      status: existing.status,
+      reason: "already_finalized",
+      now,
+    };
+  }
+
+  const existingCreatedAt = existing?.created_at ?? now;
+  const payload: Omit<PendingSession, "created_at" | "updated_at"> & {
+    created_at: string;
+    updated_at: string;
+  } = {
+    session_id: meta.session_id,
+    session_file: meta.session_file,
+    cwd: meta.cwd,
+    git_root: meta.git_root,
+    project_hash: meta.project_hash,
+    parent_session_id: meta.parent_session_id,
+    parent_session_file: meta.parent_session_file,
+    user_turn_count: meta.user_turn_count,
+    ended_at: meta.ended_at,
+    status: incomingStatus,
+    error_message: null,
+    created_at: existingCreatedAt,
+    updated_at: now,
+  };
+
+  try {
+    store.upsertPendingSession(payload);
+    const action = existing ? "updated" : "created";
+    return {
+      session_id: meta.session_id,
+      enqueued: true,
+      action,
+      status: incomingStatus,
+      now,
+    };
+  } finally {
+    store.close();
+  }
+}
+
+export interface ConsolidationStatusQueryOptions {
+  db?: SqliteDatabase;
+}
+
+export function getConsolidationStatus(
+  dbPath: string,
+  opts: ConsolidationStatusQueryOptions = {},
+): ConsolidationStatusReport {
+  const store = openConsolidationStore(dbPath, opts.db);
+  if (!store) {
+    return {
+      pending: 0,
+      processing: 0,
+      done: 0,
+      skipped: 0,
+      failed: 0,
+      stage1Count: 0,
+      lastGeneratedAt: null,
+      lastUpdatedAt: null,
+    };
+  }
+
+  try {
+    const byStatus = store.getConsolidationStatusCounts();
+    return {
+      pending: byStatus.pending,
+      processing: byStatus.processing,
+      done: byStatus.done,
+      skipped: byStatus.skipped,
+      failed: byStatus.failed,
+      stage1Count: store.getStage1Count(),
+      lastGeneratedAt: store.getLastGeneratedAt(),
+      lastUpdatedAt: store.getLastUpdatedAt(),
+    };
+  } finally {
+    store.close();
+  }
+}

package/src/consolidation/index.ts

@@ -0,0 +1,90 @@
+export type {
+  ConsolidationStatus,
+  ConsolidationStatusReport,
+  JobReport,
+  PendingSession,
+  Stage1Output,
+} from "./types.js";
+
+export {
+  getMemoryIndexStats,
+  memoryIndexSnippet,
+  readMemoryIndexCap,
+  type MemoryIndexCapOptions,
+  type MemoryIndexStats,
+} from "./memoryIndex.js";
+
+export {
+  findGitRoot,
+  lineMatchesScope,
+  parseScopeComment,
+  projectHash,
+  projectScope,
+  scopeForCwd,
+  type MemoryScope,
+} from "./scope.js";
+
+export {
+  appendConsolidationLog,
+  readRecentConsolidationLogs,
+  type ConsolidationLogEntry,
+} from "./log.js";
+
+export { withFileLock } from "./lock.js";
+
+export { enqueueSession, getConsolidationStatus } from "./enqueue.js";
+export type {
+  ConsolidationStatusQueryOptions,
+  EnqueueOptions,
+  EnqueueSessionInput,
+} from "./enqueue.js";
+
+export { openConsolidationStore } from "./stage1/store.js";
+export type { SqliteDatabase, ConsolidationStore } from "./stage1/store.js";
+
+export {
+  computeDeltaTurns,
+} from "./stage1/deltaExtract.js";
+
+export {
+  drainQueue,
+  type DrainQueueOptions,
+  type DrainQueueReport,
+} from "./stage1/drainQueue.js";
+
+export { extractSessionToStage1 } from "./stage1/extractSession.js";
+
+export {
+  runPhase2,
+  type Phase2Options,
+  type Phase2Report,
+} from "./phase2/runPhase2.js";
+
+export {
+  defaultConsolidationDbPath,
+  runConsolidate,
+  type ConsolidateOptions,
+  type ConsolidateReport,
+} from "./scheduler/runConsolidate.js";
+
+export {
+  buildLaunchdPlist,
+  launchdPlistPath,
+} from "./scheduler/launchd.js";
+
+export {
+  buildSystemdService,
+  buildSystemdTimer,
+  systemdServicePath,
+  systemdTimerPath,
+} from "./scheduler/systemd.js";
+
+export { setupSchedule } from "./scheduler/setupSchedule.js";
+
+export type {
+  ScheduleAction,
+  ScheduledFile,
+  ScheduleOptions,
+  SchedulePlatform,
+  SetupScheduleResult,
+} from "./scheduler/types.js";

package/src/consolidation/lock.ts

@@ -0,0 +1,41 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+
+const LOCK_RETRIES = 50;
+const LOCK_DELAY_MS = 100;
+
+async function sleep(ms: number): Promise<void> {
+  await new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+export async function withFileLock<T>(
+  lockPath: string,
+  fn: () => Promise<T>,
+  opts: { retries?: number; delayMs?: number } = {},
+): Promise<T> {
+  const retries = opts.retries ?? LOCK_RETRIES;
+  const delayMs = opts.delayMs ?? LOCK_DELAY_MS;
+  let handle: fs.FileHandle | null = null;
+
+  await fs.mkdir(path.dirname(lockPath), { recursive: true, mode: 0o700 });
+
+  for (let i = 0; i < retries; i++) {
+    try {
+      handle = await fs.open(lockPath, "wx");
+      break;
+    } catch {
+      await sleep(delayMs);
+    }
+  }
+
+  if (!handle) {
+    throw new Error(`could not acquire lock: ${lockPath}`);
+  }
+
+  try {
+    return await fn();
+  } finally {
+    await handle.close().catch(() => {});
+    await fs.unlink(lockPath).catch(() => {});
+  }
+}

package/src/consolidation/log.ts

@@ -0,0 +1,43 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+
+export interface ConsolidationLogEntry {
+  ts?: string;
+  phase: string;
+  [key: string]: unknown;
+}
+
+export async function appendConsolidationLog(
+  logPath: string,
+  entry: ConsolidationLogEntry,
+): Promise<void> {
+  await fs.mkdir(path.dirname(logPath), { recursive: true, mode: 0o700 });
+  const line = JSON.stringify({
+    ts: entry.ts ?? new Date().toISOString(),
+    ...entry,
+  });
+  await fs.appendFile(logPath, `${line}\n`, { encoding: "utf8", mode: 0o600 });
+}
+
+export async function readRecentConsolidationLogs(
+  logPath: string,
+  limit = 5,
+): Promise<ConsolidationLogEntry[]> {
+  let text: string;
+  try {
+    text = await fs.readFile(logPath, "utf8");
+  } catch {
+    return [];
+  }
+  return text
+    .split("\n")
+    .filter((line) => line.trim())
+    .slice(-limit)
+    .map((line) => {
+      try {
+        return JSON.parse(line) as ConsolidationLogEntry;
+      } catch {
+        return { phase: "invalid_log_line", raw: line };
+      }
+    });
+}

package/src/consolidation/memoryIndex.ts

@@ -0,0 +1,127 @@
+import fs from "node:fs";
+import fsPromises from "node:fs/promises";
+
+import {
+  lineMatchesScope,
+  type MemoryScope,
+} from "./scope.js";
+
+export interface MemoryIndexCapOptions {
+  maxLines?: number;
+  maxBytes?: number;
+  scopes?: readonly MemoryScope[];
+}
+
+export interface MemoryIndexStats {
+  path: string;
+  exists: boolean;
+  lines: number;
+  bytes: number;
+  cappedLines: number;
+  cappedBytes: number;
+}
+
+const DEFAULT_MAX_LINES = 200;
+const DEFAULT_MAX_BYTES = 25_600;
+
+function capLines(
+  text: string,
+  opts: Required<Pick<MemoryIndexCapOptions, "maxLines" | "maxBytes">> & {
+    scopes?: readonly MemoryScope[];
+  },
+): string {
+  const out: string[] = [];
+  let bytes = 0;
+  for (const line of text.split("\n")) {
+    if (out.length >= opts.maxLines) break;
+    if (!lineMatchesScope(line, opts.scopes)) continue;
+    const nextBytes = Buffer.byteLength(line + "\n", "utf8");
+    if (bytes + nextBytes > opts.maxBytes) break;
+    out.push(line);
+    bytes += nextBytes;
+  }
+  return out.join("\n").trim();
+}
+
+function normalizedOptions(opts: MemoryIndexCapOptions = {}) {
+  return {
+    maxLines: opts.maxLines ?? DEFAULT_MAX_LINES,
+    maxBytes: opts.maxBytes ?? DEFAULT_MAX_BYTES,
+    scopes: opts.scopes,
+  };
+}
+
+export function readMemoryIndexCap(
+  paths: string[],
+  opts: MemoryIndexCapOptions = {},
+): string {
+  const normalized = normalizedOptions(opts);
+  for (const p of paths) {
+    let text: string;
+    try {
+      text = fs.readFileSync(p, "utf8");
+    } catch {
+      continue;
+    }
+    const capped = capLines(text, normalized);
+    if (capped) return capped;
+  }
+  return "";
+}
+
+export async function memoryIndexSnippet(
+  paths: string[],
+  query: string,
+  opts: MemoryIndexCapOptions = {},
+): Promise<string> {
+  const q = query.trim().toLowerCase();
+  if (!q) return "";
+  const normalized = normalizedOptions(opts);
+
+  for (const p of paths) {
+    let text: string;
+    try {
+      text = await fsPromises.readFile(p, "utf8");
+    } catch {
+      continue;
+    }
+    const matches: string[] = [];
+    let bytes = 0;
+    for (const line of text.split("\n")) {
+      if (!lineMatchesScope(line, normalized.scopes)) continue;
+      if (!line.toLowerCase().includes(q)) continue;
+      const nextBytes = Buffer.byteLength(line + "\n", "utf8");
+      if (bytes + nextBytes > normalized.maxBytes) break;
+      matches.push(line);
+      bytes += nextBytes;
+      if (matches.length >= normalized.maxLines) break;
+    }
+    if (matches.length > 0) return matches.join("\n");
+  }
+  return "";
+}
+
+export function getMemoryIndexStats(
+  paths: string[],
+  opts: MemoryIndexCapOptions = {},
+): MemoryIndexStats[] {
+  const normalized = normalizedOptions(opts);
+  return paths.map((p) => {
+    let text = "";
+    let exists = true;
+    try {
+      text = fs.readFileSync(p, "utf8");
+    } catch {
+      exists = false;
+    }
+    const capped = exists ? capLines(text, normalized) : "";
+    return {
+      path: p,
+      exists,
+      lines: exists ? text.split("\n").length : 0,
+      bytes: exists ? Buffer.byteLength(text, "utf8") : 0,
+      cappedLines: capped ? capped.split("\n").length : 0,
+      cappedBytes: Buffer.byteLength(capped, "utf8"),
+    };
+  });
+}