@chendpoc/pi-memory 0.1.13 → 0.2.0

package/src/bundle/install.ts

@@ -1,206 +0,0 @@
-import { createHash } from "node:crypto";
-import fs from "node:fs/promises";
-import path from "node:path";
-
-import type { BundleManifest } from "../sidecar/bundle.js";
-
-export interface InstallBundleOptions {
-  bundleRoot: string;
-  sourceDir: string;
-  /** Number of old bundles to keep after install (default 3). */
-  retain?: number;
-}
-
-export interface InstallBundleResult {
-  bundle_ts: string;
-  bundle_version: string;
-  installed_dir: string;
-  files_copied: number;
-}
-
-/**
- * Enforces [0.4.0, 0.7.0) — producers guarantee additive minor bumps in this
- * range; breaking schema changes move to 0.7.0+ to trip this gate.
- * Ported from Kocoro internal/memory/bundle.go versionInRange.
- */
-export function versionInRange(v: string): boolean {
-  const parts = v.split(".");
-  if (parts.length !== 3) return false;
-  const [maj, min, pat] = parts.map(Number);
-  if (maj !== 0 || isNaN(min!) || isNaN(pat!)) return false;
-  if (min! < 4 || min! >= 7) return false;
-  return pat! >= 0;
-}
-
-/**
- * Keep the newest `keep` bundle dirs plus the current symlink target.
- * Best-effort — failures are silently ignored.
- * Ported from Kocoro internal/memory/bundle.go Puller.retain.
- */
-export async function retainBundles(bundleRoot: string, keep: number): Promise<void> {
-  const bundlesDir = path.join(bundleRoot, "bundles");
-  let names: string[];
-  try {
-    names = await fs.readdir(bundlesDir);
-  } catch {
-    return;
-  }
-  const dirs: string[] = [];
-  for (const name of names) {
-    try {
-      const st = await fs.stat(path.join(bundlesDir, name));
-      if (st.isDirectory()) dirs.push(name);
-    } catch { /* skip unreadable */ }
-  }
-  dirs.sort().reverse();
-  if (dirs.length <= keep) return;
-
-  let currentTarget = "";
-  try {
-    const target = await fs.readlink(path.join(bundleRoot, "current"));
-    currentTarget = path.basename(target);
-  } catch { /* no current pointer */ }
-
-  const keepSet = new Set<string>();
-  for (let i = 0; i < Math.min(keep, dirs.length); i++) {
-    keepSet.add(dirs[i]!);
-  }
-  if (currentTarget) keepSet.add(currentTarget);
-
-  for (const d of dirs) {
-    if (!keepSet.has(d)) {
-      await fs.rm(path.join(bundlesDir, d), { recursive: true, force: true }).catch(() => {});
-    }
-  }
-}
-
-/**
- * Path sandboxing rules from Kocoro bundle.go validateManifestPath (§4.2).
- */
-export function validateManifestPath(rel: string, stagingDir: string): string | null {
-  if (!rel) return "empty path";
-  if (rel.includes("\0")) return "null byte in path";
-  if (path.isAbsolute(rel)) return "absolute path";
-  const cleaned = path.normalize(rel);
-  if (
-    cleaned === ".." ||
-    cleaned.startsWith(`..${path.sep}`) ||
-    cleaned.includes(`${path.sep}..`)
-  ) {
-    return "contains parent traversal";
-  }
-  const abs = path.join(stagingDir, cleaned);
-  const cleanedAbs = path.normalize(abs);
-  const prefix = path.normalize(stagingDir) + path.sep;
-  if (!cleanedAbs.startsWith(prefix)) {
-    return "escapes staging dir";
-  }
-  return null;
-}
-
-async function sha256File(filePath: string): Promise<string> {
-  const data = await fs.readFile(filePath);
-  return createHash("sha256").update(data).digest("hex");
-}
-
-/**
- * Points <bundleRoot>/current at finalDir. On Windows uses a directory junction
- * (unprivileged, no Developer Mode needed — os.Symlink would fail with
- * ERROR_PRIVILEGE_NOT_HELD on stock hosts). On POSIX uses atomic tmp-symlink +
- * rename. Ported from Kocoro bundle_link_{unix,windows}.go.
- */
-async function swapCurrent(bundleRoot: string, finalDir: string): Promise<void> {
-  const currentLink = path.join(bundleRoot, "current");
-
-  if (process.platform === "win32") {
-    try { await fs.rm(currentLink, { recursive: true, force: true }); } catch { /* absent */ }
-    await fs.symlink(finalDir, currentLink, "junction");
-    return;
-  }
-
-  const tmpLink = path.join(bundleRoot, "current.tmp");
-  try { await fs.unlink(tmpLink); } catch { /* absent */ }
-  await fs.symlink(finalDir, tmpLink);
-  await fs.rename(tmpLink, currentLink);
-}
-
-/**
- * Install a local bundle directory into bundleRoot (staging → bundles/<ts> → current).
- * Verifies per-file sha256 when manifest lists hashes.
- */
-export async function installBundle(
-  opts: InstallBundleOptions,
-): Promise<InstallBundleResult> {
-  const sourceDir = path.resolve(opts.sourceDir);
-  const bundleRoot = path.resolve(opts.bundleRoot);
-
-  const manifestPath = path.join(sourceDir, "manifest.json");
-  const raw = await fs.readFile(manifestPath, "utf8");
-  const manifest = JSON.parse(raw) as BundleManifest;
-
-  if (!manifest.bundle_ts?.trim()) {
-    throw new Error("manifest missing bundle_ts");
-  }
-  if (!versionInRange(manifest.bundle_version ?? "")) {
-    throw new Error(
-      `bundle_version "${manifest.bundle_version}" outside supported range [0.4.0, 0.7.0)`,
-    );
-  }
-
-  await fs.mkdir(bundleRoot, { recursive: true, mode: 0o700 });
-
-  const staging = path.join(bundleRoot, "staging", manifest.bundle_ts);
-  await fs.rm(staging, { recursive: true, force: true });
-  await fs.mkdir(staging, { recursive: true, mode: 0o700 });
-
-  let filesCopied = 0;
-  for (const f of manifest.files ?? []) {
-    const err = validateManifestPath(f.path, staging);
-    if (err) {
-      await fs.rm(staging, { recursive: true, force: true });
-      throw new Error(`unsafe manifest path ${f.path}: ${err}`);
-    }
-    const src = path.join(sourceDir, path.normalize(f.path));
-    const dest = path.join(staging, path.normalize(f.path));
-    await fs.mkdir(path.dirname(dest), { recursive: true, mode: 0o700 });
-    await fs.copyFile(src, dest, fs.constants.COPYFILE_EXCL).catch(async () => {
-      await fs.copyFile(src, dest);
-    });
-
-    if (f.sha256) {
-      const got = await sha256File(dest);
-      if (got !== f.sha256) {
-        await fs.rm(staging, { recursive: true, force: true });
-        throw new Error(`sha256 mismatch on ${f.path}: got ${got} want ${f.sha256}`);
-      }
-    }
-    filesCopied++;
-  }
-
-  // Copy manifest itself if not listed in files[]
-  const manifestDest = path.join(staging, "manifest.json");
-  try {
-    await fs.access(manifestDest);
-  } catch {
-    await fs.copyFile(manifestPath, manifestDest);
-  }
-
-  const bundlesDir = path.join(bundleRoot, "bundles");
-  await fs.mkdir(bundlesDir, { recursive: true, mode: 0o700 });
-  const finalDir = path.join(bundlesDir, manifest.bundle_ts);
-  await fs.rm(finalDir, { recursive: true, force: true });
-  await fs.rename(staging, finalDir);
-  await swapCurrent(bundleRoot, finalDir);
-
-  const keep = opts.retain ?? 3;
-  if (keep > 0) {
-    await retainBundles(bundleRoot, keep);
-  }
-
-  return {
-    bundle_ts: manifest.bundle_ts,
-    bundle_version: manifest.bundle_version,
-    installed_dir: finalDir,
-    files_copied: filesCopied,
-  };
-}

package/src/cache/memoryCaches.ts

@@ -1,72 +0,0 @@
-import { LRUCache } from "lru-cache";
-import type { QueryIntent } from "../types.js";
-import type { RankedResult } from "../fallback/llmRerank.js";
-import type { SessionSearchHit } from "../fallback/sessionSearch.js";
-
-const INTENT_TTL_MS = 15 * 60_000;
-const RERANK_TTL_MS = 15 * 60_000;
-const NEGATIVE_TTL_MS = 60_000;
-
-/**
- * In-process LRU + TTL caches for pi-memory.
- *
- * - intentCache  : helper-LLM compile_memory_intents results, keyed by
- *                  normalised query text. TTL 15 min, LRU 128 entries.
- * - rerankCache  : LLM rerank results, keyed by query + hit fingerprint.
- *                  TTL 15 min, LRU 256 entries.
- * - negativeCache: queries that recently returned no usable context.
- *                  TTL 60 s, max 512 entries.
- *
- * All three caches must be invalidated together (`invalidateMemoryCaches()`)
- * after bundle reload so stale negative entries never block fresh graph data.
- */
-export const intentCache = new LRUCache<string, QueryIntent[]>({
-  max: 128,
-  ttl: INTENT_TTL_MS,
-});
-
-export const rerankCache = new LRUCache<string, RankedResult[]>({
-  max: 256,
-  ttl: RERANK_TTL_MS,
-});
-
-export const negativeCache = new LRUCache<string, true>({
-  max: 512,
-  ttl: NEGATIVE_TTL_MS,
-});
-
-function normalizeQuery(query: string): string {
-  return query.trim();
-}
-
-export function cacheKeyForIntents(query: string): string {
-  return normalizeQuery(query);
-}
-
-/**
- * Rerank cache key combines the query with a fingerprint of the hit set so
- * different FTS results for the same query get their own entry.
- */
-export function cacheKeyForRerank(query: string, hits: SessionSearchHit[]): string {
-  const hitIds = hits.map((h) => `${h.session_id}:${h.msg_index}`).join("|");
-  return `${normalizeQuery(query)}|${hitIds}`;
-}
-
-export function isNegativeCached(query: string): boolean {
-  return negativeCache.has(cacheKeyForIntents(query));
-}
-
-export function setNegativeCache(query: string): void {
-  negativeCache.set(cacheKeyForIntents(query), true);
-}
-
-export function deleteNegativeCache(query: string): void {
-  negativeCache.delete(cacheKeyForIntents(query));
-}
-
-/** Clear all three caches. Call after bundle reload or session shutdown. */
-export function invalidateMemoryCaches(): void {
-  intentCache.clear();
-  rerankCache.clear();
-  negativeCache.clear();
-}

package/src/config.ts

@@ -1,180 +0,0 @@
-import path from "node:path";
-
-import {
-  defaultBundleRoot,
-  defaultPiHome,
-  defaultSessionsDir,
-  defaultSocketPath,
-  expandPath,
-} from "./paths.js";
-
-export type MemoryProvider = "disabled" | "local" | "cloud";
-
-export type ExtractorType = "regex" | "llm";
-
-export interface TrainerConfig {
-  /** Which extractor to use (default "regex"). */
-  extractor: ExtractorType;
-  /** How many turns per LLM call when extractor is "llm" (default 10). */
-  llm_batch_size: number;
-  /** Auto-train interval: "1h"|"6h"|"12h"|"24h"|null (default null — disabled). */
-  auto_interval: string | null;
-}
-
-export interface ConsolidationScheduleConfig {
-  /** Local hour for the background consolidation job (default 3). */
-  hour: number;
-  /** Local minute for the background consolidation job (default 0). */
-  minute: number;
-  /** Structured JSONL log path (default ~/.pi/memory/consolidation.log). */
-  log_path: string;
-}
-
-export interface ConsolidationConfig {
-  /** Whether session_shutdown enqueues sessions for offline consolidation. */
-  enabled: boolean;
-  /** Minimum non-empty user turns before enqueue (default 3). */
-  min_user_turns: number;
-  /** MEMORY.md session cap by line count (default 200). */
-  memory_index_max_lines: number;
-  /** MEMORY.md session cap by UTF-8 bytes (default 25KB). */
-  memory_index_max_bytes: number;
-  /** Maximum stage1 rows selected per Phase2 run (default 20). */
-  phase2_top_n: number;
-  /** M4 prune threshold by last usage (default 90 days). */
-  max_unused_days: number;
-  /** Daily OS scheduler defaults. */
-  schedule: ConsolidationScheduleConfig;
-}
-
-export interface MemoryConfig {
-  provider: MemoryProvider;
-  tlmPath: string;
-  socketPath: string;
-  bundleRoot: string;
-  sidecarReadyTimeoutMs: number;
-  queryTimeoutMs: number;
-  clientRequestTimeoutMs: number;
-  sessionsDir: string;
-  memoryMdPaths: string[];
-  trainer: TrainerConfig;
-  consolidation: ConsolidationConfig;
-}
-
-export const defaultTrainerConfig: TrainerConfig = {
-  extractor: "regex",
-  llm_batch_size: 10,
-  auto_interval: null,
-};
-
-export function defaultConsolidationConfig(
-  overrides: Partial<ConsolidationConfig> & {
-    schedule?: Partial<ConsolidationScheduleConfig>;
-  } = {},
-): ConsolidationConfig {
-  const defaultSchedule: ConsolidationScheduleConfig = {
-    hour: 3,
-    minute: 0,
-    log_path: path.join(defaultBundleRoot(), "consolidation.log"),
-  };
-  const { schedule, ...rest } = overrides;
-  return {
-    enabled: true,
-    min_user_turns: 3,
-    memory_index_max_lines: 200,
-    memory_index_max_bytes: 25_600,
-    phase2_top_n: 20,
-    max_unused_days: 90,
-    schedule: { ...defaultSchedule, ...schedule },
-    ...rest,
-  };
-}
-
-export function defaultMemoryConfig(
-  overrides: Partial<MemoryConfig> = {},
-): MemoryConfig {
-  const {
-    trainer: trainerOverrides,
-    consolidation: consolidationOverrides,
-    ...rest
-  } = overrides;
-  return {
-    provider: "local",
-    tlmPath: "tlm",
-    socketPath: defaultSocketPath(),
-    bundleRoot: defaultBundleRoot(),
-    sidecarReadyTimeoutMs: 15_000,
-    queryTimeoutMs: 2_000,
-    clientRequestTimeoutMs: 5_000,
-    sessionsDir: defaultSessionsDir(),
-    memoryMdPaths: [path.join(defaultPiHome(), "MEMORY.md")],
-    trainer: { ...defaultTrainerConfig, ...trainerOverrides },
-    consolidation: defaultConsolidationConfig(consolidationOverrides),
-    ...rest,
-  };
-}
-
-/** Normalize user-supplied paths after JSON/env load. */
-export function normalizeMemoryConfig(
-  raw: Partial<MemoryConfig> & Record<string, unknown>,
-): MemoryConfig {
-  const base = defaultMemoryConfig();
-  const rawTrainer = (raw.trainer ?? {}) as Partial<TrainerConfig>;
-  const rawConsolidation = (raw.consolidation ?? {}) as
-    Partial<ConsolidationConfig> & {
-      schedule?: Partial<ConsolidationScheduleConfig>;
-    };
-  const rawSchedule = (rawConsolidation.schedule ?? {}) as
-    Partial<ConsolidationScheduleConfig>;
-  return {
-    provider: (raw.provider as MemoryProvider) ?? base.provider,
-    tlmPath: expandPath(String(raw.tlmPath ?? base.tlmPath)),
-    socketPath: expandPath(String(raw.socketPath ?? base.socketPath)),
-    bundleRoot: expandPath(String(raw.bundleRoot ?? base.bundleRoot)),
-    sidecarReadyTimeoutMs: Number(
-      raw.sidecarReadyTimeoutMs ?? base.sidecarReadyTimeoutMs,
-    ),
-    queryTimeoutMs: Number(raw.queryTimeoutMs ?? base.queryTimeoutMs),
-    clientRequestTimeoutMs: Number(
-      raw.clientRequestTimeoutMs ?? base.clientRequestTimeoutMs,
-    ),
-    sessionsDir: expandPath(String(raw.sessionsDir ?? base.sessionsDir)),
-    memoryMdPaths: Array.isArray(raw.memoryMdPaths)
-      ? raw.memoryMdPaths.map((p) => expandPath(String(p)))
-      : base.memoryMdPaths,
-    trainer: {
-      extractor: (rawTrainer.extractor as ExtractorType) ?? base.trainer.extractor,
-      llm_batch_size: Number(rawTrainer.llm_batch_size ?? base.trainer.llm_batch_size),
-      auto_interval: rawTrainer.auto_interval !== undefined
-        ? rawTrainer.auto_interval
-        : base.trainer.auto_interval,
-    },
-    consolidation: {
-      enabled: rawConsolidation.enabled ?? base.consolidation.enabled,
-      min_user_turns: Number(
-        rawConsolidation.min_user_turns ?? base.consolidation.min_user_turns,
-      ),
-      memory_index_max_lines: Number(
-        rawConsolidation.memory_index_max_lines ??
-          base.consolidation.memory_index_max_lines,
-      ),
-      memory_index_max_bytes: Number(
-        rawConsolidation.memory_index_max_bytes ??
-          base.consolidation.memory_index_max_bytes,
-      ),
-      phase2_top_n: Number(
-        rawConsolidation.phase2_top_n ?? base.consolidation.phase2_top_n,
-      ),
-      max_unused_days: Number(
-        rawConsolidation.max_unused_days ?? base.consolidation.max_unused_days,
-      ),
-      schedule: {
-        hour: Number(rawSchedule.hour ?? base.consolidation.schedule.hour),
-        minute: Number(rawSchedule.minute ?? base.consolidation.schedule.minute),
-        log_path: expandPath(
-          String(rawSchedule.log_path ?? base.consolidation.schedule.log_path),
-        ),
-      },
-    },
-  };
-}

package/src/consolidation/enqueue.ts

@@ -1,205 +0,0 @@
-import fs from "node:fs";
-
-import type {
-  ConsolidationStatus,
-  JobReport,
-  PendingSession,
-  ConsolidationStatusReport,
-} from "./types.js";
-import { openConsolidationStore, type SqliteDatabase } from "./stage1/store.js";
-
-export interface EnqueueSessionInput {
-  session_id: string;
-  session_file: string;
-  cwd: string;
-  git_root: string | null;
-  project_hash: string | null;
-  parent_session_id: string | null;
-  parent_session_file: string | null;
-  user_turn_count: number;
-  ended_at: string;
-}
-
-export interface EnqueueOptions {
-  enabled?: boolean;
-  minUserTurns?: number;
-  /** Injected sqlite handle for tests. */
-  db?: SqliteDatabase;
-  /** Force replacing existing done/skipped sessions. */
-  force?: boolean;
-  /** Override status for the newly enqueued job (default: pending). */
-  status?: ConsolidationStatus;
-  /** Optional now source for created_at / updated_at. */
-  now?: Date | string;
-  /** Test seam: production defaults to checking session file existence. */
-  checkFileExists?: boolean;
-}
-
-function toIso(value: Date | string | undefined): string {
-  const dt = value ? new Date(value) : new Date();
-  return dt.toISOString();
-}
-
-function isMissingFile(file: string): boolean {
-  return typeof file !== "string" || !file.trim();
-}
-
-function isAllowedStatus(status: string): status is ConsolidationStatus {
-  return status === "pending" || status === "processing" || status === "done" || status === "skipped" ||
-    status === "failed";
-}
-
-export function enqueueSession(
-  dbPath: string,
-  meta: EnqueueSessionInput,
-  opts: EnqueueOptions = {},
-): JobReport {
-  const enabled = opts.enabled !== false;
-  const minUserTurns = opts.minUserTurns ?? 3;
-  const now = toIso(opts.now);
-
-  if (!enabled) {
-    return {
-      session_id: meta.session_id,
-      enqueued: false,
-      action: "skipped",
-      status: "skipped",
-      reason: "consolidation_disabled",
-      now,
-    };
-  }
-
-  if (isMissingFile(meta.session_file)) {
-    return {
-      session_id: meta.session_id,
-      enqueued: false,
-      action: "skipped",
-      status: "skipped",
-      reason: "missing_session_file",
-      now,
-    };
-  }
-
-  if (opts.checkFileExists !== false && !fs.existsSync(meta.session_file)) {
-    return {
-      session_id: meta.session_id,
-      enqueued: false,
-      action: "skipped",
-      status: "skipped",
-      reason: "session_file_not_found",
-      now,
-    };
-  }
-
-  if (meta.user_turn_count < minUserTurns) {
-    return {
-      session_id: meta.session_id,
-      enqueued: false,
-      action: "skipped",
-      status: "skipped",
-      reason: "insufficient_user_turns",
-      now,
-    };
-  }
-
-  const store = openConsolidationStore(dbPath, opts.db);
-  if (!store) {
-    return {
-      session_id: meta.session_id,
-      enqueued: false,
-      action: "skipped",
-      status: "skipped",
-      reason: "missing_sqlite",
-      now,
-    };
-  }
-
-  const incomingStatus: ConsolidationStatus = isAllowedStatus(opts.status ?? "") ? opts.status! : "pending";
-  const existing = store.getPendingSession(meta.session_id);
-  const isFinal = existing?.status === "done" || existing?.status === "skipped";
-
-  if (existing && isFinal && !opts.force) {
-    store.close();
-    return {
-      session_id: meta.session_id,
-      enqueued: false,
-      action: "skipped",
-      status: existing.status,
-      reason: "already_finalized",
-      now,
-    };
-  }
-
-  const existingCreatedAt = existing?.created_at ?? now;
-  const payload: Omit<PendingSession, "created_at" | "updated_at"> & {
-    created_at: string;
-    updated_at: string;
-  } = {
-    session_id: meta.session_id,
-    session_file: meta.session_file,
-    cwd: meta.cwd,
-    git_root: meta.git_root,
-    project_hash: meta.project_hash,
-    parent_session_id: meta.parent_session_id,
-    parent_session_file: meta.parent_session_file,
-    user_turn_count: meta.user_turn_count,
-    ended_at: meta.ended_at,
-    status: incomingStatus,
-    error_message: null,
-    created_at: existingCreatedAt,
-    updated_at: now,
-  };
-
-  try {
-    store.upsertPendingSession(payload);
-    const action = existing ? "updated" : "created";
-    return {
-      session_id: meta.session_id,
-      enqueued: true,
-      action,
-      status: incomingStatus,
-      now,
-    };
-  } finally {
-    store.close();
-  }
-}
-
-export interface ConsolidationStatusQueryOptions {
-  db?: SqliteDatabase;
-}
-
-export function getConsolidationStatus(
-  dbPath: string,
-  opts: ConsolidationStatusQueryOptions = {},
-): ConsolidationStatusReport {
-  const store = openConsolidationStore(dbPath, opts.db);
-  if (!store) {
-    return {
-      pending: 0,
-      processing: 0,
-      done: 0,
-      skipped: 0,
-      failed: 0,
-      stage1Count: 0,
-      lastGeneratedAt: null,
-      lastUpdatedAt: null,
-    };
-  }
-
-  try {
-    const byStatus = store.getConsolidationStatusCounts();
-    return {
-      pending: byStatus.pending,
-      processing: byStatus.processing,
-      done: byStatus.done,
-      skipped: byStatus.skipped,
-      failed: byStatus.failed,
-      stage1Count: store.getStage1Count(),
-      lastGeneratedAt: store.getLastGeneratedAt(),
-      lastUpdatedAt: store.getLastUpdatedAt(),
-    };
-  } finally {
-    store.close();
-  }
-}