@chendpoc/pi-memory 0.1.13 → 0.2.0

package/src/consolidate/mergeEntries.ts

@@ -0,0 +1,22 @@
+import orderBy from "lodash/orderBy.js";
+
+import { entryDedupeKey } from "./entryKey.js";
+import type { ParsedEntry } from "../store/types.js";
+import { MEMORY_SECTIONS } from "../store/types.js";
+
+const SECTION_RANK = new Map(MEMORY_SECTIONS.map((section, index) => [section, index]));
+
+/** Rule-based dedupe before optional LLM merge. Prefer user-authored on conflict. */
+export function dedupeEntries(entries: ParsedEntry[]): ParsedEntry[] {
+  const byKey = new Map<string, ParsedEntry>();
+
+  for (const entry of entries) {
+    const key = entryDedupeKey(entry);
+    const existing = byKey.get(key);
+    if (!existing || entry.userAuthored) {
+      byKey.set(key, entry);
+    }
+  }
+
+  return orderBy([...byKey.values()], [(entry) => SECTION_RANK.get(entry.section) ?? MEMORY_SECTIONS.length]);
+}

package/src/consolidate/mergePrompt.ts

@@ -0,0 +1,46 @@
+import groupBy from "lodash/groupBy.js";
+
+import type { ParsedEntry } from "../store/types.js";
+import { formatEntryLine, formatSectionHeader } from "../store/markdown/format.js";
+import { MEMORY_SECTIONS } from "../store/types.js";
+
+export function formatEntriesForConsolidation(entries: ParsedEntry[]): string {
+  const grouped = groupBy(entries, "section");
+  const lines: string[] = [];
+
+  for (const section of MEMORY_SECTIONS) {
+    const sectionEntries = grouped[section] ?? [];
+    if (sectionEntries.length === 0) continue;
+    lines.push(formatSectionHeader(section), "");
+    for (const entry of sectionEntries) {
+      lines.push(
+        formatEntryLine({
+          id: entry.id,
+          section: entry.section,
+          content: entry.content,
+          userAuthored: entry.userAuthored,
+          timestamp: entry.timestamp,
+        }),
+      );
+    }
+    lines.push("");
+  }
+
+  return lines.join("\n").trim();
+}
+
+export function buildConsolidateMergePrompt(entriesMarkdown: string): string {
+  return `You are consolidating a durable MEMORY.md note file for a coding agent.
+
+Merge the entries below:
+- Remove exact duplicates and near-duplicates (keep the clearer wording).
+- Remove completed or obsolete TODO items.
+- NEVER remove or rewrite lines marked [user] (user-authored).
+- Keep Preferences, Conventions, Findings, and Todos factual and concise.
+- Output ONLY valid markdown with sections: ## Preferences, ## Conventions, ## Findings, ## Todos
+- Each item is a bullet line using the same format as input (with <!-- id:... --> metadata when present).
+
+<memory>
+${entriesMarkdown}
+</memory>`;
+}

package/src/consolidate/mergeWithLlm.ts

@@ -0,0 +1,39 @@
+import keyBy from "lodash/keyBy.js";
+
+import type { LlmClient } from "../adapters/llm/types.js";
+import { parseMemoryMarkdown } from "../store/markdown/parse.js";
+import type { ParsedEntry } from "../store/types.js";
+import { entryDedupeKey } from "./entryKey.js";
+import { buildConsolidateMergePrompt, formatEntriesForConsolidation } from "./mergePrompt.js";
+import { dedupeEntries } from "./mergeEntries.js";
+
+function preserveUserAuthored(original: ParsedEntry[], merged: ParsedEntry[]): ParsedEntry[] {
+  const userByContent = keyBy(
+    original.filter((entry) => entry.userAuthored),
+    entryDedupeKey,
+  );
+
+  return merged.map((entry) => {
+    const user = userByContent[entryDedupeKey(entry)];
+    if (user) {
+      return { ...entry, id: user.id, userAuthored: true, timestamp: user.timestamp };
+    }
+    return entry;
+  });
+}
+
+export async function mergeEntriesWithLlm(
+  entries: ParsedEntry[],
+  llm: LlmClient,
+  signal?: AbortSignal,
+): Promise<ParsedEntry[]> {
+  const deduped = dedupeEntries(entries);
+  if (deduped.length === 0) return deduped;
+
+  const prompt = buildConsolidateMergePrompt(formatEntriesForConsolidation(deduped));
+  const raw = await llm.complete(prompt, signal);
+  const parsed = parseMemoryMarkdown(raw, "consolidate-llm");
+  if (parsed.length === 0) return deduped;
+
+  return preserveUserAuthored(deduped, parsed);
+}

package/src/consolidate/runJob.ts

@@ -0,0 +1,81 @@
+import type { LlmClient } from "../adapters/llm/types.js";
+import { sidecarQueryCache } from "../preflight/queryCache.js";
+import { ensureSidecarRunning } from "../sidecar/sidecarManager.js";
+import { reindex } from "../sidecar/client.js";
+import { resolveSidecarPaths } from "../sidecar/paths.js";
+import type { MemoryStore } from "../store/memoryStore.js";
+
+const NOOP_LLM: LlmClient = {
+  async complete() {
+    throw new Error("LLM unavailable");
+  },
+};
+
+export type ConsolidateJobStats = {
+  entriesBefore: number;
+  entriesAfter: number;
+  overflowBefore: number;
+  overflowAfter: number;
+  indexGeneration?: number;
+};
+
+export type RunConsolidateJobOptions = {
+  store: MemoryStore;
+  agentDir: string;
+  llm?: LlmClient | null;
+  cronFired?: boolean;
+  force?: boolean;
+  reindex?: boolean;
+};
+
+export type RunConsolidateJobResult =
+  | { status: "skipped"; reason: "conditions_not_met" | "already_consolidating" }
+  | { status: "consolidated"; stats: ConsolidateJobStats }
+  | { status: "failed"; error: Error };
+
+export async function runConsolidateJob(
+  opts: RunConsolidateJobOptions,
+): Promise<RunConsolidateJobResult> {
+  const llm = opts.llm ?? NOOP_LLM;
+
+  if (!opts.force && !(await opts.store.shouldConsolidate(undefined, opts.cronFired))) {
+    return { status: "skipped", reason: "conditions_not_met" };
+  }
+
+  const before = await opts.store.getStats();
+
+  try {
+    await opts.store.consolidate(llm);
+
+    let indexGeneration: number | undefined;
+    if (opts.reindex !== false) {
+      indexGeneration = await syncSidecarIndex(opts.agentDir, opts.store);
+    }
+
+    const after = await opts.store.getStats();
+
+    return {
+      status: "consolidated",
+      stats: {
+        entriesBefore: before.entryCount,
+        entriesAfter: after.entryCount,
+        overflowBefore: before.overflowFileCount,
+        overflowAfter: after.overflowFileCount,
+        indexGeneration,
+      },
+    };
+  } catch (error) {
+    return {
+      status: "failed",
+      error: error instanceof Error ? error : new Error(String(error)),
+    };
+  }
+}
+
+async function syncSidecarIndex(agentDir: string, store: MemoryStore): Promise<number> {
+  const sidecar = resolveSidecarPaths(agentDir);
+  await ensureSidecarRunning(sidecar);
+  const result = await reindex(sidecar.socketPath, await store.exportForIndex());
+  sidecarQueryCache.onReindexComplete(agentDir, result.index_generation);
+  return result.index_generation;
+}

package/src/consolidate/scheduler.ts

@@ -0,0 +1,54 @@
+import debounce from "lodash/debounce.js";
+
+import { DEFAULT_CONSOLIDATE_CHECK_INTERVAL_MS, DEFAULT_CONSOLIDATE_DEBOUNCE_MS } from "../constants/timing.js";
+
+import type { LlmClient } from "../adapters/llm/types.js";
+import type { MemoryStore } from "../store/memoryStore.js";
+import { runConsolidateJob } from "./runJob.js";
+
+export type ConsolidateScheduler = {
+  schedule(): void;
+  runNow(opts?: { cronFired?: boolean }): Promise<void>;
+};
+
+export function createConsolidateScheduler(opts: {
+  getStore(): MemoryStore | null;
+  getAgentDir(): string | null;
+  getLlm(): LlmClient | null;
+  debounceMs?: number;
+  onComplete?(): void | Promise<void>;
+}): ConsolidateScheduler {
+  const runNow = async (runOpts: { cronFired?: boolean } = {}): Promise<void> => {
+    const store = opts.getStore();
+    const agentDir = opts.getAgentDir();
+    if (!store || !agentDir) return;
+
+    const result = await runConsolidateJob({
+      store,
+      agentDir,
+      llm: opts.getLlm(),
+      cronFired: runOpts.cronFired ?? false,
+      reindex: true,
+    });
+
+    if (result.status === "consolidated") {
+      await opts.onComplete?.();
+    }
+  };
+
+  const schedule = debounce(() => {
+    void runNow();
+  }, opts.debounceMs ?? DEFAULT_CONSOLIDATE_DEBOUNCE_MS);
+
+  return { schedule, runNow };
+}
+
+/** Periodic check for the 7-day (and other time-based) consolidate conditions. */
+export function startConsolidateInterval(
+  tick: () => void,
+  intervalMs = DEFAULT_CONSOLIDATE_CHECK_INTERVAL_MS,
+): () => void {
+  const timer = setInterval(tick, intervalMs);
+  timer.unref?.();
+  return () => clearInterval(timer);
+}

package/src/constants/chunking.ts

@@ -0,0 +1,8 @@
+/** Max characters per vector chunk before splitting a Memory Entry. */
+export const DEFAULT_CHUNK_MAX_CHARS = 512;
+
+export const MIN_CHUNK_MAX_CHARS = 128;
+export const MAX_CHUNK_MAX_CHARS = 4_000;
+
+/** Values <= 0 disable splitting (one chunk per entry, still section-prefixed). */
+export const CHUNKING_DISABLED_MAX_CHARS = 0;

package/src/constants/env.ts

@@ -0,0 +1,56 @@
+export type EmbedderProvider = "hash" | "openai" | "ollama";
+
+/** process.env keys (PI_MEMORY_*). */
+export const ENV_KEYS = {
+  EMBEDDER: "PI_MEMORY_EMBEDDER",
+  EMBED_DIM: "PI_MEMORY_EMBED_DIM",
+  OPENAI_API_KEY: "PI_MEMORY_OPENAI_API_KEY",
+  OPENAI_EMBED_MODEL: "PI_MEMORY_OPENAI_EMBED_MODEL",
+  OLLAMA_BASE_URL: "PI_MEMORY_OLLAMA_BASE_URL",
+  OLLAMA_EMBED_MODEL: "PI_MEMORY_OLLAMA_EMBED_MODEL",
+  OLLAMA_LLM_BASE_URL: "PI_MEMORY_OLLAMA_LLM_BASE_URL",
+  OLLAMA_LLM_MODEL: "PI_MEMORY_OLLAMA_LLM_MODEL",
+  HELPER_MODEL: "PI_MEMORY_HELPER_MODEL",
+  HELPER_API_KEY: "PI_MEMORY_HELPER_API_KEY",
+  OPENAI_COMPAT_BASE_URL: "PI_MEMORY_OPENAI_COMPAT_BASE_URL",
+  OPENAI_COMPAT_MODEL: "PI_MEMORY_OPENAI_COMPAT_MODEL",
+  OPENAI_COMPAT_API_KEY: "PI_MEMORY_OPENAI_COMPAT_API_KEY",
+  HTTP_TIMEOUT_MS: "PI_MEMORY_HTTP_TIMEOUT_MS",
+  PREFLIGHT_BUDGET_MS: "PI_MEMORY_PREFLIGHT_BUDGET_MS",
+  /** @deprecated Use PI_MEMORY_PREFLIGHT_BUDGET_MS */
+  PREFLIGHT_TIMEOUT_MS: "PI_MEMORY_PREFLIGHT_TIMEOUT_MS",
+  REINDEX_DEBOUNCE_MS: "PI_MEMORY_REINDEX_DEBOUNCE_MS",
+  CONSOLIDATE_DEBOUNCE_MS: "PI_MEMORY_CONSOLIDATE_DEBOUNCE_MS",
+  TOP_K: "PI_MEMORY_TOP_K",
+  MMR_LAMBDA: "PI_MEMORY_MMR_LAMBDA",
+  MIN_RELEVANCE: "PI_MEMORY_MIN_RELEVANCE",
+  CHUNK_MAX_CHARS: "PI_MEMORY_CHUNK_MAX_CHARS",
+  AGENT_DIR: "PI_MEMORY_AGENT_DIR",
+  ENV_FILE: "PI_MEMORY_ENV_FILE",
+  DEBUG: "PI_MEMORY_DEBUG",
+} as const;
+
+/** Fallback OPENAI_API_KEY when PI_MEMORY_OPENAI_API_KEY unset. */
+export const FALLBACK_OPENAI_API_KEY_ENV = "OPENAI_API_KEY";
+
+export const DEFAULT_EMBEDDER: EmbedderProvider = "hash";
+export const DEFAULT_OPENAI_EMBED_MODEL = "text-embedding-3-small";
+export const DEFAULT_OLLAMA_BASE_URL = "http://127.0.0.1:11434";
+export const DEFAULT_OLLAMA_EMBED_MODEL = "nomic-embed-text";
+
+export const DEFAULT_HELPER_PROVIDER = "deepseek";
+export const DEFAULT_HELPER_MODEL = "deepseek-v4-flash";
+export const DEFAULT_HELPER_MODEL_SPEC = `${DEFAULT_HELPER_PROVIDER}/${DEFAULT_HELPER_MODEL}`;
+
+export const DEFAULT_EMBED_DIM_FALLBACK = 1536;
+
+/** Known embedding dimensions; override with PI_MEMORY_EMBED_DIM if needed. */
+export const KNOWN_EMBED_DIMS: Record<string, number> = {
+  "text-embedding-3-small": 1536,
+  "text-embedding-3-large": 3072,
+  "nomic-embed-text": 768,
+  "mxbai-embed-large": 1024,
+  "all-minilm": 384,
+};
+
+export const DEFAULT_HASH_EMBED_DIM = 768;

package/src/constants/index.ts

@@ -0,0 +1,7 @@
+export * from "./paths.js";
+export * from "./memory.js";
+export * from "./timing.js";
+export * from "./env.js";
+export * from "./preflight.js";
+export * from "./retrieval.js";
+export * from "./security.js";

package/src/constants/memory.ts

@@ -0,0 +1,22 @@
+export const DEFAULT_MEMORY_FILE = "MEMORY.md";
+export const MEMORY_GC_FILE = ".memory_gc";
+export const COMPACTION_STATE_FILE = ".memory_compactions.json";
+export const SHUTDOWN_QUEUE_FILE = ".memory_shutdown_queue.jsonl";
+export const SHUTDOWN_PROCESSED_FILE = ".memory_shutdown_processed.json";
+export const AUTO_FILE_PREFIX = "auto-";
+
+export const OVERFLOW_POINTER_RE =
+  /^-\s*\(overflow\)\s*→\s*(auto-[\w-]+\.md)\s*(?:<!--.*?-->)?\s*$/;
+
+export const MEMORY_SECTIONS = ["Preferences", "Conventions", "Findings", "Todos"] as const;
+export type MemorySection = (typeof MEMORY_SECTIONS)[number];
+
+/** MEMORY.md line cap before spilling to auto-*.md. */
+export const DEFAULT_MAX_LINES = 150;
+
+/** Max chars for MEMORY.md fallback injection in preflight. */
+export const DEFAULT_FALLBACK_MAX_CHARS = 8_000;
+
+/** Consolidate triggers (OR). */
+export const CONSOLIDATE_OVERFLOW_FILE_THRESHOLD = 12;
+export const CONSOLIDATE_GC_INTERVAL_DAYS = 7;

package/src/constants/paths.ts

@@ -0,0 +1,26 @@
+/** Default memory data root under ~/.pi/ (MEMORY.md, sidecar, vector DB). */
+export const PI_MEMORY_DATA_SUBDIR = "pi-memory-data";
+/** pi-memory config file under ~/.pi/agent/ (not a generic dotenv at ~/.pi root). */
+export const PI_MEMORY_ENV_FILE_NAME = "pi-memory.env";
+export const PI_LOGS_SUBDIR = "logs";
+
+/** Sidecar IPC + index files (under agent dir). */
+export const SIDECAR_SOCKET_FILE = "memory.sock";
+export const SIDECAR_DB_FILE = "memory.vec.sqlite";
+export const SIDECAR_PID_SUFFIX = ".pid";
+export const SIDECAR_SPAWN_LOCK_FILE = "sidecar.spawn.lock";
+
+/** Consolidate / maintenance scheduler log basenames (under ~/.pi/logs). */
+export const CONSOLIDATE_LOG_FILE = "maintenance.log";
+export const CONSOLIDATE_ERR_LOG_FILE = "maintenance.err.log";
+
+/** OS scheduler identifiers. */
+export const LAUNCHD_LABEL = "com.pi.memory.maintenance";
+export const SCHTASKS_TASK_NAME = "PiMemoryMaintenance";
+
+export const SCHEDULER_TEMPLATE_FILES = {
+  launchd: "com.pi.memory.consolidate.plist.example",
+  crontab: "crontab.example",
+  windowsCmd: "consolidate.cmd.example",
+  windowsSchtasks: "schtasks.example.txt",
+} as const;

package/src/constants/preflight.ts

@@ -0,0 +1,13 @@
+/** Injected private memory block size cap. */
+export const PRIVATE_MEMORY_BODY_BYTE_CAP = 8 * 1024;
+
+export const PRIVATE_MEMORY_TAG = "private_memory";
+export const PRIVATE_MEMORY_OPEN = `<${PRIVATE_MEMORY_TAG}>`;
+export const PRIVATE_MEMORY_CLOSE = `</${PRIVATE_MEMORY_TAG}>`;
+
+/** Regex gate: skip episodic preflight for short generic prompts. */
+export const MEMORY_CUE_RE =
+  /(?:\b(recent|recently|last time|continue|previous|remember|recall|before|earlier)\b|之前|上次|继续|记得|回忆|刚才)/i;
+
+export const PREFLIGHT_SKIP_MIN_LENGTH = 12;
+export const PREFLIGHT_EXTRACT_MIN_LENGTH = 24;

package/src/constants/retrieval.ts

@@ -0,0 +1,10 @@
+export const DEFAULT_TOP_K = 3;
+export const CANDIDATE_POOL_MULTIPLIER = 3;
+export const MMR_LAMBDA = 0.8;
+/** Minimum cosine similarity (0–1) for a chunk to enter the candidate pool. */
+export const DEFAULT_MIN_RELEVANCE = 0.4;
+
+export const MIN_TOP_K = 1;
+export const MAX_TOP_K = 20;
+export const MIN_MMR_LAMBDA = 0;
+export const MAX_MMR_LAMBDA = 1;

package/src/constants/security.ts

@@ -0,0 +1,5 @@
+/** Unix directory mode for agent / socket parent dirs (ignored on Windows). */
+export const SECURE_DIR_MODE = 0o700;
+
+/** Unix socket file mode (ignored on Windows). */
+export const SECURE_FILE_MODE = 0o600;

package/src/constants/timing.ts

@@ -0,0 +1,46 @@
+export const MS_PER_DAY = 24 * 60 * 60 * 1000;
+
+/** HTTP / LLM adapters. */
+export const DEFAULT_HTTP_TIMEOUT_MS = 30_000;
+export const DEFAULT_LLM_COMPAT_TIMEOUT_MS = 120_000;
+
+/** Preflight episodic retrieval (shared deadline for intent + sidecar). */
+export const DEFAULT_PREFLIGHT_BUDGET_MS = 800;
+export const MIN_PREFLIGHT_BUDGET_MS = 250;
+export const MAX_PREFLIGHT_BUDGET_MS = 1_500;
+/** Minimum time reserved for sidecar embed + scan after intent extraction. */
+export const MIN_PREFLIGHT_SIDECAR_RESERVE_MS = 200;
+
+/** @deprecated Use DEFAULT_PREFLIGHT_BUDGET_MS */
+export const DEFAULT_PREFLIGHT_TIMEOUT_MS = DEFAULT_PREFLIGHT_BUDGET_MS;
+
+/** Sidecar query cache (extension process). */
+export const QUERY_CACHE_MAX_ENTRIES = 500;
+
+/** Debounced sidecar reindex after MEMORY writes. */
+export const DEFAULT_REINDEX_DEBOUNCE_MS = 500;
+
+/** Debounced consolidate check after MEMORY writes. */
+export const DEFAULT_CONSOLIDATE_DEBOUNCE_MS = 2_000;
+
+/** Session-alive periodic shouldConsolidate tick (7-day catch-up). */
+export const DEFAULT_CONSOLIDATE_CHECK_INTERVAL_MS = MS_PER_DAY;
+
+/** OS scheduler daily consolidate slot. */
+export const CONSOLIDATE_CRON_HOUR = 3;
+export const CONSOLIDATE_CRON_MINUTE = 0;
+
+/** Sidecar process lifecycle. */
+export const SIDECAR_START_TIMEOUT_MS = 5_000;
+export const SIDECAR_SPAWN_LOCK_STALE_MS = 10_000;
+export const SIDECAR_FORCE_KILL_DELAY_MS = 5_000;
+
+/** Sidecar client RPC timeouts. */
+export const SIDECAR_PING_TIMEOUT_MS = 1_000;
+export const SIDECAR_QUERY_TIMEOUT_MS = 3_000;
+export const SIDECAR_REINDEX_TIMEOUT_MS = 10_000;
+
+/** proper-lockfile on MEMORY.md. */
+export const MEMORY_LOCK_RETRIES = 5;
+export const MEMORY_LOCK_MIN_TIMEOUT_MS = 100;
+export const MEMORY_LOCK_MAX_TIMEOUT_MS = 500;