@chendpoc/pi-memory 0.1.0 → 0.1.12

package/src/service.ts

@@ -1,5 +1,8 @@
+import fs from "node:fs";
 import path from "node:path";
 import type { MemoryConfig } from "./config.js";
+import { invalidateMemoryCaches } from "./cache/memoryCaches.js";
+import { LocalGraphQuerier } from "./local/graphQuery.js";
 import { currentBundleReadable } from "./sidecar/bundle.js";
 import { SidecarClient } from "./sidecar/client.js";
 import { SidecarProcess } from "./sidecar/process.js";
@@ -16,6 +19,7 @@ import type {
 export interface MemoryServiceStatus {
   status: ServiceStatus;
   reason?: string;
+  mode?: "sidecar" | "local_graph";
   health?: HealthPayload | null;
 }
 
@@ -26,17 +30,25 @@ export interface QueryBatchResult {
 }
 
 /**
- * Mode B local memory: spawn tlm sidecar, query via Unix socket.
- * No Cloud puller — bundle must exist under bundleRoot/current.
+ * Local memory service with two query backends:
+ * 1. tlm sidecar (Unix socket) — when tlm binary is available
+ * 2. LocalGraphQuerier — direct graph.json query when tlm is missing
+ *
+ * Both require a bundle at bundleRoot/current.
  */
 export class MemoryService {
   private serviceStatus: ServiceStatus = "disabled";
   private reason = "";
+  private mode: "sidecar" | "local_graph" | null = null;
   private process: SidecarProcess | null = null;
   private client: SidecarClient | null = null;
+  private localQuerier: LocalGraphQuerier | null = null;
   private abort: AbortController | null = null;
   private scheduler: TrainScheduler | null = null;
   private sessionIndex: SessionIndex | null = null;
+  private bundleMtimes: { graph: number; manifest: number } | null = null;
+  private lastBundleCheckMs = 0;
+  private static readonly BUNDLE_CHECK_INTERVAL_MS = 5_000;
 
   constructor(private cfg: MemoryConfig) {}
 
@@ -52,6 +64,7 @@ export class MemoryService {
     return {
       status: this.serviceStatus,
       reason: this.reason || undefined,
+      mode: this.mode ?? undefined,
     };
   }
 
@@ -79,36 +92,50 @@ export class MemoryService {
 
     this.serviceStatus = "initializing";
     this.abort = new AbortController();
-    this.process = new SidecarProcess(this.cfg);
 
+    if (await this.trySidecar()) return;
+
+    if (this.tryLocalGraph()) return;
+
+    this.serviceStatus = "unavailable";
+    this.reason = "no_query_backend";
+  }
+
+  private async trySidecar(): Promise<boolean> {
+    this.process = new SidecarProcess(this.cfg);
     try {
       await this.process.resolveBinary();
     } catch {
-      this.serviceStatus = "unavailable";
-      this.reason = "tlm_binary_missing";
-      return;
+      this.process = null;
+      return false;
     }
 
     try {
       await this.process.spawn();
-      await this.process.waitReady(this.abort.signal);
+      await this.process.waitReady(this.abort!.signal);
       this.client = this.process.getClient();
       this.serviceStatus = "ready";
+      this.mode = "sidecar";
       this.reason = "";
-    } catch (err) {
-      this.serviceStatus = "unavailable";
-      this.reason =
-        err instanceof Error ? err.message : "sidecar_startup_failed";
+      return true;
+    } catch {
       await this.process.stop();
       this.process = null;
       this.client = null;
+      return false;
     }
   }
 
-  /**
-   * Start interval-based auto-training. If already running, stops and restarts.
-   * Uses config.trainer.auto_interval.
-   */
+  private tryLocalGraph(): boolean {
+    const querier = new LocalGraphQuerier(this.cfg.bundleRoot);
+    if (!querier.load()) return false;
+    this.localQuerier = querier;
+    this.serviceStatus = "ready";
+    this.mode = "local_graph";
+    this.reason = "";
+    return true;
+  }
+
   startAutoTrainer(logger?: (log: SchedulerLog) => void): void {
     this.scheduler?.stop();
     this.scheduler = createTrainScheduler(
@@ -118,17 +145,71 @@ export class MemoryService {
           sessionsDir: this.cfg.sessionsDir,
           bundleRoot: this.cfg.bundleRoot,
         },
+        onSuccess: () => { void this.notifyBundleUpdated(); },
       },
       logger,
     );
   }
 
   /**
-   * Trigger incremental session index build in the background (non-blocking).
-   * Opens (or creates) the SQLite FTS5 DB at ~/.pi/memory/sessions.db.
+   * Check if the on-disk bundle has changed and hot-reload if needed.
+   * Debounced to at most once per BUNDLE_CHECK_INTERVAL_MS to avoid excess stat calls.
+   * For local_graph, reloads LocalGraphQuerier in-process.
+   * For sidecar, issues a /bundle/reload request.
+   * Invalidates all memory caches on successful reload.
+   */
+  async ensureFreshBundle(): Promise<void> {
+    if (this.serviceStatus !== "ready") return;
+    const now = Date.now();
+    if (now - this.lastBundleCheckMs < MemoryService.BUNDLE_CHECK_INTERVAL_MS) return;
+    this.lastBundleCheckMs = now;
+
+    if (this.mode === "local_graph" && this.localQuerier) {
+      const reloaded = this.localQuerier.reloadIfStale();
+      if (reloaded) invalidateMemoryCaches();
+      return;
+    }
+
+    if (this.mode === "sidecar" && this.client) {
+      const graphPath = path.join(this.cfg.bundleRoot, "current", "graph.json");
+      const manifestPath = path.join(this.cfg.bundleRoot, "current", "manifest.json");
+      try {
+        const graphMtime = fs.statSync(graphPath).mtimeMs;
+        const manifestMtime = fs.statSync(manifestPath).mtimeMs;
+        if (!this.bundleMtimes) {
+          this.bundleMtimes = { graph: graphMtime, manifest: manifestMtime };
+          return;
+        }
+        const changed =
+          graphMtime !== this.bundleMtimes.graph ||
+          manifestMtime !== this.bundleMtimes.manifest;
+        if (!changed) return;
+        this.bundleMtimes = { graph: graphMtime, manifest: manifestMtime };
+        await this.client.reload();
+        invalidateMemoryCaches();
+      } catch {
+        /* stat or reload failure — continue with current bundle */
+      }
+    }
+  }
+
+  /**
+   * Force an immediate bundle freshness check (bypasses the debounce).
+   * Called by the auto-trainer scheduler after a successful train run.
    */
+  async notifyBundleUpdated(): Promise<void> {
+    this.lastBundleCheckMs = 0;
+    await this.ensureFreshBundle();
+  }
+
   startSessionIndex(): void {
-    const dbPath = path.join(this.cfg.bundleRoot, "sessions.db");
+    const dbDir = this.cfg.bundleRoot;
+    try {
+      fs.mkdirSync(dbDir, { recursive: true });
+    } catch {
+      return;
+    }
+    const dbPath = path.join(dbDir, "sessions.db");
     const idx = openSessionIndex(dbPath);
     if (!idx) return;
     this.sessionIndex = idx;
@@ -148,6 +229,8 @@ export class MemoryService {
     await this.process?.stop();
     this.process = null;
     this.client = null;
+    this.localQuerier = null;
+    this.mode = null;
     if (this.cfg.provider === "disabled") {
       this.serviceStatus = "disabled";
     } else {
@@ -181,22 +264,43 @@ export class MemoryService {
     errorClass: ErrorClass;
     transportError?: Error;
   }> {
-    if (this.serviceStatus !== "ready" || !this.client) {
+    if (this.serviceStatus !== "ready") {
       return { env: null, errorClass: "unavailable" };
     }
-    const timeout = AbortSignal.timeout(this.cfg.queryTimeoutMs);
-    const combined = signal
-      ? AbortSignal.any([signal, timeout])
-      : timeout;
-    return this.client.query(intent, combined);
+    await this.ensureFreshBundle();
+
+    if (this.client && this.mode === "sidecar") {
+      const timeout = AbortSignal.timeout(this.cfg.queryTimeoutMs);
+      const combined = signal
+        ? AbortSignal.any([signal, timeout])
+        : timeout;
+      return this.client.query(intent, combined);
+    }
+
+    if (this.localQuerier && this.mode === "local_graph") {
+      return this.localQuerier.query(intent);
+    }
+
+    return { env: null, errorClass: "unavailable" };
   }
 
   async health(): Promise<HealthPayload | null> {
-    if (!this.client) return null;
-    try {
-      return await this.client.health();
-    } catch {
-      return null;
+    if (this.client) {
+      try {
+        return await this.client.health();
+      } catch {
+        return null;
+      }
+    }
+    if (this.localQuerier) {
+      return {
+        ready: true,
+        compatibility: "local_graph",
+        protocol_version: 1,
+        uptime_secs: 0,
+        status_message: `local graph query (${this.localQuerier.isLoaded() ? "loaded" : "not loaded"})`,
+      };
     }
+    return null;
   }
 }

package/src/settings.ts

@@ -0,0 +1,126 @@
+import fs from "node:fs";
+import path from "node:path";
+
+import { getAgentDir } from "@earendil-works/pi-coding-agent";
+
+import {
+  defaultMemoryConfig,
+  normalizeMemoryConfig,
+  type ExtractorType,
+  type MemoryConfig,
+  type MemoryProvider,
+  type TrainerConfig,
+} from "./config.js";
+import type { OllamaConfig } from "./adapters/ollamaClient.js";
+import type { OpenAICompatConfig } from "./adapters/openaiCompatClient.js";
+
+export interface MemorySettingsFile {
+  provider?: MemoryProvider;
+  tlmPath?: string;
+  socketPath?: string;
+  bundleRoot?: string;
+  sidecarReadyTimeoutMs?: number;
+  queryTimeoutMs?: number;
+  clientRequestTimeoutMs?: number;
+  sessionsDir?: string;
+  memoryMdPaths?: string[];
+  /** LLM for intent detection, rerank, and LLM extraction. Overrides default helper model. */
+  helperModel?: string;
+  /** Ollama local API config for helper / trainer / rerank. */
+  ollama?: Partial<OllamaConfig>;
+  /** OpenAI-compatible endpoint config (vLLM, SGLang, LM Studio, etc.) */
+  vllm?: Partial<OpenAICompatConfig>;
+  trainer?: Partial<TrainerConfig>;
+}
+
+export interface LoadedMemorySettings {
+  config: MemoryConfig;
+  helperModel: string | undefined;
+  ollama: OllamaConfig | null;
+  vllm: OpenAICompatConfig | null;
+  configPath: string;
+}
+
+/** Default path: ~/.pi/agent/memory.json */
+export function defaultMemoryConfigPath(): string {
+  return path.join(getAgentDir(), "memory.json");
+}
+
+function readMemorySettingsFile(configPath = defaultMemoryConfigPath()): MemorySettingsFile {
+  try {
+    const raw = fs.readFileSync(configPath, "utf8");
+    const parsed = JSON.parse(raw) as unknown;
+    if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+      return {};
+    }
+    return parsed as MemorySettingsFile;
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code === "ENOENT") {
+      return {};
+    }
+    throw new Error(
+      `Failed to load pi-memory config from ${configPath}: ${err instanceof Error ? err.message : String(err)}`,
+    );
+  }
+}
+
+/** Load pi-memory config from ~/.pi/agent/memory.json. */
+export function loadMemorySettings(
+  overrides: Partial<MemoryConfig> = {},
+  configPath = defaultMemoryConfigPath(),
+): LoadedMemorySettings {
+  const fileSettings = readMemorySettingsFile(configPath);
+  const { helperModel, ollama, vllm, trainer, ...configFields } = fileSettings;
+
+  const config = normalizeMemoryConfig({
+    ...configFields,
+    ...(trainer ? { trainer } : {}),
+    ...overrides,
+  } as Partial<MemoryConfig> & Record<string, unknown>);
+
+  let resolvedOllama: OllamaConfig | null = null;
+  if (ollama && ollama.model) {
+    resolvedOllama = {
+      baseUrl: ollama.baseUrl?.trim() || "http://localhost:11434",
+      model: ollama.model.trim(),
+    };
+  }
+
+  let resolvedVllm: OpenAICompatConfig | null = null;
+  if (vllm && vllm.model) {
+    resolvedVllm = {
+      baseUrl: vllm.baseUrl?.trim() || "http://localhost:8000",
+      model: vllm.model.trim(),
+      apiKey: vllm.apiKey?.trim() || undefined,
+    };
+  }
+
+  return {
+    config,
+    helperModel: helperModel?.trim() || undefined,
+    ollama: resolvedOllama,
+    vllm: resolvedVllm,
+    configPath,
+  };
+}
+
+/** Convenience alias when only MemoryConfig is needed. */
+export function loadMemoryConfig(
+  overrides: Partial<MemoryConfig> = {},
+  configPath = defaultMemoryConfigPath(),
+): MemoryConfig {
+  return loadMemorySettings(overrides, configPath).config;
+}
+
+export function resolveHelperModelSpec(
+  flagValue: string | boolean | undefined,
+  settingsHelperModel: string | undefined,
+): string | undefined {
+  if (typeof flagValue === "string" && flagValue.trim()) {
+    return flagValue.trim();
+  }
+  return settingsHelperModel;
+}
+
+export { defaultMemoryConfig };
+export type { ExtractorType, MemoryProvider, TrainerConfig };

package/src/sidecar/process.ts

@@ -27,8 +27,9 @@ export class SidecarProcess {
       await access(bin, constants.X_OK);
       return bin;
     }
-    // PATH lookup: spawn with shell on unix
-    return bin;
+    const resolved = await which(bin);
+    if (!resolved) throw new Error(`${bin} not found in PATH`);
+    return resolved;
   }
 
   async spawn(): Promise<void> {
@@ -57,8 +58,8 @@ export class SidecarProcess {
       detached: process.platform !== "win32",
     });
 
-    this.child.on("error", (err) => {
-      console.error("[pi-memory] sidecar spawn error:", err.message);
+    this.child.on("error", () => {
+      /* handled by waitReady timeout */
     });
 
     this.child.unref?.();
@@ -126,6 +127,20 @@ export class SidecarProcess {
   }
 }
 
+async function which(cmd: string): Promise<string | null> {
+  const { execFileSync } = await import("node:child_process");
+  try {
+    const result = execFileSync(
+      process.platform === "win32" ? "where" : "which",
+      [cmd],
+      { encoding: "utf8", timeout: 3_000 },
+    ).trim();
+    return result.split("\n")[0]?.trim() || null;
+  } catch {
+    return null;
+  }
+}
+
 function sleep(ms: number, signal?: AbortSignal): Promise<void> {
   return new Promise((resolve, reject) => {
     if (signal?.aborted) {

package/src/tools/memoryRecall.ts

@@ -55,7 +55,7 @@ export class MemoryRecallTool {
     };
   }
 
-  async run(argsJson: string, signal?: AbortSignal): Promise<ToolResult> {
+  async run(argsJson: string, signal?: AbortSignal, onProgress?: (msg: string) => void): Promise<ToolResult> {
     const args = parseArgs(argsJson);
     if ("error" in args) {
       return { content: args.error, isError: true };
@@ -67,29 +67,53 @@ export class MemoryRecallTool {
       return { content: validation, isError: true };
     }
 
-    return this.runIntent(intent, signal);
+    return this.runIntent(intent, signal, onProgress);
   }
 
-  async runIntent(intent: QueryIntent, signal?: AbortSignal): Promise<ToolResult> {
+  async runIntent(intent: QueryIntent, signal?: AbortSignal, onProgress?: (msg: string) => void): Promise<ToolResult> {
     if (this.service.status() !== "ready") {
       return this.fallbackResult(intent, "service_unavailable", "fallback");
     }
 
-    let result = await this.service.query(intent, signal);
+    // Emit a progress message if the sidecar takes longer than 500 ms.
+    let progressTimer: ReturnType<typeof setTimeout> | null = null;
+    if (onProgress) {
+      progressTimer = setTimeout(() => {
+        onProgress("Querying episodic memory…");
+      }, 500);
+    }
+
+    let result;
+    try {
+      result = await this.service.query(intent, signal);
+    } finally {
+      if (progressTimer != null) clearTimeout(progressTimer);
+    }
+
     if (result.transportError || result.errorClass === "unavailable") {
       return this.fallbackResult(intent, "service_unavailable", "fallback");
     }
 
     if (result.errorClass === "retryable") {
-      await sleep(500, signal);
-      result = await this.service.query(intent, signal);
+      let retryTimer: ReturnType<typeof setTimeout> | null = null;
+      if (onProgress) {
+        retryTimer = setTimeout(() => onProgress("Retrying memory query…"), 500);
+      }
+      let retryResult;
+      try {
+        await sleep(500, signal);
+        retryResult = await this.service.query(intent, signal);
+      } finally {
+        if (retryTimer != null) clearTimeout(retryTimer);
+      }
       if (
-        result.transportError ||
-        result.errorClass === "unavailable" ||
-        result.errorClass === "retryable"
+        retryResult.transportError ||
+        retryResult.errorClass === "unavailable" ||
+        retryResult.errorClass === "retryable"
       ) {
         return this.fallbackResult(intent, "retryable_failed", "fallback_after_retry");
       }
+      result = retryResult;
     }
 
     if (result.errorClass === "permanent") {

package/src/trainer/scheduler.ts

@@ -5,6 +5,8 @@ export interface SchedulerConfig {
   interval: string | null;
   /** Passed through to trainBundle. */
   trainConfig?: Omit<TrainBundleConfig, "full" | "dryRun">;
+  /** Called after a successful train tick (no error). Use to notify MemoryService. */
+  onSuccess?: () => void;
 }
 
 export interface TrainScheduler {
@@ -69,6 +71,7 @@ export function createTrainScheduler(
         full: false,
         dryRun: false,
       });
+      config.onSuccess?.();
     } catch (err) {
       error = err instanceof Error ? err.message : String(err);
     }