@chendpoc/pi-memory 0.1.0 → 0.1.11

package/src/settings.ts

@@ -0,0 +1,126 @@
+import fs from "node:fs";
+import path from "node:path";
+
+import { getAgentDir } from "@earendil-works/pi-coding-agent";
+
+import {
+  defaultMemoryConfig,
+  normalizeMemoryConfig,
+  type ExtractorType,
+  type MemoryConfig,
+  type MemoryProvider,
+  type TrainerConfig,
+} from "./config.js";
+import type { OllamaConfig } from "./adapters/ollamaClient.js";
+import type { OpenAICompatConfig } from "./adapters/openaiCompatClient.js";
+
+export interface MemorySettingsFile {
+  provider?: MemoryProvider;
+  tlmPath?: string;
+  socketPath?: string;
+  bundleRoot?: string;
+  sidecarReadyTimeoutMs?: number;
+  queryTimeoutMs?: number;
+  clientRequestTimeoutMs?: number;
+  sessionsDir?: string;
+  memoryMdPaths?: string[];
+  /** LLM for intent detection, rerank, and LLM extraction. Overrides default helper model. */
+  helperModel?: string;
+  /** Ollama local API config for helper / trainer / rerank. */
+  ollama?: Partial<OllamaConfig>;
+  /** OpenAI-compatible endpoint config (vLLM, SGLang, LM Studio, etc.) */
+  vllm?: Partial<OpenAICompatConfig>;
+  trainer?: Partial<TrainerConfig>;
+}
+
+export interface LoadedMemorySettings {
+  config: MemoryConfig;
+  helperModel: string | undefined;
+  ollama: OllamaConfig | null;
+  vllm: OpenAICompatConfig | null;
+  configPath: string;
+}
+
+/** Default path: ~/.pi/agent/memory.json */
+export function defaultMemoryConfigPath(): string {
+  return path.join(getAgentDir(), "memory.json");
+}
+
+function readMemorySettingsFile(configPath = defaultMemoryConfigPath()): MemorySettingsFile {
+  try {
+    const raw = fs.readFileSync(configPath, "utf8");
+    const parsed = JSON.parse(raw) as unknown;
+    if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+      return {};
+    }
+    return parsed as MemorySettingsFile;
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code === "ENOENT") {
+      return {};
+    }
+    throw new Error(
+      `Failed to load pi-memory config from ${configPath}: ${err instanceof Error ? err.message : String(err)}`,
+    );
+  }
+}
+
+/** Load pi-memory config from ~/.pi/agent/memory.json. */
+export function loadMemorySettings(
+  overrides: Partial<MemoryConfig> = {},
+  configPath = defaultMemoryConfigPath(),
+): LoadedMemorySettings {
+  const fileSettings = readMemorySettingsFile(configPath);
+  const { helperModel, ollama, vllm, trainer, ...configFields } = fileSettings;
+
+  const config = normalizeMemoryConfig({
+    ...configFields,
+    ...(trainer ? { trainer } : {}),
+    ...overrides,
+  } as Partial<MemoryConfig> & Record<string, unknown>);
+
+  let resolvedOllama: OllamaConfig | null = null;
+  if (ollama && ollama.model) {
+    resolvedOllama = {
+      baseUrl: ollama.baseUrl?.trim() || "http://localhost:11434",
+      model: ollama.model.trim(),
+    };
+  }
+
+  let resolvedVllm: OpenAICompatConfig | null = null;
+  if (vllm && vllm.model) {
+    resolvedVllm = {
+      baseUrl: vllm.baseUrl?.trim() || "http://localhost:8000",
+      model: vllm.model.trim(),
+      apiKey: vllm.apiKey?.trim() || undefined,
+    };
+  }
+
+  return {
+    config,
+    helperModel: helperModel?.trim() || undefined,
+    ollama: resolvedOllama,
+    vllm: resolvedVllm,
+    configPath,
+  };
+}
+
+/** Convenience alias when only MemoryConfig is needed. */
+export function loadMemoryConfig(
+  overrides: Partial<MemoryConfig> = {},
+  configPath = defaultMemoryConfigPath(),
+): MemoryConfig {
+  return loadMemorySettings(overrides, configPath).config;
+}
+
+export function resolveHelperModelSpec(
+  flagValue: string | boolean | undefined,
+  settingsHelperModel: string | undefined,
+): string | undefined {
+  if (typeof flagValue === "string" && flagValue.trim()) {
+    return flagValue.trim();
+  }
+  return settingsHelperModel;
+}
+
+export { defaultMemoryConfig };
+export type { ExtractorType, MemoryProvider, TrainerConfig };

package/src/sidecar/process.ts

@@ -27,8 +27,9 @@ export class SidecarProcess {
       await access(bin, constants.X_OK);
       return bin;
     }
-    // PATH lookup: spawn with shell on unix
-    return bin;
+    const resolved = await which(bin);
+    if (!resolved) throw new Error(`${bin} not found in PATH`);
+    return resolved;
   }
 
   async spawn(): Promise<void> {
@@ -57,8 +58,8 @@ export class SidecarProcess {
       detached: process.platform !== "win32",
     });
 
-    this.child.on("error", (err) => {
-      console.error("[pi-memory] sidecar spawn error:", err.message);
+    this.child.on("error", () => {
+      /* handled by waitReady timeout */
     });
 
     this.child.unref?.();
@@ -126,6 +127,20 @@ export class SidecarProcess {
   }
 }
 
+async function which(cmd: string): Promise<string | null> {
+  const { execFileSync } = await import("node:child_process");
+  try {
+    const result = execFileSync(
+      process.platform === "win32" ? "where" : "which",
+      [cmd],
+      { encoding: "utf8", timeout: 3_000 },
+    ).trim();
+    return result.split("\n")[0]?.trim() || null;
+  } catch {
+    return null;
+  }
+}
+
 function sleep(ms: number, signal?: AbortSignal): Promise<void> {
   return new Promise((resolve, reject) => {
     if (signal?.aborted) {

package/src/trainer/sessionLoader.ts

@@ -29,6 +29,21 @@ interface PiSessionFile {
   messages?: PiSessionMessage[];
 }
 
+interface JsonlSessionHeader {
+  type: "session";
+  id?: string;
+  timestamp?: string;
+  title?: string;
+}
+
+interface JsonlMessageLine {
+  type: "message";
+  message?: {
+    role?: string;
+    content?: unknown;
+  };
+}
+
 function messageText(content: unknown): string {
   if (typeof content === "string") return content;
   if (!Array.isArray(content)) return "";
@@ -53,72 +68,143 @@ export interface SessionLoaderOptions {
 }
 
 /**
- * Scan session JSON files, parse Pi session format, optionally filter by
- * modified-after timestamp for incremental training.
+ * Collect all session files recursively (supports project subdirectories).
+ * Returns both .json and .jsonl files.
  */
-export async function loadSessions(
-  opts: SessionLoaderOptions,
-): Promise<LoadedSession[]> {
-  const { sessionsDir, modifiedAfter } = opts;
-  if (!sessionsDir.trim()) return [];
-
-  let entries: string[];
+async function collectSessionFiles(dir: string): Promise<string[]> {
+  let names: string[];
   try {
-    entries = await fs.readdir(sessionsDir);
+    names = await fs.readdir(dir);
   } catch {
     return [];
   }
 
-  const sessions: LoadedSession[] = [];
+  const files: string[] = [];
+  for (const name of names) {
+    const full = path.join(dir, name);
+    let st;
+    try { st = await fs.stat(full); } catch { continue; }
+    if (st.isDirectory()) {
+      const sub = await collectSessionFiles(full);
+      files.push(...sub);
+    } else if (st.isFile() && (name.endsWith(".json") || name.endsWith(".jsonl"))) {
+      files.push(full);
+    }
+  }
+  return files;
+}
 
-  for (const name of entries) {
-    if (!name.endsWith(".json")) continue;
-    const filePath = path.join(sessionsDir, name);
+function parseJsonlSession(raw: string, filePath: string): LoadedSession | null {
+  const lines = raw.split("\n").filter((l) => l.trim());
+  if (lines.length === 0) return null;
 
-    let st: Awaited<ReturnType<typeof fs.stat>>;
+  let header: JsonlSessionHeader | null = null;
+  const turns: SessionTurn[] = [];
+  let turnIndex = 0;
+
+  for (const line of lines) {
+    let obj: Record<string, unknown>;
     try {
-      st = await fs.stat(filePath);
+      obj = JSON.parse(line) as Record<string, unknown>;
     } catch {
       continue;
     }
-    if (!st.isFile()) continue;
 
-    if (modifiedAfter && st.mtime <= modifiedAfter) {
+    if (obj.type === "session" && !header) {
+      header = obj as unknown as JsonlSessionHeader;
       continue;
     }
 
-    let session: PiSessionFile;
+    if (obj.type === "message") {
+      const msg = (obj as unknown as JsonlMessageLine).message;
+      if (!msg?.role || !msg.content) continue;
+      if (msg.role !== "user" && msg.role !== "assistant") continue;
+      const text = messageText(msg.content);
+      if (!text.trim()) continue;
+      turns.push({ role: msg.role, content: text, turnIndex: turnIndex++ });
+    }
+  }
+
+  if (turns.length === 0) return null;
+
+  return {
+    id: header?.id ?? path.basename(filePath, path.extname(filePath)),
+    title: header?.title ?? "",
+    createdAt: header?.timestamp ?? "",
+    filePath,
+    modifiedAt: new Date(),
+    turns,
+  };
+}
+
+function parseJsonSession(raw: string, filePath: string): LoadedSession | null {
+  let session: PiSessionFile;
+  try {
+    session = JSON.parse(raw) as PiSessionFile;
+  } catch {
+    return null;
+  }
+
+  if (!session.messages || session.messages.length === 0) return null;
+
+  const turns: SessionTurn[] = [];
+  for (let i = 0; i < session.messages.length; i++) {
+    const msg = session.messages[i]!;
+    const text = messageText(msg.content);
+    if (!text.trim()) continue;
+    turns.push({ role: msg.role ?? "unknown", content: text, turnIndex: i });
+  }
+
+  if (turns.length === 0) return null;
+
+  return {
+    id: session.id ?? path.basename(filePath, ".json"),
+    title: session.title ?? "",
+    createdAt: session.created_at ?? "",
+    filePath,
+    modifiedAt: new Date(),
+    turns,
+  };
+}
+
+/**
+ * Scan session files (JSON + JSONL, recursive subdirectories), parse Pi session
+ * format, optionally filter by modified-after timestamp for incremental training.
+ */
+export async function loadSessions(
+  opts: SessionLoaderOptions,
+): Promise<LoadedSession[]> {
+  const { sessionsDir, modifiedAfter } = opts;
+  if (!sessionsDir.trim()) return [];
+
+  const filePaths = await collectSessionFiles(sessionsDir);
+  const sessions: LoadedSession[] = [];
+
+  for (const filePath of filePaths) {
+    let st: Awaited<ReturnType<typeof fs.stat>>;
     try {
-      const raw = await fs.readFile(filePath, "utf8");
-      session = JSON.parse(raw) as PiSessionFile;
+      st = await fs.stat(filePath);
     } catch {
       continue;
     }
+    if (!st.isFile()) continue;
+    if (modifiedAfter && st.mtime <= modifiedAfter) continue;
 
-    if (!session.messages || session.messages.length === 0) continue;
-
-    const turns: SessionTurn[] = [];
-    for (let i = 0; i < session.messages.length; i++) {
-      const msg = session.messages[i]!;
-      const text = messageText(msg.content);
-      if (!text.trim()) continue;
-      turns.push({
-        role: msg.role ?? "unknown",
-        content: text,
-        turnIndex: i,
-      });
+    let raw: string;
+    try {
+      raw = await fs.readFile(filePath, "utf8");
+    } catch {
+      continue;
     }
 
-    if (turns.length === 0) continue;
+    const isJsonl = filePath.endsWith(".jsonl");
+    const parsed = isJsonl
+      ? parseJsonlSession(raw, filePath)
+      : parseJsonSession(raw, filePath);
 
-    sessions.push({
-      id: session.id ?? path.basename(name, ".json"),
-      title: session.title ?? "",
-      createdAt: session.created_at ?? "",
-      filePath,
-      modifiedAt: st.mtime,
-      turns,
-    });
+    if (!parsed) continue;
+    parsed.modifiedAt = st.mtime;
+    sessions.push(parsed);
   }
 
   sessions.sort((a, b) => a.modifiedAt.getTime() - b.modifiedAt.getTime());