@chendpoc/pi-memory 0.1.0 → 0.1.11

package/src/adapters/openaiCompatClient.ts

@@ -0,0 +1,155 @@
+import http from "node:http";
+import https from "node:https";
+
+import type { LLMClient } from "../trainer/llmExtractor.js";
+import type { MemoryHelperLLM } from "../preflight/detectIntents.js";
+import type { CompileMemoryIntentsResult } from "../preflight/detectIntents.js";
+import {
+  COMPILE_MEMORY_INTENTS_PARAMETERS,
+  MEMORY_HELPER_TOOL_NAME,
+} from "../preflight/detectIntents.js";
+
+export interface OpenAICompatConfig {
+  baseUrl: string;
+  model: string;
+  apiKey?: string;
+}
+
+interface ChatMessage {
+  role: "system" | "user" | "assistant";
+  content: string;
+}
+
+interface ChatCompletionRequest {
+  model: string;
+  messages: ChatMessage[];
+  max_tokens?: number;
+  tools?: Array<{
+    type: "function";
+    function: { name: string; description: string; parameters: Record<string, unknown> };
+  }>;
+  tool_choice?: { type: "function"; function: { name: string } };
+}
+
+interface ChatCompletionResponse {
+  choices?: Array<{
+    message?: {
+      content?: string | null;
+      tool_calls?: Array<{
+        function?: { name?: string; arguments?: string };
+      }>;
+    };
+  }>;
+  error?: { message?: string };
+}
+
+async function postJSON<T>(url: URL, body: unknown, apiKey?: string): Promise<T> {
+  const mod = url.protocol === "https:" ? https : http;
+  const payload = JSON.stringify(body);
+
+  return new Promise((resolve, reject) => {
+    const headers: Record<string, string> = {
+      "Content-Type": "application/json",
+      "Content-Length": String(Buffer.byteLength(payload)),
+    };
+    if (apiKey) headers["Authorization"] = `Bearer ${apiKey}`;
+
+    const req = mod.request(url, { method: "POST", headers, timeout: 120_000 }, (res) => {
+      const chunks: Buffer[] = [];
+      res.on("data", (c) => chunks.push(c));
+      res.on("end", () => {
+        const text = Buffer.concat(chunks).toString("utf8");
+        try { resolve(JSON.parse(text) as T); }
+        catch { reject(new Error(`OpenAI-compat: invalid JSON: ${text.slice(0, 200)}`)); }
+      });
+    });
+    req.on("error", (err) => reject(new Error(`OpenAI-compat: ${err.message}`)));
+    req.on("timeout", () => { req.destroy(); reject(new Error("OpenAI-compat: timeout")); });
+    req.write(payload);
+    req.end();
+  });
+}
+
+export async function openaiCompatHealthCheck(baseUrl: string): Promise<boolean> {
+  try {
+    const url = new URL("/v1/models", baseUrl);
+    const mod = url.protocol === "https:" ? https : http;
+    return new Promise((resolve) => {
+      const req = mod.get(url, { timeout: 3_000 }, (res) => {
+        res.resume();
+        resolve(res.statusCode === 200);
+      });
+      req.on("error", () => resolve(false));
+      req.on("timeout", () => { req.destroy(); resolve(false); });
+    });
+  } catch {
+    return false;
+  }
+}
+
+export function createOpenAICompatLLMClient(cfg: OpenAICompatConfig): LLMClient {
+  return {
+    async complete(prompt: string): Promise<string> {
+      const url = new URL("/v1/chat/completions", cfg.baseUrl);
+      const body: ChatCompletionRequest = {
+        model: cfg.model,
+        messages: [{ role: "user", content: prompt }],
+        max_tokens: 8192,
+      };
+      const resp = await postJSON<ChatCompletionResponse>(url, body, cfg.apiKey);
+      if (resp.error?.message) throw new Error(`OpenAI-compat: ${resp.error.message}`);
+      const text = resp.choices?.[0]?.message?.content?.trim();
+      if (!text) throw new Error("OpenAI-compat: empty response");
+      return text;
+    },
+  };
+}
+
+export function createOpenAICompatMemoryHelper(cfg: OpenAICompatConfig): MemoryHelperLLM {
+  return {
+    async compileIntents(text: string): Promise<CompileMemoryIntentsResult> {
+      const url = new URL("/v1/chat/completions", cfg.baseUrl);
+      const body: ChatCompletionRequest = {
+        model: cfg.model,
+        messages: [
+          {
+            role: "user",
+            content: `Analyze whether the user message requires recalling private episodic memory.\n\n<message>\n${text}\n</message>`,
+          },
+        ],
+        max_tokens: 2048,
+        tools: [
+          {
+            type: "function",
+            function: {
+              name: MEMORY_HELPER_TOOL_NAME,
+              description: "Decide whether to recall private episodic memory and compile structured query intents.",
+              parameters: COMPILE_MEMORY_INTENTS_PARAMETERS as unknown as Record<string, unknown>,
+            },
+          },
+        ],
+        tool_choice: { type: "function", function: { name: MEMORY_HELPER_TOOL_NAME } },
+      };
+      const resp = await postJSON<ChatCompletionResponse>(url, body, cfg.apiKey);
+      if (resp.error?.message) throw new Error(`OpenAI-compat: ${resp.error.message}`);
+
+      const msg = resp.choices?.[0]?.message;
+      const toolCall = msg?.tool_calls?.[0]?.function;
+      if (toolCall?.name === MEMORY_HELPER_TOOL_NAME && toolCall.arguments) {
+        try {
+          return JSON.parse(toolCall.arguments) as CompileMemoryIntentsResult;
+        } catch { /* fall through */ }
+      }
+
+      const raw = msg?.content?.trim();
+      if (!raw) return { should_recall: false, intents: [] };
+
+      try {
+        const cleaned = raw.replace(/^```(?:json)?\s*/m, "").replace(/\s*```\s*$/m, "").trim();
+        return JSON.parse(cleaned) as CompileMemoryIntentsResult;
+      } catch {
+        return { should_recall: false, intents: [] };
+      }
+    },
+  };
+}

package/src/cli.ts

@@ -4,7 +4,8 @@ import path from "node:path";
 
 import { createStandaloneLLMClient } from "./adapters/piComplete.js";
 import { installBundle } from "./bundle/install.js";
-import { defaultMemoryConfig } from "./config.js";
+import type { MemoryConfig } from "./config.js";
+import { loadMemoryConfig } from "./settings.js";
 import { openSessionIndex } from "./fallback/sessionIndex.js";
 import { SidecarClient } from "./sidecar/client.js";
 import { MemoryService } from "./service.js";
@@ -13,7 +14,7 @@ import { createLLMFactExtractor } from "./trainer/llmExtractor.js";
 import { createTrainScheduler } from "./trainer/scheduler.js";
 import type { QueryIntent } from "./types.js";
 
-async function tryReloadSidecar(cfg: ReturnType<typeof defaultMemoryConfig>): Promise<void> {
+async function tryReloadSidecar(cfg: MemoryConfig): Promise<void> {
   try {
     fs.accessSync(cfg.socketPath, fs.constants.F_OK);
   } catch {
@@ -34,7 +35,7 @@ async function main(): Promise<void> {
     process.exit(0);
   }
 
-  const cfg = defaultMemoryConfig();
+  const cfg = loadMemoryConfig();
   const service = new MemoryService(cfg);
 
   if (cmd === "health") {

package/src/fallback/sessionIndex.ts

@@ -146,62 +146,100 @@ export function openSessionIndex(dbPath: string, injectedDb?: SqliteDatabase): S
     return count;
   }
 
+  async function collectFiles(dir: string): Promise<string[]> {
+    let names: string[];
+    try {
+      names = await fs.readdir(dir);
+    } catch {
+      return [];
+    }
+    const files: string[] = [];
+    for (const name of names) {
+      const full = path.join(dir, name);
+      let st;
+      try { st = await fs.stat(full); } catch { continue; }
+      if (st.isDirectory()) {
+        files.push(...await collectFiles(full));
+      } else if (st.isFile() && (name.endsWith(".json") || name.endsWith(".jsonl"))) {
+        files.push(full);
+      }
+    }
+    return files;
+  }
+
+  function parseJsonlMessages(raw: string, filePath: string): {
+    id: string; title: string; createdAt: string;
+    messages: Array<{ role: string; content: string; index: number }>;
+  } | null {
+    const lines = raw.split("\n").filter((l) => l.trim());
+    if (lines.length === 0) return null;
+    let id = path.basename(filePath, path.extname(filePath));
+    let title = "";
+    let createdAt = "";
+    const messages: Array<{ role: string; content: string; index: number }> = [];
+    let idx = 0;
+    for (const line of lines) {
+      let obj: Record<string, unknown>;
+      try { obj = JSON.parse(line) as Record<string, unknown>; } catch { continue; }
+      if (obj.type === "session") {
+        id = (obj.id as string) ?? id;
+        title = (obj.title as string) ?? "";
+        createdAt = (obj.timestamp as string) ?? "";
+        continue;
+      }
+      if (obj.type === "message") {
+        const msg = (obj as { message?: PiSessionMessage }).message;
+        if (!msg?.role || !msg.content) continue;
+        if (msg.role !== "user" && msg.role !== "assistant") continue;
+        const text = messageText(msg.content);
+        if (text.trim()) {
+          messages.push({ role: msg.role, content: text, index: idx++ });
+        }
+      }
+    }
+    if (messages.length === 0) return null;
+    return { id, title, createdAt, messages };
+  }
+
   async function loadSessionFiles(sessionsDir: string, modifiedAfter?: Date | null): Promise<Array<{
     id: string;
     title: string;
     createdAt: string;
     messages: Array<{ role: string; content: string; index: number }>;
   }>> {
-    let entries: string[];
-    try {
-      entries = await fs.readdir(sessionsDir);
-    } catch {
-      return [];
-    }
-
+    const filePaths = await collectFiles(sessionsDir);
     const results: Array<{
-      id: string;
-      title: string;
-      createdAt: string;
+      id: string; title: string; createdAt: string;
       messages: Array<{ role: string; content: string; index: number }>;
     }> = [];
 
-    for (const name of entries) {
-      if (!name.endsWith(".json")) continue;
-      const filePath = path.join(sessionsDir, name);
+    for (const filePath of filePaths) {
       let st;
-      try {
-        st = await fs.stat(filePath);
-      } catch {
-        continue;
-      }
+      try { st = await fs.stat(filePath); } catch { continue; }
       if (!st.isFile()) continue;
       if (modifiedAfter && st.mtime <= modifiedAfter) continue;
 
-      let session: PiSessionFile;
-      try {
-        const raw = await fs.readFile(filePath, "utf8");
-        session = JSON.parse(raw) as PiSessionFile;
-      } catch {
-        continue;
-      }
+      let raw: string;
+      try { raw = await fs.readFile(filePath, "utf8"); } catch { continue; }
 
-      const sessionId = session.id ?? path.basename(name, ".json");
-      const messages: Array<{ role: string; content: string; index: number }> = [];
-      for (let i = 0; i < (session.messages?.length ?? 0); i++) {
-        const msg = session.messages![i]!;
-        const text = messageText(msg.content);
-        if (text.trim()) {
-          messages.push({ role: msg.role ?? "unknown", content: text, index: i });
+      if (filePath.endsWith(".jsonl")) {
+        const parsed = parseJsonlMessages(raw, filePath);
+        if (parsed) results.push(parsed);
+      } else {
+        let session: PiSessionFile;
+        try { session = JSON.parse(raw) as PiSessionFile; } catch { continue; }
+        const sessionId = session.id ?? path.basename(filePath, ".json");
+        const messages: Array<{ role: string; content: string; index: number }> = [];
+        for (let i = 0; i < (session.messages?.length ?? 0); i++) {
+          const msg = session.messages![i]!;
+          const text = messageText(msg.content);
+          if (text.trim()) {
+            messages.push({ role: msg.role ?? "unknown", content: text, index: i });
+          }
+        }
+        if (messages.length > 0) {
+          results.push({ id: sessionId, title: session.title ?? "", createdAt: session.created_at ?? "", messages });
         }
-      }
-      if (messages.length > 0) {
-        results.push({
-          id: sessionId,
-          title: session.title ?? "",
-          createdAt: session.created_at ?? "",
-          messages,
-        });
       }
     }
     return results;

package/src/fallback/sessionSearch.ts

@@ -16,10 +16,6 @@ export interface SessionSearchHit {
 let cachedIndex: SessionIndex | null = null;
 let cachedDbPath: string | null = null;
 
-/**
- * Get or open the FTS5 session index. Returns null if better-sqlite3
- * is unavailable or the DB file doesn't exist.
- */
 function getSessionIndex(dbPath: string): SessionIndex | null {
   if (cachedIndex && cachedDbPath === dbPath) return cachedIndex;
   if (!fsSync.existsSync(dbPath)) return null;
@@ -47,8 +43,29 @@ interface PiSessionFile {
 
 const SNIPPET_MAX = 240;
 
+async function collectFiles(dir: string): Promise<string[]> {
+  let names: string[];
+  try {
+    names = await fs.readdir(dir);
+  } catch {
+    return [];
+  }
+  const files: string[] = [];
+  for (const name of names) {
+    const full = path.join(dir, name);
+    let st;
+    try { st = await fs.stat(full); } catch { continue; }
+    if (st.isDirectory()) {
+      files.push(...await collectFiles(full));
+    } else if (st.isFile() && (name.endsWith(".json") || name.endsWith(".jsonl"))) {
+      files.push(full);
+    }
+  }
+  return files;
+}
+
 /**
- * Keyword search over Pi-style session JSON files (one directory level).
+ * Keyword search over Pi-style session files (JSON + JSONL, recursive subdirectories).
  * Uses FTS5 index when available, falls back to file scan.
  * All whitespace-separated terms must match (case-insensitive AND).
  */
@@ -72,17 +89,10 @@ export async function sessionKeywordSearch(
   const terms = splitTerms(q);
   if (terms.length === 0) return [];
 
-  let entries: string[];
-  try {
-    entries = await fs.readdir(sessionsDir);
-  } catch {
-    return [];
-  }
-
+  const filePaths = await collectFiles(sessionsDir);
   const hits: SessionSearchHit[] = [];
-  for (const name of entries) {
-    if (!name.endsWith(".json")) continue;
-    const filePath = path.join(sessionsDir, name);
+
+  for (const filePath of filePaths) {
     let st;
     try {
       st = await fs.stat(filePath);
@@ -91,34 +101,104 @@ export async function sessionKeywordSearch(
     }
     if (!st.isFile()) continue;
 
-    let session: PiSessionFile;
+    let raw: string;
+    try {
+      raw = await fs.readFile(filePath, "utf8");
+    } catch {
+      continue;
+    }
+
+    if (filePath.endsWith(".jsonl")) {
+      scanJsonlFile(raw, filePath, terms, hits, limit);
+    } else {
+      scanJsonFile(raw, filePath, terms, hits, limit);
+    }
+
+    if (hits.length >= limit) return hits;
+  }
+  return hits;
+}
+
+function scanJsonFile(
+  raw: string,
+  filePath: string,
+  terms: string[],
+  hits: SessionSearchHit[],
+  limit: number,
+): void {
+  let session: PiSessionFile;
+  try {
+    session = JSON.parse(raw) as PiSessionFile;
+  } catch {
+    return;
+  }
+
+  const sessionId = session.id ?? path.basename(filePath, ".json");
+  const title = session.title ?? "";
+  const createdAt = session.created_at ?? "";
+
+  for (let i = 0; i < (session.messages?.length ?? 0); i++) {
+    const msg = session.messages![i]!;
+    const text = messageText(msg.content);
+    if (!text || !allTermsMatch(text, terms)) continue;
+    hits.push({
+      session_id: sessionId,
+      session_title: title,
+      role: msg.role ?? "unknown",
+      snippet: makeSnippet(text, terms[0]!),
+      msg_index: i,
+      created_at: createdAt,
+    });
+    if (hits.length >= limit) return;
+  }
+}
+
+function scanJsonlFile(
+  raw: string,
+  filePath: string,
+  terms: string[],
+  hits: SessionSearchHit[],
+  limit: number,
+): void {
+  const lines = raw.split("\n").filter((l) => l.trim());
+  let sessionId = path.basename(filePath, ".jsonl");
+  let title = "";
+  let createdAt = "";
+  let msgIndex = 0;
+
+  for (const line of lines) {
+    let obj: Record<string, unknown>;
     try {
-      const raw = await fs.readFile(filePath, "utf8");
-      session = JSON.parse(raw) as PiSessionFile;
+      obj = JSON.parse(line) as Record<string, unknown>;
     } catch {
       continue;
     }
 
-    const sessionId = session.id ?? path.basename(name, ".json");
-    const title = session.title ?? "";
-    const createdAt = session.created_at ?? "";
+    if (obj.type === "session") {
+      sessionId = (obj.id as string) ?? sessionId;
+      title = (obj.title as string) ?? "";
+      createdAt = (obj.timestamp as string) ?? "";
+      continue;
+    }
 
-    for (let i = 0; i < (session.messages?.length ?? 0); i++) {
-      const msg = session.messages![i]!;
+    if (obj.type === "message") {
+      const msg = (obj as { message?: PiSessionMessage }).message;
+      if (!msg?.role || !msg.content) continue;
+      if (msg.role !== "user" && msg.role !== "assistant") continue;
       const text = messageText(msg.content);
       if (!text || !allTermsMatch(text, terms)) continue;
       hits.push({
         session_id: sessionId,
         session_title: title,
-        role: msg.role ?? "unknown",
+        role: msg.role,
         snippet: makeSnippet(text, terms[0]!),
-        msg_index: i,
+        msg_index: msgIndex,
         created_at: createdAt,
       });
-      if (hits.length >= limit) return hits;
+      msgIndex++;
+      if (hits.length >= limit) return;
     }
   }
-  return hits;
 }
 
 function splitTerms(query: string): string[] {

package/src/index.ts

@@ -25,6 +25,15 @@ export {
   type MemoryProvider,
 } from "./config.js";
 
+export {
+  defaultMemoryConfigPath,
+  loadMemoryConfig,
+  loadMemorySettings,
+  resolveHelperModelSpec,
+  type LoadedMemorySettings,
+  type MemorySettingsFile,
+} from "./settings.js";
+
 export {
   defaultBundleRoot,
   defaultPiHome,
@@ -206,6 +215,23 @@ export {
 
 export { defaultSessionDbPath } from "./fallback/sessionSearch.js";
 
+export { LocalGraphQuerier } from "./local/graphQuery.js";
+
+export {
+  createOllamaLLMClient,
+  createOllamaMemoryHelper,
+  ollamaHealthCheck,
+  DEFAULT_OLLAMA_CONFIG,
+  type OllamaConfig,
+} from "./adapters/ollamaClient.js";
+
+export {
+  createOpenAICompatLLMClient,
+  createOpenAICompatMemoryHelper,
+  openaiCompatHealthCheck,
+  type OpenAICompatConfig,
+} from "./adapters/openaiCompatClient.js";
+
 export {
   rerankWithLLM,
   type RerankOptions,