@wipcomputer/memory-crystal 0.7.30 → 0.7.33

package/dist/chunk-DFQ72B7M.js

@@ -0,0 +1,248 @@
+// src/ldm.ts
+import { existsSync, mkdirSync, readFileSync, writeFileSync, copyFileSync, chmodSync, readdirSync } from "fs";
+import { join, dirname } from "path";
+import { execSync } from "child_process";
+import { fileURLToPath } from "url";
+var HOME = process.env.HOME || "";
+var LDM_ROOT = join(HOME, ".ldm");
+function loadAgentConfig(id) {
+  const cfgPath = join(LDM_ROOT, "agents", id, "config.json");
+  try {
+    if (existsSync(cfgPath)) return JSON.parse(readFileSync(cfgPath, "utf-8"));
+  } catch {
+  }
+  return null;
+}
+function saveAgentConfig(id, config) {
+  const dir = join(LDM_ROOT, "agents", id);
+  mkdirSync(dir, { recursive: true });
+  writeFileSync(join(dir, "config.json"), JSON.stringify(config, null, 2) + "\n");
+}
+function getAgentId(harnessHint) {
+  if (process.env.CRYSTAL_AGENT_ID) return process.env.CRYSTAL_AGENT_ID;
+  const agentsDir = join(LDM_ROOT, "agents");
+  if (existsSync(agentsDir)) {
+    try {
+      for (const d of readdirSync(agentsDir)) {
+        const cfg = loadAgentConfig(d);
+        if (!cfg || !cfg.agentId) continue;
+        if (!harnessHint) return cfg.agentId;
+        if (harnessHint === "claude-code" && cfg.harness === "claude-code-cli") return cfg.agentId;
+        if (harnessHint === "openclaw" && cfg.harness === "openclaw") return cfg.agentId;
+      }
+    } catch {
+    }
+  }
+  return harnessHint === "openclaw" ? "oc-lesa-mini" : "cc-mini";
+}
+function ldmPaths(agentId) {
+  const id = agentId || getAgentId();
+  const agentRoot = join(LDM_ROOT, "agents", id);
+  return {
+    root: LDM_ROOT,
+    bin: join(LDM_ROOT, "bin"),
+    secrets: join(LDM_ROOT, "secrets"),
+    state: join(LDM_ROOT, "state"),
+    config: join(LDM_ROOT, "config.json"),
+    crystalDb: join(LDM_ROOT, "memory", "crystal.db"),
+    crystalLance: join(LDM_ROOT, "memory", "lance"),
+    agentRoot,
+    transcripts: join(agentRoot, "memory", "transcripts"),
+    sessions: join(agentRoot, "memory", "sessions"),
+    daily: join(agentRoot, "memory", "daily"),
+    journals: join(agentRoot, "memory", "journals"),
+    workspace: join(agentRoot, "memory", "workspace")
+  };
+}
+function loadConfig() {
+  const configPath = join(LDM_ROOT, "config.json");
+  try {
+    if (existsSync(configPath)) {
+      return JSON.parse(readFileSync(configPath, "utf-8"));
+    }
+  } catch {
+  }
+  return null;
+}
+function saveConfig(config) {
+  const configPath = join(LDM_ROOT, "config.json");
+  writeFileSync(configPath, JSON.stringify(config, null, 2) + "\n");
+}
+function scaffoldLdm(agentId) {
+  const paths = ldmPaths(agentId);
+  mkdirSync(join(paths.root, "memory"), { recursive: true });
+  mkdirSync(paths.crystalLance, { recursive: true });
+  mkdirSync(paths.bin, { recursive: true });
+  mkdirSync(paths.secrets, { recursive: true, mode: 448 });
+  mkdirSync(paths.state, { recursive: true });
+  mkdirSync(paths.transcripts, { recursive: true });
+  mkdirSync(paths.sessions, { recursive: true });
+  mkdirSync(paths.daily, { recursive: true });
+  mkdirSync(paths.journals, { recursive: true });
+  mkdirSync(paths.workspace, { recursive: true });
+  const id = agentId || getAgentId();
+  let config = loadConfig();
+  if (!config) {
+    config = {
+      version: "1.0.0",
+      agents: [id],
+      createdAt: (/* @__PURE__ */ new Date()).toISOString(),
+      updatedAt: (/* @__PURE__ */ new Date()).toISOString()
+    };
+  } else {
+    if (!config.agents.includes(id)) {
+      config.agents.push(id);
+    }
+    config.updatedAt = (/* @__PURE__ */ new Date()).toISOString();
+  }
+  saveConfig(config);
+  return paths;
+}
+function deployCaptureScript() {
+  const paths = ldmPaths();
+  mkdirSync(paths.bin, { recursive: true });
+  const thisDir = dirname(fileURLToPath(import.meta.url));
+  let scriptSrc = join(thisDir, "crystal-capture.sh");
+  if (!existsSync(scriptSrc)) {
+    scriptSrc = join(thisDir, "..", "scripts", "crystal-capture.sh");
+  }
+  const scriptDest = join(paths.bin, "crystal-capture.sh");
+  if (!existsSync(scriptSrc)) {
+    throw new Error(`crystal-capture.sh not found at ${scriptSrc}`);
+  }
+  copyFileSync(scriptSrc, scriptDest);
+  chmodSync(scriptDest, 493);
+  return scriptDest;
+}
+var CRON_TAG = "# crystal-capture";
+var CRON_ENTRY = "* * * * * ~/.ldm/bin/crystal-capture.sh >> ~/.ldm/logs/crystal-capture.log 2>&1";
+function isCrystalCaptureLine(line) {
+  return line === CRON_TAG || line.includes("crystal-capture.sh") && line.startsWith("*");
+}
+function installCron() {
+  mkdirSync(join(HOME, ".ldm", "logs"), { recursive: true });
+  let existing = "";
+  try {
+    existing = execSync("crontab -l 2>/dev/null", { encoding: "utf8" });
+  } catch {
+  }
+  const lines = existing.split("\n").filter((line) => !isCrystalCaptureLine(line));
+  lines.push(CRON_TAG);
+  lines.push(CRON_ENTRY);
+  const newCrontab = lines.filter((l, i, arr) => !(l === "" && i === arr.length - 1)).join("\n") + "\n";
+  execSync("crontab -", { input: newCrontab, encoding: "utf8" });
+}
+function removeCron() {
+  let existing = "";
+  try {
+    existing = execSync("crontab -l 2>/dev/null", { encoding: "utf8" });
+  } catch {
+    return;
+  }
+  const lines = existing.split("\n").filter((line) => !isCrystalCaptureLine(line));
+  const newCrontab = lines.join("\n");
+  execSync("crontab -", { input: newCrontab, encoding: "utf8" });
+}
+function deployBackupScript() {
+  const paths = ldmPaths();
+  mkdirSync(paths.bin, { recursive: true });
+  const thisDir = dirname(fileURLToPath(import.meta.url));
+  let scriptSrc = join(thisDir, "ldm-backup.sh");
+  if (!existsSync(scriptSrc)) {
+    scriptSrc = join(thisDir, "..", "scripts", "ldm-backup.sh");
+  }
+  const scriptDest = join(paths.bin, "ldm-backup.sh");
+  if (!existsSync(scriptSrc)) {
+    throw new Error(`ldm-backup.sh not found at ${scriptSrc}`);
+  }
+  copyFileSync(scriptSrc, scriptDest);
+  chmodSync(scriptDest, 493);
+  return scriptDest;
+}
+function installBackupLaunchAgent() {
+  const scriptPath = join(ldmPaths().bin, "ldm-backup.sh");
+  if (!existsSync(scriptPath)) {
+    throw new Error(`Backup script not found. Run crystal init first.`);
+  }
+  const launchAgentsDir = join(HOME, "Library", "LaunchAgents");
+  mkdirSync(launchAgentsDir, { recursive: true });
+  const plistPath = join(launchAgentsDir, "ai.openclaw.ldm-backup.plist");
+  const plist = `<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+  <key>Label</key>
+  <string>ai.openclaw.ldm-backup</string>
+  <key>ProgramArguments</key>
+  <array>
+    <string>bash</string>
+    <string>${scriptPath}</string>
+  </array>
+  <key>StartCalendarInterval</key>
+  <dict>
+    <key>Hour</key>
+    <integer>3</integer>
+    <key>Minute</key>
+    <integer>0</integer>
+  </dict>
+  <key>StandardOutPath</key>
+  <string>${HOME}/.ldm/logs/ldm-backup.log</string>
+  <key>StandardErrorPath</key>
+  <string>${HOME}/.ldm/logs/ldm-backup.log</string>
+</dict>
+</plist>`;
+  writeFileSync(plistPath, plist);
+  try {
+    execSync(`launchctl unload ${plistPath} 2>/dev/null`);
+  } catch {
+  }
+  execSync(`launchctl load ${plistPath}`);
+  return plistPath;
+}
+var LEGACY_OC_DIR = join(HOME, ".openclaw");
+function resolveStatePath(filename) {
+  const paths = ldmPaths();
+  const ldmPath = join(paths.state, filename);
+  if (existsSync(ldmPath)) return ldmPath;
+  const legacyPath = join(LEGACY_OC_DIR, "memory", filename);
+  if (existsSync(legacyPath)) return legacyPath;
+  return ldmPath;
+}
+function stateWritePath(filename) {
+  const paths = ldmPaths();
+  const dir = paths.state;
+  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+  return join(dir, filename);
+}
+function resolveSecretPath(filename) {
+  const paths = ldmPaths();
+  const ldmPath = join(paths.secrets, filename);
+  if (existsSync(ldmPath)) return ldmPath;
+  const legacyPath = join(LEGACY_OC_DIR, "secrets", filename);
+  if (existsSync(legacyPath)) return legacyPath;
+  return ldmPath;
+}
+function ensureLdm(agentId) {
+  const paths = ldmPaths(agentId);
+  if (existsSync(paths.transcripts) && existsSync(paths.config)) {
+    return paths;
+  }
+  return scaffoldLdm(agentId);
+}
+
+export {
+  loadAgentConfig,
+  saveAgentConfig,
+  getAgentId,
+  ldmPaths,
+  scaffoldLdm,
+  deployCaptureScript,
+  installCron,
+  removeCron,
+  deployBackupScript,
+  installBackupLaunchAgent,
+  resolveStatePath,
+  stateWritePath,
+  resolveSecretPath,
+  ensureLdm
+};

package/dist/chunk-NX647OM3.js

@@ -0,0 +1,310 @@
+// src/llm.ts
+import { existsSync, readFileSync } from "fs";
+import { join } from "path";
+import { homedir } from "os";
+import { execSync } from "child_process";
+var samplingServer = null;
+function setSamplingServer(server) {
+  samplingServer = server;
+}
+function hasSampling() {
+  return samplingServer !== null;
+}
+var expansionCache = /* @__PURE__ */ new Map();
+var _cacheDb = null;
+var CACHE_TTL_DAYS = parseInt(process.env.CRYSTAL_CACHE_TTL_DAYS || "7", 10);
+function setLLMCacheDb(db) {
+  _cacheDb = db;
+}
+function dbCacheGet(key) {
+  if (!_cacheDb) return null;
+  try {
+    const row = _cacheDb.prepare(
+      "SELECT result FROM llm_cache WHERE cache_key = ? AND created_at > ?"
+    ).get(key, new Date(Date.now() - CACHE_TTL_DAYS * 864e5).toISOString());
+    if (row) {
+      _cacheDb.prepare("UPDATE llm_cache SET hit_count = hit_count + 1, last_hit_at = ? WHERE cache_key = ?").run((/* @__PURE__ */ new Date()).toISOString(), key);
+      return row.result;
+    }
+  } catch {
+  }
+  return null;
+}
+function dbCacheSet(key, type, query, intent, result, provider) {
+  if (!_cacheDb) return;
+  try {
+    _cacheDb.prepare(
+      "INSERT OR REPLACE INTO llm_cache (cache_key, cache_type, query, intent, result, provider, created_at, hit_count, last_hit_at) VALUES (?, ?, ?, ?, ?, ?, ?, 0, NULL)"
+    ).run(key, type, query, intent || null, result, provider, (/* @__PURE__ */ new Date()).toISOString());
+  } catch {
+  }
+}
+var detectedProvider = null;
+var detectionDone = false;
+function getOpSecret(itemName, fieldLabel) {
+  try {
+    const saTokenPath = join(homedir(), ".openclaw/secrets/op-sa-token");
+    if (!existsSync(saTokenPath)) return void 0;
+    const saToken = readFileSync(saTokenPath, "utf-8").trim();
+    const result = execSync(
+      `OP_SERVICE_ACCOUNT_TOKEN="${saToken}" op item get "${itemName}" --vault "Agent Secrets" --fields "${fieldLabel}" --reveal`,
+      { encoding: "utf-8", timeout: 5e3, stdio: ["pipe", "pipe", "pipe"] }
+    ).trim();
+    return result || void 0;
+  } catch {
+    return void 0;
+  }
+}
+async function detectProvider() {
+  if (detectionDone && detectedProvider) return detectedProvider;
+  detectionDone = true;
+  if (samplingServer) {
+    detectedProvider = { provider: "sampling", baseURL: "", apiKey: "", model: "client-selected" };
+    process.stderr.write("[memory-crystal] LLM provider: MCP Sampling (via client)\n");
+    return detectedProvider;
+  }
+  try {
+    const resp = await fetch("http://localhost:18791/v1/models", { signal: AbortSignal.timeout(1e3) });
+    if (resp.ok) {
+      const data = await resp.json();
+      const model = data?.data?.[0]?.id || "default";
+      detectedProvider = { provider: "mlx", baseURL: "http://localhost:18791/v1", apiKey: "not-needed", model };
+      process.stderr.write(`[memory-crystal] LLM provider: MLX (${model})
+`);
+      return detectedProvider;
+    }
+  } catch {
+  }
+  try {
+    const resp = await fetch("http://localhost:11434/api/tags", { signal: AbortSignal.timeout(1e3) });
+    if (resp.ok) {
+      const data = await resp.json();
+      const models = data?.models || [];
+      const embeddingOnly = ["nomic-embed-text", "mxbai-embed", "all-minilm", "snowflake-arctic-embed"];
+      const chatModel = models.find((m) => !embeddingOnly.some((e) => m.name.startsWith(e)));
+      if (chatModel) {
+        detectedProvider = { provider: "ollama", baseURL: "http://localhost:11434/v1", apiKey: "ollama", model: chatModel.name };
+        process.stderr.write(`[memory-crystal] LLM provider: Ollama (${chatModel.name})
+`);
+        return detectedProvider;
+      }
+    }
+  } catch {
+  }
+  const openaiKey = process.env.OPENAI_API_KEY || getOpSecret("OpenAI API", "api key");
+  if (openaiKey) {
+    detectedProvider = { provider: "openai", baseURL: "https://api.openai.com/v1", apiKey: openaiKey, model: "gpt-4o-mini" };
+    process.stderr.write("[memory-crystal] LLM provider: OpenAI API\n");
+    return detectedProvider;
+  }
+  const anthropicKey = process.env.ANTHROPIC_API_KEY || getOpSecret("Anthropic Auth Token - remote bunkers", "Auth Token");
+  if (anthropicKey && !anthropicKey.startsWith("sk-ant-oat")) {
+    detectedProvider = { provider: "anthropic", baseURL: "https://api.anthropic.com", apiKey: anthropicKey, model: "claude-haiku-4-5-20251001" };
+    process.stderr.write("[memory-crystal] LLM provider: Anthropic API\n");
+    return detectedProvider;
+  }
+  detectedProvider = { provider: "none", baseURL: "", apiKey: "", model: "" };
+  process.stderr.write("[memory-crystal] LLM provider: none (deep search unavailable)\n");
+  return detectedProvider;
+}
+async function chatComplete(config, messages, maxTokens = 300) {
+  if (config.provider === "sampling") {
+    return samplingComplete(messages, maxTokens);
+  }
+  if (config.provider === "anthropic") {
+    return anthropicComplete(config, messages, maxTokens);
+  }
+  const resp = await fetch(`${config.baseURL}/chat/completions`, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      "Authorization": `Bearer ${config.apiKey}`
+    },
+    body: JSON.stringify({
+      model: config.model,
+      messages,
+      max_tokens: maxTokens,
+      temperature: 0.7
+    })
+  });
+  if (!resp.ok) throw new Error(`LLM request failed: ${resp.status}`);
+  const data = await resp.json();
+  return data.choices?.[0]?.message?.content || "";
+}
+async function anthropicComplete(config, messages, maxTokens) {
+  const systemMsg = messages.find((m) => m.role === "system");
+  const userMessages = messages.filter((m) => m.role !== "system");
+  const body = {
+    model: config.model,
+    max_tokens: maxTokens,
+    messages: userMessages
+  };
+  if (systemMsg) body.system = systemMsg.content;
+  const resp = await fetch("https://api.anthropic.com/v1/messages", {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      "x-api-key": config.apiKey,
+      "anthropic-version": "2023-06-01"
+    },
+    body: JSON.stringify(body)
+  });
+  if (!resp.ok) throw new Error(`Anthropic request failed: ${resp.status}`);
+  const data = await resp.json();
+  return data.content?.[0]?.text || "";
+}
+async function samplingComplete(messages, maxTokens) {
+  if (!samplingServer) throw new Error("MCP sampling server not set");
+  const systemMsg = messages.find((m) => m.role === "system");
+  const userMessages = messages.filter((m) => m.role !== "system");
+  const result = await samplingServer.createMessage({
+    messages: userMessages.map((m) => ({
+      role: m.role,
+      content: { type: "text", text: m.content }
+    })),
+    systemPrompt: systemMsg?.content,
+    maxTokens,
+    modelPreferences: {
+      // Request cheap, fast model (Haiku-class). We don't need Opus for query expansion.
+      costPriority: 0.9,
+      speedPriority: 0.8,
+      intelligencePriority: 0.3,
+      hints: [{ name: "haiku" }]
+    }
+  });
+  if (result?.content?.type === "text") return result.content.text;
+  if (typeof result?.content === "string") return result.content;
+  return "";
+}
+var EXPAND_PROMPT = `You are a search query expander. Given a search query, generate exactly 3 variations to improve search recall.
+
+Output exactly 3 lines in this format (no other text):
+lex: <keyword-focused variation for full-text search>
+vec: <semantic variation rephrased for embedding similarity>
+hyde: <hypothetical document snippet that would answer this query>
+
+Rules:
+- Each variation must contain at least one term from the original query
+- Keep variations concise (under 30 words each)
+- lex should use specific keywords and synonyms
+- vec should rephrase the intent naturally
+- hyde should be a short passage as if answering the query`;
+async function expandQuery(query, intent) {
+  const cacheKey = intent ? `expand:${query}||${intent}` : `expand:${query}`;
+  const dbCached = dbCacheGet(cacheKey);
+  if (dbCached) {
+    try {
+      return JSON.parse(dbCached);
+    } catch {
+    }
+  }
+  const cached = expansionCache.get(cacheKey);
+  if (cached) return cached;
+  const config = await detectProvider();
+  if (config.provider === "none") return [];
+  try {
+    const intentContext = intent ? `
+Query intent: ${intent}. Use this to guide your variations toward the intended domain.` : "";
+    const result = await chatComplete(config, [
+      { role: "system", content: EXPAND_PROMPT + intentContext },
+      { role: "user", content: query }
+    ], 300);
+    const lines = result.trim().split("\n");
+    const queryLower = query.toLowerCase();
+    const queryTerms = queryLower.replace(/[^a-z0-9\s]/g, " ").split(/\s+/).filter(Boolean);
+    const hasQueryTerm = (text) => {
+      const lower = text.toLowerCase();
+      if (queryTerms.length === 0) return true;
+      return queryTerms.some((term) => lower.includes(term));
+    };
+    const variations = lines.map((line) => {
+      const colonIdx = line.indexOf(":");
+      if (colonIdx === -1) return null;
+      const type = line.slice(0, colonIdx).trim();
+      if (type !== "lex" && type !== "vec" && type !== "hyde") return null;
+      const text = line.slice(colonIdx + 1).trim();
+      if (!text || !hasQueryTerm(text)) return null;
+      return { type, text };
+    }).filter((v) => v !== null);
+    if (variations.length > 0) {
+      expansionCache.set(cacheKey, variations);
+      dbCacheSet(cacheKey, "expansion", query, intent, JSON.stringify(variations), config.provider);
+      return variations;
+    }
+  } catch (err) {
+    process.stderr.write(`[memory-crystal] Query expansion failed: ${err.message}
+`);
+  }
+  const fallback = [
+    { type: "lex", text: query },
+    { type: "vec", text: query },
+    { type: "hyde", text: `Information about ${query}` }
+  ];
+  return fallback;
+}
+var RERANK_PROMPT = `You are a search result re-ranker. Given a query and a list of text passages, rate each passage's relevance to the query.
+
+Output one line per passage in this exact format:
+<index>: <score>
+
+Where index is the passage number (0-based) and score is a float from 0.0 to 1.0.
+- 1.0 = perfectly relevant, directly answers the query
+- 0.7 = highly relevant, closely related
+- 0.4 = somewhat relevant, tangentially related
+- 0.1 = barely relevant
+- 0.0 = not relevant at all
+
+Rate ALL passages. Output nothing else.`;
+async function rerankResults(query, passages) {
+  const config = await detectProvider();
+  if (config.provider === "none") {
+    return passages.map((_, i) => ({ index: i, score: 1 - i * 0.01 }));
+  }
+  const { createHash } = await import("crypto");
+  const contentHash = createHash("sha256").update(passages.map((p) => p.slice(0, 200)).sort().join("|")).digest("hex").slice(0, 16);
+  const rerankCacheKey = `rerank:${query}||${contentHash}`;
+  const dbCachedRerank = dbCacheGet(rerankCacheKey);
+  if (dbCachedRerank) {
+    try {
+      return JSON.parse(dbCachedRerank);
+    } catch {
+    }
+  }
+  try {
+    const passageList = passages.map((p, i) => `[${i}] ${p.slice(0, 500)}`).join("\n\n");
+    const result = await chatComplete(config, [
+      { role: "system", content: RERANK_PROMPT },
+      { role: "user", content: `Query: ${query}
+
+Passages:
+${passageList}` }
+    ], 200);
+    const results = [];
+    for (const line of result.trim().split("\n")) {
+      const match = line.match(/^(\d+):\s*([\d.]+)/);
+      if (match) {
+        results.push({ index: parseInt(match[1]), score: parseFloat(match[2]) });
+      }
+    }
+    const scored = new Set(results.map((r) => r.index));
+    for (let i = 0; i < passages.length; i++) {
+      if (!scored.has(i)) results.push({ index: i, score: 0 });
+    }
+    const sorted = results.sort((a, b) => b.score - a.score);
+    dbCacheSet(rerankCacheKey, "rerank", query, void 0, JSON.stringify(sorted), config.provider);
+    return sorted;
+  } catch (err) {
+    process.stderr.write(`[memory-crystal] Reranking failed: ${err.message}
+`);
+    return passages.map((_, i) => ({ index: i, score: 1 - i * 0.01 }));
+  }
+}
+
+export {
+  setSamplingServer,
+  hasSampling,
+  setLLMCacheDb,
+  detectProvider,
+  expandQuery,
+  rerankResults
+};