@memorycrystal/crystal-memory 0.7.4

package/recall-hook.js

@@ -0,0 +1,456 @@
+#!/usr/bin/env node
+const fs = require("node:fs");
+const path = require("node:path");
+
+const OPENAI_MODEL = "text-embedding-3-small";
+const OPENAI_URL = "https://api.openai.com/v1/embeddings";
+const GEMINI_MODEL = "gemini-embedding-2-preview";
+const GEMINI_URL_BASE = "https://generativelanguage.googleapis.com/v1beta";
+const CONVEX_ACTION = "/api/action";
+const CONVEX_QUERY = "/api/query";
+const DEFAULT_LIMIT = 8;
+const RECALL_PATHS = ["crystal/recall:recallMemories"];
+
+// Session memory dedup cache: tracks memory IDs returned this session
+// Key: sessionKey, Value: Set of memoryId strings
+const sessionMemoryCache = new Map();
+const SESSION_CACHE_MAX_AGE_MS = 4 * 60 * 60 * 1000; // 4 hours
+const sessionCacheTimestamps = new Map();
+
+function getSessionMemoryIds(sessionKey) {
+  if (!sessionKey) return [];
+  // Expire old sessions
+  const ts = sessionCacheTimestamps.get(sessionKey);
+  if (ts && Date.now() - ts > SESSION_CACHE_MAX_AGE_MS) {
+    sessionMemoryCache.delete(sessionKey);
+    sessionCacheTimestamps.delete(sessionKey);
+    return [];
+  }
+  return Array.from(sessionMemoryCache.get(sessionKey) || []);
+}
+
+function addSessionMemoryIds(sessionKey, memoryIds) {
+  if (!sessionKey || !memoryIds.length) return;
+  if (!sessionMemoryCache.has(sessionKey)) {
+    sessionMemoryCache.set(sessionKey, new Set());
+    sessionCacheTimestamps.set(sessionKey, Date.now());
+  }
+  for (const id of memoryIds) sessionMemoryCache.get(sessionKey).add(id);
+}
+
+const readFileEnv = (filePath) => {
+  const values = {};
+  if (!fs.existsSync(filePath)) {
+    return values;
+  }
+  const raw = fs.readFileSync(filePath, "utf8");
+  for (const line of raw.split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith("#") || !trimmed.includes("=")) {
+      continue;
+    }
+    const [key, value = ""] = trimmed.split("=", 2);
+    const normalizedKey = key.trim();
+    values[normalizedKey] = value.trim().replace(/^"+|"+$/g, "");
+  }
+  return values;
+};
+
+const loadRuntimeEnv = () => {
+  const envCandidates = [
+    process.env.CRYSTAL_ENV_FILE,
+    path.resolve(__dirname, "..", "mcp-server", ".env"),
+    process.env.CRYSTAL_ROOT ? path.resolve(process.env.CRYSTAL_ROOT, "mcp-server", ".env") : null,
+  ].filter((entry) => typeof entry === "string" && entry.trim().length > 0);
+
+  const envFile = envCandidates.find((entry) => fs.existsSync(entry));
+
+  return {
+    ...(envFile ? readFileEnv(envFile) : {}),
+    ...process.env,
+  };
+};
+
+const readInputFromStdin = async () => {
+  const chunks = [];
+  return await new Promise((resolve) => {
+    process.stdin.setEncoding("utf8");
+    process.stdin.on("data", (chunk) => chunks.push(chunk));
+    process.stdin.on("end", () => resolve(chunks.join("")));
+  });
+};
+
+const safeGetString = (value) => (typeof value === "string" ? value : "");
+
+const parseInput = async () => {
+  const env = loadRuntimeEnv();
+  const argQuery = safeGetString(process.argv[2]);
+  const rawPayload = (await readInputFromStdin()).trim();
+
+  if (!rawPayload) {
+    return {
+      query: argQuery,
+      channel: "",
+      sessionId: "",
+      sessionKey: "",
+      mode: "",
+      env,
+    };
+  }
+
+  try {
+    const parsed = JSON.parse(rawPayload);
+    return {
+      query: safeGetString(parsed?.query) || argQuery || "",
+      channel: safeGetString(parsed?.channel),
+      sessionId: safeGetString(parsed?.sessionId),
+      sessionKey: safeGetString(parsed?.sessionKey) || safeGetString(parsed?.sessionId),
+      mode: safeGetString(parsed?.mode),
+      env,
+    };
+  } catch {
+    return {
+      query: rawPayload || argQuery,
+      channel: "",
+      sessionId: "",
+      sessionKey: "",
+      mode: "",
+      env,
+    };
+  }
+};
+
+const toConvexUrl = (value) => {
+  if (typeof value !== "string" || value.trim().length === 0) {
+    return "";
+  }
+  if (/^https?:\/\//i.test(value)) {
+    return value.replace(/\/+$/, "");
+  }
+  return `https://${value.replace(/\/+$/, "")}`;
+};
+
+const getEmbedding = async (query, env) => {
+  if (typeof query !== "string" || query.trim().length === 0) {
+    return null;
+  }
+
+  const provider = String(env.EMBEDDING_PROVIDER || "gemini").toLowerCase();
+
+  if (provider === "gemini") {
+    const geminiKey = env.GEMINI_API_KEY;
+    if (!geminiKey) {
+      return null;
+    }
+    const model = env.GEMINI_EMBEDDING_MODEL || GEMINI_MODEL;
+    const response = await fetch(`${GEMINI_URL_BASE}/models/${model}:embedContent?key=${encodeURIComponent(geminiKey)}`, {
+      method: "POST",
+      headers: {
+        "content-type": "application/json",
+      },
+      body: JSON.stringify({
+        model: `models/${model}`,
+        content: { parts: [{ text: query }] },
+      }),
+    });
+
+    if (!response.ok) {
+      return null;
+    }
+
+    const payload = await response.json().catch(() => null);
+    return Array.isArray(payload?.embedding?.values) ? payload.embedding.values : null;
+  }
+
+  const openaiKey = env.OPENAI_API_KEY;
+  if (!openaiKey) {
+    return null;
+  }
+
+  const response = await fetch(OPENAI_URL, {
+    method: "POST",
+    headers: {
+      "content-type": "application/json",
+      Authorization: `Bearer ${openaiKey}`,
+    },
+    body: JSON.stringify({
+      model: OPENAI_MODEL,
+      input: query,
+    }),
+  });
+
+  if (!response.ok) {
+    return null;
+  }
+
+  const payload = await response.json().catch(() => null);
+  return Array.isArray(payload?.data?.[0]?.embedding) ? payload.data[0].embedding : null;
+};
+
+const formatBlock = (memories) => {
+  if (!Array.isArray(memories) || memories.length === 0) {
+    return "## 🧠 Memory Crystal Memory Recall\nNo matching memories found.";
+  }
+
+  const lines = ["## 🧠 Memory Crystal Memory Recall"];
+  for (const memory of memories) {
+    const store = typeof memory.store === "string" ? memory.store.toUpperCase() : "UNKNOWN";
+    const title = typeof memory.title === "string" ? memory.title : "";
+    const content = typeof memory.content === "string" ? memory.content : "";
+    const tags = Array.isArray(memory.tags)
+      ? memory.tags.filter((tag) => typeof tag === "string" && tag.trim().length > 0).join(", ")
+      : "";
+    const strength = typeof memory.strength === "number" ? memory.strength.toFixed(2) : "0.00";
+    const confidence = typeof memory.confidence === "number" ? memory.confidence.toFixed(2) : "0.00";
+    const score = typeof memory.score === "number" ? memory.score.toFixed(2) : "0.00";
+    lines.push(`### ${store}: ${title}`);
+    lines.push(content);
+    lines.push(`Tags: ${tags.length > 0 ? tags : "none"} | Strength: ${strength} | Confidence: ${confidence} | Score: ${score}`);
+    lines.push("");
+  }
+
+  return lines.join("\n").trimEnd();
+};
+
+const normalizeMemory = (memory) => {
+  if (!memory || typeof memory !== "object") {
+    return null;
+  }
+  return {
+    memoryId: safeGetString(memory.memoryId || memory._id),
+    store: safeGetString(memory.store),
+    category: safeGetString(memory.category),
+    title: safeGetString(memory.title),
+    content: safeGetString(memory.content),
+    strength: Number.isFinite(memory.strength) ? memory.strength : 0,
+    confidence: Number.isFinite(memory.confidence) ? memory.confidence : 0,
+    tags: Array.isArray(memory.tags) ? memory.tags : [],
+    score: Number.isFinite(memory.score) ? memory.score : Number.isFinite(memory._score) ? memory._score : 0,
+    scoreValue: Number.isFinite(memory.scoreValue) ? memory.scoreValue : 0,
+  };
+};
+
+const searchMemories = async ({ embedding, query, mode, channel, sessionKey, env }) => {
+  const normalizedMode = mode && mode.trim().length > 0 ? mode.trim() : undefined;
+  // Prefer authenticated HTTP endpoint (/api/mcp/recall) when CRYSTAL_SITE + CRYSTAL_API_KEY
+  // are available. This avoids the Convex action auth requirement (ctx.auth.getUserIdentity()).
+  const crystalSite = (env.CRYSTAL_SITE || "").replace(/\/+$/, "");
+  const crystalApiKey = env.CRYSTAL_API_KEY || "";
+
+  if (crystalSite && crystalApiKey) {
+    try {
+      const payload = {
+        query: query || "",
+        limit: DEFAULT_LIMIT,
+        ...(channel ? { channel } : {}),
+        ...(sessionKey ? { sessionKey } : {}),
+        ...(normalizedMode ? { mode: normalizedMode } : {}),
+      };
+      const mcpResponse = await fetch(`${crystalSite}/api/mcp/recall`, {
+        method: "POST",
+        headers: {
+          "content-type": "application/json",
+          Authorization: `Bearer ${crystalApiKey}`,
+        },
+        body: JSON.stringify(payload),
+      });
+      if (mcpResponse.ok) {
+        const mcpPayload = await mcpResponse.json().catch(() => null);
+        const mcpMemories = Array.isArray(mcpPayload?.memories) ? mcpPayload.memories : [];
+        if (mcpMemories.length > 0) {
+          return mcpMemories;
+        }
+      }
+    } catch (_) {
+      // fall through to Convex action path
+    }
+  }
+
+  // Fallback: raw Convex action endpoint (requires auth context — only works when
+  // called from within an authenticated Convex session or from the MCP server).
+  const convexUrl = toConvexUrl(env.CONVEX_URL);
+  if (!convexUrl || !Array.isArray(embedding) || embedding.length === 0) {
+    return [];
+  }
+
+  const args = {
+    embedding,
+    limit: DEFAULT_LIMIT,
+    query: query, // enables BM25 hybrid search
+    recentMemoryIds: getSessionMemoryIds(sessionKey), // session dedup
+    ...(channel ? { channel } : {}),
+    ...(normalizedMode ? { mode: normalizedMode } : {}),
+  };
+
+  for (const path of RECALL_PATHS) {
+    const actionResponse = await fetch(`${convexUrl}${CONVEX_ACTION}`, {
+      method: "POST",
+      headers: {
+        "content-type": "application/json",
+      },
+      body: JSON.stringify({
+        path,
+        args,
+      }),
+    });
+
+    if (!actionResponse.ok) {
+      continue;
+    }
+
+    const actionPayload = await actionResponse.json().catch(() => null);
+    const actionData = actionPayload?.value ?? actionPayload;
+    const actionMemories = Array.isArray(actionData?.memories) ? actionData.memories : [];
+    if (actionMemories.length > 0) {
+      return actionMemories;
+    }
+  }
+
+  return [];
+};
+
+const fetchRecentMessages = async (channel, sessionKey, limit = 20, env) => {
+  const crystalSite = (env.CRYSTAL_SITE || "").replace(/\/+$/, "");
+  const crystalApiKey = env.CRYSTAL_API_KEY || "";
+
+  if (crystalSite && crystalApiKey) {
+    try {
+      const response = await fetch(`${crystalSite}/api/mcp/recent-messages`, {
+        method: "POST",
+        headers: {
+          "content-type": "application/json",
+          Authorization: `Bearer ${crystalApiKey}`,
+        },
+        body: JSON.stringify({
+          limit,
+          channel,
+          sessionKey,
+        }),
+      });
+      if (response.ok) {
+        const payload = await response.json().catch(() => null);
+        const messages = Array.isArray(payload?.messages) ? payload.messages : [];
+        return messages.filter((message) => message && typeof message === "object");
+      }
+    } catch (_) {
+      // fall through to raw Convex query
+    }
+  }
+
+  const convexUrl = toConvexUrl(env.CONVEX_URL);
+  if (!convexUrl) {
+    return [];
+  }
+
+  const response = await fetch(`${convexUrl}${CONVEX_QUERY}`, {
+    method: "POST",
+    headers: {
+      "content-type": "application/json",
+    },
+    body: JSON.stringify({
+      path: "crystal/messages:getRecentMessages",
+      args: {
+        limit,
+        channel,
+        sessionKey,
+      },
+    }),
+  });
+
+  if (!response.ok) {
+    return [];
+  }
+
+  const payload = await response.json().catch(() => null);
+  const data = payload?.value ?? payload;
+  const messages = Array.isArray(data)
+    ? data
+    : Array.isArray(data?.messages)
+      ? data.messages
+      : [];
+  return messages.filter((message) => message && typeof message === "object");
+};
+
+const formatRecentMessages = (messages) => {
+  if (!Array.isArray(messages) || messages.length === 0) {
+    return "";
+  }
+
+  const lines = messages.map((message) => {
+    const timestamp = typeof message.timestamp === "number" ? new Date(message.timestamp).toLocaleTimeString() : "Invalid time";
+    const role = typeof message.role === "string" ? message.role : "unknown";
+    const content = typeof message.content === "string" ? message.content : "";
+    const trimmed = content.length > 150 ? content.slice(0, 150) : content;
+    return `[${timestamp}] ${role}: ${trimmed}`;
+  });
+
+  return ["## Short-Term Memory (recent messages)", ...lines].join("\n");
+};
+
+/**
+ * Returns true if this query warrants a memory recall lookup.
+ * Skips noisy/trivial queries; forces recall for explicit memory-seeking queries.
+ */
+function shouldRecall(query) {
+  const q = (query || "").trim();
+
+  // Force recall if query explicitly seeks memory context
+  const forcePatterns = /\b(remember|recall|previously|last time|earlier|before|memory|forgot|what did we|when did we|history)\b/i;
+  if (forcePatterns.test(q)) return true;
+
+  // Skip empty or very short queries
+  if (q.length < 4) return false;
+
+  // Skip slash commands
+  if (q.startsWith("/")) return false;
+
+  // Skip pure greetings
+  if (/^(hi|hello|hey|good morning|good afternoon|good evening|howdy)[!.,\s]*$/i.test(q)) return false;
+
+  // Skip simple confirmations
+  if (/^(yes|no|ok|sure|thanks|thank you|nope|yep|yeah|nah)[!.,\s]*$/i.test(q)) return false;
+
+  // Skip pure emoji (no alphabetic characters)
+  if (!/[a-zA-Z]/.test(q)) return false;
+
+  // Skip heartbeat patterns
+  if (/HEARTBEAT|heartbeat poll/i.test(q)) return false;
+
+  return true;
+}
+
+const main = async () => {
+  const { query, mode, env, channel, sessionKey } = await parseInput();
+
+  if (!shouldRecall(query)) {
+    process.stdout.write(JSON.stringify({ injectionBlock: "", memories: [] }));
+    process.exit(0);
+  }
+  const embedding = await getEmbedding(query, env);
+  if (!embedding) {
+    process.stdout.write(JSON.stringify({ injectionBlock: "", memories: [] }));
+    return;
+  }
+
+  const memories = (await searchMemories({ embedding, query, mode, channel, sessionKey, env }))
+    .slice(0, DEFAULT_LIMIT)
+    .map(normalizeMemory)
+    .filter(Boolean);
+  addSessionMemoryIds(sessionKey, memories.map((m) => m.memoryId).filter(Boolean));
+  const recentMessages = await fetchRecentMessages(channel, sessionKey, 20, env);
+  const recentBlock = formatRecentMessages(recentMessages);
+  const longTermBlock = formatBlock(memories);
+  const injectionBlock = [recentBlock, longTermBlock].filter(Boolean).join("\n\n");
+
+  process.stdout.write(
+    JSON.stringify({
+      injectionBlock,
+      memories,
+    })
+  );
+};
+
+main().catch(() => {
+  process.stdout.write(JSON.stringify({ injectionBlock: "", memories: [] }));
+  process.exit(0);
+});

package/reinforcement.test.js

@@ -0,0 +1,105 @@
+// Tests for reinforcement injection logic.
+// We test the constants and behavior expectations directly since the
+// reinforcement logic lives inside index.js as a hook.
+
+const { getInjectionBudget } = require("./context-budget");
+
+let passed = 0;
+let failed = 0;
+
+function test(name, fn) {
+  try {
+    fn();
+    passed++;
+    console.log(`  PASS: ${name}`);
+  } catch (err) {
+    failed++;
+    console.error(`  FAIL: ${name} — ${err.message}`);
+  }
+}
+
+function assert(condition, msg) {
+  if (!condition) throw new Error(msg || "assertion failed");
+}
+
+console.log("reinforcement tests:");
+
+test("reinforcement block stays under 800 chars with long memories", () => {
+  const REINFORCEMENT_MAX_CHARS = 800;
+  const cached = [
+    { title: "A".repeat(200), content: "B".repeat(1000) },
+    { title: "C".repeat(200), content: "D".repeat(1000) },
+  ];
+
+  let block = "## Memory Reinforcement\n";
+  let charCount = block.length;
+
+  for (const mem of cached.slice(0, 2)) {
+    const title = String(mem.title || "").slice(0, 80);
+    const content = String(mem.content || "").slice(0, 300);
+    const line = `[Recall: ${title}] ${content}\n`;
+    if (charCount + line.length > REINFORCEMENT_MAX_CHARS) break;
+    block += line;
+    charCount += line.length;
+  }
+
+  assert(block.length <= REINFORCEMENT_MAX_CHARS, `block is ${block.length} chars, expected <= ${REINFORCEMENT_MAX_CHARS}`);
+});
+
+test("reinforcement only fires when turn count >= 5", () => {
+  const REINFORCEMENT_TURN_THRESHOLD = 5;
+
+  // Simulate turn counts
+  assert(3 < REINFORCEMENT_TURN_THRESHOLD, "3 turns should NOT trigger reinforcement");
+  assert(5 >= REINFORCEMENT_TURN_THRESHOLD, "5 turns should trigger reinforcement");
+  assert(10 >= REINFORCEMENT_TURN_THRESHOLD, "10 turns should trigger reinforcement");
+});
+
+test("reinforcement returns nothing when no cached recall", () => {
+  const cache = new Map();
+  const sessionKey = "test-session";
+  const cached = cache.get(sessionKey);
+  assert(!cached || cached.length === 0, "empty cache should produce no reinforcement");
+});
+
+test("reinforcement returns nothing when cache is expired", () => {
+  const SESSION_RECALL_CACHE_MAX_AGE_MS = 4 * 60 * 60 * 1000;
+  const timestamps = new Map();
+  timestamps.set("test", Date.now() - SESSION_RECALL_CACHE_MAX_AGE_MS - 1);
+  const ts = timestamps.get("test");
+  assert(Date.now() - ts > SESSION_RECALL_CACHE_MAX_AGE_MS, "expired cache should not trigger reinforcement");
+});
+
+test("reinforcement budget is within model injection budget", () => {
+  const REINFORCEMENT_MAX_CHARS = 800;
+  const smallBudget = getInjectionBudget("gpt-4o");
+  assert(REINFORCEMENT_MAX_CHARS < smallBudget.maxChars,
+    `reinforcement (${REINFORCEMENT_MAX_CHARS}) should fit within even smallest model budget (${smallBudget.maxChars})`);
+});
+
+test("cache cleanup works with Map.delete", () => {
+  const cache = new Map();
+  const timestamps = new Map();
+  cache.set("s1", [{ title: "test", content: "test" }]);
+  timestamps.set("s1", Date.now());
+  cache.delete("s1");
+  timestamps.delete("s1");
+  assert(!cache.has("s1"), "cache should be empty after delete");
+  assert(!timestamps.has("s1"), "timestamps should be empty after delete");
+});
+
+test("cache cleanup works with Map.clear", () => {
+  const cache = new Map();
+  const timestamps = new Map();
+  cache.set("s1", []);
+  cache.set("s2", []);
+  timestamps.set("s1", Date.now());
+  timestamps.set("s2", Date.now());
+  cache.clear();
+  timestamps.clear();
+  assert(cache.size === 0, "cache should be empty after clear");
+  assert(timestamps.size === 0, "timestamps should be empty after clear");
+});
+
+console.log(`\n${passed} passed, ${failed} failed`);
+if (failed > 0) process.exit(1);