@melandlabs/memory-store 1.1.6 → 1.1.7

package/dist/server/cli-shared.js

@@ -7,6 +7,345 @@ import '@melandlabs/ai/memory';
 import '@melandlabs/rag';
 import '@melandlabs/contracts';
 
+// src/search/iterative-recall.ts
+var SYSTEM_PROMPT = `You are a research assistant collecting evidence from a user's conversation history. You are NOT answering the question; only gather the most useful memory.
+
+Actions:
+search \u2014 search by concise keywords. Previously returned chunks are automatically excluded.
+Action Input: {"keywords":["keyword1","keyword2"],"date_from":"YYYY-MM-DD","date_to":"YYYY-MM-DD"}
+The date fields are optional; omit them unless an absolute date range is useful.
+
+note \u2014 save relevant results from the LAST search by their 1-based result number.
+Action Input: {"indices":[1,3]}
+
+finish \u2014 finish collecting evidence.
+Action Input: {}
+
+Workflow: search -> note -> search again -> note -> finish.
+Review every result. Call note before another search because the prior result list is replaced. Use diverse keywords on later searches. Never answer the question.
+
+Output exactly:
+Thought: one short sentence
+Action: one action name
+Action Input: valid JSON object`;
+function deriveKeywordsFromQuery(query) {
+  return query.toLowerCase().split(/[^\p{L}\p{N}]+/u).filter((token) => token.length >= 2);
+}
+function formatDateHint(dateFrom, dateTo) {
+  if (!dateFrom && !dateTo) {
+    return void 0;
+  }
+  const fromText = dateFrom ?? "the beginning";
+  const toText = dateTo ?? "now";
+  return `Restrict searches to the date range ${fromText} to ${toText}. You may still emit narrower date_from/date_to bounds when useful.`;
+}
+function buildInitialPrompt(query, dateHint) {
+  const hintText = dateHint ? `
+${dateHint}` : "";
+  return `${SYSTEM_PROMPT}
+
+Question to research: ${query}${hintText}
+Search the conversation history and collect all relevant evidence. Start with a broad keyword search.`;
+}
+function buildObservationPrompt(hits) {
+  if (hits.length === 0) {
+    return "Observation: No matching memory found.\nRespond with the next action only.";
+  }
+  const blocks = hits.map((hit, index) => {
+    const lines = [`Result ${index + 1} | score=${hit.similarity.toFixed(4)}`];
+    for (const line of hit.content.split("\n")) {
+      if (line.trim()) lines.push(line);
+    }
+    return lines.join("\n");
+  });
+  return `Observation:
+${blocks.join("\n\n")}
+
+Respond with the next action only.`;
+}
+function stripCodeFences(text) {
+  return text.replace(/```(?:json)?\s*([\s\S]*?)\s*```/g, "$1");
+}
+function extractJsonObject(text) {
+  const start = text.indexOf("{");
+  if (start === -1) {
+    return void 0;
+  }
+  let depth = 0;
+  let inString = false;
+  let escaped = false;
+  for (let i = start; i < text.length; i += 1) {
+    const ch = text[i];
+    if (escaped) {
+      escaped = false;
+      continue;
+    }
+    if (inString) {
+      if (ch === "\\") {
+        escaped = true;
+      } else if (ch === '"') {
+        inString = false;
+      }
+      continue;
+    }
+    if (ch === '"') {
+      inString = true;
+    } else if (ch === "{") {
+      depth += 1;
+    } else if (ch === "}") {
+      depth -= 1;
+      if (depth === 0) {
+        return text.slice(start, i + 1);
+      }
+    }
+  }
+  return void 0;
+}
+function parseAction(text) {
+  const cleaned = stripCodeFences(text);
+  const actionMatch = cleaned.match(/Action:\s*([\w_]+)/);
+  if (!actionMatch) return null;
+  const action = actionMatch[1].trim().toLowerCase();
+  if (action === "finish") {
+    return { type: "finish" };
+  }
+  const inputMatch = cleaned.match(/Action Input:\s*/s);
+  if (!inputMatch || inputMatch.index === void 0) return null;
+  const jsonText = extractJsonObject(cleaned.slice(inputMatch.index + inputMatch[0].length));
+  if (!jsonText) return null;
+  let parsedJson = null;
+  for (const candidate of [jsonText, jsonText.replace(/'/g, '"')]) {
+    try {
+      parsedJson = JSON.parse(candidate);
+      break;
+    } catch {
+    }
+  }
+  if (!parsedJson) return null;
+  if (action === "search") {
+    const keywords = parsedJson.keywords;
+    if (!Array.isArray(keywords) || !keywords.every((k) => typeof k === "string")) {
+      return null;
+    }
+    return {
+      type: "search",
+      keywords,
+      dateFrom: typeof parsedJson.date_from === "string" ? parsedJson.date_from : void 0,
+      dateTo: typeof parsedJson.date_to === "string" ? parsedJson.date_to : void 0
+    };
+  }
+  if (action === "note") {
+    const indices = parsedJson.indices;
+    if (!Array.isArray(indices) || !indices.every((i) => typeof i === "number")) {
+      return null;
+    }
+    return { type: "note", indices };
+  }
+  return null;
+}
+function createIterativeRecallPlanner(deps) {
+  const { complete, options = {} } = deps;
+  const defaults = {
+    maxIterations: 4,
+    searchTopK: 5,
+    disabled: false,
+    fallbackToBaseline: true
+  };
+  let lastDegraded = false;
+  let iterationsWithValidAction = 0;
+  return {
+    lastDegraded: () => lastDegraded,
+    async plan(input) {
+      const opts = {
+        ...defaults,
+        ...options,
+        ...input.options
+      };
+      lastDegraded = false;
+      iterationsWithValidAction = 0;
+      if (opts.disabled || !input.query.trim()) {
+        return { evidence: [], stats: { iterations: 0, searches: 0, notes: 0 } };
+      }
+      const messages = [
+        { role: "system", content: SYSTEM_PROMPT },
+        {
+          role: "user",
+          content: buildInitialPrompt(input.query, formatDateHint(input.dateFrom, input.dateTo))
+        }
+      ];
+      const evidence = /* @__PURE__ */ new Map();
+      const excludedIds = /* @__PURE__ */ new Set();
+      let lastHits = [];
+      let searches = 0;
+      let notes = 0;
+      let iterations = 0;
+      let completeThrew = false;
+      for (let i = 0; i < opts.maxIterations; i += 1) {
+        iterations = i + 1;
+        let reply;
+        try {
+          reply = await complete(messages.map((m) => `${m.role}: ${m.content}`).join("\n\n"));
+        } catch {
+          completeThrew = true;
+          break;
+        }
+        messages.push({ role: "assistant", content: reply });
+        const action = parseAction(reply);
+        if (!action) {
+          messages.push({
+            role: "user",
+            content: "Observation: Use exactly one valid action: search, note, or finish. Respond with the next action only."
+          });
+          continue;
+        }
+        iterationsWithValidAction += 1;
+        if (action.type === "finish") {
+          break;
+        }
+        if (action.type === "search") {
+          const result = await input.executor.search({
+            keywords: action.keywords,
+            dateFrom: action.dateFrom ?? input.dateFrom,
+            dateTo: action.dateTo ?? input.dateTo
+          });
+          lastHits = result.candidates.filter((c) => !excludedIds.has(c.id)).slice(0, opts.searchTopK);
+          for (const hit of lastHits) {
+            excludedIds.add(hit.id);
+          }
+          searches += 1;
+          messages.push({ role: "user", content: buildObservationPrompt(lastHits) });
+          continue;
+        }
+        if (action.type === "note") {
+          let saved = 0;
+          for (const index of action.indices) {
+            if (index >= 1 && index <= lastHits.length) {
+              const hit = lastHits[index - 1];
+              if (!evidence.has(hit.id)) {
+                evidence.set(hit.id, hit);
+                saved += 1;
+              }
+            }
+          }
+          notes += saved;
+          messages.push({
+            role: "user",
+            content: `Observation: Saved ${saved} new result(s). Total notes: ${evidence.size}.
+Respond with the next action only.`
+          });
+        }
+      }
+      let fallbackRan = false;
+      if (evidence.size === 0 && opts.fallbackToBaseline) {
+        fallbackRan = true;
+        if (lastHits.length > 0) {
+          for (const hit of lastHits) {
+            if (!evidence.has(hit.id)) {
+              evidence.set(hit.id, hit);
+            }
+          }
+        } else {
+          const baseline = await input.executor.search({
+            keywords: deriveKeywordsFromQuery(input.query),
+            dateFrom: input.dateFrom,
+            dateTo: input.dateTo
+          });
+          searches += 1;
+          for (const hit of baseline.candidates.slice(0, opts.searchTopK)) {
+            if (!evidence.has(hit.id)) {
+              evidence.set(hit.id, hit);
+            }
+          }
+        }
+      }
+      lastDegraded = completeThrew || iterationsWithValidAction === 0 || fallbackRan;
+      return {
+        evidence: Array.from(evidence.values()),
+        stats: { iterations, searches, notes }
+      };
+    }
+  };
+}
+
+// src/search/query-rewriter.ts
+var SYSTEM_PROMPT2 = `You are a memory retrieval assistant.
+
+The memory store contains a user's past messages, written in the first person (e.g. "I told you about my sister's wedding", "I prefer dark mode").
+
+Your job is NOT to answer the question. Your job is to rewrite the assistant's question into a short, natural question that the USER would ask their own memory log to find the relevant entry.
+
+Rules:
+- Use first person: "Did I tell you about ...?", "What did I say about ...?", "Have I mentioned ...?"
+- Do NOT address the user as "you".
+- Do NOT answer the question.
+- Output each rephrasing on its own line, prefixed with "- ".
+- Keep each rephrasing under 20 words.
+- Output exactly the requested number of lines.`;
+function buildPrompt(query, numVariants) {
+  const label = numVariants === 1 ? "1 alternative rephrasing" : `${numVariants} alternative rephrasings`;
+  return `${SYSTEM_PROMPT2}
+
+Output exactly ${label}.
+
+Example:
+Question: What is the name of my cat?
+- Did I tell you the name of my cat?
+
+Question: ${query}
+Output:`;
+}
+function sanitizeLine(text) {
+  return text.replace(/^\s*["'`]+|["'`]+\s*$/g, "").replace(/^Rewritten:\s*/i, "").trim();
+}
+function parseVariants(raw, maxVariants, original) {
+  const lines = raw.split(/\r?\n/).map((line) => sanitizeLine(line.replace(/^\s*-\s+/, ""))).filter((line) => line.length > 0);
+  if (lines.length === 0) return [];
+  const seen = /* @__PURE__ */ new Set([original.toLowerCase()]);
+  const variants = [];
+  for (const line of lines) {
+    if (variants.length >= maxVariants) break;
+    const key = line.toLowerCase();
+    if (seen.has(key)) continue;
+    seen.add(key);
+    variants.push(line);
+  }
+  return variants;
+}
+function createUserVoiceRewriter(options) {
+  const { complete, maxVariants = 1, disabled = false } = options;
+  const wantedVariants = Math.max(0, Math.floor(maxVariants));
+  let lastDegraded = false;
+  return {
+    async rewrite(input) {
+      if (disabled || !input.query || !input.query.trim()) {
+        lastDegraded = false;
+        return [input.query];
+      }
+      const original = input.query.trim();
+      if (wantedVariants === 0) {
+        lastDegraded = false;
+        return [original];
+      }
+      try {
+        const raw = await complete(buildPrompt(original, wantedVariants));
+        const variants = parseVariants(raw, wantedVariants, original);
+        if (variants.length === 0) {
+          lastDegraded = true;
+          return [original];
+        }
+        lastDegraded = false;
+        return [original, ...variants];
+      } catch {
+        lastDegraded = true;
+        return [original];
+      }
+    },
+    lastDegraded() {
+      return lastDegraded;
+    }
+  };
+}
+
 // src/storage/postgres-raw-message-factory.ts
 var factory = null;
 async function resolvePostgresFactory(env) {
@@ -158,7 +497,11 @@ function parseUnifiedArgs(argv) {
     insightsBackend: env.INSIGHTS_BACKEND ?? "none",
     insightsCollection: env.INSIGHTS_COLLECTION ?? "opencontext_insights",
     knowledgeBackend: env.KNOWLEDGE_BACKEND ?? "none",
-    knowledgeCollection: env.KNOWLEDGE_COLLECTION ?? "opencontext_knowledge"
+    knowledgeCollection: env.KNOWLEDGE_COLLECTION ?? "opencontext_knowledge",
+    reasoning: env.REASONING === "1" || env.REASONING === "true",
+    reasoningBaseUrl: env.OPENCONTEXT_LLM_BASE_URL,
+    reasoningModel: env.OPENCONTEXT_LLM_MODEL,
+    reasoningTimeoutMs: env.OPENCONTEXT_LLM_TIMEOUT_MS ? Number.parseInt(env.OPENCONTEXT_LLM_TIMEOUT_MS, 10) : void 0
   };
   for (let i = 0; i < argv.length; i += 1) {
     const arg = argv[i];
@@ -193,6 +536,21 @@ function parseUnifiedArgs(argv) {
       case "--knowledge-collection":
         args.knowledgeCollection = takeValue();
         break;
+      case "--reasoning":
+        args.reasoning = true;
+        break;
+      case "--no-reasoning":
+        args.reasoning = false;
+        break;
+      case "--reasoning-base-url":
+        args.reasoningBaseUrl = takeValue();
+        break;
+      case "--reasoning-model":
+        args.reasoningModel = takeValue();
+        break;
+      case "--reasoning-timeout-ms":
+        args.reasoningTimeoutMs = Number.parseInt(takeValue(), 10);
+        break;
     }
   }
   for (const [name, value] of [
@@ -366,6 +724,52 @@ async function buildUnified(args) {
     };
     log(`knowledge backend wired via chroma (collection=${args.knowledgeCollection})`);
   }
+  if (args.reasoning) {
+    const apiKey = process.env.OPENCONTEXT_LLM_API_KEY;
+    if (!apiKey) {
+      throw new Error(
+        "--reasoning requires OPENCONTEXT_LLM_API_KEY (and ideally OPENCONTEXT_LLM_BASE_URL / OPENCONTEXT_LLM_MODEL) in the environment"
+      );
+    }
+    const baseUrl = args.reasoningBaseUrl ?? process.env.OPENCONTEXT_LLM_BASE_URL ?? "https://openrouter.ai/api/v1";
+    const model = args.reasoningModel ?? process.env.OPENCONTEXT_LLM_MODEL ?? "openai/gpt-4o-mini";
+    const timeoutMs = args.reasoningTimeoutMs ?? 3e4;
+    const complete = async (prompt) => {
+      const controller = new AbortController();
+      const timer = setTimeout(() => controller.abort(), timeoutMs);
+      timer.unref?.();
+      try {
+        const res = await fetch(`${baseUrl}/chat/completions`, {
+          method: "POST",
+          headers: {
+            "content-type": "application/json",
+            authorization: `Bearer ${apiKey}`,
+            "HTTP-Referer": process.env.NEXT_PUBLIC_APP_URL ?? "https://opencontext.ai",
+            "X-Title": "opencontext AI"
+          },
+          body: JSON.stringify({
+            model,
+            messages: [{ role: "user", content: prompt }],
+            temperature: 0
+          }),
+          signal: controller.signal
+        });
+        if (!res.ok) throw new Error(`reasoning LLM ${res.status}: ${await res.text()}`);
+        const body = await res.json();
+        const text = body.choices?.[0]?.message?.content?.trim();
+        if (!text) throw new Error("reasoning LLM response missing choices[0].message.content");
+        return text;
+      } finally {
+        clearTimeout(timer);
+      }
+    };
+    const queryRewriter = createUserVoiceRewriter({ complete });
+    const iterativePlanner = createIterativeRecallPlanner({ complete });
+    if (!unified.reasoning) unified.reasoning = {};
+    unified.reasoning.queryRewriter = queryRewriter;
+    unified.reasoning.iterativePlanner = iterativePlanner;
+    log(`reasoning wired (model=${model}, baseUrl=${baseUrl})`);
+  }
   return unified;
 }
 function printUnifiedHelp() {
@@ -388,7 +792,26 @@ Cross-source search (wires unified.searchKnowledge / searchInsights / searchRawM
   --knowledge-backend <name>      chroma | none
                                   (env: KNOWLEDGE_BACKEND, default: none)
   --knowledge-collection <name>   Chroma collection (default: opencontext_knowledge)
-  Note: the chroma backends dynamically import @melandlabs/ai-rag.`);
+  Note: the chroma backends dynamically import @melandlabs/ai-rag.
+
+Reasoning (wires unified.reasoning.{queryRewriter, iterativePlanner}):
+  --reasoning                     Enable the LLM reasoning layer so /v1/search and
+                                  memory.search can honor reasoningStrategy:
+                                    "rewrite"    \u2014 first-person memory-check rephrase
+                                    "iterative"  \u2014 planner that searches, notes evidence,
+                                                   searches again
+                                  (env: REASONING=1, default: off)
+  --no-reasoning                  Explicitly disable even if REASONING=1 is set.
+  --reasoning-base-url <url>      OpenAI-compatible base URL for the reasoning LLM.
+                                  (env: OPENCONTEXT_LLM_BASE_URL, default: https://openrouter.ai/api/v1)
+  --reasoning-model <name>        Reasoning LLM model identifier.
+                                  (env: OPENCONTEXT_LLM_MODEL, default: openai/gpt-4o-mini)
+  --reasoning-timeout-ms <int>    Per-request timeout (default: 30000).
+
+  Required env when --reasoning is set:
+    OPENCONTEXT_LLM_API_KEY        Bearer token (no default)
+    OPENCONTEXT_LLM_BASE_URL       (optional) overrides --reasoning-base-url
+    OPENCONTEXT_LLM_MODEL          (optional) overrides --reasoning-model`);
 }
 
 export { buildUnified, parseUnifiedArgs, printUnifiedHelp };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@melandlabs/memory-store",
-  "version": "1.1.6",
+  "version": "1.1.7",
   "type": "module",
   "files": [
     "dist",
@@ -60,15 +60,15 @@
     "@modelcontextprotocol/sdk": "^1.25.3",
     "hono": "^4.6.14",
     "zod": "^4.3.6",
-    "@melandlabs/ai": "0.9.0",
-    "@melandlabs/contracts": "0.6.0",
+    "@melandlabs/ai": "0.9.1",
     "@melandlabs/env-config": "0.4.0",
+    "@melandlabs/rag": "0.3.0",
+    "@melandlabs/okf": "0.2.1",
+    "@melandlabs/contracts": "0.6.0",
     "@melandlabs/memory-consolidation": "0.5.2",
-    "@melandlabs/okf": "0.2.0",
-    "@melandlabs/sqlite": "0.5.1",
-    "@melandlabs/vsa": "0.3.0",
     "@melandlabs/shared": "0.4.0",
-    "@melandlabs/rag": "0.3.0"
+    "@melandlabs/vsa": "0.3.0",
+    "@melandlabs/sqlite": "0.5.2"
   },
   "peerDependencies": {
     "@melandlabs/ai-rag": "0.2.9",