@wipcomputer/memory-crystal 0.7.34-alpha.2 → 0.7.34-alpha.4

package/dist/search-pipeline-7UJMXPLO.js

@@ -1,280 +0,0 @@
-// src/llm.ts
-import { existsSync, readFileSync } from "fs";
-import { join } from "path";
-import { homedir } from "os";
-import { execSync } from "child_process";
-var expansionCache = /* @__PURE__ */ new Map();
-var detectedProvider = null;
-var detectionDone = false;
-function getOpSecret(itemName, fieldLabel) {
-  try {
-    const saTokenPath = join(homedir(), ".openclaw/secrets/op-sa-token");
-    if (!existsSync(saTokenPath)) return void 0;
-    const saToken = readFileSync(saTokenPath, "utf-8").trim();
-    const result = execSync(
-      `OP_SERVICE_ACCOUNT_TOKEN="${saToken}" op item get "${itemName}" --vault "Agent Secrets" --fields "${fieldLabel}" --reveal`,
-      { encoding: "utf-8", timeout: 5e3, stdio: ["pipe", "pipe", "pipe"] }
-    ).trim();
-    return result || void 0;
-  } catch {
-    return void 0;
-  }
-}
-async function detectProvider() {
-  if (detectionDone && detectedProvider) return detectedProvider;
-  detectionDone = true;
-  try {
-    const resp = await fetch("http://localhost:8080/v1/models", { signal: AbortSignal.timeout(1e3) });
-    if (resp.ok) {
-      const data = await resp.json();
-      const model = data?.data?.[0]?.id || "default";
-      detectedProvider = { provider: "mlx", baseURL: "http://localhost:8080/v1", apiKey: "not-needed", model };
-      process.stderr.write(`[memory-crystal] LLM provider: MLX (${model})
-`);
-      return detectedProvider;
-    }
-  } catch {
-  }
-  try {
-    const resp = await fetch("http://localhost:11434/api/tags", { signal: AbortSignal.timeout(1e3) });
-    if (resp.ok) {
-      const data = await resp.json();
-      const models = data?.models || [];
-      const embeddingOnly = ["nomic-embed-text", "mxbai-embed", "all-minilm", "snowflake-arctic-embed"];
-      const chatModel = models.find((m) => !embeddingOnly.some((e) => m.name.startsWith(e)));
-      if (chatModel) {
-        detectedProvider = { provider: "ollama", baseURL: "http://localhost:11434/v1", apiKey: "ollama", model: chatModel.name };
-        process.stderr.write(`[memory-crystal] LLM provider: Ollama (${chatModel.name})
-`);
-        return detectedProvider;
-      }
-    }
-  } catch {
-  }
-  const anthropicKey = process.env.ANTHROPIC_API_KEY || getOpSecret("Anthropic Auth Token - buglers lads", "credential");
-  if (anthropicKey) {
-    detectedProvider = { provider: "anthropic", baseURL: "https://api.anthropic.com", apiKey: anthropicKey, model: "claude-haiku-4-5-20251001" };
-    process.stderr.write("[memory-crystal] LLM provider: Anthropic API\n");
-    return detectedProvider;
-  }
-  const openaiKey = process.env.OPENAI_API_KEY || getOpSecret("OpenAI API", "api key");
-  if (openaiKey) {
-    detectedProvider = { provider: "openai", baseURL: "https://api.openai.com/v1", apiKey: openaiKey, model: "gpt-4o-mini" };
-    process.stderr.write("[memory-crystal] LLM provider: OpenAI API\n");
-    return detectedProvider;
-  }
-  detectedProvider = { provider: "none", baseURL: "", apiKey: "", model: "" };
-  process.stderr.write("[memory-crystal] LLM provider: none (deep search unavailable)\n");
-  return detectedProvider;
-}
-async function chatComplete(config, messages, maxTokens = 300) {
-  if (config.provider === "anthropic") {
-    return anthropicComplete(config, messages, maxTokens);
-  }
-  const resp = await fetch(`${config.baseURL}/chat/completions`, {
-    method: "POST",
-    headers: {
-      "Content-Type": "application/json",
-      "Authorization": `Bearer ${config.apiKey}`
-    },
-    body: JSON.stringify({
-      model: config.model,
-      messages,
-      max_tokens: maxTokens,
-      temperature: 0.7
-    })
-  });
-  if (!resp.ok) throw new Error(`LLM request failed: ${resp.status}`);
-  const data = await resp.json();
-  return data.choices?.[0]?.message?.content || "";
-}
-async function anthropicComplete(config, messages, maxTokens) {
-  const systemMsg = messages.find((m) => m.role === "system");
-  const userMessages = messages.filter((m) => m.role !== "system");
-  const body = {
-    model: config.model,
-    max_tokens: maxTokens,
-    messages: userMessages
-  };
-  if (systemMsg) body.system = systemMsg.content;
-  const resp = await fetch("https://api.anthropic.com/v1/messages", {
-    method: "POST",
-    headers: {
-      "Content-Type": "application/json",
-      "x-api-key": config.apiKey,
-      "anthropic-version": "2023-06-01"
-    },
-    body: JSON.stringify(body)
-  });
-  if (!resp.ok) throw new Error(`Anthropic request failed: ${resp.status}`);
-  const data = await resp.json();
-  return data.content?.[0]?.text || "";
-}
-var EXPAND_PROMPT = `You are a search query expander. Given a search query, generate exactly 3 variations to improve search recall.
-
-Output exactly 3 lines in this format (no other text):
-lex: <keyword-focused variation for full-text search>
-vec: <semantic variation rephrased for embedding similarity>
-hyde: <hypothetical document snippet that would answer this query>
-
-Rules:
-- Each variation must contain at least one term from the original query
-- Keep variations concise (under 30 words each)
-- lex should use specific keywords and synonyms
-- vec should rephrase the intent naturally
-- hyde should be a short passage as if answering the query`;
-async function expandQuery(query) {
-  const cached = expansionCache.get(query);
-  if (cached) return cached;
-  const config = await detectProvider();
-  if (config.provider === "none") return [];
-  try {
-    const result = await chatComplete(config, [
-      { role: "system", content: EXPAND_PROMPT },
-      { role: "user", content: query }
-    ], 300);
-    const lines = result.trim().split("\n");
-    const queryLower = query.toLowerCase();
-    const queryTerms = queryLower.replace(/[^a-z0-9\s]/g, " ").split(/\s+/).filter(Boolean);
-    const hasQueryTerm = (text) => {
-      const lower = text.toLowerCase();
-      if (queryTerms.length === 0) return true;
-      return queryTerms.some((term) => lower.includes(term));
-    };
-    const variations = lines.map((line) => {
-      const colonIdx = line.indexOf(":");
-      if (colonIdx === -1) return null;
-      const type = line.slice(0, colonIdx).trim();
-      if (type !== "lex" && type !== "vec" && type !== "hyde") return null;
-      const text = line.slice(colonIdx + 1).trim();
-      if (!text || !hasQueryTerm(text)) return null;
-      return { type, text };
-    }).filter((v) => v !== null);
-    if (variations.length > 0) {
-      expansionCache.set(query, variations);
-      return variations;
-    }
-  } catch (err) {
-    process.stderr.write(`[memory-crystal] Query expansion failed: ${err.message}
-`);
-  }
-  const fallback = [
-    { type: "lex", text: query },
-    { type: "vec", text: query },
-    { type: "hyde", text: `Information about ${query}` }
-  ];
-  return fallback;
-}
-var RERANK_PROMPT = `You are a search result re-ranker. Given a query and a list of text passages, rate each passage's relevance to the query.
-
-Output one line per passage in this exact format:
-<index>: <score>
-
-Where index is the passage number (0-based) and score is a float from 0.0 to 1.0.
-- 1.0 = perfectly relevant, directly answers the query
-- 0.7 = highly relevant, closely related
-- 0.4 = somewhat relevant, tangentially related
-- 0.1 = barely relevant
-- 0.0 = not relevant at all
-
-Rate ALL passages. Output nothing else.`;
-async function rerankResults(query, passages) {
-  const config = await detectProvider();
-  if (config.provider === "none") {
-    return passages.map((_, i) => ({ index: i, score: 1 - i * 0.01 }));
-  }
-  try {
-    const passageList = passages.map((p, i) => `[${i}] ${p.slice(0, 500)}`).join("\n\n");
-    const result = await chatComplete(config, [
-      { role: "system", content: RERANK_PROMPT },
-      { role: "user", content: `Query: ${query}
-
-Passages:
-${passageList}` }
-    ], 200);
-    const results = [];
-    for (const line of result.trim().split("\n")) {
-      const match = line.match(/^(\d+):\s*([\d.]+)/);
-      if (match) {
-        results.push({ index: parseInt(match[1]), score: parseFloat(match[2]) });
-      }
-    }
-    const scored = new Set(results.map((r) => r.index));
-    for (let i = 0; i < passages.length; i++) {
-      if (!scored.has(i)) results.push({ index: i, score: 0 });
-    }
-    return results.sort((a, b) => b.score - a.score);
-  } catch (err) {
-    process.stderr.write(`[memory-crystal] Reranking failed: ${err.message}
-`);
-    return passages.map((_, i) => ({ index: i, score: 1 - i * 0.01 }));
-  }
-}
-
-// src/search-pipeline.ts
-var STRONG_SIGNAL_MIN_SCORE = 0.85;
-var STRONG_SIGNAL_MIN_GAP = 0.15;
-var RERANK_CANDIDATE_LIMIT = 40;
-async function deepSearch(crystal, query, options = {}) {
-  const limit = options.limit || 5;
-  const filter = options.filter;
-  const provider = await detectProvider();
-  if (provider.provider === "none") {
-    return crystal.search(query, limit, filter);
-  }
-  const db = crystal.sqliteDb;
-  if (!db) return crystal.search(query, limit, filter);
-  const sinceDate = filter?.since ? crystal.parseSince(filter.since) : void 0;
-  const internalFilter = { ...filter, sinceDate };
-  const initialFts = crystal.searchFTS(query, 20, internalFilter);
-  const topScore = initialFts[0]?.score ?? 0;
-  const secondScore = initialFts[1]?.score ?? 0;
-  const hasStrongSignal = initialFts.length > 0 && topScore >= STRONG_SIGNAL_MIN_SCORE && topScore - secondScore >= STRONG_SIGNAL_MIN_GAP;
-  const expanded = hasStrongSignal ? [] : await expandQuery(query);
-  const allResultLists = [];
-  if (initialFts.length > 0) allResultLists.push(initialFts);
-  const [queryEmbedding] = await crystal.embed([query]);
-  const originalVec = crystal.searchVec(queryEmbedding, 30, internalFilter);
-  if (originalVec.length > 0) allResultLists.push(originalVec);
-  for (const variation of expanded) {
-    if (variation.type === "lex") {
-      const ftsResults = crystal.searchFTS(variation.text, 20, internalFilter);
-      if (ftsResults.length > 0) allResultLists.push(ftsResults);
-    } else {
-      const [embedding] = await crystal.embed([variation.text]);
-      const vecResults = crystal.searchVec(embedding, 20, internalFilter);
-      if (vecResults.length > 0) allResultLists.push(vecResults);
-    }
-  }
-  const weights = allResultLists.map((_, i) => i < 2 ? 2 : 1);
-  const fused = crystal.reciprocalRankFusion(allResultLists, weights);
-  const candidates = fused.slice(0, RERANK_CANDIDATE_LIMIT);
-  if (candidates.length === 0) return [];
-  const passages = candidates.map((c) => c.text.slice(0, 500));
-  const reranked = await rerankResults(query, passages);
-  const now = Date.now();
-  const blended = reranked.map((r) => {
-    const candidate = candidates[r.index];
-    if (!candidate) return null;
-    const rrfRank = r.index + 1;
-    let rrfWeight;
-    if (rrfRank <= 3) rrfWeight = 0.75;
-    else if (rrfRank <= 10) rrfWeight = 0.6;
-    else rrfWeight = 0.4;
-    const rrfScore = 1 / rrfRank;
-    const blendedScore = rrfWeight * rrfScore + (1 - rrfWeight) * r.score;
-    const ageDays = candidate.created_at ? (now - new Date(candidate.created_at).getTime()) / 864e5 : 0;
-    const recency = candidate.created_at ? crystal.recencyWeight(ageDays) : 1;
-    const finalScore = Math.min(blendedScore * recency * 8, 1);
-    const freshness = candidate.created_at ? crystal.freshnessLabel(ageDays) : void 0;
-    return {
-      ...candidate,
-      score: finalScore,
-      freshness
-    };
-  }).filter((r) => r !== null);
-  return blended.sort((a, b) => b.score - a.score).slice(0, limit);
-}
-export {
-  deepSearch
-};

package/dist/search-pipeline-CBV25NX7.js

@@ -1,99 +0,0 @@
-import {
-  detectProvider,
-  expandQuery,
-  rerankResults
-} from "./chunk-NX647OM3.js";
-
-// src/search-pipeline.ts
-var STRONG_SIGNAL_MIN_SCORE = 0.85;
-var STRONG_SIGNAL_MIN_GAP = 0.15;
-var DEFAULT_CANDIDATE_LIMIT = 40;
-async function deepSearch(crystal, query, options = {}) {
-  const limit = options.limit || 5;
-  const candidateLimit = options.candidateLimit || DEFAULT_CANDIDATE_LIMIT;
-  const intent = options.intent;
-  const filter = options.filter;
-  const explain = options.explain || false;
-  const provider = await detectProvider();
-  if (provider.provider === "none") {
-    return crystal.search(query, limit, filter);
-  }
-  const db = crystal.sqliteDb;
-  if (!db) return crystal.search(query, limit, filter);
-  const sinceDate = filter?.since ? crystal.parseSince(filter.since) : void 0;
-  const untilDate = filter?.until ? crystal.parseSince(filter.until) : void 0;
-  const internalFilter = { ...filter, sinceDate, untilDate };
-  const initialFts = crystal.searchFTS(query, 20, internalFilter);
-  const topScore = initialFts[0]?.score ?? 0;
-  const secondScore = initialFts[1]?.score ?? 0;
-  const hasStrongSignal = !intent && initialFts.length > 0 && topScore >= STRONG_SIGNAL_MIN_SCORE && topScore - secondScore >= STRONG_SIGNAL_MIN_GAP;
-  const expanded = hasStrongSignal ? [] : await expandQuery(query, intent);
-  const allResultLists = [];
-  if (initialFts.length > 0) allResultLists.push(initialFts);
-  const [queryEmbedding] = await crystal.embed([query]);
-  const originalVec = crystal.searchVec(queryEmbedding, 30, internalFilter);
-  if (originalVec.length > 0) allResultLists.push(originalVec);
-  for (const variation of expanded) {
-    if (variation.type === "lex") {
-      const ftsResults = crystal.searchFTS(variation.text, 20, internalFilter);
-      if (ftsResults.length > 0) allResultLists.push(ftsResults);
-    } else {
-      const [embedding] = await crystal.embed([variation.text]);
-      const vecResults = crystal.searchVec(embedding, 20, internalFilter);
-      if (vecResults.length > 0) allResultLists.push(vecResults);
-    }
-  }
-  const weights = allResultLists.map((_, i) => i < 2 ? 2 : 1);
-  const fused = crystal.reciprocalRankFusion(allResultLists, weights);
-  const candidates = fused.slice(0, candidateLimit);
-  if (candidates.length === 0) return [];
-  const ftsScoreMap = /* @__PURE__ */ new Map();
-  const vecScoreMap = /* @__PURE__ */ new Map();
-  if (explain) {
-    for (const r of initialFts) ftsScoreMap.set(r.text.slice(0, 200), r.score);
-    for (const r of originalVec) vecScoreMap.set(r.text.slice(0, 200), r.score);
-  }
-  const passages = candidates.map((c) => c.text.slice(0, 500));
-  const rerankQuery = intent ? `${intent}: ${query}` : query;
-  const reranked = await rerankResults(rerankQuery, passages);
-  const now = Date.now();
-  const blended = reranked.map((r) => {
-    const candidate = candidates[r.index];
-    if (!candidate) return null;
-    const rrfRank = r.index + 1;
-    let rrfWeight;
-    if (rrfRank <= 3) rrfWeight = 0.75;
-    else if (rrfRank <= 10) rrfWeight = 0.6;
-    else rrfWeight = 0.4;
-    const rrfScore = 1 / rrfRank;
-    const blendedScore = rrfWeight * rrfScore + (1 - rrfWeight) * r.score;
-    const ageDays = candidate.created_at ? (now - new Date(candidate.created_at).getTime()) / 864e5 : 0;
-    const recency = candidate.created_at ? crystal.recencyWeight(ageDays) : 1;
-    const finalScore = blendedScore * recency;
-    const freshness = candidate.created_at ? crystal.freshnessLabel(ageDays) : void 0;
-    const result = {
-      ...candidate,
-      score: finalScore,
-      freshness
-    };
-    if (explain) {
-      const dedup = candidate.text.slice(0, 200);
-      result.explain = {
-        fts_score: ftsScoreMap.get(dedup),
-        vec_score: vecScoreMap.get(dedup),
-        rrf_rank: rrfRank,
-        rrf_score: rrfScore,
-        rerank_score: r.score,
-        recency_weight: recency,
-        final_score: finalScore
-      };
-    }
-    return result;
-  }).filter((r) => r !== null);
-  const sorted = blended.sort((a, b) => b.score - a.score).slice(0, limit);
-  const topNormScore = sorted[0]?.score || 1;
-  return sorted.map((r) => ({ ...r, score: Math.min(r.score / topNormScore * 0.95, 0.95) }));
-}
-export {
-  deepSearch
-};

package/dist/search-pipeline-DQTRLGBH.js

@@ -1,74 +0,0 @@
-import {
-  detectProvider,
-  expandQuery,
-  rerankResults
-} from "./chunk-7IUE7ODU.js";
-import "./chunk-PNKVD2UK.js";
-
-// src/search-pipeline.ts
-var STRONG_SIGNAL_MIN_SCORE = 0.85;
-var STRONG_SIGNAL_MIN_GAP = 0.15;
-var RERANK_CANDIDATE_LIMIT = 40;
-async function deepSearch(crystal, query, options = {}) {
-  const limit = options.limit || 5;
-  const filter = options.filter;
-  const provider = await detectProvider();
-  if (provider.provider === "none") {
-    return crystal.search(query, limit, filter);
-  }
-  const db = crystal.sqliteDb;
-  if (!db) return crystal.search(query, limit, filter);
-  const sinceDate = filter?.since ? crystal.parseSince(filter.since) : void 0;
-  const internalFilter = { ...filter, sinceDate };
-  const initialFts = crystal.searchFTS(query, 20, internalFilter);
-  const topScore = initialFts[0]?.score ?? 0;
-  const secondScore = initialFts[1]?.score ?? 0;
-  const hasStrongSignal = initialFts.length > 0 && topScore >= STRONG_SIGNAL_MIN_SCORE && topScore - secondScore >= STRONG_SIGNAL_MIN_GAP;
-  const expanded = hasStrongSignal ? [] : await expandQuery(query);
-  const allResultLists = [];
-  if (initialFts.length > 0) allResultLists.push(initialFts);
-  const [queryEmbedding] = await crystal.embed([query]);
-  const originalVec = crystal.searchVec(queryEmbedding, 30, internalFilter);
-  if (originalVec.length > 0) allResultLists.push(originalVec);
-  for (const variation of expanded) {
-    if (variation.type === "lex") {
-      const ftsResults = crystal.searchFTS(variation.text, 20, internalFilter);
-      if (ftsResults.length > 0) allResultLists.push(ftsResults);
-    } else {
-      const [embedding] = await crystal.embed([variation.text]);
-      const vecResults = crystal.searchVec(embedding, 20, internalFilter);
-      if (vecResults.length > 0) allResultLists.push(vecResults);
-    }
-  }
-  const weights = allResultLists.map((_, i) => i < 2 ? 2 : 1);
-  const fused = crystal.reciprocalRankFusion(allResultLists, weights);
-  const candidates = fused.slice(0, RERANK_CANDIDATE_LIMIT);
-  if (candidates.length === 0) return [];
-  const passages = candidates.map((c) => c.text.slice(0, 500));
-  const reranked = await rerankResults(query, passages);
-  const now = Date.now();
-  const blended = reranked.map((r) => {
-    const candidate = candidates[r.index];
-    if (!candidate) return null;
-    const rrfRank = r.index + 1;
-    let rrfWeight;
-    if (rrfRank <= 3) rrfWeight = 0.75;
-    else if (rrfRank <= 10) rrfWeight = 0.6;
-    else rrfWeight = 0.4;
-    const rrfScore = 1 / rrfRank;
-    const blendedScore = rrfWeight * rrfScore + (1 - rrfWeight) * r.score;
-    const ageDays = candidate.created_at ? (now - new Date(candidate.created_at).getTime()) / 864e5 : 0;
-    const recency = candidate.created_at ? crystal.recencyWeight(ageDays) : 1;
-    const finalScore = Math.min(blendedScore * recency * 8, 1);
-    const freshness = candidate.created_at ? crystal.freshnessLabel(ageDays) : void 0;
-    return {
-      ...candidate,
-      score: finalScore,
-      freshness
-    };
-  }).filter((r) => r !== null);
-  return blended.sort((a, b) => b.score - a.score).slice(0, limit);
-}
-export {
-  deepSearch
-};