@mnexium/core 0.1.0

package/src/ai/memoryExtractionService.ts

@@ -0,0 +1,265 @@
+import type { CerebrasClient } from "../providers/cerebras";
+import type { JsonLlmClient } from "./types";
+
+export interface ExtractedClaim {
+  predicate: string;
+  object_value: string;
+  claim_type?: string;
+  confidence?: number;
+}
+
+export interface ExtractedMemory {
+  text: string;
+  kind: "fact" | "preference" | "context" | "note" | "event" | "trait";
+  importance: number;
+  confidence: number;
+  is_temporal: boolean;
+  visibility: "private" | "shared" | "public";
+  tags: string[];
+  claims: ExtractedClaim[];
+}
+
+export interface MemoryExtractionResult {
+  memories: ExtractedMemory[];
+}
+
+export interface MemoryExtractionService {
+  name: string;
+  extract(args: {
+    subject_id: string;
+    text: string;
+    force?: boolean;
+    conversation_context?: string[];
+  }): Promise<MemoryExtractionResult>;
+}
+
+export interface CreateLLMMemoryExtractionServiceOptions {
+  llm: JsonLlmClient;
+}
+
+function asBool(value: unknown, fallback = false): boolean {
+  if (typeof value === "boolean") return value;
+  if (typeof value === "string") return value.toLowerCase() === "true";
+  return fallback;
+}
+
+function asNumber(value: unknown, fallback: number, min: number, max: number): number {
+  const n = Number(value);
+  if (!Number.isFinite(n)) return fallback;
+  if (n < min) return min;
+  if (n > max) return max;
+  return n;
+}
+
+function asStringArray(value: unknown): string[] {
+  if (!Array.isArray(value)) return [];
+  return value.map((v) => String(v || "").trim()).filter(Boolean);
+}
+
+function normalizeKind(value: unknown): ExtractedMemory["kind"] {
+  const v = String(value || "").toLowerCase().trim();
+  if (v === "fact" || v === "preference" || v === "context" || v === "note" || v === "event" || v === "trait") {
+    return v;
+  }
+  return "fact";
+}
+
+function normalizeVisibility(value: unknown): ExtractedMemory["visibility"] {
+  const v = String(value || "").toLowerCase().trim();
+  if (v === "private" || v === "shared" || v === "public") return v;
+  return "private";
+}
+
+function normalizeClaim(raw: unknown): ExtractedClaim | null {
+  if (!raw || typeof raw !== "object") return null;
+  const obj = raw as Record<string, unknown>;
+  const predicate = String(obj.predicate || "").trim();
+  const objectValue = String(obj.object_value || "").trim();
+  if (!predicate || !objectValue) return null;
+  const out: ExtractedClaim = {
+    predicate,
+    object_value: objectValue,
+  };
+  const claimType = String(obj.claim_type || "").trim();
+  if (claimType) out.claim_type = claimType;
+  const confidence = Number(obj.confidence);
+  if (Number.isFinite(confidence)) out.confidence = Math.max(0, Math.min(1, confidence));
+  return out;
+}
+
+function normalizeMemory(raw: unknown): ExtractedMemory | null {
+  if (!raw || typeof raw !== "object") return null;
+  const obj = raw as Record<string, unknown>;
+  const text = String(obj.text || "").trim();
+  if (!text) return null;
+  const claimsRaw = Array.isArray(obj.claims) ? obj.claims : [];
+  const claims = claimsRaw.map(normalizeClaim).filter((v): v is ExtractedClaim => !!v);
+  return {
+    text,
+    kind: normalizeKind(obj.kind),
+    importance: asNumber(obj.importance, 50, 0, 100),
+    confidence: asNumber(obj.confidence, 0.8, 0, 1),
+    is_temporal: asBool(obj.is_temporal, false),
+    visibility: normalizeVisibility(obj.visibility),
+    tags: asStringArray(obj.tags),
+    claims,
+  };
+}
+
+function normalizePredicate(raw: string): string {
+  return raw
+    .toLowerCase()
+    .replace(/[^a-z0-9_ ]/g, "")
+    .trim()
+    .replace(/\s+/g, "_");
+}
+
+function buildSimpleClaims(text: string): ExtractedClaim[] {
+  const claims: ExtractedClaim[] = [];
+  const push = (predicate: string, value: string, claimType = "fact", confidence = 0.65) => {
+    const p = normalizePredicate(predicate);
+    const v = String(value || "").trim();
+    if (!p || !v) return;
+    const key = `${p}::${v.toLowerCase()}`;
+    if (claims.some((c) => `${c.predicate}::${c.object_value.toLowerCase()}` === key)) return;
+    claims.push({
+      predicate: p,
+      object_value: v,
+      claim_type: claimType,
+      confidence,
+    });
+  };
+
+  const patterns: Array<[RegExp, (m: RegExpMatchArray) => void]> = [
+    [/my name is\s+([^.,!?\n]+)/i, (m) => push("name", m[1], "fact", 0.9)],
+    [/i live in\s+([^.,!?\n]+)/i, (m) => push("lives_in", m[1], "fact", 0.85)],
+    [/i work at\s+([^.,!?\n]+)/i, (m) => push("works_at", m[1], "fact", 0.85)],
+    [/my favorite\s+([a-zA-Z ]+)\s+is\s+([^.,!?\n]+)/i, (m) => push(`favorite_${m[1]}`, m[2], "preference", 0.85)],
+    [/i like\s+([^.,!?\n]+)/i, (m) => push("likes", m[1], "preference", 0.7)],
+  ];
+
+  for (const [pattern, handler] of patterns) {
+    const match = text.match(pattern);
+    if (match) handler(match);
+  }
+
+  return claims;
+}
+
+function simpleExtract(args: {
+  text: string;
+  force?: boolean;
+}): MemoryExtractionResult {
+  const text = String(args.text || "").trim().replace(/\s+/g, " ");
+  if (!text) return { memories: [] };
+
+  const lower = text.toLowerCase();
+  const trivial =
+    /^(ok|thanks|thank you|cool|nice|yes|no|yep|nope|hi|hello|hey)\b/.test(lower) &&
+    text.length < 40;
+
+  if (trivial && !args.force) {
+    return { memories: [] };
+  }
+
+  const claims = buildSimpleClaims(text);
+  return {
+    memories: [
+      {
+        text: text.slice(0, 2000),
+        kind: claims.length > 0 ? "fact" : "note",
+        importance: args.force ? 70 : 50,
+        confidence: claims.length > 0 ? 0.75 : 0.6,
+        is_temporal: false,
+        visibility: "private",
+        tags: ["simple_mode"],
+        claims,
+      },
+    ],
+  };
+}
+
+export function createLLMMemoryExtractionService(
+  options: CreateLLMMemoryExtractionServiceOptions,
+): MemoryExtractionService {
+  const { llm } = options;
+
+  return {
+    name: `${llm.provider}:${llm.model}`,
+    async extract(args) {
+      const text = String(args.text || "").trim();
+      if (!text) return { memories: [] };
+
+      const force = args.force === true;
+      const contextBlock = Array.isArray(args.conversation_context) && args.conversation_context.length > 0
+        ? `\nRecent conversation:\n${args.conversation_context.map((m, i) => `${i + 1}. ${m}`).join("\n")}\n`
+        : "";
+
+      const systemPrompt = `You extract durable user memories from chat text.
+Return strict JSON:
+{
+  "memories": [
+    {
+      "text": "string",
+      "kind": "fact|preference|context|note|event|trait",
+      "importance": 0-100,
+      "confidence": 0-1,
+      "is_temporal": true|false,
+      "visibility": "private|shared|public",
+      "tags": ["string"],
+      "claims": [
+        {
+          "predicate": "string",
+          "object_value": "string",
+          "claim_type": "string",
+          "confidence": 0-1
+        }
+      ]
+    }
+  ]
+}
+
+Rules:
+- Prefer durable, user-specific memories.
+- Keep memory text concise and factual.
+- Use empty list when no durable memory exists.
+- If force=true, return at least one memory if possible.`;
+
+      const userPrompt = `${contextBlock}subject_id=${args.subject_id}\nforce=${force}\ntext:\n${text}`;
+
+      try {
+        const result = await llm.call({
+          systemPrompt,
+          userPrompt,
+          jsonMode: true,
+          timeoutMs: 4000,
+        });
+        const memoriesRaw = Array.isArray(result?.memories) ? result.memories : [];
+        const memories = memoriesRaw.map(normalizeMemory).filter((v): v is ExtractedMemory => !!v);
+        if (memories.length > 0) return { memories };
+      } catch {
+        // Fall through to simple extraction fallback.
+      }
+
+      return simpleExtract({ text, force });
+    },
+  };
+}
+
+export function createSimpleMemoryExtractionService(): MemoryExtractionService {
+  return {
+    name: "simple",
+    async extract(args) {
+      return simpleExtract(args);
+    },
+  };
+}
+
+// Compatibility helper to preserve previous import path.
+export function createCerebrasMemoryExtractionService(options: {
+  cerebras: CerebrasClient;
+}): MemoryExtractionService {
+  return createLLMMemoryExtractionService({
+    llm: options.cerebras as unknown as JsonLlmClient,
+  });
+}

package/src/ai/recallService.ts

@@ -0,0 +1,442 @@
+import type { CoreStore } from "../contracts/storage";
+import type { Memory } from "../contracts/types";
+import type { CerebrasClient } from "../providers/cerebras";
+import type { JsonLlmClient } from "./types";
+
+const CLASSIFY_TIMEOUT_MS = 2000;
+const RERANK_TIMEOUT_MS = 3000;
+
+export type RecallMode = "broad" | "direct" | "indirect" | "simple";
+
+type ScoredMemory = Memory & {
+  score: number;
+  effective_score: number;
+};
+
+export interface RecallService {
+  name: string;
+  search(args: {
+    project_id: string;
+    subject_id: string;
+    query: string;
+    limit: number;
+    min_score: number;
+    conversation_context?: string[];
+  }): Promise<{
+    memories: ScoredMemory[];
+    mode: RecallMode;
+    used_queries: string[];
+    predicates: string[];
+  }>;
+}
+
+export interface CreateLLMRecallServiceOptions {
+  store: CoreStore;
+  llm: JsonLlmClient;
+  embed: (text: string) => Promise<number[]>;
+}
+
+export interface CreateSimpleRecallServiceOptions {
+  store: CoreStore;
+  embed?: (text: string) => Promise<number[]>;
+}
+
+function asStringArray(value: unknown): string[] {
+  if (!Array.isArray(value)) return [];
+  return value
+    .map((v) => String(v || "").trim())
+    .filter(Boolean);
+}
+
+function scoreNum(v: unknown, fallback = 0): number {
+  const n = Number(v);
+  if (!Number.isFinite(n)) return fallback;
+  return n;
+}
+
+function clampInt(v: number, min: number, max: number): number {
+  if (!Number.isFinite(v)) return min;
+  if (v < min) return min;
+  if (v > max) return max;
+  return Math.floor(v);
+}
+
+function dedupeQueries(items: string[]): string[] {
+  const seen = new Set<string>();
+  const out: string[] = [];
+  for (const item of items) {
+    const key = item.toLowerCase().trim();
+    if (!key || seen.has(key)) continue;
+    seen.add(key);
+    out.push(item.trim());
+  }
+  return out;
+}
+
+function toIso(value: unknown): string {
+  const s = String(value || "");
+  const d = new Date(s);
+  if (Number.isNaN(d.getTime())) return new Date(0).toISOString();
+  return d.toISOString();
+}
+
+async function classifyRecallQuery(params: {
+  llm: JsonLlmClient;
+  query: string;
+  conversationContext: string[];
+}): Promise<{
+  mode: Exclude<RecallMode, "simple">;
+  predicates: string[];
+  searchHints: string[];
+  expandedQueries: string[];
+}> {
+  const contextBlock = params.conversationContext.length
+    ? `\nRecent conversation:\n${params.conversationContext
+        .map((m, i) => `${i + 1}. ${m}`)
+        .join("\n")}\n`
+    : "";
+
+  const systemPrompt = `You are a memory retrieval router.
+Classify a user query into:
+- broad: asks for overall summary/profile
+- direct: asks for specific personal fact
+- indirect: asks for advice where personal context helps
+
+Also extract:
+- predicates: structured fields likely needed (0-3)
+- search_hints: short keyword phrases (1-3)
+- expanded_queries: only for indirect mode (0-3)
+
+Return strict JSON:
+{
+  "mode":"broad|direct|indirect",
+  "predicates":["..."],
+  "search_hints":["..."],
+  "expanded_queries":["..."]
+}`;
+
+  const userPrompt = `${contextBlock}User message: "${params.query}"`;
+
+  try {
+    const result = await params.llm.call({
+      systemPrompt,
+      userPrompt,
+      jsonMode: true,
+      timeoutMs: CLASSIFY_TIMEOUT_MS,
+    });
+    const mode: Exclude<RecallMode, "simple"> =
+      result?.mode === "broad" || result?.mode === "direct" || result?.mode === "indirect"
+        ? result.mode
+        : "indirect";
+    return {
+      mode,
+      predicates: asStringArray(result?.predicates).slice(0, 3),
+      searchHints: asStringArray(result?.search_hints).slice(0, 3),
+      expandedQueries: asStringArray(result?.expanded_queries).slice(0, 3),
+    };
+  } catch {
+    return {
+      mode: "indirect",
+      predicates: [],
+      searchHints: [],
+      expandedQueries: [],
+    };
+  }
+}
+
+async function rerankMemories(params: {
+  llm: JsonLlmClient;
+  query: string;
+  conversationContext: string[];
+  candidates: ScoredMemory[];
+  topK: number;
+}): Promise<ScoredMemory[]> {
+  if (params.candidates.length === 0) return [];
+  const filtered = params.candidates.filter((m) => String(m.text || "").trim().length >= 10);
+  if (filtered.length === 0) return [];
+  if (filtered.length <= params.topK) return filtered;
+  const contextBlock = params.conversationContext.length
+    ? `\nRecent conversation:\n${params.conversationContext
+        .map((m, i) => `${i + 1}. ${m}`)
+        .join("\n")}\n`
+    : "";
+
+  const memories = filtered.map((m, i) => `[${i}] ${m.text}`).join("\n");
+  const systemPrompt = `You are a memory relevance judge.
+Given a user query and candidate memories, select relevant memories and rank them.
+Return strict JSON:
+{
+  "ranked": [
+    { "index": 0, "relevant": true, "score": 0.92 }
+  ]
+}
+Rules:
+- Keep only relevant=true entries.
+- score is 0..1.
+- Return at most ${params.topK} entries.`;
+  const userPrompt = `${contextBlock}User query: "${params.query}"\n\nCandidates:\n${memories}`;
+
+  try {
+    const result = await params.llm.call({
+      systemPrompt,
+      userPrompt,
+      jsonMode: true,
+      timeoutMs: RERANK_TIMEOUT_MS,
+    });
+    const ranked = Array.isArray(result?.ranked) ? result.ranked : [];
+    const selected: Array<{ idx: number; score: number }> = [];
+    for (const item of ranked) {
+      const idx = Number(item?.index);
+      if (!Number.isInteger(idx) || idx < 0 || idx >= filtered.length) continue;
+      if (item?.relevant !== true) continue;
+      selected.push({ idx, score: scoreNum(item?.score, 0.5) });
+    }
+    if (selected.length === 0) return [];
+    selected.sort((a, b) => b.score - a.score);
+    return selected.slice(0, params.topK).map((row) => {
+      const base = filtered[row.idx];
+      const rerankScore = row.score * 100;
+      return {
+        ...base,
+        effective_score: Math.max(base.effective_score, rerankScore),
+        score: Math.max(base.score, rerankScore),
+      };
+    });
+  } catch {
+    return filtered.slice(0, params.topK);
+  }
+}
+
+export function createLLMRecallService(options: CreateLLMRecallServiceOptions): RecallService {
+  const { store, llm, embed } = options;
+
+  return {
+    name: `${llm.provider}:${llm.model}`,
+    async search(args) {
+      const query = String(args.query || "").trim();
+      const limit = clampInt(args.limit || 25, 1, 200);
+      const minScore = Number.isFinite(Number(args.min_score)) ? Number(args.min_score) : 30;
+      const conversationContext = Array.isArray(args.conversation_context)
+        ? args.conversation_context.map((v) => String(v || "")).filter(Boolean).slice(-5)
+        : [];
+
+      if (!query) {
+        return {
+          memories: [],
+          mode: "indirect",
+          used_queries: [],
+          predicates: [],
+        };
+      }
+
+      const classified = await classifyRecallQuery({
+        llm,
+        query,
+        conversationContext,
+      });
+
+      if (classified.mode === "broad") {
+        const rows = await store.listMemories({
+          project_id: args.project_id,
+          subject_id: args.subject_id,
+          limit: Math.min(limit * 3, 200),
+          offset: 0,
+          include_deleted: false,
+          include_superseded: false,
+        });
+        const sorted = [...rows].sort((a, b) => {
+          const imp = Number(b.importance) - Number(a.importance);
+          if (imp !== 0) return imp;
+          return toIso(b.created_at).localeCompare(toIso(a.created_at));
+        });
+        const broadLimit = Math.max(limit, 20);
+        const memories: ScoredMemory[] = sorted.slice(0, broadLimit).map((m) => ({
+          ...m,
+          score: 100,
+          effective_score: Number(m.importance || 0),
+        }));
+        return {
+          memories,
+          mode: "broad",
+          used_queries: [query],
+          predicates: classified.predicates,
+        };
+      }
+
+      const queries = dedupeQueries([
+        query,
+        ...classified.searchHints,
+        ...(classified.mode === "indirect" ? classified.expandedQueries : []),
+      ]).slice(0, 6);
+
+      const allResults: Array<{
+        q: string;
+        rank: number;
+        rows: ScoredMemory[];
+      }> = [];
+
+      const searches = queries.map(async (q, rank) => {
+        let embedding: number[] | null = null;
+        try {
+          const emb = await embed(q);
+          embedding = Array.isArray(emb) && emb.length > 0 ? emb : null;
+        } catch {
+          embedding = null;
+        }
+        const rows = await store.searchMemories({
+          project_id: args.project_id,
+          subject_id: args.subject_id,
+          q,
+          query_embedding: embedding,
+          limit: Math.min(limit * 2, 200),
+          min_score: minScore,
+        });
+        allResults.push({
+          q,
+          rank,
+          rows: rows as ScoredMemory[],
+        });
+      });
+
+      await Promise.all(searches);
+
+      if (classified.mode === "direct" && classified.predicates.length > 0) {
+        const truth = await store.getCurrentTruth({
+          project_id: args.project_id,
+          subject_id: args.subject_id,
+        });
+        const sourceMemoryIds = truth
+          .filter((row) => classified.predicates.includes(String(row.predicate)))
+          .map((row) => row.source_memory_id)
+          .filter((id): id is string => !!id);
+        const uniqueIds = [...new Set(sourceMemoryIds)];
+        for (const memoryId of uniqueIds) {
+          const mem = await store.getMemory({
+            project_id: args.project_id,
+            id: memoryId,
+          });
+          if (!mem || mem.is_deleted || mem.status !== "active") continue;
+          allResults.push({
+            q: "__claims__",
+            rank: 0,
+            rows: [
+              {
+                ...mem,
+                score: 100,
+                effective_score: 120,
+              },
+            ],
+          });
+        }
+      }
+
+      const byId = new Map<string, ScoredMemory>();
+      for (const group of allResults) {
+        for (const row of group.rows) {
+          const qBoost = group.rank === 0 ? 1 : 1 - group.rank * 0.03;
+          const boosted: ScoredMemory = {
+            ...row,
+            score: scoreNum(row.score, 0) * qBoost,
+            effective_score: scoreNum(row.effective_score, 0) * qBoost,
+          };
+          const existing = byId.get(row.id);
+          if (!existing || boosted.effective_score > existing.effective_score) {
+            byId.set(row.id, boosted);
+          }
+        }
+      }
+
+      let candidates = [...byId.values()].sort((a, b) => b.effective_score - a.effective_score);
+
+      if (classified.mode === "direct") {
+        const hasClaimResults = allResults.some((group) => group.q === "__claims__" && group.rows.length > 0);
+        if (hasClaimResults) {
+          const directLimit = Math.min(limit, 5);
+          candidates = candidates
+            .sort((a, b) => b.effective_score - a.effective_score)
+            .slice(0, directLimit);
+        } else if (candidates.length > limit) {
+          candidates = await rerankMemories({
+            llm,
+            query,
+            conversationContext,
+            candidates,
+            topK: limit,
+          });
+        } else {
+          const directLimit = Math.min(limit, 5);
+          candidates = candidates
+            .sort((a, b) => b.effective_score - a.effective_score)
+            .slice(0, directLimit);
+        }
+      } else if (candidates.length > limit) {
+        candidates = await rerankMemories({
+          llm,
+          query,
+          conversationContext,
+          candidates,
+          topK: limit,
+        });
+      } else {
+        candidates = candidates.slice(0, limit);
+      }
+
+      return {
+        memories: candidates.slice(0, limit),
+        mode: classified.mode,
+        used_queries: queries,
+        predicates: classified.predicates,
+      };
+    },
+  };
+}
+
+export function createSimpleRecallService(options: CreateSimpleRecallServiceOptions): RecallService {
+  const { store, embed } = options;
+  return {
+    name: "simple",
+    async search(args) {
+      const query = String(args.query || "").trim();
+      const limit = clampInt(args.limit || 25, 1, 200);
+      const minScore = Number.isFinite(Number(args.min_score)) ? Number(args.min_score) : 30;
+      let embedding: number[] | null = null;
+      if (embed) {
+        try {
+          const emb = await embed(query);
+          embedding = Array.isArray(emb) && emb.length > 0 ? emb : null;
+        } catch {
+          embedding = null;
+        }
+      }
+
+      const memories = (await store.searchMemories({
+        project_id: args.project_id,
+        subject_id: args.subject_id,
+        q: query,
+        query_embedding: embedding,
+        limit,
+        min_score: minScore,
+      })) as ScoredMemory[];
+
+      return {
+        memories,
+        mode: "simple",
+        used_queries: [query],
+        predicates: [],
+      };
+    },
+  };
+}
+
+// Compatibility helper to preserve previous import path.
+export function createCerebrasRecallService(options: {
+  store: CoreStore;
+  cerebras: CerebrasClient;
+  embed: (text: string) => Promise<number[]>;
+}): RecallService {
+  return createLLMRecallService({
+    store: options.store,
+    llm: options.cerebras as unknown as JsonLlmClient,
+    embed: options.embed,
+  });
+}

package/src/ai/types.ts

@@ -0,0 +1,11 @@
+export interface JsonLlmClient {
+  provider: "cerebras" | "openai" | "unknown";
+  model: string;
+  call: (opts: {
+    systemPrompt: string;
+    userPrompt: string;
+    jsonMode?: boolean;
+    timeoutMs?: number;
+    temperature?: number;
+  }) => Promise<any | null>;
+}