@100xprompt/chitta 0.1.4 → 0.1.5

package/README.md

@@ -118,9 +118,27 @@ opencode, Kiro, Amp, Factory, Kilo, Trae). Any other MCP client: `--print` and p
 
 | Tool | Does |
 |---|---|
-| `context_ingest` | Store text → record node + **permission edges** (ACL) + **vector chunks** + **extracted concept graph** |
-| `get_context` | Retrieve ranked, cited, permission-filtered snippets |
+| `context_ingest` | Store text → record node + **permission edges** (ACL) + **vector chunks** + **extracted concept graph** + **atomic memories** |
+| `get_context` | Retrieve ranked, cited, permission-filtered snippets + the **current memory** (latest, contradiction-resolved) |
+| `context_forget` | Forget memories that are no longer true/wanted (soft-delete, within what you may see) |
+| `context_profile` | Synthesize a profile of a person/org/entity (permanent + recent facts + connections) |
 | `context_graph` | Return the knowledge graph (concepts + relationships) the user can access |
+| `context_relate` | Graph queries over the entity graph (neighbors / path / impact / central) |
+
+## Living memory (permission-aware)
+
+Beyond storing snippets, Chitta maintains a **living-memory layer** - the part most memory
+products treat as proprietary magic, here done natively and **ACL-scoped**:
+
+- **Atomic memories** - precise typed facts ("Sarah works at Meta"), not just chunks.
+- **Contradiction → versioning** - a newer single-valued fact **supersedes** the old one
+  (`works_at`: Google → Meta); recall returns the current truth, history is kept (v1→vN).
+- **Forgetting** - `context_forget` soft-deletes by description; optional TTL
+  (`CONTEXT_MEMORY_TTL_DAYS`) retires dynamic memories, static facts are exempt. It's
+  coherent: the underlying graph fact is expired too.
+- **Permission-aware throughout** - you can only recall or forget what your ACL permits,
+  across a *shared* org graph. (Most memory layers only isolate per-user pools - they have
+  no concept of "who is allowed to remember what" inside a team.)
 
 ## Run it
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@100xprompt/chitta",
-  "version": "0.1.4",
+  "version": "0.1.5",
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.0.0",
     "sqlite-vec": "^0.1.9",

package/src/embedded/index.ts

@@ -10,7 +10,7 @@ import { SqliteVecService } from "./sqlite-vec-service"
 import { LocalHashEmbeddings } from "./local-embeddings"
 import { TransformersEmbeddings, AutoEmbeddings } from "./transformers-embeddings"
 import { Ingestor, type IngestDoc } from "./ingest"
-import { DeterministicExtractor, type KnowledgeExtractor } from "./extract"
+import { DeterministicExtractor, slugify, entityId, type KnowledgeExtractor } from "./extract"
 import { Authorizer } from "./authorizer"
 import { KgqaService } from "./kgqa-service"
 import { GraphQueryService } from "./graph-query"
@@ -19,8 +19,28 @@ import type { LlmExtractor } from "./llm-extractor"
 import type { EmbeddingProvider } from "../provider"
 import type { RetrievalResponse } from "../types"
 import { hybridSearch } from "./retrieval/hybrid-retriever"
+import { cosine } from "./retrieval/passage"
 import type { SearchTrace } from "./retrieval/trace"
 
+/** A current memory surfaced to a caller (latest version, not forgotten), ACL-scoped. */
+export interface RecalledMemory {
+  memory: string
+  version: number
+  isStatic: boolean
+  updatedAt: number
+  rootId: string
+}
+
+/** A synthesized, ACL-scoped profile of one subject - the permanent facts, the recent
+ *  (dynamic) facts, and the entities it's most connected to. Supermemory's "user profile",
+ *  but for ANY principal/entity the caller is permitted to see, not just the caller. */
+export interface Profile {
+  subject: string
+  staticFacts: string[]
+  recentFacts: string[]
+  related: string[]
+}
+
 export { SqliteStore } from "./sqlite-store"
 export { SqliteGraphProvider } from "./sqlite-graph-provider"
 export { SqliteVecService } from "./sqlite-vec-service"
@@ -149,6 +169,83 @@ export function buildEmbeddedContext(opts: EmbeddedOptions = {}) {
     return hybridSearch({ retrieval, store, graph, embeddings, reranker }, query, userId, orgId, trace, limit)
   }
 
+  // LIVING MEMORY - the permission-aware atomic-memory layer (Supermemory parity, but
+  // ACL-scoped). recallMemories returns the CURRENT truth (latest version, not forgotten)
+  // about whatever the query is asking, ranked by semantic similarity, gated by the same
+  // accessible-record set the rest of retrieval uses (leak-proof by construction).
+  async function recallMemories(query: string, userId: string, orgId: string, limit = 8): Promise<RecalledMemory[]> {
+    store.memories.sweep() // lazy TTL: retire any expired dynamic memories first
+    const accessible = await graph.getAccessibleVirtualRecordIds({ userId, orgId })
+    const vids = [...new Set(Object.values(accessible))]
+    const rows = store.memories.recall(vids)
+    if (rows.length === 0) return []
+    const qv = await (embeddings.embedQuery ? embeddings.embedQuery(query) : embeddings.embedDense(query))
+    const scored = rows.map((r) => ({ r, s: r.embedding ? cosine(qv, JSON.parse(r.embedding) as number[]) : 0 }))
+    scored.sort((a, b) => b.s - a.s)
+    return scored.slice(0, limit).map(({ r }) => ({
+      memory: r.memory, version: r.version, isStatic: !!r.is_static, updatedAt: r.updated_at, rootId: r.root_id ?? r.id,
+    }))
+  }
+
+  // Forget memories matching a description (semantic similarity OR substring), within
+  // the caller's accessible set only - you can never forget what you can't see. Soft
+  // delete (history kept, excluded from recall). Returns the memory texts forgotten.
+  async function forgetMemories(query: string, userId: string, orgId: string, reason = "forgotten by user"): Promise<string[]> {
+    const accessible = await graph.getAccessibleVirtualRecordIds({ userId, orgId })
+    const vids = [...new Set(Object.values(accessible))]
+    const rows = store.memories.recall(vids)
+    if (rows.length === 0) return []
+    const q = query.trim().toLowerCase()
+    const qv = await (embeddings.embedQuery ? embeddings.embedQuery(query) : embeddings.embedDense(query))
+    const targets = rows.filter((r) => {
+      if (r.memory.toLowerCase().includes(q)) return true
+      return r.embedding ? cosine(qv, JSON.parse(r.embedding) as number[]) >= 0.6 : false
+    })
+    if (targets.length === 0) return []
+    store.memories.forget(targets.map((r) => r.id), reason)
+    // Keep the forget COHERENT across layers: also expire the underlying typed edge so
+    // KGQA / graph queries stop asserting the fact too. subject_key is `subj|pred` (a
+    // single-valued fact) or `subj|pred|obj` (multi-valued) - both carry entity ids.
+    for (const r of targets) {
+      const parts = r.subject_key.split("|")
+      if (parts.length === 2) store.expireEdges(parts[0], parts[1])
+      else if (parts.length === 3) store.expireEdges(parts[0], parts[1], parts[2])
+    }
+    return targets.map((r) => r.memory)
+  }
+
+  // How a fact evolved: the full version chain (v1 → vN) for a memory's root. ACL is
+  // enforced by the caller (recallMemories returns only accessible roots).
+  function memoryHistory(rootId: string): Array<{ memory: string; version: number; isLatest: boolean; forgotten: boolean }> {
+    return store.memories.history(rootId).map((r) => ({
+      memory: r.memory, version: r.version, isLatest: !!r.is_latest, forgotten: !!r.is_forgotten,
+    }))
+  }
+
+  // PROFILE synthesis - roll up everything currently known about one subject into a
+  // compact, structured view: permanent facts (static), recent facts (dynamic, newest
+  // first), and the entities it's most connected to. ACL-scoped (built only from the
+  // caller's accessible memories + graph). Returns null when nothing is known. This is
+  // the Supermemory "user profile" surface, generalized to any permitted entity.
+  async function buildProfile(subject: string, userId: string, orgId: string): Promise<Profile | null> {
+    store.memories.sweep()
+    const accessible = await graph.getAccessibleVirtualRecordIds({ userId, orgId })
+    const vids = [...new Set(Object.values(accessible))]
+    const rows = store.memories.recall(vids)
+    const eid = entityId(slugify(subject))
+    const prefix = `${eid}|`
+    const mine = rows.filter((r) => r.subject_key.startsWith(prefix))
+    const nb = await graphQuery.neighbors(subject, userId, orgId)
+    const related = (nb?.neighbors ?? []).slice(0, 10).map((n) => n.label)
+    if (mine.length === 0 && related.length === 0) return null
+    const staticFacts = mine.filter((r) => r.is_static).map((r) => r.memory)
+    const recentFacts = mine
+      .filter((r) => !r.is_static)
+      .sort((a, b) => b.updated_at - a.updated_at)
+      .map((r) => r.memory)
+    return { subject: nb?.entity ?? subject, staticFacts, recentFacts, related }
+  }
+
   // Same retrieval, but also returns the pipeline TRACE (for the UI's explainability).
   async function searchTraced(query: string, userId: string, orgId: string) {
     const trace: SearchTrace = { counts: { vector: 0, keyword: 0, graph: 0, fused: 0 }, reranked: false, items: [] }
@@ -171,6 +268,11 @@ export function buildEmbeddedContext(opts: EmbeddedOptions = {}) {
       const emb = await embeddings.embedDense(r.content)
       store.addChunk(r.point_id, r.virtual_record_id, r.org_id, r.content, emb)
     }
+    // Memories carry their own embeddings (for semantic recall) - re-embed them too so
+    // an embedder switch doesn't leave the memory layer in a stale vector space.
+    for (const m of store.memories.all()) {
+      store.memories.updateEmbedding(m.id, await embeddings.embedDense(m.memory))
+    }
     return rows.length
   }
 
@@ -209,6 +311,10 @@ export function buildEmbeddedContext(opts: EmbeddedOptions = {}) {
     deleteRecord,
     searchWithGraph,
     searchTraced,
+    recallMemories,
+    forgetMemories,
+    memoryHistory,
+    buildProfile,
     reindex,
     rebuildGraph,
   }

package/src/embedded/ingest.ts

@@ -8,6 +8,15 @@ import { DeterministicExtractor, stripBoilerplate, slugify, entityId, type Knowl
 import { CodeExtractor } from "./code-extractor"
 import { guardIngest } from "../security/limits"
 import { sanitizeBody, sanitizeLabel } from "../security/sanitize"
+import { consolidateTriples } from "./memory/consolidate"
+
+// Optional default TTL for dynamic memories (CONTEXT_MEMORY_TTL_DAYS). Unset ⇒ memories
+// never auto-expire; set ⇒ non-static memories get a forget_after and the TTL sweep
+// retires them. Static facts (names, birthplaces) are always exempt.
+function memoryTtlMs(): number | undefined {
+  const days = Number(process.env.CONTEXT_MEMORY_TTL_DAYS ?? 0)
+  return days > 0 ? days * 24 * 60 * 60 * 1000 : undefined
+}
 
 export interface IngestDoc {
   recordId: string
@@ -195,6 +204,23 @@ export class Ingestor {
           : await this.writeGraphFor(doc.recordId, cleanText, doc.recordName)
     }
 
+    // (5) MEMORIES: the living-memory layer. Consolidate the PRECISE typed triples the
+    // caller supplied into atomic memories (contradiction → new version, dedup, TTL).
+    // We use only the provided typed predicates - the deterministic extractor emits
+    // symmetric "relates_to" co-occurrence, which is graph signal, not an atomic fact.
+    // Inherits this record's ACL via virtualRecordId. No-op when no typed triples given.
+    if (doc.relations?.length) {
+      const typed = doc.relations.filter((r) => (r.type || "").trim().toLowerCase().replace(/\s+/g, "_") !== "relates_to")
+      if (typed.length) {
+        await consolidateTriples(this.store.memories, this.embeddings, typed, {
+          orgId: doc.orgId,
+          virtualRecordId: vid,
+          sourceRecordId: doc.recordId,
+          ttlMs: memoryTtlMs(),
+        })
+      }
+    }
+
     return { recordId: doc.recordId, chunks: chunks.length, entities }
   }
 

package/src/embedded/local-embeddings.ts

@@ -1,36 +1,84 @@
-// In-process embeddings. This deterministic hashing embedder is dependency-free
-// so the embedded stack runs and tests with zero downloads. For real semantic
-// quality in the single binary, swap in transformers.js / fastembed (ONNX bge-*)
+// In-process embeddings. This deterministic, dependency-free embedder is the default
+// (zero downloads - tests and bunx launches run offline). It is NOT a neural model, but
+// it is much stronger than a plain bag-of-words hash: it also hashes CHARACTER N-GRAMS
+// (so morphological variants overlap - "running"~"run", and typos degrade gracefully)
+// and WORD BIGRAMS (so short phrases carry signal), with signed feature hashing to
+// cancel collision bias and sublinear term weighting. For true semantic quality (real
+// synonyms, paraphrase) install @huggingface/transformers and set CONTEXT_EMBEDDINGS=real
 // - it implements the same EmbeddingProvider interface, so nothing above changes.
 
 import type { EmbeddingProvider } from "../provider"
 
-const DIM = 64
+// 256 dims (vs the old 64): fewer collisions for the richer feature set. NOTE: changing
+// this value changes the vector space - an existing DB self-heals via the embedder-drift
+// reconcile() (it detects the dim change and reindexes).
+const DIM = 256
 
 function tokens(text: string): string[] {
   return text.toLowerCase().match(/[a-z0-9]+/g) ?? []
 }
 
-function bucket(token: string): number {
+// FNV-1a → unsigned 32-bit. Used both to pick a bucket and (its high bit) a sign.
+function fnv(s: string): number {
   let h = 2166136261
-  for (let i = 0; i < token.length; i++) {
-    h ^= token.charCodeAt(i)
+  for (let i = 0; i < s.length; i++) {
+    h ^= s.charCodeAt(i)
     h = Math.imul(h, 16777619)
   }
-  return Math.abs(h) % DIM
+  return h >>> 0
+}
+
+// Signed feature hashing: bucket = h % DIM, sign = high bit → ±. Signed hashing makes
+// collisions cancel in expectation instead of always adding, so the vector is cleaner.
+function addFeature(v: number[], feat: string, weight: number): void {
+  const h = fnv(feat)
+  const idx = h % DIM
+  const sign = (h & 0x80000000) !== 0 ? 1 : -1
+  v[idx] += sign * weight
+}
+
+// Character n-grams of a token, padded so prefixes/suffixes are distinct features.
+function charNGrams(token: string, n: number): string[] {
+  const s = `#${token}#`
+  if (s.length <= n) return [s]
+  const out: string[] = []
+  for (let i = 0; i + n <= s.length; i++) out.push(s.slice(i, i + n))
+  return out
+}
+
+function embed(text: string): number[] {
+  const v = new Array(DIM).fill(0)
+  const toks = tokens(text)
+  // Sublinear term frequency: repeated tokens shouldn't dominate (1 + log count).
+  const tf = new Map<string, number>()
+  for (const t of toks) tf.set(t, (tf.get(t) ?? 0) + 1)
+  for (const [t, c] of tf) {
+    const w = 1 + Math.log(c)
+    addFeature(v, `w:${t}`, w) // whole-word feature
+    for (const g of charNGrams(t, 3)) addFeature(v, `c:${g}`, 0.5 * w) // morphology / fuzzy
+  }
+  // Word bigrams: short phrases ("new york", "machine learning") carry their own signal.
+  for (let i = 0; i + 1 < toks.length; i++) addFeature(v, `b:${toks[i]}_${toks[i + 1]}`, 0.7)
+  const norm = Math.sqrt(v.reduce((s, x) => s + x * x, 0)) || 1
+  return v.map((x) => x / norm)
 }
 
 export class LocalHashEmbeddings implements EmbeddingProvider {
   async embedDense(query: string): Promise<number[]> {
-    const v = new Array(DIM).fill(0)
-    for (const t of tokens(query)) v[bucket(t)] += 1
-    const norm = Math.sqrt(v.reduce((s, x) => s + x * x, 0)) || 1
-    return v.map((x) => x / norm)
+    return embed(query)
+  }
+
+  // Symmetric: queries and documents share the same feature space (no asymmetric prefix).
+  async embedQuery(query: string): Promise<number[]> {
+    return embed(query)
   }
 
   async embedSparse(query: string): Promise<{ indices: number[]; values: number[] }> {
     const counts = new Map<number, number>()
-    for (const t of tokens(query)) counts.set(bucket(t), (counts.get(bucket(t)) ?? 0) + 1)
+    for (const t of tokens(query)) {
+      const idx = fnv(`w:${t}`) % DIM
+      counts.set(idx, (counts.get(idx) ?? 0) + 1)
+    }
     return { indices: [...counts.keys()], values: [...counts.values()] }
   }
 }

package/src/embedded/memory/consolidate.ts

@@ -0,0 +1,135 @@
+// Consolidation - the living-memory "engine". Turns the typed triples a record
+// asserts into ATOMIC memories and decides, per fact, whether it is:
+//   • NEW       - first time we've seen this subject (create v1), or an independent
+//                 multi-valued fact (its own chain),
+//   • DUPLICATE - the exact same fact re-asserted (just refresh recency),
+//   • UPDATE    - a single-valued (functional) fact that CONTRADICTS the current one
+//                 (e.g. works_at: Google → Meta) → supersede: flip the old version's
+//                 is_latest, write a new version (+1) linked via the chain.
+// This is Supermemory's updates/extends/derives model, but grounded in our typed-graph
+// + permission model: contradictions resolve non-destructively (history kept) and the
+// whole thing inherits the source record's ACL via virtual_record_id. No LLM needed -
+// the calling model already supplied precise triples; an LLM extractor only enriches.
+
+import type { EmbeddingProvider } from "../../provider"
+import type { MemoryRepo, NewMemory } from "../store/memories"
+import { slugify, entityId } from "../extract"
+import { sanitizeText } from "../../security/sanitize"
+
+export type MemoryAction = "created" | "updated" | "duplicate"
+
+// Single-valued predicates: a subject has at most ONE current value, so a new value
+// SUPERSEDES the old (a contradiction → a new memory version). Mirrors
+// FUNCTIONAL_PREDICATES in ingest.ts (kept in sync; both describe the same semantics).
+const FUNCTIONAL = new Set([
+  "lives_in", "located_in", "based_in", "works_at", "employed_by", "ceo_of", "led_by",
+  "born_in", "current_role", "role_is", "status_is", "owns", "owned_by", "married_to",
+  "reports_to", "headquartered_in", "capital_of", "member_of",
+])
+
+// Permanent facts that should never auto-expire (TTL sweep skips is_static memories).
+const STATIC = new Set(["born_in", "capital_of", "native_of", "nationality_of", "gender_of"])
+
+export interface TripleInput {
+  from: string
+  to: string
+  type: string
+}
+
+export interface ConsolidateOpts {
+  orgId: string
+  virtualRecordId: string
+  sourceRecordId: string
+  /** Default TTL (ms from now) for dynamic memories; omitted ⇒ no expiry. */
+  ttlMs?: number
+}
+
+function newId(): string {
+  return `mem:${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 9)}`
+}
+
+/** Consolidate one atomic fact into the memory store. Returns what happened. */
+export async function consolidateFact(
+  repo: MemoryRepo,
+  embeddings: EmbeddingProvider,
+  fact: { subjectKey: string; memory: string; functional: boolean; isStatic: boolean },
+  opts: ConsolidateOpts,
+): Promise<MemoryAction> {
+  const current = repo.latestBySubject(fact.subjectKey)
+  const forgetAfter = !fact.isStatic && opts.ttlMs ? Date.now() + opts.ttlMs : null
+
+  if (!current) {
+    const id = newId()
+    const embedding = await embeddings.embedDense(fact.memory)
+    const base: NewMemory = {
+      id, orgId: opts.orgId, virtualRecordId: opts.virtualRecordId, subjectKey: fact.subjectKey,
+      memory: fact.memory, embedding, isStatic: fact.isStatic, forgetAfter,
+      version: 1, parentId: null, rootId: id, relation: null, sourceRecordId: opts.sourceRecordId,
+    }
+    repo.insert(base)
+    return "created"
+  }
+
+  if (current.memory === fact.memory) {
+    repo.touch(current.id) // exact re-assertion → just refresh recency
+    return "duplicate"
+  }
+
+  // Different value for the same subject. For a FUNCTIONAL predicate this is a
+  // contradiction → supersede with a new version. For a multi-valued predicate the
+  // subject_key already includes the object, so we never reach here for those (a
+  // different object is a different subject_key → "created"). Guard anyway.
+  if (!fact.functional) {
+    const id = newId()
+    const embedding = await embeddings.embedDense(fact.memory)
+    repo.insert({
+      id, orgId: opts.orgId, virtualRecordId: opts.virtualRecordId, subjectKey: fact.subjectKey,
+      memory: fact.memory, embedding, isStatic: fact.isStatic, forgetAfter,
+      version: 1, parentId: null, rootId: id, relation: null, sourceRecordId: opts.sourceRecordId,
+    })
+    return "created"
+  }
+
+  const id = newId()
+  const embedding = await embeddings.embedDense(fact.memory)
+  repo.markSuperseded(current.id)
+  repo.insert({
+    id, orgId: opts.orgId, virtualRecordId: opts.virtualRecordId, subjectKey: fact.subjectKey,
+    memory: fact.memory, embedding, isStatic: fact.isStatic, forgetAfter,
+    version: current.version + 1, parentId: current.id, rootId: current.root_id ?? current.id,
+    relation: "updates", sourceRecordId: opts.sourceRecordId,
+  })
+  return "updated"
+}
+
+/** Turn the typed triples a record asserts into atomic memories. Functional facts
+ *  key on (subject|predicate) so a new value supersedes; multi-valued facts key on
+ *  the full triple so distinct objects coexist and re-asserts dedup. Returns counts. */
+export async function consolidateTriples(
+  repo: MemoryRepo,
+  embeddings: EmbeddingProvider,
+  triples: TripleInput[],
+  opts: ConsolidateOpts,
+): Promise<{ created: number; updated: number; duplicate: number }> {
+  const tally = { created: 0, updated: 0, duplicate: 0 }
+  for (const t of triples) {
+    const from = sanitizeText(t.from).trim()
+    const to = sanitizeText(t.to).trim()
+    const pred = (t.type || "relates_to").trim().toLowerCase().replace(/\s+/g, "_")
+    if (!from || !to || !pred) continue
+    const subjId = entityId(slugify(from))
+    const objId = entityId(slugify(to))
+    if (!subjId || !objId) continue
+    const functional = FUNCTIONAL.has(pred)
+    const subjectKey = functional ? `${subjId}|${pred}` : `${subjId}|${pred}|${objId}`
+    const memory = `${from} ${pred.replace(/_/g, " ")} ${to}`
+    const action = await consolidateFact(
+      repo,
+      embeddings,
+      { subjectKey, memory, functional, isStatic: STATIC.has(pred) },
+      opts,
+    )
+    tally[action]++
+  }
+  return tally
+}

package/src/embedded/sqlite-store.ts

@@ -18,6 +18,7 @@ import { migrate, tryEnableExtensions, tryLoadVec } from "./store/schema"
 import * as graph from "./store/nodes-edges"
 import * as fts from "./store/fts"
 import { ChunkRepo } from "./store/chunks"
+import { MemoryRepo } from "./store/memories"
 import * as salience from "./store/salience"
 
 export type Json = Record<string, unknown>
@@ -26,6 +27,7 @@ export class SqliteStore {
   readonly db: Database
   readonly vecEnabled: boolean
   readonly ftsEnabled: boolean
+  readonly memories: MemoryRepo
   private readonly chunks: ChunkRepo
 
   constructor(path = ":memory:") {
@@ -45,6 +47,7 @@ export class SqliteStore {
     this.vecEnabled = encrypted ? false : tryLoadVec(this.db)
     this.ftsEnabled = fts.tryEnableFts(this.db)
     this.chunks = new ChunkRepo(this.db, this.vecEnabled, this.ftsEnabled, encrypted)
+    this.memories = new MemoryRepo(this.db)
   }
 
   // ── Graph: nodes & edges ────────────────────────────────────────────────
@@ -64,6 +67,10 @@ export class SqliteStore {
     return graph.supersedeEdge(this.db, src, label, keepDst, atTime)
   }
 
+  expireEdges(src: string, label: string, dst?: string): number {
+    return graph.expireEdges(this.db, src, label, dst)
+  }
+
   backfillEdgeProvenance(): number {
     return graph.backfillEdgeProvenance(this.db)
   }

package/src/embedded/store/memories.ts

@@ -0,0 +1,156 @@
+// Memory repository - SQL primitives for the living-memory layer. Pure persistence:
+// the CLASSIFICATION logic (is this new fact a contradiction → new version, or an
+// independent memory?) lives in ../memory/consolidate.ts; this file just does the
+// reads/writes. Every read is ACL-scoped by the caller passing the accessible
+// virtual_record_id set (gate-first, like the rest of the store) so no memory can
+// leak across a permission boundary - including superseded versions and forgotten rows.
+
+import { Database } from "bun:sqlite"
+import { ph } from "./schema"
+
+export interface MemoryRow {
+  id: string
+  org_id: string
+  virtual_record_id: string
+  subject_key: string
+  memory: string
+  embedding: string | null
+  is_static: number
+  is_forgotten: number
+  forget_after: number | null
+  forget_reason: string | null
+  version: number
+  parent_id: string | null
+  root_id: string | null
+  is_latest: number
+  relation: string | null
+  source_record_id: string | null
+  created_at: number
+  updated_at: number
+}
+
+export interface NewMemory {
+  id: string
+  orgId: string
+  virtualRecordId: string
+  subjectKey: string
+  memory: string
+  embedding: number[]
+  isStatic?: boolean
+  forgetAfter?: number | null
+  version?: number
+  parentId?: string | null
+  rootId?: string | null
+  relation?: string | null
+  sourceRecordId?: string | null
+}
+
+export class MemoryRepo {
+  constructor(private readonly db: Database) {}
+
+  /** The current (live) memory for a subject_key, if any. "Live" = latest, not forgotten. */
+  latestBySubject(subjectKey: string): MemoryRow | undefined {
+    return this.db
+      .query("SELECT * FROM memories WHERE subject_key = ? AND is_latest = 1 AND is_forgotten = 0 ORDER BY version DESC LIMIT 1")
+      .get(subjectKey) as MemoryRow | undefined
+  }
+
+  insert(m: NewMemory): void {
+    const now = Date.now()
+    this.db
+      .query(
+        `INSERT INTO memories
+           (id, org_id, virtual_record_id, subject_key, memory, embedding, is_static,
+            is_forgotten, forget_after, forget_reason, version, parent_id, root_id,
+            is_latest, relation, source_record_id, created_at, updated_at)
+         VALUES (?, ?, ?, ?, ?, ?, ?, 0, ?, NULL, ?, ?, ?, 1, ?, ?, ?, ?)`,
+      )
+      .run(
+        m.id,
+        m.orgId,
+        m.virtualRecordId,
+        m.subjectKey,
+        m.memory,
+        JSON.stringify(m.embedding),
+        m.isStatic ? 1 : 0,
+        m.forgetAfter ?? null,
+        m.version ?? 1,
+        m.parentId ?? null,
+        m.rootId ?? m.id,
+        m.relation ?? null,
+        m.sourceRecordId ?? null,
+        now,
+        now,
+      )
+  }
+
+  /** Close out a memory version: it is no longer the latest (a newer version supersedes it). */
+  markSuperseded(id: string): void {
+    this.db.query("UPDATE memories SET is_latest = 0, updated_at = ? WHERE id = ?").run(Date.now(), id)
+  }
+
+  /** A re-asserted identical fact: just refresh recency (no new version). */
+  touch(id: string): void {
+    this.db.query("UPDATE memories SET updated_at = ? WHERE id = ?").run(Date.now(), id)
+  }
+
+  /** Forget memories by id (soft-delete with a reason). Returns rows affected. */
+  forget(ids: string[], reason: string): number {
+    if (ids.length === 0) return 0
+    const res = this.db
+      .query(`UPDATE memories SET is_forgotten = 1, forget_reason = ?, updated_at = ? WHERE id IN (${ph(ids.length)}) AND is_forgotten = 0`)
+      .run(reason, Date.now(), ...ids)
+    return Number(res.changes)
+  }
+
+  /** TTL sweep: forget every dynamic memory whose forget_after has passed. Static
+   *  memories (names, birthplaces) are exempt. Cheap; called lazily before recall/ingest. */
+  sweep(now = Date.now()): number {
+    const res = this.db
+      .query(
+        `UPDATE memories SET is_forgotten = 1, forget_reason = 'expired (ttl)', updated_at = ?
+         WHERE is_forgotten = 0 AND is_static = 0 AND forget_after IS NOT NULL AND forget_after < ?`,
+      )
+      .run(now, now)
+    return Number(res.changes)
+  }
+
+  /** Current memories the caller may see: ACL-scoped to the accessible vids, latest
+   *  version only, not forgotten, not expired. The gate-first ACL filter - leak-proof
+   *  by construction (an inaccessible vid is never in the IN-list). */
+  recall(accessibleVids: string[], now = Date.now()): MemoryRow[] {
+    if (accessibleVids.length === 0) return []
+    return this.db
+      .query(
+        `SELECT * FROM memories
+         WHERE virtual_record_id IN (${ph(accessibleVids.length)})
+           AND is_latest = 1 AND is_forgotten = 0
+           AND (forget_after IS NULL OR forget_after > ?)
+         ORDER BY updated_at DESC`,
+      )
+      .all(...accessibleVids, now) as MemoryRow[]
+  }
+
+  /** Full version history of a memory chain (oldest → newest), for "how did this evolve". */
+  history(rootId: string): MemoryRow[] {
+    return this.db.query("SELECT * FROM memories WHERE root_id = ? ORDER BY version ASC").all(rootId) as MemoryRow[]
+  }
+
+  /** All memory rows (for reindex when the embedder dimension changes). */
+  all(): MemoryRow[] {
+    return this.db.query("SELECT id, memory FROM memories").all() as MemoryRow[]
+  }
+
+  updateEmbedding(id: string, embedding: number[]): void {
+    this.db.query("UPDATE memories SET embedding = ? WHERE id = ?").run(JSON.stringify(embedding), id)
+  }
+
+  counts(): { total: number; current: number; forgotten: number } {
+    const get = (sql: string) => (this.db.query(sql).get() as { c: number }).c
+    return {
+      total: get("SELECT count(*) c FROM memories"),
+      current: get("SELECT count(*) c FROM memories WHERE is_latest = 1 AND is_forgotten = 0"),
+      forgotten: get("SELECT count(*) c FROM memories WHERE is_forgotten = 1"),
+    }
+  }
+}

package/src/embedded/store/nodes-edges.ts

@@ -84,6 +84,19 @@ export function supersedeEdge(db: Database, src: string, label: string, keepDst:
   return Number(res.changes)
 }
 
+// Expire (close the validity interval on) live edges from `src` with `label`,
+// optionally restricted to a specific `dst`. Used when a memory is FORGOTTEN, so the
+// underlying typed-graph fact stops being asserted by KGQA / graph queries too -
+// keeping the forget coherent across both layers. Non-destructive (history kept).
+export function expireEdges(db: Database, src: string, label: string, dst?: string): number {
+  const now = Date.now()
+  const sql = dst
+    ? `UPDATE edges SET invalid_at = ?, expired_at = ? WHERE src = ? AND label = ? AND dst = ? AND expired_at IS NULL`
+    : `UPDATE edges SET invalid_at = ?, expired_at = ? WHERE src = ? AND label = ? AND expired_at IS NULL`
+  const res = dst ? db.query(sql).run(now, now, src, label, dst) : db.query(sql).run(now, now, src, label)
+  return Number(res.changes)
+}
+
 // Backfill provenance for LEGACY concept edges that predate provenance tracking
 // (migrated/older data has provenance '[]'). With per-edge ACL now fail-closed, an
 // un-provenanced edge would be hidden from everyone - so attribute each to the

package/src/embedded/store/schema.ts

@@ -57,6 +57,45 @@ export function migrate(db: Database): void {
   // sure we are the relationship is real. Added idempotently so existing DBs upgrade.
   const ecols = (db.query("PRAGMA table_info(edges)").all() as Array<{ name: string }>).map((c) => c.name)
   if (!ecols.includes("confidence")) db.exec("ALTER TABLE edges ADD COLUMN confidence REAL NOT NULL DEFAULT 1")
+  migrateMemories(db)
+}
+
+// The MEMORIES table - the living-memory layer (Supermemory-style atomic memories,
+// but permission-aware). Each row is ONE atomic fact, not a chunk. It carries the
+// version chain (root_id/parent_id/version/is_latest), the forgetting axes
+// (is_forgotten/forget_after/forget_reason), the static-vs-dynamic flag, and an ACL
+// anchor (virtual_record_id - inherits the source record's permissions, exactly like
+// chunks). A "current" memory has is_latest=1 AND is_forgotten=0. Contradictions
+// supersede (flip is_latest, +1 version) - history is never deleted. The embedding
+// makes memories semantically recallable; the subject_key groups a single-valued
+// fact's versions (functional predicate) and de-duplicates re-asserted triples.
+export function migrateMemories(db: Database): void {
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS memories (
+      id TEXT PRIMARY KEY,
+      org_id TEXT,
+      virtual_record_id TEXT,
+      subject_key TEXT,
+      memory TEXT NOT NULL,
+      embedding TEXT,
+      is_static INTEGER NOT NULL DEFAULT 0,
+      is_forgotten INTEGER NOT NULL DEFAULT 0,
+      forget_after INTEGER,
+      forget_reason TEXT,
+      version INTEGER NOT NULL DEFAULT 1,
+      parent_id TEXT,
+      root_id TEXT,
+      is_latest INTEGER NOT NULL DEFAULT 1,
+      relation TEXT,
+      source_record_id TEXT,
+      created_at INTEGER NOT NULL,
+      updated_at INTEGER NOT NULL
+    );
+    CREATE INDEX IF NOT EXISTS idx_memories_acl ON memories(virtual_record_id, is_latest, is_forgotten);
+    CREATE INDEX IF NOT EXISTS idx_memories_subject ON memories(subject_key, is_latest);
+    CREATE INDEX IF NOT EXISTS idx_memories_root ON memories(root_id);
+    CREATE INDEX IF NOT EXISTS idx_memories_source ON memories(source_record_id);
+  `)
 }
 
 // The edges table is a property-graph relation store shared by ACL (permissions/

package/src/mcp/backend.ts

@@ -20,6 +20,8 @@ export interface BackendStats {
   chunks: number
   entities: number
   relations: number
+  /** Living-memory layer counts (local mode). */
+  memories?: { total: number; current: number; forgotten: number }
 }
 
 export interface ExactAnswer {
@@ -49,6 +51,15 @@ export interface ContextBackend {
    *  complete edge set (same as context_relate), as readable fact lines. Null when no
    *  entity is named. Lets get_context reach graph-query completeness for breadth recall. */
   relatedFacts?: (q: string, limit?: number) => Promise<{ entity: string; facts: string[] } | null>
+  /** Living memory: the CURRENT truth (latest version, not forgotten) for a query,
+   *  ACL-scoped. Each item carries its version so callers can show what evolved. */
+  recallMemories?: (q: string, limit?: number) => Promise<Array<{ memory: string; version: number; isStatic: boolean }>>
+  /** Forget memories matching a description (within the caller's accessible set).
+   *  Soft-delete; returns the memory texts that were forgotten. */
+  forget?: (q: string, reason?: string) => Promise<string[]>
+  /** Synthesized, ACL-scoped profile of a subject: permanent facts, recent facts, and
+   *  most-connected entities. Null when nothing is known about it. */
+  profile?: (subject: string) => Promise<{ subject: string; staticFacts: string[]; recentFacts: string[]; related: string[] } | null>
   ingest?: (doc: IngestDoc) => Promise<{ recordId: string; chunks: number; entities: number }>
   /** The accessible knowledge graph (entities + relations). Local mode only. */
   graph?: () => Promise<KnowledgeGraph>
@@ -119,6 +130,13 @@ export function resolveBackend(): ContextBackend {
       })
       return { entity: n.entity, facts }
     },
+    // Living memory: current truth (latest, non-forgotten), ACL-scoped, version-tagged.
+    recallMemories: async (q, limit) => {
+      const mems = await ctx.recallMemories(q, ctx.userId, ctx.orgId, limit && limit > 0 ? limit : 8)
+      return mems.map((m) => ({ memory: m.memory, version: m.version, isStatic: m.isStatic }))
+    },
+    forget: (q, reason) => ctx.forgetMemories(q, ctx.userId, ctx.orgId, reason),
+    profile: (subject) => ctx.buildProfile(subject, ctx.userId, ctx.orgId),
     ingest: (doc) => ctx.authorizedIngest(ctx.userId, doc), // write-side authorization + ownership
     graph: async () => {
       const accessible = await ctx.graph.getAccessibleVirtualRecordIds({ userId: ctx.userId, orgId: ctx.orgId })
@@ -140,6 +158,7 @@ export function resolveBackend(): ContextBackend {
       chunks: count("SELECT count(*) c FROM chunks"),
       entities: count("SELECT count(*) c FROM nodes WHERE coll = 'entities'"),
       relations: count("SELECT count(*) c FROM edges WHERE label = 'relates_to'"),
+      memories: ctx.store.memories.counts(),
     }),
   }
 }

package/src/mcp/tools/context-about.ts

@@ -45,6 +45,9 @@ async function describe(backend: ContextBackend): Promise<string> {
   if (backend.stats) {
     const s = await backend.stats()
     lines.push(`- contents: ${s.records} record(s), ${s.chunks} chunk(s), ${s.entities} concept(s), ${s.relations} relationship(s)`)
+    if (s.memories !== undefined) {
+      lines.push(`- living memory: ${s.memories.current} current memor(ies), ${s.memories.forgotten} forgotten (of ${s.memories.total} total versions)`)
+    }
   }
   lines.push("", "## Tools")
   for (const t of listedTools) lines.push(`- **${t.name}** - ${t.description}`)

package/src/mcp/tools/context-forget.ts

@@ -0,0 +1,35 @@
+import type { ContextBackend } from "../backend"
+import type { ToolModule, ToolResult } from "./types"
+
+const schema = {
+  name: "context_forget",
+  description:
+    "Forget stored memories that are no longer true or wanted. USE WHEN: the user says 'forget that…', " +
+    "'that's no longer true', 'delete what you know about…', or a fact is explicitly retracted. Describe " +
+    "WHAT to forget in natural language (e.g. 'my old address', 'that I work at Google'); matching memories " +
+    "within what YOU can access are soft-deleted (history is kept, they stop appearing in recall). You can " +
+    "only ever forget what you're permitted to see. DON'T USE to correct a fact - for that just context_ingest " +
+    "the new value (a single-valued fact auto-supersedes the old one).",
+  inputSchema: {
+    type: "object" as const,
+    properties: {
+      query: { type: "string", description: "natural-language description of the memory/memories to forget" },
+      reason: { type: "string", description: "why it's being forgotten (optional, stored for audit)" },
+    },
+    required: ["query"],
+  },
+}
+
+async function handler(args: Record<string, unknown>, backend: ContextBackend): Promise<ToolResult> {
+  const query = String((args as any).query ?? "")
+  const reason = (args as any).reason ? String((args as any).reason) : undefined
+  if (!query.trim()) return { content: [{ type: "text", text: "Nothing to forget: provide a description." }], isError: true }
+  const forgotten = await backend.forget!(query, reason)
+  if (forgotten.length === 0) {
+    return { content: [{ type: "text", text: `No matching memories found to forget for "${query}".` }] }
+  }
+  const list = forgotten.map((m) => `• ${m}`).join("\n")
+  return { content: [{ type: "text", text: `Forgot ${forgotten.length} memor${forgotten.length === 1 ? "y" : "ies"}:\n${list}` }] }
+}
+
+export const contextForgetTool: ToolModule = { schema, handler, available: (b) => !!b.forget }

package/src/mcp/tools/context-profile.ts

@@ -0,0 +1,34 @@
+import type { ContextBackend } from "../backend"
+import { sanitizeText } from "../../security/sanitize"
+import type { ToolModule, ToolResult } from "./types"
+
+const schema = {
+  name: "context_profile",
+  description:
+    "Get a synthesized profile of a person, org, or any entity - the permanent facts, the most recent facts, " +
+    "and what it's most connected to, rolled up from memory. USE WHEN: 'who is X', 'what do you know about X', " +
+    "'summarize what we know about X', or before personalizing a response to someone. Returns the CURRENT truth " +
+    "(contradictions already resolved, forgotten facts excluded), permission-filtered to what you may see. " +
+    "DON'T USE for a one-off fact (use get_context) or general world knowledge.",
+  inputSchema: {
+    type: "object" as const,
+    properties: {
+      subject: { type: "string", description: "the person/org/entity to profile (name as written)" },
+    },
+    required: ["subject"],
+  },
+}
+
+async function handler(args: Record<string, unknown>, backend: ContextBackend): Promise<ToolResult> {
+  const subject = String((args as any).subject ?? "").trim()
+  if (!subject) return { content: [{ type: "text", text: "Provide a subject to profile." }], isError: true }
+  const p = await backend.profile!(subject)
+  if (!p) return { content: [{ type: "text", text: `No memory about "${subject}" (or you don't have access).` }] }
+  const out: string[] = [`Profile - ${sanitizeText(p.subject)}`]
+  if (p.staticFacts.length) out.push("", "Permanent:", ...p.staticFacts.map((f) => `• ${sanitizeText(f)}`))
+  if (p.recentFacts.length) out.push("", "Current (most recent first):", ...p.recentFacts.map((f) => `• ${sanitizeText(f)}`))
+  if (p.related.length) out.push("", `Connected to: ${p.related.map((r) => sanitizeText(r)).join(", ")}`)
+  return { content: [{ type: "text", text: out.join("\n") }] }
+}
+
+export const contextProfileTool: ToolModule = { schema, handler, available: (b) => !!b.profile }

package/src/mcp/tools/get-context.ts

@@ -63,6 +63,21 @@ async function handler(args: Record<string, unknown>, backend: ContextBackend):
     }
   }
 
+  // (2b) Living memory: the CURRENT truth (latest version, not forgotten) about the
+  // query, ACL-scoped. This is the evolving/deduped/forgetting-aware layer - it reflects
+  // contradictions already resolved (e.g. "works at Meta", not the superseded "Google").
+  // Distinct from the graph neighborhood (raw edges) and ranked snippets (raw text).
+  let memories = ""
+  if (backend.recallMemories) {
+    const mems = await backend.recallMemories(query, limit && limit > 0 ? limit : 8)
+    if (mems.length) {
+      const body = mems
+        .map((m) => `• ${sanitizeText(m.memory)}${m.version > 1 ? ` (updated, v${m.version})` : ""}`)
+        .join("\n")
+      memories = `Current memory (latest, contradictions resolved):\n${body}`
+    }
+  }
+
   // (3) Full ranked recall (vector + BM25 + GraphRAG), breadth-aware.
   const res = await backend.query(query, limit)
   const recalled =
@@ -70,7 +85,7 @@ async function handler(args: Record<string, unknown>, backend: ContextBackend):
       ? renderRecalled(res.searchResults.map((r) => ({ content: r.content, source: r.metadata.recordName ?? "untitled" })))
       : ""
 
-  const sections = [highlight, graphFacts, recalled].filter(Boolean)
+  const sections = [highlight, memories, graphFacts, recalled].filter(Boolean)
   let text: string
   if (sections.length) text = sections.join("\n\n---\n\n")
   else

package/src/mcp/tools/index.ts

@@ -6,6 +6,8 @@
 import type { ContextBackend } from "../backend"
 import { getContextTool } from "./get-context"
 import { contextIngestTool } from "./context-ingest"
+import { contextForgetTool } from "./context-forget"
+import { contextProfileTool } from "./context-profile"
 import { contextGraphTool } from "./context-graph"
 import { contextRebuildTool } from "./context-rebuild"
 import { contextRelateTool } from "./context-relate"
@@ -15,6 +17,8 @@ import type { ToolModule, ToolResult, ToolSchema } from "./types"
 const ALL: ToolModule[] = [
   getContextTool,
   contextIngestTool,
+  contextForgetTool,
+  contextProfileTool,
   contextGraphTool,
   contextRebuildTool,
   contextRelateTool,