@memstack/core 0.4.0 → 0.6.4

package/README.md

@@ -82,15 +82,74 @@ Think of it as the open-source alternative to [Mem0](https://mem0.ai/) — plugg
 
 ## Quick Start
 
+```bash
+npm install @memstack/core
+```
+
+### OpenAI
+
 ```typescript
 import { MemStack, OpenAILLMAdapter, OpenAIEmbeddingAdapter, InMemoryStorageAdapter } from "@memstack/core";
 
+const llm = new OpenAILLMAdapter({ apiKey: process.env.OPENAI_API_KEY! });
+
 const memstack = new MemStack({
-  llm: new OpenAILLMAdapter({ apiKey: process.env.OPENAI_API_KEY! }),
+  llm,
   embedding: new OpenAIEmbeddingAdapter({ apiKey: process.env.OPENAI_API_KEY! }),
   storage: new InMemoryStorageAdapter(),
 });
+```
+
+### DeepSeek (no embeddings)
+
+DeepSeek provides chat completions but has no embedding API. Use the OpenAI-compatible LLM adapter with `baseURL` and omit the embedding adapter — retrieval falls back to keyword + recency + importance ranking. You still get the full pipeline: store, summarize, prune, and compileContext.
+
+```typescript
+import { MemStack, OpenAILLMAdapter, InMemoryStorageAdapter } from "@memstack/core";
+
+const llm = new OpenAILLMAdapter({
+  apiKey: process.env.DEEPSEEK_API_KEY!,
+  baseURL: "https://api.deepseek.com/v1",
+  defaultModel: "deepseek-chat",
+});
+
+const memstack = new MemStack({
+  llm,
+  storage: new InMemoryStorageAdapter(),
+  // No embedding adapter — retrieval uses keyword matching
+});
+```
 
+### OpenRouter / Together AI / any OpenAI-compatible API
+
+Same pattern — change `baseURL` and `defaultModel`:
+
+```typescript
+// OpenRouter
+const llm = new OpenAILLMAdapter({
+  apiKey: process.env.OPENROUTER_API_KEY!,
+  baseURL: "https://openrouter.ai/api/v1",
+  defaultModel: "openai/gpt-4o-mini",
+});
+
+// Together AI
+const llm = new OpenAILLMAdapter({
+  apiKey: process.env.TOGETHER_API_KEY!,
+  baseURL: "https://api.together.xyz/v1",
+  defaultModel: "meta-llama/Llama-3.3-70B-Instruct-Turbo",
+});
+
+// Gemini (OpenAI-compatible endpoint)
+const llm = new OpenAILLMAdapter({
+  apiKey: process.env.GEMINI_API_KEY!,
+  baseURL: "https://generativelanguage.googleapis.com/v1beta/openai",
+  defaultModel: "gemini-2.0-flash",
+});
+```
+
+### Store and retrieve
+
+```typescript
 // 1. Store what happened
 await memstack.memory.store({
   actorId: "support-bot-42",
@@ -106,18 +165,21 @@ const memories = await memstack.memory.retrieve({
   strategy: "hybrid",
 });
 
-// 3. Inject into your LLM call
+// 3. Assemble an LLM-ready context
 const ctx = await memstack.memory.compileContext({
   actorId: "support-bot-42",
   maxTokens: 2000,
 });
 
-const llmResponse = await llm.complete({
+const response = await llm.complete({
   system: `You are a support bot. Here is what you remember:\n${ctx.systemPrompt}`,
   user: "The user is back and still can't log in. What do you do?",
 });
 
-// 4. Every 100 interactions, summarization kicks in automatically.
+console.log(response.text);
+// "Based on our history, the user has been experiencing 503 errors on Chrome 125..."
+
+// 4. Every 100 interactions, summarization triggers automatically.
 // Old interactions are compressed into a paragraph. Token costs stay flat.
 ```
 
@@ -192,7 +254,7 @@ No embedding adapter? `semantic` and `hybrid` fall back to keyword matching + im
 
 ### 3. Compile Context
 
-The killer feature. `compileContext()` takes the retrieval results and assembles an LLM-ready system prompt — deduplicated, sorted by recency and importance, with a token estimate so you know the cost before calling the LLM.
+`compileContext()` takes retrieval results and assembles an LLM-ready system prompt — deduplicated, sorted by recency and importance, with a token estimate so you know the cost before calling the LLM.
 
 ```typescript
 const ctx = await ms.memory.compileContext({
@@ -213,13 +275,16 @@ const ctx = await ms.memory.compileContext({
 console.log(ctx.tokenEstimate);  // ~280
 
 // Inject into your LLM call
+const currentMessage = "The user is asking about their refund status.";
 const response = await llm.complete({
   system: ctx.systemPrompt,
-  user: userMessage,
+  user: currentMessage,
 });
+
+console.log(response.text);
 ```
 
-`compileContext()` is the difference between "we have a vector DB" and "we have agent memory." It handles deduplication, token budgeting, and the recent-vs-important split that makes context useful.
+`compileContext()` handles deduplication, token budgeting, and splits context into important-vs-recent sections. Without it, you'd be concatenating raw retrieval results and risking context-window overflow.
 
 ### 4. Summarize
 
@@ -309,13 +374,29 @@ const ms = new MemStack({
 ### Support Agent
 
 ```typescript
+// detectUrgency and classifyIntent are your own business logic.
+// They could be simple keyword matchers, regex, or an LLM call.
+function detectUrgency(msg: string): number {
+  if (msg.match(/urgent|asap|immediately/i)) return 0.9;
+  if (msg.match(/error|fail|broken/i)) return 0.7;
+  return 0.5;
+}
+
+function classifyIntent(msg: string): string[] {
+  const tags: string[] = [];
+  if (msg.match(/bill|refund|charge|payment/i)) tags.push("billing");
+  if (msg.match(/error|bug|fail|crash/i)) tags.push("bug");
+  if (msg.match(/login|password|account/i)) tags.push("account");
+  return tags;
+}
+
 // Every customer message becomes a memory
 async function handleMessage(customerId: string, message: string) {
   await ms.memory.store({
     actorId: `customer:${customerId}`,
     content: message,
-    importance: detectUrgency(message), // NLP heuristic or LLM call
-    tags: classifyIntent(message),      // "billing", "bug", "account", etc.
+    importance: detectUrgency(message),
+    tags: classifyIntent(message),
   });
 
   // Retrieve everything relevant to this customer's history
@@ -339,6 +420,12 @@ async function handleMessage(customerId: string, message: string) {
 ### RAG Pipeline
 
 ```typescript
+// Suppose you have documents from your knowledge base
+const documents = [
+  { text: "Authentication uses JWT tokens with 15-minute expiry.", url: "/docs/auth", section: "security" },
+  { text: "Refunds are processed within 5-10 business days.", url: "/docs/billing", section: "billing" },
+];
+
 // Index documents as observation memories
 for (const doc of documents) {
   await ms.memory.store({
@@ -441,9 +528,61 @@ await ms.memory.retrieve({ actorId: "x", query: "login bug", strategy: "hybrid"
 
 Embeddings power semantic search. They're optional — without them, retrieval uses keyword matching.
 
-**With embeddings** (configure an `EmbeddingProvider`): each `store()` computes a vector. `retrieve()` with `"semantic"` or `"hybrid"` uses cosine similarity ranking.
+### With embeddings vs Without embeddings
 
-**Without embeddings**: everything still works — retrieval falls back to importance + recency + keyword filters. No API costs, no setup.
+**With embeddings** (`embedding` adapter configured):
+
+```typescript
+import { MemStack, OpenAILLMAdapter, OpenAIEmbeddingAdapter, InMemoryStorageAdapter } from "@memstack/core";
+
+const ms = new MemStack({
+  llm: new OpenAILLMAdapter({ apiKey: process.env.OPENAI_API_KEY! }),
+  embedding: new OpenAIEmbeddingAdapter({ apiKey: process.env.OPENAI_API_KEY! }),
+  storage: new InMemoryStorageAdapter(),
+});
+
+// store() computes a 1536-dim vector automatically
+await ms.memory.store({
+  actorId: "agent-7",
+  content: "Customer asked about refund policy for Q2 purchases.",
+});
+
+// retrieve() with "semantic" or "hybrid" uses cosine similarity
+// Query: "refund" finds the refund policy memory even though the word "refund"
+// appears differently across stored memories.
+const results = await ms.memory.retrieve({
+  actorId: "agent-7",
+  query: "how do I get my money back",
+  strategy: "semantic",
+});
+// Matches "Customer asked about refund policy" — semantic match, not keyword match.
+```
+
+**Without embeddings** (no `embedding` adapter):
+
+```typescript
+const ms = new MemStack({
+  llm: new OpenAILLMAdapter({ apiKey: process.env.OPENAI_API_KEY! }),
+  storage: new InMemoryStorageAdapter(),
+  // no embedding adapter
+});
+
+// store() works identically, just no vector computed
+await ms.memory.store({
+  actorId: "agent-7",
+  content: "Customer asked about refund policy for Q2 purchases.",
+});
+
+// retrieve() with "semantic" or "hybrid" falls back to keyword matching
+// plus importance/recency sorting. No API costs, no setup required.
+const results = await ms.memory.retrieve({
+  actorId: "agent-7",
+  query: "refund",
+  strategy: "hybrid", // falls back to keyword + importance
+});
+// Still works — finds "refund" via substring match. Less precise for
+// paraphrased queries ("money back" won't match "refund").
+```
 
 **Batch embedding:** `storeBatch()` sends all texts in one embedding API call, reducing cost and latency.
 
@@ -456,6 +595,28 @@ const ms = new MemStack({
 });
 ```
 
+### Vector dimensions and model compatibility
+
+Different embedding models produce vectors of different lengths. Cosine similarity only works between vectors of the same dimension. If you change embedding models, existing vectors become incompatible — they can't be compared to new ones.
+
+| Adapter | Default model | Dimensions |
+|---------|--------------|------------|
+| `OpenAIEmbeddingAdapter` | `text-embedding-3-small` | 1536 |
+| `OpenAIEmbeddingAdapter` | `text-embedding-3-large` | 3072 |
+| `CohereEmbeddingAdapter` | `embed-english-v3.0` | 1024 |
+| `CohereEmbeddingAdapter` | `embed-english-light-v3.0` | 384 |
+| `CohereEmbeddingAdapter` | `embed-english-v2.0` | 4096 |
+| `CohereEmbeddingAdapter` | `embed-multilingual-v3.0` | 1024 |
+
+**What happens if dimensions don't match:** If you store memories with one model (e.g., 1536 dims) then switch to another model (e.g., 1024 dims), the storage adapter receives query vectors and stored vectors of different lengths. Cosine similarity between vectors of different dimensions is undefined — results depend on the storage backend's behavior. Most will either error, return empty results, or produce meaningless scores.
+
+**Recommendation:** Pick one embedding model per storage instance and stick with it. If you need to switch models, create a new storage instance and re-embed from scratch.
+
+**DeepSeek users:** DeepSeek has no embeddings API. If you use DeepSeek as your LLM, you must either:
+1. Omit the embedding adapter and use `"recent"` or `"important"` retrieval strategies (no API costs, less precise)
+2. Pair DeepSeek with a separate embedding provider (e.g., OpenAI for embeddings, DeepSeek for chat)
+
+
 ---
 
 ## Adapters
@@ -512,6 +673,8 @@ new OpenAIEmbeddingAdapter({ apiKey: "...", baseURL: "https://api.voyageai.com/v
 
 MemStack ships with **11 production-ready storage adapters (7 experimental)** — every major backend, zero peer dependencies, all client-injected.
 
+### Production (e2e verified against real instances)
+
 **Built-in (zero external deps):**
 | Adapter | Backend | Use case |
 |---|---|---|
@@ -524,21 +687,11 @@ MemStack ships with **11 production-ready storage adapters (7 experimental)**
 | Adapter | Backend | Vector search |
 |---|---|---|
 | `PostgresStorageAdapter` | PostgreSQL + pgvector | HNSW native |
-| `SQLiteStorageAdapter` | SQLite (better-sqlite3) | In-memory cosine |
-| `TursoStorageAdapter` | Turso (libsql) | DiskANN native |
-
-**Aggregators:**
-| Adapter | Delegates to |
-|---|---|
-| `Mem0StorageAdapter` | Mem0 OSS or Cloud |
-| `ZepStorageAdapter` | Zep Cloud or Community Edition |
 
 **Vector databases:**
 | Adapter | Backend |
 |---|---|
 | `QdrantStorageAdapter` | Qdrant |
-| `PineconeStorageAdapter` | Pinecone |
-| `ChromaStorageAdapter` | ChromaDB |
 | `WeaviateStorageAdapter` | Weaviate |
 | `LanceDBStorageAdapter` | LanceDB |
 | `MongoDBStorageAdapter` | MongoDB Atlas Vector Search |
@@ -547,13 +700,28 @@ MemStack ships with **11 production-ready storage adapters (7 experimental)**
 | Adapter | Backend |
 |---|---|
 | `RedisStorageAdapter` | Redis (ioredis) |
-| `UpstashStorageAdapter` | Upstash Redis + Vector |
 
 **Graph:**
 | Adapter | Backend |
 |---|---|
 | `Neo4jStorageAdapter` | Neo4j |
 
+### Experimental (mock-tested, blocked by cloud deps or platform constraints)
+
+Available via direct source import. Not yet in the barrel export — uncomment in `src/index.ts` when e2e verified.
+
+| Adapter | Backend | Blocker |
+|---|---|---|
+| `SQLiteStorageAdapter` | SQLite (better-sqlite3) | Native binary for Node 24 |
+| `TursoStorageAdapter` | Turso (libsql) | Cloud-only (needs Turso account) |
+| `ChromaStorageAdapter` | ChromaDB | Embedding function dependency |
+| `PineconeStorageAdapter` | Pinecone | Cloud-only (needs API key) |
+| `UpstashStorageAdapter` | Upstash Redis + Vector | Cloud-only (needs API key) |
+| `Mem0StorageAdapter` | Mem0 OSS or Cloud | Cloud-only (needs API key) |
+| `ZepStorageAdapter` | Zep Cloud or CE | Cloud-only (needs API key) |
+
+> **Direct import:** `import { ChromaStorageAdapter } from "@memstack/core/src/adapters/storage/chroma.js"`
+
 **Quick-start per backend:**
 
 ```ts
@@ -561,11 +729,6 @@ MemStack ships with **11 production-ready storage adapters (7 experimental)**
 import { PostgresStorageAdapter } from "@memstack/core";
 const storage = new PostgresStorageAdapter({ connectionString: "postgres://..." });
 
-// SQLite
-import Database from "better-sqlite3";
-import { SQLiteStorageAdapter } from "@memstack/core";
-const storage = new SQLiteStorageAdapter({ db: new Database("memory.db") });
-
 // Redis
 import Redis from "ioredis";
 import { RedisStorageAdapter } from "@memstack/core";
@@ -604,26 +767,21 @@ class MyStorage implements StorageProvider {
 
 ## Backend Comparison
 
-| Backend | Vector search | Touch | Best for |
+| Backend | Vector search | Touch | Status |
 |---|---|---|---|
-| InMemory | Cosine in-memory | Yes | Testing, prototyping |
-| Disk (JSON) | Keyword + importance | Yes | Simple local persistence |
-| Markdown | Keyword + importance | No | Human-readable, git-diffable |
-| Postgres | pgvector HNSW | Yes | Production relational |
-| SQLite | Cosine in-memory | Yes | Local dev, solo apps |
-| Turso | DiskANN native | Yes | Edge/serverless |
-| Redis | RediSearch KNN (auto-detect) | Yes | Sub-5ms hot session state |
-| Upstash | Native vector (vector mode) | No | CF Workers, Vercel Edge |
-| Qdrant | ANN native | No | Best filtered search |
-| Pinecone | ANN native | No | Zero-ops managed |
-| Chroma | Native | No | LangChain prototyping |
-| Weaviate | BM25 + vector hybrid | No | Hybrid search |
-| LanceDB | DiskANN native | No | Embedded local vector |
-| MongoDB | Atlas Vector Search | No | Existing MongoDB deployments |
-| Neo4j | Neo4j vector index | No | Relationship-aware agents |
-| Hybrid | Delegates to cache/durable | If durable supports | Read-through cache pattern |
-| Mem0 | Delegates to Mem0 | No | Multi-backend via Mem0 |
-| Zep | Graphiti temporal graph | No | Temporal graph memory |
+| InMemory | Cosine in-memory | Yes | ✅ Production |
+| Disk (JSON) | Keyword + importance | Yes | ✅ Production |
+| Markdown | Keyword + importance | No | ✅ Production |
+| Postgres | pgvector HNSW | Yes | ✅ Production |
+| Redis | RediSearch KNN (auto-detect) | Yes | ✅ Production |
+| Qdrant | ANN native | No | ✅ Production |
+| Weaviate | BM25 + vector hybrid | No | ✅ Production |
+| LanceDB | DiskANN native | No | ✅ Production |
+| MongoDB | Atlas Vector Search | No | ✅ Production |
+| Neo4j | Neo4j vector index | No | ✅ Production |
+| Hybrid | Delegates to cache/durable | If durable supports | ✅ Production |
+| SQLite | Cosine in-memory | Yes | ✅ Production |
+| Hybrid | Delegates to cache/durable | If durable supports | ✅ Production |
 
 ---
 
@@ -676,7 +834,11 @@ ms.memory.dryRunPrune(strategy: PruneStrategy): Promise<{ wouldPrune: string[];
 ms.memory.count(filter?: MemoryCountFilter): Promise<number>
 ms.memory.delete(id: string): Promise<void>
 ms.memory.deleteMany(ids: string[]): Promise<number>
-ms.memory.touch(id: string): Promise<void>  // bump recency without changing content
+ms.memory.touch(id: string): Promise<void>
+ms.memory.purgeActor(actorId: string): Promise<number>
+ms.memory.merge(ids: string[]): Promise<Memory>
+ms.memory.stats(actorId?: string): Promise<MemoryStats>
+ms.memory.summarizeStream(options: SummarizeOptions): AsyncIterable<{ chunk: string; text: string }>
 ```
 
 ### Export / Import
@@ -684,6 +846,8 @@ ms.memory.touch(id: string): Promise<void>  // bump recency without changing con
 Snapshot and restore full state for persistence, backups, or migration:
 
 ```typescript
+import * as fs from "node:fs";
+
 // Save
 const snapshot = await ms.export();
 fs.writeFileSync("state.json", JSON.stringify(snapshot, null, 2));
@@ -785,6 +949,7 @@ cd memstack
 pnpm install
 
 pnpm test           # 393 tests, no external services needed
+pnpm test:e2e        # 82 E2E tests (requires Docker)
 pnpm test:watch     # Watch mode
 pnpm build          # CJS + ESM + type declarations
 pnpm check          # TypeScript type-check only
@@ -839,7 +1004,7 @@ npm login
 npm publish --access public
 ```
 
-The `@memstack` scope requires `--access public` on first publish.
+The `@memstack` scope requires `--access public`.
 
 ---
 
@@ -847,7 +1012,6 @@ The `@memstack` scope requires `--access public` on first publish.
 
 Most needed contributions:
 
-- **Docker Compose** for integration testing
 - **LLM adapters**: Google Gemini (native), Amazon Bedrock, Vertex AI
 - **Embedding adapters**: local inference (transformers.js, ONNX)
 - **Benchmarks**: retrieval quality, latency, cost comparisons