@shadowforge0/aquifer-memory 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +148 -117
- package/consumers/cli.js +63 -0
- package/consumers/mcp.js +2 -2
- package/docs/setup.md +194 -0
- package/package.json +6 -6
- package/scripts/smoke.mjs +115 -0
package/README.md
CHANGED
|
@@ -32,7 +32,7 @@ Sessions, summaries, turn-level embeddings, entity graph — all live in one dat
|
|
|
32
32
|
| **Ranking** | 3-way RRF: FTS + session embedding + turn embedding | Single vector similarity |
|
|
33
33
|
| **Knowledge graph** | Built-in entity extraction & co-occurrence | Usually separate system |
|
|
34
34
|
| **Multi-tenant** | `tenant_id` on every table, day-1 | Often an afterthought |
|
|
35
|
-
| **Dependencies** |
|
|
35
|
+
| **Dependencies** | `pg` + MCP SDK | Multiple SDKs |
|
|
36
36
|
|
|
37
37
|
### Before and after
|
|
38
38
|
|
|
@@ -48,80 +48,150 @@ Sessions, summaries, turn-level embeddings, entity graph — all live in one dat
|
|
|
48
48
|
|
|
49
49
|
---
|
|
50
50
|
|
|
51
|
-
##
|
|
51
|
+
## Requirements
|
|
52
52
|
|
|
53
|
-
|
|
53
|
+
| Component | Required? | Purpose | Example |
|
|
54
|
+
|-----------|-----------|---------|---------|
|
|
55
|
+
| Node.js >= 18 | Yes | Runtime | — |
|
|
56
|
+
| PostgreSQL 15+ | Yes | Storage for sessions, summaries, entities | Local, Docker, or managed |
|
|
57
|
+
| pgvector extension | Yes | Vector similarity search | `CREATE EXTENSION vector;` (included in `pgvector/pgvector` Docker image) |
|
|
58
|
+
| Embedding endpoint | Yes (for recall) | Turn + session embedding | Ollama `bge-m3`, OpenAI `text-embedding-3-small`, any OpenAI-compatible API |
|
|
59
|
+
| LLM endpoint | Optional | Built-in summarization during `enrich` | Ollama, OpenRouter, OpenAI — or provide your own `summaryFn` |
|
|
60
|
+
| `@modelcontextprotocol/sdk` + `zod` | Yes (for MCP server) | MCP protocol runtime | Included in dependencies — installed automatically |
|
|
54
61
|
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
62
|
+
---
|
|
63
|
+
|
|
64
|
+
## Quick Start (MCP Server)
|
|
65
|
+
|
|
66
|
+
This gets you from zero to a working MCP memory server. For library API usage, see [API Reference](#api-reference) below.
|
|
67
|
+
|
|
68
|
+
### 1. Start the stack
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
docker compose up -d
|
|
72
|
+
# Starts PostgreSQL 16 + pgvector and Ollama with bge-m3 (auto-pulled).
|
|
73
|
+
# First run takes a few minutes while Ollama downloads the model.
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
Already have PostgreSQL + pgvector and an embedding endpoint? Skip this step.
|
|
58
77
|
|
|
59
|
-
### Install
|
|
78
|
+
### 2. Install
|
|
60
79
|
|
|
61
80
|
```bash
|
|
62
81
|
npm install @shadowforge0/aquifer-memory
|
|
63
82
|
```
|
|
64
83
|
|
|
65
|
-
###
|
|
84
|
+
### 3. Configure + verify
|
|
66
85
|
|
|
67
|
-
```
|
|
68
|
-
|
|
86
|
+
```bash
|
|
87
|
+
export DATABASE_URL="postgresql://aquifer:aquifer@localhost:5432/aquifer"
|
|
88
|
+
export AQUIFER_EMBED_BASE_URL="http://localhost:11434/v1"
|
|
89
|
+
export AQUIFER_EMBED_MODEL="bge-m3"
|
|
69
90
|
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
schema: 'memory', // PG schema name (default: 'aquifer')
|
|
73
|
-
tenantId: 'default', // multi-tenant isolation
|
|
74
|
-
embed: {
|
|
75
|
-
fn: async (texts) => embeddings, // your embedding function
|
|
76
|
-
dim: 1024, // optional dimension hint
|
|
77
|
-
},
|
|
78
|
-
llm: {
|
|
79
|
-
fn: async (prompt) => text, // your LLM function (for built-in summarize)
|
|
80
|
-
},
|
|
81
|
-
entities: {
|
|
82
|
-
enabled: true,
|
|
83
|
-
scope: 'my-app', // entity namespace (default: 'default')
|
|
84
|
-
},
|
|
85
|
-
});
|
|
91
|
+
npx aquifer quickstart
|
|
92
|
+
```
|
|
86
93
|
|
|
87
|
-
|
|
88
|
-
|
|
94
|
+
`quickstart` runs migrations, commits a test session, embeds it, recalls it, and cleans up. If it prints `✓ Aquifer is working`, you're done.
|
|
95
|
+
|
|
96
|
+
### 4. Start the MCP server
|
|
97
|
+
|
|
98
|
+
```bash
|
|
99
|
+
npx aquifer mcp
|
|
89
100
|
```
|
|
90
101
|
|
|
91
|
-
|
|
102
|
+
See [.env.example](.env.example) for all env vars, or [docs/setup.md](docs/setup.md) for the full setup guide.
|
|
92
103
|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
104
|
+
---
|
|
105
|
+
|
|
106
|
+
## Environment Variables
|
|
107
|
+
|
|
108
|
+
| Variable | Required? | Purpose | Example |
|
|
109
|
+
|----------|-----------|---------|---------|
|
|
110
|
+
| `DATABASE_URL` | Yes | PostgreSQL connection string | `postgresql://user:pass@localhost:5432/mydb` |
|
|
111
|
+
| `AQUIFER_SCHEMA` | No | PG schema name (default: `aquifer`) | `memory` |
|
|
112
|
+
| `AQUIFER_TENANT_ID` | No | Multi-tenant key (default: `default`) | `my-app` |
|
|
113
|
+
| `AQUIFER_EMBED_BASE_URL` | Yes (for recall) | Embedding API base URL | `http://localhost:11434/v1` |
|
|
114
|
+
| `AQUIFER_EMBED_MODEL` | Yes (for recall) | Embedding model name | `bge-m3` |
|
|
115
|
+
| `AQUIFER_EMBED_API_KEY` | Provider-dependent | API key for hosted embedding providers | `sk-...` |
|
|
116
|
+
| `AQUIFER_EMBED_DIM` | No | Embedding dimension override (auto-detected) | `1024` |
|
|
117
|
+
| `AQUIFER_LLM_BASE_URL` | No | LLM API base URL (for built-in summarization) | `http://localhost:11434/v1` |
|
|
118
|
+
| `AQUIFER_LLM_MODEL` | No | LLM model name | `llama3.1` |
|
|
119
|
+
| `AQUIFER_LLM_API_KEY` | Provider-dependent | API key for hosted LLM providers | `sk-...` |
|
|
120
|
+
| `AQUIFER_ENTITIES_ENABLED` | No | Enable knowledge graph (default: `false`) | `true` |
|
|
121
|
+
| `AQUIFER_ENTITY_SCOPE` | No | Entity namespace (default: `default`) | `my-app` |
|
|
122
|
+
| `AQUIFER_RERANK_ENABLED` | No | Enable cross-encoder reranking | `true` |
|
|
123
|
+
| `AQUIFER_RERANK_PROVIDER` | No | Reranker provider: `tei`, `jina`, `openrouter` | `tei` |
|
|
124
|
+
| `AQUIFER_RERANK_BASE_URL` | No | Reranker endpoint | `http://localhost:8080` |
|
|
125
|
+
| `AQUIFER_AGENT_ID` | No | Default agent ID | `main` |
|
|
126
|
+
|
|
127
|
+
Full env-to-config mapping is in [consumers/shared/config.js](consumers/shared/config.js).
|
|
128
|
+
|
|
129
|
+
---
|
|
130
|
+
|
|
131
|
+
## Host Integration
|
|
132
|
+
|
|
133
|
+
MCP is the primary integration surface. Agent hosts connect to the Aquifer MCP server, which exposes four tools: `session_recall`, `session_feedback`, `memory_stats`, `memory_pending`.
|
|
134
|
+
|
|
135
|
+
| Integration | Route | Status | When to use |
|
|
136
|
+
|-------------|-------|--------|-------------|
|
|
137
|
+
| MCP server | `consumers/mcp.js` | Primary | Claude Code, OpenClaw, Codex, any MCP-capable host |
|
|
138
|
+
| Library API | `createAquifer()` | Primary | Backend apps, custom pipelines, direct Node.js usage |
|
|
139
|
+
| CLI | `consumers/cli.js` | Secondary | Operations, debugging, manual recall/backfill |
|
|
140
|
+
| OpenClaw plugin | `consumers/openclaw-plugin.js` | Compatibility only | Session capture via `before_reset` — not for tool delivery |
|
|
141
|
+
|
|
142
|
+
### Claude Code
|
|
143
|
+
|
|
144
|
+
Add to your project's `.claude.json` or user-level MCP config:
|
|
145
|
+
|
|
146
|
+
```json
|
|
147
|
+
{
|
|
148
|
+
"mcpServers": {
|
|
149
|
+
"aquifer": {
|
|
150
|
+
"type": "stdio",
|
|
151
|
+
"command": "node",
|
|
152
|
+
"args": ["/path/to/aquifer/consumers/mcp.js"],
|
|
153
|
+
"env": {
|
|
154
|
+
"DATABASE_URL": "postgresql://...",
|
|
155
|
+
"AQUIFER_EMBED_BASE_URL": "http://localhost:11434/v1",
|
|
156
|
+
"AQUIFER_EMBED_MODEL": "bge-m3"
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
}
|
|
111
161
|
```
|
|
112
162
|
|
|
113
|
-
|
|
163
|
+
Tools appear as `mcp__aquifer__session_recall`, `mcp__aquifer__session_feedback`, etc.
|
|
114
164
|
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
165
|
+
### OpenClaw
|
|
166
|
+
|
|
167
|
+
Add to `openclaw.json` under `mcp.servers`:
|
|
168
|
+
|
|
169
|
+
```json
|
|
170
|
+
{
|
|
171
|
+
"mcp": {
|
|
172
|
+
"servers": {
|
|
173
|
+
"aquifer": {
|
|
174
|
+
"command": "node",
|
|
175
|
+
"args": ["/path/to/aquifer/consumers/mcp.js"],
|
|
176
|
+
"env": {
|
|
177
|
+
"DATABASE_URL": "postgresql://...",
|
|
178
|
+
"AQUIFER_EMBED_BASE_URL": "http://localhost:11434/v1",
|
|
179
|
+
"AQUIFER_EMBED_MODEL": "bge-m3"
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
}
|
|
123
185
|
```
|
|
124
186
|
|
|
187
|
+
Tools materialize as `aquifer__session_recall`, `aquifer__session_feedback`, `aquifer__memory_stats`, `aquifer__memory_pending` (server name prefix added by the host).
|
|
188
|
+
|
|
189
|
+
The OpenClaw plugin (`consumers/openclaw-plugin.js`) is retained for session capture via `before_reset` but is **not** the recommended tool delivery path. Use MCP.
|
|
190
|
+
|
|
191
|
+
### Other MCP-capable hosts
|
|
192
|
+
|
|
193
|
+
Any host that supports MCP stdio can connect the same way — point it at `node consumers/mcp.js` with the required env vars. The MCP server is the canonical external contract.
|
|
194
|
+
|
|
125
195
|
---
|
|
126
196
|
|
|
127
197
|
## Architecture
|
|
@@ -161,8 +231,6 @@ const results = await aquifer.recall('auth middleware decision', {
|
|
|
161
231
|
└──────────────────────────────────┘
|
|
162
232
|
```
|
|
163
233
|
|
|
164
|
-
**Integration model:** MCP is the primary integration surface. Agent hosts connect to Aquifer through the MCP server (`consumers/mcp.js`), which exposes `session_recall`, `session_feedback`, `memory_stats`, and `memory_pending`. The CLI wraps the same engine for command-line use. The OpenClaw plugin (`consumers/openclaw-plugin.js`) is retained as a compatibility adapter for session capture but is not the primary tool delivery path.
|
|
165
|
-
|
|
166
234
|
### File Reference
|
|
167
235
|
|
|
168
236
|
| File | Purpose |
|
|
@@ -375,7 +443,9 @@ Closes the PostgreSQL connection pool (only if Aquifer created it).
|
|
|
375
443
|
|
|
376
444
|
## Configuration
|
|
377
445
|
|
|
378
|
-
Aquifer
|
|
446
|
+
Aquifer resolves config from three sources in priority order: config file → environment variables → programmatic overrides. See [consumers/shared/config.js](consumers/shared/config.js) for the full env-to-config mapping.
|
|
447
|
+
|
|
448
|
+
Config file is auto-discovered at `aquifer.config.json` in the working directory, or set `AQUIFER_CONFIG=/path/to/config.json`.
|
|
379
449
|
|
|
380
450
|
```javascript
|
|
381
451
|
createAquifer({
|
|
@@ -409,61 +479,6 @@ createAquifer({
|
|
|
409
479
|
|
|
410
480
|
Fallback chain: `config.entities.scope` → `'default'`.
|
|
411
481
|
|
|
412
|
-
### MCP Server (primary integration)
|
|
413
|
-
|
|
414
|
-
Agent hosts should connect through the Aquifer MCP server. For OpenClaw, add to `openclaw.json`:
|
|
415
|
-
|
|
416
|
-
```json
|
|
417
|
-
{
|
|
418
|
-
"mcp": {
|
|
419
|
-
"servers": {
|
|
420
|
-
"aquifer": {
|
|
421
|
-
"command": "node",
|
|
422
|
-
"args": ["/path/to/aquifer/consumers/mcp.js"],
|
|
423
|
-
"env": {
|
|
424
|
-
"DATABASE_URL": "postgresql://...",
|
|
425
|
-
"AQUIFER_SCHEMA": "aquifer",
|
|
426
|
-
"AQUIFER_EMBED_BASE_URL": "http://localhost:11434/v1",
|
|
427
|
-
"AQUIFER_EMBED_MODEL": "bge-m3"
|
|
428
|
-
}
|
|
429
|
-
}
|
|
430
|
-
}
|
|
431
|
-
}
|
|
432
|
-
}
|
|
433
|
-
```
|
|
434
|
-
|
|
435
|
-
Tools are exposed as `aquifer__session_recall`, `aquifer__session_feedback`, `aquifer__memory_stats`, `aquifer__memory_pending` (server name prefix is added by the host).
|
|
436
|
-
|
|
437
|
-
For Claude Code, add to `.claude.json`:
|
|
438
|
-
|
|
439
|
-
```json
|
|
440
|
-
{
|
|
441
|
-
"mcpServers": {
|
|
442
|
-
"aquifer": {
|
|
443
|
-
"type": "stdio",
|
|
444
|
-
"command": "node",
|
|
445
|
-
"args": ["/path/to/aquifer/consumers/mcp.js"]
|
|
446
|
-
}
|
|
447
|
-
}
|
|
448
|
-
}
|
|
449
|
-
```
|
|
450
|
-
|
|
451
|
-
### CLI (secondary)
|
|
452
|
-
|
|
453
|
-
For command-line use with environment variables:
|
|
454
|
-
|
|
455
|
-
```bash
|
|
456
|
-
export DATABASE_URL="postgresql://..."
|
|
457
|
-
export AQUIFER_EMBED_BASE_URL="http://localhost:11434/v1"
|
|
458
|
-
export AQUIFER_EMBED_MODEL="bge-m3"
|
|
459
|
-
export AQUIFER_ENTITIES_ENABLED=true
|
|
460
|
-
|
|
461
|
-
aquifer migrate
|
|
462
|
-
aquifer recall "search query" --limit 5
|
|
463
|
-
aquifer backfill --concurrency 3
|
|
464
|
-
aquifer stats --json
|
|
465
|
-
```
|
|
466
|
-
|
|
467
482
|
---
|
|
468
483
|
|
|
469
484
|
## Database Schema
|
|
@@ -478,6 +493,8 @@ aquifer stats --json
|
|
|
478
493
|
|
|
479
494
|
Key indexes: GIN on messages, GiST on `tsvector`, ivfflat on embeddings, B-tree on tenant/agent/timestamps.
|
|
480
495
|
|
|
496
|
+
Note: the schema uses basic ivfflat indexes suitable for development and moderate-scale use. For large deployments (100k+ embeddings), consider adding HNSW indexes — this is a future optimization area, not included out of the box.
|
|
497
|
+
|
|
481
498
|
### 002-entities.sql
|
|
482
499
|
|
|
483
500
|
| Table | Purpose |
|
|
@@ -499,15 +516,29 @@ Also adds `trust_score` column to `session_summaries` (default 0.5, range 0–1)
|
|
|
499
516
|
|
|
500
517
|
---
|
|
501
518
|
|
|
519
|
+
## Troubleshooting
|
|
520
|
+
|
|
521
|
+
**`error: type "vector" does not exist`** — pgvector extension is not installed. Run `CREATE EXTENSION IF NOT EXISTS vector;` as a superuser, or use the `pgvector/pgvector` Docker image which includes it.
|
|
522
|
+
|
|
523
|
+
**`aquifer mcp requires @modelcontextprotocol/sdk and zod`** — These are now regular dependencies and should be installed automatically. If you see this error, run `npm install` again to ensure all deps are present.
|
|
524
|
+
|
|
525
|
+
**Recall returns no results** — Make sure you've run `enrich` after `commit`. Raw sessions are not searchable until enriched (summarized + embedded). Check `aquifer stats` to see if summaries and turn embeddings exist.
|
|
526
|
+
|
|
527
|
+
**OpenClaw tools not visible** — Use `mcp.servers.aquifer` in `openclaw.json`, not the plugin. Tools appear as `aquifer__session_recall` etc. The plugin (`consumers/openclaw-plugin.js`) is for session capture only.
|
|
528
|
+
|
|
529
|
+
**Embedding provider connection refused** — Verify your `AQUIFER_EMBED_BASE_URL` is reachable. For local Ollama, make sure the server is running and the model is pulled (`ollama pull bge-m3`).
|
|
530
|
+
|
|
531
|
+
---
|
|
532
|
+
|
|
502
533
|
## Dependencies
|
|
503
534
|
|
|
504
535
|
| Package | Purpose |
|
|
505
536
|
|---------|---------|
|
|
506
537
|
| `pg` ≥ 8.13 | PostgreSQL client |
|
|
538
|
+
| `@modelcontextprotocol/sdk` ≥ 1.29 | MCP server protocol |
|
|
539
|
+
| `zod` ≥ 3.25 | Schema validation (MCP tools) |
|
|
507
540
|
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
LLM and embedding calls use raw HTTP — no SDK required.
|
|
541
|
+
LLM and embedding calls use raw HTTP — no additional SDK required.
|
|
511
542
|
|
|
512
543
|
---
|
|
513
544
|
|
package/consumers/cli.js
CHANGED
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
* Aquifer CLI
|
|
6
6
|
*
|
|
7
7
|
* Usage:
|
|
8
|
+
* aquifer quickstart Verify end-to-end setup
|
|
8
9
|
* aquifer migrate Run database migrations
|
|
9
10
|
* aquifer recall <query> [options] Search sessions
|
|
10
11
|
* aquifer backfill [options] Enrich pending sessions
|
|
@@ -163,6 +164,64 @@ async function cmdStats(aquifer, args) {
|
|
|
163
164
|
}
|
|
164
165
|
}
|
|
165
166
|
|
|
167
|
+
async function cmdQuickstart(aquifer) {
|
|
168
|
+
console.log('Aquifer quickstart — verifying end-to-end setup.\n');
|
|
169
|
+
|
|
170
|
+
// 1. Migrate
|
|
171
|
+
console.log('1/5 Running migrations...');
|
|
172
|
+
await aquifer.migrate();
|
|
173
|
+
console.log(' OK\n');
|
|
174
|
+
|
|
175
|
+
// 2. Commit
|
|
176
|
+
const sessionId = `quickstart-${Date.now()}`;
|
|
177
|
+
console.log('2/5 Committing test session...');
|
|
178
|
+
await aquifer.commit(sessionId, [
|
|
179
|
+
{ role: 'user', content: 'We decided to use PostgreSQL with pgvector for the AI memory store instead of a separate vector database.' },
|
|
180
|
+
{ role: 'assistant', content: 'Good choice. PG gives us ACID transactions, full-text search, and vector similarity all in one place.' },
|
|
181
|
+
{ role: 'user', content: 'The main advantage is turn-level embedding — we can find the exact moment a decision was made.' },
|
|
182
|
+
], { agentId: 'quickstart', source: 'quickstart' });
|
|
183
|
+
console.log(' OK\n');
|
|
184
|
+
|
|
185
|
+
// 3. Enrich (skip summary — LLM may not be configured)
|
|
186
|
+
console.log('3/5 Enriching (turn embeddings)...');
|
|
187
|
+
const enrichResult = await aquifer.enrich(sessionId, {
|
|
188
|
+
agentId: 'quickstart',
|
|
189
|
+
skipSummary: true,
|
|
190
|
+
skipEntities: true,
|
|
191
|
+
});
|
|
192
|
+
console.log(` OK — ${enrichResult.turnsEmbedded} turns embedded\n`);
|
|
193
|
+
|
|
194
|
+
// 4. Recall
|
|
195
|
+
console.log('4/5 Recalling "PostgreSQL memory store"...');
|
|
196
|
+
const results = await aquifer.recall('PostgreSQL memory store', { limit: 3 });
|
|
197
|
+
if (results.length === 0) {
|
|
198
|
+
console.error(' FAIL — no results returned. Check your embedding config.');
|
|
199
|
+
process.exitCode = 1;
|
|
200
|
+
return;
|
|
201
|
+
}
|
|
202
|
+
console.log(` OK — ${results.length} result(s), top score: ${results[0].score?.toFixed(3)}`);
|
|
203
|
+
if (results[0].matchedTurnText) {
|
|
204
|
+
console.log(` Matched: "${results[0].matchedTurnText.slice(0, 100)}..."`);
|
|
205
|
+
}
|
|
206
|
+
console.log();
|
|
207
|
+
|
|
208
|
+
// 5. Cleanup
|
|
209
|
+
console.log('5/5 Cleaning up test data...');
|
|
210
|
+
const { Pool } = require('pg');
|
|
211
|
+
const { loadConfig } = require('./shared/config');
|
|
212
|
+
const config = loadConfig();
|
|
213
|
+
const pool = new Pool({ connectionString: config.db.url });
|
|
214
|
+
const schema = config.schema || 'aquifer';
|
|
215
|
+
await pool.query(`DELETE FROM ${schema}.turn_embeddings WHERE session_id IN (SELECT id FROM ${schema}.sessions WHERE session_id = $1)`, [sessionId]);
|
|
216
|
+
await pool.query(`DELETE FROM ${schema}.session_summaries WHERE session_id IN (SELECT id FROM ${schema}.sessions WHERE session_id = $1)`, [sessionId]);
|
|
217
|
+
await pool.query(`DELETE FROM ${schema}.sessions WHERE session_id = $1`, [sessionId]);
|
|
218
|
+
await pool.end();
|
|
219
|
+
console.log(' OK\n');
|
|
220
|
+
|
|
221
|
+
console.log('✓ Aquifer is working. You can now start the MCP server:');
|
|
222
|
+
console.log(' npx aquifer mcp');
|
|
223
|
+
}
|
|
224
|
+
|
|
166
225
|
async function cmdExport(aquifer, args) {
|
|
167
226
|
const output = args.flags.output || null;
|
|
168
227
|
const limit = parseInt(args.flags.limit || '1000', 10);
|
|
@@ -201,6 +260,7 @@ async function main() {
|
|
|
201
260
|
console.log(`Usage: aquifer <command> [options]
|
|
202
261
|
|
|
203
262
|
Commands:
|
|
263
|
+
quickstart Verify end-to-end setup (migrate → commit → enrich → recall)
|
|
204
264
|
migrate Run database migrations
|
|
205
265
|
recall <query> Search sessions (requires embed config)
|
|
206
266
|
feedback Record trust feedback on a session
|
|
@@ -250,6 +310,9 @@ Options:
|
|
|
250
310
|
|
|
251
311
|
try {
|
|
252
312
|
switch (command) {
|
|
313
|
+
case 'quickstart':
|
|
314
|
+
await cmdQuickstart(aquifer);
|
|
315
|
+
break;
|
|
253
316
|
case 'migrate':
|
|
254
317
|
await cmdMigrate(aquifer);
|
|
255
318
|
break;
|
package/consumers/mcp.js
CHANGED
|
@@ -67,14 +67,14 @@ async function main() {
|
|
|
67
67
|
} catch (e) {
|
|
68
68
|
process.stderr.write(
|
|
69
69
|
'aquifer mcp requires @modelcontextprotocol/sdk and zod.\n' +
|
|
70
|
-
'
|
|
70
|
+
'These should be installed automatically. Try: npm install\n'
|
|
71
71
|
);
|
|
72
72
|
process.exit(1);
|
|
73
73
|
}
|
|
74
74
|
|
|
75
75
|
const server = new McpServer({
|
|
76
76
|
name: 'aquifer-memory',
|
|
77
|
-
version: '0.
|
|
77
|
+
version: '0.9.0',
|
|
78
78
|
});
|
|
79
79
|
|
|
80
80
|
server.tool(
|
package/docs/setup.md
ADDED
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
# Aquifer Setup Guide
|
|
2
|
+
|
|
3
|
+
This guide walks you through installing Aquifer and verifying a complete write → enrich → recall cycle. By the end, you will have a working MCP memory server that an agent host can connect to.
|
|
4
|
+
|
|
5
|
+
## Prerequisites
|
|
6
|
+
|
|
7
|
+
You need three things running before Aquifer can work:
|
|
8
|
+
|
|
9
|
+
1. **PostgreSQL 15+** with the **pgvector** extension installed
|
|
10
|
+
2. **Node.js 18+**
|
|
11
|
+
3. **An embedding endpoint** — Ollama (local), OpenAI, or any OpenAI-compatible API
|
|
12
|
+
|
|
13
|
+
## Step 1: Database
|
|
14
|
+
|
|
15
|
+
### Option A: Docker (recommended for local dev)
|
|
16
|
+
|
|
17
|
+
The repo includes a `docker-compose.yml` that starts PostgreSQL 16 with pgvector and Ollama with bge-m3 auto-pulled:
|
|
18
|
+
|
|
19
|
+
```bash
|
|
20
|
+
cd /path/to/aquifer
|
|
21
|
+
docker compose up -d
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
This gives you a database at `postgresql://aquifer:aquifer@localhost:5432/aquifer` with pgvector ready, plus an Ollama server with bge-m3 for embeddings. First run takes a few minutes while the model downloads.
|
|
25
|
+
|
|
26
|
+
### Option B: Existing PostgreSQL
|
|
27
|
+
|
|
28
|
+
Make sure pgvector is installed. Connect as a superuser and run:
|
|
29
|
+
|
|
30
|
+
```sql
|
|
31
|
+
CREATE EXTENSION IF NOT EXISTS vector;
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
If your PostgreSQL was installed from a package manager, you may need to install the pgvector package separately. See [pgvector installation](https://github.com/pgvector/pgvector#installation).
|
|
35
|
+
|
|
36
|
+
## Step 2: Install Aquifer
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
npm install @shadowforge0/aquifer-memory
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
All dependencies including MCP SDK and zod are installed automatically.
|
|
43
|
+
|
|
44
|
+
## Step 3: Configure
|
|
45
|
+
|
|
46
|
+
Aquifer reads configuration from three sources (in priority order):
|
|
47
|
+
|
|
48
|
+
1. Config file: `aquifer.config.json` in the working directory, or set `AQUIFER_CONFIG=/path/to/config.json`
|
|
49
|
+
2. Environment variables (see below)
|
|
50
|
+
3. Programmatic overrides via `createAquifer()`
|
|
51
|
+
|
|
52
|
+
### Minimum env vars for MCP recall
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
export DATABASE_URL="postgresql://aquifer:aquifer@localhost:5432/aquifer"
|
|
56
|
+
export AQUIFER_EMBED_BASE_URL="http://localhost:11434/v1"
|
|
57
|
+
export AQUIFER_EMBED_MODEL="bge-m3"
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
### Optional but common
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
# PG schema (default: aquifer) — useful for running multiple instances in one database
|
|
64
|
+
export AQUIFER_SCHEMA="aquifer"
|
|
65
|
+
|
|
66
|
+
# LLM for built-in summarization — without this, enrich requires a custom summaryFn
|
|
67
|
+
export AQUIFER_LLM_BASE_URL="http://localhost:11434/v1"
|
|
68
|
+
export AQUIFER_LLM_MODEL="llama3.1"
|
|
69
|
+
|
|
70
|
+
# Knowledge graph
|
|
71
|
+
export AQUIFER_ENTITIES_ENABLED="true"
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
Copy `.env.example` from the repo root for a full annotated list.
|
|
75
|
+
|
|
76
|
+
## Step 4: Verify everything works
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
npx aquifer quickstart
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
This single command runs migrations, commits a test session, embeds it, recalls it, and cleans up. If it prints `✓ Aquifer is working`, your setup is correct.
|
|
83
|
+
|
|
84
|
+
You can also run individual steps manually: `npx aquifer migrate`, `npx aquifer stats`, etc.
|
|
85
|
+
|
|
86
|
+
## Step 5: Start the MCP server
|
|
87
|
+
|
|
88
|
+
```bash
|
|
89
|
+
npx aquifer mcp
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
The server starts on stdio and waits for MCP client connections. There is no visible output on success — the server is ready when the process stays running without error.
|
|
93
|
+
|
|
94
|
+
### Verify with the library API (optional)
|
|
95
|
+
|
|
96
|
+
If you want to test the library directly instead of the CLI:
|
|
97
|
+
|
|
98
|
+
```javascript
|
|
99
|
+
const { createAquifer, createEmbedder } = require('@shadowforge0/aquifer-memory');
|
|
100
|
+
|
|
101
|
+
const embedder = createEmbedder({
|
|
102
|
+
provider: 'ollama',
|
|
103
|
+
ollamaUrl: 'http://localhost:11434',
|
|
104
|
+
model: 'bge-m3',
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
const aquifer = createAquifer({
|
|
108
|
+
db: process.env.DATABASE_URL,
|
|
109
|
+
schema: 'aquifer',
|
|
110
|
+
embed: { fn: (texts) => embedder.embedBatch(texts) },
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
await aquifer.migrate();
|
|
114
|
+
|
|
115
|
+
// Commit a test session
|
|
116
|
+
await aquifer.commit('test-001', [
|
|
117
|
+
{ role: 'user', content: 'We decided to use PostgreSQL for the memory store.' },
|
|
118
|
+
{ role: 'assistant', content: 'Good choice — PG gives us ACID, FTS, and pgvector in one place.' },
|
|
119
|
+
], { agentId: 'test' });
|
|
120
|
+
|
|
121
|
+
// Enrich (embed turns — summarization needs LLM config)
|
|
122
|
+
await aquifer.enrich('test-001', { agentId: 'test', skipSummary: true });
|
|
123
|
+
|
|
124
|
+
// Recall
|
|
125
|
+
const results = await aquifer.recall('PostgreSQL memory', { limit: 3 });
|
|
126
|
+
console.log('Results:', results.length); // Should be >= 1
|
|
127
|
+
|
|
128
|
+
await aquifer.close();
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
## Connecting a host
|
|
132
|
+
|
|
133
|
+
Once the MCP server is verified, connect your agent host:
|
|
134
|
+
|
|
135
|
+
### Claude Code
|
|
136
|
+
|
|
137
|
+
Add to `.claude.json` (project-level) or user-level MCP config:
|
|
138
|
+
|
|
139
|
+
```json
|
|
140
|
+
{
|
|
141
|
+
"mcpServers": {
|
|
142
|
+
"aquifer": {
|
|
143
|
+
"type": "stdio",
|
|
144
|
+
"command": "node",
|
|
145
|
+
"args": ["/absolute/path/to/aquifer/consumers/mcp.js"],
|
|
146
|
+
"env": {
|
|
147
|
+
"DATABASE_URL": "postgresql://aquifer:aquifer@localhost:5432/aquifer",
|
|
148
|
+
"AQUIFER_EMBED_BASE_URL": "http://localhost:11434/v1",
|
|
149
|
+
"AQUIFER_EMBED_MODEL": "bge-m3"
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
Tools appear as `mcp__aquifer__session_recall`, `mcp__aquifer__session_feedback`, `mcp__aquifer__memory_stats`, `mcp__aquifer__memory_pending`.
|
|
157
|
+
|
|
158
|
+
### OpenClaw
|
|
159
|
+
|
|
160
|
+
Add to `openclaw.json`:
|
|
161
|
+
|
|
162
|
+
```json
|
|
163
|
+
{
|
|
164
|
+
"mcp": {
|
|
165
|
+
"servers": {
|
|
166
|
+
"aquifer": {
|
|
167
|
+
"command": "node",
|
|
168
|
+
"args": ["/absolute/path/to/aquifer/consumers/mcp.js"],
|
|
169
|
+
"env": {
|
|
170
|
+
"DATABASE_URL": "postgresql://...",
|
|
171
|
+
"AQUIFER_EMBED_BASE_URL": "http://localhost:11434/v1",
|
|
172
|
+
"AQUIFER_EMBED_MODEL": "bge-m3"
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
Tools materialize as `aquifer__session_recall`, `aquifer__session_feedback`, `aquifer__memory_stats`, `aquifer__memory_pending`.
|
|
181
|
+
|
|
182
|
+
Do **not** use the OpenClaw plugin (`consumers/openclaw-plugin.js`) for tool delivery. The plugin is retained for session capture via `before_reset` only.
|
|
183
|
+
|
|
184
|
+
## Troubleshooting
|
|
185
|
+
|
|
186
|
+
**`error: type "vector" does not exist`** — pgvector is not installed. Use the `pgvector/pgvector` Docker image, or install the extension manually: `CREATE EXTENSION IF NOT EXISTS vector;` (requires superuser).
|
|
187
|
+
|
|
188
|
+
**`aquifer mcp requires @modelcontextprotocol/sdk and zod`** — These are regular dependencies and should be installed automatically. Run `npm install` again to ensure all deps are present.
|
|
189
|
+
|
|
190
|
+
**Recall returns empty results** — Sessions must be enriched before they are searchable. Run `npx aquifer stats` and check that summaries and/or turn embeddings exist. If not, run `npx aquifer backfill` to enrich pending sessions.
|
|
191
|
+
|
|
192
|
+
**`ECONNREFUSED` on embed calls** — Your embedding endpoint is not reachable. For Ollama: make sure it is running (`ollama serve`) and the model is pulled (`ollama pull bge-m3`).
|
|
193
|
+
|
|
194
|
+
**Enrich fails with "no LLM configured"** — The built-in summarizer needs `AQUIFER_LLM_BASE_URL` + `AQUIFER_LLM_MODEL`. Alternatively, pass `skipSummary: true` to enrich without summarization (turn embeddings still work), or provide your own `summaryFn`.
|
package/package.json
CHANGED
|
@@ -1,14 +1,16 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@shadowforge0/aquifer-memory",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "PG-native long-term memory for AI agents. Turn-level embedding, hybrid RRF ranking, optional knowledge graph.
|
|
3
|
+
"version": "0.9.0",
|
|
4
|
+
"description": "PG-native long-term memory for AI agents. Turn-level embedding, hybrid RRF ranking, optional knowledge graph. MCP server, CLI, and library API.",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"files": [
|
|
7
7
|
"index.js",
|
|
8
8
|
"core/",
|
|
9
9
|
"pipeline/",
|
|
10
10
|
"schema/",
|
|
11
|
-
"consumers/"
|
|
11
|
+
"consumers/",
|
|
12
|
+
"docs/",
|
|
13
|
+
"scripts/"
|
|
12
14
|
],
|
|
13
15
|
"bin": {
|
|
14
16
|
"aquifer": "./consumers/cli.js"
|
|
@@ -32,10 +34,8 @@
|
|
|
32
34
|
},
|
|
33
35
|
"author": "shadowforge0",
|
|
34
36
|
"dependencies": {
|
|
35
|
-
"pg": "^8.13.0"
|
|
36
|
-
},
|
|
37
|
-
"optionalDependencies": {
|
|
38
37
|
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
38
|
+
"pg": "^8.13.0",
|
|
39
39
|
"zod": "^3.25.76"
|
|
40
40
|
},
|
|
41
41
|
"engines": {
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Aquifer smoke test — validates the full write → enrich → recall cycle.
|
|
5
|
+
*
|
|
6
|
+
* Prerequisites:
|
|
7
|
+
* - DATABASE_URL set to a PostgreSQL database with pgvector
|
|
8
|
+
* - AQUIFER_EMBED_BASE_URL + AQUIFER_EMBED_MODEL set (e.g., Ollama bge-m3)
|
|
9
|
+
*
|
|
10
|
+
* Usage:
|
|
11
|
+
* node scripts/smoke.mjs
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { createRequire } from 'module';
|
|
15
|
+
const require = createRequire(import.meta.url);
|
|
16
|
+
|
|
17
|
+
const { createAquifer, createEmbedder } = require('../index.js');
|
|
18
|
+
const { loadConfig } = require('../consumers/shared/config.js');
|
|
19
|
+
|
|
20
|
+
const config = loadConfig();
|
|
21
|
+
|
|
22
|
+
if (!config.db.url) {
|
|
23
|
+
console.error('ERROR: DATABASE_URL is not set.');
|
|
24
|
+
process.exit(1);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
if (!config.embed.baseUrl || !config.embed.model) {
|
|
28
|
+
console.error('ERROR: AQUIFER_EMBED_BASE_URL and AQUIFER_EMBED_MODEL must be set.');
|
|
29
|
+
process.exit(1);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
// Build embedder
|
|
33
|
+
const isOllama = config.embed.baseUrl.includes('11434') || config.embed.baseUrl.includes('ollama');
|
|
34
|
+
const embedder = isOllama
|
|
35
|
+
? createEmbedder({
|
|
36
|
+
provider: 'ollama',
|
|
37
|
+
ollamaUrl: config.embed.baseUrl.replace(/\/v1\/?$/, ''),
|
|
38
|
+
model: config.embed.model,
|
|
39
|
+
})
|
|
40
|
+
: createEmbedder({
|
|
41
|
+
provider: 'openai',
|
|
42
|
+
openaiApiKey: config.embed.apiKey || '',
|
|
43
|
+
openaiModel: config.embed.model,
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
const aquifer = createAquifer({
|
|
47
|
+
db: config.db.url,
|
|
48
|
+
schema: config.schema || 'aquifer',
|
|
49
|
+
tenantId: config.tenantId || 'default',
|
|
50
|
+
embed: { fn: (texts) => embedder.embedBatch(texts), dim: config.embed.dim || null },
|
|
51
|
+
entities: { enabled: false },
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
const SESSION_ID = `smoke-test-${Date.now()}`;
|
|
55
|
+
|
|
56
|
+
try {
|
|
57
|
+
// 1. Migrate
|
|
58
|
+
console.log('1. Running migrations...');
|
|
59
|
+
await aquifer.migrate();
|
|
60
|
+
console.log(' OK');
|
|
61
|
+
|
|
62
|
+
// 2. Commit a test session
|
|
63
|
+
console.log('2. Committing test session...');
|
|
64
|
+
const commitResult = await aquifer.commit(SESSION_ID, [
|
|
65
|
+
{ role: 'user', content: 'We decided to use PostgreSQL with pgvector for the AI memory store instead of a separate vector database.' },
|
|
66
|
+
{ role: 'assistant', content: 'Good choice. PG gives us ACID transactions, full-text search, and vector similarity all in one place.' },
|
|
67
|
+
{ role: 'user', content: 'The main advantage is turn-level embedding — we can find the exact moment a decision was made.' },
|
|
68
|
+
], { agentId: 'smoke-test', source: 'smoke' });
|
|
69
|
+
console.log(` OK — session ${commitResult.isNew ? 'created' : 'updated'}`);
|
|
70
|
+
|
|
71
|
+
// 3. Enrich (skip summary since LLM may not be configured)
|
|
72
|
+
console.log('3. Enriching (turn embeddings, skip summary)...');
|
|
73
|
+
const enrichResult = await aquifer.enrich(SESSION_ID, {
|
|
74
|
+
agentId: 'smoke-test',
|
|
75
|
+
skipSummary: true,
|
|
76
|
+
skipEntities: true,
|
|
77
|
+
});
|
|
78
|
+
console.log(` OK — ${enrichResult.turnsEmbedded} turns embedded`);
|
|
79
|
+
|
|
80
|
+
// 4. Recall
|
|
81
|
+
console.log('4. Recalling "PostgreSQL memory store"...');
|
|
82
|
+
const results = await aquifer.recall('PostgreSQL memory store', { limit: 3 });
|
|
83
|
+
if (results.length === 0) {
|
|
84
|
+
console.error(' FAIL — no results returned');
|
|
85
|
+
process.exit(1);
|
|
86
|
+
}
|
|
87
|
+
console.log(` OK — ${results.length} result(s), top score: ${results[0].score?.toFixed(3)}`);
|
|
88
|
+
if (results[0].matchedTurnText) {
|
|
89
|
+
console.log(` Matched turn: "${results[0].matchedTurnText.slice(0, 100)}..."`);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
// 5. Stats
|
|
93
|
+
console.log('5. Checking stats...');
|
|
94
|
+
const stats = await aquifer.getStats();
|
|
95
|
+
console.log(` Sessions: ${stats.sessionTotal}, Turn embeddings: ${stats.turnEmbeddings}`);
|
|
96
|
+
|
|
97
|
+
// 6. Cleanup — remove smoke test session
|
|
98
|
+
console.log('6. Cleaning up...');
|
|
99
|
+
const { Pool } = require('pg');
|
|
100
|
+
const pool = new Pool({ connectionString: config.db.url });
|
|
101
|
+
const schema = config.schema || 'aquifer';
|
|
102
|
+
await pool.query(`DELETE FROM ${schema}.turn_embeddings WHERE session_id IN (SELECT id FROM ${schema}.sessions WHERE session_id = $1)`, [SESSION_ID]);
|
|
103
|
+
await pool.query(`DELETE FROM ${schema}.session_summaries WHERE session_id IN (SELECT id FROM ${schema}.sessions WHERE session_id = $1)`, [SESSION_ID]);
|
|
104
|
+
await pool.query(`DELETE FROM ${schema}.sessions WHERE session_id = $1`, [SESSION_ID]);
|
|
105
|
+
await pool.end();
|
|
106
|
+
console.log(' OK');
|
|
107
|
+
|
|
108
|
+
console.log('\n✓ smoke test passed');
|
|
109
|
+
} catch (err) {
|
|
110
|
+
console.error(`\n✗ smoke test failed: ${err.message}`);
|
|
111
|
+
if (err.stack) console.error(err.stack);
|
|
112
|
+
process.exit(1);
|
|
113
|
+
} finally {
|
|
114
|
+
await aquifer.close();
|
|
115
|
+
}
|