@shadowforge0/aquifer-memory 0.6.0 → 0.8.0

package/README.md

@@ -127,18 +127,28 @@ const results = await aquifer.recall('auth middleware decision', {
 ## Architecture
 
 ```
-┌─────────────────────────────────────────────────────────────┐
-│                    createAquifer (entry)                     │
-│         Config · Migration · Ingest · Recall · Enrich       │
-└────────┬──────────┬──────────┬──────────┬───────────────────┘
+┌──────────────────────────────────────────────────────────────┐
+│                     Agent Hosts                              │
+│   Claude Code · OpenClaw · Codex · OpenCode · ...            │
+└──────────────────────┬───────────────────────────────────────┘
+                       │ MCP (stdio or HTTP)
+┌──────────────────────▼───────────────────────────────────────┐
+│              Aquifer MCP Server (canonical API)               │
+│   session_recall · session_feedback · memory_stats · ...     │
+└──────────────────────┬───────────────────────────────────────┘
+                       │
+┌──────────────────────▼───────────────────────────────────────┐
+│                    createAquifer (engine)                     │
+│         Config · Migration · Ingest · Recall · Enrich        │
+└────────┬──────────┬──────────┬──────────┬────────────────────┘
          │          │          │          │
     ┌────▼───┐ ┌────▼────┐ ┌──▼───┐ ┌───▼──────────┐
     │storage │ │hybrid-  │ │entity│ │   pipeline/   │
     │  .js   │ │rank.js  │ │ .js  │ │summarize.js   │
     └────────┘ └─────────┘ └──────┘ │embed.js       │
          │                     │    │extract-ent.js │
-    ┌────▼───────────┐    ┌───▼──┐  └───────────────┘
-    │  PostgreSQL     │    │ LLM  │
+    ┌────▼───────────┐    ┌───▼──┐  │rerank.js      │
+    │  PostgreSQL     │    │ LLM  │  └───────────────┘
     │  + pgvector     │    │ API  │
     └────────────────┘    └──────┘
 
@@ -151,11 +161,13 @@ const results = await aquifer.recall('auth middleware decision', {
     └──────────────────────────────────┘
 ```
 
+**Integration model:** MCP is the primary integration surface. Agent hosts connect to Aquifer through the MCP server (`consumers/mcp.js`), which exposes `session_recall`, `session_feedback`, `memory_stats`, and `memory_pending`. The CLI wraps the same engine for command-line use. The OpenClaw plugin (`consumers/openclaw-plugin.js`) is retained as a compatibility adapter for session capture but is not the primary tool delivery path.
+
 ### File Reference
 
 | File | Purpose |
 |------|---------|
-| `index.js` | Entry point — exports `createAquifer`, `createEmbedder` |
+| `index.js` | Entry point — exports `createAquifer`, `createEmbedder`, `createReranker`, `normalizeSession` |
 | `core/aquifer.js` | Main facade: `migrate()`, `ingest()`, `recall()`, `enrich()` |
 | `core/storage.js` | Session/summary/turn CRUD, FTS search, embedding search |
 | `core/entity.js` | Entity upsert, mention tracking, relation graph, normalization |
@@ -163,6 +175,8 @@ const results = await aquifer.recall('auth middleware decision', {
 | `pipeline/summarize.js` | LLM-powered session summarization with structured output |
 | `pipeline/embed.js` | Embedding client (any OpenAI-compatible API) |
 | `pipeline/extract-entities.js` | LLM-powered entity extraction (12 types) |
+| `pipeline/rerank.js` | Cross-encoder reranking (TEI, Jina, OpenRouter) |
+| `pipeline/normalize/` | Session normalization for Claude Code / gateway noise |
 | `schema/001-base.sql` | DDL: sessions, summaries, turn_embeddings, FTS indexes |
 | `schema/002-entities.sql` | DDL: entities, mentions, relations, entity_sessions |
 | `schema/003-trust-feedback.sql` | DDL: trust_score column, session_feedback audit trail |
@@ -395,9 +409,48 @@ createAquifer({
 
 Fallback chain: `config.entities.scope` → `'default'`.
 
-### Consumers (CLI, MCP, OpenClaw plugin)
+### MCP Server (primary integration)
+
+Agent hosts should connect through the Aquifer MCP server. For OpenClaw, add to `openclaw.json`:
+
+```json
+{
+  "mcp": {
+    "servers": {
+      "aquifer": {
+        "command": "node",
+        "args": ["/path/to/aquifer/consumers/mcp.js"],
+        "env": {
+          "DATABASE_URL": "postgresql://...",
+          "AQUIFER_SCHEMA": "aquifer",
+          "AQUIFER_EMBED_BASE_URL": "http://localhost:11434/v1",
+          "AQUIFER_EMBED_MODEL": "bge-m3"
+        }
+      }
+    }
+  }
+}
+```
+
+Tools are exposed as `aquifer__session_recall`, `aquifer__session_feedback`, `aquifer__memory_stats`, `aquifer__memory_pending` (server name prefix is added by the host).
+
+For Claude Code, add to `.claude.json`:
+
+```json
+{
+  "mcpServers": {
+    "aquifer": {
+      "type": "stdio",
+      "command": "node",
+      "args": ["/path/to/aquifer/consumers/mcp.js"]
+    }
+  }
+}
+```
+
+### CLI (secondary)
 
-For consumer-based setup using environment variables instead of code:
+For command-line use with environment variables:
 
 ```bash
 export DATABASE_URL="postgresql://..."

package/consumers/cli.js

@@ -14,7 +14,6 @@
  */
 
 const { createAquiferFromConfig } = require('./shared/factory');
-const { loadConfig } = require('./shared/config');
 
 // ---------------------------------------------------------------------------
 // Argument parser (minimal, no deps)
@@ -120,30 +119,12 @@ async function cmdBackfill(aquifer, args) {
   const skipTurnEmbed = !!args.flags['skip-turn-embed'];
   const skipEntities = !!args.flags['skip-entities'];
 
-  const config = aquifer._config || {};
-  const schema = config.schema || 'aquifer';
-  const tenantId = config.tenantId || 'default';
-  const pool = aquifer._pool;
+  const pending = await aquifer.getPendingSessions({ limit });
 
-  if (!pool) {
-    console.error('Backfill requires direct pool access.');
-    process.exit(1);
-  }
-
-  const qi = (id) => `"${id}"`;
-  const { rows } = await pool.query(`
-    SELECT session_id, agent_id, processing_status
-    FROM ${qi(schema)}.sessions
-    WHERE tenant_id = $1
-      AND processing_status IN ('pending', 'failed')
-    ORDER BY started_at DESC
-    LIMIT $2
-  `, [tenantId, limit]);
-
-  console.log(`Found ${rows.length} sessions to backfill${dryRun ? ' (dry-run)' : ''}`);
+  console.log(`Found ${pending.length} sessions to backfill${dryRun ? ' (dry-run)' : ''}`);
 
   let enriched = 0, failed = 0;
-  for (const row of rows) {
+  for (const row of pending) {
     if (dryRun) {
       console.log(`  [dry-run] ${row.session_id} (${row.agent_id}) status=${row.processing_status}`);
       continue;
@@ -164,40 +145,12 @@ async function cmdBackfill(aquifer, args) {
     }
   }
 
-  console.log(`\nDone. enriched=${enriched} failed=${failed} total=${rows.length}`);
+  console.log(`\nDone. enriched=${enriched} failed=${failed} total=${pending.length}`);
   if (failed > 0) process.exitCode = 2;
 }
 
 async function cmdStats(aquifer, args) {
-  const config = aquifer._config || {};
-  const schema = config.schema || 'aquifer';
-  const tenantId = config.tenantId || 'default';
-  const pool = aquifer._pool;
-
-  if (!pool) {
-    console.error('Stats requires direct pool access.');
-    process.exit(1);
-  }
-
-  const qi = (id) => `"${id}"`;
-  const [sessions, summaries, turns, entities] = await Promise.all([
-    pool.query(`SELECT processing_status, COUNT(*)::int as count FROM ${qi(schema)}.sessions WHERE tenant_id = $1 GROUP BY processing_status`, [tenantId]),
-    pool.query(`SELECT COUNT(*)::int as count FROM ${qi(schema)}.session_summaries WHERE tenant_id = $1`, [tenantId]),
-    pool.query(`SELECT COUNT(*)::int as count FROM ${qi(schema)}.turn_embeddings WHERE tenant_id = $1`, [tenantId]),
-    pool.query(`SELECT COUNT(*)::int as count FROM ${qi(schema)}.entities WHERE tenant_id = $1`, [tenantId]).catch(() => ({ rows: [{ count: 0 }] })),
-  ]);
-
-  const timeRange = await pool.query(`SELECT MIN(started_at) as earliest, MAX(started_at) as latest FROM ${qi(schema)}.sessions WHERE tenant_id = $1`, [tenantId]);
-
-  const stats = {
-    sessions: Object.fromEntries(sessions.rows.map(r => [r.processing_status, r.count])),
-    sessionTotal: sessions.rows.reduce((s, r) => s + r.count, 0),
-    summaries: summaries.rows[0]?.count || 0,
-    turnEmbeddings: turns.rows[0]?.count || 0,
-    entities: entities.rows[0]?.count || 0,
-    earliest: timeRange.rows[0]?.earliest || null,
-    latest: timeRange.rows[0]?.latest || null,
-  };
+  const stats = await aquifer.getStats();
 
   if (args.flags.json) {
     console.log(JSON.stringify(stats, null, 2));
@@ -211,34 +164,14 @@ async function cmdStats(aquifer, args) {
 }
 
 async function cmdExport(aquifer, args) {
-  const config = aquifer._config || {};
-  const schema = config.schema || 'aquifer';
-  const tenantId = config.tenantId || 'default';
-  const pool = aquifer._pool;
   const output = args.flags.output || null;
   const limit = parseInt(args.flags.limit || '1000', 10);
 
-  if (!pool) {
-    console.error('Export requires direct pool access.');
-    process.exit(1);
-  }
-
-  const qi = (id) => `"${id}"`;
-  const where = [`s.tenant_id = $1`];
-  const params = [tenantId];
-
-  if (args.flags['agent-id']) { params.push(args.flags['agent-id']); where.push(`s.agent_id = $${params.length}`); }
-  if (args.flags.source) { params.push(args.flags.source); where.push(`s.source = $${params.length}`); }
-  params.push(limit);
-
-  const { rows } = await pool.query(`
-    SELECT s.*, ss.summary_text, ss.structured_summary
-    FROM ${qi(schema)}.sessions s
-    LEFT JOIN ${qi(schema)}.session_summaries ss ON ss.session_row_id = s.id
-    WHERE ${where.join(' AND ')}
-    ORDER BY s.started_at DESC
-    LIMIT $${params.length}
-  `, params);
+  const rows = await aquifer.exportSessions({
+    agentId: args.flags['agent-id'],
+    source: args.flags.source,
+    limit,
+  });
 
   const stream = output ? require('fs').createWriteStream(output) : process.stdout;
   for (const row of rows) {
@@ -340,7 +273,7 @@ Options:
         process.exit(1);
     }
   } finally {
-    if (aquifer._pool) await aquifer._pool.end();
+    await aquifer.close();
   }
 }
 

package/consumers/mcp.js

@@ -2,7 +2,12 @@
 'use strict';
 
 /**
- * Aquifer MCP Server — session_recall tool via Model Context Protocol.
+ * Aquifer MCP Server — canonical external contract for agent host integration.
+ *
+ * This is the primary integration surface for Aquifer. Agent hosts (Claude Code,
+ * Codex, OpenCode, etc.) should integrate through this MCP server.
+ *
+ * Tools: session_recall, session_feedback, memory_stats, memory_pending
  *
  * Usage:
  *   npx aquifer mcp
@@ -69,7 +74,7 @@ async function main() {
 
   const server = new McpServer({
     name: 'aquifer-memory',
-    version: '0.6.0',
+    version: '0.8.0',
   });
 
   server.tool(
@@ -84,6 +89,7 @@ async function main() {
       dateTo: z.string().optional().describe('End date YYYY-MM-DD'),
       entities: z.array(z.string()).optional().describe('Entity names to match'),
       entityMode: z.enum(['any', 'all']).optional().describe('"any" (default, boost) or "all" (only sessions with every entity)'),
+      mode: z.enum(['fts', 'hybrid', 'vector']).optional().describe('Recall mode: "fts" (keyword only, no embed needed), "hybrid" (default, FTS + vector), "vector" (vector only)'),
     },
     async (params) => {
       try {
@@ -100,6 +106,7 @@ async function main() {
           recallOpts.entities = params.entities;
           recallOpts.entityMode = params.entityMode || 'any';
         }
+        if (params.mode) recallOpts.mode = params.mode;
 
         const results = await aquifer.recall(params.query, recallOpts);
         const text = formatResults(results, params.query);
@@ -120,6 +127,7 @@ async function main() {
       sessionId: z.string().min(1).describe('Session ID to give feedback on'),
       verdict: z.enum(['helpful', 'unhelpful']).describe('Was the recalled session useful?'),
       note: z.string().optional().describe('Optional reason'),
+      agentId: z.string().optional().describe('Agent ID the session was stored under (e.g. "main"). Defaults to "agent" if omitted.'),
     },
     async (params) => {
       try {
@@ -127,6 +135,7 @@ async function main() {
         const result = await aquifer.feedback(params.sessionId, {
           verdict: params.verdict,
           note: params.note || undefined,
+          agentId: params.agentId || undefined,
         });
         return {
           content: [{ type: 'text', text: `Feedback: ${result.verdict} (trust ${result.trustBefore.toFixed(2)} → ${result.trustAfter.toFixed(2)})` }],
@@ -140,9 +149,64 @@ async function main() {
     }
   );
 
+  server.tool(
+    'memory_stats',
+    'Return storage statistics for the Aquifer memory store (session counts by status, summaries, turn embeddings, entities, date range).',
+    {},
+    async () => {
+      try {
+        const aquifer = getAquifer();
+        const stats = await aquifer.getStats();
+        const lines = [
+          `Sessions: ${stats.sessionTotal} total`,
+        ];
+        for (const [status, count] of Object.entries(stats.sessions)) {
+          lines.push(`  ${status}: ${count}`);
+        }
+        lines.push(`Summaries: ${stats.summaries}`);
+        lines.push(`Turn embeddings: ${stats.turnEmbeddings}`);
+        lines.push(`Entities: ${stats.entities}`);
+        if (stats.earliest) lines.push(`Date range: ${new Date(stats.earliest).toISOString().slice(0, 10)} → ${new Date(stats.latest).toISOString().slice(0, 10)}`);
+        return { content: [{ type: 'text', text: lines.join('\n') }] };
+      } catch (err) {
+        return {
+          content: [{ type: 'text', text: `memory_stats error: ${err.message}` }],
+          isError: true,
+        };
+      }
+    }
+  );
+
+  server.tool(
+    'memory_pending',
+    'List sessions with pending or failed processing status.',
+    {
+      limit: z.number().int().min(1).max(200).optional().describe('Max results (default 20)'),
+    },
+    async (params) => {
+      try {
+        const aquifer = getAquifer();
+        const rows = await aquifer.getPendingSessions({ limit: params.limit ?? 20 });
+        if (rows.length === 0) {
+          return { content: [{ type: 'text', text: 'No pending or failed sessions.' }] };
+        }
+        const lines = [`${rows.length} pending/failed session(s):\n`];
+        for (const row of rows) {
+          lines.push(`${row.session_id}  [${row.processing_status}]  agent=${row.agent_id}`);
+        }
+        return { content: [{ type: 'text', text: lines.join('\n') }] };
+      } catch (err) {
+        return {
+          content: [{ type: 'text', text: `memory_pending error: ${err.message}` }],
+          isError: true,
+        };
+      }
+    }
+  );
+
   // Graceful shutdown
   const cleanup = async () => {
-    if (_aquifer?._pool) await _aquifer._pool.end().catch(() => {});
+    if (_aquifer) await _aquifer.close().catch(() => {});
     process.exit(0);
   };
   process.on('SIGINT', cleanup);
@@ -153,7 +217,7 @@ async function main() {
 
   // Clean up pool when transport closes (stdin EOF)
   transport.onclose = async () => {
-    if (_aquifer?._pool) await _aquifer._pool.end().catch(() => {});
+    if (_aquifer) await _aquifer.close().catch(() => {});
   };
 }
 

package/consumers/openclaw-plugin.js

@@ -1,11 +1,17 @@
 'use strict';
 
 /**
- * Aquifer Memory — OpenClaw Plugin
+ * Aquifer Memory — OpenClaw Host Adapter
  *
- * Auto-captures sessions on before_reset and provides session_recall tool.
- * Install: add to openclaw.json plugins or extensions directory.
+ * Ingest adapter: auto-captures sessions on before_reset.
+ * Tool adapter: exposes session_recall/session_feedback via OpenClaw registerTool().
+ *
+ * Status: COMPATIBILITY ONLY. The official tool delivery path is mcp.servers.aquifer
+ * (see consumers/mcp.js). registerTool() exposure has OpenClaw upstream limitations
+ * that prevent reliable tool visibility. This plugin is retained for before_reset
+ * session capture; tool registration code is kept for future upstream fixes.
  *
+ * Install: add to openclaw.json plugins or extensions directory.
  * Config via plugin config, environment variables, or aquifer.config.json.
  */
 
@@ -169,6 +175,10 @@ module.exports = {
             } catch (enrichErr) {
               api.logger.warn(`[aquifer-memory] enrich failed for ${sessionId}: ${enrichErr.message}`);
             }
+          } else {
+            try {
+              await aquifer.skip(sessionId, { agentId, reason: `user_count=${norm.userCount} < min=${minUserMessages}` });
+            } catch (e) { api.logger.warn(`[aquifer-memory] skip failed for ${sessionId}: ${e.message}`); }
           }
 
           recentlyProcessed.set(dedupKey, Date.now());
@@ -189,8 +199,6 @@ module.exports = {
 
     // --- session_recall tool ---
 
-    // --- session_recall tool ---
-
     api.registerTool((ctx) => {
       if ((ctx?.sessionKey || '').includes('subagent')) return null;
 
@@ -208,6 +216,7 @@ module.exports = {
             dateTo: { type: 'string', description: 'End date YYYY-MM-DD' },
             entities: { type: 'array', items: { type: 'string' }, description: 'Entity names to match' },
             entityMode: { type: 'string', enum: ['any', 'all'], description: '"any" (default, boost) or "all" (only sessions with every entity)' },
+            mode: { type: 'string', enum: ['fts', 'hybrid', 'vector'], description: 'Recall mode: "fts" (keyword only), "hybrid" (default), "vector" (vector only)' },
           },
           required: ['query'],
         },
@@ -225,6 +234,7 @@ module.exports = {
               recallOpts.entities = params.entities;
               recallOpts.entityMode = params.entityMode || 'any';
             }
+            if (params.mode) recallOpts.mode = params.mode;
 
             const results = await aquifer.recall(params.query, recallOpts);
             const text = formatRecallResults(results);
@@ -253,14 +263,17 @@ module.exports = {
             sessionId: { type: 'string', description: 'Session ID to give feedback on' },
             verdict: { type: 'string', enum: ['helpful', 'unhelpful'], description: 'Was the recalled session useful?' },
             note: { type: 'string', description: 'Optional reason' },
+            agentId: { type: 'string', description: 'Agent ID the session was stored under (e.g. "main"). Defaults to context agent or "agent" if omitted.' },
           },
           required: ['sessionId', 'verdict'],
         },
         async execute(_toolCallId, params) {
           try {
+            const resolvedAgentId = params.agentId || ctx?.agentId || undefined;
             const result = await aquifer.feedback(params.sessionId, {
               verdict: params.verdict,
               note: params.note || undefined,
+              agentId: resolvedAgentId,
             });
             return {
               content: [{ type: 'text', text: `Feedback: ${result.verdict} (trust ${result.trustBefore.toFixed(2)} → ${result.trustAfter.toFixed(2)})` }],

package/consumers/shared/config.js

@@ -29,8 +29,19 @@ const DEFAULTS = {
     maxRetries: 3,
     temperature: 0,
   },
-  entities: { enabled: false, mergeCall: true },
+  entities: { enabled: false, mergeCall: true, scope: 'default' },
   rank: { rrf: 0.65, timeDecay: 0.25, access: 0.10, entityBoost: 0.18 },
+  rerank: {
+    enabled: false,
+    provider: null,    // 'tei' | 'jina' | 'openrouter' | 'custom'
+    baseUrl: null,     // TEI base URL
+    apiKey: null,      // Jina / OpenRouter API key
+    model: null,       // model override (Jina / OpenRouter)
+    topK: 20,
+    maxChars: 1600,
+    timeoutMs: 2000,
+    maxRetries: 1,
+  },
 };
 
 // ---------------------------------------------------------------------------
@@ -57,6 +68,15 @@ const ENV_MAP = [
   ['AQUIFER_LLM_TIMEOUT_MS',    'llm.timeoutMs',     Number],
   ['AQUIFER_LLM_TEMPERATURE',   'llm.temperature',   Number],
   ['AQUIFER_ENTITIES_ENABLED',  'entities.enabled',  Boolean],
+  ['AQUIFER_ENTITY_SCOPE',     'entities.scope'],
+  ['AQUIFER_RERANK_ENABLED',   'rerank.enabled',    Boolean],
+  ['AQUIFER_RERANK_PROVIDER',  'rerank.provider'],
+  ['AQUIFER_RERANK_BASE_URL',  'rerank.baseUrl'],
+  ['AQUIFER_RERANK_API_KEY',   'rerank.apiKey'],
+  ['AQUIFER_RERANK_MODEL',     'rerank.model'],
+  ['AQUIFER_RERANK_TOP_K',     'rerank.topK',       Number],
+  ['AQUIFER_RERANK_MAX_CHARS', 'rerank.maxChars',   Number],
+  ['AQUIFER_RERANK_TIMEOUT_MS','rerank.timeoutMs',   Number],
 ];
 
 // ---------------------------------------------------------------------------

package/consumers/shared/factory.js

@@ -1,7 +1,7 @@
 'use strict';
 
 const { Pool } = require('pg');
-const { createAquifer, createEmbedder } = require('../../index');
+const { createAquifer, createEmbedder, createReranker } = require('../../index');
 const { loadConfig } = require('./config');
 const { createLlmFn } = require('./llm');
 
@@ -57,20 +57,41 @@ function createAquiferFromConfig(overrides) {
     llmFn = createLlmFn(config.llm);
   }
 
+  // Rerank config (optional)
+  let rerankOpts = null;
+  if (config.rerank && config.rerank.enabled && config.rerank.provider) {
+    const rc = config.rerank;
+    const rerankConfig = { provider: rc.provider, topK: rc.topK, maxChars: rc.maxChars };
+    if (rc.provider === 'tei') {
+      rerankConfig.teiBaseUrl = rc.baseUrl || 'http://localhost:8080';
+      rerankConfig.timeout = rc.timeoutMs || 2000;
+      rerankConfig.maxRetries = rc.maxRetries ?? 1;
+    } else if (rc.provider === 'jina') {
+      rerankConfig.jinaApiKey = rc.apiKey;
+      if (rc.model) rerankConfig.jinaModel = rc.model;
+      rerankConfig.timeout = rc.timeoutMs || 2000;
+      rerankConfig.maxRetries = rc.maxRetries ?? 1;
+    } else if (rc.provider === 'openrouter') {
+      rerankConfig.openrouterApiKey = rc.apiKey;
+      if (rc.model) rerankConfig.model = rc.model;
+      rerankConfig.timeout = rc.timeoutMs || 5000;
+      rerankConfig.maxRetries = rc.maxRetries ?? 1;
+    }
+    rerankOpts = rerankConfig;
+  }
+
   const aquifer = createAquifer({
     db: pool,
+    ownsPool: true,
     schema: config.schema,
     tenantId: config.tenantId,
     embed: embedFn ? { fn: embedFn, dim: config.embed.dim || null } : null,
     llm: llmFn ? { fn: llmFn } : null,
     entities: config.entities,
     rank: config.rank,
+    rerank: rerankOpts,
   });
 
-  // Attach pool for lifecycle management
-  aquifer._pool = pool;
-  aquifer._config = config;
-
   return aquifer;
 }