@shadowforge0/aquifer-memory 0.7.0 → 0.9.0

package/consumers/mcp.js

@@ -2,7 +2,12 @@
 'use strict';
 
 /**
- * Aquifer MCP Server — session_recall tool via Model Context Protocol.
+ * Aquifer MCP Server — canonical external contract for agent host integration.
+ *
+ * This is the primary integration surface for Aquifer. Agent hosts (Claude Code,
+ * Codex, OpenCode, etc.) should integrate through this MCP server.
+ *
+ * Tools: session_recall, session_feedback, memory_stats, memory_pending
  *
  * Usage:
  *   npx aquifer mcp
@@ -62,14 +67,14 @@ async function main() {
   } catch (e) {
     process.stderr.write(
       'aquifer mcp requires @modelcontextprotocol/sdk and zod.\n' +
-      'Install: npm install @modelcontextprotocol/sdk zod\n'
+      'These should be installed automatically. Try: npm install\n'
     );
     process.exit(1);
   }
 
   const server = new McpServer({
     name: 'aquifer-memory',
-    version: '0.6.0',
+    version: '0.9.0',
   });
 
   server.tool(
@@ -84,6 +89,7 @@ async function main() {
       dateTo: z.string().optional().describe('End date YYYY-MM-DD'),
       entities: z.array(z.string()).optional().describe('Entity names to match'),
       entityMode: z.enum(['any', 'all']).optional().describe('"any" (default, boost) or "all" (only sessions with every entity)'),
+      mode: z.enum(['fts', 'hybrid', 'vector']).optional().describe('Recall mode: "fts" (keyword only, no embed needed), "hybrid" (default, FTS + vector), "vector" (vector only)'),
     },
     async (params) => {
       try {
@@ -100,6 +106,7 @@ async function main() {
           recallOpts.entities = params.entities;
           recallOpts.entityMode = params.entityMode || 'any';
         }
+        if (params.mode) recallOpts.mode = params.mode;
 
         const results = await aquifer.recall(params.query, recallOpts);
         const text = formatResults(results, params.query);
@@ -120,6 +127,7 @@ async function main() {
       sessionId: z.string().min(1).describe('Session ID to give feedback on'),
       verdict: z.enum(['helpful', 'unhelpful']).describe('Was the recalled session useful?'),
       note: z.string().optional().describe('Optional reason'),
+      agentId: z.string().optional().describe('Agent ID the session was stored under (e.g. "main"). Defaults to "agent" if omitted.'),
     },
     async (params) => {
       try {
@@ -127,6 +135,7 @@ async function main() {
         const result = await aquifer.feedback(params.sessionId, {
           verdict: params.verdict,
           note: params.note || undefined,
+          agentId: params.agentId || undefined,
         });
         return {
           content: [{ type: 'text', text: `Feedback: ${result.verdict} (trust ${result.trustBefore.toFixed(2)} → ${result.trustAfter.toFixed(2)})` }],
@@ -140,9 +149,64 @@ async function main() {
     }
   );
 
+  server.tool(
+    'memory_stats',
+    'Return storage statistics for the Aquifer memory store (session counts by status, summaries, turn embeddings, entities, date range).',
+    {},
+    async () => {
+      try {
+        const aquifer = getAquifer();
+        const stats = await aquifer.getStats();
+        const lines = [
+          `Sessions: ${stats.sessionTotal} total`,
+        ];
+        for (const [status, count] of Object.entries(stats.sessions)) {
+          lines.push(`  ${status}: ${count}`);
+        }
+        lines.push(`Summaries: ${stats.summaries}`);
+        lines.push(`Turn embeddings: ${stats.turnEmbeddings}`);
+        lines.push(`Entities: ${stats.entities}`);
+        if (stats.earliest) lines.push(`Date range: ${new Date(stats.earliest).toISOString().slice(0, 10)} → ${new Date(stats.latest).toISOString().slice(0, 10)}`);
+        return { content: [{ type: 'text', text: lines.join('\n') }] };
+      } catch (err) {
+        return {
+          content: [{ type: 'text', text: `memory_stats error: ${err.message}` }],
+          isError: true,
+        };
+      }
+    }
+  );
+
+  server.tool(
+    'memory_pending',
+    'List sessions with pending or failed processing status.',
+    {
+      limit: z.number().int().min(1).max(200).optional().describe('Max results (default 20)'),
+    },
+    async (params) => {
+      try {
+        const aquifer = getAquifer();
+        const rows = await aquifer.getPendingSessions({ limit: params.limit ?? 20 });
+        if (rows.length === 0) {
+          return { content: [{ type: 'text', text: 'No pending or failed sessions.' }] };
+        }
+        const lines = [`${rows.length} pending/failed session(s):\n`];
+        for (const row of rows) {
+          lines.push(`${row.session_id}  [${row.processing_status}]  agent=${row.agent_id}`);
+        }
+        return { content: [{ type: 'text', text: lines.join('\n') }] };
+      } catch (err) {
+        return {
+          content: [{ type: 'text', text: `memory_pending error: ${err.message}` }],
+          isError: true,
+        };
+      }
+    }
+  );
+
   // Graceful shutdown
   const cleanup = async () => {
-    if (_aquifer?._pool) await _aquifer._pool.end().catch(() => {});
+    if (_aquifer) await _aquifer.close().catch(() => {});
     process.exit(0);
   };
   process.on('SIGINT', cleanup);
@@ -153,7 +217,7 @@ async function main() {
 
   // Clean up pool when transport closes (stdin EOF)
   transport.onclose = async () => {
-    if (_aquifer?._pool) await _aquifer._pool.end().catch(() => {});
+    if (_aquifer) await _aquifer.close().catch(() => {});
   };
 }
 

package/consumers/openclaw-plugin.js

@@ -1,11 +1,17 @@
 'use strict';
 
 /**
- * Aquifer Memory — OpenClaw Plugin
+ * Aquifer Memory — OpenClaw Host Adapter
  *
- * Auto-captures sessions on before_reset and provides session_recall tool.
- * Install: add to openclaw.json plugins or extensions directory.
+ * Ingest adapter: auto-captures sessions on before_reset.
+ * Tool adapter: exposes session_recall/session_feedback via OpenClaw registerTool().
+ *
+ * Status: COMPATIBILITY ONLY. The official tool delivery path is mcp.servers.aquifer
+ * (see consumers/mcp.js). registerTool() exposure has OpenClaw upstream limitations
+ * that prevent reliable tool visibility. This plugin is retained for before_reset
+ * session capture; tool registration code is kept for future upstream fixes.
  *
+ * Install: add to openclaw.json plugins or extensions directory.
  * Config via plugin config, environment variables, or aquifer.config.json.
  */
 
@@ -169,6 +175,10 @@ module.exports = {
             } catch (enrichErr) {
               api.logger.warn(`[aquifer-memory] enrich failed for ${sessionId}: ${enrichErr.message}`);
             }
+          } else {
+            try {
+              await aquifer.skip(sessionId, { agentId, reason: `user_count=${norm.userCount} < min=${minUserMessages}` });
+            } catch (e) { api.logger.warn(`[aquifer-memory] skip failed for ${sessionId}: ${e.message}`); }
           }
 
           recentlyProcessed.set(dedupKey, Date.now());
@@ -189,8 +199,6 @@ module.exports = {
 
     // --- session_recall tool ---
 
-    // --- session_recall tool ---
-
     api.registerTool((ctx) => {
       if ((ctx?.sessionKey || '').includes('subagent')) return null;
 
@@ -208,6 +216,7 @@ module.exports = {
             dateTo: { type: 'string', description: 'End date YYYY-MM-DD' },
             entities: { type: 'array', items: { type: 'string' }, description: 'Entity names to match' },
             entityMode: { type: 'string', enum: ['any', 'all'], description: '"any" (default, boost) or "all" (only sessions with every entity)' },
+            mode: { type: 'string', enum: ['fts', 'hybrid', 'vector'], description: 'Recall mode: "fts" (keyword only), "hybrid" (default), "vector" (vector only)' },
           },
           required: ['query'],
         },
@@ -225,6 +234,7 @@ module.exports = {
               recallOpts.entities = params.entities;
               recallOpts.entityMode = params.entityMode || 'any';
             }
+            if (params.mode) recallOpts.mode = params.mode;
 
             const results = await aquifer.recall(params.query, recallOpts);
             const text = formatRecallResults(results);
@@ -253,14 +263,17 @@ module.exports = {
             sessionId: { type: 'string', description: 'Session ID to give feedback on' },
             verdict: { type: 'string', enum: ['helpful', 'unhelpful'], description: 'Was the recalled session useful?' },
             note: { type: 'string', description: 'Optional reason' },
+            agentId: { type: 'string', description: 'Agent ID the session was stored under (e.g. "main"). Defaults to context agent or "agent" if omitted.' },
           },
           required: ['sessionId', 'verdict'],
         },
         async execute(_toolCallId, params) {
           try {
+            const resolvedAgentId = params.agentId || ctx?.agentId || undefined;
             const result = await aquifer.feedback(params.sessionId, {
               verdict: params.verdict,
               note: params.note || undefined,
+              agentId: resolvedAgentId,
             });
             return {
               content: [{ type: 'text', text: `Feedback: ${result.verdict} (trust ${result.trustBefore.toFixed(2)} → ${result.trustAfter.toFixed(2)})` }],

package/consumers/shared/config.js

@@ -33,10 +33,10 @@ const DEFAULTS = {
   rank: { rrf: 0.65, timeDecay: 0.25, access: 0.10, entityBoost: 0.18 },
   rerank: {
     enabled: false,
-    provider: null,    // 'tei' | 'jina' | 'custom'
+    provider: null,    // 'tei' | 'jina' | 'openrouter' | 'custom'
     baseUrl: null,     // TEI base URL
-    apiKey: null,      // Jina API key
-    model: null,       // Jina model override
+    apiKey: null,      // Jina / OpenRouter API key
+    model: null,       // model override (Jina / OpenRouter)
     topK: 20,
     maxChars: 1600,
     timeoutMs: 2000,

package/consumers/shared/factory.js

@@ -71,12 +71,18 @@ function createAquiferFromConfig(overrides) {
       if (rc.model) rerankConfig.jinaModel = rc.model;
       rerankConfig.timeout = rc.timeoutMs || 2000;
       rerankConfig.maxRetries = rc.maxRetries ?? 1;
+    } else if (rc.provider === 'openrouter') {
+      rerankConfig.openrouterApiKey = rc.apiKey;
+      if (rc.model) rerankConfig.model = rc.model;
+      rerankConfig.timeout = rc.timeoutMs || 5000;
+      rerankConfig.maxRetries = rc.maxRetries ?? 1;
     }
     rerankOpts = rerankConfig;
   }
 
   const aquifer = createAquifer({
     db: pool,
+    ownsPool: true,
     schema: config.schema,
     tenantId: config.tenantId,
     embed: embedFn ? { fn: embedFn, dim: config.embed.dim || null } : null,
@@ -86,10 +92,6 @@ function createAquiferFromConfig(overrides) {
     rerank: rerankOpts,
   });
 
-  // Attach pool for lifecycle management
-  aquifer._pool = pool;
-  aquifer._config = config;
-
   return aquifer;
 }
 

package/core/aquifer.js

@@ -77,6 +77,7 @@ function createAquifer(config) {
     ownsPool = true;
   } else {
     pool = config.db;
+    ownsPool = !!config.ownsPool;  // allow factory to claim ownership
   }
 
   // Embed config (lazy — only required for recall/enrich)
@@ -99,8 +100,18 @@ function createAquifer(config) {
   const entityPromptFn = config.entities && config.entities.prompt ? config.entities.prompt : null;
   const entityScope = (config.entities && config.entities.scope) || 'default';
 
-  // FTS config (default: 'simple'; set to 'zhcfg' for Chinese tokenization)
-  const ftsConfig = config.ftsConfig || 'simple';
+  // FTS config — locked to 'simple'.
+  // The search_tsv trigger always uses to_tsvector('simple', ...), so query-time
+  // config must match.  Warn and override if someone passes anything else.
+  const _rawFtsConfig = config.ftsConfig || 'simple';
+  if (_rawFtsConfig !== 'simple') {
+    console.warn(
+      `[aquifer] ftsConfig '${_rawFtsConfig}' is not currently supported. ` +
+      `The search_tsv index is built with 'simple'; only 'simple' is valid at query time. ` +
+      `Overriding to 'simple'.`
+    );
+  }
+  const ftsConfig = 'simple';
 
   // Rank weights
   const rankWeights = {
@@ -551,7 +562,16 @@ function createAquifer(config) {
 
     async recall(query, opts = {}) {
       if (!query) return [];
-      requireEmbed('recall');
+
+      const VALID_MODES = ['fts', 'hybrid', 'vector'];
+      const mode = opts.mode !== undefined ? opts.mode : 'hybrid';
+      if (!VALID_MODES.includes(mode)) {
+        throw new Error(`Invalid recall mode: "${mode}". Must be one of: ${VALID_MODES.join(', ')}`);
+      }
+
+      if (mode === 'hybrid' || mode === 'vector') {
+        requireEmbed('recall');
+      }
 
       const {
         agentId,
@@ -582,10 +602,13 @@ function createAquifer(config) {
       const rerankTopK = rerankEnabled ? Math.max(limit, opts.rerankTopK || defaultRerankTopK) : limit;
       const fetchLimit = rerankTopK * 4;
 
-      // 1. Embed query
-      const queryVecResult = await embedFn([query]);
-      const queryVec = queryVecResult[0];
-      if (!queryVec || !queryVec.length) return []; // m3: guard empty array too
+      // 1. Embed query (only needed for hybrid/vector modes)
+      let queryVec = null;
+      if (mode === 'hybrid' || mode === 'vector') {
+        const queryVecResult = await embedFn([query]);
+        queryVec = queryVecResult[0];
+        if (!queryVec || !queryVec.length) return []; // m3: guard empty array too
+      }
 
       // 2. Entity intersection pre-filter (when entityMode === 'all')
       let candidateSessionIds = null; // null = no filter
@@ -661,17 +684,26 @@ function createAquifer(config) {
         } catch (_) { /* entity search failure non-fatal */ }
       }
 
-      // 3. Run 3 search paths in parallel
+      // 3. Run search paths in parallel (conditioned on mode)
+      const runFts = mode === 'fts' || mode === 'hybrid';
+      const runVector = mode === 'vector' || mode === 'hybrid';
+
       const [ftsRows, embRows, turnResult] = await Promise.all([
-        storage.searchSessions(pool, query, {
-          schema, tenantId, agentIds: resolvedAgentIds, source, dateFrom, dateTo, limit: fetchLimit, ftsConfig,
-        }).catch(() => []),
-        embeddingSearchSummaries(queryVec, {
-          agentIds: resolvedAgentIds, source, dateFrom, dateTo, limit: fetchLimit,
-        }).catch(() => []),
-        storage.searchTurnEmbeddings(pool, {
-          schema, tenantId, queryVec, dateFrom, dateTo, agentIds: resolvedAgentIds, source, limit: fetchLimit,
-        }).catch(() => ({ rows: [] })),
+        runFts
+          ? storage.searchSessions(pool, query, {
+              schema, tenantId, agentIds: resolvedAgentIds, source, dateFrom, dateTo, limit: fetchLimit, ftsConfig,
+            }).catch(() => [])
+          : Promise.resolve([]),
+        runVector
+          ? embeddingSearchSummaries(queryVec, {
+              agentIds: resolvedAgentIds, source, dateFrom, dateTo, limit: fetchLimit,
+            }).catch(() => [])
+          : Promise.resolve([]),
+        runVector
+          ? storage.searchTurnEmbeddings(pool, {
+              schema, tenantId, queryVec, dateFrom, dateTo, agentIds: resolvedAgentIds, source, limit: fetchLimit,
+            }).catch(() => ({ rows: [] }))
+          : Promise.resolve({ rows: [] }),
       ]);
 
       const turnRows = turnResult.rows || [];
@@ -836,6 +868,27 @@ function createAquifer(config) {
       return storage.getSession(pool, sessionId, agentId, opts, { schema, tenantId });
     },
 
+    async skip(sessionId, opts = {}) {
+      const agentId = opts.agentId || 'agent';
+      const reason = opts.reason || null;
+      // Atomic CAS: only skip if still pending (avoids race with concurrent enrich)
+      const result = await pool.query(
+        `UPDATE ${qi(schema)}.sessions
+        SET processing_status = 'skipped', processing_error = $1
+        WHERE session_id = $2 AND agent_id = $3 AND tenant_id = $4
+          AND processing_status = 'pending'
+        RETURNING id`,
+        [reason, sessionId, agentId, tenantId]
+      );
+      if (result.rows.length === 0) {
+        // Check if session exists at all
+        const existing = await storage.getSession(pool, sessionId, agentId, {}, { schema, tenantId });
+        if (!existing) throw new Error(`Session not found: ${sessionId} (agentId=${agentId})`);
+        return null; // exists but not pending — no-op
+      }
+      return { id: result.rows[0].id, sessionId, agentId, status: 'skipped' };
+    },
+
     async getSessionFull(sessionId) {
       // Try to find the session across agents by querying directly
       const result = await pool.query(
@@ -868,6 +921,93 @@ function createAquifer(config) {
         summary: sumResult.rows[0] || null,
       };
     },
+
+    // --- public config accessor ---
+
+    getConfig() {
+      return { schema, tenantId };
+    },
+
+    // --- admin query helpers ---
+
+    async getStats() {
+      const [sessions, summaries, turns, timeRange] = await Promise.all([
+        pool.query(
+          `SELECT processing_status, COUNT(*)::int as count
+          FROM ${qi(schema)}.sessions WHERE tenant_id = $1
+          GROUP BY processing_status`,
+          [tenantId]
+        ),
+        pool.query(
+          `SELECT COUNT(*)::int as count FROM ${qi(schema)}.session_summaries WHERE tenant_id = $1`,
+          [tenantId]
+        ),
+        pool.query(
+          `SELECT COUNT(*)::int as count FROM ${qi(schema)}.turn_embeddings WHERE tenant_id = $1`,
+          [tenantId]
+        ),
+        pool.query(
+          `SELECT MIN(started_at) as earliest, MAX(started_at) as latest
+          FROM ${qi(schema)}.sessions WHERE tenant_id = $1`,
+          [tenantId]
+        ),
+      ]);
+
+      let entityCount = 0;
+      try {
+        const entResult = await pool.query(
+          `SELECT COUNT(*)::int as count FROM ${qi(schema)}.entities WHERE tenant_id = $1`,
+          [tenantId]
+        );
+        entityCount = entResult.rows[0]?.count || 0;
+      } catch (_) { /* entities table may not exist */ }
+
+      return {
+        sessions: Object.fromEntries(sessions.rows.map(r => [r.processing_status, r.count])),
+        sessionTotal: sessions.rows.reduce((s, r) => s + r.count, 0),
+        summaries: summaries.rows[0]?.count || 0,
+        turnEmbeddings: turns.rows[0]?.count || 0,
+        entities: entityCount,
+        earliest: timeRange.rows[0]?.earliest || null,
+        latest: timeRange.rows[0]?.latest || null,
+      };
+    },
+
+    async getPendingSessions(opts = {}) {
+      const limit = opts.limit !== undefined ? opts.limit : 100;
+      const result = await pool.query(
+        `SELECT session_id, agent_id, processing_status
+        FROM ${qi(schema)}.sessions
+        WHERE tenant_id = $1
+          AND processing_status IN ('pending', 'failed')
+        ORDER BY started_at DESC
+        LIMIT $2`,
+        [tenantId, limit]
+      );
+      return result.rows;
+    },
+
+    async exportSessions(opts = {}) {
+      const { agentId, source, limit = 1000 } = opts;
+      const where = [`s.tenant_id = $1`];
+      const params = [tenantId];
+
+      if (agentId) { params.push(agentId); where.push(`s.agent_id = $${params.length}`); }
+      if (source) { params.push(source); where.push(`s.source = $${params.length}`); }
+      params.push(limit);
+
+      const result = await pool.query(
+        `SELECT s.session_id, s.agent_id, s.source, s.started_at, s.msg_count,
+                s.processing_status, ss.summary_text, ss.structured_summary
+        FROM ${qi(schema)}.sessions s
+        LEFT JOIN ${qi(schema)}.session_summaries ss ON ss.session_row_id = s.id
+        WHERE ${where.join(' AND ')}
+        ORDER BY s.started_at DESC
+        LIMIT $${params.length}`,
+        params
+      );
+      return result.rows;
+    },
   };
 
   return aquifer;

package/core/storage.js

@@ -31,7 +31,7 @@ const TURN_NOISE_RE = [
   /^A new session was started via \/new/,
 ];
 
-const VALID_STATUSES = new Set(['pending', 'processing', 'succeeded', 'partial', 'failed']);
+const VALID_STATUSES = new Set(['pending', 'processing', 'succeeded', 'partial', 'failed', 'skipped']);
 
 // ---------------------------------------------------------------------------
 // upsertSession
@@ -339,8 +339,17 @@ async function searchSessions(pool, query, {
   ftsConfig = 'simple',
 } = {}) {
   const clampedLimit = Math.max(1, Math.min(100, limit));
-  // Sanitize ftsConfig to prevent SQL injection (must be a valid regconfig name)
-  const safeFts = /^[a-zA-Z_][a-zA-Z0-9_]*$/.test(ftsConfig) ? ftsConfig : 'simple';
+  // FTS config is locked to 'simple' — the search_tsv trigger always uses
+  // to_tsvector('simple', ...) so query semantics must match.  Warn callers
+  // that pass a different value rather than silently honouring it.
+  if (ftsConfig !== 'simple') {
+    console.warn(
+      `[aquifer/storage] searchSessions: ftsConfig '${ftsConfig}' ignored. ` +
+      `Only 'simple' is supported (index is built with simple tokenizer). ` +
+      `Using 'simple'.`
+    );
+  }
+  const safeFts = 'simple';
 
   // Normalize agentId/agentIds
   const agentIds = rawAgentIds && rawAgentIds.length > 0