@shadowforge0/aquifer-memory 0.5.0 → 0.7.0

package/README.md

@@ -4,7 +4,7 @@
 
 **PG-native long-term memory for AI agents**
 
-*Turn-level embedding, hybrid RRF ranking, trust scoring, entity intersection, knowledge graph — all on PostgreSQL + pgvector.*
+*Turn-level embedding, hybrid RRF ranking, trust scoring, entity intersection, knowledge graph, entity scoping — all on PostgreSQL + pgvector.*
 
 [![npm version](https://img.shields.io/npm/v/@shadowforge0/aquifer-memory)](https://www.npmjs.com/package/@shadowforge0/aquifer-memory)
 [![PostgreSQL 15+](https://img.shields.io/badge/PostgreSQL-15%2B-336791)](https://www.postgresql.org/)
@@ -68,19 +68,19 @@ npm install @shadowforge0/aquifer-memory
 const { createAquifer } = require('@shadowforge0/aquifer-memory');
 
 const aquifer = createAquifer({
+  db: 'postgresql://user:pass@localhost:5432/mydb',  // connection string or pg.Pool
   schema: 'memory',                    // PG schema name (default: 'aquifer')
-  pg: {
-    connectionString: 'postgresql://user:pass@localhost:5432/mydb',
-  },
-  embedder: {
-    baseURL: 'http://localhost:11434/v1',   // Ollama
-    model: 'bge-m3',
-    apiKey: 'ollama',
+  tenantId: 'default',                 // multi-tenant isolation
+  embed: {
+    fn: async (texts) => embeddings,   // your embedding function
+    dim: 1024,                         // optional dimension hint
   },
   llm: {
-    baseURL: 'https://api.openai.com/v1',
-    model: 'gpt-4o-mini',
-    apiKey: process.env.OPENAI_API_KEY,
+    fn: async (prompt) => text,        // your LLM function (for built-in summarize)
+  },
+  entities: {
+    enabled: true,
+    scope: 'my-app',                   // entity namespace (default: 'default')
   },
 });
 
@@ -88,26 +88,36 @@ const aquifer = createAquifer({
 await aquifer.migrate();
 ```
 
-### Ingest a session
+### Write path: commit + enrich
 
 ```javascript
-await aquifer.ingest({
-  sessionId: 'conv-001',
+// 1. Store the session
+await aquifer.commit('conv-001', [
+  { role: 'user', content: 'Let me tell you about our new auth approach...' },
+  { role: 'assistant', content: 'Got it. So the plan is...' },
+], { agentId: 'main' });
+
+// 2. Enrich: summarize + embed turns + extract entities
+const result = await aquifer.enrich('conv-001', {
   agentId: 'main',
-  messages: [
-    { role: 'user', content: 'Let me tell you about our new auth approach...' },
-    { role: 'assistant', content: 'Got it. So the plan is...' },
-  ],
+  // Optional: bring your own summarize pipeline
+  summaryFn: async (msgs) => ({ summaryText, structuredSummary, entityRaw }),
+  entityParseFn: (text) => [{ name, normalizedName, type, aliases }],
+  // Optional: post-commit hook for downstream processing
+  postProcess: async (ctx) => {
+    // ctx contains session, summary, embedding, parsedEntities, etc.
+  },
 });
-// Stores session → generates summary → creates turn embeddings → extracts entities
 ```
 
-### Recall
+### Read path: recall
 
 ```javascript
 const results = await aquifer.recall('auth middleware decision', {
   agentId: 'main',
   limit: 5,
+  entities: ['auth-middleware'],       // optional: entity-aware search
+  entityMode: 'all',                   // 'any' (boost) or 'all' (hard filter)
 });
 // Returns ranked sessions with scores, using 3-way RRF fusion
 ```
@@ -257,109 +267,150 @@ Pass `schema: 'my_app'` to `createAquifer()` and all tables live under that Post
 
 ### `createAquifer(config)`
 
-Returns an Aquifer instance with the following methods:
+Returns an Aquifer instance. Config:
+
+```javascript
+{
+  db,          // pg connection string or Pool instance (required)
+  schema,      // PG schema name (default: 'aquifer')
+  tenantId,    // multi-tenant key (default: 'default')
+  embed: { fn, dim },      // embedding function (required for recall)
+  llm: { fn },             // LLM function (required for built-in summarize)
+  entities: {
+    enabled,               // enable KG (default: false)
+    scope,                 // entity namespace (default: 'default')
+    mergeCall,             // merge entity extraction into summary LLM call (default: true)
+  },
+  rank: { rrf, timeDecay, access, entityBoost },  // weight overrides
+}
+```
 
 #### `aquifer.migrate()`
 
-Runs SQL migrations (idempotent). Creates tables, indexes, and extensions.
+Runs SQL migrations (idempotent). Creates tables, indexes, triggers, and extensions.
 
-#### `aquifer.ingest(options)`
+#### `aquifer.commit(sessionId, messages, opts)`
 
-Ingests a session: stores messages, generates summary, creates turn embeddings, extracts entities.
+Stores a session. Returns `{ id, sessionId, isNew }`.
 
 ```javascript
-await aquifer.ingest({
-  sessionId: 'unique-id',
+await aquifer.commit('session-001', messages, {
   agentId: 'main',
-  source: 'api',                // optional, default 'api'
-  messages: [{ role, content }],
-  tenantId: 'default',          // optional
-  model: 'gpt-4o',             // optional metadata
-  tokensIn: 1500,              // optional
-  tokensOut: 800,              // optional
+  source: 'api',
+  sessionKey: 'optional-key',
+  model: 'gpt-4o',
+  tokensIn: 1500,
+  tokensOut: 800,
+  startedAt: isoString,
+  lastMessageAt: isoString,
 });
 ```
 
-#### `aquifer.recall(query, options)`
+#### `aquifer.enrich(sessionId, opts)`
 
-Hybrid search across sessions.
+Enriches a committed session: summarize, embed turns, extract entities. Uses optimistic locking with stale-reclaim (sessions stuck processing > 10 min are reclaimable).
+
+```javascript
+const result = await aquifer.enrich('session-001', {
+  agentId: 'main',
+  summaryFn,          // custom summarize pipeline (bypasses built-in LLM)
+  entityParseFn,      // custom entity parser
+  postProcess,        // async callback after tx commit
+  model: 'override',  // model metadata override
+  skipSummary: false,
+  skipTurnEmbed: false,
+  skipEntities: false,
+});
+// Returns: { summary, turnsEmbedded, entitiesFound, warnings, effectiveModel, postProcessError }
+```
+
+**postProcess hook**: runs after transaction commit, receives full context (session, summary, embedding, parsedEntities, etc.). Best-effort, at-most-once.
+
+#### `aquifer.recall(query, opts)`
+
+Hybrid search across sessions using 3-way RRF.
 
 ```javascript
 const results = await aquifer.recall('search query', {
   agentId: 'main',
-  limit: 10,                    // max results
-  entities: ['postgres', 'migration'],  // optional: explicit entity names
+  limit: 10,
+  entities: ['postgres', 'migration'],
   entityMode: 'all',            // 'any' (default) or 'all'
-  weights: {                    // optional: override ranking weights
-    rrf: 0.65,
-    timeDecay: 0.25,
-    access: 0.10,
-    entityBoost: 0.18,
-    openLoop: 0.08,
-  },
+  weights: { rrf, timeDecay, access, entityBoost },
 });
 // Returns: [{ sessionId, score, trustScore, summaryText, matchedTurnText, _debug, ... }]
 ```
 
-#### `aquifer.feedback(sessionId, options)`
+#### `aquifer.feedback(sessionId, opts)`
 
-Records explicit trust feedback on a session.
+Records trust feedback. Returns `{ trustBefore, trustAfter, verdict }`.
 
 ```javascript
 await aquifer.feedback('session-id', {
   verdict: 'helpful',   // or 'unhelpful'
-  agentId: 'main',      // optional
-  note: 'reason',       // optional
+  agentId: 'main',
+  note: 'reason',
 });
-// Returns: { trustBefore, trustAfter, verdict }
 ```
 
-#### `aquifer.enrich(sessionId, options)`
-
-Re-processes an existing session: regenerate summary, embeddings, and entities.
-
 #### `aquifer.close()`
 
-Closes the PostgreSQL connection pool.
+Closes the PostgreSQL connection pool (only if Aquifer created it).
 
 ---
 
 ## Configuration
 
+Aquifer takes a `db` connection (string or `pg.Pool`), plus optional `embed` and `llm` functions:
+
 ```javascript
 createAquifer({
-  // PostgreSQL schema name (all tables created under this schema)
-  schema: 'aquifer',
-
-  // PostgreSQL connection
-  pg: {
-    connectionString: 'postgresql://...',
-    // or individual: host, port, database, user, password
-    max: 10,  // pool size
-  },
-
-  // Embedding provider (any OpenAI-compatible API)
-  embedder: {
-    baseURL: 'http://localhost:11434/v1',
-    model: 'bge-m3',
-    apiKey: 'ollama',
-    dimensions: 1024,           // optional
-    timeout: 30000,             // ms, default 30s
+  db: 'postgresql://user:pass@localhost/mydb',  // or an existing pg.Pool
+  schema: 'aquifer',           // PG schema (default: 'aquifer')
+  tenantId: 'default',         // multi-tenant key
+  embed: {
+    fn: myEmbedFn,             // async (texts: string[]) => number[][]
+    dim: 1024,                 // optional dimension hint
   },
-
-  // LLM for summarization & entity extraction
   llm: {
-    baseURL: 'https://api.openai.com/v1',
-    model: 'gpt-4o-mini',
-    apiKey: process.env.OPENAI_API_KEY,
-    timeout: 60000,             // ms, default 60s
+    fn: myLlmFn,               // async (prompt: string) => string
+  },
+  entities: {
+    enabled: true,             // enable KG (default: false)
+    scope: 'my-app',           // entity namespace — decoupled from agentId
+    mergeCall: true,           // merge entity extraction into summary prompt
+  },
+  rank: {
+    rrf: 0.65,                 // FTS + embedding fusion weight
+    timeDecay: 0.25,           // recency weight
+    access: 0.10,              // access frequency weight
+    entityBoost: 0.18,         // entity match boost
   },
-
-  // Tenant isolation
-  tenantId: 'default',
 });
 ```
 
+### Entity Scope
+
+`entities.scope` defines the namespace for entity identity. The unique constraint is `(tenant_id, normalized_name, entity_scope)` — the same entity name in different scopes creates separate entities. This decouples entity identity from `agentId`, allowing multiple agents to share an entity namespace.
+
+Fallback chain: `config.entities.scope` → `'default'`.
+
+### Consumers (CLI, MCP, OpenClaw plugin)
+
+For consumer-based setup using environment variables instead of code:
+
+```bash
+export DATABASE_URL="postgresql://..."
+export AQUIFER_EMBED_BASE_URL="http://localhost:11434/v1"
+export AQUIFER_EMBED_MODEL="bge-m3"
+export AQUIFER_ENTITIES_ENABLED=true
+
+aquifer migrate
+aquifer recall "search query" --limit 5
+aquifer backfill --concurrency 3
+aquifer stats --json
+```
+
 ---
 
 ## Database Schema
@@ -378,12 +429,12 @@ Key indexes: GIN on messages, GiST on `tsvector`, ivfflat on embeddings, B-tree
 
 | Table | Purpose |
 |-------|---------|
-| `entities` | Normalized named entities with type, aliases, frequency, optional embedding |
+| `entities` | Normalized named entities with type, aliases, frequency, entity_scope, optional embedding |
 | `entity_mentions` | Entity × session join with mention count and context |
 | `entity_relations` | Co-occurrence edges (undirected, `CHECK src < dst`) |
 | `entity_sessions` | Entity-session association for boost scoring |
 
-Key indexes: trigram on entity names, GiST on embeddings, composite on tenant/agent.
+Key indexes: trigram on entity names, GiST on embeddings, unique on `(tenant_id, normalized_name, entity_scope)`.
 
 ### 003-trust-feedback.sql
 

package/consumers/cli.js

@@ -344,7 +344,12 @@ Options:
   }
 }
 
-main().catch(err => {
-  console.error(`aquifer: ${err.message}`);
-  process.exit(1);
-});
+// Export for testing; execute only when run directly
+module.exports = { parseArgs };
+
+if (require.main === module) {
+  main().catch(err => {
+    console.error(`aquifer: ${err.message}`);
+    process.exit(1);
+  });
+}

package/consumers/mcp.js

@@ -69,7 +69,7 @@ async function main() {
 
   const server = new McpServer({
     name: 'aquifer-memory',
-    version: '0.3.1',
+    version: '0.6.0',
   });
 
   server.tool(

package/consumers/shared/config.js

@@ -29,8 +29,19 @@ const DEFAULTS = {
     maxRetries: 3,
     temperature: 0,
   },
-  entities: { enabled: false, mergeCall: true },
+  entities: { enabled: false, mergeCall: true, scope: 'default' },
   rank: { rrf: 0.65, timeDecay: 0.25, access: 0.10, entityBoost: 0.18 },
+  rerank: {
+    enabled: false,
+    provider: null,    // 'tei' | 'jina' | 'custom'
+    baseUrl: null,     // TEI base URL
+    apiKey: null,      // Jina API key
+    model: null,       // Jina model override
+    topK: 20,
+    maxChars: 1600,
+    timeoutMs: 2000,
+    maxRetries: 1,
+  },
 };
 
 // ---------------------------------------------------------------------------
@@ -57,6 +68,15 @@ const ENV_MAP = [
   ['AQUIFER_LLM_TIMEOUT_MS',    'llm.timeoutMs',     Number],
   ['AQUIFER_LLM_TEMPERATURE',   'llm.temperature',   Number],
   ['AQUIFER_ENTITIES_ENABLED',  'entities.enabled',  Boolean],
+  ['AQUIFER_ENTITY_SCOPE',     'entities.scope'],
+  ['AQUIFER_RERANK_ENABLED',   'rerank.enabled',    Boolean],
+  ['AQUIFER_RERANK_PROVIDER',  'rerank.provider'],
+  ['AQUIFER_RERANK_BASE_URL',  'rerank.baseUrl'],
+  ['AQUIFER_RERANK_API_KEY',   'rerank.apiKey'],
+  ['AQUIFER_RERANK_MODEL',     'rerank.model'],
+  ['AQUIFER_RERANK_TOP_K',     'rerank.topK',       Number],
+  ['AQUIFER_RERANK_MAX_CHARS', 'rerank.maxChars',   Number],
+  ['AQUIFER_RERANK_TIMEOUT_MS','rerank.timeoutMs',   Number],
 ];
 
 // ---------------------------------------------------------------------------

package/consumers/shared/factory.js

@@ -1,7 +1,7 @@
 'use strict';
 
 const { Pool } = require('pg');
-const { createAquifer, createEmbedder } = require('../../index');
+const { createAquifer, createEmbedder, createReranker } = require('../../index');
 const { loadConfig } = require('./config');
 const { createLlmFn } = require('./llm');
 
@@ -57,6 +57,24 @@ function createAquiferFromConfig(overrides) {
     llmFn = createLlmFn(config.llm);
   }
 
+  // Rerank config (optional)
+  let rerankOpts = null;
+  if (config.rerank && config.rerank.enabled && config.rerank.provider) {
+    const rc = config.rerank;
+    const rerankConfig = { provider: rc.provider, topK: rc.topK, maxChars: rc.maxChars };
+    if (rc.provider === 'tei') {
+      rerankConfig.teiBaseUrl = rc.baseUrl || 'http://localhost:8080';
+      rerankConfig.timeout = rc.timeoutMs || 2000;
+      rerankConfig.maxRetries = rc.maxRetries ?? 1;
+    } else if (rc.provider === 'jina') {
+      rerankConfig.jinaApiKey = rc.apiKey;
+      if (rc.model) rerankConfig.jinaModel = rc.model;
+      rerankConfig.timeout = rc.timeoutMs || 2000;
+      rerankConfig.maxRetries = rc.maxRetries ?? 1;
+    }
+    rerankOpts = rerankConfig;
+  }
+
   const aquifer = createAquifer({
     db: pool,
     schema: config.schema,
@@ -65,6 +83,7 @@ function createAquiferFromConfig(overrides) {
     llm: llmFn ? { fn: llmFn } : null,
     entities: config.entities,
     rank: config.rank,
+    rerank: rerankOpts,
   });
 
   // Attach pool for lifecycle management

package/core/aquifer.js

@@ -36,6 +36,24 @@ function loadSql(filename, schema) {
   return raw.replace(/\$\{schema\}/g, qi(schema));
 }
 
+// ---------------------------------------------------------------------------
+// buildRerankDocument — assemble text for cross-encoder reranking
+// ---------------------------------------------------------------------------
+
+function buildRerankDocument(row, maxChars) {
+  let text = (row.summary_text || row.summary_snippet || '').replace(/\s+/g, ' ').trim();
+  const turn = (row.matched_turn_text || '').replace(/\s+/g, ' ').trim();
+
+  if (!text) {
+    text = turn;
+  } else if (turn && !text.includes(turn)) {
+    text = `${text}\n\nMatched turn:\n${turn}`;
+  }
+
+  if (text.length > maxChars) text = text.slice(0, maxChars);
+  return text;
+}
+
 // ---------------------------------------------------------------------------
 // createAquifer
 // ---------------------------------------------------------------------------
@@ -79,6 +97,10 @@ function createAquifer(config) {
   let entitiesEnabled = config.entities && config.entities.enabled === true;
   const mergeCall = config.entities && config.entities.mergeCall !== undefined ? config.entities.mergeCall : true;
   const entityPromptFn = config.entities && config.entities.prompt ? config.entities.prompt : null;
+  const entityScope = (config.entities && config.entities.scope) || 'default';
+
+  // FTS config (default: 'simple'; set to 'zhcfg' for Chinese tokenization)
+  const ftsConfig = config.ftsConfig || 'simple';
 
   // Rank weights
   const rankWeights = {
@@ -89,6 +111,16 @@ function createAquifer(config) {
     ...(config.rank || {}),
   };
 
+  // Reranker config (optional)
+  const rerankConfig = config.rerank || null;
+  let reranker = null;
+  if (rerankConfig) {
+    const { createReranker } = require('../pipeline/rerank');
+    reranker = createReranker(rerankConfig);
+  }
+  const defaultRerankTopK = rerankConfig ? Math.max(1, rerankConfig.topK || 20) : 0;
+  const rerankMaxChars = rerankConfig ? Math.max(200, rerankConfig.maxChars || 1600) : 0;
+
   // Source registry (in-memory)
   const sources = new Map();
 
@@ -105,7 +137,7 @@ function createAquifer(config) {
 
   // --- Helper: embed search on summaries ---
   async function embeddingSearchSummaries(queryVec, opts) {
-    const { agentId, source, dateFrom, dateTo, limit = 20 } = opts;
+    const { agentIds, source, dateFrom, dateTo, limit = 20 } = opts;
     const where = [`s.tenant_id = $1`];
     const params = [tenantId];
 
@@ -120,9 +152,9 @@ function createAquifer(config) {
       params.push(dateTo);
       where.push(`($${params.length}::date IS NULL OR s.started_at::date <= $${params.length}::date)`);
     }
-    if (agentId) {
-      params.push(agentId);
-      where.push(`s.agent_id = $${params.length}`);
+    if (agentIds && agentIds.length > 0) {
+      params.push(agentIds);
+      where.push(`s.agent_id = ANY($${params.length})`);
     }
     if (source) {
       params.push(source);
@@ -264,11 +296,14 @@ function createAquifer(config) {
       const optModel = 'model' in opts ? opts.model : undefined; // undefined = no override
 
       // 1. Optimistic lock: claim session for processing
+      //    Also reclaim stale 'processing' sessions (stuck > 10 min = likely killed process)
+      const STALE_MINUTES = 10;
       const claimResult = await pool.query(
         `UPDATE ${qi(schema)}.sessions
-        SET processing_status = 'processing'
+        SET processing_status = 'processing', processing_started_at = NOW()
         WHERE session_id = $1 AND agent_id = $2 AND tenant_id = $3
-          AND processing_status IN ('pending', 'failed')
+          AND (processing_status IN ('pending', 'failed')
+               OR (processing_status = 'processing' AND (processing_started_at IS NULL OR processing_started_at < NOW() - INTERVAL '${STALE_MINUTES} minutes')))
         RETURNING *`,
         [sessionId, agentId, tenantId]
       );
@@ -398,6 +433,7 @@ function createAquifer(config) {
                 aliases: ent.aliases,
                 type: ent.type,
                 agentId,
+                entityScope,
                 createdBy: 'aquifer',
                 occurredAt: session.started_at ? new Date(session.started_at).toISOString() : null,
               });
@@ -519,6 +555,7 @@ function createAquifer(config) {
 
       const {
         agentId,
+        agentIds: rawAgentIds,
         source,
         dateFrom,
         dateTo,
@@ -528,6 +565,12 @@ function createAquifer(config) {
         entityMode = 'any',
       } = opts;
 
+      // Normalize agentId/agentIds into a single resolved value
+      // agentIds takes precedence; agentId is sugar for agentIds: [agentId]
+      const resolvedAgentIds = rawAgentIds && rawAgentIds.length > 0
+        ? rawAgentIds
+        : (agentId ? [agentId] : null);
+
       // Validate before touching DB
       if (explicitEntities && explicitEntities.length > 0 && !entitiesEnabled) {
         throw new Error('Entities are not enabled');
@@ -535,7 +578,9 @@ function createAquifer(config) {
 
       await ensureMigrated();
 
-      const fetchLimit = limit * 4;
+      const rerankEnabled = !!reranker && opts.rerank !== false;
+      const rerankTopK = rerankEnabled ? Math.max(limit, opts.rerankTopK || defaultRerankTopK) : limit;
+      const fetchLimit = rerankTopK * 4;
 
       // 1. Embed query
       const queryVecResult = await embedFn([query]);
@@ -549,7 +594,7 @@ function createAquifer(config) {
       if (explicitEntities && explicitEntities.length > 0) {
 
         const resolved = await entity.resolveEntities(pool, {
-          schema, tenantId, names: explicitEntities, agentId,
+          schema, tenantId, names: explicitEntities, entityScope,
         });
 
         if (resolved.length === 0) return [];
@@ -594,7 +639,7 @@ function createAquifer(config) {
         // No explicit entities: existing query-text-based entity boost
         try {
           const matchedEntities = await entity.searchEntities(pool, {
-            schema, tenantId, query, agentId, limit: 10,
+            schema, tenantId, query, entityScope, limit: 10,
           });
 
           if (matchedEntities.length > 0) {
@@ -619,13 +664,13 @@ function createAquifer(config) {
       // 3. Run 3 search paths in parallel
       const [ftsRows, embRows, turnResult] = await Promise.all([
         storage.searchSessions(pool, query, {
-          schema, tenantId, agentId, source, dateFrom, dateTo, limit: fetchLimit,
+          schema, tenantId, agentIds: resolvedAgentIds, source, dateFrom, dateTo, limit: fetchLimit, ftsConfig,
         }).catch(() => []),
         embeddingSearchSummaries(queryVec, {
-          agentId, source, dateFrom, dateTo, limit: fetchLimit,
+          agentIds: resolvedAgentIds, source, dateFrom, dateTo, limit: fetchLimit,
         }).catch(() => []),
         storage.searchTurnEmbeddings(pool, {
-          schema, tenantId, queryVec, dateFrom, dateTo, agentId, source, limit: fetchLimit,
+          schema, tenantId, queryVec, dateFrom, dateTo, agentIds: resolvedAgentIds, source, limit: fetchLimit,
         }).catch(() => ({ rows: [] })),
       ]);
 
@@ -686,15 +731,45 @@ function createAquifer(config) {
         [...filteredEmb, ...filterFn(externalRows)],
         filteredTurn,
         {
-          limit,
+          limit: rerankTopK,
           weights: mergedWeights,
           entityScoreBySession,
           openLoopSet,
         },
       );
 
+      // 6b. Rerank (optional)
+      let finalRanked = ranked;
+      if (rerankEnabled && ranked.length > 1) {
+        try {
+          const docs = ranked.map(r => buildRerankDocument(r, rerankMaxChars));
+          const rerankResult = await reranker.rerank(query, docs, { topN: ranked.length });
+          const scoreMap = new Map(rerankResult.map(r => [r.index, r.score]));
+
+          finalRanked = ranked.map((r, i) => ({
+            ...r,
+            _hybridScore: r._score,
+            _rerankScore: scoreMap.has(i) ? scoreMap.get(i) : null,
+          }));
+
+          finalRanked.sort((a, b) => {
+            const aR = a._rerankScore ?? -Infinity;
+            const bR = b._rerankScore ?? -Infinity;
+            if (aR !== bR) return bR - aR;
+            return (b._hybridScore || 0) - (a._hybridScore || 0);
+          });
+          finalRanked = finalRanked.slice(0, limit);
+        } catch (rerankErr) {
+          // Fallback: use original hybrid-rank order, flag in debug
+          if (process.env.AQUIFER_DEBUG) console.error('[aquifer] rerank error:', rerankErr.message);
+          finalRanked = ranked.slice(0, limit).map(r => ({ ...r, _rerankFallback: true }));
+        }
+      } else {
+        finalRanked = ranked.slice(0, limit);
+      }
+
       // 7. Record access
-      const sessionRowIds = ranked
+      const sessionRowIds = finalRanked
         .map(r => r.id || r.session_row_id)
         .filter(Boolean);
 
@@ -705,7 +780,7 @@ function createAquifer(config) {
       }
 
       // 8. Format results
-      return ranked.map(r => ({
+      return finalRanked.map(r => ({
         sessionId: r.session_id,
         agentId: r.agent_id,
         source: r.source,
@@ -715,7 +790,7 @@ function createAquifer(config) {
         summarySnippet: r.summary_snippet || null,
         matchedTurnText: r.matched_turn_text || null,
         matchedTurnIndex: r.matched_turn_index || null,
-        score: r._score,
+        score: r._rerankScore ?? r._score,
         trustScore: r._trustScore ?? 0.5,
         _debug: {
           rrf: r._rrf,
@@ -725,6 +800,9 @@ function createAquifer(config) {
           trustScore: r._trustScore,
           trustMultiplier: r._trustMultiplier,
           openLoopBoost: r._openLoopBoost,
+          hybridScore: r._hybridScore ?? r._score,
+          rerankScore: r._rerankScore ?? null,
+          rerankFallback: r._rerankFallback || false,
         },
       }));
     },

package/core/entity.js

@@ -141,21 +141,23 @@ async function upsertEntity(pool, {
   type = 'other',
   status = 'active',
   agentId = 'main',
+  entityScope,
   createdBy,
   metadata = {},
   embedding,
   occurredAt,
 }) {
+  const scope = entityScope || agentId || 'default';
   const normalizedAliases = aliases.map(a => normalizeEntityName(a)).filter(Boolean);
   const embStr = embedding ? vecToStr(embedding) : null;
   const ts = occurredAt || new Date().toISOString();
 
   const result = await pool.query(
     `INSERT INTO ${qi(schema)}.entities
-      (tenant_id, name, normalized_name, aliases, type, status, agent_id,
+      (tenant_id, name, normalized_name, aliases, type, status, agent_id, entity_scope,
        created_by, metadata, embedding, first_seen_at, last_seen_at, frequency)
-    VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb, $10::vector, $11, $11, 1)
-    ON CONFLICT (tenant_id, normalized_name, agent_id) DO UPDATE SET
+    VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10::jsonb, $11::vector, $12, $12, 1)
+    ON CONFLICT (tenant_id, normalized_name, entity_scope) DO UPDATE SET
       frequency    = ${qi(schema)}.entities.frequency + 1,
       aliases      = ARRAY(SELECT DISTINCT unnest(${qi(schema)}.entities.aliases || EXCLUDED.aliases)),
       last_seen_at = GREATEST(${qi(schema)}.entities.last_seen_at, EXCLUDED.last_seen_at),
@@ -164,7 +166,7 @@ async function upsertEntity(pool, {
     RETURNING id, (xmax = 0) AS is_new`,
     [
       tenantId, name, normalizedName, normalizedAliases,
-      type, status, agentId,
+      type, status, agentId, scope,
       createdBy || null,
       JSON.stringify(metadata),
       embStr,
@@ -276,6 +278,7 @@ async function searchEntities(pool, {
   tenantId,
   query,
   agentId,
+  entityScope,
   limit = 10,
   similarityThreshold = 0.1,
 }) {
@@ -284,11 +287,13 @@ async function searchEntities(pool, {
   if (!normQ) return [];
 
   const escaped = _escapeIlike(normQ);
+  // Use entityScope if provided, fall back to agentId for backward compat
+  const scopeFilter = entityScope || agentId || null;
 
   const result = await pool.query(
     `SELECT
       id, name, normalized_name, aliases, type, status, frequency, agent_id,
-      last_seen_at, metadata,
+      entity_scope, last_seen_at, metadata,
       similarity(normalized_name, $1) AS name_sim
     FROM ${qi(schema)}.entities
     WHERE status = 'active'
@@ -298,10 +303,10 @@ async function searchEntities(pool, {
         OR normalized_name ILIKE '%' || $4 || '%' ESCAPE '\\'
         OR $5 = ANY(aliases)
       )
-      AND ($6::text IS NULL OR agent_id = $6)
+      AND ($6::text IS NULL OR entity_scope = $6)
     ORDER BY name_sim DESC, frequency DESC
     LIMIT $7`,
-    [normQ, tenantId, similarityThreshold, escaped, normQ, agentId || null, clampedLimit]
+    [normQ, tenantId, similarityThreshold, escaped, normQ, scopeFilter, clampedLimit]
   );
 
   return result.rows;
@@ -353,9 +358,12 @@ async function resolveEntities(pool, {
   tenantId,
   names,
   agentId = null,
+  entityScope,
   threshold = 0.1,
 }) {
   if (!names || names.length === 0) return [];
+  // Use entityScope if provided, fall back to agentId for backward compat
+  const scopeFilter = entityScope || agentId || null;
 
   const seen = new Map();
   const results = [];
@@ -376,10 +384,10 @@ async function resolveEntities(pool, {
           OR normalized_name = $2
           OR $2 = ANY(aliases)
         )
-        AND ($4::text IS NULL OR agent_id = $4)
+        AND ($4::text IS NULL OR entity_scope = $4)
       ORDER BY similarity(normalized_name, $2) DESC, frequency DESC
       LIMIT 1`,
-      [tenantId, normQ, threshold, agentId || null]
+      [tenantId, normQ, threshold, scopeFilter]
     );
 
     if (result.rows[0]) {

package/core/storage.js

@@ -331,12 +331,46 @@ async function searchSessions(pool, query, {
   schema,
   tenantId,
   agentId,
+  agentIds: rawAgentIds,
   source,
   dateFrom,  // m1: add date filtering
   dateTo,
   limit = 20,
+  ftsConfig = 'simple',
 } = {}) {
   const clampedLimit = Math.max(1, Math.min(100, limit));
+  // Sanitize ftsConfig to prevent SQL injection (must be a valid regconfig name)
+  const safeFts = /^[a-zA-Z_][a-zA-Z0-9_]*$/.test(ftsConfig) ? ftsConfig : 'simple';
+
+  // Normalize agentId/agentIds
+  const agentIds = rawAgentIds && rawAgentIds.length > 0
+    ? rawAgentIds
+    : (agentId ? [agentId] : null);
+
+  const where = [
+    `ss.search_tsv @@ plainto_tsquery('${safeFts}', $1)`,
+    `s.tenant_id = $2`,
+  ];
+  const params = [query, tenantId];
+
+  if (agentIds) {
+    params.push(agentIds);
+    where.push(`s.agent_id = ANY($${params.length})`);
+  }
+  if (source) {
+    params.push(source);
+    where.push(`s.source = $${params.length}`);
+  }
+  if (dateFrom) {
+    params.push(dateFrom);
+    where.push(`s.started_at::date >= $${params.length}::date`);
+  }
+  if (dateTo) {
+    params.push(dateTo);
+    where.push(`s.started_at::date <= $${params.length}::date`);
+  }
+  params.push(clampedLimit);
+
   const result = await pool.query(
     `SELECT
       s.id,
@@ -351,19 +385,14 @@ async function searchSessions(pool, query, {
       ss.access_count,
       ss.last_accessed_at,
       ss.trust_score,
-      ts_headline('simple', COALESCE(ss.summary_text, ''), plainto_tsquery('simple', $1)) AS summary_snippet,
-      ts_rank(ss.search_tsv, plainto_tsquery('simple', $1)) AS fts_rank
+      ts_headline('${safeFts}', COALESCE(ss.summary_text, ''), plainto_tsquery('${safeFts}', $1)) AS summary_snippet,
+      ts_rank(ss.search_tsv, plainto_tsquery('${safeFts}', $1)) AS fts_rank
     FROM ${qi(schema)}.sessions s
     LEFT JOIN ${qi(schema)}.session_summaries ss ON ss.session_row_id = s.id
-    WHERE ss.search_tsv @@ plainto_tsquery('simple', $1)
-      AND s.tenant_id = $2
-      AND ($3::text IS NULL OR s.agent_id = $3)
-      AND ($4::text IS NULL OR s.source = $4)
-      AND ($5::date IS NULL OR s.started_at::date >= $5::date)
-      AND ($6::date IS NULL OR s.started_at::date <= $6::date)
+    WHERE ${where.join(' AND ')}
     ORDER BY fts_rank DESC, s.last_message_at DESC NULLS LAST
-    LIMIT $7`,
-    [query, tenantId, agentId || null, source || null, dateFrom || null, dateTo || null, clampedLimit]
+    LIMIT $${params.length}`,
+    params
   );
   return result.rows;
 }
@@ -479,23 +508,29 @@ async function searchTurnEmbeddings(pool, {
   dateFrom,
   dateTo,
   agentId,
+  agentIds: rawAgentIds,
   source,
   limit = 15,
 }) {
   const where = ['s.tenant_id = $1'];
   const params = [tenantId];
 
+  // Normalize agentId/agentIds
+  const agentIds = rawAgentIds && rawAgentIds.length > 0
+    ? rawAgentIds
+    : (agentId ? [agentId] : null);
+
   if (dateFrom) {
     params.push(dateFrom);
-    where.push(`($${params.length}::date IS NULL OR s.started_at::date >= $${params.length}::date)`);
+    where.push(`s.started_at::date >= $${params.length}::date`);
   }
   if (dateTo) {
     params.push(dateTo);
-    where.push(`($${params.length}::date IS NULL OR s.started_at::date <= $${params.length}::date)`);
+    where.push(`s.started_at::date <= $${params.length}::date`);
   }
-  if (agentId) {
-    params.push(agentId);
-    where.push(`t.agent_id = $${params.length}`);
+  if (agentIds) {
+    params.push(agentIds);
+    where.push(`t.agent_id = ANY($${params.length})`);
   }
   if (source) {
     params.push(source);

package/index.js

@@ -2,5 +2,6 @@
 
 const { createAquifer } = require('./core/aquifer');
 const { createEmbedder } = require('./pipeline/embed');
+const { createReranker } = require('./pipeline/rerank');
 
-module.exports = { createAquifer, createEmbedder };
+module.exports = { createAquifer, createEmbedder, createReranker };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@shadowforge0/aquifer-memory",
-  "version": "0.5.0",
+  "version": "0.7.0",
   "description": "PG-native long-term memory for AI agents. Turn-level embedding, hybrid RRF ranking, optional knowledge graph. Includes CLI, MCP server, and OpenClaw plugin.",
   "main": "index.js",
   "files": [

package/pipeline/_http.js

@@ -0,0 +1,67 @@
+'use strict';
+
+const http = require('http');
+const https = require('https');
+
+// ---------------------------------------------------------------------------
+// HTTP helpers (shared by embed.js and rerank.js)
+// ---------------------------------------------------------------------------
+
+function httpRequest(url, options, body) {
+  return new Promise((resolve, reject) => {
+    const parsedUrl = new URL(url);
+    const transport = parsedUrl.protocol === 'https:' ? https : http;
+
+    // M8 fix: settled flag to prevent double-settle on timeout race
+    let settled = false;
+    const finish = (fn, val) => { if (!settled) { settled = true; fn(val); } };
+
+    const req = transport.request(parsedUrl, options, (res) => {
+      const chunks = [];
+      res.on('data', (chunk) => chunks.push(chunk));
+      res.on('end', () => {
+        if (timer) clearTimeout(timer);
+        const raw = Buffer.concat(chunks).toString();
+        if (res.statusCode < 200 || res.statusCode >= 300) {
+          finish(reject, new Error(`HTTP ${res.statusCode}: ${raw.slice(0, 500)}`));
+          return;
+        }
+        try {
+          finish(resolve, JSON.parse(raw));
+        } catch (e) {
+          finish(reject, new Error(`Invalid JSON response: ${raw.slice(0, 200)}`));
+        }
+      });
+    });
+
+    const timer = options.timeout
+      ? setTimeout(() => { req.destroy(); finish(reject, new Error('Request timeout')); }, options.timeout)
+      : null;
+
+    req.on('error', (err) => { if (timer) clearTimeout(timer); finish(reject, err); });
+    if (body) req.write(JSON.stringify(body));
+    req.end();
+  });
+}
+
+// ---------------------------------------------------------------------------
+// Retry wrapper
+// ---------------------------------------------------------------------------
+
+async function withRetry(fn, { maxRetries = 3, initialBackoffMs = 2000 } = {}) {
+  let lastErr;
+  for (let attempt = 0; attempt < maxRetries; attempt++) {
+    try {
+      return await fn();
+    } catch (err) {
+      lastErr = err;
+      if (attempt < maxRetries - 1) {
+        const delay = initialBackoffMs * Math.pow(2, attempt);
+        await new Promise(r => setTimeout(r, delay));
+      }
+    }
+  }
+  throw lastErr;
+}
+
+module.exports = { httpRequest, withRetry };

package/pipeline/embed.js

@@ -1,68 +1,6 @@
 'use strict';
 
-const http = require('http');
-const https = require('https');
-
-// ---------------------------------------------------------------------------
-// HTTP helpers
-// ---------------------------------------------------------------------------
-
-function httpRequest(url, options, body) {
-  return new Promise((resolve, reject) => {
-    const parsedUrl = new URL(url);
-    const transport = parsedUrl.protocol === 'https:' ? https : http;
-
-    // M8 fix: settled flag to prevent double-settle on timeout race
-    let settled = false;
-    const finish = (fn, val) => { if (!settled) { settled = true; fn(val); } };
-
-    const req = transport.request(parsedUrl, options, (res) => {
-      const chunks = [];
-      res.on('data', (chunk) => chunks.push(chunk));
-      res.on('end', () => {
-        if (timer) clearTimeout(timer);
-        const raw = Buffer.concat(chunks).toString();
-        if (res.statusCode < 200 || res.statusCode >= 300) {
-          finish(reject, new Error(`HTTP ${res.statusCode}: ${raw.slice(0, 500)}`));
-          return;
-        }
-        try {
-          finish(resolve, JSON.parse(raw));
-        } catch (e) {
-          finish(reject, new Error(`Invalid JSON response: ${raw.slice(0, 200)}`));
-        }
-      });
-    });
-
-    const timer = options.timeout
-      ? setTimeout(() => { req.destroy(); finish(reject, new Error('Request timeout')); }, options.timeout)
-      : null;
-
-    req.on('error', (err) => { if (timer) clearTimeout(timer); finish(reject, err); });
-    if (body) req.write(JSON.stringify(body));
-    req.end();
-  });
-}
-
-// ---------------------------------------------------------------------------
-// Retry wrapper
-// ---------------------------------------------------------------------------
-
-async function withRetry(fn, { maxRetries = 3, initialBackoffMs = 2000 } = {}) {
-  let lastErr;
-  for (let attempt = 0; attempt < maxRetries; attempt++) {
-    try {
-      return await fn();
-    } catch (err) {
-      lastErr = err;
-      if (attempt < maxRetries - 1) {
-        const delay = initialBackoffMs * Math.pow(2, attempt);
-        await new Promise(r => setTimeout(r, delay));
-      }
-    }
-  }
-  throw lastErr;
-}
+const { httpRequest, withRetry } = require('./_http');
 
 // ---------------------------------------------------------------------------
 // Ollama adapter

package/pipeline/rerank.js

@@ -0,0 +1,161 @@
+'use strict';
+
+const { httpRequest, withRetry } = require('./_http');
+
+// ---------------------------------------------------------------------------
+// Custom adapter
+// ---------------------------------------------------------------------------
+
+function validateResults(results) {
+  return results.filter(r =>
+    r && typeof r.index === 'number' && Number.isFinite(r.index)
+      && typeof r.score === 'number' && Number.isFinite(r.score)
+  );
+}
+
+function createCustomReranker(config) {
+  const fn = config.fn;
+  if (!fn) throw new Error('fn is required for custom reranker');
+
+  return {
+    async rerank(query, documents, opts = {}) {
+      if (!query || !documents || documents.length === 0) return [];
+      const topN = opts.topN || documents.length;
+      const results = await fn({ query, documents, topN });
+      if (!Array.isArray(results)) throw new Error('Custom reranker fn must return an array');
+      return validateResults(results).sort((a, b) => b.score - a.score);
+    },
+  };
+}
+
+// ---------------------------------------------------------------------------
+// TEI adapter (HuggingFace Text Embeddings Inference)
+// ---------------------------------------------------------------------------
+
+function createTEIReranker(config) {
+  const baseUrl = (config.teiBaseUrl || config.baseUrl || 'http://localhost:8080').replace(/\/+$/, '');
+  const timeout = config.timeout || 2000;
+  const maxRetries = config.maxRetries ?? 1;
+  const initialBackoffMs = config.initialBackoffMs || 250;
+
+  return {
+    async rerank(query, documents, opts = {}) {
+      if (!query || !documents || documents.length === 0) return [];
+
+      const result = await withRetry(
+        () => httpRequest(`${baseUrl}/rerank`, {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json' },
+          timeout,
+        }, { query, texts: documents, raw_scores: false }),
+        { maxRetries, initialBackoffMs },
+      );
+
+      // TEI returns array of { index, score }
+      const arr = Array.isArray(result) ? result : [];
+      return validateResults(arr.map(r => ({ index: r.index, score: r.score })))
+        .sort((a, b) => b.score - a.score);
+    },
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Jina adapter
+// ---------------------------------------------------------------------------
+
+function createJinaReranker(config) {
+  const apiKey = config.jinaApiKey;
+  if (!apiKey) throw new Error('jinaApiKey is required for Jina reranker');
+
+  const model = config.jinaModel || 'jina-reranker-v2-base-multilingual';
+  const baseUrl = (config.jinaBaseUrl || 'https://api.jina.ai/v1/rerank').replace(/\/+$/, '');
+  const timeout = config.timeout || 2000;
+  const maxRetries = config.maxRetries ?? 1;
+  const initialBackoffMs = config.initialBackoffMs || 250;
+
+  return {
+    async rerank(query, documents, opts = {}) {
+      if (!query || !documents || documents.length === 0) return [];
+      const topN = opts.topN || documents.length;
+
+      const result = await withRetry(
+        () => httpRequest(baseUrl, {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+            'Authorization': `Bearer ${apiKey}`,
+          },
+          timeout,
+        }, { model, query, documents, top_n: topN }),
+        { maxRetries, initialBackoffMs },
+      );
+
+      // Jina returns { results: [{ index, relevance_score }] }
+      const arr = result.results || [];
+      return validateResults(arr.map(r => ({ index: r.index, score: r.relevance_score })))
+        .sort((a, b) => b.score - a.score);
+    },
+  };
+}
+
+// ---------------------------------------------------------------------------
+// OpenRouter adapter (Cohere rerank etc. via OpenRouter)
+// ---------------------------------------------------------------------------
+
+function createOpenRouterReranker(config) {
+  const apiKey = config.openrouterApiKey || config.apiKey;
+  if (!apiKey) throw new Error('openrouterApiKey is required for OpenRouter reranker');
+
+  const model = config.model || 'cohere/rerank-v3.5';
+  const baseUrl = (config.openrouterBaseUrl || 'https://openrouter.ai/api/v1/rerank').replace(/\/+$/, '');
+  const timeout = config.timeout || 5000;
+  const maxRetries = config.maxRetries ?? 1;
+  const initialBackoffMs = config.initialBackoffMs || 250;
+
+  return {
+    async rerank(query, documents, opts = {}) {
+      if (!query || !documents || documents.length === 0) return [];
+      const topN = opts.topN || documents.length;
+
+      const result = await withRetry(
+        () => httpRequest(baseUrl, {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+            'Authorization': `Bearer ${apiKey}`,
+          },
+          timeout,
+        }, { model, query, documents, top_n: topN }),
+        { maxRetries, initialBackoffMs },
+      );
+
+      // OpenRouter returns { results: [{ index, relevance_score }] }
+      const arr = result.results || [];
+      return validateResults(arr.map(r => ({ index: r.index, score: r.relevance_score })))
+        .sort((a, b) => b.score - a.score);
+    },
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Factory
+// ---------------------------------------------------------------------------
+
+function createReranker(config = {}) {
+  const provider = config.provider || 'custom';
+
+  switch (provider) {
+    case 'custom':
+      return createCustomReranker(config);
+    case 'tei':
+      return createTEIReranker(config);
+    case 'jina':
+      return createJinaReranker(config);
+    case 'openrouter':
+      return createOpenRouterReranker(config);
+    default:
+      throw new Error(`Unknown rerank provider: ${provider}`);
+  }
+}
+
+module.exports = { createReranker };

package/schema/001-base.sql

@@ -24,10 +24,11 @@ CREATE TABLE IF NOT EXISTS ${schema}.sessions (
   started_at         TIMESTAMPTZ,
   ended_at           TIMESTAMPTZ,
   last_message_at    TIMESTAMPTZ,
-  processing_status  TEXT         NOT NULL DEFAULT 'pending',
-  processed_at       TIMESTAMPTZ,
-  processing_error   TEXT,
-  created_at         TIMESTAMPTZ  NOT NULL DEFAULT now(),
+  processing_status    TEXT         NOT NULL DEFAULT 'pending',
+  processing_started_at TIMESTAMPTZ,
+  processed_at         TIMESTAMPTZ,
+  processing_error     TEXT,
+  created_at           TIMESTAMPTZ  NOT NULL DEFAULT now(),
   UNIQUE (tenant_id, agent_id, session_id)
 );
 

package/schema/002-entities.sql

@@ -20,16 +20,28 @@ CREATE TABLE IF NOT EXISTS ${schema}.entities (
                     CHECK (status IN ('active','merged','deleted')),
   frequency       INT          NOT NULL DEFAULT 1,
   agent_id        TEXT         NOT NULL DEFAULT 'main',
+  entity_scope    TEXT         NOT NULL DEFAULT 'default',
   created_by      TEXT,
   metadata        JSONB        NOT NULL DEFAULT '{}',
   embedding       vector,
   first_seen_at   TIMESTAMPTZ  NOT NULL DEFAULT now(),
-  last_seen_at    TIMESTAMPTZ  NOT NULL DEFAULT now(),
-  UNIQUE (tenant_id, normalized_name, agent_id)
+  last_seen_at    TIMESTAMPTZ  NOT NULL DEFAULT now()
 );
 
-CREATE INDEX IF NOT EXISTS idx_entities_tenant_agent
-  ON ${schema}.entities (tenant_id, agent_id);
+-- Migration: add entity_scope if missing (idempotent)
+-- For upgrades: backfill from agent_id so existing data keeps its scope
+ALTER TABLE ${schema}.entities ADD COLUMN IF NOT EXISTS entity_scope TEXT DEFAULT 'default';
+UPDATE ${schema}.entities SET entity_scope = agent_id WHERE entity_scope IS NULL OR entity_scope = 'default';
+ALTER TABLE ${schema}.entities ALTER COLUMN entity_scope SET NOT NULL;
+
+-- Unique constraint: entity identity is (tenant, name, scope)
+-- Drop legacy agent-based constraint if it exists
+DROP INDEX IF EXISTS ${schema}.idx_entities_tenant_name_agent;
+CREATE UNIQUE INDEX IF NOT EXISTS idx_entities_tenant_name_scope
+  ON ${schema}.entities (tenant_id, normalized_name, entity_scope);
+
+CREATE INDEX IF NOT EXISTS idx_entities_tenant_scope
+  ON ${schema}.entities (tenant_id, entity_scope);
 
 CREATE INDEX IF NOT EXISTS idx_entities_type
   ON ${schema}.entities (type);
@@ -44,7 +56,7 @@ CREATE INDEX IF NOT EXISTS idx_entities_aliases
   ON ${schema}.entities USING GIN (aliases);
 
 CREATE INDEX IF NOT EXISTS idx_entities_active
-  ON ${schema}.entities (tenant_id, agent_id, frequency DESC)
+  ON ${schema}.entities (tenant_id, entity_scope, frequency DESC)
   WHERE status = 'active';
 
 -- =========================================================================