@shadowforge0/aquifer-memory 1.5.9 → 1.5.12

package/README.md

@@ -134,14 +134,32 @@ Need LLM summarization, the knowledge graph, OpenAI embeddings, or the reranker?
 | `AQUIFER_AGENT_ID` | No | Default agent ID | `main` |
 | `AQUIFER_MIGRATIONS_MODE` | No | Startup handshake mode: `apply` (default), `check`, `off` | `apply` |
 | `AQUIFER_MIGRATION_LOCK_TIMEOUT_MS` | No | Advisory-lock wait before `AQ_MIGRATION_LOCK_TIMEOUT` (default 30000) | `30000` |
+| `AQUIFER_INSIGHTS_DEDUP_MODE` | No | Insights semantic dedup mode: `off` (default), `shadow`, `enforce` — env wins over code for this field only, so operators can kill-switch without redeploy | `shadow` |
+| `AQUIFER_INSIGHTS_DEDUP_COSINE` | No | Cosine threshold for semantic merge (default `0.88`; warn outside `[0.75, 0.95]`) | `0.90` |
+| `AQUIFER_INSIGHTS_DEDUP_CLOSE_BAND_FROM` | No | Lower bound for close-band logging (`dedupNear`); must be below threshold (default `0.85`) | `0.82` |
 
 Full env-to-config mapping is in [consumers/shared/config.js](consumers/shared/config.js).
 
+### Insights semantic dedup (1.5.10)
+
+When a cron extractor (`scripts/extract-insights-from-recent-sessions.js`) or any other caller writes insights via `commitInsight`, the canonical-key layer (1.5.3+) dedupes rows whose `canonicalClaim + entities` hash to the same value. But LLMs don't always produce the same `canonicalClaim` across runs, so 1.5.10 adds a second tier: `title + body` are embedded, matched against `(tenant, agent, type)`-scoped active rows, and a top cosine above `AQUIFER_INSIGHTS_DEDUP_COSINE` triggers supersede (enforce) or metadata-only would-merge logging (shadow). Close-band hits (`closeBandFrom ≤ cos < threshold`) write `metadata.dedupNear` without supersede so operators can tune thresholds without committing.
+
+Recommended rollout: `shadow` for one weekly cycle, inspect `SELECT metadata->>'shadowMatch' FROM insights WHERE metadata ? 'shadowMatch'`, then flip to `enforce`. Kill-switch: `AQUIFER_INSIGHTS_DEDUP_MODE=off` and restart.
+
+Pre-1.5.3 rows with `canonical_key_v2 IS NULL` are caught by the semantic tier but skip the canonical path; a startup warn points at the one-shot backfill:
+
+```bash
+DATABASE_URL=... \
+  node scripts/backfill-canonical-key.js --schema <schema> --agent <id>
+```
+
+The script is idempotent (`WHERE canonical_key_v2 IS NULL` guard) and race-safe with live writers.
+
 ---
 
 ## Host Integration
 
-MCP is the primary integration surface. Agent hosts connect to the Aquifer MCP server, which exposes five tools: `session_recall`, `session_feedback`, `session_bootstrap`, `memory_stats`, `memory_pending`.
+MCP is the primary integration surface. Agent hosts connect to the Aquifer MCP server, which exposes six tools: `session_recall`, `session_feedback`, `feedback_stats`, `session_bootstrap`, `memory_stats`, `memory_pending`.
 
 | Integration | Route | Status | When to use |
 |-------------|-------|--------|-------------|
@@ -196,7 +214,7 @@ Add to `openclaw.json` under `mcp.servers`:
 }
 ```
 
-Tools materialize as `aquifer__session_recall`, `aquifer__session_feedback`, `aquifer__session_bootstrap`, `aquifer__memory_stats`, `aquifer__memory_pending` (server name prefix added by the host).
+Tools materialize as `aquifer__session_recall`, `aquifer__session_feedback`, `aquifer__feedback_stats`, `aquifer__session_bootstrap`, `aquifer__memory_stats`, `aquifer__memory_pending` (server name prefix added by the host).
 
 The OpenClaw plugin (`consumers/openclaw-plugin.js`) is retained for session capture via `before_reset` but is **not** the recommended tool delivery path. Use MCP.
 

package/consumers/cli.js

@@ -99,6 +99,7 @@ async function cmdRecall(aquifer, args) {
     return;
   }
 
+  const showExplain = !!args.flags.explain;
   for (let i = 0; i < results.length; i++) {
     const r = results[i];
     const ss = r.structuredSummary || {};
@@ -107,6 +108,18 @@ async function cmdRecall(aquifer, args) {
     console.log(`${i + 1}. [${r.score?.toFixed(3)}] ${title} (${date}, ${r.agentId})`);
     if (ss.overview) console.log(`   ${ss.overview.slice(0, 200)}`);
     if (r.matchedTurnText) console.log(`   > ${r.matchedTurnText.slice(0, 150)}`);
+    if (showExplain && r._debug) {
+      const d = r._debug;
+      const f = (v) => typeof v === 'number' ? v.toFixed(3) : '?';
+      const parts = [
+        `rrf=${f(d.rrf)}`, `td=${f(d.timeDecay)}`, `access=${f(d.access)}`,
+        `entity=${f(d.entityScore)}`, `trust=${f(d.trustScore)}(\u00d7${f(d.trustMultiplier)})`,
+        `ol=${f(d.openLoopBoost)}`, `\u2192 hybrid=${f(d.hybridScore)}`,
+      ];
+      if (d.rerankApplied) parts.push(`rerank=${f(d.rerankScore)}(${d.rerankReason || '?'})`);
+      else parts.push(`[rerank: off (${d.rerankReason || '?'})]`);
+      console.log(`   ${parts.join(' ')}`);
+    }
     console.log();
   }
 }
@@ -133,6 +146,22 @@ async function cmdFeedback(aquifer, args) {
   }
 }
 
+async function cmdFeedbackStats(aquifer, args) {
+  const stats = await aquifer.feedbackStats({
+    agentId: args.flags['agent-id'] || undefined,
+    dateFrom: args.flags['date-from'] || undefined,
+    dateTo: args.flags['date-to'] || undefined,
+  });
+
+  if (args.flags.json) {
+    console.log(JSON.stringify(stats, null, 2));
+  } else {
+    console.log(`Feedback: ${stats.totalFeedback} total (${stats.helpfulCount} helpful, ${stats.unhelpfulCount} unhelpful)`);
+    console.log(`Coverage: ${stats.feedbackSessions}/${stats.totalSessions} sessions rated`);
+    console.log(`Trust score: avg=${stats.trustScoreAvg} min=${stats.trustScoreMin} max=${stats.trustScoreMax}`);
+  }
+}
+
 async function cmdBackfill(aquifer, args) {
   const limit = parsePositiveInt(args.flags.limit, 100);
   const dryRun = !!args.flags['dry-run'];
@@ -318,6 +347,7 @@ Commands:
   migrate                     Run database migrations
   recall <query>              Search sessions (requires embed config)
   feedback                    Record trust feedback on a session
+  feedback-stats              Show trust feedback statistics and coverage
   backfill                    Enrich pending sessions
   stats                       Show database statistics
   export                      Export sessions as JSONL
@@ -336,6 +366,7 @@ Options:
   --session-id ID             Session ID (feedback)
   --verdict helpful|unhelpful Feedback verdict (feedback)
   --note TEXT                 Feedback note (feedback)
+  --explain                    Show score breakdown per result (recall)
   --json                      JSON output
   --dry-run                   Preview only (backfill)
   --output PATH               Output file (export)
@@ -410,6 +441,9 @@ Options:
       case 'feedback':
         await cmdFeedback(aquifer, args);
         break;
+      case 'feedback-stats':
+        await cmdFeedbackStats(aquifer, args);
+        break;
       case 'backfill':
         await cmdBackfill(aquifer, args);
         break;

package/consumers/mcp.js

@@ -7,7 +7,8 @@
  * This is the primary integration surface for Aquifer. Agent hosts (Claude Code,
  * Codex, OpenCode, etc.) should integrate through this MCP server.
  *
- * Tools: session_recall, session_feedback, memory_stats, memory_pending
+ * Tools: session_recall, session_feedback, feedback_stats,
+ * session_bootstrap, memory_stats, memory_pending
  *
  * Usage:
  *   npx aquifer mcp
@@ -32,8 +33,8 @@ function getAquifer() {
 
 const { formatRecallResults } = require('./shared/recall-format');
 
-function formatResults(results, query) {
-  return formatRecallResults(results, { query, showScore: true });
+function formatResults(results, query, explain) {
+  return formatRecallResults(results, { query, showScore: true, showExplain: !!explain });
 }
 
 // ---------------------------------------------------------------------------
@@ -74,6 +75,7 @@ async function main() {
       entities: z.array(z.string()).optional().describe('Entity names to match'),
       entityMode: z.enum(['any', 'all']).optional().describe('"any" (default, boost) or "all" (only sessions with every entity)'),
       mode: z.enum(['fts', 'hybrid', 'vector']).optional().describe('Recall mode: "fts" (keyword only, no embed needed), "hybrid" (default, FTS + vector), "vector" (vector only)'),
+      explain: z.boolean().optional().describe('Include per-result score breakdown (diagnostic)'),
     },
     async (params) => {
       try {
@@ -93,7 +95,7 @@ async function main() {
         if (params.mode) recallOpts.mode = params.mode;
 
         const results = await aquifer.recall(params.query, recallOpts);
-        const text = formatResults(results, params.query);
+        const text = formatResults(results, params.query, params.explain);
         return { content: [{ type: 'text', text }] };
       } catch (err) {
         return {
@@ -106,7 +108,7 @@ async function main() {
 
   server.tool(
     'session_feedback',
-    'Record trust feedback on a recalled session. Helpful sessions rank higher in future recalls.',
+    'After using session_recall, mark the result helpful if it directly informed your answer, or unhelpful if it was irrelevant/outdated. Include a short note. Sessions with more helpful feedback rank higher in future recalls.',
     {
       sessionId: z.string().min(1).describe('Session ID to give feedback on'),
       verdict: z.enum(['helpful', 'unhelpful']).describe('Was the recalled session useful?'),
@@ -133,6 +135,37 @@ async function main() {
     }
   );
 
+  server.tool(
+    'feedback_stats',
+    'Return trust feedback statistics: total feedback count, helpful/unhelpful breakdown, trust score distribution, and coverage.',
+    {
+      agentId: z.string().optional().describe('Filter by agent ID'),
+      dateFrom: z.string().optional().describe('Start date YYYY-MM-DD'),
+      dateTo: z.string().optional().describe('End date YYYY-MM-DD'),
+    },
+    async (params) => {
+      try {
+        const aquifer = getAquifer();
+        const stats = await aquifer.feedbackStats({
+          agentId: params.agentId || undefined,
+          dateFrom: params.dateFrom || undefined,
+          dateTo: params.dateTo || undefined,
+        });
+        const lines = [
+          `Feedback: ${stats.totalFeedback} total (${stats.helpfulCount} helpful, ${stats.unhelpfulCount} unhelpful)`,
+          `Coverage: ${stats.feedbackSessions}/${stats.totalSessions} sessions rated`,
+          `Trust score: avg=${stats.trustScoreAvg} min=${stats.trustScoreMin} max=${stats.trustScoreMax}`,
+        ];
+        return { content: [{ type: 'text', text: lines.join('\n') }] };
+      } catch (err) {
+        return {
+          content: [{ type: 'text', text: `feedback_stats error: ${err.message}` }],
+          isError: true,
+        };
+      }
+    }
+  );
+
   server.tool(
     'memory_stats',
     'Return storage statistics for the Aquifer memory store (session counts by status, summaries, turn embeddings, entities, date range).',

package/consumers/miranda/context-inject.js

@@ -14,6 +14,7 @@ function buildSessionContext({ today, agentId, focusText, todoItems, moodLine, h
     const parts = [];
     parts.push('你是 Miranda。以下是你已經知道的現況，直接用來回應，不需要讀檔或搜尋。像做 briefing——帶現況也帶判斷和建議。用散文段落，最後一句必須是結論或建議，不能是問句。若草稿有 bullet、標題、表格或問句收尾，改寫再送出。');
     parts.push('回答任何關於過去做過什麼、討論過什麼、決策過什麼的問題時，第一步用 session_recall MCP tool 查，不要用 grep、讀 log、翻檔案。工具在手上就用。');
+    parts.push('用完 session_recall 後，如果某筆結果實際幫助了你的回答，呼叫 session_feedback(sessionId, verdict="helpful")；如果結果明顯過時或錯誤，呼叫 session_feedback(sessionId, verdict="unhelpful")，帶簡短 note 說明原因。只對實際影響回答的結果回饋，不要每次 recall 都打分。');
 
     if (focusText) parts.push(`現在的焦點是${focusText}。`);
     if (handoffText) parts.push(`上一段的交接：${handoffText}`);

package/consumers/openclaw-plugin.js

@@ -214,6 +214,7 @@ function register(api) {
             entities: { type: 'array', items: { type: 'string' }, description: 'Entity names to match' },
             entityMode: { type: 'string', enum: ['any', 'all'], description: '"any" (default, boost) or "all" (only sessions with every entity)' },
             mode: { type: 'string', enum: ['fts', 'hybrid', 'vector'], description: 'Recall mode: "fts" (keyword only), "hybrid" (default), "vector" (vector only)' },
+            explain: { type: 'boolean', description: 'Include per-result score breakdown (diagnostic)' },
           },
           required: ['query'],
         },
@@ -234,7 +235,7 @@ function register(api) {
             if (params.mode) recallOpts.mode = params.mode;
 
             const results = await aquifer.recall(params.query, recallOpts);
-            const text = formatRecallResults(results);
+            const text = formatRecallResults(results, { showScore: true, showExplain: !!params.explain });
             return { content: [{ type: 'text', text }] };
           } catch (err) {
             return {
@@ -253,7 +254,7 @@ function register(api) {
 
       return {
         name: 'session_feedback',
-        description: 'Record trust feedback on a recalled session. Helpful sessions rank higher in future recalls.',
+        description: 'After using session_recall, mark the result helpful if it directly informed your answer, or unhelpful if it was irrelevant/outdated. Include a short note. Sessions with more helpful feedback rank higher in future recalls.',
         parameters: {
           type: 'object',
           properties: {
@@ -285,5 +286,44 @@ function register(api) {
       };
     }, { name: 'session_feedback' });
 
-  api.logger.info('[aquifer-memory] registered (before_reset + session_recall + session_feedback)');
+    // --- feedback_stats tool ---
+
+    api.registerTool((ctx) => {
+      if ((ctx?.sessionKey || '').includes('subagent')) return null;
+
+      return {
+        name: 'feedback_stats',
+        description: 'Return trust feedback statistics: total feedback count, helpful/unhelpful breakdown, trust score distribution, and coverage.',
+        parameters: {
+          type: 'object',
+          properties: {
+            agentId: { type: 'string', description: 'Filter by agent ID' },
+            dateFrom: { type: 'string', description: 'Start date YYYY-MM-DD' },
+            dateTo: { type: 'string', description: 'End date YYYY-MM-DD' },
+          },
+        },
+        async execute(_toolCallId, params) {
+          try {
+            const stats = await aquifer.feedbackStats({
+              agentId: params.agentId || undefined,
+              dateFrom: params.dateFrom || undefined,
+              dateTo: params.dateTo || undefined,
+            });
+            const lines = [
+              `Feedback: ${stats.totalFeedback} total (${stats.helpfulCount} helpful, ${stats.unhelpfulCount} unhelpful)`,
+              `Coverage: ${stats.feedbackSessions}/${stats.totalSessions} sessions rated`,
+              `Trust score: avg=${stats.trustScoreAvg} min=${stats.trustScoreMin} max=${stats.trustScoreMax}`,
+            ];
+            return { content: [{ type: 'text', text: lines.join('\n') }] };
+          } catch (err) {
+            return {
+              content: [{ type: 'text', text: `feedback_stats error: ${err.message}` }],
+              isError: true,
+            };
+          }
+        },
+      };
+    }, { name: 'feedback_stats' });
+
+  api.logger.info('[aquifer-memory] registered (before_reset + session_recall + session_feedback + feedback_stats)');
 }

package/consumers/shared/config.js

@@ -30,6 +30,15 @@ const DEFAULTS = {
     temperature: 0,
   },
   entities: { enabled: false, mergeCall: true, scope: 'default' },
+  insights: {
+    recallWeights: null,
+    recencyWindowDays: null,
+    dedup: {
+      mode: 'off',
+      cosineThreshold: 0.88,
+      closeBandFrom: 0.85,
+    },
+  },
   rank: { rrf: 0.65, timeDecay: 0.25, access: 0.10, entityBoost: 0.18 },
   rerank: {
     enabled: false,
@@ -75,6 +84,9 @@ const ENV_MAP = [
   ['AQUIFER_LLM_TEMPERATURE',   'llm.temperature',   Number],
   ['AQUIFER_ENTITIES_ENABLED',  'entities.enabled',  Boolean],
   ['AQUIFER_ENTITY_SCOPE',     'entities.scope'],
+  ['AQUIFER_INSIGHTS_DEDUP_MODE',             'insights.dedup.mode'],
+  ['AQUIFER_INSIGHTS_DEDUP_COSINE',           'insights.dedup.cosineThreshold', Number],
+  ['AQUIFER_INSIGHTS_DEDUP_CLOSE_BAND_FROM',  'insights.dedup.closeBandFrom',   Number],
   ['AQUIFER_RERANK_ENABLED',   'rerank.enabled',    Boolean],
   ['AQUIFER_RERANK_PROVIDER',  'rerank.provider'],
   ['AQUIFER_RERANK_BASE_URL',  'rerank.baseUrl'],
@@ -165,6 +177,14 @@ function loadConfig(opts = {}) {
     config = deepMerge(config, opts.overrides);
   }
 
+  // insights.dedup shorthand: true → enforce, false → off
+  if (config.insights && typeof config.insights.dedup === 'boolean') {
+    config.insights.dedup = {
+      ...DEFAULTS.insights.dedup,
+      mode: config.insights.dedup ? 'enforce' : 'off',
+    };
+  }
+
   return config;
 }
 

package/consumers/shared/factory.js

@@ -91,6 +91,7 @@ function createAquiferFromConfig(overrides) {
     rank: config.rank,
     rerank: rerankOpts,
     migrations: config.migrations,
+    insights: config.insights,
   });
 
   return aquifer;

package/consumers/shared/recall-format.js

@@ -66,6 +66,30 @@ const defaultRenderers = {
         if (!showScore) return null;
         return `Score: ${typeof result.score === 'number' ? result.score.toFixed(3) : '?'}`;
     },
+    explain(result, { showExplain }) {
+        if (!showExplain) return null;
+        const d = result._debug;
+        if (!d) return null;
+        const f = (v) => typeof v === 'number' ? v.toFixed(3) : '?';
+        const parts = [
+            `rrf=${f(d.rrf)}`,
+            `td=${f(d.timeDecay)}`,
+            `access=${f(d.access)}`,
+            `entity=${f(d.entityScore)}`,
+            `trust=${f(d.trustScore)}(\u00d7${f(d.trustMultiplier)})`,
+            `ol=${f(d.openLoopBoost)}`,
+            `\u2192 hybrid=${f(d.hybridScore)}`,
+        ];
+        if (d.rerankApplied) {
+            parts.push(`rerank=${f(d.rerankScore)}(${d.rerankReason || '?'})`);
+        } else {
+            parts.push(`[rerank: off (${d.rerankReason || '?'})]`);
+        }
+        if (Array.isArray(d.searchErrors) && d.searchErrors.length > 0) {
+            parts.push(`errors: ${d.searchErrors.map(e => (e && e.path) || '?').join(',')}`);
+        }
+        return `  ${parts.join(' ')}`;
+    },
     separator() {
         return '';
     },
@@ -102,6 +126,8 @@ function createRecallFormatter(overrides = {}) {
             if (matched) lines.push(matched);
             const score = r.score(res, { showScore: !!opts.showScore, ...ctx });
             if (score) lines.push(score);
+            const explain = r.explain(res, { showExplain: !!opts.showExplain, ...ctx });
+            if (explain) lines.push(explain);
             const sep = r.separator(i, ctx);
             if (sep !== null && sep !== undefined) lines.push(sep);
         }

package/core/aquifer.js

@@ -1558,6 +1558,17 @@ function createAquifer(config = {}) {
       });
     },
 
+    async feedbackStats(opts = {}) {
+      await ensureMigrated();
+      return storage.getFeedbackStats(pool, {
+        schema,
+        tenantId,
+        agentId: opts.agentId || undefined,
+        dateFrom: opts.dateFrom || undefined,
+        dateTo: opts.dateTo || undefined,
+      });
+    },
+
     // --- admin ---
 
     async getSession(sessionId, opts = {}) {
@@ -1837,6 +1848,7 @@ function createAquifer(config = {}) {
     recallWeights: (config.insights && config.insights.recallWeights) || null,
     recencyWindowDays: config.insights && Number.isFinite(config.insights.recencyWindowDays)
       ? config.insights.recencyWindowDays : undefined,
+    dedup: config.insights && config.insights.dedup ? config.insights.dedup : undefined,
   });
 
   return aquifer;