@shadowforge0/aquifer-memory 1.5.9 → 1.5.12

package/core/insights.js

@@ -26,65 +26,20 @@ const DEFAULT_RECALL_WEIGHTS = Object.freeze({
   recency: 0.10,
 });
 
+const DEFAULT_DEDUP = Object.freeze({
+  mode: 'off',
+  cosineThreshold: 0.88,
+  closeBandFrom: 0.85,
+});
+
+const VALID_DEDUP_MODES = new Set(['off', 'shadow', 'enforce']);
+
 // Recency linear decay horizon — an insight is treated as "fully recent" at
 // creation (age=0) and "zero recency" at age >= recencyWindowDays. Beyond,
 // recency contribution is clamped to 0 rather than going negative. Configurable
 // via createAquifer({ insights: { recencyWindowDays } }).
 const DEFAULT_RECENCY_WINDOW_DAYS = 90;
 
-const LEADING_PUNCT_RE = /^[\s\-_.,;:!?'"()\[\]{}@#]+/;
-const TRAILING_PUNCT_RE = /[\s\-_.,;:!?'"()\[\]{}@#]+$/;
-
-function _normalizeText(input) {
-  if (typeof input !== 'string' || !input) return '';
-  let s = input.normalize('NFKC');
-  s = s.toLowerCase();
-  s = s.replace(/\s+/g, ' ');
-  s = s.replace(LEADING_PUNCT_RE, '');
-  s = s.replace(TRAILING_PUNCT_RE, '');
-  return s;
-}
-
-function normalizeCanonicalClaim(text) {
-  return _normalizeText(text);
-}
-
-function normalizeBody(text) {
-  return _normalizeText(text);
-}
-
-function normalizeEntitySet(entities) {
-  if (!entities || !Array.isArray(entities)) return '';
-  const { normalizeEntityName } = require('./entity');
-  const normalized = entities
-    .map(e => normalizeEntityName(e))
-    .filter(Boolean);
-  const deduped = [...new Set(normalized)];
-  deduped.sort();
-  return deduped.join('|');
-}
-
-function defaultCanonicalKey({ tenantId, agentId, type, canonicalClaim, entities }) {
-  const normClaim = normalizeCanonicalClaim(canonicalClaim);
-  const normEntities = normalizeEntitySet(entities);
-  const input = `${tenantId || ''}|${agentId || ''}|${type || ''}|${normClaim}|${normEntities}`;
-  return crypto.createHash('sha256').update(input).digest('hex');
-}
-
-function defaultIdempotencyKey({
-  tenantId, agentId, type, title, body, sourceSessionIds, evidenceWindow,
-}) {
-  const sorted = (sourceSessionIds || []).slice().sort().join('|');
-  const winFrom = evidenceWindow && evidenceWindow.from ? new Date(evidenceWindow.from).toISOString() : '';
-  const winTo = evidenceWindow && evidenceWindow.to ? new Date(evidenceWindow.to).toISOString() : '';
-  // Hash must include body + window so legitimate revisions (same sessions but
-  // tightened body, or extended window) get a new key and replace the old row
-  // via supersede, not get swallowed as a duplicate.
-  return crypto.createHash('sha256')
-    .update(`${tenantId}|${agentId}|${type}|${title}|${body || ''}|${sorted}|${winFrom}|${winTo}`)
-    .digest('hex');
-}
-
 // ---------------------------------------------------------------------------
 // Canonical identity helpers (Phase 2 C1)
 //
@@ -160,6 +115,94 @@ function vecToPgLiteral(v) {
   return `[${v.join(',')}]`;
 }
 
+function truncate(input, limit) {
+  if (typeof input !== 'string') return '';
+  if (!Number.isFinite(limit) || limit < 0) return '';
+  return input.length <= limit ? input : input.slice(0, limit);
+}
+
+function truncateNormalized(input, limit) {
+  return truncate(normalizeBody(input), limit);
+}
+
+function resolveDedupConfig(dedup, embedFn) {
+  let resolved;
+  if (dedup === true) {
+    resolved = { ...DEFAULT_DEDUP, mode: 'enforce' };
+  } else if (dedup === false || dedup === undefined) {
+    resolved = { ...DEFAULT_DEDUP };
+  } else if (dedup && typeof dedup === 'object') {
+    resolved = { ...DEFAULT_DEDUP, ...dedup };
+  } else {
+    resolved = { ...DEFAULT_DEDUP };
+  }
+
+  const rawMode = typeof resolved.mode === 'string' ? resolved.mode.trim().toLowerCase() : resolved.mode;
+  if (!VALID_DEDUP_MODES.has(rawMode)) {
+    console.warn(`[aquifer] insights dedup: invalid mode ${JSON.stringify(resolved.mode)}; coercing to 'off'`);
+    resolved.mode = 'off';
+  } else {
+    resolved.mode = rawMode;
+  }
+
+  const envMode = process.env.AQUIFER_INSIGHTS_DEDUP_MODE;
+  if (typeof envMode === 'string') {
+    const normalizedEnvMode = envMode.trim().toLowerCase();
+    if (VALID_DEDUP_MODES.has(normalizedEnvMode)) {
+      resolved.mode = normalizedEnvMode;
+    }
+  }
+
+  // Reject non-numeric sentinels (null, bool, objects) BEFORE Number()
+  // coerces them to 0 — 0 would silently become a "merge everything"
+  // threshold in enforce mode.
+  let cosineThreshold;
+  if (resolved.cosineThreshold === null || resolved.cosineThreshold === undefined
+      || typeof resolved.cosineThreshold === 'boolean') {
+    console.warn(`[aquifer] insights dedup: invalid cosineThreshold ${JSON.stringify(resolved.cosineThreshold)}; defaulting to 0.88`);
+    cosineThreshold = DEFAULT_DEDUP.cosineThreshold;
+  } else {
+    cosineThreshold = Number(resolved.cosineThreshold);
+    if (!Number.isFinite(cosineThreshold)) {
+      console.warn('[aquifer] insights dedup: invalid cosineThreshold; defaulting to 0.88');
+      cosineThreshold = DEFAULT_DEDUP.cosineThreshold;
+    } else if (cosineThreshold < 0.75 || cosineThreshold > 0.95) {
+      const clamped = Math.max(0, Math.min(1, cosineThreshold));
+      console.warn(`[aquifer] insights dedup: cosineThreshold ${cosineThreshold} outside recommended [0.75,0.95]; using ${clamped}`);
+      cosineThreshold = (cosineThreshold >= 0 && cosineThreshold <= 1) ? cosineThreshold : clamped;
+    }
+  }
+  resolved.cosineThreshold = cosineThreshold;
+
+  let closeBandFrom;
+  if (resolved.closeBandFrom === null || resolved.closeBandFrom === undefined
+      || typeof resolved.closeBandFrom === 'boolean') {
+    console.warn(`[aquifer] insights dedup: invalid closeBandFrom ${JSON.stringify(resolved.closeBandFrom)}; defaulting to 0.85`);
+    closeBandFrom = DEFAULT_DEDUP.closeBandFrom;
+  } else {
+    closeBandFrom = Number(resolved.closeBandFrom);
+    if (!Number.isFinite(closeBandFrom)) {
+      console.warn('[aquifer] insights dedup: invalid closeBandFrom; defaulting to 0.85');
+      closeBandFrom = DEFAULT_DEDUP.closeBandFrom;
+    }
+  }
+  if (closeBandFrom >= resolved.cosineThreshold) {
+    const adjusted = Math.max(0, resolved.cosineThreshold - 0.03);
+    console.warn(`[aquifer] insights dedup: closeBandFrom ${closeBandFrom} must be below cosineThreshold ${resolved.cosineThreshold}; using ${adjusted}`);
+    closeBandFrom = adjusted;
+  }
+  resolved.closeBandFrom = closeBandFrom;
+
+  if (resolved.mode !== 'off') {
+    console.log(`[aquifer] insights dedup: mode=${resolved.mode} threshold=${resolved.cosineThreshold} close_band_from=${resolved.closeBandFrom}`);
+    if (!embedFn) {
+      console.warn('[aquifer] insights dedup: embedFn unavailable; semantic dedup disabled at runtime');
+    }
+  }
+
+  return Object.freeze(resolved);
+}
+
 function mapRow(row) {
   if (!row) return null;
   return {
@@ -184,7 +227,7 @@ function mapRow(row) {
   };
 }
 
-function createInsights({ pool, schema, defaultTenantId, embedFn, recallWeights, recencyWindowDays }) {
+function createInsights({ pool, schema, defaultTenantId, embedFn, recallWeights, recencyWindowDays, dedup }) {
   if (!pool) throw new Error('createInsights: pool is required');
   if (!schema) throw new Error('createInsights: schema is required');
 
@@ -192,6 +235,24 @@ function createInsights({ pool, schema, defaultTenantId, embedFn, recallWeights,
   const recencyWindow = Number.isFinite(recencyWindowDays) && recencyWindowDays > 0
     ? recencyWindowDays : DEFAULT_RECENCY_WINDOW_DAYS;
   const tbl = `${schema}.insights`;
+  const dedupConfig = resolveDedupConfig(dedup, embedFn);
+
+  if (dedupConfig.mode !== 'off') {
+    pool.query(
+      `SELECT count(*)::int AS n FROM ${tbl}
+        WHERE canonical_key_v2 IS NULL AND status = 'active'`
+    ).then(r => {
+      const n = r && r.rows && r.rows[0] ? Number(r.rows[0].n) : 0;
+      if (n > 0) {
+        console.warn(
+          `[aquifer] insights: ${n} active rows with canonical_key_v2 IS NULL. `
+          + 'Run scripts/backfill-canonical-key.js to include them in canonical dedup.'
+        );
+      }
+    }).catch(() => {
+      // non-fatal
+    });
+  }
 
   // -------------------------------------------------------------------------
   // commitInsight
@@ -283,9 +344,101 @@ function createInsights({ pool, schema, defaultTenantId, embedFn, recallWeights,
         toSupersede = Number(activeRow.id);
       }
 
-      // Optional embedding.
       let embedding = null;
-      if (embedFn) {
+      let embeddingReady = false;
+
+      if (dedupConfig.mode !== 'off' && !toSupersede && embedFn) {
+        // Embed the incoming title+body once. If this throws, the label
+        // is genuinely 'embed_failed' — the candidate SELECT never ran.
+        let embedFailed = false;
+        try {
+          const v = await embedFn([`${title}\n\n${body}`]);
+          if (Array.isArray(v) && Array.isArray(v[0])) {
+            embedding = vecToPgLiteral(v[0]);
+          }
+          embeddingReady = true;
+        } catch {
+          embedFailed = true;
+          embeddingReady = true;
+          metadata = { ...metadata, dedupSkipped: 'embed_failed' };
+        }
+
+        if (!embedFailed && embedding) {
+          // Candidate lookup. If this throws (DB error), let it bubble
+          // to the outer commitInsight try/catch → AQ_INTERNAL. Do NOT
+          // mislabel it as embed_failed.
+          const semanticLookup = await pool.query(
+            `SELECT *, 1.0 - (embedding <=> $4::vector) AS cos_sim
+               FROM ${tbl}
+              WHERE tenant_id = $1
+                AND agent_id = $2
+                AND insight_type = $3
+                AND status = 'active'
+                AND embedding IS NOT NULL
+              ORDER BY embedding <=> $4::vector
+              LIMIT 1`,
+            [tenantId, agentId, type, embedding]
+          );
+
+          if (semanticLookup.rowCount > 0) {
+            const candidate = semanticLookup.rows[0];
+            const cosine = Number(candidate.cos_sim);
+
+            if (cosine >= dedupConfig.cosineThreshold) {
+              const candidateUpper = parseUpperFromRange(candidate.evidence_window);
+              const isStaleReplay = candidateUpper
+                && new Date(toIso).getTime() < candidateUpper.getTime();
+
+              if (dedupConfig.mode === 'enforce') {
+                // Enforce path: stale-replay returns the candidate as
+                // duplicate; otherwise supersede.
+                if (isStaleReplay) {
+                  return ok({ insight: mapRow(candidate), duplicate: true });
+                }
+                toSupersede = Number(candidate.id);
+                metadata = {
+                  ...metadata,
+                  dedupVia: 'semantic',
+                  dedupCandidate: { id: Number(candidate.id), cosine },
+                };
+              } else {
+                // Shadow path: always insert the new row, always record
+                // shadowMatch metadata. staleReplay flag tells reviewers
+                // the enforce-mode twin would have returned duplicate
+                // instead of superseding.
+                metadata = {
+                  ...metadata,
+                  shadowMatch: {
+                    candidateId: Number(candidate.id),
+                    cosine,
+                    threshold: dedupConfig.cosineThreshold,
+                    candidateTitle: truncate(candidate.title, 200),
+                    candidateBody: truncateNormalized(candidate.body, 200),
+                    wouldSupersede: !isStaleReplay,
+                    staleReplay: Boolean(isStaleReplay),
+                    ranAt: new Date().toISOString(),
+                  },
+                };
+              }
+            } else if (cosine >= dedupConfig.closeBandFrom) {
+              metadata = {
+                ...metadata,
+                dedupNear: {
+                  candidateId: Number(candidate.id),
+                  cosine,
+                  threshold: dedupConfig.cosineThreshold,
+                  closeBandFrom: dedupConfig.closeBandFrom,
+                  candidateTitle: truncate(candidate.title, 200),
+                  candidateBody: truncateNormalized(candidate.body, 200),
+                },
+              };
+            }
+          }
+        }
+      }
+
+      // Optional embedding.
+      if (embedFn && !embeddingReady) {
         try {
           const v = await embedFn([`${title}\n\n${body}`]);
           if (Array.isArray(v) && Array.isArray(v[0])) embedding = vecToPgLiteral(v[0]);
@@ -485,13 +638,12 @@ function createInsights({ pool, schema, defaultTenantId, embedFn, recallWeights,
     recallInsights,
     markStale,
     supersede,
-    _internal: { defaultIdempotencyKey, vecToPgLiteral, mapRow, weights },
+    _internal: { vecToPgLiteral, mapRow, weights, dedup: dedupConfig },
   };
 }
 
 module.exports = {
   createInsights,
-  defaultIdempotencyKey,
   defaultCanonicalKey,
   normalizeCanonicalClaim,
   normalizeBody,

package/core/mcp-manifest.js

@@ -46,13 +46,17 @@ const MCP_TOOL_MANIFEST = Object.freeze([
           enum: ['fts', 'hybrid', 'vector'],
           description: 'Recall mode: "fts" (keyword only, no embed needed), "hybrid" (default, FTS + vector), "vector" (vector only)',
         },
+        explain: {
+          type: 'boolean',
+          description: 'Include per-result score breakdown (rrf, timeDecay, entity, trust, rerank). Diagnostic use only.',
+        },
       },
       required: ['query'],
     },
   },
   {
     name: 'session_feedback',
-    description: 'Record trust feedback on a recalled session. Helpful sessions rank higher in future recalls.',
+    description: 'After using session_recall, mark the result helpful if it directly informed your answer, or unhelpful if it was irrelevant/outdated. Include a short note. Sessions with more helpful feedback rank higher in future recalls.',
     inputSchema: {
       type: 'object',
       additionalProperties: false,
@@ -85,6 +89,19 @@ const MCP_TOOL_MANIFEST = Object.freeze([
       },
     },
   },
+  {
+    name: 'feedback_stats',
+    description: 'Return trust feedback statistics: total feedback count, helpful/unhelpful breakdown, trust score distribution, and coverage (how many sessions have been rated).',
+    inputSchema: {
+      type: 'object',
+      additionalProperties: false,
+      properties: {
+        agentId: { type: 'string', description: 'Filter by agent ID' },
+        dateFrom: { type: 'string', description: 'Start date YYYY-MM-DD for feedback window' },
+        dateTo: { type: 'string', description: 'End date YYYY-MM-DD for feedback window' },
+      },
+    },
+  },
   {
     name: 'session_bootstrap',
     description: 'Load recent session context for a new conversation. Returns summaries, open items, and decisions from recent sessions. Call this at the start of a conversation for continuity; use session_recall for keyword search.',

package/core/storage.js

@@ -666,6 +666,76 @@ async function recordFeedback(pool, {
   }
 }
 
+// ---------------------------------------------------------------------------
+// getFeedbackStats — aggregate feedback and trust score metrics
+// ---------------------------------------------------------------------------
+
+async function getFeedbackStats(pool, { schema, tenantId, agentId, dateFrom, dateTo }) {
+  const params = [tenantId];
+  let sessionClause = '';
+  if (agentId) {
+    params.push(agentId);
+    sessionClause += ` AND s.agent_id = $${params.length}`;
+  }
+  if (dateFrom) {
+    params.push(dateFrom);
+    sessionClause += ` AND s.started_at >= $${params.length}::date`;
+  }
+  if (dateTo) {
+    params.push(dateTo);
+    sessionClause += ` AND s.started_at < ($${params.length}::date + interval '1 day')`;
+  }
+
+  const fbQuery = `
+    WITH scoped_sessions AS (
+      SELECT s.id
+      FROM ${qi(schema)}.sessions s
+      WHERE s.tenant_id = $1${sessionClause}
+    )
+    SELECT
+      COUNT(sf.*)::int AS total,
+      COUNT(*) FILTER (WHERE sf.verdict = 'helpful')::int AS helpful,
+      COUNT(*) FILTER (WHERE sf.verdict = 'unhelpful')::int AS unhelpful,
+      COUNT(DISTINCT sf.session_row_id)::int AS rated_sessions
+    FROM scoped_sessions ss
+    LEFT JOIN ${qi(schema)}.session_feedback sf
+      ON sf.session_row_id = ss.id`;
+
+  const ssQuery = `
+    WITH scoped_sessions AS (
+      SELECT s.id
+      FROM ${qi(schema)}.sessions s
+      WHERE s.tenant_id = $1${sessionClause}
+    )
+    SELECT
+      COUNT(scoped_sessions.id)::int AS total_sessions,
+      ROUND(AVG(summary.trust_score)::numeric, 3) AS avg_ts,
+      MIN(summary.trust_score) AS min_ts,
+      MAX(summary.trust_score) AS max_ts
+    FROM scoped_sessions
+    LEFT JOIN ${qi(schema)}.session_summaries summary
+      ON summary.session_row_id = scoped_sessions.id`;
+
+  const [fbResult, ssResult] = await Promise.all([
+    pool.query(fbQuery, params),
+    pool.query(ssQuery, params),
+  ]);
+
+  const fb = fbResult.rows[0];
+  const ss = ssResult.rows[0];
+
+  return {
+    totalFeedback: fb.total,
+    helpfulCount: fb.helpful,
+    unhelpfulCount: fb.unhelpful,
+    feedbackSessions: fb.rated_sessions,
+    totalSessions: ss.total_sessions,
+    trustScoreAvg: (ss.avg_ts !== null && ss.avg_ts !== undefined) ? parseFloat(ss.avg_ts) : 0.5,
+    trustScoreMin: (ss.min_ts !== null && ss.min_ts !== undefined) ? parseFloat(ss.min_ts) : 0.5,
+    trustScoreMax: (ss.max_ts !== null && ss.max_ts !== undefined) ? parseFloat(ss.max_ts) : 0.5,
+  };
+}
+
 // ---------------------------------------------------------------------------
 // Exports
 // ---------------------------------------------------------------------------
@@ -683,4 +753,5 @@ module.exports = {
   searchTurnEmbeddings,
   searchSummaryEmbeddings,
   recordFeedback,
+  getFeedbackStats,
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@shadowforge0/aquifer-memory",
-  "version": "1.5.9",
+  "version": "1.5.12",
   "description": "PG-native long-term memory for AI agents. Turn-level embedding, hybrid RRF ranking, optional knowledge graph. MCP server, CLI, and library API.",
   "main": "index.js",
   "files": [
@@ -13,7 +13,15 @@
     "consumers/default/",
     "consumers/openclaw-ext/",
     "docs/",
-    "scripts/"
+    "scripts/backfill-canonical-key.js",
+    "scripts/diagnose-fts-zh.js",
+    "scripts/diagnose-vector.js",
+    "scripts/drop-entity-state-history.sql",
+    "scripts/drop-insights.sql",
+    "scripts/extract-insights-from-recent-sessions.js",
+    "scripts/find-dburl-hints.js",
+    "scripts/install-openclaw.sh",
+    "scripts/smoke.mjs"
   ],
   "bin": {
     "aquifer": "./consumers/cli.js"