@shadowforge0/aquifer-memory 1.0.1 → 1.0.2

package/core/aquifer.js

@@ -100,19 +100,6 @@ function createAquifer(config) {
   const entityPromptFn = config.entities && config.entities.prompt ? config.entities.prompt : null;
   const entityScope = (config.entities && config.entities.scope) || 'default';
 
-  // FTS config — locked to 'simple'.
-  // The search_tsv trigger always uses to_tsvector('simple', ...), so query-time
-  // config must match.  Warn and override if someone passes anything else.
-  const _rawFtsConfig = config.ftsConfig || 'simple';
-  if (_rawFtsConfig !== 'simple') {
-    console.warn(
-      `[aquifer] ftsConfig '${_rawFtsConfig}' is not currently supported. ` +
-      `The search_tsv index is built with 'simple'; only 'simple' is valid at query time. ` +
-      `Overriding to 'simple'.`
-    );
-  }
-  const ftsConfig = 'simple';
-
   // Rank weights
   const rankWeights = {
     rrf: 0.65,
@@ -706,7 +693,7 @@ function createAquifer(config) {
       const [ftsRows, embRows, turnResult] = await Promise.all([
         runFts
           ? storage.searchSessions(pool, query, {
-              schema, tenantId, agentIds: resolvedAgentIds, source, dateFrom, dateTo, limit: fetchLimit, ftsConfig,
+              schema, tenantId, agentIds: resolvedAgentIds, source, dateFrom, dateTo, limit: fetchLimit,
             }).catch((err) => {
               recordSearchError('fts', err);
               return [];

package/core/storage.js

@@ -211,7 +211,7 @@ async function getMessages(pool, sessionId, agentId, { schema, tenantId } = {})
 }
 
 // ---------------------------------------------------------------------------
-// searchSessions (FTS)
+// searchSessions (trigram + FTS fallback)
 // ---------------------------------------------------------------------------
 
 async function searchSessions(pool, query, {
@@ -220,34 +220,27 @@ async function searchSessions(pool, query, {
   agentId,
   agentIds: rawAgentIds,
   source,
-  dateFrom,  // m1: add date filtering
+  dateFrom,
   dateTo,
   limit = 20,
-  ftsConfig = 'simple',
 } = {}) {
   const clampedLimit = Math.max(1, Math.min(100, limit));
-  // FTS config is locked to 'simple' — the search_tsv trigger always uses
-  // to_tsvector('simple', ...) so query semantics must match.  Warn callers
-  // that pass a different value rather than silently honouring it.
-  if (ftsConfig !== 'simple') {
-    console.warn(
-      `[aquifer/storage] searchSessions: ftsConfig '${ftsConfig}' ignored. ` +
-      `Only 'simple' is supported (index is built with simple tokenizer). ` +
-      `Using 'simple'.`
-    );
-  }
-  const safeFts = 'simple';
 
   // Normalize agentId/agentIds
   const agentIds = rawAgentIds && rawAgentIds.length > 0
     ? rawAgentIds
     : (agentId ? [agentId] : null);
 
+  // Escape LIKE special characters in query
+  const likeQuery = query.replace(/[%_\\]/g, '\\$&');
+
+  // Primary: trigram ILIKE on search_text (works for CJK + Latin)
+  // Fallback: tsvector FTS (for installations without search_text populated)
   const where = [
-    `ss.search_tsv @@ plainto_tsquery('${safeFts}', $1)`,
-    `s.tenant_id = $2`,
+    `(ss.search_text ILIKE '%' || $1 || '%' OR ss.search_tsv @@ plainto_tsquery('simple', $2))`,
+    `s.tenant_id = $3`,
   ];
-  const params = [query, tenantId];
+  const params = [likeQuery, query, tenantId];
 
   if (agentIds) {
     params.push(agentIds);
@@ -281,8 +274,10 @@ async function searchSessions(pool, query, {
       ss.access_count,
       ss.last_accessed_at,
       ss.trust_score,
-      ts_headline('${safeFts}', COALESCE(ss.summary_text, ''), plainto_tsquery('${safeFts}', $1)) AS summary_snippet,
-      ts_rank(ss.search_tsv, plainto_tsquery('${safeFts}', $1)) AS fts_rank
+      CASE WHEN ss.search_text IS NOT NULL
+        THEN similarity(ss.search_text, $2)
+        ELSE ts_rank(ss.search_tsv, plainto_tsquery('simple', $2))
+      END AS fts_rank
     FROM ${qi(schema)}.sessions s
     LEFT JOIN ${qi(schema)}.session_summaries ss ON ss.session_row_id = s.id
     WHERE ${where.join(' AND ')}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@shadowforge0/aquifer-memory",
-  "version": "1.0.1",
+  "version": "1.0.2",
   "description": "PG-native long-term memory for AI agents. Turn-level embedding, hybrid RRF ranking, optional knowledge graph. MCP server, CLI, and library API.",
   "main": "index.js",
   "files": [

package/schema/001-base.sql

@@ -2,6 +2,7 @@
 -- Usage: replace ${schema} with actual schema name (e.g., 'aquifer')
 
 CREATE EXTENSION IF NOT EXISTS vector;
+CREATE EXTENSION IF NOT EXISTS pg_trgm;
 CREATE SCHEMA IF NOT EXISTS ${schema};
 
 -- =========================================================================
@@ -85,6 +86,7 @@ CREATE TABLE IF NOT EXISTS ${schema}.session_summaries (
   structured_summary       JSONB        NOT NULL DEFAULT '{}',
   embedding                vector,
   search_tsv               TSVECTOR,
+  search_text              TEXT,
   access_count             INT          NOT NULL DEFAULT 0,
   last_accessed_at         TIMESTAMPTZ,
   updated_at               TIMESTAMPTZ  NOT NULL DEFAULT now()
@@ -96,6 +98,9 @@ CREATE INDEX IF NOT EXISTS idx_summaries_tenant
 CREATE INDEX IF NOT EXISTS idx_summaries_search_tsv
   ON ${schema}.session_summaries USING GIN (search_tsv);
 
+CREATE INDEX IF NOT EXISTS idx_summaries_search_text_trgm
+  ON ${schema}.session_summaries USING GIN (search_text gin_trgm_ops);
+
 CREATE INDEX IF NOT EXISTS idx_summaries_embedding
   ON ${schema}.session_summaries (session_row_id)
   WHERE embedding IS NOT NULL;
@@ -141,6 +146,11 @@ BEGIN
     setweight(to_tsvector('simple', COALESCE(NEW.summary_text, '')), 'C') ||
     setweight(to_tsvector('simple', open_loops_text || ' ' || facts_text), 'D');
 
+  NEW.search_text :=
+    title_text || ' ' || overview_text || ' ' || topics_text || ' ' ||
+    decisions_text || ' ' || COALESCE(NEW.summary_text, '') || ' ' ||
+    open_loops_text || ' ' || facts_text;
+
   RETURN NEW;
 END;
 $$;
@@ -149,7 +159,7 @@ DROP TRIGGER IF EXISTS trg_session_summaries_search_tsv
   ON ${schema}.session_summaries;
 
 CREATE TRIGGER trg_session_summaries_search_tsv
-  BEFORE INSERT OR UPDATE OF summary_text, structured_summary
+  BEFORE INSERT OR UPDATE OF summary_text, structured_summary, search_text
   ON ${schema}.session_summaries
   FOR EACH ROW
   EXECUTE FUNCTION ${schema}.session_summaries_search_tsv_update();

package/scripts/diagnose-fts-zh.js

@@ -0,0 +1,161 @@
+'use strict';
+
+/**
+ * FTS 中文診斷：檢查 'simple' tokenizer 在實際中文資料上的表現
+ *
+ * 測試項目：
+ * 1. FTS tokenization — 實際 token 長什麼樣
+ * 2. FTS recall — 常見中文查詢的命中率
+ * 3. FTS vs vector — FTS 有沒有在幫忙還是在拖後腿
+ */
+
+const { Pool } = require('pg');
+
+const DB_URL = process.env.DATABASE_URL || 'postgresql://burk:790476@localhost:5432/openclaw_db';
+const SCHEMA = process.env.AQUIFER_SCHEMA || 'miranda';
+
+const pool = new Pool({ connectionString: DB_URL });
+
+async function run() {
+  const qi = (s) => `"${s}"`;
+
+  console.log('=== FTS 中文診斷 ===\n');
+
+  // 1. 看 token 分佈
+  console.log('--- 1. Token 分析 ---');
+  const tokenSample = await pool.query(`
+    SELECT ss.session_id,
+           array_length(tsvector_to_array(ss.search_tsv), 1) as token_count,
+           left(ss.summary_text, 80) as preview
+    FROM ${qi(SCHEMA)}.session_summaries ss
+    WHERE ss.search_tsv IS NOT NULL
+    ORDER BY ss.updated_at DESC
+    LIMIT 10
+  `);
+
+  let totalTokens = 0;
+  let sessionCount = 0;
+  for (const r of tokenSample.rows) {
+    totalTokens += r.token_count || 0;
+    sessionCount++;
+    console.log(`  ${r.session_id?.slice(0, 8)} | ${r.token_count || 0} tokens | ${r.preview}`);
+  }
+  console.log(`  avg: ${sessionCount ? Math.round(totalTokens / sessionCount) : 0} tokens/session\n`);
+
+  // 2. 看一個 session 的實際 token
+  console.log('--- 2. Token 範例（最近 session）---');
+  const tokenDetail = await pool.query(`
+    SELECT ss.session_id,
+           array_to_string(tsvector_to_array(ss.search_tsv), ' | ') as tokens
+    FROM ${qi(SCHEMA)}.session_summaries ss
+    WHERE ss.search_tsv IS NOT NULL
+    ORDER BY ss.updated_at DESC
+    LIMIT 1
+  `);
+  if (tokenDetail.rows[0]) {
+    console.log(`  session: ${tokenDetail.rows[0].session_id?.slice(0, 8)}`);
+    const tokens = tokenDetail.rows[0].tokens || '';
+    // 分類 token
+    const all = tokens.split(' | ');
+    const cjk = all.filter(t => /[\u4e00-\u9fff]/.test(t));
+    const latin = all.filter(t => /^[a-z0-9]/.test(t));
+    const other = all.filter(t => !(/[\u4e00-\u9fff]/.test(t)) && !(/^[a-z0-9]/.test(t)));
+    console.log(`  total: ${all.length} | latin: ${latin.length} | cjk: ${cjk.length} | other: ${other.length}`);
+    console.log(`  CJK tokens (前 20): ${cjk.slice(0, 20).join(' | ')}`);
+    console.log(`  Latin tokens (前 20): ${latin.slice(0, 20).join(' | ')}\n`);
+  }
+
+  // 3. 中文查詢命中率測試
+  console.log('--- 3. 中文查詢 FTS 命中率 ---');
+  const testQueries = [
+    'afterburn',
+    'bootstrap',
+    'session',
+    'recall',
+    '記憶',
+    '修復',
+    '架構',
+    '時區',
+    '去重',
+    'daily entries',
+    'OpenCode',
+    'entity',
+    'Jenny',
+    'Aquifer',
+    '消化模式',
+  ];
+
+  // 總 session 數
+  const totalResult = await pool.query(`
+    SELECT COUNT(*) as cnt FROM ${qi(SCHEMA)}.session_summaries WHERE search_tsv IS NOT NULL
+  `);
+  const totalSessions = parseInt(totalResult.rows[0].cnt);
+  console.log(`  total sessions with FTS index: ${totalSessions}\n`);
+
+  for (const q of testQueries) {
+    const ftsResult = await pool.query(`
+      SELECT COUNT(*) as cnt
+      FROM ${qi(SCHEMA)}.session_summaries ss
+      WHERE ss.search_tsv @@ plainto_tsquery('simple', $1)
+    `, [q]);
+    const ftsHits = parseInt(ftsResult.rows[0].cnt);
+
+    // 同時看 summary_text ILIKE 能找到幾筆（ground truth）
+    const ilikeResult = await pool.query(`
+      SELECT COUNT(*) as cnt
+      FROM ${qi(SCHEMA)}.session_summaries ss
+      WHERE ss.summary_text ILIKE $1
+         OR ss.structured_summary::text ILIKE $1
+    `, [`%${q}%`]);
+    const ilikeHits = parseInt(ilikeResult.rows[0].cnt);
+
+    const ftsRecall = ilikeHits > 0 ? Math.round(ftsHits / ilikeHits * 100) : (ftsHits === 0 ? 100 : 0);
+    const status = ftsHits === ilikeHits ? '✓' : (ftsHits < ilikeHits ? '✗ MISS' : '?');
+    console.log(`  "${q}" | FTS: ${ftsHits} | ILIKE: ${ilikeHits} | recall: ${ftsRecall}% | ${status}`);
+  }
+
+  // 4. FTS 對 RRF 的貢獻度
+  console.log('\n--- 4. FTS 在 hybrid search 中的貢獻度 ---');
+  // 跑幾個查詢，看 FTS 跟 vector 的 session 重疊率
+  const overlapQueries = ['afterburn', 'bootstrap', '記憶', 'recall', 'entity'];
+  for (const q of overlapQueries) {
+    const ftsResult = await pool.query(`
+      SELECT ss.session_id
+      FROM ${qi(SCHEMA)}.session_summaries ss
+      JOIN ${qi(SCHEMA)}.sessions s ON s.id = ss.session_row_id
+      WHERE ss.search_tsv @@ plainto_tsquery('simple', $1)
+        AND s.processing_status = 'succeeded'
+      ORDER BY ts_rank(ss.search_tsv, plainto_tsquery('simple', $1)) DESC
+      LIMIT 10
+    `, [q]);
+    const ftsIds = new Set(ftsResult.rows.map(r => r.session_id));
+
+    // vector search (if embedding available)
+    const embResult = await pool.query(`
+      SELECT ss.session_id
+      FROM ${qi(SCHEMA)}.session_summaries ss
+      JOIN ${qi(SCHEMA)}.sessions s ON s.id = ss.session_row_id
+      WHERE ss.embedding IS NOT NULL
+        AND s.processing_status = 'succeeded'
+      ORDER BY ss.embedding <=> (
+        SELECT ss2.embedding FROM ${qi(SCHEMA)}.session_summaries ss2
+        WHERE ss2.search_tsv @@ plainto_tsquery('simple', $1)
+        ORDER BY ts_rank(ss2.search_tsv, plainto_tsquery('simple', $1)) DESC
+        LIMIT 1
+      )
+      LIMIT 10
+    `, [q]);
+    const embIds = new Set(embResult.rows.map(r => r.session_id));
+
+    const overlap = [...ftsIds].filter(id => embIds.has(id)).length;
+    const ftsOnly = [...ftsIds].filter(id => !embIds.has(id)).length;
+    const embOnly = [...embIds].filter(id => !ftsIds.has(id)).length;
+
+    console.log(`  "${q}" | FTS top10: ${ftsIds.size} | Vec top10: ${embIds.size} | overlap: ${overlap} | FTS-only: ${ftsOnly} | Vec-only: ${embOnly}`);
+  }
+
+  await pool.end();
+  console.log('\n=== 完成 ===');
+}
+
+run().catch(err => { console.error(err); process.exit(1); });