@shadowforge0/aquifer-memory 1.0.1 → 1.0.3

package/core/aquifer.js

@@ -100,19 +100,6 @@ function createAquifer(config) {
   const entityPromptFn = config.entities && config.entities.prompt ? config.entities.prompt : null;
   const entityScope = (config.entities && config.entities.scope) || 'default';
 
-  // FTS config — locked to 'simple'.
-  // The search_tsv trigger always uses to_tsvector('simple', ...), so query-time
-  // config must match.  Warn and override if someone passes anything else.
-  const _rawFtsConfig = config.ftsConfig || 'simple';
-  if (_rawFtsConfig !== 'simple') {
-    console.warn(
-      `[aquifer] ftsConfig '${_rawFtsConfig}' is not currently supported. ` +
-      `The search_tsv index is built with 'simple'; only 'simple' is valid at query time. ` +
-      `Overriding to 'simple'.`
-    );
-  }
-  const ftsConfig = 'simple';
-
   // Rank weights
   const rankWeights = {
     rrf: 0.65,
@@ -200,21 +187,30 @@ function createAquifer(config) {
     // --- lifecycle ---
 
     async migrate() {
-      // 1. Run base DDL
-      const baseSql = loadSql('001-base.sql', schema);
-      await pool.query(baseSql);
-
-      // 2. If entities enabled, run entity DDL
-      if (entitiesEnabled) {
-        const entitySql = loadSql('002-entities.sql', schema);
-        await pool.query(entitySql);
-      }
+      // Advisory lock prevents concurrent migrations across processes.
+      // Lock key is derived from schema name to allow parallel migration
+      // of different schemas in the same database.
+      const lockKey = Buffer.from(`aquifer:${schema}`).reduce((h, b) => (h * 31 + b) & 0x7fffffff, 0);
+      await pool.query('SELECT pg_advisory_lock($1)', [lockKey]);
+      try {
+        // 1. Run base DDL
+        const baseSql = loadSql('001-base.sql', schema);
+        await pool.query(baseSql);
+
+        // 2. If entities enabled, run entity DDL
+        if (entitiesEnabled) {
+          const entitySql = loadSql('002-entities.sql', schema);
+          await pool.query(entitySql);
+        }
 
-      // 3. Trust + feedback (always, not gated by entities)
-      const trustSql = loadSql('003-trust-feedback.sql', schema);
-      await pool.query(trustSql);
+        // 3. Trust + feedback (always, not gated by entities)
+        const trustSql = loadSql('003-trust-feedback.sql', schema);
+        await pool.query(trustSql);
 
-      migrated = true;
+        migrated = true;
+      } finally {
+        await pool.query('SELECT pg_advisory_unlock($1)', [lockKey]).catch(() => {});
+      }
     },
 
     async close() {
@@ -706,7 +702,7 @@ function createAquifer(config) {
       const [ftsRows, embRows, turnResult] = await Promise.all([
         runFts
           ? storage.searchSessions(pool, query, {
-              schema, tenantId, agentIds: resolvedAgentIds, source, dateFrom, dateTo, limit: fetchLimit, ftsConfig,
+              schema, tenantId, agentIds: resolvedAgentIds, source, dateFrom, dateTo, limit: fetchLimit,
             }).catch((err) => {
               recordSearchError('fts', err);
               return [];

package/core/entity.js

@@ -222,27 +222,41 @@ async function upsertEntityRelations(pool, {
 }) {
   if (!pairs || pairs.length === 0) return { upserted: 0 };
   const ts = occurredAt || new Date().toISOString();
-  let upserted = 0;
 
+  // Filter and normalize pairs
+  const validPairs = [];
   for (const { srcEntityId, dstEntityId } of pairs) {
     if (!srcEntityId || !dstEntityId || srcEntityId === dstEntityId) continue;
+    validPairs.push({
+      lo: Math.min(srcEntityId, dstEntityId),
+      hi: Math.max(srcEntityId, dstEntityId),
+    });
+  }
 
-    const lo = Math.min(srcEntityId, dstEntityId);
-    const hi = Math.max(srcEntityId, dstEntityId);
-
-    await pool.query(
-      `INSERT INTO ${qi(schema)}.entity_relations
-        (src_entity_id, dst_entity_id, co_occurrence_count, first_seen_at, last_seen_at)
-      VALUES ($1, $2, 1, $3, $3)
-      ON CONFLICT (src_entity_id, dst_entity_id) DO UPDATE SET
-        co_occurrence_count = ${qi(schema)}.entity_relations.co_occurrence_count + 1,
-        last_seen_at        = GREATEST(${qi(schema)}.entity_relations.last_seen_at, EXCLUDED.last_seen_at)`,
-      [lo, hi, ts]
-    );
-    upserted++;
+  if (validPairs.length === 0) return { upserted: 0 };
+
+  // Batch insert: multi-row VALUES
+  const COLS_PER_ROW = 3;
+  const valueClauses = [];
+  const params = [];
+
+  for (const { lo, hi } of validPairs) {
+    const off = params.length;
+    params.push(lo, hi, ts);
+    valueClauses.push(`($${off+1}, $${off+2}, 1, $${off+3}, $${off+3})`);
   }
 
-  return { upserted };
+  await pool.query(
+    `INSERT INTO ${qi(schema)}.entity_relations
+      (src_entity_id, dst_entity_id, co_occurrence_count, first_seen_at, last_seen_at)
+    VALUES ${valueClauses.join(',\n')}
+    ON CONFLICT (src_entity_id, dst_entity_id) DO UPDATE SET
+      co_occurrence_count = ${qi(schema)}.entity_relations.co_occurrence_count + 1,
+      last_seen_at        = GREATEST(${qi(schema)}.entity_relations.last_seen_at, EXCLUDED.last_seen_at)`,
+    params
+  );
+
+  return { upserted: validPairs.length };
 }
 
 // ---------------------------------------------------------------------------

package/core/storage.js

@@ -211,7 +211,7 @@ async function getMessages(pool, sessionId, agentId, { schema, tenantId } = {})
 }
 
 // ---------------------------------------------------------------------------
-// searchSessions (FTS)
+// searchSessions (trigram + FTS fallback)
 // ---------------------------------------------------------------------------
 
 async function searchSessions(pool, query, {
@@ -220,34 +220,27 @@ async function searchSessions(pool, query, {
   agentId,
   agentIds: rawAgentIds,
   source,
-  dateFrom,  // m1: add date filtering
+  dateFrom,
   dateTo,
   limit = 20,
-  ftsConfig = 'simple',
 } = {}) {
   const clampedLimit = Math.max(1, Math.min(100, limit));
-  // FTS config is locked to 'simple' — the search_tsv trigger always uses
-  // to_tsvector('simple', ...) so query semantics must match.  Warn callers
-  // that pass a different value rather than silently honouring it.
-  if (ftsConfig !== 'simple') {
-    console.warn(
-      `[aquifer/storage] searchSessions: ftsConfig '${ftsConfig}' ignored. ` +
-      `Only 'simple' is supported (index is built with simple tokenizer). ` +
-      `Using 'simple'.`
-    );
-  }
-  const safeFts = 'simple';
 
   // Normalize agentId/agentIds
   const agentIds = rawAgentIds && rawAgentIds.length > 0
     ? rawAgentIds
     : (agentId ? [agentId] : null);
 
+  // Escape LIKE special characters in query
+  const likeQuery = query.replace(/[%_\\]/g, '\\$&');
+
+  // Primary: trigram ILIKE on search_text (works for CJK + Latin)
+  // Fallback: tsvector FTS (for installations without search_text populated)
   const where = [
-    `ss.search_tsv @@ plainto_tsquery('${safeFts}', $1)`,
-    `s.tenant_id = $2`,
+    `(ss.search_text ILIKE '%' || $1 || '%' OR ss.search_tsv @@ plainto_tsquery('simple', $2))`,
+    `s.tenant_id = $3`,
   ];
-  const params = [query, tenantId];
+  const params = [likeQuery, query, tenantId];
 
   if (agentIds) {
     params.push(agentIds);
@@ -281,8 +274,10 @@ async function searchSessions(pool, query, {
       ss.access_count,
       ss.last_accessed_at,
       ss.trust_score,
-      ts_headline('${safeFts}', COALESCE(ss.summary_text, ''), plainto_tsquery('${safeFts}', $1)) AS summary_snippet,
-      ts_rank(ss.search_tsv, plainto_tsquery('${safeFts}', $1)) AS fts_rank
+      CASE WHEN ss.search_text IS NOT NULL
+        THEN similarity(ss.search_text, $2)
+        ELSE ts_rank(ss.search_tsv, plainto_tsquery('simple', $2))
+      END AS fts_rank
     FROM ${qi(schema)}.sessions s
     LEFT JOIN ${qi(schema)}.session_summaries ss ON ss.session_row_id = s.id
     WHERE ${where.join(' AND ')}
@@ -365,32 +360,45 @@ async function upsertTurnEmbeddings(pool, sessionRowId, {
     throw new Error(`turns.length (${turns.length}) !== vectors.length (${vectors.length})`);
   }
 
+  // Batch insert: build multi-row VALUES clause
+  const COLS_PER_ROW = 10;
+  const valueClauses = [];
+  const params = [];
+
   for (let i = 0; i < turns.length; i++) {
     const t = turns[i];
     const vec = vectors[i];
     if (!vec) continue;
 
     const contentHash = crypto.createHash('sha256').update(t.text).digest('hex').slice(0, 16);
-    await pool.query(
-      `INSERT INTO ${qi(schema)}.turn_embeddings
-        (session_row_id, tenant_id, session_id, agent_id, source,
-         turn_index, message_index, role, content_text, content_hash, embedding)
-      VALUES ($1,$2,$3,$4,$5,$6,$7,'user',$8,$9,$10::vector)
-      ON CONFLICT (session_row_id, message_index) DO UPDATE SET
-        content_text = EXCLUDED.content_text,
-        content_hash = EXCLUDED.content_hash,
-        embedding = CASE
-          WHEN ${qi(schema)}.turn_embeddings.content_hash = EXCLUDED.content_hash
-          THEN ${qi(schema)}.turn_embeddings.embedding
-          ELSE EXCLUDED.embedding
-        END`,
-      [
-        sessionRowId, tenantId, sessionId, agentId, source || null,
-        t.turnIndex, t.messageIndex,
-        t.text, contentHash, vecToStr(vec),
-      ]
+    const off = params.length;
+    params.push(
+      sessionRowId, tenantId, sessionId, agentId, source || null,
+      t.turnIndex, t.messageIndex,
+      t.text, contentHash, vecToStr(vec),
+    );
+    valueClauses.push(
+      `($${off+1},$${off+2},$${off+3},$${off+4},$${off+5},$${off+6},$${off+7},'user',$${off+8},$${off+9},$${off+10}::vector)`
     );
   }
+
+  if (valueClauses.length === 0) return;
+
+  await pool.query(
+    `INSERT INTO ${qi(schema)}.turn_embeddings
+      (session_row_id, tenant_id, session_id, agent_id, source,
+       turn_index, message_index, role, content_text, content_hash, embedding)
+    VALUES ${valueClauses.join(',\n')}
+    ON CONFLICT (session_row_id, message_index) DO UPDATE SET
+      content_text = EXCLUDED.content_text,
+      content_hash = EXCLUDED.content_hash,
+      embedding = CASE
+        WHEN ${qi(schema)}.turn_embeddings.content_hash = EXCLUDED.content_hash
+        THEN ${qi(schema)}.turn_embeddings.embedding
+        ELSE EXCLUDED.embedding
+      END`,
+    params
+  );
 }
 
 // ---------------------------------------------------------------------------

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@shadowforge0/aquifer-memory",
-  "version": "1.0.1",
+  "version": "1.0.3",
   "description": "PG-native long-term memory for AI agents. Turn-level embedding, hybrid RRF ranking, optional knowledge graph. MCP server, CLI, and library API.",
   "main": "index.js",
   "files": [

package/schema/001-base.sql

@@ -2,6 +2,7 @@
 -- Usage: replace ${schema} with actual schema name (e.g., 'aquifer')
 
 CREATE EXTENSION IF NOT EXISTS vector;
+CREATE EXTENSION IF NOT EXISTS pg_trgm;
 CREATE SCHEMA IF NOT EXISTS ${schema};
 
 -- =========================================================================
@@ -85,6 +86,7 @@ CREATE TABLE IF NOT EXISTS ${schema}.session_summaries (
   structured_summary       JSONB        NOT NULL DEFAULT '{}',
   embedding                vector,
   search_tsv               TSVECTOR,
+  search_text              TEXT,
   access_count             INT          NOT NULL DEFAULT 0,
   last_accessed_at         TIMESTAMPTZ,
   updated_at               TIMESTAMPTZ  NOT NULL DEFAULT now()
@@ -96,6 +98,9 @@ CREATE INDEX IF NOT EXISTS idx_summaries_tenant
 CREATE INDEX IF NOT EXISTS idx_summaries_search_tsv
   ON ${schema}.session_summaries USING GIN (search_tsv);
 
+CREATE INDEX IF NOT EXISTS idx_summaries_search_text_trgm
+  ON ${schema}.session_summaries USING GIN (search_text gin_trgm_ops);
+
 CREATE INDEX IF NOT EXISTS idx_summaries_embedding
   ON ${schema}.session_summaries (session_row_id)
   WHERE embedding IS NOT NULL;
@@ -141,6 +146,11 @@ BEGIN
     setweight(to_tsvector('simple', COALESCE(NEW.summary_text, '')), 'C') ||
     setweight(to_tsvector('simple', open_loops_text || ' ' || facts_text), 'D');
 
+  NEW.search_text :=
+    title_text || ' ' || overview_text || ' ' || topics_text || ' ' ||
+    decisions_text || ' ' || COALESCE(NEW.summary_text, '') || ' ' ||
+    open_loops_text || ' ' || facts_text;
+
   RETURN NEW;
 END;
 $$;
@@ -149,7 +159,7 @@ DROP TRIGGER IF EXISTS trg_session_summaries_search_tsv
   ON ${schema}.session_summaries;
 
 CREATE TRIGGER trg_session_summaries_search_tsv
-  BEFORE INSERT OR UPDATE OF summary_text, structured_summary
+  BEFORE INSERT OR UPDATE OF summary_text, structured_summary, search_text
   ON ${schema}.session_summaries
   FOR EACH ROW
   EXECUTE FUNCTION ${schema}.session_summaries_search_tsv_update();

package/scripts/diagnose-fts-zh.js

@@ -0,0 +1,161 @@
+'use strict';
+
+/**
+ * FTS 中文診斷：檢查 'simple' tokenizer 在實際中文資料上的表現
+ *
+ * 測試項目：
+ * 1. FTS tokenization — 實際 token 長什麼樣
+ * 2. FTS recall — 常見中文查詢的命中率
+ * 3. FTS vs vector — FTS 有沒有在幫忙還是在拖後腿
+ */
+
+const { Pool } = require('pg');
+
+const DB_URL = process.env.DATABASE_URL || 'postgresql://burk:790476@localhost:5432/openclaw_db';
+const SCHEMA = process.env.AQUIFER_SCHEMA || 'miranda';
+
+const pool = new Pool({ connectionString: DB_URL });
+
+async function run() {
+  const qi = (s) => `"${s}"`;
+
+  console.log('=== FTS 中文診斷 ===\n');
+
+  // 1. 看 token 分佈
+  console.log('--- 1. Token 分析 ---');
+  const tokenSample = await pool.query(`
+    SELECT ss.session_id,
+           array_length(tsvector_to_array(ss.search_tsv), 1) as token_count,
+           left(ss.summary_text, 80) as preview
+    FROM ${qi(SCHEMA)}.session_summaries ss
+    WHERE ss.search_tsv IS NOT NULL
+    ORDER BY ss.updated_at DESC
+    LIMIT 10
+  `);
+
+  let totalTokens = 0;
+  let sessionCount = 0;
+  for (const r of tokenSample.rows) {
+    totalTokens += r.token_count || 0;
+    sessionCount++;
+    console.log(`  ${r.session_id?.slice(0, 8)} | ${r.token_count || 0} tokens | ${r.preview}`);
+  }
+  console.log(`  avg: ${sessionCount ? Math.round(totalTokens / sessionCount) : 0} tokens/session\n`);
+
+  // 2. 看一個 session 的實際 token
+  console.log('--- 2. Token 範例（最近 session）---');
+  const tokenDetail = await pool.query(`
+    SELECT ss.session_id,
+           array_to_string(tsvector_to_array(ss.search_tsv), ' | ') as tokens
+    FROM ${qi(SCHEMA)}.session_summaries ss
+    WHERE ss.search_tsv IS NOT NULL
+    ORDER BY ss.updated_at DESC
+    LIMIT 1
+  `);
+  if (tokenDetail.rows[0]) {
+    console.log(`  session: ${tokenDetail.rows[0].session_id?.slice(0, 8)}`);
+    const tokens = tokenDetail.rows[0].tokens || '';
+    // 分類 token
+    const all = tokens.split(' | ');
+    const cjk = all.filter(t => /[\u4e00-\u9fff]/.test(t));
+    const latin = all.filter(t => /^[a-z0-9]/.test(t));
+    const other = all.filter(t => !(/[\u4e00-\u9fff]/.test(t)) && !(/^[a-z0-9]/.test(t)));
+    console.log(`  total: ${all.length} | latin: ${latin.length} | cjk: ${cjk.length} | other: ${other.length}`);
+    console.log(`  CJK tokens (前 20): ${cjk.slice(0, 20).join(' | ')}`);
+    console.log(`  Latin tokens (前 20): ${latin.slice(0, 20).join(' | ')}\n`);
+  }
+
+  // 3. 中文查詢命中率測試
+  console.log('--- 3. 中文查詢 FTS 命中率 ---');
+  const testQueries = [
+    'afterburn',
+    'bootstrap',
+    'session',
+    'recall',
+    '記憶',
+    '修復',
+    '架構',
+    '時區',
+    '去重',
+    'daily entries',
+    'OpenCode',
+    'entity',
+    'Jenny',
+    'Aquifer',
+    '消化模式',
+  ];
+
+  // 總 session 數
+  const totalResult = await pool.query(`
+    SELECT COUNT(*) as cnt FROM ${qi(SCHEMA)}.session_summaries WHERE search_tsv IS NOT NULL
+  `);
+  const totalSessions = parseInt(totalResult.rows[0].cnt);
+  console.log(`  total sessions with FTS index: ${totalSessions}\n`);
+
+  for (const q of testQueries) {
+    const ftsResult = await pool.query(`
+      SELECT COUNT(*) as cnt
+      FROM ${qi(SCHEMA)}.session_summaries ss
+      WHERE ss.search_tsv @@ plainto_tsquery('simple', $1)
+    `, [q]);
+    const ftsHits = parseInt(ftsResult.rows[0].cnt);
+
+    // 同時看 summary_text ILIKE 能找到幾筆（ground truth）
+    const ilikeResult = await pool.query(`
+      SELECT COUNT(*) as cnt
+      FROM ${qi(SCHEMA)}.session_summaries ss
+      WHERE ss.summary_text ILIKE $1
+         OR ss.structured_summary::text ILIKE $1
+    `, [`%${q}%`]);
+    const ilikeHits = parseInt(ilikeResult.rows[0].cnt);
+
+    const ftsRecall = ilikeHits > 0 ? Math.round(ftsHits / ilikeHits * 100) : (ftsHits === 0 ? 100 : 0);
+    const status = ftsHits === ilikeHits ? '✓' : (ftsHits < ilikeHits ? '✗ MISS' : '?');
+    console.log(`  "${q}" | FTS: ${ftsHits} | ILIKE: ${ilikeHits} | recall: ${ftsRecall}% | ${status}`);
+  }
+
+  // 4. FTS 對 RRF 的貢獻度
+  console.log('\n--- 4. FTS 在 hybrid search 中的貢獻度 ---');
+  // 跑幾個查詢，看 FTS 跟 vector 的 session 重疊率
+  const overlapQueries = ['afterburn', 'bootstrap', '記憶', 'recall', 'entity'];
+  for (const q of overlapQueries) {
+    const ftsResult = await pool.query(`
+      SELECT ss.session_id
+      FROM ${qi(SCHEMA)}.session_summaries ss
+      JOIN ${qi(SCHEMA)}.sessions s ON s.id = ss.session_row_id
+      WHERE ss.search_tsv @@ plainto_tsquery('simple', $1)
+        AND s.processing_status = 'succeeded'
+      ORDER BY ts_rank(ss.search_tsv, plainto_tsquery('simple', $1)) DESC
+      LIMIT 10
+    `, [q]);
+    const ftsIds = new Set(ftsResult.rows.map(r => r.session_id));
+
+    // vector search (if embedding available)
+    const embResult = await pool.query(`
+      SELECT ss.session_id
+      FROM ${qi(SCHEMA)}.session_summaries ss
+      JOIN ${qi(SCHEMA)}.sessions s ON s.id = ss.session_row_id
+      WHERE ss.embedding IS NOT NULL
+        AND s.processing_status = 'succeeded'
+      ORDER BY ss.embedding <=> (
+        SELECT ss2.embedding FROM ${qi(SCHEMA)}.session_summaries ss2
+        WHERE ss2.search_tsv @@ plainto_tsquery('simple', $1)
+        ORDER BY ts_rank(ss2.search_tsv, plainto_tsquery('simple', $1)) DESC
+        LIMIT 1
+      )
+      LIMIT 10
+    `, [q]);
+    const embIds = new Set(embResult.rows.map(r => r.session_id));
+
+    const overlap = [...ftsIds].filter(id => embIds.has(id)).length;
+    const ftsOnly = [...ftsIds].filter(id => !embIds.has(id)).length;
+    const embOnly = [...embIds].filter(id => !ftsIds.has(id)).length;
+
+    console.log(`  "${q}" | FTS top10: ${ftsIds.size} | Vec top10: ${embIds.size} | overlap: ${overlap} | FTS-only: ${ftsOnly} | Vec-only: ${embOnly}`);
+  }
+
+  await pool.end();
+  console.log('\n=== 完成 ===');
+}
+
+run().catch(err => { console.error(err); process.exit(1); });