@chainlesschain/personal-data-hub 0.2.4 → 0.3.1

package/lib/migrations.js

@@ -245,6 +245,90 @@ const PHASE_8_DDL = [
   `CREATE INDEX IF NOT EXISTS idx_review_queue_pending ON review_queue(reviewed_at, enqueued_at)`,
 ];
 
+// Phase 16 DDL — FTS5 full-text index over events for the Vault Browser UI.
+//
+// Uses FTS5 in **external-content** mode (content='events', content_rowid='rowid')
+// so the events table remains the single source of truth — events_fts only
+// stores the inverted index, kept in sync by 3 triggers.
+//
+// Tokenizer is `trigram` (SQLite ≥3.34) which gives substring matching for
+// CJK (中文) — unicode61 (the FTS5 default) splits only on whitespace and
+// is unusable for Chinese. Trigram min query length is 3 chars (1-2 char
+// queries match nothing); the UI surfaces a hint.
+//
+// If FTS5 or the trigram tokenizer is unavailable in the local SQLite build
+// (rare with bs3mc which bundles SQLCipher 4 / SQLite 3.42+, but possible
+// in custom builds), the probe sets `_meta.fts_mode = 'like'` and the
+// migration skips the virtual table + triggers. vault.searchEvents falls
+// back to LIKE-mode (slower, no ranking, ASCII-only correct).
+//
+// Backfill runs inline inside the migration transaction; on multi-100k-row
+// vaults this can take ~5-15 seconds (one-time cost on upgrade).
+function _hasFts5Trigram(db) {
+  // Probe by trying to create a temp virtual table. We can't rely on
+  // pragma_compile_options because (a) some bs3mc builds don't surface
+  // ENABLE_FTS5 there, and (b) trigram is a tokenizer registration, not
+  // a compile option. The CREATE is the ground truth.
+  try {
+    db.exec(
+      "CREATE VIRTUAL TABLE temp._fts_probe USING fts5(x, tokenize='trigram')"
+    );
+    db.exec("DROP TABLE temp._fts_probe");
+    return true;
+  } catch (_err) {
+    return false;
+  }
+}
+
+const PHASE_16_FTS_DDL = {
+  // External-content FTS5: no row data copy, just the inverted index.
+  // Columns mirror what's worth searching in events:
+  //   content_text  — the JSON payload, searched as-is (JSON braces tokenize
+  //                   harmlessly with trigram; query "{key" never matches
+  //                   anything users type)
+  //   extra_text    — extra JSON tail (also worth searching: order numbers,
+  //                   merchant names, message bodies live here for many
+  //                   adapters)
+  //   subtype/actor/place — short flat strings, indexed for keyword filter
+  createVirtualTable: `
+    CREATE VIRTUAL TABLE events_fts USING fts5(
+      subtype, content_text, actor, place, extra_text,
+      content='events',
+      content_rowid='rowid',
+      tokenize='trigram'
+    )
+  `,
+  // After INSERT — index the new row.
+  triggerAi: `
+    CREATE TRIGGER events_ai AFTER INSERT ON events BEGIN
+      INSERT INTO events_fts(rowid, subtype, content_text, actor, place, extra_text)
+      VALUES (new.rowid, new.subtype, new.content, new.actor, new.place, new.extra);
+    END
+  `,
+  // After DELETE — remove from index. external-content delete uses the
+  // sentinel ('delete', rowid, ...all-cols) pattern.
+  triggerAd: `
+    CREATE TRIGGER events_ad AFTER DELETE ON events BEGIN
+      INSERT INTO events_fts(events_fts, rowid, subtype, content_text, actor, place, extra_text)
+      VALUES('delete', old.rowid, old.subtype, old.content, old.actor, old.place, old.extra);
+    END
+  `,
+  // After UPDATE — delete-then-insert (FTS5 external-content idiom).
+  triggerAu: `
+    CREATE TRIGGER events_au AFTER UPDATE ON events BEGIN
+      INSERT INTO events_fts(events_fts, rowid, subtype, content_text, actor, place, extra_text)
+      VALUES('delete', old.rowid, old.subtype, old.content, old.actor, old.place, old.extra);
+      INSERT INTO events_fts(rowid, subtype, content_text, actor, place, extra_text)
+      VALUES (new.rowid, new.subtype, new.content, new.actor, new.place, new.extra);
+    END
+  `,
+  // One-shot backfill of all existing rows.
+  backfill: `
+    INSERT INTO events_fts(rowid, subtype, content_text, actor, place, extra_text)
+    SELECT rowid, subtype, content, actor, place, extra FROM events
+  `,
+};
+
 const MIGRATIONS = [
   {
     version: 1,
@@ -260,6 +344,52 @@ const MIGRATIONS = [
       for (const sql of PHASE_8_DDL) db.exec(sql);
     },
   },
+  {
+    version: 3,
+    description:
+      "Phase 16 Vault Browser — events_fts FTS5 (trigram) virtual table + 3 sync triggers + backfill; LIKE fallback when FTS5 unavailable",
+    up(db) {
+      const supported = _hasFts5Trigram(db);
+      // Record the mode in _meta so the runtime can pick the right query path
+      // without re-probing every open. Set BEFORE creating tables so partial
+      // failures still leave a queryable mode marker.
+      const now = Date.now();
+      db.prepare(
+        `INSERT INTO _meta (key, value, updated_at) VALUES ('fts_mode', ?, ?)
+         ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at`
+      ).run(supported ? "fts5" : "like", now);
+      if (!supported) return;
+
+      db.exec(PHASE_16_FTS_DDL.createVirtualTable);
+      db.exec(PHASE_16_FTS_DDL.triggerAi);
+      db.exec(PHASE_16_FTS_DDL.triggerAd);
+      db.exec(PHASE_16_FTS_DDL.triggerAu);
+      db.exec(PHASE_16_FTS_DDL.backfill);
+    },
+  },
+  {
+    version: 4,
+    description:
+      "Recreate uniq_{events,persons,places,items}_source as partial indices " +
+      "(WHERE source_original_id IS NOT NULL) to match putEvent/putPerson/etc " +
+      "UPSERT ON CONFLICT WHERE clauses. Older vaults (pre commit 44c4188a8) " +
+      "may have full unique indices created via CREATE UNIQUE INDEX IF NOT " +
+      "EXISTS without the WHERE clause; the IF-NOT-EXISTS hides the schema " +
+      "drift forever, manifesting only as runtime SQLite error '2nd ON CONFLICT " +
+      "clause does not match any PRIMARY KEY or UNIQUE constraint'. Symptom on " +
+      "Android: adapter.sync fails silently, vault events table stays at 1 row " +
+      "while raw_events accumulates 1000+. Idempotent on already-correct vaults.",
+    up(db) {
+      const tables = ["events", "persons", "places", "items"];
+      for (const t of tables) {
+        db.exec(`DROP INDEX IF EXISTS uniq_${t}_source`);
+        db.exec(
+          `CREATE UNIQUE INDEX uniq_${t}_source ON ${t}(source_adapter, source_original_id) ` +
+            `WHERE source_original_id IS NOT NULL`
+        );
+      }
+    },
+  },
 ];
 
 const TARGET_VERSION = MIGRATIONS[MIGRATIONS.length - 1].version;
@@ -311,9 +441,25 @@ function getSchemaVersion(db) {
   }
 }
 
+/**
+ * Returns 'fts5' or 'like' depending on what migration 3 recorded.
+ * Pre-migration-3 vaults return 'like' as the safe default.
+ */
+function getFtsMode(db) {
+  try {
+    const row = db.prepare("SELECT value FROM _meta WHERE key = 'fts_mode'").get();
+    return row && (row.value === "fts5" || row.value === "like") ? row.value : "like";
+  } catch (_err) {
+    return "like";
+  }
+}
+
 module.exports = {
   MIGRATIONS,
   TARGET_VERSION,
   applyMigrations,
   getSchemaVersion,
+  getFtsMode,
+  // Exported for tests + driver capability checks at vault open time.
+  _hasFts5Trigram,
 };

package/lib/query-parser.js

@@ -219,6 +219,78 @@ function parseIntent(text) {
   return "list";
 }
 
+// ─── Entity-name extraction (FTS5 fulltext routing) ────────────────────
+//
+// Pull a probable entity-name candidate out of the raw question so
+// `_gatherFacts` can augment intent=list results with `vault.searchEvents`
+// (FTS5 + trigram CJK substring; LIKE fallback). Heuristic: strip every
+// known stop-pattern (time / intent / subtype / adapter / list-trigger /
+// pronoun / punct / digit) and pick the longest 2-10 char chunk that
+// remains.
+//
+// Wrong extractions are SAFE: the engine treats this as an OPTIONAL
+// augmentation — extracted-but-irrelevant terms just return 0 FTS rows
+// (wasted budget, not lost facts). Single-character Chinese names like
+// "妈" / "爸" are deliberately NOT picked up because single-char tokens
+// false-positive heavily on residual verbs (说/看/买). That's a known
+// limitation; first-pass acceptable.
+//
+// Stop-pattern order matters: multi-char compounds must run BEFORE
+// shorter alternatives so "多少钱" doesn't decay to "多少" + leftover "钱".
+
+const ENTITY_STOP_PATTERNS = [
+  // Compounds — multi-char specific tokens first
+  /(多少钱|多少次|多少个|多少家|多少人|多少张|多少部|加起来|共多少|总共)/g,
+  /(几个|几次|几条|几单)/g,
+  /(how\s+many|count\s+of)/gi,
+  // Time
+  /(今天|昨天|前天|明天|本周|这周|上周|这个礼拜|上个礼拜|这一周|上一周|本月|这月|上月|这个月|上个月|上一月|今年|去年|最近|最新)/g,
+  /\d+\s*[天周月年个]/g,
+  /\d{4}\s*年\s*\d{1,2}\s*月/g,
+  /(today|yesterday|past|recent|latest)/gi,
+  // Intent (remaining shorter forms after compounds)
+  /(多少|合计)/g,
+  /(sum|total|count|amount)/gi,
+  // Subtype keywords — compound forms first
+  /(下了几单|下了多少单|去旅游)/g,
+  /(订单|下单|买了|购买|支付|付款|花了|花费|消费|开销|金额|转账|转给|转钱|收入|工资|进账|收到|聊天|消息|聊了|对话|朋友圈|动态|去过|到过|去了|来到|出差|旅行|浏览|看了|阅读|发了)/g,
+  /(order|payment|transfer|income|message|chat|moment|post|visited|trip|browse|read|spent|spend)/gi,
+  // Adapter keywords — compound forms first
+  /(大众点评|百度地图|火车票)/g,
+  /(支付宝|微信|邮箱|邮件|淘宝|天猫|京东|拼多多|美团|高德|高铁|携程)/g,
+  /(alipay|wechat|email|imap|taobao|tmall|jingdong|jd|pdd|meituan|dianping|baidu\s*map|12306|ctrip)/gi,
+  /(deepseek|kimi|通义|智谱|混元|千帆|扣子|chatgpt|claude)/gi,
+  // List / search trigger
+  /(查一下|找一找|帮我|给我|看下|看看|看一下)/g,
+  /(列出|列表|查询|查找|查看|提到|发现)/g,
+  /(list|show|find|search)/gi,
+  // Pronouns / particles / prepositions (multi-char first, then single-char)
+  /(我们|你们|他们|什么|哪个|哪些|怎么|为什么|是否)/g,
+  /[的了吗啊呢在给到与和跟对从向是有我你他她它这那哪谁啥嘛]/g,
+  // Punctuation + whitespace
+  /[\s!?.,;:'"()，。！？；：、《》「」『』【】]+/g,
+  // Numbers
+  /\d+/g,
+];
+
+/**
+ * Extract a probable entity-name candidate from raw question text.
+ *
+ * @param {string} text
+ * @returns {string|null}  longest remaining 2-10 char chunk, or null
+ */
+function extractEntityTerm(text) {
+  if (typeof text !== "string" || text.length === 0) return null;
+  let s = text;
+  for (const re of ENTITY_STOP_PATTERNS) {
+    s = s.replace(re, " ");
+  }
+  const candidates = s.split(/\s+/).filter((t) => t.length >= 2 && t.length <= 10);
+  if (candidates.length === 0) return null;
+  candidates.sort((a, b) => b.length - a.length);
+  return candidates[0];
+}
+
 // ─── Full parser ─────────────────────────────────────────────────────────
 
 /**
@@ -250,7 +322,9 @@ module.exports = {
   parseTimeWindow,
   parseFilters,
   parseIntent,
+  extractEntityTerm,
   // exposed for tests
   SUBTYPE_KEYWORDS,
   ADAPTER_KEYWORDS,
+  ENTITY_STOP_PATTERNS,
 };

package/lib/registry.js

@@ -267,6 +267,168 @@ class AdapterRegistry {
     return this._finish(report, startedAt);
   }
 
+  /**
+   * 2026-05-24 — re-derive canonical events from raw_events archive.
+   * Use case: a past sync wrote to raw_events (putRawEvent succeeded) but
+   * putBatch failed silently (e.g. partial-index drift trap #25) → events
+   * table stuck at 0 for those entities while raw piled up. After fixing
+   * the underlying schema/code bug, call this to promote the orphan raws
+   * to canonical events without re-fetching from the source.
+   *
+   * Behaviour:
+   *   - Iterates raw_events filtered by [opts.adapter] (or all)
+   *   - For each row: lookup registered adapter → normalize(raw) →
+   *     partition valid/invalid → putBatch
+   *   - Skips raws whose adapter is not currently registered (logs
+   *     `adapter.rederive.adapter_missing` audit)
+   *   - On adapter.normalize() throw, increments invalidCount + audit
+   *
+   * Does NOT re-fetch from the source, does NOT update watermarks (raw
+   * archive timestamp is what it was), does NOT run KG/RAG sinks (those
+   * are sync-time concerns — call them via syncAll if needed).
+   *
+   * @param {object} [opts]
+   * @param {string} [opts.adapter]  Filter by adapter name; default = all
+   * @param {number} [opts.batchSize=100] Raws per partitionBatch+putBatch tx
+   * @returns {Promise<RederiveReport>}
+   *
+   * @typedef {object} RederiveReport
+   * @property {number} rawSeen        Total raw_events iterated
+   * @property {number} invalidCount   Normalize threw or partition rejected
+   * @property {number} adapterMissing Raws whose adapter not registered
+   * @property {object} entityCounts   { events, persons, places, items, topics }
+   * @property {number} durationMs
+   * @property {Array<{adapter,error,sample?}>} errors  Adapter-level errors
+   */
+  async rederive(opts = {}) {
+    const startedAt = Date.now();
+    const report = {
+      rawSeen: 0,
+      invalidCount: 0,
+      adapterMissing: 0,
+      entityCounts: { events: 0, persons: 0, places: 0, items: 0, topics: 0 },
+      durationMs: 0,
+      errors: [],
+    };
+    const batchSize = Number.isInteger(opts.batchSize) && opts.batchSize > 0
+      ? opts.batchSize
+      : 100;
+
+    // Page through raw_events in batches to avoid loading the whole table
+    // for large vaults (1000+ rows is normal). Group raws by adapter so a
+    // single putBatch tx commits per-adapter (mirrors syncAdapter shape).
+    let offset = 0;
+    let totalProcessed = 0;
+    /** @type {Map<string, Array<object>>} */
+    const buffersByAdapter = new Map();
+
+    const flushAdapter = async (adapterName) => {
+      const buffer = buffersByAdapter.get(adapterName);
+      if (!buffer || buffer.length === 0) return;
+      const adapter = this._adapters.get(adapterName);
+      if (!adapter) {
+        report.adapterMissing += buffer.length;
+        try {
+          this.vault.audit("adapter.rederive.adapter_missing", adapterName, {
+            droppedCount: buffer.length,
+          });
+        } catch (_e) { /* audit failure is non-fatal */ }
+        buffersByAdapter.set(adapterName, []);
+        return;
+      }
+      // normalize + collect into one merged batch (per adapter buffer)
+      const merged = { events: [], persons: [], places: [], items: [], topics: [] };
+      for (const raw of buffer) {
+        let normalized;
+        try {
+          normalized = adapter.normalize(raw);
+        } catch (err) {
+          report.invalidCount += 1;
+          try {
+            this.vault.audit("adapter.rederive.normalize_failed", adapterName, {
+              originalId: raw.originalId,
+              error: toError(err, "normalize").message,
+            });
+          } catch (_e) { /* audit non-fatal */ }
+          continue;
+        }
+        if (!normalized || typeof normalized !== "object") continue;
+        for (const k of ["events", "persons", "places", "items", "topics"]) {
+          if (Array.isArray(normalized[k])) merged[k].push(...normalized[k]);
+        }
+      }
+      const { valid, invalidReasons } = partitionBatch(merged);
+      if (invalidReasons.length > 0) {
+        report.invalidCount += invalidReasons.length;
+        try {
+          this.vault.audit("adapter.rederive.invalid_entities", adapterName, {
+            count: invalidReasons.length,
+            sample: invalidReasons.slice(0, 5),
+          });
+        } catch (_e) { /* audit non-fatal */ }
+      }
+      try {
+        const counts = this.vault.putBatch(valid);
+        for (const k of Object.keys(counts)) {
+          report.entityCounts[k] = (report.entityCounts[k] || 0) + counts[k];
+        }
+      } catch (err) {
+        report.errors.push({
+          adapter: adapterName,
+          error: toError(err, "putBatch").message,
+          sample: buffer.slice(0, 3).map((r) => r.originalId),
+        });
+        try {
+          this.vault.audit("adapter.rederive.put_batch_failed", adapterName, {
+            error: toError(err, "putBatch").message,
+            droppedCount: buffer.length,
+          });
+        } catch (_e) { /* audit non-fatal */ }
+      }
+      buffersByAdapter.set(adapterName, []);
+    };
+
+    while (true) {
+      const page = this.vault.queryRawEvents({
+        adapter: opts.adapter,
+        limit: batchSize,
+        offset,
+      });
+      if (page.length === 0) break;
+      offset += page.length;
+      report.rawSeen += page.length;
+      // Group by adapter into buffers, flush whenever a buffer hits batchSize.
+      for (const raw of page) {
+        let buf = buffersByAdapter.get(raw.adapter);
+        if (!buf) {
+          buf = [];
+          buffersByAdapter.set(raw.adapter, buf);
+        }
+        buf.push(raw);
+        if (buf.length >= batchSize) {
+          await flushAdapter(raw.adapter);
+        }
+      }
+      totalProcessed += page.length;
+    }
+    // Final flush of remaining per-adapter buffers
+    for (const name of Array.from(buffersByAdapter.keys())) {
+      await flushAdapter(name);
+    }
+
+    report.durationMs = Date.now() - startedAt;
+    try {
+      this.vault.audit("adapter.rederive.summary", opts.adapter || "*", {
+        rawSeen: report.rawSeen,
+        invalidCount: report.invalidCount,
+        adapterMissing: report.adapterMissing,
+        entityCounts: report.entityCounts,
+        durationMs: report.durationMs,
+      });
+    } catch (_e) { /* audit non-fatal */ }
+    return report;
+  }
+
   /**
    * Sync every registered adapter sequentially.
    * Returns an array of SyncReports in registration order.