@chainlesschain/personal-data-hub 0.2.4 → 0.3.0

package/lib/vault.js

@@ -27,8 +27,79 @@ const fs = require("node:fs");
 const path = require("node:path");
 
 const { validate } = require("./schemas");
-const { applyMigrations, getSchemaVersion } = require("./migrations");
+const { applyMigrations, getSchemaVersion, getFtsMode } = require("./migrations");
 const { isValidKeyHex } = require("./key-providers");
+const { getCategory, PREFIX_RULES } = require("./categories");
+
+// FTS5 trigram tokenizer requires queries of >= 3 chars to produce any
+// trigrams at all (single 2-char input gives zero index keys → empty result).
+// Surface this to the caller so the UI can show a hint instead of a confusing
+// "no results" state.
+const FTS5_MIN_QUERY_LEN = 3;
+
+/**
+ * Translate a user-typed FTS5 query into a safe-to-bind string. FTS5 has its
+ * own query syntax — bare operators like `OR`, `AND`, `NOT`, `(`, `:`, `*`,
+ * double-quotes have meaning. For the browser keyword box we want literal
+ * substring search, so wrap the whole input in double quotes (FTS5 phrase
+ * mode) after escaping any embedded double quotes.
+ */
+function _quoteFtsQuery(q) {
+  return '"' + String(q).replace(/"/g, '""') + '"';
+}
+
+/**
+ * Build a (sql, params) fragment matching events for the given category.
+ * Uses categories.js PREFIX_RULES as the single source of truth so a new
+ * adapter prefix only needs to be added in one place.
+ *
+ * Returns `{ sql: "(...)", params: { catN: ... } }` or
+ * `{ sql: "0=1", params: {} }` when no rule matches (unknown category).
+ */
+function _categoryToWhere(category, paramPrefix = "cat") {
+  if (typeof category !== "string" || category.length === 0) {
+    return { sql: null, params: {} };
+  }
+  const matched = PREFIX_RULES.filter((rule) => rule[1] === category);
+  // "other" is a synthetic bucket — nothing in PREFIX_RULES maps to it; an
+  // event's category is "other" iff its adapter matches no prefix. Translate
+  // that to a NOT-IN-any-prefix condition.
+  if (category === "other") {
+    const exclude = [];
+    const params = {};
+    let i = 0;
+    for (const [rule] of PREFIX_RULES) {
+      const key = `${paramPrefix}${i}`;
+      if (rule.endsWith("*")) {
+        params[key] = rule.slice(0, -1) + "%";
+        exclude.push(`source_adapter NOT LIKE @${key}`);
+      } else {
+        params[key] = rule;
+        exclude.push(`source_adapter != @${key}`);
+      }
+      i++;
+    }
+    return { sql: "(" + exclude.join(" AND ") + ")", params };
+  }
+  if (matched.length === 0) {
+    return { sql: "0=1", params: {} };
+  }
+  const conds = [];
+  const params = {};
+  let i = 0;
+  for (const [rule] of matched) {
+    const key = `${paramPrefix}${i}`;
+    if (rule.endsWith("*")) {
+      params[key] = rule.slice(0, -1) + "%";
+      conds.push(`source_adapter LIKE @${key}`);
+    } else {
+      params[key] = rule;
+      conds.push(`source_adapter = @${key}`);
+    }
+    i++;
+  }
+  return { sql: "(" + conds.join(" OR ") + ")", params };
+}
 
 // Default SQLCipher cipher (matches WCDB / mainstream SQLCipher v4).
 const DEFAULT_CIPHER = "sqlcipher";
@@ -277,6 +348,22 @@ class LocalVault {
           source = excluded.source,
           extra = excluded.extra,
           ingested_at = excluded.ingested_at,
+          confidence = excluded.confidence
+        ON CONFLICT(source_adapter, source_original_id)
+          WHERE source_original_id IS NOT NULL
+          DO UPDATE SET
+          subtype = excluded.subtype,
+          occurred_at = excluded.occurred_at,
+          duration_ms = excluded.duration_ms,
+          actor = excluded.actor,
+          participants = excluded.participants,
+          place = excluded.place,
+          items = excluded.items,
+          topics = excluded.topics,
+          content = excluded.content,
+          source = excluded.source,
+          extra = excluded.extra,
+          ingested_at = excluded.ingested_at,
           confidence = excluded.confidence`
       )
       .run({
@@ -325,6 +412,18 @@ class LocalVault {
           source = excluded.source,
           extra = excluded.extra,
           ingested_at = excluded.ingested_at,
+          confidence = excluded.confidence
+        ON CONFLICT(source_adapter, source_original_id)
+          WHERE source_original_id IS NOT NULL
+          DO UPDATE SET
+          subtype = excluded.subtype,
+          names = excluded.names,
+          identifiers = excluded.identifiers,
+          relation = excluded.relation,
+          notes = excluded.notes,
+          source = excluded.source,
+          extra = excluded.extra,
+          ingested_at = excluded.ingested_at,
           confidence = excluded.confidence`
       )
       .run({
@@ -370,6 +469,19 @@ class LocalVault {
           source = excluded.source,
           extra = excluded.extra,
           ingested_at = excluded.ingested_at,
+          confidence = excluded.confidence
+        ON CONFLICT(source_adapter, source_original_id)
+          WHERE source_original_id IS NOT NULL
+          DO UPDATE SET
+          name = excluded.name,
+          coordinates_lat = excluded.coordinates_lat,
+          coordinates_lng = excluded.coordinates_lng,
+          address = excluded.address,
+          category = excluded.category,
+          aliases = excluded.aliases,
+          source = excluded.source,
+          extra = excluded.extra,
+          ingested_at = excluded.ingested_at,
           confidence = excluded.confidence`
       )
       .run({
@@ -420,6 +532,21 @@ class LocalVault {
           source = excluded.source,
           extra = excluded.extra,
           ingested_at = excluded.ingested_at,
+          confidence = excluded.confidence
+        ON CONFLICT(source_adapter, source_original_id)
+          WHERE source_original_id IS NOT NULL
+          DO UPDATE SET
+          subtype = excluded.subtype,
+          name = excluded.name,
+          category = excluded.category,
+          price_value = excluded.price_value,
+          price_currency = excluded.price_currency,
+          merchant = excluded.merchant,
+          external_url = excluded.external_url,
+          thumbnail_local_path = excluded.thumbnail_local_path,
+          source = excluded.source,
+          extra = excluded.extra,
+          ingested_at = excluded.ingested_at,
           confidence = excluded.confidence`
       )
       .run({
@@ -546,6 +673,50 @@ class LocalVault {
       .run(adapter, originalId, capturedAt, json);
   }
 
+  /**
+   * 2026-05-24 — iterate raw_events sequentially for re-derive flow.
+   * Returns rows shaped like the original raw payload object the adapter
+   * yielded ({ originalId, capturedAt, payload }) so the caller can feed
+   * directly into adapter.normalize().
+   *
+   * @param {object} [opts]
+   * @param {string} [opts.adapter]  Filter by adapter name; default = all
+   * @param {number} [opts.limit]    Max rows; default = unlimited
+   * @param {number} [opts.offset=0] Skip first N rows
+   * @returns {Array<{adapter: string, originalId: string, capturedAt: number, payload: object}>}
+   */
+  queryRawEvents({ adapter, limit, offset = 0 } = {}) {
+    const db = this._requireOpen();
+    let sql =
+      "SELECT adapter, original_id, captured_at, payload FROM raw_events";
+    const args = [];
+    if (adapter) {
+      sql += " WHERE adapter = ?";
+      args.push(adapter);
+    }
+    sql += " ORDER BY adapter, captured_at, original_id";
+    if (Number.isInteger(limit) && limit > 0) {
+      sql += " LIMIT ? OFFSET ?";
+      args.push(limit, Number.isInteger(offset) ? offset : 0);
+    } else if (Number.isInteger(offset) && offset > 0) {
+      sql += " LIMIT -1 OFFSET ?";
+      args.push(offset);
+    }
+    const rows = db.prepare(sql).all(...args);
+    return rows.map((r) => ({
+      adapter: r.adapter,
+      originalId: r.original_id,
+      capturedAt: r.captured_at,
+      payload: (() => {
+        try {
+          return JSON.parse(r.payload);
+        } catch (_e) {
+          return r.payload; // raw string if not JSON
+        }
+      })(),
+    }));
+  }
+
   // ─── Entity reads ──────────────────────────────────────────────────────
 
   getEvent(id) {
@@ -734,6 +905,190 @@ class LocalVault {
       .map((row) => this._rowToItem(row));
   }
 
+  /**
+   * Mode (`'fts5'` or `'like'`) recorded by migration 3. Determines whether
+   * searchEvents uses the FTS5 virtual table or falls back to LIKE scans.
+   * Cached on first read.
+   */
+  ftsMode() {
+    if (!this._ftsMode) this._ftsMode = getFtsMode(this._requireOpen());
+    return this._ftsMode;
+  }
+
+  /**
+   * Full-text + faceted search over events for the Vault Browser UI.
+   *
+   * Filters (all optional, ANDed):
+   *   q         — keyword string; FTS5 phrase match if length >= 3 (or LIKE
+   *               substring match on subtype/content/actor/place/extra in
+   *               fallback mode). Shorter queries are ignored as if absent.
+   *   adapter   — exact source_adapter
+   *   category  — one of categories.CATEGORIES; expands to adapter prefix list
+   *   subtype   — exact subtype match
+   *   since     — occurred_at >= since (ms epoch)
+   *   until     — occurred_at <= until
+   *   cursor    — { occurredAt, id } from previous page's last row
+   *   limit     — default 50, max 500
+   *
+   * Pagination is cursor-based on (occurred_at DESC, id DESC) — stable under
+   * concurrent inserts (newer events appear only on re-fetch of page 1).
+   *
+   * Returns `{ rows: Event[], nextCursor: {occurredAt, id} | null, mode: 'fts5'|'like', shortQuery: boolean }`.
+   *   - shortQuery=true means the q was non-empty but below FTS5_MIN_QUERY_LEN
+   *     and was dropped — UI should hint "请输入至少 3 个字".
+   */
+  searchEvents(q = {}) {
+    const db = this._requireOpen();
+    const mode = this.ftsMode();
+    const limit = Number.isInteger(q.limit) && q.limit > 0 ? Math.min(q.limit, 500) : 50;
+
+    const where = [];
+    const params = { limit: limit + 1 }; // +1 to detect "is there a next page?"
+
+    let shortQuery = false;
+    const rawQ = typeof q.q === "string" ? q.q.trim() : "";
+
+    // Keyword filter — FTS5 path uses MATCH on events_fts; LIKE path does
+    // OR across the 5 indexed columns. Sub-min-length q in FTS5 mode is
+    // dropped silently (and reported back as shortQuery=true).
+    let joinFts = false;
+    if (rawQ.length > 0) {
+      if (mode === "fts5") {
+        if (rawQ.length >= FTS5_MIN_QUERY_LEN) {
+          joinFts = true;
+          params.q = _quoteFtsQuery(rawQ);
+          where.push("events_fts MATCH @q");
+        } else {
+          shortQuery = true;
+        }
+      } else {
+        params.qLike = "%" + rawQ + "%";
+        where.push(
+          "(subtype LIKE @qLike OR content LIKE @qLike OR actor LIKE @qLike OR place LIKE @qLike OR extra LIKE @qLike)"
+        );
+      }
+    }
+
+    if (q.adapter) {
+      where.push("source_adapter = @adapter");
+      params.adapter = q.adapter;
+    }
+    if (q.category) {
+      const { sql, params: catParams } = _categoryToWhere(q.category);
+      if (sql) {
+        where.push(sql);
+        Object.assign(params, catParams);
+      }
+    }
+    if (q.subtype) {
+      where.push("e.subtype = @subtype");
+      params.subtype = q.subtype;
+    }
+    if (Number.isFinite(q.since)) {
+      where.push("occurred_at >= @since");
+      params.since = q.since;
+    }
+    if (Number.isFinite(q.until)) {
+      where.push("occurred_at <= @until");
+      params.until = q.until;
+    }
+    // Cursor: rows strictly older than the cursor's (occurred_at, id) tuple.
+    // SQLite tuple comparison handles this natively.
+    if (q.cursor && Number.isFinite(q.cursor.occurredAt) && typeof q.cursor.id === "string") {
+      where.push("(occurred_at, e.id) < (@cursorAt, @cursorId)");
+      params.cursorAt = q.cursor.occurredAt;
+      params.cursorId = q.cursor.id;
+    }
+
+    const sql =
+      "SELECT e.* FROM events e" +
+      (joinFts ? " JOIN events_fts f ON e.rowid = f.rowid" : "") +
+      (where.length ? " WHERE " + where.join(" AND ") : "") +
+      " ORDER BY occurred_at DESC, e.id DESC LIMIT @limit";
+
+    const rowsPlusOne = db.prepare(sql).all(params);
+    const hasMore = rowsPlusOne.length > limit;
+    const rows = hasMore ? rowsPlusOne.slice(0, limit) : rowsPlusOne;
+    const events = rows.map((r) => this._rowToEvent(r));
+    const last = rows[rows.length - 1];
+    return {
+      rows: events,
+      nextCursor: hasMore && last ? { occurredAt: last.occurred_at, id: last.id } : null,
+      mode,
+      shortQuery,
+    };
+  }
+
+  /**
+   * Counts events grouped by category / adapter / subtype, honoring the
+   * same q/since/until filters as searchEvents. Powers the sidebar badges
+   * + adapter chips ("社交聊天 (123)" / "social-bilibili (45)").
+   *
+   * Returns `{ byCategory, byAdapter, bySubtype, total, mode, shortQuery }`.
+   * Each map is `{ key: count }`. Empty keys are omitted.
+   */
+  facetCounts(q = {}) {
+    const db = this._requireOpen();
+    const mode = this.ftsMode();
+    const params = {};
+    const where = [];
+
+    let shortQuery = false;
+    const rawQ = typeof q.q === "string" ? q.q.trim() : "";
+    let joinFts = false;
+    if (rawQ.length > 0) {
+      if (mode === "fts5") {
+        if (rawQ.length >= FTS5_MIN_QUERY_LEN) {
+          joinFts = true;
+          params.q = _quoteFtsQuery(rawQ);
+          where.push("events_fts MATCH @q");
+        } else {
+          shortQuery = true;
+        }
+      } else {
+        params.qLike = "%" + rawQ + "%";
+        where.push(
+          "(subtype LIKE @qLike OR content LIKE @qLike OR actor LIKE @qLike OR place LIKE @qLike OR extra LIKE @qLike)"
+        );
+      }
+    }
+    if (Number.isFinite(q.since)) {
+      where.push("occurred_at >= @since");
+      params.since = q.since;
+    }
+    if (Number.isFinite(q.until)) {
+      where.push("occurred_at <= @until");
+      params.until = q.until;
+    }
+
+    const baseFrom =
+      "FROM events e" + (joinFts ? " JOIN events_fts f ON e.rowid = f.rowid" : "");
+    const whereSql = where.length ? " WHERE " + where.join(" AND ") : "";
+
+    const adapterRows = db
+      .prepare(
+        `SELECT source_adapter AS k, COUNT(*) AS n ${baseFrom}${whereSql} GROUP BY source_adapter`
+      )
+      .all(params);
+    const subtypeRows = db
+      .prepare(`SELECT e.subtype AS k, COUNT(*) AS n ${baseFrom}${whereSql} GROUP BY e.subtype`)
+      .all(params);
+
+    const byAdapter = {};
+    const byCategory = {};
+    let total = 0;
+    for (const r of adapterRows) {
+      byAdapter[r.k] = r.n;
+      const cat = getCategory(r.k);
+      byCategory[cat] = (byCategory[cat] || 0) + r.n;
+      total += r.n;
+    }
+    const bySubtype = {};
+    for (const r of subtypeRows) bySubtype[r.k] = r.n;
+
+    return { byCategory, byAdapter, bySubtype, total, mode, shortQuery };
+  }
+
   countEvents(q = {}) {
     const where = [];
     const params = {};
@@ -1275,4 +1630,10 @@ class LocalVault {
   }
 }
 
-module.exports = { LocalVault, _internal: { loadDriver, formatDriverLoadError } };
+module.exports = {
+  LocalVault,
+  _internal: { loadDriver, formatDriverLoadError },
+  // Pure-JS helpers exported for unit testing without the native bs3mc
+  // binding (search SQL builders, category WHERE translator, FTS5 escape).
+  _searchHelpers: { _categoryToWhere, _quoteFtsQuery, FTS5_MIN_QUERY_LEN },
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chainlesschain/personal-data-hub",
-  "version": "0.2.4",
+  "version": "0.3.0",
   "description": "Personal Data Hub — UnifiedSchema + validators + KG ingest helpers for the data-back-to-the-individual middleware",
   "type": "commonjs",
   "main": "lib/index.js",
@@ -14,6 +14,7 @@
     "./migrations": "./lib/migrations.js",
     "./key-providers": "./lib/key-providers.js",
     "./adapter-spec": "./lib/adapter-spec.js",
+    "./categories": "./lib/categories.js",
     "./kg-derive": "./lib/kg-derive.js",
     "./rag-derive": "./lib/rag-derive.js",
     "./registry": "./lib/registry.js",

package/scripts/run-native-tests-sandbox.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# Run vault-search.test.js (FTS5 native integration) in an isolated sandbox.
+#
+# Background: the root node_modules bs3mc binding is built for Electron 39
+# (NODE_MODULE_VERSION 140) which doesn't match ANY Node.js version
+# (Node 24 = ABI 137, Node 25 = ABI 141). Plain `npm test` fails locally with
+# ABI mismatch. CI is unaffected because its node_modules is built fresh.
+#
+# This script:
+#   1. Copies lib/ + the target test into a temp sandbox
+#   2. Installs a separate bs3mc compiled for the CURRENT host Node ABI
+#   3. Runs vitest against it
+#
+# Idempotent; rerun any time. Sandbox lives under $TMPDIR/pdh-fts5-sandbox
+# so it survives between runs (faster re-install).
+#
+# Usage: bash packages/personal-data-hub/scripts/run-native-tests-sandbox.sh
+
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+SANDBOX="${TMPDIR:-/tmp}/pdh-fts5-sandbox"
+
+echo "==> Sandbox: $SANDBOX"
+mkdir -p "$SANDBOX/lib" "$SANDBOX/__tests__"
+
+# Sync sources every run (lib/ may have evolved since last sandbox build)
+cp -r "$ROOT/lib/." "$SANDBOX/lib/"
+cp "$ROOT/__tests__/vault-search.test.js" "$SANDBOX/__tests__/"
+
+# Minimal package.json — only the deps the target test needs.
+cat > "$SANDBOX/package.json" <<'EOF'
+{
+  "name": "pdh-fts5-sandbox",
+  "version": "0.0.0",
+  "private": true,
+  "type": "commonjs",
+  "scripts": { "test": "vitest run" },
+  "dependencies": { "better-sqlite3-multiple-ciphers": "^12.5.0" },
+  "devDependencies": { "vitest": "^4.1.5" }
+}
+EOF
+
+if [ ! -d "$SANDBOX/node_modules/better-sqlite3-multiple-ciphers/build" ]; then
+  echo "==> Installing deps (one-time, ~30-60s)"
+  (cd "$SANDBOX" && npm install --no-audit --no-fund --loglevel=warn)
+else
+  echo "==> Deps already installed (skipping npm install)"
+fi
+
+echo "==> Running tests"
+cd "$SANDBOX"
+exec node ./node_modules/vitest/vitest.mjs run