@chainlesschain/personal-data-hub 0.4.28 → 0.4.29

package/lib/adapters/social-toutiao-adb/article-reader.js

@@ -0,0 +1,202 @@
+/**
+ * Toutiao on-device article reader — recovers the user's feed/read articles
+ * from the app's local `news_article.db` (table `article`), a plaintext SQLite
+ * DB. No signing/encryption needed.
+ *
+ * Why this exists (real-device 2026-06-18, user's exported plaintext DB):
+ *   - `article` rows are the local feed cache (48 rows on the test export). The
+ *     title is NOT a column — it lives in the `share_info` JSON blob
+ *     ({title, share_url, ...}); `ext_json` is a heavier fallback. `behot_time`
+ *     is when the item surfaced; `read_timestamp>0` ⇒ actually opened;
+ *     `is_user_digg`/`is_user_repin` ⇒ engagement.
+ *   - Modest signal (feed-shown ≈ weak interest; digg/read ≈ strong), but
+ *     titled + plaintext, so it's a usable "articles I browsed" stream.
+ *
+ * Emits BROWSE events under source.adapter `social-toutiao` (the canonical
+ * adapter name, so byApp aggregation attributes correctly). Stable originalId
+ * (`social-toutiao:article:<group_id>`) → re-ingest UPDATES, not duplicates.
+ *
+ * Authorization: only on your own device/account.
+ */
+"use strict";
+
+const { newId } = require("../../ids");
+const {
+  _internals: { loadDatabaseClass },
+} = require("../social-bilibili-adb/chromium-cookies-reader");
+
+const ARTICLE_TABLE = "article";
+const READER_VERSION = "toutiao-article-0.1";
+const TITLE_SUFFIX = /\s*-\s*今日头条\s*$/;
+
+/** seconds-or-ms epoch → ms (heuristic: > 1e12 ⇒ already ms). */
+function toEpochMs(v) {
+  const n = Number(v);
+  if (!Number.isFinite(n) || n <= 0) return null;
+  return n > 1e12 ? Math.floor(n) : Math.floor(n * 1000);
+}
+
+function safeParse(s) {
+  if (typeof s !== "string" || s.length < 2) return null;
+  try {
+    return JSON.parse(s);
+  } catch (_e) {
+    return null;
+  }
+}
+
+/** Title lives in share_info.title (or ext_json.title); strip the brand suffix. */
+function extractTitle(row) {
+  const si = safeParse(row.share_info);
+  let title = si && (si.title || si.share_title);
+  if (!title) {
+    const ej = safeParse(row.ext_json);
+    title = ej && (ej.title || ej.share_title || (ej.article && ej.article.title));
+  }
+  if (typeof title !== "string" || !title.trim()) return null;
+  return title.replace(TITLE_SUFFIX, "").trim();
+}
+
+function extractUrl(row) {
+  const si = safeParse(row.share_info);
+  const u = (si && si.share_url) || row.share_url || null;
+  if (typeof u !== "string" || !u) return null;
+  // Drop the noisy share/tracking query so the same article dedups by url too.
+  return u.split("?")[0];
+}
+
+function extractCategory(row) {
+  const u = row.share_url || "";
+  const m = /[?&]category_new=([^&]+)/.exec(u);
+  return m ? decodeURIComponent(m[1]) : null;
+}
+
+/**
+ * Read article rows + parse them into structured records (pure once a Database
+ * class is injected). Newest-first.
+ *
+ * @returns {{articles: Array<{groupId,title,url,category,behotTime,readTimestamp,digg,repin}>}}
+ */
+function readToutiaoArticles(dbPath, opts = {}) {
+  const Database = opts._databaseClass || loadDatabaseClass();
+  const limit = Number.isInteger(opts.limit) && opts.limit > 0 ? opts.limit : 5000;
+  const db = new Database(dbPath, { readonly: true });
+  try {
+    const exists = db
+      .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name=?")
+      .get(ARTICLE_TABLE);
+    if (!exists) return { articles: [] };
+
+    const cols = new Set(
+      db.prepare(`PRAGMA table_info("${ARTICLE_TABLE}")`).all().map((c) => c.name),
+    );
+    const hasBehot = cols.has("behot_time");
+    const rows = db
+      .prepare(
+        `SELECT * FROM "${ARTICLE_TABLE}"${hasBehot ? " ORDER BY behot_time DESC" : ""} LIMIT ${limit}`,
+      )
+      .all();
+
+    const articles = [];
+    for (const r of rows) {
+      const groupId =
+        r.group_id != null ? String(r.group_id) : r.item_id != null ? String(r.item_id) : null;
+      if (!groupId) continue;
+      const title = extractTitle(r);
+      if (!title) continue; // untitled cache rows carry no signal
+      articles.push({
+        groupId,
+        title,
+        url: extractUrl(r),
+        category: extractCategory(r),
+        behotTime: hasBehot ? toEpochMs(r.behot_time) : null,
+        readTimestamp: cols.has("read_timestamp") ? toEpochMs(r.read_timestamp) : null,
+        digg: cols.has("is_user_digg") ? !!r.is_user_digg : false,
+        repin: cols.has("is_user_repin") ? !!r.is_user_repin : false,
+      });
+    }
+    return { articles };
+  } finally {
+    try {
+      db.close();
+    } catch (_e) {
+      /* best-effort */
+    }
+  }
+}
+
+/**
+ * Map article records → BROWSE events tagged with the toutiao source.
+ * @returns {{events: object[]}}
+ */
+function buildArticleEvents(articles, opts = {}) {
+  const now = Number.isFinite(opts.now) ? opts.now : Date.now();
+  const events = [];
+  for (const a of articles || []) {
+    if (!a || !a.groupId || !a.title) continue;
+    const occurredAt =
+      (Number.isFinite(a.readTimestamp) && a.readTimestamp) ||
+      (Number.isFinite(a.behotTime) && a.behotTime) ||
+      now;
+    events.push({
+      id: newId(),
+      type: "event",
+      subtype: "browse",
+      occurredAt,
+      actor: "person-self",
+      content: { title: a.title, text: a.title },
+      ingestedAt: now,
+      source: {
+        adapter: "social-toutiao",
+        adapterVersion: READER_VERSION,
+        originalId: `social-toutiao:article:${a.groupId}`,
+        capturedAt: occurredAt,
+        capturedBy: "sqlite",
+      },
+      extra: {
+        platform: "toutiao",
+        kind: "article",
+        groupId: a.groupId,
+        url: a.url || null,
+        category: a.category || null,
+        digg: a.digg,
+        repin: a.repin,
+        read: Number.isFinite(a.readTimestamp) && a.readTimestamp > 0,
+      },
+    });
+  }
+  return { events };
+}
+
+/**
+ * Read news_article.db and write the article BROWSE events into the vault.
+ * @param {object} vault LocalVault (must expose putBatch)
+ * @param {string} dbPath path to news_article.db
+ */
+function articlesToVault(vault, dbPath, opts = {}) {
+  if (!vault || typeof vault.putBatch !== "function") {
+    throw new TypeError("articlesToVault: vault with putBatch required");
+  }
+  if (typeof dbPath !== "string" || !dbPath) {
+    throw new TypeError("articlesToVault: dbPath required");
+  }
+  const { articles } = readToutiaoArticles(dbPath, opts);
+  const built = buildArticleEvents(articles, opts);
+  const res = built.events.length
+    ? vault.putBatch({ events: built.events })
+    : { events: 0 };
+  return {
+    ingested: res.events || 0,
+    articles: articles.length,
+    digg: articles.filter((a) => a.digg).length,
+    read: articles.filter((a) => Number.isFinite(a.readTimestamp) && a.readTimestamp > 0).length,
+  };
+}
+
+module.exports = {
+  ARTICLE_TABLE,
+  readToutiaoArticles,
+  buildArticleEvents,
+  articlesToVault,
+  _internals: { toEpochMs, extractTitle, extractUrl, extractCategory },
+};

package/lib/analysis-skills/overview.js

@@ -39,6 +39,18 @@ class OverviewSkill extends AnalysisSkill {
     if (until != null) q.until = until;
     const events = this.vault.queryEvents(q) || [];
 
+    // Accurate, uncapped app/type/total counts via SQL GROUP BY. queryEvents
+    // hard-caps at 10k rows, so deriving byApp/byType/total from `events`
+    // silently undercounts any app whose data is older than the recent-10k
+    // window — e.g. on a vault where one chat app dominates recent events,
+    // social-douyin showed 10 instead of its true 232. facetCounts honors the
+    // same since/until. (Row-derived spend/contacts/monthly stay sample-based —
+    // they need actual rows.)
+    const facets =
+      typeof this.vault.facetCounts === "function"
+        ? this.vault.facetCounts({ since, until })
+        : null;
+
     const byApp = new Map();
     const byType = new Map();
     const byMonth = new Map();
@@ -81,7 +93,11 @@ class OverviewSkill extends AnalysisSkill {
       if (citations.length < 50) citations.push(e.id);
     }
 
-    const byAppArr = [...byApp.entries()].map(([app, count]) => ({ app, count })).sort((a, b) => b.count - a.count);
+    const byAppArr = (
+      facets
+        ? Object.entries(facets.byAdapter).map(([app, count]) => ({ app, count }))
+        : [...byApp.entries()].map(([app, count]) => ({ app, count }))
+    ).sort((a, b) => b.count - a.count);
     const topContacts = [...contacts.entries()]
       .map(([personId, v]) => ({
         personId,
@@ -93,8 +109,8 @@ class OverviewSkill extends AnalysisSkill {
       .slice(0, topN);
 
     const summary = {
-      totalEvents: events.length,
-      appsActive: byApp.size,
+      totalEvents: facets ? facets.total : events.length,
+      appsActive: facets ? Object.keys(facets.byAdapter).length : byApp.size,
       period: { since: since || null, until: until || null },
       topAppName: byAppArr.length ? byAppArr[0].app : null,
     };
@@ -103,7 +119,11 @@ class OverviewSkill extends AnalysisSkill {
       skill: "analysis.overview",
       summary,
       byApp: byAppArr,
-      byType: [...byType.entries()].map(([type, count]) => ({ type, count })).sort((a, b) => b.count - a.count),
+      byType: (
+        facets
+          ? Object.entries(facets.bySubtype).map(([type, count]) => ({ type, count }))
+          : [...byType.entries()].map(([type, count]) => ({ type, count }))
+      ).sort((a, b) => b.count - a.count),
       monthlyActivity: [...byMonth.entries()].map(([monthKey, count]) => ({ monthKey, count })).sort((a, b) => a.monthKey.localeCompare(b.monthKey)),
       topContacts,
       spending: {

package/lib/analysis-skills/spending.js

@@ -30,6 +30,13 @@ const { AnalysisSkill } = require("./base");
 
 const SUPPORTED_DIMENSIONS = new Set(["merchant", "category", "counterparty", "month"]);
 
+// Event subtypes that carry content.amount (shared by the row fetch + the
+// accurate SQL-sum path). Phase 7 shopping adapters emit "order".
+const PAYMENT_SUBTYPES = [
+  "payment", "transfer", "refund", "utility",
+  "redenvelope", "investment", "income", "order",
+];
+
 class SpendingSkill extends AnalysisSkill {
   constructor(opts) {
     super({ ...opts, name: "analysis.spending" });
@@ -48,6 +55,19 @@ class SpendingSkill extends AnalysisSkill {
     const filtered = this._applyFilters(events, options);
 
     const summary = this._summarize(filtered, since, until);
+    // The row fetch caps at 5000 events PER subtype — a heavy alipay/wechat-pay
+    // user with >5000 payments would have their TOTAL silently undercounted.
+    // When no row-only filter is active (merchant text / personId / direction),
+    // recompute the headline totals from the uncapped SQL SUM. Breakdown / trend
+    // / citations stay row-sampled (they need actual rows).
+    const accurate = this._accurateTotals({ since, until }, options);
+    if (accurate) {
+      summary.totalSpend = accurate.totalSpend;
+      summary.totalIncome = accurate.totalIncome;
+      summary.netFlow = accurate.netFlow;
+      summary.eventCount = accurate.eventCount;
+      if (accurate.currency) summary.currency = accurate.currency;
+    }
     const breakdown = this._breakdown(filtered, dimension, topN);
     const trend = this._monthlyTrend(filtered);
     const citations = filtered.slice(0, 50).map((e) => e.id);
@@ -72,8 +92,7 @@ class SpendingSkill extends AnalysisSkill {
     // Phase 7 shopping adapters emit subtype="order" — must include so
     // spending aggregates cover Taobao/JD/Meituan along with Alipay
     // (payment/transfer) + Email (refund) etc.
-    const subtypes = ["payment", "transfer", "refund", "utility", "redenvelope", "investment", "income", "order"];
-    for (const subtype of subtypes) {
+    for (const subtype of PAYMENT_SUBTYPES) {
       const q = { subtype, limit: 5000 };
       if (since != null) q.since = since;
       if (until != null) q.until = until;
@@ -88,6 +107,48 @@ class SpendingSkill extends AnalysisSkill {
     return events;
   }
 
+  /**
+   * Accurate (uncapped) headline totals via vault.sumEventAmount — used only
+   * when the query has no filter SQL can't express. merchantFilter (text match
+   * on title/counterparty) and personId (participant expansion) need rows, and
+   * a direction filter changes which total/count is meaningful, so any of them
+   * → return null and fall back to the row-sampled summary. Returns null when
+   * the vault lacks sumEventAmount (older vault → original behavior).
+   */
+  _accurateTotals({ since, until }, options) {
+    if (
+      (typeof options.merchantFilter === "string" && options.merchantFilter.length > 0) ||
+      (typeof options.personId === "string" && options.personId.length > 0) ||
+      options.direction === "out" ||
+      options.direction === "in"
+    ) {
+      return null;
+    }
+    if (typeof this.vault.sumEventAmount !== "function") return null;
+    let totalSpend = 0;
+    let totalIncome = 0;
+    let eventCount = 0;
+    let currency = null;
+    for (const subtype of PAYMENT_SUBTYPES) {
+      const q = { subtype };
+      if (since != null) q.since = since;
+      if (until != null) q.until = until;
+      const r = this.vault.sumEventAmount(q);
+      if (!r) continue;
+      totalSpend += (r.byDirection && r.byDirection.out) || 0;
+      totalIncome += (r.byDirection && r.byDirection.in) || 0;
+      eventCount += r.count || 0;
+      if (!currency && r.count > 0 && r.currency) currency = r.currency;
+    }
+    return {
+      totalSpend: Math.round(totalSpend * 100) / 100,
+      totalIncome: Math.round(totalIncome * 100) / 100,
+      netFlow: Math.round((totalIncome - totalSpend) * 100) / 100,
+      eventCount,
+      currency,
+    };
+  }
+
   _applyFilters(events, options) {
     let out = events;
     if (typeof options.merchantFilter === "string" && options.merchantFilter.length > 0) {

package/lib/analysis-skills/timeline.js

@@ -63,12 +63,17 @@ class TimelineSkill extends AnalysisSkill {
   }
 
   _fetchEvents({ since, until }, limit) {
-    // Exclude inventory-snapshot events (installed-app roster + contact
-    // roster from system-data-android). They carry a synthetic
-    // collection-time occurredAt — tens of thousands of them cluster at one
-    // recent timestamp and would otherwise crowd out real activity from this
-    // chronological narrative. They remain in the vault for facet counts.
-    const q = { limit, excludeExtraKinds: ["app-snapshot", "contact-snapshot"] };
+    // Exclude inventory-snapshot + aggregate-baseline events. The snapshots
+    // (installed-app / contact roster from system-data-android) carry a
+    // synthetic collection-time occurredAt — tens of thousands cluster at one
+    // recent timestamp and would crowd out real activity. `app-usage-profile`
+    // is a single rolling aggregate (e.g. douyin "24天/108h" baseline), not a
+    // discrete activity, so it doesn't belong in a chronological narrative.
+    // All remain in the vault for facet counts / overview.
+    const q = {
+      limit,
+      excludeExtraKinds: ["app-snapshot", "contact-snapshot", "app-usage-profile"],
+    };
     if (since != null) q.since = since;
     if (until != null) q.until = until;
     const events = this.vault.queryEvents(q) || [];

package/lib/query-parser.js

@@ -127,8 +127,16 @@ function parseTimeWindow(text, now = Date.now()) {
   if (m) {
     const n = parseInt(m[1], 10);
     if (Number.isFinite(n) && n > 0) {
+      // Safe month subtraction. Naive `setMonth(getMonth()-n)` overflows on a
+      // month-end day into a shorter month (e.g. Mar 31 −1mo → "Feb 31" → Mar 3),
+      // which silently DROPS the whole previous month from the window. Pin to
+      // day 1 first, then clamp the day to the target month's length.
       const target = new Date(now);
+      const day = target.getDate();
+      target.setDate(1);
       target.setMonth(target.getMonth() - n);
+      const lastDay = new Date(target.getFullYear(), target.getMonth() + 1, 0).getDate();
+      target.setDate(Math.min(day, lastDay));
       return { since: target.getTime(), until: now };
     }
   }
@@ -152,7 +160,10 @@ const SUBTYPE_KEYWORDS = [
   { subtype: "order", patterns: [/(订单|下单|买了|购买|下了几单|下了多少单|order)/i] },
   { subtype: "payment", patterns: [/(支付|付款|花了|花费|消费|开销|payment|spent|spend)/i] },
   { subtype: "transfer", patterns: [/(转账|转给|转钱|transfer)/i] },
-  { subtype: "income", patterns: [/(收入|工资|进账|收到|income)/i] },
+  // NOTE: bare 收到 ("receive") is deliberately NOT here — you 收到 messages /
+  // packages / 红包 too, so it stole "收到多少消息" → income (income is checked
+  // before message). 收到转账 still classifies as transfer (checked earlier).
+  { subtype: "income", patterns: [/(收入|工资|进账|入账|income)/i] },
   { subtype: "message", patterns: [/(聊天|消息|聊了|对话|message|chat)/i] },
   { subtype: "post", patterns: [/(朋友圈|发了|动态|moment|post)/i] },
   { subtype: "visit", patterns: [/(去过|到过|visited|去了|来到)/i] },
@@ -201,20 +212,39 @@ function parseFilters(text) {
 
 // ─── Intent detection (sum / count / list / latest / ...) ────────────────
 
+// Amount/money words — BOTH the spend side (花/消费/开销/spent/金额) and the
+// income side (收入/进账/到账/赚/挣). A question carrying one of these plus a
+// "多少/how much" wants a SUM (sumEventAmount), not a row list.
+const AMOUNT_HINT =
+  /(花|花了|花费|消费|开销|spent|金额|多少钱|amount|收入|进账|到账|入账|赚|挣)/;
+// Count quantifier: "多少X" or "几X" for a measure word. 钱 is deliberately
+// EXCLUDED so "多少钱" routes to sum-amount, not count. Symmetric 多少/几 (the
+// old pattern had 几条/几单 but not 多少条/多少单, and 多少部 but not 几部).
+const COUNT_QUANTIFIER =
+  /(多少|几)(次|条|单|个|家|人|张|部|篇|集|本|件|笔|顿|杯)|how\s+many|count\s+of/i;
+const HOW_MUCH = /(多少钱|多少|how\s+much)/i;
+
 function parseIntent(text) {
   if (typeof text !== "string") return "list";
   if (/(总共|共多少|加起来|sum|total|合计)/.test(text)) {
-    // Distinguish amount vs count by presence of currency words.
-    if (/(花|花了|花费|消费|开销|spent|金额|多少钱|amount)/.test(text)) return "sum-amount";
+    // Distinguish amount vs count by presence of amount words (incl. income,
+    // so "总共收入多少" is sum-amount, not count).
+    if (AMOUNT_HINT.test(text)) return "sum-amount";
     return "count";
   }
-  // Count intents: 几次/条/单/个 / 多少个/家/人/张/部 / how many / count of
-  // 2026-05-21: extended "几个 X" / "多少个 X" — needed for "几个联系人"
-  // and "几个 app" which prior pattern missed (returned "list" → LLM had no
-  // hint to read authoritative TOTALS instead of the FACTS sample length).
-  if (/(多少次|几次|几条|几单|几个|多少个|多少家|多少人|多少张|多少部|how\s+many|count\s+of)/i.test(text)) {
+  // Count: 多少X / 几X for a measure word ("多少条朋友圈" / "下了几单" /
+  // "几个联系人"). Runs BEFORE the bare-sum rule so "消费了多少次" → count.
+  if (COUNT_QUANTIFIER.test(text)) {
     return "count";
   }
+  // Spend/income question without an explicit 总共/合计 — "(这个月)花了多少钱" /
+  // "在淘宝花了多少" / "这个月收入多少" / "赚了多少". The amount word + a
+  // "多少/how much" ⇒ a TOTAL. Without this these common phrasings fell through
+  // to intent=list and the engine returned a row sample, not the authoritative
+  // sumEventAmount total.
+  if (AMOUNT_HINT.test(text) && HOW_MUCH.test(text)) {
+    return "sum-amount";
+  }
   if (/(最近|最新|latest|recent)/i.test(text)) return "latest";
   return "list";
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chainlesschain/personal-data-hub",
-  "version": "0.4.28",
+  "version": "0.4.29",
   "description": "Personal Data Hub — UnifiedSchema + validators + KG ingest helpers for the data-back-to-the-individual middleware",
   "type": "commonjs",
   "main": "lib/index.js",