@chainlesschain/personal-data-hub 0.2.3 → 0.3.0

package/lib/adapters/browser-history-chrome/adapter.js

@@ -0,0 +1,247 @@
+"use strict";
+
+// BrowserHistoryChromeAdapter — desktop Chrome (and Chromium-family) data
+// adapter. Reads History.sqlite + Bookmarks JSON directly from the user's
+// own profile directory; no extension, no network, no permission prompts.
+//
+// Supported profiles auto-detected on platform:
+//   Windows: %LOCALAPPDATA%\Google\Chrome\User Data\Default
+//   macOS:   ~/Library/Application Support/Google/Chrome/Default
+//   Linux:   ~/.config/google-chrome/Default
+// `opts.profilePath` overrides — point at "Profile 1", an Edge profile,
+// or a copy of one for testing.
+//
+// Chrome locks History while running. We snapshot via fs.copyFileSync to
+// %TEMP% (carrying the WAL sidecar) and read from the copy. ~7 MB DB copies
+// in <50ms; even 100k-visit profiles iterate in seconds.
+
+const path = require("node:path");
+const fs = require("node:fs");
+
+const {
+  ENTITY_TYPES,
+  EVENT_SUBTYPES,
+  ITEM_SUBTYPES,
+  CAPTURED_BY,
+} = require("../../constants");
+
+const {
+  defaultChromeProfileDir,
+  copyHistorySnapshot,
+  cleanupHistorySnapshot,
+  readVisits,
+} = require("./chrome-db-reader");
+const { readBookmarks } = require("./bookmarks-reader");
+
+const NAME = "browser-history-chrome";
+const VERSION = "0.1.0";
+
+// The adapter is browser-agnostic — Chromium-derived browsers (Chrome / Edge /
+// Brave / Vivaldi / Arc) share the History SQLite + Bookmarks JSON schema.
+// Subclasses override `_browserConfig()` to point at a different profile root.
+class BrowserHistoryChromeAdapter {
+  constructor(opts = {}) {
+    const cfg = this._browserConfig();
+    this.name = cfg.name;
+    this.version = cfg.version;
+    this._browser = cfg.browser;
+    this.capabilities = [
+      `sync:${cfg.browser}-history-sqlite`,
+      `sync:${cfg.browser}-bookmarks-json`,
+    ];
+    this.extractMode = "file-import";
+    this.rateLimits = { perDay: 96 }; // ~once per 15 min ceiling
+    this.dataDisclosure = {
+      fields: [
+        "visits:url,title,visitTimeMs,transition,visitDurationMs,hidden",
+        "bookmarks:url,name,dateAddedMs,folderPath",
+      ],
+      sensitivity: "high",
+      legalGate: false,
+      defaultInclude: { history: true, bookmarks: true },
+    };
+    this._deps = {
+      fs: require("node:fs"),
+      defaultProfileDir: cfg.defaultProfileDir,
+    };
+    this._profileOverride = typeof opts.profilePath === "string" ? opts.profilePath : null;
+  }
+
+  _browserConfig() {
+    return {
+      name: NAME,
+      version: VERSION,
+      browser: "chrome",
+      defaultProfileDir: defaultChromeProfileDir,
+    };
+  }
+
+  _resolveProfileDir(opts) {
+    if (typeof opts?.profilePath === "string" && opts.profilePath.length > 0) {
+      return opts.profilePath;
+    }
+    if (this._profileOverride) return this._profileOverride;
+    return this._deps.defaultProfileDir();
+  }
+
+  async authenticate(ctx = {}) {
+    const dir = this._resolveProfileDir(ctx);
+    if (!dir) {
+      return {
+        ok: false,
+        reason: "PROFILE_PATH_UNRESOLVED",
+        message: `no default ${this._browser} profile dir on this platform; pass opts.profilePath`,
+      };
+    }
+    const histPath = path.join(dir, "History");
+    if (!this._deps.fs.existsSync(histPath)) {
+      return {
+        ok: false,
+        reason: "PROFILE_NOT_FOUND",
+        message: `no ${this._browser} History at ${histPath} — install ${this._browser} / open it at least once, or point opts.profilePath at a different profile`,
+      };
+    }
+    return { ok: true, mode: "file-import", profileDir: dir };
+  }
+
+  async healthCheck() {
+    const dir = this._resolveProfileDir({});
+    const ok = !!dir && this._deps.fs.existsSync(path.join(dir, "History"));
+    return { ok, lastChecked: Date.now() };
+  }
+
+  async *sync(opts = {}) {
+    const profileDir = this._resolveProfileDir(opts);
+    if (!profileDir || !this._deps.fs.existsSync(path.join(profileDir, "History"))) {
+      throw new Error(
+        `${this.name}.sync: no History at ${path.join(profileDir || "?", "History")} — set opts.profilePath`,
+      );
+    }
+
+    const includeHistory = opts.include?.history !== false;
+    const includeBookmarks = opts.include?.bookmarks !== false;
+    const limit = Number.isInteger(opts.limit) && opts.limit > 0 ? opts.limit : Infinity;
+    const capturedAt = Date.now();
+    let emitted = 0;
+
+    // History (SQLite snapshot)
+    if (includeHistory) {
+      let tmp = null;
+      try {
+        tmp = copyHistorySnapshot(profileDir, { fs: this._deps.fs });
+        for (const v of readVisits(tmp, {
+          since: opts.since,
+          limit: Number.isFinite(limit) ? limit : undefined,
+          includeHidden: opts.includeHidden === true,
+        })) {
+          if (emitted >= limit) return;
+          yield {
+            kind: "visit",
+            originalId: `${this._browser}-visit:${profileDir}:${v.visitId}`,
+            capturedAt,
+            payload: { ...v, profileDir },
+          };
+          emitted += 1;
+        }
+      } finally {
+        if (tmp) cleanupHistorySnapshot(tmp, { fs: this._deps.fs });
+      }
+    }
+
+    // Bookmarks (JSON)
+    if (includeBookmarks) {
+      for (const b of readBookmarks(profileDir, { fs: this._deps.fs })) {
+        if (emitted >= limit) return;
+        yield {
+          kind: "bookmark",
+          originalId: `${this._browser}-bookmark:${profileDir}:${b.guid || b.id || b.url}`,
+          capturedAt,
+          payload: { ...b, profileDir },
+        };
+        emitted += 1;
+      }
+    }
+  }
+
+  normalize(raw) {
+    const ingestedAt = Date.now();
+    const browser = this._browser;
+    const source = (originalId) => ({
+      adapter: this.name,
+      adapterVersion: this.version,
+      capturedAt: raw.capturedAt,
+      capturedBy: CAPTURED_BY.SQLITE,
+      originalId,
+    });
+
+    if (raw.kind === "visit") {
+      const p = raw.payload || {};
+      const url = typeof p.url === "string" ? p.url : "";
+      const title = typeof p.title === "string" && p.title.length > 0
+        ? p.title
+        : (url || "(无标题)");
+      const occurredAt = Number.isInteger(p.visitTimeMs) ? p.visitTimeMs : raw.capturedAt;
+      const event = {
+        id: `event-${browser}-visit-${p.visitId}`,
+        type: ENTITY_TYPES.EVENT,
+        subtype: EVENT_SUBTYPES.BROWSE,
+        occurredAt,
+        ingestedAt,
+        source: source(`${browser}-visit:${p.profileDir}:${p.visitId}`),
+        actor: "self",
+        content: {
+          title: title.length > 200 ? title.substring(0, 200) + "…" : title,
+          text: url,
+        },
+      };
+      if (Number.isInteger(p.visitDurationMs) && p.visitDurationMs > 0) {
+        event.durationMs = p.visitDurationMs;
+      }
+      event.extra = {
+        url,
+        transition: p.transition || null,
+        rawTransition: Number.isInteger(p.rawTransition) ? p.rawTransition : null,
+        visitCount: p.visitCount || 0,
+        typedCount: p.typedCount || 0,
+        hidden: p.hidden === true,
+        fromVisit: p.fromVisit || 0,
+        browser,
+        profileDir: p.profileDir,
+      };
+      return { events: [event], persons: [], places: [], items: [], topics: [] };
+    }
+
+    if (raw.kind === "bookmark") {
+      const p = raw.payload || {};
+      const url = typeof p.url === "string" ? p.url : "";
+      const name = typeof p.name === "string" && p.name.length > 0 ? p.name : url;
+      const stableId = p.guid || p.id || url;
+      const item = {
+        id: `item-${browser}-bookmark-${stableId}`,
+        type: ENTITY_TYPES.ITEM,
+        subtype: ITEM_SUBTYPES.LINK,
+        name,
+        category: "bookmark",
+        ingestedAt,
+        source: source(`${browser}-bookmark:${p.profileDir}:${stableId}`),
+        extra: {
+          url,
+          dateAddedMs: Number.isInteger(p.dateAddedMs) ? p.dateAddedMs : null,
+          dateLastUsedMs: Number.isInteger(p.dateLastUsedMs) ? p.dateLastUsedMs : null,
+          folderPath: typeof p.folderPath === "string" ? p.folderPath : null,
+          browser,
+          profileDir: p.profileDir,
+        },
+      };
+      return { events: [], persons: [], places: [], items: [item], topics: [] };
+    }
+
+    throw new Error(`${this.name}.normalize: unknown raw.kind=${raw.kind}`);
+  }
+}
+
+module.exports = {
+  BrowserHistoryChromeAdapter,
+  BROWSER_HISTORY_CHROME_NAME: NAME,
+  BROWSER_HISTORY_CHROME_VERSION: VERSION,
+};

package/lib/adapters/browser-history-chrome/bookmarks-reader.js

@@ -0,0 +1,79 @@
+"use strict";
+
+// bookmarks-reader — Chrome's Bookmarks file is plain JSON next to the
+// History DB. No copy-first needed; Chrome writes atomically. We walk
+// the tree depth-first so each url node carries its folder path.
+
+const fs = require("node:fs");
+const path = require("node:path");
+
+const ROOT_LABELS = {
+  bookmark_bar: "书签栏",
+  other: "其他书签",
+  synced: "移动设备书签",
+};
+
+function webkitUsStrToEpochMs(s) {
+  if (typeof s !== "string" || s.length === 0) return null;
+  // Chrome stores date_added as a decimal string of WebKit microseconds.
+  // Parse to BigInt to keep precision.
+  try {
+    const us = BigInt(s);
+    return Number((us - 11_644_473_600_000_000n) / 1000n);
+  } catch (_e) {
+    return null;
+  }
+}
+
+function* walkNode(node, folderTrail) {
+  if (!node || typeof node !== "object") return;
+  if (node.type === "url" && typeof node.url === "string") {
+    yield {
+      id: node.id,
+      guid: node.guid,
+      name: typeof node.name === "string" ? node.name : node.url,
+      url: node.url,
+      dateAddedMs: webkitUsStrToEpochMs(node.date_added),
+      dateLastUsedMs: webkitUsStrToEpochMs(node.date_last_used),
+      folderPath: folderTrail.join(" / "),
+    };
+  } else if (node.type === "folder" && Array.isArray(node.children)) {
+    const nextTrail =
+      typeof node.name === "string" && node.name.length > 0
+        ? folderTrail.concat(node.name)
+        : folderTrail;
+    for (const c of node.children) {
+      yield* walkNode(c, nextTrail);
+    }
+  }
+}
+
+// Read + iterate every bookmark url node across all three roots
+// (bookmark_bar / other / synced). Returns a generator so a callsite that
+// only wants a count can short-circuit.
+function* readBookmarks(profileDir, opts = {}) {
+  const fsMod = opts.fs || fs;
+  const file = path.join(profileDir, "Bookmarks");
+  if (!fsMod.existsSync(file)) return;
+  const text = fsMod.readFileSync(file, "utf-8");
+  const data = JSON.parse(text);
+  const roots = (data && data.roots) || {};
+  for (const [rootKey, rootNode] of Object.entries(roots)) {
+    if (!rootNode || typeof rootNode !== "object") continue;
+    const rootLabel = ROOT_LABELS[rootKey] || rootKey;
+    // Walk the root's CHILDREN directly with the localised root label as the
+    // trail; the root folder's own `name` ("Bookmarks bar" / "Other bookmarks"
+    // — Chrome's English defaults) would otherwise tack on a redundant
+    // English segment after our Chinese label.
+    if (Array.isArray(rootNode.children)) {
+      for (const c of rootNode.children) {
+        yield* walkNode(c, [rootLabel]);
+      }
+    }
+  }
+}
+
+module.exports = {
+  readBookmarks,
+  ROOT_LABELS,
+};

package/lib/adapters/browser-history-chrome/chrome-db-reader.js

@@ -0,0 +1,223 @@
+"use strict";
+
+// chrome-db-reader — opens a copy of Chrome's History SQLite and yields
+// rows. We MUST copy first; Chrome holds an exclusive lock on the live
+// file while running, and even when closed the WAL files (`-wal`, `-shm`)
+// need to come along or we'd see a stale snapshot.
+
+const fs = require("node:fs");
+const path = require("node:path");
+const os = require("node:os");
+// Dual-load: bs3mc tracks Electron's ABI 140 (runtime path), plain
+// better-sqlite3 tracks Node's ABI 127 (test path). Whichever loads
+// without NODE_MODULE_VERSION mismatch wins. Both expose the same
+// Database class for unencrypted DBs.
+function loadDatabase() {
+  for (const mod of ["better-sqlite3-multiple-ciphers", "better-sqlite3"]) {
+    let cls;
+    try {
+      // eslint-disable-next-line global-require
+      cls = require(mod);
+    } catch (_e) {
+      continue; // require failed, try next
+    }
+    // require() returns the JS class even when the native binding is
+    // ABI-mismatched; instantiation is what actually loads the .node
+    // and throws. Smoke-test with an in-memory DB.
+    try {
+      const probe = new cls(":memory:");
+      probe.close();
+      return cls;
+    } catch (_e) {
+      // ABI mismatch — try next candidate
+    }
+  }
+  throw new Error(
+    "chrome-db-reader: neither better-sqlite3-multiple-ciphers nor better-sqlite3 loaded — both ABI-mismatched",
+  );
+}
+const Database = loadDatabase();
+
+// WebKit timestamps are microseconds since 1601-01-01 UTC. Convert to
+// epoch-ms by shifting the epoch (11644473600 seconds × 1e6 µs/s).
+const WEBKIT_EPOCH_DELTA_US = 11_644_473_600_000_000n;
+function webkitUsToEpochMs(wkUs) {
+  if (wkUs == null) return null;
+  // wkUs may arrive as Number (up to 2^53) or BigInt — handle both.
+  const bn = typeof wkUs === "bigint" ? wkUs : BigInt(wkUs);
+  return Number((bn - WEBKIT_EPOCH_DELTA_US) / 1000n);
+}
+function epochMsToWebkitUs(ms) {
+  return BigInt(ms) * 1000n + WEBKIT_EPOCH_DELTA_US;
+}
+
+// Chrome transition flags (lower 8 bits of `transition`). See
+// chromium/src/components/history/core/browser/history_types.h.
+const CORE_TRANSITION_NAMES = {
+  0: "link",
+  1: "typed",
+  2: "auto_bookmark",
+  3: "auto_subframe",
+  4: "manual_subframe",
+  5: "generated",
+  6: "auto_toplevel",
+  7: "form_submit",
+  8: "reload",
+  9: "keyword",
+  10: "keyword_generated",
+};
+function decodeTransition(raw) {
+  if (!Number.isFinite(raw)) return null;
+  const core = raw & 0xff;
+  return CORE_TRANSITION_NAMES[core] || `unknown(${core})`;
+}
+
+function defaultChromeProfileDir() {
+  if (process.platform === "win32") {
+    const lad = process.env.LOCALAPPDATA;
+    if (!lad) return null;
+    return path.join(lad, "Google", "Chrome", "User Data", "Default");
+  }
+  if (process.platform === "darwin") {
+    return path.join(os.homedir(), "Library", "Application Support", "Google", "Chrome", "Default");
+  }
+  return path.join(os.homedir(), ".config", "google-chrome", "Default");
+}
+
+// Edge is Chromium under the hood — identical History/Bookmarks schema,
+// just a different on-disk profile root. Same reader code works.
+function defaultEdgeProfileDir() {
+  if (process.platform === "win32") {
+    const lad = process.env.LOCALAPPDATA;
+    if (!lad) return null;
+    return path.join(lad, "Microsoft", "Edge", "User Data", "Default");
+  }
+  if (process.platform === "darwin") {
+    return path.join(os.homedir(), "Library", "Application Support", "Microsoft Edge", "Default");
+  }
+  return path.join(os.homedir(), ".config", "microsoft-edge", "Default");
+}
+
+// Brave: another Chromium fork; same schema again. Not auto-registered unless
+// the user has Brave installed, since the wiring layer calls authenticate()
+// at the adapter level rather than the registry filtering ahead of time.
+function defaultBraveProfileDir() {
+  if (process.platform === "win32") {
+    const lad = process.env.LOCALAPPDATA;
+    if (!lad) return null;
+    return path.join(lad, "BraveSoftware", "Brave-Browser", "User Data", "Default");
+  }
+  if (process.platform === "darwin") {
+    return path.join(
+      os.homedir(),
+      "Library",
+      "Application Support",
+      "BraveSoftware",
+      "Brave-Browser",
+      "Default",
+    );
+  }
+  return path.join(os.homedir(), ".config", "BraveSoftware", "Brave-Browser", "Default");
+}
+
+// Copy the History file + any sidecar journal/WAL/SHM next to it. Returns
+// the temp path that the caller is responsible for cleaning up.
+function copyHistorySnapshot(profileDir, opts = {}) {
+  const fsMod = opts.fs || fs;
+  const src = path.join(profileDir, "History");
+  if (!fsMod.existsSync(src)) {
+    const err = new Error(`Chrome History not found at ${src}`);
+    err.code = "CHROME_HISTORY_NOT_FOUND";
+    throw err;
+  }
+  const tmp = path.join(
+    os.tmpdir(),
+    `pdh-chrome-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.db`,
+  );
+  fsMod.copyFileSync(src, tmp);
+  for (const ext of ["-journal", "-wal", "-shm"]) {
+    const w = src + ext;
+    if (fsMod.existsSync(w)) {
+      try {
+        fsMod.copyFileSync(w, tmp + ext);
+      } catch (_e) {
+        // Sidecar copy failures are non-fatal — better-sqlite3 will just
+        // see the pre-WAL state, which is what we want anyway.
+      }
+    }
+  }
+  return tmp;
+}
+
+function cleanupHistorySnapshot(tmpPath, opts = {}) {
+  const fsMod = opts.fs || fs;
+  for (const ext of ["", "-journal", "-wal", "-shm"]) {
+    try {
+      fsMod.unlinkSync(tmpPath + ext);
+    } catch (_e) {
+      // best-effort
+    }
+  }
+}
+
+// Yields visit rows in occurredAt-ascending order so the registry's
+// watermark (max occurredAt) advances monotonically across syncs.
+function* readVisits(tmpPath, opts = {}) {
+  const sinceWk = Number.isInteger(opts.since) && opts.since > 0
+    ? epochMsToWebkitUs(opts.since)
+    : 0n;
+  const limit = Number.isInteger(opts.limit) && opts.limit > 0 ? opts.limit : 200_000;
+  const includeHidden = opts.includeHidden === true;
+  const db = new Database(tmpPath, { readonly: true });
+  try {
+    // Bind sinceWk as a string — better-sqlite3 accepts BigInt only when
+    // safeIntegers is on, which we don't enable. SQLite compares numerically
+    // so passing the decimal string is safe (and avoids 2^53 truncation).
+    const stmt = db.prepare(
+      `SELECT v.id AS visit_id, v.url AS url_id, v.visit_time AS visit_time,
+              v.transition AS transition, v.visit_duration AS visit_duration,
+              v.from_visit AS from_visit, u.url AS url, u.title AS title,
+              u.visit_count AS visit_count, u.typed_count AS typed_count,
+              u.hidden AS hidden
+       FROM visits v
+       JOIN urls u ON v.url = u.id
+       WHERE v.visit_time > ?
+         ${includeHidden ? "" : "AND u.hidden = 0"}
+       ORDER BY v.visit_time ASC
+       LIMIT ?`,
+    );
+    const rows = stmt.iterate(sinceWk.toString(), limit);
+    for (const r of rows) {
+      yield {
+        visitId: r.visit_id,
+        urlId: r.url_id,
+        url: r.url,
+        title: r.title || "",
+        visitTimeMs: webkitUsToEpochMs(r.visit_time),
+        visitDurationMs: Number.isInteger(r.visit_duration)
+          ? Math.floor(r.visit_duration / 1000)
+          : 0,
+        transition: decodeTransition(r.transition),
+        rawTransition: r.transition,
+        fromVisit: r.from_visit || 0,
+        visitCount: r.visit_count || 0,
+        typedCount: r.typed_count || 0,
+        hidden: r.hidden === 1,
+      };
+    }
+  } finally {
+    db.close();
+  }
+}
+
+module.exports = {
+  defaultChromeProfileDir,
+  defaultEdgeProfileDir,
+  defaultBraveProfileDir,
+  copyHistorySnapshot,
+  cleanupHistorySnapshot,
+  readVisits,
+  webkitUsToEpochMs,
+  epochMsToWebkitUs,
+  decodeTransition,
+};

package/lib/adapters/browser-history-chrome/index.js

@@ -0,0 +1,23 @@
+"use strict";
+
+const {
+  BrowserHistoryChromeAdapter,
+  BROWSER_HISTORY_CHROME_NAME,
+  BROWSER_HISTORY_CHROME_VERSION,
+} = require("./adapter");
+const dbReader = require("./chrome-db-reader");
+const bookmarksReader = require("./bookmarks-reader");
+
+module.exports = {
+  BrowserHistoryChromeAdapter,
+  BROWSER_HISTORY_CHROME_NAME,
+  BROWSER_HISTORY_CHROME_VERSION,
+  defaultChromeProfileDir: dbReader.defaultChromeProfileDir,
+  copyHistorySnapshot: dbReader.copyHistorySnapshot,
+  cleanupHistorySnapshot: dbReader.cleanupHistorySnapshot,
+  readVisits: dbReader.readVisits,
+  readBookmarks: bookmarksReader.readBookmarks,
+  webkitUsToEpochMs: dbReader.webkitUsToEpochMs,
+  epochMsToWebkitUs: dbReader.epochMsToWebkitUs,
+  decodeTransition: dbReader.decodeTransition,
+};

package/lib/adapters/browser-history-edge/adapter.js

@@ -0,0 +1,34 @@
+"use strict";
+
+// BrowserHistoryEdgeAdapter — Microsoft Edge is Chromium under the hood,
+// so the History SQLite schema + Bookmarks JSON layout are byte-identical
+// to Chrome. Only the on-disk profile root differs. Subclass the Chrome
+// adapter and override the browser config; everything else (copy-then-read
+// snapshot, bookmark walker, normalize, etc.) is inherited.
+
+const {
+  BrowserHistoryChromeAdapter,
+} = require("../browser-history-chrome/adapter");
+const {
+  defaultEdgeProfileDir,
+} = require("../browser-history-chrome/chrome-db-reader");
+
+const NAME = "browser-history-edge";
+const VERSION = "0.1.0";
+
+class BrowserHistoryEdgeAdapter extends BrowserHistoryChromeAdapter {
+  _browserConfig() {
+    return {
+      name: NAME,
+      version: VERSION,
+      browser: "edge",
+      defaultProfileDir: defaultEdgeProfileDir,
+    };
+  }
+}
+
+module.exports = {
+  BrowserHistoryEdgeAdapter,
+  BROWSER_HISTORY_EDGE_NAME: NAME,
+  BROWSER_HISTORY_EDGE_VERSION: VERSION,
+};

package/lib/adapters/browser-history-edge/index.js

@@ -0,0 +1,13 @@
+"use strict";
+
+const {
+  BrowserHistoryEdgeAdapter,
+  BROWSER_HISTORY_EDGE_NAME,
+  BROWSER_HISTORY_EDGE_VERSION,
+} = require("./adapter");
+
+module.exports = {
+  BrowserHistoryEdgeAdapter,
+  BROWSER_HISTORY_EDGE_NAME,
+  BROWSER_HISTORY_EDGE_VERSION,
+};