@chainlesschain/personal-data-hub 0.1.0 → 0.2.0

package/lib/adapters/system-data/system-data-adapter.js

@@ -0,0 +1,344 @@
+/**
+ * SystemDataAdapter — Android system data (contacts / call log / SMS / WiFi).
+ *
+ * Phase 4.5.5. Sits on top of the forensics-bridge sidecar.
+ *
+ * Per-source pipeline (each one independent — disabling SMS doesn't break the
+ * others):
+ *
+ *   contacts:  android.pull_file → system.parse_contacts → Person stream
+ *   calllog:   android.pull_file → system.parse_calllog → Event(call) + Person stream
+ *   sms:       android.pull_file → system.parse_sms     → Event(message) + Person stream
+ *   wifi:      android.pull_file → system.parse_wifi    → Place stream
+ *
+ * Or, when `opts.dataPaths` is provided (e.g. user already adb-pulled files
+ * manually, or testing with a local fixture), skip the pull step.
+ *
+ * Privacy gating: `opts.include` decides which sub-sources run. Default per
+ * Adapter_System_Data.md §5.1 + OQ-SD1: contacts ON / calllog ON / sms OFF /
+ * wifi ON. The UI dialog re-confirms this on each sync.
+ */
+
+"use strict";
+
+const path = require("node:path");
+const os = require("node:os");
+const fs = require("node:fs");
+
+const { PythonSidecarAdapter } = require("../_python-sidecar-base");
+
+const NAME = "system-data";
+const VERSION = "0.1.0";
+
+const DEFAULT_INCLUDE = Object.freeze({
+  contacts: true,
+  calllog: true,
+  sms: false, // opt-out by default — see Adapter_System_Data.md §5.1
+  wifi: true,
+});
+
+/**
+ * Default Android system provider paths. Override via opts.remotePaths when
+ * a device uses a non-stock layout.
+ */
+const DEFAULT_REMOTE_PATHS = Object.freeze({
+  contacts:
+    "/data/data/com.android.providers.contacts/databases/contacts2.db",
+  calllog: "/data/data/com.android.providers.contacts/databases/calllog.db",
+  sms: "/data/data/com.android.providers.telephony/databases/mmssms.db",
+  wifi: "/data/misc/wifi/", // directory — pull_file works for one file, so wifi mode-A is dataPaths
+});
+
+/**
+ * Per-source workaround paths under /sdcard/Download/ for stock Android
+ * (no `adb root`) — user copies files via Termux + tsu or MT Manager.
+ */
+const SDCARD_WORKAROUND_PATHS = Object.freeze({
+  contacts: "/sdcard/Download/contacts2.db",
+  calllog: "/sdcard/Download/calllog.db",
+  sms: "/sdcard/Download/mmssms.db",
+  wifi_xml: "/sdcard/Download/WifiConfigStore.xml",
+  wifi_conf: "/sdcard/Download/wpa_supplicant.conf",
+});
+
+class SystemDataAdapter extends PythonSidecarAdapter {
+  constructor(opts) {
+    super(opts);
+    this.name = NAME;
+    this.version = VERSION;
+    this.capabilities = [
+      "sync:android-adb",
+      "sync:android-sdcard-workaround",
+      "sync:host-dataPaths",
+    ];
+    this.rateLimits = { perDay: 12 }; // system data day-to-day churn is small
+    this.dataDisclosure = {
+      fields: [
+        "contacts:name,phone,email,organization,notes,starred,photoUri",
+        "calllog:number,duration,timestamp,type,name",
+        "sms:address,body,timestamp,type,threadId,isRead",
+        "wifi:ssid,securityType,hidden",
+        // Explicitly NOT collected:
+        //   - wifi:password (never written to vault, even when present in source)
+      ],
+      sensitivity: "high", // SMS may include third-party content
+      legalGate: true, // requires explicit user agreement on third-party content
+      retentionDays: undefined, // user-controlled (no default cap)
+      notice:
+        "短信和通话记录可能包含他人电话号码或对话内容；所有数据在本机加密存储，不向任何服务器上传（含 AI 分析）。",
+      defaultInclude: { ...DEFAULT_INCLUDE },
+    };
+  }
+
+  // -----------------------------------------------------------------------
+  // PersonalDataAdapter — authenticate / healthCheck override
+  // -----------------------------------------------------------------------
+
+  /**
+   * Verify the sidecar is reachable AND there is at least one usable ADB
+   * device (unless caller signals offline-import mode by passing dataPaths).
+   *
+   * @param {object} ctx
+   * @param {object} [ctx.dataPaths] If set, ADB presence is not required.
+   * @param {string} [ctx.serial]    Optional serial; auth checks just that device.
+   */
+  async authenticate(ctx = {}) {
+    const pong = await this.supervisor.invoke("sidecar.ping", {}, { timeoutMs: 3000 });
+    if (ctx.dataPaths && Object.keys(ctx.dataPaths).length > 0) {
+      return { ok: true, mode: "offline", sidecarVersion: pong.version };
+    }
+    let devices;
+    try {
+      const out = await this.supervisor.invoke("android.list_devices", {}, { timeoutMs: 5000 });
+      devices = out.devices || [];
+    } catch (err) {
+      return {
+        ok: false,
+        reason: `android.list_devices failed: ${err.code || err.message}`,
+      };
+    }
+    const wanted = ctx.serial
+      ? devices.filter((d) => d.serial === ctx.serial)
+      : devices.filter((d) => d.state === "device");
+    if (wanted.length === 0) {
+      return {
+        ok: false,
+        reason: ctx.serial
+          ? `device "${ctx.serial}" not found or not authorized`
+          : "no authorized ADB devices attached",
+      };
+    }
+    return { ok: true, mode: "device", devices: wanted };
+  }
+
+  // -----------------------------------------------------------------------
+  // Orchestration (subclass hook)
+  // -----------------------------------------------------------------------
+
+  /**
+   * Orchestrate the 4 sub-sources sequentially.
+   *
+   * @param {object} opts
+   * @param {object} [opts.include]      Per-source enable flags (defaults: DEFAULT_INCLUDE).
+   * @param {string} [opts.serial]       Required when pulling from a live device.
+   * @param {object} [opts.dataPaths]    Pre-extracted host paths, keys:
+   *                                       {contacts, calllog, sms, wifi}.
+   * @param {object} [opts.remotePaths]  Override default device paths.
+   * @param {"normal"|"sdcard"} [opts.extractMode]
+   *                                     "normal" = pull from /data/data (root only),
+   *                                     "sdcard" = pull from /sdcard/Download (workaround).
+   * @param {string} [opts.scratchDir]   Directory for pulled DBs. Default: hub tmp.
+   * @param {(msg: object) => void} [opts.onProgress]  Forwarded as adapter-progress.
+   */
+  async _runSidecar(opts, emit) {
+    const include = { ...DEFAULT_INCLUDE, ...(opts.include || {}) };
+    const dataPaths = opts.dataPaths || {};
+    const extractMode = opts.extractMode || "normal";
+    const remotePaths =
+      extractMode === "sdcard"
+        ? {
+            contacts: SDCARD_WORKAROUND_PATHS.contacts,
+            calllog: SDCARD_WORKAROUND_PATHS.calllog,
+            sms: SDCARD_WORKAROUND_PATHS.sms,
+            wifi: SDCARD_WORKAROUND_PATHS.wifi_xml,
+          }
+        : { ...DEFAULT_REMOTE_PATHS, ...(opts.remotePaths || {}) };
+
+    const scratchDir =
+      opts.scratchDir ||
+      fs.mkdtempSync(path.join(os.tmpdir(), "system-data-sync-"));
+    fs.mkdirSync(scratchDir, { recursive: true });
+
+    const onProgress = typeof opts.onProgress === "function" ? opts.onProgress : null;
+    const tellProgress = (source, phase, extra = {}) => {
+      if (onProgress) onProgress({ source, phase, ...extra });
+    };
+
+    const sourcesRun = [];
+
+    // ─── Contacts ────────────────────────────────────────────────────────
+    let contactsLocal = dataPaths.contacts || null;
+    if (include.contacts) {
+      if (!contactsLocal) {
+        if (!opts.serial) {
+          throw new Error(
+            "system-data: contacts enabled but no serial/dataPaths.contacts provided",
+          );
+        }
+        tellProgress("contacts", "pulling");
+        const pulled = await this.supervisor.invoke(
+          "android.pull_file",
+          {
+            serial: opts.serial,
+            remote_path: remotePaths.contacts,
+            local_dir: scratchDir,
+          },
+          { timeoutMs: 60_000 },
+        );
+        contactsLocal = pulled.local;
+      }
+      tellProgress("contacts", "parsing", { dbPath: contactsLocal });
+      const r = await this.supervisor.invoke(
+        "system.parse_contacts",
+        { data_path: contactsLocal, device_serial: opts.serial || null },
+        {
+          timeoutMs: 120_000,
+          onChunk: (batch) => this._emitChunkAsRaws(batch, emit),
+          onProgress: (p) => tellProgress("contacts", "progress", p),
+        },
+      );
+      sourcesRun.push({ source: "contacts", ...r });
+    }
+
+    // ─── Call log ────────────────────────────────────────────────────────
+    if (include.calllog) {
+      let calllogLocal = dataPaths.calllog || null;
+      if (!calllogLocal) {
+        if (!opts.serial) {
+          throw new Error("system-data: calllog enabled but no serial/dataPaths.calllog");
+        }
+        tellProgress("calllog", "pulling");
+        try {
+          const pulled = await this.supervisor.invoke(
+            "android.pull_file",
+            {
+              serial: opts.serial,
+              remote_path: remotePaths.calllog,
+              local_dir: scratchDir,
+            },
+            { timeoutMs: 60_000 },
+          );
+          calllogLocal = pulled.local;
+        } catch (err) {
+          // Calls table may live in contacts2.db on pre-Android-11 builds.
+          if (err.code === "EXTRACT_PERMISSION_DENIED" && contactsLocal) {
+            calllogLocal = contactsLocal;
+          } else {
+            throw err;
+          }
+        }
+      }
+      tellProgress("calllog", "parsing", { dbPath: calllogLocal });
+      const r = await this.supervisor.invoke(
+        "system.parse_calllog",
+        {
+          data_path: calllogLocal,
+          contacts_db_path: contactsLocal,
+          device_serial: opts.serial || null,
+        },
+        {
+          timeoutMs: 180_000,
+          onChunk: (batch) => this._emitChunkAsRaws(batch, emit),
+          onProgress: (p) => tellProgress("calllog", "progress", p),
+        },
+      );
+      sourcesRun.push({ source: "calllog", ...r });
+    }
+
+    // ─── SMS ────────────────────────────────────────────────────────────
+    if (include.sms) {
+      let smsLocal = dataPaths.sms || null;
+      if (!smsLocal) {
+        if (!opts.serial) {
+          throw new Error("system-data: sms enabled but no serial/dataPaths.sms");
+        }
+        tellProgress("sms", "pulling");
+        const pulled = await this.supervisor.invoke(
+          "android.pull_file",
+          {
+            serial: opts.serial,
+            remote_path: remotePaths.sms,
+            local_dir: scratchDir,
+          },
+          { timeoutMs: 60_000 },
+        );
+        smsLocal = pulled.local;
+      }
+      tellProgress("sms", "parsing", { dbPath: smsLocal });
+      const r = await this.supervisor.invoke(
+        "system.parse_sms",
+        {
+          data_path: smsLocal,
+          contacts_db_path: contactsLocal,
+          device_serial: opts.serial || null,
+        },
+        {
+          timeoutMs: 300_000, // SMS can be 10K+ rows on long-term devices
+          onChunk: (batch) => this._emitChunkAsRaws(batch, emit),
+          onProgress: (p) => tellProgress("sms", "progress", p),
+        },
+      );
+      sourcesRun.push({ source: "sms", ...r });
+    }
+
+    // ─── WiFi ───────────────────────────────────────────────────────────
+    if (include.wifi) {
+      let wifiLocal = dataPaths.wifi || null;
+      if (!wifiLocal) {
+        // WiFi config is a single file, but two possible names. Prefer XML.
+        if (!opts.serial) {
+          throw new Error("system-data: wifi enabled but no serial/dataPaths.wifi");
+        }
+        tellProgress("wifi", "pulling");
+        try {
+          const pulled = await this.supervisor.invoke(
+            "android.pull_file",
+            {
+              serial: opts.serial,
+              remote_path: remotePaths.wifi,
+              local_dir: scratchDir,
+            },
+            { timeoutMs: 30_000 },
+          );
+          wifiLocal = path.dirname(pulled.local);
+        } catch (err) {
+          // Non-fatal — wifi often inaccessible without root. Skip this source.
+          tellProgress("wifi", "skipped", { reason: err.code || err.message });
+          return { sources: sourcesRun, scratchDir };
+        }
+      }
+      tellProgress("wifi", "parsing", { dbPath: wifiLocal });
+      const r = await this.supervisor.invoke(
+        "system.parse_wifi",
+        { data_path: wifiLocal, device_serial: opts.serial || null },
+        {
+          timeoutMs: 30_000,
+          onChunk: (batch) => this._emitChunkAsRaws(batch, emit),
+          onProgress: (p) => tellProgress("wifi", "progress", p),
+        },
+      );
+      sourcesRun.push({ source: "wifi", ...r });
+    }
+
+    return { sources: sourcesRun, scratchDir };
+  }
+}
+
+module.exports = {
+  SystemDataAdapter,
+  SYSTEM_DATA_ADAPTER_NAME: NAME,
+  SYSTEM_DATA_ADAPTER_VERSION: VERSION,
+  DEFAULT_INCLUDE,
+  DEFAULT_REMOTE_PATHS,
+  SDCARD_WORKAROUND_PATHS,
+};

package/lib/adapters/travel-12306/index.js

@@ -0,0 +1,151 @@
+/**
+ * Phase 9.2 — 12306 (China Railway) ticket adapter.
+ *
+ * Source format: 12306 doesn't have an official user export. We accept
+ * two file formats:
+ *   1. order-confirmation emails (already adapter-parsed by Phase 5 +
+ *      Phase 5.4 travel template). Phase 9.2 reads those events back
+ *      out of the vault and **re-normalizes** them into the
+ *      adapter-neutral travel schema. This is the "rich vault →
+ *      enrich" pattern.
+ *   2. user-uploaded JSON dump (e.g. exported from a 3rd-party 12306
+ *      scraper, or hand-curated). Optional.
+ *
+ * For v0.5 we focus on (2) since (1) is purely vault-side derivation
+ * the AnalysisEngine can do at query time.
+ */
+
+"use strict";
+
+const fs = require("node:fs");
+const { normalizeTravelRecord, parseChineseDateTime } = require("../travel-base");
+
+const NAME = "travel-12306";
+const VERSION = "0.5.0";
+
+class Train12306Adapter {
+  constructor(opts = {}) {
+    if (!opts.account || !opts.account.username) {
+      throw new Error("Train12306Adapter: opts.account.username required (12306 user id)");
+    }
+    this.account = opts.account;
+    this._dataPath = opts.dataPath || null;
+
+    this.name = NAME;
+    this.version = VERSION;
+    this.capabilities = ["import:json", "parse:12306-orders"];
+    this.extractMode = "file-import";
+    this.rateLimits = {};
+    this.dataDisclosure = {
+      fields: [
+        "12306:orderId / passengerName / trainNumber / fromStation / toStation / departureTime / arrivalTime / seat / price",
+      ],
+      sensitivity: "medium",
+      legalGate: false,
+    };
+  }
+
+  async authenticate() {
+    return { ok: true, account: this.account.username };
+  }
+
+  async healthCheck() {
+    return { ok: true, lastChecked: Date.now() };
+  }
+
+  async *sync(opts = {}) {
+    const dataPath = opts.dataPath || this._dataPath;
+    if (!dataPath || !fs.existsSync(dataPath)) return;
+    const buf = fs.readFileSync(dataPath, "utf-8");
+    let records;
+    try {
+      records = parseRecords(buf);
+    } catch (err) {
+      throw new Error(`Train12306Adapter: parse failed: ${err.message}`);
+    }
+    for (const r of records) {
+      yield {
+        adapter: NAME,
+        originalId: String(r.recordId || r.orderId || r.ticketNumber),
+        capturedAt: r.bookedAt || r.departureMs || Date.now(),
+        payload: { record: r },
+      };
+    }
+  }
+
+  normalize(raw) {
+    if (!raw || !raw.payload || !raw.payload.record) {
+      throw new Error("Train12306Adapter.normalize: raw.payload.record missing");
+    }
+    return normalizeTravelRecord(raw.payload.record, {
+      adapterName: NAME,
+      adapterVersion: VERSION,
+    });
+  }
+}
+
+/**
+ * Parse a 12306 dump file. Accepts either:
+ *   - JSON array of order objects
+ *   - JSON object { orders: [...] }
+ *   - JSONL (one order per line)
+ */
+function parseRecords(text) {
+  let raw;
+  try {
+    raw = JSON.parse(text);
+  } catch (_e) {
+    // Try JSONL
+    raw = text
+      .split(/\r?\n/)
+      .filter((l) => l.trim().startsWith("{"))
+      .map((l) => JSON.parse(l));
+  }
+  const orders = Array.isArray(raw) ? raw : raw.orders || [];
+  return orders.map(orderToRecord).filter(Boolean);
+}
+
+function orderToRecord(o) {
+  if (!o || typeof o !== "object") return null;
+  const recordId = o.orderId || o.ticketNumber || o.id || o.order_no;
+  if (!recordId) return null;
+  return {
+    vendorId: "12306",
+    recordId: String(recordId),
+    vehicleType: "train",
+    from: {
+      station: o.fromStation || o.from_station || o.from,
+      city: o.fromCity || o.from_city,
+    },
+    to: {
+      station: o.toStation || o.to_station || o.to,
+      city: o.toCity || o.to_city,
+    },
+    departureMs: numberOrParse(o.departureTime || o.departure_time || o.start_time),
+    arrivalMs: numberOrParse(o.arrivalTime || o.arrival_time || o.end_time),
+    carrier: "12306",
+    vehicleNumber: o.trainNumber || o.train_no || o.trainNo,
+    totalCost: o.price != null
+      ? { value: parseFloat(o.price), currency: "CNY" }
+      : null,
+    traveler: o.passengerName || o.passenger || o.name,
+    confirmationCode: o.ticketNumber || o.ticket_no || recordId,
+    bookedAt: numberOrParse(o.bookedAt || o.order_time),
+    extras: {
+      seat: o.seat || o.seatType,
+      seatNumber: o.seatNumber || o.seat_number,
+      idCardLast6: o.idLast6 || undefined, // for cross-source EntityResolver linking
+    },
+  };
+}
+
+function numberOrParse(v) {
+  if (Number.isFinite(v)) return v;
+  if (typeof v === "string") {
+    if (/^\d+$/.test(v)) return parseInt(v, 10);
+    return parseChineseDateTime(v);
+  }
+  return null;
+}
+
+module.exports = { Train12306Adapter, parseRecords, NAME, VERSION };

package/lib/adapters/travel-amap/index.js

@@ -0,0 +1,164 @@
+/**
+ * Phase 9.4 — Amap (高德地图) location history adapter.
+ *
+ * Source: Amap stores recent navigation / search history in app-local
+ * SQLite DBs. Per sjqz/parsers/amap.py, the relevant tables are:
+ *   - history_search  (search queries)
+ *   - history_route   (planned routes)
+ *   - favourites      (saved locations like 公司 / 家)
+ *
+ * Adapter extractMode is "device-pull" — relies on Phase 7.5
+ * AndroidExtractor to pull the .db files from Amap's app-private
+ * directory. For v0.5 we accept a pre-pulled local path (file-import
+ * fallback) so users without root can hand-extract via adb backup.
+ */
+
+"use strict";
+
+const fs = require("node:fs");
+const { normalizeTravelRecord, parseChineseDateTime } = require("../travel-base");
+
+const NAME = "travel-amap";
+const VERSION = "0.5.0";
+
+class AmapAdapter {
+  constructor(opts = {}) {
+    if (!opts.account || !opts.account.deviceId) {
+      throw new Error("AmapAdapter: opts.account.deviceId required");
+    }
+    this.account = opts.account;
+    this._dbPath = opts.dbPath || null;
+    this._dbDriverFactory = opts.dbDriverFactory || null;
+
+    this.name = NAME;
+    this.version = VERSION;
+    this.capabilities = ["sync:sqlite", "parse:amap-history"];
+    this.extractMode = "device-pull";
+    this.rateLimits = {};
+    this.dataDisclosure = {
+      fields: [
+        "amap:search_history (query / time / location)",
+        "amap:route_history (from / to / mode / time)",
+        "amap:favourites (name / address / coords)",
+      ],
+      sensitivity: "medium",
+      legalGate: false,
+    };
+  }
+
+  async authenticate() {
+    return { ok: true, account: this.account.deviceId };
+  }
+
+  async healthCheck() {
+    return { ok: true, lastChecked: Date.now() };
+  }
+
+  async *sync(opts = {}) {
+    const dbPath = opts.dbPath || this._dbPath;
+    if (!dbPath || !fs.existsSync(dbPath)) return;
+    const Database = this._dbDriverFactory || (() => require("better-sqlite3-multiple-ciphers"));
+    const Driver = typeof Database === "function" ? Database() : Database;
+    const db = new Driver(dbPath, { readonly: true });
+
+    try {
+      // History routes (most analytically valuable)
+      const routes = trySelect(db, "SELECT * FROM history_route LIMIT 5000")
+        || trySelect(db, "SELECT * FROM ROUTE_HISTORY LIMIT 5000")
+        || [];
+      for (const r of routes) {
+        const rec = routeRowToRecord(r);
+        if (rec) {
+          yield {
+            adapter: NAME,
+            originalId: rec.recordId,
+            capturedAt: rec.bookedAt || Date.now(),
+            payload: { record: rec, kind: "route" },
+          };
+        }
+      }
+      // History search (queries — produce trip events of type "visit")
+      const searches = trySelect(db, "SELECT * FROM history_search LIMIT 5000") || [];
+      for (const r of searches) {
+        const rec = searchRowToRecord(r);
+        if (rec) {
+          yield {
+            adapter: NAME,
+            originalId: rec.recordId,
+            capturedAt: rec.bookedAt || Date.now(),
+            payload: { record: rec, kind: "search" },
+          };
+        }
+      }
+    } finally {
+      try { db.close(); } catch (_e) {}
+    }
+  }
+
+  normalize(raw) {
+    if (!raw || !raw.payload || !raw.payload.record) {
+      throw new Error("AmapAdapter.normalize: raw.payload.record missing");
+    }
+    return normalizeTravelRecord(raw.payload.record, {
+      adapterName: NAME,
+      adapterVersion: VERSION,
+    });
+  }
+}
+
+function trySelect(db, sql) {
+  try {
+    return db.prepare(sql).all();
+  } catch (_e) {
+    return null;
+  }
+}
+
+function routeRowToRecord(row) {
+  if (!row) return null;
+  const id = row.id || row._id || row.uid || row.guid;
+  if (!id) return null;
+  return {
+    vendorId: "amap",
+    recordId: `route-${id}`,
+    vehicleType: row.mode === "drive" ? "car" : (row.mode || "trip"),
+    from: { name: row.from_name || row.fromName || row.start, lat: row.from_lat || null, lng: row.from_lng || null },
+    to: { name: row.to_name || row.toName || row.dest, lat: row.to_lat || null, lng: row.to_lng || null },
+    departureMs: numberOrParse(row.time || row.create_time || row.start_time),
+    carrier: "高德地图",
+    extras: { mode: row.mode },
+  };
+}
+
+function searchRowToRecord(row) {
+  if (!row) return null;
+  const id = row.id || row._id || row.guid;
+  if (!id) return null;
+  // Search = a "visit" intent
+  return {
+    vendorId: "amap",
+    recordId: `search-${id}`,
+    vehicleType: "visit",
+    to: { name: row.keyword || row.query || row.poiname, lat: row.lat || null, lng: row.lng || null, city: row.city },
+    departureMs: numberOrParse(row.time || row.create_time),
+    carrier: "高德地图",
+    extras: { query: row.keyword || row.query },
+  };
+}
+
+function numberOrParse(v) {
+  if (Number.isFinite(v)) {
+    // Amap timestamps are sometimes seconds — heuristic upgrade to ms
+    return v > 1e12 ? v : (v > 1e10 ? v : v * 1000);
+  }
+  if (typeof v === "string") {
+    if (/^\d+$/.test(v)) {
+      const n = parseInt(v, 10);
+      return n > 1e12 ? n : (n > 1e10 ? n : n * 1000);
+    }
+    return parseChineseDateTime(v);
+  }
+  return null;
+}
+
+module.exports = { AmapAdapter, NAME, VERSION };