@chainlesschain/personal-data-hub 0.1.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/__tests__/adapters/ai-chat-cookie-capture-spec.test.js +211 -0
- package/__tests__/adapters/ai-chat-health-checker.test.js +262 -0
- package/__tests__/adapters/ai-chat-history.test.js +396 -0
- package/__tests__/adapters/ai-chat-http-client.test.js +242 -0
- package/__tests__/adapters/ai-chat-vendors.test.js +874 -0
- package/__tests__/adapters/alipay-bill-adapter.test.js +538 -0
- package/__tests__/adapters/email-adapter.test.js +138 -1
- package/__tests__/adapters/email-classifier.test.js +347 -0
- package/__tests__/adapters/email-pdf-extractor.test.js +529 -0
- package/__tests__/adapters/email-retry-progress.test.js +294 -0
- package/__tests__/adapters/email-templates.test.js +699 -0
- package/__tests__/adapters/social-toutiao-kuaishou-scaffold.test.js +269 -0
- package/__tests__/adapters/system-data-adapter.test.js +440 -0
- package/__tests__/adapters/system-data-android-ingest.test.js +144 -0
- package/__tests__/adapters/system-data-android.test.js +387 -0
- package/__tests__/adapters/system-data-disclosure.test.js +153 -0
- package/__tests__/adapters/wechat-bootstrap.test.js +240 -0
- package/__tests__/adapters/wechat-env-probe.test.js +162 -0
- package/__tests__/adapters/wechat-frida-agent.test.js +191 -0
- package/__tests__/adapters/wechat-frida-integration.test.js +149 -0
- package/__tests__/adapters/wechat-frida-key-provider.test.js +188 -0
- package/__tests__/adapters/wechat-md5-key-provider.test.js +101 -0
- package/__tests__/analysis-skills.test.js +556 -0
- package/__tests__/analysis.test.js +329 -1
- package/__tests__/e2e/ai-chat-cross-source-journey.test.js +213 -0
- package/__tests__/e2e/full-user-journey.test.js +188 -0
- package/__tests__/entity-resolver-ingest-hook.test.js +177 -0
- package/__tests__/entity-resolver-stages.test.js +411 -0
- package/__tests__/entity-resolver-vault.test.js +246 -0
- package/__tests__/entity-resolver.test.js +526 -0
- package/__tests__/fixtures/entity-resolver-200-mock.json +96 -0
- package/__tests__/integration/ai-chat-history-registry.test.js +228 -0
- package/__tests__/integration/aichat-wizard-end-to-end.test.js +282 -0
- package/__tests__/integration/cross-adapter-pipelines.test.js +396 -0
- package/__tests__/integration/wechat-bootstrap-end-to-end.test.js +390 -0
- package/__tests__/longtail-adapters.test.js +217 -0
- package/__tests__/mobile-extractor.test.js +288 -0
- package/__tests__/registry.test.js +4 -2
- package/__tests__/shopping-adapters.test.js +296 -0
- package/__tests__/sidecar-contacts-cross-validate.test.js +163 -0
- package/__tests__/sidecar-supervisor.test.js +120 -0
- package/__tests__/social-adapters.test.js +206 -0
- package/__tests__/travel-adapters.test.js +325 -0
- package/__tests__/vault.test.js +3 -3
- package/__tests__/wechat-adapter.test.js +476 -0
- package/__tests__/whatsapp-adapter.test.js +135 -0
- package/lib/adapter-spec.js +12 -0
- package/lib/adapters/_python-sidecar-base.js +207 -0
- package/lib/adapters/ai-chat-history/ai-chat-adapter.js +374 -0
- package/lib/adapters/ai-chat-history/cookie-auth.js +109 -0
- package/lib/adapters/ai-chat-history/cookie-capture-spec.js +331 -0
- package/lib/adapters/ai-chat-history/health-checker.js +210 -0
- package/lib/adapters/ai-chat-history/http-client.js +211 -0
- package/lib/adapters/ai-chat-history/index.js +28 -0
- package/lib/adapters/ai-chat-history/schema-map.js +258 -0
- package/lib/adapters/ai-chat-history/vendor-spec.js +86 -0
- package/lib/adapters/ai-chat-history/vendors/coze.js +179 -0
- package/lib/adapters/ai-chat-history/vendors/deepseek.js +199 -0
- package/lib/adapters/ai-chat-history/vendors/doubao.js +255 -0
- package/lib/adapters/ai-chat-history/vendors/dreamina.js +174 -0
- package/lib/adapters/ai-chat-history/vendors/hunyuan.js +176 -0
- package/lib/adapters/ai-chat-history/vendors/kimi.js +182 -0
- package/lib/adapters/ai-chat-history/vendors/qianfan.js +160 -0
- package/lib/adapters/ai-chat-history/vendors/tongyi.js +193 -0
- package/lib/adapters/ai-chat-history/vendors/zhipu.js +202 -0
- package/lib/adapters/ai-chat-history/wizard-controller.js +473 -0
- package/lib/adapters/alipay-bill/alipay-bill-adapter.js +311 -0
- package/lib/adapters/alipay-bill/counterparty.js +129 -0
- package/lib/adapters/alipay-bill/csv-parser.js +217 -0
- package/lib/adapters/alipay-bill/index.js +41 -0
- package/lib/adapters/alipay-bill/zip-decryptor.js +111 -0
- package/lib/adapters/email-imap/classifier.js +495 -0
- package/lib/adapters/email-imap/email-adapter.js +419 -8
- package/lib/adapters/email-imap/index.js +42 -0
- package/lib/adapters/email-imap/pdf-extractor.js +192 -0
- package/lib/adapters/email-imap/templates/bill.js +232 -0
- package/lib/adapters/email-imap/templates/government.js +120 -0
- package/lib/adapters/email-imap/templates/index.js +78 -0
- package/lib/adapters/email-imap/templates/order.js +186 -0
- package/lib/adapters/email-imap/templates/other.js +114 -0
- package/lib/adapters/email-imap/templates/register.js +113 -0
- package/lib/adapters/email-imap/templates/travel.js +157 -0
- package/lib/adapters/email-imap/templates/utils.js +275 -0
- package/lib/adapters/email-imap/transactions.js +234 -0
- package/lib/adapters/messaging-qq/index.js +158 -0
- package/lib/adapters/messaging-telegram/index.js +142 -0
- package/lib/adapters/messaging-whatsapp/index.js +189 -0
- package/lib/adapters/shopping-base/index.js +208 -0
- package/lib/adapters/shopping-jd/index.js +150 -0
- package/lib/adapters/shopping-meituan/index.js +154 -0
- package/lib/adapters/shopping-taobao/index.js +176 -0
- package/lib/adapters/social-bilibili/index.js +171 -0
- package/lib/adapters/social-douyin/index.js +116 -0
- package/lib/adapters/social-kuaishou/index.js +237 -0
- package/lib/adapters/social-toutiao/index.js +236 -0
- package/lib/adapters/social-weibo/index.js +164 -0
- package/lib/adapters/social-xiaohongshu/index.js +96 -0
- package/lib/adapters/system-data/disclosure.js +166 -0
- package/lib/adapters/system-data/index.js +34 -0
- package/lib/adapters/system-data/system-data-adapter.js +344 -0
- package/lib/adapters/system-data-android/adapter.js +348 -0
- package/lib/adapters/system-data-android/index.js +76 -0
- package/lib/adapters/travel-12306/index.js +151 -0
- package/lib/adapters/travel-amap/index.js +164 -0
- package/lib/adapters/travel-baidu-map/index.js +162 -0
- package/lib/adapters/travel-base/index.js +240 -0
- package/lib/adapters/travel-ctrip/index.js +151 -0
- package/lib/adapters/wechat/bootstrap.js +146 -0
- package/lib/adapters/wechat/content-parser.js +326 -0
- package/lib/adapters/wechat/db-reader.js +209 -0
- package/lib/adapters/wechat/env-probe.js +218 -0
- package/lib/adapters/wechat/frida-agent/loader.js +67 -0
- package/lib/adapters/wechat/frida-agent/wechat-key-hook.js +126 -0
- package/lib/adapters/wechat/index.js +37 -0
- package/lib/adapters/wechat/key-extractor.js +158 -0
- package/lib/adapters/wechat/key-providers/frida-key-provider.js +244 -0
- package/lib/adapters/wechat/key-providers/index.js +22 -0
- package/lib/adapters/wechat/key-providers/key-provider-base.js +44 -0
- package/lib/adapters/wechat/key-providers/md5-key-provider.js +81 -0
- package/lib/adapters/wechat/normalize.js +220 -0
- package/lib/adapters/wechat/wechat-adapter.js +205 -0
- package/lib/analysis-skills/base.js +113 -0
- package/lib/analysis-skills/footprint.js +167 -0
- package/lib/analysis-skills/index.js +58 -0
- package/lib/analysis-skills/interests.js +161 -0
- package/lib/analysis-skills/relations.js +226 -0
- package/lib/analysis-skills/spending.js +219 -0
- package/lib/analysis-skills/timeline.js +167 -0
- package/lib/analysis.js +191 -2
- package/lib/entity-resolver/embedding-stage.js +198 -0
- package/lib/entity-resolver/entity-resolver.js +384 -0
- package/lib/entity-resolver/index.js +42 -0
- package/lib/entity-resolver/llm-stage.js +191 -0
- package/lib/entity-resolver/rule-stage.js +208 -0
- package/lib/entity-resolver/worker.js +149 -0
- package/lib/index.js +131 -0
- package/lib/migrations.js +73 -0
- package/lib/mobile-extractor/android.js +193 -0
- package/lib/mobile-extractor/index.js +9 -0
- package/lib/mobile-extractor/ios.js +223 -0
- package/lib/prompt-builder.js +11 -1
- package/lib/query-parser.js +7 -1
- package/lib/registry.js +42 -0
- package/lib/sidecar/index.js +15 -0
- package/lib/sidecar/supervisor.js +359 -0
- package/lib/vault.js +343 -0
- package/package.json +36 -3
- package/scripts/_make-fixture-all.js +126 -0
- package/scripts/_make-fixture-contacts.js +84 -0
- package/scripts/evaluate-entity-resolver.js +213 -0
- package/scripts/smoke-phase-5-5.js +196 -0
- package/scripts/smoke-phase-5-7.js +181 -0
- package/scripts/smoke-system-data-contacts.js +309 -0
- package/scripts/smoke-system-data.js +312 -0
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Phase 13.4 — Xiaohongshu (小红书) adapter.
|
|
3
|
+
*
|
|
4
|
+
* Per sjqz/parsers/lifestyle.py XiaohongshuParser. Tables:
|
|
5
|
+
* - note / browse_history viewed notes
|
|
6
|
+
* - liked_note / favourite collected notes
|
|
7
|
+
* - search_history queries
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
"use strict";
|
|
11
|
+
|
|
12
|
+
const fs = require("node:fs");
|
|
13
|
+
const { newId } = require("../../ids");
|
|
14
|
+
|
|
15
|
+
const NAME = "social-xiaohongshu";
|
|
16
|
+
const VERSION = "0.5.0";
|
|
17
|
+
|
|
18
|
+
class XiaohongshuAdapter {
|
|
19
|
+
constructor(opts = {}) {
|
|
20
|
+
if (!opts.account || !opts.account.uid) {
|
|
21
|
+
throw new Error("XiaohongshuAdapter: opts.account.uid required");
|
|
22
|
+
}
|
|
23
|
+
this.account = opts.account;
|
|
24
|
+
this._dbPath = opts.dbPath || null;
|
|
25
|
+
this._dbDriverFactory = opts.dbDriverFactory || null;
|
|
26
|
+
|
|
27
|
+
this.name = NAME;
|
|
28
|
+
this.version = VERSION;
|
|
29
|
+
this.capabilities = ["sync:sqlite", "parse:xhs-history"];
|
|
30
|
+
this.extractMode = "device-pull";
|
|
31
|
+
this.rateLimits = {};
|
|
32
|
+
this.dataDisclosure = {
|
|
33
|
+
fields: ["xhs:viewed_notes / liked / favourites / search_history"],
|
|
34
|
+
sensitivity: "medium",
|
|
35
|
+
legalGate: false,
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
async authenticate() { return { ok: true, account: this.account.uid }; }
|
|
40
|
+
async healthCheck() { return { ok: true, lastChecked: Date.now() }; }
|
|
41
|
+
|
|
42
|
+
async *sync(opts = {}) {
|
|
43
|
+
const dbPath = opts.dbPath || this._dbPath;
|
|
44
|
+
if (!dbPath || !fs.existsSync(dbPath)) return;
|
|
45
|
+
const Driver = this._dbDriverFactory
|
|
46
|
+
? this._dbDriverFactory()
|
|
47
|
+
: require("better-sqlite3-multiple-ciphers");
|
|
48
|
+
const db = new Driver(dbPath, { readonly: true });
|
|
49
|
+
try {
|
|
50
|
+
const histories = trySelect(db, "SELECT * FROM browse_history ORDER BY view_time DESC LIMIT 5000")
|
|
51
|
+
|| trySelect(db, "SELECT * FROM note ORDER BY view_time DESC LIMIT 5000") || [];
|
|
52
|
+
for (const row of histories) {
|
|
53
|
+
yield { adapter: NAME, originalId: `history-${row.id || row.note_id}`, capturedAt: parseTime(row.view_time), payload: { row, kind: "history" } };
|
|
54
|
+
}
|
|
55
|
+
const likes = trySelect(db, "SELECT * FROM liked_note ORDER BY like_time DESC LIMIT 5000") || [];
|
|
56
|
+
for (const row of likes) {
|
|
57
|
+
yield { adapter: NAME, originalId: `like-${row.id || row.note_id}`, capturedAt: parseTime(row.like_time), payload: { row, kind: "like" } };
|
|
58
|
+
}
|
|
59
|
+
const favs = trySelect(db, "SELECT * FROM favourite ORDER BY save_time DESC LIMIT 5000") || [];
|
|
60
|
+
for (const row of favs) {
|
|
61
|
+
yield { adapter: NAME, originalId: `fav-${row.id || row.note_id}`, capturedAt: parseTime(row.save_time), payload: { row, kind: "favourite" } };
|
|
62
|
+
}
|
|
63
|
+
} finally {
|
|
64
|
+
try { db.close(); } catch (_e) {}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
normalize(raw) {
|
|
69
|
+
const { kind, row } = raw.payload;
|
|
70
|
+
const now = Date.now();
|
|
71
|
+
const occurredAt = parseTime(row.view_time || row.like_time || row.save_time) || now;
|
|
72
|
+
const source = { adapter: NAME, adapterVersion: VERSION, originalId: raw.originalId, capturedAt: occurredAt, capturedBy: "sqlite" };
|
|
73
|
+
const subtypeMap = { history: "browse", like: "like", favourite: "like" };
|
|
74
|
+
return {
|
|
75
|
+
events: [{
|
|
76
|
+
id: newId(), type: "event",
|
|
77
|
+
subtype: subtypeMap[kind] || "browse",
|
|
78
|
+
occurredAt, actor: "person-self",
|
|
79
|
+
content: { title: row.title || row.note_title || "(no title)" },
|
|
80
|
+
ingestedAt: now, source,
|
|
81
|
+
extra: { noteId: row.note_id || null, author: row.author || row.nickname || null, kind },
|
|
82
|
+
}],
|
|
83
|
+
persons: [], places: [], items: [], topics: [],
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
function trySelect(db, sql) { try { return db.prepare(sql).all(); } catch (_e) { return null; } }
|
|
88
|
+
function parseTime(v) {
|
|
89
|
+
if (Number.isFinite(v)) return v > 1e12 ? v : v * 1000;
|
|
90
|
+
if (typeof v === "string") {
|
|
91
|
+
if (/^\d+$/.test(v)) { const n = parseInt(v, 10); return n > 1e12 ? n : n * 1000; }
|
|
92
|
+
const t = Date.parse(v); return Number.isFinite(t) ? t : null;
|
|
93
|
+
}
|
|
94
|
+
return null;
|
|
95
|
+
}
|
|
96
|
+
module.exports = { XiaohongshuAdapter, NAME, VERSION };
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dataDisclosure metadata + helpers for SystemDataAdapter — Phase 4.5.6.
|
|
3
|
+
*
|
|
4
|
+
* The metadata itself lives on the adapter (so the AdapterRegistry sees it
|
|
5
|
+
* during `assertAdapter`). This module exposes additional helpers for the
|
|
6
|
+
* desktop UI / hub layer to read:
|
|
7
|
+
*
|
|
8
|
+
* - SOURCE_DESCRIPTORS — per-source human-readable label + field list +
|
|
9
|
+
* sensitivity tier; the UI maps each to a toggle in the disclosure
|
|
10
|
+
* dialog (per Adapter_System_Data.md §5.1 mockup).
|
|
11
|
+
* - sanitizeInclude({...}) — clamp arbitrary user input to the supported
|
|
12
|
+
* boolean shape, so a stale Vue store can't smuggle non-boolean values
|
|
13
|
+
* past adapter.sync().
|
|
14
|
+
* - resolveRetentionMs({retentionDays}) — convert the user-set retention
|
|
15
|
+
* policy to milliseconds; the hub job runner uses this to schedule
|
|
16
|
+
* periodic deletes.
|
|
17
|
+
*
|
|
18
|
+
* Why a separate file rather than inline on the adapter? Three reasons:
|
|
19
|
+
* 1. The Vue layer can `require()` this without spinning up a SidecarSupervisor.
|
|
20
|
+
* 2. Tests can compare the descriptor list against the adapter's actual
|
|
21
|
+
* `dataDisclosure.fields` array, catching drift.
|
|
22
|
+
* 3. Future sources (calendar, browser history) follow the same shape so
|
|
23
|
+
* the disclosure UI is reusable.
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
"use strict";
|
|
27
|
+
|
|
28
|
+
const SOURCE_DESCRIPTORS = Object.freeze({
|
|
29
|
+
contacts: Object.freeze({
|
|
30
|
+
key: "contacts",
|
|
31
|
+
label: "通讯录",
|
|
32
|
+
fields: ["name", "phone", "email", "organization", "notes", "starred", "photoUri"],
|
|
33
|
+
sensitivity: "medium",
|
|
34
|
+
defaultEnabled: true,
|
|
35
|
+
estimate: "约 50-500 人",
|
|
36
|
+
rationale: "作为 EntityResolver 跨源 Person 主键的权威种子集",
|
|
37
|
+
}),
|
|
38
|
+
calllog: Object.freeze({
|
|
39
|
+
key: "calllog",
|
|
40
|
+
label: "通话记录",
|
|
41
|
+
fields: ["number", "duration", "timestamp", "type", "name"],
|
|
42
|
+
sensitivity: "high",
|
|
43
|
+
defaultEnabled: true,
|
|
44
|
+
estimate: "近 1 年约 1-10k 条",
|
|
45
|
+
rationale: "跨人互动时间线,独立于 app 内聊天",
|
|
46
|
+
}),
|
|
47
|
+
sms: Object.freeze({
|
|
48
|
+
key: "sms",
|
|
49
|
+
label: "短信和彩信",
|
|
50
|
+
fields: ["address", "body", "timestamp", "type", "threadId", "isRead"],
|
|
51
|
+
sensitivity: "high",
|
|
52
|
+
defaultEnabled: false, // opt-out (per OQ-SD1 + design doc §5.1)
|
|
53
|
+
estimate: "近 3 年约 5-20k 条",
|
|
54
|
+
rationale: "银行账单、验证码、物流通知的元数据源;含他人信息",
|
|
55
|
+
warning: "可能包含他人电话号码或对话内容",
|
|
56
|
+
}),
|
|
57
|
+
wifi: Object.freeze({
|
|
58
|
+
key: "wifi",
|
|
59
|
+
label: "WiFi 网络(不含密码)",
|
|
60
|
+
fields: ["ssid", "securityType", "hidden"],
|
|
61
|
+
excludedFields: ["password"],
|
|
62
|
+
sensitivity: "low",
|
|
63
|
+
defaultEnabled: true,
|
|
64
|
+
estimate: "约 5-50 个",
|
|
65
|
+
rationale: "常去地点种子(家/办公室/常去咖啡店)",
|
|
66
|
+
}),
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
const SOURCE_KEYS = Object.freeze(Object.keys(SOURCE_DESCRIPTORS));
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Clamp arbitrary user-provided `include` object to a strict boolean shape.
|
|
73
|
+
* Unknown keys are dropped; missing keys fall back to descriptor defaults.
|
|
74
|
+
*
|
|
75
|
+
* @param {object} raw user-controllable input from UI / IPC layer
|
|
76
|
+
* @returns {{contacts: boolean, calllog: boolean, sms: boolean, wifi: boolean}}
|
|
77
|
+
*/
|
|
78
|
+
function sanitizeInclude(raw) {
|
|
79
|
+
const out = {};
|
|
80
|
+
for (const key of SOURCE_KEYS) {
|
|
81
|
+
if (raw && Object.prototype.hasOwnProperty.call(raw, key)) {
|
|
82
|
+
out[key] = Boolean(raw[key]);
|
|
83
|
+
} else {
|
|
84
|
+
out[key] = SOURCE_DESCRIPTORS[key].defaultEnabled;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
return out;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Resolve a user-configured retention policy to a wall-clock millisecond
|
|
92
|
+
* threshold (rows with `ingestedAt < now - returnedMs` are eligible for
|
|
93
|
+
* background purge).
|
|
94
|
+
*
|
|
95
|
+
* @param {{retentionDays?: number}} policy
|
|
96
|
+
* @returns {number|null} null means "no retention cap" (default)
|
|
97
|
+
*/
|
|
98
|
+
function resolveRetentionMs(policy) {
|
|
99
|
+
if (!policy || policy.retentionDays == null) return null;
|
|
100
|
+
const days = Number(policy.retentionDays);
|
|
101
|
+
if (!Number.isFinite(days) || days <= 0) return null;
|
|
102
|
+
return Math.floor(days * 24 * 60 * 60 * 1000);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Cross-check that the disclosure-fields strings on the adapter actually
|
|
107
|
+
* cover every source's declared fields. Used in tests to catch the
|
|
108
|
+
* "adapter shipped a field the UI never warned about" drift bug.
|
|
109
|
+
*
|
|
110
|
+
* Returns { ok: boolean, missing: string[] } — `missing` lists
|
|
111
|
+
* "source:field" pairs that the adapter doesn't declare.
|
|
112
|
+
*/
|
|
113
|
+
function checkDisclosureCoverage(adapterFields) {
|
|
114
|
+
const declared = new Set(adapterFields || []);
|
|
115
|
+
const missing = [];
|
|
116
|
+
for (const desc of Object.values(SOURCE_DESCRIPTORS)) {
|
|
117
|
+
const expected = `${desc.key}:${desc.fields.join(",")}`;
|
|
118
|
+
const sourceHit = Array.from(declared).find((s) => s.startsWith(`${desc.key}:`));
|
|
119
|
+
if (!sourceHit) {
|
|
120
|
+
missing.push(expected);
|
|
121
|
+
continue;
|
|
122
|
+
}
|
|
123
|
+
// Verify every expected field appears in the declared string.
|
|
124
|
+
const declaredFields = new Set(
|
|
125
|
+
sourceHit.split(":", 2)[1].split(",").map((s) => s.trim()),
|
|
126
|
+
);
|
|
127
|
+
for (const f of desc.fields) {
|
|
128
|
+
if (!declaredFields.has(f)) missing.push(`${desc.key}:${f}`);
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
return { ok: missing.length === 0, missing };
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* Localized disclosure dialog payload — what the UI binds to.
|
|
136
|
+
* Returns a stable structure independent of the adapter instance, so the
|
|
137
|
+
* Vue store can be tested without the adapter wired in.
|
|
138
|
+
*/
|
|
139
|
+
function buildDisclosurePayload() {
|
|
140
|
+
return {
|
|
141
|
+
adapter: "system-data",
|
|
142
|
+
sensitivity: "high",
|
|
143
|
+
legalGate: true,
|
|
144
|
+
notice:
|
|
145
|
+
"短信和通话记录可能包含他人电话号码或对话内容;所有数据在本机加密存储,不向任何服务器上传(含 AI 分析)。",
|
|
146
|
+
sources: SOURCE_KEYS.map((key) => SOURCE_DESCRIPTORS[key]),
|
|
147
|
+
legalDeclaration:
|
|
148
|
+
[
|
|
149
|
+
"您声明:",
|
|
150
|
+
"1. 您是这部手机的合法使用者,对其上数据拥有访问权",
|
|
151
|
+
"2. 您理解短信内容可能涉及他人隐私,承诺仅在本机使用,不向任何第三方分发",
|
|
152
|
+
"3. 本工具不会将系统数据上传至云端(含 LLM 分析全部本地完成)",
|
|
153
|
+
"",
|
|
154
|
+
"不符合上述条件,请勿启用本 adapter。",
|
|
155
|
+
].join("\n"),
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
module.exports = {
|
|
160
|
+
SOURCE_DESCRIPTORS,
|
|
161
|
+
SOURCE_KEYS,
|
|
162
|
+
sanitizeInclude,
|
|
163
|
+
resolveRetentionMs,
|
|
164
|
+
checkDisclosureCoverage,
|
|
165
|
+
buildDisclosurePayload,
|
|
166
|
+
};
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
const {
|
|
4
|
+
SystemDataAdapter,
|
|
5
|
+
SYSTEM_DATA_ADAPTER_NAME,
|
|
6
|
+
SYSTEM_DATA_ADAPTER_VERSION,
|
|
7
|
+
DEFAULT_INCLUDE,
|
|
8
|
+
DEFAULT_REMOTE_PATHS,
|
|
9
|
+
SDCARD_WORKAROUND_PATHS,
|
|
10
|
+
} = require("./system-data-adapter");
|
|
11
|
+
const {
|
|
12
|
+
SOURCE_DESCRIPTORS,
|
|
13
|
+
SOURCE_KEYS,
|
|
14
|
+
sanitizeInclude,
|
|
15
|
+
resolveRetentionMs,
|
|
16
|
+
checkDisclosureCoverage,
|
|
17
|
+
buildDisclosurePayload,
|
|
18
|
+
} = require("./disclosure");
|
|
19
|
+
|
|
20
|
+
module.exports = {
|
|
21
|
+
SystemDataAdapter,
|
|
22
|
+
SYSTEM_DATA_ADAPTER_NAME,
|
|
23
|
+
SYSTEM_DATA_ADAPTER_VERSION,
|
|
24
|
+
DEFAULT_INCLUDE,
|
|
25
|
+
DEFAULT_REMOTE_PATHS,
|
|
26
|
+
SDCARD_WORKAROUND_PATHS,
|
|
27
|
+
// Disclosure helpers (Phase 4.5.6)
|
|
28
|
+
SOURCE_DESCRIPTORS,
|
|
29
|
+
SOURCE_KEYS,
|
|
30
|
+
sanitizeInclude,
|
|
31
|
+
resolveRetentionMs,
|
|
32
|
+
checkDisclosureCoverage,
|
|
33
|
+
buildDisclosurePayload,
|
|
34
|
+
};
|
|
@@ -0,0 +1,344 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SystemDataAdapter — Android system data (contacts / call log / SMS / WiFi).
|
|
3
|
+
*
|
|
4
|
+
* Phase 4.5.5. Sits on top of the forensics-bridge sidecar.
|
|
5
|
+
*
|
|
6
|
+
* Per-source pipeline (each one independent — disabling SMS doesn't break the
|
|
7
|
+
* others):
|
|
8
|
+
*
|
|
9
|
+
* contacts: android.pull_file → system.parse_contacts → Person stream
|
|
10
|
+
* calllog: android.pull_file → system.parse_calllog → Event(call) + Person stream
|
|
11
|
+
* sms: android.pull_file → system.parse_sms → Event(message) + Person stream
|
|
12
|
+
* wifi: android.pull_file → system.parse_wifi → Place stream
|
|
13
|
+
*
|
|
14
|
+
* Or, when `opts.dataPaths` is provided (e.g. user already adb-pulled files
|
|
15
|
+
* manually, or testing with a local fixture), skip the pull step.
|
|
16
|
+
*
|
|
17
|
+
* Privacy gating: `opts.include` decides which sub-sources run. Default per
|
|
18
|
+
* Adapter_System_Data.md §5.1 + OQ-SD1: contacts ON / calllog ON / sms OFF /
|
|
19
|
+
* wifi ON. The UI dialog re-confirms this on each sync.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
"use strict";
|
|
23
|
+
|
|
24
|
+
const path = require("node:path");
|
|
25
|
+
const os = require("node:os");
|
|
26
|
+
const fs = require("node:fs");
|
|
27
|
+
|
|
28
|
+
const { PythonSidecarAdapter } = require("../_python-sidecar-base");
|
|
29
|
+
|
|
30
|
+
const NAME = "system-data";
|
|
31
|
+
const VERSION = "0.1.0";
|
|
32
|
+
|
|
33
|
+
const DEFAULT_INCLUDE = Object.freeze({
|
|
34
|
+
contacts: true,
|
|
35
|
+
calllog: true,
|
|
36
|
+
sms: false, // opt-out by default — see Adapter_System_Data.md §5.1
|
|
37
|
+
wifi: true,
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Default Android system provider paths. Override via opts.remotePaths when
|
|
42
|
+
* a device uses a non-stock layout.
|
|
43
|
+
*/
|
|
44
|
+
const DEFAULT_REMOTE_PATHS = Object.freeze({
|
|
45
|
+
contacts:
|
|
46
|
+
"/data/data/com.android.providers.contacts/databases/contacts2.db",
|
|
47
|
+
calllog: "/data/data/com.android.providers.contacts/databases/calllog.db",
|
|
48
|
+
sms: "/data/data/com.android.providers.telephony/databases/mmssms.db",
|
|
49
|
+
wifi: "/data/misc/wifi/", // directory — pull_file works for one file, so wifi mode-A is dataPaths
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Per-source workaround paths under /sdcard/Download/ for stock Android
|
|
54
|
+
* (no `adb root`) — user copies files via Termux + tsu or MT Manager.
|
|
55
|
+
*/
|
|
56
|
+
const SDCARD_WORKAROUND_PATHS = Object.freeze({
|
|
57
|
+
contacts: "/sdcard/Download/contacts2.db",
|
|
58
|
+
calllog: "/sdcard/Download/calllog.db",
|
|
59
|
+
sms: "/sdcard/Download/mmssms.db",
|
|
60
|
+
wifi_xml: "/sdcard/Download/WifiConfigStore.xml",
|
|
61
|
+
wifi_conf: "/sdcard/Download/wpa_supplicant.conf",
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
class SystemDataAdapter extends PythonSidecarAdapter {
|
|
65
|
+
constructor(opts) {
|
|
66
|
+
super(opts);
|
|
67
|
+
this.name = NAME;
|
|
68
|
+
this.version = VERSION;
|
|
69
|
+
this.capabilities = [
|
|
70
|
+
"sync:android-adb",
|
|
71
|
+
"sync:android-sdcard-workaround",
|
|
72
|
+
"sync:host-dataPaths",
|
|
73
|
+
];
|
|
74
|
+
this.rateLimits = { perDay: 12 }; // system data day-to-day churn is small
|
|
75
|
+
this.dataDisclosure = {
|
|
76
|
+
fields: [
|
|
77
|
+
"contacts:name,phone,email,organization,notes,starred,photoUri",
|
|
78
|
+
"calllog:number,duration,timestamp,type,name",
|
|
79
|
+
"sms:address,body,timestamp,type,threadId,isRead",
|
|
80
|
+
"wifi:ssid,securityType,hidden",
|
|
81
|
+
// Explicitly NOT collected:
|
|
82
|
+
// - wifi:password (never written to vault, even when present in source)
|
|
83
|
+
],
|
|
84
|
+
sensitivity: "high", // SMS may include third-party content
|
|
85
|
+
legalGate: true, // requires explicit user agreement on third-party content
|
|
86
|
+
retentionDays: undefined, // user-controlled (no default cap)
|
|
87
|
+
notice:
|
|
88
|
+
"短信和通话记录可能包含他人电话号码或对话内容;所有数据在本机加密存储,不向任何服务器上传(含 AI 分析)。",
|
|
89
|
+
defaultInclude: { ...DEFAULT_INCLUDE },
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// -----------------------------------------------------------------------
|
|
94
|
+
// PersonalDataAdapter — authenticate / healthCheck override
|
|
95
|
+
// -----------------------------------------------------------------------
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Verify the sidecar is reachable AND there is at least one usable ADB
|
|
99
|
+
* device (unless caller signals offline-import mode by passing dataPaths).
|
|
100
|
+
*
|
|
101
|
+
* @param {object} ctx
|
|
102
|
+
* @param {object} [ctx.dataPaths] If set, ADB presence is not required.
|
|
103
|
+
* @param {string} [ctx.serial] Optional serial; auth checks just that device.
|
|
104
|
+
*/
|
|
105
|
+
async authenticate(ctx = {}) {
|
|
106
|
+
const pong = await this.supervisor.invoke("sidecar.ping", {}, { timeoutMs: 3000 });
|
|
107
|
+
if (ctx.dataPaths && Object.keys(ctx.dataPaths).length > 0) {
|
|
108
|
+
return { ok: true, mode: "offline", sidecarVersion: pong.version };
|
|
109
|
+
}
|
|
110
|
+
let devices;
|
|
111
|
+
try {
|
|
112
|
+
const out = await this.supervisor.invoke("android.list_devices", {}, { timeoutMs: 5000 });
|
|
113
|
+
devices = out.devices || [];
|
|
114
|
+
} catch (err) {
|
|
115
|
+
return {
|
|
116
|
+
ok: false,
|
|
117
|
+
reason: `android.list_devices failed: ${err.code || err.message}`,
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
const wanted = ctx.serial
|
|
121
|
+
? devices.filter((d) => d.serial === ctx.serial)
|
|
122
|
+
: devices.filter((d) => d.state === "device");
|
|
123
|
+
if (wanted.length === 0) {
|
|
124
|
+
return {
|
|
125
|
+
ok: false,
|
|
126
|
+
reason: ctx.serial
|
|
127
|
+
? `device "${ctx.serial}" not found or not authorized`
|
|
128
|
+
: "no authorized ADB devices attached",
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
return { ok: true, mode: "device", devices: wanted };
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
// -----------------------------------------------------------------------
|
|
135
|
+
// Orchestration (subclass hook)
|
|
136
|
+
// -----------------------------------------------------------------------
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* Orchestrate the 4 sub-sources sequentially.
|
|
140
|
+
*
|
|
141
|
+
* @param {object} opts
|
|
142
|
+
* @param {object} [opts.include] Per-source enable flags (defaults: DEFAULT_INCLUDE).
|
|
143
|
+
* @param {string} [opts.serial] Required when pulling from a live device.
|
|
144
|
+
* @param {object} [opts.dataPaths] Pre-extracted host paths, keys:
|
|
145
|
+
* {contacts, calllog, sms, wifi}.
|
|
146
|
+
* @param {object} [opts.remotePaths] Override default device paths.
|
|
147
|
+
* @param {"normal"|"sdcard"} [opts.extractMode]
|
|
148
|
+
* "normal" = pull from /data/data (root only),
|
|
149
|
+
* "sdcard" = pull from /sdcard/Download (workaround).
|
|
150
|
+
* @param {string} [opts.scratchDir] Directory for pulled DBs. Default: hub tmp.
|
|
151
|
+
* @param {(msg: object) => void} [opts.onProgress] Forwarded as adapter-progress.
|
|
152
|
+
*/
|
|
153
|
+
async _runSidecar(opts, emit) {
|
|
154
|
+
const include = { ...DEFAULT_INCLUDE, ...(opts.include || {}) };
|
|
155
|
+
const dataPaths = opts.dataPaths || {};
|
|
156
|
+
const extractMode = opts.extractMode || "normal";
|
|
157
|
+
const remotePaths =
|
|
158
|
+
extractMode === "sdcard"
|
|
159
|
+
? {
|
|
160
|
+
contacts: SDCARD_WORKAROUND_PATHS.contacts,
|
|
161
|
+
calllog: SDCARD_WORKAROUND_PATHS.calllog,
|
|
162
|
+
sms: SDCARD_WORKAROUND_PATHS.sms,
|
|
163
|
+
wifi: SDCARD_WORKAROUND_PATHS.wifi_xml,
|
|
164
|
+
}
|
|
165
|
+
: { ...DEFAULT_REMOTE_PATHS, ...(opts.remotePaths || {}) };
|
|
166
|
+
|
|
167
|
+
const scratchDir =
|
|
168
|
+
opts.scratchDir ||
|
|
169
|
+
fs.mkdtempSync(path.join(os.tmpdir(), "system-data-sync-"));
|
|
170
|
+
fs.mkdirSync(scratchDir, { recursive: true });
|
|
171
|
+
|
|
172
|
+
const onProgress = typeof opts.onProgress === "function" ? opts.onProgress : null;
|
|
173
|
+
const tellProgress = (source, phase, extra = {}) => {
|
|
174
|
+
if (onProgress) onProgress({ source, phase, ...extra });
|
|
175
|
+
};
|
|
176
|
+
|
|
177
|
+
const sourcesRun = [];
|
|
178
|
+
|
|
179
|
+
// ─── Contacts ────────────────────────────────────────────────────────
|
|
180
|
+
let contactsLocal = dataPaths.contacts || null;
|
|
181
|
+
if (include.contacts) {
|
|
182
|
+
if (!contactsLocal) {
|
|
183
|
+
if (!opts.serial) {
|
|
184
|
+
throw new Error(
|
|
185
|
+
"system-data: contacts enabled but no serial/dataPaths.contacts provided",
|
|
186
|
+
);
|
|
187
|
+
}
|
|
188
|
+
tellProgress("contacts", "pulling");
|
|
189
|
+
const pulled = await this.supervisor.invoke(
|
|
190
|
+
"android.pull_file",
|
|
191
|
+
{
|
|
192
|
+
serial: opts.serial,
|
|
193
|
+
remote_path: remotePaths.contacts,
|
|
194
|
+
local_dir: scratchDir,
|
|
195
|
+
},
|
|
196
|
+
{ timeoutMs: 60_000 },
|
|
197
|
+
);
|
|
198
|
+
contactsLocal = pulled.local;
|
|
199
|
+
}
|
|
200
|
+
tellProgress("contacts", "parsing", { dbPath: contactsLocal });
|
|
201
|
+
const r = await this.supervisor.invoke(
|
|
202
|
+
"system.parse_contacts",
|
|
203
|
+
{ data_path: contactsLocal, device_serial: opts.serial || null },
|
|
204
|
+
{
|
|
205
|
+
timeoutMs: 120_000,
|
|
206
|
+
onChunk: (batch) => this._emitChunkAsRaws(batch, emit),
|
|
207
|
+
onProgress: (p) => tellProgress("contacts", "progress", p),
|
|
208
|
+
},
|
|
209
|
+
);
|
|
210
|
+
sourcesRun.push({ source: "contacts", ...r });
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
// ─── Call log ────────────────────────────────────────────────────────
|
|
214
|
+
if (include.calllog) {
|
|
215
|
+
let calllogLocal = dataPaths.calllog || null;
|
|
216
|
+
if (!calllogLocal) {
|
|
217
|
+
if (!opts.serial) {
|
|
218
|
+
throw new Error("system-data: calllog enabled but no serial/dataPaths.calllog");
|
|
219
|
+
}
|
|
220
|
+
tellProgress("calllog", "pulling");
|
|
221
|
+
try {
|
|
222
|
+
const pulled = await this.supervisor.invoke(
|
|
223
|
+
"android.pull_file",
|
|
224
|
+
{
|
|
225
|
+
serial: opts.serial,
|
|
226
|
+
remote_path: remotePaths.calllog,
|
|
227
|
+
local_dir: scratchDir,
|
|
228
|
+
},
|
|
229
|
+
{ timeoutMs: 60_000 },
|
|
230
|
+
);
|
|
231
|
+
calllogLocal = pulled.local;
|
|
232
|
+
} catch (err) {
|
|
233
|
+
// Calls table may live in contacts2.db on pre-Android-11 builds.
|
|
234
|
+
if (err.code === "EXTRACT_PERMISSION_DENIED" && contactsLocal) {
|
|
235
|
+
calllogLocal = contactsLocal;
|
|
236
|
+
} else {
|
|
237
|
+
throw err;
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
tellProgress("calllog", "parsing", { dbPath: calllogLocal });
|
|
242
|
+
const r = await this.supervisor.invoke(
|
|
243
|
+
"system.parse_calllog",
|
|
244
|
+
{
|
|
245
|
+
data_path: calllogLocal,
|
|
246
|
+
contacts_db_path: contactsLocal,
|
|
247
|
+
device_serial: opts.serial || null,
|
|
248
|
+
},
|
|
249
|
+
{
|
|
250
|
+
timeoutMs: 180_000,
|
|
251
|
+
onChunk: (batch) => this._emitChunkAsRaws(batch, emit),
|
|
252
|
+
onProgress: (p) => tellProgress("calllog", "progress", p),
|
|
253
|
+
},
|
|
254
|
+
);
|
|
255
|
+
sourcesRun.push({ source: "calllog", ...r });
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
// ─── SMS ────────────────────────────────────────────────────────────
|
|
259
|
+
if (include.sms) {
|
|
260
|
+
let smsLocal = dataPaths.sms || null;
|
|
261
|
+
if (!smsLocal) {
|
|
262
|
+
if (!opts.serial) {
|
|
263
|
+
throw new Error("system-data: sms enabled but no serial/dataPaths.sms");
|
|
264
|
+
}
|
|
265
|
+
tellProgress("sms", "pulling");
|
|
266
|
+
const pulled = await this.supervisor.invoke(
|
|
267
|
+
"android.pull_file",
|
|
268
|
+
{
|
|
269
|
+
serial: opts.serial,
|
|
270
|
+
remote_path: remotePaths.sms,
|
|
271
|
+
local_dir: scratchDir,
|
|
272
|
+
},
|
|
273
|
+
{ timeoutMs: 60_000 },
|
|
274
|
+
);
|
|
275
|
+
smsLocal = pulled.local;
|
|
276
|
+
}
|
|
277
|
+
tellProgress("sms", "parsing", { dbPath: smsLocal });
|
|
278
|
+
const r = await this.supervisor.invoke(
|
|
279
|
+
"system.parse_sms",
|
|
280
|
+
{
|
|
281
|
+
data_path: smsLocal,
|
|
282
|
+
contacts_db_path: contactsLocal,
|
|
283
|
+
device_serial: opts.serial || null,
|
|
284
|
+
},
|
|
285
|
+
{
|
|
286
|
+
timeoutMs: 300_000, // SMS can be 10K+ rows on long-term devices
|
|
287
|
+
onChunk: (batch) => this._emitChunkAsRaws(batch, emit),
|
|
288
|
+
onProgress: (p) => tellProgress("sms", "progress", p),
|
|
289
|
+
},
|
|
290
|
+
);
|
|
291
|
+
sourcesRun.push({ source: "sms", ...r });
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
// ─── WiFi ───────────────────────────────────────────────────────────
|
|
295
|
+
if (include.wifi) {
|
|
296
|
+
let wifiLocal = dataPaths.wifi || null;
|
|
297
|
+
if (!wifiLocal) {
|
|
298
|
+
// WiFi config is a single file, but two possible names. Prefer XML.
|
|
299
|
+
if (!opts.serial) {
|
|
300
|
+
throw new Error("system-data: wifi enabled but no serial/dataPaths.wifi");
|
|
301
|
+
}
|
|
302
|
+
tellProgress("wifi", "pulling");
|
|
303
|
+
try {
|
|
304
|
+
const pulled = await this.supervisor.invoke(
|
|
305
|
+
"android.pull_file",
|
|
306
|
+
{
|
|
307
|
+
serial: opts.serial,
|
|
308
|
+
remote_path: remotePaths.wifi,
|
|
309
|
+
local_dir: scratchDir,
|
|
310
|
+
},
|
|
311
|
+
{ timeoutMs: 30_000 },
|
|
312
|
+
);
|
|
313
|
+
wifiLocal = path.dirname(pulled.local);
|
|
314
|
+
} catch (err) {
|
|
315
|
+
// Non-fatal — wifi often inaccessible without root. Skip this source.
|
|
316
|
+
tellProgress("wifi", "skipped", { reason: err.code || err.message });
|
|
317
|
+
return { sources: sourcesRun, scratchDir };
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
tellProgress("wifi", "parsing", { dbPath: wifiLocal });
|
|
321
|
+
const r = await this.supervisor.invoke(
|
|
322
|
+
"system.parse_wifi",
|
|
323
|
+
{ data_path: wifiLocal, device_serial: opts.serial || null },
|
|
324
|
+
{
|
|
325
|
+
timeoutMs: 30_000,
|
|
326
|
+
onChunk: (batch) => this._emitChunkAsRaws(batch, emit),
|
|
327
|
+
onProgress: (p) => tellProgress("wifi", "progress", p),
|
|
328
|
+
},
|
|
329
|
+
);
|
|
330
|
+
sourcesRun.push({ source: "wifi", ...r });
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
return { sources: sourcesRun, scratchDir };
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
module.exports = {
|
|
338
|
+
SystemDataAdapter,
|
|
339
|
+
SYSTEM_DATA_ADAPTER_NAME: NAME,
|
|
340
|
+
SYSTEM_DATA_ADAPTER_VERSION: VERSION,
|
|
341
|
+
DEFAULT_INCLUDE,
|
|
342
|
+
DEFAULT_REMOTE_PATHS,
|
|
343
|
+
SDCARD_WORKAROUND_PATHS,
|
|
344
|
+
};
|