@chainlesschain/personal-data-hub 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/__tests__/adapters/ai-chat-history.test.js +395 -0
- package/__tests__/adapters/ai-chat-http-client.test.js +242 -0
- package/__tests__/adapters/ai-chat-vendors.test.js +733 -0
- package/__tests__/adapters/alipay-bill-adapter.test.js +538 -0
- package/__tests__/adapters/email-adapter.test.js +138 -1
- package/__tests__/adapters/email-classifier.test.js +347 -0
- package/__tests__/adapters/email-pdf-extractor.test.js +529 -0
- package/__tests__/adapters/email-retry-progress.test.js +294 -0
- package/__tests__/adapters/email-templates.test.js +699 -0
- package/__tests__/adapters/system-data-adapter.test.js +440 -0
- package/__tests__/adapters/system-data-disclosure.test.js +153 -0
- package/__tests__/analysis-skills.test.js +409 -0
- package/__tests__/entity-resolver-ingest-hook.test.js +177 -0
- package/__tests__/entity-resolver-stages.test.js +411 -0
- package/__tests__/entity-resolver-vault.test.js +246 -0
- package/__tests__/entity-resolver.test.js +526 -0
- package/__tests__/fixtures/entity-resolver-200-mock.json +96 -0
- package/__tests__/longtail-adapters.test.js +217 -0
- package/__tests__/mobile-extractor.test.js +288 -0
- package/__tests__/shopping-adapters.test.js +296 -0
- package/__tests__/sidecar-contacts-cross-validate.test.js +163 -0
- package/__tests__/sidecar-supervisor.test.js +120 -0
- package/__tests__/social-adapters.test.js +206 -0
- package/__tests__/travel-adapters.test.js +325 -0
- package/__tests__/vault.test.js +3 -3
- package/__tests__/wechat-adapter.test.js +476 -0
- package/__tests__/whatsapp-adapter.test.js +135 -0
- package/lib/adapter-spec.js +12 -0
- package/lib/adapters/_python-sidecar-base.js +207 -0
- package/lib/adapters/ai-chat-history/ai-chat-adapter.js +335 -0
- package/lib/adapters/ai-chat-history/cookie-auth.js +109 -0
- package/lib/adapters/ai-chat-history/http-client.js +211 -0
- package/lib/adapters/ai-chat-history/index.js +28 -0
- package/lib/adapters/ai-chat-history/schema-map.js +221 -0
- package/lib/adapters/ai-chat-history/vendor-spec.js +85 -0
- package/lib/adapters/ai-chat-history/vendors/coze.js +179 -0
- package/lib/adapters/ai-chat-history/vendors/deepseek.js +199 -0
- package/lib/adapters/ai-chat-history/vendors/dreamina.js +174 -0
- package/lib/adapters/ai-chat-history/vendors/hunyuan.js +176 -0
- package/lib/adapters/ai-chat-history/vendors/kimi.js +182 -0
- package/lib/adapters/ai-chat-history/vendors/qianfan.js +160 -0
- package/lib/adapters/ai-chat-history/vendors/tongyi.js +193 -0
- package/lib/adapters/ai-chat-history/vendors/zhipu.js +202 -0
- package/lib/adapters/alipay-bill/alipay-bill-adapter.js +307 -0
- package/lib/adapters/alipay-bill/counterparty.js +129 -0
- package/lib/adapters/alipay-bill/csv-parser.js +217 -0
- package/lib/adapters/alipay-bill/index.js +41 -0
- package/lib/adapters/alipay-bill/zip-decryptor.js +111 -0
- package/lib/adapters/email-imap/classifier.js +495 -0
- package/lib/adapters/email-imap/email-adapter.js +419 -8
- package/lib/adapters/email-imap/index.js +42 -0
- package/lib/adapters/email-imap/pdf-extractor.js +192 -0
- package/lib/adapters/email-imap/templates/bill.js +232 -0
- package/lib/adapters/email-imap/templates/government.js +120 -0
- package/lib/adapters/email-imap/templates/index.js +78 -0
- package/lib/adapters/email-imap/templates/order.js +186 -0
- package/lib/adapters/email-imap/templates/other.js +114 -0
- package/lib/adapters/email-imap/templates/register.js +113 -0
- package/lib/adapters/email-imap/templates/travel.js +157 -0
- package/lib/adapters/email-imap/templates/utils.js +275 -0
- package/lib/adapters/email-imap/transactions.js +234 -0
- package/lib/adapters/messaging-qq/index.js +158 -0
- package/lib/adapters/messaging-telegram/index.js +142 -0
- package/lib/adapters/messaging-whatsapp/index.js +189 -0
- package/lib/adapters/shopping-base/index.js +208 -0
- package/lib/adapters/shopping-jd/index.js +150 -0
- package/lib/adapters/shopping-meituan/index.js +154 -0
- package/lib/adapters/shopping-taobao/index.js +176 -0
- package/lib/adapters/social-bilibili/index.js +171 -0
- package/lib/adapters/social-douyin/index.js +116 -0
- package/lib/adapters/social-weibo/index.js +164 -0
- package/lib/adapters/social-xiaohongshu/index.js +96 -0
- package/lib/adapters/system-data/disclosure.js +166 -0
- package/lib/adapters/system-data/index.js +34 -0
- package/lib/adapters/system-data/system-data-adapter.js +344 -0
- package/lib/adapters/travel-12306/index.js +151 -0
- package/lib/adapters/travel-amap/index.js +164 -0
- package/lib/adapters/travel-baidu-map/index.js +162 -0
- package/lib/adapters/travel-base/index.js +240 -0
- package/lib/adapters/travel-ctrip/index.js +151 -0
- package/lib/adapters/wechat/content-parser.js +326 -0
- package/lib/adapters/wechat/db-reader.js +209 -0
- package/lib/adapters/wechat/index.js +28 -0
- package/lib/adapters/wechat/key-extractor.js +158 -0
- package/lib/adapters/wechat/normalize.js +220 -0
- package/lib/adapters/wechat/wechat-adapter.js +205 -0
- package/lib/analysis-skills/base.js +113 -0
- package/lib/analysis-skills/footprint.js +167 -0
- package/lib/analysis-skills/index.js +58 -0
- package/lib/analysis-skills/interests.js +161 -0
- package/lib/analysis-skills/relations.js +226 -0
- package/lib/analysis-skills/spending.js +216 -0
- package/lib/analysis-skills/timeline.js +167 -0
- package/lib/entity-resolver/embedding-stage.js +198 -0
- package/lib/entity-resolver/entity-resolver.js +384 -0
- package/lib/entity-resolver/index.js +42 -0
- package/lib/entity-resolver/llm-stage.js +191 -0
- package/lib/entity-resolver/rule-stage.js +208 -0
- package/lib/entity-resolver/worker.js +149 -0
- package/lib/index.js +115 -0
- package/lib/migrations.js +73 -0
- package/lib/mobile-extractor/android.js +193 -0
- package/lib/mobile-extractor/index.js +9 -0
- package/lib/mobile-extractor/ios.js +223 -0
- package/lib/registry.js +42 -0
- package/lib/sidecar/index.js +15 -0
- package/lib/sidecar/supervisor.js +359 -0
- package/lib/vault.js +266 -0
- package/package.json +29 -3
- package/scripts/_make-fixture-all.js +126 -0
- package/scripts/_make-fixture-contacts.js +84 -0
- package/scripts/evaluate-entity-resolver.js +213 -0
- package/scripts/smoke-phase-5-5.js +196 -0
- package/scripts/smoke-phase-5-7.js +181 -0
- package/scripts/smoke-system-data-contacts.js +309 -0
- package/scripts/smoke-system-data.js +312 -0
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 通义千问 (Aliyun Tongyi) vendor adapter — Phase 10.2 wiring.
|
|
3
|
+
*
|
|
4
|
+
* Reference: docs/design/Adapter_AIChat_History.md §6.3
|
|
5
|
+
* - login https://tongyi.aliyun.com/
|
|
6
|
+
* - convs POST /dialog/conversation/list
|
|
7
|
+
* - msgs POST /dialog/conversation/messages
|
|
8
|
+
* - X-Csrf-Token header required (from XSRF-TOKEN cookie)
|
|
9
|
+
* - 阿里通用 anti-bot — conservative rateLimits.
|
|
10
|
+
*
|
|
11
|
+
* Endpoints + payload shapes are reverse-engineered and SUBJECT TO DRIFT.
|
|
12
|
+
* Tests use the same fixture-fetch pattern as DeepSeek + Kimi so wire breakage
|
|
13
|
+
* surfaces immediately when a real cookie is reconnected.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
"use strict";
|
|
17
|
+
|
|
18
|
+
const BASE = "https://tongyi.aliyun.com";
|
|
19
|
+
const CONV_LIST_PATH = "/dialog/conversation/list";
|
|
20
|
+
const MSG_LIST_PATH = "/dialog/conversation/messages";
|
|
21
|
+
const USER_INFO_PATH = "/api/user/info";
|
|
22
|
+
|
|
23
|
+
const DEFAULT_PAGE_SIZE = 30;
|
|
24
|
+
|
|
25
|
+
function _ensureClient(ctx) {
|
|
26
|
+
if (!ctx || !ctx.httpClient) {
|
|
27
|
+
throw new Error("tongyi: ctx.httpClient required");
|
|
28
|
+
}
|
|
29
|
+
return ctx.httpClient;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function _csrfHeader(session) {
|
|
33
|
+
// Tongyi reads CSRF from a cookie + echoes it back as a header.
|
|
34
|
+
const value = session && (session.get("XSRF-TOKEN") || session.get("_csrf"));
|
|
35
|
+
return value ? { "X-Csrf-Token": value, "X-Xsrf-Token": value } : {};
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
async function validateCookie(ctx) {
|
|
39
|
+
const client = _ensureClient(ctx);
|
|
40
|
+
try {
|
|
41
|
+
const data = await client.getJson(BASE + USER_INFO_PATH, {
|
|
42
|
+
session: ctx.session,
|
|
43
|
+
headers: _csrfHeader(ctx.session),
|
|
44
|
+
});
|
|
45
|
+
if (data && (data.success || data.code === 200) && (data.data || data.userId)) {
|
|
46
|
+
const userId = (data.data && (data.data.userId || data.data.uid)) || data.userId;
|
|
47
|
+
return { ok: true, userId };
|
|
48
|
+
}
|
|
49
|
+
return { ok: false, reason: "UNEXPECTED_RESPONSE_SHAPE" };
|
|
50
|
+
} catch (err) {
|
|
51
|
+
return { ok: false, reason: err.code || err.message };
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
async function *listConversations(ctx, opts = {}) {
|
|
56
|
+
const client = _ensureClient(ctx);
|
|
57
|
+
const pageSize = Number.isFinite(opts.pageSize) ? opts.pageSize : DEFAULT_PAGE_SIZE;
|
|
58
|
+
const sinceTs = opts.since && opts.since.lastUpdatedAt ? Number(opts.since.lastUpdatedAt) : 0;
|
|
59
|
+
|
|
60
|
+
let pageNum = 1;
|
|
61
|
+
while (true) {
|
|
62
|
+
const body = { pageNum, pageSize };
|
|
63
|
+
const data = await client.postJson(BASE + CONV_LIST_PATH, body, {
|
|
64
|
+
session: ctx.session,
|
|
65
|
+
headers: _csrfHeader(ctx.session),
|
|
66
|
+
});
|
|
67
|
+
const items = _extractList(data);
|
|
68
|
+
if (items.length === 0) return;
|
|
69
|
+
|
|
70
|
+
let stopped = false;
|
|
71
|
+
for (const c of items) {
|
|
72
|
+
const updatedAt = _toMs(c.gmtModified || c.updatedAt || c.gmtCreate);
|
|
73
|
+
if (sinceTs && updatedAt <= sinceTs) {
|
|
74
|
+
stopped = true;
|
|
75
|
+
break;
|
|
76
|
+
}
|
|
77
|
+
yield {
|
|
78
|
+
vendor: "tongyi",
|
|
79
|
+
originalId: String(c.sessionId || c.id),
|
|
80
|
+
title: c.summary || c.firstQuery || c.title || undefined,
|
|
81
|
+
modelName: c.modelName || c.model || undefined,
|
|
82
|
+
createdAt: _toMs(c.gmtCreate || c.createdAt),
|
|
83
|
+
updatedAt,
|
|
84
|
+
messageCount: c.messageCount || undefined,
|
|
85
|
+
archived: Boolean(c.archived),
|
|
86
|
+
extra: { agentName: c.agentName || c.botName },
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
if (stopped) return;
|
|
90
|
+
if (items.length < pageSize) return;
|
|
91
|
+
pageNum++;
|
|
92
|
+
if (pageNum > 200) return; // safety
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
async function *listMessages(ctx, conversationId, _opts = {}) {
|
|
97
|
+
const client = _ensureClient(ctx);
|
|
98
|
+
const body = { sessionId: String(conversationId), parentMsgId: "" };
|
|
99
|
+
const data = await client.postJson(BASE + MSG_LIST_PATH, body, {
|
|
100
|
+
session: ctx.session,
|
|
101
|
+
headers: _csrfHeader(ctx.session),
|
|
102
|
+
});
|
|
103
|
+
const msgs = _extractList(data);
|
|
104
|
+
|
|
105
|
+
// Tongyi returns messages mixed user/bot in their own order; we sort
|
|
106
|
+
// ascending by createTime for stable chronological yield.
|
|
107
|
+
msgs.sort((a, b) => _toMs(a.createTime || a.gmtCreate) - _toMs(b.createTime || b.gmtCreate));
|
|
108
|
+
|
|
109
|
+
for (const m of msgs) {
|
|
110
|
+
yield {
|
|
111
|
+
vendor: "tongyi",
|
|
112
|
+
originalId: String(m.msgId || m.id),
|
|
113
|
+
conversationId: String(conversationId),
|
|
114
|
+
role: _normalizeRole(m.senderType || m.role || (m.contentType === "user" ? "user" : "assistant")),
|
|
115
|
+
content: _buildContent(m),
|
|
116
|
+
createdAt: _toMs(m.createTime || m.gmtCreate),
|
|
117
|
+
parentMessageId: m.parentMsgId ? String(m.parentMsgId) : undefined,
|
|
118
|
+
modelName: m.modelName || m.model || undefined,
|
|
119
|
+
extra: m.feedback ? { feedback: m.feedback } : undefined,
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
function _extractList(data) {
|
|
125
|
+
if (!data) return [];
|
|
126
|
+
if (Array.isArray(data.data)) return data.data;
|
|
127
|
+
if (data.data && Array.isArray(data.data.list)) return data.data.list;
|
|
128
|
+
if (data.data && Array.isArray(data.data.records)) return data.data.records;
|
|
129
|
+
if (Array.isArray(data.list)) return data.list;
|
|
130
|
+
return [];
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
function _normalizeRole(r) {
|
|
134
|
+
if (r === "user" || r === "USER" || r === 1) return "user";
|
|
135
|
+
if (r === "assistant" || r === "ASSISTANT" || r === "bot" || r === 2) return "assistant";
|
|
136
|
+
if (r === "system" || r === "SYSTEM" || r === 0) return "system";
|
|
137
|
+
return r ? String(r).toLowerCase() : "assistant";
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
function _buildContent(m) {
|
|
141
|
+
// Tongyi messages have content fanned across `contents[].content` for
|
|
142
|
+
// multi-segment replies (image + text). Flatten to single text.
|
|
143
|
+
const segments = Array.isArray(m.contents) ? m.contents : null;
|
|
144
|
+
const text = segments
|
|
145
|
+
? segments.map((s) => s.content || "").filter(Boolean).join("\n")
|
|
146
|
+
: (m.content || "");
|
|
147
|
+
const content = { text };
|
|
148
|
+
|
|
149
|
+
const files = []
|
|
150
|
+
.concat(Array.isArray(m.fileList) ? m.fileList : [])
|
|
151
|
+
.concat(Array.isArray(m.imageList) ? m.imageList : []);
|
|
152
|
+
if (files.length > 0) {
|
|
153
|
+
content.attachments = files
|
|
154
|
+
.map((f) => ({
|
|
155
|
+
type: (f.type === "image" || /image/i.test(f.fileType || "")) ? "image" : "file",
|
|
156
|
+
filename: f.fileName || f.name,
|
|
157
|
+
url: f.url || f.fileUrl,
|
|
158
|
+
size: f.size,
|
|
159
|
+
mimeType: f.fileType,
|
|
160
|
+
}))
|
|
161
|
+
.filter((a) => a.url || a.filename);
|
|
162
|
+
}
|
|
163
|
+
return content;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
function _toMs(t) {
|
|
167
|
+
if (typeof t === "number") return t > 1e12 ? t : t * 1000;
|
|
168
|
+
if (typeof t === "string") {
|
|
169
|
+
const n = Number(t);
|
|
170
|
+
if (Number.isFinite(n)) return n > 1e12 ? n : n * 1000;
|
|
171
|
+
const d = Date.parse(t);
|
|
172
|
+
return Number.isFinite(d) ? d : 0;
|
|
173
|
+
}
|
|
174
|
+
return 0;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
const SPEC = {
|
|
178
|
+
name: "tongyi",
|
|
179
|
+
displayName: "通义千问",
|
|
180
|
+
androidPackage: "com.aliyun.tongyi",
|
|
181
|
+
loginUrl: "https://tongyi.aliyun.com/",
|
|
182
|
+
cookieDomains: ["tongyi.aliyun.com", ".aliyun.com"],
|
|
183
|
+
rateLimits: { perMinute: 20, minIntervalMs: 2000 }, // Aliyun anti-bot — conservative
|
|
184
|
+
|
|
185
|
+
validateCookie,
|
|
186
|
+
listConversations,
|
|
187
|
+
listMessages,
|
|
188
|
+
};
|
|
189
|
+
|
|
190
|
+
module.exports = {
|
|
191
|
+
SPEC,
|
|
192
|
+
_internal: { _toMs, _normalizeRole, _buildContent, _csrfHeader, _extractList, BASE },
|
|
193
|
+
};
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 智谱清言 (Zhipu Qingyan) vendor adapter — Phase 10.2 wiring.
|
|
3
|
+
*
|
|
4
|
+
* Reference: docs/design/Adapter_AIChat_History.md §6.4
|
|
5
|
+
* - login https://chatglm.cn/
|
|
6
|
+
* - convs GET /chatglm/backend-api/v1/conversation/list
|
|
7
|
+
* - msgs GET /chatglm/backend-api/v1/conversation/<id>
|
|
8
|
+
* - cookie chatglm_token (Bearer-style — re-projected from cookie when
|
|
9
|
+
* present; otherwise relies on cookie auth alone)
|
|
10
|
+
*
|
|
11
|
+
* GLM-4 messages have optional `tool_calls` for web-search injection; we
|
|
12
|
+
* thread those into RawMessage.extra.toolCalls so schema-map preserves them
|
|
13
|
+
* across the KG / RAG flow.
|
|
14
|
+
*
|
|
15
|
+
* Endpoint shapes are reverse-engineered + SUBJECT TO DRIFT.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
"use strict";
|
|
19
|
+
|
|
20
|
+
const BASE = "https://chatglm.cn";
|
|
21
|
+
const CONV_LIST_PATH = "/chatglm/backend-api/v1/conversation/list";
|
|
22
|
+
const CONV_DETAIL_PATH = "/chatglm/backend-api/v1/conversation/";
|
|
23
|
+
const USER_INFO_PATH = "/chatglm/backend-api/v1/user/info";
|
|
24
|
+
|
|
25
|
+
const DEFAULT_PAGE_SIZE = 30;
|
|
26
|
+
|
|
27
|
+
function _ensureClient(ctx) {
|
|
28
|
+
if (!ctx || !ctx.httpClient) {
|
|
29
|
+
throw new Error("zhipu: ctx.httpClient required");
|
|
30
|
+
}
|
|
31
|
+
return ctx.httpClient;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function _authHeader(session) {
|
|
35
|
+
const token = session && session.get("chatglm_token");
|
|
36
|
+
return token ? { Authorization: `Bearer ${token}` } : {};
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
async function validateCookie(ctx) {
|
|
40
|
+
const client = _ensureClient(ctx);
|
|
41
|
+
try {
|
|
42
|
+
const data = await client.getJson(BASE + USER_INFO_PATH, {
|
|
43
|
+
session: ctx.session,
|
|
44
|
+
headers: _authHeader(ctx.session),
|
|
45
|
+
});
|
|
46
|
+
if (data && data.status === 0 && data.result) {
|
|
47
|
+
return { ok: true, userId: data.result.user_id || data.result.id };
|
|
48
|
+
}
|
|
49
|
+
if (data && (data.user_id || data.id)) {
|
|
50
|
+
return { ok: true, userId: data.user_id || data.id };
|
|
51
|
+
}
|
|
52
|
+
return { ok: false, reason: "UNEXPECTED_RESPONSE_SHAPE" };
|
|
53
|
+
} catch (err) {
|
|
54
|
+
return { ok: false, reason: err.code || err.message };
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
async function *listConversations(ctx, opts = {}) {
|
|
59
|
+
const client = _ensureClient(ctx);
|
|
60
|
+
const pageSize = Number.isFinite(opts.pageSize) ? opts.pageSize : DEFAULT_PAGE_SIZE;
|
|
61
|
+
const sinceTs = opts.since && opts.since.lastUpdatedAt ? Number(opts.since.lastUpdatedAt) : 0;
|
|
62
|
+
|
|
63
|
+
let page = 1;
|
|
64
|
+
while (true) {
|
|
65
|
+
const url = new URL(BASE + CONV_LIST_PATH);
|
|
66
|
+
url.searchParams.set("page", String(page));
|
|
67
|
+
url.searchParams.set("page_size", String(pageSize));
|
|
68
|
+
|
|
69
|
+
const data = await client.getJson(url.toString(), {
|
|
70
|
+
session: ctx.session,
|
|
71
|
+
headers: _authHeader(ctx.session),
|
|
72
|
+
});
|
|
73
|
+
const list = _extractList(data);
|
|
74
|
+
if (list.length === 0) return;
|
|
75
|
+
|
|
76
|
+
let stopped = false;
|
|
77
|
+
for (const c of list) {
|
|
78
|
+
const updatedAt = _toMs(c.update_time || c.updateTime || c.create_time);
|
|
79
|
+
if (sinceTs && updatedAt <= sinceTs) {
|
|
80
|
+
stopped = true;
|
|
81
|
+
break;
|
|
82
|
+
}
|
|
83
|
+
yield {
|
|
84
|
+
vendor: "zhipu",
|
|
85
|
+
originalId: String(c.conversation_id || c.id),
|
|
86
|
+
title: c.title || c.summary || undefined,
|
|
87
|
+
modelName: c.model || c.model_name || undefined,
|
|
88
|
+
createdAt: _toMs(c.create_time || c.createTime),
|
|
89
|
+
updatedAt,
|
|
90
|
+
messageCount: c.message_count || undefined,
|
|
91
|
+
archived: Boolean(c.archived),
|
|
92
|
+
extra: { assistantId: c.assistant_id },
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
if (stopped) return;
|
|
96
|
+
if (list.length < pageSize) return;
|
|
97
|
+
page++;
|
|
98
|
+
if (page > 200) return;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
async function *listMessages(ctx, conversationId, _opts = {}) {
|
|
103
|
+
const client = _ensureClient(ctx);
|
|
104
|
+
const url = BASE + CONV_DETAIL_PATH + encodeURIComponent(conversationId);
|
|
105
|
+
const data = await client.getJson(url, {
|
|
106
|
+
session: ctx.session,
|
|
107
|
+
headers: _authHeader(ctx.session),
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
// GLM detail returns { result: { messages: [...] } } or { messages: [...] }.
|
|
111
|
+
let msgs = [];
|
|
112
|
+
if (data && data.result && Array.isArray(data.result.messages)) msgs = data.result.messages;
|
|
113
|
+
else if (data && Array.isArray(data.messages)) msgs = data.messages;
|
|
114
|
+
else if (data && data.result && Array.isArray(data.result.history)) msgs = data.result.history;
|
|
115
|
+
|
|
116
|
+
msgs.sort((a, b) => _toMs(a.create_time || a.timestamp) - _toMs(b.create_time || b.timestamp));
|
|
117
|
+
|
|
118
|
+
for (const m of msgs) {
|
|
119
|
+
yield {
|
|
120
|
+
vendor: "zhipu",
|
|
121
|
+
originalId: String(m.id || m.message_id),
|
|
122
|
+
conversationId: String(conversationId),
|
|
123
|
+
role: _normalizeRole(m.role || m.sender),
|
|
124
|
+
content: _buildContent(m),
|
|
125
|
+
createdAt: _toMs(m.create_time || m.timestamp),
|
|
126
|
+
parentMessageId: m.parent_id ? String(m.parent_id) : undefined,
|
|
127
|
+
modelName: m.model || undefined,
|
|
128
|
+
extra: {
|
|
129
|
+
toolCalls: Array.isArray(m.tool_calls) ? m.tool_calls : undefined,
|
|
130
|
+
searchQuery: m.search_query,
|
|
131
|
+
},
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function _extractList(data) {
|
|
137
|
+
if (!data) return [];
|
|
138
|
+
if (Array.isArray(data.result)) return data.result;
|
|
139
|
+
if (data.result && Array.isArray(data.result.list)) return data.result.list;
|
|
140
|
+
if (data.result && Array.isArray(data.result.conversations)) return data.result.conversations;
|
|
141
|
+
if (Array.isArray(data.conversations)) return data.conversations;
|
|
142
|
+
if (Array.isArray(data.list)) return data.list;
|
|
143
|
+
return [];
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
function _normalizeRole(r) {
|
|
147
|
+
if (r === "user" || r === "USER") return "user";
|
|
148
|
+
if (r === "assistant" || r === "ASSISTANT" || r === "bot") return "assistant";
|
|
149
|
+
if (r === "system" || r === "SYSTEM") return "system";
|
|
150
|
+
if (r === "tool" || r === "TOOL") return "tool";
|
|
151
|
+
return r ? String(r).toLowerCase() : "assistant";
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
function _buildContent(m) {
|
|
155
|
+
const content = { text: m.content || m.text || "" };
|
|
156
|
+
if (Array.isArray(m.attachments) && m.attachments.length > 0) {
|
|
157
|
+
content.attachments = m.attachments
|
|
158
|
+
.map((a) => ({
|
|
159
|
+
type: (a.type === "image" || /image/i.test(a.mime_type || "")) ? "image" : "file",
|
|
160
|
+
filename: a.name || a.file_name,
|
|
161
|
+
url: a.url || a.download_url,
|
|
162
|
+
size: a.size,
|
|
163
|
+
mimeType: a.mime_type,
|
|
164
|
+
}))
|
|
165
|
+
.filter((a) => a.url || a.filename);
|
|
166
|
+
}
|
|
167
|
+
if (Array.isArray(m.images) && m.images.length > 0) {
|
|
168
|
+
content.attachments = (content.attachments || []).concat(
|
|
169
|
+
m.images.map((i) => ({ type: "image", url: i.url || i, mimeType: "image/png" })),
|
|
170
|
+
);
|
|
171
|
+
}
|
|
172
|
+
return content;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
function _toMs(t) {
|
|
176
|
+
if (typeof t === "number") return t > 1e12 ? t : t * 1000;
|
|
177
|
+
if (typeof t === "string") {
|
|
178
|
+
const n = Number(t);
|
|
179
|
+
if (Number.isFinite(n)) return n > 1e12 ? n : n * 1000;
|
|
180
|
+
const d = Date.parse(t);
|
|
181
|
+
return Number.isFinite(d) ? d : 0;
|
|
182
|
+
}
|
|
183
|
+
return 0;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
const SPEC = {
|
|
187
|
+
name: "zhipu",
|
|
188
|
+
displayName: "智谱清言",
|
|
189
|
+
androidPackage: "com.zhipuai.qingyan",
|
|
190
|
+
loginUrl: "https://chatglm.cn/",
|
|
191
|
+
cookieDomains: ["chatglm.cn", ".chatglm.cn"],
|
|
192
|
+
rateLimits: { perMinute: 20, minIntervalMs: 2000 },
|
|
193
|
+
|
|
194
|
+
validateCookie,
|
|
195
|
+
listConversations,
|
|
196
|
+
listMessages,
|
|
197
|
+
};
|
|
198
|
+
|
|
199
|
+
module.exports = {
|
|
200
|
+
SPEC,
|
|
201
|
+
_internal: { _toMs, _normalizeRole, _buildContent, _authHeader, _extractList, BASE },
|
|
202
|
+
};
|
|
@@ -0,0 +1,307 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AlipayBillAdapter — Phase 6 of the Personal Data Hub.
|
|
3
|
+
*
|
|
4
|
+
* **Not a server-pull adapter** — Alipay has no public API. Users export
|
|
5
|
+
* a CSV bill from the Alipay app (我的 → 账单 → 开具交易流水证明 → 发到
|
|
6
|
+
* 邮箱), then drop the resulting `alipay_record_*.zip` into our UI.
|
|
7
|
+
*
|
|
8
|
+
* The adapter's `sync()` therefore takes an explicit `csvPath` (or
|
|
9
|
+
* `zipPath` + password) opt rather than auto-fetching. Registry calls
|
|
10
|
+
* with no opt → no-op (returns immediately). UI drives sync per-file.
|
|
11
|
+
*
|
|
12
|
+
* Watermark: Alipay CSVs are full-month exports; no incremental
|
|
13
|
+
* server-side state. We dedupe via `source.originalId = txId` so re-
|
|
14
|
+
* importing the same CSV produces 0 new events. Watermark is only
|
|
15
|
+
* informational ("last imported file hash + row count").
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
"use strict";
|
|
19
|
+
|
|
20
|
+
const fs = require("node:fs");
|
|
21
|
+
const crypto = require("node:crypto");
|
|
22
|
+
|
|
23
|
+
const { EVENT_SUBTYPES, PERSON_SUBTYPES, CAPTURED_BY } = require("../../constants");
|
|
24
|
+
const { newId } = require("../../ids");
|
|
25
|
+
const { parseAlipayCsvBuffer } = require("./csv-parser");
|
|
26
|
+
const { extractCsvFromZip } = require("./zip-decryptor");
|
|
27
|
+
const {
|
|
28
|
+
classifyCounterparty,
|
|
29
|
+
counterpartyToPersonId,
|
|
30
|
+
} = require("./counterparty");
|
|
31
|
+
|
|
32
|
+
const NAME = "alipay-bill";
|
|
33
|
+
const VERSION = "0.1.0"; // Phase 6 — initial CSV-import adapter
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Map Alipay's `类型` string → UnifiedSchema Event.subtype.
|
|
37
|
+
* Per design doc §4.4.
|
|
38
|
+
*/
|
|
39
|
+
function mapAlipayTypeToSubtype(alipayType, direction) {
|
|
40
|
+
const t = String(alipayType || "");
|
|
41
|
+
if (t.includes("转账")) return "transfer";
|
|
42
|
+
if (t.includes("退款")) return "refund";
|
|
43
|
+
if (t.includes("理财") || t.includes("余额宝")) return "investment";
|
|
44
|
+
if (t.includes("红包")) return "redenvelope";
|
|
45
|
+
if (t.includes("缴费")) return "utility";
|
|
46
|
+
if (t.includes("交易关闭") || t.includes("交易失败")) return "cancelled";
|
|
47
|
+
return direction === "收入" ? "income" : "payment";
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
class AlipayBillAdapter {
|
|
51
|
+
constructor(opts) {
|
|
52
|
+
if (!opts || typeof opts !== "object") {
|
|
53
|
+
throw new Error("AlipayBillAdapter: opts required");
|
|
54
|
+
}
|
|
55
|
+
const account = opts.account;
|
|
56
|
+
if (!account || typeof account !== "object") {
|
|
57
|
+
throw new Error("AlipayBillAdapter: opts.account required");
|
|
58
|
+
}
|
|
59
|
+
if (typeof account.email !== "string" || account.email.length === 0) {
|
|
60
|
+
throw new Error("AlipayBillAdapter: account.email required (Alipay account identifier)");
|
|
61
|
+
}
|
|
62
|
+
this.account = account;
|
|
63
|
+
// ZIP password (= 身份证后 6 位 by default). Optional — if the user's
|
|
64
|
+
// export is unencrypted (rare) or they extract manually first, pass
|
|
65
|
+
// csvPath at sync() time.
|
|
66
|
+
this._zipPassword = typeof opts.zipPassword === "string" ? opts.zipPassword : null;
|
|
67
|
+
// Test seams
|
|
68
|
+
this._csvParser = typeof opts.csvParser === "function" ? opts.csvParser : parseAlipayCsvBuffer;
|
|
69
|
+
this._zipExtractor = typeof opts.zipExtractor === "function" ? opts.zipExtractor : extractCsvFromZip;
|
|
70
|
+
|
|
71
|
+
this.name = NAME;
|
|
72
|
+
this.version = VERSION;
|
|
73
|
+
this.capabilities = ["import:csv-zip", "parse:transactions"];
|
|
74
|
+
this.rateLimits = {};
|
|
75
|
+
this.dataDisclosure = {
|
|
76
|
+
fields: [
|
|
77
|
+
"alipay:txId, createdAt, paidAt, counterparty, itemName, amount, direction, status, note",
|
|
78
|
+
],
|
|
79
|
+
sensitivity: "high",
|
|
80
|
+
legalGate: false,
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
async authenticate(_ctx = {}) {
|
|
85
|
+
// No server auth — adapter is always "ok" once configured.
|
|
86
|
+
return { ok: true, account: this.account.email, provider: "alipay-bill" };
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
async healthCheck() {
|
|
90
|
+
return { ok: true, lastChecked: Date.now() };
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* `sync()` here is driven by an explicit file path. When called with
|
|
95
|
+
* no zipPath/csvPath the adapter emits 0 events (waiting for user
|
|
96
|
+
* action). Registry's syncAll() will hit this case for periodic
|
|
97
|
+
* checks — same as Phase 5 EmailAdapter handles authcode-not-set.
|
|
98
|
+
*
|
|
99
|
+
* @param {object} opts
|
|
100
|
+
* @param {string} [opts.zipPath] full path to alipay_record_*.zip
|
|
101
|
+
* @param {string} [opts.csvPath] full path to a pre-extracted .csv
|
|
102
|
+
* @param {string} [opts.zipPassword] overrides constructor zipPassword
|
|
103
|
+
* @param {Function} [opts.onProgress]
|
|
104
|
+
*/
|
|
105
|
+
async *sync(opts = {}) {
|
|
106
|
+
const zipPath = typeof opts.zipPath === "string" ? opts.zipPath : null;
|
|
107
|
+
const csvPath = typeof opts.csvPath === "string" ? opts.csvPath : null;
|
|
108
|
+
if (!zipPath && !csvPath) {
|
|
109
|
+
// Idle — no file to import this run
|
|
110
|
+
return;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
const onProgress = typeof opts.onProgress === "function" ? opts.onProgress : null;
|
|
114
|
+
const emit = (phase, payload = {}) => {
|
|
115
|
+
if (!onProgress) return;
|
|
116
|
+
try { onProgress({ phase, adapter: NAME, ...payload }); } catch (_e) {}
|
|
117
|
+
};
|
|
118
|
+
|
|
119
|
+
emit("opening", { zipPath, csvPath });
|
|
120
|
+
|
|
121
|
+
let csvBuffer;
|
|
122
|
+
let sourceFile;
|
|
123
|
+
if (zipPath) {
|
|
124
|
+
const password = typeof opts.zipPassword === "string" ? opts.zipPassword : this._zipPassword;
|
|
125
|
+
const out = await this._zipExtractor(zipPath, { password });
|
|
126
|
+
csvBuffer = out.buffer;
|
|
127
|
+
sourceFile = `${zipPath}::${out.filename}`;
|
|
128
|
+
} else {
|
|
129
|
+
csvBuffer = fs.readFileSync(csvPath);
|
|
130
|
+
sourceFile = csvPath;
|
|
131
|
+
}
|
|
132
|
+
const fileSha256 = crypto.createHash("sha256").update(csvBuffer).digest("hex");
|
|
133
|
+
emit("parsing", { sourceFile, fileSha256, bytes: csvBuffer.length });
|
|
134
|
+
|
|
135
|
+
const parsed = this._csvParser(csvBuffer);
|
|
136
|
+
emit("parsed", {
|
|
137
|
+
sourceFile,
|
|
138
|
+
encoding: parsed.encoding,
|
|
139
|
+
rows: parsed.rows.length,
|
|
140
|
+
header: parsed.header,
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
let yielded = 0;
|
|
144
|
+
for (const row of parsed.rows) {
|
|
145
|
+
emit("row", { current: yielded + 1, total: parsed.rows.length, txId: row.txId });
|
|
146
|
+
yield this._rowToRawEvent(row, {
|
|
147
|
+
sourceFile,
|
|
148
|
+
fileSha256,
|
|
149
|
+
accountEmail: this.account.email,
|
|
150
|
+
importedAt: Date.now(),
|
|
151
|
+
billPeriod: parsed.header,
|
|
152
|
+
});
|
|
153
|
+
yielded += 1;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
emit("done", { yielded, sourceFile });
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* normalize(raw) → NormalizedBatch (Event + Persons + Items).
|
|
161
|
+
* Per design doc §5.4.
|
|
162
|
+
*/
|
|
163
|
+
normalize(raw) {
|
|
164
|
+
if (!raw || typeof raw !== "object" || !raw.payload) {
|
|
165
|
+
throw new Error("AlipayBillAdapter.normalize: missing raw or raw.payload");
|
|
166
|
+
}
|
|
167
|
+
const row = raw.payload.row;
|
|
168
|
+
if (!row || typeof row !== "object") {
|
|
169
|
+
throw new Error("AlipayBillAdapter.normalize: payload.row missing");
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
// Parse the amount and timestamps
|
|
173
|
+
const amount = parseFloat(row.amount);
|
|
174
|
+
const occurredAt = parseAlipayDateTime(row.paidAt) || parseAlipayDateTime(row.createdAt) || raw.capturedAt || Date.now();
|
|
175
|
+
|
|
176
|
+
// Counterparty → Person (with stable id for dedup)
|
|
177
|
+
const counterpartyId = counterpartyToPersonId(row.counterparty);
|
|
178
|
+
const counterpartyKind = classifyCounterparty(row.counterparty);
|
|
179
|
+
const direction = row.direction === "收入" ? "in" : "out";
|
|
180
|
+
|
|
181
|
+
const subtype = mapAlipayTypeToSubtype(row.alipayType, row.direction);
|
|
182
|
+
const eventId = newId();
|
|
183
|
+
|
|
184
|
+
// Skip closed / failed transactions — they polluted vault with
|
|
185
|
+
// "transaction never happened" rows. Mark as cancelled instead.
|
|
186
|
+
const isCancelled = subtype === "cancelled" || /关闭|失败/.test(row.status || "");
|
|
187
|
+
|
|
188
|
+
const ingestedAt = Date.now();
|
|
189
|
+
const source = {
|
|
190
|
+
adapter: NAME,
|
|
191
|
+
adapterVersion: VERSION,
|
|
192
|
+
originalId: row.txId,
|
|
193
|
+
capturedAt: raw.capturedAt || occurredAt,
|
|
194
|
+
capturedBy: CAPTURED_BY ? (CAPTURED_BY.EXPORT || "export") : "export",
|
|
195
|
+
};
|
|
196
|
+
|
|
197
|
+
const event = {
|
|
198
|
+
id: eventId,
|
|
199
|
+
type: "event",
|
|
200
|
+
subtype: isCancelled ? "cancelled" : subtype,
|
|
201
|
+
occurredAt,
|
|
202
|
+
actor: direction === "out" ? "person-self" : counterpartyId,
|
|
203
|
+
participants: [counterpartyId, "person-self"].filter(Boolean),
|
|
204
|
+
content: {
|
|
205
|
+
title: row.itemName || row.alipayType || row.counterparty,
|
|
206
|
+
...(row.note ? { text: row.note } : {}),
|
|
207
|
+
amount: {
|
|
208
|
+
value: Number.isFinite(amount) ? amount : 0,
|
|
209
|
+
currency: "CNY",
|
|
210
|
+
direction,
|
|
211
|
+
},
|
|
212
|
+
},
|
|
213
|
+
ingestedAt,
|
|
214
|
+
source,
|
|
215
|
+
extra: {
|
|
216
|
+
alipayType: row.alipayType,
|
|
217
|
+
sourceChannel: row.sourceChannel || undefined,
|
|
218
|
+
merchantOrderNumber: row.merchantOrderNumber || undefined,
|
|
219
|
+
txStatus: row.status,
|
|
220
|
+
serviceFee: parseFloat(row.serviceFee || "0") || 0,
|
|
221
|
+
refundedAmount: parseFloat(row.refundedAmount || "0") || 0,
|
|
222
|
+
fundStatus: row.fundStatus || undefined,
|
|
223
|
+
accountEmail: raw.payload.accountEmail,
|
|
224
|
+
fileSha256: raw.payload.fileSha256,
|
|
225
|
+
billPeriod: raw.payload.billPeriod || undefined,
|
|
226
|
+
counterpartyKind,
|
|
227
|
+
...(counterpartyKind === "unknown" ? { needsResolve: true } : {}),
|
|
228
|
+
},
|
|
229
|
+
};
|
|
230
|
+
|
|
231
|
+
const persons = [{
|
|
232
|
+
id: counterpartyId,
|
|
233
|
+
type: "person",
|
|
234
|
+
subtype: counterpartyKind === "contact"
|
|
235
|
+
? (PERSON_SUBTYPES ? (PERSON_SUBTYPES.CONTACT || "contact") : "contact")
|
|
236
|
+
: (PERSON_SUBTYPES ? (PERSON_SUBTYPES.MERCHANT || "merchant") : "merchant"),
|
|
237
|
+
names: [row.counterparty || "(unknown)"],
|
|
238
|
+
identifiers: {},
|
|
239
|
+
ingestedAt,
|
|
240
|
+
source,
|
|
241
|
+
extra: {
|
|
242
|
+
...(counterpartyKind === "unknown" ? { needsResolve: true } : {}),
|
|
243
|
+
firstSeenAt: occurredAt,
|
|
244
|
+
},
|
|
245
|
+
}];
|
|
246
|
+
|
|
247
|
+
// Item (only when an itemName is present and not just an alipayType)
|
|
248
|
+
const items = [];
|
|
249
|
+
if (row.itemName && row.itemName !== row.alipayType) {
|
|
250
|
+
items.push({
|
|
251
|
+
id: newId(),
|
|
252
|
+
type: "item",
|
|
253
|
+
subtype: "product",
|
|
254
|
+
name: row.itemName,
|
|
255
|
+
price: { value: amount, currency: "CNY" },
|
|
256
|
+
merchant: counterpartyId,
|
|
257
|
+
ingestedAt,
|
|
258
|
+
source,
|
|
259
|
+
extra: {
|
|
260
|
+
sourceEventId: eventId,
|
|
261
|
+
},
|
|
262
|
+
});
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
return { events: [event], persons, places: [], items, topics: [] };
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
_rowToRawEvent(row, ctx) {
|
|
269
|
+
return {
|
|
270
|
+
adapter: NAME,
|
|
271
|
+
originalId: row.txId,
|
|
272
|
+
capturedAt: parseAlipayDateTime(row.paidAt) || parseAlipayDateTime(row.createdAt) || ctx.importedAt,
|
|
273
|
+
payload: {
|
|
274
|
+
row,
|
|
275
|
+
accountEmail: ctx.accountEmail,
|
|
276
|
+
sourceFile: ctx.sourceFile,
|
|
277
|
+
fileSha256: ctx.fileSha256,
|
|
278
|
+
importedAt: ctx.importedAt,
|
|
279
|
+
billPeriod: ctx.billPeriod,
|
|
280
|
+
},
|
|
281
|
+
};
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
// ─── helpers ────────────────────────────────────────────────────────────
|
|
286
|
+
|
|
287
|
+
/**
|
|
288
|
+
* Parse "2024-04-01 09:23:13" → ms epoch (local time). Alipay timestamps
|
|
289
|
+
* are local-time strings (no timezone marker). Returns null on parse
|
|
290
|
+
* failure.
|
|
291
|
+
*/
|
|
292
|
+
function parseAlipayDateTime(s) {
|
|
293
|
+
if (typeof s !== "string" || s.length === 0) return null;
|
|
294
|
+
// Replace space with T so Date can parse it
|
|
295
|
+
const iso = s.replace(" ", "T");
|
|
296
|
+
const d = new Date(iso);
|
|
297
|
+
const t = d.getTime();
|
|
298
|
+
return Number.isFinite(t) ? t : null;
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
module.exports = {
|
|
302
|
+
AlipayBillAdapter,
|
|
303
|
+
mapAlipayTypeToSubtype,
|
|
304
|
+
parseAlipayDateTime,
|
|
305
|
+
NAME,
|
|
306
|
+
VERSION,
|
|
307
|
+
};
|