dsh-session-cloud 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cordis.patch.yml +9 -0
- package/lib/client.js +1912 -0
- package/lib/client.js.map +7 -0
- package/lib/config.d.ts +60 -0
- package/lib/config.js +61 -0
- package/lib/crypto/envelope.d.ts +62 -0
- package/lib/crypto/envelope.js +162 -0
- package/lib/dsh-types.d.ts +69 -0
- package/lib/dsh-types.js +1 -0
- package/lib/index.d.ts +11 -0
- package/lib/index.js +485 -0
- package/lib/status.d.ts +17 -0
- package/lib/status.js +65 -0
- package/lib/sync/engine.d.ts +85 -0
- package/lib/sync/engine.js +379 -0
- package/lib/sync/frames.d.ts +35 -0
- package/lib/sync/frames.js +121 -0
- package/lib/sync/http-client.d.ts +44 -0
- package/lib/sync/http-client.js +87 -0
- package/lib/sync/kdf.d.ts +14 -0
- package/lib/sync/kdf.js +102 -0
- package/lib/sync/restore.d.ts +136 -0
- package/lib/sync/restore.js +295 -0
- package/lib/sync/state.d.ts +89 -0
- package/lib/sync/state.js +117 -0
- package/package.json +62 -0
|
@@ -0,0 +1,379 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 上行同步引擎(docs/02 §4.2、§10)。
|
|
3
|
+
*
|
|
4
|
+
* 职责:变更发现(listSnapshots revision 对比 + 定时兜底)→ 按字节偏移读新增
|
|
5
|
+
* → zstd 帧边界扫描取完整帧 → 链式 GCM 分段加密 → append 上传(409 幂等判定)
|
|
6
|
+
* → meta 覆写 → 推进本地 state。与 cordis 解耦,纯类便于单测。
|
|
7
|
+
*/
|
|
8
|
+
import fsp from 'node:fs/promises';
|
|
9
|
+
import { LogEncryptor, encryptMeta, verifyTailSegment } from '../crypto/envelope.js';
|
|
10
|
+
import { detectEncoding, emptyFold, foldFrame, scanZstdFrames } from './frames.js';
|
|
11
|
+
import { ProtocolError } from './http-client.js';
|
|
12
|
+
const BACKOFF_INITIAL_MS = 1000;
|
|
13
|
+
const BACKOFF_MAX_MS = 5 * 60 * 1000;
|
|
14
|
+
/** flush 钩子触发 scan 的合并窗口:一轮对话多次 flush 只做一次变更发现。 */
|
|
15
|
+
const SCAN_DEBOUNCE_MS = 500;
|
|
16
|
+
/** 409 幂等校验不通过:写入方冲突(疑似同名设备的另一台机器)。 */
|
|
17
|
+
export class ConflictError extends Error {
|
|
18
|
+
constructor(message) {
|
|
19
|
+
super(message);
|
|
20
|
+
this.name = 'ConflictError';
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
export function logKey(device, sessionId) {
|
|
24
|
+
return `sessions/${device}/${sessionId}.log.enc`;
|
|
25
|
+
}
|
|
26
|
+
export function metaKey(device, sessionId) {
|
|
27
|
+
return `sessions/${device}/${sessionId}.meta.enc`;
|
|
28
|
+
}
|
|
29
|
+
export function freshSessionState() {
|
|
30
|
+
return {
|
|
31
|
+
uploadedBytes: 0,
|
|
32
|
+
remoteBytes: 0,
|
|
33
|
+
lastSegmentIndex: -1,
|
|
34
|
+
lastTag: '',
|
|
35
|
+
localRevision: '',
|
|
36
|
+
eventCount: 0,
|
|
37
|
+
updatedAt: 0,
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
export class SyncEngine {
|
|
41
|
+
options;
|
|
42
|
+
scheduled = new Map();
|
|
43
|
+
latest = new Map();
|
|
44
|
+
inFlight = new Set();
|
|
45
|
+
rerunRequested = new Set();
|
|
46
|
+
backoff = new Map();
|
|
47
|
+
scanTimer;
|
|
48
|
+
disposed = false;
|
|
49
|
+
now;
|
|
50
|
+
constructor(options) {
|
|
51
|
+
this.options = options;
|
|
52
|
+
this.now = options.now ?? Date.now;
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* 外部触发的变更发现入口(session/flush 钩子接线,M5):把窗口内的多次
|
|
56
|
+
* 触发合并为一次 scan。listener 必须轻——flush 是 awaited durability
|
|
57
|
+
* checkpoint,重活留给 scan/防抖队列,这里只拨一个 timer。
|
|
58
|
+
*/
|
|
59
|
+
requestScan() {
|
|
60
|
+
if (this.disposed || !this.options.getConfig().autoUpload)
|
|
61
|
+
return;
|
|
62
|
+
if (this.scanTimer !== undefined)
|
|
63
|
+
return;
|
|
64
|
+
this.scanTimer = setTimeout(() => {
|
|
65
|
+
this.scanTimer = undefined;
|
|
66
|
+
void this.scan();
|
|
67
|
+
}, SCAN_DEBOUNCE_MS);
|
|
68
|
+
this.scanTimer.unref?.();
|
|
69
|
+
}
|
|
70
|
+
/** 一轮变更发现:revision 对比,变化的进防抖队列。 */
|
|
71
|
+
async scan() {
|
|
72
|
+
if (this.disposed || !this.options.getConfig().autoUpload)
|
|
73
|
+
return;
|
|
74
|
+
let snapshots;
|
|
75
|
+
try {
|
|
76
|
+
snapshots = await this.options.persistence.listSnapshots();
|
|
77
|
+
}
|
|
78
|
+
catch (error) {
|
|
79
|
+
this.emit({ kind: 'error', sessionId: '', message: `listSnapshots failed: ${String(error)}` });
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
for (const snapshot of snapshots) {
|
|
83
|
+
if (!this.needsSync(snapshot))
|
|
84
|
+
continue;
|
|
85
|
+
this.schedule(snapshot);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
/** 变更判定:revision 与 state 不一致(或处于退避待重试)才需要同步。 */
|
|
89
|
+
needsSync(snapshot) {
|
|
90
|
+
const state = this.options.state.getSession(snapshot.header.id);
|
|
91
|
+
if (state?.status === 'conflict' || state?.status === 'skipped-plain')
|
|
92
|
+
return false;
|
|
93
|
+
const wait = this.backoff.get(snapshot.header.id);
|
|
94
|
+
if (wait !== undefined && wait.revision !== snapshot.revision) {
|
|
95
|
+
// 会话有新变更:重置退避立即重试(docs/02 §10「下一次会话变更重置退避」)
|
|
96
|
+
this.backoff.delete(snapshot.header.id);
|
|
97
|
+
return true;
|
|
98
|
+
}
|
|
99
|
+
if (state?.localRevision === snapshot.revision && wait === undefined)
|
|
100
|
+
return false;
|
|
101
|
+
return true;
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* 立即全部同步(设置卡「立即全部同步」按钮,docs/01 §5.3):
|
|
105
|
+
* 用户显式触发,无视 autoUpload 开关;全量扫描后把所有待同步会话
|
|
106
|
+
* 调度进队列并 await 完成(复用 schedule/flush 防抖-冲刷机制)。
|
|
107
|
+
*/
|
|
108
|
+
async syncAllNow() {
|
|
109
|
+
if (this.disposed)
|
|
110
|
+
return { ok: 0, failed: 0 };
|
|
111
|
+
// 用户显式触发:重置全部退避(手动同步的语义至少等同一次会话变更)
|
|
112
|
+
this.backoff.clear();
|
|
113
|
+
let snapshots;
|
|
114
|
+
try {
|
|
115
|
+
snapshots = await this.options.persistence.listSnapshots();
|
|
116
|
+
}
|
|
117
|
+
catch (error) {
|
|
118
|
+
return { ok: 0, failed: 1, firstError: `listSnapshots failed: ${String(error)}`.slice(0, 300) };
|
|
119
|
+
}
|
|
120
|
+
const targets = snapshots.filter((snapshot) => this.needsSync(snapshot));
|
|
121
|
+
for (const snapshot of targets)
|
|
122
|
+
this.schedule(snapshot);
|
|
123
|
+
await this.flush();
|
|
124
|
+
let ok = 0;
|
|
125
|
+
let failed = 0;
|
|
126
|
+
let firstError;
|
|
127
|
+
for (const snapshot of targets) {
|
|
128
|
+
const state = this.options.state.getSession(snapshot.header.id);
|
|
129
|
+
if (state?.status === 'error' || state?.status === 'conflict') {
|
|
130
|
+
failed += 1;
|
|
131
|
+
firstError ??= `${snapshot.header.id}: ${state.error ?? ''}`.slice(0, 300);
|
|
132
|
+
}
|
|
133
|
+
else {
|
|
134
|
+
ok += 1;
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
return { ok, failed, firstError };
|
|
138
|
+
}
|
|
139
|
+
/** 同会话变更在防抖窗口内合并为一次上传。 */
|
|
140
|
+
schedule(snapshot) {
|
|
141
|
+
if (this.disposed)
|
|
142
|
+
return;
|
|
143
|
+
const id = snapshot.header.id;
|
|
144
|
+
this.latest.set(id, snapshot);
|
|
145
|
+
const existing = this.scheduled.get(id);
|
|
146
|
+
if (existing)
|
|
147
|
+
clearTimeout(existing.timer);
|
|
148
|
+
const timer = setTimeout(() => {
|
|
149
|
+
this.scheduled.delete(id);
|
|
150
|
+
void this.pump(id);
|
|
151
|
+
}, this.options.getConfig().uploadDebounceMs);
|
|
152
|
+
timer.unref?.();
|
|
153
|
+
this.scheduled.set(id, { snapshot, timer });
|
|
154
|
+
}
|
|
155
|
+
/** 进程退出前冲刷:立即执行所有防抖中的会话并等待在途上传结束。 */
|
|
156
|
+
async flush() {
|
|
157
|
+
const ids = [...this.scheduled.keys()];
|
|
158
|
+
for (const { timer } of this.scheduled.values())
|
|
159
|
+
clearTimeout(timer);
|
|
160
|
+
this.scheduled.clear();
|
|
161
|
+
await Promise.all(ids.map((id) => this.pump(id)));
|
|
162
|
+
}
|
|
163
|
+
async dispose() {
|
|
164
|
+
if (this.scanTimer !== undefined) {
|
|
165
|
+
clearTimeout(this.scanTimer);
|
|
166
|
+
this.scanTimer = undefined;
|
|
167
|
+
}
|
|
168
|
+
await this.flush(); // 退出前强制冲刷在途批次
|
|
169
|
+
this.disposed = true;
|
|
170
|
+
// 等在途上传收尾(dispose 不发起新同步)
|
|
171
|
+
while (this.inFlight.size > 0) {
|
|
172
|
+
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
/** 启动:解除 conflict 状态(配置如设备名可能已更换,允许重试),然后做一轮变更发现。 */
|
|
176
|
+
async start() {
|
|
177
|
+
for (const [id, state] of this.options.state.entries()) {
|
|
178
|
+
if (state.status === 'conflict' || state.status === 'error') {
|
|
179
|
+
await this.options.state.putSession(id, { ...state, status: 'ok', error: undefined });
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
await this.scan();
|
|
183
|
+
}
|
|
184
|
+
/** 每会话串行泵:在途期间到来的变更置 rerun 标记,结束后立即再跑一轮。 */
|
|
185
|
+
async pump(sessionId) {
|
|
186
|
+
if (this.disposed)
|
|
187
|
+
return;
|
|
188
|
+
if (this.inFlight.has(sessionId)) {
|
|
189
|
+
this.rerunRequested.add(sessionId);
|
|
190
|
+
return;
|
|
191
|
+
}
|
|
192
|
+
this.inFlight.add(sessionId);
|
|
193
|
+
try {
|
|
194
|
+
do {
|
|
195
|
+
this.rerunRequested.delete(sessionId);
|
|
196
|
+
const snapshot = this.latest.get(sessionId);
|
|
197
|
+
if (!snapshot)
|
|
198
|
+
break;
|
|
199
|
+
await this.syncSession(snapshot);
|
|
200
|
+
} while (this.rerunRequested.has(sessionId) && !this.disposed);
|
|
201
|
+
}
|
|
202
|
+
finally {
|
|
203
|
+
this.inFlight.delete(sessionId);
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
emit(event) {
|
|
207
|
+
this.options.onEvent?.(event);
|
|
208
|
+
}
|
|
209
|
+
async syncSession(snapshot) {
|
|
210
|
+
const id = snapshot.header.id;
|
|
211
|
+
const state = this.options.state.getSession(id) ?? freshSessionState();
|
|
212
|
+
try {
|
|
213
|
+
await this.syncSessionOnce(snapshot, state);
|
|
214
|
+
this.backoff.delete(id);
|
|
215
|
+
}
|
|
216
|
+
catch (error) {
|
|
217
|
+
if (error instanceof ConflictError) {
|
|
218
|
+
state.status = 'conflict';
|
|
219
|
+
state.error = error.message;
|
|
220
|
+
await this.options.state.putSession(id, state);
|
|
221
|
+
this.emit({ kind: 'conflict', sessionId: id, message: error.message });
|
|
222
|
+
return;
|
|
223
|
+
}
|
|
224
|
+
const prev = this.backoff.get(id);
|
|
225
|
+
const delayMs = Math.min(prev ? prev.delayMs * 2 : BACKOFF_INITIAL_MS, BACKOFF_MAX_MS);
|
|
226
|
+
this.backoff.set(id, { delayMs, nextRetryAt: this.now() + delayMs, revision: snapshot.revision });
|
|
227
|
+
state.status = 'error';
|
|
228
|
+
state.error = String(error);
|
|
229
|
+
await this.options.state.putSession(id, state);
|
|
230
|
+
await this.options.state.setGlobal({ lastError: String(error) });
|
|
231
|
+
this.emit({ kind: 'error', sessionId: id, message: String(error) });
|
|
232
|
+
// 退避到点后由下一轮 scan 兜底重试(revision 与 state 不一致会再次 schedule)
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
async syncSessionOnce(snapshot, state) {
|
|
236
|
+
const { header, revision } = snapshot;
|
|
237
|
+
const id = header.id;
|
|
238
|
+
const { client, state: store } = this.options;
|
|
239
|
+
const wait = this.backoff.get(id);
|
|
240
|
+
if (wait && this.now() < wait.nextRetryAt)
|
|
241
|
+
return;
|
|
242
|
+
const location = this.options.persistence.locate(header);
|
|
243
|
+
if (!location) {
|
|
244
|
+
// 无单文件 artifact 的后端(如 SQLite):不同步,标记后不再扫描
|
|
245
|
+
state.status = 'skipped-plain';
|
|
246
|
+
state.error = 'persistence backend exposes no file artifact';
|
|
247
|
+
state.localRevision = revision;
|
|
248
|
+
await store.putSession(id, state);
|
|
249
|
+
return;
|
|
250
|
+
}
|
|
251
|
+
const stat = await fsp.stat(location.path);
|
|
252
|
+
if (stat.size < state.uploadedBytes) {
|
|
253
|
+
// 文件变短(防御:正常不可达)——回退全量重传
|
|
254
|
+
Object.assign(state, freshSessionState());
|
|
255
|
+
}
|
|
256
|
+
// 明文 profile 检测(仅首传判断,之后由 state 记住)
|
|
257
|
+
if (state.uploadedBytes === 0 && state.lastSegmentIndex === -1) {
|
|
258
|
+
const handle = await fsp.open(location.path, 'r');
|
|
259
|
+
try {
|
|
260
|
+
const prefix = Buffer.alloc(4);
|
|
261
|
+
await handle.read(prefix, 0, 4, 0);
|
|
262
|
+
if (detectEncoding(prefix) !== 'zstd') {
|
|
263
|
+
state.status = 'skipped-plain';
|
|
264
|
+
state.error = 'session artifact is not zstd-compressed (compression: none profile)';
|
|
265
|
+
state.localRevision = revision;
|
|
266
|
+
await store.putSession(id, state);
|
|
267
|
+
this.emit({ kind: 'skipped-plain', sessionId: id });
|
|
268
|
+
return;
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
finally {
|
|
272
|
+
await handle.close();
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
const newBytes = await readTail(location.path, state.uploadedBytes);
|
|
276
|
+
if (newBytes.length === 0) {
|
|
277
|
+
state.localRevision = revision;
|
|
278
|
+
await store.putSession(id, state);
|
|
279
|
+
return;
|
|
280
|
+
}
|
|
281
|
+
const scan = scanZstdFrames(newBytes);
|
|
282
|
+
if (scan.frames.length === 0) {
|
|
283
|
+
// 没有完整帧(写入进行中),等下一窗口
|
|
284
|
+
state.localRevision = revision;
|
|
285
|
+
await store.putSession(id, state);
|
|
286
|
+
return;
|
|
287
|
+
}
|
|
288
|
+
const last = scan.frames[scan.frames.length - 1];
|
|
289
|
+
const plaintext = newBytes.subarray(0, last.end);
|
|
290
|
+
// 链式分段加密(从 state 恢复段序号与前段 tag)
|
|
291
|
+
const nextIndex = state.lastSegmentIndex + 1;
|
|
292
|
+
const encryptor = new LogEncryptor(this.options.key, this.options.salt, id, nextIndex, state.lastTag ? Buffer.from(state.lastTag, 'hex') : undefined);
|
|
293
|
+
const segment = encryptor.append(plaintext);
|
|
294
|
+
const payload = state.remoteBytes === 0 ? Buffer.concat([encryptor.objectHeader(), segment]) : segment;
|
|
295
|
+
const key = logKey(this.options.device, id);
|
|
296
|
+
try {
|
|
297
|
+
if (state.remoteBytes === 0) {
|
|
298
|
+
await client.overwrite(key, payload);
|
|
299
|
+
}
|
|
300
|
+
else {
|
|
301
|
+
await client.append(key, state.remoteBytes, payload);
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
catch (error) {
|
|
305
|
+
if (!(error instanceof ProtocolError) || error.status !== 409)
|
|
306
|
+
throw error;
|
|
307
|
+
// 409 幂等判定:通过则不重传、直接随下方流程推进 state
|
|
308
|
+
await this.resolve409(key, id, nextIndex, state, segment, plaintext, error.currentLength);
|
|
309
|
+
}
|
|
310
|
+
// meta 折叠:解压本段全部帧
|
|
311
|
+
const fold = emptyFold();
|
|
312
|
+
for (const frame of scan.frames) {
|
|
313
|
+
await foldFrame(newBytes.subarray(frame.start, frame.end), fold);
|
|
314
|
+
}
|
|
315
|
+
state.uploadedBytes += plaintext.length;
|
|
316
|
+
state.remoteBytes += payload.length;
|
|
317
|
+
state.lastSegmentIndex = nextIndex;
|
|
318
|
+
state.lastTag = encryptor.lastTag.toString('hex');
|
|
319
|
+
state.localRevision = revision;
|
|
320
|
+
state.eventCount += fold.eventCount;
|
|
321
|
+
if (fold.title !== undefined)
|
|
322
|
+
state.title = fold.title;
|
|
323
|
+
state.updatedAt = Math.max(state.updatedAt, fold.updatedAt);
|
|
324
|
+
state.status = 'ok';
|
|
325
|
+
state.error = undefined;
|
|
326
|
+
const meta = {
|
|
327
|
+
sessionId: id,
|
|
328
|
+
title: state.title ?? '',
|
|
329
|
+
createdAt: header.createdAt,
|
|
330
|
+
updatedAt: state.updatedAt || header.createdAt,
|
|
331
|
+
eventCount: state.eventCount,
|
|
332
|
+
formatVersion: header.version,
|
|
333
|
+
cwd: header.cwd ?? '',
|
|
334
|
+
device: this.options.device,
|
|
335
|
+
};
|
|
336
|
+
await client.overwrite(metaKey(this.options.device, id), encryptMeta(this.options.key, this.options.salt, id, meta));
|
|
337
|
+
await store.putSession(id, state);
|
|
338
|
+
await store.setGlobal({ lastSyncAt: this.now(), lastError: undefined });
|
|
339
|
+
this.emit({ kind: 'uploaded', sessionId: id, bytes: payload.length });
|
|
340
|
+
}
|
|
341
|
+
/**
|
|
342
|
+
* 409 幂等判定(docs/02 §10):服务端长度恰等于「state 偏移 + 待传段长」时,
|
|
343
|
+
* 下载末段做链式 AAD 校验并比对明文——通过则视为上次上传已成功(不重复上传);
|
|
344
|
+
* 否则判定写入方冲突,抛 ConflictError 停止该会话同步,不做盲目全量覆写。
|
|
345
|
+
*/
|
|
346
|
+
async resolve409(key, sessionId, segmentIndex, state, segment, plaintext, serverLength) {
|
|
347
|
+
const expected = state.remoteBytes + segment.length;
|
|
348
|
+
if (serverLength === undefined || serverLength !== expected) {
|
|
349
|
+
throw new ConflictError(`409 on ${key}: server length ${serverLength ?? '?'} != state ${state.remoteBytes} + segment ${segment.length}`);
|
|
350
|
+
}
|
|
351
|
+
const tail = await this.options.client.downloadRange(key, serverLength - segment.length, serverLength - 1);
|
|
352
|
+
const prevTag = state.lastTag ? Buffer.from(state.lastTag, 'hex') : null;
|
|
353
|
+
let decoded;
|
|
354
|
+
try {
|
|
355
|
+
decoded = verifyTailSegment(this.options.key, sessionId, segmentIndex, prevTag, tail);
|
|
356
|
+
}
|
|
357
|
+
catch {
|
|
358
|
+
throw new ConflictError(`409 on ${key}: tail segment failed chained-AAD verification`);
|
|
359
|
+
}
|
|
360
|
+
if (!decoded.equals(plaintext)) {
|
|
361
|
+
throw new ConflictError(`409 on ${key}: tail segment plaintext differs from local`);
|
|
362
|
+
}
|
|
363
|
+
// 幂等确认:上次上传其实成功了,只是 state 未推进
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
async function readTail(file, offset) {
|
|
367
|
+
const handle = await fsp.open(file, 'r');
|
|
368
|
+
try {
|
|
369
|
+
const stat = await handle.stat();
|
|
370
|
+
if (stat.size <= offset)
|
|
371
|
+
return Buffer.alloc(0);
|
|
372
|
+
const buffer = Buffer.alloc(stat.size - offset);
|
|
373
|
+
await handle.read(buffer, 0, buffer.length, offset);
|
|
374
|
+
return buffer;
|
|
375
|
+
}
|
|
376
|
+
finally {
|
|
377
|
+
await handle.close();
|
|
378
|
+
}
|
|
379
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { constants, zstdCompress, zstdDecompress } from 'node:zlib';
|
|
2
|
+
export declare const ZSTD_MAGIC = 4247762216;
|
|
3
|
+
export declare const zstdCompressAsync: typeof zstdCompress.__promisify__;
|
|
4
|
+
export declare const zstdDecompressAsync: typeof zstdDecompress.__promisify__;
|
|
5
|
+
/** 与 dsh 一致:压缩帧带 checksum(compressZstdFrame 的 CHECKSUM_OPTIONS)。 */
|
|
6
|
+
export declare const ZSTD_CHECKSUM_OPTIONS: {
|
|
7
|
+
params: {
|
|
8
|
+
[constants.ZSTD_c_checksumFlag]: number;
|
|
9
|
+
};
|
|
10
|
+
};
|
|
11
|
+
export interface ZstdFrameRange {
|
|
12
|
+
start: number;
|
|
13
|
+
end: number;
|
|
14
|
+
}
|
|
15
|
+
export interface ZstdFrameScan {
|
|
16
|
+
frames: ZstdFrameRange[];
|
|
17
|
+
tornStart?: number;
|
|
18
|
+
}
|
|
19
|
+
export declare function scanZstdFrames(buffer: Buffer, maxFrames?: number): ZstdFrameScan;
|
|
20
|
+
/** 会话文件的物理编码检测:zstd 魔数或明文(`compression: none` profile)。 */
|
|
21
|
+
export declare function detectEncoding(prefix: Buffer): 'zstd' | 'plain';
|
|
22
|
+
/** 事件折叠累积器:标题 latest-wins,updatedAt 取最大事件时间,事件按解码后计数。 */
|
|
23
|
+
export interface MetaFold {
|
|
24
|
+
title?: string;
|
|
25
|
+
updatedAt: number;
|
|
26
|
+
eventCount: number;
|
|
27
|
+
}
|
|
28
|
+
export declare function emptyFold(): MetaFold;
|
|
29
|
+
/**
|
|
30
|
+
* 把一段完整帧解压后的 JSONL 明文折叠进 meta 累积器。
|
|
31
|
+
* 会话首帧的首行是 header 行(type 'session'),跳过不计数。
|
|
32
|
+
*/
|
|
33
|
+
export declare function foldPlaintext(plaintext: Buffer, acc: MetaFold): void;
|
|
34
|
+
/** 解压一个完整帧并折叠。 */
|
|
35
|
+
export declare function foldFrame(frameBytes: Buffer, acc: MetaFold): Promise<void>;
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* zstd 帧结构扫描与事件折叠(M2 上行链路只读本地会话文件)。
|
|
3
|
+
*
|
|
4
|
+
* scanZstdFrames 复刻自 dsh(packages/session/session-persistence-jsonl/src/zstd.ts),
|
|
5
|
+
* 纯结构扫描不解压;帧边界即 dsh 的提交边界(每事件批次一个完整帧)。
|
|
6
|
+
* 事件折叠按存储行解码规则自行计数:chunk 行(text-chunks 等)一行打包多个事件,
|
|
7
|
+
* 按 data.texts/data.args 长度计数(与 decodeStorageRecord 的展开数一致)。
|
|
8
|
+
*/
|
|
9
|
+
import { promisify } from 'node:util';
|
|
10
|
+
import { constants, zstdCompress, zstdDecompress } from 'node:zlib';
|
|
11
|
+
export const ZSTD_MAGIC = 0xfd2fb528;
|
|
12
|
+
export const zstdCompressAsync = promisify(zstdCompress);
|
|
13
|
+
export const zstdDecompressAsync = promisify(zstdDecompress);
|
|
14
|
+
/** 与 dsh 一致:压缩帧带 checksum(compressZstdFrame 的 CHECKSUM_OPTIONS)。 */
|
|
15
|
+
export const ZSTD_CHECKSUM_OPTIONS = { params: { [constants.ZSTD_c_checksumFlag]: 1 } };
|
|
16
|
+
export function scanZstdFrames(buffer, maxFrames = Number.POSITIVE_INFINITY) {
|
|
17
|
+
const frames = [];
|
|
18
|
+
let offset = 0;
|
|
19
|
+
while (offset < buffer.length) {
|
|
20
|
+
const start = offset;
|
|
21
|
+
if (buffer.length - offset < 4)
|
|
22
|
+
return { frames, tornStart: start };
|
|
23
|
+
if (buffer.readUInt32LE(offset) !== ZSTD_MAGIC) {
|
|
24
|
+
throw new Error(`corrupt Zstandard session log: invalid frame magic at byte ${offset}`);
|
|
25
|
+
}
|
|
26
|
+
offset += 4;
|
|
27
|
+
if (offset === buffer.length)
|
|
28
|
+
return { frames, tornStart: start };
|
|
29
|
+
const descriptor = buffer.readUInt8(offset);
|
|
30
|
+
offset += 1;
|
|
31
|
+
if ((descriptor & 0x18) !== 0) {
|
|
32
|
+
throw new Error(`corrupt Zstandard session log: reserved frame-header bit at byte ${offset - 1}`);
|
|
33
|
+
}
|
|
34
|
+
const contentSizeFlag = descriptor >>> 6;
|
|
35
|
+
const singleSegment = (descriptor & 0x20) !== 0;
|
|
36
|
+
const checksum = (descriptor & 0x04) !== 0;
|
|
37
|
+
const dictionaryFlag = descriptor & 0x03;
|
|
38
|
+
const dictionaryBytes = dictionaryFlag === 3 ? 4 : dictionaryFlag;
|
|
39
|
+
const contentSizeBytes = contentSizeFlag === 0 ? (singleSegment ? 1 : 0) : 1 << contentSizeFlag;
|
|
40
|
+
const remainingHeaderBytes = (singleSegment ? 0 : 1) + dictionaryBytes + contentSizeBytes;
|
|
41
|
+
if (buffer.length - offset < remainingHeaderBytes)
|
|
42
|
+
return { frames, tornStart: start };
|
|
43
|
+
offset += remainingHeaderBytes;
|
|
44
|
+
for (;;) {
|
|
45
|
+
if (buffer.length - offset < 3)
|
|
46
|
+
return { frames, tornStart: start };
|
|
47
|
+
const blockHeader = buffer.readUIntLE(offset, 3);
|
|
48
|
+
offset += 3;
|
|
49
|
+
const lastBlock = (blockHeader & 1) !== 0;
|
|
50
|
+
const blockType = (blockHeader >>> 1) & 0x03;
|
|
51
|
+
const blockSize = blockHeader >>> 3;
|
|
52
|
+
if (blockType === 0x03) {
|
|
53
|
+
throw new Error(`corrupt Zstandard session log: reserved block type at byte ${offset - 3}`);
|
|
54
|
+
}
|
|
55
|
+
const payloadBytes = blockType === 0x01 ? 1 : blockSize;
|
|
56
|
+
if (buffer.length - offset < payloadBytes)
|
|
57
|
+
return { frames, tornStart: start };
|
|
58
|
+
offset += payloadBytes;
|
|
59
|
+
if (lastBlock)
|
|
60
|
+
break;
|
|
61
|
+
}
|
|
62
|
+
if (checksum) {
|
|
63
|
+
if (buffer.length - offset < 4)
|
|
64
|
+
return { frames, tornStart: start };
|
|
65
|
+
offset += 4;
|
|
66
|
+
}
|
|
67
|
+
frames.push({ start, end: offset });
|
|
68
|
+
if (frames.length === maxFrames)
|
|
69
|
+
return { frames };
|
|
70
|
+
}
|
|
71
|
+
return { frames };
|
|
72
|
+
}
|
|
73
|
+
/** 会话文件的物理编码检测:zstd 魔数或明文(`compression: none` profile)。 */
|
|
74
|
+
export function detectEncoding(prefix) {
|
|
75
|
+
return prefix.length >= 4 && prefix.readUInt32LE(0) === ZSTD_MAGIC ? 'zstd' : 'plain';
|
|
76
|
+
}
|
|
77
|
+
export function emptyFold() {
|
|
78
|
+
return { updatedAt: 0, eventCount: 0 };
|
|
79
|
+
}
|
|
80
|
+
const CHUNK_ROW_TYPES = new Set(['text-chunks', 'reasoning-chunks', 'tool-call-chunks']);
|
|
81
|
+
/**
|
|
82
|
+
* 把一段完整帧解压后的 JSONL 明文折叠进 meta 累积器。
|
|
83
|
+
* 会话首帧的首行是 header 行(type 'session'),跳过不计数。
|
|
84
|
+
*/
|
|
85
|
+
export function foldPlaintext(plaintext, acc) {
|
|
86
|
+
const text = plaintext.toString('utf8');
|
|
87
|
+
for (const line of text.split('\n')) {
|
|
88
|
+
if (line.length === 0)
|
|
89
|
+
continue;
|
|
90
|
+
const row = JSON.parse(line);
|
|
91
|
+
const type = row.type;
|
|
92
|
+
if (type === 'session')
|
|
93
|
+
continue; // header 行
|
|
94
|
+
if (typeof type !== 'string')
|
|
95
|
+
throw new Error('corrupt event row: missing type');
|
|
96
|
+
if (CHUNK_ROW_TYPES.has(type)) {
|
|
97
|
+
const data = row.data;
|
|
98
|
+
const count = (data.texts ?? data.args ?? []).length;
|
|
99
|
+
acc.eventCount += count;
|
|
100
|
+
const time0 = typeof row.time0 === 'number' ? row.time0 : undefined;
|
|
101
|
+
if (time0 !== undefined) {
|
|
102
|
+
const dt = Array.isArray(data.dt) ? data.dt.reduce((a, b) => a + b, 0) : 0;
|
|
103
|
+
acc.updatedAt = Math.max(acc.updatedAt, time0 + dt);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
else {
|
|
107
|
+
acc.eventCount += 1;
|
|
108
|
+
if (typeof row.time === 'number')
|
|
109
|
+
acc.updatedAt = Math.max(acc.updatedAt, row.time);
|
|
110
|
+
if (type === 'session/title') {
|
|
111
|
+
const data = row.data;
|
|
112
|
+
if (typeof data.title === 'string')
|
|
113
|
+
acc.title = data.title;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
/** 解压一个完整帧并折叠。 */
|
|
119
|
+
export async function foldFrame(frameBytes, acc) {
|
|
120
|
+
foldPlaintext(await zstdDecompressAsync(frameBytes), acc);
|
|
121
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dsh-cloud-sync protocol v1 客户端(docs/03-server-protocol.md §4/§7)。
|
|
3
|
+
* 仅依赖全局 fetch,无第三方依赖。
|
|
4
|
+
*/
|
|
5
|
+
export interface RemoteObject {
|
|
6
|
+
key: string;
|
|
7
|
+
size: number;
|
|
8
|
+
revision: string;
|
|
9
|
+
lastModified: string;
|
|
10
|
+
}
|
|
11
|
+
/** 带状态码的协议错误;409 额外携带服务端当前长度。 */
|
|
12
|
+
export declare class ProtocolError extends Error {
|
|
13
|
+
readonly status: number;
|
|
14
|
+
readonly currentLength?: number | undefined;
|
|
15
|
+
constructor(status: number, message: string, currentLength?: number | undefined);
|
|
16
|
+
}
|
|
17
|
+
export interface SyncClientOptions {
|
|
18
|
+
serverUrl: string;
|
|
19
|
+
token: string;
|
|
20
|
+
/** 用户名(可选):多用户服务端的配对校验,非空时随请求发送 X-DSH-User。 */
|
|
21
|
+
username?: string;
|
|
22
|
+
timeoutMs?: number;
|
|
23
|
+
}
|
|
24
|
+
export declare class SyncClient {
|
|
25
|
+
private base;
|
|
26
|
+
private token;
|
|
27
|
+
private username;
|
|
28
|
+
private timeoutMs;
|
|
29
|
+
constructor(options: SyncClientOptions);
|
|
30
|
+
private request;
|
|
31
|
+
/** 列表(可选前缀过滤),按 key 字典序。 */
|
|
32
|
+
list(prefix?: string): Promise<RemoteObject[]>;
|
|
33
|
+
/** 全量下载;key 不存在抛 ProtocolError(404)。 */
|
|
34
|
+
download(key: string): Promise<Buffer>;
|
|
35
|
+
/** Range 下载 [start, end](含端点)。 */
|
|
36
|
+
downloadRange(key: string, start: number, end: number): Promise<Buffer>;
|
|
37
|
+
/** 全量覆写(首传、meta 更新)。 */
|
|
38
|
+
overwrite(key: string, content: Buffer): Promise<void>;
|
|
39
|
+
/** 追加;offset 与服务端当前长度不一致抛 ProtocolError(409, currentLength)。 */
|
|
40
|
+
append(key: string, offset: number, content: Buffer): Promise<void>;
|
|
41
|
+
delete(key: string): Promise<void>;
|
|
42
|
+
/** 连通性测试:列表接口通且鉴权通过即成功(healthz 不鉴权,不用)。 */
|
|
43
|
+
ping(): Promise<void>;
|
|
44
|
+
}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dsh-cloud-sync protocol v1 客户端(docs/03-server-protocol.md §4/§7)。
|
|
3
|
+
* 仅依赖全局 fetch,无第三方依赖。
|
|
4
|
+
*/
|
|
5
|
+
/** 带状态码的协议错误;409 额外携带服务端当前长度。 */
|
|
6
|
+
export class ProtocolError extends Error {
|
|
7
|
+
status;
|
|
8
|
+
currentLength;
|
|
9
|
+
constructor(status, message, currentLength) {
|
|
10
|
+
super(message);
|
|
11
|
+
this.status = status;
|
|
12
|
+
this.currentLength = currentLength;
|
|
13
|
+
this.name = 'ProtocolError';
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
export class SyncClient {
|
|
17
|
+
base;
|
|
18
|
+
token;
|
|
19
|
+
username;
|
|
20
|
+
timeoutMs;
|
|
21
|
+
constructor(options) {
|
|
22
|
+
this.base = options.serverUrl.replace(/\/+$/, '');
|
|
23
|
+
this.token = options.token;
|
|
24
|
+
this.username = options.username || undefined;
|
|
25
|
+
this.timeoutMs = options.timeoutMs ?? 15000;
|
|
26
|
+
}
|
|
27
|
+
async request(method, path, body, headers = {}) {
|
|
28
|
+
const response = await fetch(`${this.base}${path}`, {
|
|
29
|
+
method,
|
|
30
|
+
headers: {
|
|
31
|
+
Authorization: `Bearer ${this.token}`,
|
|
32
|
+
...(this.username ? { 'X-DSH-User': this.username } : {}),
|
|
33
|
+
...(body ? { 'Content-Type': 'application/octet-stream' } : {}),
|
|
34
|
+
...headers,
|
|
35
|
+
},
|
|
36
|
+
body: body,
|
|
37
|
+
signal: AbortSignal.timeout(this.timeoutMs),
|
|
38
|
+
});
|
|
39
|
+
const responseBody = Buffer.from(await response.arrayBuffer());
|
|
40
|
+
if (!response.ok) {
|
|
41
|
+
let currentLength;
|
|
42
|
+
if (response.status === 409) {
|
|
43
|
+
try {
|
|
44
|
+
currentLength = JSON.parse(responseBody.toString('utf8')).currentLength;
|
|
45
|
+
}
|
|
46
|
+
catch {
|
|
47
|
+
// 哑存储规范要求 409 带 currentLength;缺失按无长度处理
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
throw new ProtocolError(response.status, `${method} ${path} → ${response.status}`, currentLength);
|
|
51
|
+
}
|
|
52
|
+
return { status: response.status, headers: response.headers, body: responseBody };
|
|
53
|
+
}
|
|
54
|
+
/** 列表(可选前缀过滤),按 key 字典序。 */
|
|
55
|
+
async list(prefix) {
|
|
56
|
+
const query = prefix ? `?prefix=${encodeURIComponent(prefix)}` : '';
|
|
57
|
+
const { body } = await this.request('GET', `/v1/sessions${query}`);
|
|
58
|
+
return JSON.parse(body.toString('utf8')).objects;
|
|
59
|
+
}
|
|
60
|
+
/** 全量下载;key 不存在抛 ProtocolError(404)。 */
|
|
61
|
+
async download(key) {
|
|
62
|
+
const { body } = await this.request('GET', `/v1/${key}`);
|
|
63
|
+
return body;
|
|
64
|
+
}
|
|
65
|
+
/** Range 下载 [start, end](含端点)。 */
|
|
66
|
+
async downloadRange(key, start, end) {
|
|
67
|
+
const { body } = await this.request('GET', `/v1/${key}`, undefined, {
|
|
68
|
+
Range: `bytes=${start}-${end}`,
|
|
69
|
+
});
|
|
70
|
+
return body;
|
|
71
|
+
}
|
|
72
|
+
/** 全量覆写(首传、meta 更新)。 */
|
|
73
|
+
async overwrite(key, content) {
|
|
74
|
+
await this.request('PUT', `/v1/${key}`, content);
|
|
75
|
+
}
|
|
76
|
+
/** 追加;offset 与服务端当前长度不一致抛 ProtocolError(409, currentLength)。 */
|
|
77
|
+
async append(key, offset, content) {
|
|
78
|
+
await this.request('PUT', `/v1/${key}`, content, { 'X-Append-Offset': String(offset) });
|
|
79
|
+
}
|
|
80
|
+
async delete(key) {
|
|
81
|
+
await this.request('DELETE', `/v1/${key}`);
|
|
82
|
+
}
|
|
83
|
+
/** 连通性测试:列表接口通且鉴权通过即成功(healthz 不鉴权,不用)。 */
|
|
84
|
+
async ping() {
|
|
85
|
+
await this.list('sessions/');
|
|
86
|
+
}
|
|
87
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { type SyncClient } from './http-client.js';
|
|
2
|
+
export interface KdfDeps {
|
|
3
|
+
client: SyncClient;
|
|
4
|
+
device: string;
|
|
5
|
+
passphrase: string;
|
|
6
|
+
/** 采纳他机 sidecar 时回调(日志/状态行提示) */
|
|
7
|
+
onAdopt?: (message: string) => void;
|
|
8
|
+
}
|
|
9
|
+
export interface KdfKey {
|
|
10
|
+
key: Buffer;
|
|
11
|
+
salt: Buffer;
|
|
12
|
+
}
|
|
13
|
+
/** kdf 引导(见模块注释的解析顺序)。 */
|
|
14
|
+
export declare function bootstrapKdf(deps: KdfDeps): Promise<KdfKey>;
|