@twinsearth/w2m-dsh-plugin 2.0.7 → 2.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +3 -3
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +22 -5
- package/TEST-CASES-JOINT-FLEET-2.0.8.md +119 -0
- package/TEST-PLAN-JOINT-FLEET-2.0.8.md +99 -0
- package/codex/mcp-servers.toml +1 -1
- package/docs/DSH-UPGRADE-2.0.8.md +129 -0
- package/docs/TRANSFER-LAYER.md +162 -0
- package/docs/acceptance/2.0.8/RELEASE-2080.json +21 -0
- package/docs/acceptance/2.0.9/RELEASE-2090.json +24 -0
- package/docs/adr/0002-p2p-transport.md +67 -0
- package/docs/plans/IM2A-2.2.2-PLAN.md +293 -0
- package/lib/client.js +30 -4
- package/package.json +5 -3
- package/plugin.json +1 -1
- package/src/agent/agent.mjs +7 -0
- package/src/agent/chacha.mjs +93 -0
- package/src/agent/crypto.mjs +64 -29
- package/src/agent/discovery.mjs +81 -0
- package/src/agent/iroh-sidecar.mjs +149 -0
- package/src/agent/p2p-node.mjs +11 -1
- package/src/agent/p2p-race.mjs +105 -0
- package/src/agent/p2p.mjs +105 -18
- package/src/agent/path-scheduler.mjs +106 -0
- package/src/agent/transfer/artifact-ref-v3.mjs +93 -0
- package/src/agent/transfer/ferry.mjs +206 -0
- package/src/agent/transfer/index.mjs +4 -0
- package/src/agent/transfer/manifest.mjs +92 -0
- package/src/agent/transfer/object-store.mjs +144 -0
- package/src/agent/transfer/publish.mjs +76 -0
- package/src/agent/transfer/reference.mjs +97 -0
- package/src/agent/transfer/router.mjs +80 -0
- package/src/agent/transfer/want-block.mjs +135 -0
- package/src/agent/transport.mjs +146 -0
- package/src/game/xiangqi.mjs +217 -0
- package/src/plugin/fleet-binding.mjs +105 -0
- package/src/plugin/tools.mjs +34 -2
package/src/agent/p2p.mjs
CHANGED
|
@@ -114,6 +114,42 @@ export const DEFAULT_WINDOW = 32;
|
|
|
114
114
|
export const DEFAULT_RTO_MS = 300;
|
|
115
115
|
export const MAX_RTO_MS = 3000;
|
|
116
116
|
|
|
117
|
+
/**
|
|
118
|
+
* RFC 6298 RTT estimator. The old channel used a constant 300 ms RTO, which is
|
|
119
|
+
* wrong on both a LAN (too slow) and a lossy WAN (too fast). This keeps SRTT and
|
|
120
|
+
* RTTVAR and clamps the result; callers still apply per-attempt exponential
|
|
121
|
+
* backoff on top.
|
|
122
|
+
*/
|
|
123
|
+
export class RttEstimator {
|
|
124
|
+
constructor({ initialRtoMs = DEFAULT_RTO_MS, minRtoMs = 100, maxRtoMs = MAX_RTO_MS } = {}) {
|
|
125
|
+
this.srtt = null;
|
|
126
|
+
this.rttvar = null;
|
|
127
|
+
this.minRtoMs = minRtoMs;
|
|
128
|
+
this.maxRtoMs = maxRtoMs;
|
|
129
|
+
this.rtoMs = Math.min(maxRtoMs, Math.max(minRtoMs, initialRtoMs));
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
sample(rttMs) {
|
|
133
|
+
if (!Number.isFinite(rttMs) || rttMs < 0) return this.rtoMs;
|
|
134
|
+
if (this.srtt === null) {
|
|
135
|
+
this.srtt = rttMs;
|
|
136
|
+
this.rttvar = rttMs / 2;
|
|
137
|
+
} else {
|
|
138
|
+
this.rttvar = 0.75 * this.rttvar + 0.25 * Math.abs(this.srtt - rttMs);
|
|
139
|
+
this.srtt = 0.875 * this.srtt + 0.125 * rttMs;
|
|
140
|
+
}
|
|
141
|
+
this.rtoMs = Math.min(this.maxRtoMs, Math.max(this.minRtoMs, this.srtt + 4 * this.rttvar));
|
|
142
|
+
return this.rtoMs;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
reset() {
|
|
146
|
+
this.srtt = null;
|
|
147
|
+
this.rttvar = null;
|
|
148
|
+
this.rtoMs = Math.min(this.maxRtoMs, Math.max(this.minRtoMs, DEFAULT_RTO_MS));
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
|
|
117
153
|
/** Retransmits per fragment before the message is declared undeliverable. */
|
|
118
154
|
export const DEFAULT_MAX_ATTEMPTS = 8;
|
|
119
155
|
|
|
@@ -985,9 +1021,14 @@ export class P2PChannel extends EventEmitter {
|
|
|
985
1021
|
this.session = options.session;
|
|
986
1022
|
this.tuning = {
|
|
987
1023
|
window: options.tuning?.window ?? DEFAULT_WINDOW,
|
|
1024
|
+
maxWindow: options.tuning?.maxWindow ?? 256,
|
|
988
1025
|
rtoMs: options.tuning?.rtoMs ?? DEFAULT_RTO_MS,
|
|
989
1026
|
maxAttempts: options.tuning?.maxAttempts ?? DEFAULT_MAX_ATTEMPTS,
|
|
990
1027
|
};
|
|
1028
|
+
/** Adaptive send window: starts at the tuning value, grows on completed messages. */
|
|
1029
|
+
this.window = this.tuning.window;
|
|
1030
|
+
/** RTT/RTO measured from ACK arrival, instead of the fixed 300 ms default. */
|
|
1031
|
+
this.rtt = new RttEstimator({ initialRtoMs: this.tuning.rtoMs, maxRtoMs: MAX_RTO_MS });
|
|
991
1032
|
|
|
992
1033
|
const crypto = options.crypto ?? null;
|
|
993
1034
|
/** @type {CryptoSession|null} */
|
|
@@ -1150,11 +1191,25 @@ export class P2PChannel extends EventEmitter {
|
|
|
1150
1191
|
const entry = this.outgoing.get(messageId);
|
|
1151
1192
|
if (!entry) return;
|
|
1152
1193
|
entry.acked.add(index);
|
|
1194
|
+
entry.inFlight.delete(index);
|
|
1195
|
+
// Karn: a fragment that was retransmitted has an ambiguous RTT, so only the
|
|
1196
|
+
// first transmission is sampled.
|
|
1197
|
+
if (entry.sentAt.has(index) && !entry.retransmitted.has(index)) {
|
|
1198
|
+
this.rtt.sample(Date.now() - entry.sentAt.get(index));
|
|
1199
|
+
entry.sentAt.delete(index);
|
|
1200
|
+
}
|
|
1153
1201
|
if (entry.acked.size === entry.fragments.length) {
|
|
1154
1202
|
this.outgoing.delete(messageId);
|
|
1155
1203
|
clearTimeout(entry.timer);
|
|
1204
|
+
// A completed message is the only honest RTT-free signal that the path holds more
|
|
1205
|
+
// than the previous window: grow conservatively, and halve on a retransmit timeout.
|
|
1206
|
+
this.window = Math.min(this.tuning.maxWindow, this.window + Math.max(1, Math.floor(this.window / 8)));
|
|
1156
1207
|
entry.resolve();
|
|
1208
|
+
return;
|
|
1157
1209
|
}
|
|
1210
|
+
// ACK-driven progress: the next window leaves as soon as a slot frees, instead of
|
|
1211
|
+
// waiting for the RTO timer (the 0.12 MiB/s ceiling).
|
|
1212
|
+
if (entry.pump !== null) entry.pump();
|
|
1158
1213
|
}
|
|
1159
1214
|
|
|
1160
1215
|
/**
|
|
@@ -1207,6 +1262,10 @@ export class P2PChannel extends EventEmitter {
|
|
|
1207
1262
|
const entry = {
|
|
1208
1263
|
fragments,
|
|
1209
1264
|
acked: new Set(),
|
|
1265
|
+
inFlight: new Set(),
|
|
1266
|
+
sentAt: new Map(),
|
|
1267
|
+
retransmitted: new Set(),
|
|
1268
|
+
pump: null,
|
|
1210
1269
|
resolve,
|
|
1211
1270
|
reject,
|
|
1212
1271
|
attempts: new Map(),
|
|
@@ -1215,45 +1274,73 @@ export class P2PChannel extends EventEmitter {
|
|
|
1215
1274
|
|
|
1216
1275
|
const pump = () => {
|
|
1217
1276
|
if (this.closed) return;
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
for (const index of pending) {
|
|
1227
|
-
const attempt = (entry.attempts.get(index) ?? 0) + 1;
|
|
1228
|
-
entry.attempts.set(index, attempt);
|
|
1277
|
+
// Send only what the current window has room for. inFlight is the honest
|
|
1278
|
+
// count of fragments the peer has not acknowledged yet; re-sending those
|
|
1279
|
+
// is the timer's job, not pump's.
|
|
1280
|
+
let budget = Math.max(1, this.window - entry.inFlight.size);
|
|
1281
|
+
for (let i = 0; i < fragments.length && budget > 0; i += 1) {
|
|
1282
|
+
if (entry.acked.has(i) || entry.inFlight.has(i)) continue;
|
|
1283
|
+
const attempt = (entry.attempts.get(i) ?? 0) + 1;
|
|
1284
|
+
entry.attempts.set(i, attempt);
|
|
1229
1285
|
if (attempt > this.tuning.maxAttempts) {
|
|
1230
1286
|
this.outgoing.delete(messageId);
|
|
1287
|
+
clearTimeout(entry.timer);
|
|
1288
|
+
this.window = Math.max(8, Math.floor(this.window / 2));
|
|
1231
1289
|
reject(
|
|
1232
1290
|
new Error(
|
|
1233
|
-
`P2P_FRAGMENT_UNACKED: fragment ${
|
|
1291
|
+
`P2P_FRAGMENT_UNACKED: fragment ${i}/${fragments.length} of message ${messageId} ` +
|
|
1234
1292
|
`after ${this.tuning.maxAttempts} attempts`,
|
|
1235
1293
|
),
|
|
1236
1294
|
);
|
|
1237
1295
|
return;
|
|
1238
1296
|
}
|
|
1239
|
-
|
|
1297
|
+
entry.inFlight.add(i);
|
|
1298
|
+
if (!entry.sentAt.has(i)) entry.sentAt.set(i, Date.now());
|
|
1299
|
+
budget -= 1;
|
|
1300
|
+
const body = Buffer.alloc(12 + fragments[i].length);
|
|
1240
1301
|
body.writeUInt32BE(messageId, 0);
|
|
1241
|
-
body.writeUInt32BE(
|
|
1302
|
+
body.writeUInt32BE(i, 4);
|
|
1242
1303
|
body.writeUInt32BE(fragments.length, 8);
|
|
1243
|
-
fragments[
|
|
1304
|
+
fragments[i].copy(body, 12);
|
|
1244
1305
|
this.sendFrame(KIND.DATA, body);
|
|
1245
1306
|
}
|
|
1246
1307
|
|
|
1247
|
-
|
|
1248
|
-
const
|
|
1308
|
+
if (entry.inFlight.size === 0) return;
|
|
1309
|
+
const worst = Math.max(...[...entry.inFlight].map((i) => entry.attempts.get(i) ?? 1));
|
|
1310
|
+
const delay = Math.min(this.rtt.rtoMs * 2 ** (worst - 1), MAX_RTO_MS);
|
|
1249
1311
|
// One timer per message, not one per channel: a shared field means a second
|
|
1250
1312
|
// concurrent send silently cancels the first one's retransmission schedule.
|
|
1251
1313
|
clearTimeout(entry.timer);
|
|
1252
1314
|
entry.timer = setTimeout(() => {
|
|
1253
|
-
if (this.outgoing.has(messageId))
|
|
1315
|
+
if (!this.outgoing.has(messageId)) return;
|
|
1316
|
+
// Retransmit the fragments that are still unacknowledged, then schedule again.
|
|
1317
|
+
for (const i of entry.inFlight) {
|
|
1318
|
+
entry.retransmitted.add(i);
|
|
1319
|
+
const attempt = (entry.attempts.get(i) ?? 0) + 1;
|
|
1320
|
+
entry.attempts.set(i, attempt);
|
|
1321
|
+
if (attempt > this.tuning.maxAttempts) {
|
|
1322
|
+
this.outgoing.delete(messageId);
|
|
1323
|
+
this.window = Math.max(8, Math.floor(this.window / 2));
|
|
1324
|
+
reject(
|
|
1325
|
+
new Error(
|
|
1326
|
+
`P2P_FRAGMENT_UNACKED: fragment ${i}/${fragments.length} of message ${messageId} ` +
|
|
1327
|
+
`after ${this.tuning.maxAttempts} attempts`,
|
|
1328
|
+
),
|
|
1329
|
+
);
|
|
1330
|
+
return;
|
|
1331
|
+
}
|
|
1332
|
+
const body = Buffer.alloc(12 + fragments[i].length);
|
|
1333
|
+
body.writeUInt32BE(messageId, 0);
|
|
1334
|
+
body.writeUInt32BE(i, 4);
|
|
1335
|
+
body.writeUInt32BE(fragments.length, 8);
|
|
1336
|
+
fragments[i].copy(body, 12);
|
|
1337
|
+
this.sendFrame(KIND.DATA, body);
|
|
1338
|
+
}
|
|
1339
|
+
pump();
|
|
1254
1340
|
}, delay);
|
|
1255
1341
|
entry.timer.unref?.();
|
|
1256
1342
|
};
|
|
1343
|
+
entry.pump = pump;
|
|
1257
1344
|
|
|
1258
1345
|
pump();
|
|
1259
1346
|
});
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* PathScheduler: score candidate paths, and switch only on sustained evidence.
|
|
3
|
+
*
|
|
4
|
+
* This is deliberately pure policy: no sockets, no timers beyond the overlap
|
|
5
|
+
* deadline it reports. The caller opens the challenger, keeps the old path for
|
|
6
|
+
* `keepUntilMs` (make-before-break), then closes it.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
export const PATH_KIND = Object.freeze({
|
|
10
|
+
IROH_DIRECT: 'iroh-direct',
|
|
11
|
+
UDP_DIRECT: 'udp-direct',
|
|
12
|
+
LIBP2P: 'libp2p',
|
|
13
|
+
IROH_RELAY: 'iroh-relay',
|
|
14
|
+
CONTROL_PLANE: 'control-plane',
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
/** Higher is better. The type weight dominates; latency and loss refine it. */
|
|
18
|
+
export const PATH_WEIGHT = Object.freeze({
|
|
19
|
+
[PATH_KIND.IROH_DIRECT]: 5,
|
|
20
|
+
[PATH_KIND.UDP_DIRECT]: 4,
|
|
21
|
+
[PATH_KIND.LIBP2P]: 3,
|
|
22
|
+
[PATH_KIND.IROH_RELAY]: 2,
|
|
23
|
+
[PATH_KIND.CONTROL_PLANE]: 1,
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
export const SCHEDULER_CODE = Object.freeze({
|
|
27
|
+
OK: 'PATH_OK',
|
|
28
|
+
NOT_ENCRYPTED: 'PATH_NOT_ENCRYPTED',
|
|
29
|
+
NO_CANDIDATES: 'PATH_NO_CANDIDATES',
|
|
30
|
+
SWITCH_PENDING: 'PATH_SWITCH_PENDING',
|
|
31
|
+
ALL_FAILED: 'ALL_TRANSPORTS_FAILED',
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* @param {{kind: string, encrypted: boolean, firstFrameRttMs: number, lossRate: number}} path
|
|
36
|
+
* @returns {number} score; encrypted paths outrank plaintext by a wide margin
|
|
37
|
+
*/
|
|
38
|
+
export function scorePath({ kind, encrypted, firstFrameRttMs = 0, lossRate = 0 }) {
|
|
39
|
+
const weight = PATH_WEIGHT[kind] ?? 0;
|
|
40
|
+
if (encrypted !== true) return -1;
|
|
41
|
+
if (!Number.isFinite(weight) || weight <= 0) return -1;
|
|
42
|
+
const rtt = Number.isFinite(firstFrameRttMs) ? Math.max(0, firstFrameRttMs) : 1000;
|
|
43
|
+
const loss = Number.isFinite(lossRate) ? Math.min(1, Math.max(0, lossRate)) : 1;
|
|
44
|
+
return weight * 1000 - rtt - loss * 2000;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* A path switch is allowed only when:
|
|
49
|
+
* - the challenger is encrypted;
|
|
50
|
+
* - two consecutive samples beat the current path by > 20%;
|
|
51
|
+
* - the challenger's kind is at least as preferred as the current one
|
|
52
|
+
* (no switching down to a relay while a direct path is healthy).
|
|
53
|
+
*/
|
|
54
|
+
export class PathScheduler {
|
|
55
|
+
constructor({ overlapMs = 5000, improvement = 1.2, requiredSamples = 2, now = Date.now } = {}) {
|
|
56
|
+
this.overlapMs = overlapMs;
|
|
57
|
+
this.improvement = improvement;
|
|
58
|
+
this.requiredSamples = requiredSamples;
|
|
59
|
+
this.now = now;
|
|
60
|
+
this.current = null;
|
|
61
|
+
this.challenger = null;
|
|
62
|
+
this.leading = 0;
|
|
63
|
+
this.lastPlan = null;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
setCurrent(path) {
|
|
67
|
+
this.current = path;
|
|
68
|
+
this.challenger = null;
|
|
69
|
+
this.leading = 0;
|
|
70
|
+
return this.current;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** Record one challenger sample; returns a switch plan or null. */
|
|
74
|
+
observe(path) {
|
|
75
|
+
if (path?.encrypted !== true) return null;
|
|
76
|
+
const challengerScore = scorePath(path);
|
|
77
|
+
const currentScore = this.current === null ? Number.NEGATIVE_INFINITY : scorePath(this.current);
|
|
78
|
+
if (currentScore === Number.NEGATIVE_INFINITY) {
|
|
79
|
+
this.current = path;
|
|
80
|
+
return { switch: false, reason: 'first-path', path };
|
|
81
|
+
}
|
|
82
|
+
const sameChallenger = this.challenger !== null && this.challenger.id === path.id;
|
|
83
|
+
if (challengerScore > currentScore * this.improvement) {
|
|
84
|
+
this.challenger = path;
|
|
85
|
+
this.leading = sameChallenger ? this.leading + 1 : 1;
|
|
86
|
+
if (this.leading < this.requiredSamples) return null;
|
|
87
|
+
if ((PATH_WEIGHT[path.kind] ?? 0) < (PATH_WEIGHT[this.current.kind] ?? 0)) return null;
|
|
88
|
+
const plan = { switch: true, from: this.current, to: path, keepUntilMs: this.now() + this.overlapMs, reason: 'better-path' };
|
|
89
|
+
this.lastPlan = plan;
|
|
90
|
+
this.current = path;
|
|
91
|
+
this.challenger = null;
|
|
92
|
+
this.leading = 0;
|
|
93
|
+
return plan;
|
|
94
|
+
}
|
|
95
|
+
this.challenger = path;
|
|
96
|
+
this.leading = 0;
|
|
97
|
+
return null;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** Close the loser only after the overlap window has elapsed. */
|
|
101
|
+
breakOld(plan, atMs = this.now()) {
|
|
102
|
+
if (plan?.switch !== true) return { ok: false, code: SCHEDULER_CODE.NOT_ENCRYPTED };
|
|
103
|
+
if (atMs < plan.keepUntilMs) return { ok: false, code: SCHEDULER_CODE.SWITCH_PENDING, closeAtMs: plan.keepUntilMs };
|
|
104
|
+
return { ok: true, close: plan.from };
|
|
105
|
+
}
|
|
106
|
+
}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Optional ArtifactRef v3 (feature-flagged; v2 remains the default on the wire).
|
|
3
|
+
*
|
|
4
|
+
* v3 adds an explicit hash algorithm, a Merkle root and a SHA-256 root so a
|
|
5
|
+
* reader can cross-check the two aggregations. BLAKE3 is not bundled and Node
|
|
6
|
+
* has no native implementation: when it is unavailable, creation falls back to
|
|
7
|
+
* SHA-256 and records that decision instead of pretending BLAKE3 was used.
|
|
8
|
+
*/
|
|
9
|
+
import { createHash } from 'node:crypto';
|
|
10
|
+
|
|
11
|
+
export const REF_V3_VERSION = 3;
|
|
12
|
+
|
|
13
|
+
export const REF_V3_CODE = Object.freeze({
|
|
14
|
+
OK: 'REF_V3_OK',
|
|
15
|
+
BAD_SHAPE: 'REF_V3_BAD_SHAPE',
|
|
16
|
+
UNSUPPORTED_ALG: 'REF_V3_UNSUPPORTED_ALG',
|
|
17
|
+
BLAKE3_UNAVAILABLE: 'XFER_BLAKE3_UNAVAILABLE',
|
|
18
|
+
INCONSISTENT: 'REF_V3_INCONSISTENT',
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
const sha256 = (buffer) => createHash('sha256').update(buffer).digest();
|
|
22
|
+
const hex64 = (value) => {
|
|
23
|
+
if (typeof value !== 'string') return null;
|
|
24
|
+
const raw = value.startsWith('sha256:') ? value.slice('sha256:'.length) : value;
|
|
25
|
+
return /^[0-9a-f]{64}$/.test(raw) ? raw : null;
|
|
26
|
+
};
|
|
27
|
+
const isHex64 = (value) => hex64(value) !== null;
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Merkle root over chunk digests: leaves are the existing 32-byte chunk hashes;
|
|
31
|
+
* internal nodes are SHA-256(0x01 || left || right); the final root is
|
|
32
|
+
* SHA-256(0x02 || root). An odd node is promoted, never duplicated.
|
|
33
|
+
*/
|
|
34
|
+
export function merkleRootSha256(chunkHashesHex) {
|
|
35
|
+
if (!Array.isArray(chunkHashesHex) || chunkHashesHex.length === 0) return null;
|
|
36
|
+
let level = chunkHashesHex.map((hex) => {
|
|
37
|
+
if (!isHex64(hex)) throw new TypeError('chunk hash must be 64 lowercase hex characters');
|
|
38
|
+
return Buffer.from(hex, 'hex');
|
|
39
|
+
});
|
|
40
|
+
while (level.length > 1) {
|
|
41
|
+
const next = [];
|
|
42
|
+
for (let i = 0; i < level.length; i += 2) {
|
|
43
|
+
if (i + 1 >= level.length) { next.push(level[i]); continue; }
|
|
44
|
+
next.push(sha256(Buffer.concat([Buffer.from([0x01]), level[i], level[i + 1]])));
|
|
45
|
+
}
|
|
46
|
+
level = next;
|
|
47
|
+
}
|
|
48
|
+
return sha256(Buffer.concat([Buffer.from([0x02]), level[0]])).toString('hex');
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* @param {{rootCid: string, sha256Root: string, merkleRoot: string, hashAlg?: string,
|
|
53
|
+
* chunkBytes: number, chunkCount: number, size: number, originPeer?: string|null,
|
|
54
|
+
* blake3Available?: boolean}} input
|
|
55
|
+
* @returns {{ok: true, ref: object} | {ok: false, code: string, detail?: string}}
|
|
56
|
+
*/
|
|
57
|
+
export function createRefV3(input) {
|
|
58
|
+
const requestedAlg = input?.hashAlg === 'blake3' ? 'blake3' : 'sha256';
|
|
59
|
+
const effectiveAlg = requestedAlg === 'blake3' && input?.blake3Available !== true ? 'sha256' : requestedAlg;
|
|
60
|
+
const ref = {
|
|
61
|
+
version: REF_V3_VERSION,
|
|
62
|
+
rootCid: hex64(input?.rootCid) ?? input?.rootCid,
|
|
63
|
+
sha256Root: hex64(input?.sha256Root) ?? input?.sha256Root,
|
|
64
|
+
merkleRoot: input?.merkleRoot,
|
|
65
|
+
hashAlg: effectiveAlg,
|
|
66
|
+
...(effectiveAlg !== requestedAlg ? { fallbackFrom: requestedAlg } : {}),
|
|
67
|
+
chunkBytes: input?.chunkBytes,
|
|
68
|
+
chunkCount: input?.chunkCount,
|
|
69
|
+
size: input?.size,
|
|
70
|
+
originPeer: typeof input?.originPeer === 'string' ? input.originPeer : null,
|
|
71
|
+
};
|
|
72
|
+
return verifyRefV3(ref, { blake3Available: input?.blake3Available === true });
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Validate v3 shape and invariants. A BLAKE3 v3 ref cannot be verified on a
|
|
77
|
+
* runtime without BLAKE3: that is named, not silently downgraded.
|
|
78
|
+
*/
|
|
79
|
+
export function verifyRefV3(ref, { blake3Available = false } = {}) {
|
|
80
|
+
if (ref === null || typeof ref !== 'object') return { ok: false, code: REF_V3_CODE.BAD_SHAPE, detail: 'ref must be an object' };
|
|
81
|
+
if (ref.version !== REF_V3_VERSION) return { ok: false, code: REF_V3_CODE.BAD_SHAPE, detail: 'version must be 3' };
|
|
82
|
+
if (!isHex64(ref.rootCid) || !isHex64(ref.sha256Root) || !isHex64(ref.merkleRoot)) {
|
|
83
|
+
return { ok: false, code: REF_V3_CODE.BAD_SHAPE, detail: 'rootCid, sha256Root and merkleRoot must be 64-hex' };
|
|
84
|
+
}
|
|
85
|
+
if (ref.hashAlg !== 'sha256' && ref.hashAlg !== 'blake3') return { ok: false, code: REF_V3_CODE.UNSUPPORTED_ALG, detail: String(ref.hashAlg) };
|
|
86
|
+
if (ref.hashAlg === 'blake3' && blake3Available !== true) return { ok: false, code: REF_V3_CODE.BLAKE3_UNAVAILABLE };
|
|
87
|
+
for (const name of ['chunkBytes', 'chunkCount', 'size']) {
|
|
88
|
+
if (!Number.isInteger(ref[name]) || ref[name] < (name === 'size' ? 0 : 1)) return { ok: false, code: REF_V3_CODE.BAD_SHAPE, detail: name };
|
|
89
|
+
}
|
|
90
|
+
if (ref.chunkCount !== Math.ceil(ref.size / ref.chunkBytes)) return { ok: false, code: REF_V3_CODE.INCONSISTENT, detail: 'chunkCount != ceil(size/chunkBytes)' };
|
|
91
|
+
if (ref.hashAlg === 'sha256' && hex64(ref.rootCid) !== hex64(ref.sha256Root)) return { ok: false, code: REF_V3_CODE.INCONSISTENT, detail: 'sha256 rootCid must equal sha256Root' };
|
|
92
|
+
return { ok: true, code: REF_V3_CODE.OK, ref, effectiveAlg: ref.hashAlg };
|
|
93
|
+
}
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Artifact Ferry v1(本地最小闭环):chunkFile → ArtifactRef → sink 接收端。
|
|
3
|
+
*
|
|
4
|
+
* 不变式:
|
|
5
|
+
* · 控制面只持引用(ref/rootCid/size/chunkCount),字节永远在本机或 sink;
|
|
6
|
+
* · 发送端流式分块,内存与文件大小无关;每块 SHA-256;
|
|
7
|
+
* · 接收端逐块校验、位图持久化、可断点续传;finalize 重算根才原子落盘;
|
|
8
|
+
* · 路径只允许 sink 根下的相对路径(拒绝绝对路径/.. /盘符/UNC)。
|
|
9
|
+
* 本阶段不做远端 P2P 拉取:`pull` 明确返回 P2P_UNAVAILABLE,绝不回落 relay blob。
|
|
10
|
+
*/
|
|
11
|
+
import { createHash } from 'node:crypto';
|
|
12
|
+
import {
|
|
13
|
+
closeSync, existsSync, fsyncSync, ftruncateSync, mkdirSync, openSync, readdirSync, readFileSync,
|
|
14
|
+
readSync, renameSync, statSync, writeFileSync, writeSync,
|
|
15
|
+
} from 'node:fs';
|
|
16
|
+
import { basename, isAbsolute, join, relative, resolve, sep } from 'node:path';
|
|
17
|
+
|
|
18
|
+
import { createManifest, chunkLengths, chunkRange } from './manifest.mjs';
|
|
19
|
+
|
|
20
|
+
const sha256 = (buf) => createHash('sha256').update(buf).digest('hex');
|
|
21
|
+
const ROOT = /^sha256:[0-9a-f]{64}$/;
|
|
22
|
+
const readJson = (file) => JSON.parse(readFileSync(file, 'utf8').replace(/^\uFEFF/, ''));
|
|
23
|
+
|
|
24
|
+
export function safeRelative(input) {
|
|
25
|
+
if (typeof input !== 'string' || input.trim() === '') return { ok: false, code: 'PATH_REQUIRED' };
|
|
26
|
+
const raw = input.replace(/\\/g, '/');
|
|
27
|
+
if (raw.includes('\0')) return { ok: false, code: 'PATH_NUL' };
|
|
28
|
+
if (isAbsolute(input) || /^[A-Za-z]:/.test(raw) || raw.startsWith('//')) return { ok: false, code: 'PATH_ABSOLUTE' };
|
|
29
|
+
const parts = raw.split('/').filter((part) => part !== '' && part !== '.');
|
|
30
|
+
if (parts.length === 0 || parts.some((part) => part === '..')) return { ok: false, code: 'PATH_TRAVERSAL' };
|
|
31
|
+
return { ok: true, relative: parts.join('/') };
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** 流式分块:只读一块、算一块的哈希;返回 rootCid 与块清单。 */
|
|
35
|
+
export function chunkFile(filePath, { chunkBytes = 1024 * 1024 } = {}) {
|
|
36
|
+
const stat = statSync(filePath);
|
|
37
|
+
const plan = chunkLengths(stat.size, chunkBytes);
|
|
38
|
+
if (plan.ok !== true) return plan;
|
|
39
|
+
const fd = openSync(filePath, 'r');
|
|
40
|
+
const chunks = [];
|
|
41
|
+
try {
|
|
42
|
+
for (let index = 0; index < plan.count; index += 1) {
|
|
43
|
+
const length = index === plan.count - 1 ? plan.lastChunkBytes : plan.chunkBytes;
|
|
44
|
+
const buf = Buffer.alloc(length);
|
|
45
|
+
const read = readSync(fd, buf, 0, length, index * plan.chunkBytes);
|
|
46
|
+
if (read !== length) return { ok: false, code: 'XFER_READ_SHORT' };
|
|
47
|
+
const digest = sha256(buf);
|
|
48
|
+
chunks.push({ index, cid: 'sha256:' + digest, sha256: digest, size: length });
|
|
49
|
+
}
|
|
50
|
+
} finally {
|
|
51
|
+
closeSync(fd);
|
|
52
|
+
}
|
|
53
|
+
const built = createManifest({
|
|
54
|
+
sizeBytes: stat.size,
|
|
55
|
+
chunkBytes: plan.chunkBytes,
|
|
56
|
+
chunkHashes: chunks.map((chunk) => chunk.sha256),
|
|
57
|
+
objectId: sha256(`${stat.size}:${plan.chunkBytes}:${chunks.map((c) => c.sha256).join('')}`).slice(0, 32),
|
|
58
|
+
kind: 'file',
|
|
59
|
+
sinkKind: 'cache',
|
|
60
|
+
relPath: basename(filePath),
|
|
61
|
+
});
|
|
62
|
+
if (built.ok !== true) return built;
|
|
63
|
+
return {
|
|
64
|
+
ok: true,
|
|
65
|
+
rootCid: 'sha256:' + built.manifest.root_sha256,
|
|
66
|
+
rootHex: built.manifest.root_sha256,
|
|
67
|
+
size: stat.size,
|
|
68
|
+
chunkBytes: plan.chunkBytes,
|
|
69
|
+
chunkCount: plan.count,
|
|
70
|
+
chunks,
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** 控制面引用:只含元数据,不含字节、块清单或路径。 */
|
|
75
|
+
export function createRefV2({ rootCid, size, chunkCount, chunkBytes, originPeer, sinkHint = 'workspace://artifacts', mediaType = 'application/octet-stream', refId = null }) {
|
|
76
|
+
if (typeof rootCid !== 'string' || !ROOT.test(rootCid)) return { ok: false, code: 'ARTIFACT_ROOT_INVALID' };
|
|
77
|
+
if (!Number.isSafeInteger(size) || size < 0) return { ok: false, code: 'ARTIFACT_SIZE_INVALID' };
|
|
78
|
+
if (!Number.isSafeInteger(chunkCount) || chunkCount < 0) return { ok: false, code: 'ARTIFACT_CHUNKS_INVALID' };
|
|
79
|
+
if (!Number.isSafeInteger(chunkBytes) || chunkBytes <= 0) return { ok: false, code: 'ARTIFACT_CHUNK_SIZE_INVALID' };
|
|
80
|
+
if (typeof originPeer !== 'string' || originPeer === '') return { ok: false, code: 'ARTIFACT_ORIGIN_REQUIRED' };
|
|
81
|
+
return { ok: true, ref: { v: 2, ref: refId ?? rootCid, rootCid, size, chunkCount, chunkBytes, mediaType, originPeer, sinkHint } };
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
export function verifyRefV2(ref) {
|
|
85
|
+
if (ref === null || typeof ref !== 'object' || ref.v !== 2) return { ok: false, code: 'ARTIFACT_REF_INVALID' };
|
|
86
|
+
if (typeof ref.ref !== 'string' || ref.ref === '') return { ok: false, code: 'ARTIFACT_REF_INVALID' };
|
|
87
|
+
if (typeof ref.rootCid !== 'string' || !ROOT.test(ref.rootCid)) return { ok: false, code: 'ARTIFACT_ROOT_INVALID' };
|
|
88
|
+
if (!Number.isSafeInteger(ref.size) || ref.size < 0 || !Number.isSafeInteger(ref.chunkCount) || ref.chunkCount < 0) return { ok: false, code: 'ARTIFACT_REF_INVALID' };
|
|
89
|
+
const plan = chunkLengths(ref.size, ref.chunkBytes);
|
|
90
|
+
if (plan.ok !== true) return plan;
|
|
91
|
+
if (plan.count !== ref.chunkCount) return { ok: false, code: 'ARTIFACT_REF_INCONSISTENT', detail: { expectedChunks: plan.count, chunkCount: ref.chunkCount } };
|
|
92
|
+
if (typeof ref.originPeer !== 'string' || ref.originPeer === '') return { ok: false, code: 'ARTIFACT_ORIGIN_REQUIRED' };
|
|
93
|
+
return { ok: true };
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** 本地引用索引(控制面的本机替身;relay 发布是下一阶段接线)。 */
|
|
97
|
+
export function createRefIndex({ dir }) {
|
|
98
|
+
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
99
|
+
const fileOf = (refId) => join(dir, sha256(String(refId)) + '.json');
|
|
100
|
+
return {
|
|
101
|
+
publish(ref) {
|
|
102
|
+
const checked = verifyRefV2(ref);
|
|
103
|
+
if (checked.ok !== true) return checked;
|
|
104
|
+
const file = fileOf(ref.ref);
|
|
105
|
+
writeFileSync(file, JSON.stringify(ref), { encoding: 'utf8', mode: 0o600 });
|
|
106
|
+
return { ok: true, ref, path: file };
|
|
107
|
+
},
|
|
108
|
+
get(refId) {
|
|
109
|
+
try {
|
|
110
|
+
const ref = readJson(fileOf(refId));
|
|
111
|
+
const checked = verifyRefV2(ref);
|
|
112
|
+
return checked.ok === true ? { ok: true, ref } : checked;
|
|
113
|
+
} catch { return { ok: false, code: 'ARTIFACT_REF_NOT_FOUND' }; }
|
|
114
|
+
},
|
|
115
|
+
list() {
|
|
116
|
+
return { ok: true, refs: readdirSync(dir).filter((n) => n.endsWith('.json')).map((n) => { try { return readJson(join(dir, n)); } catch { return null; } }).filter(Boolean) };
|
|
117
|
+
},
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* 接收端:位图持久化 + 逐块校验 + finalize 根校验后原子落盘。
|
|
123
|
+
* @param {{stateDir:string, sinkRoot:string, ref:object, destination:string, chunkBytes?:number}} input
|
|
124
|
+
*/
|
|
125
|
+
export function createReceiver({ stateDir, sinkRoot, ref, destination, chunkBytes = null }) {
|
|
126
|
+
const checked = verifyRefV2(ref);
|
|
127
|
+
if (checked.ok !== true) return checked;
|
|
128
|
+
const safe = safeRelative(destination);
|
|
129
|
+
if (safe.ok !== true) return safe;
|
|
130
|
+
const root = resolve(sinkRoot);
|
|
131
|
+
const target = resolve(root, safe.relative);
|
|
132
|
+
const rel = relative(root, target);
|
|
133
|
+
if (rel === '' || rel.startsWith('..') || isAbsolute(rel) || rel.split(sep).includes('..')) return { ok: false, code: 'PATH_TRAVERSAL' };
|
|
134
|
+
const transferId = sha256(ref.ref + '|' + ref.rootCid + '|' + ref.size).slice(0, 32);
|
|
135
|
+
const dir = join(stateDir, transferId);
|
|
136
|
+
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
137
|
+
const bitmapPath = join(dir, 'bitmap.json');
|
|
138
|
+
const partPath = join(dir, 'data.part');
|
|
139
|
+
const chunk = Number.isInteger(chunkBytes) ? chunkBytes : ref.chunkBytes;
|
|
140
|
+
let bitmap = null;
|
|
141
|
+
if (existsSync(bitmapPath)) {
|
|
142
|
+
try { bitmap = readJson(bitmapPath); } catch { bitmap = null; }
|
|
143
|
+
if (bitmap !== null && (bitmap.rootCid !== ref.rootCid || bitmap.size !== ref.size || bitmap.chunkBytes !== chunk)) return { ok: false, code: 'XFER_SESSION_CONFLICT' };
|
|
144
|
+
}
|
|
145
|
+
if (bitmap === null) {
|
|
146
|
+
const fd = openSync(partPath, 'w+', 0o600);
|
|
147
|
+
try { ftruncateSync(fd, ref.size); } finally { closeSync(fd); }
|
|
148
|
+
bitmap = { v: 1, transferId, rootCid: ref.rootCid, size: ref.size, chunkBytes: chunk, chunkCount: ref.chunkCount, chunks: ref.chunkCount, chunk_bytes: chunk, last_chunk_bytes: ref.chunkCount === 0 ? 0 : ref.size - (ref.chunkCount - 1) * chunk, hashes: Array(ref.chunkCount).fill(null), destination: safe.relative, updatedAt: new Date().toISOString() };
|
|
149
|
+
writeFileSync(bitmapPath, JSON.stringify(bitmap), { encoding: 'utf8', mode: 0o600 });
|
|
150
|
+
}
|
|
151
|
+
const save = () => { bitmap.updatedAt = new Date().toISOString(); const tmp = bitmapPath + '.tmp-' + process.pid; writeFileSync(tmp, JSON.stringify(bitmap), { encoding: 'utf8', mode: 0o600 }); renameSync(tmp, bitmapPath); };
|
|
152
|
+
let finalized = false;
|
|
153
|
+
return {
|
|
154
|
+
ok: true,
|
|
155
|
+
transferId,
|
|
156
|
+
have() {
|
|
157
|
+
return new Set(bitmap.hashes.map((h, i) => (h === null ? null : i)).filter((i) => i !== null));
|
|
158
|
+
},
|
|
159
|
+
progress() {
|
|
160
|
+
const received = bitmap.hashes.map((h, i) => (h === null ? null : i)).filter((i) => i !== null);
|
|
161
|
+
return { total: bitmap.chunkCount, received: received.length, complete: received.length === bitmap.chunkCount, bytes: bitmap.hashes.reduce((sum, h, i) => sum + (h === null ? 0 : i === bitmap.chunkCount - 1 ? bitmap.size - i * chunk : chunk), 0) };
|
|
162
|
+
},
|
|
163
|
+
writeChunk(index, data, expectedSha256) {
|
|
164
|
+
if (finalized) return { ok: false, code: 'XFER_ALREADY_FINALIZED' };
|
|
165
|
+
const range = chunkRange(bitmap, index);
|
|
166
|
+
if (range === null) return { ok: false, code: 'XFER_CHUNK_INDEX_INVALID' };
|
|
167
|
+
const buf = Buffer.isBuffer(data) ? data : Buffer.from(data ?? '');
|
|
168
|
+
if (buf.length !== range.length) return { ok: false, code: 'XFER_CHUNK_LENGTH_INVALID', detail: { expected: range.length, got: buf.length } };
|
|
169
|
+
const digest = sha256(buf);
|
|
170
|
+
if (typeof expectedSha256 !== 'string' || expectedSha256 !== digest) return { ok: false, code: 'XFER_CHUNK_HASH_MISMATCH', retry: index };
|
|
171
|
+
if (bitmap.hashes[index] !== null && bitmap.hashes[index] !== digest) return { ok: false, code: 'XFER_CHUNK_CONFLICT', recoverable: true, retry: index };
|
|
172
|
+
if (bitmap.hashes[index] === digest) return { ok: true, duplicate: true, index };
|
|
173
|
+
const fd = openSync(partPath, 'r+');
|
|
174
|
+
try { writeSync(fd, buf, 0, buf.length, range.offset); } finally { closeSync(fd); }
|
|
175
|
+
bitmap.hashes[index] = digest;
|
|
176
|
+
save();
|
|
177
|
+
return { ok: true, index, sha256: digest, writtenBytes: buf.length };
|
|
178
|
+
},
|
|
179
|
+
finalizeTransfer(rootCid = ref.rootCid) {
|
|
180
|
+
if (rootCid !== ref.rootCid) return { ok: false, code: 'XFER_ROOT_MISMATCH' };
|
|
181
|
+
if (bitmap.hashes.some((h) => h === null)) return { ok: false, code: 'XFER_CHUNKS_MISSING', detail: { missing: bitmap.hashes.filter((h) => h === null).length } };
|
|
182
|
+
const built = createManifest({ sizeBytes: bitmap.size, chunkBytes: bitmap.chunkBytes, chunkHashes: bitmap.hashes, objectId: transferId, kind: 'file', sinkKind: 'cache', relPath: safe.relative });
|
|
183
|
+
if (built.ok !== true) return built;
|
|
184
|
+
if ('sha256:' + built.manifest.root_sha256 !== rootCid) return { ok: false, code: 'XFER_ROOT_MISMATCH' };
|
|
185
|
+
mkdirSync(resolve(target, '..'), { recursive: true, mode: 0o700 });
|
|
186
|
+
const fd = openSync(partPath, 'r+');
|
|
187
|
+
try { fsyncSync(fd); } finally { closeSync(fd); }
|
|
188
|
+
renameSync(partPath, target);
|
|
189
|
+
finalized = true;
|
|
190
|
+
try { writeFileSync(join(dir, 'done.json'), JSON.stringify({ transferId, rootCid, size: bitmap.size, path: target, at: new Date().toISOString() }), { encoding: 'utf8', mode: 0o600 }); } catch { /* receipt is best effort */ }
|
|
191
|
+
return { ok: true, path: target, verified: true, bytes: bitmap.size, transferId };
|
|
192
|
+
},
|
|
193
|
+
/** Explicit recovery for a conflicting source: drop the claimed block so a correct source can win. */
|
|
194
|
+
evictChunk(index) {
|
|
195
|
+
if (!Number.isInteger(index) || index < 0 || index >= bitmap.chunkCount) return { ok: false, code: 'XFER_CHUNK_INDEX_INVALID' };
|
|
196
|
+
if (bitmap.hashes[index] === null) return { ok: true, index, alreadyMissing: true };
|
|
197
|
+
bitmap.hashes[index] = null;
|
|
198
|
+
save();
|
|
199
|
+
return { ok: true, index, evicted: true };
|
|
200
|
+
},
|
|
201
|
+
/** 远端 P2P 拉取尚未接线:显式失败,不回落 relay。 */
|
|
202
|
+
async pullFrom(originPeer) {
|
|
203
|
+
return { ok: false, code: 'P2P_UNAVAILABLE', detail: { originPeer, why: 'want-block transport is a later slice; relay fallback is forbidden' } };
|
|
204
|
+
},
|
|
205
|
+
};
|
|
206
|
+
}
|