@twinsearth/w2m-dsh-plugin 2.0.8 → 2.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +3 -3
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +8 -0
- package/TEST-CASES-JOINT-FLEET-2.0.8.md +119 -0
- package/TEST-PLAN-JOINT-FLEET-2.0.8.md +99 -0
- package/codex/mcp-servers.toml +1 -1
- package/docs/DSH-UPGRADE-2.0.8.md +129 -0
- package/docs/TRANSFER-LAYER.md +162 -0
- package/docs/acceptance/2.0.9/RELEASE-2090.json +24 -0
- package/docs/adr/0002-p2p-transport.md +67 -0
- package/lib/client.js +29 -3
- package/package.json +5 -3
- package/plugin.json +1 -1
- package/src/agent/agent.mjs +7 -0
- package/src/agent/discovery.mjs +81 -0
- package/src/agent/iroh-sidecar.mjs +149 -0
- package/src/agent/p2p-node.mjs +11 -1
- package/src/agent/p2p-race.mjs +105 -0
- package/src/agent/p2p.mjs +105 -18
- package/src/agent/path-scheduler.mjs +106 -0
- package/src/agent/transfer/artifact-ref-v3.mjs +93 -0
- package/src/agent/transfer/ferry.mjs +12 -1
- package/src/agent/transfer/router.mjs +80 -0
- package/src/agent/transfer/want-block.mjs +46 -0
- package/src/agent/transport.mjs +146 -0
- package/src/plugin/tools.mjs +35 -25
package/src/agent/p2p.mjs
CHANGED
|
@@ -114,6 +114,42 @@ export const DEFAULT_WINDOW = 32;
|
|
|
114
114
|
export const DEFAULT_RTO_MS = 300;
|
|
115
115
|
export const MAX_RTO_MS = 3000;
|
|
116
116
|
|
|
117
|
+
/**
|
|
118
|
+
* RFC 6298 RTT estimator. The old channel used a constant 300 ms RTO, which is
|
|
119
|
+
* wrong on both a LAN (too slow) and a lossy WAN (too fast). This keeps SRTT and
|
|
120
|
+
* RTTVAR and clamps the result; callers still apply per-attempt exponential
|
|
121
|
+
* backoff on top.
|
|
122
|
+
*/
|
|
123
|
+
export class RttEstimator {
|
|
124
|
+
constructor({ initialRtoMs = DEFAULT_RTO_MS, minRtoMs = 100, maxRtoMs = MAX_RTO_MS } = {}) {
|
|
125
|
+
this.srtt = null;
|
|
126
|
+
this.rttvar = null;
|
|
127
|
+
this.minRtoMs = minRtoMs;
|
|
128
|
+
this.maxRtoMs = maxRtoMs;
|
|
129
|
+
this.rtoMs = Math.min(maxRtoMs, Math.max(minRtoMs, initialRtoMs));
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
sample(rttMs) {
|
|
133
|
+
if (!Number.isFinite(rttMs) || rttMs < 0) return this.rtoMs;
|
|
134
|
+
if (this.srtt === null) {
|
|
135
|
+
this.srtt = rttMs;
|
|
136
|
+
this.rttvar = rttMs / 2;
|
|
137
|
+
} else {
|
|
138
|
+
this.rttvar = 0.75 * this.rttvar + 0.25 * Math.abs(this.srtt - rttMs);
|
|
139
|
+
this.srtt = 0.875 * this.srtt + 0.125 * rttMs;
|
|
140
|
+
}
|
|
141
|
+
this.rtoMs = Math.min(this.maxRtoMs, Math.max(this.minRtoMs, this.srtt + 4 * this.rttvar));
|
|
142
|
+
return this.rtoMs;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
reset() {
|
|
146
|
+
this.srtt = null;
|
|
147
|
+
this.rttvar = null;
|
|
148
|
+
this.rtoMs = Math.min(this.maxRtoMs, Math.max(this.minRtoMs, DEFAULT_RTO_MS));
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
|
|
117
153
|
/** Retransmits per fragment before the message is declared undeliverable. */
|
|
118
154
|
export const DEFAULT_MAX_ATTEMPTS = 8;
|
|
119
155
|
|
|
@@ -985,9 +1021,14 @@ export class P2PChannel extends EventEmitter {
|
|
|
985
1021
|
this.session = options.session;
|
|
986
1022
|
this.tuning = {
|
|
987
1023
|
window: options.tuning?.window ?? DEFAULT_WINDOW,
|
|
1024
|
+
maxWindow: options.tuning?.maxWindow ?? 256,
|
|
988
1025
|
rtoMs: options.tuning?.rtoMs ?? DEFAULT_RTO_MS,
|
|
989
1026
|
maxAttempts: options.tuning?.maxAttempts ?? DEFAULT_MAX_ATTEMPTS,
|
|
990
1027
|
};
|
|
1028
|
+
/** Adaptive send window: starts at the tuning value, grows on completed messages. */
|
|
1029
|
+
this.window = this.tuning.window;
|
|
1030
|
+
/** RTT/RTO measured from ACK arrival, instead of the fixed 300 ms default. */
|
|
1031
|
+
this.rtt = new RttEstimator({ initialRtoMs: this.tuning.rtoMs, maxRtoMs: MAX_RTO_MS });
|
|
991
1032
|
|
|
992
1033
|
const crypto = options.crypto ?? null;
|
|
993
1034
|
/** @type {CryptoSession|null} */
|
|
@@ -1150,11 +1191,25 @@ export class P2PChannel extends EventEmitter {
|
|
|
1150
1191
|
const entry = this.outgoing.get(messageId);
|
|
1151
1192
|
if (!entry) return;
|
|
1152
1193
|
entry.acked.add(index);
|
|
1194
|
+
entry.inFlight.delete(index);
|
|
1195
|
+
// Karn: a fragment that was retransmitted has an ambiguous RTT, so only the
|
|
1196
|
+
// first transmission is sampled.
|
|
1197
|
+
if (entry.sentAt.has(index) && !entry.retransmitted.has(index)) {
|
|
1198
|
+
this.rtt.sample(Date.now() - entry.sentAt.get(index));
|
|
1199
|
+
entry.sentAt.delete(index);
|
|
1200
|
+
}
|
|
1153
1201
|
if (entry.acked.size === entry.fragments.length) {
|
|
1154
1202
|
this.outgoing.delete(messageId);
|
|
1155
1203
|
clearTimeout(entry.timer);
|
|
1204
|
+
// A completed message is the only honest RTT-free signal that the path holds more
|
|
1205
|
+
// than the previous window: grow conservatively, and halve on a retransmit timeout.
|
|
1206
|
+
this.window = Math.min(this.tuning.maxWindow, this.window + Math.max(1, Math.floor(this.window / 8)));
|
|
1156
1207
|
entry.resolve();
|
|
1208
|
+
return;
|
|
1157
1209
|
}
|
|
1210
|
+
// ACK-driven progress: the next window leaves as soon as a slot frees, instead of
|
|
1211
|
+
// waiting for the RTO timer (the 0.12 MiB/s ceiling).
|
|
1212
|
+
if (entry.pump !== null) entry.pump();
|
|
1158
1213
|
}
|
|
1159
1214
|
|
|
1160
1215
|
/**
|
|
@@ -1207,6 +1262,10 @@ export class P2PChannel extends EventEmitter {
|
|
|
1207
1262
|
const entry = {
|
|
1208
1263
|
fragments,
|
|
1209
1264
|
acked: new Set(),
|
|
1265
|
+
inFlight: new Set(),
|
|
1266
|
+
sentAt: new Map(),
|
|
1267
|
+
retransmitted: new Set(),
|
|
1268
|
+
pump: null,
|
|
1210
1269
|
resolve,
|
|
1211
1270
|
reject,
|
|
1212
1271
|
attempts: new Map(),
|
|
@@ -1215,45 +1274,73 @@ export class P2PChannel extends EventEmitter {
|
|
|
1215
1274
|
|
|
1216
1275
|
const pump = () => {
|
|
1217
1276
|
if (this.closed) return;
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
for (const index of pending) {
|
|
1227
|
-
const attempt = (entry.attempts.get(index) ?? 0) + 1;
|
|
1228
|
-
entry.attempts.set(index, attempt);
|
|
1277
|
+
// Send only what the current window has room for. inFlight is the honest
|
|
1278
|
+
// count of fragments the peer has not acknowledged yet; re-sending those
|
|
1279
|
+
// is the timer's job, not pump's.
|
|
1280
|
+
let budget = Math.max(1, this.window - entry.inFlight.size);
|
|
1281
|
+
for (let i = 0; i < fragments.length && budget > 0; i += 1) {
|
|
1282
|
+
if (entry.acked.has(i) || entry.inFlight.has(i)) continue;
|
|
1283
|
+
const attempt = (entry.attempts.get(i) ?? 0) + 1;
|
|
1284
|
+
entry.attempts.set(i, attempt);
|
|
1229
1285
|
if (attempt > this.tuning.maxAttempts) {
|
|
1230
1286
|
this.outgoing.delete(messageId);
|
|
1287
|
+
clearTimeout(entry.timer);
|
|
1288
|
+
this.window = Math.max(8, Math.floor(this.window / 2));
|
|
1231
1289
|
reject(
|
|
1232
1290
|
new Error(
|
|
1233
|
-
`P2P_FRAGMENT_UNACKED: fragment ${
|
|
1291
|
+
`P2P_FRAGMENT_UNACKED: fragment ${i}/${fragments.length} of message ${messageId} ` +
|
|
1234
1292
|
`after ${this.tuning.maxAttempts} attempts`,
|
|
1235
1293
|
),
|
|
1236
1294
|
);
|
|
1237
1295
|
return;
|
|
1238
1296
|
}
|
|
1239
|
-
|
|
1297
|
+
entry.inFlight.add(i);
|
|
1298
|
+
if (!entry.sentAt.has(i)) entry.sentAt.set(i, Date.now());
|
|
1299
|
+
budget -= 1;
|
|
1300
|
+
const body = Buffer.alloc(12 + fragments[i].length);
|
|
1240
1301
|
body.writeUInt32BE(messageId, 0);
|
|
1241
|
-
body.writeUInt32BE(
|
|
1302
|
+
body.writeUInt32BE(i, 4);
|
|
1242
1303
|
body.writeUInt32BE(fragments.length, 8);
|
|
1243
|
-
fragments[
|
|
1304
|
+
fragments[i].copy(body, 12);
|
|
1244
1305
|
this.sendFrame(KIND.DATA, body);
|
|
1245
1306
|
}
|
|
1246
1307
|
|
|
1247
|
-
|
|
1248
|
-
const
|
|
1308
|
+
if (entry.inFlight.size === 0) return;
|
|
1309
|
+
const worst = Math.max(...[...entry.inFlight].map((i) => entry.attempts.get(i) ?? 1));
|
|
1310
|
+
const delay = Math.min(this.rtt.rtoMs * 2 ** (worst - 1), MAX_RTO_MS);
|
|
1249
1311
|
// One timer per message, not one per channel: a shared field means a second
|
|
1250
1312
|
// concurrent send silently cancels the first one's retransmission schedule.
|
|
1251
1313
|
clearTimeout(entry.timer);
|
|
1252
1314
|
entry.timer = setTimeout(() => {
|
|
1253
|
-
if (this.outgoing.has(messageId))
|
|
1315
|
+
if (!this.outgoing.has(messageId)) return;
|
|
1316
|
+
// Retransmit the fragments that are still unacknowledged, then schedule again.
|
|
1317
|
+
for (const i of entry.inFlight) {
|
|
1318
|
+
entry.retransmitted.add(i);
|
|
1319
|
+
const attempt = (entry.attempts.get(i) ?? 0) + 1;
|
|
1320
|
+
entry.attempts.set(i, attempt);
|
|
1321
|
+
if (attempt > this.tuning.maxAttempts) {
|
|
1322
|
+
this.outgoing.delete(messageId);
|
|
1323
|
+
this.window = Math.max(8, Math.floor(this.window / 2));
|
|
1324
|
+
reject(
|
|
1325
|
+
new Error(
|
|
1326
|
+
`P2P_FRAGMENT_UNACKED: fragment ${i}/${fragments.length} of message ${messageId} ` +
|
|
1327
|
+
`after ${this.tuning.maxAttempts} attempts`,
|
|
1328
|
+
),
|
|
1329
|
+
);
|
|
1330
|
+
return;
|
|
1331
|
+
}
|
|
1332
|
+
const body = Buffer.alloc(12 + fragments[i].length);
|
|
1333
|
+
body.writeUInt32BE(messageId, 0);
|
|
1334
|
+
body.writeUInt32BE(i, 4);
|
|
1335
|
+
body.writeUInt32BE(fragments.length, 8);
|
|
1336
|
+
fragments[i].copy(body, 12);
|
|
1337
|
+
this.sendFrame(KIND.DATA, body);
|
|
1338
|
+
}
|
|
1339
|
+
pump();
|
|
1254
1340
|
}, delay);
|
|
1255
1341
|
entry.timer.unref?.();
|
|
1256
1342
|
};
|
|
1343
|
+
entry.pump = pump;
|
|
1257
1344
|
|
|
1258
1345
|
pump();
|
|
1259
1346
|
});
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* PathScheduler: score candidate paths, and switch only on sustained evidence.
|
|
3
|
+
*
|
|
4
|
+
* This is deliberately pure policy: no sockets, no timers beyond the overlap
|
|
5
|
+
* deadline it reports. The caller opens the challenger, keeps the old path for
|
|
6
|
+
* `keepUntilMs` (make-before-break), then closes it.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
export const PATH_KIND = Object.freeze({
|
|
10
|
+
IROH_DIRECT: 'iroh-direct',
|
|
11
|
+
UDP_DIRECT: 'udp-direct',
|
|
12
|
+
LIBP2P: 'libp2p',
|
|
13
|
+
IROH_RELAY: 'iroh-relay',
|
|
14
|
+
CONTROL_PLANE: 'control-plane',
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
/** Higher is better. The type weight dominates; latency and loss refine it. */
|
|
18
|
+
export const PATH_WEIGHT = Object.freeze({
|
|
19
|
+
[PATH_KIND.IROH_DIRECT]: 5,
|
|
20
|
+
[PATH_KIND.UDP_DIRECT]: 4,
|
|
21
|
+
[PATH_KIND.LIBP2P]: 3,
|
|
22
|
+
[PATH_KIND.IROH_RELAY]: 2,
|
|
23
|
+
[PATH_KIND.CONTROL_PLANE]: 1,
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
export const SCHEDULER_CODE = Object.freeze({
|
|
27
|
+
OK: 'PATH_OK',
|
|
28
|
+
NOT_ENCRYPTED: 'PATH_NOT_ENCRYPTED',
|
|
29
|
+
NO_CANDIDATES: 'PATH_NO_CANDIDATES',
|
|
30
|
+
SWITCH_PENDING: 'PATH_SWITCH_PENDING',
|
|
31
|
+
ALL_FAILED: 'ALL_TRANSPORTS_FAILED',
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* @param {{kind: string, encrypted: boolean, firstFrameRttMs: number, lossRate: number}} path
|
|
36
|
+
* @returns {number} score; encrypted paths outrank plaintext by a wide margin
|
|
37
|
+
*/
|
|
38
|
+
export function scorePath({ kind, encrypted, firstFrameRttMs = 0, lossRate = 0 }) {
|
|
39
|
+
const weight = PATH_WEIGHT[kind] ?? 0;
|
|
40
|
+
if (encrypted !== true) return -1;
|
|
41
|
+
if (!Number.isFinite(weight) || weight <= 0) return -1;
|
|
42
|
+
const rtt = Number.isFinite(firstFrameRttMs) ? Math.max(0, firstFrameRttMs) : 1000;
|
|
43
|
+
const loss = Number.isFinite(lossRate) ? Math.min(1, Math.max(0, lossRate)) : 1;
|
|
44
|
+
return weight * 1000 - rtt - loss * 2000;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* A path switch is allowed only when:
|
|
49
|
+
* - the challenger is encrypted;
|
|
50
|
+
* - two consecutive samples beat the current path by > 20%;
|
|
51
|
+
* - the challenger's kind is at least as preferred as the current one
|
|
52
|
+
* (no switching down to a relay while a direct path is healthy).
|
|
53
|
+
*/
|
|
54
|
+
export class PathScheduler {
|
|
55
|
+
constructor({ overlapMs = 5000, improvement = 1.2, requiredSamples = 2, now = Date.now } = {}) {
|
|
56
|
+
this.overlapMs = overlapMs;
|
|
57
|
+
this.improvement = improvement;
|
|
58
|
+
this.requiredSamples = requiredSamples;
|
|
59
|
+
this.now = now;
|
|
60
|
+
this.current = null;
|
|
61
|
+
this.challenger = null;
|
|
62
|
+
this.leading = 0;
|
|
63
|
+
this.lastPlan = null;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
setCurrent(path) {
|
|
67
|
+
this.current = path;
|
|
68
|
+
this.challenger = null;
|
|
69
|
+
this.leading = 0;
|
|
70
|
+
return this.current;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** Record one challenger sample; returns a switch plan or null. */
|
|
74
|
+
observe(path) {
|
|
75
|
+
if (path?.encrypted !== true) return null;
|
|
76
|
+
const challengerScore = scorePath(path);
|
|
77
|
+
const currentScore = this.current === null ? Number.NEGATIVE_INFINITY : scorePath(this.current);
|
|
78
|
+
if (currentScore === Number.NEGATIVE_INFINITY) {
|
|
79
|
+
this.current = path;
|
|
80
|
+
return { switch: false, reason: 'first-path', path };
|
|
81
|
+
}
|
|
82
|
+
const sameChallenger = this.challenger !== null && this.challenger.id === path.id;
|
|
83
|
+
if (challengerScore > currentScore * this.improvement) {
|
|
84
|
+
this.challenger = path;
|
|
85
|
+
this.leading = sameChallenger ? this.leading + 1 : 1;
|
|
86
|
+
if (this.leading < this.requiredSamples) return null;
|
|
87
|
+
if ((PATH_WEIGHT[path.kind] ?? 0) < (PATH_WEIGHT[this.current.kind] ?? 0)) return null;
|
|
88
|
+
const plan = { switch: true, from: this.current, to: path, keepUntilMs: this.now() + this.overlapMs, reason: 'better-path' };
|
|
89
|
+
this.lastPlan = plan;
|
|
90
|
+
this.current = path;
|
|
91
|
+
this.challenger = null;
|
|
92
|
+
this.leading = 0;
|
|
93
|
+
return plan;
|
|
94
|
+
}
|
|
95
|
+
this.challenger = path;
|
|
96
|
+
this.leading = 0;
|
|
97
|
+
return null;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** Close the loser only after the overlap window has elapsed. */
|
|
101
|
+
breakOld(plan, atMs = this.now()) {
|
|
102
|
+
if (plan?.switch !== true) return { ok: false, code: SCHEDULER_CODE.NOT_ENCRYPTED };
|
|
103
|
+
if (atMs < plan.keepUntilMs) return { ok: false, code: SCHEDULER_CODE.SWITCH_PENDING, closeAtMs: plan.keepUntilMs };
|
|
104
|
+
return { ok: true, close: plan.from };
|
|
105
|
+
}
|
|
106
|
+
}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Optional ArtifactRef v3 (feature-flagged; v2 remains the default on the wire).
|
|
3
|
+
*
|
|
4
|
+
* v3 adds an explicit hash algorithm, a Merkle root and a SHA-256 root so a
|
|
5
|
+
* reader can cross-check the two aggregations. BLAKE3 is not bundled and Node
|
|
6
|
+
* has no native implementation: when it is unavailable, creation falls back to
|
|
7
|
+
* SHA-256 and records that decision instead of pretending BLAKE3 was used.
|
|
8
|
+
*/
|
|
9
|
+
import { createHash } from 'node:crypto';
|
|
10
|
+
|
|
11
|
+
export const REF_V3_VERSION = 3;
|
|
12
|
+
|
|
13
|
+
export const REF_V3_CODE = Object.freeze({
|
|
14
|
+
OK: 'REF_V3_OK',
|
|
15
|
+
BAD_SHAPE: 'REF_V3_BAD_SHAPE',
|
|
16
|
+
UNSUPPORTED_ALG: 'REF_V3_UNSUPPORTED_ALG',
|
|
17
|
+
BLAKE3_UNAVAILABLE: 'XFER_BLAKE3_UNAVAILABLE',
|
|
18
|
+
INCONSISTENT: 'REF_V3_INCONSISTENT',
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
const sha256 = (buffer) => createHash('sha256').update(buffer).digest();
|
|
22
|
+
const hex64 = (value) => {
|
|
23
|
+
if (typeof value !== 'string') return null;
|
|
24
|
+
const raw = value.startsWith('sha256:') ? value.slice('sha256:'.length) : value;
|
|
25
|
+
return /^[0-9a-f]{64}$/.test(raw) ? raw : null;
|
|
26
|
+
};
|
|
27
|
+
const isHex64 = (value) => hex64(value) !== null;
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Merkle root over chunk digests: leaves are the existing 32-byte chunk hashes;
|
|
31
|
+
* internal nodes are SHA-256(0x01 || left || right); the final root is
|
|
32
|
+
* SHA-256(0x02 || root). An odd node is promoted, never duplicated.
|
|
33
|
+
*/
|
|
34
|
+
export function merkleRootSha256(chunkHashesHex) {
|
|
35
|
+
if (!Array.isArray(chunkHashesHex) || chunkHashesHex.length === 0) return null;
|
|
36
|
+
let level = chunkHashesHex.map((hex) => {
|
|
37
|
+
if (!isHex64(hex)) throw new TypeError('chunk hash must be 64 lowercase hex characters');
|
|
38
|
+
return Buffer.from(hex, 'hex');
|
|
39
|
+
});
|
|
40
|
+
while (level.length > 1) {
|
|
41
|
+
const next = [];
|
|
42
|
+
for (let i = 0; i < level.length; i += 2) {
|
|
43
|
+
if (i + 1 >= level.length) { next.push(level[i]); continue; }
|
|
44
|
+
next.push(sha256(Buffer.concat([Buffer.from([0x01]), level[i], level[i + 1]])));
|
|
45
|
+
}
|
|
46
|
+
level = next;
|
|
47
|
+
}
|
|
48
|
+
return sha256(Buffer.concat([Buffer.from([0x02]), level[0]])).toString('hex');
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* @param {{rootCid: string, sha256Root: string, merkleRoot: string, hashAlg?: string,
|
|
53
|
+
* chunkBytes: number, chunkCount: number, size: number, originPeer?: string|null,
|
|
54
|
+
* blake3Available?: boolean}} input
|
|
55
|
+
* @returns {{ok: true, ref: object} | {ok: false, code: string, detail?: string}}
|
|
56
|
+
*/
|
|
57
|
+
export function createRefV3(input) {
|
|
58
|
+
const requestedAlg = input?.hashAlg === 'blake3' ? 'blake3' : 'sha256';
|
|
59
|
+
const effectiveAlg = requestedAlg === 'blake3' && input?.blake3Available !== true ? 'sha256' : requestedAlg;
|
|
60
|
+
const ref = {
|
|
61
|
+
version: REF_V3_VERSION,
|
|
62
|
+
rootCid: hex64(input?.rootCid) ?? input?.rootCid,
|
|
63
|
+
sha256Root: hex64(input?.sha256Root) ?? input?.sha256Root,
|
|
64
|
+
merkleRoot: input?.merkleRoot,
|
|
65
|
+
hashAlg: effectiveAlg,
|
|
66
|
+
...(effectiveAlg !== requestedAlg ? { fallbackFrom: requestedAlg } : {}),
|
|
67
|
+
chunkBytes: input?.chunkBytes,
|
|
68
|
+
chunkCount: input?.chunkCount,
|
|
69
|
+
size: input?.size,
|
|
70
|
+
originPeer: typeof input?.originPeer === 'string' ? input.originPeer : null,
|
|
71
|
+
};
|
|
72
|
+
return verifyRefV3(ref, { blake3Available: input?.blake3Available === true });
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Validate v3 shape and invariants. A BLAKE3 v3 ref cannot be verified on a
|
|
77
|
+
* runtime without BLAKE3: that is named, not silently downgraded.
|
|
78
|
+
*/
|
|
79
|
+
export function verifyRefV3(ref, { blake3Available = false } = {}) {
|
|
80
|
+
if (ref === null || typeof ref !== 'object') return { ok: false, code: REF_V3_CODE.BAD_SHAPE, detail: 'ref must be an object' };
|
|
81
|
+
if (ref.version !== REF_V3_VERSION) return { ok: false, code: REF_V3_CODE.BAD_SHAPE, detail: 'version must be 3' };
|
|
82
|
+
if (!isHex64(ref.rootCid) || !isHex64(ref.sha256Root) || !isHex64(ref.merkleRoot)) {
|
|
83
|
+
return { ok: false, code: REF_V3_CODE.BAD_SHAPE, detail: 'rootCid, sha256Root and merkleRoot must be 64-hex' };
|
|
84
|
+
}
|
|
85
|
+
if (ref.hashAlg !== 'sha256' && ref.hashAlg !== 'blake3') return { ok: false, code: REF_V3_CODE.UNSUPPORTED_ALG, detail: String(ref.hashAlg) };
|
|
86
|
+
if (ref.hashAlg === 'blake3' && blake3Available !== true) return { ok: false, code: REF_V3_CODE.BLAKE3_UNAVAILABLE };
|
|
87
|
+
for (const name of ['chunkBytes', 'chunkCount', 'size']) {
|
|
88
|
+
if (!Number.isInteger(ref[name]) || ref[name] < (name === 'size' ? 0 : 1)) return { ok: false, code: REF_V3_CODE.BAD_SHAPE, detail: name };
|
|
89
|
+
}
|
|
90
|
+
if (ref.chunkCount !== Math.ceil(ref.size / ref.chunkBytes)) return { ok: false, code: REF_V3_CODE.INCONSISTENT, detail: 'chunkCount != ceil(size/chunkBytes)' };
|
|
91
|
+
if (ref.hashAlg === 'sha256' && hex64(ref.rootCid) !== hex64(ref.sha256Root)) return { ok: false, code: REF_V3_CODE.INCONSISTENT, detail: 'sha256 rootCid must equal sha256Root' };
|
|
92
|
+
return { ok: true, code: REF_V3_CODE.OK, ref, effectiveAlg: ref.hashAlg };
|
|
93
|
+
}
|
|
@@ -86,6 +86,9 @@ export function verifyRefV2(ref) {
|
|
|
86
86
|
if (typeof ref.ref !== 'string' || ref.ref === '') return { ok: false, code: 'ARTIFACT_REF_INVALID' };
|
|
87
87
|
if (typeof ref.rootCid !== 'string' || !ROOT.test(ref.rootCid)) return { ok: false, code: 'ARTIFACT_ROOT_INVALID' };
|
|
88
88
|
if (!Number.isSafeInteger(ref.size) || ref.size < 0 || !Number.isSafeInteger(ref.chunkCount) || ref.chunkCount < 0) return { ok: false, code: 'ARTIFACT_REF_INVALID' };
|
|
89
|
+
const plan = chunkLengths(ref.size, ref.chunkBytes);
|
|
90
|
+
if (plan.ok !== true) return plan;
|
|
91
|
+
if (plan.count !== ref.chunkCount) return { ok: false, code: 'ARTIFACT_REF_INCONSISTENT', detail: { expectedChunks: plan.count, chunkCount: ref.chunkCount } };
|
|
89
92
|
if (typeof ref.originPeer !== 'string' || ref.originPeer === '') return { ok: false, code: 'ARTIFACT_ORIGIN_REQUIRED' };
|
|
90
93
|
return { ok: true };
|
|
91
94
|
}
|
|
@@ -165,7 +168,7 @@ export function createReceiver({ stateDir, sinkRoot, ref, destination, chunkByte
|
|
|
165
168
|
if (buf.length !== range.length) return { ok: false, code: 'XFER_CHUNK_LENGTH_INVALID', detail: { expected: range.length, got: buf.length } };
|
|
166
169
|
const digest = sha256(buf);
|
|
167
170
|
if (typeof expectedSha256 !== 'string' || expectedSha256 !== digest) return { ok: false, code: 'XFER_CHUNK_HASH_MISMATCH', retry: index };
|
|
168
|
-
if (bitmap.hashes[index] !== null && bitmap.hashes[index] !== digest) return { ok: false, code: 'XFER_CHUNK_CONFLICT' };
|
|
171
|
+
if (bitmap.hashes[index] !== null && bitmap.hashes[index] !== digest) return { ok: false, code: 'XFER_CHUNK_CONFLICT', recoverable: true, retry: index };
|
|
169
172
|
if (bitmap.hashes[index] === digest) return { ok: true, duplicate: true, index };
|
|
170
173
|
const fd = openSync(partPath, 'r+');
|
|
171
174
|
try { writeSync(fd, buf, 0, buf.length, range.offset); } finally { closeSync(fd); }
|
|
@@ -187,6 +190,14 @@ export function createReceiver({ stateDir, sinkRoot, ref, destination, chunkByte
|
|
|
187
190
|
try { writeFileSync(join(dir, 'done.json'), JSON.stringify({ transferId, rootCid, size: bitmap.size, path: target, at: new Date().toISOString() }), { encoding: 'utf8', mode: 0o600 }); } catch { /* receipt is best effort */ }
|
|
188
191
|
return { ok: true, path: target, verified: true, bytes: bitmap.size, transferId };
|
|
189
192
|
},
|
|
193
|
+
/** Explicit recovery for a conflicting source: drop the claimed block so a correct source can win. */
|
|
194
|
+
evictChunk(index) {
|
|
195
|
+
if (!Number.isInteger(index) || index < 0 || index >= bitmap.chunkCount) return { ok: false, code: 'XFER_CHUNK_INDEX_INVALID' };
|
|
196
|
+
if (bitmap.hashes[index] === null) return { ok: true, index, alreadyMissing: true };
|
|
197
|
+
bitmap.hashes[index] = null;
|
|
198
|
+
save();
|
|
199
|
+
return { ok: true, index, evicted: true };
|
|
200
|
+
},
|
|
190
201
|
/** 远端 P2P 拉取尚未接线:显式失败,不回落 relay。 */
|
|
191
202
|
async pullFrom(originPeer) {
|
|
192
203
|
return { ok: false, code: 'P2P_UNAVAILABLE', detail: { originPeer, why: 'want-block transport is a later slice; relay fallback is forbidden' } };
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* P0-a: route `xfer.*` frames that arrive on a real P2PChannel.
|
|
3
|
+
*
|
|
4
|
+
* The router owns no transport: it is handed the frame and a channel, and it
|
|
5
|
+
* uses the existing chunk source/client primitives. Registering a source or a
|
|
6
|
+
* receiver is explicit, so an unregistered frame is dropped (never guessed).
|
|
7
|
+
* One listener per channel is the caller's job; this router never attaches one.
|
|
8
|
+
*/
|
|
9
|
+
import { createChunkSource, createWantClient } from './want-block.mjs';
|
|
10
|
+
|
|
11
|
+
export function createXferFrameRouter() {
|
|
12
|
+
const sources = new Map(); // rootCid -> { chunked, filePath, transferId, source }
|
|
13
|
+
const receivers = new Map(); // transferId -> { receiver, client }
|
|
14
|
+
|
|
15
|
+
const parse = (payload) => {
|
|
16
|
+
if (Buffer.isBuffer(payload) || typeof payload === 'string') {
|
|
17
|
+
try { return JSON.parse(Buffer.from(payload).toString('utf8')); } catch { return null; }
|
|
18
|
+
}
|
|
19
|
+
return payload !== null && typeof payload === 'object' ? payload : null;
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
return {
|
|
23
|
+
registerSource(rootCid, { chunked, filePath, transferId = null } = {}) {
|
|
24
|
+
if (typeof rootCid !== 'string' || rootCid === '') return { ok: false, code: 'XFER_ROOT_INVALID' };
|
|
25
|
+
sources.set(rootCid, { chunked, filePath, transferId, source: null, sourceTransferId: null });
|
|
26
|
+
return { ok: true };
|
|
27
|
+
},
|
|
28
|
+
registerReceiver(transferId, receiver, rootCid) {
|
|
29
|
+
if (typeof transferId !== 'string' || transferId === '') return { ok: false, code: 'XFER_TRANSFER_INVALID' };
|
|
30
|
+
const client = createWantClient({ transferId, rootCid, receiver });
|
|
31
|
+
if (client.ok !== true) return client;
|
|
32
|
+
receivers.set(transferId, { receiver, client });
|
|
33
|
+
return { ok: true };
|
|
34
|
+
},
|
|
35
|
+
unregister(rootCid) { sources.delete(rootCid); },
|
|
36
|
+
has(rootCid) { return sources.has(rootCid); },
|
|
37
|
+
/** Send the next want window for a registered receiver over a channel. */
|
|
38
|
+
request(transferId, channel) {
|
|
39
|
+
const entry = receivers.get(transferId);
|
|
40
|
+
if (entry === undefined || typeof channel?.send !== 'function') return { ok: false, code: 'XFER_RECEIVER_UNKNOWN' };
|
|
41
|
+
const missing = entry.client.missing();
|
|
42
|
+
if (missing.length === 0) return { ok: true, requested: 0 };
|
|
43
|
+
const frames = entry.client.requestFrames(missing);
|
|
44
|
+
for (const frame of frames) channel.send(JSON.stringify(frame));
|
|
45
|
+
return { ok: true, requested: missing.length };
|
|
46
|
+
},
|
|
47
|
+
/**
|
|
48
|
+
* @returns {boolean} true when the frame belonged to this layer (handled or refused)
|
|
49
|
+
*/
|
|
50
|
+
handle(payload, { channel } = {}) {
|
|
51
|
+
const frame = parse(payload);
|
|
52
|
+
if (frame === null || typeof frame.type !== 'string' || !frame.type.startsWith('xfer.')) return false;
|
|
53
|
+
if (typeof channel?.send !== 'function') return true;
|
|
54
|
+
if (frame.type === 'xfer.want') {
|
|
55
|
+
const entry = sources.get(frame.rootCid) ?? sources.get(frame.transferId);
|
|
56
|
+
if (entry === undefined) return true; // unknown source: drop, do not leak why
|
|
57
|
+
if (entry.source === null || entry.sourceTransferId !== frame.transferId) {
|
|
58
|
+
entry.source = createChunkSource({ chunked: entry.chunked, filePath: entry.filePath, transferId: frame.transferId });
|
|
59
|
+
entry.sourceTransferId = frame.transferId;
|
|
60
|
+
}
|
|
61
|
+
const served = entry.source.handle(frame);
|
|
62
|
+
if (served.ok !== true) { channel.send(JSON.stringify({ type: 'xfer.error', transferId: frame.transferId, code: served.code })); return true; }
|
|
63
|
+
for (const outgoing of served.frames) channel.send(JSON.stringify(outgoing));
|
|
64
|
+
return true;
|
|
65
|
+
}
|
|
66
|
+
if (frame.type === 'xfer.chunk') {
|
|
67
|
+
const entry = receivers.get(frame.transferId);
|
|
68
|
+
if (entry === undefined) return true;
|
|
69
|
+
entry.receiver.writeChunk(frame.index, Buffer.from(String(frame.data), 'base64'), frame.sha256);
|
|
70
|
+
return true;
|
|
71
|
+
}
|
|
72
|
+
if (frame.type === 'xfer.error') {
|
|
73
|
+
const entry = receivers.get(frame.transferId);
|
|
74
|
+
if (entry !== undefined) entry.lastError = frame.code ?? 'XFER_PEER_ERROR';
|
|
75
|
+
return true;
|
|
76
|
+
}
|
|
77
|
+
return true;
|
|
78
|
+
},
|
|
79
|
+
};
|
|
80
|
+
}
|
|
@@ -87,3 +87,49 @@ export function verifyChunkFrame(frame) {
|
|
|
87
87
|
if (digest !== frame.sha256) return { ok: false, code: 'XFER_CHUNK_HASH_MISMATCH' };
|
|
88
88
|
return { ok: true, data };
|
|
89
89
|
}
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* P0-a: run the chunk source over a MessageChannel (P2PChannel / iroh sidecar)
|
|
94
|
+
* instead of a bare dgram socket. Frames stay JSON for wire compatibility; the
|
|
95
|
+
* channel supplies encryption, ACK and fragmentation.
|
|
96
|
+
*/
|
|
97
|
+
export function attachChunkSource(channel, { chunked, filePath, transferId = null }) {
|
|
98
|
+
const source = createChunkSource({ chunked, filePath, transferId });
|
|
99
|
+
if (source.ok !== true) return source;
|
|
100
|
+
const handler = (payload) => {
|
|
101
|
+
let frame = null;
|
|
102
|
+
try { frame = JSON.parse(Buffer.from(payload).toString('utf8')); } catch { return; }
|
|
103
|
+
const served = source.handle(frame);
|
|
104
|
+
if (served.ok !== true) { channel.send(JSON.stringify({ type: ERROR, transferId: frame?.transferId ?? null, code: served.code })); return; }
|
|
105
|
+
for (const outgoing of served.frames) channel.send(JSON.stringify(outgoing));
|
|
106
|
+
};
|
|
107
|
+
channel.on('message', handler);
|
|
108
|
+
return { ok: true, close: () => channel.off?.('message', handler) };
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* P0-a: receiver side over a MessageChannel. request() sends the next window;
|
|
113
|
+
* inbound chunk frames are validated and written exactly as in the UDP path.
|
|
114
|
+
*/
|
|
115
|
+
export function attachWantClient(channel, { transferId, rootCid, receiver }) {
|
|
116
|
+
const client = createWantClient({ transferId, rootCid, receiver });
|
|
117
|
+
if (client.ok !== true) return client;
|
|
118
|
+
const handler = (payload) => {
|
|
119
|
+
let frame = null;
|
|
120
|
+
try { frame = JSON.parse(Buffer.from(payload).toString('utf8')); } catch { return; }
|
|
121
|
+
client.accept(frame);
|
|
122
|
+
};
|
|
123
|
+
channel.on('message', handler);
|
|
124
|
+
return {
|
|
125
|
+
ok: true,
|
|
126
|
+
client,
|
|
127
|
+
request() {
|
|
128
|
+
const missing = client.missing();
|
|
129
|
+
if (missing.length === 0) return 0;
|
|
130
|
+
channel.send(JSON.stringify(client.requestFrames(missing)[0]));
|
|
131
|
+
return missing.length;
|
|
132
|
+
},
|
|
133
|
+
close: () => channel.off?.('message', handler),
|
|
134
|
+
};
|
|
135
|
+
}
|