@securecoms/agent-bridge 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin.js +257 -45
- package/dist/bridge-agent.d.ts +58 -0
- package/dist/index.js +257 -45
- package/package.json +1 -1
package/dist/bin.js
CHANGED
|
@@ -1387,6 +1387,18 @@ var LeafRosterMismatchError = class extends Error {
|
|
|
1387
1387
|
this.name = "LeafRosterMismatchError";
|
|
1388
1388
|
}
|
|
1389
1389
|
};
|
|
1390
|
+
var AmbiguousOwnLeafError = class extends Error {
|
|
1391
|
+
deviceId;
|
|
1392
|
+
totalLeaves;
|
|
1393
|
+
candidateLeaves;
|
|
1394
|
+
constructor(deviceId, totalLeaves, candidateLeaves) {
|
|
1395
|
+
super(`removeDeviceFromLiveGroup: cannot identify device ${deviceId}'s leaf \u2014 the identity owns ${totalLeaves} leaves and ${candidateLeaves} remain after excluding this committer's own. Leaf credentials carry the identity, not the device, so the target is genuinely ambiguous. Refusing rather than evicting a device that was never revoked.`);
|
|
1396
|
+
this.deviceId = deviceId;
|
|
1397
|
+
this.totalLeaves = totalLeaves;
|
|
1398
|
+
this.candidateLeaves = candidateLeaves;
|
|
1399
|
+
this.name = "AmbiguousOwnLeafError";
|
|
1400
|
+
}
|
|
1401
|
+
};
|
|
1390
1402
|
var HealAddAfterRemoveError = class extends Error {
|
|
1391
1403
|
constructor(cause) {
|
|
1392
1404
|
super("healDivergedMember: Add failed after Remove already landed \u2014 member detached, re-run heal to recover");
|
|
@@ -1406,7 +1418,7 @@ var ForgedCredentialError = class extends Error {
|
|
|
1406
1418
|
this.name = "ForgedCredentialError";
|
|
1407
1419
|
}
|
|
1408
1420
|
};
|
|
1409
|
-
var MlsGroupKeyProvider = class {
|
|
1421
|
+
var MlsGroupKeyProvider = class _MlsGroupKeyProvider {
|
|
1410
1422
|
o;
|
|
1411
1423
|
suite;
|
|
1412
1424
|
managers = /* @__PURE__ */ new Map();
|
|
@@ -1556,9 +1568,25 @@ var MlsGroupKeyProvider = class {
|
|
|
1556
1568
|
* Snapshot of per-channel decrypt-error / stuck state (ADR-0002 observability).
|
|
1557
1569
|
* Surfaces a wedged channel that would otherwise "look idle": the daemon serves
|
|
1558
1570
|
* it on GET /v1/metrics and the web shell reads it via a selector. A channel is
|
|
1559
|
-
* `stuck` once it has had `stuckThreshold`
|
|
1560
|
-
* intervening successful decrypt. Returns a plain object keyed by
|
|
1561
|
-
* absence of a key means zero errors observed for that channel.
|
|
1571
|
+
* `stuck` once it has had `stuckThreshold` GENUINELY-WEDGED decrypt failures
|
|
1572
|
+
* with no intervening successful decrypt. Returns a plain object keyed by
|
|
1573
|
+
* channelId; absence of a key means zero errors observed for that channel.
|
|
1574
|
+
*
|
|
1575
|
+
* `stuck` gates on `wedgedRun`, NOT `consecutiveErrors` — the same correction
|
|
1576
|
+
* the quarantine gate already got (P0a). `consecutiveErrors` is bumped at the
|
|
1577
|
+
* top of the catch for EVERY failure, INCLUDING the benign past-epoch
|
|
1578
|
+
* duplicate-commit churn that the `permanentlyDead` path immediately and
|
|
1579
|
+
* safely auto-drains. Deriving the user-visible health signal from it meant
|
|
1580
|
+
* ordinary at-least-once / committer-race traffic painted a perfectly healthy
|
|
1581
|
+
* channel red.
|
|
1582
|
+
*
|
|
1583
|
+
* That is the prod symptom of 2026-07-31: a freshly-welcomed federated partner
|
|
1584
|
+
* was queued the very Add commit its Welcome already contained. It joined the
|
|
1585
|
+
* correct group at the correct epoch, correctly identified the replayed commit
|
|
1586
|
+
* as dead and drained it — and still showed "This channel is stuck", because
|
|
1587
|
+
* the drain happened AFTER the error had been counted. `wedgedRun` counts only
|
|
1588
|
+
* rows left UN-acked (actually stuck), and a clean decrypt clears it, so a
|
|
1589
|
+
* genuine wedge still reports exactly as before.
|
|
1562
1590
|
*/
|
|
1563
1591
|
errorStats() {
|
|
1564
1592
|
const out = {};
|
|
@@ -1567,7 +1595,7 @@ var MlsGroupKeyProvider = class {
|
|
|
1567
1595
|
decryptErrors: e.decryptErrors,
|
|
1568
1596
|
consecutiveErrors: e.consecutiveErrors,
|
|
1569
1597
|
lastErrorAt: e.lastErrorAt,
|
|
1570
|
-
stuck: e.
|
|
1598
|
+
stuck: e.wedgedRun >= this.stuckThreshold,
|
|
1571
1599
|
quarantined: e.quarantined
|
|
1572
1600
|
};
|
|
1573
1601
|
}
|
|
@@ -2232,6 +2260,25 @@ var MlsGroupKeyProvider = class {
|
|
|
2232
2260
|
* `removed_device_id` (instead of removed_identity_id); the server resolves the
|
|
2233
2261
|
* device's identity, asserts it matches, and purges that ONE device.
|
|
2234
2262
|
*/
|
|
2263
|
+
/**
|
|
2264
|
+
* This committer's OWN leaf index in the given group's ratchet tree, or null
|
|
2265
|
+
* when it cannot be read.
|
|
2266
|
+
*
|
|
2267
|
+
* `MLSGroupManager.state` is marked `@internal` but is a declared (non-private)
|
|
2268
|
+
* field, and `state.privatePath.leafIndex` is the same value ts-mls itself uses
|
|
2269
|
+
* as the sender leaf when protecting application data. Reading it here avoids a
|
|
2270
|
+
* change to `@agentvault/crypto`; if a public accessor is added later this
|
|
2271
|
+
* should delegate to it.
|
|
2272
|
+
*
|
|
2273
|
+
* Returns null rather than throwing so every caller must decide explicitly what
|
|
2274
|
+
* an unknown own-leaf means. The one caller today FAILS CLOSED on null by
|
|
2275
|
+
* falling back to the positional path only for FOREIGN identities — it never
|
|
2276
|
+
* uses null to justify guessing among an own identity's leaves.
|
|
2277
|
+
*/
|
|
2278
|
+
ownLeafIndex(mgr) {
|
|
2279
|
+
const idx = mgr.state?.privatePath?.leafIndex;
|
|
2280
|
+
return typeof idx === "number" ? idx : null;
|
|
2281
|
+
}
|
|
2235
2282
|
async removeDeviceFromLiveGroup(channelId, deviceId, opts) {
|
|
2236
2283
|
return this.locks.run(channelId, () => this.evictOnThrow(channelId, () => this.removeDeviceFromLiveGroupUnlocked(channelId, deviceId, opts, 0)));
|
|
2237
2284
|
}
|
|
@@ -2244,7 +2291,17 @@ var MlsGroupKeyProvider = class {
|
|
|
2244
2291
|
if (leaves.length !== opts.orderedDeviceIds.length) {
|
|
2245
2292
|
throw new LeafRosterMismatchError();
|
|
2246
2293
|
}
|
|
2247
|
-
|
|
2294
|
+
let leafIndex;
|
|
2295
|
+
const ownLeaf = this.ownLeafIndex(mgr);
|
|
2296
|
+
if (ownLeaf !== null && leaves.includes(ownLeaf)) {
|
|
2297
|
+
const others = leaves.filter((i) => i !== ownLeaf);
|
|
2298
|
+
if (others.length !== 1) {
|
|
2299
|
+
throw new AmbiguousOwnLeafError(deviceId, leaves.length, others.length);
|
|
2300
|
+
}
|
|
2301
|
+
leafIndex = others[0];
|
|
2302
|
+
} else {
|
|
2303
|
+
leafIndex = leaves[pos];
|
|
2304
|
+
}
|
|
2248
2305
|
const init = await this.o.backend.initConnectChannelGroup(channelId, [this.o.deviceId]);
|
|
2249
2306
|
const priorEpoch = Number(mgr.epoch);
|
|
2250
2307
|
const { commit } = await mgr.removeMembers([leafIndex]);
|
|
@@ -2497,7 +2554,7 @@ var MlsGroupKeyProvider = class {
|
|
|
2497
2554
|
});
|
|
2498
2555
|
await this.o.store.save(channelId, mgr.exportState());
|
|
2499
2556
|
this.managers.set(channelId, mgr);
|
|
2500
|
-
|
|
2557
|
+
return { staleMemberDeviceIds: reinit.stale_member_device_ids ?? [] };
|
|
2501
2558
|
});
|
|
2502
2559
|
}
|
|
2503
2560
|
/** Returns true when an in-memory MLS manager is live for the channel.
|
|
@@ -2573,7 +2630,60 @@ var MlsGroupKeyProvider = class {
|
|
|
2573
2630
|
* ⇒ refuse, never call verify), then structural signer check, then attestation
|
|
2574
2631
|
* verify against the NON-EMPTY pinned key, then post-join group-id assertion.
|
|
2575
2632
|
*/
|
|
2633
|
+
/** C4 (split-brain Task 4): back-off for REFUSED supersedes, keyed
|
|
2634
|
+
* `${channelId}:${groupId}`. A refused supersede leaves its row un-acked by
|
|
2635
|
+
* design (it is not a stray), but pre-back-off that meant a full re-verify —
|
|
2636
|
+
* including a fetchResetAttestation HTTP round-trip — on EVERY receive poll,
|
|
2637
|
+
* forever, for a request that can never succeed (the nine production split
|
|
2638
|
+
* channels all carry NULL-attestation legacy groups).
|
|
2639
|
+
*
|
|
2640
|
+
* ⚠️ ONLY POST-FETCH refusals enter the back-off. The step-0 pin guard and
|
|
2641
|
+
* the retired-group check run BEFORE fetchResetAttestation, so they
|
|
2642
|
+
* contribute zero HTTP to the storm — and the pin guard's precondition is
|
|
2643
|
+
* TRANSIENT (cold owner cache / momentarily anomalous roster; the bridge
|
|
2644
|
+
* warms the cache before receiveOnce for exactly this reason). Backing a
|
|
2645
|
+
* pre-fetch refusal off buys nothing and converts a one-cycle stall into a
|
|
2646
|
+
* ten-minute outage on a valid recovery.
|
|
2647
|
+
*
|
|
2648
|
+
* Of the backed-off refusals: no-attestation, unparseable content,
|
|
2649
|
+
* channel/group mismatch and post-join mismatch are DETERMINISTIC per
|
|
2650
|
+
* (channel, group) — the attestation columns are written at group creation
|
|
2651
|
+
* and never change. Signer-mismatch, signature-verify and no-bundle are
|
|
2652
|
+
* ASSUMED stable for the window; they are pin- or bundle-dependent, but by
|
|
2653
|
+
* the time they run the fetch has already been paid, so suppressing the
|
|
2654
|
+
* window bounds the storm at the cost of a ≤10-minute delay if the pin or
|
|
2655
|
+
* bundle changes. Note the suppression means the reason is not RE-OBSERVED
|
|
2656
|
+
* during the window — the underlying precondition can change freely; the
|
|
2657
|
+
* window is a cost bound, not a claim of immutability.
|
|
2658
|
+
*
|
|
2659
|
+
* The reason lives in the VALUE for observability; the clearing rule lives
|
|
2660
|
+
* in the KEY: the owner minting a FIXED replacement group arrives with a
|
|
2661
|
+
* NEW group id — a different key, tried immediately. The un-openable path
|
|
2662
|
+
* (verified Welcome, KP mismatch) is EXEMPT: onSupersedeUnopenable's own
|
|
2663
|
+
* cooldown governs it. In-memory only: a reload/restart costs one extra
|
|
2664
|
+
* fetch per stranded row — reload-bounded, not poll-bounded. */
|
|
2665
|
+
supersedeRefusals = /* @__PURE__ */ new Map();
|
|
2666
|
+
static SUPERSEDE_REFUSAL_BACKOFF_MS = 10 * 6e4;
|
|
2667
|
+
/** True while (channelId, groupId) is inside its refusal back-off window.
|
|
2668
|
+
* @internal exposed for diagnostics only — not part of the stable surface. */
|
|
2669
|
+
supersedeBackoffActive(channelId, groupId) {
|
|
2670
|
+
const rec = this.supersedeRefusals.get(`${channelId}:${groupId}`);
|
|
2671
|
+
return rec !== void 0 && Date.now() < rec.until;
|
|
2672
|
+
}
|
|
2576
2673
|
async trySupersede(channelId, row, heldGroupId) {
|
|
2674
|
+
const refuse = (msg) => {
|
|
2675
|
+
this.recordDecryptError(row, new Error(msg));
|
|
2676
|
+
const now = Date.now();
|
|
2677
|
+
for (const [k, v] of this.supersedeRefusals) {
|
|
2678
|
+
if (now >= v.until)
|
|
2679
|
+
this.supersedeRefusals.delete(k);
|
|
2680
|
+
}
|
|
2681
|
+
this.supersedeRefusals.set(`${channelId}:${row.group_id}`, {
|
|
2682
|
+
reason: msg,
|
|
2683
|
+
until: now + _MlsGroupKeyProvider.SUPERSEDE_REFUSAL_BACKOFF_MS
|
|
2684
|
+
});
|
|
2685
|
+
return false;
|
|
2686
|
+
};
|
|
2577
2687
|
const owner = this.o.onResolvePinnedOwner ? await this.o.onResolvePinnedOwner(channelId) : null;
|
|
2578
2688
|
if (!owner || !owner.publicKeyHex) {
|
|
2579
2689
|
this.recordDecryptError(row, new Error("supersede refused: no pinned owner key (step-0 guard)"));
|
|
@@ -2586,33 +2696,27 @@ var MlsGroupKeyProvider = class {
|
|
|
2586
2696
|
}
|
|
2587
2697
|
const att = await this.o.backend.fetchResetAttestation(row.group_id);
|
|
2588
2698
|
if (!att) {
|
|
2589
|
-
|
|
2590
|
-
return false;
|
|
2699
|
+
return refuse(`supersede refused: no attestation for group ${row.group_id}`);
|
|
2591
2700
|
}
|
|
2592
2701
|
if (att.signerIdentityId !== owner.identityId) {
|
|
2593
|
-
|
|
2594
|
-
return false;
|
|
2702
|
+
return refuse("supersede refused: signer is not the pinned owner");
|
|
2595
2703
|
}
|
|
2596
2704
|
let content;
|
|
2597
2705
|
try {
|
|
2598
2706
|
content = JSON.parse(att.signedContent);
|
|
2599
2707
|
} catch {
|
|
2600
|
-
|
|
2601
|
-
return false;
|
|
2708
|
+
return refuse("supersede refused: unparseable signed_content");
|
|
2602
2709
|
}
|
|
2603
2710
|
if (content.channel_id !== row.connect_channel_id || content.new_group_id !== row.group_id) {
|
|
2604
|
-
|
|
2605
|
-
return false;
|
|
2711
|
+
return refuse("supersede refused: attestation channel/group mismatch");
|
|
2606
2712
|
}
|
|
2607
2713
|
const ok = await verifyApprovalArtifact({ content, signature: att.signatureHex, signerPublicKey: owner.publicKeyHex, createdAt: "", version: 1 }, owner.publicKeyHex);
|
|
2608
2714
|
if (!ok) {
|
|
2609
|
-
|
|
2610
|
-
return false;
|
|
2715
|
+
return refuse("supersede refused: signature does not verify against pinned owner key");
|
|
2611
2716
|
}
|
|
2612
2717
|
const bundle = await this.o.store.loadPendingBundle(this.o.deviceId);
|
|
2613
2718
|
if (!bundle) {
|
|
2614
|
-
|
|
2615
|
-
return false;
|
|
2719
|
+
return refuse("supersede refused: no pending KeyPackage bundle to open the new Welcome");
|
|
2616
2720
|
}
|
|
2617
2721
|
return await this.locks.run(channelId, async () => {
|
|
2618
2722
|
const mgr = new MLSGroupManager();
|
|
@@ -2637,14 +2741,14 @@ var MlsGroupKeyProvider = class {
|
|
|
2637
2741
|
return false;
|
|
2638
2742
|
}
|
|
2639
2743
|
if (new TextDecoder().decode(mgr.groupId) !== row.group_id) {
|
|
2640
|
-
|
|
2641
|
-
return false;
|
|
2744
|
+
return refuse("supersede refused: post-join group id != attested new_group_id");
|
|
2642
2745
|
}
|
|
2643
2746
|
this.evictChannelState(channelId);
|
|
2644
2747
|
await this.o.store.delete(channelId);
|
|
2645
2748
|
await this.o.store.discardStaging(channelId);
|
|
2646
2749
|
await this.o.store.addRetiredGroupId(channelId, heldGroupId);
|
|
2647
2750
|
this.errors.delete(channelId);
|
|
2751
|
+
this.supersedeRefusals.delete(`${channelId}:${row.group_id}`);
|
|
2648
2752
|
this.managers.set(channelId, mgr);
|
|
2649
2753
|
await this.o.store.save(channelId, mgr.exportState());
|
|
2650
2754
|
await this.o.backend.ackDelivery(this.o.deviceId, [row.queue_id]);
|
|
@@ -2681,9 +2785,15 @@ var MlsGroupKeyProvider = class {
|
|
|
2681
2785
|
if (heldGroupId !== null && heldGroupId === row.group_id) {
|
|
2682
2786
|
handshakeAcked.push(row.queue_id);
|
|
2683
2787
|
} else if (heldGroupId !== null && heldGroupId !== row.group_id) {
|
|
2684
|
-
|
|
2685
|
-
|
|
2686
|
-
|
|
2788
|
+
if (this.supersedeBackoffActive(wcid, row.group_id)) {
|
|
2789
|
+
if (typeof process !== "undefined" && process.env?.SECURECOMS_TRACE_DELIVERY) {
|
|
2790
|
+
console.warn(`[trace] welcome ch=${wcid.slice(0, 8)} g=${row.group_id.slice(0, 8)} -> SKIP(refusal back-off)`);
|
|
2791
|
+
}
|
|
2792
|
+
} else {
|
|
2793
|
+
const ok = await this.trySupersede(wcid, row, heldGroupId);
|
|
2794
|
+
if (ok)
|
|
2795
|
+
supersededChannelIds.push(wcid);
|
|
2796
|
+
}
|
|
2687
2797
|
}
|
|
2688
2798
|
} else if (typeof process !== "undefined" && process.env?.SECURECOMS_TRACE_DELIVERY) {
|
|
2689
2799
|
console.warn(`[trace] welcome ch=${wcid.slice(0, 8)} rowGroup=${(row.group_id || "-").slice(0, 8)} -> NO-LOCAL-STATE (leave un-acked for join path)`);
|
|
@@ -3386,19 +3496,8 @@ var RestMlsBackend = class {
|
|
|
3386
3496
|
created: j.created === void 0 ? true : Boolean(j.created)
|
|
3387
3497
|
};
|
|
3388
3498
|
}
|
|
3389
|
-
async reinitConnectChannelGroup(
|
|
3390
|
-
|
|
3391
|
-
if (!committer)
|
|
3392
|
-
throw new Error("reinitConnectChannelGroup: empty member set");
|
|
3393
|
-
const j = await this.post(`/api/v1/mls/connect-channels/${channelId}/reinit`, {
|
|
3394
|
-
member_device_ids: memberDeviceIds,
|
|
3395
|
-
committer_device_id: committer
|
|
3396
|
-
});
|
|
3397
|
-
return {
|
|
3398
|
-
group_id: String(j.group_id),
|
|
3399
|
-
current_epoch: Number(j.current_epoch),
|
|
3400
|
-
member_device_ids: Array.isArray(j.member_device_ids) ? j.member_device_ids.map(String) : []
|
|
3401
|
-
};
|
|
3499
|
+
async reinitConnectChannelGroup() {
|
|
3500
|
+
throw new Error("reinitConnectChannelGroup was removed (it minted unattested MLS groups that orphaned channel members permanently). Use resetReinitConnectChannelGroup with an owner attestation.");
|
|
3402
3501
|
}
|
|
3403
3502
|
async resetReinitConnectChannelGroup(channelId, memberDeviceIds, att) {
|
|
3404
3503
|
const committer = memberDeviceIds[0];
|
|
@@ -3416,7 +3515,11 @@ var RestMlsBackend = class {
|
|
|
3416
3515
|
return {
|
|
3417
3516
|
group_id: String(j.group_id),
|
|
3418
3517
|
current_epoch: Number(j.current_epoch),
|
|
3419
|
-
member_device_ids: Array.isArray(j.member_device_ids) ? j.member_device_ids.map(String) : []
|
|
3518
|
+
member_device_ids: Array.isArray(j.member_device_ids) ? j.member_device_ids.map(String) : [],
|
|
3519
|
+
// Absent on a backend that predates the dropped-member report; an empty
|
|
3520
|
+
// list is the correct read of "the server told us nothing", since the old
|
|
3521
|
+
// server also never dropped anyone for an expired KeyPackage.
|
|
3522
|
+
stale_member_device_ids: Array.isArray(j.stale_member_device_ids) ? j.stale_member_device_ids.map(String) : []
|
|
3420
3523
|
};
|
|
3421
3524
|
}
|
|
3422
3525
|
async fetchKeyPackages(deviceIds) {
|
|
@@ -5355,11 +5458,33 @@ var BridgeAgent = class _BridgeAgent {
|
|
|
5355
5458
|
return;
|
|
5356
5459
|
}
|
|
5357
5460
|
this.providerCapabilityMissing = false;
|
|
5358
|
-
|
|
5359
|
-
|
|
5360
|
-
|
|
5361
|
-
|
|
5362
|
-
|
|
5461
|
+
const enclaves = await this.enclavesToCover();
|
|
5462
|
+
this.enclaveListEverSucceeded.add(this.cfg.enclaveId);
|
|
5463
|
+
const seen = /* @__PURE__ */ new Set();
|
|
5464
|
+
const chans = [];
|
|
5465
|
+
const failed = [];
|
|
5466
|
+
let listedAny = false;
|
|
5467
|
+
for (const encId of enclaves) {
|
|
5468
|
+
try {
|
|
5469
|
+
for (const c of await this.listChannels(encId)) {
|
|
5470
|
+
if (seen.has(c.channel_id)) continue;
|
|
5471
|
+
seen.add(c.channel_id);
|
|
5472
|
+
chans.push(c);
|
|
5473
|
+
}
|
|
5474
|
+
listedAny = true;
|
|
5475
|
+
this.enclaveListEverSucceeded.add(encId);
|
|
5476
|
+
this.enclaveListWarned.delete(encId);
|
|
5477
|
+
} catch (e) {
|
|
5478
|
+
failed.push(encId);
|
|
5479
|
+
if (!this.enclaveListWarned.has(encId)) {
|
|
5480
|
+
this.enclaveListWarned.add(encId);
|
|
5481
|
+
console.warn(`[bridge] selfHealSweep: listChannels failed for enclave ${encId}; skipping (warn-once per streak):`, e);
|
|
5482
|
+
}
|
|
5483
|
+
}
|
|
5484
|
+
}
|
|
5485
|
+
this.sweepEnclavesFailed = failed;
|
|
5486
|
+
if (!listedAny) {
|
|
5487
|
+
console.warn("[bridge] selfHealSweep aborted: every enclave listing failed");
|
|
5363
5488
|
return;
|
|
5364
5489
|
}
|
|
5365
5490
|
for (const c of chans) {
|
|
@@ -5705,7 +5830,7 @@ var BridgeAgent = class _BridgeAgent {
|
|
|
5705
5830
|
* the other enclaves in the set.
|
|
5706
5831
|
*/
|
|
5707
5832
|
async refreshChannelMeta() {
|
|
5708
|
-
const enclaves =
|
|
5833
|
+
const enclaves = await this.enclavesToCover();
|
|
5709
5834
|
const now = Date.now();
|
|
5710
5835
|
for (const encId of enclaves) {
|
|
5711
5836
|
try {
|
|
@@ -5718,6 +5843,89 @@ var BridgeAgent = class _BridgeAgent {
|
|
|
5718
5843
|
}
|
|
5719
5844
|
}
|
|
5720
5845
|
}
|
|
5846
|
+
/** Cache for the server-side enclave enumeration (see enclavesToCover). */
|
|
5847
|
+
enclaveEnumCache = null;
|
|
5848
|
+
/** Enclaves whose channel listing failed on the NEWEST sweep (audit HIGH:
|
|
5849
|
+
* skip-one deleted the sweep's partial-coverage signal — observations_stale
|
|
5850
|
+
* is a global max, so one healthy enclave masked a failed one). Surfaced on
|
|
5851
|
+
* health() so partial coverage is never silent. */
|
|
5852
|
+
sweepEnclavesFailed = [];
|
|
5853
|
+
/** Enclave ids whose channel listing has EVER succeeded this process, seeded
|
|
5854
|
+
* with HOME (home is always expected to list; without the seed, a home
|
|
5855
|
+
* failure right after a restart would read as "never succeeded" and slip
|
|
5856
|
+
* through the expected-gap branch — weaker than the old abort-on-failure).
|
|
5857
|
+
* A failure of a member of this set is a REGRESSION and degrades health; a
|
|
5858
|
+
* failure of a never-succeeded enclave (partner: tenant-scoped 404, permanent
|
|
5859
|
+
* pending backend authz) is an EXPECTED gap — reported, not degrading, or
|
|
5860
|
+
* every federated agent would peg degraded forever. */
|
|
5861
|
+
enclaveListEverSucceeded = /* @__PURE__ */ new Set();
|
|
5862
|
+
/** Warn-once-per-failure-streak, keyed by enclave id (cleared on success):
|
|
5863
|
+
* a permanently-404ing partner enclave must not log 1440 lines a day. */
|
|
5864
|
+
enclaveListWarned = /* @__PURE__ */ new Set();
|
|
5865
|
+
static ENCLAVE_ENUM_TTL_MS = 5 * 6e4;
|
|
5866
|
+
enclaveEnumWarned = false;
|
|
5867
|
+
/**
|
|
5868
|
+
* C1 (split-brain Task 3): every enclave this agent must COVER — for the
|
|
5869
|
+
* self-heal sweep and for channel-meta refresh alike. ONE helper on purpose:
|
|
5870
|
+
* these two call sites had drifted (refreshChannelMeta looped home+learned,
|
|
5871
|
+
* selfHealSweep looped nothing), and the drift is exactly what left both
|
|
5872
|
+
* production agents with zero recovery coverage in their second enclave.
|
|
5873
|
+
*
|
|
5874
|
+
* The set is SERVER enumeration ∪ home ∪ store-learned:
|
|
5875
|
+
*
|
|
5876
|
+
* - SERVER (`GET /api/v1/enclaves`, membership-scoped, federated included)
|
|
5877
|
+
* is the authoritative source. The store CANNOT be: channel_meta's only
|
|
5878
|
+
* writers are listChannels itself (circular — you learn enclave B by
|
|
5879
|
+
* having already listed B) and the receive pump's successfully-decrypted
|
|
5880
|
+
* loop, and a FORKED agent decrypts nothing, so the agent that most needs
|
|
5881
|
+
* sweeping in enclave B is precisely the one whose store never learned B.
|
|
5882
|
+
* - home + learned stay in the union as the degraded-mode floor: if the
|
|
5883
|
+
* enumeration endpoint is down, we sweep what we can rather than nothing,
|
|
5884
|
+
* and we warn (once) so the gap is diagnosable.
|
|
5885
|
+
*
|
|
5886
|
+
* The server result is cached for ENCLAVE_ENUM_TTL_MS so per-ingest-cycle
|
|
5887
|
+
* callers (refreshChannelMeta) do not add an HTTP round-trip per cycle;
|
|
5888
|
+
* membership changes are picked up within one TTL, which matches the sweep
|
|
5889
|
+
* cadence this exists to serve.
|
|
5890
|
+
*
|
|
5891
|
+
* ⚠️ Residual gap, stated: the enumeration RETURNS cross-tenant partner
|
|
5892
|
+
* enclaves, but `list_channels_in_enclave` is tenant-scoped and 404s them
|
|
5893
|
+
* under this agent's bearer — so a partner enclave in this set still yields
|
|
5894
|
+
* no channels. Callers must treat a per-enclave listing failure as skip-one,
|
|
5895
|
+
* never abort-all. Closing that needs backend authz work, not daemon work.
|
|
5896
|
+
*/
|
|
5897
|
+
async enclavesToCover() {
|
|
5898
|
+
const now = Date.now();
|
|
5899
|
+
if (!this.enclaveEnumCache || now - this.enclaveEnumCache.at > _BridgeAgent.ENCLAVE_ENUM_TTL_MS) {
|
|
5900
|
+
try {
|
|
5901
|
+
const url = `${this.cfg.apiBase}/api/v1/enclaves?caller_identity_id=${encodeURIComponent(this.cfg.identityId)}`;
|
|
5902
|
+
const res = await this.fetchImpl(url, {
|
|
5903
|
+
headers: { authorization: `Bearer ${this.cfg.apiKey}` }
|
|
5904
|
+
});
|
|
5905
|
+
if (!res.ok) throw new Error(`list_enclaves failed: ${res.status}`);
|
|
5906
|
+
const rows = await res.json();
|
|
5907
|
+
this.enclaveEnumCache = { ids: rows.map((r) => r.enclave_id), at: Date.now() };
|
|
5908
|
+
this.enclaveEnumWarned = false;
|
|
5909
|
+
} catch (e) {
|
|
5910
|
+
if (!this.enclaveEnumWarned) {
|
|
5911
|
+
this.enclaveEnumWarned = true;
|
|
5912
|
+
console.warn(
|
|
5913
|
+
"[bridge] enclave enumeration failed; sweep coverage degraded to home+learned enclaves:",
|
|
5914
|
+
e
|
|
5915
|
+
);
|
|
5916
|
+
}
|
|
5917
|
+
}
|
|
5918
|
+
}
|
|
5919
|
+
const fresh = this.enclaveEnumCache && Date.now() - this.enclaveEnumCache.at <= _BridgeAgent.ENCLAVE_ENUM_TTL_MS;
|
|
5920
|
+
if (fresh) {
|
|
5921
|
+
return /* @__PURE__ */ new Set([this.cfg.enclaveId, ...this.enclaveEnumCache.ids]);
|
|
5922
|
+
}
|
|
5923
|
+
return /* @__PURE__ */ new Set([
|
|
5924
|
+
this.cfg.enclaveId,
|
|
5925
|
+
...this.store.distinctEnclaveIds(),
|
|
5926
|
+
...this.enclaveEnumCache?.ids ?? []
|
|
5927
|
+
]);
|
|
5928
|
+
}
|
|
5721
5929
|
/** A live roster for a channel, or undefined if missing OR expired past the TTL. */
|
|
5722
5930
|
freshRoster(channelId) {
|
|
5723
5931
|
const hit = this.channelRoster.get(channelId);
|
|
@@ -6314,6 +6522,9 @@ var BridgeAgent = class _BridgeAgent {
|
|
|
6314
6522
|
const stalePeriodMs = _BridgeAgent.SELF_HEAL_SWEEP_INTERVAL_MS + _BridgeAgent.SELF_HEAL_SWEEP_JITTER_MS;
|
|
6315
6523
|
const observationsStale = this.providerCapabilityMissing || channels.length === 0 || Date.now() - newest > stalePeriodMs;
|
|
6316
6524
|
const divergedChannels = channels.filter((o) => o.diverged === true).length;
|
|
6525
|
+
const sweepCoverageLost = this.sweepEnclavesFailed.some(
|
|
6526
|
+
(id) => this.enclaveListEverSucceeded.has(id)
|
|
6527
|
+
);
|
|
6317
6528
|
const liveChannels = channels.map((o) => ({
|
|
6318
6529
|
...o,
|
|
6319
6530
|
last_decrypt_at: this.provider.lastDecryptAtFor?.(o.channel_id) ?? o.last_decrypt_at
|
|
@@ -6328,8 +6539,9 @@ var BridgeAgent = class _BridgeAgent {
|
|
|
6328
6539
|
observations_stale: observationsStale,
|
|
6329
6540
|
diverged_channels: divergedChannels,
|
|
6330
6541
|
// A2: the truthful roll-up gates read instead of `status` (D-3).
|
|
6331
|
-
degraded: divergedChannels > 0 || observationsStale,
|
|
6542
|
+
degraded: divergedChannels > 0 || observationsStale || sweepCoverageLost,
|
|
6332
6543
|
channels: liveChannels,
|
|
6544
|
+
sweep_enclaves_failed: [...this.sweepEnclavesFailed],
|
|
6333
6545
|
identity_id: this.cfg.identityId,
|
|
6334
6546
|
joined_channels: joined,
|
|
6335
6547
|
buffered: this.buffer.length,
|
package/dist/bridge-agent.d.ts
CHANGED
|
@@ -674,6 +674,58 @@ export declare class BridgeAgent {
|
|
|
674
674
|
* the other enclaves in the set.
|
|
675
675
|
*/
|
|
676
676
|
private refreshChannelMeta;
|
|
677
|
+
/** Cache for the server-side enclave enumeration (see enclavesToCover). */
|
|
678
|
+
private enclaveEnumCache;
|
|
679
|
+
/** Enclaves whose channel listing failed on the NEWEST sweep (audit HIGH:
|
|
680
|
+
* skip-one deleted the sweep's partial-coverage signal — observations_stale
|
|
681
|
+
* is a global max, so one healthy enclave masked a failed one). Surfaced on
|
|
682
|
+
* health() so partial coverage is never silent. */
|
|
683
|
+
private sweepEnclavesFailed;
|
|
684
|
+
/** Enclave ids whose channel listing has EVER succeeded this process, seeded
|
|
685
|
+
* with HOME (home is always expected to list; without the seed, a home
|
|
686
|
+
* failure right after a restart would read as "never succeeded" and slip
|
|
687
|
+
* through the expected-gap branch — weaker than the old abort-on-failure).
|
|
688
|
+
* A failure of a member of this set is a REGRESSION and degrades health; a
|
|
689
|
+
* failure of a never-succeeded enclave (partner: tenant-scoped 404, permanent
|
|
690
|
+
* pending backend authz) is an EXPECTED gap — reported, not degrading, or
|
|
691
|
+
* every federated agent would peg degraded forever. */
|
|
692
|
+
private readonly enclaveListEverSucceeded;
|
|
693
|
+
/** Warn-once-per-failure-streak, keyed by enclave id (cleared on success):
|
|
694
|
+
* a permanently-404ing partner enclave must not log 1440 lines a day. */
|
|
695
|
+
private readonly enclaveListWarned;
|
|
696
|
+
private static readonly ENCLAVE_ENUM_TTL_MS;
|
|
697
|
+
private enclaveEnumWarned;
|
|
698
|
+
/**
|
|
699
|
+
* C1 (split-brain Task 3): every enclave this agent must COVER — for the
|
|
700
|
+
* self-heal sweep and for channel-meta refresh alike. ONE helper on purpose:
|
|
701
|
+
* these two call sites had drifted (refreshChannelMeta looped home+learned,
|
|
702
|
+
* selfHealSweep looped nothing), and the drift is exactly what left both
|
|
703
|
+
* production agents with zero recovery coverage in their second enclave.
|
|
704
|
+
*
|
|
705
|
+
* The set is SERVER enumeration ∪ home ∪ store-learned:
|
|
706
|
+
*
|
|
707
|
+
* - SERVER (`GET /api/v1/enclaves`, membership-scoped, federated included)
|
|
708
|
+
* is the authoritative source. The store CANNOT be: channel_meta's only
|
|
709
|
+
* writers are listChannels itself (circular — you learn enclave B by
|
|
710
|
+
* having already listed B) and the receive pump's successfully-decrypted
|
|
711
|
+
* loop, and a FORKED agent decrypts nothing, so the agent that most needs
|
|
712
|
+
* sweeping in enclave B is precisely the one whose store never learned B.
|
|
713
|
+
* - home + learned stay in the union as the degraded-mode floor: if the
|
|
714
|
+
* enumeration endpoint is down, we sweep what we can rather than nothing,
|
|
715
|
+
* and we warn (once) so the gap is diagnosable.
|
|
716
|
+
*
|
|
717
|
+
* The server result is cached for ENCLAVE_ENUM_TTL_MS so per-ingest-cycle
|
|
718
|
+
* callers (refreshChannelMeta) do not add an HTTP round-trip per cycle;
|
|
719
|
+
* membership changes are picked up within one TTL, which matches the sweep
|
|
720
|
+
* cadence this exists to serve.
|
|
721
|
+
*
|
|
722
|
+
* ⚠️ Residual gap, stated: the enumeration RETURNS cross-tenant partner
|
|
723
|
+
* enclaves, but `list_channels_in_enclave` is tenant-scoped and 404s them
|
|
724
|
+
* under this agent's bearer — so a partner enclave in this set still yields
|
|
725
|
+
* no channels. Callers must treat a per-enclave listing failure as skip-one,
|
|
726
|
+
* never abort-all. Closing that needs backend authz work, not daemon work.
|
|
727
|
+
*/
|
|
728
|
+
private enclavesToCover;
|
|
677
729
|
/** A live roster for a channel, or undefined if missing OR expired past the TTL. */
|
|
678
730
|
private freshRoster;
|
|
679
731
|
/**
|
|
@@ -836,6 +888,12 @@ export declare class BridgeAgent {
|
|
|
836
888
|
degraded: boolean;
|
|
837
889
|
/** M6: per-channel observations, newest sweep. Read from cache; no I/O. */
|
|
838
890
|
channels: ChannelObservation[];
|
|
891
|
+
/** C1: enclaves whose channel listing failed on the newest sweep. Non-empty
|
|
892
|
+
* means partial coverage — those enclaves' channels were NOT probed. A
|
|
893
|
+
* PREVIOUSLY-SUCCEEDING (or home) entry also flips `degraded`; a
|
|
894
|
+
* never-succeeded entry (partner enclave, tenant-scoped 404) is reported
|
|
895
|
+
* here without degrading. */
|
|
896
|
+
sweep_enclaves_failed: string[];
|
|
839
897
|
/** Optional memory-adapter counters (e.g. Hermes provider); present only when the active provider exposes stats(). */
|
|
840
898
|
hermes_no_session?: number;
|
|
841
899
|
hermes_skipped?: number;
|
package/dist/index.js
CHANGED
|
@@ -1284,6 +1284,18 @@ var LeafRosterMismatchError = class extends Error {
|
|
|
1284
1284
|
this.name = "LeafRosterMismatchError";
|
|
1285
1285
|
}
|
|
1286
1286
|
};
|
|
1287
|
+
var AmbiguousOwnLeafError = class extends Error {
|
|
1288
|
+
deviceId;
|
|
1289
|
+
totalLeaves;
|
|
1290
|
+
candidateLeaves;
|
|
1291
|
+
constructor(deviceId, totalLeaves, candidateLeaves) {
|
|
1292
|
+
super(`removeDeviceFromLiveGroup: cannot identify device ${deviceId}'s leaf \u2014 the identity owns ${totalLeaves} leaves and ${candidateLeaves} remain after excluding this committer's own. Leaf credentials carry the identity, not the device, so the target is genuinely ambiguous. Refusing rather than evicting a device that was never revoked.`);
|
|
1293
|
+
this.deviceId = deviceId;
|
|
1294
|
+
this.totalLeaves = totalLeaves;
|
|
1295
|
+
this.candidateLeaves = candidateLeaves;
|
|
1296
|
+
this.name = "AmbiguousOwnLeafError";
|
|
1297
|
+
}
|
|
1298
|
+
};
|
|
1287
1299
|
var HealAddAfterRemoveError = class extends Error {
|
|
1288
1300
|
constructor(cause) {
|
|
1289
1301
|
super("healDivergedMember: Add failed after Remove already landed \u2014 member detached, re-run heal to recover");
|
|
@@ -1303,7 +1315,7 @@ var ForgedCredentialError = class extends Error {
|
|
|
1303
1315
|
this.name = "ForgedCredentialError";
|
|
1304
1316
|
}
|
|
1305
1317
|
};
|
|
1306
|
-
var MlsGroupKeyProvider = class {
|
|
1318
|
+
var MlsGroupKeyProvider = class _MlsGroupKeyProvider {
|
|
1307
1319
|
o;
|
|
1308
1320
|
suite;
|
|
1309
1321
|
managers = /* @__PURE__ */ new Map();
|
|
@@ -1453,9 +1465,25 @@ var MlsGroupKeyProvider = class {
|
|
|
1453
1465
|
* Snapshot of per-channel decrypt-error / stuck state (ADR-0002 observability).
|
|
1454
1466
|
* Surfaces a wedged channel that would otherwise "look idle": the daemon serves
|
|
1455
1467
|
* it on GET /v1/metrics and the web shell reads it via a selector. A channel is
|
|
1456
|
-
* `stuck` once it has had `stuckThreshold`
|
|
1457
|
-
* intervening successful decrypt. Returns a plain object keyed by
|
|
1458
|
-
* absence of a key means zero errors observed for that channel.
|
|
1468
|
+
* `stuck` once it has had `stuckThreshold` GENUINELY-WEDGED decrypt failures
|
|
1469
|
+
* with no intervening successful decrypt. Returns a plain object keyed by
|
|
1470
|
+
* channelId; absence of a key means zero errors observed for that channel.
|
|
1471
|
+
*
|
|
1472
|
+
* `stuck` gates on `wedgedRun`, NOT `consecutiveErrors` — the same correction
|
|
1473
|
+
* the quarantine gate already got (P0a). `consecutiveErrors` is bumped at the
|
|
1474
|
+
* top of the catch for EVERY failure, INCLUDING the benign past-epoch
|
|
1475
|
+
* duplicate-commit churn that the `permanentlyDead` path immediately and
|
|
1476
|
+
* safely auto-drains. Deriving the user-visible health signal from it meant
|
|
1477
|
+
* ordinary at-least-once / committer-race traffic painted a perfectly healthy
|
|
1478
|
+
* channel red.
|
|
1479
|
+
*
|
|
1480
|
+
* That is the prod symptom of 2026-07-31: a freshly-welcomed federated partner
|
|
1481
|
+
* was queued the very Add commit its Welcome already contained. It joined the
|
|
1482
|
+
* correct group at the correct epoch, correctly identified the replayed commit
|
|
1483
|
+
* as dead and drained it — and still showed "This channel is stuck", because
|
|
1484
|
+
* the drain happened AFTER the error had been counted. `wedgedRun` counts only
|
|
1485
|
+
* rows left UN-acked (actually stuck), and a clean decrypt clears it, so a
|
|
1486
|
+
* genuine wedge still reports exactly as before.
|
|
1459
1487
|
*/
|
|
1460
1488
|
errorStats() {
|
|
1461
1489
|
const out = {};
|
|
@@ -1464,7 +1492,7 @@ var MlsGroupKeyProvider = class {
|
|
|
1464
1492
|
decryptErrors: e.decryptErrors,
|
|
1465
1493
|
consecutiveErrors: e.consecutiveErrors,
|
|
1466
1494
|
lastErrorAt: e.lastErrorAt,
|
|
1467
|
-
stuck: e.
|
|
1495
|
+
stuck: e.wedgedRun >= this.stuckThreshold,
|
|
1468
1496
|
quarantined: e.quarantined
|
|
1469
1497
|
};
|
|
1470
1498
|
}
|
|
@@ -2129,6 +2157,25 @@ var MlsGroupKeyProvider = class {
|
|
|
2129
2157
|
* `removed_device_id` (instead of removed_identity_id); the server resolves the
|
|
2130
2158
|
* device's identity, asserts it matches, and purges that ONE device.
|
|
2131
2159
|
*/
|
|
2160
|
+
/**
|
|
2161
|
+
* This committer's OWN leaf index in the given group's ratchet tree, or null
|
|
2162
|
+
* when it cannot be read.
|
|
2163
|
+
*
|
|
2164
|
+
* `MLSGroupManager.state` is marked `@internal` but is a declared (non-private)
|
|
2165
|
+
* field, and `state.privatePath.leafIndex` is the same value ts-mls itself uses
|
|
2166
|
+
* as the sender leaf when protecting application data. Reading it here avoids a
|
|
2167
|
+
* change to `@agentvault/crypto`; if a public accessor is added later this
|
|
2168
|
+
* should delegate to it.
|
|
2169
|
+
*
|
|
2170
|
+
* Returns null rather than throwing so every caller must decide explicitly what
|
|
2171
|
+
* an unknown own-leaf means. The one caller today FAILS CLOSED on null by
|
|
2172
|
+
* falling back to the positional path only for FOREIGN identities — it never
|
|
2173
|
+
* uses null to justify guessing among an own identity's leaves.
|
|
2174
|
+
*/
|
|
2175
|
+
ownLeafIndex(mgr) {
|
|
2176
|
+
const idx = mgr.state?.privatePath?.leafIndex;
|
|
2177
|
+
return typeof idx === "number" ? idx : null;
|
|
2178
|
+
}
|
|
2132
2179
|
async removeDeviceFromLiveGroup(channelId, deviceId, opts) {
|
|
2133
2180
|
return this.locks.run(channelId, () => this.evictOnThrow(channelId, () => this.removeDeviceFromLiveGroupUnlocked(channelId, deviceId, opts, 0)));
|
|
2134
2181
|
}
|
|
@@ -2141,7 +2188,17 @@ var MlsGroupKeyProvider = class {
|
|
|
2141
2188
|
if (leaves.length !== opts.orderedDeviceIds.length) {
|
|
2142
2189
|
throw new LeafRosterMismatchError();
|
|
2143
2190
|
}
|
|
2144
|
-
|
|
2191
|
+
let leafIndex;
|
|
2192
|
+
const ownLeaf = this.ownLeafIndex(mgr);
|
|
2193
|
+
if (ownLeaf !== null && leaves.includes(ownLeaf)) {
|
|
2194
|
+
const others = leaves.filter((i) => i !== ownLeaf);
|
|
2195
|
+
if (others.length !== 1) {
|
|
2196
|
+
throw new AmbiguousOwnLeafError(deviceId, leaves.length, others.length);
|
|
2197
|
+
}
|
|
2198
|
+
leafIndex = others[0];
|
|
2199
|
+
} else {
|
|
2200
|
+
leafIndex = leaves[pos];
|
|
2201
|
+
}
|
|
2145
2202
|
const init = await this.o.backend.initConnectChannelGroup(channelId, [this.o.deviceId]);
|
|
2146
2203
|
const priorEpoch = Number(mgr.epoch);
|
|
2147
2204
|
const { commit } = await mgr.removeMembers([leafIndex]);
|
|
@@ -2394,7 +2451,7 @@ var MlsGroupKeyProvider = class {
|
|
|
2394
2451
|
});
|
|
2395
2452
|
await this.o.store.save(channelId, mgr.exportState());
|
|
2396
2453
|
this.managers.set(channelId, mgr);
|
|
2397
|
-
|
|
2454
|
+
return { staleMemberDeviceIds: reinit.stale_member_device_ids ?? [] };
|
|
2398
2455
|
});
|
|
2399
2456
|
}
|
|
2400
2457
|
/** Returns true when an in-memory MLS manager is live for the channel.
|
|
@@ -2470,7 +2527,60 @@ var MlsGroupKeyProvider = class {
|
|
|
2470
2527
|
* ⇒ refuse, never call verify), then structural signer check, then attestation
|
|
2471
2528
|
* verify against the NON-EMPTY pinned key, then post-join group-id assertion.
|
|
2472
2529
|
*/
|
|
2530
|
+
/** C4 (split-brain Task 4): back-off for REFUSED supersedes, keyed
|
|
2531
|
+
* `${channelId}:${groupId}`. A refused supersede leaves its row un-acked by
|
|
2532
|
+
* design (it is not a stray), but pre-back-off that meant a full re-verify —
|
|
2533
|
+
* including a fetchResetAttestation HTTP round-trip — on EVERY receive poll,
|
|
2534
|
+
* forever, for a request that can never succeed (the nine production split
|
|
2535
|
+
* channels all carry NULL-attestation legacy groups).
|
|
2536
|
+
*
|
|
2537
|
+
* ⚠️ ONLY POST-FETCH refusals enter the back-off. The step-0 pin guard and
|
|
2538
|
+
* the retired-group check run BEFORE fetchResetAttestation, so they
|
|
2539
|
+
* contribute zero HTTP to the storm — and the pin guard's precondition is
|
|
2540
|
+
* TRANSIENT (cold owner cache / momentarily anomalous roster; the bridge
|
|
2541
|
+
* warms the cache before receiveOnce for exactly this reason). Backing a
|
|
2542
|
+
* pre-fetch refusal off buys nothing and converts a one-cycle stall into a
|
|
2543
|
+
* ten-minute outage on a valid recovery.
|
|
2544
|
+
*
|
|
2545
|
+
* Of the backed-off refusals: no-attestation, unparseable content,
|
|
2546
|
+
* channel/group mismatch and post-join mismatch are DETERMINISTIC per
|
|
2547
|
+
* (channel, group) — the attestation columns are written at group creation
|
|
2548
|
+
* and never change. Signer-mismatch, signature-verify and no-bundle are
|
|
2549
|
+
* ASSUMED stable for the window; they are pin- or bundle-dependent, but by
|
|
2550
|
+
* the time they run the fetch has already been paid, so suppressing the
|
|
2551
|
+
* window bounds the storm at the cost of a ≤10-minute delay if the pin or
|
|
2552
|
+
* bundle changes. Note the suppression means the reason is not RE-OBSERVED
|
|
2553
|
+
* during the window — the underlying precondition can change freely; the
|
|
2554
|
+
* window is a cost bound, not a claim of immutability.
|
|
2555
|
+
*
|
|
2556
|
+
* The reason lives in the VALUE for observability; the clearing rule lives
|
|
2557
|
+
* in the KEY: the owner minting a FIXED replacement group arrives with a
|
|
2558
|
+
* NEW group id — a different key, tried immediately. The un-openable path
|
|
2559
|
+
* (verified Welcome, KP mismatch) is EXEMPT: onSupersedeUnopenable's own
|
|
2560
|
+
* cooldown governs it. In-memory only: a reload/restart costs one extra
|
|
2561
|
+
* fetch per stranded row — reload-bounded, not poll-bounded. */
|
|
2562
|
+
supersedeRefusals = /* @__PURE__ */ new Map();
|
|
2563
|
+
static SUPERSEDE_REFUSAL_BACKOFF_MS = 10 * 6e4;
|
|
2564
|
+
/** True while (channelId, groupId) is inside its refusal back-off window.
|
|
2565
|
+
* @internal exposed for diagnostics only — not part of the stable surface. */
|
|
2566
|
+
supersedeBackoffActive(channelId, groupId) {
|
|
2567
|
+
const rec = this.supersedeRefusals.get(`${channelId}:${groupId}`);
|
|
2568
|
+
return rec !== void 0 && Date.now() < rec.until;
|
|
2569
|
+
}
|
|
2473
2570
|
async trySupersede(channelId, row, heldGroupId) {
|
|
2571
|
+
const refuse = (msg) => {
|
|
2572
|
+
this.recordDecryptError(row, new Error(msg));
|
|
2573
|
+
const now = Date.now();
|
|
2574
|
+
for (const [k, v] of this.supersedeRefusals) {
|
|
2575
|
+
if (now >= v.until)
|
|
2576
|
+
this.supersedeRefusals.delete(k);
|
|
2577
|
+
}
|
|
2578
|
+
this.supersedeRefusals.set(`${channelId}:${row.group_id}`, {
|
|
2579
|
+
reason: msg,
|
|
2580
|
+
until: now + _MlsGroupKeyProvider.SUPERSEDE_REFUSAL_BACKOFF_MS
|
|
2581
|
+
});
|
|
2582
|
+
return false;
|
|
2583
|
+
};
|
|
2474
2584
|
const owner = this.o.onResolvePinnedOwner ? await this.o.onResolvePinnedOwner(channelId) : null;
|
|
2475
2585
|
if (!owner || !owner.publicKeyHex) {
|
|
2476
2586
|
this.recordDecryptError(row, new Error("supersede refused: no pinned owner key (step-0 guard)"));
|
|
@@ -2483,33 +2593,27 @@ var MlsGroupKeyProvider = class {
|
|
|
2483
2593
|
}
|
|
2484
2594
|
const att = await this.o.backend.fetchResetAttestation(row.group_id);
|
|
2485
2595
|
if (!att) {
|
|
2486
|
-
|
|
2487
|
-
return false;
|
|
2596
|
+
return refuse(`supersede refused: no attestation for group ${row.group_id}`);
|
|
2488
2597
|
}
|
|
2489
2598
|
if (att.signerIdentityId !== owner.identityId) {
|
|
2490
|
-
|
|
2491
|
-
return false;
|
|
2599
|
+
return refuse("supersede refused: signer is not the pinned owner");
|
|
2492
2600
|
}
|
|
2493
2601
|
let content;
|
|
2494
2602
|
try {
|
|
2495
2603
|
content = JSON.parse(att.signedContent);
|
|
2496
2604
|
} catch {
|
|
2497
|
-
|
|
2498
|
-
return false;
|
|
2605
|
+
return refuse("supersede refused: unparseable signed_content");
|
|
2499
2606
|
}
|
|
2500
2607
|
if (content.channel_id !== row.connect_channel_id || content.new_group_id !== row.group_id) {
|
|
2501
|
-
|
|
2502
|
-
return false;
|
|
2608
|
+
return refuse("supersede refused: attestation channel/group mismatch");
|
|
2503
2609
|
}
|
|
2504
2610
|
const ok = await verifyApprovalArtifact({ content, signature: att.signatureHex, signerPublicKey: owner.publicKeyHex, createdAt: "", version: 1 }, owner.publicKeyHex);
|
|
2505
2611
|
if (!ok) {
|
|
2506
|
-
|
|
2507
|
-
return false;
|
|
2612
|
+
return refuse("supersede refused: signature does not verify against pinned owner key");
|
|
2508
2613
|
}
|
|
2509
2614
|
const bundle = await this.o.store.loadPendingBundle(this.o.deviceId);
|
|
2510
2615
|
if (!bundle) {
|
|
2511
|
-
|
|
2512
|
-
return false;
|
|
2616
|
+
return refuse("supersede refused: no pending KeyPackage bundle to open the new Welcome");
|
|
2513
2617
|
}
|
|
2514
2618
|
return await this.locks.run(channelId, async () => {
|
|
2515
2619
|
const mgr = new MLSGroupManager();
|
|
@@ -2534,14 +2638,14 @@ var MlsGroupKeyProvider = class {
|
|
|
2534
2638
|
return false;
|
|
2535
2639
|
}
|
|
2536
2640
|
if (new TextDecoder().decode(mgr.groupId) !== row.group_id) {
|
|
2537
|
-
|
|
2538
|
-
return false;
|
|
2641
|
+
return refuse("supersede refused: post-join group id != attested new_group_id");
|
|
2539
2642
|
}
|
|
2540
2643
|
this.evictChannelState(channelId);
|
|
2541
2644
|
await this.o.store.delete(channelId);
|
|
2542
2645
|
await this.o.store.discardStaging(channelId);
|
|
2543
2646
|
await this.o.store.addRetiredGroupId(channelId, heldGroupId);
|
|
2544
2647
|
this.errors.delete(channelId);
|
|
2648
|
+
this.supersedeRefusals.delete(`${channelId}:${row.group_id}`);
|
|
2545
2649
|
this.managers.set(channelId, mgr);
|
|
2546
2650
|
await this.o.store.save(channelId, mgr.exportState());
|
|
2547
2651
|
await this.o.backend.ackDelivery(this.o.deviceId, [row.queue_id]);
|
|
@@ -2578,9 +2682,15 @@ var MlsGroupKeyProvider = class {
|
|
|
2578
2682
|
if (heldGroupId !== null && heldGroupId === row.group_id) {
|
|
2579
2683
|
handshakeAcked.push(row.queue_id);
|
|
2580
2684
|
} else if (heldGroupId !== null && heldGroupId !== row.group_id) {
|
|
2581
|
-
|
|
2582
|
-
|
|
2583
|
-
|
|
2685
|
+
if (this.supersedeBackoffActive(wcid, row.group_id)) {
|
|
2686
|
+
if (typeof process !== "undefined" && process.env?.SECURECOMS_TRACE_DELIVERY) {
|
|
2687
|
+
console.warn(`[trace] welcome ch=${wcid.slice(0, 8)} g=${row.group_id.slice(0, 8)} -> SKIP(refusal back-off)`);
|
|
2688
|
+
}
|
|
2689
|
+
} else {
|
|
2690
|
+
const ok = await this.trySupersede(wcid, row, heldGroupId);
|
|
2691
|
+
if (ok)
|
|
2692
|
+
supersededChannelIds.push(wcid);
|
|
2693
|
+
}
|
|
2584
2694
|
}
|
|
2585
2695
|
} else if (typeof process !== "undefined" && process.env?.SECURECOMS_TRACE_DELIVERY) {
|
|
2586
2696
|
console.warn(`[trace] welcome ch=${wcid.slice(0, 8)} rowGroup=${(row.group_id || "-").slice(0, 8)} -> NO-LOCAL-STATE (leave un-acked for join path)`);
|
|
@@ -3283,19 +3393,8 @@ var RestMlsBackend = class {
|
|
|
3283
3393
|
created: j.created === void 0 ? true : Boolean(j.created)
|
|
3284
3394
|
};
|
|
3285
3395
|
}
|
|
3286
|
-
async reinitConnectChannelGroup(
|
|
3287
|
-
|
|
3288
|
-
if (!committer)
|
|
3289
|
-
throw new Error("reinitConnectChannelGroup: empty member set");
|
|
3290
|
-
const j = await this.post(`/api/v1/mls/connect-channels/${channelId}/reinit`, {
|
|
3291
|
-
member_device_ids: memberDeviceIds,
|
|
3292
|
-
committer_device_id: committer
|
|
3293
|
-
});
|
|
3294
|
-
return {
|
|
3295
|
-
group_id: String(j.group_id),
|
|
3296
|
-
current_epoch: Number(j.current_epoch),
|
|
3297
|
-
member_device_ids: Array.isArray(j.member_device_ids) ? j.member_device_ids.map(String) : []
|
|
3298
|
-
};
|
|
3396
|
+
async reinitConnectChannelGroup() {
|
|
3397
|
+
throw new Error("reinitConnectChannelGroup was removed (it minted unattested MLS groups that orphaned channel members permanently). Use resetReinitConnectChannelGroup with an owner attestation.");
|
|
3299
3398
|
}
|
|
3300
3399
|
async resetReinitConnectChannelGroup(channelId, memberDeviceIds, att) {
|
|
3301
3400
|
const committer = memberDeviceIds[0];
|
|
@@ -3313,7 +3412,11 @@ var RestMlsBackend = class {
|
|
|
3313
3412
|
return {
|
|
3314
3413
|
group_id: String(j.group_id),
|
|
3315
3414
|
current_epoch: Number(j.current_epoch),
|
|
3316
|
-
member_device_ids: Array.isArray(j.member_device_ids) ? j.member_device_ids.map(String) : []
|
|
3415
|
+
member_device_ids: Array.isArray(j.member_device_ids) ? j.member_device_ids.map(String) : [],
|
|
3416
|
+
// Absent on a backend that predates the dropped-member report; an empty
|
|
3417
|
+
// list is the correct read of "the server told us nothing", since the old
|
|
3418
|
+
// server also never dropped anyone for an expired KeyPackage.
|
|
3419
|
+
stale_member_device_ids: Array.isArray(j.stale_member_device_ids) ? j.stale_member_device_ids.map(String) : []
|
|
3317
3420
|
};
|
|
3318
3421
|
}
|
|
3319
3422
|
async fetchKeyPackages(deviceIds) {
|
|
@@ -5211,11 +5314,33 @@ var BridgeAgent = class _BridgeAgent {
|
|
|
5211
5314
|
return;
|
|
5212
5315
|
}
|
|
5213
5316
|
this.providerCapabilityMissing = false;
|
|
5214
|
-
|
|
5215
|
-
|
|
5216
|
-
|
|
5217
|
-
|
|
5218
|
-
|
|
5317
|
+
const enclaves = await this.enclavesToCover();
|
|
5318
|
+
this.enclaveListEverSucceeded.add(this.cfg.enclaveId);
|
|
5319
|
+
const seen = /* @__PURE__ */ new Set();
|
|
5320
|
+
const chans = [];
|
|
5321
|
+
const failed = [];
|
|
5322
|
+
let listedAny = false;
|
|
5323
|
+
for (const encId of enclaves) {
|
|
5324
|
+
try {
|
|
5325
|
+
for (const c of await this.listChannels(encId)) {
|
|
5326
|
+
if (seen.has(c.channel_id)) continue;
|
|
5327
|
+
seen.add(c.channel_id);
|
|
5328
|
+
chans.push(c);
|
|
5329
|
+
}
|
|
5330
|
+
listedAny = true;
|
|
5331
|
+
this.enclaveListEverSucceeded.add(encId);
|
|
5332
|
+
this.enclaveListWarned.delete(encId);
|
|
5333
|
+
} catch (e) {
|
|
5334
|
+
failed.push(encId);
|
|
5335
|
+
if (!this.enclaveListWarned.has(encId)) {
|
|
5336
|
+
this.enclaveListWarned.add(encId);
|
|
5337
|
+
console.warn(`[bridge] selfHealSweep: listChannels failed for enclave ${encId}; skipping (warn-once per streak):`, e);
|
|
5338
|
+
}
|
|
5339
|
+
}
|
|
5340
|
+
}
|
|
5341
|
+
this.sweepEnclavesFailed = failed;
|
|
5342
|
+
if (!listedAny) {
|
|
5343
|
+
console.warn("[bridge] selfHealSweep aborted: every enclave listing failed");
|
|
5219
5344
|
return;
|
|
5220
5345
|
}
|
|
5221
5346
|
for (const c of chans) {
|
|
@@ -5561,7 +5686,7 @@ var BridgeAgent = class _BridgeAgent {
|
|
|
5561
5686
|
* the other enclaves in the set.
|
|
5562
5687
|
*/
|
|
5563
5688
|
async refreshChannelMeta() {
|
|
5564
|
-
const enclaves =
|
|
5689
|
+
const enclaves = await this.enclavesToCover();
|
|
5565
5690
|
const now = Date.now();
|
|
5566
5691
|
for (const encId of enclaves) {
|
|
5567
5692
|
try {
|
|
@@ -5574,6 +5699,89 @@ var BridgeAgent = class _BridgeAgent {
|
|
|
5574
5699
|
}
|
|
5575
5700
|
}
|
|
5576
5701
|
}
|
|
5702
|
+
/** Cache for the server-side enclave enumeration (see enclavesToCover). */
|
|
5703
|
+
enclaveEnumCache = null;
|
|
5704
|
+
/** Enclaves whose channel listing failed on the NEWEST sweep (audit HIGH:
|
|
5705
|
+
* skip-one deleted the sweep's partial-coverage signal — observations_stale
|
|
5706
|
+
* is a global max, so one healthy enclave masked a failed one). Surfaced on
|
|
5707
|
+
* health() so partial coverage is never silent. */
|
|
5708
|
+
sweepEnclavesFailed = [];
|
|
5709
|
+
/** Enclave ids whose channel listing has EVER succeeded this process, seeded
|
|
5710
|
+
* with HOME (home is always expected to list; without the seed, a home
|
|
5711
|
+
* failure right after a restart would read as "never succeeded" and slip
|
|
5712
|
+
* through the expected-gap branch — weaker than the old abort-on-failure).
|
|
5713
|
+
* A failure of a member of this set is a REGRESSION and degrades health; a
|
|
5714
|
+
* failure of a never-succeeded enclave (partner: tenant-scoped 404, permanent
|
|
5715
|
+
* pending backend authz) is an EXPECTED gap — reported, not degrading, or
|
|
5716
|
+
* every federated agent would peg degraded forever. */
|
|
5717
|
+
enclaveListEverSucceeded = /* @__PURE__ */ new Set();
|
|
5718
|
+
/** Warn-once-per-failure-streak, keyed by enclave id (cleared on success):
|
|
5719
|
+
* a permanently-404ing partner enclave must not log 1440 lines a day. */
|
|
5720
|
+
enclaveListWarned = /* @__PURE__ */ new Set();
|
|
5721
|
+
static ENCLAVE_ENUM_TTL_MS = 5 * 6e4;
|
|
5722
|
+
enclaveEnumWarned = false;
|
|
5723
|
+
/**
|
|
5724
|
+
* C1 (split-brain Task 3): every enclave this agent must COVER — for the
|
|
5725
|
+
* self-heal sweep and for channel-meta refresh alike. ONE helper on purpose:
|
|
5726
|
+
* these two call sites had drifted (refreshChannelMeta looped home+learned,
|
|
5727
|
+
* selfHealSweep looped nothing), and the drift is exactly what left both
|
|
5728
|
+
* production agents with zero recovery coverage in their second enclave.
|
|
5729
|
+
*
|
|
5730
|
+
* The set is SERVER enumeration ∪ home ∪ store-learned:
|
|
5731
|
+
*
|
|
5732
|
+
* - SERVER (`GET /api/v1/enclaves`, membership-scoped, federated included)
|
|
5733
|
+
* is the authoritative source. The store CANNOT be: channel_meta's only
|
|
5734
|
+
* writers are listChannels itself (circular — you learn enclave B by
|
|
5735
|
+
* having already listed B) and the receive pump's successfully-decrypted
|
|
5736
|
+
* loop, and a FORKED agent decrypts nothing, so the agent that most needs
|
|
5737
|
+
* sweeping in enclave B is precisely the one whose store never learned B.
|
|
5738
|
+
* - home + learned stay in the union as the degraded-mode floor: if the
|
|
5739
|
+
* enumeration endpoint is down, we sweep what we can rather than nothing,
|
|
5740
|
+
* and we warn (once) so the gap is diagnosable.
|
|
5741
|
+
*
|
|
5742
|
+
* The server result is cached for ENCLAVE_ENUM_TTL_MS so per-ingest-cycle
|
|
5743
|
+
* callers (refreshChannelMeta) do not add an HTTP round-trip per cycle;
|
|
5744
|
+
* membership changes are picked up within one TTL, which matches the sweep
|
|
5745
|
+
* cadence this exists to serve.
|
|
5746
|
+
*
|
|
5747
|
+
* ⚠️ Residual gap, stated: the enumeration RETURNS cross-tenant partner
|
|
5748
|
+
* enclaves, but `list_channels_in_enclave` is tenant-scoped and 404s them
|
|
5749
|
+
* under this agent's bearer — so a partner enclave in this set still yields
|
|
5750
|
+
* no channels. Callers must treat a per-enclave listing failure as skip-one,
|
|
5751
|
+
* never abort-all. Closing that needs backend authz work, not daemon work.
|
|
5752
|
+
*/
|
|
5753
|
+
async enclavesToCover() {
|
|
5754
|
+
const now = Date.now();
|
|
5755
|
+
if (!this.enclaveEnumCache || now - this.enclaveEnumCache.at > _BridgeAgent.ENCLAVE_ENUM_TTL_MS) {
|
|
5756
|
+
try {
|
|
5757
|
+
const url = `${this.cfg.apiBase}/api/v1/enclaves?caller_identity_id=${encodeURIComponent(this.cfg.identityId)}`;
|
|
5758
|
+
const res = await this.fetchImpl(url, {
|
|
5759
|
+
headers: { authorization: `Bearer ${this.cfg.apiKey}` }
|
|
5760
|
+
});
|
|
5761
|
+
if (!res.ok) throw new Error(`list_enclaves failed: ${res.status}`);
|
|
5762
|
+
const rows = await res.json();
|
|
5763
|
+
this.enclaveEnumCache = { ids: rows.map((r) => r.enclave_id), at: Date.now() };
|
|
5764
|
+
this.enclaveEnumWarned = false;
|
|
5765
|
+
} catch (e) {
|
|
5766
|
+
if (!this.enclaveEnumWarned) {
|
|
5767
|
+
this.enclaveEnumWarned = true;
|
|
5768
|
+
console.warn(
|
|
5769
|
+
"[bridge] enclave enumeration failed; sweep coverage degraded to home+learned enclaves:",
|
|
5770
|
+
e
|
|
5771
|
+
);
|
|
5772
|
+
}
|
|
5773
|
+
}
|
|
5774
|
+
}
|
|
5775
|
+
const fresh = this.enclaveEnumCache && Date.now() - this.enclaveEnumCache.at <= _BridgeAgent.ENCLAVE_ENUM_TTL_MS;
|
|
5776
|
+
if (fresh) {
|
|
5777
|
+
return /* @__PURE__ */ new Set([this.cfg.enclaveId, ...this.enclaveEnumCache.ids]);
|
|
5778
|
+
}
|
|
5779
|
+
return /* @__PURE__ */ new Set([
|
|
5780
|
+
this.cfg.enclaveId,
|
|
5781
|
+
...this.store.distinctEnclaveIds(),
|
|
5782
|
+
...this.enclaveEnumCache?.ids ?? []
|
|
5783
|
+
]);
|
|
5784
|
+
}
|
|
5577
5785
|
/** A live roster for a channel, or undefined if missing OR expired past the TTL. */
|
|
5578
5786
|
freshRoster(channelId) {
|
|
5579
5787
|
const hit = this.channelRoster.get(channelId);
|
|
@@ -6170,6 +6378,9 @@ var BridgeAgent = class _BridgeAgent {
|
|
|
6170
6378
|
const stalePeriodMs = _BridgeAgent.SELF_HEAL_SWEEP_INTERVAL_MS + _BridgeAgent.SELF_HEAL_SWEEP_JITTER_MS;
|
|
6171
6379
|
const observationsStale = this.providerCapabilityMissing || channels.length === 0 || Date.now() - newest > stalePeriodMs;
|
|
6172
6380
|
const divergedChannels = channels.filter((o) => o.diverged === true).length;
|
|
6381
|
+
const sweepCoverageLost = this.sweepEnclavesFailed.some(
|
|
6382
|
+
(id) => this.enclaveListEverSucceeded.has(id)
|
|
6383
|
+
);
|
|
6173
6384
|
const liveChannels = channels.map((o) => ({
|
|
6174
6385
|
...o,
|
|
6175
6386
|
last_decrypt_at: this.provider.lastDecryptAtFor?.(o.channel_id) ?? o.last_decrypt_at
|
|
@@ -6184,8 +6395,9 @@ var BridgeAgent = class _BridgeAgent {
|
|
|
6184
6395
|
observations_stale: observationsStale,
|
|
6185
6396
|
diverged_channels: divergedChannels,
|
|
6186
6397
|
// A2: the truthful roll-up gates read instead of `status` (D-3).
|
|
6187
|
-
degraded: divergedChannels > 0 || observationsStale,
|
|
6398
|
+
degraded: divergedChannels > 0 || observationsStale || sweepCoverageLost,
|
|
6188
6399
|
channels: liveChannels,
|
|
6400
|
+
sweep_enclaves_failed: [...this.sweepEnclavesFailed],
|
|
6189
6401
|
identity_id: this.cfg.identityId,
|
|
6190
6402
|
joined_channels: joined,
|
|
6191
6403
|
buffered: this.buffer.length,
|