@securecoms/agent-bridge 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -0
- package/connectors/openclaw/gate.js +41 -0
- package/connectors/openclaw/gate.test.ts +54 -0
- package/connectors/openclaw/openclaw-entry.gate.test.ts +321 -0
- package/connectors/openclaw/openclaw-entry.js +602 -0
- package/connectors/openclaw/openclaw-entry.smoke.test.ts +30 -0
- package/connectors/openclaw/openclaw.plugin.json +25 -0
- package/connectors/openclaw/package.json +18 -0
- package/dist/bin.d.ts +4 -1
- package/dist/bin.js +1216 -76
- package/dist/bridge-agent.d.ts +178 -5
- package/dist/config.d.ts +1 -0
- package/dist/index.js +478 -41
- package/dist/preflight-upgrade.d.cts +2 -0
- package/dist/preflight.cjs +31 -8
- package/dist/service/config.d.ts +1 -0
- package/dist/service/connection-file.d.ts +33 -0
- package/dist/service/fleet.d.ts +27 -0
- package/dist/service/install-prep.d.ts +69 -0
- package/dist/service/resolve-node.d.ts +13 -0
- package/dist/service/status.d.ts +25 -0
- package/dist/service/wire.d.ts +76 -0
- package/dist/ws-server.d.ts +1 -1
- package/package.json +5 -4
package/dist/index.js
CHANGED
|
@@ -1,3 +1,12 @@
|
|
|
1
|
+
// ../connect-core/dist/cadence.js
|
|
2
|
+
var SELF_HEAL_SWEEP_INTERVAL_MS = 9e4;
|
|
3
|
+
var SELF_HEAL_SWEEP_JITTER_MS = 3e4;
|
|
4
|
+
var REQUEST_WELCOME_COOLDOWN_MS = 12e4;
|
|
5
|
+
var WORST_CASE_RESIGNAL_MS = Math.ceil(REQUEST_WELCOME_COOLDOWN_MS / SELF_HEAL_SWEEP_INTERVAL_MS) * (SELF_HEAL_SWEEP_INTERVAL_MS + SELF_HEAL_SWEEP_JITTER_MS);
|
|
6
|
+
function isResignalDue(now, lastAt, cooldownMs = REQUEST_WELCOME_COOLDOWN_MS) {
|
|
7
|
+
return now - lastAt >= cooldownMs;
|
|
8
|
+
}
|
|
9
|
+
|
|
1
10
|
// ../connect-core/dist/envelope/types.js
|
|
2
11
|
var MESSAGE_TYPES = [
|
|
3
12
|
"text",
|
|
@@ -1326,6 +1335,23 @@ var MlsGroupKeyProvider = class {
|
|
|
1326
1335
|
* server delete-and-replaces. Keyed by deviceId (a provider has one), SEPARATE
|
|
1327
1336
|
* from `locks` (channel-keyed) so it never deadlocks against a held channel lock. */
|
|
1328
1337
|
publishLock = new MlsOperationQueue();
|
|
1338
|
+
/**
|
|
1339
|
+
* M6 (ADR-0013 Decision H): wall-clock ms of the last SUCCESSFUL decrypt per
|
|
1340
|
+
* channel. This is the only evidence-based liveness signal the member has —
|
|
1341
|
+
* every other membership indicator in the daemon (`joinState`, a persisted
|
|
1342
|
+
* store row, a leaf in the tree) is a structural artifact that a forked
|
|
1343
|
+
* channel keeps asserting long after it stopped being able to read anything.
|
|
1344
|
+
*
|
|
1345
|
+
* ⚠️ Deliberately written OUTSIDE the `errors` map, and written BEFORE
|
|
1346
|
+
* {@link recordDecryptSuccess}'s `if (!e) return` early-exit. That guard is
|
|
1347
|
+
* correct for its own purpose (there is no error run to clear for a channel
|
|
1348
|
+
* that never errored) but it means a healthy channel decrypting normally
|
|
1349
|
+
* never touches `errors` at all. Keying this timestamp off that map would
|
|
1350
|
+
* leave it null for exactly the channels that are working — a field that
|
|
1351
|
+
* reports "no successful decrypt" for a healthy channel is worse than no
|
|
1352
|
+
* field, because it inverts the signal it exists to give.
|
|
1353
|
+
*/
|
|
1354
|
+
lastDecryptAt = /* @__PURE__ */ new Map();
|
|
1329
1355
|
/**
|
|
1330
1356
|
* Per-channel decrypt-error / stuck tracker (ADR-0002). Bounded by the number
|
|
1331
1357
|
* of channels this device touches (channels are bounded; an entry is created
|
|
@@ -1390,6 +1416,7 @@ var MlsGroupKeyProvider = class {
|
|
|
1390
1416
|
* re-wedges (the onWedged guard is reset so the hook can fire again).
|
|
1391
1417
|
*/
|
|
1392
1418
|
recordDecryptSuccess(channelId) {
|
|
1419
|
+
this.lastDecryptAt.set(channelId, Date.now());
|
|
1393
1420
|
const e = this.errors.get(channelId);
|
|
1394
1421
|
if (!e)
|
|
1395
1422
|
return;
|
|
@@ -1537,7 +1564,9 @@ var MlsGroupKeyProvider = class {
|
|
|
1537
1564
|
* equals newDeviceId (the same device re-admitted).
|
|
1538
1565
|
*/
|
|
1539
1566
|
async admitGuestDevice(channelId, g) {
|
|
1540
|
-
await this.addMemberToLiveGroup(channelId, g.newDeviceId
|
|
1567
|
+
await this.addMemberToLiveGroup(channelId, g.newDeviceId, {
|
|
1568
|
+
knownLeafDevices: g.priorDeviceId ? [g.priorDeviceId] : []
|
|
1569
|
+
});
|
|
1541
1570
|
if (g.priorDeviceId && g.priorDeviceId !== g.newDeviceId) {
|
|
1542
1571
|
const orderedDeviceIds = g.orderedDeviceIds ?? [g.priorDeviceId, g.newDeviceId];
|
|
1543
1572
|
await this.removeDeviceFromLiveGroup(channelId, g.priorDeviceId, {
|
|
@@ -1562,6 +1591,85 @@ var MlsGroupKeyProvider = class {
|
|
|
1562
1591
|
async currentEpoch(channelId) {
|
|
1563
1592
|
return Number((await this.manager(channelId)).epoch);
|
|
1564
1593
|
}
|
|
1594
|
+
/**
|
|
1595
|
+
* Converge a BEHIND member in place by replaying the commits it missed,
|
|
1596
|
+
* instead of tearing down its session and being re-admitted.
|
|
1597
|
+
*
|
|
1598
|
+
* Ported from gen2 (#440). This is the primitive SecureComs was missing, and
|
|
1599
|
+
* its absence is why divergence here has been effectively unrecoverable: the
|
|
1600
|
+
* only client-side route back was Remove+Add, which gen2 treats as the
|
|
1601
|
+
* LAST RESORT and rate-limits hard, because they measured it as a divergence
|
|
1602
|
+
* SOURCE (#360 — epoch 1204 in ~6 minutes; "the re-admitter's churn IS the
|
|
1603
|
+
* bug, not the member").
|
|
1604
|
+
*
|
|
1605
|
+
* Replay is strictly cheaper and safer than re-admission:
|
|
1606
|
+
* - no re-key, no roster change, no epoch movement at all
|
|
1607
|
+
* - it cannot storm the group, because it only READS
|
|
1608
|
+
* - it preserves the member's leaf, so it cannot create a duplicate
|
|
1609
|
+
*
|
|
1610
|
+
* The server already implements the whole contract
|
|
1611
|
+
* (`GET /mls/groups/{id}/sync?last_epoch=N` → `{action, messages}`) and has
|
|
1612
|
+
* since before this change. Nothing consumed it. That is the entire defect.
|
|
1613
|
+
*
|
|
1614
|
+
* Returns true when the member converged to the server's epoch. Returns false
|
|
1615
|
+
* when the caller should fall back to re-admission — the gap exceeded the
|
|
1616
|
+
* server's REJOIN_THRESHOLD, there was nothing to apply, or a commit would
|
|
1617
|
+
* not apply (which means the state is genuinely FORKED, not merely behind,
|
|
1618
|
+
* and replay cannot fix a fork).
|
|
1619
|
+
*/
|
|
1620
|
+
async resyncByReplay(channelId, groupId) {
|
|
1621
|
+
if (!await this.hasPersistedChannelState(channelId))
|
|
1622
|
+
return false;
|
|
1623
|
+
let from;
|
|
1624
|
+
try {
|
|
1625
|
+
from = await this.currentEpoch(channelId);
|
|
1626
|
+
} catch {
|
|
1627
|
+
return false;
|
|
1628
|
+
}
|
|
1629
|
+
const resp = await this.o.backend.syncGroup(groupId, from, this.o.deviceId);
|
|
1630
|
+
if (resp.action !== "replay")
|
|
1631
|
+
return false;
|
|
1632
|
+
const commits = resp.messages.filter((m) => m.message_type === "commit").sort((a, b) => a.epoch - b.epoch);
|
|
1633
|
+
if (commits.length === 0)
|
|
1634
|
+
return false;
|
|
1635
|
+
let applied = 0;
|
|
1636
|
+
try {
|
|
1637
|
+
await this.locks.run(channelId, async () => {
|
|
1638
|
+
const mgr = await this.manager(channelId);
|
|
1639
|
+
for (const c of commits) {
|
|
1640
|
+
if (c.epoch <= Number(mgr.epoch))
|
|
1641
|
+
continue;
|
|
1642
|
+
await mgr.decrypt(hexToBytes(c.payload));
|
|
1643
|
+
applied++;
|
|
1644
|
+
}
|
|
1645
|
+
if (applied > 0)
|
|
1646
|
+
await this.o.store.save(channelId, mgr.exportState());
|
|
1647
|
+
});
|
|
1648
|
+
} catch (e) {
|
|
1649
|
+
console.warn(`[mls] resyncByReplay ${channelId.slice(0, 8)}: commit failed to apply after ${applied} \u2014 state is forked, not behind:`, e?.message);
|
|
1650
|
+
return false;
|
|
1651
|
+
}
|
|
1652
|
+
if (applied > 0) {
|
|
1653
|
+
this.recordDecryptSuccess(channelId);
|
|
1654
|
+
}
|
|
1655
|
+
return applied > 0;
|
|
1656
|
+
}
|
|
1657
|
+
/**
|
|
1658
|
+
* M6 (ADR-0013 Decision H): wall-clock ms of the last successful decrypt for
|
|
1659
|
+
* `channelId`, or `undefined` if this process has never decrypted anything on
|
|
1660
|
+
* it. Synchronous and allocation-free — it is read from the `/healthz` path,
|
|
1661
|
+
* which must never fan out work, because the conditions it exists to report
|
|
1662
|
+
* are exactly the conditions under which fanned-out work hangs.
|
|
1663
|
+
*
|
|
1664
|
+
* `undefined` is genuinely ambiguous and callers MUST NOT read it as
|
|
1665
|
+
* "broken": it also covers a freshly-restarted daemon and a healthy channel
|
|
1666
|
+
* that has simply been quiet. It is evidence of liveness when present, never
|
|
1667
|
+
* evidence of divergence when absent. Divergence is decided by the held-vs-
|
|
1668
|
+
* server epoch comparison, not by this field.
|
|
1669
|
+
*/
|
|
1670
|
+
lastDecryptAtFor(channelId) {
|
|
1671
|
+
return this.lastDecryptAt.get(channelId);
|
|
1672
|
+
}
|
|
1565
1673
|
/**
|
|
1566
1674
|
* Run a committer op and, on ANY throw that escapes it, evict the channel's
|
|
1567
1675
|
* cached MLSGroupManager.
|
|
@@ -1785,7 +1893,7 @@ var MlsGroupKeyProvider = class {
|
|
|
1785
1893
|
if (!bundle)
|
|
1786
1894
|
throw new Error("ensureSession: call publishKeyPackage() first");
|
|
1787
1895
|
if (opts.forceReinit) {
|
|
1788
|
-
const others = opts.otherDeviceIds ?? [];
|
|
1896
|
+
const others = [...new Set(opts.otherDeviceIds ?? [])].filter((d) => d !== this.o.deviceId);
|
|
1789
1897
|
const reinit = await this.o.backend.reinitConnectChannelGroup(channelId, [this.o.deviceId, ...others]);
|
|
1790
1898
|
const mgr = new MLSGroupManager();
|
|
1791
1899
|
await mgr.createGroup(new TextEncoder().encode(reinit.group_id), bundle);
|
|
@@ -1815,7 +1923,7 @@ var MlsGroupKeyProvider = class {
|
|
|
1815
1923
|
throw new BootstrapRaceLostError();
|
|
1816
1924
|
const mgr = new MLSGroupManager();
|
|
1817
1925
|
await mgr.createGroup(new TextEncoder().encode(init.group_id), bundle);
|
|
1818
|
-
const others = opts.otherDeviceIds ?? [];
|
|
1926
|
+
const others = [...new Set(opts.otherDeviceIds ?? [])].filter((d) => d !== this.o.deviceId);
|
|
1819
1927
|
const kps = await this.o.backend.fetchKeyPackages(others);
|
|
1820
1928
|
const pkgs = others.map((d) => {
|
|
1821
1929
|
const hex = kps[d];
|
|
@@ -1866,17 +1974,38 @@ var MlsGroupKeyProvider = class {
|
|
|
1866
1974
|
* full member roster (unioning in any device ids we pass). That avoids a
|
|
1867
1975
|
* parallel group_id-by-channel resolver.
|
|
1868
1976
|
*/
|
|
1869
|
-
async addMemberToLiveGroup(channelId, newDeviceId) {
|
|
1870
|
-
return this.locks.run(channelId, () => this.evictOnThrow(channelId, () => this.addMemberToLiveGroupUnlocked(channelId, newDeviceId, 0)));
|
|
1977
|
+
async addMemberToLiveGroup(channelId, newDeviceId, opts) {
|
|
1978
|
+
return this.locks.run(channelId, () => this.evictOnThrow(channelId, () => this.addMemberToLiveGroupUnlocked(channelId, newDeviceId, 0, opts)));
|
|
1871
1979
|
}
|
|
1872
1980
|
async addMemberToLiveGroupUnlocked(channelId, newDeviceId, _attempt, opts) {
|
|
1873
1981
|
const mgr = await this.manager(channelId);
|
|
1874
|
-
const init = await this.o.backend.initConnectChannelGroup(channelId, [this.o.deviceId, newDeviceId]);
|
|
1875
1982
|
const kps = await this.o.backend.fetchKeyPackages([newDeviceId]);
|
|
1876
1983
|
const hex = kps[newDeviceId];
|
|
1877
1984
|
if (!hex)
|
|
1878
1985
|
throw new Error(`addMemberToLiveGroup: no published KeyPackage for device ${newDeviceId}`);
|
|
1879
1986
|
const pkg = MLSGroupManager.deserializeKeyPackage(hexToBytes(hex));
|
|
1987
|
+
if (_attempt === 0) {
|
|
1988
|
+
let adds;
|
|
1989
|
+
try {
|
|
1990
|
+
adds = (await this.o.backend.getCommitterWork(channelId)).adds;
|
|
1991
|
+
} catch (e) {
|
|
1992
|
+
throw new Error(`addMemberToLiveGroup: committer-work unreachable for channel ${channelId} \u2014 refusing plain Add for device ${newDeviceId} (fail closed): ` + (e instanceof Error ? e.message : String(e)));
|
|
1993
|
+
}
|
|
1994
|
+
if (!adds.includes(newDeviceId)) {
|
|
1995
|
+
const cred = pkg.leafNode.credential;
|
|
1996
|
+
const claimed = cred.credentialType === "basic" ? new TextDecoder().decode(cred.identity) : null;
|
|
1997
|
+
const leaves = claimed === null ? null : mgr.leafIndicesForIdentity(claimed);
|
|
1998
|
+
if (leaves === null || leaves.length > 0) {
|
|
1999
|
+
const known = opts?.knownLeafDevices ? [...new Set(opts.knownLeafDevices)] : null;
|
|
2000
|
+
const swapSafe = leaves !== null && known !== null && !known.includes(newDeviceId) && known.length === leaves.length;
|
|
2001
|
+
if (!swapSafe) {
|
|
2002
|
+
console.warn(`addMemberToLiveGroup: device ${newDeviceId.slice(0, 8)} is outside the committer-work add-delta and its identity already holds a leaf \u2014 terminal skip (no commit)`);
|
|
2003
|
+
return;
|
|
2004
|
+
}
|
|
2005
|
+
}
|
|
2006
|
+
}
|
|
2007
|
+
}
|
|
2008
|
+
const init = await this.o.backend.initConnectChannelGroup(channelId, [this.o.deviceId, newDeviceId]);
|
|
1880
2009
|
if (opts?.expectedIdentityId) {
|
|
1881
2010
|
const cred = pkg.leafNode.credential;
|
|
1882
2011
|
const claimed = cred.credentialType === "basic" ? new TextDecoder().decode(cred.identity) : "";
|
|
@@ -2215,6 +2344,7 @@ var MlsGroupKeyProvider = class {
|
|
|
2215
2344
|
await this.o.store.delete(channelId);
|
|
2216
2345
|
await this.o.store.discardStaging(channelId);
|
|
2217
2346
|
this.errors.delete(channelId);
|
|
2347
|
+
this.lastDecryptAt.delete(channelId);
|
|
2218
2348
|
});
|
|
2219
2349
|
}
|
|
2220
2350
|
/** Owner-attested reset supersede (owner/web side, Task 9). Mirrors the
|
|
@@ -2280,6 +2410,25 @@ var MlsGroupKeyProvider = class {
|
|
|
2280
2410
|
return true;
|
|
2281
2411
|
return await this.o.store.load(channelId) !== null;
|
|
2282
2412
|
}
|
|
2413
|
+
/**
|
|
2414
|
+
* Returns the group id our persisted (or live in-memory) MLS state holds for
|
|
2415
|
+
* `channelId`, or null when there is no state to read, or when the state
|
|
2416
|
+
* exists but a loaded manager can't be constructed from it. Read-only —
|
|
2417
|
+
* `manager()` only loads from the store and caches in memory; it makes no
|
|
2418
|
+
* backend call and writes nothing to the store. Mirrors the same
|
|
2419
|
+
* held-vs-row comparison the Welcome receive path already does inline
|
|
2420
|
+
* (see the `heldGroupId` decode in the `message_type === "welcome"` branch
|
|
2421
|
+
* of `receiveOnce`) so callers can ask "is my held state for THIS group?"
|
|
2422
|
+
* without duplicating that decode logic.
|
|
2423
|
+
*/
|
|
2424
|
+
async heldGroupIdFor(channelId) {
|
|
2425
|
+
try {
|
|
2426
|
+
const mgr = await this.manager(channelId);
|
|
2427
|
+
return new TextDecoder().decode(mgr.groupId);
|
|
2428
|
+
} catch {
|
|
2429
|
+
return null;
|
|
2430
|
+
}
|
|
2431
|
+
}
|
|
2283
2432
|
/**
|
|
2284
2433
|
* Return the number of leaves the given identity holds in the live tree for
|
|
2285
2434
|
* `channelId`. Loads the manager from the store if not already cached.
|
|
@@ -2630,6 +2779,15 @@ function parseServerFrame(frame) {
|
|
|
2630
2779
|
}
|
|
2631
2780
|
|
|
2632
2781
|
// ../connect-sdk-ts/dist/transport/ws-pings.js
|
|
2782
|
+
function base64UrlNoPad(s) {
|
|
2783
|
+
const bytes = new TextEncoder().encode(s);
|
|
2784
|
+
let bin = "";
|
|
2785
|
+
for (const b of bytes)
|
|
2786
|
+
bin += String.fromCharCode(b);
|
|
2787
|
+
const b64 = typeof btoa === "function" ? btoa(bin) : Buffer.from(bytes).toString("base64");
|
|
2788
|
+
return b64.replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");
|
|
2789
|
+
}
|
|
2790
|
+
var BEARER_SUBPROTOCOL = "securecoms.bearer.v1";
|
|
2633
2791
|
function openDeliveryPings(o) {
|
|
2634
2792
|
const base = o.backoffBaseMs ?? 500;
|
|
2635
2793
|
const cap = o.backoffCapMs ?? 8e3;
|
|
@@ -2778,8 +2936,7 @@ function openDeliveryPings(o) {
|
|
|
2778
2936
|
}
|
|
2779
2937
|
if (closed || gen !== connectGen)
|
|
2780
2938
|
return;
|
|
2781
|
-
const
|
|
2782
|
-
const ws = new o.wsImpl(`${o.url}${sep}token=${encodeURIComponent(token)}`);
|
|
2939
|
+
const ws = new o.wsImpl(o.url, [BEARER_SUBPROTOCOL, base64UrlNoPad(token)]);
|
|
2783
2940
|
sock = ws;
|
|
2784
2941
|
ws.onopen = () => {
|
|
2785
2942
|
push({ reconnected: true });
|
|
@@ -3089,6 +3246,22 @@ var RestMlsBackend = class {
|
|
|
3089
3246
|
});
|
|
3090
3247
|
return String(j.id);
|
|
3091
3248
|
}
|
|
3249
|
+
async syncGroup(groupId, lastEpoch, deviceId) {
|
|
3250
|
+
const deviceParam = deviceId ? `&device_id=${encodeURIComponent(deviceId)}` : "";
|
|
3251
|
+
const j = await this.get(`/api/v1/mls/groups/${groupId}/sync?last_epoch=${encodeURIComponent(String(lastEpoch))}${deviceParam}`);
|
|
3252
|
+
const action = j?.action === "rejoin" ? "rejoin" : "replay";
|
|
3253
|
+
const messages = Array.isArray(j?.messages) ? j.messages : [];
|
|
3254
|
+
return {
|
|
3255
|
+
action,
|
|
3256
|
+
messages: messages.map((m) => ({
|
|
3257
|
+
id: String(m.id),
|
|
3258
|
+
epoch: Number(m.epoch),
|
|
3259
|
+
message_type: String(m.message_type),
|
|
3260
|
+
sender_device_id: String(m.sender_device_id ?? ""),
|
|
3261
|
+
payload: String(m.payload ?? "")
|
|
3262
|
+
}))
|
|
3263
|
+
};
|
|
3264
|
+
}
|
|
3092
3265
|
async getConnectChannelGroup(channelId) {
|
|
3093
3266
|
const j = await this.get(`/api/v1/mls/connect-channels/${channelId}/group`);
|
|
3094
3267
|
return {
|
|
@@ -4278,6 +4451,22 @@ var InvalidPathError = class extends Error {
|
|
|
4278
4451
|
if (cause !== void 0) this.cause = cause;
|
|
4279
4452
|
}
|
|
4280
4453
|
};
|
|
4454
|
+
var InvalidMessageIdError = class extends Error {
|
|
4455
|
+
constructor(messageId) {
|
|
4456
|
+
super(`message_id must be a UUID, got: ${messageId}`);
|
|
4457
|
+
this.messageId = messageId;
|
|
4458
|
+
this.name = "InvalidMessageIdError";
|
|
4459
|
+
}
|
|
4460
|
+
};
|
|
4461
|
+
var UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
4462
|
+
function isUuid(s) {
|
|
4463
|
+
return UUID_RE.test(s);
|
|
4464
|
+
}
|
|
4465
|
+
function resolveMessageId(messageId) {
|
|
4466
|
+
if (messageId === void 0) return crypto.randomUUID();
|
|
4467
|
+
if (!isUuid(messageId)) throw new InvalidMessageIdError(messageId);
|
|
4468
|
+
return messageId;
|
|
4469
|
+
}
|
|
4281
4470
|
var AgentPausedSendError = class extends Error {
|
|
4282
4471
|
constructor(channelId) {
|
|
4283
4472
|
super(`agent is paused in channel ${channelId}; send suppressed (no self-heal, no retry)`);
|
|
@@ -4478,10 +4667,75 @@ var BridgeAgent = class _BridgeAgent {
|
|
|
4478
4667
|
* counter is incremented. Surfaced on health()/metrics().
|
|
4479
4668
|
*/
|
|
4480
4669
|
failedAttachments = 0;
|
|
4670
|
+
/**
|
|
4671
|
+
* Task 8: count of live AUTHENTICATED /v1/ws sessions, so a later `status`
|
|
4672
|
+
* verb can tell "bridge process up" (the app's green dot, which only
|
|
4673
|
+
* proves bridge<->backend) apart from "a harness is actually attached"
|
|
4674
|
+
* (a real consumer holding an authenticated session). Only ws-server.ts
|
|
4675
|
+
* mutates this, and only after authedUpgrade() passes — an unauthenticated
|
|
4676
|
+
* socket, or the HTTP /healthz probe, must never move this counter.
|
|
4677
|
+
* Surfaced on health()/metrics().
|
|
4678
|
+
*/
|
|
4679
|
+
authSessions = 0;
|
|
4680
|
+
/**
|
|
4681
|
+
* Task 3 (audit M3): count of newest-wins evictions of a prior /v1/ws
|
|
4682
|
+
* session. The daemon's WS is single-consumer newest-wins, so
|
|
4683
|
+
* `agent_sessions` alone can stay >=1 while two connectors flap the same
|
|
4684
|
+
* identity in and out — this counter makes that split-brain visible.
|
|
4685
|
+
* Only ws-server.ts mutates this, via noteSessionEvicted(). Surfaced on
|
|
4686
|
+
* health()/status.
|
|
4687
|
+
*/
|
|
4688
|
+
evictions = 0;
|
|
4481
4689
|
/** Per-channel timestamp of last requestWelcome call, for cooldown enforcement. */
|
|
4482
4690
|
lastWelcomeRequest = /* @__PURE__ */ new Map();
|
|
4483
4691
|
/** Minimum gap between requestWelcome calls for the same channel (2 minutes). */
|
|
4484
|
-
|
|
4692
|
+
/**
|
|
4693
|
+
* M6 (ADR-0013 Decision H): the sweep's per-channel observation cache, and the
|
|
4694
|
+
* only thing {@link health} reports divergence from.
|
|
4695
|
+
*
|
|
4696
|
+
* Written by `selfHealSweep` in a `finally`, read synchronously by `health()`.
|
|
4697
|
+
* `health()` must NEVER fan out network calls of its own: a health endpoint
|
|
4698
|
+
* that does I/O hangs under exactly the conditions it exists to report, and
|
|
4699
|
+
* the conditions here are "the delivery service is unreachable or lying".
|
|
4700
|
+
*/
|
|
4701
|
+
observations = /* @__PURE__ */ new Map();
|
|
4702
|
+
/**
|
|
4703
|
+
* A3 (dr-evil must-fix #3): true when the sweep found the provider missing a
|
|
4704
|
+
* required capability (stale SDK dist under a newer bridge dist) and ABORTED.
|
|
4705
|
+
* Folded into `observations_stale` (and therefore `degraded`) on health():
|
|
4706
|
+
* a bridge that cannot heal must never present trustworthy-looking data.
|
|
4707
|
+
*/
|
|
4708
|
+
providerCapabilityMissing = false;
|
|
4709
|
+
/**
|
|
4710
|
+
* ADR-0013 Decision D: the wipe's OWN anti-thrash bound, deliberately
|
|
4711
|
+
* separate from {@link REQUEST_WELCOME_COOLDOWN_MS} and separately keyed.
|
|
4712
|
+
*
|
|
4713
|
+
* Two bounds exist because they protect different things and must never
|
|
4714
|
+
* share a gate. The request cooldown protects the single committer from
|
|
4715
|
+
* churn; deferring an ask costs a little latency. The wipe bound only stops
|
|
4716
|
+
* the member thrashing its own session; skipping a wipe costs the member its
|
|
4717
|
+
* ONLY route back, permanently. Sharing one map is what starved the repair.
|
|
4718
|
+
*/
|
|
4719
|
+
static WIPE_ANTI_THRASH_MS = 6e5;
|
|
4720
|
+
/**
|
|
4721
|
+
* ADR-0013 Decision A (wipe-first): the SHORTER re-wipe floor used only when
|
|
4722
|
+
* the fork is PROVEN (the self-heal sweep's resyncByReplay returned false).
|
|
4723
|
+
*
|
|
4724
|
+
* The 600s WIPE_ANTI_THRASH_MS is the right duty cycle for the reactive
|
|
4725
|
+
* onWedged path, but it is a TRAP for a proven-forked member: while the member
|
|
4726
|
+
* holds forked state the owner's re-Welcome is stray-acked (mls.ts, rowGroup ==
|
|
4727
|
+
* heldGroup) and destroyed, so the member never returns to no-state and the
|
|
4728
|
+
* Welcome never lands via the first-join path. Suppressing the re-wipe for a
|
|
4729
|
+
* full 10 minutes means every owner Welcome in that window is destroyed — the
|
|
4730
|
+
* observed hermes flap. This floor lets a proven-forked member re-wipe promptly
|
|
4731
|
+
* (return to no-state so the next Welcome opens) while still bounding the loop:
|
|
4732
|
+
* ~one 25s member ingest poll + margin, which preserves Decision D ("cannot
|
|
4733
|
+
* loop") because two wipes still cannot fire inside a single poll interval.
|
|
4734
|
+
*/
|
|
4735
|
+
static PROVEN_FORK_REWIPE_FLOOR_MS = 3e4;
|
|
4736
|
+
/** Per-channel timestamp of the last state repudiation (Decision D). */
|
|
4737
|
+
lastWipe = /* @__PURE__ */ new Map();
|
|
4738
|
+
static REQUEST_WELCOME_COOLDOWN_MS = REQUEST_WELCOME_COOLDOWN_MS;
|
|
4485
4739
|
// H1: bound the graceful-shutdown drain so a hung in-flight op can't block exit.
|
|
4486
4740
|
static GRACEFUL_DRAIN_MS = 5e3;
|
|
4487
4741
|
/**
|
|
@@ -4503,13 +4757,13 @@ var BridgeAgent = class _BridgeAgent {
|
|
|
4503
4757
|
*/
|
|
4504
4758
|
selfHealSweepEnabled = process.env.SECURECOMS_SELF_HEAL_SWEEP !== "false";
|
|
4505
4759
|
/** Task 12: proactive sweep cadence (in addition to the once-at-boot fire). */
|
|
4506
|
-
static SELF_HEAL_SWEEP_INTERVAL_MS =
|
|
4760
|
+
static SELF_HEAL_SWEEP_INTERVAL_MS = SELF_HEAL_SWEEP_INTERVAL_MS;
|
|
4507
4761
|
// C3: add up to this much random jitter to BOTH the boot sweep delay and each
|
|
4508
4762
|
// periodic interval, so a fleet of daemons restarting/sweeping together does not
|
|
4509
4763
|
// fire re-Welcome signals at the SAME instant and thundering-herd the single
|
|
4510
4764
|
// committer (the observed 409 commit-lease saturation). Decorrelation only —
|
|
4511
4765
|
// C1 already removed the restart KP-rotation storm that caused the herd.
|
|
4512
|
-
static SELF_HEAL_SWEEP_JITTER_MS =
|
|
4766
|
+
static SELF_HEAL_SWEEP_JITTER_MS = SELF_HEAL_SWEEP_JITTER_MS;
|
|
4513
4767
|
selfHealTimer;
|
|
4514
4768
|
bootSweepTimer;
|
|
4515
4769
|
rng;
|
|
@@ -4648,7 +4902,8 @@ var BridgeAgent = class _BridgeAgent {
|
|
|
4648
4902
|
await this.provider.joinAndReconcile(channelId, this.cfg.enclaveId);
|
|
4649
4903
|
this.joinState.set(channelId, "JOINED");
|
|
4650
4904
|
return true;
|
|
4651
|
-
} catch {
|
|
4905
|
+
} catch (e) {
|
|
4906
|
+
console.warn(`[bridge] reconcileJoin ${channelId} failed:`, e);
|
|
4652
4907
|
this.joinState.set(channelId, "PENDING");
|
|
4653
4908
|
return false;
|
|
4654
4909
|
}
|
|
@@ -4706,20 +4961,40 @@ var BridgeAgent = class _BridgeAgent {
|
|
|
4706
4961
|
* SDK's onWedged caller. Single-shot per wedge episode — full rejoin
|
|
4707
4962
|
* reliability is P1, not this hook's job.
|
|
4708
4963
|
*/
|
|
4709
|
-
async healWedgedChannel(channelId) {
|
|
4964
|
+
async healWedgedChannel(channelId, provenForked = false) {
|
|
4710
4965
|
try {
|
|
4711
4966
|
const now = performance.now();
|
|
4712
|
-
const last = this.lastWelcomeRequest.get(channelId) ?? -Infinity;
|
|
4713
|
-
if (now - last < _BridgeAgent.REQUEST_WELCOME_COOLDOWN_MS) return;
|
|
4714
4967
|
const g = await this.backend.getConnectChannelGroup(channelId);
|
|
4715
|
-
if (!g?.group_id) return;
|
|
4716
|
-
this.
|
|
4717
|
-
|
|
4718
|
-
|
|
4719
|
-
|
|
4968
|
+
if (!g?.group_id) return false;
|
|
4969
|
+
const lastWipe = this.lastWipe.get(channelId) ?? -Infinity;
|
|
4970
|
+
const floor = provenForked ? _BridgeAgent.PROVEN_FORK_REWIPE_FLOOR_MS : _BridgeAgent.WIPE_ANTI_THRASH_MS;
|
|
4971
|
+
let repudiated = false;
|
|
4972
|
+
if (now - lastWipe >= floor) {
|
|
4973
|
+
if (await this.provider.hasPersistedChannelState(channelId)) {
|
|
4974
|
+
await this.provider.resetChannelForRejoin(channelId);
|
|
4975
|
+
repudiated = true;
|
|
4976
|
+
}
|
|
4977
|
+
this.lastWipe.set(channelId, now);
|
|
4978
|
+
} else {
|
|
4979
|
+
console.warn(
|
|
4980
|
+
`[bridge] healWedgedChannel ${channelId}: wipe suppressed by ${provenForked ? "proven-fork re-wipe floor" : "anti-thrash bound"} (${Math.round((floor - (now - lastWipe)) / 1e3)}s remaining) \u2014 NOT repudiated`
|
|
4981
|
+
);
|
|
4982
|
+
}
|
|
4983
|
+
this.joinState.delete(channelId);
|
|
4984
|
+
if (!repudiated && await this.provider.hasPersistedChannelState(channelId)) {
|
|
4985
|
+
console.warn(
|
|
4986
|
+
`[bridge] healWedgedChannel ${channelId}: wipe suppressed and state still held \u2014 NOT requesting (the answer would be stray-acked and would cost the group two commits)`
|
|
4987
|
+
);
|
|
4988
|
+
return false;
|
|
4720
4989
|
}
|
|
4990
|
+
const last = this.lastWelcomeRequest.get(channelId) ?? -Infinity;
|
|
4991
|
+
if (!isResignalDue(now, last, _BridgeAgent.REQUEST_WELCOME_COOLDOWN_MS)) return repudiated;
|
|
4992
|
+
this.lastWelcomeRequest.set(channelId, now);
|
|
4721
4993
|
await this.signalHeldKpWelcome(g.group_id);
|
|
4722
|
-
|
|
4994
|
+
return repudiated;
|
|
4995
|
+
} catch (e) {
|
|
4996
|
+
console.warn(`[bridge] healWedgedChannel ${channelId} failed:`, e);
|
|
4997
|
+
return false;
|
|
4723
4998
|
}
|
|
4724
4999
|
}
|
|
4725
5000
|
/**
|
|
@@ -4741,7 +5016,7 @@ var BridgeAgent = class _BridgeAgent {
|
|
|
4741
5016
|
try {
|
|
4742
5017
|
const now = performance.now();
|
|
4743
5018
|
const last = this.lastWelcomeRequest.get(channelId) ?? -Infinity;
|
|
4744
|
-
if (now
|
|
5019
|
+
if (!isResignalDue(now, last, _BridgeAgent.REQUEST_WELCOME_COOLDOWN_MS)) return false;
|
|
4745
5020
|
this.lastWelcomeRequest.set(channelId, now);
|
|
4746
5021
|
await this.signalHeldKpWelcome(groupId);
|
|
4747
5022
|
return true;
|
|
@@ -4749,6 +5024,19 @@ var BridgeAgent = class _BridgeAgent {
|
|
|
4749
5024
|
return false;
|
|
4750
5025
|
}
|
|
4751
5026
|
}
|
|
5027
|
+
/**
|
|
5028
|
+
* Task 2 (MLS convergence slice): true when we hold persisted state for
|
|
5029
|
+
* `channelId` whose group id does NOT match `serverGroupId` — the group
|
|
5030
|
+
* the server currently says is live for that channel. Holding SOME state
|
|
5031
|
+
* is not proof of health; it is proof of health only for the group the
|
|
5032
|
+
* server currently attests. A null held group id (state present but the
|
|
5033
|
+
* manager can't be loaded from it) is treated as diverged too — we can't
|
|
5034
|
+
* confirm health, so we ask rather than silently trusting stale state.
|
|
5035
|
+
*/
|
|
5036
|
+
async isChannelDiverged(channelId, serverGroupId) {
|
|
5037
|
+
const held = await this.provider.heldGroupIdFor(channelId);
|
|
5038
|
+
return held !== serverGroupId;
|
|
5039
|
+
}
|
|
4752
5040
|
/**
|
|
4753
5041
|
* Task 12: proactive self-heal sweep. ingestOnce's Task-11 divergence
|
|
4754
5042
|
* branch only fires reactively — off a delivery ROW that already arrived
|
|
@@ -4757,11 +5045,22 @@ var BridgeAgent = class _BridgeAgent {
|
|
|
4757
5045
|
* first thing lost. This sweep walks every member channel — rooms AND DMs
|
|
4758
5046
|
* (H2: a wiped, idle DM with no traffic since the wipe otherwise never
|
|
4759
5047
|
* proactively re-Welcomes, so it stays dark until the owner happens to send)
|
|
4760
|
-
* — fired once at boot + on SELF_HEAL_SWEEP_INTERVAL_MS
|
|
4761
|
-
*
|
|
4762
|
-
*
|
|
4763
|
-
*
|
|
4764
|
-
* the
|
|
5048
|
+
* — fired once at boot + on SELF_HEAL_SWEEP_INTERVAL_MS.
|
|
5049
|
+
*
|
|
5050
|
+
* Task 2: holding persisted state is NOT proof of health by itself — it is
|
|
5051
|
+
* proof of health only for the group the server currently says is live for
|
|
5052
|
+
* the channel. The prior rule ("I have persisted state ⇒ healthy ⇒ don't
|
|
5053
|
+
* ask") silently stopped filing welcome-requests for any channel that had
|
|
5054
|
+
* been superseded (e.g. a server-side reset/rotation) while the member
|
|
5055
|
+
* still held state for the OLD group: it could hold state, hold the WRONG
|
|
5056
|
+
* state, and never say anything. So the server's current group_id is
|
|
5057
|
+
* fetched FIRST via the READ-ONLY getConnectChannelGroup probe (never
|
|
5058
|
+
* initConnectChannelGroup, which would side-effect a member_device_ids
|
|
5059
|
+
* union + commit), then the branch is: persisted state whose group id
|
|
5060
|
+
* matches the server's current group ⇒ healthy, skip; persisted state for
|
|
5061
|
+
* a STALE group id ⇒ diverged, signal; no persisted state at all ⇒
|
|
5062
|
+
* reconcile-first (consume an already-queued Welcome) before concluding
|
|
5063
|
+
* divergence. All three paths go through the same per-channel cooldown.
|
|
4765
5064
|
*
|
|
4766
5065
|
* Each channel is wrapped in its own try/catch: `listChannels` filters at
|
|
4767
5066
|
* the identity level while `requestWelcome` (via request_welcome) gates at
|
|
@@ -4770,25 +5069,114 @@ var BridgeAgent = class _BridgeAgent {
|
|
|
4770
5069
|
* must not abort the sweep for every other channel.
|
|
4771
5070
|
*/
|
|
4772
5071
|
async selfHealSweep() {
|
|
5072
|
+
if (typeof this.provider.resyncByReplay !== "function" || typeof this.provider.heldGroupIdFor !== "function") {
|
|
5073
|
+
this.providerCapabilityMissing = true;
|
|
5074
|
+
console.error(
|
|
5075
|
+
"[bridge] selfHealSweep ABORTED: provider is missing required capabilities (resyncByReplay/heldGroupIdFor) \u2014 stale SDK dist? Rebuild SDK dist THEN bridge dist. Healing is NOT running; observations are reported STALE."
|
|
5076
|
+
);
|
|
5077
|
+
return;
|
|
5078
|
+
}
|
|
5079
|
+
this.providerCapabilityMissing = false;
|
|
4773
5080
|
let chans;
|
|
4774
5081
|
try {
|
|
4775
5082
|
chans = await this.listChannels();
|
|
4776
|
-
} catch {
|
|
5083
|
+
} catch (e) {
|
|
5084
|
+
console.warn("[bridge] selfHealSweep aborted: listChannels failed:", e);
|
|
4777
5085
|
return;
|
|
4778
5086
|
}
|
|
4779
5087
|
for (const c of chans) {
|
|
5088
|
+
const obs = {
|
|
5089
|
+
channel_id: c.channel_id,
|
|
5090
|
+
state: "probe_failed",
|
|
5091
|
+
held_group_id: null,
|
|
5092
|
+
server_group_id: null,
|
|
5093
|
+
held_epoch: null,
|
|
5094
|
+
server_epoch: null,
|
|
5095
|
+
diverged: null,
|
|
5096
|
+
last_decrypt_at: this.provider.lastDecryptAtFor?.(c.channel_id) ?? null,
|
|
5097
|
+
observed_at: Date.now()
|
|
5098
|
+
};
|
|
4780
5099
|
try {
|
|
4781
|
-
if (await this.provider.hasPersistedChannelState(c.channel_id)) continue;
|
|
4782
|
-
await this.reconcileJoin(c.channel_id);
|
|
4783
|
-
if (await this.provider.hasPersistedChannelState(c.channel_id)) continue;
|
|
4784
5100
|
const g = await this.backend.getConnectChannelGroup(c.channel_id);
|
|
4785
|
-
|
|
5101
|
+
obs.server_group_id = g?.group_id ?? null;
|
|
5102
|
+
obs.server_epoch = g?.current_epoch ?? null;
|
|
5103
|
+
if (!g?.group_id) {
|
|
5104
|
+
obs.state = "group_unresolvable";
|
|
5105
|
+
continue;
|
|
5106
|
+
}
|
|
5107
|
+
const hasState = await this.provider.hasPersistedChannelState(c.channel_id);
|
|
5108
|
+
if (hasState) {
|
|
5109
|
+
obs.held_group_id = await this.provider.heldGroupIdFor(c.channel_id).catch(() => null);
|
|
5110
|
+
obs.held_epoch = await this.provider.currentEpoch(c.channel_id).catch(() => null);
|
|
5111
|
+
if (!await this.isChannelDiverged(c.channel_id, g.group_id)) {
|
|
5112
|
+
if (obs.held_epoch === null) {
|
|
5113
|
+
obs.state = "probe_failed";
|
|
5114
|
+
obs.diverged = null;
|
|
5115
|
+
continue;
|
|
5116
|
+
}
|
|
5117
|
+
const behind = typeof obs.held_epoch === "number" && typeof obs.server_epoch === "number" && obs.held_epoch < obs.server_epoch;
|
|
5118
|
+
obs.state = behind ? "behind" : "healthy";
|
|
5119
|
+
obs.diverged = behind;
|
|
5120
|
+
if (behind) {
|
|
5121
|
+
const prev = this.observations.get(c.channel_id);
|
|
5122
|
+
const notAdvancing = prev != null && prev.held_epoch === obs.held_epoch;
|
|
5123
|
+
if (notAdvancing) {
|
|
5124
|
+
obs.state = "stuck";
|
|
5125
|
+
if (await this.provider.resyncByReplay(c.channel_id, g.group_id)) {
|
|
5126
|
+
obs.held_epoch = await this.provider.currentEpoch(c.channel_id).catch(() => null);
|
|
5127
|
+
obs.last_decrypt_at = this.provider.lastDecryptAtFor?.(c.channel_id) ?? null;
|
|
5128
|
+
if (obs.held_epoch === null) {
|
|
5129
|
+
obs.state = "probe_failed";
|
|
5130
|
+
obs.diverged = null;
|
|
5131
|
+
continue;
|
|
5132
|
+
}
|
|
5133
|
+
obs.state = "healthy";
|
|
5134
|
+
obs.diverged = false;
|
|
5135
|
+
console.warn(
|
|
5136
|
+
`[bridge] selfHealSweep: ${c.channel_id} caught up by REPLAY to epoch ${obs.held_epoch} (no re-admit, no epoch churn)`
|
|
5137
|
+
);
|
|
5138
|
+
continue;
|
|
5139
|
+
}
|
|
5140
|
+
const repudiated = await this.healWedgedChannel(c.channel_id, true);
|
|
5141
|
+
console.warn(
|
|
5142
|
+
`[bridge] selfHealSweep: ${c.channel_id} STUCK at epoch ${obs.held_epoch} (server=${obs.server_epoch}) across two sweeps \u2014 ${repudiated ? "repudiated and re-requested" : "repair SUPPRESSED (anti-thrash); state unchanged"}`
|
|
5143
|
+
);
|
|
5144
|
+
}
|
|
5145
|
+
continue;
|
|
5146
|
+
}
|
|
5147
|
+
console.debug(
|
|
5148
|
+
`[bridge] selfHealSweep: ${c.channel_id} ${behind ? `BEHIND (held=${obs.held_epoch} server=${obs.server_epoch}) \u2014 reporting only, action predicate is group-id` : "healthy, held group matches server"} \u2014 skipping`
|
|
5149
|
+
);
|
|
5150
|
+
continue;
|
|
5151
|
+
}
|
|
5152
|
+
obs.state = "diverged";
|
|
5153
|
+
obs.diverged = true;
|
|
5154
|
+
console.warn(`[bridge] selfHealSweep: ${c.channel_id} holds STALE group state \u2014 signalling re-Welcome`);
|
|
5155
|
+
} else {
|
|
5156
|
+
obs.state = "no_local_state";
|
|
5157
|
+
obs.diverged = false;
|
|
5158
|
+
await this.reconcileJoin(c.channel_id);
|
|
5159
|
+
if (await this.provider.hasPersistedChannelState(c.channel_id)) {
|
|
5160
|
+
obs.held_group_id = await this.provider.heldGroupIdFor(c.channel_id).catch(() => null);
|
|
5161
|
+
obs.held_epoch = await this.provider.currentEpoch(c.channel_id).catch(() => null);
|
|
5162
|
+
if (obs.held_epoch === null) {
|
|
5163
|
+
obs.state = "probe_failed";
|
|
5164
|
+
obs.diverged = null;
|
|
5165
|
+
continue;
|
|
5166
|
+
}
|
|
5167
|
+
obs.state = "healthy";
|
|
5168
|
+
continue;
|
|
5169
|
+
}
|
|
5170
|
+
}
|
|
4786
5171
|
const now = performance.now();
|
|
4787
5172
|
const last = this.lastWelcomeRequest.get(c.channel_id) ?? -Infinity;
|
|
4788
|
-
if (now
|
|
5173
|
+
if (!isResignalDue(now, last, _BridgeAgent.REQUEST_WELCOME_COOLDOWN_MS)) continue;
|
|
4789
5174
|
this.lastWelcomeRequest.set(c.channel_id, now);
|
|
4790
5175
|
await this.signalHeldKpWelcome(g.group_id);
|
|
4791
|
-
} catch {
|
|
5176
|
+
} catch (e) {
|
|
5177
|
+
console.warn(`[bridge] selfHealSweep: channel ${c.channel_id} failed:`, e);
|
|
5178
|
+
} finally {
|
|
5179
|
+
this.observations.set(c.channel_id, obs);
|
|
4792
5180
|
}
|
|
4793
5181
|
}
|
|
4794
5182
|
}
|
|
@@ -4843,7 +5231,7 @@ var BridgeAgent = class _BridgeAgent {
|
|
|
4843
5231
|
if (meta && !meta.is_dm) this.enforceRateFloor("room", channelId);
|
|
4844
5232
|
const joined = await this.reconcileJoin(channelId);
|
|
4845
5233
|
if (!joined) throw new ChannelPendingError(channelId);
|
|
4846
|
-
const id = messageId
|
|
5234
|
+
const id = resolveMessageId(messageId);
|
|
4847
5235
|
const sendStart = performance.now();
|
|
4848
5236
|
await this.sendWithSelfHealTerminal(
|
|
4849
5237
|
this.provider,
|
|
@@ -4880,7 +5268,7 @@ var BridgeAgent = class _BridgeAgent {
|
|
|
4880
5268
|
this.enforceRateFloor("a2a", channelId);
|
|
4881
5269
|
const joined = await this.reconcileJoin(channelId);
|
|
4882
5270
|
if (!joined) throw new ChannelPendingError(channelId);
|
|
4883
|
-
const id = messageId
|
|
5271
|
+
const id = resolveMessageId(messageId);
|
|
4884
5272
|
const sendStart = performance.now();
|
|
4885
5273
|
await this.sendWithSelfHealTerminal(
|
|
4886
5274
|
this.provider,
|
|
@@ -5421,11 +5809,12 @@ var BridgeAgent = class _BridgeAgent {
|
|
|
5421
5809
|
if (await this.provider.hasPersistedChannelState(cid)) continue;
|
|
5422
5810
|
const now = performance.now();
|
|
5423
5811
|
const last = this.lastWelcomeRequest.get(cid) ?? -Infinity;
|
|
5424
|
-
if (now
|
|
5812
|
+
if (!isResignalDue(now, last, _BridgeAgent.REQUEST_WELCOME_COOLDOWN_MS)) continue;
|
|
5425
5813
|
this.lastWelcomeRequest.set(cid, now);
|
|
5426
5814
|
try {
|
|
5427
5815
|
await this.signalHeldKpWelcome(gid);
|
|
5428
|
-
} catch {
|
|
5816
|
+
} catch (e) {
|
|
5817
|
+
console.warn(`[bridge] ingestOnce: signalHeldKpWelcome for ${cid} failed:`, e);
|
|
5429
5818
|
}
|
|
5430
5819
|
}
|
|
5431
5820
|
} catch {
|
|
@@ -5614,11 +6003,55 @@ var BridgeAgent = class _BridgeAgent {
|
|
|
5614
6003
|
const w = this.waiters.splice(0);
|
|
5615
6004
|
for (const r of w) r();
|
|
5616
6005
|
}
|
|
6006
|
+
/**
|
|
6007
|
+
* Task 8: record a newly authenticated /v1/ws session. Called by
|
|
6008
|
+
* ws-server.ts exactly once per connection, AFTER authedUpgrade() passes
|
|
6009
|
+
* (never for a socket closed 1008 for bad auth, and never for the HTTP
|
|
6010
|
+
* /healthz probe, which is not a /v1/ws session at all).
|
|
6011
|
+
*/
|
|
6012
|
+
noteSessionOpen() {
|
|
6013
|
+
this.authSessions++;
|
|
6014
|
+
}
|
|
6015
|
+
/**
|
|
6016
|
+
* Task 8: record an authenticated /v1/ws session ending. Clamped at 0 so a
|
|
6017
|
+
* stray extra close (e.g. a double-fired cleanup) can't go negative and
|
|
6018
|
+
* poison the next open/close cycle.
|
|
6019
|
+
*/
|
|
6020
|
+
noteSessionClose() {
|
|
6021
|
+
this.authSessions = Math.max(0, this.authSessions - 1);
|
|
6022
|
+
}
|
|
6023
|
+
/**
|
|
6024
|
+
* Task 3 (audit M3): record a prior /v1/ws session being evicted by a
|
|
6025
|
+
* newer connection for the same identity (newest-wins). Called by
|
|
6026
|
+
* ws-server.ts right after prior.cleanup() in the eviction path.
|
|
6027
|
+
*/
|
|
6028
|
+
noteSessionEvicted() {
|
|
6029
|
+
this.evictions++;
|
|
6030
|
+
}
|
|
5617
6031
|
health() {
|
|
5618
6032
|
let joined = 0;
|
|
5619
6033
|
for (const s of this.joinState.values()) if (s === "JOINED") joined++;
|
|
6034
|
+
const channels = [...this.observations.values()];
|
|
6035
|
+
const newest = channels.reduce((m, o) => Math.max(m, o.observed_at), 0);
|
|
6036
|
+
const stalePeriodMs = _BridgeAgent.SELF_HEAL_SWEEP_INTERVAL_MS + _BridgeAgent.SELF_HEAL_SWEEP_JITTER_MS;
|
|
6037
|
+
const observationsStale = this.providerCapabilityMissing || channels.length === 0 || Date.now() - newest > stalePeriodMs;
|
|
6038
|
+
const divergedChannels = channels.filter((o) => o.diverged === true).length;
|
|
6039
|
+
const liveChannels = channels.map((o) => ({
|
|
6040
|
+
...o,
|
|
6041
|
+
last_decrypt_at: this.provider.lastDecryptAtFor?.(o.channel_id) ?? o.last_decrypt_at
|
|
6042
|
+
}));
|
|
5620
6043
|
return {
|
|
6044
|
+
// Deliberately still unconditional. `agent-bridge status` and any external
|
|
6045
|
+
// monitor read this field, and a monitor that restarts non-ok bridges would
|
|
6046
|
+
// loop forever on a diverged agent. The additive fields below land first so
|
|
6047
|
+
// the data can be trusted; flipping this to "degraded" is its own change.
|
|
5621
6048
|
status: "ok",
|
|
6049
|
+
sweep_enabled: this.selfHealSweepEnabled,
|
|
6050
|
+
observations_stale: observationsStale,
|
|
6051
|
+
diverged_channels: divergedChannels,
|
|
6052
|
+
// A2: the truthful roll-up gates read instead of `status` (D-3).
|
|
6053
|
+
degraded: divergedChannels > 0 || observationsStale,
|
|
6054
|
+
channels: liveChannels,
|
|
5622
6055
|
identity_id: this.cfg.identityId,
|
|
5623
6056
|
joined_channels: joined,
|
|
5624
6057
|
buffered: this.buffer.length,
|
|
@@ -5628,6 +6061,8 @@ var BridgeAgent = class _BridgeAgent {
|
|
|
5628
6061
|
permanently_skipped: this.provider.permanentlySkippedCount(),
|
|
5629
6062
|
failed_attachments: this.failedAttachments,
|
|
5630
6063
|
delivery_pings: this.pingsReceived,
|
|
6064
|
+
agent_sessions: this.authSessions,
|
|
6065
|
+
evictions: this.evictions,
|
|
5631
6066
|
uptime_s: this.startedAt ? Math.floor((Date.now() - this.startedAt) / 1e3) : 0,
|
|
5632
6067
|
version: BRIDGE_VERSION,
|
|
5633
6068
|
...this.memory.stats?.() ?? {}
|
|
@@ -5987,7 +6422,7 @@ function createHttpServer(agent, opts) {
|
|
|
5987
6422
|
if (method === "POST" && path === "/v1/send") {
|
|
5988
6423
|
const parsed = sendSchema.safeParse(JSON.parse(await readBody(req, SEND_CAP)));
|
|
5989
6424
|
if (!parsed.success) return jsonError(res, "invalid_request", parsed.error.issues);
|
|
5990
|
-
const r = parsed.data.a2a ? await agent.sendA2A(parsed.data.channel_id, parsed.data.a2a
|
|
6425
|
+
const r = parsed.data.a2a ? await agent.sendA2A(parsed.data.channel_id, parsed.data.a2a) : await agent.send(parsed.data.channel_id, parsed.data.text);
|
|
5991
6426
|
return json(res, 200, r);
|
|
5992
6427
|
}
|
|
5993
6428
|
if (method === "POST" && path === "/v1/send-attachment") {
|
|
@@ -6118,12 +6553,14 @@ function attachAgentWebSocket(server, agent, opts) {
|
|
|
6118
6553
|
sock.close(1008, "unauthorized");
|
|
6119
6554
|
return;
|
|
6120
6555
|
}
|
|
6556
|
+
agent.noteSessionOpen();
|
|
6121
6557
|
if (current && current.sock !== sock) {
|
|
6122
6558
|
const prior = current;
|
|
6123
6559
|
if (prior.sock.readyState === WebSocket.OPEN) {
|
|
6124
6560
|
prior.sock.close(1e3, "replaced by newer connection");
|
|
6125
6561
|
}
|
|
6126
6562
|
prior.cleanup();
|
|
6563
|
+
agent.noteSessionEvicted();
|
|
6127
6564
|
}
|
|
6128
6565
|
sock.isAlive = true;
|
|
6129
6566
|
sock.on("pong", () => {
|
|
@@ -6153,6 +6590,7 @@ function attachAgentWebSocket(server, agent, opts) {
|
|
|
6153
6590
|
const cleanup = () => {
|
|
6154
6591
|
if (cleaned) return;
|
|
6155
6592
|
cleaned = true;
|
|
6593
|
+
agent.noteSessionClose();
|
|
6156
6594
|
unsubscribe();
|
|
6157
6595
|
if (inflight.size) {
|
|
6158
6596
|
agent.requeueFront([...inflight.values()]);
|
|
@@ -6239,9 +6677,8 @@ async function handleInbound(sock, agent, data, inflight) {
|
|
|
6239
6677
|
sendJson(sock, wsErrorFrame("invalid_request"));
|
|
6240
6678
|
return;
|
|
6241
6679
|
}
|
|
6242
|
-
const idempotencyKey = typeof m.idempotency_key === "string" ? m.idempotency_key : void 0;
|
|
6243
6680
|
try {
|
|
6244
|
-
const r = await agent.send(m.channel_id, m.text
|
|
6681
|
+
const r = await agent.send(m.channel_id, m.text);
|
|
6245
6682
|
sendJson(sock, { type: "sent", message_id: r.message_id });
|
|
6246
6683
|
} catch (e) {
|
|
6247
6684
|
if (e instanceof ChannelPendingError) {
|