@securecoms/agent-bridge 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/connectors/openclaw/gate.js +41 -0
- package/connectors/openclaw/gate.test.ts +54 -0
- package/connectors/openclaw/openclaw-entry.gate.test.ts +321 -0
- package/connectors/openclaw/openclaw-entry.js +602 -0
- package/connectors/openclaw/openclaw-entry.smoke.test.ts +30 -0
- package/connectors/openclaw/openclaw.plugin.json +25 -0
- package/connectors/openclaw/package.json +18 -0
- package/dist/bin.d.ts +4 -1
- package/dist/bin.js +1063 -70
- package/dist/bridge-agent.d.ts +178 -5
- package/dist/index.js +468 -39
- package/dist/preflight-upgrade.d.cts +2 -0
- package/dist/preflight.cjs +31 -8
- package/dist/service/connection-file.d.ts +33 -0
- package/dist/service/fleet.d.ts +27 -0
- package/dist/service/harness-wire.d.ts +21 -0
- package/dist/service/install-prep.d.ts +5 -7
- package/dist/service/resolve-node.d.ts +13 -0
- package/dist/service/status.d.ts +25 -0
- package/dist/service/wire.d.ts +76 -0
- package/dist/ws-server.d.ts +1 -1
- package/package.json +5 -4
package/dist/index.js
CHANGED
|
@@ -1,3 +1,12 @@
|
|
|
1
|
+
// ../connect-core/dist/cadence.js
|
|
2
|
+
var SELF_HEAL_SWEEP_INTERVAL_MS = 9e4;
|
|
3
|
+
var SELF_HEAL_SWEEP_JITTER_MS = 3e4;
|
|
4
|
+
var REQUEST_WELCOME_COOLDOWN_MS = 12e4;
|
|
5
|
+
var WORST_CASE_RESIGNAL_MS = Math.ceil(REQUEST_WELCOME_COOLDOWN_MS / SELF_HEAL_SWEEP_INTERVAL_MS) * (SELF_HEAL_SWEEP_INTERVAL_MS + SELF_HEAL_SWEEP_JITTER_MS);
|
|
6
|
+
function isResignalDue(now, lastAt, cooldownMs = REQUEST_WELCOME_COOLDOWN_MS) {
|
|
7
|
+
return now - lastAt >= cooldownMs;
|
|
8
|
+
}
|
|
9
|
+
|
|
1
10
|
// ../connect-core/dist/envelope/types.js
|
|
2
11
|
var MESSAGE_TYPES = [
|
|
3
12
|
"text",
|
|
@@ -1326,6 +1335,23 @@ var MlsGroupKeyProvider = class {
|
|
|
1326
1335
|
* server delete-and-replaces. Keyed by deviceId (a provider has one), SEPARATE
|
|
1327
1336
|
* from `locks` (channel-keyed) so it never deadlocks against a held channel lock. */
|
|
1328
1337
|
publishLock = new MlsOperationQueue();
|
|
1338
|
+
/**
|
|
1339
|
+
* M6 (ADR-0013 Decision H): wall-clock ms of the last SUCCESSFUL decrypt per
|
|
1340
|
+
* channel. This is the only evidence-based liveness signal the member has —
|
|
1341
|
+
* every other membership indicator in the daemon (`joinState`, a persisted
|
|
1342
|
+
* store row, a leaf in the tree) is a structural artifact that a forked
|
|
1343
|
+
* channel keeps asserting long after it stopped being able to read anything.
|
|
1344
|
+
*
|
|
1345
|
+
* ⚠️ Deliberately written OUTSIDE the `errors` map, and written BEFORE
|
|
1346
|
+
* {@link recordDecryptSuccess}'s `if (!e) return` early-exit. That guard is
|
|
1347
|
+
* correct for its own purpose (there is no error run to clear for a channel
|
|
1348
|
+
* that never errored) but it means a healthy channel decrypting normally
|
|
1349
|
+
* never touches `errors` at all. Keying this timestamp off that map would
|
|
1350
|
+
* leave it null for exactly the channels that are working — a field that
|
|
1351
|
+
* reports "no successful decrypt" for a healthy channel is worse than no
|
|
1352
|
+
* field, because it inverts the signal it exists to give.
|
|
1353
|
+
*/
|
|
1354
|
+
lastDecryptAt = /* @__PURE__ */ new Map();
|
|
1329
1355
|
/**
|
|
1330
1356
|
* Per-channel decrypt-error / stuck tracker (ADR-0002). Bounded by the number
|
|
1331
1357
|
* of channels this device touches (channels are bounded; an entry is created
|
|
@@ -1390,6 +1416,7 @@ var MlsGroupKeyProvider = class {
|
|
|
1390
1416
|
* re-wedges (the onWedged guard is reset so the hook can fire again).
|
|
1391
1417
|
*/
|
|
1392
1418
|
recordDecryptSuccess(channelId) {
|
|
1419
|
+
this.lastDecryptAt.set(channelId, Date.now());
|
|
1393
1420
|
const e = this.errors.get(channelId);
|
|
1394
1421
|
if (!e)
|
|
1395
1422
|
return;
|
|
@@ -1537,7 +1564,9 @@ var MlsGroupKeyProvider = class {
|
|
|
1537
1564
|
* equals newDeviceId (the same device re-admitted).
|
|
1538
1565
|
*/
|
|
1539
1566
|
async admitGuestDevice(channelId, g) {
|
|
1540
|
-
await this.addMemberToLiveGroup(channelId, g.newDeviceId
|
|
1567
|
+
await this.addMemberToLiveGroup(channelId, g.newDeviceId, {
|
|
1568
|
+
knownLeafDevices: g.priorDeviceId ? [g.priorDeviceId] : []
|
|
1569
|
+
});
|
|
1541
1570
|
if (g.priorDeviceId && g.priorDeviceId !== g.newDeviceId) {
|
|
1542
1571
|
const orderedDeviceIds = g.orderedDeviceIds ?? [g.priorDeviceId, g.newDeviceId];
|
|
1543
1572
|
await this.removeDeviceFromLiveGroup(channelId, g.priorDeviceId, {
|
|
@@ -1562,6 +1591,85 @@ var MlsGroupKeyProvider = class {
|
|
|
1562
1591
|
async currentEpoch(channelId) {
|
|
1563
1592
|
return Number((await this.manager(channelId)).epoch);
|
|
1564
1593
|
}
|
|
1594
|
+
/**
|
|
1595
|
+
* Converge a BEHIND member in place by replaying the commits it missed,
|
|
1596
|
+
* instead of tearing down its session and being re-admitted.
|
|
1597
|
+
*
|
|
1598
|
+
* Ported from gen2 (#440). This is the primitive SecureComs was missing, and
|
|
1599
|
+
* its absence is why divergence here has been effectively unrecoverable: the
|
|
1600
|
+
* only client-side route back was Remove+Add, which gen2 treats as the
|
|
1601
|
+
* LAST RESORT and rate-limits hard, because they measured it as a divergence
|
|
1602
|
+
* SOURCE (#360 — epoch 1204 in ~6 minutes; "the re-admitter's churn IS the
|
|
1603
|
+
* bug, not the member").
|
|
1604
|
+
*
|
|
1605
|
+
* Replay is strictly cheaper and safer than re-admission:
|
|
1606
|
+
* - no re-key, no roster change, no epoch movement at all
|
|
1607
|
+
* - it cannot storm the group, because it only READS
|
|
1608
|
+
* - it preserves the member's leaf, so it cannot create a duplicate
|
|
1609
|
+
*
|
|
1610
|
+
* The server already implements the whole contract
|
|
1611
|
+
* (`GET /mls/groups/{id}/sync?last_epoch=N` → `{action, messages}`) and has
|
|
1612
|
+
* since before this change. Nothing consumed it. That is the entire defect.
|
|
1613
|
+
*
|
|
1614
|
+
* Returns true when the member converged to the server's epoch. Returns false
|
|
1615
|
+
* when the caller should fall back to re-admission — the gap exceeded the
|
|
1616
|
+
* server's REJOIN_THRESHOLD, there was nothing to apply, or a commit would
|
|
1617
|
+
* not apply (which means the state is genuinely FORKED, not merely behind,
|
|
1618
|
+
* and replay cannot fix a fork).
|
|
1619
|
+
*/
|
|
1620
|
+
async resyncByReplay(channelId, groupId) {
|
|
1621
|
+
if (!await this.hasPersistedChannelState(channelId))
|
|
1622
|
+
return false;
|
|
1623
|
+
let from;
|
|
1624
|
+
try {
|
|
1625
|
+
from = await this.currentEpoch(channelId);
|
|
1626
|
+
} catch {
|
|
1627
|
+
return false;
|
|
1628
|
+
}
|
|
1629
|
+
const resp = await this.o.backend.syncGroup(groupId, from, this.o.deviceId);
|
|
1630
|
+
if (resp.action !== "replay")
|
|
1631
|
+
return false;
|
|
1632
|
+
const commits = resp.messages.filter((m) => m.message_type === "commit").sort((a, b) => a.epoch - b.epoch);
|
|
1633
|
+
if (commits.length === 0)
|
|
1634
|
+
return false;
|
|
1635
|
+
let applied = 0;
|
|
1636
|
+
try {
|
|
1637
|
+
await this.locks.run(channelId, async () => {
|
|
1638
|
+
const mgr = await this.manager(channelId);
|
|
1639
|
+
for (const c of commits) {
|
|
1640
|
+
if (c.epoch <= Number(mgr.epoch))
|
|
1641
|
+
continue;
|
|
1642
|
+
await mgr.decrypt(hexToBytes(c.payload));
|
|
1643
|
+
applied++;
|
|
1644
|
+
}
|
|
1645
|
+
if (applied > 0)
|
|
1646
|
+
await this.o.store.save(channelId, mgr.exportState());
|
|
1647
|
+
});
|
|
1648
|
+
} catch (e) {
|
|
1649
|
+
console.warn(`[mls] resyncByReplay ${channelId.slice(0, 8)}: commit failed to apply after ${applied} \u2014 state is forked, not behind:`, e?.message);
|
|
1650
|
+
return false;
|
|
1651
|
+
}
|
|
1652
|
+
if (applied > 0) {
|
|
1653
|
+
this.recordDecryptSuccess(channelId);
|
|
1654
|
+
}
|
|
1655
|
+
return applied > 0;
|
|
1656
|
+
}
|
|
1657
|
+
/**
|
|
1658
|
+
* M6 (ADR-0013 Decision H): wall-clock ms of the last successful decrypt for
|
|
1659
|
+
* `channelId`, or `undefined` if this process has never decrypted anything on
|
|
1660
|
+
* it. Synchronous and allocation-free — it is read from the `/healthz` path,
|
|
1661
|
+
* which must never fan out work, because the conditions it exists to report
|
|
1662
|
+
* are exactly the conditions under which fanned-out work hangs.
|
|
1663
|
+
*
|
|
1664
|
+
* `undefined` is genuinely ambiguous and callers MUST NOT read it as
|
|
1665
|
+
* "broken": it also covers a freshly-restarted daemon and a healthy channel
|
|
1666
|
+
* that has simply been quiet. It is evidence of liveness when present, never
|
|
1667
|
+
* evidence of divergence when absent. Divergence is decided by the held-vs-
|
|
1668
|
+
* server epoch comparison, not by this field.
|
|
1669
|
+
*/
|
|
1670
|
+
lastDecryptAtFor(channelId) {
|
|
1671
|
+
return this.lastDecryptAt.get(channelId);
|
|
1672
|
+
}
|
|
1565
1673
|
/**
|
|
1566
1674
|
* Run a committer op and, on ANY throw that escapes it, evict the channel's
|
|
1567
1675
|
* cached MLSGroupManager.
|
|
@@ -1785,7 +1893,7 @@ var MlsGroupKeyProvider = class {
|
|
|
1785
1893
|
if (!bundle)
|
|
1786
1894
|
throw new Error("ensureSession: call publishKeyPackage() first");
|
|
1787
1895
|
if (opts.forceReinit) {
|
|
1788
|
-
const others = opts.otherDeviceIds ?? [];
|
|
1896
|
+
const others = [...new Set(opts.otherDeviceIds ?? [])].filter((d) => d !== this.o.deviceId);
|
|
1789
1897
|
const reinit = await this.o.backend.reinitConnectChannelGroup(channelId, [this.o.deviceId, ...others]);
|
|
1790
1898
|
const mgr = new MLSGroupManager();
|
|
1791
1899
|
await mgr.createGroup(new TextEncoder().encode(reinit.group_id), bundle);
|
|
@@ -1815,7 +1923,7 @@ var MlsGroupKeyProvider = class {
|
|
|
1815
1923
|
throw new BootstrapRaceLostError();
|
|
1816
1924
|
const mgr = new MLSGroupManager();
|
|
1817
1925
|
await mgr.createGroup(new TextEncoder().encode(init.group_id), bundle);
|
|
1818
|
-
const others = opts.otherDeviceIds ?? [];
|
|
1926
|
+
const others = [...new Set(opts.otherDeviceIds ?? [])].filter((d) => d !== this.o.deviceId);
|
|
1819
1927
|
const kps = await this.o.backend.fetchKeyPackages(others);
|
|
1820
1928
|
const pkgs = others.map((d) => {
|
|
1821
1929
|
const hex = kps[d];
|
|
@@ -1866,17 +1974,38 @@ var MlsGroupKeyProvider = class {
|
|
|
1866
1974
|
* full member roster (unioning in any device ids we pass). That avoids a
|
|
1867
1975
|
* parallel group_id-by-channel resolver.
|
|
1868
1976
|
*/
|
|
1869
|
-
async addMemberToLiveGroup(channelId, newDeviceId) {
|
|
1870
|
-
return this.locks.run(channelId, () => this.evictOnThrow(channelId, () => this.addMemberToLiveGroupUnlocked(channelId, newDeviceId, 0)));
|
|
1977
|
+
async addMemberToLiveGroup(channelId, newDeviceId, opts) {
|
|
1978
|
+
return this.locks.run(channelId, () => this.evictOnThrow(channelId, () => this.addMemberToLiveGroupUnlocked(channelId, newDeviceId, 0, opts)));
|
|
1871
1979
|
}
|
|
1872
1980
|
async addMemberToLiveGroupUnlocked(channelId, newDeviceId, _attempt, opts) {
|
|
1873
1981
|
const mgr = await this.manager(channelId);
|
|
1874
|
-
const init = await this.o.backend.initConnectChannelGroup(channelId, [this.o.deviceId, newDeviceId]);
|
|
1875
1982
|
const kps = await this.o.backend.fetchKeyPackages([newDeviceId]);
|
|
1876
1983
|
const hex = kps[newDeviceId];
|
|
1877
1984
|
if (!hex)
|
|
1878
1985
|
throw new Error(`addMemberToLiveGroup: no published KeyPackage for device ${newDeviceId}`);
|
|
1879
1986
|
const pkg = MLSGroupManager.deserializeKeyPackage(hexToBytes(hex));
|
|
1987
|
+
if (_attempt === 0) {
|
|
1988
|
+
let adds;
|
|
1989
|
+
try {
|
|
1990
|
+
adds = (await this.o.backend.getCommitterWork(channelId)).adds;
|
|
1991
|
+
} catch (e) {
|
|
1992
|
+
throw new Error(`addMemberToLiveGroup: committer-work unreachable for channel ${channelId} \u2014 refusing plain Add for device ${newDeviceId} (fail closed): ` + (e instanceof Error ? e.message : String(e)));
|
|
1993
|
+
}
|
|
1994
|
+
if (!adds.includes(newDeviceId)) {
|
|
1995
|
+
const cred = pkg.leafNode.credential;
|
|
1996
|
+
const claimed = cred.credentialType === "basic" ? new TextDecoder().decode(cred.identity) : null;
|
|
1997
|
+
const leaves = claimed === null ? null : mgr.leafIndicesForIdentity(claimed);
|
|
1998
|
+
if (leaves === null || leaves.length > 0) {
|
|
1999
|
+
const known = opts?.knownLeafDevices ? [...new Set(opts.knownLeafDevices)] : null;
|
|
2000
|
+
const swapSafe = leaves !== null && known !== null && !known.includes(newDeviceId) && known.length === leaves.length;
|
|
2001
|
+
if (!swapSafe) {
|
|
2002
|
+
console.warn(`addMemberToLiveGroup: device ${newDeviceId.slice(0, 8)} is outside the committer-work add-delta and its identity already holds a leaf \u2014 terminal skip (no commit)`);
|
|
2003
|
+
return;
|
|
2004
|
+
}
|
|
2005
|
+
}
|
|
2006
|
+
}
|
|
2007
|
+
}
|
|
2008
|
+
const init = await this.o.backend.initConnectChannelGroup(channelId, [this.o.deviceId, newDeviceId]);
|
|
1880
2009
|
if (opts?.expectedIdentityId) {
|
|
1881
2010
|
const cred = pkg.leafNode.credential;
|
|
1882
2011
|
const claimed = cred.credentialType === "basic" ? new TextDecoder().decode(cred.identity) : "";
|
|
@@ -2215,6 +2344,7 @@ var MlsGroupKeyProvider = class {
|
|
|
2215
2344
|
await this.o.store.delete(channelId);
|
|
2216
2345
|
await this.o.store.discardStaging(channelId);
|
|
2217
2346
|
this.errors.delete(channelId);
|
|
2347
|
+
this.lastDecryptAt.delete(channelId);
|
|
2218
2348
|
});
|
|
2219
2349
|
}
|
|
2220
2350
|
/** Owner-attested reset supersede (owner/web side, Task 9). Mirrors the
|
|
@@ -2280,6 +2410,25 @@ var MlsGroupKeyProvider = class {
|
|
|
2280
2410
|
return true;
|
|
2281
2411
|
return await this.o.store.load(channelId) !== null;
|
|
2282
2412
|
}
|
|
2413
|
+
/**
|
|
2414
|
+
* Returns the group id our persisted (or live in-memory) MLS state holds for
|
|
2415
|
+
* `channelId`, or null when there is no state to read, or when the state
|
|
2416
|
+
* exists but a loaded manager can't be constructed from it. Read-only —
|
|
2417
|
+
* `manager()` only loads from the store and caches in memory; it makes no
|
|
2418
|
+
* backend call and writes nothing to the store. Mirrors the same
|
|
2419
|
+
* held-vs-row comparison the Welcome receive path already does inline
|
|
2420
|
+
* (see the `heldGroupId` decode in the `message_type === "welcome"` branch
|
|
2421
|
+
* of `receiveOnce`) so callers can ask "is my held state for THIS group?"
|
|
2422
|
+
* without duplicating that decode logic.
|
|
2423
|
+
*/
|
|
2424
|
+
async heldGroupIdFor(channelId) {
|
|
2425
|
+
try {
|
|
2426
|
+
const mgr = await this.manager(channelId);
|
|
2427
|
+
return new TextDecoder().decode(mgr.groupId);
|
|
2428
|
+
} catch {
|
|
2429
|
+
return null;
|
|
2430
|
+
}
|
|
2431
|
+
}
|
|
2283
2432
|
/**
|
|
2284
2433
|
* Return the number of leaves the given identity holds in the live tree for
|
|
2285
2434
|
* `channelId`. Loads the manager from the store if not already cached.
|
|
@@ -3097,6 +3246,22 @@ var RestMlsBackend = class {
|
|
|
3097
3246
|
});
|
|
3098
3247
|
return String(j.id);
|
|
3099
3248
|
}
|
|
3249
|
+
async syncGroup(groupId, lastEpoch, deviceId) {
|
|
3250
|
+
const deviceParam = deviceId ? `&device_id=${encodeURIComponent(deviceId)}` : "";
|
|
3251
|
+
const j = await this.get(`/api/v1/mls/groups/${groupId}/sync?last_epoch=${encodeURIComponent(String(lastEpoch))}${deviceParam}`);
|
|
3252
|
+
const action = j?.action === "rejoin" ? "rejoin" : "replay";
|
|
3253
|
+
const messages = Array.isArray(j?.messages) ? j.messages : [];
|
|
3254
|
+
return {
|
|
3255
|
+
action,
|
|
3256
|
+
messages: messages.map((m) => ({
|
|
3257
|
+
id: String(m.id),
|
|
3258
|
+
epoch: Number(m.epoch),
|
|
3259
|
+
message_type: String(m.message_type),
|
|
3260
|
+
sender_device_id: String(m.sender_device_id ?? ""),
|
|
3261
|
+
payload: String(m.payload ?? "")
|
|
3262
|
+
}))
|
|
3263
|
+
};
|
|
3264
|
+
}
|
|
3100
3265
|
async getConnectChannelGroup(channelId) {
|
|
3101
3266
|
const j = await this.get(`/api/v1/mls/connect-channels/${channelId}/group`);
|
|
3102
3267
|
return {
|
|
@@ -4286,6 +4451,22 @@ var InvalidPathError = class extends Error {
|
|
|
4286
4451
|
if (cause !== void 0) this.cause = cause;
|
|
4287
4452
|
}
|
|
4288
4453
|
};
|
|
4454
|
+
var InvalidMessageIdError = class extends Error {
|
|
4455
|
+
constructor(messageId) {
|
|
4456
|
+
super(`message_id must be a UUID, got: ${messageId}`);
|
|
4457
|
+
this.messageId = messageId;
|
|
4458
|
+
this.name = "InvalidMessageIdError";
|
|
4459
|
+
}
|
|
4460
|
+
};
|
|
4461
|
+
var UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
4462
|
+
function isUuid(s) {
|
|
4463
|
+
return UUID_RE.test(s);
|
|
4464
|
+
}
|
|
4465
|
+
function resolveMessageId(messageId) {
|
|
4466
|
+
if (messageId === void 0) return crypto.randomUUID();
|
|
4467
|
+
if (!isUuid(messageId)) throw new InvalidMessageIdError(messageId);
|
|
4468
|
+
return messageId;
|
|
4469
|
+
}
|
|
4289
4470
|
var AgentPausedSendError = class extends Error {
|
|
4290
4471
|
constructor(channelId) {
|
|
4291
4472
|
super(`agent is paused in channel ${channelId}; send suppressed (no self-heal, no retry)`);
|
|
@@ -4486,10 +4667,75 @@ var BridgeAgent = class _BridgeAgent {
|
|
|
4486
4667
|
* counter is incremented. Surfaced on health()/metrics().
|
|
4487
4668
|
*/
|
|
4488
4669
|
failedAttachments = 0;
|
|
4670
|
+
/**
|
|
4671
|
+
* Task 8: count of live AUTHENTICATED /v1/ws sessions, so a later `status`
|
|
4672
|
+
* verb can tell "bridge process up" (the app's green dot, which only
|
|
4673
|
+
* proves bridge<->backend) apart from "a harness is actually attached"
|
|
4674
|
+
* (a real consumer holding an authenticated session). Only ws-server.ts
|
|
4675
|
+
* mutates this, and only after authedUpgrade() passes — an unauthenticated
|
|
4676
|
+
* socket, or the HTTP /healthz probe, must never move this counter.
|
|
4677
|
+
* Surfaced on health()/metrics().
|
|
4678
|
+
*/
|
|
4679
|
+
authSessions = 0;
|
|
4680
|
+
/**
|
|
4681
|
+
* Task 3 (audit M3): count of newest-wins evictions of a prior /v1/ws
|
|
4682
|
+
* session. The daemon's WS is single-consumer newest-wins, so
|
|
4683
|
+
* `agent_sessions` alone can stay >=1 while two connectors flap the same
|
|
4684
|
+
* identity in and out — this counter makes that split-brain visible.
|
|
4685
|
+
* Only ws-server.ts mutates this, via noteSessionEvicted(). Surfaced on
|
|
4686
|
+
* health()/status.
|
|
4687
|
+
*/
|
|
4688
|
+
evictions = 0;
|
|
4489
4689
|
/** Per-channel timestamp of last requestWelcome call, for cooldown enforcement. */
|
|
4490
4690
|
lastWelcomeRequest = /* @__PURE__ */ new Map();
|
|
4491
4691
|
/** Minimum gap between requestWelcome calls for the same channel (2 minutes). */
|
|
4492
|
-
|
|
4692
|
+
/**
|
|
4693
|
+
* M6 (ADR-0013 Decision H): the sweep's per-channel observation cache, and the
|
|
4694
|
+
* only thing {@link health} reports divergence from.
|
|
4695
|
+
*
|
|
4696
|
+
* Written by `selfHealSweep` in a `finally`, read synchronously by `health()`.
|
|
4697
|
+
* `health()` must NEVER fan out network calls of its own: a health endpoint
|
|
4698
|
+
* that does I/O hangs under exactly the conditions it exists to report, and
|
|
4699
|
+
* the conditions here are "the delivery service is unreachable or lying".
|
|
4700
|
+
*/
|
|
4701
|
+
observations = /* @__PURE__ */ new Map();
|
|
4702
|
+
/**
|
|
4703
|
+
* A3 (dr-evil must-fix #3): true when the sweep found the provider missing a
|
|
4704
|
+
* required capability (stale SDK dist under a newer bridge dist) and ABORTED.
|
|
4705
|
+
* Folded into `observations_stale` (and therefore `degraded`) on health():
|
|
4706
|
+
* a bridge that cannot heal must never present trustworthy-looking data.
|
|
4707
|
+
*/
|
|
4708
|
+
providerCapabilityMissing = false;
|
|
4709
|
+
/**
|
|
4710
|
+
* ADR-0013 Decision D: the wipe's OWN anti-thrash bound, deliberately
|
|
4711
|
+
* separate from {@link REQUEST_WELCOME_COOLDOWN_MS} and separately keyed.
|
|
4712
|
+
*
|
|
4713
|
+
* Two bounds exist because they protect different things and must never
|
|
4714
|
+
* share a gate. The request cooldown protects the single committer from
|
|
4715
|
+
* churn; deferring an ask costs a little latency. The wipe bound only stops
|
|
4716
|
+
* the member thrashing its own session; skipping a wipe costs the member its
|
|
4717
|
+
* ONLY route back, permanently. Sharing one map is what starved the repair.
|
|
4718
|
+
*/
|
|
4719
|
+
static WIPE_ANTI_THRASH_MS = 6e5;
|
|
4720
|
+
/**
|
|
4721
|
+
* ADR-0013 Decision A (wipe-first): the SHORTER re-wipe floor used only when
|
|
4722
|
+
* the fork is PROVEN (the self-heal sweep's resyncByReplay returned false).
|
|
4723
|
+
*
|
|
4724
|
+
* The 600s WIPE_ANTI_THRASH_MS is the right duty cycle for the reactive
|
|
4725
|
+
* onWedged path, but it is a TRAP for a proven-forked member: while the member
|
|
4726
|
+
* holds forked state the owner's re-Welcome is stray-acked (mls.ts, rowGroup ==
|
|
4727
|
+
* heldGroup) and destroyed, so the member never returns to no-state and the
|
|
4728
|
+
* Welcome never lands via the first-join path. Suppressing the re-wipe for a
|
|
4729
|
+
* full 10 minutes means every owner Welcome in that window is destroyed — the
|
|
4730
|
+
* observed hermes flap. This floor lets a proven-forked member re-wipe promptly
|
|
4731
|
+
* (return to no-state so the next Welcome opens) while still bounding the loop:
|
|
4732
|
+
* ~one 25s member ingest poll + margin, which preserves Decision D ("cannot
|
|
4733
|
+
* loop") because two wipes still cannot fire inside a single poll interval.
|
|
4734
|
+
*/
|
|
4735
|
+
static PROVEN_FORK_REWIPE_FLOOR_MS = 3e4;
|
|
4736
|
+
/** Per-channel timestamp of the last state repudiation (Decision D). */
|
|
4737
|
+
lastWipe = /* @__PURE__ */ new Map();
|
|
4738
|
+
static REQUEST_WELCOME_COOLDOWN_MS = REQUEST_WELCOME_COOLDOWN_MS;
|
|
4493
4739
|
// H1: bound the graceful-shutdown drain so a hung in-flight op can't block exit.
|
|
4494
4740
|
static GRACEFUL_DRAIN_MS = 5e3;
|
|
4495
4741
|
/**
|
|
@@ -4511,13 +4757,13 @@ var BridgeAgent = class _BridgeAgent {
|
|
|
4511
4757
|
*/
|
|
4512
4758
|
selfHealSweepEnabled = process.env.SECURECOMS_SELF_HEAL_SWEEP !== "false";
|
|
4513
4759
|
/** Task 12: proactive sweep cadence (in addition to the once-at-boot fire). */
|
|
4514
|
-
static SELF_HEAL_SWEEP_INTERVAL_MS =
|
|
4760
|
+
static SELF_HEAL_SWEEP_INTERVAL_MS = SELF_HEAL_SWEEP_INTERVAL_MS;
|
|
4515
4761
|
// C3: add up to this much random jitter to BOTH the boot sweep delay and each
|
|
4516
4762
|
// periodic interval, so a fleet of daemons restarting/sweeping together does not
|
|
4517
4763
|
// fire re-Welcome signals at the SAME instant and thundering-herd the single
|
|
4518
4764
|
// committer (the observed 409 commit-lease saturation). Decorrelation only —
|
|
4519
4765
|
// C1 already removed the restart KP-rotation storm that caused the herd.
|
|
4520
|
-
static SELF_HEAL_SWEEP_JITTER_MS =
|
|
4766
|
+
static SELF_HEAL_SWEEP_JITTER_MS = SELF_HEAL_SWEEP_JITTER_MS;
|
|
4521
4767
|
selfHealTimer;
|
|
4522
4768
|
bootSweepTimer;
|
|
4523
4769
|
rng;
|
|
@@ -4656,7 +4902,8 @@ var BridgeAgent = class _BridgeAgent {
|
|
|
4656
4902
|
await this.provider.joinAndReconcile(channelId, this.cfg.enclaveId);
|
|
4657
4903
|
this.joinState.set(channelId, "JOINED");
|
|
4658
4904
|
return true;
|
|
4659
|
-
} catch {
|
|
4905
|
+
} catch (e) {
|
|
4906
|
+
console.warn(`[bridge] reconcileJoin ${channelId} failed:`, e);
|
|
4660
4907
|
this.joinState.set(channelId, "PENDING");
|
|
4661
4908
|
return false;
|
|
4662
4909
|
}
|
|
@@ -4714,20 +4961,40 @@ var BridgeAgent = class _BridgeAgent {
|
|
|
4714
4961
|
* SDK's onWedged caller. Single-shot per wedge episode — full rejoin
|
|
4715
4962
|
* reliability is P1, not this hook's job.
|
|
4716
4963
|
*/
|
|
4717
|
-
async healWedgedChannel(channelId) {
|
|
4964
|
+
async healWedgedChannel(channelId, provenForked = false) {
|
|
4718
4965
|
try {
|
|
4719
4966
|
const now = performance.now();
|
|
4720
|
-
const last = this.lastWelcomeRequest.get(channelId) ?? -Infinity;
|
|
4721
|
-
if (now - last < _BridgeAgent.REQUEST_WELCOME_COOLDOWN_MS) return;
|
|
4722
4967
|
const g = await this.backend.getConnectChannelGroup(channelId);
|
|
4723
|
-
if (!g?.group_id) return;
|
|
4724
|
-
this.
|
|
4725
|
-
|
|
4726
|
-
|
|
4727
|
-
|
|
4968
|
+
if (!g?.group_id) return false;
|
|
4969
|
+
const lastWipe = this.lastWipe.get(channelId) ?? -Infinity;
|
|
4970
|
+
const floor = provenForked ? _BridgeAgent.PROVEN_FORK_REWIPE_FLOOR_MS : _BridgeAgent.WIPE_ANTI_THRASH_MS;
|
|
4971
|
+
let repudiated = false;
|
|
4972
|
+
if (now - lastWipe >= floor) {
|
|
4973
|
+
if (await this.provider.hasPersistedChannelState(channelId)) {
|
|
4974
|
+
await this.provider.resetChannelForRejoin(channelId);
|
|
4975
|
+
repudiated = true;
|
|
4976
|
+
}
|
|
4977
|
+
this.lastWipe.set(channelId, now);
|
|
4978
|
+
} else {
|
|
4979
|
+
console.warn(
|
|
4980
|
+
`[bridge] healWedgedChannel ${channelId}: wipe suppressed by ${provenForked ? "proven-fork re-wipe floor" : "anti-thrash bound"} (${Math.round((floor - (now - lastWipe)) / 1e3)}s remaining) \u2014 NOT repudiated`
|
|
4981
|
+
);
|
|
4982
|
+
}
|
|
4983
|
+
this.joinState.delete(channelId);
|
|
4984
|
+
if (!repudiated && await this.provider.hasPersistedChannelState(channelId)) {
|
|
4985
|
+
console.warn(
|
|
4986
|
+
`[bridge] healWedgedChannel ${channelId}: wipe suppressed and state still held \u2014 NOT requesting (the answer would be stray-acked and would cost the group two commits)`
|
|
4987
|
+
);
|
|
4988
|
+
return false;
|
|
4728
4989
|
}
|
|
4990
|
+
const last = this.lastWelcomeRequest.get(channelId) ?? -Infinity;
|
|
4991
|
+
if (!isResignalDue(now, last, _BridgeAgent.REQUEST_WELCOME_COOLDOWN_MS)) return repudiated;
|
|
4992
|
+
this.lastWelcomeRequest.set(channelId, now);
|
|
4729
4993
|
await this.signalHeldKpWelcome(g.group_id);
|
|
4730
|
-
|
|
4994
|
+
return repudiated;
|
|
4995
|
+
} catch (e) {
|
|
4996
|
+
console.warn(`[bridge] healWedgedChannel ${channelId} failed:`, e);
|
|
4997
|
+
return false;
|
|
4731
4998
|
}
|
|
4732
4999
|
}
|
|
4733
5000
|
/**
|
|
@@ -4749,7 +5016,7 @@ var BridgeAgent = class _BridgeAgent {
|
|
|
4749
5016
|
try {
|
|
4750
5017
|
const now = performance.now();
|
|
4751
5018
|
const last = this.lastWelcomeRequest.get(channelId) ?? -Infinity;
|
|
4752
|
-
if (now
|
|
5019
|
+
if (!isResignalDue(now, last, _BridgeAgent.REQUEST_WELCOME_COOLDOWN_MS)) return false;
|
|
4753
5020
|
this.lastWelcomeRequest.set(channelId, now);
|
|
4754
5021
|
await this.signalHeldKpWelcome(groupId);
|
|
4755
5022
|
return true;
|
|
@@ -4757,6 +5024,19 @@ var BridgeAgent = class _BridgeAgent {
|
|
|
4757
5024
|
return false;
|
|
4758
5025
|
}
|
|
4759
5026
|
}
|
|
5027
|
+
/**
|
|
5028
|
+
* Task 2 (MLS convergence slice): true when we hold persisted state for
|
|
5029
|
+
* `channelId` whose group id does NOT match `serverGroupId` — the group
|
|
5030
|
+
* the server currently says is live for that channel. Holding SOME state
|
|
5031
|
+
* is not proof of health; it is proof of health only for the group the
|
|
5032
|
+
* server currently attests. A null held group id (state present but the
|
|
5033
|
+
* manager can't be loaded from it) is treated as diverged too — we can't
|
|
5034
|
+
* confirm health, so we ask rather than silently trusting stale state.
|
|
5035
|
+
*/
|
|
5036
|
+
async isChannelDiverged(channelId, serverGroupId) {
|
|
5037
|
+
const held = await this.provider.heldGroupIdFor(channelId);
|
|
5038
|
+
return held !== serverGroupId;
|
|
5039
|
+
}
|
|
4760
5040
|
/**
|
|
4761
5041
|
* Task 12: proactive self-heal sweep. ingestOnce's Task-11 divergence
|
|
4762
5042
|
* branch only fires reactively — off a delivery ROW that already arrived
|
|
@@ -4765,11 +5045,22 @@ var BridgeAgent = class _BridgeAgent {
|
|
|
4765
5045
|
* first thing lost. This sweep walks every member channel — rooms AND DMs
|
|
4766
5046
|
* (H2: a wiped, idle DM with no traffic since the wipe otherwise never
|
|
4767
5047
|
* proactively re-Welcomes, so it stays dark until the owner happens to send)
|
|
4768
|
-
* — fired once at boot + on SELF_HEAL_SWEEP_INTERVAL_MS
|
|
4769
|
-
*
|
|
4770
|
-
*
|
|
4771
|
-
*
|
|
4772
|
-
* the
|
|
5048
|
+
* — fired once at boot + on SELF_HEAL_SWEEP_INTERVAL_MS.
|
|
5049
|
+
*
|
|
5050
|
+
* Task 2: holding persisted state is NOT proof of health by itself — it is
|
|
5051
|
+
* proof of health only for the group the server currently says is live for
|
|
5052
|
+
* the channel. The prior rule ("I have persisted state ⇒ healthy ⇒ don't
|
|
5053
|
+
* ask") silently stopped filing welcome-requests for any channel that had
|
|
5054
|
+
* been superseded (e.g. a server-side reset/rotation) while the member
|
|
5055
|
+
* still held state for the OLD group: it could hold state, hold the WRONG
|
|
5056
|
+
* state, and never say anything. So the server's current group_id is
|
|
5057
|
+
* fetched FIRST via the READ-ONLY getConnectChannelGroup probe (never
|
|
5058
|
+
* initConnectChannelGroup, which would side-effect a member_device_ids
|
|
5059
|
+
* union + commit), then the branch is: persisted state whose group id
|
|
5060
|
+
* matches the server's current group ⇒ healthy, skip; persisted state for
|
|
5061
|
+
* a STALE group id ⇒ diverged, signal; no persisted state at all ⇒
|
|
5062
|
+
* reconcile-first (consume an already-queued Welcome) before concluding
|
|
5063
|
+
* divergence. All three paths go through the same per-channel cooldown.
|
|
4773
5064
|
*
|
|
4774
5065
|
* Each channel is wrapped in its own try/catch: `listChannels` filters at
|
|
4775
5066
|
* the identity level while `requestWelcome` (via request_welcome) gates at
|
|
@@ -4778,25 +5069,114 @@ var BridgeAgent = class _BridgeAgent {
|
|
|
4778
5069
|
* must not abort the sweep for every other channel.
|
|
4779
5070
|
*/
|
|
4780
5071
|
async selfHealSweep() {
|
|
5072
|
+
if (typeof this.provider.resyncByReplay !== "function" || typeof this.provider.heldGroupIdFor !== "function") {
|
|
5073
|
+
this.providerCapabilityMissing = true;
|
|
5074
|
+
console.error(
|
|
5075
|
+
"[bridge] selfHealSweep ABORTED: provider is missing required capabilities (resyncByReplay/heldGroupIdFor) \u2014 stale SDK dist? Rebuild SDK dist THEN bridge dist. Healing is NOT running; observations are reported STALE."
|
|
5076
|
+
);
|
|
5077
|
+
return;
|
|
5078
|
+
}
|
|
5079
|
+
this.providerCapabilityMissing = false;
|
|
4781
5080
|
let chans;
|
|
4782
5081
|
try {
|
|
4783
5082
|
chans = await this.listChannels();
|
|
4784
|
-
} catch {
|
|
5083
|
+
} catch (e) {
|
|
5084
|
+
console.warn("[bridge] selfHealSweep aborted: listChannels failed:", e);
|
|
4785
5085
|
return;
|
|
4786
5086
|
}
|
|
4787
5087
|
for (const c of chans) {
|
|
5088
|
+
const obs = {
|
|
5089
|
+
channel_id: c.channel_id,
|
|
5090
|
+
state: "probe_failed",
|
|
5091
|
+
held_group_id: null,
|
|
5092
|
+
server_group_id: null,
|
|
5093
|
+
held_epoch: null,
|
|
5094
|
+
server_epoch: null,
|
|
5095
|
+
diverged: null,
|
|
5096
|
+
last_decrypt_at: this.provider.lastDecryptAtFor?.(c.channel_id) ?? null,
|
|
5097
|
+
observed_at: Date.now()
|
|
5098
|
+
};
|
|
4788
5099
|
try {
|
|
4789
|
-
if (await this.provider.hasPersistedChannelState(c.channel_id)) continue;
|
|
4790
|
-
await this.reconcileJoin(c.channel_id);
|
|
4791
|
-
if (await this.provider.hasPersistedChannelState(c.channel_id)) continue;
|
|
4792
5100
|
const g = await this.backend.getConnectChannelGroup(c.channel_id);
|
|
4793
|
-
|
|
5101
|
+
obs.server_group_id = g?.group_id ?? null;
|
|
5102
|
+
obs.server_epoch = g?.current_epoch ?? null;
|
|
5103
|
+
if (!g?.group_id) {
|
|
5104
|
+
obs.state = "group_unresolvable";
|
|
5105
|
+
continue;
|
|
5106
|
+
}
|
|
5107
|
+
const hasState = await this.provider.hasPersistedChannelState(c.channel_id);
|
|
5108
|
+
if (hasState) {
|
|
5109
|
+
obs.held_group_id = await this.provider.heldGroupIdFor(c.channel_id).catch(() => null);
|
|
5110
|
+
obs.held_epoch = await this.provider.currentEpoch(c.channel_id).catch(() => null);
|
|
5111
|
+
if (!await this.isChannelDiverged(c.channel_id, g.group_id)) {
|
|
5112
|
+
if (obs.held_epoch === null) {
|
|
5113
|
+
obs.state = "probe_failed";
|
|
5114
|
+
obs.diverged = null;
|
|
5115
|
+
continue;
|
|
5116
|
+
}
|
|
5117
|
+
const behind = typeof obs.held_epoch === "number" && typeof obs.server_epoch === "number" && obs.held_epoch < obs.server_epoch;
|
|
5118
|
+
obs.state = behind ? "behind" : "healthy";
|
|
5119
|
+
obs.diverged = behind;
|
|
5120
|
+
if (behind) {
|
|
5121
|
+
const prev = this.observations.get(c.channel_id);
|
|
5122
|
+
const notAdvancing = prev != null && prev.held_epoch === obs.held_epoch;
|
|
5123
|
+
if (notAdvancing) {
|
|
5124
|
+
obs.state = "stuck";
|
|
5125
|
+
if (await this.provider.resyncByReplay(c.channel_id, g.group_id)) {
|
|
5126
|
+
obs.held_epoch = await this.provider.currentEpoch(c.channel_id).catch(() => null);
|
|
5127
|
+
obs.last_decrypt_at = this.provider.lastDecryptAtFor?.(c.channel_id) ?? null;
|
|
5128
|
+
if (obs.held_epoch === null) {
|
|
5129
|
+
obs.state = "probe_failed";
|
|
5130
|
+
obs.diverged = null;
|
|
5131
|
+
continue;
|
|
5132
|
+
}
|
|
5133
|
+
obs.state = "healthy";
|
|
5134
|
+
obs.diverged = false;
|
|
5135
|
+
console.warn(
|
|
5136
|
+
`[bridge] selfHealSweep: ${c.channel_id} caught up by REPLAY to epoch ${obs.held_epoch} (no re-admit, no epoch churn)`
|
|
5137
|
+
);
|
|
5138
|
+
continue;
|
|
5139
|
+
}
|
|
5140
|
+
const repudiated = await this.healWedgedChannel(c.channel_id, true);
|
|
5141
|
+
console.warn(
|
|
5142
|
+
`[bridge] selfHealSweep: ${c.channel_id} STUCK at epoch ${obs.held_epoch} (server=${obs.server_epoch}) across two sweeps \u2014 ${repudiated ? "repudiated and re-requested" : "repair SUPPRESSED (anti-thrash); state unchanged"}`
|
|
5143
|
+
);
|
|
5144
|
+
}
|
|
5145
|
+
continue;
|
|
5146
|
+
}
|
|
5147
|
+
console.debug(
|
|
5148
|
+
`[bridge] selfHealSweep: ${c.channel_id} ${behind ? `BEHIND (held=${obs.held_epoch} server=${obs.server_epoch}) \u2014 reporting only, action predicate is group-id` : "healthy, held group matches server"} \u2014 skipping`
|
|
5149
|
+
);
|
|
5150
|
+
continue;
|
|
5151
|
+
}
|
|
5152
|
+
obs.state = "diverged";
|
|
5153
|
+
obs.diverged = true;
|
|
5154
|
+
console.warn(`[bridge] selfHealSweep: ${c.channel_id} holds STALE group state \u2014 signalling re-Welcome`);
|
|
5155
|
+
} else {
|
|
5156
|
+
obs.state = "no_local_state";
|
|
5157
|
+
obs.diverged = false;
|
|
5158
|
+
await this.reconcileJoin(c.channel_id);
|
|
5159
|
+
if (await this.provider.hasPersistedChannelState(c.channel_id)) {
|
|
5160
|
+
obs.held_group_id = await this.provider.heldGroupIdFor(c.channel_id).catch(() => null);
|
|
5161
|
+
obs.held_epoch = await this.provider.currentEpoch(c.channel_id).catch(() => null);
|
|
5162
|
+
if (obs.held_epoch === null) {
|
|
5163
|
+
obs.state = "probe_failed";
|
|
5164
|
+
obs.diverged = null;
|
|
5165
|
+
continue;
|
|
5166
|
+
}
|
|
5167
|
+
obs.state = "healthy";
|
|
5168
|
+
continue;
|
|
5169
|
+
}
|
|
5170
|
+
}
|
|
4794
5171
|
const now = performance.now();
|
|
4795
5172
|
const last = this.lastWelcomeRequest.get(c.channel_id) ?? -Infinity;
|
|
4796
|
-
if (now
|
|
5173
|
+
if (!isResignalDue(now, last, _BridgeAgent.REQUEST_WELCOME_COOLDOWN_MS)) continue;
|
|
4797
5174
|
this.lastWelcomeRequest.set(c.channel_id, now);
|
|
4798
5175
|
await this.signalHeldKpWelcome(g.group_id);
|
|
4799
|
-
} catch {
|
|
5176
|
+
} catch (e) {
|
|
5177
|
+
console.warn(`[bridge] selfHealSweep: channel ${c.channel_id} failed:`, e);
|
|
5178
|
+
} finally {
|
|
5179
|
+
this.observations.set(c.channel_id, obs);
|
|
4800
5180
|
}
|
|
4801
5181
|
}
|
|
4802
5182
|
}
|
|
@@ -4851,7 +5231,7 @@ var BridgeAgent = class _BridgeAgent {
|
|
|
4851
5231
|
if (meta && !meta.is_dm) this.enforceRateFloor("room", channelId);
|
|
4852
5232
|
const joined = await this.reconcileJoin(channelId);
|
|
4853
5233
|
if (!joined) throw new ChannelPendingError(channelId);
|
|
4854
|
-
const id = messageId
|
|
5234
|
+
const id = resolveMessageId(messageId);
|
|
4855
5235
|
const sendStart = performance.now();
|
|
4856
5236
|
await this.sendWithSelfHealTerminal(
|
|
4857
5237
|
this.provider,
|
|
@@ -4888,7 +5268,7 @@ var BridgeAgent = class _BridgeAgent {
|
|
|
4888
5268
|
this.enforceRateFloor("a2a", channelId);
|
|
4889
5269
|
const joined = await this.reconcileJoin(channelId);
|
|
4890
5270
|
if (!joined) throw new ChannelPendingError(channelId);
|
|
4891
|
-
const id = messageId
|
|
5271
|
+
const id = resolveMessageId(messageId);
|
|
4892
5272
|
const sendStart = performance.now();
|
|
4893
5273
|
await this.sendWithSelfHealTerminal(
|
|
4894
5274
|
this.provider,
|
|
@@ -5429,11 +5809,12 @@ var BridgeAgent = class _BridgeAgent {
|
|
|
5429
5809
|
if (await this.provider.hasPersistedChannelState(cid)) continue;
|
|
5430
5810
|
const now = performance.now();
|
|
5431
5811
|
const last = this.lastWelcomeRequest.get(cid) ?? -Infinity;
|
|
5432
|
-
if (now
|
|
5812
|
+
if (!isResignalDue(now, last, _BridgeAgent.REQUEST_WELCOME_COOLDOWN_MS)) continue;
|
|
5433
5813
|
this.lastWelcomeRequest.set(cid, now);
|
|
5434
5814
|
try {
|
|
5435
5815
|
await this.signalHeldKpWelcome(gid);
|
|
5436
|
-
} catch {
|
|
5816
|
+
} catch (e) {
|
|
5817
|
+
console.warn(`[bridge] ingestOnce: signalHeldKpWelcome for ${cid} failed:`, e);
|
|
5437
5818
|
}
|
|
5438
5819
|
}
|
|
5439
5820
|
} catch {
|
|
@@ -5622,11 +6003,55 @@ var BridgeAgent = class _BridgeAgent {
|
|
|
5622
6003
|
const w = this.waiters.splice(0);
|
|
5623
6004
|
for (const r of w) r();
|
|
5624
6005
|
}
|
|
6006
|
+
/**
|
|
6007
|
+
* Task 8: record a newly authenticated /v1/ws session. Called by
|
|
6008
|
+
* ws-server.ts exactly once per connection, AFTER authedUpgrade() passes
|
|
6009
|
+
* (never for a socket closed 1008 for bad auth, and never for the HTTP
|
|
6010
|
+
* /healthz probe, which is not a /v1/ws session at all).
|
|
6011
|
+
*/
|
|
6012
|
+
noteSessionOpen() {
|
|
6013
|
+
this.authSessions++;
|
|
6014
|
+
}
|
|
6015
|
+
/**
|
|
6016
|
+
* Task 8: record an authenticated /v1/ws session ending. Clamped at 0 so a
|
|
6017
|
+
* stray extra close (e.g. a double-fired cleanup) can't go negative and
|
|
6018
|
+
* poison the next open/close cycle.
|
|
6019
|
+
*/
|
|
6020
|
+
noteSessionClose() {
|
|
6021
|
+
this.authSessions = Math.max(0, this.authSessions - 1);
|
|
6022
|
+
}
|
|
6023
|
+
/**
|
|
6024
|
+
* Task 3 (audit M3): record a prior /v1/ws session being evicted by a
|
|
6025
|
+
* newer connection for the same identity (newest-wins). Called by
|
|
6026
|
+
* ws-server.ts right after prior.cleanup() in the eviction path.
|
|
6027
|
+
*/
|
|
6028
|
+
noteSessionEvicted() {
|
|
6029
|
+
this.evictions++;
|
|
6030
|
+
}
|
|
5625
6031
|
health() {
|
|
5626
6032
|
let joined = 0;
|
|
5627
6033
|
for (const s of this.joinState.values()) if (s === "JOINED") joined++;
|
|
6034
|
+
const channels = [...this.observations.values()];
|
|
6035
|
+
const newest = channels.reduce((m, o) => Math.max(m, o.observed_at), 0);
|
|
6036
|
+
const stalePeriodMs = _BridgeAgent.SELF_HEAL_SWEEP_INTERVAL_MS + _BridgeAgent.SELF_HEAL_SWEEP_JITTER_MS;
|
|
6037
|
+
const observationsStale = this.providerCapabilityMissing || channels.length === 0 || Date.now() - newest > stalePeriodMs;
|
|
6038
|
+
const divergedChannels = channels.filter((o) => o.diverged === true).length;
|
|
6039
|
+
const liveChannels = channels.map((o) => ({
|
|
6040
|
+
...o,
|
|
6041
|
+
last_decrypt_at: this.provider.lastDecryptAtFor?.(o.channel_id) ?? o.last_decrypt_at
|
|
6042
|
+
}));
|
|
5628
6043
|
return {
|
|
6044
|
+
// Deliberately still unconditional. `agent-bridge status` and any external
|
|
6045
|
+
// monitor read this field, and a monitor that restarts non-ok bridges would
|
|
6046
|
+
// loop forever on a diverged agent. The additive fields below land first so
|
|
6047
|
+
// the data can be trusted; flipping this to "degraded" is its own change.
|
|
5629
6048
|
status: "ok",
|
|
6049
|
+
sweep_enabled: this.selfHealSweepEnabled,
|
|
6050
|
+
observations_stale: observationsStale,
|
|
6051
|
+
diverged_channels: divergedChannels,
|
|
6052
|
+
// A2: the truthful roll-up gates read instead of `status` (D-3).
|
|
6053
|
+
degraded: divergedChannels > 0 || observationsStale,
|
|
6054
|
+
channels: liveChannels,
|
|
5630
6055
|
identity_id: this.cfg.identityId,
|
|
5631
6056
|
joined_channels: joined,
|
|
5632
6057
|
buffered: this.buffer.length,
|
|
@@ -5636,6 +6061,8 @@ var BridgeAgent = class _BridgeAgent {
|
|
|
5636
6061
|
permanently_skipped: this.provider.permanentlySkippedCount(),
|
|
5637
6062
|
failed_attachments: this.failedAttachments,
|
|
5638
6063
|
delivery_pings: this.pingsReceived,
|
|
6064
|
+
agent_sessions: this.authSessions,
|
|
6065
|
+
evictions: this.evictions,
|
|
5639
6066
|
uptime_s: this.startedAt ? Math.floor((Date.now() - this.startedAt) / 1e3) : 0,
|
|
5640
6067
|
version: BRIDGE_VERSION,
|
|
5641
6068
|
...this.memory.stats?.() ?? {}
|
|
@@ -5995,7 +6422,7 @@ function createHttpServer(agent, opts) {
|
|
|
5995
6422
|
if (method === "POST" && path === "/v1/send") {
|
|
5996
6423
|
const parsed = sendSchema.safeParse(JSON.parse(await readBody(req, SEND_CAP)));
|
|
5997
6424
|
if (!parsed.success) return jsonError(res, "invalid_request", parsed.error.issues);
|
|
5998
|
-
const r = parsed.data.a2a ? await agent.sendA2A(parsed.data.channel_id, parsed.data.a2a
|
|
6425
|
+
const r = parsed.data.a2a ? await agent.sendA2A(parsed.data.channel_id, parsed.data.a2a) : await agent.send(parsed.data.channel_id, parsed.data.text);
|
|
5999
6426
|
return json(res, 200, r);
|
|
6000
6427
|
}
|
|
6001
6428
|
if (method === "POST" && path === "/v1/send-attachment") {
|
|
@@ -6126,12 +6553,14 @@ function attachAgentWebSocket(server, agent, opts) {
|
|
|
6126
6553
|
sock.close(1008, "unauthorized");
|
|
6127
6554
|
return;
|
|
6128
6555
|
}
|
|
6556
|
+
agent.noteSessionOpen();
|
|
6129
6557
|
if (current && current.sock !== sock) {
|
|
6130
6558
|
const prior = current;
|
|
6131
6559
|
if (prior.sock.readyState === WebSocket.OPEN) {
|
|
6132
6560
|
prior.sock.close(1e3, "replaced by newer connection");
|
|
6133
6561
|
}
|
|
6134
6562
|
prior.cleanup();
|
|
6563
|
+
agent.noteSessionEvicted();
|
|
6135
6564
|
}
|
|
6136
6565
|
sock.isAlive = true;
|
|
6137
6566
|
sock.on("pong", () => {
|
|
@@ -6161,6 +6590,7 @@ function attachAgentWebSocket(server, agent, opts) {
|
|
|
6161
6590
|
const cleanup = () => {
|
|
6162
6591
|
if (cleaned) return;
|
|
6163
6592
|
cleaned = true;
|
|
6593
|
+
agent.noteSessionClose();
|
|
6164
6594
|
unsubscribe();
|
|
6165
6595
|
if (inflight.size) {
|
|
6166
6596
|
agent.requeueFront([...inflight.values()]);
|
|
@@ -6247,9 +6677,8 @@ async function handleInbound(sock, agent, data, inflight) {
|
|
|
6247
6677
|
sendJson(sock, wsErrorFrame("invalid_request"));
|
|
6248
6678
|
return;
|
|
6249
6679
|
}
|
|
6250
|
-
const idempotencyKey = typeof m.idempotency_key === "string" ? m.idempotency_key : void 0;
|
|
6251
6680
|
try {
|
|
6252
|
-
const r = await agent.send(m.channel_id, m.text
|
|
6681
|
+
const r = await agent.send(m.channel_id, m.text);
|
|
6253
6682
|
sendJson(sock, { type: "sent", message_id: r.message_id });
|
|
6254
6683
|
} catch (e) {
|
|
6255
6684
|
if (e instanceof ChannelPendingError) {
|