@securecoms/agent-bridge 0.36.0 → 0.38.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -0
- package/connectors/hermes/__pycache__/securecoms_gate.cpython-310.pyc +0 -0
- package/connectors/hermes/__pycache__/securecoms_transport.cpython-310.pyc +0 -0
- package/connectors/hermes/adapter.py +424 -32
- package/connectors/hermes/securecoms_transport.py +97 -1
- package/connectors/hermes/tests/test_adapter_wiring.py +887 -0
- package/connectors/hermes/tests/test_securecoms_transport.py +127 -0
- package/connectors/openclaw/openclaw-entry.contract.test.ts +96 -0
- package/connectors/openclaw/openclaw-entry.inbound.test.ts +228 -1
- package/connectors/openclaw/openclaw-entry.js +128 -3
- package/dist/attachments.d.ts +33 -1
- package/dist/bin.js +4607 -1020
- package/dist/bridge-agent.d.ts +129 -0
- package/dist/grants/mcp-policy.d.ts +7 -0
- package/dist/index.js +4577 -990
- package/dist/version.d.ts +1 -1
- package/dist/ws-server.d.ts +1 -1
- package/package.json +1 -1
|
@@ -45,7 +45,12 @@ def _install_gateway_stub():
|
|
|
45
45
|
self.value = value
|
|
46
46
|
|
|
47
47
|
class MessageType(Enum):
|
|
48
|
+
# Real Hermes gateway/platforms/event.py MessageType (subset).
|
|
48
49
|
TEXT = "text"
|
|
50
|
+
PHOTO = "photo"
|
|
51
|
+
VIDEO = "video"
|
|
52
|
+
VOICE = "voice"
|
|
53
|
+
DOCUMENT = "document"
|
|
49
54
|
|
|
50
55
|
@dataclass
|
|
51
56
|
class MessageEvent:
|
|
@@ -61,6 +66,9 @@ def _install_gateway_stub():
|
|
|
61
66
|
# Real Hermes: may this event resolve gateway commands / control
|
|
62
67
|
# prompts? False keeps untrusted payload text conversational.
|
|
63
68
|
allow_gateway_control: bool = True
|
|
69
|
+
# Real Hermes: local file paths (or URLs) of inbound media + mimes.
|
|
70
|
+
media_urls: list = field(default_factory=list)
|
|
71
|
+
media_types: list = field(default_factory=list)
|
|
64
72
|
|
|
65
73
|
@dataclass
|
|
66
74
|
class SendResult:
|
|
@@ -466,6 +474,7 @@ def fresh_gate(monkeypatch):
|
|
|
466
474
|
monkeypatch.setattr(adapter, "_HARNESS_VERSION", None)
|
|
467
475
|
monkeypatch.setattr(adapter, "_REFUSAL_NOTICE_SENT", set())
|
|
468
476
|
monkeypatch.setattr(adapter, "_FRAMES_WITHOUT_AUTHORIZATION", 0)
|
|
477
|
+
monkeypatch.setattr(adapter, "_ATTACHMENTS_DROPPED", 0)
|
|
469
478
|
|
|
470
479
|
|
|
471
480
|
class TestToolGate:
|
|
@@ -1107,3 +1116,881 @@ class TestConnectSignatureContract:
|
|
|
1107
1116
|
code, _message, retryable = a.fatal_errors[-1]
|
|
1108
1117
|
assert code == "unwired"
|
|
1109
1118
|
assert retryable is False
|
|
1119
|
+
|
|
1120
|
+
|
|
1121
|
+
PNG = b"\x89PNG\r\n\x1a\n\x00\x00\x00\r"
|
|
1122
|
+
JPEG = b"\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01"
|
|
1123
|
+
|
|
1124
|
+
|
|
1125
|
+
@pytest.fixture
|
|
1126
|
+
def att_dir(tmp_path):
|
|
1127
|
+
d = tmp_path / "attachments" / "dm"
|
|
1128
|
+
d.mkdir(parents=True)
|
|
1129
|
+
return d
|
|
1130
|
+
|
|
1131
|
+
|
|
1132
|
+
def _att(att_dir, name, data=JPEG, mime="image/jpeg"):
|
|
1133
|
+
p = att_dir / f"m-{name}"
|
|
1134
|
+
p.write_bytes(data)
|
|
1135
|
+
return {"filename": name, "mime": mime, "size": len(data), "path": str(p), "sha256": "x"}
|
|
1136
|
+
|
|
1137
|
+
|
|
1138
|
+
class TestInboundAttachments:
|
|
1139
|
+
async def _handled(self, monkeypatch, frames, channels=None, refresh=None):
|
|
1140
|
+
monkeypatch.setattr(adapter, "_TOOL_VETO_ACTIVE", True)
|
|
1141
|
+
a, _ = make_adapter(monkeypatch)
|
|
1142
|
+
a._bound_identity = "A"
|
|
1143
|
+
if channels:
|
|
1144
|
+
a._channels.update(channels)
|
|
1145
|
+
if refresh is not None:
|
|
1146
|
+
a._session = object()
|
|
1147
|
+
|
|
1148
|
+
async def fake_refresh():
|
|
1149
|
+
a._channels.update(refresh)
|
|
1150
|
+
|
|
1151
|
+
a._refresh_channels = fake_refresh
|
|
1152
|
+
handled = []
|
|
1153
|
+
|
|
1154
|
+
async def handler(event):
|
|
1155
|
+
handled.append(event)
|
|
1156
|
+
|
|
1157
|
+
a.set_message_handler(handler)
|
|
1158
|
+
ws = await _pump_frames(a, frames)
|
|
1159
|
+
return handled, ws
|
|
1160
|
+
|
|
1161
|
+
@pytest.mark.asyncio
|
|
1162
|
+
async def test_image_frame_becomes_a_photo_event_with_the_local_path(self, monkeypatch, fresh_gate, att_dir):
|
|
1163
|
+
a = _att(att_dir, "IMG_3375.jpeg")
|
|
1164
|
+
handled, ws = await self._handled(monkeypatch, [
|
|
1165
|
+
_frame("m1", "owner", "dm", "what is this?", OWNER_AUTH, attachments=[a]),
|
|
1166
|
+
])
|
|
1167
|
+
assert len(handled) == 1
|
|
1168
|
+
ev = handled[0]
|
|
1169
|
+
assert ev.message_type == adapter.MessageType.PHOTO
|
|
1170
|
+
assert ev.media_urls == [a["path"]]
|
|
1171
|
+
assert ev.media_types == ["image/jpeg"]
|
|
1172
|
+
assert ev.text == "what is this?"
|
|
1173
|
+
assert [f for f in ws.sent if f.get("type") == "ack"][0]["message_id"] == "m1"
|
|
1174
|
+
assert adapter._ATTACHMENTS_DROPPED == 0
|
|
1175
|
+
|
|
1176
|
+
@pytest.mark.asyncio
|
|
1177
|
+
async def test_image_only_frame_still_runs_a_turn(self, monkeypatch, fresh_gate, att_dir):
|
|
1178
|
+
a = _att(att_dir, "a.png", PNG, "image/png")
|
|
1179
|
+
handled, _ = await self._handled(monkeypatch, [_frame("m1", "owner", "dm", "", OWNER_AUTH, attachments=[a])])
|
|
1180
|
+
assert len(handled) == 1
|
|
1181
|
+
assert handled[0].text == ""
|
|
1182
|
+
assert handled[0].message_type == adapter.MessageType.PHOTO
|
|
1183
|
+
assert handled[0].media_urls == [a["path"]]
|
|
1184
|
+
|
|
1185
|
+
@pytest.mark.asyncio
|
|
1186
|
+
async def test_a_pdf_is_dropped_and_counted(self, monkeypatch, fresh_gate, att_dir):
|
|
1187
|
+
a = _att(att_dir, "r.pdf", b"%PDF-1.7\n....", "application/pdf")
|
|
1188
|
+
handled, _ = await self._handled(monkeypatch, [_frame("m1", "owner", "dm", "", OWNER_AUTH, attachments=[a])])
|
|
1189
|
+
assert handled[0].message_type == adapter.MessageType.TEXT
|
|
1190
|
+
assert handled[0].media_urls == []
|
|
1191
|
+
assert adapter._ATTACHMENTS_DROPPED == 1
|
|
1192
|
+
|
|
1193
|
+
@pytest.mark.asyncio
|
|
1194
|
+
async def test_bad_path_is_dropped_and_the_turn_is_text(self, monkeypatch, fresh_gate, att_dir):
|
|
1195
|
+
a = {**_att(att_dir, "x.jpg"), "path": "/etc/passwd"}
|
|
1196
|
+
handled, _ = await self._handled(monkeypatch, [_frame("m1", "owner", "dm", "look", OWNER_AUTH, attachments=[a])])
|
|
1197
|
+
assert len(handled) == 1
|
|
1198
|
+
assert handled[0].message_type == adapter.MessageType.TEXT
|
|
1199
|
+
assert handled[0].media_urls == []
|
|
1200
|
+
assert handled[0].media_types == []
|
|
1201
|
+
|
|
1202
|
+
@pytest.mark.asyncio
|
|
1203
|
+
async def test_text_only_frame_is_unchanged(self, monkeypatch, fresh_gate):
|
|
1204
|
+
handled, _ = await self._handled(monkeypatch, [_frame("m1", "owner", "dm", "hi", OWNER_AUTH)])
|
|
1205
|
+
ev = handled[0]
|
|
1206
|
+
assert ev.text == "hi"
|
|
1207
|
+
assert ev.message_type == adapter.MessageType.TEXT
|
|
1208
|
+
assert ev.media_urls == [] and ev.media_types == []
|
|
1209
|
+
assert adapter._ATTACHMENTS_DROPPED == 0
|
|
1210
|
+
|
|
1211
|
+
@pytest.mark.asyncio
|
|
1212
|
+
async def test_ungranted_sender_image_keeps_the_command_escape(self, monkeypatch, fresh_gate, att_dir):
|
|
1213
|
+
a = _att(att_dir, "a.jpg")
|
|
1214
|
+
handled, _ = await self._handled(monkeypatch, [
|
|
1215
|
+
_frame("m1", "member", "dm", "/reset", _no_grant("team_room"), attachments=[a]),
|
|
1216
|
+
])
|
|
1217
|
+
ev = handled[0]
|
|
1218
|
+
assert ev.text == adapter.securecoms_authz.COMMAND_ESCAPE + "/reset"
|
|
1219
|
+
assert ev.allow_gateway_control is False
|
|
1220
|
+
assert ev.media_urls == [a["path"]]
|
|
1221
|
+
|
|
1222
|
+
|
|
1223
|
+
class TestChatTypeDefault:
|
|
1224
|
+
"""Review r1 M3: an uncached channel is a group (per-user session), never a DM."""
|
|
1225
|
+
|
|
1226
|
+
async def _chat_type(self, monkeypatch, **kw):
|
|
1227
|
+
handled, _ = await TestInboundAttachments()._handled(
|
|
1228
|
+
monkeypatch, [_frame("m1", "owner", "c1", "hi", OWNER_AUTH)], **kw)
|
|
1229
|
+
return handled[0].source.chat_type
|
|
1230
|
+
|
|
1231
|
+
@pytest.mark.asyncio
|
|
1232
|
+
async def test_uncached_channel_is_a_group(self, monkeypatch, fresh_gate):
|
|
1233
|
+
assert await self._chat_type(monkeypatch) == "group"
|
|
1234
|
+
|
|
1235
|
+
@pytest.mark.asyncio
|
|
1236
|
+
async def test_cached_dm_is_still_a_dm(self, monkeypatch, fresh_gate):
|
|
1237
|
+
assert await self._chat_type(monkeypatch, channels={"c1": {"name": "c1", "is_dm": True}}) == "dm"
|
|
1238
|
+
|
|
1239
|
+
@pytest.mark.asyncio
|
|
1240
|
+
async def test_cached_room_is_a_group(self, monkeypatch, fresh_gate):
|
|
1241
|
+
assert await self._chat_type(monkeypatch, channels={"c1": {"name": "c1", "is_dm": False}}) == "group"
|
|
1242
|
+
|
|
1243
|
+
@pytest.mark.asyncio
|
|
1244
|
+
async def test_a_dm_created_after_connect_is_looked_up_first(self, monkeypatch, fresh_gate):
|
|
1245
|
+
assert await self._chat_type(monkeypatch, refresh={"c1": {"name": "c1", "is_dm": True}}) == "dm"
|
|
1246
|
+
|
|
1247
|
+
@pytest.mark.asyncio
|
|
1248
|
+
async def test_an_unknown_channel_is_looked_up_at_most_once_a_minute(self, monkeypatch, fresh_gate):
|
|
1249
|
+
monkeypatch.setattr(adapter, "_TOOL_VETO_ACTIVE", True)
|
|
1250
|
+
a, _ = make_adapter(monkeypatch)
|
|
1251
|
+
a._bound_identity = "A"
|
|
1252
|
+
a._session = object()
|
|
1253
|
+
calls = []
|
|
1254
|
+
|
|
1255
|
+
async def fake_refresh():
|
|
1256
|
+
calls.append(1)
|
|
1257
|
+
|
|
1258
|
+
a._refresh_channels = fake_refresh
|
|
1259
|
+
clock = [1000.0]
|
|
1260
|
+
monkeypatch.setattr(adapter.time, "monotonic", lambda: clock[0])
|
|
1261
|
+
handled = []
|
|
1262
|
+
|
|
1263
|
+
async def handler(event):
|
|
1264
|
+
handled.append(event)
|
|
1265
|
+
|
|
1266
|
+
a.set_message_handler(handler)
|
|
1267
|
+
await _pump_frames(a, [_frame(f"m{i}", "owner", "c1", "hi", OWNER_AUTH) for i in range(3)])
|
|
1268
|
+
assert len(calls) == 1
|
|
1269
|
+
assert [e.source.chat_type for e in handled] == ["group"] * 3
|
|
1270
|
+
clock[0] += adapter._UNKNOWN_CHANNEL_RETRY_S
|
|
1271
|
+
await _pump_frames(a, [_frame("m9", "owner", "c1", "hi", OWNER_AUTH)])
|
|
1272
|
+
assert len(calls) == 2
|
|
1273
|
+
# another unknown channel has its own budget
|
|
1274
|
+
await _pump_frames(a, [_frame("n1", "owner", "c2", "hi", OWNER_AUTH)])
|
|
1275
|
+
assert len(calls) == 3
|
|
1276
|
+
assert adapter._UNKNOWN_CHANNEL_RETRY_S == 60.0
|
|
1277
|
+
|
|
1278
|
+
@pytest.mark.asyncio
|
|
1279
|
+
async def test_a_slow_channel_refresh_is_bounded(self, monkeypatch, fresh_gate):
|
|
1280
|
+
a, _ = make_adapter(monkeypatch)
|
|
1281
|
+
monkeypatch.setattr(adapter, "_CHANNEL_REFRESH_TIMEOUT_S", 0.05)
|
|
1282
|
+
|
|
1283
|
+
async def slow_fetch():
|
|
1284
|
+
await asyncio.sleep(30)
|
|
1285
|
+
|
|
1286
|
+
a._fetch_channels = slow_fetch
|
|
1287
|
+
await asyncio.wait_for(a._refresh_channels(), timeout=2) # returns, never raises
|
|
1288
|
+
assert adapter._CHANNEL_REFRESH_TIMEOUT_S == 0.05
|
|
1289
|
+
|
|
1290
|
+
def test_refresh_timeout_default_is_short(self):
|
|
1291
|
+
assert adapter._CHANNEL_REFRESH_TIMEOUT_S == 5.0
|
|
1292
|
+
|
|
1293
|
+
@pytest.mark.asyncio
|
|
1294
|
+
async def test_get_chat_info_defaults_to_group(self, monkeypatch, fresh_gate):
|
|
1295
|
+
a, _ = make_adapter(monkeypatch)
|
|
1296
|
+
|
|
1297
|
+
async def no_refresh():
|
|
1298
|
+
return None
|
|
1299
|
+
|
|
1300
|
+
a._refresh_channels = no_refresh
|
|
1301
|
+
assert (await a.get_chat_info("unknown"))["type"] == "group"
|
|
1302
|
+
|
|
1303
|
+
|
|
1304
|
+
# ---------------------------------------------------------------------------
|
|
1305
|
+
# Ack on durable acceptance + redelivery dedup.
|
|
1306
|
+
#
|
|
1307
|
+
# _HermesSessions mirrors the Hermes v2026.9.24 paths the adapter relies on
|
|
1308
|
+
# (gateway/platforms/base.py unless noted):
|
|
1309
|
+
# - handle_message (3950) resets event._gateway_accepted (3953); an idle
|
|
1310
|
+
# session starts a tracked task and sets it True (3990, 3862, 3876);
|
|
1311
|
+
# - a busy session runs a bypass command (/stop, /approve) inline, leaving
|
|
1312
|
+
# _gateway_accepted False (_handle_message_while_active, 3995-4013; /stop
|
|
1313
|
+
# cancels the owner task, cancel_session_processing 3891), and queues
|
|
1314
|
+
# anything else in _pending_messages with _gateway_accepted True (4058-4064);
|
|
1315
|
+
# - _process_message_background (4438) calls on_processing_start, awaits the
|
|
1316
|
+
# runner, then on_processing_complete with SUCCESS / FAILURE / CANCELLED on
|
|
1317
|
+
# every exit (4520-4540), and hands a queued follow-up to a fresh drain
|
|
1318
|
+
# task (_spawn_drain_task 4560);
|
|
1319
|
+
# - cancel_background_tasks (4593) cancels every tracked task on shutdown.
|
|
1320
|
+
# _Runner mirrors the runner side: it persists the active-turn marker and only
|
|
1321
|
+
# then publishes event._gateway_active_turn_token (gateway/run_inbound.py
|
|
1322
|
+
# 1754-1767, called at gateway/run_turn.py:2090 before the model call), and
|
|
1323
|
+
# clears it when the turn ends (run_inbound.py:1382, 1769-1795).
|
|
1324
|
+
# F2DaemonWS mirrors the daemon (ws-server.ts F2): at most one message per
|
|
1325
|
+
# channel in flight; the next one for that channel goes out only after the ack.
|
|
1326
|
+
# The module stub above (inline handle_message, no _gateway_accepted) is the
|
|
1327
|
+
# "this Hermes has none of it" fallback case.
|
|
1328
|
+
# ---------------------------------------------------------------------------
|
|
1329
|
+
|
|
1330
|
+
|
|
1331
|
+
class ProcessingOutcome(Enum):
|
|
1332
|
+
SUCCESS = "success"
|
|
1333
|
+
FAILURE = "failure"
|
|
1334
|
+
CANCELLED = "cancelled"
|
|
1335
|
+
|
|
1336
|
+
|
|
1337
|
+
_BYPASS = ("/stop", "/approve")
|
|
1338
|
+
|
|
1339
|
+
|
|
1340
|
+
class _HermesSessions:
|
|
1341
|
+
async def handle_message(self, event):
|
|
1342
|
+
event._gateway_accepted = False
|
|
1343
|
+
if not self._message_handler:
|
|
1344
|
+
return
|
|
1345
|
+
key = self._event_session_key(event)
|
|
1346
|
+
# On-entry self-heal (base.py:3983-3985, _heal_stale_session_lock 3849):
|
|
1347
|
+
# a guard whose recorded owner task already exited is cleared.
|
|
1348
|
+
owner = self._session_tasks.get(key)
|
|
1349
|
+
if key in self._active_sessions and owner is not None and owner.done():
|
|
1350
|
+
self._active_sessions.pop(key, None)
|
|
1351
|
+
self._pending_messages.pop(key, None)
|
|
1352
|
+
self._session_tasks.pop(key, None)
|
|
1353
|
+
if key in self._active_sessions:
|
|
1354
|
+
cmd = (event.text or "").strip().split(" ")[0]
|
|
1355
|
+
if cmd in _BYPASS:
|
|
1356
|
+
self.inline_commands.append(cmd)
|
|
1357
|
+
if cmd == "/stop":
|
|
1358
|
+
await self.cancel_session_processing(key)
|
|
1359
|
+
else:
|
|
1360
|
+
self.approvals.setdefault(key, asyncio.get_running_loop().create_future()).set_result(True)
|
|
1361
|
+
return
|
|
1362
|
+
# merge_pending_message_event(merge_text=True) (base.py:1739-1781):
|
|
1363
|
+
# TEXT appends to the queued event, which keeps its own message_id.
|
|
1364
|
+
existing = self._pending_messages.get(key)
|
|
1365
|
+
if existing is None:
|
|
1366
|
+
self._pending_messages[key] = event
|
|
1367
|
+
else:
|
|
1368
|
+
existing.text = existing.text + "\n" + event.text
|
|
1369
|
+
event._gateway_accepted = True
|
|
1370
|
+
return
|
|
1371
|
+
self._active_sessions[key] = asyncio.Event()
|
|
1372
|
+
self._track(key, asyncio.create_task(self._process_message_background(event, key)))
|
|
1373
|
+
event._gateway_accepted = True
|
|
1374
|
+
|
|
1375
|
+
def _track(self, key, task):
|
|
1376
|
+
self._session_tasks[key] = task
|
|
1377
|
+
self._background_tasks.add(task)
|
|
1378
|
+
task.add_done_callback(self._background_tasks.discard)
|
|
1379
|
+
|
|
1380
|
+
async def _process_message_background(self, event, key):
|
|
1381
|
+
try:
|
|
1382
|
+
await self._run_processing_hook("on_processing_start", event)
|
|
1383
|
+
await self._message_handler(event)
|
|
1384
|
+
await self._run_processing_hook("on_processing_complete", event, ProcessingOutcome.SUCCESS)
|
|
1385
|
+
except asyncio.CancelledError:
|
|
1386
|
+
await self._run_processing_hook("on_processing_complete", event, ProcessingOutcome.CANCELLED)
|
|
1387
|
+
raise
|
|
1388
|
+
except Exception:
|
|
1389
|
+
await self._run_processing_hook("on_processing_complete", event, ProcessingOutcome.FAILURE)
|
|
1390
|
+
finally:
|
|
1391
|
+
pending = self._pending_messages.pop(key, None)
|
|
1392
|
+
if pending is not None and not self._shutting_down:
|
|
1393
|
+
self._track(key, asyncio.create_task(self._process_message_background(pending, key)))
|
|
1394
|
+
else:
|
|
1395
|
+
self._active_sessions.pop(key, None)
|
|
1396
|
+
self._session_tasks.pop(key, None)
|
|
1397
|
+
|
|
1398
|
+
async def on_processing_start(self, event):
|
|
1399
|
+
pass
|
|
1400
|
+
|
|
1401
|
+
async def _run_processing_hook(self, hook_name, *args, **kwargs):
|
|
1402
|
+
# base.py:3470-3477: a failing hook never breaks message flow.
|
|
1403
|
+
hook = getattr(self, hook_name, None)
|
|
1404
|
+
if not callable(hook):
|
|
1405
|
+
return
|
|
1406
|
+
try:
|
|
1407
|
+
await hook(*args, **kwargs)
|
|
1408
|
+
except Exception:
|
|
1409
|
+
pass
|
|
1410
|
+
|
|
1411
|
+
async def cancel_session_processing(self, key):
|
|
1412
|
+
task = self._session_tasks.pop(key, None)
|
|
1413
|
+
if task is not None and not task.done():
|
|
1414
|
+
task.cancel()
|
|
1415
|
+
await asyncio.gather(task, return_exceptions=True)
|
|
1416
|
+
self._active_sessions.pop(key, None)
|
|
1417
|
+
|
|
1418
|
+
async def cancel_background_tasks(self):
|
|
1419
|
+
self._shutting_down = True
|
|
1420
|
+
tasks = [t for t in self._background_tasks if not t.done()]
|
|
1421
|
+
for t in tasks:
|
|
1422
|
+
t.cancel()
|
|
1423
|
+
await asyncio.gather(*tasks, return_exceptions=True)
|
|
1424
|
+
|
|
1425
|
+
def _event_session_key(self, event):
|
|
1426
|
+
s = event.source
|
|
1427
|
+
return f"agent:main:securecoms:{s.chat_type}:{s.chat_id}:{s.user_id}"
|
|
1428
|
+
|
|
1429
|
+
|
|
1430
|
+
class _Runner:
|
|
1431
|
+
"""The gateway runner's message handler. marker_gates hold a turn before
|
|
1432
|
+
its marker is persisted (lease wait); turn_gates hold it after (the model
|
|
1433
|
+
call). A turn whose text names 'approval' waits for an /approve."""
|
|
1434
|
+
|
|
1435
|
+
def __init__(self, adapter_ref):
|
|
1436
|
+
self.a = adapter_ref
|
|
1437
|
+
self.started = []
|
|
1438
|
+
self.marker_gates = {}
|
|
1439
|
+
self.turn_gates = {}
|
|
1440
|
+
self.raise_before_marker = set()
|
|
1441
|
+
|
|
1442
|
+
def gate(self, kind, mid):
|
|
1443
|
+
return getattr(self, kind).setdefault(mid, asyncio.Event())
|
|
1444
|
+
|
|
1445
|
+
def release(self, kind, mid):
|
|
1446
|
+
self.gate(kind, mid).set()
|
|
1447
|
+
|
|
1448
|
+
async def __call__(self, event):
|
|
1449
|
+
mid = event.message_id
|
|
1450
|
+
self.started.append(mid)
|
|
1451
|
+
if mid in self.marker_gates:
|
|
1452
|
+
await self.marker_gates[mid].wait()
|
|
1453
|
+
if mid in self.raise_before_marker:
|
|
1454
|
+
raise RuntimeError("history unreadable")
|
|
1455
|
+
event._gateway_active_turn_token = "tok-" + mid # persisted, then published
|
|
1456
|
+
try:
|
|
1457
|
+
if "approval" in (event.text or ""):
|
|
1458
|
+
key = self.a._event_session_key(event)
|
|
1459
|
+
await self.a.approvals.setdefault(key, asyncio.get_running_loop().create_future())
|
|
1460
|
+
else:
|
|
1461
|
+
await self.gate("turn_gates", mid).wait()
|
|
1462
|
+
finally:
|
|
1463
|
+
del event._gateway_active_turn_token
|
|
1464
|
+
|
|
1465
|
+
|
|
1466
|
+
class F2DaemonWS:
|
|
1467
|
+
def __init__(self, frames):
|
|
1468
|
+
self.frames = list(frames)
|
|
1469
|
+
self.next = 0
|
|
1470
|
+
self.inflight = {} # channel_id -> message_id
|
|
1471
|
+
self.delivered = []
|
|
1472
|
+
self.sent = []
|
|
1473
|
+
self.closed = False
|
|
1474
|
+
self._acked = asyncio.Event()
|
|
1475
|
+
|
|
1476
|
+
def __aiter__(self):
|
|
1477
|
+
return self
|
|
1478
|
+
|
|
1479
|
+
async def __anext__(self):
|
|
1480
|
+
while True:
|
|
1481
|
+
if self.next >= len(self.frames):
|
|
1482
|
+
return _FakeWSMsg(aiohttp.WSMsgType.CLOSED)
|
|
1483
|
+
f = self.frames[self.next]
|
|
1484
|
+
if f["channel_id"] not in self.inflight:
|
|
1485
|
+
self.next += 1
|
|
1486
|
+
self.inflight[f["channel_id"]] = f.get("message_id")
|
|
1487
|
+
self.delivered.append(f.get("message_id"))
|
|
1488
|
+
return _FakeWSMsg(aiohttp.WSMsgType.TEXT, json.dumps(f))
|
|
1489
|
+
self._acked.clear()
|
|
1490
|
+
await self._acked.wait()
|
|
1491
|
+
|
|
1492
|
+
async def send_json(self, payload):
|
|
1493
|
+
self.sent.append(payload)
|
|
1494
|
+
if payload.get("type") in ("ack", "nack"):
|
|
1495
|
+
for ch, mid in list(self.inflight.items()):
|
|
1496
|
+
if mid == payload.get("message_id"):
|
|
1497
|
+
del self.inflight[ch]
|
|
1498
|
+
self._acked.set()
|
|
1499
|
+
|
|
1500
|
+
async def close(self):
|
|
1501
|
+
self.closed = True
|
|
1502
|
+
|
|
1503
|
+
|
|
1504
|
+
def make_hermes_adapter(monkeypatch):
|
|
1505
|
+
class HermesFaithfulAdapter(_HermesSessions, adapter.SecureComsAdapter):
|
|
1506
|
+
pass
|
|
1507
|
+
|
|
1508
|
+
monkeypatch.setattr(adapter, "_read_file_text", make_fs({}))
|
|
1509
|
+
a = HermesFaithfulAdapter(types.SimpleNamespace(extra={}))
|
|
1510
|
+
a._active_sessions = {}
|
|
1511
|
+
a._pending_messages = {}
|
|
1512
|
+
a._session_tasks = {}
|
|
1513
|
+
a._background_tasks = set()
|
|
1514
|
+
a._shutting_down = False
|
|
1515
|
+
a.inline_commands = []
|
|
1516
|
+
a.approvals = {}
|
|
1517
|
+
a._bound_identity = "A"
|
|
1518
|
+
runner = _Runner(a)
|
|
1519
|
+
a.set_message_handler(runner)
|
|
1520
|
+
return a, runner
|
|
1521
|
+
|
|
1522
|
+
|
|
1523
|
+
async def _settle():
|
|
1524
|
+
for _ in range(20):
|
|
1525
|
+
await asyncio.sleep(0)
|
|
1526
|
+
|
|
1527
|
+
|
|
1528
|
+
async def _drain_waiters(a, timeout=2.0):
|
|
1529
|
+
async def drain():
|
|
1530
|
+
while a._ack_waiters:
|
|
1531
|
+
await asyncio.gather(*list(a._ack_waiters))
|
|
1532
|
+
|
|
1533
|
+
await asyncio.wait_for(drain(), timeout=timeout)
|
|
1534
|
+
|
|
1535
|
+
|
|
1536
|
+
async def _pump(a, ws, timeout=2.0):
|
|
1537
|
+
a._ws = ws
|
|
1538
|
+
await asyncio.wait_for(a._pump(), timeout=timeout)
|
|
1539
|
+
|
|
1540
|
+
|
|
1541
|
+
def _acks(ws):
|
|
1542
|
+
return [f["message_id"] for f in ws.sent if f.get("type") == "ack"]
|
|
1543
|
+
|
|
1544
|
+
|
|
1545
|
+
def _nacks(ws):
|
|
1546
|
+
return [f["message_id"] for f in ws.sent if f.get("type") == "nack"]
|
|
1547
|
+
|
|
1548
|
+
|
|
1549
|
+
def _turns_done(a):
|
|
1550
|
+
return [t for t in a._background_tasks if not t.done()] == []
|
|
1551
|
+
|
|
1552
|
+
|
|
1553
|
+
@pytest.fixture
|
|
1554
|
+
def ack_state(monkeypatch, fresh_gate):
|
|
1555
|
+
monkeypatch.setattr(adapter, "_TOOL_VETO_ACTIVE", True)
|
|
1556
|
+
# raising=False so a pre-change adapter fails these tests on behaviour, not setup.
|
|
1557
|
+
monkeypatch.setattr(adapter, "_TURN_TRACKING_WARNED", False, raising=False)
|
|
1558
|
+
|
|
1559
|
+
|
|
1560
|
+
class TestAckOnAcceptance:
|
|
1561
|
+
@pytest.mark.asyncio
|
|
1562
|
+
async def test_ack_waits_for_the_durable_marker_not_for_the_turn_end(self, monkeypatch, ack_state):
|
|
1563
|
+
a, r = make_hermes_adapter(monkeypatch)
|
|
1564
|
+
r.gate("marker_gates", "m1")
|
|
1565
|
+
ws = F2DaemonWS([_frame("m1", "owner", "c1", "hi", OWNER_AUTH)])
|
|
1566
|
+
await _pump(a, ws)
|
|
1567
|
+
await _settle()
|
|
1568
|
+
assert r.started == ["m1"]
|
|
1569
|
+
assert _acks(ws) == [] # running, but no durable marker yet
|
|
1570
|
+
r.release("marker_gates", "m1")
|
|
1571
|
+
await _drain_waiters(a)
|
|
1572
|
+
assert _acks(ws) == ["m1"]
|
|
1573
|
+
assert not _turns_done(a) # acked while the turn is still running
|
|
1574
|
+
r.release("turn_gates", "m1")
|
|
1575
|
+
await a.cancel_background_tasks()
|
|
1576
|
+
|
|
1577
|
+
@pytest.mark.asyncio
|
|
1578
|
+
@pytest.mark.parametrize("command", ["/stop", "/approve"])
|
|
1579
|
+
async def test_a_mid_turn_command_in_the_same_channel_is_delivered_and_runs(
|
|
1580
|
+
self, monkeypatch, ack_state, command
|
|
1581
|
+
):
|
|
1582
|
+
# H1: with the daemon's one-in-flight-per-channel rule, the command is
|
|
1583
|
+
# delivered only once the running turn's frame is acked. It must get
|
|
1584
|
+
# through while that turn is still running.
|
|
1585
|
+
a, r = make_hermes_adapter(monkeypatch)
|
|
1586
|
+
first = "long job" if command == "/stop" else "needs approval"
|
|
1587
|
+
ws = F2DaemonWS([_frame("m1", "owner", "c1", first, OWNER_AUTH),
|
|
1588
|
+
_frame("m2", "owner", "c1", command, OWNER_AUTH)])
|
|
1589
|
+
await _pump(a, ws) # times out if m1's ack waits for the turn to end
|
|
1590
|
+
assert ws.delivered == ["m1", "m2"]
|
|
1591
|
+
assert a.inline_commands == [command]
|
|
1592
|
+
await _drain_waiters(a)
|
|
1593
|
+
await _settle()
|
|
1594
|
+
assert _acks(ws) == ["m1", "m2"]
|
|
1595
|
+
assert _turns_done(a) # /stop cancelled the turn; /approve let it finish
|
|
1596
|
+
assert r.started == ["m1"]
|
|
1597
|
+
|
|
1598
|
+
@pytest.mark.asyncio
|
|
1599
|
+
async def test_a_turn_that_fails_before_its_marker_is_acked(self, monkeypatch, ack_state):
|
|
1600
|
+
a, r = make_hermes_adapter(monkeypatch)
|
|
1601
|
+
r.raise_before_marker.add("m1")
|
|
1602
|
+
ws = F2DaemonWS([_frame("m1", "owner", "c1", "hi", OWNER_AUTH)])
|
|
1603
|
+
await _pump(a, ws)
|
|
1604
|
+
await _drain_waiters(a, timeout=1) # on the outcome, not the 10 s bound
|
|
1605
|
+
assert _acks(ws) == ["m1"] # a poison turn is never redelivered forever
|
|
1606
|
+
|
|
1607
|
+
@pytest.mark.asyncio
|
|
1608
|
+
async def test_dropped_frames_ack_at_once(self, monkeypatch, ack_state):
|
|
1609
|
+
monkeypatch.setattr(adapter, "_TOOL_VETO_ACTIVE", False) # ungranted turns are refused
|
|
1610
|
+
a, r = make_hermes_adapter(monkeypatch)
|
|
1611
|
+
|
|
1612
|
+
async def no_send(*args, **kwargs):
|
|
1613
|
+
return None
|
|
1614
|
+
|
|
1615
|
+
a.send = no_send
|
|
1616
|
+
ws = F2DaemonWS([
|
|
1617
|
+
_frame("echo", "A", "c1", "my own send", OWNER_AUTH), # echo guard
|
|
1618
|
+
_frame("refused", "member", "room", "hi", _no_grant("team_room")), # no grant, no veto
|
|
1619
|
+
])
|
|
1620
|
+
await _pump(a, ws)
|
|
1621
|
+
assert r.started == []
|
|
1622
|
+
assert _acks(ws) == ["echo", "refused"]
|
|
1623
|
+
assert not a._ack_waiters
|
|
1624
|
+
|
|
1625
|
+
@pytest.mark.asyncio
|
|
1626
|
+
async def test_a_redelivery_before_acceptance_is_not_dispatched_again(self, monkeypatch, ack_state):
|
|
1627
|
+
a, r = make_hermes_adapter(monkeypatch)
|
|
1628
|
+
r.gate("marker_gates", "m1")
|
|
1629
|
+
f = _frame("m1", "owner", "c1", "hi", OWNER_AUTH)
|
|
1630
|
+
ws1 = F2DaemonWS([f])
|
|
1631
|
+
await _pump(a, ws1)
|
|
1632
|
+
await _settle()
|
|
1633
|
+
ws2 = F2DaemonWS([f]) # socket flap: the daemon redelivers the unacked frame
|
|
1634
|
+
await _pump(a, ws2)
|
|
1635
|
+
assert r.started == ["m1"]
|
|
1636
|
+
assert _acks(ws2) == []
|
|
1637
|
+
r.release("marker_gates", "m1")
|
|
1638
|
+
await _drain_waiters(a)
|
|
1639
|
+
assert r.started == ["m1"]
|
|
1640
|
+
assert _acks(ws1) == []
|
|
1641
|
+
assert _acks(ws2) == ["m1"] # the original's ack, on the current socket
|
|
1642
|
+
r.release("turn_gates", "m1")
|
|
1643
|
+
await a.cancel_background_tasks()
|
|
1644
|
+
|
|
1645
|
+
@pytest.mark.asyncio
|
|
1646
|
+
async def test_a_redelivery_after_the_ack_is_acked_without_a_turn(self, monkeypatch, ack_state):
|
|
1647
|
+
a, r = make_hermes_adapter(monkeypatch)
|
|
1648
|
+
r.release("turn_gates", "m1")
|
|
1649
|
+
f = _frame("m1", "owner", "c1", "hi", OWNER_AUTH)
|
|
1650
|
+
ws1 = F2DaemonWS([f])
|
|
1651
|
+
await _pump(a, ws1)
|
|
1652
|
+
await _drain_waiters(a)
|
|
1653
|
+
assert _acks(ws1) == ["m1"]
|
|
1654
|
+
ws2 = F2DaemonWS([f])
|
|
1655
|
+
await _pump(a, ws2)
|
|
1656
|
+
assert r.started == ["m1"]
|
|
1657
|
+
assert _acks(ws2) == ["m1"]
|
|
1658
|
+
assert not a._ack_waiters
|
|
1659
|
+
|
|
1660
|
+
@pytest.mark.asyncio
|
|
1661
|
+
async def test_no_socket_at_acceptance_then_the_redelivery_is_acked(self, monkeypatch, ack_state):
|
|
1662
|
+
a, r = make_hermes_adapter(monkeypatch)
|
|
1663
|
+
r.gate("marker_gates", "m1")
|
|
1664
|
+
f = _frame("m1", "owner", "c1", "hi", OWNER_AUTH)
|
|
1665
|
+
await _pump(a, F2DaemonWS([f]))
|
|
1666
|
+
await _settle()
|
|
1667
|
+
a._ws = None # link down when Hermes accepts: the ack is dropped
|
|
1668
|
+
r.release("marker_gates", "m1")
|
|
1669
|
+
await _drain_waiters(a)
|
|
1670
|
+
ws2 = F2DaemonWS([f])
|
|
1671
|
+
await _pump(a, ws2)
|
|
1672
|
+
assert r.started == ["m1"]
|
|
1673
|
+
assert _acks(ws2) == ["m1"]
|
|
1674
|
+
r.release("turn_gates", "m1")
|
|
1675
|
+
await a.cancel_background_tasks()
|
|
1676
|
+
|
|
1677
|
+
@pytest.mark.asyncio
|
|
1678
|
+
async def test_a_frame_without_a_message_id_is_not_deduped(self, monkeypatch, ack_state):
|
|
1679
|
+
a, r = make_hermes_adapter(monkeypatch)
|
|
1680
|
+
f = {k: v for k, v in _frame("x", "owner", "c1", "hi", OWNER_AUTH).items() if k != "message_id"}
|
|
1681
|
+
g = dict(f, channel_id="c2")
|
|
1682
|
+
ws = F2DaemonWS([f, g])
|
|
1683
|
+
await _pump(a, ws)
|
|
1684
|
+
await _drain_waiters(a)
|
|
1685
|
+
assert len(r.started) == 2
|
|
1686
|
+
assert _acks(ws) == [None, None] # unchanged wire shape for an id-less frame
|
|
1687
|
+
await a.cancel_background_tasks()
|
|
1688
|
+
|
|
1689
|
+
@pytest.mark.asyncio
|
|
1690
|
+
async def test_two_channels_progress_independently(self, monkeypatch, ack_state):
|
|
1691
|
+
a, r = make_hermes_adapter(monkeypatch)
|
|
1692
|
+
r.gate("marker_gates", "slow")
|
|
1693
|
+
ws = F2DaemonWS([_frame("slow", "owner", "c1", "hi", OWNER_AUTH),
|
|
1694
|
+
_frame("fast", "owner", "c2", "hi", OWNER_AUTH)])
|
|
1695
|
+
await _pump(a, ws)
|
|
1696
|
+
await _settle()
|
|
1697
|
+
assert r.started == ["slow", "fast"]
|
|
1698
|
+
await _settle()
|
|
1699
|
+
assert _acks(ws) == ["fast"]
|
|
1700
|
+
r.release("marker_gates", "slow")
|
|
1701
|
+
await _drain_waiters(a)
|
|
1702
|
+
assert _acks(ws) == ["fast", "slow"]
|
|
1703
|
+
await a.cancel_background_tasks()
|
|
1704
|
+
|
|
1705
|
+
@pytest.mark.asyncio
|
|
1706
|
+
async def test_a_follow_up_queued_behind_a_busy_session_is_acked_at_once(self, monkeypatch, ack_state):
|
|
1707
|
+
a, r = make_hermes_adapter(monkeypatch)
|
|
1708
|
+
ws = F2DaemonWS([_frame("m1", "owner", "c1", "one", OWNER_AUTH),
|
|
1709
|
+
_frame("m2", "owner", "c1", "two", OWNER_AUTH)])
|
|
1710
|
+
await _pump(a, ws) # m1 accepted -> acked -> m2 delivered mid-turn and queued
|
|
1711
|
+
assert ws.delivered == ["m1", "m2"]
|
|
1712
|
+
assert a._pending_messages # m2 waits behind m1 in Hermes's pending slot
|
|
1713
|
+
assert _acks(ws) == ["m1", "m2"] # no wait for a turn that may be minutes away
|
|
1714
|
+
assert not a._ack_waiters
|
|
1715
|
+
r.release("turn_gates", "m1") # m1 ends; the drain turn still runs m2
|
|
1716
|
+
await _settle()
|
|
1717
|
+
assert r.started == ["m1", "m2"]
|
|
1718
|
+
await a.cancel_background_tasks()
|
|
1719
|
+
|
|
1720
|
+
@pytest.mark.asyncio
|
|
1721
|
+
async def test_a_stop_after_a_queued_follow_up_is_dispatched_promptly(self, monkeypatch, ack_state):
|
|
1722
|
+
# Round-2 M1: the queued follow-up must not hold the channel for the
|
|
1723
|
+
# 10 s acceptance bound, or the /stop behind it waits that long.
|
|
1724
|
+
assert adapter._ACCEPT_WAIT_S == 10.0
|
|
1725
|
+
a, r = make_hermes_adapter(monkeypatch)
|
|
1726
|
+
ws = F2DaemonWS([_frame("m1", "owner", "c1", "long job", OWNER_AUTH),
|
|
1727
|
+
_frame("m2", "owner", "c1", "also do X", OWNER_AUTH),
|
|
1728
|
+
_frame("m3", "owner", "c1", "/stop", OWNER_AUTH)])
|
|
1729
|
+
loop = asyncio.get_running_loop()
|
|
1730
|
+
t0 = loop.time()
|
|
1731
|
+
await _pump(a, ws, timeout=2.0)
|
|
1732
|
+
assert loop.time() - t0 < 1.0
|
|
1733
|
+
assert ws.delivered == ["m1", "m2", "m3"]
|
|
1734
|
+
assert a.inline_commands == ["/stop"]
|
|
1735
|
+
assert _acks(ws) == ["m1", "m2", "m3"]
|
|
1736
|
+
await a.cancel_background_tasks()
|
|
1737
|
+
|
|
1738
|
+
@pytest.mark.asyncio
|
|
1739
|
+
async def test_a_stale_session_guard_does_not_count_as_busy(self, monkeypatch, ack_state):
|
|
1740
|
+
# Hermes heals a guard whose owner task exited and starts a fresh turn,
|
|
1741
|
+
# so the event is not "queued behind a busy session": wait for its marker.
|
|
1742
|
+
a, r = make_hermes_adapter(monkeypatch)
|
|
1743
|
+
f = _frame("m1", "owner", "c1", "hi", OWNER_AUTH)
|
|
1744
|
+
done = asyncio.get_running_loop().create_future()
|
|
1745
|
+
done.set_result(None)
|
|
1746
|
+
key = "agent:main:securecoms:group:c1:owner"
|
|
1747
|
+
a._active_sessions[key] = asyncio.Event()
|
|
1748
|
+
a._session_tasks[key] = done
|
|
1749
|
+
r.gate("marker_gates", "m1")
|
|
1750
|
+
ws = F2DaemonWS([f])
|
|
1751
|
+
await _pump(a, ws)
|
|
1752
|
+
await _settle()
|
|
1753
|
+
assert r.started == ["m1"]
|
|
1754
|
+
assert _acks(ws) == [] # its own turn: the ack waits for the marker
|
|
1755
|
+
r.release("marker_gates", "m1")
|
|
1756
|
+
await _drain_waiters(a)
|
|
1757
|
+
assert _acks(ws) == ["m1"]
|
|
1758
|
+
await a.cancel_background_tasks()
|
|
1759
|
+
|
|
1760
|
+
@pytest.mark.asyncio
|
|
1761
|
+
async def test_a_turn_never_accepted_is_acked_at_the_bound(self, monkeypatch, ack_state):
|
|
1762
|
+
monkeypatch.setattr(adapter, "_ACCEPT_WAIT_S", 0.05)
|
|
1763
|
+
a, r = make_hermes_adapter(monkeypatch)
|
|
1764
|
+
r.gate("marker_gates", "m1") # e.g. stuck waiting for the turn lease
|
|
1765
|
+
ws = F2DaemonWS([_frame("m1", "owner", "c1", "hi", OWNER_AUTH)])
|
|
1766
|
+
await _pump(a, ws)
|
|
1767
|
+
await _drain_waiters(a)
|
|
1768
|
+
assert _acks(ws) == ["m1"]
|
|
1769
|
+
await a.cancel_background_tasks()
|
|
1770
|
+
|
|
1771
|
+
def test_the_bound_default(self):
|
|
1772
|
+
assert adapter._ACCEPT_WAIT_S == 10.0
|
|
1773
|
+
|
|
1774
|
+
@pytest.mark.asyncio
|
|
1775
|
+
async def test_a_turn_cancelled_by_shutdown_before_acceptance_stays_unacked(self, monkeypatch, ack_state):
|
|
1776
|
+
a, r = make_hermes_adapter(monkeypatch)
|
|
1777
|
+
r.gate("marker_gates", "m1")
|
|
1778
|
+
f = _frame("m1", "owner", "c1", "hi", OWNER_AUTH)
|
|
1779
|
+
ws = F2DaemonWS([f])
|
|
1780
|
+
await _pump(a, ws)
|
|
1781
|
+
await _settle()
|
|
1782
|
+
await a.cancel_background_tasks() # Hermes shutdown cancels the owner task
|
|
1783
|
+
await _drain_waiters(a)
|
|
1784
|
+
assert _acks(ws) == [] # no marker, so no Hermes resume: the daemon must redeliver
|
|
1785
|
+
assert _nacks(ws) == ["m1"] # failed back at once, not held until the socket closes
|
|
1786
|
+
a._shutting_down = False
|
|
1787
|
+
ws2 = F2DaemonWS([f])
|
|
1788
|
+
await _pump(a, ws2)
|
|
1789
|
+
assert r.started == ["m1", "m1"] # the ledger forgot it: the redelivery runs
|
|
1790
|
+
await a.cancel_background_tasks()
|
|
1791
|
+
|
|
1792
|
+
@pytest.mark.asyncio
|
|
1793
|
+
async def test_a_turn_cancelled_after_acceptance_was_already_acked(self, monkeypatch, ack_state):
|
|
1794
|
+
a, r = make_hermes_adapter(monkeypatch)
|
|
1795
|
+
ws = F2DaemonWS([_frame("m1", "owner", "c1", "hi", OWNER_AUTH)])
|
|
1796
|
+
await _pump(a, ws)
|
|
1797
|
+
await _drain_waiters(a)
|
|
1798
|
+
assert _acks(ws) == ["m1"]
|
|
1799
|
+
await a.cancel_background_tasks() # recovery is Hermes's own from here
|
|
1800
|
+
assert _acks(ws) == ["m1"]
|
|
1801
|
+
|
|
1802
|
+
@pytest.mark.asyncio
|
|
1803
|
+
async def test_disconnect_cancels_the_wait_and_forgets_the_id(self, monkeypatch, ack_state):
|
|
1804
|
+
a, r = make_hermes_adapter(monkeypatch)
|
|
1805
|
+
r.gate("marker_gates", "m1")
|
|
1806
|
+
f = _frame("m1", "owner", "c1", "hi", OWNER_AUTH)
|
|
1807
|
+
ws = F2DaemonWS([f])
|
|
1808
|
+
await _pump(a, ws)
|
|
1809
|
+
await _settle()
|
|
1810
|
+
waiters = list(a._ack_waiters)
|
|
1811
|
+
assert len(waiters) == 1
|
|
1812
|
+
await asyncio.wait_for(a.disconnect(), timeout=2)
|
|
1813
|
+
assert all(t.cancelled() for t in waiters)
|
|
1814
|
+
assert not a._ack_waiters
|
|
1815
|
+
await a.cancel_background_tasks()
|
|
1816
|
+
r.release("marker_gates", "m1")
|
|
1817
|
+
assert _acks(ws) == []
|
|
1818
|
+
# M3: the same adapter instance, reconnected, runs the redelivery.
|
|
1819
|
+
a._shutting_down = False
|
|
1820
|
+
a._stopped = False
|
|
1821
|
+
ws2 = F2DaemonWS([f])
|
|
1822
|
+
await _pump(a, ws2)
|
|
1823
|
+
assert r.started == ["m1", "m1"]
|
|
1824
|
+
await a.cancel_background_tasks()
|
|
1825
|
+
|
|
1826
|
+
@pytest.mark.asyncio
|
|
1827
|
+
async def test_a_dispatch_error_forgets_the_id_so_the_redelivery_runs(self, monkeypatch, ack_state):
|
|
1828
|
+
a, r = make_hermes_adapter(monkeypatch)
|
|
1829
|
+
calls = []
|
|
1830
|
+
real = a._dispatch
|
|
1831
|
+
|
|
1832
|
+
async def flaky(frame):
|
|
1833
|
+
calls.append(frame["message_id"])
|
|
1834
|
+
if len(calls) == 1:
|
|
1835
|
+
raise RuntimeError("boom")
|
|
1836
|
+
return await real(frame)
|
|
1837
|
+
|
|
1838
|
+
a._dispatch = flaky
|
|
1839
|
+
r.release("turn_gates", "m1")
|
|
1840
|
+
f = _frame("m1", "owner", "c1", "hi", OWNER_AUTH)
|
|
1841
|
+
ws1 = F2DaemonWS([f])
|
|
1842
|
+
with pytest.raises(RuntimeError):
|
|
1843
|
+
await _pump(a, ws1) # unchanged: the supervisor reconnects, no ack
|
|
1844
|
+
assert _acks(ws1) == []
|
|
1845
|
+
ws2 = F2DaemonWS([f])
|
|
1846
|
+
await _pump(a, ws2)
|
|
1847
|
+
await _drain_waiters(a)
|
|
1848
|
+
assert calls == ["m1", "m1"]
|
|
1849
|
+
assert _acks(ws2) == ["m1"]
|
|
1850
|
+
|
|
1851
|
+
@pytest.mark.asyncio
|
|
1852
|
+
async def test_an_ack_dropped_during_a_long_outage_still_blocks_a_rerun(self, monkeypatch, ack_state):
|
|
1853
|
+
# Round-2 M2: link down for longer than the dedup TTL.
|
|
1854
|
+
a, r = make_hermes_adapter(monkeypatch)
|
|
1855
|
+
clock = [0.0]
|
|
1856
|
+
a._deliveries._now = lambda: clock[0]
|
|
1857
|
+
r.gate("marker_gates", "m1")
|
|
1858
|
+
f = _frame("m1", "owner", "c1", "hi", OWNER_AUTH)
|
|
1859
|
+
await _pump(a, F2DaemonWS([f]))
|
|
1860
|
+
await _settle()
|
|
1861
|
+
a._ws = None # link down when Hermes accepts: the ack cannot go out
|
|
1862
|
+
r.release("marker_gates", "m1")
|
|
1863
|
+
await _drain_waiters(a)
|
|
1864
|
+
assert list(a._unsent_acks) == ["m1"]
|
|
1865
|
+
r.release("turn_gates", "m1") # the turn ends: a re-run would start a fresh turn
|
|
1866
|
+
await _settle()
|
|
1867
|
+
assert _turns_done(a)
|
|
1868
|
+
clock[0] += adapter._DEDUP_TTL_S * 3
|
|
1869
|
+
for i in range(adapter._DEDUP_MAX_ENTRIES + 5): # and LRU pressure
|
|
1870
|
+
a._deliveries.begin(f"x{i}")
|
|
1871
|
+
a._deliveries.finish(f"x{i}")
|
|
1872
|
+
ws2 = F2DaemonWS([f])
|
|
1873
|
+
await _pump(a, ws2)
|
|
1874
|
+
await _settle()
|
|
1875
|
+
assert r.started == ["m1"] # not re-run
|
|
1876
|
+
assert _acks(ws2) == ["m1"]
|
|
1877
|
+
assert not a._unsent_acks
|
|
1878
|
+
|
|
1879
|
+
@pytest.mark.asyncio
|
|
1880
|
+
async def test_dropped_acks_are_flushed_on_the_next_connect(self, monkeypatch, ack_state):
|
|
1881
|
+
a, r = make_hermes_adapter(monkeypatch)
|
|
1882
|
+
r.gate("marker_gates", "m1")
|
|
1883
|
+
await _pump(a, F2DaemonWS([_frame("m1", "owner", "c1", "hi", OWNER_AUTH)]))
|
|
1884
|
+
await _settle()
|
|
1885
|
+
a._ws = None
|
|
1886
|
+
r.release("marker_gates", "m1")
|
|
1887
|
+
await _drain_waiters(a)
|
|
1888
|
+
files = {CONNECTIONS_DIR + "/A.json": conn_file(identity_id="A"), "/tok/A": "token-A"}
|
|
1889
|
+
monkeypatch.setattr(adapter, "_read_file_text", make_fs(files))
|
|
1890
|
+
monkeypatch.setattr(adapter, "_CONNECTIONS_DIR", CONNECTIONS_DIR)
|
|
1891
|
+
ws = FakeWS()
|
|
1892
|
+
session = FakeSession(healthz_by_url={"http://127.0.0.1:9001/healthz": {"identity_id": "A"}}, ws=ws)
|
|
1893
|
+
monkeypatch.setattr(adapter, "_new_client_session", lambda: session)
|
|
1894
|
+
assert await a._open_ws() is True
|
|
1895
|
+
assert [f["type"] for f in ws.sent][:2] == ["hello", "ack"]
|
|
1896
|
+
assert _acks(ws) == ["m1"]
|
|
1897
|
+
assert not a._unsent_acks
|
|
1898
|
+
r.release("turn_gates", "m1")
|
|
1899
|
+
await a.cancel_background_tasks()
|
|
1900
|
+
|
|
1901
|
+
@pytest.mark.asyncio
|
|
1902
|
+
async def test_an_ack_send_that_raises_is_kept_for_the_next_connect(self, monkeypatch, ack_state):
|
|
1903
|
+
a, r = make_hermes_adapter(monkeypatch)
|
|
1904
|
+
|
|
1905
|
+
class BrokenWS(FakeWS):
|
|
1906
|
+
async def send_json(self, payload):
|
|
1907
|
+
raise ConnectionResetError("gone")
|
|
1908
|
+
|
|
1909
|
+
a._ws = BrokenWS()
|
|
1910
|
+
await a._send_ack("m9")
|
|
1911
|
+
assert list(a._unsent_acks) == ["m9"]
|
|
1912
|
+
a._ws = ws = FakeWS()
|
|
1913
|
+
await a._flush_unsent_acks()
|
|
1914
|
+
assert _acks(ws) == ["m9"]
|
|
1915
|
+
assert not a._unsent_acks
|
|
1916
|
+
|
|
1917
|
+
@pytest.mark.asyncio
|
|
1918
|
+
async def test_without_the_hermes_acceptance_signal_it_acks_at_dispatch_and_warns_once(
|
|
1919
|
+
self, monkeypatch, ack_state, caplog
|
|
1920
|
+
):
|
|
1921
|
+
a, _ = make_adapter(monkeypatch) # the plain stub: inline handler, no _gateway_accepted
|
|
1922
|
+
a._bound_identity = "A"
|
|
1923
|
+
seen = []
|
|
1924
|
+
|
|
1925
|
+
async def handler(event):
|
|
1926
|
+
seen.append(event.message_id)
|
|
1927
|
+
|
|
1928
|
+
a.set_message_handler(handler)
|
|
1929
|
+
with caplog.at_level("WARNING", logger=adapter.logger.name):
|
|
1930
|
+
ws = await _pump_frames(a, [_frame("m1", "owner", "c1", "hi", OWNER_AUTH),
|
|
1931
|
+
_frame("m2", "owner", "c2", "hi", OWNER_AUTH)])
|
|
1932
|
+
assert seen == ["m1", "m2"]
|
|
1933
|
+
assert _acks(ws) == ["m1", "m2"]
|
|
1934
|
+
assert not a._ack_waiters
|
|
1935
|
+
warned = [r for r in caplog.records if "acceptance signal unavailable" in r.getMessage()]
|
|
1936
|
+
assert len(warned) == 1
|
|
1937
|
+
|
|
1938
|
+
|
|
1939
|
+
class TestDeliveryLedger:
|
|
1940
|
+
def _ledger(self, max_entries=3, ttl=10.0):
|
|
1941
|
+
clock = [0.0]
|
|
1942
|
+
return adapter._DeliveryLedger(max_entries, ttl, lambda: clock[0]), clock
|
|
1943
|
+
|
|
1944
|
+
def test_states(self):
|
|
1945
|
+
led, _ = self._ledger()
|
|
1946
|
+
assert led.begin("a") == "new"
|
|
1947
|
+
assert led.begin("a") == "in_flight"
|
|
1948
|
+
led.finish("a")
|
|
1949
|
+
assert led.begin("a") == "done"
|
|
1950
|
+
led.forget("a")
|
|
1951
|
+
assert led.begin("a") == "new"
|
|
1952
|
+
|
|
1953
|
+
def test_lru_bound_evicts_the_oldest_completed_entry(self):
|
|
1954
|
+
led, _ = self._ledger(max_entries=3)
|
|
1955
|
+
for mid in ("a", "b", "c", "d"):
|
|
1956
|
+
led.begin(mid)
|
|
1957
|
+
led.finish(mid)
|
|
1958
|
+
assert len(led) == 3
|
|
1959
|
+
assert led.begin("a") == "new" # evicted, oldest first
|
|
1960
|
+
assert led.begin("d") == "done"
|
|
1961
|
+
|
|
1962
|
+
def test_in_flight_entries_are_never_evicted(self):
|
|
1963
|
+
led, _ = self._ledger(max_entries=2)
|
|
1964
|
+
led.begin("live")
|
|
1965
|
+
for mid in ("a", "b", "c"):
|
|
1966
|
+
led.begin(mid)
|
|
1967
|
+
led.finish(mid)
|
|
1968
|
+
assert led.begin("live") == "in_flight"
|
|
1969
|
+
assert len(led) <= 3
|
|
1970
|
+
|
|
1971
|
+
def test_completed_entries_expire_after_the_ttl_in_flight_ones_do_not(self):
|
|
1972
|
+
led, clock = self._ledger(ttl=10.0)
|
|
1973
|
+
led.begin("done")
|
|
1974
|
+
led.finish("done")
|
|
1975
|
+
led.begin("live")
|
|
1976
|
+
clock[0] = 11.0
|
|
1977
|
+
assert led.begin("done") == "new"
|
|
1978
|
+
assert led.begin("live") == "in_flight"
|
|
1979
|
+
|
|
1980
|
+
def test_pinned_entries_survive_ttl_and_lru_until_unpinned(self):
|
|
1981
|
+
led, clock = self._ledger(max_entries=2, ttl=10.0)
|
|
1982
|
+
led.begin("p")
|
|
1983
|
+
led.finish("p")
|
|
1984
|
+
led.pin("p")
|
|
1985
|
+
for mid in ("a", "b", "c"):
|
|
1986
|
+
led.begin(mid)
|
|
1987
|
+
led.finish(mid)
|
|
1988
|
+
clock[0] = 100.0
|
|
1989
|
+
assert led.begin("p") == "done"
|
|
1990
|
+
led.unpin("p")
|
|
1991
|
+
clock[0] = 200.0
|
|
1992
|
+
assert led.begin("p") == "new"
|
|
1993
|
+
|
|
1994
|
+
def test_defaults(self):
|
|
1995
|
+
assert adapter._DEDUP_MAX_ENTRIES == 2048
|
|
1996
|
+
assert adapter._DEDUP_TTL_S == 600.0
|