kijito-tools 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/NOTICE +6 -0
- package/README.md +220 -0
- package/bin/cli.js +34 -0
- package/install.sh +161 -0
- package/package.json +50 -0
- package/providers/_shared/wake-core.mjs +358 -0
- package/providers/claude/CLAUDE.md.snippet +31 -0
- package/providers/claude/install.sh +186 -0
- package/providers/claude/scripts/arm-session.sh +66 -0
- package/providers/claude/scripts/claude-armed.sh +101 -0
- package/providers/claude/scripts/heartbeat-watchdog.sh +209 -0
- package/providers/claude/scripts/inbox-selftest.sh +277 -0
- package/providers/claude/scripts/kijito-persona-lib.sh +94 -0
- package/providers/claude/scripts/kijito-qa-pass.sh +12 -0
- package/providers/claude/scripts/lifecycle-lib.sh +167 -0
- package/providers/claude/scripts/myctx.sh +46 -0
- package/providers/claude/scripts/self-clear.sh +134 -0
- package/providers/claude/scripts/session-autosend.sh +85 -0
- package/providers/claude/scripts/session-catchup-hint.sh +330 -0
- package/providers/claude/scripts/statusline-context.sh +67 -0
- package/providers/claude/skills/kijito-qa-memory/SKILL.md +110 -0
- package/providers/claude/skills/kijito-recall/SKILL.md +53 -0
- package/providers/claude/skills/kijito-start/SKILL.md +122 -0
- package/providers/claude/wiring/README.md +93 -0
- package/providers/claude/wiring/kijito-heartbeat@.service +21 -0
- package/providers/codex/README.md +75 -0
- package/providers/codex/codex-kijito-parity-plan.md +265 -0
- package/providers/codex/docs-codex-setup.md +64 -0
- package/providers/codex/install.mjs +184 -0
- package/providers/codex/n0-capability-probe-protocol.md +472 -0
- package/providers/codex/n0-harness/README.md +57 -0
- package/providers/codex/n0-harness/cli.mjs +58 -0
- package/providers/codex/n0-harness/evidence-manifest.mjs +77 -0
- package/providers/codex/n0-harness/fixture.mjs +279 -0
- package/providers/codex/n0-harness/lib.mjs +141 -0
- package/providers/codex/n0-harness/manifest.mjs +36 -0
- package/providers/codex/n0-harness/oracle.mjs +464 -0
- package/providers/codex/n0-harness/parser.mjs +200 -0
- package/providers/codex/n0-harness/prompt.mjs +43 -0
- package/providers/codex/n0-harness/snapshot.mjs +120 -0
- package/providers/codex/n0-harness/specimen.mjs +37 -0
- package/providers/codex/notify/kijito-notify-count.mjs +117 -0
- package/providers/codex/notify/kijito-notify-count.test.mjs +76 -0
- package/providers/codex/plans/gate4-battery-log.md +184 -0
- package/providers/codex/plans/gate4-battery-protocol.md +43 -0
- package/providers/codex/plans/hive-user-first-plan-QA.md +34 -0
- package/providers/codex/plans/hive-user-first-plan.md +247 -0
- package/providers/codex/release-manifest.json +53 -0
- package/providers/codex/same-chat-continuation-plan-gate.md +235 -0
- package/providers/codex/same-chat-continuation-plan.md +458 -0
- package/providers/codex/skills/kijito-qa-memory/SKILL.md +193 -0
- package/providers/codex/skills/kijito-qa-memory/agents/openai.yaml +4 -0
- package/providers/codex/skills/kijito-recall/SKILL.md +53 -0
- package/providers/codex/skills/kijito-start/SKILL.md +173 -0
- package/providers/codex/skills/kijito-start/agents/openai.yaml +4 -0
- package/providers/codex/test/n0-cli-refusal.test.mjs +31 -0
- package/providers/codex/test/n0-counterexample-matrix.test.mjs +118 -0
- package/providers/codex/test/n0-guard-census-core.mjs +1372 -0
- package/providers/codex/test/n0-guard-census.json +35479 -0
- package/providers/codex/test/n0-guard-census.mjs +31 -0
- package/providers/codex/test/n0-guard-census.test.mjs +374 -0
- package/providers/codex/test/n0-guard-counterexamples.mjs +1068 -0
- package/providers/codex/test/n0-guard-independent-projection.mjs +70 -0
- package/providers/codex/test/n0-guard-independent-projection.test.mjs +53 -0
- package/providers/codex/test/n0-guard-manifest-author.mjs +424 -0
- package/providers/codex/test/n0-guard-mutation-operators.test.mjs +244 -0
- package/providers/codex/test/n0-guard-mutation-runner.mjs +599 -0
- package/providers/codex/test/n0-guard-remainder-classes.test.mjs +314 -0
- package/providers/codex/test/n0-guard-sanitizer-pairs.test.mjs +68 -0
- package/providers/codex/test/n0-guard-shared-context-predicates.test.mjs +84 -0
- package/providers/codex/test/n0-harness.test.mjs +553 -0
- package/providers/codex/test/n0-notable-regressions.test.mjs +70 -0
- package/providers/codex/test/n0-subject-boundary.mjs +97 -0
- package/providers/codex/test/n0-subject-boundary.test.mjs +72 -0
- package/providers/codex/test/same-chat-plan-preflight.sh +167 -0
- package/providers/codex/tools/refresh-manifest.mjs +81 -0
- package/providers/codex/wake-helper/TRANSPORT-NOTES.md +25 -0
- package/providers/codex/wake-helper/integration.test.mjs +285 -0
- package/providers/codex/wake-helper/kijito-wake-helper.mjs +488 -0
- package/providers/codex/wake-helper/kijito-wake-helper.test.mjs +128 -0
- package/providers/codex/wake-helper/mock-daemon.mjs +64 -0
- package/providers/codex/wake-helper/status-probe.mjs +45 -0
- package/providers/codex/wake-helper/ws-uds.mjs +153 -0
- package/providers/monitor/.github/workflows/publish-npm.yml +38 -0
- package/providers/monitor/.github/workflows/publish-pypi.yml +30 -0
- package/providers/monitor/CHANGELOG.md +465 -0
- package/providers/monitor/IMPORT-PROVENANCE.md +68 -0
- package/providers/monitor/LICENSE +202 -0
- package/providers/monitor/NOTICE +7 -0
- package/providers/monitor/OPAQUE-OUTPUT-ENFORCEMENT.md +45 -0
- package/providers/monitor/README.md +571 -0
- package/providers/monitor/RELEASING.md +125 -0
- package/providers/monitor/arm-hive-monitor.sh +13 -0
- package/providers/monitor/bin/cli.js +43 -0
- package/providers/monitor/com.kijito.inbox-monitor.plist.template +59 -0
- package/providers/monitor/docs/DESIGN.md +984 -0
- package/providers/monitor/kijito-inbox-monitor@.service.template +95 -0
- package/providers/monitor/kijito_inbox_monitor.py +3666 -0
- package/providers/monitor/package.json +35 -0
- package/providers/monitor/pyproject.toml +42 -0
- package/providers/monitor/scripts/mutation-check.py +399 -0
- package/providers/monitor/scripts/prepublish-gate.sh +284 -0
- package/providers/monitor/test_kijito_monitor.py +5609 -0
|
@@ -0,0 +1,3666 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Kijito Inbox Monitor - client-side liveness watcher for your Kijito inbox.
|
|
3
|
+
|
|
4
|
+
A standalone, zero-dependency (Python stdlib only) process that polls your Kijito inbox at api.kijito.ai and emits
|
|
5
|
+
one event per new message into whatever harness is running - NDJSON on stdout and/or by exec-ing a command per
|
|
6
|
+
event. It keeps a *running* agent's inbox live by waking it BETWEEN tool calls (the LLM-UX inbox-liveness fix). It
|
|
7
|
+
is NOT a server.
|
|
8
|
+
|
|
9
|
+
Authentication is required: set $KIJITOMON_TOKEN (or --token-file) to your Kijito API token. POSIX target
|
|
10
|
+
(Linux/macOS); on Windows it runs interval-only (no SIGUSR1 seam, no flock). See docs/DESIGN.md for the design.
|
|
11
|
+
"""
|
|
12
|
+
import argparse
|
|
13
|
+
import datetime
|
|
14
|
+
import errno
|
|
15
|
+
import hashlib
|
|
16
|
+
import http.client
|
|
17
|
+
import json
|
|
18
|
+
import os
|
|
19
|
+
import select
|
|
20
|
+
import signal
|
|
21
|
+
import socket
|
|
22
|
+
import ssl
|
|
23
|
+
import stat
|
|
24
|
+
import subprocess
|
|
25
|
+
import sys
|
|
26
|
+
import tempfile
|
|
27
|
+
import time
|
|
28
|
+
import urllib.parse
|
|
29
|
+
import urllib.request
|
|
30
|
+
|
|
31
|
+
try:
|
|
32
|
+
import fcntl # POSIX only
|
|
33
|
+
except ImportError: # pragma: no cover - Windows
|
|
34
|
+
fcntl = None
|
|
35
|
+
|
|
36
|
+
__version__ = "0.5.0"
|
|
37
|
+
SOURCE = "kijito-inbox"
|
|
38
|
+
# A named User-Agent is REQUIRED: api.kijito.ai is fronted by a WAF that 403s the default Python-urllib UA.
|
|
39
|
+
USER_AGENT = "kijito-inbox-monitor/%s" % __version__
|
|
40
|
+
KIJITO_BASE = "https://api.kijito.ai"
|
|
41
|
+
INBOX_URL = KIJITO_BASE + "/api/inbox"
|
|
42
|
+
PERSONAS_URL = KIJITO_BASE + "/api/personas"
|
|
43
|
+
NOTIFY_PENDING_URL = KIJITO_BASE + "/api/notify/pending"
|
|
44
|
+
EXEC_TIMEOUT = 10
|
|
45
|
+
HTTP_TIMEOUT = 5 # per-request timeout default (normal fetches)
|
|
46
|
+
LONGPOLL_SLACK = 10 # client socket timeout = server hold (--wait) + this, so a half-open hold is always detected
|
|
47
|
+
LONGPOLL_BACKOFF_CAP = 30 # cap (s) on exponential backoff between failed long-poll attempts
|
|
48
|
+
PIN_TRACKING_CAP = 5000 # max delivered ids remembered above a pinned watermark (bounds the state file)
|
|
49
|
+
WALK_BACK_MAX_PAGES = 50 # page budget for an authoritative backward walk over an omitted span
|
|
50
|
+
BROKEN_SINK_RETRY_S = 30 # cooldown before re-trying a persona sink we refused; the refusal's RELEASE
|
|
51
|
+
# condition, so removing a hostile path recovers without a restart (re-audit 10, H2)
|
|
52
|
+
IS_POSIX = os.name == "posix"
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
56
|
+
# Errors
|
|
57
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
58
|
+
class FatalConfig(Exception):
|
|
59
|
+
"""A fatal startup/config error → exit non-zero (NOT a per-poll liveness failure)."""
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
63
|
+
# §7.3 Canonical identity (computed BEFORE DNS resolution; trivial URL variations must not flip it)
|
|
64
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
65
|
+
def canonical_identity(url):
|
|
66
|
+
p = urllib.parse.urlsplit(url)
|
|
67
|
+
scheme = (p.scheme or "http").lower()
|
|
68
|
+
host = (p.hostname or "").lower()
|
|
69
|
+
port = p.port or (443 if scheme == "https" else 80)
|
|
70
|
+
path = (p.path or "/").rstrip("/") or "/"
|
|
71
|
+
# sort query params; the constant mark_read is excluded so its presence can't flip identity.
|
|
72
|
+
# Use LISTS (not tuples) so the identity is JSON-round-trip stable - a persisted identity reloads
|
|
73
|
+
# as lists, and the freshly-computed one must compare EQUAL (tuples would reload as lists → spurious
|
|
74
|
+
# mismatch → restart-resume silently re-baselines, defeating the state-file).
|
|
75
|
+
q = sorted([k, v] for k, v in urllib.parse.parse_qsl(p.query, keep_blank_values=True) if k != "mark_read")
|
|
76
|
+
return [scheme, host, port, path, q]
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
80
|
+
# Connection hardening - resolve-once + pin the IP (no TOCTOU re-resolve), and never follow redirects.
|
|
81
|
+
# The destination is the fixed Kijito API host, so there is no user-supplied URL to guard; pinning + no-redirect
|
|
82
|
+
# remain as defense-in-depth against DNS games and redirect surprises.
|
|
83
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
84
|
+
def resolve_and_pin(host, port):
|
|
85
|
+
"""Resolve host and return the first IP to pin the connection to (no re-resolve at connect time = no TOCTOU)."""
|
|
86
|
+
try:
|
|
87
|
+
infos = socket.getaddrinfo(host, port, proto=socket.IPPROTO_TCP)
|
|
88
|
+
except socket.gaierror as e:
|
|
89
|
+
raise FatalConfig("cannot resolve host %r: %s" % (host, e))
|
|
90
|
+
return infos[0][4][0]
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class _PinnedHTTPConnection(http.client.HTTPConnection):
|
|
94
|
+
def __init__(self, host, pinned_ip=None, timeout=HTTP_TIMEOUT, **kw):
|
|
95
|
+
super().__init__(host, timeout=timeout, **kw)
|
|
96
|
+
self._pinned_ip = pinned_ip
|
|
97
|
+
|
|
98
|
+
def connect(self):
|
|
99
|
+
ip = self._pinned_ip or self.host
|
|
100
|
+
self.sock = socket.create_connection((ip, self.port), self.timeout)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
class _PinnedHTTPSConnection(http.client.HTTPSConnection):
|
|
104
|
+
def __init__(self, host, pinned_ip=None, timeout=HTTP_TIMEOUT, **kw):
|
|
105
|
+
super().__init__(host, timeout=timeout, **kw)
|
|
106
|
+
self._pinned_ip = pinned_ip
|
|
107
|
+
|
|
108
|
+
def connect(self):
|
|
109
|
+
ip = self._pinned_ip or self.host
|
|
110
|
+
sock = socket.create_connection((ip, self.port), self.timeout)
|
|
111
|
+
ctx = self._context or ssl.create_default_context()
|
|
112
|
+
# connect to the pinned IP but verify the cert against the real hostname (SNI preserved)
|
|
113
|
+
self.sock = ctx.wrap_socket(sock, server_hostname=self.host)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
class _NoRedirect(urllib.request.HTTPRedirectHandler):
|
|
117
|
+
"""Redirects are never followed - a redirect is treated as an unhealthy poll."""
|
|
118
|
+
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
119
|
+
return None
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def build_opener(pinned_ip):
|
|
123
|
+
class _PinnedHTTPHandler(urllib.request.HTTPHandler):
|
|
124
|
+
def http_open(self, req):
|
|
125
|
+
return self.do_open(lambda h, **kw: _PinnedHTTPConnection(h, pinned_ip=pinned_ip, **kw), req)
|
|
126
|
+
|
|
127
|
+
class _PinnedHTTPSHandler(urllib.request.HTTPSHandler):
|
|
128
|
+
def https_open(self, req):
|
|
129
|
+
return self.do_open(lambda h, **kw: _PinnedHTTPSConnection(h, pinned_ip=pinned_ip, **kw), req)
|
|
130
|
+
|
|
131
|
+
return urllib.request.build_opener(_NoRedirect, _PinnedHTTPHandler, _PinnedHTTPSHandler)
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
135
|
+
# §5 http-poll adapter - peek + shape-validate + classify healthy/failure
|
|
136
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
137
|
+
def _is_int(v):
|
|
138
|
+
"""A REAL integer. `bool` is a subclass of int in Python, so True would otherwise satisfy every
|
|
139
|
+
isinstance(x, int) check in this file and then behave as 1 - a malformed row id, a malformed
|
|
140
|
+
size_dropped and a malformed persisted cursor all slipped through that way (Loom re-audit 6)."""
|
|
141
|
+
return isinstance(v, int) and not isinstance(v, bool)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
_MISSING = object() # "the server did not send this field at all", distinct from an explicit null
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
class Poll:
|
|
148
|
+
"""Result of one fetch. ok=True → HEALTHY (items is the validated list). ok=False → liveness FAILURE.
|
|
149
|
+
|
|
150
|
+
`omitted` carries the server's OWN declaration that this window is incomplete. The inbox endpoint
|
|
151
|
+
returns the NEWEST messages that fit a count limit AND an aggregate content budget, and reports what
|
|
152
|
+
it left out via truncated / size_truncated / size_dropped. Discarding those fields is how a bounded
|
|
153
|
+
window turns into permanent mail loss: items the server omitted are never emitted, and the cursor
|
|
154
|
+
then advances past them. The truncation is not silent in the DATA - only in the handling of it.
|
|
155
|
+
"""
|
|
156
|
+
def __init__(self, ok, items=None, reason=None, status=None, redirected=False, omitted=0,
|
|
157
|
+
omitted_exact=True, next_before_id=None, continuation_ok=True, consistent=True,
|
|
158
|
+
unread_not_shown=None):
|
|
159
|
+
self.ok = ok
|
|
160
|
+
self.items = items
|
|
161
|
+
self.reason = reason
|
|
162
|
+
self.status = status
|
|
163
|
+
self.redirected = redirected
|
|
164
|
+
self.omitted = omitted # >0 iff the server said this window is incomplete
|
|
165
|
+
self.omitted_exact = omitted_exact # False => `omitted` is only a LOWER BOUND, never closable by count
|
|
166
|
+
self.next_before_id = next_before_id # backward cursor; None when nothing older was withheld
|
|
167
|
+
# False when the server's continuation was ABSENT or MALFORMED - i.e. it never answered. Distinct
|
|
168
|
+
# from next_before_id=None, which is the server AFFIRMING there is nothing older. A walk may treat
|
|
169
|
+
# only the affirmation as terminal; silence is a contract violation and must pin.
|
|
170
|
+
self.continuation_ok = continuation_ok
|
|
171
|
+
# False when the window's OWN TWO HALVES disagree - see fetch_from_payload(). A window that
|
|
172
|
+
# contradicts itself cannot be believed in either direction, so it can neither close a span nor
|
|
173
|
+
# be walked through; it PINS. Both directions of the contradiction are covered, not just the
|
|
174
|
+
# one that happens to have been seen in the wild.
|
|
175
|
+
self.consistent = consistent
|
|
176
|
+
# Unread mail the server holds that this response did NOT hand us. None = the server did not say
|
|
177
|
+
# (older API), which is NOT the same as 0 - see _hidden_unread() for why that distinction is the
|
|
178
|
+
# whole safety property of this field.
|
|
179
|
+
self.unread_not_shown = unread_not_shown
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def fetch(opener, url, headers):
|
|
183
|
+
"""One peek fetch. Returns a Poll. A poll is HEALTHY iff 2xx AND parses AND shape-valid (§5)."""
|
|
184
|
+
req = urllib.request.Request(url, headers=headers, method="GET")
|
|
185
|
+
try:
|
|
186
|
+
with opener.open(req, timeout=HTTP_TIMEOUT) as resp:
|
|
187
|
+
status = resp.status
|
|
188
|
+
body = resp.read()
|
|
189
|
+
except urllib.error.HTTPError as e:
|
|
190
|
+
# _NoRedirect makes 3xx raise here as well as 4xx/5xx
|
|
191
|
+
if 300 <= e.code < 400:
|
|
192
|
+
return Poll(False, reason="redirect", status=e.code, redirected=True)
|
|
193
|
+
return Poll(False, reason="http %d" % e.code, status=e.code)
|
|
194
|
+
except (urllib.error.URLError, socket.timeout, ConnectionError, OSError) as e:
|
|
195
|
+
return Poll(False, reason="unreachable: %s" % e)
|
|
196
|
+
if not (200 <= status < 300):
|
|
197
|
+
return Poll(False, reason="http %d" % status, status=status)
|
|
198
|
+
try:
|
|
199
|
+
data = json.loads(body)
|
|
200
|
+
except (ValueError, UnicodeDecodeError) as e:
|
|
201
|
+
return Poll(False, reason="parse-fail: %s" % e, status=status)
|
|
202
|
+
if not isinstance(data, dict):
|
|
203
|
+
return Poll(False, reason="shape-invalid: body is not an object", status=status)
|
|
204
|
+
# ONE implementation of the body contract, shared with the tests. Two copies of a rule this subtle is
|
|
205
|
+
# two chances to get it wrong, and the tests would then be exercising the copy production does not use.
|
|
206
|
+
return fetch_from_payload(data, status=status)
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def fetch_from_payload(data, status=200):
|
|
210
|
+
"""Build a Poll from an already-decoded body. The validation path fetch() uses, exposed so tests can
|
|
211
|
+
exercise the CONTRACT (absent vs null vs malformed continuation) rather than construct Polls by hand -
|
|
212
|
+
a hand-built Poll bypasses exactly the checks under test."""
|
|
213
|
+
items = data.get("result")
|
|
214
|
+
if not isinstance(items, list):
|
|
215
|
+
return Poll(False, reason="shape-invalid: result is not a list", status=status)
|
|
216
|
+
seen_ids = set()
|
|
217
|
+
for m in items:
|
|
218
|
+
if not isinstance(m, dict) or not _is_int(m.get("id")):
|
|
219
|
+
# `bool` is a subclass of int, so an id of True would otherwise pass and then compare as 1.
|
|
220
|
+
return Poll(False, reason="shape-invalid: row missing integer id", status=status)
|
|
221
|
+
if m["id"] in seen_ids:
|
|
222
|
+
# A page cannot legitimately carry the same id twice, and the cursor logic dedupes only
|
|
223
|
+
# against what it has ALREADY delivered - so a repeat inside one window is emitted twice.
|
|
224
|
+
return Poll(False, reason="shape-invalid: duplicate id %s in one page" % m["id"], status=status)
|
|
225
|
+
seen_ids.add(m["id"])
|
|
226
|
+
n, exact = _declared_omissions(data)
|
|
227
|
+
nb_raw = data.get("next_before_id", _MISSING)
|
|
228
|
+
if nb_raw is None:
|
|
229
|
+
nb, nb_ok = None, True
|
|
230
|
+
elif isinstance(nb_raw, int) and not isinstance(nb_raw, bool) and nb_raw >= 0:
|
|
231
|
+
nb, nb_ok = nb_raw, True
|
|
232
|
+
else:
|
|
233
|
+
nb, nb_ok = None, False
|
|
234
|
+
# THE OMISSION DECLARATION AND THE CONTINUATION ARE TWO HALVES OF ONE STATEMENT, and the server emits
|
|
235
|
+
# them from a SINGLE expression - `next_before_id = oldest_row if (has_more or size_dropped) else None`
|
|
236
|
+
# (Kijito web_api.py, commented "present exactly when mail was withheld"). So a window withheld rows IF
|
|
237
|
+
# AND ONLY IF it hands back a continuation, and either half contradicting the other is a contract
|
|
238
|
+
# violation, not a quirk to interpret:
|
|
239
|
+
# withheld AND terminal -> "I hid rows" + "there is nothing older" (Loom re-audit 6, HIGH 3)
|
|
240
|
+
# withheld NOTHING AND more -> "I hid nothing" + "there is more" (Loom re-audit 7, HIGH 4)
|
|
241
|
+
# The second is not merely the theoretical twin of the first; it follows from how the window is BUILT.
|
|
242
|
+
# A page returns every older row that FIT, so if it withheld nothing there is nothing older left for a
|
|
243
|
+
# continuation to point at. Believing the "I hid nothing" half advances the cursor over whatever the
|
|
244
|
+
# other half says is still there, which is the silent-loss direction.
|
|
245
|
+
# VERIFIED against the live API across 14 pages, including the case that could have made this rule
|
|
246
|
+
# pin production forever: a page returning EXACTLY `limit` rows with more behind it declares
|
|
247
|
+
# truncated=True (limit=4 -> next_before_id=1032), while one that exactly exhausts the mailbox
|
|
248
|
+
# declares nothing and terminates (limit=5 -> next_before_id=null). The server never leaves a
|
|
249
|
+
# complete window pointing onward, so this check cannot fire on healthy traffic.
|
|
250
|
+
consistent = True
|
|
251
|
+
if nb_ok:
|
|
252
|
+
if n and nb is None:
|
|
253
|
+
consistent = False
|
|
254
|
+
elif not n and nb is not None:
|
|
255
|
+
consistent = False
|
|
256
|
+
uns = data.get("unread_not_shown")
|
|
257
|
+
# A non-int (absent, null, a string, a float) means the server made NO statement. Coercing that to 0
|
|
258
|
+
# would manufacture a "nothing is hidden" assertion out of silence - the exact inversion this field
|
|
259
|
+
# exists to avoid. Negative is nonsense from a count, so it is also treated as no statement.
|
|
260
|
+
return Poll(True, items=items, status=status, omitted=n, omitted_exact=exact,
|
|
261
|
+
next_before_id=nb, continuation_ok=nb_ok, consistent=consistent,
|
|
262
|
+
unread_not_shown=uns if isinstance(uns, int) and not isinstance(uns, bool) and uns >= 0
|
|
263
|
+
else None)
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def _declared_omissions(data):
|
|
267
|
+
"""How many messages the server says it left out of this window (0 if it says none).
|
|
268
|
+
|
|
269
|
+
Returns (count, exact). `exact` is False when the server signalled a truncation WITHOUT saying how
|
|
270
|
+
many rows it withheld - then `count` is only a LOWER BOUND, and no amount of recovered mail can prove
|
|
271
|
+
the span empty, because there is no number to reach. A gap with an inexact count must stay pinned
|
|
272
|
+
until an authoritative backward read can walk it; counting rows against a lower bound would let one
|
|
273
|
+
recovered message "close" an unbounded hole.
|
|
274
|
+
|
|
275
|
+
An alarm that invents losses is as corrosive as one that hides them, so this must not round in
|
|
276
|
+
either direction. THREE DISTINCT SIGNALS, and conflating them is wrong BOTH ways:
|
|
277
|
+
truncated=True -> rows withheld by the COUNT limit, quantity NOT stated -> inexact.
|
|
278
|
+
size_dropped=N -> exactly N rows withheld by the content budget -> exact.
|
|
279
|
+
size_truncated=True, size_dropped=0 -> a lone oversized message had its BODY clipped. No row was
|
|
280
|
+
withheld, so this contributes NOTHING. Verified live: a
|
|
281
|
+
limit=3 request returns truncated=True with size_dropped=0
|
|
282
|
+
and rows genuinely missing, while an oversized single message
|
|
283
|
+
reports size_truncated with nothing dropped.
|
|
284
|
+
"""
|
|
285
|
+
n, exact = 0, True
|
|
286
|
+
trunc = data.get("truncated", _MISSING)
|
|
287
|
+
if trunc is True:
|
|
288
|
+
n, exact = n + 1, False # count-limit truncation never states a quantity
|
|
289
|
+
elif trunc is not _MISSING and trunc is not False:
|
|
290
|
+
# A truncation flag that is neither true nor false is UNINTERPRETABLE, and reading it as "no
|
|
291
|
+
# omission" is the one direction that loses mail. Treat it as an unquantified withholding.
|
|
292
|
+
n, exact = max(n, 1), False
|
|
293
|
+
dropped = data.get("size_dropped")
|
|
294
|
+
if _is_int(dropped):
|
|
295
|
+
n += max(dropped, 0)
|
|
296
|
+
else:
|
|
297
|
+
st = data.get("size_truncated", _MISSING)
|
|
298
|
+
if st is True:
|
|
299
|
+
n, exact = max(n, 1), False # size truncation with no number at all
|
|
300
|
+
elif st is not _MISSING and st is not False:
|
|
301
|
+
n, exact = max(n, 1), False # same rule: an uninterpretable flag is not a denial
|
|
302
|
+
return (n, exact)
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
# Memory count per persona, refreshed on every directory fetch. Used by the stranded-mail check to ask
|
|
306
|
+
# "does anyone actually OWN this inbox", which survives a directory that lists every registered recipient.
|
|
307
|
+
# None (not 0) means the server did not report a count, so the check must not infer anything from it.
|
|
308
|
+
_PERSONA_MEMORY_COUNTS = {}
|
|
309
|
+
|
|
310
|
+
# Read count per persona (mail_total - unread), refreshed on every directory fetch. This is the REAL
|
|
311
|
+
# "is anyone consuming this inbox" signal, replacing the memory-count proxy for in-directory inboxes:
|
|
312
|
+
# ownership (ever authored one memory) is MONOTONIC and permanently immunises an inbox, so a typo-variant
|
|
313
|
+
# that ever received one memory (e.g. 'rvier', a variant of 'river') became invisible even while it held
|
|
314
|
+
# unread mail nobody reads. read==0 says the inbox has never been consumed - exact, no threshold.
|
|
315
|
+
# None means the server did not report both fields, so the check degrades to the memory-count signal
|
|
316
|
+
# rather than reading an unknown as zero.
|
|
317
|
+
_PERSONA_READ_COUNTS = {}
|
|
318
|
+
|
|
319
|
+
# Declared `retired` flag per persona from /api/personas. It is the DECLARED classification that separates
|
|
320
|
+
# clearable debris from a real-but-dormant inbox among inboxes that read==0: retired => loud debris,
|
|
321
|
+
# not-retired (or undeclared) => quiet dormant. None means the server did not report it → treated as
|
|
322
|
+
# not-retired (quiet), because loudly declaring an inbox clearable on absent data is the dangerous
|
|
323
|
+
# direction. A boolean, never a threshold.
|
|
324
|
+
_PERSONA_RETIRED = {}
|
|
325
|
+
|
|
326
|
+
# Declared `write_only` flag per persona from /api/personas. TRUE = an inbox that is undrained BY DESIGN:
|
|
327
|
+
# a real member whose mail is consumed through another surface (a human reading sessions/digests, never
|
|
328
|
+
# the box itself - `jason` is the live case), so read==0 is expected forever and must NEVER alarm. This
|
|
329
|
+
# is a FACT the API declares, not a policy: the classifier derives the alarm tier from it (write_only =>
|
|
330
|
+
# quiet), so if alarm policy ever changes the fact stays true. It is INDEPENDENT of `retired` - a
|
|
331
|
+
# write_only inbox is live, the opposite of clearable debris. None/absent => treated as not-write-only,
|
|
332
|
+
# so the producer can ship before the API populates the field with zero behaviour change. A boolean.
|
|
333
|
+
_PERSONA_WRITE_ONLY = {}
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def _row_memory_count(row):
|
|
337
|
+
"""Memories owned by this persona, or None if the server did not say.
|
|
338
|
+
|
|
339
|
+
Prefers the top-level `memory_count`. Deliberately does NOT fall back to summing `projects[].count`:
|
|
340
|
+
project counts exclude GLOBAL-scoped memories, so a persona whose memories are all global sums to
|
|
341
|
+
zero and looks unowned. Measured live: maestro sums to 0 across projects but owns 61 memories; the
|
|
342
|
+
same gap exists for codex, ladybug, leadgen, omniview, quill, sterling and vellum. Summing the wrong
|
|
343
|
+
field would have made the alarm cry wolf about half the fleet.
|
|
344
|
+
"""
|
|
345
|
+
n = row.get("memory_count")
|
|
346
|
+
return n if isinstance(n, int) and n >= 0 else None
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def _row_read_count(row):
|
|
350
|
+
"""Messages this persona has READ (mail_total - unread), or None if the server did not report both.
|
|
351
|
+
|
|
352
|
+
Same tri-state discipline as _row_memory_count: an UNKNOWN read count (either field missing/uninteger)
|
|
353
|
+
is None, never 0 - the stranded check must degrade to the memory-count signal instead of reading an
|
|
354
|
+
unknown as "never consumed". bool is excluded explicitly (isinstance(True, int) is True in Python), and
|
|
355
|
+
a negative result (unread somehow exceeding mail_total) is treated as unknown rather than trusted.
|
|
356
|
+
"""
|
|
357
|
+
total = row.get("mail_total")
|
|
358
|
+
unread = row.get("unread")
|
|
359
|
+
if isinstance(total, bool) or not isinstance(total, int) or total < 0:
|
|
360
|
+
return None
|
|
361
|
+
if isinstance(unread, bool) or not isinstance(unread, int) or unread < 0:
|
|
362
|
+
return None
|
|
363
|
+
read = total - unread
|
|
364
|
+
return read if read >= 0 else None
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
def _row_retired(row):
|
|
368
|
+
"""The persona's declared `retired` flag as a strict bool, or None if the server did not report it.
|
|
369
|
+
|
|
370
|
+
Only a genuine bool counts; anything else (absent, null, a string) is None = no declaration, which the
|
|
371
|
+
stranded partition treats as NOT retired (quiet/dormant). Declaring an inbox clearable debris - the
|
|
372
|
+
LOUD tier - must rest on a positive declaration, never on the absence of one.
|
|
373
|
+
"""
|
|
374
|
+
r = row.get("retired")
|
|
375
|
+
return r if isinstance(r, bool) else None
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
def _row_write_only(row):
|
|
379
|
+
"""The persona's declared `write_only` flag as a strict bool, or None if the server did not report it.
|
|
380
|
+
|
|
381
|
+
Same tri-state discipline as _row_retired: only a genuine bool is a declaration; absent/null/a string
|
|
382
|
+
is None = undeclared, which the partition treats as NOT write-only (no suppression). Quieting an inbox
|
|
383
|
+
that holds unread mail must rest on a POSITIVE declaration that it is undrained by design, never on the
|
|
384
|
+
absence of one - the mirror of the retired rule, so an absent field can never silence a real backlog.
|
|
385
|
+
"""
|
|
386
|
+
w = row.get("write_only")
|
|
387
|
+
return w if isinstance(w, bool) else None
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
def fetch_personas(opener, headers):
|
|
391
|
+
"""Fetch the account persona directory for default/explicit all-persona mode."""
|
|
392
|
+
req = urllib.request.Request(PERSONAS_URL, headers=headers, method="GET")
|
|
393
|
+
try:
|
|
394
|
+
with opener.open(req, timeout=HTTP_TIMEOUT) as resp:
|
|
395
|
+
if not (200 <= resp.status < 300):
|
|
396
|
+
raise FatalConfig("/api/personas returned http %d" % resp.status)
|
|
397
|
+
data = json.loads(resp.read())
|
|
398
|
+
except FatalConfig:
|
|
399
|
+
raise
|
|
400
|
+
except Exception as e:
|
|
401
|
+
raise FatalConfig("cannot fetch /api/personas for --all-personas: %s" % e)
|
|
402
|
+
rows = data.get("result") if isinstance(data, dict) else None
|
|
403
|
+
if not isinstance(rows, list):
|
|
404
|
+
raise FatalConfig("/api/personas shape-invalid: result is not a list")
|
|
405
|
+
personas = []
|
|
406
|
+
for row in rows:
|
|
407
|
+
if isinstance(row, dict) and isinstance(row.get("persona"), str) and row["persona"]:
|
|
408
|
+
personas.append(row["persona"])
|
|
409
|
+
_PERSONA_MEMORY_COUNTS[row["persona"]] = _row_memory_count(row)
|
|
410
|
+
_PERSONA_READ_COUNTS[row["persona"]] = _row_read_count(row)
|
|
411
|
+
_PERSONA_RETIRED[row["persona"]] = _row_retired(row)
|
|
412
|
+
_PERSONA_WRITE_ONLY[row["persona"]] = _row_write_only(row)
|
|
413
|
+
if not personas:
|
|
414
|
+
raise FatalConfig("/api/personas returned no personas")
|
|
415
|
+
return personas
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
# Urgent unread per persona, from the SAME row the unread count comes from - no extra request. Kept
|
|
419
|
+
# separately from `counts` so the fast-path arithmetic is untouched. A sender marking a message urgent is
|
|
420
|
+
# the closest thing the hive has to a declared expectation of attention, which makes it the one signal
|
|
421
|
+
# that can distinguish "idle by design" from "nobody is coming" without asking the silent party.
|
|
422
|
+
_URGENT_UNREAD = {}
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
def _parse_unread_rows(data):
|
|
426
|
+
"""Parse a /api/notify/pending body into {persona: unread}, or None if the shape is invalid.
|
|
427
|
+
A persona with zero unread is ABSENT from the list → callers treat absent as 0.
|
|
428
|
+
|
|
429
|
+
Also records `unread_urgent` into _URGENT_UNREAD as a side table. The endpoint hands it over on every
|
|
430
|
+
tick and it was previously discarded; a signal you already receive and throw away is the cheapest kind
|
|
431
|
+
of blindness.
|
|
432
|
+
"""
|
|
433
|
+
rows = data.get("result") if isinstance(data, dict) else None
|
|
434
|
+
if not isinstance(rows, list):
|
|
435
|
+
return None
|
|
436
|
+
counts = {}
|
|
437
|
+
for row in rows:
|
|
438
|
+
if isinstance(row, dict) and isinstance(row.get("persona"), str):
|
|
439
|
+
u = row.get("unread")
|
|
440
|
+
counts[row["persona"]] = u if isinstance(u, int) else 0
|
|
441
|
+
ug = row.get("unread_urgent")
|
|
442
|
+
# Absent (an older server) means NO STATEMENT, not zero - the same tri-state discipline as
|
|
443
|
+
# §5.2. Recording a 0 we were never told would assert "nothing is escalated" on no evidence.
|
|
444
|
+
if isinstance(ug, int) and not isinstance(ug, bool) and ug >= 0:
|
|
445
|
+
_URGENT_UNREAD[row["persona"]] = ug
|
|
446
|
+
return counts
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
def fetch_unread_counts(opener, count_url, headers):
|
|
450
|
+
"""§9 fast-path pre-check: GET /api/notify/pending once and fan the counts out in-process.
|
|
451
|
+
|
|
452
|
+
Returns (available, {persona: unread_count}). available=False if the endpoint is absent / non-2xx / bad shape →
|
|
453
|
+
callers fall back to the full inbox-list poll. Response: {"result":[{persona,unread,unread_urgent}]}.
|
|
454
|
+
"""
|
|
455
|
+
req = urllib.request.Request(count_url, headers=headers, method="GET")
|
|
456
|
+
try:
|
|
457
|
+
with opener.open(req, timeout=HTTP_TIMEOUT) as resp:
|
|
458
|
+
if not (200 <= resp.status < 300):
|
|
459
|
+
return (False, {})
|
|
460
|
+
data = json.loads(resp.read())
|
|
461
|
+
except Exception:
|
|
462
|
+
return (False, {})
|
|
463
|
+
counts = _parse_unread_rows(data)
|
|
464
|
+
if counts is None:
|
|
465
|
+
return (False, {})
|
|
466
|
+
return (True, counts)
|
|
467
|
+
|
|
468
|
+
|
|
469
|
+
def fetch_unread_counts_longpoll(opener, headers, wait, cursor):
|
|
470
|
+
"""Long-poll variant of the fast-path. GET /api/notify/pending?wait=<sec>[&cursor=<opaque>].
|
|
471
|
+
|
|
472
|
+
The server holds the request up to `wait` seconds, returning the instant the account's mail-state advances
|
|
473
|
+
beyond `cursor` (else on timeout). Returns (available, {persona: unread}, cursor):
|
|
474
|
+
- `cursor` is the server's OPAQUE token to echo on the next call - NEVER parse it.
|
|
475
|
+
- available=False on any connection error / non-2xx / bad shape → the caller falls back to the full inbox poll
|
|
476
|
+
and RECONNECTS WITH THE SAME cursor (lossless resume across a wifi/NAT/Cloudflare/server-restart drop).
|
|
477
|
+
- cursor is None when the server did NOT long-poll (no `cursor` field): the endpoint predates long-poll, so the
|
|
478
|
+
caller interval-polls. This makes the client safe to ship BEFORE the server supports it - it interval-polls
|
|
479
|
+
today and auto-upgrades to instant the moment a cursor starts coming back, no redeploy.
|
|
480
|
+
The client socket timeout is wait+LONGPOLL_SLACK so a half-open held connection is detected, never hung.
|
|
481
|
+
"""
|
|
482
|
+
q = {"wait": str(wait)}
|
|
483
|
+
if cursor is not None:
|
|
484
|
+
q["cursor"] = cursor
|
|
485
|
+
url = NOTIFY_PENDING_URL + "?" + urllib.parse.urlencode(q)
|
|
486
|
+
req = urllib.request.Request(url, headers=headers, method="GET")
|
|
487
|
+
try:
|
|
488
|
+
with opener.open(req, timeout=wait + LONGPOLL_SLACK) as resp:
|
|
489
|
+
if not (200 <= resp.status < 300):
|
|
490
|
+
return (False, {}, cursor)
|
|
491
|
+
data = json.loads(resp.read())
|
|
492
|
+
except Exception:
|
|
493
|
+
return (False, {}, cursor) # keep the old cursor → next attempt resumes losslessly
|
|
494
|
+
counts = _parse_unread_rows(data)
|
|
495
|
+
if counts is None:
|
|
496
|
+
return (False, {}, cursor)
|
|
497
|
+
new_cursor = data.get("cursor")
|
|
498
|
+
if not isinstance(new_cursor, str) or not new_cursor:
|
|
499
|
+
new_cursor = None # server didn't long-poll → caller interval-polls (forward/back-compat)
|
|
500
|
+
return (True, counts, new_cursor)
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
504
|
+
# §6 Emit
|
|
505
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
506
|
+
def _now_iso():
|
|
507
|
+
return datetime.datetime.now(datetime.timezone.utc).isoformat()
|
|
508
|
+
|
|
509
|
+
|
|
510
|
+
def _safe_text(s):
|
|
511
|
+
"""Return `s` in a form that can ALWAYS be written to a UTF-8 file and put in a child's environment.
|
|
512
|
+
|
|
513
|
+
A CONTENT BYTE MUST NEVER BECOME A DELIVERY FAILURE. Two shapes arrive from real message bodies and
|
|
514
|
+
neither is our bug to have opinions about:
|
|
515
|
+
· a NUL, which cannot appear in an environment value at all (subprocess raises ValueError);
|
|
516
|
+
· a lone surrogate, which cannot be encoded to UTF-8 (json.dumps(ensure_ascii=False) passes it
|
|
517
|
+
straight through, and the file write then raises UnicodeEncodeError).
|
|
518
|
+
Both used to matter less because emit failures were swallowed. With the round-7 delivery gate they
|
|
519
|
+
would matter enormously: the exec path would report a permanent non-delivery and WEDGE THE WATERMARK
|
|
520
|
+
on that one message forever, and the file path would raise straight out of poll_once and crash the
|
|
521
|
+
producer - which under a KeepAlive supervisor is a crash LOOP, since the same message is refetched
|
|
522
|
+
every time. Trading a silent skip for a permanent stall is the failure this project keeps re-learning
|
|
523
|
+
(see the corruption pin), so the fix is to make the event REPRESENTABLE rather than to fail on it.
|
|
524
|
+
"""
|
|
525
|
+
try:
|
|
526
|
+
s.encode("utf-8")
|
|
527
|
+
except UnicodeEncodeError:
|
|
528
|
+
s = s.encode("utf-8", "replace").decode("utf-8")
|
|
529
|
+
return s.replace("\x00", "") if "\x00" in s else s
|
|
530
|
+
|
|
531
|
+
|
|
532
|
+
class RotatingFileSink:
|
|
533
|
+
"""Owns the events-log fd and rotates it by size IN-PROCESS, so the writer reopens after its OWN rename.
|
|
534
|
+
|
|
535
|
+
Why this exists: a launchd StandardOutPath fd is NEVER reopened by launchd when an external rotator
|
|
536
|
+
(newsyslog) renames the file - the producer would keep appending to the orphaned inode while a `tail -F`
|
|
537
|
+
consumer follows the new empty file → SILENT blinding (the exact failure class this tool fights). Owning
|
|
538
|
+
the fd here and reopening after our OWN rename closes that hole with no external dependency and no sudo;
|
|
539
|
+
consumers just tail -F by name. max_bytes <= 0 disables rotation (unbounded)."""
|
|
540
|
+
def __init__(self, path, max_bytes, keep):
|
|
541
|
+
self.path = path
|
|
542
|
+
self.max_bytes = max_bytes
|
|
543
|
+
self.keep = max(1, keep)
|
|
544
|
+
self._fh = None
|
|
545
|
+
self._pending = False # bytes written that are not known to be on stable storage yet
|
|
546
|
+
self._sync_failed = False # an fsync we can never retry (the fd was rotated away) failed
|
|
547
|
+
self._dir_pending = False # a directory ENTRY changed (create/rotate) and is not durable yet
|
|
548
|
+
self._broken = None # non-None => the sink is unusable; write() reports a FAILED delivery
|
|
549
|
+
self._open()
|
|
550
|
+
|
|
551
|
+
def _open(self):
|
|
552
|
+
# Never abandon a live handle: any path that reopens without closing first leaks an fd per
|
|
553
|
+
# rotation, which in a long-lived producer is unbounded. Defensive rather than reactive - the
|
|
554
|
+
# callers currently all close first, and this makes that non-load-bearing.
|
|
555
|
+
if self._fh is not None:
|
|
556
|
+
try:
|
|
557
|
+
self._fh.close()
|
|
558
|
+
except OSError:
|
|
559
|
+
pass
|
|
560
|
+
self._fh = None
|
|
561
|
+
dirn = os.path.dirname(os.path.abspath(self.path)) or "."
|
|
562
|
+
_makedirs_private(dirn)
|
|
563
|
+
existed = os.path.lexists(self.path) # lexists: a dangling SYMLINK counts as present, and must
|
|
564
|
+
self._fh = _open_private(self.path, "a", encoding="utf-8")
|
|
565
|
+
# EVERY persisted artifact, not just the one we opened (Loom re-audit 9, H2): rotated archives
|
|
566
|
+
# written by an older version keep their 0644 forever otherwise, because they are never reopened.
|
|
567
|
+
# THE RANGE COMES FROM THE DIRECTORY, NOT FROM `keep` (Loom re-audit 10, M4). A bound derived from
|
|
568
|
+
# CURRENT retention cannot reach an artifact left by a LARGER FORMER retention - shrinking keep
|
|
569
|
+
# from 10 to 5 stranded .7 at 0644 permanently - and an increment-until-absent scan would stop at
|
|
570
|
+
# the first hole a hand-deleted archive leaves. Listing is the only bound config cannot outlive.
|
|
571
|
+
# ★ THE VERDICT IS DELIBERATELY IGNORED HERE (loom's class, half A - "who consumes this?"). It is
|
|
572
|
+
# consumed by _repair_mode itself, which warns per file. Escalating would be wrong in both
|
|
573
|
+
# directions: these are ARCHIVES, not the live sink, so refusing to open the events file because a
|
|
574
|
+
# months-old archive is unreadable converts a stale-permission leak into a total delivery outage.
|
|
575
|
+
for archive in self._archive_paths():
|
|
576
|
+
_repair_mode(archive)
|
|
577
|
+
if not existed:
|
|
578
|
+
# A NEW FILE NEEDS ITS DIRECTORY ENTRY SYNCED, NOT JUST ITS BYTES (Loom re-audit 8, HIGH 2).
|
|
579
|
+
# fsync on the fd makes the CONTENT durable; the NAME lives in the directory. Deferred to
|
|
580
|
+
# sync() so it lands before the cursor that acknowledges these events is persisted.
|
|
581
|
+
self._dir_pending = True
|
|
582
|
+
|
|
583
|
+
def _archive_paths(self):
|
|
584
|
+
"""Every rotated archive of this sink that EXISTS RIGHT NOW, found by listing the directory.
|
|
585
|
+
|
|
586
|
+
Deliberately not `range(1, keep + 2)`: that bound is CURRENT config, and the artifacts most likely
|
|
587
|
+
to be left at a permissive mode are exactly the ones a FORMER, larger retention wrote (Loom
|
|
588
|
+
re-audit 10, M4). Matches `<basename>.<digits>` only, so the `.lock` sidecar is never touched.
|
|
589
|
+
"""
|
|
590
|
+
d = os.path.dirname(os.path.abspath(self.path)) or "."
|
|
591
|
+
base = os.path.basename(self.path)
|
|
592
|
+
try:
|
|
593
|
+
names = os.listdir(d)
|
|
594
|
+
except OSError:
|
|
595
|
+
return [] # unreadable directory: nothing to enumerate, and _open still has to proceed
|
|
596
|
+
return sorted(os.path.join(d, n) for n in names
|
|
597
|
+
if n.startswith(base + ".") and n[len(base) + 1:].isdigit())
|
|
598
|
+
|
|
599
|
+
def _reopen_or_break(self):
|
|
600
|
+
"""Reopen after a rotation. A failure here must NOT escape the poll loop (Loom re-audit 9,
|
|
601
|
+
MEDIUM): an exception out of write() unwinds through poll_once and, under a KeepAlive supervisor,
|
|
602
|
+
is a crash loop. It becomes a broken sink instead, which write() reports as a failed delivery, so
|
|
603
|
+
the cursor holds and the mail is re-delivered when the sink recovers."""
|
|
604
|
+
try:
|
|
605
|
+
self._open()
|
|
606
|
+
self._broken = None
|
|
607
|
+
except OSError as e:
|
|
608
|
+
self._broken = str(e)
|
|
609
|
+
self._fh = None
|
|
610
|
+
sys.stderr.write("kijito-inbox-monitor: WARNING events sink %s is unusable (%s); holding the "
|
|
611
|
+
"cursor until it recovers\n" % (self.path, e))
|
|
612
|
+
|
|
613
|
+
def write(self, line):
|
|
614
|
+
"""Append one event line. Returns True IFF the line reached the file.
|
|
615
|
+
|
|
616
|
+
A FAILED write must never be reported as a delivery. The cursor that acknowledges an event is
|
|
617
|
+
persisted from the same poll, so swallowing an OSError here would advance the watermark over a
|
|
618
|
+
message nobody received - the exact silent loss this tool exists to prevent, arriving through
|
|
619
|
+
the emit path instead of the fetch path.
|
|
620
|
+
"""
|
|
621
|
+
if self._broken is not None or self._fh is None:
|
|
622
|
+
self._reopen_or_break() # try to recover, silently on success
|
|
623
|
+
if self._broken is not None or self._fh is None:
|
|
624
|
+
return False
|
|
625
|
+
try:
|
|
626
|
+
self._fh.write(line)
|
|
627
|
+
self._fh.flush()
|
|
628
|
+
except (OSError, UnicodeError, ValueError) as e:
|
|
629
|
+
# UnicodeError/ValueError are belt-and-braces behind _safe_text(): an event that still cannot
|
|
630
|
+
# be encoded must be a FAILED DELIVERY (loud, retried, visible on stderr) and never an
|
|
631
|
+
# exception escaping poll_once, which under a KeepAlive supervisor is a silent crash loop.
|
|
632
|
+
sys.stderr.write("kijito-inbox-monitor: WARNING event write FAILED, holding the cursor: %s\n" % e)
|
|
633
|
+
return False
|
|
634
|
+
self._pending = True
|
|
635
|
+
self._maybe_rotate()
|
|
636
|
+
return True
|
|
637
|
+
|
|
638
|
+
def sync(self):
|
|
639
|
+
"""Force written events onto stable storage. Returns True IFF they are durable.
|
|
640
|
+
|
|
641
|
+
THE DURABILITY BARRIER (Loom re-audit 7, MEDIUM). flush() only moves bytes from Python's buffer
|
|
642
|
+
into the kernel's; a power loss between the flush and the writeback loses them. The state file
|
|
643
|
+
IS fsynced, so without this the CURSOR can outlive the EVENT it acknowledges - the watcher comes
|
|
644
|
+
back believing it delivered mail that no consumer ever saw, and never fetches it again. Ordering,
|
|
645
|
+
not just syncing, is what matters: event durable BEFORE cursor durable.
|
|
646
|
+
"""
|
|
647
|
+
failed_earlier, self._sync_failed = self._sync_failed, False
|
|
648
|
+
ok = True
|
|
649
|
+
if self._dir_pending:
|
|
650
|
+
# THE NAME AS WELL AS THE BYTES. Syncing only the fd leaves a cursor that can outlive the
|
|
651
|
+
# PATHNAME of the events it acknowledges - and the state file may live in a DIFFERENT
|
|
652
|
+
# directory (--state-file and --events-file-template are independent), so syncing the state
|
|
653
|
+
# directory proves nothing about this one.
|
|
654
|
+
if _fsync_dir(os.path.dirname(os.path.abspath(self.path)) or "."):
|
|
655
|
+
self._dir_pending = False
|
|
656
|
+
else:
|
|
657
|
+
sys.stderr.write("kijito-inbox-monitor: WARNING could not fsync the events directory; "
|
|
658
|
+
"holding the cursor\n")
|
|
659
|
+
ok = False
|
|
660
|
+
if self._pending:
|
|
661
|
+
try:
|
|
662
|
+
self._fh.flush()
|
|
663
|
+
os.fsync(self._fh.fileno())
|
|
664
|
+
self._pending = False
|
|
665
|
+
except OSError as e:
|
|
666
|
+
sys.stderr.write("kijito-inbox-monitor: WARNING event fsync FAILED, holding the cursor: %s\n" % e)
|
|
667
|
+
ok = False
|
|
668
|
+
return ok and not failed_earlier
|
|
669
|
+
|
|
670
|
+
def _maybe_rotate(self):
|
|
671
|
+
if self.max_bytes <= 0:
|
|
672
|
+
return
|
|
673
|
+
try:
|
|
674
|
+
size = os.fstat(self._fh.fileno()).st_size
|
|
675
|
+
except OSError:
|
|
676
|
+
return
|
|
677
|
+
if size < self.max_bytes:
|
|
678
|
+
return
|
|
679
|
+
# Sync BEFORE the rename: after os.replace this fd names the archive, so a later sync() cannot
|
|
680
|
+
# make these bytes durable. If it fails, remember it - the next sync() must report non-durable
|
|
681
|
+
# once (holding the cursor for one poll) rather than silently losing the signal.
|
|
682
|
+
if not self.sync():
|
|
683
|
+
self._sync_failed = True
|
|
684
|
+
try:
|
|
685
|
+
try:
|
|
686
|
+
self._fh.close()
|
|
687
|
+
except OSError as e: # a close() failure is a real error, not a reason to unwind the poll
|
|
688
|
+
sys.stderr.write("kijito-inbox-monitor: WARNING closing %s during rotation failed: %s\n"
|
|
689
|
+
% (self.path, e))
|
|
690
|
+
oldest = "%s.%d" % (self.path, self.keep)
|
|
691
|
+
if os.path.exists(oldest):
|
|
692
|
+
os.remove(oldest)
|
|
693
|
+
for i in range(self.keep - 1, 0, -1):
|
|
694
|
+
src = "%s.%d" % (self.path, i)
|
|
695
|
+
if os.path.exists(src):
|
|
696
|
+
os.replace(src, "%s.%d" % (self.path, i + 1))
|
|
697
|
+
if os.path.exists(self.path):
|
|
698
|
+
os.replace(self.path, "%s.1" % self.path)
|
|
699
|
+
except OSError as e:
|
|
700
|
+
sys.stderr.write("kijito-inbox-monitor: WARNING log rotation failed (non-fatal): %s\n" % e)
|
|
701
|
+
finally:
|
|
702
|
+
# Every rename above rewrote directory ENTRIES; none of them is durable until the directory
|
|
703
|
+
# itself is synced (Loom re-audit 8, HIGH 2).
|
|
704
|
+
self._dir_pending = True
|
|
705
|
+
self._reopen_or_break() # reopen by NAME - a tail -F consumer follows us onto the fresh file
|
|
706
|
+
|
|
707
|
+
def close(self):
|
|
708
|
+
if self._fh is not None:
|
|
709
|
+
try:
|
|
710
|
+
self._fh.close()
|
|
711
|
+
except OSError as e:
|
|
712
|
+
sys.stderr.write("kijito-inbox-monitor: WARNING closing %s failed: %s\n" % (self.path, e))
|
|
713
|
+
finally:
|
|
714
|
+
self._fh = None
|
|
715
|
+
|
|
716
|
+
|
|
717
|
+
_BROKEN_SINK = object() # a sink that exists in config but cannot be written to SAFELY
|
|
718
|
+
|
|
719
|
+
_BASE62 = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
|
|
720
|
+
|
|
721
|
+
|
|
722
|
+
def _wake_nonce(event_id):
|
|
723
|
+
"""The wake nonce: 11 base62 characters DERIVED from the event_id, never minted beside it.
|
|
724
|
+
|
|
725
|
+
WHY DERIVED, AND WHY THIS IS THE LOAD-BEARING DECISION.
|
|
726
|
+
The consumer-side wake ledger needs a nonce with "recompute-asserted uniqueness". The obvious
|
|
727
|
+
reading -- mint a fresh random one per emission -- silently introduces a SECOND identity that
|
|
728
|
+
CONTRADICTS the one this emitter already has. `_event_id` deliberately gives a `new` event the
|
|
729
|
+
same id across a restart, a re-delivery after state loss, and two watchers of the same inbox,
|
|
730
|
+
because a duplicated MESSAGE is duplicated WORK. A per-emission random nonce would call those
|
|
731
|
+
two different wakes; the consumer would find no queue entry containing the second one, score it
|
|
732
|
+
LOST, and PAGE -- on precisely the recovery path this producer exists to survive.
|
|
733
|
+
|
|
734
|
+
⚠️ ERRATUM ON "a re-delivery after state loss" ABOVE (disclosed 2026-08-05, found by a drill).
|
|
735
|
+
That phrase names an EMITTER capability and a WATCHER trigger as if they were one thing; they
|
|
736
|
+
live in different components. The emitter does handle a re-delivery correctly when one occurs.
|
|
737
|
+
But the WATCHER does not produce one by losing its state file: an absent state file BASELINES to
|
|
738
|
+
the newest visible id (see poll_once's absent-state branch), so the backlog is skipped, not
|
|
739
|
+
re-emitted. Wiping the state file was MEASURED not to re-deliver. The path that does reach it is
|
|
740
|
+
an UNACKNOWLEDGED delivery -- a refused sink, a non-zero --exec -- where the cursor is held below
|
|
741
|
+
the message and the next poll re-delivers it. Do not cite the state-loss case as evidence that
|
|
742
|
+
re-delivery works: it is the one case that cannot reach it.
|
|
743
|
+
|
|
744
|
+
⇒ Deriving from the event_id makes the nonce stable exactly where the event_id is stable and
|
|
745
|
+
distinct exactly where it is distinct. Signal events already get a per-emission-unique id
|
|
746
|
+
(`<persona>:<event>:<run>-<seq>`, 64 bits of per-run entropy), so ONE rule serves both families
|
|
747
|
+
and neither family's meaning changes. "Recompute-asserted" also becomes literally true: any
|
|
748
|
+
auditor recomputes this from the event_id in the same row.
|
|
749
|
+
|
|
750
|
+
★ THE RULING'S STRONGEST GROUND (river): random DESTROYS information at the producer -- "this is
|
|
751
|
+
the same work re-delivered" becomes unrecoverable downstream because the identity that would
|
|
752
|
+
have said so was never minted. Derived merely DEFERS a decision to the consumer, where a missing
|
|
753
|
+
outcome column can supply it. Between two schemes that each have a false-page mode, take the one
|
|
754
|
+
whose defect is repairable.
|
|
755
|
+
|
|
756
|
+
⚠️ 11 IS FORCED, NOT CHOSEN: the spec wants >=64 bits in <=11 base62 chars. 10 chars = 59.54 bits
|
|
757
|
+
(fails the floor), 11 = 65.50 (fits), 12 breaks the ceiling. There is no slack in either direction.
|
|
758
|
+
|
|
759
|
+
⛔ THIS IS AN ATTRIBUTION LABEL, NOT A CAPABILITY. It is deterministic and therefore GUESSABLE by
|
|
760
|
+
anyone who knows the event_id. Nothing may treat nonce-presence as evidence of authenticity; a
|
|
761
|
+
forger able to write transcript rows already has what it needs and gains nothing from this value.
|
|
762
|
+
If a consumer ever requires an UNGUESSABLE nonce, this derivation is wrong for it and the choice
|
|
763
|
+
must be revisited rather than patched.
|
|
764
|
+
|
|
765
|
+
⚠️ IDENTIFIES A WAKE, NOT A DELIVERY. Two different panes delivered the same message carry the
|
|
766
|
+
SAME nonce -- correctly, it is the same work. Consumer ledgers must therefore key rows on
|
|
767
|
+
(nonce, session_id), never on the nonce alone, or two panes' deliveries collide into one row.
|
|
768
|
+
"""
|
|
769
|
+
v = int.from_bytes(hashlib.sha256(event_id.encode("utf-8")).digest(), "big")
|
|
770
|
+
out = []
|
|
771
|
+
for _ in range(11):
|
|
772
|
+
out.append(_BASE62[v % 62])
|
|
773
|
+
v //= 62
|
|
774
|
+
return "".join(out)
|
|
775
|
+
|
|
776
|
+
|
|
777
|
+
def _clock_map():
|
|
778
|
+
"""Map our two SEMANTICS onto this platform's constants. Returns {key: (semantic_name, const)}.
|
|
779
|
+
|
|
780
|
+
⛔⛔ THE KEYS NAME SEMANTICS, NOT OS CONSTANTS, AND THE TWO DISAGREE ACROSS PLATFORMS.
|
|
781
|
+
monotonic := DOES NOT advance while the machine is not executing, and NEVER steps
|
|
782
|
+
boottime := DOES advance while the machine is not executing, and NEVER steps
|
|
783
|
+
|
|
784
|
+
On Linux those are CLOCK_MONOTONIC and CLOCK_BOOTTIME, and the names coincide with the meanings.
|
|
785
|
+
ON DARWIN THEY DO NOT, AND THERE ARE TWO SEPARATE TRAPS, BOTH SILENT:
|
|
786
|
+
|
|
787
|
+
CLOCK_UPTIME_RAW EXCLUDES sleep, raw -> carries Linux CLOCK_MONOTONIC's semantic
|
|
788
|
+
CLOCK_MONOTONIC_RAW INCLUDES sleep, raw -> carries Linux CLOCK_BOOTTIME's semantic
|
|
789
|
+
CLOCK_MONOTONIC INCLUDES sleep BUT IS CALENDAR-DERIVED: measured 2026-08-15 it read
|
|
790
|
+
EXACTLY wall - kern.boottime (201341.498, to 3 decimals), so it absorbs
|
|
791
|
+
NTP adjustments to the wall clock -- it can sit BELOW CLOCK_UPTIME_RAW
|
|
792
|
+
CLOCK_BOOTTIME does not exist
|
|
793
|
+
|
|
794
|
+
Measured on the real Mac (2026-08-05): the sleep-including clocks ran 18.20 h ahead of
|
|
795
|
+
CLOCK_UPTIME_RAW -- a difference that IS the accumulated sleep, matching an independent
|
|
796
|
+
kern.boottime derivation to two decimals.
|
|
797
|
+
|
|
798
|
+
An earlier version of this function read CLOCK_MONOTONIC on every platform and omitted boottime
|
|
799
|
+
where the constant was missing. On a Mac that emits the SLEEP-INCLUDING clock under the key
|
|
800
|
+
`monotonic`, and drops the sleep-excluding quantity entirely -- so a consumer differencing
|
|
801
|
+
wall against `monotonic` measures ~0 freeze forever, on every Mac-emitted row, with nothing
|
|
802
|
+
raising. The bug is invisible to a Linux test suite by construction: there, the names are honest.
|
|
803
|
+
|
|
804
|
+
The NEXT version sourced Darwin's boottime from CLOCK_MONOTONIC -- right direction, wrong clock:
|
|
805
|
+
on a fresh-uptime Mac whose wall clock NTP-stepped back ~8.3 s after boot, it emitted
|
|
806
|
+
boottime 4957.865 < monotonic 4966.194 (measured 2026-08-14), violating the definitional
|
|
807
|
+
invariant boottime >= monotonic that consumers difference against. The never-steps half of each
|
|
808
|
+
semantic is as load-bearing as the sleep half, so BOTH must come from RAW clocks:
|
|
809
|
+
CLOCK_MONOTONIC_RAW and CLOCK_UPTIME_RAW share one tick source (mach_continuous_time vs
|
|
810
|
+
mach_absolute_time), so boottime >= monotonic holds by construction.
|
|
811
|
+
|
|
812
|
+
⇒ Dispatch on the SEMANTIC and record which constant supplied it (see _emission_stamps), so the
|
|
813
|
+
mapping is auditable from the row instead of being a property of the reader's assumptions.
|
|
814
|
+
"""
|
|
815
|
+
m = {}
|
|
816
|
+
if hasattr(time, "CLOCK_UPTIME_RAW"): # Darwin: the sleep-EXCLUDING raw clock
|
|
817
|
+
m["monotonic"] = ("CLOCK_UPTIME_RAW", time.CLOCK_UPTIME_RAW)
|
|
818
|
+
elif hasattr(time, "CLOCK_MONOTONIC"): # Linux: names and meanings coincide
|
|
819
|
+
m["monotonic"] = ("CLOCK_MONOTONIC", time.CLOCK_MONOTONIC)
|
|
820
|
+
if hasattr(time, "CLOCK_BOOTTIME"): # Linux: the sleep-INCLUDING clock
|
|
821
|
+
m["boottime"] = ("CLOCK_BOOTTIME", time.CLOCK_BOOTTIME)
|
|
822
|
+
elif hasattr(time, "CLOCK_UPTIME_RAW") and hasattr(time, "CLOCK_MONOTONIC_RAW"):
|
|
823
|
+
# Darwin: the sleep-INCLUDING raw clock. NOT CLOCK_MONOTONIC -- that one is
|
|
824
|
+
# calendar-derived and absorbs NTP steps (see above). Gate on UPTIME_RAW's presence:
|
|
825
|
+
# it identifies the platform as Darwin, where MONOTONIC_RAW includes sleep, rather than
|
|
826
|
+
# a Linux box, where CLOCK_MONOTONIC_RAW EXCLUDES suspend and would be the wrong clock.
|
|
827
|
+
m["boottime"] = ("CLOCK_MONOTONIC_RAW", time.CLOCK_MONOTONIC_RAW)
|
|
828
|
+
return m
|
|
829
|
+
|
|
830
|
+
|
|
831
|
+
def _quarantine_inverted_stamps(stamps, src):
|
|
832
|
+
"""The emission-chokepoint canary: quarantine a (monotonic, boottime) pair that reads inverted.
|
|
833
|
+
|
|
834
|
+
boottime >= monotonic is definitional -- boottime is monotonic plus the time the machine did
|
|
835
|
+
not execute. A pair that reads inverted means the platform mapping above is WRONG (the defect
|
|
836
|
+
this module actually shipped: Darwin boottime from calendar-derived CLOCK_MONOTONIC read 8.3 s
|
|
837
|
+
below monotonic at fresh uptime), and the boottime value is then a NON-measurement of its
|
|
838
|
+
semantic. Publishing it would poison every consumer that differences boottime against
|
|
839
|
+
monotonic to measure dwell -- so the pair is split at the chokepoint: boottime is REMOVED from
|
|
840
|
+
the stamp set (omitted, never faked -- absence stays legible as absence) and the rejected
|
|
841
|
+
reading is preserved LOUDLY under `clock_defect`, so the row itself reports the broken mapping
|
|
842
|
+
instead of feeding it to consumers as data. Mutates stamps/src in place; returns stamps.
|
|
843
|
+
"""
|
|
844
|
+
if "monotonic" in stamps and "boottime" in stamps and stamps["boottime"] < stamps["monotonic"]:
|
|
845
|
+
stamps["clock_defect"] = {
|
|
846
|
+
"kind": "boottime_below_monotonic",
|
|
847
|
+
"boottime": stamps.pop("boottime"),
|
|
848
|
+
"boottime_src": src.pop("boottime", None),
|
|
849
|
+
"monotonic": stamps["monotonic"],
|
|
850
|
+
}
|
|
851
|
+
return stamps
|
|
852
|
+
|
|
853
|
+
|
|
854
|
+
def _emission_stamps():
|
|
855
|
+
"""The producer-emission stamp: a coherent set of clocks read at one instant.
|
|
856
|
+
|
|
857
|
+
Three readings because none answers alone:
|
|
858
|
+
wall - comparable across hosts and to every other timestamp in the system, but it STEPS
|
|
859
|
+
(NTP, hypervisor time sync), so a wall delta is not an elapsed time.
|
|
860
|
+
monotonic - never steps, but STOPS while the machine is not executing.
|
|
861
|
+
boottime - like monotonic, except it keeps counting while the machine is not executing.
|
|
862
|
+
|
|
863
|
+
Differencing them across two events is what makes dwell measurable rather than assumed:
|
|
864
|
+
(wall delta - monotonic delta) over an interval is the time the machine DID NOT EXECUTE, which is
|
|
865
|
+
the difference between "this wake sat in a queue for three hours" and "the host was frozen".
|
|
866
|
+
Measured on this seat, the two are routinely confused: 72.79 h of hypervisor freeze presented as
|
|
867
|
+
ordinary elapsed wall time, with BOOTTIME - MONOTONIC reading exactly 0.00 s throughout, because a
|
|
868
|
+
hypervisor pause stops the guest's clocks TOGETHER and the guest is not running to notice.
|
|
869
|
+
|
|
870
|
+
`src` records which OS constant supplied each semantic, so a consumer can AUDIT the mapping from
|
|
871
|
+
the row rather than assuming the platform's names mean what they say -- see _clock_map(), where
|
|
872
|
+
Darwin's do not. A key is OMITTED, never faked, where its semantic is genuinely unavailable: a
|
|
873
|
+
fabricated value is indistinguishable from a real zero-freeze reading, which is the exact failure
|
|
874
|
+
these fields exist to detect. The same rule governs a pair the platform hands us inverted --
|
|
875
|
+
_quarantine_inverted_stamps drops the boottime reading and reports the defect on the row.
|
|
876
|
+
"""
|
|
877
|
+
stamps = {"wall": _now_iso()}
|
|
878
|
+
src = {}
|
|
879
|
+
for key, (const_name, const) in _clock_map().items():
|
|
880
|
+
stamps[key] = round(time.clock_gettime(const), 6)
|
|
881
|
+
src[key] = const_name
|
|
882
|
+
_quarantine_inverted_stamps(stamps, src)
|
|
883
|
+
if src:
|
|
884
|
+
stamps["src"] = src
|
|
885
|
+
return stamps
|
|
886
|
+
|
|
887
|
+
|
|
888
|
+
# ----------------------------------------------------------------------------------------------------------------
|
|
889
|
+
# §7.1b WAKE CLASS - so a consumer's filter stops having to learn every new event name (v16 phase 1)
|
|
890
|
+
# ----------------------------------------------------------------------------------------------------------------
|
|
891
|
+
# THE DEFECT THIS CLOSES. Every consumer in the fleet filters on a NAME ALLOWLIST -
|
|
892
|
+
# `"event": ?"(new|alert|recovered)"`. So every diagnostic this module added to kill a silent failure
|
|
893
|
+
# was ITSELF silent: state_corrupt, baseline_skipped, seed_ahead, replay_capped and persona_added
|
|
894
|
+
# matched nobody's filter, and a running `grep` never re-reads its argv, so they stayed invisible even
|
|
895
|
+
# after the docs were fixed. cadence's statement of it: "a diagnostic added to kill a silent failure is
|
|
896
|
+
# itself silent unless the consumer's filter learned its name."
|
|
897
|
+
#
|
|
898
|
+
# THE FIX IS A CLASS THE CONSUMER MATCHES STRUCTURALLY, so a new kind is covered the day it is added
|
|
899
|
+
# rather than the day every seat is re-armed. Three rules make it structural rather than cosmetic:
|
|
900
|
+
#
|
|
901
|
+
# 1. AN UNCLASSIFIED KIND WAKES. The default is `diagnostic`, NOT `liveness` - a kind whose author
|
|
902
|
+
# forgot to classify it wakes people, so the omission is visible immediately instead of silently
|
|
903
|
+
# muting a channel. You cannot FALL INTO the suppressing value; it must be typed deliberately.
|
|
904
|
+
# 2. `liveness` IS A CLOSED SET, ASSERTED BY A TEST. Its exclusion is load-bearing: `heartbeat` fires
|
|
905
|
+
# every 900 s, and `armed`'s exclusion is why "I was not woken" does not mean "nothing arrived".
|
|
906
|
+
# A small, deliberately-frozen suppression set is the one place an allowlist is correct.
|
|
907
|
+
# 3. ONE TABLE, NEXT TO THE CHOKEPOINT. Not a classification scattered across construction sites -
|
|
908
|
+
# that is precisely how the consumer-side allowlist rotted in the first place.
|
|
909
|
+
#
|
|
910
|
+
# ⚠️ PHASE 1 ONLY. The producer stamps; `event` is UNTOUCHED, so every existing filter keeps working
|
|
911
|
+
# byte-for-byte and there is no flag day. Consumers switch to `wake_class` per seat, at each owner's
|
|
912
|
+
# pace (phase 2), and the name allowlist dies only when none of them match on `event` (phase 3).
|
|
913
|
+
# ⛔ A consumer that matches `wake_class` against a producer that does not emit it matches NOTHING -
|
|
914
|
+
# a fleet-wide wake outage delivered by the fix for a wake outage. Hence producer FIRST, always.
|
|
915
|
+
WAKE_CLASS_MAIL = "mail" # a real inbox message
|
|
916
|
+
WAKE_CLASS_DIAGNOSTIC = "diagnostic" # the producer is reporting something wrong or surprising
|
|
917
|
+
WAKE_CLASS_LIVENESS = "liveness" # routine "I am alive" ticks - the ONLY suppressing value
|
|
918
|
+
|
|
919
|
+
# The suppression set, closed and frozen. Adding a member here silently mutes a channel, so a test
|
|
920
|
+
# asserts this exact membership and a third member fails the suite.
|
|
921
|
+
_LIVENESS_KINDS = frozenset({"heartbeat", "armed"})
|
|
922
|
+
|
|
923
|
+
# Every kind this module can emit. `new` is the only mail; everything that is not mail and not
|
|
924
|
+
# liveness is a diagnostic, INCLUDING kinds absent from this table (see _wake_class).
|
|
925
|
+
_WAKE_CLASS_BY_KIND = {
|
|
926
|
+
"new": WAKE_CLASS_MAIL,
|
|
927
|
+
"alert": WAKE_CLASS_DIAGNOSTIC,
|
|
928
|
+
"recovered": WAKE_CLASS_DIAGNOSTIC,
|
|
929
|
+
"state_corrupt": WAKE_CLASS_DIAGNOSTIC,
|
|
930
|
+
"baseline_skipped": WAKE_CLASS_DIAGNOSTIC,
|
|
931
|
+
"seed_ahead": WAKE_CLASS_DIAGNOSTIC,
|
|
932
|
+
"replay_capped": WAKE_CLASS_DIAGNOSTIC,
|
|
933
|
+
# persona_added is listed EXPLICITLY rather than left to the default. It would reach the right
|
|
934
|
+
# answer either way, and that is the problem: correct-by-accident is not correct. The catch-all
|
|
935
|
+
# exists for kinds nobody has thought of, not for kinds we know about and did not write down.
|
|
936
|
+
"persona_added": WAKE_CLASS_DIAGNOSTIC,
|
|
937
|
+
"heartbeat": WAKE_CLASS_LIVENESS,
|
|
938
|
+
"armed": WAKE_CLASS_LIVENESS,
|
|
939
|
+
}
|
|
940
|
+
|
|
941
|
+
|
|
942
|
+
def _wake_class(kind):
|
|
943
|
+
"""Classify an event kind. An UNKNOWN kind is a `diagnostic`, which means it WAKES.
|
|
944
|
+
|
|
945
|
+
Fail toward visible noise, never toward a silently muted channel: a kind added without a
|
|
946
|
+
classification is a mistake, and the failure mode of a mistake should be "someone got woken and
|
|
947
|
+
asked why", not "a channel went quiet and nobody noticed for a month". That direction is the whole
|
|
948
|
+
reason this field is worth having, so it is asserted by a test that CONSTRUCTS an unknown kind
|
|
949
|
+
rather than by reading this line - a default that is never exercised is a default nobody tested.
|
|
950
|
+
"""
|
|
951
|
+
return _WAKE_CLASS_BY_KIND.get(kind, WAKE_CLASS_DIAGNOSTIC)
|
|
952
|
+
|
|
953
|
+
|
|
954
|
+
class Emitter:
|
|
955
|
+
def __init__(self, mode, exec_cmd, content_chars, no_content, sink=None, suppress_authors=None,
|
|
956
|
+
sink_template=None, max_bytes=0, keep=5):
|
|
957
|
+
self.mode = mode
|
|
958
|
+
self.exec_cmd = exec_cmd
|
|
959
|
+
self.content_chars = content_chars
|
|
960
|
+
self.no_content = no_content
|
|
961
|
+
self.sink = sink # single shared RotatingFileSink (--events-file), else None (→ stdout)
|
|
962
|
+
self.suppress_authors = set(suppress_authors or []) # drop self-echo 'new' events from these authors
|
|
963
|
+
# --events-file-template: one OWNED RotatingFileSink PER PERSONA, so a session subscribes to ONLY its
|
|
964
|
+
# own mail by `tail -F events.<persona>.ndjson` - no shared-file grep to invent (the inbox-liveness LLM-UX problem).
|
|
965
|
+
self.sink_template = sink_template
|
|
966
|
+
self._max_bytes = max_bytes
|
|
967
|
+
self._keep = keep
|
|
968
|
+
self._sinks_by_persona = {}
|
|
969
|
+
# key -> monotonic deadline before which we will not retry. NOT a set (Loom re-audit 10, H2):
|
|
970
|
+
# membership alone has no release condition, so removing a hostile symlink never recovered without
|
|
971
|
+
# a restart. WHAT CLEARS THIS: the deadline expiring and the reopen SUCCEEDING (see _sink_for).
|
|
972
|
+
self._broken_sinks = {}
|
|
973
|
+
# §6.3 event-id namespace. A BARE counter would restart at 1 on every process start and hand
|
|
974
|
+
# old ids to new events - a consumer that had already seen them would drop live mail, which is
|
|
975
|
+
# worse than the duplicate it was meant to prevent. Namespacing the counter with a per-run token
|
|
976
|
+
# makes ids unique across restarts by construction; 8 random bytes keep that true even for a
|
|
977
|
+
# supervisor restarting the producer thousands of times.
|
|
978
|
+
self._run = "%016x" % int.from_bytes(os.urandom(8), "big")
|
|
979
|
+
self._seq = 0
|
|
980
|
+
|
|
981
|
+
def _sink_for(self, persona):
|
|
982
|
+
"""Route an event to its persona's sink (template mode), the single shared sink, or stdout (None).
|
|
983
|
+
|
|
984
|
+
A sink we cannot create SAFELY returns the BROKEN sentinel, never None: None means "no sink
|
|
985
|
+
configured, write to stdout", and falling through to stdout because a path looked like a symlink
|
|
986
|
+
would print the mail we just refused to file. One persona's bad path must also not take the
|
|
987
|
+
others down, so it is contained here rather than raised.
|
|
988
|
+
"""
|
|
989
|
+
if self.sink_template is None:
|
|
990
|
+
return self.sink
|
|
991
|
+
key = persona or "_all" # events with no persona (e.g. a bare --url target) land in one _all file
|
|
992
|
+
s = self._sinks_by_persona.get(key)
|
|
993
|
+
if s is None:
|
|
994
|
+
# A REFUSAL IS A COOLDOWN, NOT A VERDICT (Loom re-audit 10, H2). Caching the refusal with no
|
|
995
|
+
# release meant a persona whose path was briefly hostile stayed undeliverable for the life of
|
|
996
|
+
# the process: the operator removed the symlink, the fault was gone, and mail kept being held
|
|
997
|
+
# with nothing left to fix. A permanent fail-closed is the same bug as a fail-open, facing the
|
|
998
|
+
# other way (invariant 2: every pin must be dischargeable).
|
|
999
|
+
retry_at = self._broken_sinks.get(key)
|
|
1000
|
+
if retry_at is not None and _monotonic() < retry_at:
|
|
1001
|
+
return _BROKEN_SINK
|
|
1002
|
+
path = self.sink_template.replace("{persona}", _state_safe_persona(key))
|
|
1003
|
+
try:
|
|
1004
|
+
s = RotatingFileSink(path, self._max_bytes, self._keep)
|
|
1005
|
+
except OSError as e:
|
|
1006
|
+
self._broken_sinks[key] = _monotonic() + BROKEN_SINK_RETRY_S
|
|
1007
|
+
_warn_persona_once(key, "cannot open an events sink for %r safely (%s); its mail will be "
|
|
1008
|
+
"held, not written elsewhere" % (key, e))
|
|
1009
|
+
return _BROKEN_SINK
|
|
1010
|
+
if self._broken_sinks.pop(key, None) is not None:
|
|
1011
|
+
# Re-arm the suppressed warning so a LATER break is reported instead of silently
|
|
1012
|
+
# inheriting this one's suppression, and say so - a recovery nobody can see is the same
|
|
1013
|
+
# invisibility this tool exists to remove.
|
|
1014
|
+
_clear_persona_warning(key)
|
|
1015
|
+
sys.stderr.write("kijito-inbox-monitor: events sink for persona %r recovered; its held "
|
|
1016
|
+
"mail will be delivered\n" % key)
|
|
1017
|
+
self._sinks_by_persona[key] = s
|
|
1018
|
+
return s
|
|
1019
|
+
|
|
1020
|
+
def close(self):
|
|
1021
|
+
if self.sink is not None:
|
|
1022
|
+
self.sink.close()
|
|
1023
|
+
for s in self._sinks_by_persona.values():
|
|
1024
|
+
s.close()
|
|
1025
|
+
|
|
1026
|
+
def sync(self, persona=None):
|
|
1027
|
+
"""Make THIS persona's written events durable. Returns True IFF they are on stable storage.
|
|
1028
|
+
|
|
1029
|
+
Called by the watcher BEFORE it persists a cursor that acknowledges those events. exec-per-event
|
|
1030
|
+
has no sink of ours to sync (the consumer owns its own durability, and its exit status is the
|
|
1031
|
+
acknowledgement), and a stdout stream is a pipe we do not own - both answer True.
|
|
1032
|
+
|
|
1033
|
+
SCOPED TO ONE PERSONA on purpose. The Emitter is shared by every watch target, so syncing all
|
|
1034
|
+
sinks would let ONE persona's failing sink retract every OTHER persona's deliveries - a full
|
|
1035
|
+
directory's worth of duplicate storms caused by a stream nobody was reading. In template mode
|
|
1036
|
+
each persona owns its sink; in single-file mode there is one shared sink and syncing it is
|
|
1037
|
+
correct for whichever target asks.
|
|
1038
|
+
"""
|
|
1039
|
+
ok = True
|
|
1040
|
+
if self.sink is not None:
|
|
1041
|
+
ok = self.sink.sync() and ok
|
|
1042
|
+
if self.sink_template is not None:
|
|
1043
|
+
if (persona or "_all") in self._broken_sinks:
|
|
1044
|
+
return False # nothing was written, so nothing is durable; hold the cursor
|
|
1045
|
+
s = self._sinks_by_persona.get(persona or "_all")
|
|
1046
|
+
if s is not None:
|
|
1047
|
+
ok = s.sync() and ok
|
|
1048
|
+
return ok
|
|
1049
|
+
|
|
1050
|
+
def _clip(self, content):
|
|
1051
|
+
if self.no_content:
|
|
1052
|
+
return None
|
|
1053
|
+
s = "" if content is None else str(content)
|
|
1054
|
+
return s[: self.content_chars]
|
|
1055
|
+
|
|
1056
|
+
def _event_id(self, event):
|
|
1057
|
+
"""A producer-owned identity for this event (§6.3). Never derived from the serialised bytes.
|
|
1058
|
+
|
|
1059
|
+
TWO KINDS OF IDENTITY, because `new` and the signals need opposite things:
|
|
1060
|
+
|
|
1061
|
+
· `new` carries the MESSAGE's identity - persona plus the server's message id. The same message
|
|
1062
|
+
therefore always gets the same event id: across a restart, across a re-delivery after state
|
|
1063
|
+
loss, and across two watchers of the same inbox. That is what makes exactly-once processing
|
|
1064
|
+
possible on the consumer side, and it is the case that matters, because a duplicated message
|
|
1065
|
+
is duplicated WORK while a duplicated signal is only noise.
|
|
1066
|
+
|
|
1067
|
+
· everything else is a SIGNAL, and gets an id unique to this emission. A recurrence is a
|
|
1068
|
+
genuinely different event - a second outage is a second thing you want to see - so signals
|
|
1069
|
+
must NOT collapse into their earlier selves. Repeated announcements of an UNCHANGED condition
|
|
1070
|
+
are suppressed at the source instead (the alarms are edge-triggered and self-clearing), which
|
|
1071
|
+
is where that belongs.
|
|
1072
|
+
|
|
1073
|
+
Deliberately not a hash of the emitted line: byte-hashing couples the consumer to our
|
|
1074
|
+
formatting, so a change to key order, spacing or content clipping silently changes the dedupe
|
|
1075
|
+
key and re-delivers old events.
|
|
1076
|
+
"""
|
|
1077
|
+
persona = event.get("persona") or "_"
|
|
1078
|
+
if event.get("event") == "new" and isinstance(event.get("id"), int):
|
|
1079
|
+
return "%s:new:%d" % (persona, event["id"])
|
|
1080
|
+
self._seq += 1
|
|
1081
|
+
return "%s:%s:%s-%d" % (persona, event.get("event") or "_", self._run, self._seq)
|
|
1082
|
+
|
|
1083
|
+
def emit(self, event):
|
|
1084
|
+
"""Deliver one event. Returns True IFF delivery was ACKNOWLEDGED.
|
|
1085
|
+
|
|
1086
|
+
DELIVERY IS ACKNOWLEDGED, NOT ASSUMED (Loom re-audit 7, HIGH 1). The return value is what lets
|
|
1087
|
+
the watcher hold its cursor below a message it could not hand over. Before this, emit() swallowed
|
|
1088
|
+
every failure and the cursor advanced regardless, so a consumer whose --exec exited non-zero -
|
|
1089
|
+
the wake hook that is the entire point of exec mode - never saw that message again, and the
|
|
1090
|
+
watcher reported success. Anything other than True here means "not acknowledged": the message
|
|
1091
|
+
will be re-delivered rather than dropped, because a duplicate is recoverable and a skip is not.
|
|
1092
|
+
|
|
1093
|
+
`event` is a dict already containing event/source/ts and type-specific fields.
|
|
1094
|
+
"""
|
|
1095
|
+
# Stamped HERE, the single chokepoint every event passes through, rather than in the
|
|
1096
|
+
# convenience constructors: a future event kind added elsewhere cannot forget to carry one.
|
|
1097
|
+
event["event_id"] = self._event_id(event)
|
|
1098
|
+
# The nonce is DERIVED from the event_id, never minted beside it - so it must be computed
|
|
1099
|
+
# after it, and it inherits its identity semantics exactly. See _wake_nonce().
|
|
1100
|
+
event["nonce"] = _wake_nonce(event["event_id"])
|
|
1101
|
+
# Stamp 1 of the three-stamp wake ledger, all three clocks read together so they are a
|
|
1102
|
+
# COHERENT triple. `ts` is deliberately left alone: it is stamped in the convenience
|
|
1103
|
+
# constructors, microseconds earlier, and consumers already depend on it.
|
|
1104
|
+
event["emitted"] = _emission_stamps()
|
|
1105
|
+
# Classified HERE for the same reason event_id is: a future kind added at some other
|
|
1106
|
+
# construction site cannot forget to carry one, because it does not get a choice. `event`
|
|
1107
|
+
# itself is untouched, so every filter that matches on the NAME keeps working unchanged -
|
|
1108
|
+
# which is what makes phase 1 safe to land without coordinating a single consumer.
|
|
1109
|
+
event["wake_class"] = _wake_class(event.get("event"))
|
|
1110
|
+
if self.mode == "stdout-jsonl":
|
|
1111
|
+
# Sanitised at the SERIALISED line, so one call covers every field an event can carry -
|
|
1112
|
+
# content, `from`, an alarm `reason` built from server data - rather than each of them.
|
|
1113
|
+
line = _safe_text(json.dumps(event, ensure_ascii=False)) + "\n"
|
|
1114
|
+
sink = self._sink_for(event.get("persona"))
|
|
1115
|
+
if sink is _BROKEN_SINK:
|
|
1116
|
+
return False # a failed delivery: hold the cursor, never divert the mail
|
|
1117
|
+
if sink is not None:
|
|
1118
|
+
return sink.write(line)
|
|
1119
|
+
try:
|
|
1120
|
+
sys.stdout.write(line)
|
|
1121
|
+
sys.stdout.flush()
|
|
1122
|
+
except OSError as e:
|
|
1123
|
+
sys.stderr.write("kijito-inbox-monitor: WARNING stdout write FAILED, holding the cursor: %s\n" % e)
|
|
1124
|
+
return False
|
|
1125
|
+
return True
|
|
1126
|
+
else: # exec-per-event
|
|
1127
|
+
env = dict(os.environ)
|
|
1128
|
+
env["KIJITOMON_EVENT"] = str(event.get("event", ""))
|
|
1129
|
+
env["KIJITOMON_SOURCE"] = str(event.get("source", ""))
|
|
1130
|
+
env["KIJITOMON_TS"] = str(event.get("ts", ""))
|
|
1131
|
+
env["KIJITOMON_EVENT_ID"] = str(event.get("event_id", ""))
|
|
1132
|
+
# TRANSMITTED, NEVER RE-DERIVED (river's ruling, 2026-08-05, on a gap a drill measured).
|
|
1133
|
+
# The nonce was stamped in emit() and reached the ndjson wire, but never the exec env - so the
|
|
1134
|
+
# ONE channel our docs point consumers at first could not see the identity that says "this is
|
|
1135
|
+
# the same work re-delivered". It is derivable from KIJITOMON_EVENT_ID, and that is exactly the
|
|
1136
|
+
# hazard: re-derivation is a SECOND IMPLEMENTATION of sha256 + base62 + a pinned alphabet + an
|
|
1137
|
+
# 11-char truncation, and two implementations diverge. The unpinned alphabet has ALREADY
|
|
1138
|
+
# manufactured one false integrity alarm against correct data. Worse, the divergence surfaces
|
|
1139
|
+
# in SOMEBODY ELSE'S detector: a consumer whose derivation is slightly off splices a token that
|
|
1140
|
+
# matches no enqueue row, and D1 pages a delivery pathology that does not exist.
|
|
1141
|
+
# ⇒ DUPLICATE INSTRUMENTS, TRANSMIT DATA. For an instrument, divergence is a safety property;
|
|
1142
|
+
# for a shared identifier, divergence IS the defect. The discriminator is whether the thing
|
|
1143
|
+
# is a MEASUREMENT or a VALUE. This is a value: it must be IDENTICAL in two processes.
|
|
1144
|
+
# No capability is disclosed by passing it - the nonce is an attribution label, deterministic
|
|
1145
|
+
# and therefore already guessable from the event_id sitting beside it.
|
|
1146
|
+
env["KIJITOMON_NONCE"] = str(event.get("nonce", ""))
|
|
1147
|
+
keymap = {
|
|
1148
|
+
"id": "KIJITOMON_ID", "from": "KIJITOMON_FROM", "content": "KIJITOMON_CONTENT",
|
|
1149
|
+
"created": "KIJITOMON_CREATED", "cursor": "KIJITOMON_CURSOR",
|
|
1150
|
+
"persona": "KIJITOMON_PERSONA",
|
|
1151
|
+
"reason": "KIJITOMON_REASON", "consecutive_failures": "KIJITOMON_FAILURES",
|
|
1152
|
+
"seeded": "KIJITOMON_SEEDED", "current_max": "KIJITOMON_CURRENT_MAX",
|
|
1153
|
+
"capped_to": "KIJITOMON_CAPPED_TO", "dropped": "KIJITOMON_DROPPED",
|
|
1154
|
+
"stranded_inboxes": "KIJITOMON_STRANDED",
|
|
1155
|
+
"dormant_inboxes": "KIJITOMON_DORMANT",
|
|
1156
|
+
}
|
|
1157
|
+
for k, envname in keymap.items():
|
|
1158
|
+
if k in event and event[k] is not None:
|
|
1159
|
+
v = event[k]
|
|
1160
|
+
# A list is comma-joined, not str()'d: a Python repr ("['a', 'b']") is unusable from a
|
|
1161
|
+
# shell consumer, and exec-per-event is the portable primitive people reach for first.
|
|
1162
|
+
env[envname] = _safe_text(",".join(str(x) for x in v) if isinstance(v, list) else str(v))
|
|
1163
|
+
try:
|
|
1164
|
+
r = subprocess.run(self.exec_cmd, shell=True, env=env, timeout=EXEC_TIMEOUT, check=False)
|
|
1165
|
+
except subprocess.TimeoutExpired:
|
|
1166
|
+
# A timeout is NOT a delivery. The command may well have run - so the consumer must be
|
|
1167
|
+
# idempotent - but we have no acknowledgement, and inventing one is how mail disappears.
|
|
1168
|
+
sys.stderr.write("kijito-inbox-monitor: exec TIMED OUT, holding the cursor: %s\n" % self.exec_cmd)
|
|
1169
|
+
return False
|
|
1170
|
+
except Exception as e:
|
|
1171
|
+
sys.stderr.write("kijito-inbox-monitor: exec FAILED to run, holding the cursor: %s\n" % e)
|
|
1172
|
+
return False
|
|
1173
|
+
if r.returncode != 0:
|
|
1174
|
+
sys.stderr.write("kijito-inbox-monitor: exec exited %d, holding the cursor (the event will be "
|
|
1175
|
+
"re-delivered): %s\n" % (r.returncode, self.exec_cmd))
|
|
1176
|
+
return False
|
|
1177
|
+
return True
|
|
1178
|
+
|
|
1179
|
+
# convenience constructors (carry the canonical fields; ts stamped at emit time)
|
|
1180
|
+
def new(self, m):
|
|
1181
|
+
"""Emit one `new` event. Returns True IFF the message is ACKNOWLEDGED (see emit())."""
|
|
1182
|
+
if self.suppress_authors and m.get("from") in self.suppress_authors:
|
|
1183
|
+
# --suppress-author: don't wake on an event WE authored (self-echo noise). This is a
|
|
1184
|
+
# deliberate POLICY drop, so it counts as acknowledged - the cursor must still advance, or
|
|
1185
|
+
# suppressing an author would pin the watermark forever on that author's next message.
|
|
1186
|
+
return True
|
|
1187
|
+
ev = {"event": "new", "source": SOURCE, "ts": _now_iso(), "id": m.get("id"),
|
|
1188
|
+
"from": m.get("from"), "created": m.get("created")}
|
|
1189
|
+
if m.get("_persona"):
|
|
1190
|
+
ev["persona"] = m.get("_persona")
|
|
1191
|
+
c = self._clip(m.get("content"))
|
|
1192
|
+
if c is not None:
|
|
1193
|
+
ev["content"] = c
|
|
1194
|
+
return self.emit(ev)
|
|
1195
|
+
|
|
1196
|
+
def lifecycle(self, event, **fields):
|
|
1197
|
+
ev = {"event": event, "source": SOURCE, "ts": _now_iso()}
|
|
1198
|
+
ev.update(fields)
|
|
1199
|
+
return self.emit(ev)
|
|
1200
|
+
|
|
1201
|
+
|
|
1202
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
1203
|
+
# §7.3 State file (canonical identity + flock + atomic write + resume)
|
|
1204
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
1205
|
+
# A state file that EXISTS but cannot be trusted. Distinct from None (genuinely absent) because the two
|
|
1206
|
+
# demand opposite behaviour: absent means baseline, corrupt means fail closed and re-emit.
|
|
1207
|
+
CORRUPT_STATE = object()
|
|
1208
|
+
|
|
1209
|
+
|
|
1210
|
+
PRIVATE_FILE_MODE = 0o600 # event streams and lock sidecars carry/guard message content
|
|
1211
|
+
PRIVATE_DIR_MODE = 0o700
|
|
1212
|
+
|
|
1213
|
+
|
|
1214
|
+
class InsecureFile(OSError):
|
|
1215
|
+
"""A path we were about to write MAIL into is not something we are willing to write mail into."""
|
|
1216
|
+
|
|
1217
|
+
|
|
1218
|
+
def _assert_private_fd(fd, path):
|
|
1219
|
+
"""Fail CLOSED unless this fd is a REGULAR file, owned by US, at exactly 0600 (Loom re-audit 9, H1/H2).
|
|
1220
|
+
|
|
1221
|
+
My round-8 repair was best-effort - it warned on failure and wrote anyway - on the reasoning that "a
|
|
1222
|
+
file we do not own must not crash the watcher". That reasoning is exactly backwards for a file we are
|
|
1223
|
+
about to append PRIVATE MAIL to: loom's repro left a pre-existing 0666 file at 0666 and delivered mail
|
|
1224
|
+
into it. Refusing to write is the only safe answer, and the caller turns that into a FAILED DELIVERY,
|
|
1225
|
+
so the cursor holds and nothing is lost.
|
|
1226
|
+
"""
|
|
1227
|
+
st = os.fstat(fd)
|
|
1228
|
+
if not stat.S_ISREG(st.st_mode):
|
|
1229
|
+
raise InsecureFile("%s is not a regular file" % path)
|
|
1230
|
+
if st.st_uid != os.geteuid():
|
|
1231
|
+
raise InsecureFile("%s is owned by uid %d, not by us (uid %d)" % (path, st.st_uid, os.geteuid()))
|
|
1232
|
+
cur = st.st_mode & 0o777
|
|
1233
|
+
if cur != PRIVATE_FILE_MODE:
|
|
1234
|
+
# EXACTLY 0600, not merely "no group/other bits" - a 0700 event file kept its execute bit under
|
|
1235
|
+
# the old `st_mode & 0o077` test, which is a mode nothing here should ever have (re-audit 9, H2).
|
|
1236
|
+
try:
|
|
1237
|
+
os.fchmod(fd, PRIVATE_FILE_MODE)
|
|
1238
|
+
except OSError as e:
|
|
1239
|
+
# Typed, so a caller reading the log can tell "we refused this path" from "the disk broke".
|
|
1240
|
+
raise InsecureFile("%s is %o and cannot be tightened to 0600: %s" % (path, cur, e))
|
|
1241
|
+
again = os.fstat(fd).st_mode & 0o777
|
|
1242
|
+
if again != PRIVATE_FILE_MODE:
|
|
1243
|
+
raise InsecureFile("%s is %o and could not be tightened to 0600 (now %o)" % (path, cur, again))
|
|
1244
|
+
sys.stderr.write("kijito-inbox-monitor: tightened %s from %o to 0600 (it carries message content "
|
|
1245
|
+
"and was reachable by other local users)\n" % (path, cur))
|
|
1246
|
+
|
|
1247
|
+
|
|
1248
|
+
def _open_private(path, mode="a", encoding=None):
|
|
1249
|
+
"""Open a file that must never be readable by anyone else, refusing anything suspicious.
|
|
1250
|
+
|
|
1251
|
+
THE EVENT STREAM CARRIES MESSAGE BODIES. A plain open() takes the process umask (022 by default), so
|
|
1252
|
+
every events.<persona>.ndjson was created 0644 - world-readable private hive mail, verified live
|
|
1253
|
+
(re-audit 8, H1). But the FIRST repair was itself unsafe (re-audit 9, H1), and worse than the leak it
|
|
1254
|
+
fixed: it FOLLOWED SYMLINKS, so it chmod'ed and appended mail to whatever a link pointed at, and a
|
|
1255
|
+
DANGLING link created its target in another directory entirely. A passive disclosure had been turned
|
|
1256
|
+
into an active write primitive.
|
|
1257
|
+
· O_NOFOLLOW - the final component must not be a symlink. (No TOCTOU window: the check is the open.)
|
|
1258
|
+
· owner + regular-file, checked on the FD we already hold, never by a second path lookup.
|
|
1259
|
+
· fail CLOSED - callers convert InsecureFile into a failed delivery, never a crash and never a write.
|
|
1260
|
+
"""
|
|
1261
|
+
# O_NONBLOCK matters as much as O_NOFOLLOW here: opening a FIFO for writing BLOCKS until a reader
|
|
1262
|
+
# appears, so a FIFO planted at the events path would HANG the watcher forever - silently, with no
|
|
1263
|
+
# crash to notice and no events to miss noticing. (Found when the regular-file test hung the suite.)
|
|
1264
|
+
# On a regular file O_NONBLOCK is a no-op, so it costs nothing on the path we actually take.
|
|
1265
|
+
flags = (os.O_WRONLY | os.O_CREAT | os.O_APPEND
|
|
1266
|
+
| getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0))
|
|
1267
|
+
try:
|
|
1268
|
+
fd = os.open(path, flags, PRIVATE_FILE_MODE)
|
|
1269
|
+
except OSError as e:
|
|
1270
|
+
# ELOOP here means the path IS a symlink - report it as what it is, not as a generic open failure.
|
|
1271
|
+
raise InsecureFile("refusing to open %s: %s" % (path, e))
|
|
1272
|
+
try:
|
|
1273
|
+
_assert_private_fd(fd, path)
|
|
1274
|
+
except OSError:
|
|
1275
|
+
os.close(fd)
|
|
1276
|
+
raise
|
|
1277
|
+
return os.fdopen(fd, mode, encoding=encoding) if encoding else os.fdopen(fd, mode)
|
|
1278
|
+
|
|
1279
|
+
|
|
1280
|
+
def _repair_mode(path):
|
|
1281
|
+
"""Tighten an EXISTING artifact to 0600 in place. Returns True if it is now safe.
|
|
1282
|
+
|
|
1283
|
+
Repairing only the file we happen to open leaves every OTHER persisted artifact exactly as it was -
|
|
1284
|
+
loom found pre-existing rotated archives still at 0644 after the round-8 fix (re-audit 9, H2). Opened
|
|
1285
|
+
O_NOFOLLOW and validated on the fd, for the same reason as _open_private.
|
|
1286
|
+
"""
|
|
1287
|
+
try:
|
|
1288
|
+
fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0))
|
|
1289
|
+
except FileNotFoundError:
|
|
1290
|
+
return True # nothing there is nothing to leak
|
|
1291
|
+
except OSError as e:
|
|
1292
|
+
sys.stderr.write("kijito-inbox-monitor: WARNING refusing to repair %s: %s\n" % (path, e))
|
|
1293
|
+
return False
|
|
1294
|
+
try:
|
|
1295
|
+
_assert_private_fd(fd, path)
|
|
1296
|
+
return True
|
|
1297
|
+
except OSError as e:
|
|
1298
|
+
sys.stderr.write("kijito-inbox-monitor: WARNING %s may be readable by other local users: %s\n"
|
|
1299
|
+
% (path, e))
|
|
1300
|
+
return False
|
|
1301
|
+
finally:
|
|
1302
|
+
os.close(fd)
|
|
1303
|
+
|
|
1304
|
+
|
|
1305
|
+
def _makedirs_private(path):
|
|
1306
|
+
"""Create EVERY missing level 0700, and warn about an existing level anyone else can write.
|
|
1307
|
+
|
|
1308
|
+
os.makedirs(mode=...) applies the mode to the LEAF only; intermediate directories get the umask
|
|
1309
|
+
default, so a nested path left its parents 0755 (Loom re-audit 9, MEDIUM). An EXISTING directory is
|
|
1310
|
+
still not re-permissioned - silently changing a path the operator already owns is not ours to do - but
|
|
1311
|
+
a group/world-WRITABLE one is reported, because that is the condition under which someone else can
|
|
1312
|
+
swap a file for a symlink underneath us. (_open_private then refuses it, which is the real defence;
|
|
1313
|
+
this is the warning that tells you why events stopped.)
|
|
1314
|
+
"""
|
|
1315
|
+
path = os.path.abspath(path)
|
|
1316
|
+
missing = []
|
|
1317
|
+
cur = path
|
|
1318
|
+
while not os.path.isdir(cur):
|
|
1319
|
+
missing.append(cur)
|
|
1320
|
+
parent = os.path.dirname(cur)
|
|
1321
|
+
if parent == cur:
|
|
1322
|
+
break
|
|
1323
|
+
cur = parent
|
|
1324
|
+
for d in reversed(missing):
|
|
1325
|
+
try:
|
|
1326
|
+
os.mkdir(d, PRIVATE_DIR_MODE)
|
|
1327
|
+
except FileExistsError:
|
|
1328
|
+
pass
|
|
1329
|
+
# Check the WHOLE ancestor chain, not only the levels we created - loom's point was that an EXISTING
|
|
1330
|
+
# directory is never validated, and an existing one is exactly where a hostile path would already be.
|
|
1331
|
+
# A sticky directory (/tmp, mode 1777) is excluded: the sticky bit is precisely what makes a shared
|
|
1332
|
+
# writable directory safe, and warning about it would train the reader to ignore this line.
|
|
1333
|
+
seen = path
|
|
1334
|
+
while True:
|
|
1335
|
+
try:
|
|
1336
|
+
st = os.stat(seen)
|
|
1337
|
+
except OSError:
|
|
1338
|
+
break
|
|
1339
|
+
if (st.st_mode & 0o022) and not (st.st_mode & stat.S_ISVTX):
|
|
1340
|
+
sys.stderr.write("kijito-inbox-monitor: WARNING directory %s is writable by other local users "
|
|
1341
|
+
"(mode %o); files there can be swapped for symlinks underneath us\n"
|
|
1342
|
+
% (seen, st.st_mode & 0o777))
|
|
1343
|
+
parent = os.path.dirname(seen)
|
|
1344
|
+
if parent == seen:
|
|
1345
|
+
break
|
|
1346
|
+
seen = parent
|
|
1347
|
+
|
|
1348
|
+
|
|
1349
|
+
def _fsync_dir(path):
|
|
1350
|
+
"""fsync a DIRECTORY so a rename inside it is durable. Returns True on success.
|
|
1351
|
+
|
|
1352
|
+
os.replace is atomic for a concurrent READER, but atomicity is not durability: after a power loss
|
|
1353
|
+
the new file's contents can be on disk while the directory entry still names the old inode - i.e. a
|
|
1354
|
+
silently OLDER cursor. Syncing the file alone (which is all we did) does not cover the rename.
|
|
1355
|
+
"""
|
|
1356
|
+
try:
|
|
1357
|
+
fd = os.open(path, os.O_RDONLY)
|
|
1358
|
+
except OSError:
|
|
1359
|
+
return False
|
|
1360
|
+
try:
|
|
1361
|
+
os.fsync(fd)
|
|
1362
|
+
return True
|
|
1363
|
+
except OSError:
|
|
1364
|
+
return False
|
|
1365
|
+
finally:
|
|
1366
|
+
os.close(fd)
|
|
1367
|
+
|
|
1368
|
+
|
|
1369
|
+
def identity_migratable(stored, current):
|
|
1370
|
+
"""True iff a persisted identity differs from the current one ONLY BY THE CASE OF A QUERY VALUE.
|
|
1371
|
+
|
|
1372
|
+
THE CASE-ONLY MIGRATION (Loom re-audit 7, HIGH 3). The state PATH casefolds the persona
|
|
1373
|
+
(_state_safe_persona - the local filesystem is case-insensitive, so it must), while the IDENTITY
|
|
1374
|
+
embeds the persona with its original case, straight from the directory. So one file, written when
|
|
1375
|
+
the directory spelled the persona `Loom`, is reloaded by a run that discovered `loom` - the identity
|
|
1376
|
+
compares UNEQUAL, load() reports ABSENT, and absent BASELINES to the newest visible id, skipping
|
|
1377
|
+
everything since the lost cursor. A cursor destroyed by a spelling change is exactly the silent skip
|
|
1378
|
+
the state file exists to prevent.
|
|
1379
|
+
|
|
1380
|
+
★ THIS IS A THIRD LAYER, NOT A HARMONISATION OF THE OTHER TWO (see CaseAsymmetryInvariantTest, and
|
|
1381
|
+
do not "simplify" it into them). The SERVER's inbox namespace stays case-SENSITIVE - `Loom` and
|
|
1382
|
+
`loom` remain distinct inboxes and a variant holding mail is still alarmed on as stranded. What this
|
|
1383
|
+
says is narrower and follows from the path layer: because the path already collapses the variants,
|
|
1384
|
+
ONE state file can only ever describe ONE of them, so a casefold-equal identity in THAT file is the
|
|
1385
|
+
same watched source spelled differently - a migration to accept and rewrite, not a different source
|
|
1386
|
+
to baseline over.
|
|
1387
|
+
|
|
1388
|
+
Deliberately strict about WHAT may differ: scheme, host, port and path must match EXACTLY, and so
|
|
1389
|
+
must every query KEY. Only the query VALUE is compared case-insensitively. Nothing here invents
|
|
1390
|
+
case-insensitivity for a URL path or for a host we did not already lowercase.
|
|
1391
|
+
"""
|
|
1392
|
+
if not (isinstance(stored, list) and len(stored) == 5 and isinstance(current, list) and len(current) == 5):
|
|
1393
|
+
return False
|
|
1394
|
+
if stored[:4] != current[:4]:
|
|
1395
|
+
return False
|
|
1396
|
+
sq, cq = stored[4], current[4]
|
|
1397
|
+
if not (isinstance(sq, list) and isinstance(cq, list)) or len(sq) != len(cq):
|
|
1398
|
+
return False
|
|
1399
|
+
for s, c in zip(sq, cq):
|
|
1400
|
+
if not (isinstance(s, (list, tuple)) and len(s) == 2):
|
|
1401
|
+
return False
|
|
1402
|
+
if str(s[0]) != str(c[0]) or str(s[1]).casefold() != str(c[1]).casefold():
|
|
1403
|
+
return False
|
|
1404
|
+
return True
|
|
1405
|
+
|
|
1406
|
+
|
|
1407
|
+
class StateFile:
|
|
1408
|
+
def __init__(self, path, identity):
|
|
1409
|
+
self.path = path
|
|
1410
|
+
self.identity = identity
|
|
1411
|
+
self._lockf = None
|
|
1412
|
+
# Set by lock() when the state file could not be PROVEN private, and consumed by load(), which
|
|
1413
|
+
# then fails closed. WHAT CLEARS THIS: nothing within the process - the condition is a property of
|
|
1414
|
+
# the path on disk, re-evaluated from scratch on the next start. Recorded explicitly because "no
|
|
1415
|
+
# release condition" is only acceptable when it is the ANSWER, not when it is an oversight.
|
|
1416
|
+
self.unsafe = False
|
|
1417
|
+
|
|
1418
|
+
def lock(self):
|
|
1419
|
+
if not IS_POSIX or fcntl is None:
|
|
1420
|
+
return # Windows: no lock (documented; run a single instance)
|
|
1421
|
+
dirn = os.path.dirname(os.path.abspath(self.path)) or "."
|
|
1422
|
+
_makedirs_private(dirn)
|
|
1423
|
+
# Lock a DEDICATED .lock SIDECAR, never the state-file itself: save() replaces the state-file's inode
|
|
1424
|
+
# (mkstemp + os.replace) on every poll, which would orphan a flock held on it and let a second watcher
|
|
1425
|
+
# lock the new inode freely. The sidecar is never replaced, so the flock persists for the process
|
|
1426
|
+
# lifetime. flock is advisory + auto-released by the OS on exit (no stale lockfile to clean).
|
|
1427
|
+
if not _repair_mode(self.path): # the state file itself, if an older version left it permissive
|
|
1428
|
+
# CONSUME THE VERDICT (Loom re-audit 10, H1). This was a bare statement, so a state file we
|
|
1429
|
+
# could not prove private was then trusted anyway. It is the highest-value file here to
|
|
1430
|
+
# subvert: whoever controls the CURSOR controls which mail counts as already delivered, and a
|
|
1431
|
+
# cursor moved FORWARD is silent, permanent mail loss - the single failure this tool exists to
|
|
1432
|
+
# prevent. Deliberately NOT fatal: one persona's hostile path must not take the whole producer
|
|
1433
|
+
# down (the same reasoning as _sink_for). load() fails closed on it instead, which routes into
|
|
1434
|
+
# the existing, tested "present but untrustworthy" path rather than inventing a new one.
|
|
1435
|
+
self.unsafe = True
|
|
1436
|
+
self._lockf = _open_private(self.path + ".lock", "a+")
|
|
1437
|
+
try:
|
|
1438
|
+
fcntl.flock(self._lockf.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
1439
|
+
except OSError:
|
|
1440
|
+
# Close the sidecar we just opened before failing. Leaving it open leaks an fd per refused
|
|
1441
|
+
# lock - and persona rediscovery runs every tick, so a persona whose file is held by another
|
|
1442
|
+
# watcher would leak one fd per attempt for the life of the process.
|
|
1443
|
+
self.unlock()
|
|
1444
|
+
raise FatalConfig("state-file in use (another watcher holds the lock): %s" % self.path)
|
|
1445
|
+
|
|
1446
|
+
def load(self):
|
|
1447
|
+
"""Return the resumed state on a VALID identity-matching file; None if genuinely ABSENT.
|
|
1448
|
+
|
|
1449
|
+
Raises FatalConfig on a present-but-unreadable path, and returns the CORRUPT sentinel on a file
|
|
1450
|
+
that EXISTS but cannot be trusted.
|
|
1451
|
+
|
|
1452
|
+
ABSENT AND CORRUPT ARE NOT THE SAME ANSWER (Loom re-audit 5, HIGH 2). Both used to return None, so
|
|
1453
|
+
a garbled state file was indistinguishable from a first launch - and a first launch BASELINES to
|
|
1454
|
+
the newest visible id, silently skipping every message between the lost cursor and now. That is a
|
|
1455
|
+
permanent, invisible loss produced by the one event most likely to accompany a crash. A file that
|
|
1456
|
+
is present but unparseable is EVIDENCE THAT A CURSOR EXISTED, so it must fail closed and re-emit
|
|
1457
|
+
rather than fail open and skip. Duplicates are recoverable; skips are not.
|
|
1458
|
+
"""
|
|
1459
|
+
if self.unsafe:
|
|
1460
|
+
# The verdict lock() computed and used to discard. Present-but-not-provably-ours is exactly
|
|
1461
|
+
# the "evidence a cursor existed, but not one we can trust" case: fail closed, re-emit.
|
|
1462
|
+
sys.stderr.write("kijito-inbox-monitor: WARNING state-file %s could not be proven private; "
|
|
1463
|
+
"refusing to trust its cursor and failing closed\n" % self.path)
|
|
1464
|
+
return CORRUPT_STATE
|
|
1465
|
+
# LET THE O_NOFOLLOW OPEN BE THE EXISTENCE TEST (Loom re-audit 10, H1). The WRITE path was given
|
|
1466
|
+
# O_NOFOLLOW in re-audit 9 and the READ path was left behind, so a symlink planted at the state
|
|
1467
|
+
# path was followed and its target read as our own state. os.path.exists() follows symlinks too,
|
|
1468
|
+
# so it was answering for the TARGET rather than the link - and being a second path lookup it
|
|
1469
|
+
# opened a TOCTOU window between the check and the open. One syscall now settles both questions.
|
|
1470
|
+
try:
|
|
1471
|
+
fd = os.open(self.path,
|
|
1472
|
+
os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0))
|
|
1473
|
+
except FileNotFoundError:
|
|
1474
|
+
return None # genuinely ABSENT - the one case that may baseline
|
|
1475
|
+
except OSError as e:
|
|
1476
|
+
# PRESENT, but not something we are willing to read: a symlink (ELOOP), a FIFO, a directory.
|
|
1477
|
+
# Still EVIDENCE THAT A CURSOR EXISTED, so it takes the same fail-closed answer as an
|
|
1478
|
+
# unparseable file. Reading it would be worse than not resuming; baselining would be worst.
|
|
1479
|
+
sys.stderr.write("kijito-inbox-monitor: WARNING state-file %s exists but is not a file we will "
|
|
1480
|
+
"read (%s); failing closed and re-emitting rather than baselining\n"
|
|
1481
|
+
% (self.path, e))
|
|
1482
|
+
return CORRUPT_STATE
|
|
1483
|
+
try:
|
|
1484
|
+
f = os.fdopen(fd, "r")
|
|
1485
|
+
except OSError as e:
|
|
1486
|
+
os.close(fd) # never leak the descriptor we just took (re-audit 10, L6's class)
|
|
1487
|
+
raise FatalConfig("state-file unreadable: %s" % e)
|
|
1488
|
+
with f:
|
|
1489
|
+
try:
|
|
1490
|
+
raw = f.read()
|
|
1491
|
+
except OSError as e:
|
|
1492
|
+
raise FatalConfig("state-file unreadable: %s" % e)
|
|
1493
|
+
if not raw.strip():
|
|
1494
|
+
# PRESENT BUT EMPTY IS NOT ABSENT (Loom re-audit 6, HIGH 4). A zero-byte file is still
|
|
1495
|
+
# evidence that a cursor existed here; treating it as a first launch baselines over
|
|
1496
|
+
# everything since. Same fail-open shape as an unparseable file, same answer.
|
|
1497
|
+
sys.stderr.write("kijito-inbox-monitor: WARNING state-file is present but EMPTY; refusing to "
|
|
1498
|
+
"baseline over it: %s\n" % self.path)
|
|
1499
|
+
return CORRUPT_STATE
|
|
1500
|
+
try:
|
|
1501
|
+
d = json.loads(raw)
|
|
1502
|
+
cursor = d["cursor"]
|
|
1503
|
+
state = d["state"]
|
|
1504
|
+
failures = d["consecutive_failures"]
|
|
1505
|
+
ident = d["identity"]
|
|
1506
|
+
except (ValueError, KeyError, TypeError):
|
|
1507
|
+
sys.stderr.write("kijito-inbox-monitor: WARNING state-file is present but unparseable; refusing to "
|
|
1508
|
+
"baseline over it (that would silently skip everything since the lost cursor): "
|
|
1509
|
+
"%s\n" % self.path)
|
|
1510
|
+
return CORRUPT_STATE
|
|
1511
|
+
if not ((cursor is None or _is_int(cursor)) and state in ("UP", "DOWN")
|
|
1512
|
+
and _is_int(failures)):
|
|
1513
|
+
sys.stderr.write("kijito-inbox-monitor: WARNING state-file has a valid envelope but invalid "
|
|
1514
|
+
"fields; refusing to baseline over it: %s\n" % self.path)
|
|
1515
|
+
return CORRUPT_STATE
|
|
1516
|
+
if ident != self.identity:
|
|
1517
|
+
if identity_migratable(ident, self.identity):
|
|
1518
|
+
# CASE-ONLY MIGRATION, not a different source - see identity_migratable(). Resume the
|
|
1519
|
+
# cursor; the next save() rewrites the file with the current spelling, so this converges
|
|
1520
|
+
# after one poll instead of re-warning forever.
|
|
1521
|
+
sys.stderr.write("kijito-inbox-monitor: state-file identity differs only by case (%r -> %r); "
|
|
1522
|
+
"MIGRATING it rather than re-baselining (re-baselining would skip every "
|
|
1523
|
+
"message since the stored cursor): %s\n" % (ident, self.identity, self.path))
|
|
1524
|
+
else:
|
|
1525
|
+
sys.stderr.write("kijito-inbox-monitor: WARNING state-file identity mismatch (%r != %r) - NOT "
|
|
1526
|
+
"resuming its cursor; re-baselining to avoid a silently-blind watcher.\n"
|
|
1527
|
+
% (ident, self.identity))
|
|
1528
|
+
return None
|
|
1529
|
+
# EVERY PERSISTED FIELD IS READ STRICTLY, AND ANYTHING UNRECOGNISED FAILS CLOSED (Loom re-audit 7,
|
|
1530
|
+
# HIGH 2). The pin's own flags were read with `d.get(k) is True`, so a JSON `1` - the shape a
|
|
1531
|
+
# hand-edit, a jq one-liner or another language's serialiser produces - normalised to False and
|
|
1532
|
+
# SILENTLY UNPINNED the watermark, letting the replay cap cross the very span the pin was
|
|
1533
|
+
# protecting. `pin_evidence_intact` had the mirror bug (`is False`, so `0` read as intact), and
|
|
1534
|
+
# ints-that-are-bools were accepted as row ids and as gap_alerted. A malformed field is EVIDENCE
|
|
1535
|
+
# THE FILE CANNOT BE TRUSTED, so the honest answer is CORRUPT_STATE - which arms below the visible
|
|
1536
|
+
# window and re-emits it - never a quietly permissive default.
|
|
1537
|
+
strict_ok = True
|
|
1538
|
+
|
|
1539
|
+
def _flag(key):
|
|
1540
|
+
"""Strict tri-state read of a persisted boolean: (value, ok)."""
|
|
1541
|
+
nonlocal strict_ok
|
|
1542
|
+
v = d.get(key, _MISSING)
|
|
1543
|
+
if v is _MISSING or v is None:
|
|
1544
|
+
return False
|
|
1545
|
+
if v is True or v is False:
|
|
1546
|
+
return v
|
|
1547
|
+
sys.stderr.write("kijito-inbox-monitor: WARNING state-file field %r is not a boolean (%r); refusing "
|
|
1548
|
+
"to interpret it: %s\n" % (key, v, self.path))
|
|
1549
|
+
strict_ok = False
|
|
1550
|
+
return False
|
|
1551
|
+
|
|
1552
|
+
# Ids already emitted ABOVE a pinned watermark. Absent in files written by older versions, which is
|
|
1553
|
+
# exactly the forward-compat case: an empty set just means "nothing pinned", the pre-pinning behaviour.
|
|
1554
|
+
alerted = d.get("gap_alerted", _MISSING)
|
|
1555
|
+
if alerted is _MISSING or alerted is None:
|
|
1556
|
+
alerted = None
|
|
1557
|
+
elif _is_int(alerted):
|
|
1558
|
+
pass
|
|
1559
|
+
else:
|
|
1560
|
+
sys.stderr.write("kijito-inbox-monitor: WARNING state-file 'gap_alerted' is not an integer (%r); "
|
|
1561
|
+
"refusing to interpret it: %s\n" % (alerted, self.path))
|
|
1562
|
+
alerted, strict_ok = None, False
|
|
1563
|
+
release_at = d.get("pin_release_at", _MISSING)
|
|
1564
|
+
if release_at is _MISSING or release_at is None:
|
|
1565
|
+
release_at = None
|
|
1566
|
+
elif _is_int(release_at):
|
|
1567
|
+
pass
|
|
1568
|
+
else:
|
|
1569
|
+
sys.stderr.write("kijito-inbox-monitor: WARNING state-file 'pin_release_at' is not an integer (%r); "
|
|
1570
|
+
"refusing to interpret it: %s\n" % (release_at, self.path))
|
|
1571
|
+
release_at, strict_ok = None, False
|
|
1572
|
+
raw = d.get("emitted_above")
|
|
1573
|
+
if raw is None:
|
|
1574
|
+
emitted, intact = set(), True # no pin was in force; the ordinary case
|
|
1575
|
+
elif isinstance(raw, list) and all(_is_int(i) for i in raw):
|
|
1576
|
+
# _is_int, not isinstance(i, int): `true` in this list would otherwise become the id 1 and
|
|
1577
|
+
# suppress a real message 1 for the life of the pin.
|
|
1578
|
+
emitted, intact = set(raw), True
|
|
1579
|
+
else:
|
|
1580
|
+
# CORRUPT PIN STATE MUST FAIL CLOSED. Loading it as an empty set silently UNPINS: the watcher
|
|
1581
|
+
# would then think nothing was outstanding, let the replay cap jump the cursor over the very
|
|
1582
|
+
# span the pin was protecting, and lose it. We cannot know which ids were delivered, so we
|
|
1583
|
+
# keep the pin (empty tracking) and mark the evidence unusable - the gap can then only be
|
|
1584
|
+
# closed by an authoritative read, never by counting.
|
|
1585
|
+
sys.stderr.write("kijito-inbox-monitor: WARNING state-file 'emitted_above' is malformed; "
|
|
1586
|
+
"keeping the watermark PINNED with no delivery tracking rather than "
|
|
1587
|
+
"silently unpinning: %s\n" % self.path)
|
|
1588
|
+
emitted, intact = set(), False
|
|
1589
|
+
# A recorded gap alert with no pin tracking is itself inconsistent: something was pinned when the
|
|
1590
|
+
# file was written. Treat it the same way - hold the pin rather than assume it resolved.
|
|
1591
|
+
if alerted is not None and not emitted and intact and raw is None:
|
|
1592
|
+
intact = False
|
|
1593
|
+
# THE PIN'S OWN STATE IS PERSISTED (Loom re-audit 6, HIGH 1). It used to be inferred from
|
|
1594
|
+
# `emitted_above`, which is empty in exactly the case that matters - a corrupt-state pin, where
|
|
1595
|
+
# nothing has been tracked yet. So a restart lost the pin, the replay cap was free again, and the
|
|
1596
|
+
# very span the pin was protecting got crossed on the first poll. A pin that does not survive a
|
|
1597
|
+
# restart is not a pin; the crash is when you need it.
|
|
1598
|
+
# Read strictly: `0` used to slip past `is False` and leave the evidence marked INTACT, which is
|
|
1599
|
+
# the fail-OPEN direction on the one field that says "stop trusting your own view of this span".
|
|
1600
|
+
if d.get("pin_evidence_intact", _MISSING) is not _MISSING and not _flag("pin_evidence_intact"):
|
|
1601
|
+
intact = False
|
|
1602
|
+
pin_forced = _flag("pin_forced")
|
|
1603
|
+
state_corrupt = _flag("state_corrupt")
|
|
1604
|
+
if not strict_ok:
|
|
1605
|
+
sys.stderr.write("kijito-inbox-monitor: WARNING state-file has malformed pin fields; treating the "
|
|
1606
|
+
"whole file as CORRUPT (fail closed) rather than resuming a state we cannot "
|
|
1607
|
+
"read: %s\n" % self.path)
|
|
1608
|
+
return CORRUPT_STATE
|
|
1609
|
+
# Whether the unread-not-shown alarm is currently ANNOUNCED. Absent (older file) reads as False:
|
|
1610
|
+
# a re-announce after an upgrade costs one event and is honest about the current condition,
|
|
1611
|
+
# whereas defaulting to True would silence a live condition for the rest of the run.
|
|
1612
|
+
hidden = d.get("unread_hidden") is True
|
|
1613
|
+
return {"cursor": cursor, "state": state, "failures": failures, "emitted_above": emitted,
|
|
1614
|
+
"gap_alerted": alerted, "pin_evidence_intact": intact,
|
|
1615
|
+
"pin_forced": pin_forced, "pin_release_at": release_at,
|
|
1616
|
+
"state_corrupt": state_corrupt, "unread_hidden": hidden}
|
|
1617
|
+
|
|
1618
|
+
def unlock(self):
|
|
1619
|
+
"""Release the single-writer flock and close the sidecar fd.
|
|
1620
|
+
|
|
1621
|
+
The OS drops an flock when the process exits, so this is hygiene rather than correctness - but an
|
|
1622
|
+
fd held for a target that is torn down is a genuine leak in a long-lived process, and it is what
|
|
1623
|
+
surfaced as the suite's two ResourceWarnings (Loom re-audit 7, item 7).
|
|
1624
|
+
"""
|
|
1625
|
+
if self._lockf is not None:
|
|
1626
|
+
try:
|
|
1627
|
+
self._lockf.close()
|
|
1628
|
+
finally:
|
|
1629
|
+
self._lockf = None
|
|
1630
|
+
|
|
1631
|
+
def save(self, cursor, state, failures, emitted_above=None, gap_alerted=None,
|
|
1632
|
+
pin_forced=False, pin_evidence_intact=True, state_corrupt=False, pin_release_at=None,
|
|
1633
|
+
unread_hidden=False):
|
|
1634
|
+
"""Persist the cursor. Returns True IFF the write is DURABLE (Loom re-audit 8, HIGH 3).
|
|
1635
|
+
|
|
1636
|
+
The directory fsync used to be called and its answer thrown away, so a failure returned success
|
|
1637
|
+
with no diagnostic: the cursor was written and its durability merely assumed. The failure
|
|
1638
|
+
direction is re-delivery rather than loss - a reverted state file replays mail - but a watcher
|
|
1639
|
+
that cannot tell you it failed to persist will keep not telling you, and a disk failing this way
|
|
1640
|
+
is exactly the condition nobody notices.
|
|
1641
|
+
"""
|
|
1642
|
+
if not IS_POSIX:
|
|
1643
|
+
return True # best-effort; skip on Windows
|
|
1644
|
+
d = {"identity": self.identity, "cursor": cursor, "state": state, "consecutive_failures": failures}
|
|
1645
|
+
# Persisted so a RESTART cannot re-emit what we already delivered above a pinned watermark.
|
|
1646
|
+
# Without this, failing closed would trade silent loss for a duplicate storm on every restart.
|
|
1647
|
+
if emitted_above:
|
|
1648
|
+
d["emitted_above"] = sorted(emitted_above)
|
|
1649
|
+
# Persisted too, so a restart does not re-announce a gap it already announced.
|
|
1650
|
+
if gap_alerted is not None:
|
|
1651
|
+
d["gap_alerted"] = gap_alerted
|
|
1652
|
+
# The pin's own state, persisted rather than inferred. `emitted_above` is EMPTY for a
|
|
1653
|
+
# corrupt-state pin, so inferring from it silently dropped exactly the pin that matters.
|
|
1654
|
+
if pin_forced:
|
|
1655
|
+
d["pin_forced"] = True
|
|
1656
|
+
if not pin_evidence_intact:
|
|
1657
|
+
d["pin_evidence_intact"] = False
|
|
1658
|
+
if state_corrupt:
|
|
1659
|
+
d["state_corrupt"] = True
|
|
1660
|
+
# The floor that RELEASES a corruption pin. Persisted with the pin itself: a pin whose release
|
|
1661
|
+
# condition does not survive a restart is a pin that can never clear.
|
|
1662
|
+
if pin_release_at is not None:
|
|
1663
|
+
d["pin_release_at"] = pin_release_at
|
|
1664
|
+
# Same reason, for the unread-not-shown alarm: KeepAlive restarts a crashing producer, and an
|
|
1665
|
+
# un-persisted suppression would turn a crash loop into a wake storm on a condition nobody can
|
|
1666
|
+
# act on any faster for being told twice.
|
|
1667
|
+
if unread_hidden:
|
|
1668
|
+
d["unread_hidden"] = True
|
|
1669
|
+
dirn = os.path.dirname(os.path.abspath(self.path)) or "."
|
|
1670
|
+
# BOTH OF THESE ARE INSIDE THE GUARD, and they did not used to be (drill, 2026-08-05).
|
|
1671
|
+
# This function builds a careful "written but not provably durable" path - _fsync_dir fails ->
|
|
1672
|
+
# return False -> _state_not_durable() announces -> the producer keeps running on the in-memory
|
|
1673
|
+
# cursor. But `_makedirs_private` and `mkstemp` sat OUTSIDE the try, so the MOST ORDINARY way a
|
|
1674
|
+
# state write actually fails - an unwritable state DIRECTORY - raised EACCES before any of that,
|
|
1675
|
+
# escaped save() entirely, and reached only the last-resort top-level handler: exit 2.
|
|
1676
|
+
# ⇒ THE CONSEQUENCE WAS A WHOLE CATEGORY WORSE THAN THE ONE THIS CODE PREPARED FOR. Supervisors
|
|
1677
|
+
# restart us (launchd KeepAlive; systemd Restart=always, RestartSec=15), and a producer that
|
|
1678
|
+
# dies before persisting its cursor re-delivers the same mail on every respawn. Measured on the
|
|
1679
|
+
# drill specimen: 4 wakes/min for ONE message, forever, cursor frozen - a duplicate storm
|
|
1680
|
+
# arriving through the very path built to prevent one.
|
|
1681
|
+
# ★ A LAST-RESORT HANDLER PLUS SUPERVISED AUTO-RESTART CONVERTS ANY UNGUARDED FAULT INTO A
|
|
1682
|
+
# PERIODIC STORM. Spend the guard budget per-fault; do not delegate it to the supervisor.
|
|
1683
|
+
# The top handler's own comment already said reaching it means a guard is missing.
|
|
1684
|
+
try:
|
|
1685
|
+
_makedirs_private(dirn)
|
|
1686
|
+
fd, tmp = tempfile.mkstemp(dir=dirn, prefix=".kijmon-", suffix=".tmp")
|
|
1687
|
+
except OSError as e:
|
|
1688
|
+
sys.stderr.write("kijito-inbox-monitor: WARNING cannot create a temp file in the state "
|
|
1689
|
+
"directory %s (%s); the cursor is NOT being persisted, so a restart will "
|
|
1690
|
+
"replay mail from an older cursor. Continuing on the in-memory cursor "
|
|
1691
|
+
"rather than exiting - a crash here is restarted into a re-delivery "
|
|
1692
|
+
"loop\n" % (dirn, e))
|
|
1693
|
+
return False
|
|
1694
|
+
try:
|
|
1695
|
+
with os.fdopen(fd, "w") as f:
|
|
1696
|
+
json.dump(d, f)
|
|
1697
|
+
f.flush()
|
|
1698
|
+
os.fsync(f.fileno())
|
|
1699
|
+
os.replace(tmp, self.path)
|
|
1700
|
+
# The rename needs its own sync - see _fsync_dir(). Without it the cursor's DURABILITY story
|
|
1701
|
+
# stops one level short of the thing that makes it visible. ITS ANSWER IS RETURNED, not
|
|
1702
|
+
# discarded: a check whose result nobody reads is not a check.
|
|
1703
|
+
if not _fsync_dir(dirn):
|
|
1704
|
+
sys.stderr.write("kijito-inbox-monitor: WARNING state-file directory fsync FAILED for %s; "
|
|
1705
|
+
"the cursor is written but its durability is UNPROVEN (a crash may replay "
|
|
1706
|
+
"mail from an older cursor)\n" % dirn)
|
|
1707
|
+
return False
|
|
1708
|
+
return True
|
|
1709
|
+
except OSError as e:
|
|
1710
|
+
sys.stderr.write("kijito-inbox-monitor: WARNING state-file write failed (non-fatal): %s\n" % e)
|
|
1711
|
+
try:
|
|
1712
|
+
os.unlink(tmp)
|
|
1713
|
+
except OSError:
|
|
1714
|
+
pass
|
|
1715
|
+
return False
|
|
1716
|
+
|
|
1717
|
+
|
|
1718
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
1719
|
+
# §10 SIGUSR1 self-pipe (POSIX) + clean shutdown
|
|
1720
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
1721
|
+
class WakeSeam:
|
|
1722
|
+
def __init__(self):
|
|
1723
|
+
self.r = self.w = None
|
|
1724
|
+
self.stop = False
|
|
1725
|
+
|
|
1726
|
+
def install(self):
|
|
1727
|
+
if not IS_POSIX:
|
|
1728
|
+
return
|
|
1729
|
+
self.r, self.w = socket.socketpair()
|
|
1730
|
+
self.r.setblocking(False)
|
|
1731
|
+
self.w.setblocking(False)
|
|
1732
|
+
signal.set_wakeup_fd(self.w.fileno())
|
|
1733
|
+
# a real (no-op) handler must be installed or the default disposition terminates the process
|
|
1734
|
+
signal.signal(signal.SIGUSR1, lambda *_: None)
|
|
1735
|
+
# clean shutdown: flip stop flag and let select wake (set_wakeup_fd writes the byte)
|
|
1736
|
+
for sig in (signal.SIGTERM, signal.SIGINT):
|
|
1737
|
+
signal.signal(sig, self._on_stop)
|
|
1738
|
+
|
|
1739
|
+
def _on_stop(self, *_):
|
|
1740
|
+
self.stop = True
|
|
1741
|
+
|
|
1742
|
+
def drain(self):
|
|
1743
|
+
if self.r is None:
|
|
1744
|
+
return
|
|
1745
|
+
try:
|
|
1746
|
+
while True:
|
|
1747
|
+
if not self.r.recv(4096):
|
|
1748
|
+
break
|
|
1749
|
+
except (BlockingIOError, OSError):
|
|
1750
|
+
pass
|
|
1751
|
+
|
|
1752
|
+
def wait(self, timeout):
|
|
1753
|
+
"""Block up to timeout, returning early if a signal byte arrives. Drain happens at the next poll start."""
|
|
1754
|
+
if self.r is None:
|
|
1755
|
+
# Windows / no seam: plain sleep, but stay interruptible-ish via short slices
|
|
1756
|
+
end = _monotonic() + timeout
|
|
1757
|
+
while _monotonic() < end and not self.stop:
|
|
1758
|
+
time_sleep(min(0.5, end - _monotonic()))
|
|
1759
|
+
return
|
|
1760
|
+
try:
|
|
1761
|
+
select.select([self.r], [], [], timeout)
|
|
1762
|
+
except (InterruptedError, OSError):
|
|
1763
|
+
pass
|
|
1764
|
+
|
|
1765
|
+
|
|
1766
|
+
def _monotonic():
|
|
1767
|
+
import time as _t
|
|
1768
|
+
return _t.monotonic()
|
|
1769
|
+
|
|
1770
|
+
|
|
1771
|
+
def time_sleep(s):
|
|
1772
|
+
import time as _t
|
|
1773
|
+
_t.sleep(max(0.0, s))
|
|
1774
|
+
|
|
1775
|
+
|
|
1776
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
1777
|
+
# Core watcher
|
|
1778
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
1779
|
+
def build_headers(args):
|
|
1780
|
+
"""Resolve the required Kijito API token. --token-file wins over $KIJITOMON_TOKEN; missing token is fatal.
|
|
1781
|
+
|
|
1782
|
+
Every request carries a named User-Agent - the Kijito API WAF rejects the default Python-urllib UA with 403.
|
|
1783
|
+
"""
|
|
1784
|
+
headers = {"User-Agent": USER_AGENT}
|
|
1785
|
+
token = None
|
|
1786
|
+
if args.token_file: # --token-file wins over env
|
|
1787
|
+
try:
|
|
1788
|
+
with open(args.token_file) as f:
|
|
1789
|
+
token = f.read().strip()
|
|
1790
|
+
except OSError as e:
|
|
1791
|
+
raise FatalConfig("--token-file unreadable: %s" % e)
|
|
1792
|
+
elif os.environ.get("KIJITOMON_TOKEN"):
|
|
1793
|
+
token = os.environ["KIJITOMON_TOKEN"].strip()
|
|
1794
|
+
if not token:
|
|
1795
|
+
raise FatalConfig("no Kijito API token - set $KIJITOMON_TOKEN or pass --token-file (get a token from "
|
|
1796
|
+
"your Kijito account)")
|
|
1797
|
+
if args.auth_header:
|
|
1798
|
+
headers[args.auth_header] = token
|
|
1799
|
+
else:
|
|
1800
|
+
headers["Authorization"] = "Bearer %s" % token
|
|
1801
|
+
return headers
|
|
1802
|
+
|
|
1803
|
+
|
|
1804
|
+
def persona_url(persona):
|
|
1805
|
+
return "%s?persona=%s&mark_read=false" % (INBOX_URL, urllib.parse.quote(persona))
|
|
1806
|
+
|
|
1807
|
+
|
|
1808
|
+
def make_opener_for(url):
|
|
1809
|
+
p = urllib.parse.urlsplit(url)
|
|
1810
|
+
host = p.hostname or ""
|
|
1811
|
+
port = p.port or (443 if p.scheme == "https" else 80)
|
|
1812
|
+
pinned = resolve_and_pin(host, port)
|
|
1813
|
+
return build_opener(pinned)
|
|
1814
|
+
|
|
1815
|
+
|
|
1816
|
+
def _state_path_for_persona(base_path, persona):
|
|
1817
|
+
if not base_path or not persona:
|
|
1818
|
+
return base_path
|
|
1819
|
+
root, ext = os.path.splitext(base_path)
|
|
1820
|
+
safe = _state_safe_persona(persona)
|
|
1821
|
+
base = os.path.basename(root).casefold()
|
|
1822
|
+
if base == safe or base.endswith("." + safe):
|
|
1823
|
+
return base_path
|
|
1824
|
+
return root + "." + safe + (ext or ".json")
|
|
1825
|
+
|
|
1826
|
+
|
|
1827
|
+
def _state_safe_persona(persona):
|
|
1828
|
+
"""Map a persona to a filename component - CASEFOLDED, deliberately.
|
|
1829
|
+
|
|
1830
|
+
macOS (APFS) and Windows are case-INSENSITIVE, so 'Claude-chat' and 'claude-chat' name the SAME
|
|
1831
|
+
file. Deriving the path from the raw name made the producer block on its OWN flock every tick for
|
|
1832
|
+
a case-variant persona, and left that persona with no event stream at all - a SILENT wake gap,
|
|
1833
|
+
which is the exact failure this tool exists to prevent. Matching case-insensitively here is the
|
|
1834
|
+
filesystem half of the fix; the persona's ORIGINAL case is preserved for the API (persona_url),
|
|
1835
|
+
i.e. case-insensitive match, case-preserving display.
|
|
1836
|
+
"""
|
|
1837
|
+
return "".join(c if (c.isalnum() or c in "._-") else "_" for c in persona.casefold())
|
|
1838
|
+
|
|
1839
|
+
|
|
1840
|
+
_WARNED_PERSONAS = set()
|
|
1841
|
+
|
|
1842
|
+
|
|
1843
|
+
def _warn_persona_once(persona, text):
|
|
1844
|
+
"""Emit a per-persona warning at most ONCE per process.
|
|
1845
|
+
|
|
1846
|
+
Persona discovery runs every tick, so a condition that cannot resolve itself (a state file held by
|
|
1847
|
+
another watcher, an unusable path) otherwise grows stderr without bound: one observed 3-day run had
|
|
1848
|
+
20,079 of 20,129 stderr lines from a single repeated warning, which buries every other diagnostic.
|
|
1849
|
+
"""
|
|
1850
|
+
key = persona.casefold()
|
|
1851
|
+
if key in _WARNED_PERSONAS:
|
|
1852
|
+
return
|
|
1853
|
+
_WARNED_PERSONAS.add(key)
|
|
1854
|
+
sys.stderr.write("kijito-inbox-monitor: WARNING %s (further warnings for %r suppressed)\n"
|
|
1855
|
+
% (text, persona))
|
|
1856
|
+
|
|
1857
|
+
|
|
1858
|
+
def _clear_persona_warning(persona):
|
|
1859
|
+
"""WHAT CLEARS THIS: the condition the warning described actually recovering.
|
|
1860
|
+
|
|
1861
|
+
Without a release, suppress-once is itself an instance of loom's class - a state set and never
|
|
1862
|
+
cleared - and it fails in the dangerous direction: a persona whose sink broke, recovered, then broke
|
|
1863
|
+
AGAIN would be silently suppressed forever, so the second outage arrives with no diagnostic at all.
|
|
1864
|
+
Called from the recovery path, never on a timer: the warning is suppressed exactly as long as the
|
|
1865
|
+
condition it reported is still true.
|
|
1866
|
+
"""
|
|
1867
|
+
_WARNED_PERSONAS.discard(persona.casefold())
|
|
1868
|
+
|
|
1869
|
+
|
|
1870
|
+
def requested_personas(args, opener, headers):
|
|
1871
|
+
# ⚠️ CASE-INSENSITIVE DEDUPE, THE SAME RULE AS new_personas() (re-audit 11, F3). This used to be an
|
|
1872
|
+
# EXACT `p not in personas`, so `--persona Loom --persona loom` survived as two entries - but
|
|
1873
|
+
# _state_safe_persona() CASEFOLDS the state path, so both resolve to ONE state file. The second
|
|
1874
|
+
# flock then raises FatalConfig("state-file in use") out of the UNCAUGHT list comprehension in
|
|
1875
|
+
# run(), and the producer refuses to start FOR EVERY PERSONA. That breaks the containment rule this
|
|
1876
|
+
# file states twice ("one persona's hostile path must not take the whole producer down"), and the
|
|
1877
|
+
# error blamed "another watcher" when the collision was with itself.
|
|
1878
|
+
# Keeps the FIRST spelling seen, exactly like new_personas(), so nothing about the normal path moves.
|
|
1879
|
+
personas = []
|
|
1880
|
+
seen = set()
|
|
1881
|
+
|
|
1882
|
+
def add(p):
|
|
1883
|
+
key = p.casefold()
|
|
1884
|
+
if p and key not in seen:
|
|
1885
|
+
seen.add(key)
|
|
1886
|
+
personas.append(p)
|
|
1887
|
+
|
|
1888
|
+
for p in (p.strip() for p in args.persona or []):
|
|
1889
|
+
add(p)
|
|
1890
|
+
for group in args.personas or []:
|
|
1891
|
+
for p in (part.strip() for part in group.split(",")):
|
|
1892
|
+
add(p)
|
|
1893
|
+
if args.all_personas or not personas:
|
|
1894
|
+
for p in fetch_personas(opener, headers):
|
|
1895
|
+
add(p)
|
|
1896
|
+
return personas
|
|
1897
|
+
|
|
1898
|
+
|
|
1899
|
+
def watches_all_personas(args):
|
|
1900
|
+
return args.all_personas or not (args.persona or args.personas)
|
|
1901
|
+
|
|
1902
|
+
|
|
1903
|
+
def new_personas(existing, discovered):
|
|
1904
|
+
# Case-INSENSITIVE: a case-variant of a persona we already watch is the SAME inbox and (on a
|
|
1905
|
+
# case-insensitive filesystem) the same state file - adopting it again self-deadlocks. Also
|
|
1906
|
+
# collapses variants within `discovered`, keeping the first spelling seen.
|
|
1907
|
+
seen = {p.casefold() for p in existing}
|
|
1908
|
+
out = []
|
|
1909
|
+
for p in discovered:
|
|
1910
|
+
key = p.casefold()
|
|
1911
|
+
if key not in seen:
|
|
1912
|
+
seen.add(key)
|
|
1913
|
+
out.append(p)
|
|
1914
|
+
return out
|
|
1915
|
+
|
|
1916
|
+
|
|
1917
|
+
class WatchTarget:
|
|
1918
|
+
def __init__(self, persona, url, opener, headers, args, emitter):
|
|
1919
|
+
self.persona = persona
|
|
1920
|
+
self.url = url
|
|
1921
|
+
self.opener = opener
|
|
1922
|
+
self.headers = headers
|
|
1923
|
+
self.args = args
|
|
1924
|
+
self.emitter = emitter
|
|
1925
|
+
self.identity = canonical_identity(url)
|
|
1926
|
+
self.state_file = None
|
|
1927
|
+
self.cursor = None
|
|
1928
|
+
self.fsm_state = "UP"
|
|
1929
|
+
self.failures = 0
|
|
1930
|
+
self.armed = False
|
|
1931
|
+
self.fast_path = False
|
|
1932
|
+
self.last_unread = None
|
|
1933
|
+
self.skips = 0
|
|
1934
|
+
self.first_poll = True
|
|
1935
|
+
self.last_heartbeat = _monotonic()
|
|
1936
|
+
# FAIL-CLOSED state. `cursor` is a CONFIRMED-CONTIGUOUS watermark: everything at or below it is
|
|
1937
|
+
# known delivered. When the server admits it hid messages above the cursor, the watermark PINS
|
|
1938
|
+
# rather than stepping over them, and ids emitted above the pin are remembered here so liveness
|
|
1939
|
+
# (delivering what we can see) does not cost us duplicates. Both are persisted.
|
|
1940
|
+
self.emitted_above = set()
|
|
1941
|
+
self.gap_alerted = None # the pinned watermark we have already alerted on, so pinning does not spam
|
|
1942
|
+
# False once we can no longer reason about the pinned span - tracking overflowed, or the persisted
|
|
1943
|
+
# pin state was corrupt. A gap can then only be closed by an authoritative read, never by counting.
|
|
1944
|
+
self.pin_evidence_intact = True
|
|
1945
|
+
self.pin_forced = False # hold a pin whose tracking we lost, so nothing can jump the watermark
|
|
1946
|
+
self.state_corrupt = False # a state file was PRESENT but unusable: arm fail-closed, and say so
|
|
1947
|
+
# The floor at which a corruption pin may RELEASE. A corrupt-state arm parks the watermark one
|
|
1948
|
+
# BELOW the window it re-emits, so the ordinary release test (a complete window reaching back to
|
|
1949
|
+
# at-or-below the watermark) is unsatisfiable by construction - the pin carries the floor that
|
|
1950
|
+
# discharges it instead. Persisted, because a release condition that dies on restart is not one.
|
|
1951
|
+
self.pin_release_at = None
|
|
1952
|
+
self.delivery_blocked = False # an emit failed; the cursor is held below it until it succeeds
|
|
1953
|
+
self.state_not_durable = False # the last cursor write could not be proven durable
|
|
1954
|
+
# Is the unread-not-shown alarm currently ANNOUNCED? Keyed on the CONDITION, so it self-clears
|
|
1955
|
+
# (§5.2). One flag per target, so the key is per-inbox and EXACT - a case-variant persona is a
|
|
1956
|
+
# different WatchTarget with a different flag, and cannot hold this one's alarm down.
|
|
1957
|
+
self.unread_hidden = False
|
|
1958
|
+
# Does the persona DIRECTORY know this inbox? Optimistic by default: see poll_once() for why this
|
|
1959
|
+
# alarm fails OPEN where the stranded-mail alarm fails closed.
|
|
1960
|
+
self.directory_backed = True
|
|
1961
|
+
|
|
1962
|
+
self.count_url = NOTIFY_PENDING_URL
|
|
1963
|
+
cp = urllib.parse.urlsplit(url)
|
|
1964
|
+
self.unread_persona = dict(urllib.parse.parse_qsl(cp.query)).get("persona") or persona
|
|
1965
|
+
|
|
1966
|
+
state_path = _state_path_for_persona(args.state_file, persona)
|
|
1967
|
+
if state_path:
|
|
1968
|
+
self.state_file = StateFile(state_path, self.identity)
|
|
1969
|
+
if not args.self_test:
|
|
1970
|
+
self.state_file.lock()
|
|
1971
|
+
loaded = self.state_file.load()
|
|
1972
|
+
if loaded is CORRUPT_STATE:
|
|
1973
|
+
# A file that EXISTS but cannot be parsed is EVIDENCE A CURSOR EXISTED. Baselining
|
|
1974
|
+
# here would step over every message between that lost cursor and now, invisibly.
|
|
1975
|
+
# So fail closed: keep no cursor, force the pin so the watermark cannot jump, and
|
|
1976
|
+
# mark the evidence unusable. The first poll then emits everything visible (the
|
|
1977
|
+
# replay cap is bypassed while pinned) and the gap is announced rather than buried.
|
|
1978
|
+
self.state_corrupt = True
|
|
1979
|
+
self.pin_forced = True
|
|
1980
|
+
self.pin_evidence_intact = False
|
|
1981
|
+
elif loaded is not None:
|
|
1982
|
+
self.cursor = loaded["cursor"]
|
|
1983
|
+
self.fsm_state, self.failures = loaded["state"], loaded["failures"]
|
|
1984
|
+
self.emitted_above = loaded["emitted_above"]
|
|
1985
|
+
self.gap_alerted = loaded["gap_alerted"]
|
|
1986
|
+
self.pin_evidence_intact = loaded["pin_evidence_intact"]
|
|
1987
|
+
self.state_corrupt = loaded["state_corrupt"]
|
|
1988
|
+
self.pin_release_at = loaded["pin_release_at"]
|
|
1989
|
+
# A persisted forced pin is authoritative; the inference from missing tracking is only
|
|
1990
|
+
# a fallback for files written before the flag existed.
|
|
1991
|
+
self.pin_forced = loaded["pin_forced"] or not loaded["pin_evidence_intact"]
|
|
1992
|
+
self.unread_hidden = loaded["unread_hidden"]
|
|
1993
|
+
if args.seed_at is not None:
|
|
1994
|
+
self.cursor = args.seed_at
|
|
1995
|
+
|
|
1996
|
+
def self_test(self):
|
|
1997
|
+
poll = fetch(self.opener, self.url, self.headers)
|
|
1998
|
+
reach_ok = poll.ok
|
|
1999
|
+
label = self.persona or self.url
|
|
2000
|
+
sys.stderr.write("self-test[%s]: source %s (%s)\n" % (
|
|
2001
|
+
label, "REACHABLE+healthy" if reach_ok else "UNHEALTHY", poll.reason or "ok"
|
|
2002
|
+
))
|
|
2003
|
+
# CONSUME THE EMITTER'S ANSWER (Loom re-audit 10, M3). This was `emit_ok = True` with the call as a
|
|
2004
|
+
# bare statement, flipping only on an EXCEPTION - but emit() reports a failed delivery by RETURNING
|
|
2005
|
+
# False, which is its documented, non-exceptional path (a refused sink, a failed write, a non-zero
|
|
2006
|
+
# --exec). So a self-test against a sink that had just refused the write printed emit=OK. The one
|
|
2007
|
+
# surface whose entire job is to tell an operator "this works before you trust it" was itself an
|
|
2008
|
+
# instance of the class it exists to detect - which is why a sweep starts with the diagnostics.
|
|
2009
|
+
try:
|
|
2010
|
+
emit_ok = bool(self.emitter.new({"id": 0, "from": "self-test", "content": "synthetic emit OK",
|
|
2011
|
+
"created": _now_iso(), "_persona": self.persona}))
|
|
2012
|
+
except Exception as e:
|
|
2013
|
+
emit_ok = False
|
|
2014
|
+
sys.stderr.write("self-test[%s]: emit FAILED: %s\n" % (label, e))
|
|
2015
|
+
sys.stderr.write("self-test[%s]: emit=%s reachable=%s\n" % (
|
|
2016
|
+
label, "OK" if emit_ok else "FAIL", reach_ok
|
|
2017
|
+
))
|
|
2018
|
+
return reach_ok and emit_ok
|
|
2019
|
+
|
|
2020
|
+
def lifecycle(self, event, **fields):
|
|
2021
|
+
"""Emit a lifecycle event. RETURNS whether it was delivered (Loom/river re-audit 11, F1).
|
|
2022
|
+
|
|
2023
|
+
This used to drop `Emitter.lifecycle`'s bool on the floor, which made every caller structurally
|
|
2024
|
+
unable to know whether the thing it had just recorded as announced was in fact announced.
|
|
2025
|
+
"""
|
|
2026
|
+
if self.persona:
|
|
2027
|
+
fields["persona"] = self.persona
|
|
2028
|
+
return self.emitter.lifecycle(event, **fields)
|
|
2029
|
+
|
|
2030
|
+
def _alarm(self, event, log_text, **fields):
|
|
2031
|
+
"""Emit an edge event and GUARANTEE it reaches a human, returning whether it was DELIVERED.
|
|
2032
|
+
|
|
2033
|
+
`log_text` is the stderr wording ONLY; it is deliberately NOT injected into the event, so this
|
|
2034
|
+
adds no field to any event's schema (§6.1). Callers that want `reason` on the wire pass it in
|
|
2035
|
+
**fields like any other field.
|
|
2036
|
+
|
|
2037
|
+
THE DEFECT THIS CLOSES (re-audit 11, F1): every alarm committed its "already alarmed" state
|
|
2038
|
+
BEFORE emitting and discarded the emit's answer, so an alarm that was never delivered was never
|
|
2039
|
+
re-raised - not after the channel recovered, and not after a restart, because `gap_alerted` is
|
|
2040
|
+
PERSISTED. Mail was never at risk (the cursor holds correctly throughout); it was the ALARMS
|
|
2041
|
+
that vanished, which is worse than it sounds because the tool's headline promise is that it
|
|
2042
|
+
pins LOUDLY rather than in silence.
|
|
2043
|
+
|
|
2044
|
+
The fallback is stderr, NOT a retry down the event channel - that channel is the thing that
|
|
2045
|
+
just failed, and reporting a fault through the faulty channel is how the fault stays invisible
|
|
2046
|
+
(DESIGN.md §176, the same reasoning as _delivery_failed). Deliberately does NOT gate the cursor
|
|
2047
|
+
on a lifecycle event: DESIGN.md §170 says lifecycle events are not acknowledged and not gated,
|
|
2048
|
+
and that stays true. "Do not gate the watermark on it" and "do not record that you alarmed when
|
|
2049
|
+
you did not" are different propositions, and only the first was ever documented.
|
|
2050
|
+
"""
|
|
2051
|
+
delivered = self.lifecycle(event, **fields)
|
|
2052
|
+
if not delivered:
|
|
2053
|
+
sys.stderr.write("kijito-inbox-monitor: %s EVENT UNDELIVERED (persona %r): %s\n"
|
|
2054
|
+
% (event.upper(), self.persona, log_text))
|
|
2055
|
+
return delivered
|
|
2056
|
+
|
|
2057
|
+
def _pin_release_floor(self):
|
|
2058
|
+
"""The reach a COMPLETE window must achieve to discharge a pin.
|
|
2059
|
+
|
|
2060
|
+
Normally the watermark itself: a window reaching back to at-or-below the cursor visibly spans
|
|
2061
|
+
everything we have not confirmed. A CORRUPTION pin is the exception - it parks the watermark one
|
|
2062
|
+
below the window it re-emits, so that window's own floor is always cursor+1 and the ordinary test
|
|
2063
|
+
can never be met by it (Loom re-audit 7, HIGH 5: the pin never cleared, the cursor never moved,
|
|
2064
|
+
and the window was re-delivered on every poll for the life of the process AND across restarts).
|
|
2065
|
+
Taking the MAX keeps the ordinary rule exactly as strict as it was - a recorded floor can only
|
|
2066
|
+
ever be the span we re-emitted, never something below the watermark.
|
|
2067
|
+
"""
|
|
2068
|
+
floor = self.cursor or 0
|
|
2069
|
+
if self.pin_release_at is not None:
|
|
2070
|
+
floor = max(floor, self.pin_release_at)
|
|
2071
|
+
return floor
|
|
2072
|
+
|
|
2073
|
+
def _release_pin(self):
|
|
2074
|
+
"""Discharge a forced pin AND everything that was holding it up.
|
|
2075
|
+
|
|
2076
|
+
ONE implementation, because there are TWO authoritative proofs (a completed backward walk, and a
|
|
2077
|
+
complete window reaching the release floor) and they must leave identical state. They did not:
|
|
2078
|
+
the walk path cleared only `pin_forced`, so a released corruption pin went on persisting
|
|
2079
|
+
`state_corrupt: true` and its `pin_release_at` for the life of the file - a state file still
|
|
2080
|
+
describing a corruption that had been fully recovered. Found by running the repro against the
|
|
2081
|
+
LIVE api.kijito.ai and reading what was actually written back, which no fixture asserted.
|
|
2082
|
+
"""
|
|
2083
|
+
self.pin_forced = False
|
|
2084
|
+
self.pin_release_at = None
|
|
2085
|
+
self.state_corrupt = False
|
|
2086
|
+
|
|
2087
|
+
def _state_not_durable(self):
|
|
2088
|
+
if self.state_not_durable:
|
|
2089
|
+
return
|
|
2090
|
+
self.state_not_durable = True
|
|
2091
|
+
sys.stderr.write("kijito-inbox-monitor: WARNING the cursor for persona %r was written but its "
|
|
2092
|
+
"durability is UNPROVEN; a crash may replay mail from an older cursor (further "
|
|
2093
|
+
"reports suppressed until it persists cleanly)\n" % self.persona)
|
|
2094
|
+
|
|
2095
|
+
def _state_durable_again(self):
|
|
2096
|
+
if not self.state_not_durable:
|
|
2097
|
+
return
|
|
2098
|
+
self.state_not_durable = False
|
|
2099
|
+
sys.stderr.write("kijito-inbox-monitor: cursor persistence for persona %r recovered\n" % self.persona)
|
|
2100
|
+
|
|
2101
|
+
def _delivery_failed(self, mid):
|
|
2102
|
+
"""Report a failed hand-off ONCE, and say what the watcher is doing about it.
|
|
2103
|
+
|
|
2104
|
+
Deliberately stderr and NOT an `alert` event: the event channel is the thing that just failed, so
|
|
2105
|
+
an alarm about it would be routed through the broken pipe (exec mode re-runs the same failing
|
|
2106
|
+
command; sink mode writes to the file that just refused a write). Reporting a fault down the
|
|
2107
|
+
faulty channel is how the fault stays invisible. Keyed on the condition and self-clearing, like
|
|
2108
|
+
every other alarm here.
|
|
2109
|
+
"""
|
|
2110
|
+
if self.delivery_blocked:
|
|
2111
|
+
return
|
|
2112
|
+
self.delivery_blocked = True
|
|
2113
|
+
sys.stderr.write("kijito-inbox-monitor: WARNING delivery of message %s to persona %r FAILED; HOLDING the "
|
|
2114
|
+
"cursor below it so it is re-delivered rather than skipped (further reports "
|
|
2115
|
+
"suppressed until delivery recovers)\n" % (mid, self.persona))
|
|
2116
|
+
|
|
2117
|
+
def _delivery_recovered(self):
|
|
2118
|
+
if not self.delivery_blocked:
|
|
2119
|
+
return
|
|
2120
|
+
self.delivery_blocked = False
|
|
2121
|
+
sys.stderr.write("kijito-inbox-monitor: delivery to persona %r recovered; the cursor is advancing "
|
|
2122
|
+
"again\n" % self.persona)
|
|
2123
|
+
|
|
2124
|
+
def _uncovered_gap(self, poll, items):
|
|
2125
|
+
"""(cursor, window_floor, omitted) iff omitted mail may sit ABOVE the cursor, else None.
|
|
2126
|
+
|
|
2127
|
+
THE DISCRIMINATOR, and it is the whole reason this is not a permanent alarm:
|
|
2128
|
+
window_floor <= cursor -> the window reaches back PAST what we already emitted, so every
|
|
2129
|
+
omitted message is BELOW the cursor and was already delivered. Safe.
|
|
2130
|
+
window_floor > cursor -> the window starts above the cursor while the server says it dropped
|
|
2131
|
+
things, so the uncovered span (cursor, window_floor) may hold mail
|
|
2132
|
+
we have never emitted. Unsafe.
|
|
2133
|
+
In steady state the long-poll keeps the backlog to a message or two, so the window always reaches
|
|
2134
|
+
back and this returns None - no behaviour change. It fires after an outage or a burst, which is
|
|
2135
|
+
exactly when a bounded window starts hiding things.
|
|
2136
|
+
"""
|
|
2137
|
+
declared, exact = poll.omitted, poll.omitted_exact
|
|
2138
|
+
if not poll.continuation_ok:
|
|
2139
|
+
# SILENCE IS NOT AN ANSWER HERE EITHER. A window whose `next_before_id` is ABSENT or
|
|
2140
|
+
# MALFORMED has told us nothing about whether it withheld rows, so its "I omitted nothing"
|
|
2141
|
+
# cannot be taken as an assertion - the two fields are one statement and half of it is
|
|
2142
|
+
# unreadable. The WALK has refused to read that silence as exhaustion since Loom re-audit 5
|
|
2143
|
+
# (HIGH 1); the gap check never got the same rule, so a server that garbled the field while
|
|
2144
|
+
# declaring no omission advanced the watermark over anything it was hiding, silently and
|
|
2145
|
+
# with no alert. Same defect, one layer over. (Found by re-reading round 7 adversarially.)
|
|
2146
|
+
declared, exact = max(declared, 1), False
|
|
2147
|
+
if not poll.consistent:
|
|
2148
|
+
# A SELF-CONTRADICTORY WINDOW IS AN OMISSION WE CANNOT COUNT (Loom re-audit 7, HIGH 4). This
|
|
2149
|
+
# check used to read `poll.omitted` alone and never looked at the continuation at all, so a
|
|
2150
|
+
# window declaring "I withheld nothing" while handing back a cursor for older mail was taken
|
|
2151
|
+
# at its word - and the watermark stepped over whatever the continuation was pointing at.
|
|
2152
|
+
# There is no number to reach here, so it enters as an UNQUANTIFIED withholding: closable
|
|
2153
|
+
# only by a backward walk that exhausts the span, never by arithmetic.
|
|
2154
|
+
declared, exact = max(declared, 1), False
|
|
2155
|
+
if not declared or self.cursor is None or not self.armed or not items:
|
|
2156
|
+
return None
|
|
2157
|
+
floor = min(m["id"] for m in items)
|
|
2158
|
+
if floor <= self.cursor:
|
|
2159
|
+
return None
|
|
2160
|
+
return (self.cursor, floor, declared, exact)
|
|
2161
|
+
|
|
2162
|
+
def _hidden_unread(self, poll):
|
|
2163
|
+
"""Does the server hold unread mail this window did not show us? True / False / None (NO CLAIM).
|
|
2164
|
+
|
|
2165
|
+
§5.2 A CHEAP ALARM SIGNAL - deliberately NOT a coverage mechanism. `unread_not_shown` is a COUNT
|
|
2166
|
+
with no cursor of its own, so it can say THAT something is out of view but never WHICH rows;
|
|
2167
|
+
coverage stays with the backward walk, which terminates (§5.1). This answers only the alarm
|
|
2168
|
+
question "is there unread mail I cannot see", where a superset is the right answer because you
|
|
2169
|
+
want to know regardless of WHY the mail is absent.
|
|
2170
|
+
|
|
2171
|
+
THE TRAP, AND IT INVERTS THE OBVIOUS READING OF A ZERO. The server computes this field ONLY
|
|
2172
|
+
when it withheld something; otherwise it is 0 BY CONSTRUCTION. So `== 0` does NOT assert "no
|
|
2173
|
+
unread mail exists". VERIFIED LIVE against api.kijito.ai on a real inbox holding 4 unread:
|
|
2174
|
+
newest page next_before_id=1179 unread_not_shown=0 <- computed, and truly nothing hidden
|
|
2175
|
+
walk page next_before_id=1145 unread_not_shown=4 <- the whole inbox's unread, not this window's
|
|
2176
|
+
terminal page next_before_id=null unread_not_shown=0 <- 0 WITH 4 UNREAD SITTING ABOVE IT
|
|
2177
|
+
Reading that last 0 as "clear" is the false-negative this method exists to refuse. A FALSE
|
|
2178
|
+
assertion is therefore only avoidable by requiring POSITIVE evidence for the negative answer,
|
|
2179
|
+
never by trusting the number - which is why the two False branches below are justified by
|
|
2180
|
+
DIFFERENT facts and are not the redundancy they look like.
|
|
2181
|
+
|
|
2182
|
+
EVALUATE ONLY ON THE NEWEST-PAGE POLL. On a backward-walk page `next_before_id is None` means
|
|
2183
|
+
merely "nothing OLDER than this page", not "nothing outside this window" - the terminal-page row
|
|
2184
|
+
above is exactly that case. poll_once() calls this with the un-cursored poll only, so the walk
|
|
2185
|
+
pages structurally cannot reach it.
|
|
2186
|
+
"""
|
|
2187
|
+
n = poll.unread_not_shown
|
|
2188
|
+
if n is None:
|
|
2189
|
+
return None # server made no statement (older API) -> assert nothing in either direction
|
|
2190
|
+
if n > 0:
|
|
2191
|
+
return True # unread mail exists that this response did not include
|
|
2192
|
+
if poll.next_before_id is not None:
|
|
2193
|
+
return False # the 0 was genuinely COMPUTED against a withheld remainder
|
|
2194
|
+
if poll.omitted == 0:
|
|
2195
|
+
return False # complete window: nothing older exists and nothing was withheld
|
|
2196
|
+
# Contradictory: rows were declared omitted, yet no cursor leads to them. The 0 is unexplained,
|
|
2197
|
+
# so make no claim rather than report a clear we cannot justify.
|
|
2198
|
+
return None
|
|
2199
|
+
|
|
2200
|
+
def _walk_back(self, from_id, stop_at):
|
|
2201
|
+
"""Page BACKWARD over (stop_at, from_id) and return (rows, covered).
|
|
2202
|
+
|
|
2203
|
+
This is the AUTHORITATIVE way to read an omitted span, and it replaces the unread_only
|
|
2204
|
+
heuristic entirely. Two properties the heuristic never had:
|
|
2205
|
+
· it reaches messages someone has already READ - the exact rows unread_only structurally
|
|
2206
|
+
cannot see, and the ones most likely to be hidden in an old span;
|
|
2207
|
+
· it TERMINATES, so the span can be declared covered by exhaustion rather than by counting
|
|
2208
|
+
recovered rows against a number the server may never have stated.
|
|
2209
|
+
That is what makes an INEXACT omission count closable at all.
|
|
2210
|
+
|
|
2211
|
+
Contract (river, api main @249e2b3): pass the OLDEST id you were returned as `before_id` and
|
|
2212
|
+
repeat until the page is empty or `next_before_id` is null. OMIT the parameter for the newest
|
|
2213
|
+
page - 0 is a REAL cursor, not "no cursor". A malformed cursor is a hard 400, so a bug here
|
|
2214
|
+
fails loudly instead of silently re-serving the newest page.
|
|
2215
|
+
|
|
2216
|
+
`covered` is True only if the walk reached stop_at or ran out of older messages. A walk cut
|
|
2217
|
+
short by the page budget returns False, and the caller must keep the watermark pinned: a
|
|
2218
|
+
partial walk proves nothing, and claiming otherwise is the very failure this replaced.
|
|
2219
|
+
|
|
2220
|
+
THE CHAIN IS VALIDATED STRICTLY, NOT ASSUMED (Loom re-audit 5, HIGH 1). Coverage-by-exhaustion
|
|
2221
|
+
is only as good as the chain being a real chain, so every link is checked before it is trusted:
|
|
2222
|
+
· the continuation must BE AN ANSWER. A missing or malformed `next_before_id` is not an
|
|
2223
|
+
end-of-chain, it is silence, and reading silence as "nothing older" hands back coverage the
|
|
2224
|
+
server never asserted.
|
|
2225
|
+
· the continuation must EQUAL THE OLDEST ROW WE WERE HANDED. The contract is "pass the oldest
|
|
2226
|
+
id you were returned"; a server whose continuation points BELOW that is skipping the rows in
|
|
2227
|
+
between, and following it walks straight over them while reporting success.
|
|
2228
|
+
Neither check can be satisfied by accident, and both fail to PIN, which is the safe direction.
|
|
2229
|
+
"""
|
|
2230
|
+
sep = "&" if "?" in self.url else "?"
|
|
2231
|
+
rows, cursor, pages = [], from_id, 0
|
|
2232
|
+
while pages < WALK_BACK_MAX_PAGES:
|
|
2233
|
+
pages += 1
|
|
2234
|
+
poll = fetch(self.opener, "%s%sbefore_id=%d" % (self.url, sep, cursor), self.headers)
|
|
2235
|
+
if not poll.ok:
|
|
2236
|
+
return (rows, False) # transient failure: no claim either way
|
|
2237
|
+
if not poll.continuation_ok:
|
|
2238
|
+
# Absent or malformed continuation: the server did not answer. NOT exhaustion.
|
|
2239
|
+
return (rows, False)
|
|
2240
|
+
batch = poll.items or []
|
|
2241
|
+
rows.extend(batch)
|
|
2242
|
+
# VALIDATE THE PAGE BEFORE TAKING ANY COVERAGE FROM IT. Every check below rejects a page
|
|
2243
|
+
# whose own account of itself does not hold together; a page that fails one of them cannot
|
|
2244
|
+
# be trusted to have handed back the rows it appears to contain, so it may not close a span
|
|
2245
|
+
# even when it seems to reach the watermark.
|
|
2246
|
+
if not poll.consistent:
|
|
2247
|
+
# SELF-CONTRADICTORY PAGE, IN EITHER DIRECTION. Withheld-rows + "nothing older" (Loom
|
|
2248
|
+
# re-audit 6, HIGH 3) and withheld-nothing + "there is more" (Loom re-audit 7, HIGH 4)
|
|
2249
|
+
# are the same defect facing opposite ways: the two halves of the page's declaration
|
|
2250
|
+
# disagree, so believing EITHER half steps over what the other one just asserted.
|
|
2251
|
+
return (rows, False)
|
|
2252
|
+
if not batch:
|
|
2253
|
+
if poll.next_before_id is not None:
|
|
2254
|
+
# EMPTY PAGE CLAIMING THERE IS MORE (Loom re-audit 6, HIGH 2). It returned nothing
|
|
2255
|
+
# while pointing further back, so the range it covered is unobserved - and because
|
|
2256
|
+
# the oldest-row check has no row to check, following the pointer walks straight
|
|
2257
|
+
# over that range and still reports the span covered.
|
|
2258
|
+
return (rows, False)
|
|
2259
|
+
return (rows, True) # empty AND affirmed terminal: the chain genuinely ends
|
|
2260
|
+
oldest = min(m["id"] for m in batch)
|
|
2261
|
+
if poll.next_before_id is not None and poll.next_before_id != oldest:
|
|
2262
|
+
# The chain skips rows between `oldest` and the continuation. Following it would
|
|
2263
|
+
# walk over them and still report the span covered.
|
|
2264
|
+
# Checked BEFORE the reach-back return below: a page that reaches the watermark while
|
|
2265
|
+
# skipping rows is still a page whose row set we cannot vouch for, and taking coverage
|
|
2266
|
+
# from it would be trusting the one page we just caught misdescribing itself.
|
|
2267
|
+
return (rows, False)
|
|
2268
|
+
if oldest <= stop_at:
|
|
2269
|
+
return (rows, True) # walked back past the watermark: span fully seen
|
|
2270
|
+
if poll.next_before_id is None:
|
|
2271
|
+
return (rows, True) # server AFFIRMS there is nothing older
|
|
2272
|
+
if poll.next_before_id >= cursor:
|
|
2273
|
+
return (rows, False) # cursor not advancing; refuse to spin
|
|
2274
|
+
cursor = poll.next_before_id
|
|
2275
|
+
return (rows, False) # budget exhausted before reaching the watermark
|
|
2276
|
+
|
|
2277
|
+
def poll_once(self, counts_available=False, unread_counts=None):
|
|
2278
|
+
args = self.args
|
|
2279
|
+
unread_counts = unread_counts or {}
|
|
2280
|
+
|
|
2281
|
+
skip_full = False
|
|
2282
|
+
if self.armed and self.fast_path and not args.no_fast_path and self.unread_persona:
|
|
2283
|
+
if counts_available:
|
|
2284
|
+
unread = unread_counts.get(self.unread_persona, 0)
|
|
2285
|
+
increased = unread > self.last_unread if self.last_unread is not None else True
|
|
2286
|
+
self.last_unread = unread
|
|
2287
|
+
if not increased and self.skips < args.resync_every:
|
|
2288
|
+
skip_full = True
|
|
2289
|
+
self.skips += 1
|
|
2290
|
+
# unavailable (transient) → fall through to the full inbox-list poll (the baseline)
|
|
2291
|
+
|
|
2292
|
+
if skip_full:
|
|
2293
|
+
# count endpoint reachable + no unread increase = a HEALTHY poll with no new items
|
|
2294
|
+
if self.fsm_state == "DOWN":
|
|
2295
|
+
# THE RECOVERY EDGE IS THE SAME DEFECT FACING THE OTHER WAY (argus, re-audit 11 - a
|
|
2296
|
+
# site the review did not name). Committing "UP" and discarding the emit means a
|
|
2297
|
+
# consumer that saw the DOWN alert never learns the source came back: it is left
|
|
2298
|
+
# holding an alarm it can NEVER clear, because this edge is crossed exactly once.
|
|
2299
|
+
# The transition must commit (it is the FSM); the announcement gets stderr.
|
|
2300
|
+
self.fsm_state = "UP"
|
|
2301
|
+
self._alarm("recovered", "source recovered", cursor=self.cursor)
|
|
2302
|
+
self.failures = 0
|
|
2303
|
+
else:
|
|
2304
|
+
self.skips = 0
|
|
2305
|
+
poll = fetch(self.opener, self.url, self.headers)
|
|
2306
|
+
|
|
2307
|
+
if poll.status == 404 and (self.first_poll or args.self_test):
|
|
2308
|
+
raise FatalConfig("inbox endpoint 404 (hive disabled?) - fatal at startup")
|
|
2309
|
+
if poll.status == 401 and (self.first_poll or args.self_test):
|
|
2310
|
+
raise FatalConfig("inbox endpoint 401 (bad or missing token) - fatal at startup")
|
|
2311
|
+
|
|
2312
|
+
if poll.ok:
|
|
2313
|
+
recovered = False
|
|
2314
|
+
if self.fsm_state == "DOWN":
|
|
2315
|
+
self.fsm_state = "UP"
|
|
2316
|
+
recovered = True
|
|
2317
|
+
self.failures = 0
|
|
2318
|
+
|
|
2319
|
+
items = poll.items
|
|
2320
|
+
# §5.4 Record who AUTHORED what, from the window we already have. Done before any cursor
|
|
2321
|
+
# or dedup logic: authorship is evidence about the SENDER and is worth collecting whether
|
|
2322
|
+
# or not the message is new to US - a message we have already delivered still proves its
|
|
2323
|
+
# author was alive when they sent it.
|
|
2324
|
+
note_authorship(items)
|
|
2325
|
+
note_observation_floor(self.persona, items)
|
|
2326
|
+
diag = None
|
|
2327
|
+
new_items = []
|
|
2328
|
+
do_arm = not self.armed
|
|
2329
|
+
|
|
2330
|
+
if do_arm:
|
|
2331
|
+
if self.cursor is None and self.state_corrupt:
|
|
2332
|
+
# Fail CLOSED: arm BELOW everything visible and EMIT the whole window, rather than
|
|
2333
|
+
# baselining to the newest id and skipping the lost span in silence. The replay cap
|
|
2334
|
+
# is deliberately not applied - it exists to stop a huge first-run backlog, and
|
|
2335
|
+
# here every visible message is one we may already owe someone.
|
|
2336
|
+
self.cursor = min((m["id"] for m in items), default=0) - 1
|
|
2337
|
+
new_items = sorted(items, key=lambda m: m["id"])
|
|
2338
|
+
# THE PIN NOW CARRIES ITS OWN RELEASE FLOOR (Loom re-audit 7, HIGH 5) - see
|
|
2339
|
+
# _pin_release_floor(). Set here, and again below if this first window was empty.
|
|
2340
|
+
diag = ("state_corrupt", {"armed_at": self.cursor,
|
|
2341
|
+
"reason": "state file present but unusable; re-emitting the "
|
|
2342
|
+
"visible window instead of baselining over it"})
|
|
2343
|
+
elif self.cursor is None:
|
|
2344
|
+
self.cursor = max((m["id"] for m in items), default=0)
|
|
2345
|
+
# ⛔ AN ABSENT STATE FILE MEANS TWO THINGS THAT DEMAND OPPOSITE BEHAVIOUR, AND
|
|
2346
|
+
# NOTHING HERE CAN TELL THEM APART. A genuine first launch must baseline - never
|
|
2347
|
+
# flood a new agent with inbox history. A LOST state file must not: everything
|
|
2348
|
+
# since the vanished cursor is owed to someone. The branch above distinguishes
|
|
2349
|
+
# exists-but-corrupt, because a file that is present is EVIDENCE a cursor existed.
|
|
2350
|
+
# Absence leaves no such evidence, so the baseline stands - but it no longer
|
|
2351
|
+
# happens QUIETLY. (Found by assay's state-wipe drill, 2026-08-05: a wiped state
|
|
2352
|
+
# file skipped an unread message with no bounce and no record. Clause 5's rule is
|
|
2353
|
+
# "fail open HONESTLY, never silently" - the honesty is the part that was missing.)
|
|
2354
|
+
#
|
|
2355
|
+
# ⚠️ NOT re-emitting: the anti-flood behaviour is deliberate and unchanged. This
|
|
2356
|
+
# only converts a silent skip into an announced one.
|
|
2357
|
+
#
|
|
2358
|
+
# The unread COUNT is per-persona and comes from a different endpoint; items carry
|
|
2359
|
+
# no per-message unread flag, so this cannot say WHICH of the skipped messages are
|
|
2360
|
+
# unread - only how many the persona holds. Stated, not glossed.
|
|
2361
|
+
if items:
|
|
2362
|
+
# Keyed on unread_persona, NOT persona: it honours an explicit ?persona=
|
|
2363
|
+
# in the watch URL and falls back to persona otherwise (:1749), and it is
|
|
2364
|
+
# what both existing count consumers use (:2069, :2491). Keying this one
|
|
2365
|
+
# differently would diverge exactly when a watch URL carries the override
|
|
2366
|
+
# - rare, and therefore the kind of divergence that survives a long time.
|
|
2367
|
+
u = unread_counts.get(self.unread_persona) if counts_available else None
|
|
2368
|
+
# An UNKNOWN unread count must not be read as zero - that assumption is the
|
|
2369
|
+
# whole defect, one level up. Silent only when the count is KNOWN to be 0.
|
|
2370
|
+
if u != 0:
|
|
2371
|
+
diag = ("baseline_skipped", {
|
|
2372
|
+
"armed_at": self.cursor,
|
|
2373
|
+
"skipped": len(items),
|
|
2374
|
+
"id_range": [min(m["id"] for m in items), max(m["id"] for m in items)],
|
|
2375
|
+
"unread_held": u if u is not None else "unknown",
|
|
2376
|
+
"reason": "no state file: baselined to the newest visible id rather than "
|
|
2377
|
+
"re-emitting. If this was a LOST state file rather than a first "
|
|
2378
|
+
"launch, these messages will never raise a wake - they remain "
|
|
2379
|
+
"unread and readable in the inbox, but nothing will announce them",
|
|
2380
|
+
})
|
|
2381
|
+
else:
|
|
2382
|
+
current_max = max((m["id"] for m in items), default=0)
|
|
2383
|
+
# A RESTORED PIN SURVIVES ARMING. `emitted_above` is only ever non-empty when a
|
|
2384
|
+
# previous run pinned the watermark below an unresolved gap, so both branches below
|
|
2385
|
+
# must respect it: the replay cap would otherwise jump the cursor straight over the
|
|
2386
|
+
# gap on the first poll after a restart, silently erasing it, and the replay count
|
|
2387
|
+
# would double-count mail we already delivered.
|
|
2388
|
+
# `pin_forced` covers the case where the pin is real but its tracking was lost,
|
|
2389
|
+
# so an empty emitted_above must NOT read as "nothing was pinned".
|
|
2390
|
+
pinned_on_load = bool(self.emitted_above) or self.pin_forced
|
|
2391
|
+
n = sum(1 for m in items if m["id"] > self.cursor)
|
|
2392
|
+
if self.cursor > current_max:
|
|
2393
|
+
diag = ("seed_ahead", {"seeded": self.cursor, "current_max": current_max})
|
|
2394
|
+
elif n > args.max_replay and not pinned_on_load:
|
|
2395
|
+
diag = ("replay_capped", {"capped_to": current_max, "dropped": n})
|
|
2396
|
+
self.cursor = current_max
|
|
2397
|
+
self.emitted_above = set()
|
|
2398
|
+
else:
|
|
2399
|
+
new_items = sorted((m for m in items
|
|
2400
|
+
if m["id"] > self.cursor
|
|
2401
|
+
and m["id"] not in self.emitted_above),
|
|
2402
|
+
key=lambda m: m["id"])
|
|
2403
|
+
self.armed = True
|
|
2404
|
+
else:
|
|
2405
|
+
# `emitted_above` is normally empty. It is non-empty only while the watermark is PINNED
|
|
2406
|
+
# below an unresolved gap, and it is what lets us keep delivering visible mail without
|
|
2407
|
+
# re-delivering it on every subsequent poll.
|
|
2408
|
+
new_items = sorted((m for m in items
|
|
2409
|
+
if m["id"] > self.cursor and m["id"] not in self.emitted_above),
|
|
2410
|
+
key=lambda m: m["id"])
|
|
2411
|
+
|
|
2412
|
+
# THE CORRUPTION PIN'S RELEASE FLOOR, in ONE place so the arming poll and a later one
|
|
2413
|
+
# cannot disagree (Loom re-audit 7, HIGH 5). A corrupt arm parks the watermark at
|
|
2414
|
+
# min(visible)-1 so it can re-emit the whole window; that makes the ordinary release test
|
|
2415
|
+
# - a complete window reaching back to at-or-below the watermark - unsatisfiable by
|
|
2416
|
+
# construction, because the reach IS min(visible) and min(visible) > min(visible)-1. So
|
|
2417
|
+
# the pin recorded the floor it must reach back to instead. Also set on a LATER poll when
|
|
2418
|
+
# the arming window was EMPTY: there was no floor to record then, and leaving it unset
|
|
2419
|
+
# would freeze the watermark exactly as before, one poll further on.
|
|
2420
|
+
if self.pin_forced and self.state_corrupt and self.pin_release_at is None and items:
|
|
2421
|
+
self.pin_release_at = min(m["id"] for m in items)
|
|
2422
|
+
if recovered:
|
|
2423
|
+
# Same recovery edge as the fast path above, and the commit is 60+ lines earlier
|
|
2424
|
+
# (`fsm_state = "UP"; recovered = True`), which is why a block-local detector could
|
|
2425
|
+
# not pair them - it was found by reading, prompted by its twin.
|
|
2426
|
+
self._alarm("recovered", "source recovered", cursor=self.cursor)
|
|
2427
|
+
if diag:
|
|
2428
|
+
self.lifecycle(diag[0], **diag[1])
|
|
2429
|
+
if do_arm:
|
|
2430
|
+
self.lifecycle("armed", cursor=self.cursor)
|
|
2431
|
+
# §5.1 A BOUNDED WINDOW MUST NOT SILENTLY SWALLOW MAIL.
|
|
2432
|
+
# The server returns the NEWEST messages that fit, and declares what it left out. If it
|
|
2433
|
+
# omitted anything AND the window does not reach back to our cursor, un-emitted mail can
|
|
2434
|
+
# be sitting in the uncovered gap - and advancing the cursor past it loses it forever.
|
|
2435
|
+
window_cursor = self.cursor # the watermark AS THIS WINDOW SAW IT, before any advance
|
|
2436
|
+
gap = self._uncovered_gap(poll, items)
|
|
2437
|
+
pinned = False
|
|
2438
|
+
release_earned = False
|
|
2439
|
+
if gap is not None:
|
|
2440
|
+
cursor_at, window_floor, omitted, omitted_exact = gap
|
|
2441
|
+
visible = {m["id"] for m in items}
|
|
2442
|
+
# Count ONLY rows the visible window did not already contain and that sit above the
|
|
2443
|
+
# watermark. Counting every returned row lets a retry that echoes the same suffix be
|
|
2444
|
+
# reported as a recovery that never happened - a false success, worse than a loud failure.
|
|
2445
|
+
# Walk the span BACKWARD from the window floor down to the watermark. Coverage is
|
|
2446
|
+
# proven by exhausting the chain, not by counting rows against a number - which is
|
|
2447
|
+
# why this closes an INEXACT omission count that no amount of counting could.
|
|
2448
|
+
walked, covered = self._walk_back(window_floor, cursor_at)
|
|
2449
|
+
unseen = [m for m in walked
|
|
2450
|
+
if m["id"] > (self.cursor or 0) and m["id"] not in visible
|
|
2451
|
+
and m["id"] not in self.emitted_above]
|
|
2452
|
+
gap_recovered = [m for m in unseen if cursor_at < m["id"] < window_floor]
|
|
2453
|
+
known = {m["id"] for m in new_items}
|
|
2454
|
+
for m in unseen:
|
|
2455
|
+
if m["id"] not in known:
|
|
2456
|
+
new_items.append(m)
|
|
2457
|
+
known.add(m["id"])
|
|
2458
|
+
new_items.sort(key=lambda m: m["id"])
|
|
2459
|
+
|
|
2460
|
+
# FAIL CLOSED unless there is POSITIVE evidence the span is accounted for. Recovery here
|
|
2461
|
+
# is a heuristic (unread_only), not an authoritative backward page, so silence from it
|
|
2462
|
+
# proves nothing. Treat the gap as closed only when the reconciling window was itself
|
|
2463
|
+
# COMPLETE (it declared no omissions of its own) and it yielded at least as many
|
|
2464
|
+
# previously-unseen rows as the server said it withheld. Anything less pins the
|
|
2465
|
+
# watermark: stepping over would make the next poll see floor<=cursor, declare itself
|
|
2466
|
+
# safe, and bury the omission permanently.
|
|
2467
|
+
# CLOSURE BY EXHAUSTION, not by arithmetic. A completed backward walk has SEEN the
|
|
2468
|
+
# whole span, so the omission count - exact or not - stops mattering. A walk cut
|
|
2469
|
+
# short proves nothing and keeps the watermark pinned.
|
|
2470
|
+
# `pin_evidence_intact` still gates: once tracking has overflowed we cannot tell a
|
|
2471
|
+
# recovered row from one we delivered and forgot, so we do not trust our own view of
|
|
2472
|
+
# what is new until the walk itself re-establishes it.
|
|
2473
|
+
closed = covered and (self.pin_evidence_intact or bool(walked))
|
|
2474
|
+
if closed and not self.pin_evidence_intact:
|
|
2475
|
+
# An authoritative read re-establishes ground truth, so the span is knowable again.
|
|
2476
|
+
self.pin_evidence_intact = True
|
|
2477
|
+
# RELEASE THE FORCED PIN (Loom re-audit 5, MEDIUM) - but not here, and not yet. A
|
|
2478
|
+
# forced pin was held because tracking was lost, and a COMPLETED walk is the
|
|
2479
|
+
# authoritative evidence that replaces it; leaving it set froze the watermark
|
|
2480
|
+
# permanently. The DECISION is made here, where the evidence is; the ACT is deferred
|
|
2481
|
+
# until after delivery, because a pin must not be discharged on a poll that failed to
|
|
2482
|
+
# hand over what it was holding (see the release site below).
|
|
2483
|
+
release_earned = closed
|
|
2484
|
+
pinned = not closed
|
|
2485
|
+
# Alert identity is the PINNED WATERMARK, not the window floor. The floor drifts upward
|
|
2486
|
+
# as new mail arrives, so keying on it re-fires for what is the same unresolved span;
|
|
2487
|
+
# the watermark is stable for exactly as long as the gap is unresolved. Persisted, so a
|
|
2488
|
+
# restart does not re-announce it either.
|
|
2489
|
+
if pinned and self.gap_alerted != cursor_at:
|
|
2490
|
+
# ★ LATCH ONLY ON DELIVERY (re-audit 11, F1). `gap_alerted` is a PURE
|
|
2491
|
+
# ANNOUNCEMENT MARKER - it drives no behaviour, it only records "this span was
|
|
2492
|
+
# announced" - and it is PERSISTED, so committing it before the emit meant an
|
|
2493
|
+
# undelivered alarm was suppressed for the life of the span AND across restarts.
|
|
2494
|
+
# The gap condition is re-derived every poll, so re-raising costs nothing and
|
|
2495
|
+
# self-clears the moment it is genuinely delivered.
|
|
2496
|
+
gap_reason = ("bounded-window: server omitted %d message(s) and the window "
|
|
2497
|
+
"started at id %s above cursor %s; a backward walk recovered %d "
|
|
2498
|
+
"from inside the span but did not reach the watermark, so "
|
|
2499
|
+
"it stays PINNED at %s"
|
|
2500
|
+
% (omitted, window_floor, cursor_at, len(gap_recovered), cursor_at))
|
|
2501
|
+
if self._alarm("alert", gap_reason,
|
|
2502
|
+
reason=gap_reason,
|
|
2503
|
+
omitted=omitted, window_floor=window_floor, cursor_at=cursor_at,
|
|
2504
|
+
reconciled=len(gap_recovered), pinned=True):
|
|
2505
|
+
self.gap_alerted = cursor_at
|
|
2506
|
+
|
|
2507
|
+
# DELIVERY IS ACKNOWLEDGED, NOT ASSUMED (Loom re-audit 7, HIGH 1). The cursor IS the
|
|
2508
|
+
# acknowledgement - once it advances past an id, that message is never fetched again - so
|
|
2509
|
+
# it may only advance over messages the emitter actually DELIVERED. An --exec that exits
|
|
2510
|
+
# non-zero, times out, or cannot be spawned used to advance it anyway: the wake hook that
|
|
2511
|
+
# is the entire point of exec mode failed, the message was never retried, and the watcher
|
|
2512
|
+
# reported success. Delivery stops at the FIRST failure so a consumer never sees message
|
|
2513
|
+
# N+1 before a retried N; the guarantee is at-least-once IN ORDER, because a duplicate is
|
|
2514
|
+
# recoverable and a skip is not.
|
|
2515
|
+
delivered, blocked_at = set(), None
|
|
2516
|
+
for m in new_items:
|
|
2517
|
+
if blocked_at is not None:
|
|
2518
|
+
break
|
|
2519
|
+
row = dict(m)
|
|
2520
|
+
row["_persona"] = self.persona
|
|
2521
|
+
if self.emitter.new(row) is True:
|
|
2522
|
+
delivered.add(m["id"])
|
|
2523
|
+
else:
|
|
2524
|
+
blocked_at = m["id"]
|
|
2525
|
+
self._delivery_failed(m["id"])
|
|
2526
|
+
# THE DURABILITY BARRIER (Loom re-audit 7, MEDIUM): the event must be on stable storage
|
|
2527
|
+
# BEFORE the cursor that acknowledges it. If the sink cannot be synced, NOTHING emitted
|
|
2528
|
+
# this poll counts as delivered - the acknowledgement is retracted wholesale rather than
|
|
2529
|
+
# left half-true.
|
|
2530
|
+
if delivered and not self.emitter.sync(self.persona):
|
|
2531
|
+
blocked_at = min(delivered) if blocked_at is None else min(blocked_at, min(delivered))
|
|
2532
|
+
delivered = set()
|
|
2533
|
+
if blocked_at is None:
|
|
2534
|
+
self._delivery_recovered()
|
|
2535
|
+
|
|
2536
|
+
# §5.2 UNREAD MAIL WE CANNOT SEE. Fires on the FALSE->TRUE edge and releases itself when
|
|
2537
|
+
# the condition clears, so it needs no ack: an ack would let someone silence "there is
|
|
2538
|
+
# mail you are not being shown" while it was still true.
|
|
2539
|
+
hidden = self._hidden_unread(poll)
|
|
2540
|
+
if hidden is True:
|
|
2541
|
+
# Routed like the stranded-mail alarm, but failing the OPPOSITE way on purpose. That
|
|
2542
|
+
# one withholds when the directory is unknown because alarming would flag EVERY
|
|
2543
|
+
# persona; this one concerns the target's OWN inbox, so the worst case of firing is a
|
|
2544
|
+
# line in a stream nobody reads, while the worst case of withholding is the silent
|
|
2545
|
+
# wake gap this whole tool exists to prevent. Suppressed only for an inbox the
|
|
2546
|
+
# directory positively does not know - and the flag is then left UNSET so the alarm
|
|
2547
|
+
# can still announce itself if that inbox later becomes directory-backed.
|
|
2548
|
+
if not self.unread_hidden and self.directory_backed:
|
|
2549
|
+
floor = min((m["id"] for m in items), default=None)
|
|
2550
|
+
unread_reason = (
|
|
2551
|
+
"unread-not-shown: the server reports %d unread message(s) in this "
|
|
2552
|
+
"inbox that this window did not include. OBSERVATION, NOT A "
|
|
2553
|
+
"DIAGNOSIS: the count covers unread mail ANYWHERE in the inbox, "
|
|
2554
|
+
"including messages already delivered to this stream that the agent "
|
|
2555
|
+
"has not read, so it is not by itself evidence of missed mail. "
|
|
2556
|
+
"Coverage of an un-emitted span is proven by the backward walk, "
|
|
2557
|
+
"never by this count." % poll.unread_not_shown)
|
|
2558
|
+
# ★ LATCH ONLY ON DELIVERY, and go through _alarm for the stderr fallback
|
|
2559
|
+
# (re-audit 11, F1 - this feature predates that rule and was written against the
|
|
2560
|
+
# old `lifecycle` + commit-first shape). `unread_hidden` is a PURE ANNOUNCEMENT
|
|
2561
|
+
# LATCH: it drives no behaviour, it only records "this condition was announced",
|
|
2562
|
+
# and it is PERSISTED - so committing it before the emit would suppress an
|
|
2563
|
+
# UNDELIVERED alarm for the life of the condition AND across restarts. The
|
|
2564
|
+
# condition is re-derived every poll, so re-raising costs nothing and self-clears
|
|
2565
|
+
# the moment it is genuinely delivered. THE DISCRIMINATING QUESTION: does
|
|
2566
|
+
# this state DRIVE behaviour, or does it only RECORD that something was
|
|
2567
|
+
# announced? Behavioural state must commit either way; a pure announcement
|
|
2568
|
+
# latch must commit only on delivery. They take opposite answers.
|
|
2569
|
+
if self._alarm("alert", unread_reason,
|
|
2570
|
+
reason=unread_reason,
|
|
2571
|
+
unread_not_shown=poll.unread_not_shown,
|
|
2572
|
+
window_floor=floor, cursor_at=window_cursor,
|
|
2573
|
+
# The discriminating FACT, left for the reader to interpret: when
|
|
2574
|
+
# the window reaches back past the watermark, everything above it
|
|
2575
|
+
# is visible, so the unseen unread can only be mail already
|
|
2576
|
+
# delivered.
|
|
2577
|
+
above_watermark=(None if floor is None or window_cursor is None
|
|
2578
|
+
else floor > window_cursor)):
|
|
2579
|
+
self.unread_hidden = True
|
|
2580
|
+
elif hidden is False:
|
|
2581
|
+
self.unread_hidden = False # condition cleared -> re-arm, so a recurrence is announced
|
|
2582
|
+
# hidden is None -> the server made no statement; hold the current state and claim nothing
|
|
2583
|
+
|
|
2584
|
+
# ★ A PIN IS NOT DISCHARGED ON A POLL THAT COULD NOT DELIVER (found by adversarially
|
|
2585
|
+
# re-reading my own round-7 work, the way loom would). Both proofs answer "did the SERVER
|
|
2586
|
+
# withhold anything" - neither says a word about whether WE handed the window over. On a
|
|
2587
|
+
# corrupt-state arm the watermark sits at min(visible)-1, so releasing while delivery was
|
|
2588
|
+
# blocked threw away the release floor AND the state_corrupt flag while the cursor was
|
|
2589
|
+
# still parked below the whole mailbox. A restart then re-forced the pin from the
|
|
2590
|
+
# surviving pin_evidence_intact=False - now with NO floor - and `reach <= cursor` is
|
|
2591
|
+
# unsatisfiable when the cursor sits below the oldest message that exists. Measured: the
|
|
2592
|
+
# watermark froze at 99 forever and correctness fell back entirely onto emitted_above
|
|
2593
|
+
# growing without bound. Holding the pin one more poll costs nothing; the delivery gate
|
|
2594
|
+
# already holds the cursor, and the pin self-clears the moment delivery recovers.
|
|
2595
|
+
if release_earned and blocked_at is None:
|
|
2596
|
+
self._release_pin()
|
|
2597
|
+
|
|
2598
|
+
if not pinned:
|
|
2599
|
+
# A COMPLETE window that reaches back past the watermark proves everything above it is
|
|
2600
|
+
# visible, so a leftover pin can be released even when there is nothing NEW to emit.
|
|
2601
|
+
# Gating this on `new_items` left a restored pin stuck forever whenever the window
|
|
2602
|
+
# contained only ids we had already delivered - the exact state a restart lands in.
|
|
2603
|
+
reach = min((m["id"] for m in items), default=None)
|
|
2604
|
+
# `continuation_ok` belongs here for the same reason it belongs in the gap check:
|
|
2605
|
+
# a window that did not answer "is there more?" cannot be the PROOF that there is not.
|
|
2606
|
+
complete = (poll.omitted == 0 and poll.consistent and poll.continuation_ok
|
|
2607
|
+
and reach is not None and reach <= self._pin_release_floor())
|
|
2608
|
+
if self.pin_forced and complete and blocked_at is None:
|
|
2609
|
+
# The other authoritative proof: nothing was withheld AND the window reaches back
|
|
2610
|
+
# past the watermark, so there is no span left to be uncertain about. Without this
|
|
2611
|
+
# a forced pin that never sees a gap again could never clear, and the watermark
|
|
2612
|
+
# would stay frozen for the life of the process.
|
|
2613
|
+
self._release_pin()
|
|
2614
|
+
if self.pin_forced:
|
|
2615
|
+
high = None # still forced: the watermark holds
|
|
2616
|
+
else:
|
|
2617
|
+
high = max(sorted(delivered)
|
|
2618
|
+
+ ([max(m["id"] for m in items)] if complete else []), default=None)
|
|
2619
|
+
if blocked_at is not None and high is not None:
|
|
2620
|
+
# THE GATE ITSELF: never acknowledge past a message we could not hand over, not
|
|
2621
|
+
# even via a complete window. `complete` proves the SERVER hid nothing; it says
|
|
2622
|
+
# nothing about whether WE delivered what it showed us.
|
|
2623
|
+
high = min(high, blocked_at - 1)
|
|
2624
|
+
if high is not None and high > (self.cursor or 0):
|
|
2625
|
+
self.cursor = high
|
|
2626
|
+
# Watermark moved, so anything at or below it is confirmed and needs no tracking.
|
|
2627
|
+
self.emitted_above = {i for i in self.emitted_above if i > self.cursor}
|
|
2628
|
+
if not self.emitted_above:
|
|
2629
|
+
self.gap_alerted = None
|
|
2630
|
+
self.pin_evidence_intact = True
|
|
2631
|
+
|
|
2632
|
+
# EVERY DELIVERED ID THE WATERMARK DOES NOT COVER IS REMEMBERED, whatever left it
|
|
2633
|
+
# uncovered (Loom re-audit 7, HIGH 5). This used to live inside the `pinned` branch
|
|
2634
|
+
# alone, so the OTHER ways of not advancing - a forced pin with no gap in sight, a
|
|
2635
|
+
# delivery that failed further up the batch - delivered mail and then forgot they had.
|
|
2636
|
+
# The corruption pin hit exactly that: it could not advance and it recorded nothing, so
|
|
2637
|
+
# it re-emitted its entire window on every poll and every restart, forever.
|
|
2638
|
+
uncovered = {i for i in delivered if i > (self.cursor or 0)}
|
|
2639
|
+
if uncovered:
|
|
2640
|
+
self.emitted_above.update(uncovered)
|
|
2641
|
+
if len(self.emitted_above) > PIN_TRACKING_CAP:
|
|
2642
|
+
# A pin that cannot clear would otherwise grow this set - and the state file -
|
|
2643
|
+
# without bound. Keep the NEWEST ids (the ones a future window can still show us,
|
|
2644
|
+
# and therefore the ones that could be re-emitted) and drop the oldest.
|
|
2645
|
+
keep = sorted(self.emitted_above)[-PIN_TRACKING_CAP:]
|
|
2646
|
+
dropped = len(self.emitted_above) - len(keep)
|
|
2647
|
+
self.emitted_above = set(keep)
|
|
2648
|
+
# ONCE WE HAVE FORGOTTEN A DELIVERED ID, WE CAN NO LONGER REASON ABOUT THIS SPAN.
|
|
2649
|
+
# A forgotten id reappearing in a reconcile looks "previously unseen", so it would
|
|
2650
|
+
# both re-emit AND be counted as recovery - manufacturing evidence out of our own
|
|
2651
|
+
# amnesia. From here the gap can only be closed by an authoritative read.
|
|
2652
|
+
if self.pin_evidence_intact:
|
|
2653
|
+
# ⚠️ THIS COMMIT IS NOT AN ANNOUNCEMENT LATCH - it is a CORRECTNESS state that
|
|
2654
|
+
# governs how the span may ever be closed, so it MUST be committed whether or
|
|
2655
|
+
# not the alarm is delivered. Only the ANNOUNCEMENT needs the second channel,
|
|
2656
|
+
# which _alarm provides (re-audit 11, F1). Getting this backwards - refusing to
|
|
2657
|
+
# record evidence loss because a sink was broken - would trade a lost alarm for
|
|
2658
|
+
# a lost invariant.
|
|
2659
|
+
self.pin_evidence_intact = False
|
|
2660
|
+
# A durable event, not just stderr: this is a correctness degradation somebody
|
|
2661
|
+
# has to act on, and stderr is not something a consumer watches. _alarm still
|
|
2662
|
+
# falls back to stderr, so an undelivered one is not silent.
|
|
2663
|
+
pin_reason = ("bounded-window: pin at cursor %s outlived its tracking "
|
|
2664
|
+
"budget and forgot %d delivered id(s). Some mail may be "
|
|
2665
|
+
"re-emitted, and this span can no longer be closed by "
|
|
2666
|
+
"reconciliation - it needs an authoritative backward read"
|
|
2667
|
+
% (self.cursor, dropped))
|
|
2668
|
+
self._alarm("alert", pin_reason, reason=pin_reason,
|
|
2669
|
+
cursor_at=self.cursor, forgot=dropped,
|
|
2670
|
+
pinned=True, evidence_lost=True)
|
|
2671
|
+
|
|
2672
|
+
else:
|
|
2673
|
+
self.failures += 1
|
|
2674
|
+
if self.failures == args.alert_after and self.fsm_state == "UP":
|
|
2675
|
+
# THE DEAD-MAN'S SWITCH. The FSM transition MUST commit (it drives the whole
|
|
2676
|
+
# liveness model, and the firing condition is an EQUALITY on `failures`, so a
|
|
2677
|
+
# reverted transition would never re-fire - the edge is crossed exactly once).
|
|
2678
|
+
# So the state commits and the ANNOUNCEMENT gets the guaranteed second channel
|
|
2679
|
+
# (re-audit 11, F1/A1). Before this, a broken sink meant the source could go down
|
|
2680
|
+
# and NOTHING was ever emitted or logged - the one event README sells as the
|
|
2681
|
+
# dead-man's switch, silently absent.
|
|
2682
|
+
self.fsm_state = "DOWN"
|
|
2683
|
+
down_reason = poll.reason or "unreachable"
|
|
2684
|
+
self._alarm("alert", "source is DOWN: %s" % down_reason,
|
|
2685
|
+
reason=down_reason,
|
|
2686
|
+
consecutive_failures=self.failures,
|
|
2687
|
+
seconds=self.failures * args.poll_seconds)
|
|
2688
|
+
|
|
2689
|
+
if self.state_file is not None:
|
|
2690
|
+
durable = self.state_file.save(self.cursor, self.fsm_state, self.failures,
|
|
2691
|
+
self.emitted_above, self.gap_alerted,
|
|
2692
|
+
pin_forced=self.pin_forced,
|
|
2693
|
+
pin_evidence_intact=self.pin_evidence_intact,
|
|
2694
|
+
state_corrupt=self.state_corrupt,
|
|
2695
|
+
pin_release_at=self.pin_release_at,
|
|
2696
|
+
unread_hidden=self.unread_hidden)
|
|
2697
|
+
# ★ CONSUME THE ANSWER (Loom re-audit 9, MEDIUM). Round 8 taught me to RETURN a durability
|
|
2698
|
+
# status; this is the same defect one layer out - I produced an answer and then discarded it
|
|
2699
|
+
# at the call site, which is the exact thing the previous round was about. A cursor whose
|
|
2700
|
+
# persistence is unproven means a crash may replay mail, and the harm is the SILENCE.
|
|
2701
|
+
if durable is False:
|
|
2702
|
+
self._state_not_durable()
|
|
2703
|
+
else:
|
|
2704
|
+
self._state_durable_again()
|
|
2705
|
+
|
|
2706
|
+
# §9 enable the fast-path once - on the first healthy poll where the count endpoint is available.
|
|
2707
|
+
# (Single enable point; the max-id cursor stays the source of truth for WHAT to emit, unread is only
|
|
2708
|
+
# the wake TRIGGER, so a late/again enable is harmless.)
|
|
2709
|
+
if self.armed and not self.fast_path and not args.no_fast_path and self.unread_persona and counts_available:
|
|
2710
|
+
self.fast_path = True
|
|
2711
|
+
self.last_unread = unread_counts.get(self.unread_persona, 0)
|
|
2712
|
+
|
|
2713
|
+
if args.heartbeat and (_monotonic() - self.last_heartbeat) >= args.heartbeat:
|
|
2714
|
+
self.lifecycle("heartbeat", cursor=self.cursor)
|
|
2715
|
+
self.last_heartbeat = _monotonic()
|
|
2716
|
+
|
|
2717
|
+
self.first_poll = False
|
|
2718
|
+
|
|
2719
|
+
|
|
2720
|
+
def build_persona_target(persona, opener_by_origin, headers, args, emitter):
|
|
2721
|
+
url = persona_url(persona)
|
|
2722
|
+
origin = urllib.parse.urlsplit(url).netloc
|
|
2723
|
+
opener = opener_by_origin.get(origin)
|
|
2724
|
+
if opener is None:
|
|
2725
|
+
opener = make_opener_for(url)
|
|
2726
|
+
opener_by_origin[origin] = opener
|
|
2727
|
+
return WatchTarget(persona, url, opener, headers, args, emitter)
|
|
2728
|
+
|
|
2729
|
+
|
|
2730
|
+
def discover_persona_targets(args, headers, emitter, targets, opener_by_origin, directory_opener):
|
|
2731
|
+
current = [t.persona for t in targets if t.persona]
|
|
2732
|
+
discovered = fetch_personas(directory_opener, headers)
|
|
2733
|
+
added = []
|
|
2734
|
+
# `discovered` is returned as well as used: it is the DIRECTORY namespace, which the stranded-mail
|
|
2735
|
+
# check diffs the inbox namespace against. Fetched here already, so the check costs no extra request.
|
|
2736
|
+
for persona in new_personas(current, discovered):
|
|
2737
|
+
try:
|
|
2738
|
+
target = build_persona_target(persona, opener_by_origin, headers, args, emitter)
|
|
2739
|
+
except (FatalConfig, OSError) as e:
|
|
2740
|
+
# ★ THE CATCH'S TYPE MUST COVER THE THROW, not merely exist.
|
|
2741
|
+
# This arm read as containment for two rounds and was not: `build_persona_target` reaches
|
|
2742
|
+
# `StateFile.lock()` -> `_open_private(path + ".lock")`, which raises `InsecureFile` - and
|
|
2743
|
+
# `InsecureFile` subclasses **OSError, not FatalConfig** (deliberately, so the SINK path can
|
|
2744
|
+
# turn it into a failed delivery instead of a crash). So ONE persona with a hostile or
|
|
2745
|
+
# un-tightenable lock sidecar escaped this arm and killed the whole producer. `InsecureFile`
|
|
2746
|
+
# is raised by security code and SOUNDS like a config fatality, which is exactly why
|
|
2747
|
+
# "FatalConfig covers it" was the natural and wrong assumption. A guard naming the wrong
|
|
2748
|
+
# exception type is indistinguishable at a glance from one that works.
|
|
2749
|
+
_warn_persona_once(persona, "cannot add persona %r: %s" % (persona, e))
|
|
2750
|
+
continue
|
|
2751
|
+
targets.append(target)
|
|
2752
|
+
added.append(persona)
|
|
2753
|
+
target.lifecycle("persona_added")
|
|
2754
|
+
return added, discovered
|
|
2755
|
+
|
|
2756
|
+
|
|
2757
|
+
def discover_from_counts(args, counts, targets, opener_by_origin, headers, emitter):
|
|
2758
|
+
"""Add a watch target for any persona that appears in the notify counts (i.e. has mail) but isn't watched yet.
|
|
2759
|
+
|
|
2760
|
+
This is how a NEW persona is picked up within one tick of receiving mail - for free from the long-poll / fast-path
|
|
2761
|
+
counts we already fetch - instead of waiting for the periodic /api/personas rescan. Only auto-adds in all-personas
|
|
2762
|
+
mode; an explicit --persona/--personas subset stays fixed."""
|
|
2763
|
+
if not watches_all_personas(args):
|
|
2764
|
+
return []
|
|
2765
|
+
# Case-INSENSITIVE membership: see new_personas(). The counts come from the INBOX namespace, which
|
|
2766
|
+
# can legitimately hold a name the persona DIRECTORY does not (that divergence is what stranded mail
|
|
2767
|
+
# in the first place), so this is the path where case-variants actually show up.
|
|
2768
|
+
current = {t.persona.casefold() for t in targets if t.persona}
|
|
2769
|
+
added = []
|
|
2770
|
+
for persona in counts:
|
|
2771
|
+
if persona and persona.casefold() not in current:
|
|
2772
|
+
try:
|
|
2773
|
+
target = build_persona_target(persona, opener_by_origin, headers, args, emitter)
|
|
2774
|
+
except (FatalConfig, OSError) as e:
|
|
2775
|
+
# Same widening, same reason as discover_persona_targets: `InsecureFile` is an OSError,
|
|
2776
|
+
# so a FatalConfig-only arm never contained it. This is the LATE-ADD path a
|
|
2777
|
+
# brand-new persona arrives on, so it is reached by anyone who can get a name into the
|
|
2778
|
+
# inbox counts - the containment matters more here, not less.
|
|
2779
|
+
_warn_persona_once(persona, "cannot add persona %r from counts: %s" % (persona, e))
|
|
2780
|
+
continue
|
|
2781
|
+
targets.append(target)
|
|
2782
|
+
added.append(persona)
|
|
2783
|
+
current.add(persona.casefold())
|
|
2784
|
+
target.lifecycle("persona_added")
|
|
2785
|
+
return added
|
|
2786
|
+
|
|
2787
|
+
|
|
2788
|
+
def refresh_directory_backing(args, directory, targets):
|
|
2789
|
+
"""Mark each target with whether the persona DIRECTORY knows its inbox (§5.2 alarm routing).
|
|
2790
|
+
|
|
2791
|
+
Refreshed EVERY tick rather than stamped when a target is built, because a brand-new persona's first
|
|
2792
|
+
mail arrives through discover_from_counts BEFORE the periodic /api/personas rescan sees it: a
|
|
2793
|
+
creation-time flag would brand a perfectly real persona as unbacked and then never revisit it, since
|
|
2794
|
+
rediscovery skips personas already watched.
|
|
2795
|
+
|
|
2796
|
+
Compared EXACTLY, never casefolded - the same asymmetry as stranded_inboxes(). The server's inbox
|
|
2797
|
+
namespace is case-SENSITIVE, so a case-variant is a DIFFERENT inbox and must not inherit the real
|
|
2798
|
+
one's backing.
|
|
2799
|
+
|
|
2800
|
+
Two no-ops, both deliberate: an explicit --persona/--personas subset was hand-picked by an operator
|
|
2801
|
+
who is by definition consuming those streams, and an EMPTY directory is missing data rather than
|
|
2802
|
+
evidence of absence. In both cases the existing (optimistic) value stands.
|
|
2803
|
+
"""
|
|
2804
|
+
if not watches_all_personas(args) or not directory:
|
|
2805
|
+
return
|
|
2806
|
+
known = {p for p in directory if p}
|
|
2807
|
+
for target in targets:
|
|
2808
|
+
if target.persona:
|
|
2809
|
+
target.directory_backed = target.persona in known
|
|
2810
|
+
|
|
2811
|
+
|
|
2812
|
+
# §5.4 Attributable authorship, observed for free. Every inbox window the watcher already fetches carries a
|
|
2813
|
+
# `from` on every row, so watching all personas means seeing who AUTHORED what without a single extra request.
|
|
2814
|
+
# {persona: (highest message id it authored, that message's created stamp)}.
|
|
2815
|
+
_LAST_AUTHORED = {}
|
|
2816
|
+
|
|
2817
|
+
|
|
2818
|
+
def note_authorship(items):
|
|
2819
|
+
"""Record the newest message id seen from each AUTHOR (§5.4).
|
|
2820
|
+
|
|
2821
|
+
Attributable in the way inbox-read and `/api/presence` are not: only B produces B's outbound, and no
|
|
2822
|
+
third party can manufacture or erase it by reading something. That is the whole reason this signal is
|
|
2823
|
+
worth collecting - a liveness check built on a bit any observer can flip is not a check.
|
|
2824
|
+
|
|
2825
|
+
Keyed EXACTLY, never casefolded: the server's persona namespace is case-SENSITIVE, so `Loom` and `loom`
|
|
2826
|
+
are different identities and must not merge (the identity half of the case asymmetry, §5.3).
|
|
2827
|
+
|
|
2828
|
+
Only the newest window is needed. Backward-walk rows are always OLDER than the window floor they were
|
|
2829
|
+
reached from, so they cannot raise a maximum; skipping them costs no evidence.
|
|
2830
|
+
"""
|
|
2831
|
+
for m in items or []:
|
|
2832
|
+
who = m.get("from")
|
|
2833
|
+
mid = m.get("id")
|
|
2834
|
+
if not isinstance(who, str) or not who or not isinstance(mid, int):
|
|
2835
|
+
continue
|
|
2836
|
+
prev = _LAST_AUTHORED.get(who)
|
|
2837
|
+
# Compare by `id`, NEVER by `created`: timestamps are stamped pre-lock while ids are assigned under
|
|
2838
|
+
# it, so two concurrent senders can carry timestamps in the opposite order from their ids.
|
|
2839
|
+
if prev is None or mid > prev[0]:
|
|
2840
|
+
_LAST_AUTHORED[who] = (mid, m.get("created"))
|
|
2841
|
+
|
|
2842
|
+
|
|
2843
|
+
# Set once, when the watch loop starts. Everything before it is INVISIBLE to this process: the table is built
|
|
2844
|
+
# from windows observed since then, so a question about earlier activity must answer UNKNOWN rather than
|
|
2845
|
+
# "none". Without this floor a fresh producer would report every persona as inactive for one tick.
|
|
2846
|
+
_OBSERVED_SINCE = None
|
|
2847
|
+
|
|
2848
|
+
# Per-inbox coverage: {persona: the lowest id ever seen in THAT inbox's window}. A persona's outbound can
|
|
2849
|
+
# land in ANY inbox, so a claim that they have authored nothing is only as good as the WORST-covered inbox.
|
|
2850
|
+
_INBOX_FLOORS = {}
|
|
2851
|
+
|
|
2852
|
+
|
|
2853
|
+
def note_observation_floor(persona, items):
|
|
2854
|
+
ids = [m["id"] for m in (items or []) if isinstance(m.get("id"), int)]
|
|
2855
|
+
if not ids:
|
|
2856
|
+
return
|
|
2857
|
+
low = min(ids)
|
|
2858
|
+
cur = _INBOX_FLOORS.get(persona)
|
|
2859
|
+
if cur is None or low < cur:
|
|
2860
|
+
_INBOX_FLOORS[persona] = low
|
|
2861
|
+
|
|
2862
|
+
|
|
2863
|
+
def observation_floor_id():
|
|
2864
|
+
"""The id below which "nobody authored anything" CANNOT be asserted. MAXIMUM, deliberately.
|
|
2865
|
+
|
|
2866
|
+
The tempting version is the minimum - the oldest message we have laid eyes on anywhere - and it is
|
|
2867
|
+
WRONG in the dangerous direction. Each inbox window reaches back only as far as its own floor, so
|
|
2868
|
+
between the lowest and highest floor there are inboxes we have NOT seen into. A message authored in
|
|
2869
|
+
that span, addressed to a poorly-covered inbox, is invisible to us; reporting "no activity" there is
|
|
2870
|
+
a silence we did not observe.
|
|
2871
|
+
Caught on live data: the watcher had seen ids down to 1160 (in one inbox) while another inbox's window
|
|
2872
|
+
only reached 1179, so a question about id 1165 looked answerable and was not.
|
|
2873
|
+
Taking the maximum can only make us answer NOT-OBSERVABLE more often, which is the safe direction.
|
|
2874
|
+
"""
|
|
2875
|
+
return max(_INBOX_FLOORS.values()) if _INBOX_FLOORS else None
|
|
2876
|
+
|
|
2877
|
+
|
|
2878
|
+
def evaluate_activity(report, persona, floor_id):
|
|
2879
|
+
"""Answer the question from a PUBLISHED report (§5.4). True / False / None (NOT OBSERVABLE).
|
|
2880
|
+
|
|
2881
|
+
Takes the report rather than reading module state, so the identical logic serves the running watcher
|
|
2882
|
+
and a one-shot `--check-activity` in a separate process. A second implementation of a tri-state this
|
|
2883
|
+
subtle is a second chance to get it wrong.
|
|
2884
|
+
|
|
2885
|
+
The tri-state is the whole point, and it is the same discipline as §5.2: absence of evidence is
|
|
2886
|
+
evidence of absence only if you were actually watching.
|
|
2887
|
+
"""
|
|
2888
|
+
if not isinstance(report, dict) or not report.get("observed_since"):
|
|
2889
|
+
return None # nothing has been observed at all
|
|
2890
|
+
seen = (report.get("last_authored") or {}).get(persona)
|
|
2891
|
+
if isinstance(seen, dict) and isinstance(seen.get("id"), int) and seen["id"] > floor_id:
|
|
2892
|
+
return True # positive evidence, and positive evidence needs no floor
|
|
2893
|
+
floor = report.get("observation_floor_id")
|
|
2894
|
+
if not isinstance(floor, int) or floor_id < floor:
|
|
2895
|
+
return None # the question predates what this report can speak for
|
|
2896
|
+
return False
|
|
2897
|
+
|
|
2898
|
+
|
|
2899
|
+
def current_activity_report():
|
|
2900
|
+
"""The in-process view, in the same shape write_activity_file() publishes."""
|
|
2901
|
+
return {
|
|
2902
|
+
"observed_since": _OBSERVED_SINCE,
|
|
2903
|
+
"observation_floor_id": observation_floor_id(),
|
|
2904
|
+
"last_authored": {p: {"id": i, "created": c} for p, (i, c) in _LAST_AUTHORED.items()},
|
|
2905
|
+
}
|
|
2906
|
+
|
|
2907
|
+
|
|
2908
|
+
def activity_since(persona, floor_id):
|
|
2909
|
+
"""Has `persona` authored anything after `floor_id`? True / False / None (NOT OBSERVABLE)."""
|
|
2910
|
+
return evaluate_activity(current_activity_report(), persona, floor_id)
|
|
2911
|
+
|
|
2912
|
+
|
|
2913
|
+
# Words that assert a CAUSE this data cannot distinguish. Deadlocked, unreachable and thinking-hard all look
|
|
2914
|
+
# identical here and need opposite remedies - a deadlock wants a ping, an unreachable member wants a human to
|
|
2915
|
+
# restart its bridge. Enforced by a test, not just documented, because a rule that lives only in prose does
|
|
2916
|
+
# not run.
|
|
2917
|
+
FORBIDDEN_DIAGNOSES = ("deadlock", "stuck", "wedged", "dead", "down", "offline", "crashed", "hung")
|
|
2918
|
+
|
|
2919
|
+
|
|
2920
|
+
def activity_observation(persona, floor_id, waits, last_evidence=None, report=None):
|
|
2921
|
+
"""The exact text a waiter emits about the member it is waiting on (§5.4). OBSERVATION ONLY.
|
|
2922
|
+
|
|
2923
|
+
Carries the wait count and the last-evidence stamp alongside the finding, so a reader can judge
|
|
2924
|
+
magnitude without a second query - a bare "no activity" invites the reader to supply the diagnosis
|
|
2925
|
+
themselves, which is the failure this wording exists to prevent.
|
|
2926
|
+
"""
|
|
2927
|
+
rep = report if report is not None else current_activity_report()
|
|
2928
|
+
row = (rep.get("last_authored") or {}).get(persona)
|
|
2929
|
+
seen = (row.get("id"), row.get("created")) if isinstance(row, dict) else None
|
|
2930
|
+
evidence = last_evidence if last_evidence is not None else (seen[1] if seen else None)
|
|
2931
|
+
tail = ("; %s's last observed message was %s" % (persona, evidence)) if evidence else (
|
|
2932
|
+
"; no message from %s has been observed at all" % persona)
|
|
2933
|
+
return ("no activity from %s since your message at id %s; you have waited %d heartbeat(s)%s "
|
|
2934
|
+
"(checked: authored mail. This states what was OBSERVED, not why: not-yet-read, unable to "
|
|
2935
|
+
"receive, and still working are indistinguishable from here and need different responses.)"
|
|
2936
|
+
% (persona, floor_id, waits, tail))
|
|
2937
|
+
|
|
2938
|
+
|
|
2939
|
+
def check_activity(path, persona, floor_id, waits):
|
|
2940
|
+
"""One-shot `--check-activity`: read a published report and answer for ONE persona (§5.4).
|
|
2941
|
+
|
|
2942
|
+
Exit codes are the contract, because this is meant to be called from a shell heartbeat:
|
|
2943
|
+
0 evidence of activity (nothing to report)
|
|
2944
|
+
1 no activity in a span we actually covered -> the observation is printed
|
|
2945
|
+
2 NOT OBSERVABLE / unusable report -> print why; assert nothing
|
|
2946
|
+
2 is deliberately distinct from 1. Collapsing them would turn "I was not watching" into "they were
|
|
2947
|
+
silent", which is the false assertion this whole signal is built to refuse.
|
|
2948
|
+
"""
|
|
2949
|
+
try:
|
|
2950
|
+
with open(path) as f:
|
|
2951
|
+
report = json.load(f)
|
|
2952
|
+
except (OSError, ValueError) as e:
|
|
2953
|
+
sys.stderr.write("kijito-inbox-monitor: activity report unreadable (%s): %s\n" % (path, e))
|
|
2954
|
+
return 2
|
|
2955
|
+
verdict = evaluate_activity(report, persona, floor_id)
|
|
2956
|
+
if verdict is True:
|
|
2957
|
+
row = (report.get("last_authored") or {}).get(persona) or {}
|
|
2958
|
+
sys.stdout.write("active: %s authored id %s at %s\n"
|
|
2959
|
+
% (persona, row.get("id"), row.get("created")))
|
|
2960
|
+
return 0
|
|
2961
|
+
if verdict is None:
|
|
2962
|
+
sys.stdout.write("not observable: this report cannot speak about id %s for %s "
|
|
2963
|
+
"(observed since %s, floor id %s). No claim either way.\n"
|
|
2964
|
+
% (floor_id, persona, report.get("observed_since"),
|
|
2965
|
+
report.get("observation_floor_id")))
|
|
2966
|
+
return 2
|
|
2967
|
+
sys.stdout.write(activity_observation(persona, floor_id, waits, report=report) + "\n")
|
|
2968
|
+
return 1
|
|
2969
|
+
|
|
2970
|
+
|
|
2971
|
+
def write_activity_file(path, now_iso=None):
|
|
2972
|
+
"""Publish the authorship table so any harness can evaluate the predicate without inventing a scan.
|
|
2973
|
+
|
|
2974
|
+
This exists to keep consumers OUT of the dangerous shape. Answering "has B sent anything" from a client
|
|
2975
|
+
otherwise means polling every persona's inbox on a timer, where one missing `mark_read=false` destroys
|
|
2976
|
+
read-state fleet-wide. The watcher already holds the answer, gathered safely.
|
|
2977
|
+
"""
|
|
2978
|
+
d = {
|
|
2979
|
+
"observed_since": _OBSERVED_SINCE,
|
|
2980
|
+
"observation_floor_id": observation_floor_id(),
|
|
2981
|
+
"updated": now_iso or _now_iso(),
|
|
2982
|
+
# A question about anything at or below observation_floor_id is NOT ANSWERABLE from this file.
|
|
2983
|
+
"last_authored": {p: {"id": i, "created": c} for p, (i, c) in sorted(_LAST_AUTHORED.items())},
|
|
2984
|
+
# Personas with mail a SENDER escalated. Published alongside authorship because the pair is what
|
|
2985
|
+
# separates "idle by design" from "nobody is coming": urgency is an expectation someone declared,
|
|
2986
|
+
# and silence only means something once something was expected. A persona absent here was not
|
|
2987
|
+
# reported on, which is not the same as zero.
|
|
2988
|
+
"urgent_unread": {p: n for p, n in sorted(_URGENT_UNREAD.items()) if n},
|
|
2989
|
+
}
|
|
2990
|
+
dirn = os.path.dirname(os.path.abspath(path)) or "."
|
|
2991
|
+
try:
|
|
2992
|
+
os.makedirs(dirn, exist_ok=True)
|
|
2993
|
+
fd, tmp = tempfile.mkstemp(dir=dirn, prefix=".kijmon-act-", suffix=".tmp")
|
|
2994
|
+
with os.fdopen(fd, "w") as f:
|
|
2995
|
+
json.dump(d, f)
|
|
2996
|
+
os.replace(tmp, path) # atomic: a reader never sees a half-written table
|
|
2997
|
+
except OSError as e:
|
|
2998
|
+
sys.stderr.write("kijito-inbox-monitor: WARNING activity-file write failed (non-fatal): %s\n" % e)
|
|
2999
|
+
|
|
3000
|
+
|
|
3001
|
+
def has_consumer_evidence(persona):
|
|
3002
|
+
"""POSITIVE evidence that a real agent stands behind this persona name (§5.6).
|
|
3003
|
+
|
|
3004
|
+
Deliberately the SAME shape as the stranded-mail ownership predicate and river's broadcast eligibility
|
|
3005
|
+
rule, because the three answer one question - "is anyone actually there?" - and two predicates for one
|
|
3006
|
+
question drift apart and then disagree about the same inbox.
|
|
3007
|
+
|
|
3008
|
+
Evidence is positive: authorship we OBSERVED, or memories the directory says they own. A count of NONE
|
|
3009
|
+
is not reported rather than reported-zero, and no data is not evidence of absence, so an unreported
|
|
3010
|
+
count leaves the persona eligible. Only a positively-stated zero with no observed authorship excludes.
|
|
3011
|
+
"""
|
|
3012
|
+
if persona in _LAST_AUTHORED:
|
|
3013
|
+
return True # we watched them write something
|
|
3014
|
+
n = _PERSONA_MEMORY_COUNTS.get(persona)
|
|
3015
|
+
if n is None:
|
|
3016
|
+
return True # the server said nothing; do not infer absence
|
|
3017
|
+
return n > 0
|
|
3018
|
+
|
|
3019
|
+
|
|
3020
|
+
def deliverable_watchers(directory, targets):
|
|
3021
|
+
"""Which watchers should receive an account-level alarm (§5.6).
|
|
3022
|
+
|
|
3023
|
+
Directory membership alone routes alarms into the streams of long-dead test personas - the same defect
|
|
3024
|
+
as a broadcast amplifying phantoms - so eligibility needs evidence of a consumer, not just a name.
|
|
3025
|
+
|
|
3026
|
+
FAILS OPEN, and that matters more than the filtering: if the predicate would leave NOBODY, every
|
|
3027
|
+
directory watcher is used instead. An alarm delivered to a stream nobody reads costs one line; an alarm
|
|
3028
|
+
delivered to NOBODY is the silent failure this tool exists to prevent, and a filter that can silence
|
|
3029
|
+
every recipient at once is a worse bug than the noise it removes.
|
|
3030
|
+
"""
|
|
3031
|
+
known = {p for p in (directory or ()) if p}
|
|
3032
|
+
candidates = sorted({t.persona for t in targets if t.persona and t.persona in known})
|
|
3033
|
+
live = [p for p in candidates if has_consumer_evidence(p)]
|
|
3034
|
+
return live or candidates
|
|
3035
|
+
|
|
3036
|
+
|
|
3037
|
+
_REPORTED_URGENT_QUIET = set()
|
|
3038
|
+
|
|
3039
|
+
# write_only members holding urgent unread, surfaced QUIETLY this process (a stderr NOTICE, and an
|
|
3040
|
+
# informational `urgent_unanswered_write_only` field on any loud urgent alert), never firing the loud
|
|
3041
|
+
# alarm on their own. Kept SEPARATE from _REPORTED_URGENT_QUIET so the loud and quiet tiers cannot gag
|
|
3042
|
+
# one another, and re-armed by intersection_update so a member re-entering the tier is surfaced again.
|
|
3043
|
+
# Same once-per-member, EXACT-keyed discipline as _REPORTED_STRANDED / _REPORTED_DORMANT.
|
|
3044
|
+
_REPORTED_URGENT_WO = set()
|
|
3045
|
+
|
|
3046
|
+
|
|
3047
|
+
def urgent_unanswered(directory):
|
|
3048
|
+
"""Directory personas holding SENDER-ESCALATED mail while showing no observed activity (§5.5).
|
|
3049
|
+
|
|
3050
|
+
THE PREDICATE, and the reason this alarm is buildable at all:
|
|
3051
|
+
unread_urgent > 0 AND activity_since(persona) is False
|
|
3052
|
+
An "is this agent stuck" alarm normally cannot exist, because an agent idle BY DESIGN and an agent that
|
|
3053
|
+
is wedged look identical from outside - so it fires on every dormant persona and rots into noise. What
|
|
3054
|
+
breaks the tie is a declared EXPECTATION, and `unread_urgent` is one: not the recipient declaring
|
|
3055
|
+
liveness, but a SENDER declaring that this needs attention now. Silence only means something once
|
|
3056
|
+
something was expected.
|
|
3057
|
+
|
|
3058
|
+
Both halves must be POSITIVE. `activity_since` is a tri-state and only `False` counts - a NOT-OBSERVABLE
|
|
3059
|
+
answer means the watcher was not running for the span in question, and reporting that as silence is the
|
|
3060
|
+
fabrication this whole signal exists to refuse.
|
|
3061
|
+
|
|
3062
|
+
Restricted to DIRECTORY personas on purpose, which keeps this disjoint from the stranded-mail alarm:
|
|
3063
|
+
that one is for inboxes nobody OWNS, this one is for real members who are not responding. Two alarms
|
|
3064
|
+
with two philosophies drift apart and then disagree about the same inbox.
|
|
3065
|
+
"""
|
|
3066
|
+
out = []
|
|
3067
|
+
floor = observation_floor_id()
|
|
3068
|
+
if floor is None:
|
|
3069
|
+
return out # nothing observed at all: assert nothing
|
|
3070
|
+
known = {p for p in (directory or ()) if p}
|
|
3071
|
+
for persona, n in sorted(_URGENT_UNREAD.items()):
|
|
3072
|
+
if n and persona in known and activity_since(persona, floor) is False:
|
|
3073
|
+
out.append((persona, n))
|
|
3074
|
+
return out
|
|
3075
|
+
|
|
3076
|
+
|
|
3077
|
+
def _urgent_writeonly_detail(persona, n):
|
|
3078
|
+
"""Name a write_only member holding urgent unread, for the QUIET (non-waking) channel.
|
|
3079
|
+
|
|
3080
|
+
A write_only inbox is undrained BY DESIGN - drained via ANOTHER surface (for `jason`, largely the
|
|
3081
|
+
digest) - so a sender's URGENT flag on it does not mean the member is unresponsive HERE, and firing
|
|
3082
|
+
the loud "nobody is answering escalated mail" alarm on it is the same false-positive class write_only
|
|
3083
|
+
exists to kill (assay ruling 5612). But the COUNT must stay visible so the surface that actually
|
|
3084
|
+
drains the box can still act on it - QUIET, never INVISIBLE. Mirrors _dormant_detail.
|
|
3085
|
+
"""
|
|
3086
|
+
return ("%s (%d urgent unread; held by write_only member - drained via another surface, "
|
|
3087
|
+
"not unanswered here)" % (persona, n))
|
|
3088
|
+
|
|
3089
|
+
|
|
3090
|
+
def report_urgent_unanswered(directory, targets, emitter):
|
|
3091
|
+
"""Emit the §5.5 observation. Self-clears when EITHER half of the predicate clears; never an ack.
|
|
3092
|
+
|
|
3093
|
+
An ack would let someone silence "nobody is answering escalated mail" while it stayed true, which is
|
|
3094
|
+
how a dead-letter surface rots. Releasing the suppression the moment the condition lifts means a
|
|
3095
|
+
recurrence is announced again without anyone having to remember to reset anything.
|
|
3096
|
+
|
|
3097
|
+
write_only members are partitioned OUT of the loud tier and surfaced QUIETLY (a stderr NOTICE + an
|
|
3098
|
+
informational `urgent_unanswered_write_only` field), mirroring the stranded/dormant split: their inbox
|
|
3099
|
+
is undrained BY DESIGN (drained via another surface), so a sender's URGENT flag does not make THEM
|
|
3100
|
+
unresponsive here - it is the same false-positive class write_only exists to kill, in the sibling
|
|
3101
|
+
alarm. The count stays NAMED on the quiet channel so the draining surface (the digest) can still read
|
|
3102
|
+
it - quiet, not invisible (assay ruling 5612).
|
|
3103
|
+
"""
|
|
3104
|
+
directory = directory or ()
|
|
3105
|
+
current = urgent_unanswered(directory)
|
|
3106
|
+
# Partition by the DECLARED write_only fact, exactly as _partition_stranded does. `is True` is strict:
|
|
3107
|
+
# an undeclared or False flag leaves the member in the LOUD tier unchanged (graceful degradation, the
|
|
3108
|
+
# same tri-state _row_write_only guarantees).
|
|
3109
|
+
wo_quiet = [(p, n) for p, n in current if _PERSONA_WRITE_ONLY.get(p) is True]
|
|
3110
|
+
alerting = [(p, n) for p, n in current if _PERSONA_WRITE_ONLY.get(p) is not True]
|
|
3111
|
+
_REPORTED_URGENT_QUIET.intersection_update({p for p, _ in alerting}) # release: leaving re-arms alarm
|
|
3112
|
+
_REPORTED_URGENT_WO.intersection_update({p for p, _ in wo_quiet})
|
|
3113
|
+
fresh = [(p, n) for p, n in alerting if p not in _REPORTED_URGENT_QUIET]
|
|
3114
|
+
fresh_wo = [(p, n) for p, n in wo_quiet if p not in _REPORTED_URGENT_WO]
|
|
3115
|
+
# QUIET-BUT-NAMED tier: a stderr NOTICE is a non-waking channel (the event-stream grep filters
|
|
3116
|
+
# new|alert|recovered, which stderr is not), so a write_only member's urgent count goes on the record
|
|
3117
|
+
# without ever waking an agent - and independently of whether any loud member exists this tick. Once
|
|
3118
|
+
# per member, exactly like the dormant tier.
|
|
3119
|
+
for persona, n in fresh_wo:
|
|
3120
|
+
_REPORTED_URGENT_WO.add(persona)
|
|
3121
|
+
sys.stderr.write(
|
|
3122
|
+
"kijito-inbox-monitor: NOTICE urgent-unanswered write_only (quiet, not alarmed) - %s "
|
|
3123
|
+
"(further notices for %r suppressed)\n" % (_urgent_writeonly_detail(persona, n), persona))
|
|
3124
|
+
if not fresh:
|
|
3125
|
+
# The write_only tier NEVER fires the loud alert on its own - the whole point of the split. It has
|
|
3126
|
+
# already been recorded on stderr above; there is no unanswered non-write_only member to announce.
|
|
3127
|
+
return []
|
|
3128
|
+
detail = []
|
|
3129
|
+
for persona, n in fresh:
|
|
3130
|
+
_REPORTED_URGENT_QUIET.add(persona)
|
|
3131
|
+
seen = _LAST_AUTHORED.get(persona)
|
|
3132
|
+
detail.append("%s (%d urgent unread; %s)" % (
|
|
3133
|
+
persona, n,
|
|
3134
|
+
("last observed message %s" % seen[1]) if seen else "no message from them observed at all"))
|
|
3135
|
+
# One summarising event per watcher, exactly as the stranded alarm does - discovering several at once
|
|
3136
|
+
# must not become a wake storm. Routed by evidence of a consumer (§5.6), not by directory membership
|
|
3137
|
+
# alone, so the alert does not land in long-dead test personas' streams. The freshly-surfaced
|
|
3138
|
+
# write_only members ride along as an INFORMATIONAL `urgent_unanswered_write_only` field (mirroring
|
|
3139
|
+
# `dormant_inboxes`): a digest consumer already filtering `alert` sees the held count without the alarm
|
|
3140
|
+
# having fired on their account. Attached ONLY when non-empty - an absent field means "no statement",
|
|
3141
|
+
# the same tri-state discipline the exec layer relies on.
|
|
3142
|
+
extra = {"urgent_unanswered_write_only": [p for p, _ in fresh_wo]} if fresh_wo else {}
|
|
3143
|
+
for watcher in deliverable_watchers(directory, targets):
|
|
3144
|
+
emitter.lifecycle(
|
|
3145
|
+
"alert", persona=watcher,
|
|
3146
|
+
reason=("urgent-unanswered: %d member(s) hold mail a sender marked URGENT while no activity "
|
|
3147
|
+
"from them has been observed: %s. OBSERVATION, NOT A DIAGNOSIS: not-yet-read, unable "
|
|
3148
|
+
"to receive, and still working are indistinguishable from here and need different "
|
|
3149
|
+
"responses. Checked: authored mail." % (len(fresh), ", ".join(detail))),
|
|
3150
|
+
urgent_unanswered=[p for p, _ in fresh],
|
|
3151
|
+
**extra)
|
|
3152
|
+
return [p for p, _ in fresh]
|
|
3153
|
+
|
|
3154
|
+
|
|
3155
|
+
_REPORTED_STRANDED = set()
|
|
3156
|
+
|
|
3157
|
+
# Dormant inboxes already surfaced (quietly) this process. Same once-per-inbox discipline and the same
|
|
3158
|
+
# EXACT (never casefolded) keying as _REPORTED_STRANDED, re-armed by intersection_update so a re-dormancy
|
|
3159
|
+
# after a rescue is surfaced again. Kept separate from _REPORTED_STRANDED so the loud and quiet tiers
|
|
3160
|
+
# never gag one another.
|
|
3161
|
+
_REPORTED_DORMANT = set()
|
|
3162
|
+
|
|
3163
|
+
|
|
3164
|
+
def stranded_inboxes(directory, counts):
|
|
3165
|
+
"""Inboxes holding unread mail that the persona DIRECTORY does not know about.
|
|
3166
|
+
|
|
3167
|
+
Two namespaces exist and are populated by different paths: the DIRECTORY (who exists) and the INBOX
|
|
3168
|
+
(who can receive). When they diverge, mail lands in an inbox that nobody owns and nothing watches -
|
|
3169
|
+
it is never delivered, and nothing reports it, so the sender sees success and the recipient sees
|
|
3170
|
+
nothing. Both cases observed in the wild had this shape: a case-variant of a live persona, and a
|
|
3171
|
+
group-looking name ('all') with no broadcast semantics behind it. One held a substantive reply for
|
|
3172
|
+
14 days before anyone noticed.
|
|
3173
|
+
|
|
3174
|
+
Compared EXACTLY, deliberately NOT casefolded. The SERVER's inbox namespace is case-SENSITIVE -
|
|
3175
|
+
verified: the 'Claude-chat' inbox held a different message set from 'claude-chat' - so a case-variant
|
|
3176
|
+
is a real, DISTINCT inbox holding real mail, and casefolding here would hide the very incident this
|
|
3177
|
+
check exists to catch.
|
|
3178
|
+
|
|
3179
|
+
Note the deliberate asymmetry with _state_safe_persona(), which DOES casefold: the local filesystem
|
|
3180
|
+
is case-INSENSITIVE and cannot hold two state files for the two names, so the watcher can never adopt
|
|
3181
|
+
the variant. The rules are complementary rather than contradictory - the variant is unwatchable
|
|
3182
|
+
locally AND unwatched remotely, which is exactly why it has to be alarmed on instead of adopted.
|
|
3183
|
+
|
|
3184
|
+
TWO SIGNALS, because directory membership alone stopped being sufficient. A server may build its
|
|
3185
|
+
directory as a UNION that includes every registered RECIPIENT - and a recipient is registered the
|
|
3186
|
+
moment anyone sends to that name, typo included. On such a server every future phantom is "in the
|
|
3187
|
+
directory" instantly and absence can never fire again. So an in-directory inbox also counts as
|
|
3188
|
+
stranded when it holds mail while NEVER HAVING BEEN CONSUMED - read == mail_total - unread == 0.
|
|
3189
|
+
|
|
3190
|
+
read==0 REPLACES the older "owns ZERO memories" proxy for the in-directory case, because that proxy
|
|
3191
|
+
was MONOTONIC: authoring a single memory immunised an inbox forever, so a typo-variant that ever
|
|
3192
|
+
received one memory ('rvier', 'settest', 'qa-e2e' were live examples) went invisible while its mail
|
|
3193
|
+
piled up unread. read count is not monotonic - it tracks whether anyone is ACTUALLY consuming the
|
|
3194
|
+
inbox now. Where the server reports no read data at all this degrades to the original memory-count
|
|
3195
|
+
proxy rather than guessing, and an unknown read is never read as zero.
|
|
3196
|
+
|
|
3197
|
+
stranded_inboxes() returns only the LOUD tier: names the directory doesn't know, plus in-directory
|
|
3198
|
+
inboxes never consumed that are DECLARED `retired` (clearable debris). In-directory inboxes never
|
|
3199
|
+
consumed that are NOT declared retired are real-but-dormant; they are returned by dormant_inboxes()
|
|
3200
|
+
and surfaced quietly instead, so a live member who simply never reads a broadcast inbox does not ride
|
|
3201
|
+
the loud alarm. The partition is exact on both axes (read==0 and the boolean `retired`); no threshold.
|
|
3202
|
+
"""
|
|
3203
|
+
return _partition_stranded(directory, counts)[0]
|
|
3204
|
+
|
|
3205
|
+
|
|
3206
|
+
def _partition_stranded(directory, counts):
|
|
3207
|
+
"""Split inboxes-holding-unread into (loud, dormant). Single classifier so the two tiers cannot drift.
|
|
3208
|
+
|
|
3209
|
+
For each inbox with unread mail (checked in this order):
|
|
3210
|
+
- name not in the directory -> LOUD (signal 1, unchanged)
|
|
3211
|
+
- in directory, write_only is True -> DORMANT (undrained by design; always quiet)
|
|
3212
|
+
- in directory, read data UNKNOWN, memory_count==0 -> LOUD (degrade to the original proxy)
|
|
3213
|
+
- in directory, read > 0 -> not stranded (actively consumed)
|
|
3214
|
+
- in directory, read == 0, retired is True -> LOUD (declared clearable debris)
|
|
3215
|
+
- in directory, read == 0, retired False/undeclared -> DORMANT (real-but-idle; quiet)
|
|
3216
|
+
read = mail_total - unread, both from the /api/personas row via _PERSONA_READ_COUNTS. Compared and
|
|
3217
|
+
classified EXACTLY, never casefolded - the same case-sensitivity invariant as the rest of this check.
|
|
3218
|
+
write_only is checked BEFORE read, because an undrained-by-design inbox is quiet regardless of its
|
|
3219
|
+
read count - its read==0 (or unknown read) is the EXPECTED steady state, not evidence of a fault.
|
|
3220
|
+
That is the fix for a live member whose box the proxy would otherwise flag LOUD: the human's own
|
|
3221
|
+
inbox `jason`, in the directory with unknown read and zero memories, was riding the loud alarm every
|
|
3222
|
+
tick until write_only declared it undrained-by-design. FACT declared by the API, policy derived here.
|
|
3223
|
+
"""
|
|
3224
|
+
known = {p for p in directory if p}
|
|
3225
|
+
loud, dormant = [], []
|
|
3226
|
+
for p in sorted(counts):
|
|
3227
|
+
if not p or not counts.get(p):
|
|
3228
|
+
continue
|
|
3229
|
+
if p not in known:
|
|
3230
|
+
loud.append(p) # signal 1: no owner in the directory
|
|
3231
|
+
continue
|
|
3232
|
+
if _PERSONA_WRITE_ONLY.get(p) is True:
|
|
3233
|
+
dormant.append(p) # undrained BY DESIGN (the human's box) -> always quiet, any read
|
|
3234
|
+
continue
|
|
3235
|
+
read = _PERSONA_READ_COUNTS.get(p)
|
|
3236
|
+
if read is None:
|
|
3237
|
+
# No read data for this persona: degrade to the original ownership proxy. An UNKNOWN read
|
|
3238
|
+
# count must never be treated as zero, so we consult memory_count exactly as before.
|
|
3239
|
+
if _PERSONA_MEMORY_COUNTS.get(p) == 0:
|
|
3240
|
+
loud.append(p)
|
|
3241
|
+
continue
|
|
3242
|
+
if read > 0:
|
|
3243
|
+
continue # someone is consuming it - not stranded at all
|
|
3244
|
+
# read == 0: this inbox has never been consumed. Partition by the DECLARED retired flag.
|
|
3245
|
+
if _PERSONA_RETIRED.get(p) is True:
|
|
3246
|
+
loud.append(p) # declared clearable debris -> loud, exactly like today
|
|
3247
|
+
else:
|
|
3248
|
+
dormant.append(p) # real-but-dormant -> quiet, must NOT ride the loud alarm
|
|
3249
|
+
return loud, dormant
|
|
3250
|
+
|
|
3251
|
+
|
|
3252
|
+
def dormant_inboxes(directory, counts):
|
|
3253
|
+
"""In-directory inboxes never consumed (read==0) but NOT declared `retired` - the QUIET tier.
|
|
3254
|
+
|
|
3255
|
+
Separated from stranded_inboxes() on purpose: these are real members who simply do not read a
|
|
3256
|
+
broadcast inbox (measured live: omniview/sterling/vellum/maestro hold hundreds of memories with
|
|
3257
|
+
read==0). Alarming on them loudly would flood the very alert consumers rely on, so they are surfaced
|
|
3258
|
+
quietly (a stderr NOTICE, and an informational `dormant_inboxes` field on any loud alert) and never
|
|
3259
|
+
fire an alert on their own.
|
|
3260
|
+
"""
|
|
3261
|
+
return _partition_stranded(directory, counts)[1]
|
|
3262
|
+
|
|
3263
|
+
|
|
3264
|
+
def _stranded_detail(persona, directory, counts):
|
|
3265
|
+
"""Describe one stranded inbox, naming its twin when it is a case-variant.
|
|
3266
|
+
|
|
3267
|
+
'case-variant of known persona X' is a far more actionable diagnosis than 'unknown inbox': it tells
|
|
3268
|
+
the operator the mail was meant for a real person and how it went astray.
|
|
3269
|
+
"""
|
|
3270
|
+
twin = next((d for d in sorted(directory)
|
|
3271
|
+
if d and d != persona and d.casefold() == persona.casefold()), None)
|
|
3272
|
+
if twin is not None:
|
|
3273
|
+
return "%s (%s unread; case-variant of known persona %r)" % (persona, counts.get(persona), twin)
|
|
3274
|
+
in_dir = persona in set(directory)
|
|
3275
|
+
if in_dir and _PERSONA_READ_COUNTS.get(persona) == 0 and _PERSONA_RETIRED.get(persona) is True:
|
|
3276
|
+
return ("%s (%s unread; never consumed (read 0) and declared retired, so it is clearable debris)"
|
|
3277
|
+
% (persona, counts.get(persona)))
|
|
3278
|
+
if in_dir and _PERSONA_MEMORY_COUNTS.get(persona) == 0:
|
|
3279
|
+
return "%s (%s unread; registered as a recipient but owns no memories, so nobody works as it)" % (
|
|
3280
|
+
persona, counts.get(persona))
|
|
3281
|
+
return "%s (%s unread)" % (persona, counts.get(persona))
|
|
3282
|
+
|
|
3283
|
+
|
|
3284
|
+
def _dormant_detail(persona, counts):
|
|
3285
|
+
"""Describe one DORMANT inbox: a real member never observed reading it. Quiet, not an alarm.
|
|
3286
|
+
|
|
3287
|
+
Deliberately does NOT diagnose it as clearable - a dormant inbox is a live persona that simply is not
|
|
3288
|
+
reading here, the opposite of debris, and mislabelling it would invite deleting a real member's mail.
|
|
3289
|
+
A write_only member is named as such: its read==0 is by design, not merely unobserved.
|
|
3290
|
+
"""
|
|
3291
|
+
if _PERSONA_WRITE_ONLY.get(persona) is True:
|
|
3292
|
+
return "%s (%s unread; in the directory and declared write_only - undrained BY DESIGN (drained via another surface), never debris)" % (
|
|
3293
|
+
persona, counts.get(persona))
|
|
3294
|
+
return "%s (%s unread; in the directory but never consumed (read 0), not declared retired)" % (
|
|
3295
|
+
persona, counts.get(persona))
|
|
3296
|
+
|
|
3297
|
+
|
|
3298
|
+
def report_stranded_inboxes(directory, counts, targets, emitter):
|
|
3299
|
+
"""Alarm on undelivered mail: an inbox RECEIVING while nobody owns or watches it.
|
|
3300
|
+
|
|
3301
|
+
Reported at most once per inbox per process, and summarised into ONE event per watcher rather than
|
|
3302
|
+
one per (watcher, inbox), so discovering a backlog cannot turn into a wake storm.
|
|
3303
|
+
|
|
3304
|
+
Routed ONLY to watchers backed by a real DIRECTORY persona. This is not a formality: a stranded inbox
|
|
3305
|
+
holds mail, so discover_from_counts() gives it a watch target and an event stream of its own - and
|
|
3306
|
+
routing the alarm to every target would therefore write it straight into the unconsumed stream whose
|
|
3307
|
+
unconsumed-ness is the fault being reported. Producing an event there is not delivering it.
|
|
3308
|
+
|
|
3309
|
+
The event is an `alert` (not a new event name) so consumers already filtering `alert` surface it
|
|
3310
|
+
without being rearmed; a fresh event name would itself have gone unwatched, because a running
|
|
3311
|
+
`grep` never re-reads its argv. That is not hypothetical: the diagnostics this module emits
|
|
3312
|
+
(state_corrupt, baseline_skipped, seed_ahead, replay_capped, persona_added) ARE fresh names, and
|
|
3313
|
+
every one of them was invisible to every seated consumer until their filters were widened by hand.
|
|
3314
|
+
"""
|
|
3315
|
+
if not directory:
|
|
3316
|
+
return [] # unknown directory: alarming would flag EVERY persona. No data is not evidence of a fault.
|
|
3317
|
+
loud, dormant = _partition_stranded(directory, counts)
|
|
3318
|
+
# RELEASE the suppression for anything no longer in its tier, so the signal can fire AGAIN if that inbox
|
|
3319
|
+
# is later re-stranded / re-dormant. Suppressing for the process lifetime made "reported once" mean
|
|
3320
|
+
# "reported once ever", which silently contradicted the documented self-clearing behaviour: an inbox
|
|
3321
|
+
# that was rescued and then stranded a second time would never be announced.
|
|
3322
|
+
#
|
|
3323
|
+
# Keyed EXACTLY, not casefolded - the same asymmetry as stranded_inboxes() itself. The server's inbox
|
|
3324
|
+
# namespace is case-sensitive, so 'Claude-chat' and 'claude-chat' are DIFFERENT inboxes; sharing one
|
|
3325
|
+
# suppression key between them lets either one hold the other's alarm down.
|
|
3326
|
+
_REPORTED_STRANDED.intersection_update(loud)
|
|
3327
|
+
_REPORTED_DORMANT.intersection_update(dormant)
|
|
3328
|
+
fresh = [p for p in loud if p not in _REPORTED_STRANDED]
|
|
3329
|
+
fresh_dormant = [p for p in dormant if p not in _REPORTED_DORMANT]
|
|
3330
|
+
|
|
3331
|
+
# DORMANT tier (real-but-idle members, not declared retired): recorded QUIETLY and independently of the
|
|
3332
|
+
# loud alarm. A stderr NOTICE is a non-waking channel (the event-stream grep filters new|alert|recovered,
|
|
3333
|
+
# which stderr is not), so a dormant inbox is put on the record without ever waking an agent - and this
|
|
3334
|
+
# happens whether or not any loud inbox exists this tick. Once per inbox, exactly like the loud tier.
|
|
3335
|
+
for persona in fresh_dormant:
|
|
3336
|
+
_REPORTED_DORMANT.add(persona)
|
|
3337
|
+
sys.stderr.write(
|
|
3338
|
+
"kijito-inbox-monitor: NOTICE dormant inbox (quiet, not alarmed) - %s (further notices for %r "
|
|
3339
|
+
"suppressed)\n" % (_dormant_detail(persona, counts), persona))
|
|
3340
|
+
|
|
3341
|
+
if not fresh:
|
|
3342
|
+
# The DORMANT tier NEVER fires the loud alert on its own - that is the whole point of the split.
|
|
3343
|
+
# It has already been recorded on stderr above; there is no loud debris/unknown inbox to announce.
|
|
3344
|
+
return []
|
|
3345
|
+
for persona in fresh:
|
|
3346
|
+
_REPORTED_STRANDED.add(persona)
|
|
3347
|
+
sys.stderr.write(
|
|
3348
|
+
"kijito-inbox-monitor: ALERT stranded mail - %s is not a known persona, so no agent consumes its "
|
|
3349
|
+
"mail (further reports for %r suppressed)\n" % (_stranded_detail(persona, directory, counts), persona))
|
|
3350
|
+
detail = ", ".join(_stranded_detail(p, directory, counts) for p in fresh)
|
|
3351
|
+
# Same routing rule as the urgent-unanswered alarm (§5.6) - one predicate for "is anyone there",
|
|
3352
|
+
# because two would drift apart and disagree about the same inbox. The freshly-detected dormant inboxes
|
|
3353
|
+
# ride along as an INFORMATIONAL `dormant_inboxes` field: consumers already filtering `alert` see them
|
|
3354
|
+
# without being rearmed, but they never caused this alert to fire (only `fresh` loud did).
|
|
3355
|
+
# Attach the informational field ONLY when there is something to say - an absent field means "no
|
|
3356
|
+
# statement", the same tri-state discipline the exec layer relies on ("absent fields are simply
|
|
3357
|
+
# omitted, not defaulted"), so an empty dormant list is left off rather than shipped as [].
|
|
3358
|
+
extra = {"dormant_inboxes": list(fresh_dormant)} if fresh_dormant else {}
|
|
3359
|
+
for watcher in deliverable_watchers(directory, targets):
|
|
3360
|
+
emitter.lifecycle("alert", persona=watcher,
|
|
3361
|
+
reason="stranded-mail: %d inbox(es) receiving mail nobody watches: %s" % (len(fresh), detail),
|
|
3362
|
+
stranded_inboxes=list(fresh),
|
|
3363
|
+
**extra)
|
|
3364
|
+
return fresh
|
|
3365
|
+
|
|
3366
|
+
|
|
3367
|
+
def run(args):
|
|
3368
|
+
headers = build_headers(args)
|
|
3369
|
+
sink = None
|
|
3370
|
+
sink_template = None
|
|
3371
|
+
if not args.self_test and args.emit == "stdout-jsonl":
|
|
3372
|
+
if args.events_file_template:
|
|
3373
|
+
sink_template = args.events_file_template # one sink per persona (lazily created on first event)
|
|
3374
|
+
elif args.events_file:
|
|
3375
|
+
sink = RotatingFileSink(args.events_file, args.max_bytes, args.keep_logs)
|
|
3376
|
+
emitter = Emitter(args.emit, args.exec, args.content_chars, args.no_content, sink=sink,
|
|
3377
|
+
suppress_authors=args.suppress_author, sink_template=sink_template,
|
|
3378
|
+
max_bytes=args.max_bytes, keep=args.keep_logs)
|
|
3379
|
+
opener_by_origin = {}
|
|
3380
|
+
|
|
3381
|
+
directory_opener = make_opener_for(PERSONAS_URL)
|
|
3382
|
+
personas = requested_personas(args, directory_opener, headers)
|
|
3383
|
+
if not personas:
|
|
3384
|
+
raise FatalConfig("at least one persona is required")
|
|
3385
|
+
# ★ THE STARTUP PATH NEEDS THE SAME CONTAINMENT AS THE LATE-ADD PATHS, and it is the one place the
|
|
3386
|
+
# original fix note did not name (it specified the two discover arms plus main()). An
|
|
3387
|
+
# arm in main() only converts the traceback into a clean exit: the producer STILL dies, so one
|
|
3388
|
+
# persona's hostile lock sidecar still stops every other persona's mail. That is precisely the
|
|
3389
|
+
# property this fix exists to deny, so containment belongs HERE, per-persona, exactly like
|
|
3390
|
+
# discover_persona_targets. This used to be a bare list comprehension with no try at all.
|
|
3391
|
+
targets = []
|
|
3392
|
+
for p in personas:
|
|
3393
|
+
try:
|
|
3394
|
+
targets.append(build_persona_target(p, opener_by_origin, headers, args, emitter))
|
|
3395
|
+
except (FatalConfig, OSError) as e:
|
|
3396
|
+
_warn_persona_once(p, "cannot watch persona %r: %s" % (p, e))
|
|
3397
|
+
if not targets:
|
|
3398
|
+
# FAIL CLOSED. Skipping a persona is a real degradation, and skipping ALL of them would leave a
|
|
3399
|
+
# process that is up, heartbeat-less and watching nothing - the silent-success shape this repo
|
|
3400
|
+
# keeps finding. A watcher with no targets must not look like a running watcher.
|
|
3401
|
+
raise FatalConfig("no persona could be watched: every one of %d target(s) failed to initialise "
|
|
3402
|
+
"(see the warnings above)" % len(personas))
|
|
3403
|
+
# The DIRECTORY namespace, kept separate from `targets` on purpose: targets also accumulate personas
|
|
3404
|
+
# discovered from the inbox counts, so diffing against targets would silently absorb the very phantom
|
|
3405
|
+
# inboxes the stranded-mail check exists to find.
|
|
3406
|
+
directory_personas = list(personas) if watches_all_personas(args) else []
|
|
3407
|
+
|
|
3408
|
+
# ---- self-test (§7.2): run once, exit -------------------------------------------------------------------------
|
|
3409
|
+
if args.self_test:
|
|
3410
|
+
ok = True
|
|
3411
|
+
for target in targets:
|
|
3412
|
+
ok = target.self_test() and ok
|
|
3413
|
+
return 0 if ok else 1
|
|
3414
|
+
|
|
3415
|
+
seam = WakeSeam()
|
|
3416
|
+
seam.install()
|
|
3417
|
+
global _OBSERVED_SINCE
|
|
3418
|
+
_OBSERVED_SINCE = _now_iso() # §5.4 nothing before this instant is observable to this process
|
|
3419
|
+
rediscover_at = _monotonic() + args.rediscover_every
|
|
3420
|
+
cursor = None # opaque long-poll cursor (the server's max-message-id token) echoed on each call
|
|
3421
|
+
lp_backoff = 0 # exponential backoff (s) between FAILED long-poll attempts; 0 while healthy
|
|
3422
|
+
|
|
3423
|
+
while not seam.stop:
|
|
3424
|
+
seam.drain() # read-and-clear at START of poll (§10)
|
|
3425
|
+
if watches_all_personas(args) and directory_opener is not None and _monotonic() >= rediscover_at:
|
|
3426
|
+
try:
|
|
3427
|
+
_, discovered = discover_persona_targets(
|
|
3428
|
+
args, headers, emitter, targets, opener_by_origin, directory_opener)
|
|
3429
|
+
if discovered:
|
|
3430
|
+
directory_personas = discovered
|
|
3431
|
+
except FatalConfig as e:
|
|
3432
|
+
sys.stderr.write("kijito-inbox-monitor: WARNING persona rediscovery failed: %s\n" % e)
|
|
3433
|
+
rediscover_at = _monotonic() + args.rediscover_every
|
|
3434
|
+
|
|
3435
|
+
counts_available = False
|
|
3436
|
+
unread_counts = {}
|
|
3437
|
+
held = False # True iff this iteration was a real server-HELD long-poll (it already provided the wait)
|
|
3438
|
+
count_target = next((t for t in targets if t.unread_persona), None)
|
|
3439
|
+
if count_target is not None and not args.no_fast_path:
|
|
3440
|
+
if args.wait > 0:
|
|
3441
|
+
counts_available, unread_counts, new_cursor = fetch_unread_counts_longpoll(
|
|
3442
|
+
count_target.opener, headers, args.wait, cursor)
|
|
3443
|
+
if counts_available:
|
|
3444
|
+
lp_backoff = 0
|
|
3445
|
+
if new_cursor is not None:
|
|
3446
|
+
cursor = new_cursor # real long-poll: advance the cursor; the hold WAS the wait
|
|
3447
|
+
held = True
|
|
3448
|
+
# new_cursor is None → server doesn't long-poll (yet) → interval-poll via the sleep below
|
|
3449
|
+
else:
|
|
3450
|
+
# drop / blip / outage: back off, resume the SAME cursor next time (lossless), and this tick
|
|
3451
|
+
# falls through to per-target full inbox polls (the by-message-id correctness backstop).
|
|
3452
|
+
lp_backoff = min((lp_backoff * 2) or 1, LONGPOLL_BACKOFF_CAP)
|
|
3453
|
+
else:
|
|
3454
|
+
counts_available, unread_counts = fetch_unread_counts(
|
|
3455
|
+
count_target.opener, count_target.count_url, headers)
|
|
3456
|
+
|
|
3457
|
+
if counts_available:
|
|
3458
|
+
discover_from_counts(args, unread_counts, targets, opener_by_origin, headers, emitter)
|
|
3459
|
+
if not args.no_stranded_alerts:
|
|
3460
|
+
report_stranded_inboxes(directory_personas, unread_counts, targets, emitter)
|
|
3461
|
+
refresh_directory_backing(args, directory_personas, targets)
|
|
3462
|
+
for target in targets:
|
|
3463
|
+
target.poll_once(counts_available, unread_counts)
|
|
3464
|
+
# AFTER the polls, so this tick's authorship is already recorded - evaluating before them would
|
|
3465
|
+
# judge a member silent using a view that predates the very message proving they are not.
|
|
3466
|
+
# ITS OWN FLAG, NOT THE STRANDED ONE (ladybug review of c6e1699): these are different severities
|
|
3467
|
+
# with different audiences, and the stranded flag's own documented advice is "set this if you keep
|
|
3468
|
+
# deliberate test inboxes" - following that must not silently disable the higher-severity alarm
|
|
3469
|
+
# about real members. Coupling them made the safe-sounding instruction the dangerous one.
|
|
3470
|
+
if counts_available and not args.no_urgent_alerts:
|
|
3471
|
+
report_urgent_unanswered(directory_personas, targets, emitter)
|
|
3472
|
+
if args.activity_file:
|
|
3473
|
+
write_activity_file(args.activity_file)
|
|
3474
|
+
if seam.stop:
|
|
3475
|
+
break
|
|
3476
|
+
if held:
|
|
3477
|
+
continue # the server-held long-poll already supplied the inter-poll wait - loop straight back
|
|
3478
|
+
seam.wait(lp_backoff if lp_backoff else args.poll_seconds)
|
|
3479
|
+
|
|
3480
|
+
emitter.close()
|
|
3481
|
+
for target in targets:
|
|
3482
|
+
if target.state_file is not None:
|
|
3483
|
+
target.state_file.unlock()
|
|
3484
|
+
return 0
|
|
3485
|
+
|
|
3486
|
+
|
|
3487
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
3488
|
+
# CLI
|
|
3489
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
3490
|
+
def build_parser():
|
|
3491
|
+
p = argparse.ArgumentParser(
|
|
3492
|
+
prog="kijito-inbox-monitor",
|
|
3493
|
+
description="Watch your Kijito inbox and emit one event per new message. NOTE: emitting is not waking - a "
|
|
3494
|
+
"bare `tail` of the events file captures but does NOT wake your agent. To actually wake on new "
|
|
3495
|
+
"mail, use --emit exec-per-event with a command that pokes your agent loop, or run the tail "
|
|
3496
|
+
"under your harness's streaming/notification consumer. See the README 'Waking your agent'.")
|
|
3497
|
+
p.add_argument("--persona", action="append",
|
|
3498
|
+
help="Kijito persona whose inbox to watch. Repeat for multi-persona mode.")
|
|
3499
|
+
p.add_argument("--personas", action="append",
|
|
3500
|
+
help="Comma-separated personas to watch, e.g. codex,river,ladybug.")
|
|
3501
|
+
p.add_argument("--all-personas", action="store_true",
|
|
3502
|
+
help="Watch every persona in your Kijito account (default).")
|
|
3503
|
+
p.add_argument("--no-stranded-alerts", action="store_true",
|
|
3504
|
+
help="do not alarm on mail sitting in an inbox that is not a known persona. The alarm is "
|
|
3505
|
+
"ON by default because such mail is UNDELIVERABLE and nothing else reports it; set "
|
|
3506
|
+
"this only if you keep deliberate test inboxes and expect the alarm. It silences "
|
|
3507
|
+
"ONLY this alarm - urgent-unanswered has its own flag (--no-urgent-alerts).")
|
|
3508
|
+
p.add_argument("--no-urgent-alerts", action="store_true",
|
|
3509
|
+
help="do not alarm on escalated (URGENT) mail that a known member is not answering. The "
|
|
3510
|
+
"alarm is ON by default. Deliberately a SEPARATE flag from --no-stranded-alerts: "
|
|
3511
|
+
"silencing a low-severity alarm about inboxes nobody owns must not also silence a "
|
|
3512
|
+
"higher-severity one about real members who are not responding.")
|
|
3513
|
+
p.add_argument("--rediscover-every", type=int, default=600,
|
|
3514
|
+
help="In all-persona mode, re-scan your account every N seconds and add newly-created personas "
|
|
3515
|
+
"(default 600, min 1). Explicit persona subsets are not expanded.")
|
|
3516
|
+
p.add_argument("--poll-seconds", type=int, default=60,
|
|
3517
|
+
help="Interval (s) between polls when long-poll is off/unsupported (default 60).")
|
|
3518
|
+
p.add_argument("--wait", type=int, default=50,
|
|
3519
|
+
help="Long-poll hold (s) requested from /api/notify/pending so new mail wakes the watcher "
|
|
3520
|
+
"near-instantly at ~the same request rate (default 50; the server clamps to its own max). "
|
|
3521
|
+
"0 disables long-poll → plain interval polling at --poll-seconds. If the server doesn't "
|
|
3522
|
+
"support long-poll, the client auto-falls back to interval polling (no redeploy needed). "
|
|
3523
|
+
"Clean shutdown during a held poll can take up to --wait seconds (a supervisor's SIGKILL "
|
|
3524
|
+
"mid-hold is safe - state is persisted every cycle).")
|
|
3525
|
+
p.add_argument("--alert-after", type=int, default=3, help="Consecutive failures before an alert (min 1).")
|
|
3526
|
+
p.add_argument("--emit", choices=("stdout-jsonl", "exec-per-event"), default="stdout-jsonl")
|
|
3527
|
+
p.add_argument("--exec", help="Command to run per event (required iff --emit exec-per-event).")
|
|
3528
|
+
p.add_argument("--suppress-author", action="append",
|
|
3529
|
+
help="Do not emit 'new' events authored by this persona (repeatable) - drops the self-echo you "
|
|
3530
|
+
"get when watching all personas AND sending mail. Liveness events are unaffected.")
|
|
3531
|
+
p.add_argument("--content-chars", type=int, default=220)
|
|
3532
|
+
p.add_argument("--no-content", action="store_true", help="Omit message content entirely (opaque mode).")
|
|
3533
|
+
p.add_argument("--events-file",
|
|
3534
|
+
help="Write NDJSON events to this file (an OWNED, size-rotated fd) instead of stdout - the "
|
|
3535
|
+
"supervised-producer mode that survives log rotation. Consumers tail -F it. "
|
|
3536
|
+
"Only applies to --emit stdout-jsonl.")
|
|
3537
|
+
p.add_argument("--events-file-template",
|
|
3538
|
+
help="Per-persona supervised mode: write EACH persona's events to its OWN owned, size-rotated "
|
|
3539
|
+
"file, e.g. ~/.cache/kijito-inbox-monitor/events.{persona}.ndjson - a session then subscribes "
|
|
3540
|
+
"to only its own mail with `tail -F events.<persona>.ndjson`, no filtering. Must contain "
|
|
3541
|
+
"'{persona}'. Mutually exclusive with --events-file.")
|
|
3542
|
+
p.add_argument("--max-bytes", type=int, default=5_000_000,
|
|
3543
|
+
help="Rotate the events file(s) once one reaches N bytes (default 5000000; <=0 disables).")
|
|
3544
|
+
p.add_argument("--keep-logs", type=int, default=5,
|
|
3545
|
+
help="How many rotated --events-file archives to keep (default 5, min 1).")
|
|
3546
|
+
p.add_argument("--seed-at", type=int, help="Cursor seed = last-handled id (overrides a state-file cursor).")
|
|
3547
|
+
p.add_argument("--max-replay", type=int, default=50, help="Cap on a re-arm backlog before fast-forwarding.")
|
|
3548
|
+
p.add_argument("--state-file",
|
|
3549
|
+
help="Persist+resume cursor/FSM; single-writer locked. Kijito persona targets derive one "
|
|
3550
|
+
"file per persona from this base path. Recommended w/ a supervisor.")
|
|
3551
|
+
p.add_argument("--heartbeat", type=int, help="Emit a heartbeat event every N seconds (external dead-man's-switch).")
|
|
3552
|
+
p.add_argument("--activity-file",
|
|
3553
|
+
help="Publish who AUTHORED mail most recently, as JSON, refreshed each tick. Lets a "
|
|
3554
|
+
"harness answer 'has X been active since my message?' from data this watcher "
|
|
3555
|
+
"already collects, instead of polling every inbox itself. Off by default.")
|
|
3556
|
+
p.add_argument("--safe-persona", metavar="PERSONA",
|
|
3557
|
+
help="Print the FILENAME COMPONENT this producer derives from PERSONA, then exit 0. "
|
|
3558
|
+
"This is the ONE place the persona->filename rule lives: any other program that "
|
|
3559
|
+
"needs to name a persona's events/state file (the SessionStart hook, "
|
|
3560
|
+
"producer-health.sh, docs) must ask HERE rather than re-implement it. Three "
|
|
3561
|
+
"re-implementations had already drifted (beta feedback #14/#16, row M290): the "
|
|
3562
|
+
"rule CASEFOLDS and accepts any UNICODE alphanumeric, so 'Loom' and 'Omega' are "
|
|
3563
|
+
"exactly the names a hand-written [^A-Za-z0-9._-] filter gets wrong -- and it gets "
|
|
3564
|
+
"them wrong INVISIBLY on a case-insensitive filesystem. A pure string function: no "
|
|
3565
|
+
"token, no network, no state file.")
|
|
3566
|
+
p.add_argument("--check-activity", metavar="PERSONA",
|
|
3567
|
+
help="One-shot: read --activity-file and report whether PERSONA has authored anything "
|
|
3568
|
+
"since --since-id. Exits 0 active, 1 no activity in a covered span (prints the "
|
|
3569
|
+
"observation), 2 NOT OBSERVABLE. Reads only; no token or network needed.")
|
|
3570
|
+
p.add_argument("--since-id", type=int,
|
|
3571
|
+
help="With --check-activity: the message id you are awaiting a reply to.")
|
|
3572
|
+
p.add_argument("--waits", type=int, default=1,
|
|
3573
|
+
help="With --check-activity: how many of your own heartbeats you have waited "
|
|
3574
|
+
"(reported verbatim, so a reader can judge magnitude). Default 1.")
|
|
3575
|
+
p.add_argument("--auth-header", help="Header NAME for the token (default Authorization: Bearer).")
|
|
3576
|
+
p.add_argument("--token-file", help="File holding the auth token (wins over $KIJITOMON_TOKEN).")
|
|
3577
|
+
p.add_argument("--no-fast-path", action="store_true",
|
|
3578
|
+
help="Disable the /api/notify/pending unread pre-check; always full-poll the inbox list.")
|
|
3579
|
+
p.add_argument("--resync-every", type=int, default=10,
|
|
3580
|
+
help="Fast-path safety floor: force a full inbox poll after at most N consecutive cheap "
|
|
3581
|
+
"skips, so a stale/wrong unread count can never blind the watcher (default 10, min 1).")
|
|
3582
|
+
p.add_argument("--self-test", action="store_true", help="Probe + synthetic emit, then exit (run before trusting).")
|
|
3583
|
+
return p
|
|
3584
|
+
|
|
3585
|
+
|
|
3586
|
+
def validate_args(args):
|
|
3587
|
+
if args.alert_after < 1:
|
|
3588
|
+
raise FatalConfig("--alert-after must be >= 1")
|
|
3589
|
+
if args.resync_every < 1:
|
|
3590
|
+
raise FatalConfig("--resync-every must be >= 1")
|
|
3591
|
+
if args.rediscover_every < 1:
|
|
3592
|
+
raise FatalConfig("--rediscover-every must be >= 1")
|
|
3593
|
+
if args.emit == "exec-per-event" and not args.exec:
|
|
3594
|
+
raise FatalConfig("--exec is required when --emit exec-per-event")
|
|
3595
|
+
if args.emit != "exec-per-event" and args.exec:
|
|
3596
|
+
sys.stderr.write("kijito-inbox-monitor: WARNING --exec ignored (emit mode is %s)\n" % args.emit)
|
|
3597
|
+
if args.poll_seconds < 1:
|
|
3598
|
+
raise FatalConfig("--poll-seconds must be >= 1") # 0 → a select(timeout=0) busy-loop hammering the source
|
|
3599
|
+
if args.wait < 0:
|
|
3600
|
+
raise FatalConfig("--wait must be >= 0 (0 disables long-poll)")
|
|
3601
|
+
if args.wait > 0 and args.no_fast_path:
|
|
3602
|
+
sys.stderr.write("kijito-inbox-monitor: WARNING --wait ignored with --no-fast-path (long-poll is part of "
|
|
3603
|
+
"the fast-path)\n")
|
|
3604
|
+
if args.heartbeat is not None and args.heartbeat < 1:
|
|
3605
|
+
raise FatalConfig("--heartbeat must be >= 1")
|
|
3606
|
+
if args.content_chars < 0:
|
|
3607
|
+
raise FatalConfig("--content-chars must be >= 0")
|
|
3608
|
+
if args.max_replay < 0:
|
|
3609
|
+
raise FatalConfig("--max-replay must be >= 0")
|
|
3610
|
+
if args.keep_logs < 1:
|
|
3611
|
+
raise FatalConfig("--keep-logs must be >= 1")
|
|
3612
|
+
if args.events_file and args.events_file_template:
|
|
3613
|
+
raise FatalConfig("--events-file and --events-file-template are mutually exclusive")
|
|
3614
|
+
if args.events_file_template and "{persona}" not in args.events_file_template:
|
|
3615
|
+
raise FatalConfig("--events-file-template must contain the '{persona}' placeholder")
|
|
3616
|
+
if (args.events_file or args.events_file_template) and args.emit != "stdout-jsonl":
|
|
3617
|
+
sys.stderr.write("kijito-inbox-monitor: WARNING --events-file/-template ignored (emit mode is %s)\n" % args.emit)
|
|
3618
|
+
if args.seed_at is not None:
|
|
3619
|
+
single = len(args.persona or []) == 1 and not args.personas and not args.all_personas
|
|
3620
|
+
if not single:
|
|
3621
|
+
raise FatalConfig("--seed-at requires a single --persona target, "
|
|
3622
|
+
"not multi-persona/all-personas - each persona has its own cursor")
|
|
3623
|
+
|
|
3624
|
+
|
|
3625
|
+
def main(argv=None):
|
|
3626
|
+
args = build_parser().parse_args(argv)
|
|
3627
|
+
# A pure read of an existing report: no token, no network, no state file, no watch loop. Placed
|
|
3628
|
+
# before validate_args so a heartbeat can call it without satisfying the watcher's own config.
|
|
3629
|
+
# A pure string transform, deliberately reachable with NO other configuration: every caller that
|
|
3630
|
+
# needs the filename rule must be able to ask for it, or it will guess again (row M290).
|
|
3631
|
+
if args.safe_persona is not None:
|
|
3632
|
+
if not args.safe_persona:
|
|
3633
|
+
sys.stderr.write("kijito-inbox-monitor: FATAL --safe-persona needs a non-empty PERSONA\n")
|
|
3634
|
+
return 2
|
|
3635
|
+
sys.stdout.write(_state_safe_persona(args.safe_persona) + "\n")
|
|
3636
|
+
return 0
|
|
3637
|
+
if args.check_activity:
|
|
3638
|
+
if not args.activity_file:
|
|
3639
|
+
sys.stderr.write("kijito-inbox-monitor: FATAL --check-activity requires --activity-file\n")
|
|
3640
|
+
return 2
|
|
3641
|
+
if args.since_id is None:
|
|
3642
|
+
sys.stderr.write("kijito-inbox-monitor: FATAL --check-activity requires --since-id "
|
|
3643
|
+
"(the message id you are waiting on a reply to)\n")
|
|
3644
|
+
return 2
|
|
3645
|
+
return check_activity(args.activity_file, args.check_activity, args.since_id, args.waits)
|
|
3646
|
+
try:
|
|
3647
|
+
validate_args(args)
|
|
3648
|
+
return run(args)
|
|
3649
|
+
except FatalConfig as e:
|
|
3650
|
+
sys.stderr.write("kijito-inbox-monitor: FATAL %s\n" % e)
|
|
3651
|
+
return 2
|
|
3652
|
+
except OSError as e:
|
|
3653
|
+
# THE BACKSTOP, and deliberately only that. `InsecureFile` is an OSError, so before
|
|
3654
|
+
# this arm an escaping one exited via a TRACEBACK - which under launchd KeepAlive means a crash
|
|
3655
|
+
# loop with the cause buried in monitor.err rather than a stated fatal condition. Containment
|
|
3656
|
+
# that keeps the OTHER personas running lives at the three per-persona sites; this arm exists so
|
|
3657
|
+
# that ANY OSError that still reaches the top exits with a diagnosis instead of a stack trace.
|
|
3658
|
+
# It must stay LAST-RESORT: if this is what caught your fault, a per-persona guard was missing.
|
|
3659
|
+
sys.stderr.write("kijito-inbox-monitor: FATAL unhandled file/OS error: %s\n" % e)
|
|
3660
|
+
return 2
|
|
3661
|
+
except KeyboardInterrupt:
|
|
3662
|
+
return 0
|
|
3663
|
+
|
|
3664
|
+
|
|
3665
|
+
if __name__ == "__main__":
|
|
3666
|
+
sys.exit(main())
|