kijito-tools 0.2.6 → 0.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/providers/monitor/CHANGELOG.md +39 -0
- package/providers/monitor/README.md +4 -3
- package/providers/monitor/UPSTREAM +11 -11
- package/providers/monitor/docs/DESIGN.md +46 -13
- package/providers/monitor/kijito_inbox_monitor.py +188 -14
- package/providers/monitor/package.json +1 -1
- package/providers/monitor/pyproject.toml +1 -1
- package/providers/monitor/scripts/mutation-check.py +30 -3
- package/providers/monitor/test_kijito_monitor.py +396 -2
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "kijito-tools",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.8",
|
|
4
4
|
"description": "Installer for kijito-tools: copies the context-tracking, session catch-up, and self-clear scripts plus the Kijito skills into ~/.claude.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"type": "commonjs",
|
|
@@ -3,6 +3,45 @@
|
|
|
3
3
|
All notable changes to kijito-inbox-monitor are documented in this file.
|
|
4
4
|
The format is based on Keep a Changelog, and this project follows Semantic Versioning.
|
|
5
5
|
|
|
6
|
+
## [0.5.9] - 2026-09-26
|
|
7
|
+
|
|
8
|
+
### Fixed
|
|
9
|
+
- **A short server restart no longer raises an `alert`/`recovered` pair** (the dead-man `alert` now needs a
|
|
10
|
+
MEASURED span, not only a failure count: `--alert-floor-seconds`, default `(--alert-after - 1) *
|
|
11
|
+
--poll-seconds` = 120 s; DESIGN.md §7.1 rev 10). In long-poll mode a failed poll is retried after 1/2/4 s,
|
|
12
|
+
so `--alert-after 3` was reached ~3-7 s into any outage: every routine server restart (a ~10-20 s window
|
|
13
|
+
of edge 502s) woke the consumer twice for no mail. `seconds` is now the measured span and `floor_seconds`
|
|
14
|
+
names the floor; the first-failure stamp (`down_since`) is persisted so a supervisor restart mid-outage
|
|
15
|
+
resumes the span. Plain interval polling is unchanged (the default floor is what N failed polls take at
|
|
16
|
+
the configured interval); `--alert-floor-seconds 0` restores the count-only edge.
|
|
17
|
+
In long-poll mode the row's `consecutive_failures` is usually well above `--alert-after` (retries keep
|
|
18
|
+
counting until the floor is reached). A `down_since` that is not a finite, positive number makes the state
|
|
19
|
+
file CORRUPT, like any other malformed field (including a huge integer literal, which would otherwise crash
|
|
20
|
+
the strict read itself).
|
|
21
|
+
|
|
22
|
+
### Added
|
|
23
|
+
- **A failed long-poll honours the server's retry hint.** When the server answers a long-poll with an error
|
|
24
|
+
that carries `Retry-After` (delta-seconds or HTTP-date) or a JSON `retry_after_seconds`, the next attempt
|
|
25
|
+
waits at least that long (rounded up, capped at 120 s) instead of retrying at 1/2/4 s into a restarting
|
|
26
|
+
server. The hint only paces retries; the alert decision is still the measured floor.
|
|
27
|
+
|
|
28
|
+
## [0.5.8] - 2026-09-26
|
|
29
|
+
|
|
30
|
+
### Fixed
|
|
31
|
+
- **Runs on native Windows** (reported from Windows 11, CPython 3.14). Four problems, each of which stopped
|
|
32
|
+
the producer or corrupted delivery there:
|
|
33
|
+
- Startup crashed with `AttributeError: module 'os' has no attribute 'geteuid'`. The private-file guard's
|
|
34
|
+
owner and 0600 checks are POSIX-only now; on Windows it still refuses anything that is not a regular
|
|
35
|
+
file.
|
|
36
|
+
- A new events file never became durable. Windows cannot open a directory to fsync it, so every sync
|
|
37
|
+
reported failure and the cursor was held forever, re-delivering the same mail. The directory fsync is
|
|
38
|
+
skipped on Windows (NTFS journals directory metadata).
|
|
39
|
+
- Every directory level printed a "writable by other local users" warning, because Windows reports
|
|
40
|
+
directories as mode 0777. That warning is POSIX-only now.
|
|
41
|
+
- On a cp1252 console or Git Bash pipe, `--help` crashed with `UnicodeEncodeError`, and so did any event
|
|
42
|
+
whose message body had a character outside cp1252 (an emoji, CJK) - mid-delivery. stdout is now
|
|
43
|
+
written as UTF-8, which JSON Lines requires anyway, and the help text is ASCII.
|
|
44
|
+
|
|
6
45
|
## [0.5.7] - 2026-09-25
|
|
7
46
|
|
|
8
47
|
### Added
|
|
@@ -295,7 +295,7 @@ Each line of the events file (and each `exec-per-event` invocation) is one event
|
|
|
295
295
|
|---------|---------|----------------------|
|
|
296
296
|
| `armed` | emitted once per persona on the first healthy poll (baseline set) | `KIJITOMON_CURSOR` |
|
|
297
297
|
| `new` | a new inbox message | `KIJITOMON_ID`, `KIJITOMON_FROM`, `KIJITOMON_CONTENT`, `KIJITOMON_CREATED`, `KIJITOMON_PERSONA` |
|
|
298
|
-
| `alert` | the source has been
|
|
298
|
+
| `alert` | the source has been failing for `--alert-after` polls **and** for a measured `--alert-floor-seconds` (dead-man; `seconds` is the measured span), **or** mail is stranded in an inbox nobody watches, **or** the server holds unread mail this window did not show (all below) | `KIJITOMON_REASON`, `KIJITOMON_FAILURES`, `KIJITOMON_SECONDS`, `KIJITOMON_FLOOR_SECONDS`, `KIJITOMON_STRANDED` |
|
|
299
299
|
| `recovered` | the source came back after an `alert` | `KIJITOMON_CURSOR` |
|
|
300
300
|
| `still_unread` | mail already announced as `new` is still **unread** `--still-unread-after` seconds (default 2 h) after it was sent. One event per poll names every such message; each message is reminded at most once per window and at most `--still-unread-max` (default 3) times; never for retired, reserved or write_only inboxes; nothing on the first poll after a restart. Read what you have handled with `mark_read=true` and it stops. | `KIJITOMON_IDS`, `KIJITOMON_OLDEST_AGE`, `KIJITOMON_REASON` |
|
|
301
301
|
| `heartbeat` | optional liveness tick (`--heartbeat N`) | `KIJITOMON_CURSOR` |
|
|
@@ -549,9 +549,10 @@ four sat above it. Only the newest page's count answers the question "is there u
|
|
|
549
549
|
| `--state-file-template PATH` | Like `--state-file`, with `{persona}` filled in by the producer (the component `--safe-persona` prints). Use it in a supervisor unit so the unit never spells a persona into a path. |
|
|
550
550
|
| `--token-file-template PATH` | Like `--token-file`, with `{persona}` filled in by the producer for the one `--persona` target. |
|
|
551
551
|
| `--safe-persona NAME` | Print the filename component the producer uses for NAME, then exit. Anything that needs to name a persona's files should ask this rather than re-implement the rule. |
|
|
552
|
-
| `--wait N` | Long-poll hold (s) requested from the server so new mail wakes the watcher near-instantly at ~the same request rate (default 50; `0` disables). Falls back to interval polling against a server that doesn't support it, and auto-upgrades when it does. |
|
|
553
|
-
| `--poll-seconds N` | Interval between polls when long-poll is off/unsupported (default 60). |
|
|
552
|
+
| `--wait N` | Long-poll hold (s) requested from the server so new mail wakes the watcher near-instantly at ~the same request rate (default 50; `0` disables). Falls back to interval polling against a server that doesn't support it, and auto-upgrades when it does. A failed long-poll waits at least as long as the server's `Retry-After` (or `retry_after_seconds`) asks, capped at 120 s. |
|
|
553
|
+
| `--poll-seconds N` | Interval between polls when long-poll is off/unsupported (default 60). Also sets the default alert floor in every mode: `(--alert-after - 1) * --poll-seconds` (see `--alert-floor-seconds`). |
|
|
554
554
|
| `--alert-after N` | Consecutive failures before an `alert` (default 3, min 1). A single transient failure is normal. |
|
|
555
|
+
| `--alert-floor-seconds N` | Measured seconds the source must have been failing before the `alert` fires, in addition to the failure count (min 0). Default `(--alert-after - 1) * --poll-seconds` - what N consecutive failed polls take at the configured interval - so long-poll's fast retries (1/2/4 s after a failure) cannot alert on a 10-15 s restart. `0` restores the count-only edge. The alert's `seconds` is the measured span, never a nominal product of flags. |
|
|
555
556
|
| `--heartbeat N` | Emit a `heartbeat` every N seconds (external dead-man's switch). |
|
|
556
557
|
| `--content-chars N` / `--no-content` | Truncate (default 220) or omit message content. |
|
|
557
558
|
| `--suppress-author P` | Don't emit `new` events authored by persona P (repeatable); drops self-echo when watching all personas. |
|
|
@@ -1,26 +1,26 @@
|
|
|
1
1
|
# The upstream release providers/monitor is a byte-exact copy of (row M314).
|
|
2
2
|
# Written by scripts/import-monitor.sh; verified by tests/vendored_monitor_test.sh. Never hand-edit.
|
|
3
3
|
repo=KijitoAI/kijito-inbox-monitor
|
|
4
|
-
sha=
|
|
5
|
-
tree=
|
|
6
|
-
tag=v0.5.
|
|
4
|
+
sha=685341bdffcffdec6f8d0c4ce849fd93d7185928
|
|
5
|
+
tree=a6ca2adec2156c5f9c6a25786ea45a65d8572c09
|
|
6
|
+
tag=v0.5.9
|
|
7
7
|
file=100644 8d88ccb0fd09b9144505f985b4f882db278fac1f .github/workflows/publish-npm.yml
|
|
8
8
|
file=100644 19ce3471a2cffa689be9d261049c1e74e3e7a25e .github/workflows/publish-pypi.yml
|
|
9
9
|
file=100644 4534290a5dd36d18d07a782b784b6b0ad5ee38fc .gitignore
|
|
10
|
-
file=100644
|
|
10
|
+
file=100644 d4204200a9f90a7fdecc4c97711c94e063a3856e CHANGELOG.md
|
|
11
11
|
file=100644 d645695673349e3947e8e5ae42332d0ac3164cd7 LICENSE
|
|
12
12
|
file=100644 f6b0e5634f3fd4156c1c000efb31578bfb068cf3 NOTICE
|
|
13
|
-
file=100644
|
|
13
|
+
file=100644 e74216a2bd031dd876ca84c4bcf29b9246b2fe67 README.md
|
|
14
14
|
file=100644 72f0c148f1e721ac1bb4a38e5cae690355774314 RELEASING.md
|
|
15
15
|
file=100755 fca1203267cb726736b0c8f239e2d74c40a7a895 arm-hive-monitor.sh
|
|
16
16
|
file=100755 bbe3f07de13dab8a8a7a4798481156ae4a46ab8f bin/cli.js
|
|
17
17
|
file=100644 fffa8bd23749a321285bd942a3f4f4a98936129c com.kijito.inbox-monitor.plist.template
|
|
18
|
-
file=100644
|
|
18
|
+
file=100644 0dd1e5227afa4572b2324fc5982431b7519f5cea docs/DESIGN.md
|
|
19
19
|
file=100644 4f5e53a8633e60e6c57c451e28546f8af8783fe8 kijito-inbox-monitor@.service.template
|
|
20
|
-
file=100644
|
|
21
|
-
file=100644
|
|
22
|
-
file=100644
|
|
20
|
+
file=100644 957c2dff0b2afb08ef52fb35ebde42e6fb896414 kijito_inbox_monitor.py
|
|
21
|
+
file=100644 805473fa67b7e4934f9a3100803478b9a1213060 package.json
|
|
22
|
+
file=100644 bf4529771d5564b4a29a766c76724fcdf8a13f76 pyproject.toml
|
|
23
23
|
file=100755 b34a5c775385ecfaefd4b374c1fc1aa082a85d14 scripts/migrate-systemd-unit.sh
|
|
24
|
-
file=100755
|
|
24
|
+
file=100755 9caeb0c317ce53c1a6669d1cce7dd41ce8a6d912 scripts/mutation-check.py
|
|
25
25
|
file=100755 acba6196213736922fc4f7135749386d85d172a6 scripts/prepublish-gate.sh
|
|
26
|
-
file=100644
|
|
26
|
+
file=100644 6a483356f839292caa207a4be418437b7b38b8ac test_kijito_monitor.py
|
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
# Kijito Inbox Monitor: Design & Implementation Spec
|
|
2
2
|
|
|
3
|
-
**Updated:** 2026-
|
|
3
|
+
**Updated:** 2026-09-26 (rev 10: §7.1 the dead-man edge needs a MEASURED span, not only a failure count -
|
|
4
|
+
`--alert-floor-seconds`, default `(alert_after - 1) * poll_seconds`; `seconds` on the `alert` is now that
|
|
5
|
+
measurement and `floor_seconds` names the floor; a failed poll's `Retry-After` paces the next retry. Before
|
|
6
|
+
this, the count-only edge fired ~3-7 s into any short server restart, and `seconds` read a nominal 90.)
|
|
7
|
+
Rev 9 (2026-08-15: §6.5 Darwin `boottime` re-sourced from `CLOCK_MONOTONIC_RAW` - Darwin's
|
|
4
8
|
`CLOCK_MONOTONIC` is calendar-derived and read below `CLOCK_UPTIME_RAW` at fresh uptime - plus the
|
|
5
9
|
inverted-pair quarantine at the emit chokepoint).
|
|
6
10
|
Rev 8 (2026-07-25): the bounded-window / delivery-acknowledgement contracts, from seven rounds of
|
|
@@ -28,7 +32,9 @@ no pip installs) that polls the Kijito inbox and emits one event per new message
|
|
|
28
32
|
running, as NDJSON on stdout and/or exec-a-command-per-event. It is the client-side liveness watcher: it
|
|
29
33
|
keeps a running agent's inbox live by waking it between tool calls. It is not a server, and not a
|
|
30
34
|
notification service. POSIX target (Linux/macOS); Windows runs interval-only (no SIGUSR1 seam, no flock,
|
|
31
|
-
per §10/§7.3).
|
|
35
|
+
per §10/§7.3). On Windows the private-file guard checks only "regular file" (there is no POSIX owner or
|
|
36
|
+
mode; access is the profile's inherited ACL), a directory fsync is skipped (Windows cannot open a directory
|
|
37
|
+
to fsync it; NTFS journals the metadata), and stdout is written as UTF-8 whatever the console code page.
|
|
32
38
|
|
|
33
39
|
## 2. The problem
|
|
34
40
|
|
|
@@ -291,7 +297,7 @@ One object per line; every event carries `event`, `source`, `ts` (emit-time UTC
|
|
|
291
297
|
```
|
|
292
298
|
{"event":"new", "source":"kijito-inbox","ts":"<iso>","id":246,"from":"river","content":"<≤N or omitted>","created":"<iso>"}
|
|
293
299
|
{"event":"armed", "source":"kijito-inbox","ts":"<iso>","cursor":250}
|
|
294
|
-
{"event":"alert", "source":"kijito-inbox","ts":"<iso>","reason":"unreachable","consecutive_failures":3,"seconds":
|
|
300
|
+
{"event":"alert", "source":"kijito-inbox","ts":"<iso>","reason":"unreachable","consecutive_failures":3,"seconds":121,"floor_seconds":120}
|
|
295
301
|
{"event":"recovered", "source":"kijito-inbox","ts":"<iso>","cursor":250}
|
|
296
302
|
{"event":"still_unread","source":"kijito-inbox","ts":"<iso>","ids":[240,246],"senders":["river"],"oldest_age_seconds":9000,"after_seconds":7200,"reason":"..."} # M229 backstop, bounded
|
|
297
303
|
{"event":"heartbeat", "source":"kijito-inbox","ts":"<iso>","cursor":250} # only if --heartbeat; cursor may be null
|
|
@@ -299,12 +305,14 @@ One object per line; every event carries `event`, `source`, `ts` (emit-time UTC
|
|
|
299
305
|
{"event":"replay_capped","source":"kijito-inbox","ts":"<iso>","capped_to":539,"dropped":389} # backlog > --max-replay (§7.0)
|
|
300
306
|
```
|
|
301
307
|
- `new` carries `id`, `from`, `content`, `created`. `content` is a silent hard cut to `--content-chars` (default 220),
|
|
302
|
-
with no marker; or it is omitted with `--no-content`. `seconds` in `alert` is **
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
+
with no marker; or it is omitted with `--no-content`. `seconds` in the reachability `alert` is the **measured**
|
|
309
|
+
monotonic span from the first failure of the run to the failure that crossed the edge (rev 10; it used to be the
|
|
310
|
+
nominal `consecutive_failures * poll_seconds`, routinely an order of magnitude off - `seconds:90` against an 11 s
|
|
311
|
+
outage). `floor_seconds` is the floor the run had to reach (§7.1). `ts - seconds` is a LOWER bound on how long the
|
|
312
|
+
source had been failing not the onset: the stamp is taken when the first failing
|
|
313
|
+
fetch RETURNS, so detection lag (a blackholed `--wait` hold plus its socket timeout, or a plain-poll interval)
|
|
314
|
+
precedes it; and after a supervisor restart the span resumes from the persisted stamp, so it can include time
|
|
315
|
+
nobody was observing (a forward wall-clock step at boot lengthens it; a backward one is clamped to zero).
|
|
308
316
|
- **Within-poll emit order (deterministic, total):** `alert`/`recovered` (FSM edge), then `replay_capped`/`seed_ahead`,
|
|
309
317
|
then `armed`, then `new` (ascending id), then `heartbeat`. So `armed`/`recovered` set `cursor` before any `new`/`heartbeat`
|
|
310
318
|
in the same cycle, which means `recovered.cursor` is non-null whenever a baseline has occurred (a `recovered` on a poll
|
|
@@ -574,12 +582,30 @@ peek, edge-alerts, and the replay cap).
|
|
|
574
582
|
### 7.1 Liveness alert FSM (dead-man's-switch)
|
|
575
583
|
States are **UP** (default) and **DOWN**; `consecutive_failures` counts from 0. A "failure" is any non-healthy poll (§5).
|
|
576
584
|
- **Healthy poll:** set `consecutive_failures = 0`; if DOWN, go UP and emit one `recovered`.
|
|
577
|
-
- **Failure:**
|
|
578
|
-
|
|
579
|
-
|
|
585
|
+
- **Failure:** on the first failure of a run stamp `down_since` (monotonic + wall clock); `consecutive_failures += 1`;
|
|
586
|
+
the UP-to-DOWN edge is crossed on the first failure at which BOTH `consecutive_failures >= --alert-after` AND the
|
|
587
|
+
measured span since `down_since` `>= floor` hold while state is UP, where `floor = --alert-floor-seconds` if given,
|
|
588
|
+
else `(--alert-after - 1) * --poll-seconds` (rev 10). Then set DOWN and emit one `alert` carrying the measured
|
|
589
|
+
`seconds` and `floor_seconds`. (The `state==UP` guard is what makes it edge-once; a resumed `state==DOWN` never
|
|
590
|
+
re-crosses the edge, so there is no duplicate `alert`.) WHY A FLOOR: in long-poll mode a failed poll is retried
|
|
591
|
+
after 1/2/4 s, so the count alone was reached ~3 s into any outage and every planned restart alerted (measured
|
|
592
|
+
on 11-20 s server restarts); the default floor is exactly what N failed polls take at the
|
|
593
|
+
configured interval, so plain interval polling keeps alerting on the N-th failure. A run under the floor emits
|
|
594
|
+
neither edge. `down_since` (wall) is persisted with `consecutive_failures` (§7.3) so a supervisor restart
|
|
595
|
+
mid-outage resumes the span rather than restarting the floor; an older file without it measures from the restart.
|
|
580
596
|
- `alert`/`recovered` are per-edge: a run may alert, recover, then alert again. A sub-threshold blip emits neither.
|
|
581
597
|
- `--alert-after` has a minimum of 1 (0 is rejected) and a default of 3 (a single transient failure is normal,
|
|
582
|
-
bouncing in ~1-2s). SIGUSR1-triggered polls
|
|
598
|
+
bouncing in ~1-2s). SIGUSR1-triggered polls count toward `consecutive_failures` like any other; they do not
|
|
599
|
+
advance the measured span, so triggered polls alone cannot cross the floor - the floor is time, by design.
|
|
600
|
+
In long-poll mode `consecutive_failures` on the row is therefore usually well above `--alert-after` (the
|
|
601
|
+
1/2/4/8/16/30 s retries keep counting until the floor is reached); the count says how many retries were made,
|
|
602
|
+
the floor says when the edge was allowed.
|
|
603
|
+
- **Server retry hint (rev 10).** A failed long-poll that carries the server's own hint - a `Retry-After` header
|
|
604
|
+
(delta-seconds or HTTP-date) or a JSON `retry_after_seconds` field, header first - waits AT LEAST that long
|
|
605
|
+
before the next attempt (rounded up, clamped to 0..120 s; the exponential 1/2/4/8/16/30 s backoff still
|
|
606
|
+
applies when it is longer). The hint only paces retries; it never touches the alert decision, which stays
|
|
607
|
+
the measured floor above. A later failure with no hint clears it, so a stale hint cannot pace an unrelated
|
|
608
|
+
failure.
|
|
583
609
|
|
|
584
610
|
### 7.2 `--self-test`: runs once and exits (no poll loop)
|
|
585
611
|
Runs for the selected persona(s) (default: every persona in the account). (a) One real peek-mode (`mark_read=false`)
|
|
@@ -599,11 +625,18 @@ read-state-neutral (DONE-WHEN #5 holds after self-test).
|
|
|
599
625
|
`/api/notify/pending` read succeeded (a persona missing from a good response is a real 0 - the server omits
|
|
600
626
|
personas with nothing pending); a poll without that read writes no `unread`, so its absence means UNKNOWN,
|
|
601
627
|
never zero. A reader should also judge freshness from the file's mtime: the file is rewritten every poll.
|
|
628
|
+
`down_since` (number: wall-clock epoch of the first failure of the current run, written only while
|
|
629
|
+
`consecutive_failures > 0`; §7.1 rev 10).
|
|
602
630
|
- **Every persisted field is read STRICTLY, and anything unrecognised fails CLOSED** (Loom re-audit 7, HIGH 2).
|
|
603
631
|
Booleans must be JSON booleans and integers must be real integers - a JSON `1` for `pin_forced` used to
|
|
604
632
|
normalise to `false` and silently UNPIN the watermark, letting the replay cap cross the very span the pin was
|
|
605
633
|
protecting; `pin_evidence_intact: 0` had the mirror bug. A malformed field is evidence the file cannot be
|
|
606
634
|
trusted, so it is treated as CORRUPT (below), never as a permissive default. `true` is not a message id.
|
|
635
|
+
`down_since` is the one float: it must be a finite, positive number (JSON's `-Infinity`/`NaN`/`1e400` all
|
|
636
|
+
parse, a 400-digit integer literal makes `math.isfinite` itself raise, and an infinite span would raise at the
|
|
637
|
+
alert edge - a crash loop fed by the file the supervisor re-reads; so the value is normalised through `float()`
|
|
638
|
+
with overflow caught, then checked). A bool, string, zero, negative or non-finite value makes the WHOLE file
|
|
639
|
+
CORRUPT (cursor discarded, pin forced), the same consequence as any other malformed field.
|
|
607
640
|
- **Canonical identity (`<canonical-id>`)** is computed before DNS resolution so trivial URL variations don't flip
|
|
608
641
|
it. From the effective inbox URL, it is the tuple `(scheme.lower(), host.lower(), effective_port, path,
|
|
609
642
|
sorted(query_params except the constant mark_read))`. Normalize by stripping a trailing `/` on path, filling the
|
|
@@ -11,10 +11,12 @@ Authentication is required: set $KIJITOMON_TOKEN (or --token-file) to your Kijit
|
|
|
11
11
|
"""
|
|
12
12
|
import argparse
|
|
13
13
|
import datetime
|
|
14
|
+
import email.utils
|
|
14
15
|
import errno
|
|
15
16
|
import hashlib
|
|
16
17
|
import http.client
|
|
17
18
|
import json
|
|
19
|
+
import math
|
|
18
20
|
import os
|
|
19
21
|
import select
|
|
20
22
|
import signal
|
|
@@ -33,7 +35,7 @@ try:
|
|
|
33
35
|
except ImportError: # pragma: no cover - Windows
|
|
34
36
|
fcntl = None
|
|
35
37
|
|
|
36
|
-
__version__ = "0.5.
|
|
38
|
+
__version__ = "0.5.9"
|
|
37
39
|
SOURCE = "kijito-inbox"
|
|
38
40
|
# A named User-Agent is REQUIRED: api.kijito.ai is fronted by a WAF that 403s the default Python-urllib UA.
|
|
39
41
|
USER_AGENT = "kijito-inbox-monitor/%s" % __version__
|
|
@@ -45,6 +47,7 @@ EXEC_TIMEOUT = 10
|
|
|
45
47
|
HTTP_TIMEOUT = 5 # per-request timeout default (normal fetches)
|
|
46
48
|
LONGPOLL_SLACK = 10 # client socket timeout = server hold (--wait) + this, so a half-open hold is always detected
|
|
47
49
|
LONGPOLL_BACKOFF_CAP = 30 # cap (s) on exponential backoff between failed long-poll attempts
|
|
50
|
+
RETRY_AFTER_CAP = 120 # cap (s) on a server-stated Retry-After; a larger hint is clamped, never obeyed blindly
|
|
48
51
|
PIN_TRACKING_CAP = 5000 # max delivered ids remembered above a pinned watermark (bounds the state file)
|
|
49
52
|
WALK_BACK_MAX_PAGES = 50 # page budget for an authoritative backward walk over an omitted span
|
|
50
53
|
BROKEN_SINK_RETRY_S = 30 # cooldown before re-trying a persona sink we refused; the refusal's RELEASE
|
|
@@ -494,6 +497,59 @@ def fetch_unread_counts(opener, count_url, headers):
|
|
|
494
497
|
return (True, counts)
|
|
495
498
|
|
|
496
499
|
|
|
500
|
+
def next_longpoll_backoff(previous, hint):
|
|
501
|
+
"""Seconds to wait before retrying a FAILED long-poll: exponential from 1 s, capped at
|
|
502
|
+
LONGPOLL_BACKOFF_CAP, and never shorter than the server's own Retry-After hint (already clamped)."""
|
|
503
|
+
backoff = min((previous * 2) or 1, LONGPOLL_BACKOFF_CAP)
|
|
504
|
+
if hint is not None:
|
|
505
|
+
backoff = max(backoff, hint)
|
|
506
|
+
return backoff
|
|
507
|
+
|
|
508
|
+
|
|
509
|
+
# The retry hint of the LAST fetch_unread_counts_longpoll call: seconds (int) the server asked us to wait, or
|
|
510
|
+
# None. A side channel rather than a fourth return value so the function's contract (and every caller and
|
|
511
|
+
# test double of it) is unchanged; the main loop resets it before each call and reads it on failure.
|
|
512
|
+
_RETRY_HINT = {"seconds": None}
|
|
513
|
+
|
|
514
|
+
|
|
515
|
+
def _retry_after_seconds(headers, body=None):
|
|
516
|
+
"""The server's own retry hint on a FAILED response, in whole seconds, or None when it gave none.
|
|
517
|
+
|
|
518
|
+
Two carriers, header first: `Retry-After` (delta-seconds, or an HTTP-date) and a JSON body field
|
|
519
|
+
`retry_after_seconds` (the edge's restart page carries both). A hint only ever LENGTHENS the wait
|
|
520
|
+
before the next retry - it is pacing, never a reason to alert - and it is clamped to
|
|
521
|
+
[0, RETRY_AFTER_CAP] so a hostile or mistaken value cannot park the producer.
|
|
522
|
+
"""
|
|
523
|
+
secs = None
|
|
524
|
+
raw = None
|
|
525
|
+
try:
|
|
526
|
+
raw = headers.get("Retry-After") if headers is not None else None
|
|
527
|
+
except Exception:
|
|
528
|
+
raw = None
|
|
529
|
+
if isinstance(raw, str) and raw.strip():
|
|
530
|
+
raw = raw.strip()
|
|
531
|
+
if raw.isdigit():
|
|
532
|
+
secs = int(raw)
|
|
533
|
+
else:
|
|
534
|
+
try:
|
|
535
|
+
when = email.utils.parsedate_to_datetime(raw)
|
|
536
|
+
now = datetime.datetime.now(datetime.timezone.utc)
|
|
537
|
+
secs = int(math.ceil((when - now).total_seconds()))
|
|
538
|
+
except (TypeError, ValueError, IndexError, OverflowError):
|
|
539
|
+
secs = None
|
|
540
|
+
if secs is None and body:
|
|
541
|
+
try:
|
|
542
|
+
data = json.loads(body)
|
|
543
|
+
except (ValueError, UnicodeDecodeError, RecursionError):
|
|
544
|
+
data = None
|
|
545
|
+
v = data.get("retry_after_seconds") if isinstance(data, dict) else None
|
|
546
|
+
if isinstance(v, (int, float)) and not isinstance(v, bool) and math.isfinite(v):
|
|
547
|
+
secs = int(math.ceil(v))
|
|
548
|
+
if secs is None:
|
|
549
|
+
return None
|
|
550
|
+
return max(0, min(secs, RETRY_AFTER_CAP))
|
|
551
|
+
|
|
552
|
+
|
|
497
553
|
def fetch_unread_counts_longpoll(opener, headers, wait, cursor):
|
|
498
554
|
"""Long-poll variant of the fast-path. GET /api/notify/pending?wait=<sec>[&cursor=<opaque>].
|
|
499
555
|
|
|
@@ -512,11 +568,22 @@ def fetch_unread_counts_longpoll(opener, headers, wait, cursor):
|
|
|
512
568
|
q["cursor"] = cursor
|
|
513
569
|
url = NOTIFY_PENDING_URL + "?" + urllib.parse.urlencode(q)
|
|
514
570
|
req = urllib.request.Request(url, headers=headers, method="GET")
|
|
571
|
+
_RETRY_HINT["seconds"] = None
|
|
515
572
|
try:
|
|
516
573
|
with opener.open(req, timeout=wait + LONGPOLL_SLACK) as resp:
|
|
517
574
|
if not (200 <= resp.status < 300):
|
|
575
|
+
_RETRY_HINT["seconds"] = _retry_after_seconds(getattr(resp, "headers", None))
|
|
518
576
|
return (False, {}, cursor)
|
|
519
577
|
data = json.loads(resp.read())
|
|
578
|
+
except urllib.error.HTTPError as e:
|
|
579
|
+
# A 502/503 during a server restart. Read the edge's own "retry in N s" so the next attempt is
|
|
580
|
+
# paced by it; a bounded read, because this body is an error page, not data.
|
|
581
|
+
try:
|
|
582
|
+
body = e.read(4096)
|
|
583
|
+
except Exception:
|
|
584
|
+
body = None
|
|
585
|
+
_RETRY_HINT["seconds"] = _retry_after_seconds(e.headers, body)
|
|
586
|
+
return (False, {}, cursor)
|
|
520
587
|
except Exception:
|
|
521
588
|
return (False, {}, cursor) # keep the old cursor → next attempt resumes losslessly
|
|
522
589
|
counts = _parse_unread_rows(data)
|
|
@@ -1182,6 +1249,7 @@ class Emitter:
|
|
|
1182
1249
|
"created": "KIJITOMON_CREATED", "cursor": "KIJITOMON_CURSOR",
|
|
1183
1250
|
"persona": "KIJITOMON_PERSONA",
|
|
1184
1251
|
"reason": "KIJITOMON_REASON", "consecutive_failures": "KIJITOMON_FAILURES",
|
|
1252
|
+
"seconds": "KIJITOMON_SECONDS", "floor_seconds": "KIJITOMON_FLOOR_SECONDS",
|
|
1185
1253
|
"seeded": "KIJITOMON_SEEDED", "current_max": "KIJITOMON_CURRENT_MAX",
|
|
1186
1254
|
"capped_to": "KIJITOMON_CAPPED_TO", "dropped": "KIJITOMON_DROPPED",
|
|
1187
1255
|
"stranded_inboxes": "KIJITOMON_STRANDED",
|
|
@@ -1261,6 +1329,12 @@ def _assert_private_fd(fd, path):
|
|
|
1261
1329
|
st = os.fstat(fd)
|
|
1262
1330
|
if not stat.S_ISREG(st.st_mode):
|
|
1263
1331
|
raise InsecureFile("%s is not a regular file" % path)
|
|
1332
|
+
if not IS_POSIX:
|
|
1333
|
+
# WINDOWS HAS NO POSIX OWNER OR MODE BITS TO CHECK (praetor, Windows 11 native, 2026-09-26: os.geteuid
|
|
1334
|
+
# does not exist there, so this line crashed the producer at startup). Access there is an ACL, and a
|
|
1335
|
+
# file created under the user's profile inherits a user-only ACL; st_mode reports a synthetic
|
|
1336
|
+
# 0666/0444 that says nothing about who can read it. The regular-file check above still holds.
|
|
1337
|
+
return
|
|
1264
1338
|
if st.st_uid != os.geteuid():
|
|
1265
1339
|
raise InsecureFile("%s is owned by uid %d, not by us (uid %d)" % (path, st.st_uid, os.geteuid()))
|
|
1266
1340
|
cur = st.st_mode & 0o777
|
|
@@ -1364,6 +1438,10 @@ def _makedirs_private(path):
|
|
|
1364
1438
|
# directory is never validated, and an existing one is exactly where a hostile path would already be.
|
|
1365
1439
|
# A sticky directory (/tmp, mode 1777) is excluded: the sticky bit is precisely what makes a shared
|
|
1366
1440
|
# writable directory safe, and warning about it would train the reader to ignore this line.
|
|
1441
|
+
# POSIX only: Windows reports every directory as 0777 (access is an ACL), so the warning would fire on
|
|
1442
|
+
# every level of every path and say nothing true.
|
|
1443
|
+
if not IS_POSIX:
|
|
1444
|
+
return
|
|
1367
1445
|
seen = path
|
|
1368
1446
|
while True:
|
|
1369
1447
|
try:
|
|
@@ -1386,7 +1464,13 @@ def _fsync_dir(path):
|
|
|
1386
1464
|
os.replace is atomic for a concurrent READER, but atomicity is not durability: after a power loss
|
|
1387
1465
|
the new file's contents can be on disk while the directory entry still names the old inode - i.e. a
|
|
1388
1466
|
silently OLDER cursor. Syncing the file alone (which is all we did) does not cover the rename.
|
|
1467
|
+
|
|
1468
|
+
Windows cannot open a directory for fsync at all, so this always failed there and the events sink held
|
|
1469
|
+
its cursor forever on a new file. NTFS journals directory metadata itself, so there is nothing further
|
|
1470
|
+
to do: report success.
|
|
1389
1471
|
"""
|
|
1472
|
+
if not IS_POSIX:
|
|
1473
|
+
return True
|
|
1390
1474
|
try:
|
|
1391
1475
|
fd = os.open(path, os.O_RDONLY)
|
|
1392
1476
|
except OSError:
|
|
@@ -1603,6 +1687,29 @@ class StateFile:
|
|
|
1603
1687
|
sys.stderr.write("kijito-inbox-monitor: WARNING state-file 'pin_release_at' is not an integer (%r); "
|
|
1604
1688
|
"refusing to interpret it: %s\n" % (release_at, self.path))
|
|
1605
1689
|
release_at, strict_ok = None, False
|
|
1690
|
+
# §7.1 wall-clock stamp of the first failure of a DOWN run. Absent in older files. Read strictly:
|
|
1691
|
+
# a bool is not a timestamp (the same `1`/`true` confusion as the pin flags above).
|
|
1692
|
+
down_since = d.get("down_since", _MISSING)
|
|
1693
|
+
if down_since is _MISSING or down_since is None:
|
|
1694
|
+
down_since = None
|
|
1695
|
+
else:
|
|
1696
|
+
# json.loads accepts `-Infinity`/`NaN`, turns 1e400 into inf, and hands back a Python int for a
|
|
1697
|
+
# 400-digit literal - on which math.isfinite ITSELF raises OverflowError. Any of those escaping
|
|
1698
|
+
# load() is a traceback at startup or at the alert edge; under a supervisor that is a crash loop
|
|
1699
|
+
# fed by the file it re-reads on every restart. So: normalise through float() with the overflow
|
|
1700
|
+
# caught, then require finite and positive. A stamp is a finite positive epoch or it is nothing.
|
|
1701
|
+
fv = None
|
|
1702
|
+
if isinstance(down_since, (int, float)) and not isinstance(down_since, bool):
|
|
1703
|
+
try:
|
|
1704
|
+
fv = float(down_since)
|
|
1705
|
+
except OverflowError:
|
|
1706
|
+
fv = None
|
|
1707
|
+
if fv is not None and math.isfinite(fv) and fv > 0:
|
|
1708
|
+
down_since = fv
|
|
1709
|
+
else:
|
|
1710
|
+
sys.stderr.write("kijito-inbox-monitor: WARNING state-file 'down_since' is not a finite positive "
|
|
1711
|
+
"number (%r); refusing to interpret it: %s\n" % (down_since, self.path))
|
|
1712
|
+
down_since, strict_ok = None, False
|
|
1606
1713
|
raw = d.get("emitted_above")
|
|
1607
1714
|
if raw is None:
|
|
1608
1715
|
emitted, intact = set(), True # no pin was in force; the ordinary case
|
|
@@ -1655,7 +1762,8 @@ class StateFile:
|
|
|
1655
1762
|
return {"cursor": cursor, "state": state, "failures": failures, "emitted_above": emitted,
|
|
1656
1763
|
"gap_alerted": alerted, "pin_evidence_intact": intact,
|
|
1657
1764
|
"pin_forced": pin_forced, "pin_release_at": release_at,
|
|
1658
|
-
"state_corrupt": state_corrupt, "unread_hidden": hidden, "unread": unread
|
|
1765
|
+
"state_corrupt": state_corrupt, "unread_hidden": hidden, "unread": unread,
|
|
1766
|
+
"down_since": down_since}
|
|
1659
1767
|
|
|
1660
1768
|
def unlock(self):
|
|
1661
1769
|
"""Release the single-writer flock and close the sidecar fd.
|
|
@@ -1672,7 +1780,7 @@ class StateFile:
|
|
|
1672
1780
|
|
|
1673
1781
|
def save(self, cursor, state, failures, emitted_above=None, gap_alerted=None,
|
|
1674
1782
|
pin_forced=False, pin_evidence_intact=True, state_corrupt=False, pin_release_at=None,
|
|
1675
|
-
unread_hidden=False, unread=None):
|
|
1783
|
+
unread_hidden=False, unread=None, down_since=None):
|
|
1676
1784
|
"""Persist the cursor. Returns True IFF the write is DURABLE (Loom re-audit 8, HIGH 3).
|
|
1677
1785
|
|
|
1678
1786
|
The directory fsync used to be called and its answer thrown away, so a failure returned success
|
|
@@ -1713,6 +1821,10 @@ class StateFile:
|
|
|
1713
1821
|
# Omitted when this poll had no count, so its absence means "unknown", never "zero".
|
|
1714
1822
|
if unread is not None:
|
|
1715
1823
|
d["unread"] = unread
|
|
1824
|
+
# §7.1 the wall-clock stamp of the first failure of the current DOWN run, so a supervisor restart
|
|
1825
|
+
# mid-outage resumes the measured span instead of restarting the floor from zero.
|
|
1826
|
+
if down_since is not None:
|
|
1827
|
+
d["down_since"] = down_since
|
|
1716
1828
|
dirn = os.path.dirname(os.path.abspath(self.path)) or "."
|
|
1717
1829
|
# BOTH OF THESE ARE INSIDE THE GUARD, and they did not used to be (drill, 2026-08-05).
|
|
1718
1830
|
# This function builds a careful "written but not provably durable" path - _fsync_dir fails ->
|
|
@@ -1810,6 +1922,20 @@ class WakeSeam:
|
|
|
1810
1922
|
pass
|
|
1811
1923
|
|
|
1812
1924
|
|
|
1925
|
+
def alert_floor_seconds(args):
|
|
1926
|
+
"""§7.1 the MEASURED span a failing run must reach before the dead-man `alert` fires.
|
|
1927
|
+
|
|
1928
|
+
`--alert-floor-seconds` when given; otherwise `(--alert-after - 1) * --poll-seconds`, which is exactly
|
|
1929
|
+
how long `--alert-after` consecutive failed polls take at the configured interval - so plain interval
|
|
1930
|
+
polling still alerts on the N-th failure, while long-poll mode (which retries after 1/2/4 s on a
|
|
1931
|
+
failure) can no longer reach the edge in ~3 s. Before this, the count-only edge fired on every
|
|
1932
|
+
routine server restart, 11 s windows included.
|
|
1933
|
+
"""
|
|
1934
|
+
if args.alert_floor_seconds is not None:
|
|
1935
|
+
return args.alert_floor_seconds
|
|
1936
|
+
return (args.alert_after - 1) * args.poll_seconds
|
|
1937
|
+
|
|
1938
|
+
|
|
1813
1939
|
def _monotonic():
|
|
1814
1940
|
import time as _t
|
|
1815
1941
|
return _t.monotonic()
|
|
@@ -2008,6 +2134,7 @@ class WatchTarget:
|
|
|
2008
2134
|
self.cursor = None
|
|
2009
2135
|
self.fsm_state = "UP"
|
|
2010
2136
|
self.failures = 0
|
|
2137
|
+
self.down_since = None # (monotonic, wall) of the FIRST failure of the current run; None while healthy
|
|
2011
2138
|
self.armed = False
|
|
2012
2139
|
self.fast_path = False
|
|
2013
2140
|
self.last_unread = None
|
|
@@ -2066,6 +2193,15 @@ class WatchTarget:
|
|
|
2066
2193
|
elif loaded is not None:
|
|
2067
2194
|
self.cursor = loaded["cursor"]
|
|
2068
2195
|
self.fsm_state, self.failures = loaded["state"], loaded["failures"]
|
|
2196
|
+
# §7.1 the measured floor survives a restart mid-outage: the persisted wall-clock stamp of
|
|
2197
|
+
# the first failure supplies the span already elapsed. Absent (a file written by an older
|
|
2198
|
+
# version) the run is measured from now - the conservative direction (later alert, never
|
|
2199
|
+
# a spurious one).
|
|
2200
|
+
self.down_since = None
|
|
2201
|
+
if self.failures:
|
|
2202
|
+
wall = loaded.get("down_since")
|
|
2203
|
+
elapsed = max(0.0, time.time() - wall) if wall is not None else 0.0
|
|
2204
|
+
self.down_since = (_monotonic() - elapsed, wall if wall is not None else time.time())
|
|
2069
2205
|
self.emitted_above = loaded["emitted_above"]
|
|
2070
2206
|
self.gap_alerted = loaded["gap_alerted"]
|
|
2071
2207
|
self.pin_evidence_intact = loaded["pin_evidence_intact"]
|
|
@@ -2453,6 +2589,7 @@ class WatchTarget:
|
|
|
2453
2589
|
self.fsm_state = "UP"
|
|
2454
2590
|
self._alarm("recovered", "source recovered", cursor=self.cursor)
|
|
2455
2591
|
self.failures = 0
|
|
2592
|
+
self.down_since = None
|
|
2456
2593
|
else:
|
|
2457
2594
|
self.skips = 0
|
|
2458
2595
|
poll = fetch(self.opener, self.url, self.headers)
|
|
@@ -2468,6 +2605,7 @@ class WatchTarget:
|
|
|
2468
2605
|
self.fsm_state = "UP"
|
|
2469
2606
|
recovered = True
|
|
2470
2607
|
self.failures = 0
|
|
2608
|
+
self.down_since = None
|
|
2471
2609
|
|
|
2472
2610
|
items = poll.items
|
|
2473
2611
|
# §5.4 Record who AUTHORED what, from the window we already have. Done before any cursor
|
|
@@ -2831,21 +2969,30 @@ class WatchTarget:
|
|
|
2831
2969
|
pinned=True, evidence_lost=True)
|
|
2832
2970
|
|
|
2833
2971
|
else:
|
|
2972
|
+
if self.failures == 0 or self.down_since is None:
|
|
2973
|
+
# First failure of this run: stamp it. Everything the floor measures starts here.
|
|
2974
|
+
self.down_since = (_monotonic(), time.time())
|
|
2834
2975
|
self.failures += 1
|
|
2835
|
-
|
|
2976
|
+
span = _monotonic() - self.down_since[0]
|
|
2977
|
+
floor = alert_floor_seconds(args)
|
|
2978
|
+
if self.failures >= args.alert_after and span >= floor and self.fsm_state == "UP":
|
|
2836
2979
|
# THE DEAD-MAN'S SWITCH. The FSM transition MUST commit (it drives the whole
|
|
2837
|
-
# liveness model
|
|
2838
|
-
# reverted transition would never re-fire
|
|
2839
|
-
#
|
|
2840
|
-
#
|
|
2841
|
-
#
|
|
2842
|
-
#
|
|
2980
|
+
# liveness model; the `fsm_state == "UP"` guard is what makes the edge cross exactly
|
|
2981
|
+
# once, so a reverted transition would never re-fire). So the state commits and the
|
|
2982
|
+
# ANNOUNCEMENT gets the guaranteed second channel (re-audit 11, F1/A1). Before this,
|
|
2983
|
+
# a broken sink meant the source could go down and NOTHING was ever emitted or
|
|
2984
|
+
# logged - the one event README sells as the dead-man's switch, silently absent.
|
|
2985
|
+
# §7.1 the edge needs BOTH the count and the MEASURED span (`>=`, not `==`: in
|
|
2986
|
+
# long-poll mode the count is reached in ~3 s and the floor is what waits). `seconds`
|
|
2987
|
+
# is now that measured span - it used to be the nominal failures * poll_seconds and
|
|
2988
|
+
# read 90 against outages of 11 s.
|
|
2843
2989
|
self.fsm_state = "DOWN"
|
|
2844
2990
|
down_reason = poll.reason or "unreachable"
|
|
2845
2991
|
self._alarm("alert", "source is DOWN: %s" % down_reason,
|
|
2846
2992
|
reason=down_reason,
|
|
2847
2993
|
consecutive_failures=self.failures,
|
|
2848
|
-
seconds=
|
|
2994
|
+
seconds=int(round(span)),
|
|
2995
|
+
floor_seconds=int(floor))
|
|
2849
2996
|
|
|
2850
2997
|
if self.state_file is not None:
|
|
2851
2998
|
durable = self.state_file.save(self.cursor, self.fsm_state, self.failures,
|
|
@@ -2855,7 +3002,8 @@ class WatchTarget:
|
|
|
2855
3002
|
state_corrupt=self.state_corrupt,
|
|
2856
3003
|
pin_release_at=self.pin_release_at,
|
|
2857
3004
|
unread_hidden=self.unread_hidden,
|
|
2858
|
-
unread=self.observed_unread
|
|
3005
|
+
unread=self.observed_unread,
|
|
3006
|
+
down_since=(self.down_since[1] if self.down_since else None))
|
|
2859
3007
|
# ★ CONSUME THE ANSWER (Loom re-audit 9, MEDIUM). Round 8 taught me to RETURN a durability
|
|
2860
3008
|
# status; this is the same defect one layer out - I produced an answer and then discarded it
|
|
2861
3009
|
# at the call site, which is the exact thing the previous round was about. A cursor whose
|
|
@@ -3623,6 +3771,7 @@ def run(args):
|
|
|
3623
3771
|
count_target = next((t for t in targets if t.unread_persona), None)
|
|
3624
3772
|
if count_target is not None and not args.no_fast_path:
|
|
3625
3773
|
if args.wait > 0:
|
|
3774
|
+
_RETRY_HINT["seconds"] = None
|
|
3626
3775
|
counts_available, unread_counts, new_cursor = fetch_unread_counts_longpoll(
|
|
3627
3776
|
count_target.opener, headers, args.wait, cursor)
|
|
3628
3777
|
if counts_available:
|
|
@@ -3634,7 +3783,7 @@ def run(args):
|
|
|
3634
3783
|
else:
|
|
3635
3784
|
# drop / blip / outage: back off, resume the SAME cursor next time (lossless), and this tick
|
|
3636
3785
|
# falls through to per-target full inbox polls (the by-message-id correctness backstop).
|
|
3637
|
-
lp_backoff =
|
|
3786
|
+
lp_backoff = next_longpoll_backoff(lp_backoff, _RETRY_HINT["seconds"])
|
|
3638
3787
|
else:
|
|
3639
3788
|
counts_available, unread_counts = fetch_unread_counts(
|
|
3640
3789
|
count_target.opener, count_target.count_url, headers)
|
|
@@ -3703,11 +3852,16 @@ def build_parser():
|
|
|
3703
3852
|
p.add_argument("--wait", type=int, default=50,
|
|
3704
3853
|
help="Long-poll hold (s) requested from /api/notify/pending so new mail wakes the watcher "
|
|
3705
3854
|
"near-instantly at ~the same request rate (default 50; the server clamps to its own max). "
|
|
3706
|
-
"0 disables long-poll
|
|
3855
|
+
"0 disables long-poll -> plain interval polling at --poll-seconds. If the server doesn't "
|
|
3707
3856
|
"support long-poll, the client auto-falls back to interval polling (no redeploy needed). "
|
|
3708
3857
|
"Clean shutdown during a held poll can take up to --wait seconds (a supervisor's SIGKILL "
|
|
3709
3858
|
"mid-hold is safe - state is persisted every cycle).")
|
|
3710
3859
|
p.add_argument("--alert-after", type=int, default=3, help="Consecutive failures before an alert (min 1).")
|
|
3860
|
+
p.add_argument("--alert-floor-seconds", type=int, default=None,
|
|
3861
|
+
help="Measured seconds the source must have been failing before an `alert` fires, in addition "
|
|
3862
|
+
"to --alert-after failures (min 0). Default: (--alert-after - 1) * --poll-seconds - what "
|
|
3863
|
+
"N consecutive failed polls take at the configured interval - so long-poll's fast retries "
|
|
3864
|
+
"cannot alert on an 11 s restart. 0 restores the count-only edge.")
|
|
3711
3865
|
p.add_argument("--emit", choices=("stdout-jsonl", "exec-per-event"), default="stdout-jsonl")
|
|
3712
3866
|
p.add_argument("--exec", help="Command to run per event (required iff --emit exec-per-event).")
|
|
3713
3867
|
p.add_argument("--suppress-author", action="append",
|
|
@@ -3787,6 +3941,8 @@ def build_parser():
|
|
|
3787
3941
|
def validate_args(args):
|
|
3788
3942
|
if args.alert_after < 1:
|
|
3789
3943
|
raise FatalConfig("--alert-after must be >= 1")
|
|
3944
|
+
if args.alert_floor_seconds is not None and args.alert_floor_seconds < 0:
|
|
3945
|
+
raise FatalConfig("--alert-floor-seconds must be >= 0")
|
|
3790
3946
|
if args.resync_every < 1:
|
|
3791
3947
|
raise FatalConfig("--resync-every must be >= 1")
|
|
3792
3948
|
if args.rediscover_every < 1:
|
|
@@ -3840,7 +3996,25 @@ def validate_args(args):
|
|
|
3840
3996
|
"not multi-persona/all-personas - each persona has its own cursor")
|
|
3841
3997
|
|
|
3842
3998
|
|
|
3999
|
+
def _utf8_stdout():
|
|
4000
|
+
"""Write stdout as UTF-8 whatever the locale says (praetor, Windows 11 native, 2026-09-26).
|
|
4001
|
+
|
|
4002
|
+
A Windows console or a Git Bash pipe reports cp1252, so `--help` (a non-ASCII character) and, far worse,
|
|
4003
|
+
any EVENT carrying a message body outside cp1252 (an emoji, CJK) raised UnicodeEncodeError - a
|
|
4004
|
+
ValueError, which the stdout sink's OSError guard does not catch, so it killed the producer mid-delivery.
|
|
4005
|
+
JSON Lines is UTF-8 by definition, so that is the right encoding for every consumer. No-op where stdout
|
|
4006
|
+
is already UTF-8 or cannot be reconfigured.
|
|
4007
|
+
"""
|
|
4008
|
+
enc = (getattr(sys.stdout, "encoding", None) or "").lower().replace("-", "").replace("_", "")
|
|
4009
|
+
if enc != "utf8" and hasattr(sys.stdout, "reconfigure"):
|
|
4010
|
+
try:
|
|
4011
|
+
sys.stdout.reconfigure(encoding="utf-8")
|
|
4012
|
+
except (ValueError, OSError):
|
|
4013
|
+
pass
|
|
4014
|
+
|
|
4015
|
+
|
|
3843
4016
|
def main(argv=None):
|
|
4017
|
+
_utf8_stdout()
|
|
3844
4018
|
args = build_parser().parse_args(argv)
|
|
3845
4019
|
# A pure read of an existing report: no token, no network, no state file, no watch loop. Placed
|
|
3846
4020
|
# before validate_args so a heartbeat can call it without satisfying the watcher's own config.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "kijito-inbox-monitor",
|
|
3
|
-
"version": "0.5.
|
|
3
|
+
"version": "0.5.9",
|
|
4
4
|
"description": "Pointer to the Python package kijito-inbox-monitor. A zero-dependency watcher that wakes a running agent when new Kijito inbox mail arrives. Install the real tool with pipx/uv/pip.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"type": "commonjs",
|
|
@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
|
|
|
7
7
|
|
|
8
8
|
[project]
|
|
9
9
|
name = "kijito-inbox-monitor"
|
|
10
|
-
version = "0.5.
|
|
10
|
+
version = "0.5.9"
|
|
11
11
|
description = "Zero-dependency watcher that wakes a running agent when new Kijito inbox mail arrives."
|
|
12
12
|
readme = "README.md"
|
|
13
13
|
requires-python = ">=3.9"
|
|
@@ -281,7 +281,8 @@ M=[
|
|
|
281
281
|
' self._alarm("alert", "source is DOWN: %s" % down_reason,\n'
|
|
282
282
|
' reason=down_reason,\n'
|
|
283
283
|
' consecutive_failures=self.failures,\n'
|
|
284
|
-
' seconds=
|
|
284
|
+
' seconds=int(round(span)),\n'
|
|
285
|
+
' floor_seconds=int(floor))',
|
|
285
286
|
' pass'),
|
|
286
287
|
("L11-F1: the DOWN alert loses its stderr fallback (undelivered == silent again)",
|
|
287
288
|
' sys.stderr.write("kijito-inbox-monitor: %s EVENT UNDELIVERED (persona %r): %s\\n"\n'
|
|
@@ -332,6 +333,27 @@ M=[
|
|
|
332
333
|
("A1: the urgent-unanswered alarm is re-coupled to --no-stranded-alerts (one flag silences both again)",
|
|
333
334
|
" if counts_available and not args.no_urgent_alerts:",
|
|
334
335
|
" if counts_available and not args.no_stranded_alerts:"),
|
|
336
|
+
# Row M375 (bug #19): a ~20 s server restart woke consumers twice (alert + recovered) because the
|
|
337
|
+
# dead-man edge was a failure COUNT reached in ~7 s of long-poll retries. The floor and the retry hint
|
|
338
|
+
# are the fix; each mutant below removes exactly one piece of it.
|
|
339
|
+
("M375a: the default alert floor is zero again (count-only edge: a 20 s restart alerts)",
|
|
340
|
+
" return (args.alert_after - 1) * args.poll_seconds",
|
|
341
|
+
" return 0"),
|
|
342
|
+
("M375b: the fast-path recovery announces `recovered` after a run that never alerted",
|
|
343
|
+
" # count endpoint reachable + no unread increase = a HEALTHY poll with no new items\n if self.fsm_state == \"DOWN\":",
|
|
344
|
+
" # count endpoint reachable + no unread increase = a HEALTHY poll with no new items\n if self.fsm_state == \"DOWN\" or self.failures:"),
|
|
345
|
+
("M375c: a 502's Retry-After is read and thrown away",
|
|
346
|
+
" _RETRY_HINT[\"seconds\"] = _retry_after_seconds(e.headers, body)",
|
|
347
|
+
" pass"),
|
|
348
|
+
("M375d: the backoff ignores the server's retry hint",
|
|
349
|
+
" backoff = max(backoff, hint)",
|
|
350
|
+
" pass"),
|
|
351
|
+
("M375e: the retry hint is not clamped (a hostile Retry-After parks the producer)",
|
|
352
|
+
" return max(0, min(secs, RETRY_AFTER_CAP))",
|
|
353
|
+
" return max(0, secs)"),
|
|
354
|
+
("M375f: a stale retry hint survives into the next, unrelated failure",
|
|
355
|
+
" req = urllib.request.Request(url, headers=headers, method=\"GET\")\n _RETRY_HINT[\"seconds\"] = None\n",
|
|
356
|
+
" req = urllib.request.Request(url, headers=headers, method=\"GET\")\n"),
|
|
335
357
|
]
|
|
336
358
|
def run(src):
|
|
337
359
|
# RELEASE WHAT WE ACQUIRE (Loom re-audit 10, L6). The temp tree was never removed and the source file
|
|
@@ -340,7 +362,12 @@ def run(src):
|
|
|
340
362
|
# both is not merely untidy - it is the harness exhibiting the class it is meant to detect.
|
|
341
363
|
d=tempfile.mkdtemp()
|
|
342
364
|
try:
|
|
343
|
-
|
|
365
|
+
# COPY THE WHOLE TREE, not two files. The suite also reads the README and the supervisor templates
|
|
366
|
+
# (the state-file-name and opaque-output agreement tests), so a two-file copy made the BASELINE red
|
|
367
|
+
# and the gate refused to run at all - a harness that cannot run proves nothing, green or red.
|
|
368
|
+
shutil.copytree(os.path.dirname(SRC), os.path.join(d, "t"),
|
|
369
|
+
ignore=shutil.ignore_patterns(".git", "__pycache__", "node_modules", ".venv"))
|
|
370
|
+
d=os.path.join(d, "t")
|
|
344
371
|
with open(os.path.join(d,"kijito_inbox_monitor.py"),"w") as fh:
|
|
345
372
|
fh.write(src)
|
|
346
373
|
# PIN THE WARNING FILTER. Inherited PYTHONWARNINGS=error turns a mutant's leaked fd into an ERROR,
|
|
@@ -356,7 +383,7 @@ def run(src):
|
|
|
356
383
|
return None, "HUNG"
|
|
357
384
|
return p.returncode, p.stderr
|
|
358
385
|
finally:
|
|
359
|
-
shutil.rmtree(d, ignore_errors=True)
|
|
386
|
+
shutil.rmtree(os.path.dirname(d) if d.endswith(os.sep + "t") else d, ignore_errors=True)
|
|
360
387
|
with open(SRC) as fh: base=fh.read()
|
|
361
388
|
rc,_=run(base)
|
|
362
389
|
if rc!=0: print("BASELINE NOT GREEN"); sys.exit(1)
|
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import datetime
|
|
2
|
+
import email.message
|
|
3
|
+
import email.utils
|
|
1
4
|
import io
|
|
2
5
|
import json
|
|
3
6
|
import os
|
|
@@ -277,6 +280,11 @@ class ValidationGuardTest(unittest.TestCase):
|
|
|
277
280
|
with self.assertRaises(km.FatalConfig):
|
|
278
281
|
km.validate_args(self._args(["--persona", "argus", "--poll-seconds", "0"]))
|
|
279
282
|
|
|
283
|
+
def test_alert_floor_seconds_must_not_be_negative(self):
|
|
284
|
+
with self.assertRaises(km.FatalConfig):
|
|
285
|
+
km.validate_args(self._args(["--persona", "argus", "--alert-floor-seconds", "-1"]))
|
|
286
|
+
km.validate_args(self._args(["--persona", "argus", "--alert-floor-seconds", "0"])) # must not raise
|
|
287
|
+
|
|
280
288
|
def test_seed_at_rejected_in_multipersona(self):
|
|
281
289
|
with self.assertRaises(km.FatalConfig):
|
|
282
290
|
km.validate_args(self._args(["--all-personas", "--seed-at", "5"]))
|
|
@@ -1718,6 +1726,7 @@ class BoundedWindowEndToEndTest(unittest.TestCase):
|
|
|
1718
1726
|
all_personas = False
|
|
1719
1727
|
state_file_template = None
|
|
1720
1728
|
alert_after = 3
|
|
1729
|
+
alert_floor_seconds = None
|
|
1721
1730
|
poll_seconds = 60
|
|
1722
1731
|
heartbeat = 0
|
|
1723
1732
|
max_replay = 50
|
|
@@ -1730,6 +1739,7 @@ class BoundedWindowEndToEndTest(unittest.TestCase):
|
|
|
1730
1739
|
t.persona, t.url, t.headers = "argus", "http://x/api/inbox?persona=argus", {}
|
|
1731
1740
|
t.opener, t.emitter, t.args = None, emitter, self.FullArgs()
|
|
1732
1741
|
t.cursor, t.armed, t.fsm_state, t.failures = cursor, True, "UP", 0
|
|
1742
|
+
t.down_since = None
|
|
1733
1743
|
t.state_file = t.last_unread = None
|
|
1734
1744
|
t.fast_path = False
|
|
1735
1745
|
t.skips = t.first_poll = 0
|
|
@@ -1982,6 +1992,7 @@ class CorruptPinStateTest(unittest.TestCase):
|
|
|
1982
1992
|
t.emitter, t.args = em, BoundedWindowEndToEndTest.FullArgs()
|
|
1983
1993
|
t.args.max_replay = 1
|
|
1984
1994
|
t.cursor, t.armed, t.fsm_state, t.failures = 100, False, "UP", 0
|
|
1995
|
+
t.down_since = None
|
|
1985
1996
|
t.state_file = t.last_unread = None
|
|
1986
1997
|
t.fast_path = False
|
|
1987
1998
|
t.skips = t.first_poll = 0
|
|
@@ -2702,6 +2713,15 @@ class ExecEnvTest(unittest.TestCase):
|
|
|
2702
2713
|
self.assertEqual(env["KIJITOMON_ID"], "41")
|
|
2703
2714
|
self.assertEqual(env["KIJITOMON_FROM"], "river")
|
|
2704
2715
|
|
|
2716
|
+
def test_the_reachability_alert_exports_its_measured_span_and_floor(self):
|
|
2717
|
+
# §7.1 rev 10: `seconds` is a measurement now, so an exec consumer gets it (and the floor it
|
|
2718
|
+
# cleared) instead of having to infer onset from flags it cannot see.
|
|
2719
|
+
env = self._env_for({"event": "alert", "source": "kijito-inbox", "ts": "t", "persona": "argus",
|
|
2720
|
+
"reason": "http 502", "consecutive_failures": 7, "seconds": 61,
|
|
2721
|
+
"floor_seconds": 60})
|
|
2722
|
+
self.assertEqual((env["KIJITOMON_FAILURES"], env["KIJITOMON_SECONDS"], env["KIJITOMON_FLOOR_SECONDS"]),
|
|
2723
|
+
("7", "61", "60"))
|
|
2724
|
+
|
|
2705
2725
|
|
|
2706
2726
|
class WarnOncePerPersonaTest(unittest.TestCase):
|
|
2707
2727
|
def setUp(self):
|
|
@@ -4608,6 +4628,7 @@ class Loom11AlarmDeliveryTest(unittest.TestCase):
|
|
|
4608
4628
|
t.opener, t.emitter = None, emitter
|
|
4609
4629
|
t.args = BoundedWindowEndToEndTest.FullArgs()
|
|
4610
4630
|
t.cursor, t.armed, t.fsm_state, t.failures = cursor, True, fsm, 0
|
|
4631
|
+
t.down_since = None
|
|
4611
4632
|
t.state_file = t.last_unread = None
|
|
4612
4633
|
t.fast_path = False
|
|
4613
4634
|
t.skips = t.first_poll = 0
|
|
@@ -4631,22 +4652,47 @@ class Loom11AlarmDeliveryTest(unittest.TestCase):
|
|
|
4631
4652
|
def _down_fetch(reason="http 502"):
|
|
4632
4653
|
return lambda opener, url, headers: km.Poll(False, reason=reason)
|
|
4633
4654
|
|
|
4655
|
+
def _clock(self, start=1000.0):
|
|
4656
|
+
"""Take over km._monotonic for ONE test; returns the mutable [now]. §7.1 the alert floor is a
|
|
4657
|
+
MEASURED span, so a liveness test has to say WHEN each poll happens, not only how many."""
|
|
4658
|
+
clock = [start]
|
|
4659
|
+
real = km._monotonic
|
|
4660
|
+
km._monotonic = lambda: clock[0]
|
|
4661
|
+
self.addCleanup(lambda: setattr(km, "_monotonic", real))
|
|
4662
|
+
return clock
|
|
4663
|
+
|
|
4664
|
+
def _run_at(self, t, fetch_fn, clock, at):
|
|
4665
|
+
"""poll_once at each monotonic instant in `at` (seconds after the clock's start)."""
|
|
4666
|
+
orig, km.fetch = km.fetch, fetch_fn
|
|
4667
|
+
try:
|
|
4668
|
+
for when in at:
|
|
4669
|
+
clock[0] = 1000.0 + when
|
|
4670
|
+
t.poll_once()
|
|
4671
|
+
finally:
|
|
4672
|
+
km.fetch = orig
|
|
4673
|
+
|
|
4634
4674
|
# ---- F2: the dead-man's switch had NO test. These are it. --------------------------------------
|
|
4635
4675
|
def test_the_liveness_DOWN_alert_IS_EMITTED_after_alert_after_failures(self):
|
|
4636
4676
|
# THE GAP A1 EXPOSED: deleting this alert entirely left all 242 tests green, because nothing
|
|
4637
4677
|
# asserted the one event README sells as the dead-man's switch.
|
|
4638
4678
|
em = self.Recorder()
|
|
4679
|
+
clock = self._clock()
|
|
4639
4680
|
t = self._target(em)
|
|
4640
|
-
|
|
4681
|
+
# §7.1: alert_after failures AT THE CONFIGURED CADENCE (poll_seconds apart) - the count AND the
|
|
4682
|
+
# measured floor ((alert_after - 1) * poll_seconds) are both reached on the third poll.
|
|
4683
|
+
self._run_at(t, self._down_fetch(), clock, at=[0, 60, 120])
|
|
4641
4684
|
alerts = [f for e, f in em.events if e == "alert"]
|
|
4642
4685
|
self.assertEqual(len(alerts), 1, "the source went down and no alert was emitted")
|
|
4643
4686
|
self.assertEqual(alerts[0]["reason"], "http 502")
|
|
4644
4687
|
self.assertEqual(alerts[0]["consecutive_failures"], t.args.alert_after)
|
|
4688
|
+
self.assertEqual(alerts[0]["seconds"], 120) # MEASURED, not failures * poll_seconds
|
|
4689
|
+
self.assertEqual(alerts[0]["floor_seconds"], 120)
|
|
4645
4690
|
self.assertEqual(t.fsm_state, "DOWN")
|
|
4646
4691
|
|
|
4647
4692
|
def test_the_DOWN_alert_does_not_fire_before_the_threshold(self):
|
|
4648
4693
|
em = self.Recorder()
|
|
4649
4694
|
t = self._target(em)
|
|
4695
|
+
t.args.alert_floor_seconds = 0 # floor off, so this test is about the COUNT guard alone
|
|
4650
4696
|
self._run(t, self._down_fetch(), times=t.args.alert_after - 1)
|
|
4651
4697
|
self.assertEqual([f for e, f in em.events if e == "alert"], [])
|
|
4652
4698
|
self.assertEqual(t.fsm_state, "UP")
|
|
@@ -4656,9 +4702,10 @@ class Loom11AlarmDeliveryTest(unittest.TestCase):
|
|
|
4656
4702
|
# edge is crossed exactly once and a reverted transition would never re-fire). So the
|
|
4657
4703
|
# announcement gets the guaranteed second channel instead.
|
|
4658
4704
|
em = self.Recorder(deliver=False)
|
|
4705
|
+
clock = self._clock()
|
|
4659
4706
|
t = self._target(em)
|
|
4660
4707
|
buf = _capture_stderr(self)
|
|
4661
|
-
self.
|
|
4708
|
+
self._run_at(t, self._down_fetch(), clock, at=[0, 60, 120])
|
|
4662
4709
|
self.assertEqual(t.fsm_state, "DOWN")
|
|
4663
4710
|
self.assertIn("UNDELIVERED", buf.getvalue())
|
|
4664
4711
|
self.assertIn("http 502", buf.getvalue())
|
|
@@ -4743,6 +4790,273 @@ def _capture_stderr(test):
|
|
|
4743
4790
|
return buf
|
|
4744
4791
|
|
|
4745
4792
|
|
|
4793
|
+
|
|
4794
|
+
class RetryAfterPacingTest(unittest.TestCase):
|
|
4795
|
+
"""A failed long-poll is paced by the server's own Retry-After hint (row M375, bug #19).
|
|
4796
|
+
|
|
4797
|
+
The edge's restart page says "restarting; retry shortly" and carries `Retry-After` plus a JSON
|
|
4798
|
+
`retry_after_seconds`. The hint only ever LENGTHENS the wait before the next retry, is clamped to
|
|
4799
|
+
RETRY_AFTER_CAP, and never feeds the alert decision - that stays the measured floor's job.
|
|
4800
|
+
"""
|
|
4801
|
+
class H(dict):
|
|
4802
|
+
def get(self, k, d=None):
|
|
4803
|
+
return dict.get(self, k, d)
|
|
4804
|
+
|
|
4805
|
+
def test_header_delta_seconds(self):
|
|
4806
|
+
self.assertEqual(km._retry_after_seconds(self.H({"Retry-After": "15"})), 15)
|
|
4807
|
+
self.assertEqual(km._retry_after_seconds(self.H({"Retry-After": " 0 "})), 0)
|
|
4808
|
+
|
|
4809
|
+
def test_header_http_date(self):
|
|
4810
|
+
when = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(seconds=30)
|
|
4811
|
+
got = km._retry_after_seconds(self.H({"Retry-After": email.utils.format_datetime(when, usegmt=True)}))
|
|
4812
|
+
self.assertTrue(28 <= got <= 31, got)
|
|
4813
|
+
past = email.utils.format_datetime(when - datetime.timedelta(hours=1), usegmt=True)
|
|
4814
|
+
self.assertEqual(km._retry_after_seconds(self.H({"Retry-After": past})), 0) # clamped, never negative
|
|
4815
|
+
|
|
4816
|
+
def test_body_field_when_no_header(self):
|
|
4817
|
+
body = json.dumps({"error": "edge_bad_gateway", "retry_after_seconds": 7.2}).encode()
|
|
4818
|
+
self.assertEqual(km._retry_after_seconds(self.H(), body), 8) # rounded UP, never shorter
|
|
4819
|
+
self.assertEqual(km._retry_after_seconds(self.H({"Retry-After": "3"}), body), 3) # header wins
|
|
4820
|
+
|
|
4821
|
+
def test_garbage_and_absent_hints_are_none(self):
|
|
4822
|
+
for h, b in [(None, None), (self.H(), None), (self.H({"Retry-After": "soon"}), None),
|
|
4823
|
+
(self.H({"Retry-After": "-5"}), None), (self.H(), b"<html>502</html>"),
|
|
4824
|
+
(self.H(), json.dumps({"retry_after_seconds": True}).encode()),
|
|
4825
|
+
(self.H(), json.dumps({"retry_after_seconds": "9"}).encode()),
|
|
4826
|
+
(self.H(), b'{"retry_after_seconds": NaN}'), (self.H(), json.dumps([1]).encode())]:
|
|
4827
|
+
self.assertIsNone(km._retry_after_seconds(h, b), (h, b))
|
|
4828
|
+
|
|
4829
|
+
def test_hint_is_clamped_to_the_cap(self):
|
|
4830
|
+
self.assertEqual(km._retry_after_seconds(self.H({"Retry-After": "86400"})), km.RETRY_AFTER_CAP)
|
|
4831
|
+
self.assertEqual(km._retry_after_seconds(self.H(), b'{"retry_after_seconds": 1e308}'), km.RETRY_AFTER_CAP)
|
|
4832
|
+
|
|
4833
|
+
def test_longpoll_502_records_the_hint_and_keeps_the_cursor(self):
|
|
4834
|
+
body = io.BytesIO(json.dumps({"error": "edge_bad_gateway", "retry_after_seconds": 10}).encode())
|
|
4835
|
+
err = urllib.error.HTTPError(km.NOTIFY_PENDING_URL, 502, "Bad Gateway",
|
|
4836
|
+
email.message.Message(), body)
|
|
4837
|
+
err.headers["Retry-After"] = "12"
|
|
4838
|
+
available, counts, cursor = km.fetch_unread_counts_longpoll(FakeOpener(exc=err), {}, 50, "keep")
|
|
4839
|
+
self.assertEqual((available, counts, cursor), (False, {}, "keep"))
|
|
4840
|
+
self.assertEqual(km._RETRY_HINT["seconds"], 12)
|
|
4841
|
+
# a later failure WITHOUT a hint clears it - a stale hint must never pace an unrelated failure
|
|
4842
|
+
km.fetch_unread_counts_longpoll(FakeOpener(exc=urllib.error.URLError("dropped")), {}, 50, "keep")
|
|
4843
|
+
self.assertIsNone(km._RETRY_HINT["seconds"])
|
|
4844
|
+
|
|
4845
|
+
def test_backoff_never_shorter_than_the_hint_and_exponential_without_one(self):
|
|
4846
|
+
seq, b = [], 0
|
|
4847
|
+
for _ in range(7):
|
|
4848
|
+
b = km.next_longpoll_backoff(b, None)
|
|
4849
|
+
seq.append(b)
|
|
4850
|
+
self.assertEqual(seq, [1, 2, 4, 8, 16, 30, 30])
|
|
4851
|
+
self.assertEqual(km.next_longpoll_backoff(0, 15), 15)
|
|
4852
|
+
self.assertEqual(km.next_longpoll_backoff(15, 15), 30)
|
|
4853
|
+
self.assertEqual(km.next_longpoll_backoff(30, 90), 90)
|
|
4854
|
+
|
|
4855
|
+
|
|
4856
|
+
class MeasuredAlertFloorTest(unittest.TestCase):
|
|
4857
|
+
"""§7.1 THE DEAD-MAN EDGE NEEDS A MEASURED SPAN, NOT ONLY A FAILURE COUNT.
|
|
4858
|
+
|
|
4859
|
+
With `--wait` the loop retries a failed long-poll after 1/2/4 s, so `--alert-after 3` was reached
|
|
4860
|
+
~3 s into ANY outage and every routine server restart - 11 s edge windows included - alerted, while `seconds`
|
|
4861
|
+
reported the nominal 90. The floor is now (alert_after - 1) * poll_seconds of MEASURED time since the
|
|
4862
|
+
first failure (or --alert-floor-seconds), and `seconds` is that measurement.
|
|
4863
|
+
"""
|
|
4864
|
+
# Borrow the liveness fixtures rather than subclass: subclassing re-runs the parent's tests.
|
|
4865
|
+
Recorder = Loom11AlarmDeliveryTest.Recorder
|
|
4866
|
+
_target = Loom11AlarmDeliveryTest._target
|
|
4867
|
+
_down_fetch = staticmethod(Loom11AlarmDeliveryTest._down_fetch)
|
|
4868
|
+
_clock = Loom11AlarmDeliveryTest._clock
|
|
4869
|
+
_run_at = Loom11AlarmDeliveryTest._run_at
|
|
4870
|
+
|
|
4871
|
+
def test_the_count_alone_does_not_cross_the_edge_the_measured_floor_does(self):
|
|
4872
|
+
em = self.Recorder()
|
|
4873
|
+
clock = self._clock()
|
|
4874
|
+
t = self._target(em)
|
|
4875
|
+
# long-poll retry shape: failures at +0, +1, +3 - the count is reached in 3 s
|
|
4876
|
+
self._run_at(t, self._down_fetch(), clock, at=[0, 1, 3])
|
|
4877
|
+
self.assertEqual([f for e, f in em.events if e == "alert"], [], "count-only edge is the defect")
|
|
4878
|
+
self.assertEqual((t.fsm_state, t.failures), ("UP", 3))
|
|
4879
|
+
self._run_at(t, self._down_fetch(), clock, at=[7, 15, 31, 63])
|
|
4880
|
+
self.assertEqual([f for e, f in em.events if e == "alert"], [], "still under the 120 s floor")
|
|
4881
|
+
self._run_at(t, self._down_fetch(), clock, at=[120])
|
|
4882
|
+
alerts = [f for e, f in em.events if e == "alert"]
|
|
4883
|
+
self.assertEqual(len(alerts), 1)
|
|
4884
|
+
self.assertEqual(alerts[0]["consecutive_failures"], 8)
|
|
4885
|
+
self.assertEqual(alerts[0]["seconds"], 120)
|
|
4886
|
+
self.assertEqual(alerts[0]["floor_seconds"], 120)
|
|
4887
|
+
self.assertEqual(t.fsm_state, "DOWN")
|
|
4888
|
+
# edge-once: more failures past the floor never re-alert
|
|
4889
|
+
self._run_at(t, self._down_fetch(), clock, at=[130, 200])
|
|
4890
|
+
self.assertEqual(len([f for e, f in em.events if e == "alert"]), 1)
|
|
4891
|
+
|
|
4892
|
+
def test_a_restart_bracket_under_the_floor_is_silent_and_leaves_no_debt(self):
|
|
4893
|
+
em = self.Recorder()
|
|
4894
|
+
clock = self._clock()
|
|
4895
|
+
t = self._target(em)
|
|
4896
|
+
self._run_at(t, self._down_fetch(), clock, at=[0, 1, 3, 7])
|
|
4897
|
+
self._run_at(t, BoundedWindowEndToEndTest()._fetch([{"id": 101}], 0), clock, at=[12])
|
|
4898
|
+
self.assertEqual([e for e, f in em.events if e in ("alert", "recovered")], [],
|
|
4899
|
+
"an 11 s bracket must produce neither edge")
|
|
4900
|
+
self.assertEqual((t.fsm_state, t.failures, t.down_since), ("UP", 0, None))
|
|
4901
|
+
self.assertEqual(em.new_ids, [101]) # and the mail from the healthy poll still flows
|
|
4902
|
+
|
|
4903
|
+
def test_bug19_a_20s_restart_of_502s_raises_zero_wake_events(self):
|
|
4904
|
+
# Row M375 / bug #19: a ~20 s server restart answered every poll with 502 while the producer
|
|
4905
|
+
# retried at its long-poll backoff (1, 2, 4, 8 s). Before the floor that was alert + recovered -
|
|
4906
|
+
# two wakes, no mail. Now: nothing at all, and the mail after the restart still flows.
|
|
4907
|
+
em = self.Recorder()
|
|
4908
|
+
clock = self._clock()
|
|
4909
|
+
t = self._target(em)
|
|
4910
|
+
self._run_at(t, self._down_fetch(), clock, at=[0, 1, 3, 7, 15])
|
|
4911
|
+
self._run_at(t, BoundedWindowEndToEndTest()._fetch([{"id": 101}], 0), clock, at=[20])
|
|
4912
|
+
wake = [e for e, f in em.events if km._wake_class(e) == km.WAKE_CLASS_DIAGNOSTIC]
|
|
4913
|
+
self.assertEqual(wake, [], "a 20 s restart must wake nobody")
|
|
4914
|
+
self.assertEqual((t.fsm_state, t.failures, t.down_since), ("UP", 0, None))
|
|
4915
|
+
self.assertEqual(em.new_ids, [101])
|
|
4916
|
+
|
|
4917
|
+
def test_bug19_a_20s_restart_recovering_on_the_count_fast_path_is_silent_too(self):
|
|
4918
|
+
# The same restart, but the first healthy tick is the /api/notify/pending fast path (no unread
|
|
4919
|
+
# increase, so the full inbox poll is skipped) - the path a long-polling producer usually
|
|
4920
|
+
# recovers through. It must not announce a recovery nobody was told about.
|
|
4921
|
+
em = self.Recorder()
|
|
4922
|
+
clock = self._clock()
|
|
4923
|
+
t = self._target(em)
|
|
4924
|
+
t.args.no_fast_path = False
|
|
4925
|
+
t.fast_path, t.last_unread = True, 0
|
|
4926
|
+
self._run_at(t, self._down_fetch(), clock, at=[0, 1, 3, 7, 15])
|
|
4927
|
+
self.assertEqual(t.failures, 5)
|
|
4928
|
+
clock[0] = 1000.0 + 20
|
|
4929
|
+
t.poll_once(counts_available=True, unread_counts={"argus": 0})
|
|
4930
|
+
self.assertEqual([e for e, f in em.events if km._wake_class(e) == km.WAKE_CLASS_DIAGNOSTIC], [])
|
|
4931
|
+
self.assertEqual((t.fsm_state, t.failures, t.down_since), ("UP", 0, None))
|
|
4932
|
+
|
|
4933
|
+
def test_an_outage_past_the_floor_raises_exactly_one_alert_then_one_recovered(self):
|
|
4934
|
+
em = self.Recorder()
|
|
4935
|
+
clock = self._clock()
|
|
4936
|
+
t = self._target(em)
|
|
4937
|
+
self._run_at(t, self._down_fetch(), clock, at=[0, 1, 3, 7, 15, 31, 61, 91, 121, 151])
|
|
4938
|
+
self._run_at(t, BoundedWindowEndToEndTest()._fetch([{"id": 101}], 0), clock, at=[160, 170])
|
|
4939
|
+
self.assertEqual([e for e, f in em.events if e in ("alert", "recovered")], ["alert", "recovered"])
|
|
4940
|
+
self.assertEqual(t.fsm_state, "UP")
|
|
4941
|
+
|
|
4942
|
+
def test_plain_poll_cadence_alerts_on_the_Nth_failure_exactly_as_before(self):
|
|
4943
|
+
em = self.Recorder()
|
|
4944
|
+
clock = self._clock()
|
|
4945
|
+
t = self._target(em)
|
|
4946
|
+
self._run_at(t, self._down_fetch(), clock, at=[0, 60])
|
|
4947
|
+
self.assertEqual([f for e, f in em.events if e == "alert"], [])
|
|
4948
|
+
self._run_at(t, self._down_fetch(), clock, at=[120])
|
|
4949
|
+
alerts = [f for e, f in em.events if e == "alert"]
|
|
4950
|
+
self.assertEqual([a["consecutive_failures"] for a in alerts], [3])
|
|
4951
|
+
|
|
4952
|
+
def test_alert_floor_seconds_override_is_honoured_and_zero_restores_count_only(self):
|
|
4953
|
+
em = self.Recorder()
|
|
4954
|
+
clock = self._clock()
|
|
4955
|
+
t = self._target(em)
|
|
4956
|
+
t.args.alert_floor_seconds = 5
|
|
4957
|
+
self._run_at(t, self._down_fetch(), clock, at=[0, 1, 3])
|
|
4958
|
+
self.assertEqual([f for e, f in em.events if e == "alert"], [])
|
|
4959
|
+
self._run_at(t, self._down_fetch(), clock, at=[6])
|
|
4960
|
+
alerts = [f for e, f in em.events if e == "alert"]
|
|
4961
|
+
self.assertEqual((alerts[0]["seconds"], alerts[0]["floor_seconds"]), (6, 5))
|
|
4962
|
+
# 0 = the pre-fix count-only edge, for anyone who wants it back
|
|
4963
|
+
em2 = self.Recorder()
|
|
4964
|
+
t2 = self._target(em2)
|
|
4965
|
+
t2.args.alert_floor_seconds = 0
|
|
4966
|
+
self._run_at(t2, self._down_fetch(), clock, at=[300, 301, 303])
|
|
4967
|
+
self.assertEqual([f["consecutive_failures"] for e, f in em2.events if e == "alert"], [3])
|
|
4968
|
+
|
|
4969
|
+
def test_a_run_restored_by_WatchTarget_init_resumes_the_measured_span(self):
|
|
4970
|
+
# EXERCISES the restore path in __init__ (not a restatement of it - the review found that
|
|
4971
|
+
# deleting the block, or flipping its sign, left the suite green when the fixture hand-set the
|
|
4972
|
+
# attributes). A supervisor restart mid-outage rebuilds the target from the state file: the
|
|
4973
|
+
# persisted wall stamp says the run began 100 s ago, so the floor (120 s) is reached 20 s after
|
|
4974
|
+
# the restart, not 120 s after it.
|
|
4975
|
+
d = tempfile.mkdtemp()
|
|
4976
|
+
self.addCleanup(lambda: __import__("shutil").rmtree(d, ignore_errors=True))
|
|
4977
|
+
base = os.path.join(d, "state.json")
|
|
4978
|
+
url = "http://x/api/inbox?persona=argus"
|
|
4979
|
+
clock = self._clock()
|
|
4980
|
+
km.StateFile(km._state_path_for_persona(base, "argus"), km.canonical_identity(url)).save(
|
|
4981
|
+
100, "UP", 2, down_since=time.time() - 100.0)
|
|
4982
|
+
|
|
4983
|
+
class A(BoundedWindowEndToEndTest.FullArgs):
|
|
4984
|
+
state_file = base
|
|
4985
|
+
seed_at = None
|
|
4986
|
+
em = self.Recorder()
|
|
4987
|
+
t = km.WatchTarget("argus", url, None, {}, A(), em)
|
|
4988
|
+
self.addCleanup(t.state_file.unlock)
|
|
4989
|
+
self.assertEqual(t.failures, 2)
|
|
4990
|
+
self.assertAlmostEqual(t.down_since[0], 1000.0 - 100.0, delta=2.0) # mapped onto the patched clock
|
|
4991
|
+
self._run_at(t, self._down_fetch(), clock, at=[0]) # span ~100 < 120
|
|
4992
|
+
self.assertEqual([f for e, f in em.events if e == "alert"], [])
|
|
4993
|
+
self._run_at(t, self._down_fetch(), clock, at=[21]) # span ~121 >= 120
|
|
4994
|
+
alerts = [f for e, f in em.events if e == "alert"]
|
|
4995
|
+
self.assertEqual(alerts[0]["consecutive_failures"], 4)
|
|
4996
|
+
self.assertGreaterEqual(alerts[0]["seconds"], 120)
|
|
4997
|
+
# and the healthy poll that follows clears the persisted stamp
|
|
4998
|
+
self._run_at(t, BoundedWindowEndToEndTest()._fetch([{"id": 101}], 0), clock, at=[30])
|
|
4999
|
+
with open(km._state_path_for_persona(base, "argus")) as f:
|
|
5000
|
+
self.assertNotIn("down_since", json.load(f))
|
|
5001
|
+
|
|
5002
|
+
def test_an_older_state_file_without_the_stamp_measures_from_the_restart(self):
|
|
5003
|
+
d = tempfile.mkdtemp()
|
|
5004
|
+
self.addCleanup(lambda: __import__("shutil").rmtree(d, ignore_errors=True))
|
|
5005
|
+
base = os.path.join(d, "state.json")
|
|
5006
|
+
url = "http://x/api/inbox?persona=argus"
|
|
5007
|
+
clock = self._clock()
|
|
5008
|
+
km.StateFile(km._state_path_for_persona(base, "argus"), km.canonical_identity(url)).save(100, "UP", 2)
|
|
5009
|
+
|
|
5010
|
+
class A(BoundedWindowEndToEndTest.FullArgs):
|
|
5011
|
+
state_file = base
|
|
5012
|
+
seed_at = None
|
|
5013
|
+
em = self.Recorder()
|
|
5014
|
+
t = km.WatchTarget("argus", url, None, {}, A(), em)
|
|
5015
|
+
self.addCleanup(t.state_file.unlock)
|
|
5016
|
+
self.assertEqual(t.failures, 2)
|
|
5017
|
+
self.assertAlmostEqual(t.down_since[0], 1000.0, delta=2.0) # the run starts NOW
|
|
5018
|
+
self._run_at(t, self._down_fetch(), clock, at=[0, 60, 119])
|
|
5019
|
+
self.assertEqual([f for e, f in em.events if e == "alert"], [], "conservative: never earlier")
|
|
5020
|
+
self._run_at(t, self._down_fetch(), clock, at=[120])
|
|
5021
|
+
self.assertEqual(len([f for e, f in em.events if e == "alert"]), 1)
|
|
5022
|
+
|
|
5023
|
+
def test_the_first_failure_stamp_round_trips_through_the_state_file_and_is_read_strictly(self):
|
|
5024
|
+
d = tempfile.mkdtemp()
|
|
5025
|
+
self.addCleanup(lambda: __import__("shutil").rmtree(d, ignore_errors=True))
|
|
5026
|
+
path = os.path.join(d, "s.json")
|
|
5027
|
+
sf = km.StateFile(path, "idx")
|
|
5028
|
+
self.assertTrue(sf.save(100, "UP", 2, down_since=1234.5))
|
|
5029
|
+
self.assertEqual(sf.load()["down_since"], 1234.5)
|
|
5030
|
+
self.assertTrue(sf.save(100, "UP", 0)) # healthy: the stamp is not written
|
|
5031
|
+
with open(path) as f:
|
|
5032
|
+
self.assertNotIn("down_since", json.load(f))
|
|
5033
|
+
self.assertIsNone(sf.load()["down_since"])
|
|
5034
|
+
with open(path, "w") as f: # a non-number is not a timestamp
|
|
5035
|
+
json.dump({"identity": "idx", "cursor": 100, "state": "UP", "consecutive_failures": 2,
|
|
5036
|
+
"down_since": "yesterday"}, f)
|
|
5037
|
+
_capture_stderr(self)
|
|
5038
|
+
self.assertIs(sf.load(), km.CORRUPT_STATE)
|
|
5039
|
+
with open(path, "w") as f: # and neither is a bool
|
|
5040
|
+
json.dump({"identity": "idx", "cursor": 100, "state": "UP", "consecutive_failures": 2,
|
|
5041
|
+
"down_since": True}, f)
|
|
5042
|
+
self.assertIs(sf.load(), km.CORRUPT_STATE)
|
|
5043
|
+
# json accepts every one of these; the 400-digit int is the one math.isfinite itself chokes on
|
|
5044
|
+
# (OverflowError, not False), and 0/-0.0 are finite but no stamp - a run cannot have begun at epoch 0.
|
|
5045
|
+
for bad in ("-Infinity", "Infinity", "NaN", "-5", "1e400", "1" + "0" * 400, "0", "-0.0"):
|
|
5046
|
+
with open(path, "w") as f:
|
|
5047
|
+
f.write('{"identity": "idx", "cursor": 100, "state": "UP", "consecutive_failures": 2, '
|
|
5048
|
+
'"down_since": %s}' % bad)
|
|
5049
|
+
self.assertIs(sf.load(), km.CORRUPT_STATE, bad) # an infinite span would raise at the edge
|
|
5050
|
+
|
|
5051
|
+
def test_the_default_floor_is_derived_from_the_two_flags(self):
|
|
5052
|
+
class A:
|
|
5053
|
+
alert_after, poll_seconds, alert_floor_seconds = 3, 30, None
|
|
5054
|
+
self.assertEqual(km.alert_floor_seconds(A()), 60) # the VM units: 3 polls at 30 s
|
|
5055
|
+
A.alert_floor_seconds = 7
|
|
5056
|
+
self.assertEqual(km.alert_floor_seconds(A()), 7)
|
|
5057
|
+
A.alert_after, A.alert_floor_seconds = 1, None
|
|
5058
|
+
self.assertEqual(km.alert_floor_seconds(A()), 0) # --alert-after 1 keeps its meaning
|
|
5059
|
+
|
|
4746
5060
|
class Loom10ClassSweepTest(unittest.TestCase):
|
|
4747
5061
|
"""Loom re-audit 10 - the CLASS, swept rather than patched one finding at a time.
|
|
4748
5062
|
|
|
@@ -6274,3 +6588,83 @@ class PersistedUnreadCountTest(unittest.TestCase):
|
|
|
6274
6588
|
|
|
6275
6589
|
def test_zero_loads(self):
|
|
6276
6590
|
self.assertEqual(self._load({"unread": 0})["unread"], 0)
|
|
6591
|
+
|
|
6592
|
+
|
|
6593
|
+
class WindowsNativeTest(unittest.TestCase):
|
|
6594
|
+
"""praetor's report (Windows 11 native, CPython 3.14, 2026-09-26): the producer crashed at startup on
|
|
6595
|
+
`os.geteuid` (POSIX-only) in _assert_private_fd, and `--help` crashed on a cp1252 console. The next two
|
|
6596
|
+
walls after that were `_fsync_dir` (Windows cannot open a directory to fsync it, so the events sink held
|
|
6597
|
+
its cursor forever on a new file) and a spurious "writable by other users" warning (Windows reports
|
|
6598
|
+
every directory as 0777). Windows is SIMULATED here: IS_POSIX False and os.geteuid removed."""
|
|
6599
|
+
|
|
6600
|
+
def setUp(self):
|
|
6601
|
+
self._d = tempfile.TemporaryDirectory()
|
|
6602
|
+
self.addCleanup(self._d.cleanup)
|
|
6603
|
+
self._posix, km.IS_POSIX = km.IS_POSIX, False
|
|
6604
|
+
self.addCleanup(setattr, km, "IS_POSIX", self._posix)
|
|
6605
|
+
self._geteuid = getattr(os, "geteuid", None)
|
|
6606
|
+
if self._geteuid is not None:
|
|
6607
|
+
del os.geteuid
|
|
6608
|
+
self.addCleanup(setattr, os, "geteuid", self._geteuid)
|
|
6609
|
+
|
|
6610
|
+
def test_a_regular_file_passes_without_geteuid_and_is_not_chmodded(self):
|
|
6611
|
+
p = os.path.join(self._d.name, "events.jsonl")
|
|
6612
|
+
with open(p, "w"):
|
|
6613
|
+
pass
|
|
6614
|
+
os.chmod(p, 0o644)
|
|
6615
|
+
fd = os.open(p, os.O_RDONLY)
|
|
6616
|
+
try:
|
|
6617
|
+
km._assert_private_fd(fd, p) # crashed with AttributeError before the fix
|
|
6618
|
+
finally:
|
|
6619
|
+
os.close(fd)
|
|
6620
|
+
self.assertEqual(os.stat(p).st_mode & 0o777, 0o644, "no POSIX mode repair is attempted off POSIX")
|
|
6621
|
+
|
|
6622
|
+
def test_a_non_regular_file_is_still_refused(self):
|
|
6623
|
+
fd = os.open(self._d.name, os.O_RDONLY)
|
|
6624
|
+
try:
|
|
6625
|
+
with self.assertRaises(km.InsecureFile):
|
|
6626
|
+
km._assert_private_fd(fd, self._d.name)
|
|
6627
|
+
finally:
|
|
6628
|
+
os.close(fd)
|
|
6629
|
+
|
|
6630
|
+
def test_fsync_dir_reports_success(self):
|
|
6631
|
+
self.assertTrue(km._fsync_dir(os.path.join(self._d.name, "no-such-dir")))
|
|
6632
|
+
|
|
6633
|
+
def test_no_writable_directory_warning_off_posix(self):
|
|
6634
|
+
d = os.path.join(self._d.name, "open")
|
|
6635
|
+
os.mkdir(d)
|
|
6636
|
+
os.chmod(d, 0o777)
|
|
6637
|
+
buf = _capture_stderr(self)
|
|
6638
|
+
km._makedirs_private(os.path.join(d, "sub"))
|
|
6639
|
+
self.assertNotIn("writable by other local users", buf.getvalue())
|
|
6640
|
+
|
|
6641
|
+
def test_the_events_file_sink_writes_and_syncs(self):
|
|
6642
|
+
# End to end: before the fix a new events file made sync() return False on every poll (the
|
|
6643
|
+
# directory fsync), so the cursor never advanced and mail was re-delivered forever.
|
|
6644
|
+
sink = km.RotatingFileSink(os.path.join(self._d.name, "argus.jsonl"), 0, 3)
|
|
6645
|
+
self.addCleanup(sink.close)
|
|
6646
|
+
self.assertTrue(sink.write('{"event": "new", "content": "caf\\u00e9 \\u2713"}\n'))
|
|
6647
|
+
self.assertTrue(sink.sync())
|
|
6648
|
+
|
|
6649
|
+
|
|
6650
|
+
class Utf8StdoutTest(unittest.TestCase):
|
|
6651
|
+
"""A cp1252 stdout (a Windows console, a Git Bash pipe) must neither crash `--help` nor an event whose
|
|
6652
|
+
message body is outside cp1252. Real subprocesses, because the defect is the interpreter's own encoding."""
|
|
6653
|
+
|
|
6654
|
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
6655
|
+
|
|
6656
|
+
def _run(self, code):
|
|
6657
|
+
env = dict(os.environ, PYTHONIOENCODING="cp1252")
|
|
6658
|
+
env.pop("PYTHONUTF8", None)
|
|
6659
|
+
return subprocess.run([sys.executable, "-c", code], cwd=self.HERE, env=env, capture_output=True, timeout=60)
|
|
6660
|
+
|
|
6661
|
+
def test_help_does_not_crash_on_cp1252(self):
|
|
6662
|
+
r = self._run("import kijito_inbox_monitor as km, sys\ntry:\n km.main(['--help'])\nexcept SystemExit as e:\n sys.exit(e.code)")
|
|
6663
|
+
self.assertEqual(r.returncode, 0, r.stderr.decode("utf-8", "replace")[-300:])
|
|
6664
|
+
|
|
6665
|
+
def test_an_event_outside_cp1252_is_delivered_as_utf8(self):
|
|
6666
|
+
r = self._run("import kijito_inbox_monitor as km\nkm._utf8_stdout()\n"
|
|
6667
|
+
"em = km.Emitter('stdout-jsonl', None, 220, False)\n"
|
|
6668
|
+
"assert em.lifecycle('armed', persona='\\u6f22\\U0001F600') is True")
|
|
6669
|
+
self.assertEqual(r.returncode, 0, r.stderr.decode("utf-8", "replace")[-300:])
|
|
6670
|
+
self.assertIn("\u6f22\U0001F600", r.stdout.decode("utf-8"))
|