cctally 1.112.0 → 1.114.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/CHANGELOG.md +27 -0
  2. package/README.md +4 -5
  3. package/bin/_cctally_cache.py +1683 -485
  4. package/bin/_cctally_core.py +223 -1
  5. package/bin/_cctally_dashboard.py +96 -18
  6. package/bin/_cctally_dashboard_perf.py +128 -9
  7. package/bin/_cctally_dashboard_sources.py +267 -204
  8. package/bin/_cctally_db.py +861 -18
  9. package/bin/_cctally_diagnosis_sources.py +22 -4
  10. package/bin/_cctally_doctor.py +136 -18
  11. package/bin/_cctally_five_hour.py +2 -2
  12. package/bin/_cctally_forecast.py +27 -6
  13. package/bin/_cctally_journal.py +67 -5
  14. package/bin/_cctally_milestone_history.py +82 -9
  15. package/bin/_cctally_percent_breakdown.py +3 -2
  16. package/bin/_cctally_quota.py +201 -33
  17. package/bin/_cctally_record.py +14 -0
  18. package/bin/_cctally_rederive.py +23 -9
  19. package/bin/_cctally_source_analytics.py +60 -15
  20. package/bin/_cctally_statusline.py +17 -0
  21. package/bin/_cctally_store.py +80 -5
  22. package/bin/_cctally_tui.py +4 -3
  23. package/bin/_cctally_weekrefs.py +128 -21
  24. package/bin/_lib_cache_report.py +35 -6
  25. package/bin/_lib_codex_conversation.py +32 -0
  26. package/bin/_lib_codex_conversation_query.py +164 -30
  27. package/bin/_lib_conversation_query.py +95 -56
  28. package/bin/_lib_conversation_retention.py +1605 -640
  29. package/bin/_lib_cost_provenance.py +53 -2
  30. package/bin/_lib_doctor.py +263 -69
  31. package/bin/_lib_fmt.py +11 -0
  32. package/bin/_lib_merge_gate.py +469 -51
  33. package/bin/_lib_perf.py +27 -1
  34. package/bin/_lib_pricing.py +209 -8
  35. package/bin/_lib_pricing_check.py +20 -4
  36. package/bin/_lib_reclaim_planner.py +792 -0
  37. package/bin/_lib_rederive.py +225 -77
  38. package/bin/_lib_snapshot_cache.py +21 -0
  39. package/bin/_lib_sqlite_close.py +81 -0
  40. package/bin/_lib_subscription_weeks.py +100 -17
  41. package/bin/_lib_tick_stats.py +217 -17
  42. package/bin/_lib_wal_checkpoint.py +1077 -0
  43. package/bin/_lib_write_budget.py +483 -0
  44. package/bin/_lib_write_io.py +703 -0
  45. package/bin/cctally +8 -0
  46. package/dashboard/static/assets/{ConversationsView-CEZ-BUyi.js → ConversationsView-Bc8LCOAZ.js} +1 -1
  47. package/dashboard/static/assets/{DoctorModal-CE1oH_Zy.js → DoctorModal-D--qVBwe.js} +1 -1
  48. package/dashboard/static/assets/ModalRoot-DLPgyAM_.js +1 -0
  49. package/dashboard/static/assets/{ProjectsDrillPanel-B3DZoCeD.js → ProjectsDrillPanel-z53a4v4M.js} +1 -1
  50. package/dashboard/static/assets/{SourceDetailModal-CHp9EgLP.js → SourceDetailModal-Bw0T8W-W.js} +1 -1
  51. package/dashboard/static/assets/{UpdateModal-5iKST6jo.js → UpdateModal-D5AKx4Yw.js} +1 -1
  52. package/dashboard/static/assets/index-BKITk1CT.css +1 -0
  53. package/dashboard/static/assets/{index-Cm0VTxJC.js → index-BXDOzSng.js} +3 -3
  54. package/dashboard/static/assets/{outlineNavigation-DkXGL8J6.js → outlineNavigation-JObY7cUY.js} +1 -1
  55. package/dashboard/static/dashboard.html +2 -2
  56. package/package.json +6 -1
  57. package/dashboard/static/assets/ModalRoot-D5Y8IFUG.js +0 -1
  58. package/dashboard/static/assets/index-Dt4tqf43.css +0 -1
@@ -0,0 +1,703 @@
1
+ """Process disk-write telemetry for the dashboard (#901 spec §5.5).
2
+
3
+ Stdlib-only leaf. It reads ONE kernel-accounted process write counter —
4
+ macOS `proc_pid_rusage(RUSAGE_INFO_V2).ri_diskio_byteswritten` through a
5
+ correctly declared ctypes ABI, Linux `/proc/self/io` `write_bytes` — and never
6
+ substitutes zero, file growth or `ru_oublock` for a counter it could not read:
7
+ an unreadable value is `None` with a reason from `UNAVAILABLE_REASONS`. These
8
+ are process writes, not NAND writes.
9
+
10
+ One cumulative timeline is sampled at tick, conversation-pass and maintenance
11
+ boundaries. An interval's delta includes whatever ran concurrently; it carries
12
+ overlap metadata and is never summed across loops. Sustained rates come only
13
+ from the timeline, through `_lib_write_budget`.
14
+
15
+ It also owns the dashboard instance descriptor: an owner-readable file written
16
+ once at startup and removed at clean shutdown, through which `cctally doctor`
17
+ discovers a running instance. It is endpoint discovery, never a measurement.
18
+ """
19
+ from __future__ import annotations
20
+
21
+ import array
22
+ import dataclasses
23
+ import datetime as dt
24
+ import ipaddress
25
+ import json
26
+ import os
27
+ import pathlib
28
+ import re
29
+ import secrets
30
+ import sys
31
+ import threading
32
+ import time
33
+
34
+ import _lib_write_budget as _budget
35
+
36
+ UTC = dt.timezone.utc
37
+ UNAVAILABLE_REASONS = ("unsupported_platform", "counter_error", "counter_reset")
38
+ WRITE_STATUSES = ("ok",) + UNAVAILABLE_REASONS + ("not_sampled",)
39
+ SOURCE_DARWIN = "proc_pid_rusage.ri_diskio_byteswritten"
40
+ SOURCE_LINUX = "proc.io.write_bytes"
41
+ SCOPE = "process"
42
+ TIMELINE_CAPACITY = 96
43
+ TIMELINE_BUCKET_NS = 5 * 1_000_000_000
44
+ EXCLUSION_CAPACITY = 16
45
+ #: The hysteresis advances at most this often; far below its 60 s spacing.
46
+ HYSTERESIS_EVALUATION_SPACING_NS = 10 * 1_000_000_000
47
+ DESCRIPTOR_SCHEMA_VERSION = 1
48
+ DESCRIPTOR_DIRNAME = "run"
49
+ DESCRIPTOR_PREFIX = "dashboard-"
50
+ _INSTANCE_ID_RE = re.compile(r"^[0-9a-f]{16}$")
51
+
52
+ _RUSAGE_INFO_V2 = 2
53
+ #: `struct rusage_info_v2` (<sys/resource.h>): a 16-byte uuid then eighteen
54
+ #: uint64 fields, in this order.
55
+ _RUSAGE_V2_FIELDS = (
56
+ "ri_user_time", "ri_system_time", "ri_pkg_idle_wkups",
57
+ "ri_interrupt_wkups", "ri_pageins", "ri_wired_size", "ri_resident_size",
58
+ "ri_phys_footprint", "ri_proc_start_abstime", "ri_proc_exit_abstime",
59
+ "ri_child_user_time", "ri_child_system_time", "ri_child_pkg_idle_wkups",
60
+ "ri_child_interrupt_wkups", "ri_child_pageins",
61
+ "ri_child_elapsed_abstime", "ri_diskio_bytesread",
62
+ "ri_diskio_byteswritten",
63
+ )
64
+ _RUSAGE_TYPE = None
65
+ _PROC_PID_RUSAGE = None
66
+
67
+
68
+ def _rusage_type():
69
+ """ctypes is imported lazily: this leaf is reachable from hook paths."""
70
+ global _RUSAGE_TYPE
71
+ if _RUSAGE_TYPE is None:
72
+ import ctypes
73
+
74
+ class RusageInfoV2(ctypes.Structure):
75
+ _fields_ = [("ri_uuid", ctypes.c_uint8 * 16)] + [
76
+ (name, ctypes.c_uint64) for name in _RUSAGE_V2_FIELDS]
77
+
78
+ _RUSAGE_TYPE = RusageInfoV2
79
+ return _RUSAGE_TYPE
80
+
81
+
82
+ def _proc_pid_rusage():
83
+ global _PROC_PID_RUSAGE
84
+ if _PROC_PID_RUSAGE is None:
85
+ import ctypes
86
+
87
+ lib = ctypes.CDLL("/usr/lib/libSystem.B.dylib", use_errno=True)
88
+ fn = lib.proc_pid_rusage
89
+ fn.argtypes = (ctypes.c_int, ctypes.c_int, ctypes.c_void_p)
90
+ fn.restype = ctypes.c_int
91
+ _PROC_PID_RUSAGE = fn
92
+ return _PROC_PID_RUSAGE
93
+
94
+
95
+ def darwin_rusage(pid):
96
+ import ctypes
97
+
98
+ info = _rusage_type()()
99
+ rc = _proc_pid_rusage()(int(pid), _RUSAGE_INFO_V2, ctypes.byref(info))
100
+ if rc != 0:
101
+ raise OSError(ctypes.get_errno(), f"proc_pid_rusage({pid}) failed")
102
+ return info
103
+
104
+
105
+ def darwin_write_bytes(pid=None) -> int:
106
+ target = os.getpid() if pid is None else int(pid)
107
+ return int(darwin_rusage(target).ri_diskio_byteswritten)
108
+
109
+
110
+ def linux_write_bytes(pid=None, *, opener=open) -> int:
111
+ path = "/proc/self/io" if pid is None else f"/proc/{int(pid)}/io"
112
+ with opener(path, "r", encoding="ascii") as fh:
113
+ for line in fh:
114
+ key, _, value = line.partition(":")
115
+ if key.strip() == "write_bytes":
116
+ return int(value.strip())
117
+ raise OSError(f"write_bytes missing from {path}")
118
+
119
+
120
+ @dataclasses.dataclass(frozen=True)
121
+ class CounterReading:
122
+ value: "int | None"
123
+ status: str
124
+ source: "str | None"
125
+
126
+
127
+ class ProcessWriteCounter:
128
+ """One process's cumulative write counter, with reset detection."""
129
+
130
+ def __init__(self, *, platform=None, pid=None, darwin_reader=None,
131
+ linux_reader=None):
132
+ self._platform = sys.platform if platform is None else platform
133
+ self._pid = pid
134
+ self._darwin = darwin_reader or darwin_write_bytes
135
+ self._linux = linux_reader or linux_write_bytes
136
+ self._last = None
137
+ self._lock = threading.Lock()
138
+
139
+ @property
140
+ def source(self) -> "str | None":
141
+ if self._platform == "darwin":
142
+ return SOURCE_DARWIN
143
+ if self._platform.startswith("linux"):
144
+ return SOURCE_LINUX
145
+ return None
146
+
147
+ def read(self) -> CounterReading:
148
+ source = self.source
149
+ if source is None:
150
+ return CounterReading(None, "unsupported_platform", None)
151
+ reader = self._darwin if source == SOURCE_DARWIN else self._linux
152
+ # #901 Amendment 19 PR-2: the observation and its comparison with the
153
+ # previous one are ONE step. With the reader outside the lock, a thread
154
+ # could observe 100, lose the race to store it to a thread that then
155
+ # observed and stored 110, and report a false `counter_reset` for a
156
+ # counter that only grew.
157
+ with self._lock:
158
+ try:
159
+ value = int(reader(self._pid))
160
+ except FileNotFoundError:
161
+ return CounterReading(None, "unsupported_platform", source)
162
+ except (OSError, ValueError, TypeError, AttributeError):
163
+ return CounterReading(None, "counter_error", source)
164
+ if value < 0:
165
+ return CounterReading(None, "counter_error", source)
166
+ last, self._last = self._last, value
167
+ if last is not None and value < last:
168
+ return CounterReading(None, "counter_reset", source)
169
+ return CounterReading(value, "ok", source)
170
+
171
+
172
+ def _read_proc_stat(pid) -> str:
173
+ with open(f"/proc/{int(pid)}/stat", "r", encoding="ascii",
174
+ errors="replace") as fh:
175
+ return fh.read()
176
+
177
+
178
+ def process_start_identity(pid, *, platform=None, darwin_rusage_fn=None,
179
+ proc_stat_reader=None) -> "str | None":
180
+ """A value that changes when `pid` is reused; None when unknowable."""
181
+ platform = sys.platform if platform is None else platform
182
+ try:
183
+ if platform == "darwin":
184
+ info = (darwin_rusage_fn or darwin_rusage)(pid)
185
+ return f"darwin:{int(info.ri_proc_start_abstime)}"
186
+ if platform.startswith("linux"):
187
+ text = (proc_stat_reader or _read_proc_stat)(pid)
188
+ fields = text.rsplit(")", 1)[1].split()
189
+ # Field 22 (starttime) is index 19 after the `(comm)` field.
190
+ return f"linux:{int(fields[19])}"
191
+ except (OSError, ValueError, IndexError, AttributeError):
192
+ return None
193
+ return None
194
+
195
+
196
+ @dataclasses.dataclass(frozen=True)
197
+ class Interval:
198
+ token: int
199
+ kind: str
200
+ thread: int
201
+ started_ns: int
202
+ start_bytes: "int | None"
203
+ start_status: str
204
+
205
+
206
+ @dataclasses.dataclass(frozen=True)
207
+ class IntervalObservation:
208
+ kind: str
209
+ started_ns: int
210
+ ended_ns: int
211
+ start_bytes: "int | None"
212
+ end_bytes: "int | None"
213
+ process_write_bytes: "int | None"
214
+ status: str
215
+ overlap: bool
216
+
217
+
218
+ class _State:
219
+ def __init__(self, counter=None, clock_ns=None, wall_s=None):
220
+ self.counter = counter or ProcessWriteCounter()
221
+ self.clock_ns = clock_ns or time.monotonic_ns
222
+ self.wall_s = wall_s or time.time
223
+ self.t = array.array("q")
224
+ self.b = array.array("q")
225
+ self.p = array.array("q")
226
+ self.ex = tuple(array.array("q") for _ in range(6))
227
+ self.ledger = array.array("q")
228
+ self.ledger_observed = False
229
+ self.publications = 0
230
+ self.warm_admitted_ns = None
231
+ self.cold_startup_bytes = None
232
+ self.last_status = "not_sampled"
233
+ self.open = {}
234
+ self.next_token = 1
235
+ self.hysteresis = _budget.HysteresisState()
236
+ self.last_hysteresis_ns = None
237
+ self.local_instance = None
238
+ # #901 Q9: the latest reclaim attempt the planner refused, in memory
239
+ # only (a refusal writes nothing durable); None after a chunk runs.
240
+ self.reclaim_refusal = None
241
+
242
+
243
+ _LOCK = threading.Lock()
244
+ _STATE = _State()
245
+
246
+
247
+ def _counter_lock(state) -> "threading.Lock | None":
248
+ return getattr(state.counter, "_lock", None)
249
+
250
+
251
+ def _before_fork() -> None:
252
+ """PR-6: a fork waits for any sample in flight (`_LOCK`, then the
253
+ counter's lock, the order `_sample_locked` takes them), so neither is
254
+ inherited held by a thread the child does not have."""
255
+ _LOCK.acquire()
256
+ lock = _counter_lock(_STATE)
257
+ if lock is not None:
258
+ lock.acquire()
259
+
260
+
261
+ def _after_fork_in_parent() -> None:
262
+ lock = _counter_lock(_STATE)
263
+ if lock is not None:
264
+ lock.release()
265
+ _LOCK.release()
266
+
267
+
268
+ def _after_fork_in_child() -> None:
269
+ """The child measures its own process: released locks, a fresh state."""
270
+ global _LOCK, _STATE
271
+ old = _STATE
272
+ counter = old.counter
273
+ if type(counter) is ProcessWriteCounter:
274
+ counter = None
275
+ else:
276
+ lock = _counter_lock(old)
277
+ if lock is not None:
278
+ lock.release()
279
+ _LOCK = threading.Lock()
280
+ _STATE = _State(counter, old.clock_ns, old.wall_s)
281
+
282
+
283
+ if hasattr(os, "register_at_fork"):
284
+ os.register_at_fork(before=_before_fork,
285
+ after_in_parent=_after_fork_in_parent,
286
+ after_in_child=_after_fork_in_child)
287
+
288
+
289
+ def reset_for_tests(*, counter=None, clock_ns=None, wall_s=None) -> None:
290
+ global _STATE
291
+ with _LOCK:
292
+ _STATE = _State(counter, clock_ns, wall_s)
293
+
294
+
295
+ def _append_sample_locked(now_ns: int, reading: CounterReading) -> None:
296
+ state = _STATE
297
+ state.last_status = reading.status
298
+ value = -1 if reading.value is None else int(reading.value)
299
+ bucket = now_ns // TIMELINE_BUCKET_NS
300
+ if len(state.t) and state.t[-1] // TIMELINE_BUCKET_NS == bucket:
301
+ state.t[-1] = now_ns
302
+ state.b[-1] = value
303
+ state.p[-1] = state.publications
304
+ return
305
+ state.t.append(now_ns)
306
+ state.b.append(value)
307
+ state.p.append(state.publications)
308
+ while len(state.t) > TIMELINE_CAPACITY:
309
+ del state.t[0]
310
+ del state.b[0]
311
+ del state.p[0]
312
+
313
+
314
+ def _sample_locked(state) -> "tuple[int, CounterReading]":
315
+ """Take the timestamp, read the counter and append the sample as ONE
316
+ step under `_LOCK` (901-CD-001). With the clock and the reader outside
317
+ the lock, a thread could stamp 4.999 s, pause while another sampled at
318
+ 5.001 s, then file its larger reading under the earlier bucket: sorted by
319
+ time the timeline decreased, and the statistic reported a false
320
+ `counter_reset`. The counter's own lock never takes `_LOCK`."""
321
+ now = state.clock_ns()
322
+ reading = state.counter.read()
323
+ _append_sample_locked(now, reading)
324
+ return now, reading
325
+
326
+
327
+ def read_counter() -> CounterReading:
328
+ """Read and record one sample (a loop wakeup or a phase boundary)."""
329
+ state = _STATE
330
+ with _LOCK:
331
+ _, reading = _sample_locked(state)
332
+ return reading
333
+
334
+
335
+ def begin_interval(kind: str) -> Interval:
336
+ state = _STATE
337
+ thread = threading.get_ident()
338
+ with _LOCK:
339
+ now, reading = _sample_locked(state)
340
+ token = state.next_token
341
+ state.next_token += 1
342
+ overlapped = False
343
+ for entry in state.open.values():
344
+ if entry[0] != thread:
345
+ entry[1] = True
346
+ overlapped = True
347
+ state.open[token] = [thread, overlapped]
348
+ return Interval(token, str(kind), thread, now, reading.value,
349
+ reading.status)
350
+
351
+
352
+ def end_interval(interval: Interval) -> IntervalObservation:
353
+ state = _STATE
354
+ with _LOCK:
355
+ now, reading = _sample_locked(state)
356
+ entry = state.open.pop(interval.token, None)
357
+ overlap = bool(entry[1]) if entry is not None else False
358
+ if (reading.status == "ok" and interval.start_status == "ok"
359
+ and interval.start_bytes is not None
360
+ and reading.value >= interval.start_bytes):
361
+ delta, status = reading.value - interval.start_bytes, "ok"
362
+ elif reading.status != "ok":
363
+ delta, status = None, reading.status
364
+ elif interval.start_status != "ok":
365
+ delta, status = None, interval.start_status
366
+ else:
367
+ delta, status = None, "counter_reset"
368
+ return IntervalObservation(
369
+ kind=interval.kind, started_ns=interval.started_ns, ended_ns=now,
370
+ start_bytes=interval.start_bytes, end_bytes=reading.value,
371
+ process_write_bytes=delta, status=status, overlap=overlap)
372
+
373
+
374
+ def mark_deletion(observation: IntervalObservation, *, rows: int) -> None:
375
+ """Record one deletion operation's BEGIN-to-COMMIT interval."""
376
+ state = _STATE
377
+ wall = int(state.wall_s())
378
+ with _LOCK:
379
+ columns = state.ex
380
+ for column, value in zip(columns, (
381
+ observation.started_ns, observation.ended_ns,
382
+ -1 if observation.start_bytes is None else observation.start_bytes,
383
+ -1 if observation.end_bytes is None else observation.end_bytes,
384
+ max(0, int(rows)), wall)):
385
+ column.append(int(value))
386
+ while len(columns[0]) > EXCLUSION_CAPACITY:
387
+ for column in columns:
388
+ del column[0]
389
+
390
+
391
+ def _samples_locked():
392
+ state = _STATE
393
+ return tuple(
394
+ _budget.CounterSample(t, None if b < 0 else b, p)
395
+ for t, b, p in zip(state.t, state.b, state.p))
396
+
397
+
398
+ def _deletions_locked():
399
+ s, e, sb, eb, rows, wall = _STATE.ex
400
+ return tuple(
401
+ _budget.DeletionInterval(a, b, None if c < 0 else c,
402
+ None if d < 0 else d, r, w)
403
+ for a, b, c, d, r, w in zip(s, e, sb, eb, rows, wall))
404
+
405
+
406
+ def _steady_locked(now_ns):
407
+ state = _STATE
408
+ return _budget.steady_statistic(
409
+ _samples_locked(), _deletions_locked(), now_ns=now_ns,
410
+ warm_admitted_ns=state.warm_admitted_ns,
411
+ counter_status=state.last_status)
412
+
413
+
414
+ def note_publication(*, cold: bool) -> None:
415
+ """One completed main publication: count it, sample, admit, evaluate."""
416
+ state = _STATE
417
+ with _LOCK:
418
+ state.publications += 1
419
+ now, reading = _sample_locked(state)
420
+ if state.warm_admitted_ns is None and not cold:
421
+ state.warm_admitted_ns = now
422
+ state.cold_startup_bytes = reading.value
423
+ if (state.last_hysteresis_ns is None
424
+ or now - state.last_hysteresis_ns
425
+ >= HYSTERESIS_EVALUATION_SPACING_NS):
426
+ state.hysteresis = _budget.advance(
427
+ state.hysteresis, _steady_locked(now), _budget.LIMITS, now)
428
+ state.last_hysteresis_ns = now
429
+
430
+
431
+ def evaluate(now_ns=None, limits=None):
432
+ """`(SteadyStatistic, HysteresisState)` now; never moves the hysteresis."""
433
+ state = _STATE
434
+ now = state.clock_ns() if now_ns is None else int(now_ns)
435
+ with _LOCK:
436
+ return _steady_locked(now), state.hysteresis
437
+
438
+
439
+ def set_maintenance_record(record) -> None:
440
+ """Hold the latest durable charge ledger, compactly (hour, charged,
441
+ largest) — the source of `writeIo.maintenance` in this process."""
442
+ buckets = _budget.parse_ledger(
443
+ record.get("ledger") if isinstance(record, dict) else None)
444
+ flat = array.array("q")
445
+ for bucket in buckets:
446
+ flat.extend((int(bucket.hour.timestamp()), bucket.charged,
447
+ bucket.largest))
448
+ with _LOCK:
449
+ _STATE.ledger = flat
450
+ _STATE.ledger_observed = True
451
+
452
+
453
+ def maintenance_ledger() -> "list[dict]":
454
+ with _LOCK:
455
+ flat = list(_STATE.ledger)
456
+ out = []
457
+ for i in range(0, len(flat), 3):
458
+ hour = dt.datetime.fromtimestamp(flat[i], UTC)
459
+ out.append({"hour": _budget.iso_z(hour), "charged": flat[i + 1],
460
+ "largest": flat[i + 2]})
461
+ return out
462
+
463
+
464
+ def register_local_instance(*, instance_id, host, port, pid=None) -> None:
465
+ with _LOCK:
466
+ _STATE.local_instance = {
467
+ "instanceId": str(instance_id), "host": str(host),
468
+ "port": int(port), "pid": os.getpid() if pid is None else int(pid)}
469
+
470
+
471
+ def clear_local_instance() -> None:
472
+ with _LOCK:
473
+ _STATE.local_instance = None
474
+
475
+
476
+ def local_instance() -> "dict | None":
477
+ with _LOCK:
478
+ local = _STATE.local_instance
479
+ return None if local is None else dict(local)
480
+
481
+
482
+ def state_for_budget() -> tuple:
483
+ """Every object this module retains, for the 65,536-byte state test."""
484
+ state = _STATE
485
+ return (state.t, state.b, state.p, state.ledger, state.ex[0],
486
+ state.ex[1:], state.open, state.hysteresis, state.local_instance,
487
+ state.cold_startup_bytes, state.last_status, state.reclaim_refusal)
488
+
489
+
490
+ #: The reason strings a reclaim refusal may publish; anything else is
491
+ #: published as "unknown" so no free text reaches `writeIo`.
492
+ RECLAIM_REFUSAL_REASON_MAX_CHARS = 48
493
+
494
+
495
+ def set_reclaim_refusal(reason, at_utc) -> None:
496
+ """Publish the latest planner refusal of this process's reclaim (spec
497
+ §5.4 "Refusal": the in-memory pass record, never a durable write)."""
498
+ text = reason if (isinstance(reason, str) and reason.isascii()
499
+ and 0 < len(reason) <= RECLAIM_REFUSAL_REASON_MAX_CHARS
500
+ and all(c.islower() or c == "_" for c in reason)
501
+ ) else "unknown"
502
+ with _LOCK:
503
+ _STATE.reclaim_refusal = (text, _budget.iso_z(at_utc))
504
+
505
+
506
+ def clear_reclaim_refusal() -> None:
507
+ with _LOCK:
508
+ _STATE.reclaim_refusal = None
509
+
510
+
511
+ def reclaim_refusal() -> "dict | None":
512
+ with _LOCK:
513
+ refusal = _STATE.reclaim_refusal
514
+ return None if refusal is None else {"reason": refusal[0],
515
+ "at": refusal[1]}
516
+
517
+
518
+ # ── instance descriptor (spec §5.6) ───────────────────────────────────────
519
+
520
+ def new_instance_id() -> str:
521
+ return secrets.token_hex(8)
522
+
523
+
524
+ def descriptor_dir(app_dir) -> pathlib.Path:
525
+ return pathlib.Path(app_dir) / DESCRIPTOR_DIRNAME
526
+
527
+
528
+ def write_instance_descriptor(app_dir, *, instance_id, host, port, pid=None,
529
+ now_utc=None) -> pathlib.Path:
530
+ """Write once at startup: owner-readable directory 0700, file 0600."""
531
+ pid = os.getpid() if pid is None else int(pid)
532
+ directory = descriptor_dir(app_dir)
533
+ directory.mkdir(parents=True, exist_ok=True)
534
+ os.chmod(directory, 0o700)
535
+ payload = {
536
+ "schemaVersion": DESCRIPTOR_SCHEMA_VERSION,
537
+ "instanceId": str(instance_id),
538
+ "pid": pid,
539
+ "processStart": process_start_identity(pid),
540
+ "host": str(host),
541
+ "port": int(port),
542
+ "startedAt": _budget.iso_z(now_utc or dt.datetime.now(UTC)),
543
+ }
544
+ target = directory / f"{DESCRIPTOR_PREFIX}{instance_id}.json"
545
+ tmp = directory / f".{target.name}.{os.getpid()}.tmp"
546
+ fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
547
+ with os.fdopen(fd, "w", encoding="utf-8") as fh:
548
+ json.dump(payload, fh, sort_keys=True)
549
+ fh.write("\n")
550
+ os.replace(tmp, target)
551
+ return target
552
+
553
+
554
+ def remove_instance_descriptor(app_dir, instance_id) -> None:
555
+ try:
556
+ (descriptor_dir(app_dir)
557
+ / f"{DESCRIPTOR_PREFIX}{instance_id}.json").unlink()
558
+ except FileNotFoundError:
559
+ pass
560
+
561
+
562
+ def _valid_descriptor(data) -> bool:
563
+ if not isinstance(data, dict):
564
+ return False
565
+ if data.get("schemaVersion") != DESCRIPTOR_SCHEMA_VERSION:
566
+ return False
567
+ instance_id = data.get("instanceId")
568
+ pid = data.get("pid")
569
+ port = data.get("port")
570
+ start = data.get("processStart")
571
+ return (isinstance(instance_id, str) and bool(_INSTANCE_ID_RE.match(instance_id))
572
+ and isinstance(pid, int) and not isinstance(pid, bool) and pid > 0
573
+ and isinstance(port, int) and not isinstance(port, bool)
574
+ and 1 <= port <= 65535
575
+ and isinstance(data.get("host"), str)
576
+ and (start is None or isinstance(start, str)))
577
+
578
+
579
+ def read_instance_descriptors(app_dir) -> "list[dict]":
580
+ try:
581
+ paths = sorted(descriptor_dir(app_dir).glob(f"{DESCRIPTOR_PREFIX}*.json"))
582
+ except OSError:
583
+ return []
584
+ out = []
585
+ for path in paths:
586
+ try:
587
+ data = json.loads(path.read_text(encoding="utf-8"))
588
+ except (OSError, ValueError):
589
+ continue
590
+ if _valid_descriptor(data):
591
+ out.append(data)
592
+ return out
593
+
594
+
595
+ def _pid_alive(pid) -> bool:
596
+ try:
597
+ os.kill(int(pid), 0)
598
+ except ProcessLookupError:
599
+ return False
600
+ except PermissionError:
601
+ return True
602
+ return True
603
+
604
+
605
+ def descriptor_is_live(descriptor, *, alive=None, identity=None) -> bool:
606
+ """The descriptor's pid is alive AND is still the process that wrote it.
607
+
608
+ #901 Amendment 19 PR-10: "still the process" must be PROVEN. A descriptor
609
+ that recorded no start identity, or whose pid's identity cannot be read
610
+ now, cannot tell its writer from a later process that reused the pid, so
611
+ it is not live; only a readable identity equal to the recorded one is."""
612
+ pid = descriptor["pid"]
613
+ if not (alive or _pid_alive)(pid):
614
+ return False
615
+ recorded = descriptor.get("processStart")
616
+ if not isinstance(recorded, str) or not recorded:
617
+ return False
618
+ current = (identity or process_start_identity)(pid)
619
+ return current is not None and current == recorded
620
+
621
+
622
+ def remove_stale_descriptors(app_dir) -> int:
623
+ """Dashboard startup only: remove descriptors of dead or reused pids."""
624
+ removed = 0
625
+ for descriptor in read_instance_descriptors(app_dir):
626
+ if not descriptor_is_live(descriptor):
627
+ remove_instance_descriptor(app_dir, descriptor["instanceId"])
628
+ removed += 1
629
+ return removed
630
+
631
+
632
+ def loopback_target(bind_host) -> "str | None":
633
+ """The IP-literal loopback address that reaches a dashboard bound to
634
+ `bind_host`, or None when no loopback address does."""
635
+ host = str(bind_host or "").strip()
636
+ if host in ("", "0.0.0.0", "127.0.0.1", "localhost", "loopback"):
637
+ return "127.0.0.1"
638
+ if host in ("::", "::1"):
639
+ return "::1"
640
+ try:
641
+ address = ipaddress.ip_address(host)
642
+ except ValueError:
643
+ return None
644
+ return host if address.is_loopback else None
645
+
646
+
647
+ # ── the `/api/debug/backend` `writeIo` object (spec §5.5) ─────────────────
648
+
649
+ def write_io_payload(*, now_ns=None, now_utc=None, limits=None) -> dict:
650
+ """The camelCase `writeIo` object: the trailing five-minute statistic,
651
+ its budget verdict (the hysteresis state, never advanced here), the
652
+ excluded deletion intervals, cold startup and the maintenance ledger.
653
+ Numbers are `None`, never zero, whenever the statistic cannot stand."""
654
+ state = _STATE
655
+ limits = limits or _budget.LIMITS
656
+ if state.last_status == "not_sampled":
657
+ read_counter()
658
+ now = state.clock_ns() if now_ns is None else int(now_ns)
659
+ wall = (dt.datetime.fromtimestamp(state.wall_s(), UTC)
660
+ if now_utc is None else now_utc.astimezone(UTC))
661
+ with _LOCK:
662
+ stat = _steady_locked(now)
663
+ hysteresis = state.hysteresis
664
+ last_status = state.last_status
665
+ cold = state.cold_startup_bytes
666
+ admitted = state.warm_admitted_ns is not None
667
+ local = None if state.local_instance is None else dict(state.local_instance)
668
+ observed = state.ledger_observed
669
+ verdict, reasons = _budget.steady_verdict(stat, hysteresis, limits)
670
+ maintenance = (_budget.maintenance_statistic(maintenance_ledger(), wall)
671
+ .as_wire() if observed else None)
672
+ last_wall = stat.last_excluded_wall_s
673
+ return {
674
+ "status": "ok" if last_status == "ok" else "unavailable",
675
+ "reason": None if last_status == "ok" else last_status,
676
+ "source": state.counter.source,
677
+ "scope": SCOPE,
678
+ "instanceId": None if local is None else local["instanceId"],
679
+ "sampledAt": _budget.iso_z(wall),
680
+ "windowSeconds": (None if stat.window_seconds is None
681
+ else round(stat.window_seconds, 3)),
682
+ "bytesWritten": stat.bytes_written,
683
+ "bytesPerMinute": stat.bytes_per_minute,
684
+ "tickCount": stat.publications,
685
+ "meanBytesPerTick": stat.mean_bytes_per_publication,
686
+ "excludedDeletions": {
687
+ "operations": stat.excluded_operations,
688
+ "rows": stat.excluded_rows,
689
+ "bytes": stat.excluded_bytes,
690
+ "lastEndedAt": (None if last_wall is None else _budget.iso_z(
691
+ dt.datetime.fromtimestamp(last_wall, UTC))),
692
+ },
693
+ "coldStartup": {"bytes": cold, "admitted": admitted},
694
+ "maintenance": maintenance,
695
+ "reclaimRefusal": reclaim_refusal(),
696
+ "budget": {
697
+ "policyVersion": limits.policy_version,
698
+ "verdict": verdict,
699
+ "reasons": list(reasons),
700
+ "bytesPerMinuteLimit": limits.bytes_per_minute,
701
+ "meanBytesPerTickLimit": limits.bytes_per_publication,
702
+ },
703
+ }