cctally 1.112.0 → 1.114.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/README.md +4 -5
- package/bin/_cctally_cache.py +1683 -485
- package/bin/_cctally_core.py +223 -1
- package/bin/_cctally_dashboard.py +96 -18
- package/bin/_cctally_dashboard_perf.py +128 -9
- package/bin/_cctally_dashboard_sources.py +267 -204
- package/bin/_cctally_db.py +861 -18
- package/bin/_cctally_diagnosis_sources.py +22 -4
- package/bin/_cctally_doctor.py +136 -18
- package/bin/_cctally_five_hour.py +2 -2
- package/bin/_cctally_forecast.py +27 -6
- package/bin/_cctally_journal.py +67 -5
- package/bin/_cctally_milestone_history.py +82 -9
- package/bin/_cctally_percent_breakdown.py +3 -2
- package/bin/_cctally_quota.py +201 -33
- package/bin/_cctally_record.py +14 -0
- package/bin/_cctally_rederive.py +23 -9
- package/bin/_cctally_source_analytics.py +60 -15
- package/bin/_cctally_statusline.py +17 -0
- package/bin/_cctally_store.py +80 -5
- package/bin/_cctally_tui.py +4 -3
- package/bin/_cctally_weekrefs.py +128 -21
- package/bin/_lib_cache_report.py +35 -6
- package/bin/_lib_codex_conversation.py +32 -0
- package/bin/_lib_codex_conversation_query.py +164 -30
- package/bin/_lib_conversation_query.py +95 -56
- package/bin/_lib_conversation_retention.py +1605 -640
- package/bin/_lib_cost_provenance.py +53 -2
- package/bin/_lib_doctor.py +263 -69
- package/bin/_lib_fmt.py +11 -0
- package/bin/_lib_merge_gate.py +469 -51
- package/bin/_lib_perf.py +27 -1
- package/bin/_lib_pricing.py +209 -8
- package/bin/_lib_pricing_check.py +20 -4
- package/bin/_lib_reclaim_planner.py +792 -0
- package/bin/_lib_rederive.py +225 -77
- package/bin/_lib_snapshot_cache.py +21 -0
- package/bin/_lib_sqlite_close.py +81 -0
- package/bin/_lib_subscription_weeks.py +100 -17
- package/bin/_lib_tick_stats.py +217 -17
- package/bin/_lib_wal_checkpoint.py +1077 -0
- package/bin/_lib_write_budget.py +483 -0
- package/bin/_lib_write_io.py +703 -0
- package/bin/cctally +8 -0
- package/dashboard/static/assets/{ConversationsView-CEZ-BUyi.js → ConversationsView-Bc8LCOAZ.js} +1 -1
- package/dashboard/static/assets/{DoctorModal-CE1oH_Zy.js → DoctorModal-D--qVBwe.js} +1 -1
- package/dashboard/static/assets/ModalRoot-DLPgyAM_.js +1 -0
- package/dashboard/static/assets/{ProjectsDrillPanel-B3DZoCeD.js → ProjectsDrillPanel-z53a4v4M.js} +1 -1
- package/dashboard/static/assets/{SourceDetailModal-CHp9EgLP.js → SourceDetailModal-Bw0T8W-W.js} +1 -1
- package/dashboard/static/assets/{UpdateModal-5iKST6jo.js → UpdateModal-D5AKx4Yw.js} +1 -1
- package/dashboard/static/assets/index-BKITk1CT.css +1 -0
- package/dashboard/static/assets/{index-Cm0VTxJC.js → index-BXDOzSng.js} +3 -3
- package/dashboard/static/assets/{outlineNavigation-DkXGL8J6.js → outlineNavigation-JObY7cUY.js} +1 -1
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +6 -1
- package/dashboard/static/assets/ModalRoot-D5Y8IFUG.js +0 -1
- package/dashboard/static/assets/index-Dt4tqf43.css +0 -1
|
@@ -0,0 +1,703 @@
|
|
|
1
|
+
"""Process disk-write telemetry for the dashboard (#901 spec §5.5).
|
|
2
|
+
|
|
3
|
+
Stdlib-only leaf. It reads ONE kernel-accounted process write counter —
|
|
4
|
+
macOS `proc_pid_rusage(RUSAGE_INFO_V2).ri_diskio_byteswritten` through a
|
|
5
|
+
correctly declared ctypes ABI, Linux `/proc/self/io` `write_bytes` — and never
|
|
6
|
+
substitutes zero, file growth or `ru_oublock` for a counter it could not read:
|
|
7
|
+
an unreadable value is `None` with a reason from `UNAVAILABLE_REASONS`. These
|
|
8
|
+
are process writes, not NAND writes.
|
|
9
|
+
|
|
10
|
+
One cumulative timeline is sampled at tick, conversation-pass and maintenance
|
|
11
|
+
boundaries. An interval's delta includes whatever ran concurrently; it carries
|
|
12
|
+
overlap metadata and is never summed across loops. Sustained rates come only
|
|
13
|
+
from the timeline, through `_lib_write_budget`.
|
|
14
|
+
|
|
15
|
+
It also owns the dashboard instance descriptor: an owner-readable file written
|
|
16
|
+
once at startup and removed at clean shutdown, through which `cctally doctor`
|
|
17
|
+
discovers a running instance. It is endpoint discovery, never a measurement.
|
|
18
|
+
"""
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import array
|
|
22
|
+
import dataclasses
|
|
23
|
+
import datetime as dt
|
|
24
|
+
import ipaddress
|
|
25
|
+
import json
|
|
26
|
+
import os
|
|
27
|
+
import pathlib
|
|
28
|
+
import re
|
|
29
|
+
import secrets
|
|
30
|
+
import sys
|
|
31
|
+
import threading
|
|
32
|
+
import time
|
|
33
|
+
|
|
34
|
+
import _lib_write_budget as _budget
|
|
35
|
+
|
|
36
|
+
UTC = dt.timezone.utc
|
|
37
|
+
UNAVAILABLE_REASONS = ("unsupported_platform", "counter_error", "counter_reset")
|
|
38
|
+
WRITE_STATUSES = ("ok",) + UNAVAILABLE_REASONS + ("not_sampled",)
|
|
39
|
+
SOURCE_DARWIN = "proc_pid_rusage.ri_diskio_byteswritten"
|
|
40
|
+
SOURCE_LINUX = "proc.io.write_bytes"
|
|
41
|
+
SCOPE = "process"
|
|
42
|
+
TIMELINE_CAPACITY = 96
|
|
43
|
+
TIMELINE_BUCKET_NS = 5 * 1_000_000_000
|
|
44
|
+
EXCLUSION_CAPACITY = 16
|
|
45
|
+
#: The hysteresis advances at most this often; far below its 60 s spacing.
|
|
46
|
+
HYSTERESIS_EVALUATION_SPACING_NS = 10 * 1_000_000_000
|
|
47
|
+
DESCRIPTOR_SCHEMA_VERSION = 1
|
|
48
|
+
DESCRIPTOR_DIRNAME = "run"
|
|
49
|
+
DESCRIPTOR_PREFIX = "dashboard-"
|
|
50
|
+
_INSTANCE_ID_RE = re.compile(r"^[0-9a-f]{16}$")
|
|
51
|
+
|
|
52
|
+
_RUSAGE_INFO_V2 = 2
|
|
53
|
+
#: `struct rusage_info_v2` (<sys/resource.h>): a 16-byte uuid then eighteen
|
|
54
|
+
#: uint64 fields, in this order.
|
|
55
|
+
_RUSAGE_V2_FIELDS = (
|
|
56
|
+
"ri_user_time", "ri_system_time", "ri_pkg_idle_wkups",
|
|
57
|
+
"ri_interrupt_wkups", "ri_pageins", "ri_wired_size", "ri_resident_size",
|
|
58
|
+
"ri_phys_footprint", "ri_proc_start_abstime", "ri_proc_exit_abstime",
|
|
59
|
+
"ri_child_user_time", "ri_child_system_time", "ri_child_pkg_idle_wkups",
|
|
60
|
+
"ri_child_interrupt_wkups", "ri_child_pageins",
|
|
61
|
+
"ri_child_elapsed_abstime", "ri_diskio_bytesread",
|
|
62
|
+
"ri_diskio_byteswritten",
|
|
63
|
+
)
|
|
64
|
+
_RUSAGE_TYPE = None
|
|
65
|
+
_PROC_PID_RUSAGE = None
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _rusage_type():
|
|
69
|
+
"""ctypes is imported lazily: this leaf is reachable from hook paths."""
|
|
70
|
+
global _RUSAGE_TYPE
|
|
71
|
+
if _RUSAGE_TYPE is None:
|
|
72
|
+
import ctypes
|
|
73
|
+
|
|
74
|
+
class RusageInfoV2(ctypes.Structure):
|
|
75
|
+
_fields_ = [("ri_uuid", ctypes.c_uint8 * 16)] + [
|
|
76
|
+
(name, ctypes.c_uint64) for name in _RUSAGE_V2_FIELDS]
|
|
77
|
+
|
|
78
|
+
_RUSAGE_TYPE = RusageInfoV2
|
|
79
|
+
return _RUSAGE_TYPE
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _proc_pid_rusage():
|
|
83
|
+
global _PROC_PID_RUSAGE
|
|
84
|
+
if _PROC_PID_RUSAGE is None:
|
|
85
|
+
import ctypes
|
|
86
|
+
|
|
87
|
+
lib = ctypes.CDLL("/usr/lib/libSystem.B.dylib", use_errno=True)
|
|
88
|
+
fn = lib.proc_pid_rusage
|
|
89
|
+
fn.argtypes = (ctypes.c_int, ctypes.c_int, ctypes.c_void_p)
|
|
90
|
+
fn.restype = ctypes.c_int
|
|
91
|
+
_PROC_PID_RUSAGE = fn
|
|
92
|
+
return _PROC_PID_RUSAGE
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def darwin_rusage(pid):
|
|
96
|
+
import ctypes
|
|
97
|
+
|
|
98
|
+
info = _rusage_type()()
|
|
99
|
+
rc = _proc_pid_rusage()(int(pid), _RUSAGE_INFO_V2, ctypes.byref(info))
|
|
100
|
+
if rc != 0:
|
|
101
|
+
raise OSError(ctypes.get_errno(), f"proc_pid_rusage({pid}) failed")
|
|
102
|
+
return info
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def darwin_write_bytes(pid=None) -> int:
|
|
106
|
+
target = os.getpid() if pid is None else int(pid)
|
|
107
|
+
return int(darwin_rusage(target).ri_diskio_byteswritten)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def linux_write_bytes(pid=None, *, opener=open) -> int:
|
|
111
|
+
path = "/proc/self/io" if pid is None else f"/proc/{int(pid)}/io"
|
|
112
|
+
with opener(path, "r", encoding="ascii") as fh:
|
|
113
|
+
for line in fh:
|
|
114
|
+
key, _, value = line.partition(":")
|
|
115
|
+
if key.strip() == "write_bytes":
|
|
116
|
+
return int(value.strip())
|
|
117
|
+
raise OSError(f"write_bytes missing from {path}")
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
@dataclasses.dataclass(frozen=True)
|
|
121
|
+
class CounterReading:
|
|
122
|
+
value: "int | None"
|
|
123
|
+
status: str
|
|
124
|
+
source: "str | None"
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
class ProcessWriteCounter:
|
|
128
|
+
"""One process's cumulative write counter, with reset detection."""
|
|
129
|
+
|
|
130
|
+
def __init__(self, *, platform=None, pid=None, darwin_reader=None,
|
|
131
|
+
linux_reader=None):
|
|
132
|
+
self._platform = sys.platform if platform is None else platform
|
|
133
|
+
self._pid = pid
|
|
134
|
+
self._darwin = darwin_reader or darwin_write_bytes
|
|
135
|
+
self._linux = linux_reader or linux_write_bytes
|
|
136
|
+
self._last = None
|
|
137
|
+
self._lock = threading.Lock()
|
|
138
|
+
|
|
139
|
+
@property
|
|
140
|
+
def source(self) -> "str | None":
|
|
141
|
+
if self._platform == "darwin":
|
|
142
|
+
return SOURCE_DARWIN
|
|
143
|
+
if self._platform.startswith("linux"):
|
|
144
|
+
return SOURCE_LINUX
|
|
145
|
+
return None
|
|
146
|
+
|
|
147
|
+
def read(self) -> CounterReading:
|
|
148
|
+
source = self.source
|
|
149
|
+
if source is None:
|
|
150
|
+
return CounterReading(None, "unsupported_platform", None)
|
|
151
|
+
reader = self._darwin if source == SOURCE_DARWIN else self._linux
|
|
152
|
+
# #901 Amendment 19 PR-2: the observation and its comparison with the
|
|
153
|
+
# previous one are ONE step. With the reader outside the lock, a thread
|
|
154
|
+
# could observe 100, lose the race to store it to a thread that then
|
|
155
|
+
# observed and stored 110, and report a false `counter_reset` for a
|
|
156
|
+
# counter that only grew.
|
|
157
|
+
with self._lock:
|
|
158
|
+
try:
|
|
159
|
+
value = int(reader(self._pid))
|
|
160
|
+
except FileNotFoundError:
|
|
161
|
+
return CounterReading(None, "unsupported_platform", source)
|
|
162
|
+
except (OSError, ValueError, TypeError, AttributeError):
|
|
163
|
+
return CounterReading(None, "counter_error", source)
|
|
164
|
+
if value < 0:
|
|
165
|
+
return CounterReading(None, "counter_error", source)
|
|
166
|
+
last, self._last = self._last, value
|
|
167
|
+
if last is not None and value < last:
|
|
168
|
+
return CounterReading(None, "counter_reset", source)
|
|
169
|
+
return CounterReading(value, "ok", source)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _read_proc_stat(pid) -> str:
|
|
173
|
+
with open(f"/proc/{int(pid)}/stat", "r", encoding="ascii",
|
|
174
|
+
errors="replace") as fh:
|
|
175
|
+
return fh.read()
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def process_start_identity(pid, *, platform=None, darwin_rusage_fn=None,
|
|
179
|
+
proc_stat_reader=None) -> "str | None":
|
|
180
|
+
"""A value that changes when `pid` is reused; None when unknowable."""
|
|
181
|
+
platform = sys.platform if platform is None else platform
|
|
182
|
+
try:
|
|
183
|
+
if platform == "darwin":
|
|
184
|
+
info = (darwin_rusage_fn or darwin_rusage)(pid)
|
|
185
|
+
return f"darwin:{int(info.ri_proc_start_abstime)}"
|
|
186
|
+
if platform.startswith("linux"):
|
|
187
|
+
text = (proc_stat_reader or _read_proc_stat)(pid)
|
|
188
|
+
fields = text.rsplit(")", 1)[1].split()
|
|
189
|
+
# Field 22 (starttime) is index 19 after the `(comm)` field.
|
|
190
|
+
return f"linux:{int(fields[19])}"
|
|
191
|
+
except (OSError, ValueError, IndexError, AttributeError):
|
|
192
|
+
return None
|
|
193
|
+
return None
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
@dataclasses.dataclass(frozen=True)
|
|
197
|
+
class Interval:
|
|
198
|
+
token: int
|
|
199
|
+
kind: str
|
|
200
|
+
thread: int
|
|
201
|
+
started_ns: int
|
|
202
|
+
start_bytes: "int | None"
|
|
203
|
+
start_status: str
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
@dataclasses.dataclass(frozen=True)
|
|
207
|
+
class IntervalObservation:
|
|
208
|
+
kind: str
|
|
209
|
+
started_ns: int
|
|
210
|
+
ended_ns: int
|
|
211
|
+
start_bytes: "int | None"
|
|
212
|
+
end_bytes: "int | None"
|
|
213
|
+
process_write_bytes: "int | None"
|
|
214
|
+
status: str
|
|
215
|
+
overlap: bool
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
class _State:
|
|
219
|
+
def __init__(self, counter=None, clock_ns=None, wall_s=None):
|
|
220
|
+
self.counter = counter or ProcessWriteCounter()
|
|
221
|
+
self.clock_ns = clock_ns or time.monotonic_ns
|
|
222
|
+
self.wall_s = wall_s or time.time
|
|
223
|
+
self.t = array.array("q")
|
|
224
|
+
self.b = array.array("q")
|
|
225
|
+
self.p = array.array("q")
|
|
226
|
+
self.ex = tuple(array.array("q") for _ in range(6))
|
|
227
|
+
self.ledger = array.array("q")
|
|
228
|
+
self.ledger_observed = False
|
|
229
|
+
self.publications = 0
|
|
230
|
+
self.warm_admitted_ns = None
|
|
231
|
+
self.cold_startup_bytes = None
|
|
232
|
+
self.last_status = "not_sampled"
|
|
233
|
+
self.open = {}
|
|
234
|
+
self.next_token = 1
|
|
235
|
+
self.hysteresis = _budget.HysteresisState()
|
|
236
|
+
self.last_hysteresis_ns = None
|
|
237
|
+
self.local_instance = None
|
|
238
|
+
# #901 Q9: the latest reclaim attempt the planner refused, in memory
|
|
239
|
+
# only (a refusal writes nothing durable); None after a chunk runs.
|
|
240
|
+
self.reclaim_refusal = None
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
_LOCK = threading.Lock()
|
|
244
|
+
_STATE = _State()
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def _counter_lock(state) -> "threading.Lock | None":
|
|
248
|
+
return getattr(state.counter, "_lock", None)
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def _before_fork() -> None:
|
|
252
|
+
"""PR-6: a fork waits for any sample in flight (`_LOCK`, then the
|
|
253
|
+
counter's lock, the order `_sample_locked` takes them), so neither is
|
|
254
|
+
inherited held by a thread the child does not have."""
|
|
255
|
+
_LOCK.acquire()
|
|
256
|
+
lock = _counter_lock(_STATE)
|
|
257
|
+
if lock is not None:
|
|
258
|
+
lock.acquire()
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def _after_fork_in_parent() -> None:
|
|
262
|
+
lock = _counter_lock(_STATE)
|
|
263
|
+
if lock is not None:
|
|
264
|
+
lock.release()
|
|
265
|
+
_LOCK.release()
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def _after_fork_in_child() -> None:
|
|
269
|
+
"""The child measures its own process: released locks, a fresh state."""
|
|
270
|
+
global _LOCK, _STATE
|
|
271
|
+
old = _STATE
|
|
272
|
+
counter = old.counter
|
|
273
|
+
if type(counter) is ProcessWriteCounter:
|
|
274
|
+
counter = None
|
|
275
|
+
else:
|
|
276
|
+
lock = _counter_lock(old)
|
|
277
|
+
if lock is not None:
|
|
278
|
+
lock.release()
|
|
279
|
+
_LOCK = threading.Lock()
|
|
280
|
+
_STATE = _State(counter, old.clock_ns, old.wall_s)
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
if hasattr(os, "register_at_fork"):
|
|
284
|
+
os.register_at_fork(before=_before_fork,
|
|
285
|
+
after_in_parent=_after_fork_in_parent,
|
|
286
|
+
after_in_child=_after_fork_in_child)
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def reset_for_tests(*, counter=None, clock_ns=None, wall_s=None) -> None:
|
|
290
|
+
global _STATE
|
|
291
|
+
with _LOCK:
|
|
292
|
+
_STATE = _State(counter, clock_ns, wall_s)
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def _append_sample_locked(now_ns: int, reading: CounterReading) -> None:
|
|
296
|
+
state = _STATE
|
|
297
|
+
state.last_status = reading.status
|
|
298
|
+
value = -1 if reading.value is None else int(reading.value)
|
|
299
|
+
bucket = now_ns // TIMELINE_BUCKET_NS
|
|
300
|
+
if len(state.t) and state.t[-1] // TIMELINE_BUCKET_NS == bucket:
|
|
301
|
+
state.t[-1] = now_ns
|
|
302
|
+
state.b[-1] = value
|
|
303
|
+
state.p[-1] = state.publications
|
|
304
|
+
return
|
|
305
|
+
state.t.append(now_ns)
|
|
306
|
+
state.b.append(value)
|
|
307
|
+
state.p.append(state.publications)
|
|
308
|
+
while len(state.t) > TIMELINE_CAPACITY:
|
|
309
|
+
del state.t[0]
|
|
310
|
+
del state.b[0]
|
|
311
|
+
del state.p[0]
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
def _sample_locked(state) -> "tuple[int, CounterReading]":
|
|
315
|
+
"""Take the timestamp, read the counter and append the sample as ONE
|
|
316
|
+
step under `_LOCK` (901-CD-001). With the clock and the reader outside
|
|
317
|
+
the lock, a thread could stamp 4.999 s, pause while another sampled at
|
|
318
|
+
5.001 s, then file its larger reading under the earlier bucket: sorted by
|
|
319
|
+
time the timeline decreased, and the statistic reported a false
|
|
320
|
+
`counter_reset`. The counter's own lock never takes `_LOCK`."""
|
|
321
|
+
now = state.clock_ns()
|
|
322
|
+
reading = state.counter.read()
|
|
323
|
+
_append_sample_locked(now, reading)
|
|
324
|
+
return now, reading
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def read_counter() -> CounterReading:
|
|
328
|
+
"""Read and record one sample (a loop wakeup or a phase boundary)."""
|
|
329
|
+
state = _STATE
|
|
330
|
+
with _LOCK:
|
|
331
|
+
_, reading = _sample_locked(state)
|
|
332
|
+
return reading
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def begin_interval(kind: str) -> Interval:
|
|
336
|
+
state = _STATE
|
|
337
|
+
thread = threading.get_ident()
|
|
338
|
+
with _LOCK:
|
|
339
|
+
now, reading = _sample_locked(state)
|
|
340
|
+
token = state.next_token
|
|
341
|
+
state.next_token += 1
|
|
342
|
+
overlapped = False
|
|
343
|
+
for entry in state.open.values():
|
|
344
|
+
if entry[0] != thread:
|
|
345
|
+
entry[1] = True
|
|
346
|
+
overlapped = True
|
|
347
|
+
state.open[token] = [thread, overlapped]
|
|
348
|
+
return Interval(token, str(kind), thread, now, reading.value,
|
|
349
|
+
reading.status)
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def end_interval(interval: Interval) -> IntervalObservation:
|
|
353
|
+
state = _STATE
|
|
354
|
+
with _LOCK:
|
|
355
|
+
now, reading = _sample_locked(state)
|
|
356
|
+
entry = state.open.pop(interval.token, None)
|
|
357
|
+
overlap = bool(entry[1]) if entry is not None else False
|
|
358
|
+
if (reading.status == "ok" and interval.start_status == "ok"
|
|
359
|
+
and interval.start_bytes is not None
|
|
360
|
+
and reading.value >= interval.start_bytes):
|
|
361
|
+
delta, status = reading.value - interval.start_bytes, "ok"
|
|
362
|
+
elif reading.status != "ok":
|
|
363
|
+
delta, status = None, reading.status
|
|
364
|
+
elif interval.start_status != "ok":
|
|
365
|
+
delta, status = None, interval.start_status
|
|
366
|
+
else:
|
|
367
|
+
delta, status = None, "counter_reset"
|
|
368
|
+
return IntervalObservation(
|
|
369
|
+
kind=interval.kind, started_ns=interval.started_ns, ended_ns=now,
|
|
370
|
+
start_bytes=interval.start_bytes, end_bytes=reading.value,
|
|
371
|
+
process_write_bytes=delta, status=status, overlap=overlap)
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def mark_deletion(observation: IntervalObservation, *, rows: int) -> None:
|
|
375
|
+
"""Record one deletion operation's BEGIN-to-COMMIT interval."""
|
|
376
|
+
state = _STATE
|
|
377
|
+
wall = int(state.wall_s())
|
|
378
|
+
with _LOCK:
|
|
379
|
+
columns = state.ex
|
|
380
|
+
for column, value in zip(columns, (
|
|
381
|
+
observation.started_ns, observation.ended_ns,
|
|
382
|
+
-1 if observation.start_bytes is None else observation.start_bytes,
|
|
383
|
+
-1 if observation.end_bytes is None else observation.end_bytes,
|
|
384
|
+
max(0, int(rows)), wall)):
|
|
385
|
+
column.append(int(value))
|
|
386
|
+
while len(columns[0]) > EXCLUSION_CAPACITY:
|
|
387
|
+
for column in columns:
|
|
388
|
+
del column[0]
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
def _samples_locked():
|
|
392
|
+
state = _STATE
|
|
393
|
+
return tuple(
|
|
394
|
+
_budget.CounterSample(t, None if b < 0 else b, p)
|
|
395
|
+
for t, b, p in zip(state.t, state.b, state.p))
|
|
396
|
+
|
|
397
|
+
|
|
398
|
+
def _deletions_locked():
|
|
399
|
+
s, e, sb, eb, rows, wall = _STATE.ex
|
|
400
|
+
return tuple(
|
|
401
|
+
_budget.DeletionInterval(a, b, None if c < 0 else c,
|
|
402
|
+
None if d < 0 else d, r, w)
|
|
403
|
+
for a, b, c, d, r, w in zip(s, e, sb, eb, rows, wall))
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
def _steady_locked(now_ns):
|
|
407
|
+
state = _STATE
|
|
408
|
+
return _budget.steady_statistic(
|
|
409
|
+
_samples_locked(), _deletions_locked(), now_ns=now_ns,
|
|
410
|
+
warm_admitted_ns=state.warm_admitted_ns,
|
|
411
|
+
counter_status=state.last_status)
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def note_publication(*, cold: bool) -> None:
|
|
415
|
+
"""One completed main publication: count it, sample, admit, evaluate."""
|
|
416
|
+
state = _STATE
|
|
417
|
+
with _LOCK:
|
|
418
|
+
state.publications += 1
|
|
419
|
+
now, reading = _sample_locked(state)
|
|
420
|
+
if state.warm_admitted_ns is None and not cold:
|
|
421
|
+
state.warm_admitted_ns = now
|
|
422
|
+
state.cold_startup_bytes = reading.value
|
|
423
|
+
if (state.last_hysteresis_ns is None
|
|
424
|
+
or now - state.last_hysteresis_ns
|
|
425
|
+
>= HYSTERESIS_EVALUATION_SPACING_NS):
|
|
426
|
+
state.hysteresis = _budget.advance(
|
|
427
|
+
state.hysteresis, _steady_locked(now), _budget.LIMITS, now)
|
|
428
|
+
state.last_hysteresis_ns = now
|
|
429
|
+
|
|
430
|
+
|
|
431
|
+
def evaluate(now_ns=None, limits=None):
|
|
432
|
+
"""`(SteadyStatistic, HysteresisState)` now; never moves the hysteresis."""
|
|
433
|
+
state = _STATE
|
|
434
|
+
now = state.clock_ns() if now_ns is None else int(now_ns)
|
|
435
|
+
with _LOCK:
|
|
436
|
+
return _steady_locked(now), state.hysteresis
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
def set_maintenance_record(record) -> None:
|
|
440
|
+
"""Hold the latest durable charge ledger, compactly (hour, charged,
|
|
441
|
+
largest) — the source of `writeIo.maintenance` in this process."""
|
|
442
|
+
buckets = _budget.parse_ledger(
|
|
443
|
+
record.get("ledger") if isinstance(record, dict) else None)
|
|
444
|
+
flat = array.array("q")
|
|
445
|
+
for bucket in buckets:
|
|
446
|
+
flat.extend((int(bucket.hour.timestamp()), bucket.charged,
|
|
447
|
+
bucket.largest))
|
|
448
|
+
with _LOCK:
|
|
449
|
+
_STATE.ledger = flat
|
|
450
|
+
_STATE.ledger_observed = True
|
|
451
|
+
|
|
452
|
+
|
|
453
|
+
def maintenance_ledger() -> "list[dict]":
|
|
454
|
+
with _LOCK:
|
|
455
|
+
flat = list(_STATE.ledger)
|
|
456
|
+
out = []
|
|
457
|
+
for i in range(0, len(flat), 3):
|
|
458
|
+
hour = dt.datetime.fromtimestamp(flat[i], UTC)
|
|
459
|
+
out.append({"hour": _budget.iso_z(hour), "charged": flat[i + 1],
|
|
460
|
+
"largest": flat[i + 2]})
|
|
461
|
+
return out
|
|
462
|
+
|
|
463
|
+
|
|
464
|
+
def register_local_instance(*, instance_id, host, port, pid=None) -> None:
|
|
465
|
+
with _LOCK:
|
|
466
|
+
_STATE.local_instance = {
|
|
467
|
+
"instanceId": str(instance_id), "host": str(host),
|
|
468
|
+
"port": int(port), "pid": os.getpid() if pid is None else int(pid)}
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
def clear_local_instance() -> None:
|
|
472
|
+
with _LOCK:
|
|
473
|
+
_STATE.local_instance = None
|
|
474
|
+
|
|
475
|
+
|
|
476
|
+
def local_instance() -> "dict | None":
|
|
477
|
+
with _LOCK:
|
|
478
|
+
local = _STATE.local_instance
|
|
479
|
+
return None if local is None else dict(local)
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
def state_for_budget() -> tuple:
|
|
483
|
+
"""Every object this module retains, for the 65,536-byte state test."""
|
|
484
|
+
state = _STATE
|
|
485
|
+
return (state.t, state.b, state.p, state.ledger, state.ex[0],
|
|
486
|
+
state.ex[1:], state.open, state.hysteresis, state.local_instance,
|
|
487
|
+
state.cold_startup_bytes, state.last_status, state.reclaim_refusal)
|
|
488
|
+
|
|
489
|
+
|
|
490
|
+
#: The reason strings a reclaim refusal may publish; anything else is
|
|
491
|
+
#: published as "unknown" so no free text reaches `writeIo`.
|
|
492
|
+
RECLAIM_REFUSAL_REASON_MAX_CHARS = 48
|
|
493
|
+
|
|
494
|
+
|
|
495
|
+
def set_reclaim_refusal(reason, at_utc) -> None:
|
|
496
|
+
"""Publish the latest planner refusal of this process's reclaim (spec
|
|
497
|
+
§5.4 "Refusal": the in-memory pass record, never a durable write)."""
|
|
498
|
+
text = reason if (isinstance(reason, str) and reason.isascii()
|
|
499
|
+
and 0 < len(reason) <= RECLAIM_REFUSAL_REASON_MAX_CHARS
|
|
500
|
+
and all(c.islower() or c == "_" for c in reason)
|
|
501
|
+
) else "unknown"
|
|
502
|
+
with _LOCK:
|
|
503
|
+
_STATE.reclaim_refusal = (text, _budget.iso_z(at_utc))
|
|
504
|
+
|
|
505
|
+
|
|
506
|
+
def clear_reclaim_refusal() -> None:
|
|
507
|
+
with _LOCK:
|
|
508
|
+
_STATE.reclaim_refusal = None
|
|
509
|
+
|
|
510
|
+
|
|
511
|
+
def reclaim_refusal() -> "dict | None":
|
|
512
|
+
with _LOCK:
|
|
513
|
+
refusal = _STATE.reclaim_refusal
|
|
514
|
+
return None if refusal is None else {"reason": refusal[0],
|
|
515
|
+
"at": refusal[1]}
|
|
516
|
+
|
|
517
|
+
|
|
518
|
+
# ── instance descriptor (spec §5.6) ───────────────────────────────────────
|
|
519
|
+
|
|
520
|
+
def new_instance_id() -> str:
|
|
521
|
+
return secrets.token_hex(8)
|
|
522
|
+
|
|
523
|
+
|
|
524
|
+
def descriptor_dir(app_dir) -> pathlib.Path:
|
|
525
|
+
return pathlib.Path(app_dir) / DESCRIPTOR_DIRNAME
|
|
526
|
+
|
|
527
|
+
|
|
528
|
+
def write_instance_descriptor(app_dir, *, instance_id, host, port, pid=None,
|
|
529
|
+
now_utc=None) -> pathlib.Path:
|
|
530
|
+
"""Write once at startup: owner-readable directory 0700, file 0600."""
|
|
531
|
+
pid = os.getpid() if pid is None else int(pid)
|
|
532
|
+
directory = descriptor_dir(app_dir)
|
|
533
|
+
directory.mkdir(parents=True, exist_ok=True)
|
|
534
|
+
os.chmod(directory, 0o700)
|
|
535
|
+
payload = {
|
|
536
|
+
"schemaVersion": DESCRIPTOR_SCHEMA_VERSION,
|
|
537
|
+
"instanceId": str(instance_id),
|
|
538
|
+
"pid": pid,
|
|
539
|
+
"processStart": process_start_identity(pid),
|
|
540
|
+
"host": str(host),
|
|
541
|
+
"port": int(port),
|
|
542
|
+
"startedAt": _budget.iso_z(now_utc or dt.datetime.now(UTC)),
|
|
543
|
+
}
|
|
544
|
+
target = directory / f"{DESCRIPTOR_PREFIX}{instance_id}.json"
|
|
545
|
+
tmp = directory / f".{target.name}.{os.getpid()}.tmp"
|
|
546
|
+
fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
|
547
|
+
with os.fdopen(fd, "w", encoding="utf-8") as fh:
|
|
548
|
+
json.dump(payload, fh, sort_keys=True)
|
|
549
|
+
fh.write("\n")
|
|
550
|
+
os.replace(tmp, target)
|
|
551
|
+
return target
|
|
552
|
+
|
|
553
|
+
|
|
554
|
+
def remove_instance_descriptor(app_dir, instance_id) -> None:
|
|
555
|
+
try:
|
|
556
|
+
(descriptor_dir(app_dir)
|
|
557
|
+
/ f"{DESCRIPTOR_PREFIX}{instance_id}.json").unlink()
|
|
558
|
+
except FileNotFoundError:
|
|
559
|
+
pass
|
|
560
|
+
|
|
561
|
+
|
|
562
|
+
def _valid_descriptor(data) -> bool:
|
|
563
|
+
if not isinstance(data, dict):
|
|
564
|
+
return False
|
|
565
|
+
if data.get("schemaVersion") != DESCRIPTOR_SCHEMA_VERSION:
|
|
566
|
+
return False
|
|
567
|
+
instance_id = data.get("instanceId")
|
|
568
|
+
pid = data.get("pid")
|
|
569
|
+
port = data.get("port")
|
|
570
|
+
start = data.get("processStart")
|
|
571
|
+
return (isinstance(instance_id, str) and bool(_INSTANCE_ID_RE.match(instance_id))
|
|
572
|
+
and isinstance(pid, int) and not isinstance(pid, bool) and pid > 0
|
|
573
|
+
and isinstance(port, int) and not isinstance(port, bool)
|
|
574
|
+
and 1 <= port <= 65535
|
|
575
|
+
and isinstance(data.get("host"), str)
|
|
576
|
+
and (start is None or isinstance(start, str)))
|
|
577
|
+
|
|
578
|
+
|
|
579
|
+
def read_instance_descriptors(app_dir) -> "list[dict]":
|
|
580
|
+
try:
|
|
581
|
+
paths = sorted(descriptor_dir(app_dir).glob(f"{DESCRIPTOR_PREFIX}*.json"))
|
|
582
|
+
except OSError:
|
|
583
|
+
return []
|
|
584
|
+
out = []
|
|
585
|
+
for path in paths:
|
|
586
|
+
try:
|
|
587
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
588
|
+
except (OSError, ValueError):
|
|
589
|
+
continue
|
|
590
|
+
if _valid_descriptor(data):
|
|
591
|
+
out.append(data)
|
|
592
|
+
return out
|
|
593
|
+
|
|
594
|
+
|
|
595
|
+
def _pid_alive(pid) -> bool:
|
|
596
|
+
try:
|
|
597
|
+
os.kill(int(pid), 0)
|
|
598
|
+
except ProcessLookupError:
|
|
599
|
+
return False
|
|
600
|
+
except PermissionError:
|
|
601
|
+
return True
|
|
602
|
+
return True
|
|
603
|
+
|
|
604
|
+
|
|
605
|
+
def descriptor_is_live(descriptor, *, alive=None, identity=None) -> bool:
|
|
606
|
+
"""The descriptor's pid is alive AND is still the process that wrote it.
|
|
607
|
+
|
|
608
|
+
#901 Amendment 19 PR-10: "still the process" must be PROVEN. A descriptor
|
|
609
|
+
that recorded no start identity, or whose pid's identity cannot be read
|
|
610
|
+
now, cannot tell its writer from a later process that reused the pid, so
|
|
611
|
+
it is not live; only a readable identity equal to the recorded one is."""
|
|
612
|
+
pid = descriptor["pid"]
|
|
613
|
+
if not (alive or _pid_alive)(pid):
|
|
614
|
+
return False
|
|
615
|
+
recorded = descriptor.get("processStart")
|
|
616
|
+
if not isinstance(recorded, str) or not recorded:
|
|
617
|
+
return False
|
|
618
|
+
current = (identity or process_start_identity)(pid)
|
|
619
|
+
return current is not None and current == recorded
|
|
620
|
+
|
|
621
|
+
|
|
622
|
+
def remove_stale_descriptors(app_dir) -> int:
|
|
623
|
+
"""Dashboard startup only: remove descriptors of dead or reused pids."""
|
|
624
|
+
removed = 0
|
|
625
|
+
for descriptor in read_instance_descriptors(app_dir):
|
|
626
|
+
if not descriptor_is_live(descriptor):
|
|
627
|
+
remove_instance_descriptor(app_dir, descriptor["instanceId"])
|
|
628
|
+
removed += 1
|
|
629
|
+
return removed
|
|
630
|
+
|
|
631
|
+
|
|
632
|
+
def loopback_target(bind_host) -> "str | None":
|
|
633
|
+
"""The IP-literal loopback address that reaches a dashboard bound to
|
|
634
|
+
`bind_host`, or None when no loopback address does."""
|
|
635
|
+
host = str(bind_host or "").strip()
|
|
636
|
+
if host in ("", "0.0.0.0", "127.0.0.1", "localhost", "loopback"):
|
|
637
|
+
return "127.0.0.1"
|
|
638
|
+
if host in ("::", "::1"):
|
|
639
|
+
return "::1"
|
|
640
|
+
try:
|
|
641
|
+
address = ipaddress.ip_address(host)
|
|
642
|
+
except ValueError:
|
|
643
|
+
return None
|
|
644
|
+
return host if address.is_loopback else None
|
|
645
|
+
|
|
646
|
+
|
|
647
|
+
# ── the `/api/debug/backend` `writeIo` object (spec §5.5) ─────────────────
|
|
648
|
+
|
|
649
|
+
def write_io_payload(*, now_ns=None, now_utc=None, limits=None) -> dict:
|
|
650
|
+
"""The camelCase `writeIo` object: the trailing five-minute statistic,
|
|
651
|
+
its budget verdict (the hysteresis state, never advanced here), the
|
|
652
|
+
excluded deletion intervals, cold startup and the maintenance ledger.
|
|
653
|
+
Numbers are `None`, never zero, whenever the statistic cannot stand."""
|
|
654
|
+
state = _STATE
|
|
655
|
+
limits = limits or _budget.LIMITS
|
|
656
|
+
if state.last_status == "not_sampled":
|
|
657
|
+
read_counter()
|
|
658
|
+
now = state.clock_ns() if now_ns is None else int(now_ns)
|
|
659
|
+
wall = (dt.datetime.fromtimestamp(state.wall_s(), UTC)
|
|
660
|
+
if now_utc is None else now_utc.astimezone(UTC))
|
|
661
|
+
with _LOCK:
|
|
662
|
+
stat = _steady_locked(now)
|
|
663
|
+
hysteresis = state.hysteresis
|
|
664
|
+
last_status = state.last_status
|
|
665
|
+
cold = state.cold_startup_bytes
|
|
666
|
+
admitted = state.warm_admitted_ns is not None
|
|
667
|
+
local = None if state.local_instance is None else dict(state.local_instance)
|
|
668
|
+
observed = state.ledger_observed
|
|
669
|
+
verdict, reasons = _budget.steady_verdict(stat, hysteresis, limits)
|
|
670
|
+
maintenance = (_budget.maintenance_statistic(maintenance_ledger(), wall)
|
|
671
|
+
.as_wire() if observed else None)
|
|
672
|
+
last_wall = stat.last_excluded_wall_s
|
|
673
|
+
return {
|
|
674
|
+
"status": "ok" if last_status == "ok" else "unavailable",
|
|
675
|
+
"reason": None if last_status == "ok" else last_status,
|
|
676
|
+
"source": state.counter.source,
|
|
677
|
+
"scope": SCOPE,
|
|
678
|
+
"instanceId": None if local is None else local["instanceId"],
|
|
679
|
+
"sampledAt": _budget.iso_z(wall),
|
|
680
|
+
"windowSeconds": (None if stat.window_seconds is None
|
|
681
|
+
else round(stat.window_seconds, 3)),
|
|
682
|
+
"bytesWritten": stat.bytes_written,
|
|
683
|
+
"bytesPerMinute": stat.bytes_per_minute,
|
|
684
|
+
"tickCount": stat.publications,
|
|
685
|
+
"meanBytesPerTick": stat.mean_bytes_per_publication,
|
|
686
|
+
"excludedDeletions": {
|
|
687
|
+
"operations": stat.excluded_operations,
|
|
688
|
+
"rows": stat.excluded_rows,
|
|
689
|
+
"bytes": stat.excluded_bytes,
|
|
690
|
+
"lastEndedAt": (None if last_wall is None else _budget.iso_z(
|
|
691
|
+
dt.datetime.fromtimestamp(last_wall, UTC))),
|
|
692
|
+
},
|
|
693
|
+
"coldStartup": {"bytes": cold, "admitted": admitted},
|
|
694
|
+
"maintenance": maintenance,
|
|
695
|
+
"reclaimRefusal": reclaim_refusal(),
|
|
696
|
+
"budget": {
|
|
697
|
+
"policyVersion": limits.policy_version,
|
|
698
|
+
"verdict": verdict,
|
|
699
|
+
"reasons": list(reasons),
|
|
700
|
+
"bytesPerMinuteLimit": limits.bytes_per_minute,
|
|
701
|
+
"meanBytesPerTickLimit": limits.bytes_per_publication,
|
|
702
|
+
},
|
|
703
|
+
}
|