cctally 1.95.4 → 1.96.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +71 -0
- package/README.md +4 -5
- package/bin/_cctally_config.py +46 -22
- package/bin/_cctally_core.py +125 -42
- package/bin/_cctally_dashboard.py +258 -100
- package/bin/_cctally_dashboard_cache_report.py +24 -1
- package/bin/_cctally_dashboard_envelope.py +11 -0
- package/bin/_cctally_dashboard_share.py +38 -4
- package/bin/_cctally_db.py +1 -1
- package/bin/_cctally_journal.py +272 -20
- package/bin/_cctally_parser.py +14 -3
- package/bin/_cctally_refresh.py +12 -2
- package/bin/_cctally_reporting.py +12 -8
- package/bin/_cctally_share.py +86 -12
- package/bin/_cctally_store.py +26 -21
- package/bin/_cctally_tui.py +11 -5
- package/bin/_lib_dashboard_settings_contract.py +86 -0
- package/bin/_lib_journal.py +4 -2
- package/bin/_lib_render.py +47 -20
- package/bin/_lib_share.py +191 -48
- package/bin/cctally +5 -1
- package/dashboard/static/assets/index-Bhr5gZ14.js +97 -0
- package/dashboard/static/assets/index-DfN_fsLZ.css +1 -0
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +2 -1
- package/dashboard/static/assets/index-BvCbpJJA.css +0 -1
- package/dashboard/static/assets/index-CRZMxeYI.js +0 -97
|
@@ -1283,6 +1283,14 @@ def _share_source_selection(req: dict) -> tuple[str, bool]:
|
|
|
1283
1283
|
return source, explicit
|
|
1284
1284
|
|
|
1285
1285
|
|
|
1286
|
+
def _share_json_bool(payload: dict, key: str, *, default: bool = False) -> bool:
|
|
1287
|
+
"""Read a strict JSON boolean instead of applying Python truthiness."""
|
|
1288
|
+
value = payload.get(key, default)
|
|
1289
|
+
if not isinstance(value, bool):
|
|
1290
|
+
raise TypeError(key)
|
|
1291
|
+
return value
|
|
1292
|
+
|
|
1293
|
+
|
|
1286
1294
|
_ACCOUNT_KEY_RE = re.compile(r"[0-9a-f]{32}|unattributed|\*")
|
|
1287
1295
|
|
|
1288
1296
|
|
|
@@ -2182,6 +2190,14 @@ def _handle_share_render_post_impl(handler) -> None:
|
|
|
2182
2190
|
"field": "options.theme",
|
|
2183
2191
|
})
|
|
2184
2192
|
return
|
|
2193
|
+
try:
|
|
2194
|
+
reveal = _share_json_bool(options, "reveal_projects")
|
|
2195
|
+
except TypeError:
|
|
2196
|
+
handler._respond_json(400, {
|
|
2197
|
+
"error": "reveal_projects must be a boolean",
|
|
2198
|
+
"field": "options.reveal_projects",
|
|
2199
|
+
})
|
|
2200
|
+
return
|
|
2185
2201
|
# `top_n` may be explicit-null when the UI's Top-N input is
|
|
2186
2202
|
# cleared (Knobs.tsx:43); treat null as "use template default"
|
|
2187
2203
|
# rather than 400-ing every preview/export until the user types
|
|
@@ -2258,7 +2274,6 @@ def _handle_share_render_post_impl(handler) -> None:
|
|
|
2258
2274
|
# a default belongs here; it must resolve to anonymize, matching
|
|
2259
2275
|
# /api/share/compose, which already defaulted closed. The kernel itself
|
|
2260
2276
|
# has no default at all, so a fourth site cannot get this wrong.
|
|
2261
|
-
reveal = bool(options.get("reveal_projects", False))
|
|
2262
2277
|
# No pre-scrub: the kernel's `render()` / `compose()` own the privacy
|
|
2263
2278
|
# contract and require RAW snapshots (#503 S1). Pre-scrubbing here
|
|
2264
2279
|
# renumbers aliases on the legacy path, and in the `source=all` branch it
|
|
@@ -2409,7 +2424,14 @@ def _handle_share_compose_post_impl(handler) -> None:
|
|
|
2409
2424
|
theme = req.get("theme", "light")
|
|
2410
2425
|
fmt = req.get("format", "html")
|
|
2411
2426
|
no_branding = bool(req.get("no_branding", False))
|
|
2412
|
-
|
|
2427
|
+
try:
|
|
2428
|
+
reveal_projects = _share_json_bool(req, "reveal_projects")
|
|
2429
|
+
except TypeError:
|
|
2430
|
+
handler._respond_json(400, {
|
|
2431
|
+
"error": "reveal_projects must be a boolean",
|
|
2432
|
+
"field": "reveal_projects",
|
|
2433
|
+
})
|
|
2434
|
+
return
|
|
2413
2435
|
sections_in = req.get("sections")
|
|
2414
2436
|
if not isinstance(title, str) or not title:
|
|
2415
2437
|
handler._respond_json(400, {"error": "missing title", "field": "title"})
|
|
@@ -2769,7 +2791,13 @@ def _handle_share_presets_post_impl(handler) -> None:
|
|
|
2769
2791
|
# #503 S3 §1. Absent means false, which is the fail-safe direction and is
|
|
2770
2792
|
# what makes this compatible with a caller written before the field
|
|
2771
2793
|
# existed: an unwitting save can no longer destroy a stored recipe.
|
|
2772
|
-
|
|
2794
|
+
try:
|
|
2795
|
+
overwrite = _share_json_bool(req, "overwrite")
|
|
2796
|
+
except TypeError:
|
|
2797
|
+
handler._respond_json(400, {
|
|
2798
|
+
"error": "overwrite must be a boolean", "field": "overwrite",
|
|
2799
|
+
})
|
|
2800
|
+
return
|
|
2773
2801
|
|
|
2774
2802
|
saved_at = _share_now_utc_iso()
|
|
2775
2803
|
record = {
|
|
@@ -2867,7 +2895,13 @@ def _handle_share_presets_rename_post_impl(handler) -> None:
|
|
|
2867
2895
|
panel = req.get("panel")
|
|
2868
2896
|
from_name = req.get("from_name")
|
|
2869
2897
|
to_name = req.get("to_name")
|
|
2870
|
-
|
|
2898
|
+
try:
|
|
2899
|
+
overwrite = _share_json_bool(req, "overwrite")
|
|
2900
|
+
except TypeError:
|
|
2901
|
+
handler._respond_json(400, {
|
|
2902
|
+
"error": "overwrite must be a boolean", "field": "overwrite",
|
|
2903
|
+
})
|
|
2904
|
+
return
|
|
2871
2905
|
|
|
2872
2906
|
if not isinstance(panel, str) or not panel:
|
|
2873
2907
|
handler._respond_json(400, {
|
package/bin/_cctally_db.py
CHANGED
|
@@ -9290,7 +9290,7 @@ def _process_start_identity(pid: int) -> "str | None":
|
|
|
9290
9290
|
stderr=subprocess.DEVNULL,
|
|
9291
9291
|
text=True,
|
|
9292
9292
|
check=False,
|
|
9293
|
-
env={**os.environ, "LC_ALL": "C"},
|
|
9293
|
+
env={**os.environ, "LC_ALL": "C", "TZ": "UTC"},
|
|
9294
9294
|
)
|
|
9295
9295
|
except OSError:
|
|
9296
9296
|
return None
|
package/bin/_cctally_journal.py
CHANGED
|
@@ -35,6 +35,7 @@ import os
|
|
|
35
35
|
import pathlib
|
|
36
36
|
import signal
|
|
37
37
|
import sqlite3
|
|
38
|
+
import stat
|
|
38
39
|
import sys
|
|
39
40
|
import time
|
|
40
41
|
from dataclasses import dataclass, field, replace as _dc_replace
|
|
@@ -326,6 +327,134 @@ def _utc_now() -> dt.datetime:
|
|
|
326
327
|
return dt.datetime.now(dt.timezone.utc)
|
|
327
328
|
|
|
328
329
|
|
|
330
|
+
def _observation_segment_month(name: str) -> tuple[int, int] | None:
|
|
331
|
+
"""Return a strict ``(year, month)`` for one monthly observation name."""
|
|
332
|
+
prefix = _lib_journal.SEGMENT_PREFIX
|
|
333
|
+
suffix = ".jsonl"
|
|
334
|
+
if not name.startswith(prefix) or not name.endswith(suffix):
|
|
335
|
+
return None
|
|
336
|
+
stamp = name[len(prefix):-len(suffix)]
|
|
337
|
+
if len(stamp) != 7 or stamp[4] != "-":
|
|
338
|
+
return None
|
|
339
|
+
year_text, month_text = stamp.split("-", 1)
|
|
340
|
+
ascii_digits = frozenset("0123456789")
|
|
341
|
+
if (
|
|
342
|
+
any(char not in ascii_digits for char in year_text)
|
|
343
|
+
or any(char not in ascii_digits for char in month_text)
|
|
344
|
+
):
|
|
345
|
+
return None
|
|
346
|
+
year = int(year_text)
|
|
347
|
+
month = int(month_text)
|
|
348
|
+
if year < 1 or not 1 <= month <= 12:
|
|
349
|
+
return None
|
|
350
|
+
return (year, month)
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
def _future_segment_refusal(
|
|
354
|
+
journal_dir, seg_name: str, future_name: str, reason: str
|
|
355
|
+
) -> JournalAppendTargetStale:
|
|
356
|
+
return JournalAppendTargetStale(
|
|
357
|
+
f"append target {seg_name} is blocked by later segment {future_name} "
|
|
358
|
+
f"in {journal_dir}: {reason}; automatic recovery is limited to empty "
|
|
359
|
+
"regular files. Preserve the future segment and either wait for its "
|
|
360
|
+
"UTC month or merge its records forward, verify `cctally db rebuild "
|
|
361
|
+
"--db stats`, and only then remove it"
|
|
362
|
+
)
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def _list_append_target_candidates(journal_dir, seg_name: str) -> list[str]:
|
|
366
|
+
"""Canonical-looking entries, including types recovery must reject.
|
|
367
|
+
|
|
368
|
+
``list_segments`` intentionally exposes readable regular segment files to
|
|
369
|
+
journal consumers. Append validation has a stricter responsibility: a
|
|
370
|
+
directory, symlink, or uninspectable path with a canonical-looking future
|
|
371
|
+
name must fail closed instead of disappearing from the recovery decision.
|
|
372
|
+
"""
|
|
373
|
+
try:
|
|
374
|
+
names = [
|
|
375
|
+
entry.name
|
|
376
|
+
for entry in journal_dir.iterdir()
|
|
377
|
+
if entry.name.endswith(".jsonl")
|
|
378
|
+
and (
|
|
379
|
+
entry.name.startswith(_lib_journal.BOOTSTRAP_PREFIX)
|
|
380
|
+
or entry.name.startswith(_lib_journal.SEGMENT_PREFIX)
|
|
381
|
+
)
|
|
382
|
+
]
|
|
383
|
+
except OSError as exc:
|
|
384
|
+
raise JournalAppendTargetStale(
|
|
385
|
+
f"append target {seg_name} cannot inspect journal entries in "
|
|
386
|
+
f"{journal_dir} ({exc}); automatic recovery is limited to "
|
|
387
|
+
"inspectable empty regular files"
|
|
388
|
+
) from exc
|
|
389
|
+
return sorted(names, key=_lib_journal.segment_sort_key)
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
def _remove_empty_future_segments(
|
|
393
|
+
journal_dir, seg_name: str, segments: list[str]
|
|
394
|
+
) -> list[str]:
|
|
395
|
+
"""Remove a restored empty future tail while holding ``journal.lock``.
|
|
396
|
+
|
|
397
|
+
The caller has already established that ``seg_name`` is stale. Every later
|
|
398
|
+
segment is inspected before any unlink, so a mixed empty/data-bearing tail
|
|
399
|
+
remains untouched. All supported writers take the same leaf lock; this is
|
|
400
|
+
therefore a complete concurrency boundary for journal mutations.
|
|
401
|
+
"""
|
|
402
|
+
target_month = _observation_segment_month(seg_name)
|
|
403
|
+
if target_month is None:
|
|
404
|
+
return segments
|
|
405
|
+
target_key = _lib_journal.segment_sort_key(seg_name)
|
|
406
|
+
later = [
|
|
407
|
+
name
|
|
408
|
+
for name in segments
|
|
409
|
+
if _lib_journal.segment_sort_key(name) > target_key
|
|
410
|
+
]
|
|
411
|
+
if not later:
|
|
412
|
+
return segments
|
|
413
|
+
|
|
414
|
+
removable: list[pathlib.Path] = []
|
|
415
|
+
for name in later:
|
|
416
|
+
month = _observation_segment_month(name)
|
|
417
|
+
path = journal_dir / name
|
|
418
|
+
if month is None or month <= target_month:
|
|
419
|
+
raise _future_segment_refusal(
|
|
420
|
+
journal_dir, seg_name, name, "the segment name is not a later UTC month"
|
|
421
|
+
)
|
|
422
|
+
try:
|
|
423
|
+
metadata = path.lstat()
|
|
424
|
+
except OSError as exc:
|
|
425
|
+
raise _future_segment_refusal(
|
|
426
|
+
journal_dir, seg_name, name, f"the segment cannot be inspected ({exc})"
|
|
427
|
+
) from exc
|
|
428
|
+
if not stat.S_ISREG(metadata.st_mode) or metadata.st_size != 0:
|
|
429
|
+
detail = (
|
|
430
|
+
f"the segment retains {metadata.st_size} bytes"
|
|
431
|
+
if stat.S_ISREG(metadata.st_mode)
|
|
432
|
+
else "the path is not a regular file"
|
|
433
|
+
)
|
|
434
|
+
raise _future_segment_refusal(
|
|
435
|
+
journal_dir, seg_name, name, detail
|
|
436
|
+
)
|
|
437
|
+
removable.append(path)
|
|
438
|
+
|
|
439
|
+
removed = False
|
|
440
|
+
try:
|
|
441
|
+
for path in removable:
|
|
442
|
+
try:
|
|
443
|
+
path.unlink()
|
|
444
|
+
removed = True
|
|
445
|
+
except OSError as exc:
|
|
446
|
+
raise _future_segment_refusal(
|
|
447
|
+
journal_dir,
|
|
448
|
+
seg_name,
|
|
449
|
+
path.name,
|
|
450
|
+
f"the empty segment cannot be removed ({exc})",
|
|
451
|
+
) from exc
|
|
452
|
+
finally:
|
|
453
|
+
if removed:
|
|
454
|
+
_fsync_dir(journal_dir)
|
|
455
|
+
return _list_append_target_candidates(journal_dir, seg_name)
|
|
456
|
+
|
|
457
|
+
|
|
329
458
|
def _validate_append_target(journal_dir, seg_name: str) -> None:
|
|
330
459
|
"""Refuse an append whose target is not the canonically-last segment.
|
|
331
460
|
|
|
@@ -343,8 +472,13 @@ def _validate_append_target(journal_dir, seg_name: str) -> None:
|
|
|
343
472
|
a writer retry against a freshly resolved target, where silently
|
|
344
473
|
redirecting a planned correction group would move it out from under a
|
|
345
474
|
caller that had already reasoned about its placement (#496 S5b §2.4).
|
|
475
|
+
|
|
476
|
+
A restored future tail is the one non-transient case (#512). Empty regular
|
|
477
|
+
future segments carry no durable truth, so they are removed and the target
|
|
478
|
+
is revalidated. Any data-bearing, malformed, or non-regular later segment
|
|
479
|
+
still fails closed with the manual recovery boundary in the error.
|
|
346
480
|
"""
|
|
347
|
-
segments =
|
|
481
|
+
segments = _list_append_target_candidates(journal_dir, seg_name)
|
|
348
482
|
if not segments:
|
|
349
483
|
return
|
|
350
484
|
if seg_name == segments[-1]:
|
|
@@ -354,6 +488,14 @@ def _validate_append_target(journal_dir, seg_name: str) -> None:
|
|
|
354
488
|
[*segments, seg_name], key=_lib_journal.segment_sort_key)
|
|
355
489
|
if provisional[-1] == seg_name:
|
|
356
490
|
return
|
|
491
|
+
segments = _remove_empty_future_segments(journal_dir, seg_name, segments)
|
|
492
|
+
if not segments or seg_name == segments[-1]:
|
|
493
|
+
return
|
|
494
|
+
if seg_name not in segments:
|
|
495
|
+
provisional = sorted(
|
|
496
|
+
[*segments, seg_name], key=_lib_journal.segment_sort_key)
|
|
497
|
+
if provisional[-1] == seg_name:
|
|
498
|
+
return
|
|
357
499
|
raise JournalAppendTargetStale(
|
|
358
500
|
f"append target {seg_name} is not the canonically-last segment "
|
|
359
501
|
f"({segments[-1]}) in {journal_dir}; re-resolve and retry"
|
|
@@ -819,7 +961,7 @@ def plan_segment_elision(segments, high_water):
|
|
|
819
961
|
summaries=summaries, covered=covered, verdict=reason)
|
|
820
962
|
|
|
821
963
|
|
|
822
|
-
def _refill_elided_quota_raw(quota_raw, gaps):
|
|
964
|
+
def _refill_elided_quota_raw(quota_raw, gaps, *, failures=None):
|
|
823
965
|
"""``(stream, complete)`` — ``quota_raw`` with the elided observations back.
|
|
824
966
|
|
|
825
967
|
Reached only when the pass elided and the cache leg's own coverage verdict
|
|
@@ -840,7 +982,10 @@ def _refill_elided_quota_raw(quota_raw, gaps):
|
|
|
840
982
|
|
|
841
983
|
``complete`` is False when any elided segment could not be re-read IN FULL.
|
|
842
984
|
The caller must then drop its covered boundary: the observations this stream
|
|
843
|
-
is missing are exactly the ones a minted certificate would claim.
|
|
985
|
+
is missing are exactly the ones a minted certificate would claim. When a
|
|
986
|
+
``failures`` list is supplied, every caught exception appends its segment
|
|
987
|
+
name and exception class so the rebuild record distinguishes transient I/O
|
|
988
|
+
failures from implementation defects.
|
|
844
989
|
|
|
845
990
|
**A short read is detected by counting, not by catching.**
|
|
846
991
|
`_iter_segment_lines` reads until its own `read()` returns nothing, so a
|
|
@@ -851,6 +996,8 @@ def _refill_elided_quota_raw(quota_raw, gaps):
|
|
|
851
996
|
"""
|
|
852
997
|
by_index: dict = {}
|
|
853
998
|
complete = True
|
|
999
|
+
if failures is None:
|
|
1000
|
+
failures = []
|
|
854
1001
|
for name, index, extent, expected_lines in gaps:
|
|
855
1002
|
recovered = []
|
|
856
1003
|
seen = 0
|
|
@@ -861,7 +1008,7 @@ def _refill_elided_quota_raw(quota_raw, gaps):
|
|
|
861
1008
|
record = _lib_journal.decode_line(raw)
|
|
862
1009
|
if record is not None and _is_codex_quota_obs(record):
|
|
863
1010
|
recovered.append(raw)
|
|
864
|
-
except Exception:
|
|
1011
|
+
except Exception as exc:
|
|
865
1012
|
# A vanished segment self-heals, because the pinned vector no longer
|
|
866
1013
|
# matches the journal and the certificate is invalid the moment it
|
|
867
1014
|
# is read. A transient read error on an UNCHANGED file does not: the
|
|
@@ -874,6 +1021,10 @@ def _refill_elided_quota_raw(quota_raw, gaps):
|
|
|
874
1021
|
# call site nor its caller catches anything, so one that escaped
|
|
875
1022
|
# would abort the whole rebuild over a re-derivable optimization.
|
|
876
1023
|
complete = False
|
|
1024
|
+
failures.append({
|
|
1025
|
+
"segment": str(name),
|
|
1026
|
+
"error": type(exc).__name__,
|
|
1027
|
+
})
|
|
877
1028
|
continue
|
|
878
1029
|
if seen != int(expected_lines):
|
|
879
1030
|
complete = False
|
|
@@ -4635,6 +4786,7 @@ def _validate_excluded_derived_fks(conn, spec, row) -> None:
|
|
|
4635
4786
|
def _full_effective_selection(hw, accumulators=None):
|
|
4636
4787
|
records = []
|
|
4637
4788
|
evidence = []
|
|
4789
|
+
coordinates = {}
|
|
4638
4790
|
prior_high_water = None
|
|
4639
4791
|
if hw is not None:
|
|
4640
4792
|
for segment, offset, raw in _read_range(None, hw):
|
|
@@ -4645,16 +4797,25 @@ def _full_effective_selection(hw, accumulators=None):
|
|
|
4645
4797
|
prior_high_water,
|
|
4646
4798
|
evidence,
|
|
4647
4799
|
)
|
|
4800
|
+
# Only correction records can establish a completed-to-tainted
|
|
4801
|
+
# transition. Keep their exact sequence coordinates without
|
|
4802
|
+
# adding one tuple for every observation in a full fallback.
|
|
4803
|
+
if record.get("t") in {"correction", "correction_batch"}:
|
|
4804
|
+
coordinates[len(records)] = (
|
|
4805
|
+
segment, offset + len(raw) + 1)
|
|
4648
4806
|
records.append(record)
|
|
4649
4807
|
prior_high_water = (segment, offset + len(raw) + 1)
|
|
4650
4808
|
cutover_claude = resolve_cutover_claude_account()
|
|
4651
4809
|
for record in records:
|
|
4652
4810
|
_normalize_legacy_account_stamp(record, cutover_claude)
|
|
4653
|
-
|
|
4811
|
+
selected = _lib_journal.resolve_effective_events(
|
|
4654
4812
|
records,
|
|
4655
4813
|
protocol_prefix_evidence=evidence,
|
|
4656
4814
|
accumulators=accumulators,
|
|
4657
4815
|
)
|
|
4816
|
+
if accumulators is not None:
|
|
4817
|
+
accumulators["coordinates"] = coordinates
|
|
4818
|
+
return selected
|
|
4658
4819
|
|
|
4659
4820
|
|
|
4660
4821
|
def _selector_generation_matches(conn, state) -> bool:
|
|
@@ -4765,15 +4926,12 @@ def _coordinate_covers(covered, target) -> bool:
|
|
|
4765
4926
|
#: — is decided BEFORE the gap read and the next successful tick writes the
|
|
4766
4927
|
#: realigned prefix. One refusal is not the shape this cap exists for.
|
|
4767
4928
|
#:
|
|
4768
|
-
#: The shape it exists for is
|
|
4769
|
-
#:
|
|
4770
|
-
#:
|
|
4771
|
-
#:
|
|
4772
|
-
#:
|
|
4773
|
-
#:
|
|
4774
|
-
#: durable prefix. Every later tick would then re-read and re-decode a
|
|
4775
|
-
#: monotonically growing range, which is the whole-prefix read spec §3.3 exists
|
|
4776
|
-
#: to prevent under another name.
|
|
4929
|
+
#: The shape it exists for is any refusal that repeats. A completed-to-tainted
|
|
4930
|
+
#: batch now signals from both the incremental and full-selection paths (#510),
|
|
4931
|
+
#: but other conservative full-selection fallbacks can still leave durable
|
|
4932
|
+
#: selector coverage behind the applied cursor. Every later tick would then
|
|
4933
|
+
#: re-read and re-decode a monotonically growing range, which is the whole-prefix
|
|
4934
|
+
#: read spec §3.3 exists to prevent under another name.
|
|
4777
4935
|
#:
|
|
4778
4936
|
#: Past the cap the tick degrades to full selection like every other degraded
|
|
4779
4937
|
#: case, and `_observe_selector_desynchronization` reports the gap and this cap
|
|
@@ -5059,6 +5217,42 @@ def _raise_taint_transition(transition, coordinates) -> None:
|
|
|
5059
5217
|
)
|
|
5060
5218
|
|
|
5061
5219
|
|
|
5220
|
+
def _full_taint_transition(conn, selection, accumulators):
|
|
5221
|
+
"""Find a stored corrected batch withdrawn by a full selection (#510)."""
|
|
5222
|
+
stored_batches = {
|
|
5223
|
+
row[0] for row in conn.execute(
|
|
5224
|
+
"SELECT DISTINCT batch_id FROM journal_effective_events "
|
|
5225
|
+
"WHERE batch_id IS NOT NULL"
|
|
5226
|
+
)
|
|
5227
|
+
}
|
|
5228
|
+
stale_batches = stored_batches - set(selection.completed_batches)
|
|
5229
|
+
if not stale_batches:
|
|
5230
|
+
return None
|
|
5231
|
+
|
|
5232
|
+
fold = accumulators["fold"]
|
|
5233
|
+
transitions = []
|
|
5234
|
+
for batch_id, _kind, fingerprint in fold.violations:
|
|
5235
|
+
if batch_id not in stale_batches:
|
|
5236
|
+
continue
|
|
5237
|
+
causal = fold.violation_available_after.get(fingerprint)
|
|
5238
|
+
if causal is not None:
|
|
5239
|
+
transitions.append(
|
|
5240
|
+
_lib_selector_state.TaintTransition(
|
|
5241
|
+
batch_id=batch_id,
|
|
5242
|
+
causal_sequence=causal,
|
|
5243
|
+
)
|
|
5244
|
+
)
|
|
5245
|
+
if not transitions:
|
|
5246
|
+
raise JournalError(
|
|
5247
|
+
"a previously completed correction batch is no longer effective "
|
|
5248
|
+
"but its causal taint record could not be resolved"
|
|
5249
|
+
)
|
|
5250
|
+
return min(
|
|
5251
|
+
transitions,
|
|
5252
|
+
key=lambda item: (item.causal_sequence, item.batch_id),
|
|
5253
|
+
)
|
|
5254
|
+
|
|
5255
|
+
|
|
5062
5256
|
def _correction_commit_high_water(batch_id, hw=None):
|
|
5063
5257
|
"""Return the exact end offset of one completed-batch commit marker.
|
|
5064
5258
|
|
|
@@ -5289,6 +5483,9 @@ def _preflight_live_events(
|
|
|
5289
5483
|
recovery_eligible=True,
|
|
5290
5484
|
kind=CORRECTION_KIND_NEWLY_COMPLETED,
|
|
5291
5485
|
)
|
|
5486
|
+
transition = _full_taint_transition(conn, full, accumulators)
|
|
5487
|
+
if transition is not None:
|
|
5488
|
+
_raise_taint_transition(transition, accumulators["coordinates"])
|
|
5292
5489
|
return to_apply
|
|
5293
5490
|
|
|
5294
5491
|
|
|
@@ -6833,6 +7030,10 @@ def _pending_publication_owes_nothing(destination, state) -> bool:
|
|
|
6833
7030
|
|
|
6834
7031
|
A marker written before the mechanism field existed reads as `replace`,
|
|
6835
7032
|
which is what those binaries did.
|
|
7033
|
+
|
|
7034
|
+
A replace marker with no usable scratch path proves neither outcome. It
|
|
7035
|
+
therefore owes a verdict: fail closed instead of discarding the marker as
|
|
7036
|
+
though the predecessor were proven.
|
|
6836
7037
|
"""
|
|
6837
7038
|
if str(state.get("mechanism") or "replace") == "in_place":
|
|
6838
7039
|
return in_place_publication_proven_predecessor(destination, state)
|
|
@@ -8406,7 +8607,9 @@ def _rebuild_quota_cache_leg_raw(
|
|
|
8406
8607
|
# built. Elision then costs one read in the racy case rather than an
|
|
8407
8608
|
# unmaterialized observation under a certificate claiming coverage.
|
|
8408
8609
|
before = len(quota_raw)
|
|
8409
|
-
|
|
8610
|
+
refill_failures: list = []
|
|
8611
|
+
quota_raw, refilled = _refill_elided_quota_raw(
|
|
8612
|
+
quota_raw, elision_gaps, failures=refill_failures)
|
|
8410
8613
|
if not refilled:
|
|
8411
8614
|
# A segment this pass elided could not be re-read, so its
|
|
8412
8615
|
# observations are absent from the stream about to be replayed.
|
|
@@ -8421,6 +8624,7 @@ def _rebuild_quota_cache_leg_raw(
|
|
|
8421
8624
|
"segments": len(elision_gaps),
|
|
8422
8625
|
"observations": len(quota_raw) - before,
|
|
8423
8626
|
"complete": refilled,
|
|
8627
|
+
"failures": refill_failures,
|
|
8424
8628
|
}
|
|
8425
8629
|
return _run_bounded_recovery(
|
|
8426
8630
|
quota_raw, file_accounts, cutover_claude, counters,
|
|
@@ -9785,9 +9989,55 @@ def _export_quota_obs() -> list:
|
|
|
9785
9989
|
return out
|
|
9786
9990
|
|
|
9787
9991
|
|
|
9788
|
-
def _cutover_segment_name(
|
|
9789
|
-
|
|
9790
|
-
|
|
9992
|
+
def _cutover_segment_name(
|
|
9993
|
+
now_utc: dt.datetime, *, existing: "list[str] | tuple[str, ...]" = ()
|
|
9994
|
+
) -> str:
|
|
9995
|
+
"""Mint a bootstrap name that sorts after every published bootstrap.
|
|
9996
|
+
|
|
9997
|
+
Wall time supplies the ordinary name. A crash orphan can be later than a
|
|
9998
|
+
retry after a backward clock step, so a non-newest candidate is re-minted
|
|
9999
|
+
one microsecond after the canonically newest published bootstrap (#509).
|
|
10000
|
+
Fail closed when that name is not parseable or cannot be advanced: writing
|
|
10001
|
+
behind it would leave the segment outside the cursor, while reusing its name
|
|
10002
|
+
would overwrite append-only journal truth.
|
|
10003
|
+
"""
|
|
10004
|
+
def _name(value: dt.datetime) -> str:
|
|
10005
|
+
ts = value.astimezone(dt.timezone.utc).strftime("%Y%m%dT%H%M%S_%f")
|
|
10006
|
+
return f"{_lib_journal.BOOTSTRAP_PREFIX}{ts}.jsonl"
|
|
10007
|
+
|
|
10008
|
+
candidate = _name(now_utc)
|
|
10009
|
+
bootstraps = [
|
|
10010
|
+
name for name in existing
|
|
10011
|
+
if name.startswith(_lib_journal.BOOTSTRAP_PREFIX)
|
|
10012
|
+
]
|
|
10013
|
+
if not bootstraps:
|
|
10014
|
+
return candidate
|
|
10015
|
+
newest = max(bootstraps, key=_lib_journal.segment_sort_key)
|
|
10016
|
+
if (_lib_journal.segment_sort_key(candidate)
|
|
10017
|
+
> _lib_journal.segment_sort_key(newest)):
|
|
10018
|
+
return candidate
|
|
10019
|
+
|
|
10020
|
+
prefix = _lib_journal.BOOTSTRAP_PREFIX
|
|
10021
|
+
suffix = ".jsonl"
|
|
10022
|
+
try:
|
|
10023
|
+
if not newest.endswith(suffix):
|
|
10024
|
+
raise ValueError("missing .jsonl suffix")
|
|
10025
|
+
newest_at = dt.datetime.strptime(
|
|
10026
|
+
newest[len(prefix):-len(suffix)], "%Y%m%dT%H%M%S_%f"
|
|
10027
|
+
).replace(tzinfo=dt.timezone.utc)
|
|
10028
|
+
reminted = _name(newest_at + dt.timedelta(microseconds=1))
|
|
10029
|
+
except (OverflowError, ValueError) as exc:
|
|
10030
|
+
raise JournalError(
|
|
10031
|
+
"cannot mint a cutover bootstrap after the canonically newest "
|
|
10032
|
+
f"published segment {newest!r}"
|
|
10033
|
+
) from exc
|
|
10034
|
+
if (_lib_journal.segment_sort_key(reminted)
|
|
10035
|
+
<= _lib_journal.segment_sort_key(newest)):
|
|
10036
|
+
raise JournalError(
|
|
10037
|
+
"cutover bootstrap remint did not advance canonical order past "
|
|
10038
|
+
f"{newest!r}"
|
|
10039
|
+
)
|
|
10040
|
+
return reminted
|
|
9791
10041
|
|
|
9792
10042
|
|
|
9793
10043
|
def _encode_bootstrap_lines(lines: list) -> bytes:
|
|
@@ -9920,7 +10170,8 @@ def run_cutover(conn, *, now_utc: dt.datetime | None = None) -> "str | None":
|
|
|
9920
10170
|
rows byte for byte, so it reuses that orphan rather than publishing a twin
|
|
9921
10171
|
(#496 S5 §3) — the retry is now idempotent on disk, not only on fold. When
|
|
9922
10172
|
the retry's export genuinely differs, no digest matches and a new segment is
|
|
9923
|
-
|
|
10173
|
+
minted after every published bootstrap, including after a backward clock
|
|
10174
|
+
step (#509)."""
|
|
9924
10175
|
if now_utc is None:
|
|
9925
10176
|
now_utc = dt.datetime.now(dt.timezone.utc)
|
|
9926
10177
|
epoch = _cctally_core.STATS_INDEX_EPOCH
|
|
@@ -9946,7 +10197,8 @@ def run_cutover(conn, *, now_utc: dt.datetime | None = None) -> "str | None":
|
|
|
9946
10197
|
reuse = _reusable_bootstrap(
|
|
9947
10198
|
hashlib.sha256(blob).hexdigest(), len(blob))
|
|
9948
10199
|
if reuse is None:
|
|
9949
|
-
seg_name = _cutover_segment_name(
|
|
10200
|
+
seg_name = _cutover_segment_name(
|
|
10201
|
+
now_utc, existing=list_segments())
|
|
9950
10202
|
seg_size = _write_bootstrap_segment(seg_name, blob)
|
|
9951
10203
|
else:
|
|
9952
10204
|
seg_name, seg_size = reuse
|
package/bin/_cctally_parser.py
CHANGED
|
@@ -2015,7 +2015,9 @@ def _build_cache_report_parser(subparsers, name, *, help_text, xref=None, fixed_
|
|
|
2015
2015
|
default=15,
|
|
2016
2016
|
dest="anomaly_threshold_pp",
|
|
2017
2017
|
help="Claude cache %% drop threshold (percentage points) vs. a trailing "
|
|
2018
|
-
"median. Default: 15."
|
|
2018
|
+
"median. Default: 15. This flag does NOT read config.json:"
|
|
2019
|
+
" cache_report.anomaly_threshold_pp, which the dashboard writes"
|
|
2020
|
+
" and the dashboard and TUI read.",
|
|
2019
2021
|
)
|
|
2020
2022
|
pc.add_argument(
|
|
2021
2023
|
"--anomaly-window-days",
|
|
@@ -2464,10 +2466,14 @@ def _build_config_parser(subparsers, name, *, help_text, xref=None):
|
|
|
2464
2466
|
name,
|
|
2465
2467
|
help=help_text,
|
|
2466
2468
|
formatter_class=CLIHelpFormatter,
|
|
2469
|
+
# The key count is interpolated from the runtime tuple, never
|
|
2470
|
+
# written as a literal: a frozen number is exactly the drift #513
|
|
2471
|
+
# exists to remove, and the next key added would make this text
|
|
2472
|
+
# wrong with no check to catch it.
|
|
2467
2473
|
description=textwrap.dedent("""\
|
|
2468
2474
|
Manage cctally user preferences in ~/.local/share/cctally/config.json.
|
|
2469
2475
|
|
|
2470
|
-
|
|
2476
|
+
Commonly set keys:
|
|
2471
2477
|
display.tz Display timezone. Values: 'local' (default; host
|
|
2472
2478
|
zone via the OS locale), 'utc', or any IANA name
|
|
2473
2479
|
like 'America/New_York'. Per-call --tz flag on
|
|
@@ -2478,13 +2484,18 @@ def _build_config_parser(subparsers, name, *, help_text, xref=None):
|
|
|
2478
2484
|
loopback-only), 'lan' (binds 0.0.0.0 —
|
|
2479
2485
|
LAN-accessible), or any literal IP / hostname.
|
|
2480
2486
|
|
|
2487
|
+
Those three are a sample, not the set. {count} keys are settable in
|
|
2488
|
+
total; run `cctally config get` to list every one with its current
|
|
2489
|
+
value, or see docs/commands/config.md for values, defaults, and
|
|
2490
|
+
which keys the dashboard can write.
|
|
2491
|
+
|
|
2481
2492
|
Examples:
|
|
2482
2493
|
cctally config get
|
|
2483
2494
|
cctally config get display.tz
|
|
2484
2495
|
cctally config set display.tz America/New_York
|
|
2485
2496
|
cctally config set dashboard.bind lan
|
|
2486
2497
|
cctally config unset dashboard.bind
|
|
2487
|
-
"""),
|
|
2498
|
+
""").format(count=len(c.ALLOWED_CONFIG_KEYS)),
|
|
2488
2499
|
)
|
|
2489
2500
|
cfg_sub = cfg_p.add_subparsers(dest="action", required=True)
|
|
2490
2501
|
cfg_get = cfg_sub.add_parser("get", help="Print current value(s)")
|
package/bin/_cctally_refresh.py
CHANGED
|
@@ -487,16 +487,26 @@ def _get_oauth_usage_config(cfg: dict) -> dict:
|
|
|
487
487
|
# Statusline cache bust + freshness + rate-limit handler
|
|
488
488
|
# =========================================================================
|
|
489
489
|
|
|
490
|
-
|
|
490
|
+
# Kept as a module alias for backward compatibility with readers that import
|
|
491
|
+
# the name. The AUTHORITATIVE value is _cctally_core.STATUSLINE_OAUTH_CACHE_PATH,
|
|
492
|
+
# resolved at CALL time below.
|
|
493
|
+
_STATUSLINE_OAUTH_CACHE = _cctally_core.STATUSLINE_OAUTH_CACHE_PATH
|
|
491
494
|
|
|
492
495
|
|
|
493
|
-
def _bust_statusline_cache(path: str =
|
|
496
|
+
def _bust_statusline_cache(path: str | None = None) -> str:
|
|
494
497
|
"""Best-effort delete of the statusline OAuth cache file.
|
|
495
498
|
|
|
496
499
|
Returns one of: ``"busted"`` (file existed and was removed),
|
|
497
500
|
``"absent"`` (file did not exist), ``"error"`` (delete failed for
|
|
498
501
|
a non-FileNotFoundError reason — logged via eprint, does NOT raise).
|
|
502
|
+
|
|
503
|
+
``path`` defaults to None and is resolved from the kernel here rather than
|
|
504
|
+
in the signature: a default argument binds at import, so it could not be
|
|
505
|
+
redirected by patching either constant, and the only defence was a stub
|
|
506
|
+
replacing this whole function at each of ten call sites (#529 S4).
|
|
499
507
|
"""
|
|
508
|
+
if path is None:
|
|
509
|
+
path = _cctally_core.STATUSLINE_OAUTH_CACHE_PATH
|
|
500
510
|
try:
|
|
501
511
|
os.remove(path)
|
|
502
512
|
return "busted"
|
|
@@ -139,6 +139,7 @@ def cmd_daily(args: argparse.Namespace) -> int:
|
|
|
139
139
|
snap = c._build_daily_snapshot(
|
|
140
140
|
view, period_start=range_start, period_end=range_end,
|
|
141
141
|
display_tz=display_tz_str, version=c._share_resolve_version(),
|
|
142
|
+
since_explicit=getattr(args, "since", None) is not None,
|
|
142
143
|
)
|
|
143
144
|
if args.order == "desc":
|
|
144
145
|
snap = dataclasses.replace(snap, rows=tuple(reversed(snap.rows)))
|
|
@@ -153,14 +154,13 @@ def cmd_daily(args: argparse.Namespace) -> int:
|
|
|
153
154
|
)
|
|
154
155
|
json_groups: list = []
|
|
155
156
|
table_groups: list = []
|
|
156
|
-
#
|
|
157
|
-
#
|
|
158
|
-
#
|
|
159
|
-
#
|
|
160
|
-
#
|
|
161
|
-
#
|
|
162
|
-
#
|
|
163
|
-
# unique json_label, so section headers stay distinct too.
|
|
157
|
+
# The shared disambiguation kernel progressively qualifies paths
|
|
158
|
+
# and is total for indistinguishable inputs. Keep a defensive
|
|
159
|
+
# per-group counter anyway: JSON object keys must never rely on a
|
|
160
|
+
# label formatter remaining injective, or `projects[label] = ...`
|
|
161
|
+
# would silently overwrite an earlier group. The table label
|
|
162
|
+
# derives from the now-unique json_label, so section headers stay
|
|
163
|
+
# distinct too.
|
|
164
164
|
# `json_label`s are unique by construction (the `(#N)` counter
|
|
165
165
|
# above). Table labels, however, can re-collide: `_alias_for`
|
|
166
166
|
# matches on `display_key` first, so a basename alias like
|
|
@@ -248,6 +248,7 @@ def cmd_daily(args: argparse.Namespace) -> int:
|
|
|
248
248
|
period_end=range_end,
|
|
249
249
|
display_tz=display_tz_str,
|
|
250
250
|
version=c._share_resolve_version(),
|
|
251
|
+
since_explicit=getattr(args, "since", None) is not None,
|
|
251
252
|
)
|
|
252
253
|
if args.order == "desc":
|
|
253
254
|
snap = dataclasses.replace(snap, rows=tuple(reversed(snap.rows)))
|
|
@@ -322,6 +323,7 @@ def cmd_monthly(args: argparse.Namespace) -> int:
|
|
|
322
323
|
period_end=range_end,
|
|
323
324
|
display_tz=display_tz_str,
|
|
324
325
|
version=c._share_resolve_version(),
|
|
326
|
+
since_explicit=getattr(args, "since", None) is not None,
|
|
325
327
|
)
|
|
326
328
|
if args.order == "desc":
|
|
327
329
|
snap = dataclasses.replace(snap, rows=tuple(reversed(snap.rows)))
|
|
@@ -456,6 +458,7 @@ def cmd_weekly(args: argparse.Namespace) -> int:
|
|
|
456
458
|
display_tz=display_tz_str,
|
|
457
459
|
version=c._share_resolve_version(),
|
|
458
460
|
breakdown_model=bool(getattr(args, "breakdown", False)),
|
|
461
|
+
since_explicit=getattr(args, "since", None) is not None,
|
|
459
462
|
)
|
|
460
463
|
if args.order == "desc":
|
|
461
464
|
snap = dataclasses.replace(snap, rows=tuple(reversed(snap.rows)))
|
|
@@ -613,6 +616,7 @@ def cmd_session(args: argparse.Namespace) -> int:
|
|
|
613
616
|
version=c._share_resolve_version(),
|
|
614
617
|
top_n=top_n,
|
|
615
618
|
tz=tz,
|
|
619
|
+
since_explicit=getattr(args, "since", None) is not None,
|
|
616
620
|
)
|
|
617
621
|
c._share_render_and_emit(snap, args)
|
|
618
622
|
return 0
|