cctally 1.114.0 → 1.116.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +41 -0
- package/bin/_cctally_account.py +32 -8
- package/bin/_cctally_cache.py +4491 -295
- package/bin/_cctally_completion.py +1355 -0
- package/bin/_cctally_config.py +37 -0
- package/bin/_cctally_core.py +282 -9
- package/bin/_cctally_dashboard.py +8 -11
- package/bin/_cctally_dashboard_conversation.py +292 -23
- package/bin/_cctally_dashboard_envelope.py +42 -35
- package/bin/_cctally_dashboard_sources.py +18 -9
- package/bin/_cctally_db.py +401 -9
- package/bin/_cctally_doctor.py +469 -27
- package/bin/_cctally_five_hour.py +166 -1049
- package/bin/_cctally_forecast.py +81 -9
- package/bin/_cctally_journal.py +5076 -177
- package/bin/_cctally_journal_repair.py +110 -19
- package/bin/_cctally_oauth_samples.py +280 -0
- package/bin/_cctally_parser.py +47 -0
- package/bin/_cctally_project.py +8 -2
- package/bin/_cctally_quarantine.py +662 -0
- package/bin/_cctally_quota.py +96 -12
- package/bin/_cctally_quota_model.py +391 -83
- package/bin/_cctally_record.py +1088 -269
- package/bin/_cctally_rederive.py +854 -192
- package/bin/_cctally_refresh.py +113 -13
- package/bin/_cctally_source_analytics.py +23 -2
- package/bin/_cctally_statusline.py +7062 -505
- package/bin/_cctally_store.py +1244 -91
- package/bin/_cctally_transcript.py +8 -7
- package/bin/_cctally_tui.py +119 -60
- package/bin/_cctally_update.py +25 -0
- package/bin/_lib_account_binding.py +152 -0
- package/bin/_lib_account_protocol.py +569 -0
- package/bin/_lib_aggregators.py +4 -0
- package/bin/_lib_cache_coverage.py +13 -1
- package/bin/_lib_cache_writer_lock.py +19 -3
- package/bin/_lib_clone_copy.py +177 -0
- package/bin/_lib_codex_conversation_query.py +133 -19
- package/bin/_lib_codex_conversation_watch.py +59 -14
- package/bin/_lib_codex_drain_admission.py +214 -0
- package/bin/_lib_codex_metadata.py +18 -1
- package/bin/_lib_codex_publication_barrier.py +277 -0
- package/bin/_lib_codex_rollout_container.py +502 -0
- package/bin/_lib_conversation_dispatch.py +3 -0
- package/bin/_lib_conversation_query.py +11 -3
- package/bin/_lib_dashboard_sources.py +57 -15
- package/bin/_lib_detector_access_registry.py +4038 -0
- package/bin/_lib_doctor.py +753 -1
- package/bin/_lib_five_hour.py +126 -0
- package/bin/_lib_ingest_frontier.py +414 -20
- package/bin/_lib_journal.py +261 -10
- package/bin/_lib_journal_router.py +166 -48
- package/bin/_lib_jsonl.py +5 -4
- package/bin/_lib_legacy_spool.py +126 -0
- package/bin/_lib_legacy_writer_evidence.py +136 -0
- package/bin/_lib_meter_rate_change.py +10 -0
- package/bin/_lib_oauth_samples.py +238 -0
- package/bin/_lib_outcome_quarantine.py +690 -0
- package/bin/_lib_pricing.py +7 -2
- package/bin/_lib_pricing_check.py +0 -12
- package/bin/_lib_provider_activity.py +770 -0
- package/bin/_lib_quarantine.py +155 -0
- package/bin/_lib_quota_calibration.py +53 -8
- package/bin/_lib_quota_copy.py +2 -1
- package/bin/_lib_quota_model.py +384 -52
- package/bin/_lib_rebuild_progress.py +146 -0
- package/bin/_lib_record.py +22 -0
- package/bin/_lib_rederive.py +393 -8
- package/bin/_lib_segment_summary.py +35 -2
- package/bin/_lib_snapshot_cache.py +43 -1
- package/bin/_lib_stale_proof.py +708 -0
- package/bin/_lib_statusline_candidates.py +356 -27
- package/bin/_lib_worker_launches.py +315 -0
- package/bin/cctally +29 -12
- package/dashboard/static/assets/{ConversationsView-Bc8LCOAZ.js → ConversationsView-Dflb_myA.js} +1 -1
- package/dashboard/static/assets/DoctorModal-DFX5CfIs.js +1 -0
- package/dashboard/static/assets/{ModalRoot-DLPgyAM_.js → ModalRoot-HlxoagIa.js} +1 -1
- package/dashboard/static/assets/{ProjectsDrillPanel-z53a4v4M.js → ProjectsDrillPanel-C3PJy1xx.js} +1 -1
- package/dashboard/static/assets/{SourceDetailModal-Bw0T8W-W.js → SourceDetailModal-BczsWKhD.js} +1 -1
- package/dashboard/static/assets/{UpdateModal-D5AKx4Yw.js → UpdateModal-Os-Lo5cs.js} +1 -1
- package/dashboard/static/assets/{index-BXDOzSng.js → index-CYZ3NLqF.js} +3 -3
- package/dashboard/static/assets/{outlineNavigation-JObY7cUY.js → outlineNavigation-k-zIif6W.js} +1 -1
- package/dashboard/static/dashboard.html +1 -1
- package/package.json +20 -2
- package/bin/_lib_merge_gate.py +0 -725
- package/dashboard/static/assets/DoctorModal-D--qVBwe.js +0 -1
|
@@ -65,6 +65,7 @@ from _lib_dashboard_json import (
|
|
|
65
65
|
encode_dashboard_json_bytes_capped,
|
|
66
66
|
)
|
|
67
67
|
from _lib_retained_size import retained_size_bytes
|
|
68
|
+
import _lib_codex_rollout_container as _rc
|
|
68
69
|
|
|
69
70
|
|
|
70
71
|
def _advance_live_tail_accounting(handler, changed, sync, label):
|
|
@@ -443,40 +444,280 @@ def _cached_file_sigs(conn, paths):
|
|
|
443
444
|
return out
|
|
444
445
|
|
|
445
446
|
|
|
447
|
+
#: The cursor columns a Codex live-tail signature is computed from (#968 5.9).
|
|
448
|
+
_CODEX_WATCH_SIG_COLUMNS = (
|
|
449
|
+
"path, size_bytes, container_kind, container_size_bytes, mtime_ns, "
|
|
450
|
+
"device_id, inode, pending_container_fingerprint")
|
|
451
|
+
|
|
452
|
+
|
|
453
|
+
def _codex_watch_kernel():
|
|
454
|
+
return sys.modules["cctally"]._load_sibling("_lib_codex_conversation_watch")
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
def _codex_acknowledged_sig(row):
|
|
458
|
+
"""The live-tail signature one ``codex_conversation_source_files`` row
|
|
459
|
+
acknowledges (#968 spec 5.9).
|
|
460
|
+
|
|
461
|
+
A container already recorded as pending (or unverified) work is
|
|
462
|
+
acknowledged at its recorded fingerprint: the verification executor owns
|
|
463
|
+
it, and rescheduling it on every poll is the non-convergence I9 forbids.
|
|
464
|
+
Otherwise the accepted container's signature: ``("plain", size)`` for
|
|
465
|
+
plain, which is today's committed size, or the container identity for a
|
|
466
|
+
zstd container.
|
|
467
|
+
"""
|
|
468
|
+
_path, size, kind, csize, mtime_ns, device, inode, pending = row
|
|
469
|
+
if pending:
|
|
470
|
+
fingerprint = _rc.ContainerFingerprint.from_json(pending)
|
|
471
|
+
if fingerprint is not None:
|
|
472
|
+
return fingerprint.watch_sig()
|
|
473
|
+
return _codex_watch_kernel().committed_watch_sig(
|
|
474
|
+
kind, size, csize, mtime_ns, device, inode)
|
|
475
|
+
|
|
476
|
+
|
|
446
477
|
def _codex_cached_file_sigs(conn, paths):
|
|
447
|
-
"""{path:
|
|
448
|
-
Codex analogue of ``_cached_file_sigs`` (spec
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
Codex live-tail watch against the cache's own committed
|
|
452
|
-
during an ingest is re-detected next cycle (the per-path
|
|
453
|
-
invariant — the S6 physical mutation sequence is at most
|
|
454
|
-
certificate, never this cursor's replacement)."""
|
|
478
|
+
"""{path: acknowledged signature} from ``codex_conversation_source_files``
|
|
479
|
+
for the given paths — the Codex analogue of ``_cached_file_sigs`` (spec
|
|
480
|
+
§5.2), in the container-aware shape the Codex observation seam returns
|
|
481
|
+
(#968 spec 5.9). Paths with no row are absent → treated as changed.
|
|
482
|
+
Baselines the Codex live-tail watch against the cache's own committed
|
|
483
|
+
cursor so growth during an ingest is re-detected next cycle (the per-path
|
|
484
|
+
committed-cursor invariant — the S6 physical mutation sequence is at most
|
|
485
|
+
an extra certificate, never this cursor's replacement)."""
|
|
455
486
|
out = {}
|
|
456
487
|
if not paths:
|
|
457
488
|
return out
|
|
458
489
|
placeholders = ",".join("?" for _ in paths)
|
|
459
490
|
try:
|
|
460
491
|
rows = conn.execute(
|
|
461
|
-
f"SELECT
|
|
492
|
+
f"SELECT {_CODEX_WATCH_SIG_COLUMNS} "
|
|
493
|
+
f"FROM codex_conversation_source_files "
|
|
462
494
|
f"WHERE path IN ({placeholders})", list(paths)).fetchall()
|
|
463
495
|
except sqlite3.OperationalError:
|
|
464
496
|
return out
|
|
465
|
-
for
|
|
466
|
-
out[
|
|
497
|
+
for row in rows:
|
|
498
|
+
out[row[0]] = _codex_acknowledged_sig(row)
|
|
467
499
|
return out
|
|
468
500
|
|
|
469
501
|
|
|
470
502
|
def _codex_all_committed_sizes(conn):
|
|
471
|
-
"""{path:
|
|
472
|
-
both the ``known_paths`` diff set and the ``committed_sizes``
|
|
473
|
-
baseline
|
|
503
|
+
"""{path: acknowledged signature} for EVERY tracked Codex file — fed to the
|
|
504
|
+
frontier as both the ``known_paths`` diff set and the ``committed_sizes``
|
|
505
|
+
growth baseline, which the kernel compares with ``codex_sig_advanced``
|
|
506
|
+
(#968 spec 5.9). A plain SELECT (no lock), so it never widens the SSE lock
|
|
507
|
+
scope."""
|
|
474
508
|
try:
|
|
475
509
|
rows = conn.execute(
|
|
476
|
-
"SELECT
|
|
510
|
+
f"SELECT {_CODEX_WATCH_SIG_COLUMNS} "
|
|
511
|
+
"FROM codex_conversation_source_files").fetchall()
|
|
477
512
|
except sqlite3.OperationalError:
|
|
478
513
|
return {}
|
|
479
|
-
return {
|
|
514
|
+
return {row[0]: _codex_acknowledged_sig(row) for row in rows}
|
|
515
|
+
|
|
516
|
+
|
|
517
|
+
def _codex_seal_token(conn):
|
|
518
|
+
"""The publication barrier's seal token (Task 11's ``SEAL_SEQ_KEY``), read
|
|
519
|
+
through the attached cache; ``None`` when unreadable, which never re-arms
|
|
520
|
+
anything."""
|
|
521
|
+
barrier = sys.modules["cctally"]._load_sibling(
|
|
522
|
+
"_lib_codex_publication_barrier")
|
|
523
|
+
try:
|
|
524
|
+
row = conn.execute(
|
|
525
|
+
"SELECT value FROM cache_db.cache_meta WHERE key=?",
|
|
526
|
+
(barrier.SEAL_SEQ_KEY,)).fetchone()
|
|
527
|
+
except sqlite3.Error:
|
|
528
|
+
return None
|
|
529
|
+
return None if row is None else row[0]
|
|
530
|
+
|
|
531
|
+
|
|
532
|
+
@dataclasses.dataclass
|
|
533
|
+
class _CodexWatchIngest:
|
|
534
|
+
"""Stream-local observations, separate from the durable ingest result.
|
|
535
|
+
|
|
536
|
+
``stats.targeted_clean`` remains authoritative for ingest completion. A
|
|
537
|
+
held container can acknowledge its observed signature without certifying
|
|
538
|
+
any accounting or transcript work as complete (T12-INTEG01).
|
|
539
|
+
"""
|
|
540
|
+
|
|
541
|
+
stats: object
|
|
542
|
+
accounting_blocked: bool = False
|
|
543
|
+
acknowledged: dict = dataclasses.field(default_factory=dict)
|
|
544
|
+
observation_visible: bool = False
|
|
545
|
+
|
|
546
|
+
def __getattr__(self, name):
|
|
547
|
+
return getattr(self.stats, name)
|
|
548
|
+
|
|
549
|
+
|
|
550
|
+
def _codex_observation_hold_only(stats, *, executor_owned=False):
|
|
551
|
+
"""Recognize container holds without overlooking an actual ingest loss.
|
|
552
|
+
|
|
553
|
+
This is only admission to a stream-local observation acknowledgment,
|
|
554
|
+
never a replacement for ``targeted_clean`` or a frontier certificate.
|
|
555
|
+
Unknown skipped reasons and every failed/contended ingest stay dirty.
|
|
556
|
+
"""
|
|
557
|
+
if any(getattr(stats, name, False) for name in (
|
|
558
|
+
"lock_contended", "deferred_reason", "files_failed",
|
|
559
|
+
"files_deferred_torn", "files_deferred_reset", "maintenance_failed",
|
|
560
|
+
"budget_exhausted", "drain_yielded", "rebuild_refused")):
|
|
561
|
+
return False
|
|
562
|
+
if getattr(stats, "barrier_retry", False) and not executor_owned:
|
|
563
|
+
return False
|
|
564
|
+
skipped = getattr(stats, "container_skipped", {})
|
|
565
|
+
return (bool(getattr(stats, "container_pending", 0) or any(skipped.values()))
|
|
566
|
+
and set(skipped) <= {
|
|
567
|
+
"decoder-unavailable", "decode-error", "read-error",
|
|
568
|
+
"unstable-stat", "verification-pending", "verification-unverified"})
|
|
569
|
+
|
|
570
|
+
|
|
571
|
+
def _codex_watch_probes(conn):
|
|
572
|
+
"""Container observations and committed cursors for both Codex polls.
|
|
573
|
+
|
|
574
|
+
Pending fingerprints acknowledge executor-owned work. A container held
|
|
575
|
+
without a recorded fingerprint acknowledges the pre-ingest observation,
|
|
576
|
+
so replacement during ingest is still detected. These are observation
|
|
577
|
+
acknowledgments only: durable pending work remains dirty.
|
|
578
|
+
|
|
579
|
+
A scoped accounting hold cannot run transcript ingest. It acknowledges
|
|
580
|
+
the observation quietly and retries once accounting accepts that exact
|
|
581
|
+
container. Failed or contended accounting remains unacknowledged. Plain
|
|
582
|
+
barrier-deferred files wait at their pre-ingest observation until a real
|
|
583
|
+
file change or a readable new seal token re-arms them once.
|
|
584
|
+
"""
|
|
585
|
+
observed: dict = {}
|
|
586
|
+
deferred: dict = {} # path -> (pre-ingest signature, seal token)
|
|
587
|
+
accounting_wait: dict = {} # path -> observed container awaiting accounting
|
|
588
|
+
rearmed: set = set()
|
|
589
|
+
|
|
590
|
+
def accounting_accepted(path, sig):
|
|
591
|
+
try:
|
|
592
|
+
row = conn.execute(
|
|
593
|
+
f"SELECT {_CODEX_WATCH_SIG_COLUMNS} "
|
|
594
|
+
"FROM cache_db.codex_session_files "
|
|
595
|
+
"WHERE path=? AND container_state='accepted'", (path,)).fetchone()
|
|
596
|
+
except sqlite3.Error:
|
|
597
|
+
return False
|
|
598
|
+
return row is not None and _codex_acknowledged_sig(row) == sig
|
|
599
|
+
|
|
600
|
+
def executor_owns_observations(paths):
|
|
601
|
+
# Publication-unit planning can encounter the same pending container
|
|
602
|
+
# as phase one and set barrier_retry. Acknowledge only when EVERY
|
|
603
|
+
# changed path has durable executor ownership of the exact observation;
|
|
604
|
+
# a failed plain/accepted contributor must keep the whole poll retrying.
|
|
605
|
+
if not paths:
|
|
606
|
+
return False
|
|
607
|
+
marks = ",".join("?" for _ in paths)
|
|
608
|
+
try:
|
|
609
|
+
rows = conn.execute(
|
|
610
|
+
"SELECT path,pending_container_fingerprint "
|
|
611
|
+
"FROM codex_conversation_source_files "
|
|
612
|
+
f"WHERE path IN ({marks}) AND container_state='pending'",
|
|
613
|
+
list(paths)).fetchall()
|
|
614
|
+
except sqlite3.Error:
|
|
615
|
+
return False
|
|
616
|
+
if len(rows) != len(paths):
|
|
617
|
+
return False
|
|
618
|
+
for path, pending in rows:
|
|
619
|
+
fingerprint = _rc.ContainerFingerprint.from_json(pending)
|
|
620
|
+
before = observed.get(path)
|
|
621
|
+
if (fingerprint is None or before is None
|
|
622
|
+
or before[0] != _rc.ZSTD
|
|
623
|
+
or fingerprint.watch_sig() != before):
|
|
624
|
+
return False
|
|
625
|
+
return True
|
|
626
|
+
|
|
627
|
+
def stat_fn(path):
|
|
628
|
+
sig = _rc.observe_watch_sig(path)
|
|
629
|
+
observed[path] = sig
|
|
630
|
+
waiting = accounting_wait.get(path)
|
|
631
|
+
if waiting is not None:
|
|
632
|
+
if sig != waiting:
|
|
633
|
+
del accounting_wait[path]
|
|
634
|
+
elif accounting_accepted(path, sig):
|
|
635
|
+
del accounting_wait[path]
|
|
636
|
+
rearmed.add(path)
|
|
637
|
+
held = deferred.get(path)
|
|
638
|
+
if held is not None:
|
|
639
|
+
token = _codex_seal_token(conn)
|
|
640
|
+
if sig != held[0] or (token is not None and token != held[1]):
|
|
641
|
+
del deferred[path]
|
|
642
|
+
if sig == held[0]:
|
|
643
|
+
rearmed.add(path)
|
|
644
|
+
if path in rearmed and sig is not None:
|
|
645
|
+
return ("rearm",) + tuple(sig)
|
|
646
|
+
return sig
|
|
647
|
+
|
|
648
|
+
def committed_sig(path):
|
|
649
|
+
rearmed.discard(path) # reached only after a clean ingest
|
|
650
|
+
held = deferred.get(path)
|
|
651
|
+
if held is not None:
|
|
652
|
+
return held[0]
|
|
653
|
+
acknowledged = _codex_cached_file_sigs(conn, [path]).get(path)
|
|
654
|
+
before = observed.get(path)
|
|
655
|
+
if (before is not None and before[0] == _rc.ZSTD
|
|
656
|
+
and acknowledged != before):
|
|
657
|
+
return before
|
|
658
|
+
return acknowledged
|
|
659
|
+
|
|
660
|
+
def settle_ingest(ingest):
|
|
661
|
+
def settled(changed):
|
|
662
|
+
token = _codex_seal_token(conn)
|
|
663
|
+
result = ingest(changed)
|
|
664
|
+
if isinstance(result, _CodexWatchIngest):
|
|
665
|
+
stats = result.stats
|
|
666
|
+
else:
|
|
667
|
+
stats = result
|
|
668
|
+
result = _CodexWatchIngest(stats)
|
|
669
|
+
clean = getattr(stats, "targeted_clean", False)
|
|
670
|
+
executor_owned = (
|
|
671
|
+
getattr(stats, "barrier_retry", False)
|
|
672
|
+
and not result.accounting_blocked
|
|
673
|
+
and not getattr(stats, "barrier_files_deferred", 0)
|
|
674
|
+
and executor_owns_observations(changed))
|
|
675
|
+
held_only = _codex_observation_hold_only(
|
|
676
|
+
stats, executor_owned=executor_owned)
|
|
677
|
+
if not clean and not held_only:
|
|
678
|
+
return result
|
|
679
|
+
acknowledged = _codex_cached_file_sigs(conn, list(changed))
|
|
680
|
+
plain_deferred = 0
|
|
681
|
+
for path in changed:
|
|
682
|
+
before = observed.get(path)
|
|
683
|
+
if before is None:
|
|
684
|
+
continue
|
|
685
|
+
if before[0] == _rc.ZSTD and held_only:
|
|
686
|
+
# The pass actually observed this container, even if the
|
|
687
|
+
# verification executor still owes its durable work.
|
|
688
|
+
result.acknowledged[path] = before
|
|
689
|
+
rearmed.discard(path)
|
|
690
|
+
if result.accounting_blocked:
|
|
691
|
+
accounting_wait[path] = before
|
|
692
|
+
elif (not result.accounting_blocked
|
|
693
|
+
and getattr(stats, "barrier_files_deferred", 0)
|
|
694
|
+
and before[0] != _rc.ZSTD
|
|
695
|
+
and _codex_watch_kernel().codex_sig_advanced(
|
|
696
|
+
before, acknowledged.get(path))):
|
|
697
|
+
# Phase-one deferrals can publish in phase two. Only a
|
|
698
|
+
# committed cursor behind the probe still owes this work;
|
|
699
|
+
# ingestion may already have consumed a later append.
|
|
700
|
+
deferred[path] = (before, token)
|
|
701
|
+
result.acknowledged[path] = before
|
|
702
|
+
plain_deferred += 1
|
|
703
|
+
elif not result.accounting_blocked and not clean:
|
|
704
|
+
# A mixed pass may have committed a plain contributor
|
|
705
|
+
# while holding a container. Keep its committed cursor.
|
|
706
|
+
sig = acknowledged.get(path)
|
|
707
|
+
if sig is not None:
|
|
708
|
+
result.acknowledged[path] = sig
|
|
709
|
+
if (plain_deferred == len(changed)
|
|
710
|
+
and not getattr(stats, "targeted_visible", False)):
|
|
711
|
+
stats.targeted_visible = False
|
|
712
|
+
result.observation_visible = (
|
|
713
|
+
not clean and bool(result.acknowledged)
|
|
714
|
+
and not result.accounting_blocked
|
|
715
|
+
and not getattr(stats, "barrier_retry", False)
|
|
716
|
+
and bool(getattr(stats, "targeted_visible", True)))
|
|
717
|
+
return result
|
|
718
|
+
return settled
|
|
719
|
+
|
|
720
|
+
return stat_fn, committed_sig, settle_ingest
|
|
480
721
|
|
|
481
722
|
|
|
482
723
|
def _codex_classified_paths(conn, paths):
|
|
@@ -1220,7 +1461,7 @@ def _send_sse_headers(handler) -> None:
|
|
|
1220
1461
|
|
|
1221
1462
|
def _run_conversation_events_stream(
|
|
1222
1463
|
handler, conn, *, passive, tail_data, resolve, ingest, cached_sigs,
|
|
1223
|
-
discovery_step=None,
|
|
1464
|
+
discovery_step=None, stat_fn=None, committed_sig=None,
|
|
1224
1465
|
) -> None:
|
|
1225
1466
|
"""Shared per-conversation live-tail SSE loop (spec §5.2), driving both the
|
|
1226
1467
|
bare (``sessionId``) and qualified (``conversationKey``) streams from ONE
|
|
@@ -1236,9 +1477,18 @@ def _run_conversation_events_stream(
|
|
|
1236
1477
|
~10-cycle Codex child-discovery frontier (``None`` for Claude — a bare or
|
|
1237
1478
|
qualified-Claude stream never runs it). The cache lock is held ONLY inside
|
|
1238
1479
|
``ingest`` / ``discovery_step``, never across a sleep (the #297 WAL
|
|
1239
|
-
discipline).
|
|
1480
|
+
discipline).
|
|
1481
|
+
|
|
1482
|
+
``stat_fn`` / ``committed_sig`` are the provider's observation seam
|
|
1483
|
+
(#968 spec 5.9): Codex passes ``_codex_watch_probes`` so both poll sites
|
|
1484
|
+
compare one container-aware signature shape; ``None`` keeps the kernel's
|
|
1485
|
+
literal-path size probe and the committed-cursor lookup through
|
|
1486
|
+
``cached_sigs`` (Claude, unchanged).
|
|
1487
|
+
"""
|
|
1240
1488
|
import time as _time
|
|
1241
1489
|
watch = sys.modules["cctally"]._load_sibling("_lib_conversation_watch")
|
|
1490
|
+
probe = stat_fn or watch.file_sig
|
|
1491
|
+
committed = committed_sig or (lambda p: cached_sigs([p]).get(p))
|
|
1242
1492
|
tail_frame = (
|
|
1243
1493
|
"event: tail\ndata: " + encode_dashboard_json(tail_data) + "\n\n"
|
|
1244
1494
|
).encode("utf-8")
|
|
@@ -1286,12 +1536,26 @@ def _run_conversation_events_stream(
|
|
|
1286
1536
|
if _request_peer_gone(handler, timeout=_LIVE_TAIL_POLL_INTERVAL):
|
|
1287
1537
|
return
|
|
1288
1538
|
cycles += 1
|
|
1289
|
-
changed = watch.changed_paths(files, seen)
|
|
1539
|
+
changed = watch.changed_paths(files, seen, probe)
|
|
1290
1540
|
if changed:
|
|
1291
1541
|
_time.sleep(_LIVE_TAIL_DEBOUNCE)
|
|
1542
|
+
observation = None
|
|
1543
|
+
|
|
1544
|
+
def poll_ingest(changed):
|
|
1545
|
+
nonlocal observation
|
|
1546
|
+
observation = ingest(changed)
|
|
1547
|
+
return observation
|
|
1548
|
+
|
|
1292
1549
|
new_seen, emitted = watch.watch_step(
|
|
1293
|
-
files, seen, ingest_fn=
|
|
1294
|
-
committed_sig_fn=
|
|
1550
|
+
files, seen, stat_fn=probe, ingest_fn=poll_ingest,
|
|
1551
|
+
committed_sig_fn=committed)
|
|
1552
|
+
if isinstance(observation, _CodexWatchIngest):
|
|
1553
|
+
# A held container was observed, not successfully ingested.
|
|
1554
|
+
# Keep that distinction out of the generic watch kernel and
|
|
1555
|
+
# out of every durable cursor/certification decision.
|
|
1556
|
+
new_seen = dict(new_seen)
|
|
1557
|
+
new_seen.update(observation.acknowledged)
|
|
1558
|
+
emitted = emitted or observation.observation_visible
|
|
1295
1559
|
seen = new_seen
|
|
1296
1560
|
if emitted:
|
|
1297
1561
|
handler.wfile.write(tail_frame)
|
|
@@ -1643,7 +1907,8 @@ def _qualified_conversation_events(
|
|
|
1643
1907
|
finally:
|
|
1644
1908
|
accounting.close()
|
|
1645
1909
|
if not accounting_stats.targeted_clean:
|
|
1646
|
-
return
|
|
1910
|
+
return _CodexWatchIngest(
|
|
1911
|
+
accounting_stats, accounting_blocked=True)
|
|
1647
1912
|
writer = sys.modules["_cctally_dashboard"].open_conversations_db()
|
|
1648
1913
|
try:
|
|
1649
1914
|
stats = sync_codex_conversations(writer, only_paths=set(changed))
|
|
@@ -1658,6 +1923,8 @@ def _qualified_conversation_events(
|
|
|
1658
1923
|
return stats
|
|
1659
1924
|
|
|
1660
1925
|
cached = lambda paths: _codex_cached_file_sigs(conn, paths)
|
|
1926
|
+
stat_fn, committed_sig, settle_ingest = _codex_watch_probes(conn)
|
|
1927
|
+
_ingest = settle_ingest(_ingest)
|
|
1661
1928
|
discovery = (
|
|
1662
1929
|
None if account_key is not None
|
|
1663
1930
|
else _make_codex_discovery_step(handler, conn, key, cq_codex)
|
|
@@ -1694,6 +1961,7 @@ def _qualified_conversation_events(
|
|
|
1694
1961
|
return stats
|
|
1695
1962
|
|
|
1696
1963
|
cached = lambda paths: _cached_file_sigs(conn, paths)
|
|
1964
|
+
stat_fn = committed_sig = None
|
|
1697
1965
|
discovery = None
|
|
1698
1966
|
|
|
1699
1967
|
try:
|
|
@@ -1701,7 +1969,8 @@ def _qualified_conversation_events(
|
|
|
1701
1969
|
handler, conn, passive=passive,
|
|
1702
1970
|
tail_data={"conversationKey": key},
|
|
1703
1971
|
resolve=_resolve, ingest=_ingest, cached_sigs=cached,
|
|
1704
|
-
discovery_step=discovery
|
|
1972
|
+
discovery_step=discovery, stat_fn=stat_fn,
|
|
1973
|
+
committed_sig=committed_sig)
|
|
1705
1974
|
except (BrokenPipeError, ConnectionResetError,
|
|
1706
1975
|
ConnectionAbortedError, socket.timeout):
|
|
1707
1976
|
pass
|
|
@@ -96,6 +96,7 @@ from _cctally_dashboard_cache_report import _cache_report_snapshot_to_dict
|
|
|
96
96
|
# copy was already exactly this — collapse to the single definition).
|
|
97
97
|
_ensure_sibling_loaded("_lib_json_envelope")
|
|
98
98
|
from _lib_json_envelope import _iso_z
|
|
99
|
+
from _lib_five_hour import latest_five_hour_reading
|
|
99
100
|
|
|
100
101
|
|
|
101
102
|
def _model_breakdowns_to_models(model_breakdowns: list[dict[str, Any]],
|
|
@@ -216,9 +217,11 @@ def _select_current_block_for_envelope(
|
|
|
216
217
|
Delta semantics:
|
|
217
218
|
- Non-crossed block: ``current_used_pct - seven_day_pct_at_block_start``
|
|
218
219
|
(the natural "how much 7d% has changed during this 5h block" read), where
|
|
219
|
-
that axis is the
|
|
220
|
-
|
|
221
|
-
|
|
220
|
+
that axis is the block's stored ``seven_day_pct_at_block_start``. #835's
|
|
221
|
+
retirement band, which published ``None`` for a value within a point of
|
|
222
|
+
one an in-place weekly credit retired, is removed (#840): stale
|
|
223
|
+
contributors are refused before publication, and a stored value that is
|
|
224
|
+
already wrong is corrected by ``db rederive``'s reviewed ``retract``.
|
|
222
225
|
- Crossed block (``crossed_seven_day_reset == 1``): the block straddles
|
|
223
226
|
a weekly reset, so the natural delta would be dominated by the
|
|
224
227
|
reset itself (e.g. −94pp) rather than the user's actual burn-rate.
|
|
@@ -235,9 +238,17 @@ def _select_current_block_for_envelope(
|
|
|
235
238
|
# statement of that key. Its `week_start_at` is a byte copy of the
|
|
236
239
|
# non-held row it was derived from, so scoping the post-reset lookup
|
|
237
240
|
# below by it is unaffected.
|
|
241
|
+
#
|
|
242
|
+
# #825/#832 spec 4.7 revision 7: the current window is the newest five-hour
|
|
243
|
+
# READING's, through the one shared selection under the same as-of and
|
|
244
|
+
# account restrictions. A retired replica (its five-hour fields cleared in
|
|
245
|
+
# place by a five-hour credit) or a weekly-only reading can be the newest
|
|
246
|
+
# row and names no window. The newest row still supplies `week_start_at`
|
|
247
|
+
# for the post-reset lookup below: that is weekly evidence, selected
|
|
248
|
+
# independently.
|
|
238
249
|
snap = conn.execute(
|
|
239
250
|
"""
|
|
240
|
-
SELECT
|
|
251
|
+
SELECT week_start_at
|
|
241
252
|
FROM weekly_usage_snapshots
|
|
242
253
|
WHERE captured_at_utc <= ?
|
|
243
254
|
AND (? IS NULL OR account_key = ?)
|
|
@@ -246,7 +257,11 @@ def _select_current_block_for_envelope(
|
|
|
246
257
|
""",
|
|
247
258
|
(_iso_z(now_utc), account_key, account_key),
|
|
248
259
|
).fetchone()
|
|
249
|
-
if snap is None
|
|
260
|
+
if snap is None:
|
|
261
|
+
return None
|
|
262
|
+
reading = latest_five_hour_reading(
|
|
263
|
+
conn, account_key=account_key, as_of_utc=_iso_z(now_utc))
|
|
264
|
+
if reading is None:
|
|
250
265
|
return None
|
|
251
266
|
|
|
252
267
|
# The account supplied by `_tui_build_current_week` is the owner of the
|
|
@@ -268,21 +283,17 @@ def _select_current_block_for_envelope(
|
|
|
268
283
|
# Account-qualified selection binds the block and its downstream milestone
|
|
269
284
|
# read to the displayed weekly account; the unqualified order remains the
|
|
270
285
|
# historical <=1-real-account fallback for published byte stability.
|
|
271
|
-
# #834 S1 (#835) Gate A R8
|
|
272
|
-
#
|
|
273
|
-
#
|
|
274
|
-
#
|
|
275
|
-
#
|
|
276
|
-
#
|
|
277
|
-
#
|
|
278
|
-
#
|
|
279
|
-
#
|
|
280
|
-
#
|
|
281
|
-
#
|
|
282
|
-
# instant it feeds is discarded with the end axis. It stays because the
|
|
283
|
-
# helper's contract is a block ROW, and pruning this SELECT to the subset the
|
|
284
|
-
# helper happens to read today would couple this route to the helper's
|
|
285
|
-
# internals. Round four's comment implied necessity instead.
|
|
286
|
+
# #834 S1 (#835) Gate A R8 selected `first_observed_at_utc`,
|
|
287
|
+
# `five_hour_resets_at` and `last_observed_at_utc` for the retirement band's
|
|
288
|
+
# week resolution. Since the band's removal (#840) the axes helper below
|
|
289
|
+
# reads only the stored `seven_day_pct_at_block_start` /
|
|
290
|
+
# `seven_day_pct_at_block_end` through `block.get(...)`, so none of the three
|
|
291
|
+
# is needed by it; they stay because the helper's contract is a block ROW,
|
|
292
|
+
# and pruning this SELECT to the subset the helper happens to read today
|
|
293
|
+
# would couple this route to the helper's internals.
|
|
294
|
+
# `seven_day_pct_at_block_end` is not selected: this route publishes the
|
|
295
|
+
# start axis alone and discards the helper's second return value, which is
|
|
296
|
+
# then `None`.
|
|
286
297
|
block = conn.execute(
|
|
287
298
|
"""
|
|
288
299
|
SELECT id, account_key,
|
|
@@ -298,7 +309,7 @@ def _select_current_block_for_envelope(
|
|
|
298
309
|
ORDER BY id ASC
|
|
299
310
|
LIMIT 1
|
|
300
311
|
""",
|
|
301
|
-
(
|
|
312
|
+
(reading.window_key, account_key, account_key,
|
|
302
313
|
_iso_z(now_utc)),
|
|
303
314
|
).fetchone()
|
|
304
315
|
if block is None:
|
|
@@ -306,14 +317,14 @@ def _select_current_block_for_envelope(
|
|
|
306
317
|
|
|
307
318
|
crossed = bool(block["crossed_seven_day_reset"])
|
|
308
319
|
# #834 S1 (#835) Gate A R8: THE THIRD CONSUMER of the two derived weekly axes
|
|
309
|
-
# goes through the one
|
|
310
|
-
# `cmd_five_hour_breakdown` use.
|
|
311
|
-
# in-place weekly credit
|
|
312
|
-
#
|
|
313
|
-
#
|
|
314
|
-
#
|
|
315
|
-
# `
|
|
316
|
-
#
|
|
320
|
+
# goes through the one shared axes helper, the same one `cmd_five_hour_blocks`
|
|
321
|
+
# and `cmd_five_hour_breakdown` use. #835 made that helper a read-time filter
|
|
322
|
+
# that withheld a value within a point of one an in-place weekly credit
|
|
323
|
+
# retired; #840 removed that retirement band, so the helper now returns the
|
|
324
|
+
# block's stored axes. A stale contributor is refused before it is
|
|
325
|
+
# published, and a stored value that is already wrong is corrected by
|
|
326
|
+
# `db rederive`'s reviewed `retract`, never by a percentage heuristic here.
|
|
327
|
+
# `sqlite3.Row` has no `.get()`, so the row is converted.
|
|
317
328
|
#
|
|
318
329
|
# Neither field is rendered by any client today: both are declared in
|
|
319
330
|
# `dashboard/web/src/types/envelope.ts` and used in one vitest module,
|
|
@@ -2180,18 +2191,14 @@ def snapshot_to_envelope(snap: "DataSnapshot", *,
|
|
|
2180
2191
|
_ld = sys.modules["cctally"]._load_sibling("_lib_doctor")
|
|
2181
2192
|
_doc_state = sys.modules["cctally"].doctor_gather_state(now_utc=now_utc, runtime_bind=runtime_bind)
|
|
2182
2193
|
_doc_report = _ld.run_checks(_doc_state)
|
|
2183
|
-
doctor_envelope =
|
|
2184
|
-
"severity": _doc_report.overall_severity,
|
|
2185
|
-
"counts": dict(_doc_report.counts),
|
|
2186
|
-
"generated_at": _ld._iso_z(_doc_report.generated_at),
|
|
2187
|
-
"fingerprint": _ld.fingerprint(_doc_report),
|
|
2188
|
-
}
|
|
2194
|
+
doctor_envelope = _ld.envelope_block(_doc_report)
|
|
2189
2195
|
except Exception as exc: # noqa: BLE001 — never crash SSE on doctor failure
|
|
2190
2196
|
doctor_envelope = {
|
|
2191
2197
|
"severity": "fail",
|
|
2192
2198
|
"counts": {"ok": 0, "warn": 0, "fail": 1},
|
|
2193
2199
|
"generated_at": _iso_z(now_utc),
|
|
2194
2200
|
"fingerprint": "sha1:" + ("0" * 40),
|
|
2201
|
+
"rebuild_worker": None,
|
|
2195
2202
|
"_error": f"{type(exc).__name__}: {exc}",
|
|
2196
2203
|
}
|
|
2197
2204
|
|
|
@@ -86,7 +86,10 @@ from _lib_codex_pools import (
|
|
|
86
86
|
from _lib_codex_conversation import _display_title as _codex_display_title
|
|
87
87
|
from _lib_fmt import stable_sum
|
|
88
88
|
from _lib_aggregators import _aggregate_codex_buckets, codex_path_scope
|
|
89
|
-
from _lib_five_hour import
|
|
89
|
+
from _lib_five_hour import (
|
|
90
|
+
_FIVE_HOUR_JITTER_FLOOR_SECONDS,
|
|
91
|
+
latest_five_hour_reading,
|
|
92
|
+
)
|
|
90
93
|
from _lib_source_analytics import (
|
|
91
94
|
QualifiedCodexEntry,
|
|
92
95
|
assign_collision_safe_project_labels,
|
|
@@ -6328,6 +6331,11 @@ def _claude_accounts_wire(
|
|
|
6328
6331
|
# load-bearing, which is the same reason every other weekly consumer
|
|
6329
6332
|
# excludes held rows rather than trusting it.
|
|
6330
6333
|
#
|
|
6334
|
+
# The five-hour figure is the account's newest five-hour READING
|
|
6335
|
+
# (#825/#832 spec 4.7 revision 7, the one shared selection): a retired
|
|
6336
|
+
# replica or a weekly-only reading can be the newest row and carries
|
|
6337
|
+
# no five-hour value, so "the latest row" is not the latest reading.
|
|
6338
|
+
#
|
|
6331
6339
|
# None is still returned only when the account retained NO row at all,
|
|
6332
6340
|
# because the caller uses that to decide whether the unattributed
|
|
6333
6341
|
# bucket appears.
|
|
@@ -6338,17 +6346,18 @@ def _claude_accounts_wire(
|
|
|
6338
6346
|
"ORDER BY captured_at_utc DESC LIMIT 1",
|
|
6339
6347
|
(key,),
|
|
6340
6348
|
).fetchone()
|
|
6341
|
-
five_hour = stats_conn
|
|
6342
|
-
"SELECT five_hour_percent FROM weekly_usage_snapshots "
|
|
6343
|
-
"WHERE account_key=? "
|
|
6344
|
-
"ORDER BY captured_at_utc DESC LIMIT 1",
|
|
6345
|
-
(key,),
|
|
6346
|
-
).fetchone()
|
|
6349
|
+
five_hour = latest_five_hour_reading(stats_conn, account_key=key)
|
|
6347
6350
|
if weekly is None and five_hour is None:
|
|
6348
|
-
|
|
6351
|
+
retained = stats_conn.execute(
|
|
6352
|
+
"SELECT 1 FROM weekly_usage_snapshots WHERE account_key=? "
|
|
6353
|
+
"LIMIT 1",
|
|
6354
|
+
(key,),
|
|
6355
|
+
).fetchone()
|
|
6356
|
+
if retained is None:
|
|
6357
|
+
return None
|
|
6349
6358
|
return (
|
|
6350
6359
|
None if weekly is None else weekly[0],
|
|
6351
|
-
None if five_hour is None else five_hour
|
|
6360
|
+
None if five_hour is None else five_hour.percent,
|
|
6352
6361
|
None if weekly is None else weekly[1],
|
|
6353
6362
|
)
|
|
6354
6363
|
|