cctally 1.108.0 → 1.110.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +147 -0
- package/README.md +6 -6
- package/bin/_cctally_account.py +4 -0
- package/bin/_cctally_alerts.py +6 -2
- package/bin/_cctally_cache.py +1652 -166
- package/bin/_cctally_config.py +3 -2
- package/bin/_cctally_core.py +594 -33
- package/bin/_cctally_dashboard.py +1585 -201
- package/bin/_cctally_dashboard_conversation.py +494 -48
- package/bin/_cctally_dashboard_envelope.py +167 -11
- package/bin/_cctally_dashboard_share.py +154 -36
- package/bin/_cctally_dashboard_sources.py +2615 -422
- package/bin/_cctally_db.py +678 -5
- package/bin/_cctally_diagnosis_sources.py +238 -26
- package/bin/_cctally_doctor.py +99 -18
- package/bin/_cctally_five_hour.py +1295 -40
- package/bin/_cctally_forecast.py +124 -28
- package/bin/_cctally_journal.py +408 -38
- package/bin/_cctally_milestone_history.py +150 -14
- package/bin/_cctally_parser.py +5 -0
- package/bin/_cctally_percent_breakdown.py +6 -1
- package/bin/_cctally_project.py +48 -76
- package/bin/_cctally_quota.py +39 -7
- package/bin/_cctally_quota_model.py +17 -2
- package/bin/_cctally_record.py +1647 -443
- package/bin/_cctally_rederive.py +899 -30
- package/bin/_cctally_refresh.py +2 -1
- package/bin/_cctally_share.py +24 -5
- package/bin/_cctally_source_analytics.py +1035 -78
- package/bin/_cctally_statusline.py +83 -19
- package/bin/_cctally_store.py +61 -4
- package/bin/_cctally_transcript.py +27 -2
- package/bin/_cctally_tui.py +264 -76
- package/bin/_cctally_weekrefs.py +28 -5
- package/bin/_lib_blocks.py +127 -20
- package/bin/_lib_cache_report.py +20 -5
- package/bin/_lib_codex_conversation_query.py +297 -64
- package/bin/_lib_codex_hooks.py +102 -0
- package/bin/_lib_codex_metadata.py +207 -0
- package/bin/_lib_conversation.py +119 -0
- package/bin/_lib_conversation_dispatch.py +31 -1
- package/bin/_lib_conversation_query.py +315 -51
- package/bin/_lib_conversation_retention.py +413 -25
- package/bin/_lib_dashboard_sources.py +267 -7
- package/bin/_lib_diagnosis.py +10 -0
- package/bin/_lib_diff_kernel.py +12 -1
- package/bin/_lib_doctor.py +230 -8
- package/bin/_lib_ingest_frontier.py +1366 -49
- package/bin/_lib_journal.py +19 -2
- package/bin/_lib_merge_gate.py +307 -0
- package/bin/_lib_pricing.py +588 -29
- package/bin/_lib_record.py +130 -12
- package/bin/_lib_rederive.py +903 -2
- package/bin/_lib_render.py +9 -3
- package/bin/_lib_retained_size.py +27 -2
- package/bin/_lib_segment_summary.py +66 -8
- package/bin/_lib_share_templates.py +28 -18
- package/bin/_lib_snapshot_cache.py +115 -1
- package/bin/_lib_source_retry.py +320 -0
- package/bin/_lib_statusline_candidates.py +400 -36
- package/bin/_lib_tick_stats.py +119 -0
- package/bin/_lib_view_models.py +256 -76
- package/bin/cctally +69 -0
- package/dashboard/static/assets/ConversationsView-BB9vUiL6.js +72 -0
- package/dashboard/static/assets/{DoctorModal-CWn3U4Wl.js → DoctorModal-DFY2iPDf.js} +1 -1
- package/dashboard/static/assets/ModalRoot-DvRKHbQE.js +1 -0
- package/dashboard/static/assets/ProjectsDrillPanel-DAv39uSo.js +1 -0
- package/dashboard/static/assets/SourceDetailModal-DSAWo2Fp.js +1 -0
- package/dashboard/static/assets/{UpdateModal-D3m8GG6V.js → UpdateModal-R_9dOVnh.js} +3 -3
- package/dashboard/static/assets/dashboardStream.shared-worker-DbG6Ef0T.js +1 -0
- package/dashboard/static/assets/index-COhxe3vU.css +1 -0
- package/dashboard/static/assets/index-DdU62sov.js +13 -0
- package/dashboard/static/assets/outlineNavigation-BOCLcRa6.js +9 -0
- package/dashboard/static/assets/useKeymap-CW7Up0Ay.js +1 -0
- package/dashboard/static/dashboard.html +3 -3
- package/package.json +4 -1
- package/dashboard/static/assets/ConversationsView-BOSaBRtu.js +0 -72
- package/dashboard/static/assets/ModalRoot-SR070V6c.js +0 -1
- package/dashboard/static/assets/ProjectsDrillPanel-QLI9i5mZ.js +0 -1
- package/dashboard/static/assets/SourceDetailModal-pv0wlFxl.js +0 -1
- package/dashboard/static/assets/dashboardStream.shared-worker-1XTMV3nr.js +0 -1
- package/dashboard/static/assets/index-BHu4mxd8.css +0 -1
- package/dashboard/static/assets/index-CIWsbux3.js +0 -13
- package/dashboard/static/assets/outlineNavigation-CJvKqmLV.js +0 -9
- package/dashboard/static/assets/useKeymap-ffqsS5G0.js +0 -1
|
@@ -278,6 +278,7 @@ import secrets
|
|
|
278
278
|
import shutil
|
|
279
279
|
import signal as _signal
|
|
280
280
|
import socket
|
|
281
|
+
import select
|
|
281
282
|
import sqlite3
|
|
282
283
|
import sys
|
|
283
284
|
import threading
|
|
@@ -297,7 +298,7 @@ from _lib_retained_size import retained_size_bytes
|
|
|
297
298
|
|
|
298
299
|
|
|
299
300
|
class _QuietThreadingHTTPServer(ThreadingHTTPServer):
|
|
300
|
-
"""
|
|
301
|
+
"""Bounded HTTP threads with quiet client-disconnect handling.
|
|
301
302
|
|
|
302
303
|
A backgrounded/closed/reloaded dashboard tab hangs up mid-response, so the
|
|
303
304
|
per-request thread's socket write raises one of the "peer went away"
|
|
@@ -313,6 +314,111 @@ class _QuietThreadingHTTPServer(ThreadingHTTPServer):
|
|
|
313
314
|
"""
|
|
314
315
|
|
|
315
316
|
daemon_threads = True
|
|
317
|
+
request_queue_size = 256
|
|
318
|
+
max_request_threads = 96
|
|
319
|
+
_reject_workers_count = 4
|
|
320
|
+
_reject_queue_size = 256
|
|
321
|
+
_busy_body = b'{"error":"server busy"}'
|
|
322
|
+
_busy_response = (
|
|
323
|
+
b"HTTP/1.1 503 Service Unavailable\r\n"
|
|
324
|
+
b"Content-Type: application/json\r\n"
|
|
325
|
+
b"Cache-Control: no-store\r\n"
|
|
326
|
+
b"Connection: close\r\n"
|
|
327
|
+
+ f"Content-Length: {len(_busy_body)}\r\n\r\n".encode("ascii")
|
|
328
|
+
+ _busy_body
|
|
329
|
+
)
|
|
330
|
+
|
|
331
|
+
def __init__(self, *args, **kwargs):
|
|
332
|
+
self._request_slots = threading.BoundedSemaphore(
|
|
333
|
+
self.max_request_threads)
|
|
334
|
+
self._reject_queue: queue.Queue[socket.socket] = queue.Queue(
|
|
335
|
+
maxsize=self._reject_queue_size)
|
|
336
|
+
self._reject_stop = threading.Event()
|
|
337
|
+
self._reject_start_lock = threading.Lock()
|
|
338
|
+
self._reject_workers: list[threading.Thread] = []
|
|
339
|
+
super().__init__(*args, **kwargs)
|
|
340
|
+
|
|
341
|
+
def _reject_request(self, request: socket.socket) -> None:
|
|
342
|
+
try:
|
|
343
|
+
# Read only the bounded request headers so a close with unread
|
|
344
|
+
# input does not reset a client before it sees the 503. Slow peers
|
|
345
|
+
# are confined to a fixed reject worker, never the accept loop.
|
|
346
|
+
request.settimeout(0.25)
|
|
347
|
+
received = bytearray()
|
|
348
|
+
while b"\r\n\r\n" not in received and len(received) < 8192:
|
|
349
|
+
chunk = request.recv(min(8192 - len(received), 4096))
|
|
350
|
+
if not chunk:
|
|
351
|
+
break
|
|
352
|
+
received.extend(chunk)
|
|
353
|
+
request.sendall(self._busy_response)
|
|
354
|
+
except OSError:
|
|
355
|
+
pass # A peer that disconnected cannot receive the overload reply.
|
|
356
|
+
finally:
|
|
357
|
+
self.shutdown_request(request)
|
|
358
|
+
|
|
359
|
+
def _reject_loop(self) -> None:
|
|
360
|
+
while not self._reject_stop.is_set():
|
|
361
|
+
try:
|
|
362
|
+
request = self._reject_queue.get(timeout=0.1)
|
|
363
|
+
except queue.Empty:
|
|
364
|
+
continue
|
|
365
|
+
try:
|
|
366
|
+
self._reject_request(request)
|
|
367
|
+
finally:
|
|
368
|
+
self._reject_queue.task_done()
|
|
369
|
+
|
|
370
|
+
def _queue_rejection(self, request: socket.socket) -> None:
|
|
371
|
+
with self._reject_start_lock:
|
|
372
|
+
if not self._reject_workers:
|
|
373
|
+
for index in range(self._reject_workers_count):
|
|
374
|
+
worker = threading.Thread(
|
|
375
|
+
target=self._reject_loop,
|
|
376
|
+
name=f"cctally-http-reject-{index}", daemon=True,
|
|
377
|
+
)
|
|
378
|
+
worker.start()
|
|
379
|
+
self._reject_workers.append(worker)
|
|
380
|
+
try:
|
|
381
|
+
self._reject_queue.put_nowait(request)
|
|
382
|
+
except queue.Full:
|
|
383
|
+
# This last-resort reply is nonblocking. It keeps the accept loop
|
|
384
|
+
# responsive even if the bounded reject queue also saturates.
|
|
385
|
+
try:
|
|
386
|
+
request.setblocking(False)
|
|
387
|
+
request.send(self._busy_response)
|
|
388
|
+
except OSError:
|
|
389
|
+
pass
|
|
390
|
+
finally:
|
|
391
|
+
self.shutdown_request(request)
|
|
392
|
+
|
|
393
|
+
def process_request(self, request, client_address):
|
|
394
|
+
if not self._request_slots.acquire(blocking=False):
|
|
395
|
+
self._queue_rejection(request)
|
|
396
|
+
return
|
|
397
|
+
try:
|
|
398
|
+
super().process_request(request, client_address)
|
|
399
|
+
except BaseException:
|
|
400
|
+
self._request_slots.release()
|
|
401
|
+
self.shutdown_request(request)
|
|
402
|
+
raise
|
|
403
|
+
|
|
404
|
+
def process_request_thread(self, request, client_address):
|
|
405
|
+
try:
|
|
406
|
+
super().process_request_thread(request, client_address)
|
|
407
|
+
finally:
|
|
408
|
+
self._request_slots.release()
|
|
409
|
+
|
|
410
|
+
def server_close(self):
|
|
411
|
+
self._reject_stop.set()
|
|
412
|
+
while True:
|
|
413
|
+
try:
|
|
414
|
+
request = self._reject_queue.get_nowait()
|
|
415
|
+
except queue.Empty:
|
|
416
|
+
break
|
|
417
|
+
self.shutdown_request(request)
|
|
418
|
+
self._reject_queue.task_done()
|
|
419
|
+
for worker in self._reject_workers:
|
|
420
|
+
worker.join(timeout=0.5)
|
|
421
|
+
super().server_close()
|
|
316
422
|
|
|
317
423
|
def handle_error(self, request, client_address):
|
|
318
424
|
exc = sys.exc_info()[1]
|
|
@@ -332,6 +438,30 @@ class _DiagnosisFlight:
|
|
|
332
438
|
result: Any = None
|
|
333
439
|
error: BaseException | None = None
|
|
334
440
|
callers: int = 0
|
|
441
|
+
deadline: float = 0.0
|
|
442
|
+
|
|
443
|
+
|
|
444
|
+
class _DiagnosisOverloaded(Exception):
|
|
445
|
+
"""The bounded diagnosis admission has no room for another caller."""
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
class _DiagnosisTimedOut(Exception):
|
|
449
|
+
"""A diagnosis did not finish within its flight's deadline."""
|
|
450
|
+
|
|
451
|
+
|
|
452
|
+
class _DiagnosisClientGone(Exception):
|
|
453
|
+
"""The requesting peer disconnected while its diagnosis was pending."""
|
|
454
|
+
|
|
455
|
+
|
|
456
|
+
def _diagnosis_peer_gone(connection: socket.socket) -> bool:
|
|
457
|
+
"""Check a waiting GET socket without consuming any request bytes."""
|
|
458
|
+
try:
|
|
459
|
+
if not select.select([connection], [], [], 0)[0]:
|
|
460
|
+
return False
|
|
461
|
+
return not connection.recv(
|
|
462
|
+
1, socket.MSG_PEEK | getattr(socket, "MSG_DONTWAIT", 0))
|
|
463
|
+
except (OSError, ValueError):
|
|
464
|
+
return True
|
|
335
465
|
|
|
336
466
|
|
|
337
467
|
def _diagnosis_flight_key(
|
|
@@ -360,48 +490,155 @@ class _DiagnosisAdmission:
|
|
|
360
490
|
never retained, so this adds no stale report cache or periodic work.
|
|
361
491
|
"""
|
|
362
492
|
|
|
363
|
-
def __init__(self
|
|
493
|
+
def __init__(self, *, max_callers: int = 32, max_per_key: int = 16,
|
|
494
|
+
max_keys: int = 8, deadline_seconds: float = 15.0) -> None:
|
|
495
|
+
if min(max_callers, max_per_key, max_keys) < 1 or deadline_seconds <= 0:
|
|
496
|
+
raise ValueError("diagnosis admission limits must be positive")
|
|
364
497
|
self._changed = threading.Condition()
|
|
365
498
|
self._admission = threading.BoundedSemaphore(1)
|
|
366
499
|
self._flights: dict[tuple[Any, bool, bool], _DiagnosisFlight] = {}
|
|
500
|
+
self._callers = 0
|
|
501
|
+
self._max_callers = max_callers
|
|
502
|
+
self._max_per_key = max_per_key
|
|
503
|
+
self._max_keys = max_keys
|
|
504
|
+
self._deadline_seconds = deadline_seconds
|
|
505
|
+
|
|
506
|
+
def _finish(self, key, flight, *, result=None, error=None) -> None:
|
|
507
|
+
with self._changed:
|
|
508
|
+
if flight.done.is_set():
|
|
509
|
+
return
|
|
510
|
+
flight.result = result
|
|
511
|
+
flight.error = error
|
|
512
|
+
flight.done.set()
|
|
513
|
+
if self._flights.get(key) is flight:
|
|
514
|
+
del self._flights[key]
|
|
515
|
+
self._changed.notify_all()
|
|
367
516
|
|
|
368
|
-
def
|
|
517
|
+
def _produce(self, key, flight, build, bounded_build) -> None:
|
|
518
|
+
remaining = max(0.0, flight.deadline - time.monotonic())
|
|
519
|
+
acquired = self._admission.acquire(timeout=remaining)
|
|
520
|
+
if not acquired:
|
|
521
|
+
self._finish(key, flight, error=_DiagnosisTimedOut())
|
|
522
|
+
return
|
|
523
|
+
try:
|
|
524
|
+
# A queued flight may have expired while a prior producer held the
|
|
525
|
+
# single process-wide slot. It must never start a late build.
|
|
526
|
+
if flight.done.is_set() or time.monotonic() >= flight.deadline:
|
|
527
|
+
self._finish(key, flight, error=_DiagnosisTimedOut())
|
|
528
|
+
return
|
|
529
|
+
try:
|
|
530
|
+
remaining = max(0.0, flight.deadline - time.monotonic())
|
|
531
|
+
result = (
|
|
532
|
+
bounded_build(remaining)
|
|
533
|
+
if bounded_build is not None else build()
|
|
534
|
+
)
|
|
535
|
+
except TimeoutError:
|
|
536
|
+
self._finish(key, flight, error=_DiagnosisTimedOut())
|
|
537
|
+
except BaseException as exc:
|
|
538
|
+
self._finish(
|
|
539
|
+
key, flight,
|
|
540
|
+
error=(_DiagnosisTimedOut()
|
|
541
|
+
if time.monotonic() >= flight.deadline else exc),
|
|
542
|
+
)
|
|
543
|
+
else:
|
|
544
|
+
if time.monotonic() >= flight.deadline:
|
|
545
|
+
self._finish(key, flight, error=_DiagnosisTimedOut())
|
|
546
|
+
else:
|
|
547
|
+
self._finish(key, flight, result=result)
|
|
548
|
+
finally:
|
|
549
|
+
# Production passes a killable process build, so reaching the
|
|
550
|
+
# deadline retires that process tree before this slot is released.
|
|
551
|
+
# Direct in-process builds remain available only to focused unit
|
|
552
|
+
# callers whose producer is known to terminate.
|
|
553
|
+
self._admission.release()
|
|
554
|
+
|
|
555
|
+
def run(self, key: tuple[Any, bool, bool], build, *, peer_gone=None,
|
|
556
|
+
bounded_build=None):
|
|
369
557
|
with self._changed:
|
|
370
558
|
flight = self._flights.get(key)
|
|
371
559
|
owner = flight is None
|
|
560
|
+
if self._callers >= self._max_callers:
|
|
561
|
+
raise _DiagnosisOverloaded()
|
|
372
562
|
if owner:
|
|
373
|
-
|
|
563
|
+
if len(self._flights) >= self._max_keys:
|
|
564
|
+
raise _DiagnosisOverloaded()
|
|
565
|
+
flight = _DiagnosisFlight(
|
|
566
|
+
deadline=time.monotonic() + self._deadline_seconds,
|
|
567
|
+
)
|
|
374
568
|
self._flights[key] = flight
|
|
569
|
+
elif flight.callers >= self._max_per_key:
|
|
570
|
+
raise _DiagnosisOverloaded()
|
|
375
571
|
flight.callers += 1
|
|
572
|
+
self._callers += 1
|
|
376
573
|
self._changed.notify_all()
|
|
377
|
-
|
|
378
|
-
if not owner:
|
|
379
|
-
flight.done.wait()
|
|
380
|
-
if flight.error is not None:
|
|
381
|
-
raise flight.error
|
|
382
|
-
return flight.result
|
|
383
|
-
|
|
384
574
|
try:
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
575
|
+
if owner:
|
|
576
|
+
try:
|
|
577
|
+
threading.Thread(
|
|
578
|
+
target=self._produce,
|
|
579
|
+
args=(key, flight, build, bounded_build),
|
|
580
|
+
name="cctally-diagnosis", daemon=True,
|
|
581
|
+
).start()
|
|
582
|
+
except BaseException as exc:
|
|
583
|
+
self._finish(key, flight, error=exc)
|
|
584
|
+
while not flight.done.is_set():
|
|
585
|
+
if peer_gone is not None and peer_gone():
|
|
586
|
+
raise _DiagnosisClientGone()
|
|
587
|
+
remaining = flight.deadline - time.monotonic()
|
|
588
|
+
if remaining <= 0:
|
|
589
|
+
break
|
|
590
|
+
flight.done.wait(min(0.1, remaining))
|
|
591
|
+
if not flight.done.is_set():
|
|
592
|
+
self._finish(key, flight, error=_DiagnosisTimedOut())
|
|
391
593
|
with self._changed:
|
|
392
|
-
flight.error
|
|
393
|
-
|
|
594
|
+
if flight.error is not None:
|
|
595
|
+
raise flight.error
|
|
596
|
+
return flight.result
|
|
394
597
|
finally:
|
|
395
598
|
with self._changed:
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
flight.done.set()
|
|
599
|
+
flight.callers -= 1
|
|
600
|
+
self._callers -= 1
|
|
399
601
|
self._changed.notify_all()
|
|
400
602
|
|
|
401
603
|
|
|
402
604
|
_DIAGNOSIS_ADMISSION = _DiagnosisAdmission()
|
|
403
605
|
|
|
404
606
|
|
|
607
|
+
class _StaticValidatorCache:
|
|
608
|
+
"""Bounded validators for bytes already served by this process."""
|
|
609
|
+
|
|
610
|
+
def __init__(self, max_entries: int = 128) -> None:
|
|
611
|
+
self._lock = threading.Lock()
|
|
612
|
+
self._entries: OrderedDict[tuple[pathlib.Path, bool],
|
|
613
|
+
tuple[tuple[int, ...], str]] = OrderedDict()
|
|
614
|
+
self._max_entries = max_entries
|
|
615
|
+
|
|
616
|
+
@staticmethod
|
|
617
|
+
def fingerprint(stat_result: os.stat_result) -> tuple[int, ...]:
|
|
618
|
+
# ctime catches a rewrite that restores size and mtime. Device/inode
|
|
619
|
+
# distinguishes replacement of a mutable-name shell during a build.
|
|
620
|
+
return (stat_result.st_dev, stat_result.st_ino, stat_result.st_size,
|
|
621
|
+
stat_result.st_mtime_ns, stat_result.st_ctime_ns)
|
|
622
|
+
|
|
623
|
+
def get(self, key, fingerprint) -> str | None:
|
|
624
|
+
with self._lock:
|
|
625
|
+
entry = self._entries.get(key)
|
|
626
|
+
if entry is None or entry[0] != fingerprint:
|
|
627
|
+
return None
|
|
628
|
+
self._entries.move_to_end(key)
|
|
629
|
+
return entry[1]
|
|
630
|
+
|
|
631
|
+
def put(self, key, fingerprint, etag: str) -> None:
|
|
632
|
+
with self._lock:
|
|
633
|
+
self._entries[key] = (fingerprint, etag)
|
|
634
|
+
self._entries.move_to_end(key)
|
|
635
|
+
if len(self._entries) > self._max_entries:
|
|
636
|
+
self._entries.popitem(last=False)
|
|
637
|
+
|
|
638
|
+
|
|
639
|
+
_STATIC_VALIDATORS = _StaticValidatorCache()
|
|
640
|
+
|
|
641
|
+
|
|
405
642
|
def _cctally():
|
|
406
643
|
"""Resolve the current ``cctally`` module at call-time (spec §5.5)."""
|
|
407
644
|
return sys.modules["cctally"]
|
|
@@ -475,6 +712,10 @@ from _lib_aggregators import (
|
|
|
475
712
|
_finalize_bucket,
|
|
476
713
|
)
|
|
477
714
|
from _lib_fmt import stable_sum
|
|
715
|
+
# The MODULE, not just the names: #714's `pricing_snapshot_context` has to be
|
|
716
|
+
# reached through the live module object so a test that swaps the snapshot is
|
|
717
|
+
# seen by the request, build and sync entries below.
|
|
718
|
+
import _lib_pricing
|
|
478
719
|
from _lib_pricing import (_calculate_entry_cost, _chip_for_model,
|
|
479
720
|
_short_model_name, claude_usage_dict)
|
|
480
721
|
from _lib_five_hour import _canonical_5h_window_key, _round_to_ten_minutes
|
|
@@ -505,6 +746,14 @@ from _cctally_cache import (
|
|
|
505
746
|
open_cache_db as _raw_open_cache_db,
|
|
506
747
|
open_conversations_db, sync_cache, sync_claude_conversations,
|
|
507
748
|
sync_codex_conversations,
|
|
749
|
+
# #780 read-route admission. Imported here so the conversation route module
|
|
750
|
+
# reaches them the same way it reaches `open_conversations_db` — through
|
|
751
|
+
# `sys.modules["_cctally_dashboard"]`, which is the seam its tests patch.
|
|
752
|
+
open_conversations_db_readonly,
|
|
753
|
+
probe_conversations_maintenance_free,
|
|
754
|
+
ConversationReaderUnavailable, MaintenanceInProgress,
|
|
755
|
+
SchemaBehind, SchemaAhead, LegacyBridgePending,
|
|
756
|
+
pricing_refusal_cause_phrase,
|
|
508
757
|
_prune_orphaned_cache_entries,
|
|
509
758
|
)
|
|
510
759
|
from _lib_snapshot_cache import (
|
|
@@ -657,6 +906,7 @@ from _cctally_dashboard_envelope import (
|
|
|
657
906
|
_envelope_rows_projected,
|
|
658
907
|
_envelope_rows_project_budget,
|
|
659
908
|
_ENVELOPE_AXIS_MAPPERS,
|
|
909
|
+
_alert_account_resolver,
|
|
660
910
|
_build_alerts_envelope_array,
|
|
661
911
|
_build_meter_rate_change_array,
|
|
662
912
|
_model_breakdowns_to_models,
|
|
@@ -734,18 +984,27 @@ def _codex_totals_wire(totals) -> dict[str, Any]:
|
|
|
734
984
|
}
|
|
735
985
|
|
|
736
986
|
|
|
737
|
-
def
|
|
738
|
-
"""Open
|
|
987
|
+
def _codex_detail_context(snapshot):
|
|
988
|
+
"""Open ONE sync-free Codex read context for a detail route (#815).
|
|
989
|
+
|
|
990
|
+
Yields exactly one ``DashboardReadContext`` and nothing else. Each resource
|
|
991
|
+
then loads only what it reads.
|
|
992
|
+
|
|
993
|
+
The retired ``_codex_detail_inputs`` loaded a full year of qualified Codex
|
|
994
|
+
accounting — ``range_start = now_utc - timedelta(days=365)`` — plus a
|
|
995
|
+
35-day, 1,000-row quota read, before either builder selected a row.
|
|
996
|
+
Measured on the production cache at S9 kickoff, every one of the 216,165
|
|
997
|
+
``codex_session_entries`` rows falls inside that year predicate, so the
|
|
998
|
+
year bounded nothing and each detail click loaded the whole corpus. The
|
|
999
|
+
block builder received no qualified entries at all and read none, and the
|
|
1000
|
+
third value that function yielded — ``_codex_entries_from_qualified(
|
|
1001
|
+
qualified)`` — was consumed by neither builder.
|
|
1002
|
+
"""
|
|
739
1003
|
from _cctally_cache import open_cache_db
|
|
740
1004
|
from _cctally_dashboard_sources import (
|
|
741
|
-
DASHBOARD_QUOTA_OBSERVATION_LIMIT,
|
|
742
|
-
DASHBOARD_QUOTA_RECENT_DAYS,
|
|
743
1005
|
DashboardReadContext,
|
|
744
|
-
_codex_entries_from_qualified,
|
|
745
1006
|
resolve_dashboard_source_semantics,
|
|
746
1007
|
)
|
|
747
|
-
from _cctally_quota import load_codex_quota_observations
|
|
748
|
-
from _cctally_source_analytics import load_qualified_codex_entries
|
|
749
1008
|
|
|
750
1009
|
now_utc = getattr(snapshot, "generated_at", None) or _command_as_of()
|
|
751
1010
|
if now_utc.tzinfo is None or now_utc.utcoffset() is None:
|
|
@@ -780,74 +1039,176 @@ def _codex_detail_inputs(snapshot):
|
|
|
780
1039
|
codex_quota_projected_thresholds=semantics.codex_quota_projected_thresholds,
|
|
781
1040
|
cache_report_anomaly_threshold_pp=semantics.cache_report_anomaly_threshold_pp,
|
|
782
1041
|
)
|
|
783
|
-
|
|
784
|
-
range_start,
|
|
785
|
-
now_utc + dt.timedelta(microseconds=1),
|
|
786
|
-
speed=semantics.speed,
|
|
787
|
-
sync=False,
|
|
788
|
-
cache_conn=cache_conn,
|
|
789
|
-
)
|
|
790
|
-
entries = _codex_entries_from_qualified(qualified)
|
|
791
|
-
active_roots = tuple(sorted(
|
|
792
|
-
str(row[0]) for row in cache_conn.execute(
|
|
793
|
-
"SELECT source_root_key FROM codex_source_roots"
|
|
794
|
-
)
|
|
795
|
-
))
|
|
796
|
-
observations = load_codex_quota_observations(
|
|
797
|
-
source_root_keys=active_roots,
|
|
798
|
-
cache_conn=cache_conn,
|
|
799
|
-
captured_at_or_after=(
|
|
800
|
-
now_utc - dt.timedelta(days=DASHBOARD_QUOTA_RECENT_DAYS)
|
|
801
|
-
),
|
|
802
|
-
active_at=now_utc,
|
|
803
|
-
max_rows=DASHBOARD_QUOTA_OBSERVATION_LIMIT,
|
|
804
|
-
)
|
|
805
|
-
yield context, qualified, entries, observations
|
|
1042
|
+
yield context
|
|
806
1043
|
finally:
|
|
807
1044
|
cache_conn.close()
|
|
808
1045
|
stats_conn.close()
|
|
809
1046
|
|
|
810
1047
|
|
|
811
|
-
def
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
1048
|
+
def _codex_generation_metadata_state(snapshot) -> str:
|
|
1049
|
+
"""The FROZEN generation's typed verdict on qualified project attribution.
|
|
1050
|
+
|
|
1051
|
+
#769 S6 / #781. `metadata_availability` used to be set only by
|
|
1052
|
+
`_codex_partial_source_detail`, and that branch was selected by
|
|
1053
|
+
`QualifiedMetadataUnavailable` escaping the expensive builder. A path that
|
|
1054
|
+
never calls the builder can never raise it, so the status is decided here
|
|
1055
|
+
instead, from what the published generation already carries.
|
|
1056
|
+
|
|
1057
|
+
#834 S2 (#828, #829) repoints it from the Projects CAPABILITY to the typed
|
|
1058
|
+
`metadata_health`, for two reasons. The capability is decided over the
|
|
1059
|
+
generation's ~30-day accounting window while both detail routes read a
|
|
1060
|
+
YEAR, so a malformed row aged thirty-one to three hundred and sixty-five
|
|
1061
|
+
days set no flag at all and the page rendered in full with that row's cost
|
|
1062
|
+
silently absent. And the capability is one boolean, so a deterministically
|
|
1063
|
+
unqualifiable row and a health read that FAILED published the same partial
|
|
1064
|
+
explanation — the reader was told to rebuild the cache over a transient
|
|
1065
|
+
SQLite error.
|
|
1066
|
+
|
|
1067
|
+
An absent or unreadable carrier answers `transient_read_failure` rather
|
|
1068
|
+
than `healthy`: the route reaches this only after `source_detail_lookup`
|
|
1069
|
+
read the same state, so it is defensive, and a claim of complete
|
|
1070
|
+
attribution is the wrong thing to guess.
|
|
1071
|
+
"""
|
|
1072
|
+
try:
|
|
1073
|
+
health = snapshot.source_bundle.sources["codex"].metadata_health
|
|
1074
|
+
except (AttributeError, KeyError, TypeError):
|
|
1075
|
+
return "transient_read_failure"
|
|
1076
|
+
if not isinstance(health, Mapping):
|
|
1077
|
+
return "transient_read_failure"
|
|
1078
|
+
state = health.get("state")
|
|
1079
|
+
return state if isinstance(state, str) and state else "transient_read_failure"
|
|
1080
|
+
|
|
1081
|
+
|
|
1082
|
+
def _codex_generation_metadata_partial(snapshot) -> bool:
|
|
1083
|
+
"""Whether this generation's attribution is anything other than healthy."""
|
|
1084
|
+
return _codex_generation_metadata_state(snapshot) != "healthy"
|
|
1085
|
+
|
|
1086
|
+
|
|
1087
|
+
#: What each non-healthy state tells the reader. #834 S2 (#829): these are two
|
|
1088
|
+
#: different situations with two different remedies, and before the typed
|
|
1089
|
+
#: carrier they published the same sentence — so a reader met by a transient
|
|
1090
|
+
#: SQLite error was told to rebuild their Codex cache.
|
|
1091
|
+
_METADATA_PARTIAL_REASONS = {
|
|
1092
|
+
"malformed_row_partial": "Project metadata is unavailable for this item.",
|
|
1093
|
+
"transient_read_failure": (
|
|
1094
|
+
"This build could not check project metadata health; it will retry on "
|
|
1095
|
+
"the next refresh."
|
|
1096
|
+
),
|
|
1097
|
+
}
|
|
1098
|
+
#: An unrecognized state is explained as a transient failure rather than left
|
|
1099
|
+
#: unexplained, matching `_codex_generation_metadata_state`'s own fallback.
|
|
1100
|
+
_METADATA_UNKNOWN_REASON = _METADATA_PARTIAL_REASONS["transient_read_failure"]
|
|
838
1101
|
|
|
839
1102
|
|
|
840
|
-
def
|
|
841
|
-
|
|
1103
|
+
def _metadata_detail_fields(state: str, *, missing: bool) -> dict[str, Any]:
|
|
1104
|
+
"""The two metadata keys every Codex detail carries, healthy or not.
|
|
1105
|
+
|
|
1106
|
+
#834 S2 (#829). Healthy details used to OMIT both keys, and
|
|
1107
|
+
`tests/test_codex_session_detail_rows.py` asserted their absence. They are
|
|
1108
|
+
explicitly null now, which is a deliberate wire change: a client cannot
|
|
1109
|
+
tell an omitted key from a key this build could not fill, and that is the
|
|
1110
|
+
exact ambiguity the typed health result exists to end. The schema bump to
|
|
1111
|
+
12 carries it.
|
|
1112
|
+
"""
|
|
1113
|
+
if state == "healthy" or not missing:
|
|
1114
|
+
return {"metadata_availability": None, "metadata_reason": None}
|
|
1115
|
+
return {
|
|
1116
|
+
"metadata_availability": "partial",
|
|
1117
|
+
"metadata_reason": _METADATA_PARTIAL_REASONS.get(
|
|
1118
|
+
state, _METADATA_UNKNOWN_REASON),
|
|
1119
|
+
}
|
|
1120
|
+
|
|
1121
|
+
|
|
1122
|
+
def _build_codex_session_detail(snapshot, row: Mapping, *,
|
|
1123
|
+
key: str) -> dict[str, Any]:
|
|
1124
|
+
"""Build the session detail from the row the route already looked up.
|
|
1125
|
+
|
|
1126
|
+
#769 S6 / #781. `source_detail_lookup` reads the frozen published bundle
|
|
1127
|
+
with no I/O, matches `data["sessions"]["rows"]` by key and enforces account
|
|
1128
|
+
ownership. The published row is a strict superset of the accounting fields
|
|
1129
|
+
this detail returns, so no accounting query is required — which also means
|
|
1130
|
+
no refresh can mix a newer database row into an older published row.
|
|
1131
|
+
|
|
1132
|
+
The retired implementation reloaded a full year of qualified Codex entries,
|
|
1133
|
+
rebuilt every session row and linearly rescanned them, recomputing
|
|
1134
|
+
`dashboard_resource_key` per row, to find the same row again. It also built
|
|
1135
|
+
that key from `row.codex_root` raw while `_session_wire` builds it from
|
|
1136
|
+
`identity_path(row.codex_root)`; with an alias active the two constructions
|
|
1137
|
+
named different rows and the request 404ed. Serving from the published row
|
|
1138
|
+
leaves exactly one construction.
|
|
1139
|
+
|
|
1140
|
+
No read of `private_session_labels` is introduced. That map is populated
|
|
1141
|
+
unconditionally and request privacy is applied later to request-local
|
|
1142
|
+
envelope copies, so reading it here would bypass the gate. The label
|
|
1143
|
+
continues to arrive through the route's existing row merge.
|
|
1144
|
+
"""
|
|
1145
|
+
if not isinstance(row, Mapping) or not row:
|
|
1146
|
+
raise SourceResourceNotFound()
|
|
1147
|
+
detail: dict[str, Any] = {
|
|
1148
|
+
"detail_kind": "codex_session",
|
|
1149
|
+
"key": key,
|
|
1150
|
+
"last_activity": row.get("last_activity"),
|
|
1151
|
+
"cost_usd": row.get("cost_usd"),
|
|
1152
|
+
"input_tokens": row.get("input_tokens"),
|
|
1153
|
+
"cached_input_tokens": row.get("cached_input_tokens"),
|
|
1154
|
+
"output_tokens": row.get("output_tokens"),
|
|
1155
|
+
"reasoning_output_tokens": row.get("reasoning_output_tokens"),
|
|
1156
|
+
"total_tokens": row.get("total_tokens"),
|
|
1157
|
+
"models": list(row.get("models") or ()),
|
|
1158
|
+
"model_breakdowns": _codex_model_breakdowns(
|
|
1159
|
+
row.get("model_breakdowns") or ()),
|
|
1160
|
+
}
|
|
1161
|
+
# The same rule the partial branch applied, over the same two inputs: a
|
|
1162
|
+
# session is partial only when the generation's attribution is incomplete
|
|
1163
|
+
# AND this row resolved no project of its own. Both keys are now always
|
|
1164
|
+
# present, null when the session is not partial.
|
|
1165
|
+
detail.update(_metadata_detail_fields(
|
|
1166
|
+
_codex_generation_metadata_state(snapshot),
|
|
1167
|
+
missing=not row.get("project"),
|
|
1168
|
+
))
|
|
1169
|
+
return detail
|
|
1170
|
+
|
|
1171
|
+
|
|
1172
|
+
def _build_codex_project_detail(context, *, key: str) -> dict[str, Any]:
|
|
1173
|
+
"""Serve one Codex project from that project's OWN accounting rows (#815).
|
|
1174
|
+
|
|
1175
|
+
The published project row cannot replace this read, unlike the session
|
|
1176
|
+
case: it carries neither `models` nor a per-session breakdown, and it is
|
|
1177
|
+
built over roughly thirty calendar days while this detail is built over a
|
|
1178
|
+
year, so substituting it would silently change every number reported.
|
|
1179
|
+
What the read stops doing is loading the OTHER projects' rows.
|
|
1180
|
+
|
|
1181
|
+
`opaque_project_key` is not reversible and `dashboard_resource_key` is a
|
|
1182
|
+
second non-reversible digest over it, so neither identifier becomes a SQL
|
|
1183
|
+
predicate. The scoped reader re-derives each candidate's opaque key over
|
|
1184
|
+
the metadata tables and this matcher decides, which is why it is a
|
|
1185
|
+
predicate rather than a value.
|
|
1186
|
+
"""
|
|
842
1187
|
from _lib_source_analytics import build_codex_project_result
|
|
1188
|
+
from _cctally_source_analytics import load_codex_project_scoped_entries
|
|
843
1189
|
|
|
1190
|
+
qualified = load_codex_project_scoped_entries(
|
|
1191
|
+
context.range_start,
|
|
1192
|
+
context.now_utc + dt.timedelta(microseconds=1),
|
|
1193
|
+
speed=context.speed,
|
|
1194
|
+
matches_project_key=lambda candidate: (
|
|
1195
|
+
dashboard_resource_key("project", "codex", candidate) == key
|
|
1196
|
+
),
|
|
1197
|
+
cache_conn=context.cache_conn,
|
|
1198
|
+
)
|
|
1199
|
+
# `blocks` and `allocation_entries` are GONE, and removing them is what
|
|
1200
|
+
# makes a scoped read legitimate rather than a silent numeric change. They
|
|
1201
|
+
# feed only `quota_attributions`, which this detail has never emitted.
|
|
1202
|
+
# There is genuine cross-project allocation: `allocation_entries` supplies
|
|
1203
|
+
# the whole root-and-window cost denominator while the project's own
|
|
1204
|
+
# entries supply the numerator, so passing the scoped population as
|
|
1205
|
+
# `allocation_entries` would set the project's quota share to 100%.
|
|
1206
|
+
# Removing the argument is correct; narrowing it is not.
|
|
844
1207
|
result = build_codex_project_result(
|
|
845
1208
|
qualified,
|
|
846
1209
|
range_start=context.range_start,
|
|
847
1210
|
range_end=context.now_utc + dt.timedelta(microseconds=1),
|
|
848
|
-
blocks=build_blocks(observations),
|
|
849
1211
|
as_of=context.now_utc,
|
|
850
|
-
allocation_entries=qualified,
|
|
851
1212
|
include_breakdown=True,
|
|
852
1213
|
)
|
|
853
1214
|
data = result.data
|
|
@@ -889,29 +1250,125 @@ def _build_codex_project_detail(context, qualified, observations, *, key: str) -
|
|
|
889
1250
|
raise SourceResourceNotFound()
|
|
890
1251
|
|
|
891
1252
|
|
|
892
|
-
def _build_codex_block_detail(context,
|
|
1253
|
+
def _build_codex_block_detail(context, *, key: str,
|
|
1254
|
+
row: "Mapping | None" = None) -> dict[str, Any]:
|
|
1255
|
+
"""Serve one Codex quota block from its OWN physical group (#815).
|
|
1256
|
+
|
|
1257
|
+
The candidate is resolved FIRST and the evidence load is scoped to it,
|
|
1258
|
+
which inverts the previous ordering for a reason: the candidate supplies
|
|
1259
|
+
the coordinates the load is scoped to.
|
|
1260
|
+
|
|
1261
|
+
The retired shape loaded the last 35 days capped at 1,000 rows before this
|
|
1262
|
+
ran. That bound is on CARDINALITY, not on the answer — it can return rows
|
|
1263
|
+
belonging to windows nobody asked about while dropping rows belonging to
|
|
1264
|
+
the window that was asked about, and ``block.observations``,
|
|
1265
|
+
``percent_milestones``, ``quota_freshness`` and ``forecast_quota`` are then
|
|
1266
|
+
all recomputed from a set missing members. A group larger than the cap, or
|
|
1267
|
+
an already-reset window sorted behind an active one, starved the requested
|
|
1268
|
+
window completely and the route answered a spurious 404.
|
|
1269
|
+
"""
|
|
1270
|
+
import _lib_accounts
|
|
893
1271
|
from _lib_quota import build_blocks, forecast_quota, percent_milestones, quota_freshness
|
|
894
|
-
from _cctally_quota import assert_projection_readable
|
|
1272
|
+
from _cctally_quota import assert_projection_readable, load_codex_quota_observations
|
|
1273
|
+
from _lib_quota_ledger import loading_unit_from_identity, snap_equivalent_raw_groups
|
|
895
1274
|
|
|
1275
|
+
published = row if isinstance(row, Mapping) else {}
|
|
896
1276
|
# BEFORE the SQL, per #496 S5b section 4.7.
|
|
897
1277
|
assert_projection_readable(context.stats_conn)
|
|
1278
|
+
# PREDICATED ON THE PUBLISHED ROW. The candidate read was `ORDER BY
|
|
1279
|
+
# resets_at_utc DESC … LIMIT 250` with no predicate, so a request for a
|
|
1280
|
+
# block beyond the 250th most recent was not found at all — and the
|
|
1281
|
+
# account constraint below would have shared that cap across accounts.
|
|
1282
|
+
# `source_detail_lookup` has already validated and returned the published
|
|
1283
|
+
# row, and `_codex_quota_block_rows` builds both that row's `resets_at`
|
|
1284
|
+
# field and the resource key from the SAME stored `resets_at_utc`, and its
|
|
1285
|
+
# `window_minutes` from the same stored column. A match therefore REQUIRES
|
|
1286
|
+
# both to be equal, so filtering on them cannot reject a block the
|
|
1287
|
+
# unfiltered scan would have matched.
|
|
1288
|
+
predicate = ""
|
|
1289
|
+
params: tuple[object, ...] = ()
|
|
1290
|
+
published_reset = published.get("resets_at")
|
|
1291
|
+
published_minutes = published.get("window_minutes")
|
|
1292
|
+
if isinstance(published_reset, str) and published_reset:
|
|
1293
|
+
try:
|
|
1294
|
+
minutes = int(published_minutes)
|
|
1295
|
+
except (TypeError, ValueError):
|
|
1296
|
+
minutes = None
|
|
1297
|
+
if minutes is not None:
|
|
1298
|
+
predicate = " AND resets_at_utc=? AND window_minutes=?"
|
|
1299
|
+
params = (published_reset, minutes)
|
|
898
1300
|
rows = context.stats_conn.execute(
|
|
899
1301
|
"SELECT source_root_key, logical_limit_key, observed_slot, window_minutes, "
|
|
900
|
-
"limit_name, resets_at_utc, current_percent, orphaned_at "
|
|
901
|
-
"FROM quota_window_blocks WHERE source='codex' "
|
|
902
|
-
"ORDER BY resets_at_utc DESC, source_root_key, logical_limit_key, observed_slot LIMIT 250"
|
|
1302
|
+
"limit_name, resets_at_utc, current_percent, orphaned_at, account_key "
|
|
1303
|
+
"FROM quota_window_blocks WHERE source='codex'" + predicate + " "
|
|
1304
|
+
"ORDER BY resets_at_utc DESC, source_root_key, logical_limit_key, observed_slot LIMIT 250",
|
|
1305
|
+
params,
|
|
903
1306
|
).fetchall()
|
|
1307
|
+
# THE ACCOUNT IS PART OF THE BLOCK'S UNIQUENESS BUT NOT OF THE KEY. Two
|
|
1308
|
+
# accounts agreeing on root, limit key, slot, window and reset produce one
|
|
1309
|
+
# resource key, and the `ORDER BY` alone decided which one this loop
|
|
1310
|
+
# returned — so a request qualified to A could be answered with B's block.
|
|
1311
|
+
# The published row carries `account_key` only under decoration, and
|
|
1312
|
+
# decoration is active only above one REAL account, which is also the only
|
|
1313
|
+
# condition under which two blocks can collide. A single-account store
|
|
1314
|
+
# therefore takes no predicate and its payload stays byte-identical.
|
|
1315
|
+
owner = published.get("account_key")
|
|
904
1316
|
matched = None
|
|
905
|
-
for
|
|
1317
|
+
for candidate_row in rows:
|
|
906
1318
|
candidate = dashboard_resource_key(
|
|
907
|
-
"block", "codex",
|
|
1319
|
+
"block", "codex", candidate_row[0], candidate_row[1],
|
|
1320
|
+
candidate_row[2], candidate_row[3], candidate_row[5],
|
|
908
1321
|
)
|
|
909
|
-
if candidate
|
|
910
|
-
|
|
911
|
-
|
|
1322
|
+
if candidate != key:
|
|
1323
|
+
continue
|
|
1324
|
+
# The SAME coercion the wire applies (`_codex_quota_block_rows` builds
|
|
1325
|
+
# `block_account` as `str(block_account or UNATTRIBUTED)`). The column
|
|
1326
|
+
# is `TEXT NOT NULL DEFAULT 'unattributed'` so a NULL is unreachable
|
|
1327
|
+
# today, but coercing it to `""` here while the wire published
|
|
1328
|
+
# `unattributed` would make the two sides disagree and 404.
|
|
1329
|
+
if owner is not None and str(
|
|
1330
|
+
candidate_row[8] or _lib_accounts.UNATTRIBUTED
|
|
1331
|
+
) != str(owner):
|
|
1332
|
+
continue
|
|
1333
|
+
matched = candidate_row
|
|
1334
|
+
break
|
|
912
1335
|
if matched is None:
|
|
913
1336
|
raise SourceResourceNotFound()
|
|
1337
|
+
matched_account = str(matched[8] or _lib_accounts.UNATTRIBUTED)
|
|
914
1338
|
reset_at = parse_iso_datetime(str(matched[5]), "codex.block.resets_at")
|
|
1339
|
+
# THE ORDER IS LOAD-BEARING. The loader matches RAW stored coordinates, and
|
|
1340
|
+
# rows are stored under whichever spelling the provider sent, so the
|
|
1341
|
+
# interpreted block tuple has to be converted to its loading unit and then
|
|
1342
|
+
# closed under the `window_minutes` snap before it can name a stored row.
|
|
1343
|
+
# Passing the interpreted tuple straight through matches nothing and turns
|
|
1344
|
+
# this route into a silent 404 generator.
|
|
1345
|
+
groups = snap_equivalent_raw_groups((
|
|
1346
|
+
loading_unit_from_identity(
|
|
1347
|
+
source_root_key=str(matched[0]),
|
|
1348
|
+
logical_limit_key=str(matched[1]),
|
|
1349
|
+
observed_slot=str(matched[2]),
|
|
1350
|
+
window_minutes=int(matched[3]),
|
|
1351
|
+
canonical_reset_iso=str(matched[5]),
|
|
1352
|
+
),
|
|
1353
|
+
))
|
|
1354
|
+
# The whole ACCOUNT-INDEPENDENT group, never pre-filtered by account, so
|
|
1355
|
+
# `adopt_unidentified_observations` still sees every witness.
|
|
1356
|
+
#
|
|
1357
|
+
# `source_root_keys` is the matched block's own root, which NARROWS the
|
|
1358
|
+
# loader's root-scoped operator-attribution overlay rather than avoiding
|
|
1359
|
+
# it. Spec §9 residual 4 stays open: the overlay still runs, and on the
|
|
1360
|
+
# production Codex shape — one root per account — the set it reaches is
|
|
1361
|
+
# IDENTICAL to what it reached before, so the narrowing buys nothing on a
|
|
1362
|
+
# single-root store. It bounds the overlay only where several roots exist.
|
|
1363
|
+
# Spec §3.3 would also have permitted skipping the overlay under a
|
|
1364
|
+
# validated non-weekly scope; scoping it is the more conservative of the
|
|
1365
|
+
# two, and it is why the residual is narrowed and not closed.
|
|
1366
|
+
observations = load_codex_quota_observations(
|
|
1367
|
+
source_root_keys=(str(matched[0]),),
|
|
1368
|
+
cache_conn=context.cache_conn,
|
|
1369
|
+
active_at=context.now_utc,
|
|
1370
|
+
physical_groups=groups,
|
|
1371
|
+
)
|
|
915
1372
|
matching_observations = tuple(
|
|
916
1373
|
observation for observation in observations
|
|
917
1374
|
if observation.identity.source_root_key == matched[0]
|
|
@@ -935,8 +1392,29 @@ def _build_codex_block_detail(context, observations, *, key: str) -> dict[str, A
|
|
|
935
1392
|
# anchor), so both sides now agree.
|
|
936
1393
|
and observation.canonical_resets_at == reset_at
|
|
937
1394
|
)
|
|
1395
|
+
# THE ACCOUNT SELECTS THE BLOCK, NOT ONLY THE CANDIDATE ROW. The load above
|
|
1396
|
+
# is deliberately account-independent so `adopt_unidentified_observations`
|
|
1397
|
+
# sees every witness, and `build_blocks` keys each block on the full
|
|
1398
|
+
# `QuotaWindowIdentity`, which carries `account_key` (`_lib_quota.py:375`).
|
|
1399
|
+
# One physical window observed by two accounts therefore yields TWO blocks
|
|
1400
|
+
# at the same `resets_at`, and matching on the reset alone returned
|
|
1401
|
+
# whichever `identity_sort_key` ordered first — the account ASCENDING
|
|
1402
|
+
# (`:547`). The route then rendered the matched row's `current_percent`
|
|
1403
|
+
# beside the OTHER account's `observations`, and with them its
|
|
1404
|
+
# `milestones`, `forecast` and `freshness`, all of which derive from
|
|
1405
|
+
# `block.observations` below.
|
|
1406
|
+
#
|
|
1407
|
+
# The predicate is the matched candidate row's own account rather than the
|
|
1408
|
+
# published `owner`, because `matched[8]` is defined on every path —
|
|
1409
|
+
# including an undecorated store and a direct builder call with no
|
|
1410
|
+
# published row — and because it is the same row that supplies the rendered
|
|
1411
|
+
# `current_percent`, which is what makes the payload internally consistent.
|
|
938
1412
|
block = next(
|
|
939
|
-
(
|
|
1413
|
+
(
|
|
1414
|
+
item for item in build_blocks(matching_observations)
|
|
1415
|
+
if item.resets_at == reset_at
|
|
1416
|
+
and item.identity.account_key == matched_account
|
|
1417
|
+
),
|
|
940
1418
|
None,
|
|
941
1419
|
)
|
|
942
1420
|
if block is None:
|
|
@@ -980,28 +1458,102 @@ def _build_codex_block_detail(context, observations, *, key: str) -> dict[str, A
|
|
|
980
1458
|
}
|
|
981
1459
|
|
|
982
1460
|
|
|
983
|
-
def _build_codex_source_detail(snapshot, *, resource: str, key: str
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
1461
|
+
def _build_codex_source_detail(snapshot, *, resource: str, key: str,
|
|
1462
|
+
row: "Mapping | None" = None) -> dict[str, Any]:
|
|
1463
|
+
if resource == "session":
|
|
1464
|
+
# #769 S6 / #781. The session detail is served from the published row,
|
|
1465
|
+
# so this resource opens no read context at all. #769 S9 / #815 then
|
|
1466
|
+
# removed the year-long qualified load from the other two: the block
|
|
1467
|
+
# route never read what it loaded, and the project route reads only its
|
|
1468
|
+
# own project's rows through `load_codex_project_scoped_entries`. No
|
|
1469
|
+
# route pays a year-long unscoped accounting read any more.
|
|
1470
|
+
return _build_codex_session_detail(snapshot, row or {}, key=key)
|
|
1471
|
+
# THE DEGRADED PATH IS PRESERVED DELIBERATELY (#815, spec §3.1). The
|
|
1472
|
+
# year-long qualified load these routes used to share was value-dead for
|
|
1473
|
+
# the block route but not BEHAVIOUR-dead: a `QualifiedMetadataUnavailable`
|
|
1474
|
+
# raised while qualifying is caught around this builder and routes the
|
|
1475
|
+
# request into the published-row partial fallback, so on a store with
|
|
1476
|
+
# incomplete Codex metadata both routes degrade today. Deleting the load
|
|
1477
|
+
# would have removed that silently. The frozen generation already decided
|
|
1478
|
+
# the same question, so it is consulted here instead of being provoked
|
|
1479
|
+
# through a read neither route needs.
|
|
1480
|
+
#
|
|
1481
|
+
# THE GATE COVERS THE PROJECT ROUTE TOO, and deliberately. Restricting it
|
|
1482
|
+
# to the block route, as spec §3.1 literally prescribes, would leave a
|
|
1483
|
+
# malformed accounting row outside the resolved project raising nothing at
|
|
1484
|
+
# all, which makes the disclosure loss larger rather than smaller.
|
|
1485
|
+
#
|
|
1486
|
+
# WHAT THE TWO WINDOWS ACTUALLY ARE. The generation's accounting window is
|
|
1487
|
+
# `context.range_start`, roughly thirty calendar days from
|
|
1488
|
+
# `resolve_shared_range(..., n=30)` and at most about thirty-one once the
|
|
1489
|
+
# configured Codex budget month widens it. That is a SUBSET of this
|
|
1490
|
+
# detail's 365 days, not a superset. Because
|
|
1491
|
+
# `_CODEX_PROJECT_METADATA_HEALTH_SQL` mirrors the qualifier's own
|
|
1492
|
+
# acceptance rule over the same rows, a partial generation always implies
|
|
1493
|
+
# the retired detail read would already have raised, so no request that
|
|
1494
|
+
# used to render a full page degrades to a partial one now.
|
|
1495
|
+
#
|
|
1496
|
+
# THE NARROWING IS IN THE OTHER DIRECTION, and spec §9 records it: a
|
|
1497
|
+
# malformed row aged thirty-one to three hundred and sixty-five days used
|
|
1498
|
+
# to degrade both detail routes through the year-long load, and it now sets
|
|
1499
|
+
# no flag at all — the page renders in full with that row's cost silently
|
|
1500
|
+
# absent from the totals.
|
|
1501
|
+
# The carrier's own verdict has to TRAVEL with this raise, because this
|
|
1502
|
+
# exception is manufactured from the generation rather than thrown by a
|
|
1503
|
+
# read. Left to the constructor default (`transient=True`, chosen because
|
|
1504
|
+
# an unclassified read failure is safer read as transient), a
|
|
1505
|
+
# `malformed_row_partial` generation published the transient retry sentence
|
|
1506
|
+
# on every project detail while its own status chip advised the cache
|
|
1507
|
+
# rebuild — two surfaces naming opposite remedies for one cause, and the
|
|
1508
|
+
# retry was never going to arrive, because `retryable` on that carrier is
|
|
1509
|
+
# false. Every deterministic raise site in `_cctally_source_analytics`
|
|
1510
|
+
# states `transient` explicitly; so must this one.
|
|
1511
|
+
carrier_state = _codex_generation_metadata_state(snapshot)
|
|
1512
|
+
if carrier_state != "healthy":
|
|
1513
|
+
raise sys.modules["_cctally_source_analytics"].QualifiedMetadataUnavailable(
|
|
1514
|
+
"Codex qualified project metadata is unavailable",
|
|
1515
|
+
transient=carrier_state == "transient_read_failure",
|
|
1516
|
+
)
|
|
1517
|
+
for context in _codex_detail_context(snapshot):
|
|
1518
|
+
detail = (
|
|
1519
|
+
_build_codex_project_detail(context, key=key)
|
|
1520
|
+
if resource == "project"
|
|
1521
|
+
else _build_codex_block_detail(context, key=key, row=row)
|
|
1522
|
+
)
|
|
1523
|
+
# Only a HEALTHY generation reaches here — the predicate above raises
|
|
1524
|
+
# otherwise — so both keys are published explicitly null rather than
|
|
1525
|
+
# omitted. #834 S2 (#829): an omitted key and a key this build could
|
|
1526
|
+
# not fill look identical to a client.
|
|
1527
|
+
detail.update(_metadata_detail_fields("healthy", missing=False))
|
|
1528
|
+
return detail
|
|
990
1529
|
raise SourceResourceNotFound()
|
|
991
1530
|
|
|
992
1531
|
|
|
993
1532
|
def _codex_partial_source_detail(snapshot, row: Mapping, *, resource: str,
|
|
994
|
-
key: str
|
|
1533
|
+
key: str,
|
|
1534
|
+
read_failure_transient: "bool | None" = None,
|
|
1535
|
+
) -> dict[str, Any]:
|
|
995
1536
|
"""Keep a published cache-only row usable when project metadata is partial."""
|
|
996
1537
|
detail = _source_safe_native_detail(
|
|
997
1538
|
row, source="codex", resource=resource, key=key,
|
|
998
1539
|
)
|
|
999
1540
|
metadata_missing = resource != "session" or not row.get("project")
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1541
|
+
# The CAUSE decides here, not the generation carrier. A project or block
|
|
1542
|
+
# reaches this fallback because THIS request's year-long live read raised,
|
|
1543
|
+
# and a healthy published carrier cannot speak to that: it measured a
|
|
1544
|
+
# different read over a different window at a different instant. Consulting
|
|
1545
|
+
# it published `metadata_availability: null` over the truncated cache-only
|
|
1546
|
+
# payload this function exists to serve, so a degraded serve reported
|
|
1547
|
+
# complete attribution. `transient` distinguishes the two remedies exactly
|
|
1548
|
+
# as it does on the generation path.
|
|
1549
|
+
if read_failure_transient is None:
|
|
1550
|
+
state = _codex_generation_metadata_state(snapshot)
|
|
1551
|
+
else:
|
|
1552
|
+
state = (
|
|
1553
|
+
"transient_read_failure" if read_failure_transient
|
|
1554
|
+
else "malformed_row_partial"
|
|
1555
|
+
)
|
|
1556
|
+
detail.update(_metadata_detail_fields(state, missing=metadata_missing))
|
|
1005
1557
|
if resource == "session":
|
|
1006
1558
|
detail.setdefault("models", [])
|
|
1007
1559
|
detail.setdefault("model_breakdowns", [])
|
|
@@ -1194,6 +1746,7 @@ PROJECT_WINDOW_WEEKS_CHOICES = (1, 4, 8, 12)
|
|
|
1194
1746
|
|
|
1195
1747
|
def resolve_aggregate_project_window_weeks(
|
|
1196
1748
|
requested: int, *, shared_start_at: object, current_week_start_at: object,
|
|
1749
|
+
interval_starts: object = None,
|
|
1197
1750
|
) -> int:
|
|
1198
1751
|
"""The drill window a row published by the All ranking must resolve with.
|
|
1199
1752
|
|
|
@@ -1233,9 +1786,34 @@ def resolve_aggregate_project_window_weeks(
|
|
|
1233
1786
|
)
|
|
1234
1787
|
except ValueError:
|
|
1235
1788
|
return requested
|
|
1789
|
+
# #750 S4 §4.7: a choice's span comes from the SELECTED INTERVAL
|
|
1790
|
+
# IDENTITIES when the caller has them. `cw_start - 7 * (weeks - 1)` assumes
|
|
1791
|
+
# every cycle is seven calendar days, and a credited week adds a cycle
|
|
1792
|
+
# WITHOUT adding seven days — so the arithmetic can decide that four cycles
|
|
1793
|
+
# cover a shared start the last four emitted intervals do not reach. The
|
|
1794
|
+
# instants are `projects.trend.weeks[].week_start_at`, oldest first. Absent
|
|
1795
|
+
# or unparseable, the seven-day walk stands: it is what an envelope
|
|
1796
|
+
# predating that field supports, and it is unchanged there.
|
|
1797
|
+
parsed_starts: list = []
|
|
1798
|
+
if isinstance(interval_starts, (list, tuple)):
|
|
1799
|
+
for value in interval_starts:
|
|
1800
|
+
if not isinstance(value, str):
|
|
1801
|
+
continue
|
|
1802
|
+
try:
|
|
1803
|
+
parsed_starts.append(parse_iso_datetime(
|
|
1804
|
+
value, "projects.trend.week_start_at"))
|
|
1805
|
+
except ValueError:
|
|
1806
|
+
continue
|
|
1807
|
+
parsed_starts.sort()
|
|
1808
|
+
|
|
1809
|
+
def _span_start(weeks: int) -> "dt.datetime":
|
|
1810
|
+
if len(parsed_starts) >= weeks:
|
|
1811
|
+
return parsed_starts[-weeks]
|
|
1812
|
+
return cw_start - dt.timedelta(days=7 * (weeks - 1))
|
|
1813
|
+
|
|
1236
1814
|
covering = [
|
|
1237
1815
|
weeks for weeks in PROJECT_WINDOW_WEEKS_CHOICES
|
|
1238
|
-
if
|
|
1816
|
+
if _span_start(weeks) <= shared_start
|
|
1239
1817
|
]
|
|
1240
1818
|
if not covering:
|
|
1241
1819
|
return max(PROJECT_WINDOW_WEEKS_CHOICES[-1], requested)
|
|
@@ -1272,12 +1850,18 @@ def _project_window_weeks_for_key(snapshot, key: str, requested: int) -> int:
|
|
|
1272
1850
|
return requested
|
|
1273
1851
|
env = getattr(snapshot, "projects_envelope", None)
|
|
1274
1852
|
current = env.get("current_week") if isinstance(env, Mapping) else None
|
|
1853
|
+
trend = env.get("trend") if isinstance(env, Mapping) else None
|
|
1854
|
+
weeks = trend.get("weeks") if isinstance(trend, Mapping) else None
|
|
1275
1855
|
return resolve_aggregate_project_window_weeks(
|
|
1276
1856
|
requested,
|
|
1277
1857
|
shared_start_at=_published_aggregate_start_at(snapshot),
|
|
1278
1858
|
current_week_start_at=(
|
|
1279
1859
|
current.get("week_start_at") if isinstance(current, Mapping) else None
|
|
1280
1860
|
),
|
|
1861
|
+
interval_starts=[
|
|
1862
|
+
w.get("week_start_at") for w in (weeks or ())
|
|
1863
|
+
if isinstance(w, Mapping)
|
|
1864
|
+
],
|
|
1281
1865
|
)
|
|
1282
1866
|
|
|
1283
1867
|
|
|
@@ -1468,7 +2052,9 @@ def build_source_detail(*, snapshot, source: str, resource: str,
|
|
|
1468
2052
|
window_weeks=window_weeks,
|
|
1469
2053
|
)
|
|
1470
2054
|
try:
|
|
1471
|
-
detail = _build_codex_source_detail(
|
|
2055
|
+
detail = _build_codex_source_detail(
|
|
2056
|
+
snapshot, resource=resource, key=key, row=row,
|
|
2057
|
+
)
|
|
1472
2058
|
if resource == "session":
|
|
1473
2059
|
for field in ("label", "project", "started_at", "duration_min"):
|
|
1474
2060
|
detail[field] = row.get(field)
|
|
@@ -1481,9 +2067,10 @@ def build_source_detail(*, snapshot, source: str, resource: str,
|
|
|
1481
2067
|
):
|
|
1482
2068
|
detail[field] = row.get(field)
|
|
1483
2069
|
return detail
|
|
1484
|
-
except sys.modules["_cctally_source_analytics"].QualifiedMetadataUnavailable:
|
|
2070
|
+
except sys.modules["_cctally_source_analytics"].QualifiedMetadataUnavailable as exc:
|
|
1485
2071
|
return _codex_partial_source_detail(
|
|
1486
2072
|
snapshot, row, resource=resource, key=key,
|
|
2073
|
+
read_failure_transient=bool(getattr(exc, "transient", True)),
|
|
1487
2074
|
)
|
|
1488
2075
|
|
|
1489
2076
|
_ensure_sibling_loaded("_cctally_dashboard_share")
|
|
@@ -1685,6 +2272,13 @@ def _make_dashboard_run_iteration(
|
|
|
1685
2272
|
|
|
1686
2273
|
def run_iteration(batch=None) -> dict:
|
|
1687
2274
|
warnings: list = []
|
|
2275
|
+
# #714: the whole snapshot build runs on ONE pricing revision. A swap
|
|
2276
|
+
# landing part-way through would publish a snapshot whose panels were
|
|
2277
|
+
# priced from two different tables.
|
|
2278
|
+
with _lib_pricing.pricing_snapshot_context():
|
|
2279
|
+
return _run_iteration_body(batch, warnings)
|
|
2280
|
+
|
|
2281
|
+
def _run_iteration_body(batch, warnings: list) -> dict:
|
|
1688
2282
|
if batch is not None and batch[1] and not skip_sync:
|
|
1689
2283
|
with sync_lock:
|
|
1690
2284
|
result = _refresh_usage_inproc()
|
|
@@ -1850,6 +2444,215 @@ def _dashboard_sync_loop(
|
|
|
1850
2444
|
sleep(min(0.1, max(0.0, deadline - now)))
|
|
1851
2445
|
|
|
1852
2446
|
|
|
2447
|
+
def _dashboard_startup_schema_migration(*, run_derivations: bool = True) -> None:
|
|
2448
|
+
"""Advance both stores to head once, at startup, in EVERY mode (#780).
|
|
2449
|
+
|
|
2450
|
+
Under `--no-sync` nothing else will: `_dashboard_self_heal_orphans` is a
|
|
2451
|
+
no-op with `skip_sync=True` and `_make_conversation_sync_thread` returns
|
|
2452
|
+
None, so no process in that mode ever opens the store write-capable. The
|
|
2453
|
+
read-only reader refuses a store that is behind head — correctly, because
|
|
2454
|
+
it must never migrate from a request thread — and without an owner here a
|
|
2455
|
+
`--no-sync` dashboard would serve a degraded conversation surface forever
|
|
2456
|
+
with nothing able to fix it.
|
|
2457
|
+
|
|
2458
|
+
This performs the SCHEMA work only: one full open, which applies the schema
|
|
2459
|
+
and runs the migration dispatcher, and then closes. It ingests nothing, so
|
|
2460
|
+
it does not reintroduce the writing that `--no-sync` exists to suppress.
|
|
2461
|
+
It never raises: a store that cannot be opened leaves the reader degraded,
|
|
2462
|
+
which is the state it was already in.
|
|
2463
|
+
|
|
2464
|
+
#769 S6 / #802: "the SCHEMA work only" was not true of the open itself.
|
|
2465
|
+
`_open_conversations_db_unlocked` runs `_import_legacy_conversation_rows`
|
|
2466
|
+
and `_ensure_codex_conversation_contract` AFTER the dispatcher, as two
|
|
2467
|
+
separate calls the dispatcher does not own, and the second consumes
|
|
2468
|
+
`conversation_rebuild_codex_pending` and rebuilds the retained Codex
|
|
2469
|
+
corpus — the measured 135-second startup. `run_derivations=False` bounds
|
|
2470
|
+
this open to what the docstring already claimed. The caller ALSO sets the
|
|
2471
|
+
process-level policy, because the read-route fallback and the live-tail
|
|
2472
|
+
opener each reach the same full opener from a request thread.
|
|
2473
|
+
"""
|
|
2474
|
+
try:
|
|
2475
|
+
conn = sys.modules["cctally"].open_conversations_db(
|
|
2476
|
+
run_derivations=run_derivations)
|
|
2477
|
+
except Exception as exc: # noqa: BLE001
|
|
2478
|
+
eprint(f"[conversations] startup schema migration skipped: {exc}")
|
|
2479
|
+
return
|
|
2480
|
+
try:
|
|
2481
|
+
conn.close()
|
|
2482
|
+
except Exception: # noqa: BLE001
|
|
2483
|
+
pass
|
|
2484
|
+
|
|
2485
|
+
|
|
2486
|
+
#: The one-shot wake-up dispatch state. `_SCHEMA_WAKE_THREAD` is the migration
|
|
2487
|
+
#: thread currently in flight, or None, and `_SCHEMA_WAKE_LOCK` serializes the
|
|
2488
|
+
#: check-and-start so eight simultaneous schema-behind requests start one
|
|
2489
|
+
#: migration rather than eight.
|
|
2490
|
+
_SCHEMA_WAKE_LOCK = threading.Lock()
|
|
2491
|
+
_SCHEMA_WAKE_THREAD: "threading.Thread | None" = None
|
|
2492
|
+
|
|
2493
|
+
|
|
2494
|
+
def _reset_schema_wake_state() -> None:
|
|
2495
|
+
"""Drop the recorded wake-up thread. Test seam only."""
|
|
2496
|
+
global _SCHEMA_WAKE_THREAD
|
|
2497
|
+
with _SCHEMA_WAKE_LOCK:
|
|
2498
|
+
_SCHEMA_WAKE_THREAD = None
|
|
2499
|
+
|
|
2500
|
+
|
|
2501
|
+
def _join_schema_wake_thread(timeout: float = 30.0) -> None:
|
|
2502
|
+
"""Wait for an in-flight wake-up to finish. Test seam only."""
|
|
2503
|
+
with _SCHEMA_WAKE_LOCK:
|
|
2504
|
+
thread = _SCHEMA_WAKE_THREAD
|
|
2505
|
+
if thread is not None:
|
|
2506
|
+
thread.join(timeout)
|
|
2507
|
+
|
|
2508
|
+
|
|
2509
|
+
def _wake_schema_writer() -> None:
|
|
2510
|
+
"""`SCHEMA_WAKE_HOOK`: a reader found a store behind head (#780).
|
|
2511
|
+
|
|
2512
|
+
The reader raised rather than migrating, because a request thread must
|
|
2513
|
+
never run the dispatcher. This performs the same schema-only open the
|
|
2514
|
+
startup owner does, so the next request is admitted.
|
|
2515
|
+
|
|
2516
|
+
IT MUST NOT DO THAT WORK HERE. `_gate_reader_schema` calls this hook
|
|
2517
|
+
inline, on the request thread, immediately before that thread returns its
|
|
2518
|
+
degraded envelope — and the work is a full `open_conversations_db()`
|
|
2519
|
+
including `_run_pending_migrations`, measured at 39.55 s for migration 009
|
|
2520
|
+
plus its backfill on the production-shaped store. Running it here holds the
|
|
2521
|
+
request open for that long and contradicts both the spec ("The opener never
|
|
2522
|
+
migrates from a request thread") and the retry comment in this file. So the
|
|
2523
|
+
hook dispatches onto one daemon thread and returns.
|
|
2524
|
+
|
|
2525
|
+
One at a time. Every schema-behind request reaches this hook, so an
|
|
2526
|
+
unguarded dispatch would start one full open per request against a store
|
|
2527
|
+
that is already behind, which adds write contention rather than progress.
|
|
2528
|
+
The guard is per-flight, not permanent: once the thread finishes, a store
|
|
2529
|
+
that falls behind again can wake a writer again.
|
|
2530
|
+
|
|
2531
|
+
Best-effort and silent on failure either way: the reader has already
|
|
2532
|
+
answered with its typed degraded envelope.
|
|
2533
|
+
|
|
2534
|
+
THE START IS INSIDE THE LOCK, and that is the whole guard. `is_alive()` is
|
|
2535
|
+
False for a thread that has been constructed but not started yet, so a
|
|
2536
|
+
start outside the lock leaves a window in which a second caller reads a
|
|
2537
|
+
non-None, not-alive thread, passes the guard, and dispatches the second
|
|
2538
|
+
39.55-second migration this exists to prevent. Holding the lock across the
|
|
2539
|
+
start costs only the microseconds `Thread.start` takes to hand off; the
|
|
2540
|
+
migration itself runs on the new thread, outside the lock.
|
|
2541
|
+
"""
|
|
2542
|
+
global _SCHEMA_WAKE_THREAD
|
|
2543
|
+
with _SCHEMA_WAKE_LOCK:
|
|
2544
|
+
if _SCHEMA_WAKE_THREAD is not None and _SCHEMA_WAKE_THREAD.is_alive():
|
|
2545
|
+
return
|
|
2546
|
+
thread = threading.Thread(
|
|
2547
|
+
target=_dashboard_startup_schema_migration,
|
|
2548
|
+
name="cctally-schema-wake",
|
|
2549
|
+
daemon=True,
|
|
2550
|
+
)
|
|
2551
|
+
_SCHEMA_WAKE_THREAD = thread
|
|
2552
|
+
try:
|
|
2553
|
+
thread.start()
|
|
2554
|
+
except RuntimeError as exc: # interpreter shutdown, thread limit
|
|
2555
|
+
eprint(f"[conversations] schema wake-up not dispatched ({exc})")
|
|
2556
|
+
_SCHEMA_WAKE_THREAD = None
|
|
2557
|
+
|
|
2558
|
+
|
|
2559
|
+
#: How often the pricing watch stats the deployed pricing file. A monotonic
|
|
2560
|
+
#: cadence, because a wall-clock one would re-poll or stall across a system
|
|
2561
|
+
#: clock change.
|
|
2562
|
+
_PRICING_WATCH_INTERVAL_S = 60.0
|
|
2563
|
+
|
|
2564
|
+
|
|
2565
|
+
class _PricingWatch:
|
|
2566
|
+
"""Detect a replaced `_lib_pricing.py` and adopt it in process (#714).
|
|
2567
|
+
|
|
2568
|
+
An installer overwrites the deployed file under a running dashboard.
|
|
2569
|
+
Nothing about that reaches this process: every CLI invocation re-imports
|
|
2570
|
+
and picks the new tables up, and only a long-lived process holds the old
|
|
2571
|
+
ones.
|
|
2572
|
+
|
|
2573
|
+
STAT FIRST, PARSE SECOND. The poll compares an inode/size/mtime signature
|
|
2574
|
+
and parses only when it changed, so the steady state is one `stat` a
|
|
2575
|
+
minute rather than an `ast.parse` of a 1,400-line file.
|
|
2576
|
+
|
|
2577
|
+
NEVER RAISES OUT OF `poll_once`. The file it reads is one an installer may
|
|
2578
|
+
be part-way through writing, so a half-written file is the ordinary case
|
|
2579
|
+
rather than an exceptional one; a raising poll would end the thread and
|
|
2580
|
+
leave the process permanently stale, which is the very condition this
|
|
2581
|
+
exists to clear.
|
|
2582
|
+
"""
|
|
2583
|
+
|
|
2584
|
+
def __init__(self, source_path=None, interval_s=None):
|
|
2585
|
+
self.source_path = pathlib.Path(
|
|
2586
|
+
source_path if source_path is not None
|
|
2587
|
+
else sys.modules["_lib_pricing"].__file__)
|
|
2588
|
+
self.interval_s = (_PRICING_WATCH_INTERVAL_S if interval_s is None
|
|
2589
|
+
else interval_s)
|
|
2590
|
+
self._signature = None
|
|
2591
|
+
self._stop = threading.Event()
|
|
2592
|
+
|
|
2593
|
+
def _current_signature(self):
|
|
2594
|
+
try:
|
|
2595
|
+
st = self.source_path.stat()
|
|
2596
|
+
except OSError:
|
|
2597
|
+
return None
|
|
2598
|
+
return (st.st_dev, st.st_ino, st.st_size, st.st_mtime_ns)
|
|
2599
|
+
|
|
2600
|
+
def poll_once(self) -> bool:
|
|
2601
|
+
"""One check. Returns True iff a new snapshot was adopted."""
|
|
2602
|
+
try:
|
|
2603
|
+
signature = self._current_signature()
|
|
2604
|
+
if signature is None or signature == self._signature:
|
|
2605
|
+
return False
|
|
2606
|
+
self._signature = signature
|
|
2607
|
+
pricing = sys.modules["_lib_pricing"]
|
|
2608
|
+
candidate = pricing.read_pricing_snapshot_from_source(
|
|
2609
|
+
self.source_path)
|
|
2610
|
+
if not pricing.adopt_pricing_candidate(candidate):
|
|
2611
|
+
return False
|
|
2612
|
+
eprint("[pricing] adopted pricing revision "
|
|
2613
|
+
f"{candidate.snapshot_date} without restarting")
|
|
2614
|
+
return True
|
|
2615
|
+
except Exception: # noqa: BLE001
|
|
2616
|
+
# Including `PricingCandidateRejected`: a rejected candidate is a
|
|
2617
|
+
# normal outcome, and the signature is already recorded so the
|
|
2618
|
+
# same bad file is not re-parsed every minute.
|
|
2619
|
+
return False
|
|
2620
|
+
|
|
2621
|
+
def run(self) -> None:
|
|
2622
|
+
while not self._stop.wait(self.interval_s):
|
|
2623
|
+
self.poll_once()
|
|
2624
|
+
|
|
2625
|
+
def stop(self) -> None:
|
|
2626
|
+
self._stop.set()
|
|
2627
|
+
|
|
2628
|
+
|
|
2629
|
+
_PRICING_WATCH: "_PricingWatch | None" = None
|
|
2630
|
+
|
|
2631
|
+
|
|
2632
|
+
def _start_pricing_watch() -> "_PricingWatch | None":
|
|
2633
|
+
"""Own the pricing detector on a lifecycle thread that runs in BOTH modes.
|
|
2634
|
+
|
|
2635
|
+
Deliberately not `_DashboardSyncThread` and not the conversation sync
|
|
2636
|
+
thread: `cmd_dashboard` disables both under `--no-sync`, so a detector in
|
|
2637
|
+
either would silently have no owner exactly where a long-lived process is
|
|
2638
|
+
most likely to run.
|
|
2639
|
+
"""
|
|
2640
|
+
global _PRICING_WATCH
|
|
2641
|
+
if _PRICING_WATCH is not None:
|
|
2642
|
+
return _PRICING_WATCH
|
|
2643
|
+
try:
|
|
2644
|
+
watch = _PricingWatch()
|
|
2645
|
+
watch.poll_once() # establish the baseline signature; adopts nothing
|
|
2646
|
+
thread = threading.Thread(
|
|
2647
|
+
target=watch.run, name="cctally-pricing-watch", daemon=True)
|
|
2648
|
+
thread.start()
|
|
2649
|
+
except Exception as exc: # noqa: BLE001
|
|
2650
|
+
eprint(f"[pricing] watch not started ({exc})")
|
|
2651
|
+
return None
|
|
2652
|
+
_PRICING_WATCH = watch
|
|
2653
|
+
return watch
|
|
2654
|
+
|
|
2655
|
+
|
|
1853
2656
|
def _dashboard_maybe_prune_retention() -> None:
|
|
1854
2657
|
"""#313 P3 (F7): throttled transcript retention prune driven from the
|
|
1855
2658
|
independent conversation sync thread. Opens a dedicated transcript
|
|
@@ -1869,6 +2672,7 @@ def _dashboard_maybe_prune_retention() -> None:
|
|
|
1869
2672
|
conn,
|
|
1870
2673
|
now_utc=dt.datetime.now(dt.timezone.utc),
|
|
1871
2674
|
retention_days=retention_days,
|
|
2675
|
+
record_phase=_lib_tick_stats.record_maintenance_phase,
|
|
1872
2676
|
)
|
|
1873
2677
|
finally:
|
|
1874
2678
|
conn.close()
|
|
@@ -1981,6 +2785,16 @@ def _conversation_sync_pass() -> str:
|
|
|
1981
2785
|
documented contract is a throttled prune driven by this thread, not a prune
|
|
1982
2786
|
conditional on a successful ingest.
|
|
1983
2787
|
|
|
2788
|
+
#780 makes the prune's own phases visible without moving them out of this
|
|
2789
|
+
measured body. The plan asked for retention to leave the measured body; it
|
|
2790
|
+
must not, because F5 above is exactly the rule that anything left outside it
|
|
2791
|
+
is work outside the duty denominator, and reclaim is real work this thread
|
|
2792
|
+
performs. What the reclaim actually needed was its own reporting, not its
|
|
2793
|
+
own denominator: `delete`, `reclaim` and `checkpoint` are each recorded in
|
|
2794
|
+
`_lib_tick_stats`'s separate maintenance ring with a duration, an outcome
|
|
2795
|
+
and a pending flag, so a long reclaim can no longer hide inside one
|
|
2796
|
+
`conversation_sync status="ok"` while the duty bound still covers it.
|
|
2797
|
+
|
|
1984
2798
|
The returned status describes the OPEN and the two SYNCS, and nothing else.
|
|
1985
2799
|
`_dashboard_maybe_prune_retention`'s outcome is discarded here and that
|
|
1986
2800
|
function ends in `except Exception: pass`, so a pass that ingested cleanly
|
|
@@ -1998,7 +2812,18 @@ def _conversation_sync_pass() -> str:
|
|
|
1998
2812
|
therefore NO SECOND OPEN ATTEMPT AFTER A FAILED OPEN — stated that narrowly
|
|
1999
2813
|
because a successful pass opens the store twice by design, once here and
|
|
2000
2814
|
once inside the prune.
|
|
2815
|
+
|
|
2816
|
+
#714: the whole pass runs on ONE pricing revision. This pass writes
|
|
2817
|
+
materialized cost, so a swap landing between the Claude leg and the Codex
|
|
2818
|
+
leg would derive one store's rollup from two revisions — the ordered-write
|
|
2819
|
+
guard would not even see it, because both halves are the same process.
|
|
2001
2820
|
"""
|
|
2821
|
+
with _lib_pricing.pricing_snapshot_context():
|
|
2822
|
+
return _conversation_sync_pass_body()
|
|
2823
|
+
|
|
2824
|
+
|
|
2825
|
+
def _conversation_sync_pass_body() -> str:
|
|
2826
|
+
"""The body of `_conversation_sync_pass`, run under its pricing context."""
|
|
2002
2827
|
try:
|
|
2003
2828
|
conn = open_conversations_db()
|
|
2004
2829
|
except (OSError, sqlite3.DatabaseError) as exc:
|
|
@@ -2252,6 +3077,47 @@ def _dashboard_self_heal_orphans(*, skip_sync):
|
|
|
2252
3077
|
# Invalidation must never turn a successful prune into a failure;
|
|
2253
3078
|
# a stale-cache tick is self-corrected once the signature next moves.
|
|
2254
3079
|
pass
|
|
3080
|
+
if result is not None and result.prune_refused:
|
|
3081
|
+
# #729: the deletions committed and the rollup re-derive that should
|
|
3082
|
+
# have followed them did not. The cache invalidation above still runs,
|
|
3083
|
+
# because those caches are keyed on the deletions and the deletions
|
|
3084
|
+
# happened; this adds the two things a refusal needs on top.
|
|
3085
|
+
#
|
|
3086
|
+
# The certificate goes because the prune mutates the transcript store
|
|
3087
|
+
# OUTSIDE a sync pass, so no stats object reaches
|
|
3088
|
+
# `provider_sync_certifiable` and nothing else would notice. Claude
|
|
3089
|
+
# only: the orphan prune surface is Claude-only, and Codex certifies on
|
|
3090
|
+
# its own evidence.
|
|
3091
|
+
# #769 S3: name the cause the prune actually recorded. Since #728 a
|
|
3092
|
+
# refusal has three possible causes and only one of them is a pricing
|
|
3093
|
+
# version difference.
|
|
3094
|
+
eprint(
|
|
3095
|
+
f"[dashboard] orphan prune: the conversation rollup re-derive was "
|
|
3096
|
+
f"refused for {result.prune_refused_files} file(s) — "
|
|
3097
|
+
f"{pricing_refusal_cause_phrase(result.prune_refused_state)}"
|
|
3098
|
+
f". The rollup backfill is armed for the next authorized process."
|
|
3099
|
+
)
|
|
3100
|
+
try:
|
|
3101
|
+
frontier = getattr(_conversation_sync_pass, "_frontier", None)
|
|
3102
|
+
if frontier is not None:
|
|
3103
|
+
# #769 S3 A9 — the bound this drop carries, stated. The pop
|
|
3104
|
+
# happens on the sync-loop thread while the conversation sync
|
|
3105
|
+
# thread reads the same dict. Both operations are single
|
|
3106
|
+
# bytecodes under the GIL, so neither can observe a torn
|
|
3107
|
+
# mapping; what the drop cannot do is reach a plan the other
|
|
3108
|
+
# thread already captured. A pass that read its certificate
|
|
3109
|
+
# microseconds before this line proceeds against a certificate
|
|
3110
|
+
# this drop has invalidated, for that one pass. The next pass
|
|
3111
|
+
# re-reads and finds nothing, so the window is one pass wide
|
|
3112
|
+
# and self-closing — which is why this is a comment rather than
|
|
3113
|
+
# a lock. Widening it into a lock would mean holding one across
|
|
3114
|
+
# a whole conversation sync pass.
|
|
3115
|
+
frontier.drop_provider("claude")
|
|
3116
|
+
except Exception:
|
|
3117
|
+
# Same posture as the invalidation above: never turn a prune that
|
|
3118
|
+
# did commit its deletions into a raised exception on this
|
|
3119
|
+
# best-effort background path.
|
|
3120
|
+
pass
|
|
2255
3121
|
return result
|
|
2256
3122
|
|
|
2257
3123
|
|
|
@@ -3737,12 +4603,14 @@ def _dashboard_build_weekly_periods(conn: "sqlite3.Connection",
|
|
|
3737
4603
|
for r in reversed(view.rows):
|
|
3738
4604
|
sw = sw_by_window.get((r.week_start_at, r.week_end_at))
|
|
3739
4605
|
if sw is not None:
|
|
3740
|
-
#
|
|
3741
|
-
#
|
|
3742
|
-
#
|
|
3743
|
-
#
|
|
3744
|
-
#
|
|
3745
|
-
|
|
4606
|
+
# The label is NOT re-derived here any more (#750 S4 §3.2a).
|
|
4607
|
+
# `build_weekly_view` already builds it from
|
|
4608
|
+
# `sw.display_start_date` and then applies D5's collision suffix
|
|
4609
|
+
# to the segments of a credited week that render one calendar
|
|
4610
|
+
# day. Overwriting it with a bare `%m-%d` here would strip that
|
|
4611
|
+
# suffix and put the two cycles back under one label — which is
|
|
4612
|
+
# what this panel is supposed to stop doing.
|
|
4613
|
+
#
|
|
3746
4614
|
# is_current keys on segment identity on both sides of the
|
|
3747
4615
|
# comparison; display_start_date may diverge for reset-event
|
|
3748
4616
|
# weeks but that is intentional — display vs. lookup are kept
|
|
@@ -3764,10 +4632,146 @@ def _dashboard_build_weekly_periods(conn: "sqlite3.Connection",
|
|
|
3764
4632
|
return rows[:n]
|
|
3765
4633
|
|
|
3766
4634
|
|
|
4635
|
+
def _retained_facts_from_rows(parent, children) -> "dict[str, Any] | None":
|
|
4636
|
+
"""Shape one parent row plus its child rows into retained facts.
|
|
4637
|
+
|
|
4638
|
+
Returns None when the retained facts fail their own internal
|
|
4639
|
+
invariant: the parent and its children are two tables read at
|
|
4640
|
+
request time with no transaction that makes them agree, so a child
|
|
4641
|
+
set that does not sum to its parent, or a parent with no children at
|
|
4642
|
+
all, is a disagreement in stored data rather than a reason to fail
|
|
4643
|
+
the request. Raising there returned an HTTP 500 to the client, which
|
|
4644
|
+
is the failure `367761bf4` was written to eliminate; a parent with
|
|
4645
|
+
no children would also have served `entries_count: 0` beside a
|
|
4646
|
+
nonzero `cost_usd`.
|
|
4647
|
+
"""
|
|
4648
|
+
if not children:
|
|
4649
|
+
return None
|
|
4650
|
+
parent_cost = float(parent["total_cost_usd"] or 0.0)
|
|
4651
|
+
child_cost = stable_sum(float(c["cost_usd"] or 0.0) for c in children)
|
|
4652
|
+
if abs(child_cost - parent_cost) >= 1e-9:
|
|
4653
|
+
return None
|
|
4654
|
+
return {
|
|
4655
|
+
"cost_usd": parent_cost,
|
|
4656
|
+
"input_tokens": int(parent["total_input_tokens"] or 0),
|
|
4657
|
+
"output_tokens": int(parent["total_output_tokens"] or 0),
|
|
4658
|
+
"cache_creation_tokens": int(parent["total_cache_create_tokens"] or 0),
|
|
4659
|
+
"cache_read_tokens": int(parent["total_cache_read_tokens"] or 0),
|
|
4660
|
+
"entries_count": sum(int(c["entry_count"] or 0) for c in children),
|
|
4661
|
+
"model_breakdowns": [
|
|
4662
|
+
{"modelName": c["model"], "cost": float(c["cost_usd"] or 0.0)}
|
|
4663
|
+
for c in children
|
|
4664
|
+
],
|
|
4665
|
+
}
|
|
4666
|
+
|
|
4667
|
+
|
|
4668
|
+
def _retained_block_facts_many(conn, blocks) -> "dict[int, dict[str, Any]]":
|
|
4669
|
+
"""The frozen parent and child facts for every journal-stamped closed
|
|
4670
|
+
block in `blocks`, keyed by ``id(block)``.
|
|
4671
|
+
|
|
4672
|
+
A closed block carrying a `journal_id` is immutable: later
|
|
4673
|
+
observations and newly discovered cache rows cannot rewrite its
|
|
4674
|
+
totals or its child membership (#399). A reader that recomputes from
|
|
4675
|
+
the cache therefore reports something the rest of the estate does not
|
|
4676
|
+
agree with, which is what `/api/block` did and what the Blocks panel
|
|
4677
|
+
behind it did (#751a review P2-2).
|
|
4678
|
+
|
|
4679
|
+
A block is absent from the result whenever the retained facts are not
|
|
4680
|
+
unambiguously its own — no row, no stamp, still open, more than one
|
|
4681
|
+
account's row on the same physical window, or facts that fail the
|
|
4682
|
+
internal invariant above. Recomputing is the honest answer there; a
|
|
4683
|
+
guess between two accounts' rows is not.
|
|
4684
|
+
|
|
4685
|
+
Two queries regardless of how many blocks are asked about, because
|
|
4686
|
+
the Blocks panel asks about a whole week of them on every tick.
|
|
4687
|
+
"""
|
|
4688
|
+
wanted: "dict[tuple[int, int], list[Any]]" = {}
|
|
4689
|
+
for block in blocks:
|
|
4690
|
+
start = getattr(block, "start_time", None)
|
|
4691
|
+
end = getattr(block, "end_time", None)
|
|
4692
|
+
if start is None or end is None or getattr(block, "is_gap", False):
|
|
4693
|
+
continue
|
|
4694
|
+
key = (
|
|
4695
|
+
int(start.astimezone(dt.timezone.utc).timestamp()),
|
|
4696
|
+
int(end.astimezone(dt.timezone.utc).timestamp()),
|
|
4697
|
+
)
|
|
4698
|
+
wanted.setdefault(key, []).append(block)
|
|
4699
|
+
if not wanted:
|
|
4700
|
+
return {}
|
|
4701
|
+
start_epochs = sorted({key[0] for key in wanted})
|
|
4702
|
+
try:
|
|
4703
|
+
placeholders = ",".join("?" for _ in start_epochs)
|
|
4704
|
+
parent_rows = conn.execute(
|
|
4705
|
+
f"""
|
|
4706
|
+
SELECT id,
|
|
4707
|
+
unixepoch(block_start_at) AS bs_epoch,
|
|
4708
|
+
unixepoch(five_hour_resets_at) AS rs_epoch,
|
|
4709
|
+
total_input_tokens, total_output_tokens,
|
|
4710
|
+
total_cache_create_tokens, total_cache_read_tokens,
|
|
4711
|
+
total_cost_usd
|
|
4712
|
+
FROM five_hour_blocks
|
|
4713
|
+
WHERE is_closed = 1
|
|
4714
|
+
AND journal_id IS NOT NULL
|
|
4715
|
+
AND unixepoch(block_start_at) IN ({placeholders})
|
|
4716
|
+
""",
|
|
4717
|
+
start_epochs,
|
|
4718
|
+
).fetchall()
|
|
4719
|
+
parents: "dict[tuple[int, int], list[Any]]" = {}
|
|
4720
|
+
for row in parent_rows:
|
|
4721
|
+
parents.setdefault(
|
|
4722
|
+
(int(row["bs_epoch"]), int(row["rs_epoch"])), [],
|
|
4723
|
+
).append(row)
|
|
4724
|
+
block_ids = [
|
|
4725
|
+
int(rows[0]["id"])
|
|
4726
|
+
for key, rows in parents.items()
|
|
4727
|
+
if key in wanted and len(rows) == 1
|
|
4728
|
+
]
|
|
4729
|
+
children_by_block: "dict[int, list[Any]]" = {}
|
|
4730
|
+
if block_ids:
|
|
4731
|
+
placeholders = ",".join("?" for _ in block_ids)
|
|
4732
|
+
child_rows = conn.execute(
|
|
4733
|
+
f"SELECT block_id, model, cost_usd, entry_count "
|
|
4734
|
+
f" FROM five_hour_block_models "
|
|
4735
|
+
f" WHERE block_id IN ({placeholders}) "
|
|
4736
|
+
f" ORDER BY cost_usd DESC, model ASC",
|
|
4737
|
+
block_ids,
|
|
4738
|
+
).fetchall()
|
|
4739
|
+
for row in child_rows:
|
|
4740
|
+
children_by_block.setdefault(
|
|
4741
|
+
int(row["block_id"]), [],
|
|
4742
|
+
).append(row)
|
|
4743
|
+
except sqlite3.DatabaseError:
|
|
4744
|
+
return {}
|
|
4745
|
+
out: "dict[int, dict[str, Any]]" = {}
|
|
4746
|
+
for key, claimants in wanted.items():
|
|
4747
|
+
rows = parents.get(key) or []
|
|
4748
|
+
if len(rows) != 1:
|
|
4749
|
+
continue
|
|
4750
|
+
parent = rows[0]
|
|
4751
|
+
facts = _retained_facts_from_rows(
|
|
4752
|
+
parent, children_by_block.get(int(parent["id"]), []),
|
|
4753
|
+
)
|
|
4754
|
+
if facts is None:
|
|
4755
|
+
continue
|
|
4756
|
+
for block in claimants:
|
|
4757
|
+
out[id(block)] = facts
|
|
4758
|
+
return out
|
|
4759
|
+
|
|
4760
|
+
|
|
4761
|
+
def _retained_block_facts(conn, block: "Block") -> "dict[str, Any] | None":
|
|
4762
|
+
"""The frozen parent and child facts for one journal-stamped closed
|
|
4763
|
+
block, or None. Single-block entry point onto
|
|
4764
|
+
``_retained_block_facts_many``; the withholding rules are stated
|
|
4765
|
+
there and are deliberately stated only once.
|
|
4766
|
+
"""
|
|
4767
|
+
return _retained_block_facts_many(conn, [block]).get(id(block))
|
|
4768
|
+
|
|
4769
|
+
|
|
3767
4770
|
def _build_block_detail(block: "Block",
|
|
3768
4771
|
entries: "list[UsageEntry]",
|
|
3769
4772
|
*,
|
|
3770
|
-
display_tz: "ZoneInfo | None" = None
|
|
4773
|
+
display_tz: "ZoneInfo | None" = None,
|
|
4774
|
+
frozen_facts: "dict[str, Any] | None" = None) -> "dict[str, Any]":
|
|
3771
4775
|
"""Build the JSON payload for ``GET /api/block/:start_at``.
|
|
3772
4776
|
|
|
3773
4777
|
Pure function over a single Block plus its constituent entries (already
|
|
@@ -3799,29 +4803,65 @@ def _build_block_detail(block: "Block",
|
|
|
3799
4803
|
"cum": running,
|
|
3800
4804
|
})
|
|
3801
4805
|
|
|
3802
|
-
# Reconcile invariant: per-entry sum equals Block.cost_usd within
|
|
3803
|
-
# Off only if pricing-dict drifted between block aggregation
|
|
4806
|
+
# Reconcile invariant: per-entry sum equals Block.cost_usd within
|
|
4807
|
+
# 1e-9. Off only if pricing-dict drifted between block aggregation
|
|
4808
|
+
# and now. Both sides derive in-process from the one entry list the
|
|
4809
|
+
# grouping pass assigned to this block, so the invariant holds
|
|
4810
|
+
# whether or not the headline below is served from retained facts.
|
|
4811
|
+
# Restricting it to the computed path would drop it for the whole
|
|
4812
|
+
# journal-stamped closed population — the population #751a most
|
|
4813
|
+
# affects — which is why it sits above the split (#769 S2 review
|
|
4814
|
+
# P1-2, prior art `367761bf4`).
|
|
3804
4815
|
assert abs(running - block.cost_usd) < 1e-9, (
|
|
3805
4816
|
f"_build_block_detail reconcile mismatch: per-entry sum "
|
|
3806
4817
|
f"{running!r} vs block.cost_usd {block.cost_usd!r}"
|
|
3807
4818
|
)
|
|
3808
4819
|
|
|
3809
|
-
|
|
3810
|
-
|
|
3811
|
-
|
|
3812
|
-
|
|
3813
|
-
|
|
3814
|
-
|
|
4820
|
+
if frozen_facts is None:
|
|
4821
|
+
headline = {
|
|
4822
|
+
"cost_usd": block.cost_usd,
|
|
4823
|
+
"input_tokens": block.input_tokens,
|
|
4824
|
+
"output_tokens": block.output_tokens,
|
|
4825
|
+
"cache_creation_tokens": block.cache_creation_tokens,
|
|
4826
|
+
"cache_read_tokens": block.cache_read_tokens,
|
|
4827
|
+
"entries_count": block.entries_count,
|
|
4828
|
+
}
|
|
4829
|
+
model_breakdowns = [
|
|
4830
|
+
{"modelName": name, "cost": cost}
|
|
4831
|
+
for name, cost in sorted(
|
|
4832
|
+
per_model_cost.items(), key=lambda kv: -kv[1],
|
|
4833
|
+
)
|
|
4834
|
+
]
|
|
4835
|
+
else:
|
|
4836
|
+
# The frozen path renders retained facts, so the per-entry sum is
|
|
4837
|
+
# NOT expected to equal them — the cache may have grown since the
|
|
4838
|
+
# close, and that growth is exactly what a frozen block ignores.
|
|
4839
|
+
# The retained set's own internal invariant (children sum to their
|
|
4840
|
+
# parent) is checked in `_retained_block_facts`, which withholds
|
|
4841
|
+
# the facts rather than raising here: a live read route must not
|
|
4842
|
+
# return a 500 because two stored tables disagree.
|
|
4843
|
+
headline = {
|
|
4844
|
+
key: frozen_facts[key] for key in (
|
|
4845
|
+
"cost_usd", "input_tokens", "output_tokens",
|
|
4846
|
+
"cache_creation_tokens", "cache_read_tokens",
|
|
4847
|
+
"entries_count",
|
|
4848
|
+
)
|
|
4849
|
+
}
|
|
4850
|
+
model_breakdowns = list(frozen_facts["model_breakdowns"])
|
|
4851
|
+
|
|
4852
|
+
models = _model_breakdowns_to_models(
|
|
4853
|
+
model_breakdowns, headline["cost_usd"],
|
|
4854
|
+
)
|
|
3815
4855
|
|
|
3816
4856
|
# Cache-hit %: cache_read / (input + cache_creation + cache_read) * 100.
|
|
3817
4857
|
# Same denominator as the canonical _compute_cache_hit_percent helper —
|
|
3818
4858
|
# cache_creation contributes to the input-side token count and must be
|
|
3819
4859
|
# in the divisor or the ratio inflates toward 100% on heavy-cache blocks.
|
|
3820
|
-
denom = (
|
|
3821
|
-
+
|
|
3822
|
-
+
|
|
4860
|
+
denom = (headline["input_tokens"]
|
|
4861
|
+
+ headline["cache_creation_tokens"]
|
|
4862
|
+
+ headline["cache_read_tokens"])
|
|
3823
4863
|
cache_hit_pct = (
|
|
3824
|
-
(
|
|
4864
|
+
(headline["cache_read_tokens"] / denom) * 100.0 if denom > 0 else None
|
|
3825
4865
|
)
|
|
3826
4866
|
|
|
3827
4867
|
# Active blocks expose burn_rate / projection in snake_case (the Block
|
|
@@ -3856,14 +4896,21 @@ def _build_block_detail(block: "Block",
|
|
|
3856
4896
|
display_tz, fmt="%H:%M %b %d", suffix=True,
|
|
3857
4897
|
),
|
|
3858
4898
|
|
|
3859
|
-
"entries_count":
|
|
3860
|
-
"cost_usd":
|
|
3861
|
-
"total_tokens":
|
|
3862
|
-
|
|
3863
|
-
|
|
3864
|
-
|
|
3865
|
-
"
|
|
4899
|
+
"entries_count": headline["entries_count"],
|
|
4900
|
+
"cost_usd": headline["cost_usd"],
|
|
4901
|
+
"total_tokens": (headline["input_tokens"]
|
|
4902
|
+
+ headline["output_tokens"]
|
|
4903
|
+
+ headline["cache_creation_tokens"]
|
|
4904
|
+
+ headline["cache_read_tokens"]),
|
|
4905
|
+
"input_tokens": headline["input_tokens"],
|
|
4906
|
+
"output_tokens": headline["output_tokens"],
|
|
4907
|
+
"cache_creation_tokens": headline["cache_creation_tokens"],
|
|
4908
|
+
"cache_read_tokens": headline["cache_read_tokens"],
|
|
3866
4909
|
"cache_hit_pct": cache_hit_pct,
|
|
4910
|
+
# Which body of evidence the numbers above came from. A frozen
|
|
4911
|
+
# closed block reports its retained facts; everything else is
|
|
4912
|
+
# computed from the entries this block owns.
|
|
4913
|
+
"facts_source": "computed" if frozen_facts is None else "retained",
|
|
3867
4914
|
|
|
3868
4915
|
"models": models,
|
|
3869
4916
|
"burn_rate": burn_rate_out,
|
|
@@ -3934,11 +4981,79 @@ def _dashboard_build_blocks_view(conn: "sqlite3.Connection",
|
|
|
3934
4981
|
display_tz=display_tz,
|
|
3935
4982
|
mode="auto",
|
|
3936
4983
|
)
|
|
4984
|
+
view = _blocks_view_with_retained_facts(conn, view)
|
|
3937
4985
|
return _blocks_view_overlapping_week(
|
|
3938
4986
|
view, week_start_at=week_start_at, week_end_at=week_end_at,
|
|
3939
4987
|
)
|
|
3940
4988
|
|
|
3941
4989
|
|
|
4990
|
+
def _blocks_view_with_retained_facts(conn, view):
|
|
4991
|
+
"""Serve each journal-stamped CLOSED block from the facts retained at
|
|
4992
|
+
its close, rather than from cache contents that have moved on.
|
|
4993
|
+
|
|
4994
|
+
Spec line 174 binds readers, plural: `/api/block` was named because
|
|
4995
|
+
it was the reader that recomputed, not because it is the only reader
|
|
4996
|
+
the requirement binds. The Blocks panel behind that route recomputed
|
|
4997
|
+
too, so clicking a row changed the number — on an install written
|
|
4998
|
+
before #751a because the retained figure is the double-counted one,
|
|
4999
|
+
and on any install because the retained figure stops at the block's
|
|
5000
|
+
last observation while a recompute runs to the reset (#769 S2 review
|
|
5001
|
+
P2-2).
|
|
5002
|
+
|
|
5003
|
+
Applied here in the dashboard adapter rather than in
|
|
5004
|
+
``build_blocks_view``: that builder is shared with ``cctally
|
|
5005
|
+
blocks``, which is a ccusage drop-in computing from JSONL and has no
|
|
5006
|
+
stats connection to read. `cctally five-hour-blocks` already renders
|
|
5007
|
+
the same materialized ``total_cost_usd``, so this brings the panel
|
|
5008
|
+
into agreement with it rather than away from it.
|
|
5009
|
+
|
|
5010
|
+
Applied BEFORE ``_blocks_view_overlapping_week``, which re-derives
|
|
5011
|
+
the footer totals from the retained blocks, so rows, ``aggregated``
|
|
5012
|
+
and totals all move together.
|
|
5013
|
+
"""
|
|
5014
|
+
if conn is None:
|
|
5015
|
+
return view
|
|
5016
|
+
blocks = list(view.aggregated)
|
|
5017
|
+
facts_by_block = _retained_block_facts_many(conn, blocks)
|
|
5018
|
+
if not facts_by_block:
|
|
5019
|
+
return view
|
|
5020
|
+
rows_by_start = {r.start_at: r for r in view.rows}
|
|
5021
|
+
for index, block in enumerate(blocks):
|
|
5022
|
+
facts = facts_by_block.get(id(block))
|
|
5023
|
+
if facts is None:
|
|
5024
|
+
continue
|
|
5025
|
+
tokens = (facts["input_tokens"] + facts["output_tokens"]
|
|
5026
|
+
+ facts["cache_creation_tokens"]
|
|
5027
|
+
+ facts["cache_read_tokens"])
|
|
5028
|
+
blocks[index] = dataclasses.replace(
|
|
5029
|
+
block,
|
|
5030
|
+
entries_count=facts["entries_count"],
|
|
5031
|
+
input_tokens=facts["input_tokens"],
|
|
5032
|
+
output_tokens=facts["output_tokens"],
|
|
5033
|
+
cache_creation_tokens=facts["cache_creation_tokens"],
|
|
5034
|
+
cache_read_tokens=facts["cache_read_tokens"],
|
|
5035
|
+
total_tokens=tokens,
|
|
5036
|
+
cost_usd=facts["cost_usd"],
|
|
5037
|
+
models=[mb["modelName"] for mb in facts["model_breakdowns"]],
|
|
5038
|
+
)
|
|
5039
|
+
iso = block.start_time.astimezone(dt.timezone.utc).isoformat()
|
|
5040
|
+
row = rows_by_start.get(iso)
|
|
5041
|
+
if row is not None:
|
|
5042
|
+
rows_by_start[iso] = dataclasses.replace(
|
|
5043
|
+
row,
|
|
5044
|
+
cost_usd=facts["cost_usd"],
|
|
5045
|
+
models=_model_breakdowns_to_models(
|
|
5046
|
+
facts["model_breakdowns"], facts["cost_usd"],
|
|
5047
|
+
),
|
|
5048
|
+
facts_source="retained",
|
|
5049
|
+
)
|
|
5050
|
+
return dataclasses.replace(
|
|
5051
|
+
view,
|
|
5052
|
+
rows=tuple(rows_by_start[r.start_at] for r in view.rows),
|
|
5053
|
+
aggregated=tuple(blocks),
|
|
5054
|
+
)
|
|
5055
|
+
|
|
5056
|
+
|
|
3942
5057
|
def _blocks_view_overlapping_week(view, *, week_start_at, week_end_at):
|
|
3943
5058
|
"""Retain only the blocks whose interval overlaps
|
|
3944
5059
|
``[week_start_at, week_end_at)``, keeping each retained block's FULL
|
|
@@ -4358,18 +5473,59 @@ class _ProjectsWeekGrid:
|
|
|
4358
5473
|
end, never ``start + 7d``.
|
|
4359
5474
|
"""
|
|
4360
5475
|
|
|
4361
|
-
__slots__ = ("starts", "ends", "_end_by_start", "_start_by_date"
|
|
5476
|
+
__slots__ = ("starts", "ends", "_end_by_start", "_start_by_date",
|
|
5477
|
+
"_segment_by_start", "segments")
|
|
4362
5478
|
|
|
4363
|
-
def __init__(self, bounds: "list[tuple[dt.datetime, dt.datetime]]"
|
|
5479
|
+
def __init__(self, bounds: "list[tuple[dt.datetime, dt.datetime]]",
|
|
5480
|
+
segments=None):
|
|
4364
5481
|
ordered = sorted(bounds, key=lambda b: b[0])
|
|
4365
5482
|
self.starts = [b[0] for b in ordered]
|
|
4366
5483
|
self.ends = [b[1] for b in ordered]
|
|
4367
5484
|
self._end_by_start = {s: e for s, e in ordered}
|
|
5485
|
+
# #750 S4 §2.3: the grid keeps the emitted `SubWeek` beside each
|
|
5486
|
+
# interval instead of being handed bare `(start, end)` tuples, so the
|
|
5487
|
+
# segment's own identity survives into the percentage read. Flattening
|
|
5488
|
+
# it away is what left the panel unable to tell two cycles of one
|
|
5489
|
+
# credited week apart (#737).
|
|
5490
|
+
self.segments = list(segments or ())
|
|
5491
|
+
self._segment_by_start = {}
|
|
5492
|
+
for sw in self.segments:
|
|
5493
|
+
try:
|
|
5494
|
+
key = parse_iso_datetime(
|
|
5495
|
+
sw.start_ts, "projects segment.start_ts",
|
|
5496
|
+
).astimezone(dt.timezone.utc)
|
|
5497
|
+
except (TypeError, ValueError):
|
|
5498
|
+
continue
|
|
5499
|
+
self._segment_by_start[key] = sw
|
|
4368
5500
|
# `weekly_usage_snapshots.week_start_date` is the date-only lookup
|
|
4369
|
-
# key a legacy row carries when it has no `week_start_at`.
|
|
4370
|
-
#
|
|
4371
|
-
#
|
|
4372
|
-
|
|
5501
|
+
# key a legacy row carries when it has no `week_start_at`. A date that
|
|
5502
|
+
# resolves to MORE THAN ONE interval is genuinely ambiguous — both
|
|
5503
|
+
# cycles of a credited week share it — so it resolves to nothing
|
|
5504
|
+
# rather than to a guess (#737). `tests/test_620_projects_unresolvable
|
|
5505
|
+
# _week.py` already requires an unresolvable week to contribute
|
|
5506
|
+
# nothing rather than be attributed somewhere.
|
|
5507
|
+
self._start_by_date = {}
|
|
5508
|
+
_ambiguous: set = set()
|
|
5509
|
+
for s in self.starts:
|
|
5510
|
+
if s.date() in self._start_by_date:
|
|
5511
|
+
_ambiguous.add(s.date())
|
|
5512
|
+
self._start_by_date[s.date()] = s
|
|
5513
|
+
for day in _ambiguous:
|
|
5514
|
+
self._start_by_date.pop(day, None)
|
|
5515
|
+
|
|
5516
|
+
def segment_for(self, start: "dt.datetime"):
|
|
5517
|
+
"""The emitted `SubWeek` whose start is ``start``, or None."""
|
|
5518
|
+
return self._segment_by_start.get(start)
|
|
5519
|
+
|
|
5520
|
+
def segments_for_week(self, start_date):
|
|
5521
|
+
"""Every segment of the WEEK whose shared `start_date` is given.
|
|
5522
|
+
|
|
5523
|
+
`start_date` is the join key into
|
|
5524
|
+
`weekly_usage_snapshots.week_start_date`, which epic invariant 3
|
|
5525
|
+
keeps identical across a credited week's cycles. It identifies the
|
|
5526
|
+
week; `segment_key` identifies the cycle.
|
|
5527
|
+
"""
|
|
5528
|
+
return [sw for sw in self.segments if sw.start_date == start_date]
|
|
4373
5529
|
|
|
4374
5530
|
def __bool__(self) -> bool:
|
|
4375
5531
|
return bool(self.starts)
|
|
@@ -4396,7 +5552,12 @@ class _ProjectsWeekGrid:
|
|
|
4396
5552
|
def start_for_date(self, day: "dt.date") -> "dt.datetime | None":
|
|
4397
5553
|
"""The interval start whose own date is ``day``, for a legacy
|
|
4398
5554
|
snapshot row that carries ``week_start_date`` but no
|
|
4399
|
-
``week_start_at``.
|
|
5555
|
+
``week_start_at``.
|
|
5556
|
+
|
|
5557
|
+
``None`` when ``day`` resolves to more than one interval. That is a
|
|
5558
|
+
credited week, whose cycles share the date, and picking the later one
|
|
5559
|
+
would attribute a whole week's spend to whichever cycle happened to
|
|
5560
|
+
sort last (#750 S4 §2.3 / #737)."""
|
|
4400
5561
|
return self._start_by_date.get(day)
|
|
4401
5562
|
|
|
4402
5563
|
def end_for(self, start: "dt.datetime") -> "dt.datetime":
|
|
@@ -4477,6 +5638,7 @@ def _projects_week_grid(
|
|
|
4477
5638
|
# fallback still renders a coherent (if approximate) window.
|
|
4478
5639
|
return None
|
|
4479
5640
|
bounds: "list[tuple[dt.datetime, dt.datetime]]" = []
|
|
5641
|
+
kept: list = []
|
|
4480
5642
|
for sw in subweeks:
|
|
4481
5643
|
try:
|
|
4482
5644
|
s = parse_iso_datetime(
|
|
@@ -4489,18 +5651,25 @@ def _projects_week_grid(
|
|
|
4489
5651
|
continue
|
|
4490
5652
|
if e > s:
|
|
4491
5653
|
bounds.append((s, e))
|
|
5654
|
+
# The segment record travels with its interval (#750 S4 §2.3).
|
|
5655
|
+
kept.append(sw)
|
|
4492
5656
|
if not bounds:
|
|
4493
5657
|
return None
|
|
4494
|
-
return _ProjectsWeekGrid(bounds)
|
|
5658
|
+
return _ProjectsWeekGrid(bounds, kept)
|
|
4495
5659
|
|
|
4496
5660
|
|
|
4497
5661
|
def _projects_week_label(week_start: "dt.datetime") -> str:
|
|
4498
|
-
"""Render a `
|
|
5662
|
+
"""Render a `Mon DD` label for the trend chart x-axis and grid columns.
|
|
5663
|
+
|
|
5664
|
+
The `wk ` prefix is gone (#750 S4 D6). These columns are billing CYCLES,
|
|
5665
|
+
not weeks — a credited week contributes one column per cycle — and the
|
|
5666
|
+
panel heading plus the pills' unit label carry the noun instead. Dropping
|
|
5667
|
+
it also frees the width D5's collision time suffix needs.
|
|
4499
5668
|
|
|
4500
|
-
|
|
4501
|
-
|
|
5669
|
+
UTC-anchored so JSON output is tz-agnostic. The FIELD name stays
|
|
5670
|
+
`week_label`: it is a frozen wire name.
|
|
4502
5671
|
"""
|
|
4503
|
-
return
|
|
5672
|
+
return week_start.strftime("%b %d")
|
|
4504
5673
|
|
|
4505
5674
|
|
|
4506
5675
|
def _projects_iter_session_entries(conn: "sqlite3.Connection",
|
|
@@ -6060,33 +7229,32 @@ def _build_projects_envelope(
|
|
|
6060
7229
|
# (bin/cctally:1162-1168) and the doctor credited-week check
|
|
6061
7230
|
# (bin/cctally:8706-8714).
|
|
6062
7231
|
#
|
|
6063
|
-
#
|
|
6064
|
-
#
|
|
6065
|
-
#
|
|
6066
|
-
#
|
|
6067
|
-
#
|
|
6068
|
-
#
|
|
6069
|
-
#
|
|
6070
|
-
#
|
|
6071
|
-
#
|
|
6072
|
-
#
|
|
7232
|
+
# #750 S4 §2.2: the SQL no longer ranks rows away before Python sees
|
|
7233
|
+
# them. `ROW_NUMBER() OVER (PARTITION BY week_start_date ...)` kept
|
|
7234
|
+
# exactly ONE row per calendar date, and both cycles of a credited week
|
|
7235
|
+
# share that date, so the panel could never give the second cycle its own
|
|
7236
|
+
# reading however the Python below resolved intervals — #731's defect in
|
|
7237
|
+
# a different surface. Every anchored NON-NULL row now crosses into
|
|
7238
|
+
# Python, carrying `captured_at_utc` and `id`, and the shared reducer
|
|
7239
|
+
# applies the segment capture bounds and picks the latest inside each
|
|
7240
|
+
# cycle. NULL rows never erased the prior known value in the former
|
|
7241
|
+
# ascending fold, so they are still excluded. The date bounds keep
|
|
7242
|
+
# historical status-line ticks out of the scan. Do not LIMIT before
|
|
7243
|
+
# Python resolves legacy date keys: an unresolvable key must not evict a
|
|
7244
|
+
# valid rendered-week row (#620 S1 A3).
|
|
6073
7245
|
weekly_pct_by_week: dict[dt.datetime, float] = {}
|
|
6074
7246
|
try:
|
|
6075
7247
|
cur = conn.execute(
|
|
6076
|
-
"WITH ranked AS ("
|
|
6077
7248
|
" SELECT week_start_date, week_start_at, weekly_percent,"
|
|
6078
|
-
" captured_at_utc, id
|
|
6079
|
-
" ROW_NUMBER() OVER ("
|
|
6080
|
-
" PARTITION BY week_start_date"
|
|
6081
|
-
" ORDER BY captured_at_utc DESC, id DESC"
|
|
6082
|
-
" ) AS latest_rank"
|
|
7249
|
+
" captured_at_utc, id"
|
|
6083
7250
|
" FROM weekly_usage_snapshots"
|
|
7251
|
+
# Held rows excluded: these rows feed `latest_usage_by_segment`,
|
|
7252
|
+
# which is a weekly read, and the pre-fetch must select the same
|
|
7253
|
+
# population the reducer would select for itself (#769 S11, #824).
|
|
6084
7254
|
" WHERE week_start_date >= ? AND week_start_date < ?"
|
|
6085
7255
|
" AND date(week_start_date) IS NOT NULL"
|
|
6086
7256
|
" AND weekly_percent IS NOT NULL"
|
|
6087
|
-
|
|
6088
|
-
" SELECT week_start_date, week_start_at, weekly_percent"
|
|
6089
|
-
" FROM ranked WHERE latest_rank = 1"
|
|
7257
|
+
+ _cctally_core.weekly_held_exclusion(conn) +
|
|
6090
7258
|
" ORDER BY captured_at_utc ASC, id ASC",
|
|
6091
7259
|
(
|
|
6092
7260
|
since_dt.date().isoformat(),
|
|
@@ -6098,7 +7266,13 @@ def _build_projects_envelope(
|
|
|
6098
7266
|
# No weekly_usage_snapshots table — leaves attributed_pct = None
|
|
6099
7267
|
# throughout (acceptable per spec §2.7).
|
|
6100
7268
|
rows = []
|
|
6101
|
-
|
|
7269
|
+
# (segment_key, captured_at_utc, id, weekly_percent) for the reducer, plus
|
|
7270
|
+
# the interval each segment key belongs to so the result can be keyed the
|
|
7271
|
+
# way this function's consumers already key it.
|
|
7272
|
+
reducer_rows: list = []
|
|
7273
|
+
start_by_segment_key: dict = {}
|
|
7274
|
+
fallback_rows: list = []
|
|
7275
|
+
for week_date_str, week_start_at, weekly_pct, captured_at, row_id in rows:
|
|
6102
7276
|
try:
|
|
6103
7277
|
wd = dt.date.fromisoformat(week_date_str)
|
|
6104
7278
|
except (TypeError, ValueError):
|
|
@@ -6127,11 +7301,53 @@ def _build_projects_envelope(
|
|
|
6127
7301
|
wstart = grid.start_for_date(wd)
|
|
6128
7302
|
if wstart is None:
|
|
6129
7303
|
continue
|
|
6130
|
-
|
|
6131
|
-
|
|
6132
|
-
|
|
6133
|
-
|
|
6134
|
-
|
|
7304
|
+
segment = grid.segment_for(wstart)
|
|
7305
|
+
if segment is None:
|
|
7306
|
+
# An interval the grid knows without a segment record behind
|
|
7307
|
+
# it (the seven-day padding `window_ending_at` produces). It
|
|
7308
|
+
# cannot carry capture bounds, so it keeps the last-capture
|
|
7309
|
+
# rule the fold already applied.
|
|
7310
|
+
fallback_rows.append((wstart, captured_at, row_id, weekly_pct))
|
|
7311
|
+
continue
|
|
7312
|
+
# The row's `week_start_at` anchor names the WEEK, never the
|
|
7313
|
+
# cycle: every capture of a credited week carries the week's
|
|
7314
|
+
# unchanged anchor, so resolving it to one interval would hand
|
|
7315
|
+
# both cycles' observations to the head and leave the tail empty.
|
|
7316
|
+
# The row is offered to every segment of that week and the
|
|
7317
|
+
# reducer's half-open capture bounds decide which cycle owns it —
|
|
7318
|
+
# exactly one does, because the bounds are contiguous and
|
|
7319
|
+
# disjoint (#750 S4 §1.2).
|
|
7320
|
+
for sibling in grid.segments_for_week(segment.start_date):
|
|
7321
|
+
try:
|
|
7322
|
+
sibling_start = parse_iso_datetime(
|
|
7323
|
+
sibling.start_ts, "projects segment.start_ts",
|
|
7324
|
+
).astimezone(dt.timezone.utc)
|
|
7325
|
+
except (TypeError, ValueError):
|
|
7326
|
+
continue
|
|
7327
|
+
start_by_segment_key[sibling.segment_key] = sibling_start
|
|
7328
|
+
reducer_rows.append(
|
|
7329
|
+
(sibling.segment_key, captured_at, row_id, weekly_pct))
|
|
7330
|
+
continue
|
|
7331
|
+
# No anchor anywhere in the store: the genuine Monday fallback.
|
|
7332
|
+
wstart = _projects_week_start_monday_utc(dt.datetime.combine(
|
|
7333
|
+
wd, dt.time(0, 0, 0), tzinfo=dt.timezone.utc,
|
|
7334
|
+
))
|
|
7335
|
+
fallback_rows.append((wstart, captured_at, row_id, weekly_pct))
|
|
7336
|
+
|
|
7337
|
+
# The bound-and-latest decision is the SHARED one (#750 S4 §2.1): the CLI
|
|
7338
|
+
# and this panel resolve a cycle's percentage through one implementation,
|
|
7339
|
+
# so they cannot answer the same question differently. The rows were
|
|
7340
|
+
# fetched and resolved here because only this function knows the interval
|
|
7341
|
+
# grid; the reducer is handed them rather than opening a second read
|
|
7342
|
+
# against a connection that already holds a transaction.
|
|
7343
|
+
if reducer_rows and grid is not None and grid.segments:
|
|
7344
|
+
for segment_key, pct in c.latest_usage_by_segment(
|
|
7345
|
+
conn, grid.segments, account_key=None, rows=reducer_rows,
|
|
7346
|
+
).items():
|
|
7347
|
+
wstart = start_by_segment_key.get(segment_key)
|
|
7348
|
+
if wstart is not None and pct is not None:
|
|
7349
|
+
weekly_pct_by_week[wstart] = float(pct)
|
|
7350
|
+
for wstart, _captured, _row_id, weekly_pct in fallback_rows:
|
|
6135
7351
|
if weekly_pct is not None:
|
|
6136
7352
|
weekly_pct_by_week[wstart] = float(weekly_pct)
|
|
6137
7353
|
|
|
@@ -6206,6 +7422,11 @@ def _build_projects_envelope(
|
|
|
6206
7422
|
wpct = weekly_pct_by_week.get(w)
|
|
6207
7423
|
trend_weeks_blocks.append({
|
|
6208
7424
|
"week_start_date": w.date().isoformat(),
|
|
7425
|
+
# #750 S4 §3.1/§3.5: the instant, additive, so a consumer can
|
|
7426
|
+
# tell two cycles of a credited week apart. `week_start_date` is
|
|
7427
|
+
# the shared join key and cannot; `ProjectsTrendChart` keys its
|
|
7428
|
+
# x-axis on this.
|
|
7429
|
+
"week_start_at": _iso_z(w),
|
|
6209
7430
|
"week_label": _projects_week_label(w),
|
|
6210
7431
|
"total_cost_usd": total_cost_by_week.get(w, 0.0),
|
|
6211
7432
|
"total_pct": wpct,
|
|
@@ -7112,8 +8333,9 @@ def _stats_ro_guarded():
|
|
|
7112
8333
|
|
|
7113
8334
|
return _cctally_store.stats_open_guarded(
|
|
7114
8335
|
_cctally_core.DB_PATH,
|
|
7115
|
-
|
|
7116
|
-
|
|
8336
|
+
# #778: forward the opener's `cached_statements=0`.
|
|
8337
|
+
connect=lambda p, **kw: sqlite3.connect(
|
|
8338
|
+
f"{pathlib.Path(p).as_uri()}?mode=ro", uri=True, **kw),
|
|
7117
8339
|
)
|
|
7118
8340
|
|
|
7119
8341
|
|
|
@@ -7225,6 +8447,15 @@ def _debug_retained_memory(hub, *, main_frontier_stats=None) -> dict:
|
|
|
7225
8447
|
source_stats.get("measurementPending", 1)),
|
|
7226
8448
|
"measurementErrorCount": int(
|
|
7227
8449
|
source_stats.get("measurementErrorCount", 0)),
|
|
8450
|
+
# A WRITE-TIME admission refusal, which is a different event from
|
|
8451
|
+
# `fallbackCount`: that counts an admission pass finding an
|
|
8452
|
+
# oversized bucket already resident, this counts an entry refused
|
|
8453
|
+
# by the write that would have retained it. `_debug_memory_owner`
|
|
8454
|
+
# returns a fixed six-key set, so without this line the counter
|
|
8455
|
+
# reached neither `/api/debug/backend` nor `cctally dashboard-perf`
|
|
8456
|
+
# and a refusal nobody could see was a diagnostic nobody had.
|
|
8457
|
+
"admissionRefusalCount": int(
|
|
8458
|
+
source_stats.get("admissionRefusalCount", 0)),
|
|
7228
8459
|
"workerAlive": int(source_stats.get("workerAlive", 0)),
|
|
7229
8460
|
})
|
|
7230
8461
|
|
|
@@ -7379,6 +8610,19 @@ _GET_ROUTES = (
|
|
|
7379
8610
|
("scope", "endpoint.conversation_detail"), True),
|
|
7380
8611
|
)
|
|
7381
8612
|
|
|
8613
|
+
#: Handlers that block for the life of a client connection rather than for one
|
|
8614
|
+
#: response. `_dispatch` must NOT pin a pricing revision around these: the pin
|
|
8615
|
+
#: would last as long as the stream, so a viewer who left a tab open would keep
|
|
8616
|
+
#: pricing from whichever revision was live when the tab connected — the exact
|
|
8617
|
+
#: staleness #714 removes. Both are streams that carry no cost of their own;
|
|
8618
|
+
#: the envelopes they forward are priced by the build that produced them, which
|
|
8619
|
+
#: has its own context.
|
|
8620
|
+
_STREAMING_ROUTE_HANDLERS = frozenset({
|
|
8621
|
+
"_serve_api_events",
|
|
8622
|
+
"_handle_get_update_stream",
|
|
8623
|
+
"_handle_get_conversation_events",
|
|
8624
|
+
})
|
|
8625
|
+
|
|
7382
8626
|
_POST_ROUTES = (
|
|
7383
8627
|
("exact", "/api/auth", "_handle_post_auth", None, False),
|
|
7384
8628
|
("exact", "/api/sync", "_handle_post_sync", None, False),
|
|
@@ -7514,6 +8758,13 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
|
|
|
7514
8758
|
eleven path-taking handlers. (NOTE: returning a bool, not ``fn()`` —
|
|
7515
8759
|
the void handlers all return None, so a None-return contract could not
|
|
7516
8760
|
tell "handled" from "unmatched".)
|
|
8761
|
+
|
|
8762
|
+
#714: every matched route runs inside a pricing-snapshot context, so a
|
|
8763
|
+
response is never assembled from two pricing revisions when an
|
|
8764
|
+
installer replaces `_lib_pricing.py` mid-request. The two long-lived
|
|
8765
|
+
streaming routes are excluded — a context entered there would pin one
|
|
8766
|
+
revision for the whole life of the stream, which is minutes to hours
|
|
8767
|
+
and the opposite of what the reload exists to achieve.
|
|
7517
8768
|
"""
|
|
7518
8769
|
path = self.path.split("?", 1)[0]
|
|
7519
8770
|
for kind, pattern, name, perf, wants_path in table:
|
|
@@ -7527,14 +8778,19 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
|
|
|
7527
8778
|
continue
|
|
7528
8779
|
fn = getattr(self, name)
|
|
7529
8780
|
args = (path,) if wants_path else ()
|
|
7530
|
-
if
|
|
7531
|
-
|
|
7532
|
-
|
|
7533
|
-
|
|
7534
|
-
|
|
7535
|
-
|
|
7536
|
-
with self._perf_gate().phase(perf[1]):
|
|
8781
|
+
if name in _STREAMING_ROUTE_HANDLERS:
|
|
8782
|
+
pinned = contextlib.nullcontext()
|
|
8783
|
+
else:
|
|
8784
|
+
pinned = _lib_pricing.pricing_snapshot_context()
|
|
8785
|
+
with pinned:
|
|
8786
|
+
if perf is None:
|
|
7537
8787
|
fn(*args)
|
|
8788
|
+
elif perf[0] == "scope":
|
|
8789
|
+
with self._perf_scope(perf[1]):
|
|
8790
|
+
fn(*args)
|
|
8791
|
+
else: # "phase"
|
|
8792
|
+
with self._perf_gate().phase(perf[1]):
|
|
8793
|
+
fn(*args)
|
|
7538
8794
|
return True
|
|
7539
8795
|
return False
|
|
7540
8796
|
|
|
@@ -8049,6 +9305,14 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
|
|
|
8049
9305
|
"conversation_sync": [
|
|
8050
9306
|
r.as_wire() for r in tick_state.conversation_records
|
|
8051
9307
|
],
|
|
9308
|
+
# #780: the maintenance phases, published beside the sync ring
|
|
9309
|
+
# rather than inside it. A reclaim that ran for its whole budget
|
|
9310
|
+
# used to be indistinguishable from a fast one, because both
|
|
9311
|
+
# were one `conversation_sync` record reporting `ok`. Empty is
|
|
9312
|
+
# a reachable steady state, so the key is always present.
|
|
9313
|
+
"maintenance": [
|
|
9314
|
+
r.as_wire() for r in tick_state.maintenance_records
|
|
9315
|
+
],
|
|
8052
9316
|
},
|
|
8053
9317
|
"tracing": {
|
|
8054
9318
|
"requested": requested,
|
|
@@ -9078,7 +10342,31 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
|
|
|
9078
10342
|
primary_model="claude-sonnet-4-6",
|
|
9079
10343
|
)
|
|
9080
10344
|
status = _dispatch_alert_notification(payload, mode="test")
|
|
9081
|
-
|
|
10345
|
+
# Dispatch and alerts.log keep the builder's internal snake_case key.
|
|
10346
|
+
# The response is an AlertEntry-shaped toast preview, so its public
|
|
10347
|
+
# account fields must follow the same R8 gate as real envelope rows.
|
|
10348
|
+
# Work on a copy: removing the key from `payload` would also change
|
|
10349
|
+
# what the native notifier and test log receive.
|
|
10350
|
+
response_alert = {k: v for k, v in payload.items() if k != "account_key"}
|
|
10351
|
+
provider = (
|
|
10352
|
+
"codex" if axis == "codex_budget" or (
|
|
10353
|
+
axis == "projected" and metric == "codex_budget_usd"
|
|
10354
|
+
) else "claude"
|
|
10355
|
+
)
|
|
10356
|
+
if _cctally_core.DB_PATH.exists():
|
|
10357
|
+
try:
|
|
10358
|
+
conn = _stats_ro_guarded()
|
|
10359
|
+
try:
|
|
10360
|
+
response_alert.update(_alert_account_resolver(conn)(
|
|
10361
|
+
provider, payload.get("account_key"),
|
|
10362
|
+
))
|
|
10363
|
+
finally:
|
|
10364
|
+
conn.close()
|
|
10365
|
+
except (OSError, sqlite3.Error):
|
|
10366
|
+
# A rehearsal still works without a readable account registry;
|
|
10367
|
+
# no writable DB open or synthetic milestone is needed.
|
|
10368
|
+
pass
|
|
10369
|
+
self._respond_json(200, {"alert": response_alert, "dispatch": status})
|
|
9082
10370
|
|
|
9083
10371
|
|
|
9084
10372
|
# ---- share endpoints (spec §5.1) — thin delegators to the F1 sibling ----
|
|
@@ -9173,15 +10461,6 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
|
|
|
9173
10461
|
return False
|
|
9174
10462
|
|
|
9175
10463
|
def _serve_static_file(self, path: pathlib.Path, ctype: str) -> None:
|
|
9176
|
-
try:
|
|
9177
|
-
body = path.read_bytes()
|
|
9178
|
-
except FileNotFoundError:
|
|
9179
|
-
self.send_error(404, "not found")
|
|
9180
|
-
return
|
|
9181
|
-
except IsADirectoryError:
|
|
9182
|
-
self.send_error(404, "not found")
|
|
9183
|
-
return
|
|
9184
|
-
|
|
9185
10464
|
compressible = (
|
|
9186
10465
|
ctype.startswith("text/")
|
|
9187
10466
|
or ctype.startswith("application/javascript")
|
|
@@ -9190,15 +10469,54 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
|
|
|
9190
10469
|
gzip_on = compressible and _accepts_gzip(
|
|
9191
10470
|
self.headers.get("Accept-Encoding")
|
|
9192
10471
|
)
|
|
9193
|
-
encoded = gzip.compress(body, compresslevel=6, mtime=0) if gzip_on else body
|
|
9194
|
-
etag = f'"{hashlib.sha256(encoded).hexdigest()}"'
|
|
9195
10472
|
cache_control = (
|
|
9196
10473
|
"public, max-age=31536000, immutable"
|
|
9197
10474
|
if self._is_hashed_static_asset(path)
|
|
9198
10475
|
else "no-cache"
|
|
9199
10476
|
)
|
|
10477
|
+
validator = self.headers.get("If-None-Match")
|
|
10478
|
+
key = (path, gzip_on)
|
|
10479
|
+
try:
|
|
10480
|
+
before = _STATIC_VALIDATORS.fingerprint(path.stat())
|
|
10481
|
+
except (FileNotFoundError, IsADirectoryError):
|
|
10482
|
+
self.send_error(404, "not found")
|
|
10483
|
+
return
|
|
10484
|
+
|
|
10485
|
+
# Only a validator derived from bytes previously served for this exact
|
|
10486
|
+
# encoding and unchanged file identity can answer before I/O. In
|
|
10487
|
+
# particular, a mutable dashboard.html replacement invalidates this.
|
|
10488
|
+
if validator:
|
|
10489
|
+
warm_etag = _STATIC_VALIDATORS.get(key, before)
|
|
10490
|
+
if warm_etag and self._etag_matches(validator, warm_etag):
|
|
10491
|
+
self.send_response(304)
|
|
10492
|
+
self.send_header("ETag", warm_etag)
|
|
10493
|
+
self.send_header("Cache-Control", cache_control)
|
|
10494
|
+
if compressible:
|
|
10495
|
+
self.send_header("Vary", "Accept-Encoding")
|
|
10496
|
+
if gzip_on:
|
|
10497
|
+
self.send_header("Content-Encoding", "gzip")
|
|
10498
|
+
self.end_headers()
|
|
10499
|
+
return
|
|
9200
10500
|
|
|
9201
|
-
|
|
10501
|
+
try:
|
|
10502
|
+
body = path.read_bytes()
|
|
10503
|
+
except FileNotFoundError:
|
|
10504
|
+
self.send_error(404, "not found")
|
|
10505
|
+
return
|
|
10506
|
+
except IsADirectoryError:
|
|
10507
|
+
self.send_error(404, "not found")
|
|
10508
|
+
return
|
|
10509
|
+
|
|
10510
|
+
encoded = gzip.compress(body, compresslevel=6, mtime=0) if gzip_on else body
|
|
10511
|
+
etag = f'"{hashlib.sha256(encoded).hexdigest()}"'
|
|
10512
|
+
try:
|
|
10513
|
+
after = _STATIC_VALIDATORS.fingerprint(path.stat())
|
|
10514
|
+
except (FileNotFoundError, IsADirectoryError):
|
|
10515
|
+
after = None
|
|
10516
|
+
if after == before:
|
|
10517
|
+
_STATIC_VALIDATORS.put(key, before, etag)
|
|
10518
|
+
|
|
10519
|
+
if after == before and self._etag_matches(validator, etag):
|
|
9202
10520
|
self.send_response(304)
|
|
9203
10521
|
self.send_header("ETag", etag)
|
|
9204
10522
|
self.send_header("Cache-Control", cache_control)
|
|
@@ -9563,12 +10881,7 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
|
|
|
9563
10881
|
scope, transcripts_visible, reveal,
|
|
9564
10882
|
)
|
|
9565
10883
|
|
|
9566
|
-
def
|
|
9567
|
-
report = sources.build_diagnosis(
|
|
9568
|
-
scope,
|
|
9569
|
-
measured_at=now_utc,
|
|
9570
|
-
transcripts_visible=transcripts_visible,
|
|
9571
|
-
)
|
|
10884
|
+
def _project_diagnosis(report):
|
|
9572
10885
|
scopes = {
|
|
9573
10886
|
result.source: diagnosis._scope_for(
|
|
9574
10887
|
sources, scope, result.source,
|
|
@@ -9587,10 +10900,43 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
|
|
|
9587
10900
|
)
|
|
9588
10901
|
return status, body
|
|
9589
10902
|
|
|
10903
|
+
def _prepare_diagnosis():
|
|
10904
|
+
return _project_diagnosis(sources.build_diagnosis(
|
|
10905
|
+
scope,
|
|
10906
|
+
measured_at=now_utc,
|
|
10907
|
+
transcripts_visible=transcripts_visible,
|
|
10908
|
+
))
|
|
10909
|
+
|
|
10910
|
+
def _prepare_bounded_diagnosis(remaining):
|
|
10911
|
+
return _project_diagnosis(
|
|
10912
|
+
sources.build_diagnosis_bounded(
|
|
10913
|
+
scope,
|
|
10914
|
+
measured_at=now_utc,
|
|
10915
|
+
transcripts_visible=transcripts_visible,
|
|
10916
|
+
timeout_seconds=remaining,
|
|
10917
|
+
)
|
|
10918
|
+
)
|
|
10919
|
+
|
|
9590
10920
|
status, body = _DIAGNOSIS_ADMISSION.run(
|
|
9591
10921
|
flight_key,
|
|
9592
10922
|
_prepare_diagnosis,
|
|
10923
|
+
peer_gone=lambda: _diagnosis_peer_gone(self.connection),
|
|
10924
|
+
bounded_build=_prepare_bounded_diagnosis,
|
|
9593
10925
|
)
|
|
10926
|
+
except _DiagnosisClientGone:
|
|
10927
|
+
return
|
|
10928
|
+
except _DiagnosisOverloaded:
|
|
10929
|
+
self._send_diagnosis_json(503, {
|
|
10930
|
+
"error": "diagnosis busy; retry shortly",
|
|
10931
|
+
"code": "diagnosis_overloaded",
|
|
10932
|
+
})
|
|
10933
|
+
return
|
|
10934
|
+
except _DiagnosisTimedOut:
|
|
10935
|
+
self._send_diagnosis_json(503, {
|
|
10936
|
+
"error": "diagnosis timed out; retry shortly",
|
|
10937
|
+
"code": "diagnosis_timeout",
|
|
10938
|
+
})
|
|
10939
|
+
return
|
|
9594
10940
|
except _DiagnosisSelectorError as exc:
|
|
9595
10941
|
self._send_diagnosis_json(
|
|
9596
10942
|
400, {"error": exc.message, "code": exc.code})
|
|
@@ -9986,8 +11332,22 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
|
|
|
9986
11332
|
load_config(), type(self).display_tz_pref_override
|
|
9987
11333
|
)
|
|
9988
11334
|
)
|
|
11335
|
+
# A journal-stamped closed block is frozen, so its detail
|
|
11336
|
+
# is read rather than recomputed from cache contents that
|
|
11337
|
+
# may have grown since the close (#399, #751a).
|
|
11338
|
+
frozen_facts = None
|
|
11339
|
+
try:
|
|
11340
|
+
_stats = open_db()
|
|
11341
|
+
except Exception:
|
|
11342
|
+
_stats = None
|
|
11343
|
+
if _stats is not None:
|
|
11344
|
+
try:
|
|
11345
|
+
frozen_facts = _retained_block_facts(_stats, target)
|
|
11346
|
+
finally:
|
|
11347
|
+
_stats.close()
|
|
9989
11348
|
detail = _build_block_detail(
|
|
9990
11349
|
target, block_entries, display_tz=_detail_tz,
|
|
11350
|
+
frozen_facts=frozen_facts,
|
|
9991
11351
|
)
|
|
9992
11352
|
status = 200
|
|
9993
11353
|
body = encode_dashboard_json_bytes(detail, ensure_ascii=False)
|
|
@@ -11062,10 +12422,34 @@ def cmd_dashboard(args: argparse.Namespace) -> int:
|
|
|
11062
12422
|
# the initial render already excludes stale sessions from a deleted
|
|
11063
12423
|
# worktree (rather than showing them until the first periodic tick).
|
|
11064
12424
|
# Gated off under --no-sync (a frozen dashboard mutates nothing).
|
|
12425
|
+
# #780: arm the reader's writer wake-up before any route can serve, then
|
|
12426
|
+
# advance the schema once. Both run in EVERY mode, including --no-sync.
|
|
12427
|
+
_cctally_cache_module = _cctally()._load_sibling("_cctally_cache")
|
|
12428
|
+
_cctally_cache_module.SCHEMA_WAKE_HOOK = _wake_schema_writer
|
|
12429
|
+
# #769 S6 / #802: set the PROCESS-level derivation policy before the first
|
|
12430
|
+
# open, so the read-route fallback and the live-tail opener inherit it.
|
|
12431
|
+
# A startup-only flag would freeze nothing, because both of those reach the
|
|
12432
|
+
# same full opener from a request thread and would consume the marker.
|
|
12433
|
+
_cctally_cache_module.set_conversation_derivations_suppressed(
|
|
12434
|
+
bool(args.no_sync))
|
|
12435
|
+
_dashboard_startup_schema_migration(run_derivations=not bool(args.no_sync))
|
|
12436
|
+
# #714: the pricing-file watch, in EVERY mode for the same reason. Both
|
|
12437
|
+
# `_DashboardSyncThread` and `_make_conversation_sync_thread` are disabled
|
|
12438
|
+
# under `--no-sync`, so a detector placed in either would have no owner in
|
|
12439
|
+
# the one mode a long-lived read-only dashboard runs in.
|
|
12440
|
+
_start_pricing_watch()
|
|
11065
12441
|
_heal = _dashboard_self_heal_orphans(skip_sync=bool(args.no_sync))
|
|
11066
12442
|
if _heal is not None and _heal.pruned_files:
|
|
11067
12443
|
print(f"dashboard: pruned {_heal.pruned_files} orphaned cache file(s) "
|
|
11068
12444
|
f"from removed sessions on startup", flush=True)
|
|
12445
|
+
if _heal is not None and _heal.prune_refused:
|
|
12446
|
+
# #729: on stdout beside the pruned-files line, because the operator
|
|
12447
|
+
# reading the startup banner is the one who can act on it. The heal
|
|
12448
|
+
# helper has already armed the backfill and dropped the certificate.
|
|
12449
|
+
print(f"dashboard: the conversation rollup re-derive was refused for "
|
|
12450
|
+
f"{_heal.prune_refused_files} pruned file(s) — "
|
|
12451
|
+
f"{pricing_refusal_cause_phrase(_heal.prune_refused_state)}",
|
|
12452
|
+
flush=True)
|
|
11069
12453
|
|
|
11070
12454
|
# #709: background retained-size verifiers own fail-closed completion.
|
|
11071
12455
|
# Bind both owners to the same publisher lock before the initial build so
|