cctally 1.60.0 → 1.62.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/bin/_cctally_cache.py +479 -54
- package/bin/_cctally_core.py +8 -0
- package/bin/_cctally_dashboard.py +1467 -234
- package/bin/_cctally_db.py +27 -1
- package/bin/_cctally_doctor.py +2 -0
- package/bin/_cctally_forecast.py +24 -2
- package/bin/_cctally_parser.py +28 -2
- package/bin/_cctally_tui.py +708 -16
- package/bin/_cctally_update.py +49 -9
- package/bin/_cctally_weekrefs.py +101 -2
- package/bin/_lib_aggregators.py +98 -71
- package/bin/_lib_cache_report.py +455 -67
- package/bin/_lib_doctor.py +10 -0
- package/bin/_lib_pricing.py +31 -4
- package/bin/_lib_snapshot_cache.py +1797 -0
- package/bin/_lib_view_models.py +107 -12
- package/bin/cctally +57 -0
- package/dashboard/static/assets/index-0jzYm75p.css +1 -0
- package/dashboard/static/assets/{index-DQH6UPc_.js → index-D_Ylyqsf.js} +1 -1
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +2 -1
- package/dashboard/static/assets/index-uGwZSssU.css +0 -1
|
@@ -247,7 +247,7 @@ import urllib.request
|
|
|
247
247
|
import webbrowser as _wb
|
|
248
248
|
from dataclasses import dataclass, field, replace
|
|
249
249
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
250
|
-
from typing import Any
|
|
250
|
+
from typing import Any, NamedTuple
|
|
251
251
|
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
|
|
252
252
|
|
|
253
253
|
|
|
@@ -316,7 +316,35 @@ from _lib_subscription_weeks import _compute_subscription_weeks
|
|
|
316
316
|
from _lib_blocks import _group_entries_into_blocks
|
|
317
317
|
from _cctally_config import save_config, _load_config_unlocked
|
|
318
318
|
from _cctally_db import _render_migration_error_banner
|
|
319
|
-
from _cctally_cache import
|
|
319
|
+
from _cctally_cache import (
|
|
320
|
+
get_entries, iter_entries, iter_entries_with_id, open_cache_db, sync_cache,
|
|
321
|
+
_prune_orphaned_cache_entries,
|
|
322
|
+
)
|
|
323
|
+
from _lib_snapshot_cache import (
|
|
324
|
+
build_cached_group_a,
|
|
325
|
+
bump_generation,
|
|
326
|
+
cached_bugk_segment,
|
|
327
|
+
reset_bugk_segment_state,
|
|
328
|
+
reset_cache_report_state,
|
|
329
|
+
reset_group_a_state,
|
|
330
|
+
reset_projects_env_state,
|
|
331
|
+
reset_session_cache_state,
|
|
332
|
+
reset_weekref_cost_state,
|
|
333
|
+
BugKSegment,
|
|
334
|
+
_bugk_key,
|
|
335
|
+
_max_id as _snapshot_max_id,
|
|
336
|
+
_reset_sig as _snapshot_reset_sig,
|
|
337
|
+
)
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
# #268 Group A cached-bucket path master switch. Normally True: the
|
|
341
|
+
# calendar builders (daily / monthly / weekly) assemble their per-bucket
|
|
342
|
+
# aggregates through the module-level BucketCache and recompute only the
|
|
343
|
+
# current/dirty slice. Flip to False to force the from-scratch wide-fetch
|
|
344
|
+
# path (the pre-#268 behavior) — the parity tests toggle it to prove the
|
|
345
|
+
# cached and from-scratch rebuilds are byte-identical, and the builders
|
|
346
|
+
# fall back to it automatically if the cache DB can't be opened.
|
|
347
|
+
_GROUP_A_CACHE_ENABLED = True
|
|
320
348
|
|
|
321
349
|
|
|
322
350
|
# === Module-level back-ref shims for helpers that STAY in bin/cctally ======
|
|
@@ -351,6 +379,84 @@ def _make_run_sync_now_locked(*args, **kwargs):
|
|
|
351
379
|
return sys.modules["cctally"]._make_run_sync_now_locked(*args, **kwargs)
|
|
352
380
|
|
|
353
381
|
|
|
382
|
+
def _register_faulthandler_sigusr1():
|
|
383
|
+
"""Register a SIGUSR1 handler that dumps all-thread tracebacks (#268 M6.1).
|
|
384
|
+
|
|
385
|
+
So any future dashboard spin (a sync thread stuck in a rebuild) is
|
|
386
|
+
self-diagnosing — ``kill -USR1 <pid>`` prints every thread's stack to stderr
|
|
387
|
+
— without needing a root py-spy. Guarded for platforms lacking SIGUSR1
|
|
388
|
+
(Windows), where it is a silent no-op. Idempotent; returns True when the
|
|
389
|
+
handler was registered, False when SIGUSR1 is unavailable."""
|
|
390
|
+
import faulthandler
|
|
391
|
+
import signal
|
|
392
|
+
if not hasattr(signal, "SIGUSR1"):
|
|
393
|
+
return False
|
|
394
|
+
faulthandler.register(signal.SIGUSR1, all_threads=True)
|
|
395
|
+
return True
|
|
396
|
+
|
|
397
|
+
|
|
398
|
+
def _dashboard_self_heal_orphans(*, skip_sync):
|
|
399
|
+
"""Prune removed-worktree orphans from the cache using a throwaway
|
|
400
|
+
connection. No-op under frozen (--no-sync) mode. Non-blocking on the
|
|
401
|
+
cache lock (retries on the next cadence if contended). Never raises.
|
|
402
|
+
|
|
403
|
+
#268 M5.2 (spec §7 / Codex F4): a prune that actually deleted rows rewrites
|
|
404
|
+
history in place — which `MAX(id)` alone can't detect (deleting a NON-max
|
|
405
|
+
row leaves the max unchanged). So on any real deletion, bump the
|
|
406
|
+
cache-generation counter (a composite-signature leg, so the next rebuild
|
|
407
|
+
can't idle-short-circuit) AND clear the Group A / session caches, forcing a
|
|
408
|
+
correct cold recompute on the next tick. The prune runs AFTER the tick's
|
|
409
|
+
publish on the same sync thread, so the very next tick recomputes."""
|
|
410
|
+
if skip_sync:
|
|
411
|
+
return None
|
|
412
|
+
try:
|
|
413
|
+
conn = open_cache_db()
|
|
414
|
+
try:
|
|
415
|
+
result = _prune_orphaned_cache_entries(conn, lock_timeout=None)
|
|
416
|
+
finally:
|
|
417
|
+
conn.close()
|
|
418
|
+
except Exception:
|
|
419
|
+
return None
|
|
420
|
+
if result is not None and (result.pruned_entries or result.pruned_files):
|
|
421
|
+
try:
|
|
422
|
+
bump_generation()
|
|
423
|
+
reset_group_a_state()
|
|
424
|
+
reset_session_cache_state()
|
|
425
|
+
# #269 M3.2 (spec §6): the shared per-weekref immutable-cost cache
|
|
426
|
+
# (B1 trend/weekly-history + B3 forecast) rides the SAME prune-site
|
|
427
|
+
# clear — a non-max deletion the reconcile's max-id regression check
|
|
428
|
+
# can't catch.
|
|
429
|
+
reset_weekref_cost_state()
|
|
430
|
+
# #272: the per-day cache-report cache rides the same prune-site
|
|
431
|
+
# clear — a non-max deletion inside a closed day the reconcile's
|
|
432
|
+
# max-id / mutation-seq regression check can't catch.
|
|
433
|
+
reset_cache_report_state()
|
|
434
|
+
# #269 M4.5 (spec §14 Win 2): the projects-envelope per-(project,
|
|
435
|
+
# week) cache rides the same prune-site clear for the same reason.
|
|
436
|
+
reset_projects_env_state()
|
|
437
|
+
# #271 M3 (spec §18): the Bug-K pre-credit segment cache rides the
|
|
438
|
+
# same prune-site clear — a non-max deletion inside a closed
|
|
439
|
+
# pre-credit window the reconcile's max-id regression check can't
|
|
440
|
+
# catch.
|
|
441
|
+
reset_bugk_segment_state()
|
|
442
|
+
# #269 (final reviewer): the per-(project, week) cache clear above is
|
|
443
|
+
# NOT sufficient on its own. `_build_projects_envelope` consults the
|
|
444
|
+
# whole-envelope memo `_PROJECTS_ENV_MEMO` FIRST — and its memo_key is
|
|
445
|
+
# `(max_id, max_wus_id, entry_mutation_seq, cw_key, weeks_back)`, which
|
|
446
|
+
# carries NO generation counter. A prune that deletes only NON-max
|
|
447
|
+
# `session_entries` rows leaves `max_id` unchanged, so the memo_key
|
|
448
|
+
# still matches and the memo would stale-serve the pre-prune envelope
|
|
449
|
+
# (still showing the deleted project + its cost) before the fresh
|
|
450
|
+
# per-week cache path is ever reached. Clear the memo too so a real
|
|
451
|
+
# prune actually changes the envelope output.
|
|
452
|
+
_projects_reset_memo()
|
|
453
|
+
except Exception:
|
|
454
|
+
# Invalidation must never turn a successful prune into a failure;
|
|
455
|
+
# a stale-cache tick is self-corrected once the signature next moves.
|
|
456
|
+
pass
|
|
457
|
+
return result
|
|
458
|
+
|
|
459
|
+
|
|
354
460
|
def _build_forecast_json_payload(*args, **kwargs):
|
|
355
461
|
return sys.modules["cctally"]._build_forecast_json_payload(*args, **kwargs)
|
|
356
462
|
|
|
@@ -1978,6 +2084,77 @@ def _cache_report_load_kernel():
|
|
|
1978
2084
|
return sys.modules["cctally"]._load_sibling("_lib_cache_report")
|
|
1979
2085
|
|
|
1980
2086
|
|
|
2087
|
+
def _cache_report_needed_closed_dates(since, now_utc, bucket_tz):
|
|
2088
|
+
"""The CLOSED display-tz dates a full-window cache-report build needs (#272 §6).
|
|
2089
|
+
|
|
2090
|
+
Every display-tz calendar date overlapping ``[since, now_utc]`` EXCEPT the
|
|
2091
|
+
current open day (Codex-2: a ``since``-straddling window yields up to
|
|
2092
|
+
``window_days + 1`` display dates, and classification runs over the full
|
|
2093
|
+
set BEFORE the ``days`` cap). Returned sorted ascending. The ``have_all``
|
|
2094
|
+
gate requires every one of these to be cached before serving the warm
|
|
2095
|
+
(today-only-fetch) path. A genuinely activity-free calendar day produces
|
|
2096
|
+
no per-day fold row, so the cold/miss populate branch stores an
|
|
2097
|
+
``is_empty`` sentinel (``empty_cached_day``) for every needed closed date
|
|
2098
|
+
the fold didn't emit — registering the quiet day as a cache HIT (the same
|
|
2099
|
+
computed-empty registration ``projects_env_week_put`` does for an empty
|
|
2100
|
+
week). That keeps ``have_all`` reachable, so a weekend/vacation gap day
|
|
2101
|
+
does NOT permanently defeat the warm path; the sentinel is skipped in the
|
|
2102
|
+
restitch, so the output stays byte-identical to from-scratch.
|
|
2103
|
+
"""
|
|
2104
|
+
today_key = now_utc.astimezone(bucket_tz).strftime("%Y-%m-%d")
|
|
2105
|
+
d = since.astimezone(bucket_tz).date()
|
|
2106
|
+
end_date = now_utc.astimezone(bucket_tz).date()
|
|
2107
|
+
out: list = []
|
|
2108
|
+
while d <= end_date:
|
|
2109
|
+
key = d.strftime("%Y-%m-%d")
|
|
2110
|
+
if key != today_key:
|
|
2111
|
+
out.append(key)
|
|
2112
|
+
d += dt.timedelta(days=1)
|
|
2113
|
+
return out
|
|
2114
|
+
|
|
2115
|
+
|
|
2116
|
+
def _day_start(date_iso, bucket_tz):
|
|
2117
|
+
"""The aware-UTC instant of ``date_iso``'s local midnight in ``bucket_tz`` (#272 §6).
|
|
2118
|
+
|
|
2119
|
+
Lower-bounds the warm-path current-day fetch so it returns exactly the
|
|
2120
|
+
entries ``_aggregate_cache_by_day`` buckets to ``date_iso`` (which buckets
|
|
2121
|
+
via ``entry.timestamp.astimezone(bucket_tz)``). DST-correct: attaching the
|
|
2122
|
+
zone to the naive local-midnight then converting to UTC lets ``ZoneInfo``
|
|
2123
|
+
pick the right offset for that wall time.
|
|
2124
|
+
"""
|
|
2125
|
+
naive = dt.datetime.strptime(date_iso, "%Y-%m-%d")
|
|
2126
|
+
return naive.replace(tzinfo=bucket_tz).astimezone(dt.timezone.utc)
|
|
2127
|
+
|
|
2128
|
+
|
|
2129
|
+
def _cache_report_empty(
|
|
2130
|
+
*, today_iso, window_days, anomaly_threshold_pp, anomaly_window_days,
|
|
2131
|
+
):
|
|
2132
|
+
"""The empty (no in-window entries) ``CacheReportSnapshot`` — factored so the
|
|
2133
|
+
warm and cold builder paths share one ``is_empty`` return (#272 §6)."""
|
|
2134
|
+
empty_today = CacheReportTodaySpotlight(
|
|
2135
|
+
date=today_iso,
|
|
2136
|
+
cache_hit_percent=0.0,
|
|
2137
|
+
baseline_median_percent=None,
|
|
2138
|
+
delta_pp=None,
|
|
2139
|
+
net_usd=0.0, saved_usd=0.0, wasted_usd=0.0,
|
|
2140
|
+
anomaly_triggered=False,
|
|
2141
|
+
anomaly_reasons=(),
|
|
2142
|
+
baseline_daily_row_count=0,
|
|
2143
|
+
)
|
|
2144
|
+
return CacheReportSnapshot(
|
|
2145
|
+
window_days=window_days,
|
|
2146
|
+
anomaly_threshold_pp=anomaly_threshold_pp,
|
|
2147
|
+
anomaly_window_days=anomaly_window_days,
|
|
2148
|
+
today=empty_today,
|
|
2149
|
+
days=(), by_project=(), by_model=(),
|
|
2150
|
+
seven_day_net_usd=0.0,
|
|
2151
|
+
seven_day_anomaly_count=0,
|
|
2152
|
+
fourteen_day_counterfactual_usd=0.0,
|
|
2153
|
+
fourteen_day_efficiency_ratio=0.0,
|
|
2154
|
+
is_empty=True,
|
|
2155
|
+
)
|
|
2156
|
+
|
|
2157
|
+
|
|
1981
2158
|
def build_cache_report_snapshot(
|
|
1982
2159
|
*,
|
|
1983
2160
|
now_utc: dt.datetime,
|
|
@@ -1985,91 +2162,167 @@ def build_cache_report_snapshot(
|
|
|
1985
2162
|
anomaly_window_days: int,
|
|
1986
2163
|
display_tz: "ZoneInfo | None",
|
|
1987
2164
|
skip_sync: bool = False,
|
|
2165
|
+
use_cache_report_cache: bool = False,
|
|
1988
2166
|
) -> CacheReportSnapshot:
|
|
1989
2167
|
"""Build the ``cache_report`` envelope field from the session-entry cache.
|
|
1990
2168
|
|
|
1991
2169
|
Pulls entries via ``get_claude_session_entries`` (uses the cache when
|
|
1992
2170
|
warm, falls back to direct-JSONL parse on cache miss / lock
|
|
1993
|
-
contention — same chain the CLI uses).
|
|
1994
|
-
|
|
1995
|
-
|
|
2171
|
+
contention — same chain the CLI uses). Aggregates per closed day via
|
|
2172
|
+
``_lib_cache_report.build_cached_days`` (served from the per-day cache
|
|
2173
|
+
when ``use_cache_report_cache`` and reconciled), reconstructs fresh
|
|
2174
|
+
rows (``reconstruct_cache_row``), runs the cross-row
|
|
2175
|
+
``classify_and_summarize`` + ``combine_day_project_partials`` restitch,
|
|
2176
|
+
and shapes the result into a frozen ``CacheReportSnapshot``.
|
|
1996
2177
|
|
|
1997
2178
|
``window_days`` is hardcoded at 14 in v1 (spec §6.1 hardcodes
|
|
1998
2179
|
``anomaly_window_days`` too; ``anomaly_threshold_pp`` is the only
|
|
1999
2180
|
user-configurable knob). F10 from spec §10 tracks making the window
|
|
2000
2181
|
configurable, plus the UI-copy work it'd require.
|
|
2182
|
+
|
|
2183
|
+
**Per-day cache (#272 §6).** With ``use_cache_report_cache=True`` (set by
|
|
2184
|
+
the dashboard sync thread AFTER ``reconcile_cache_report_cache`` succeeds),
|
|
2185
|
+
the CLOSED days are served from the immutable per-day cache and only the
|
|
2186
|
+
current (open) day is fetched + folded fresh — the warm-tick win. The
|
|
2187
|
+
reconcile (in ``_tui_build_snapshot``) owns invalidation; this builder only
|
|
2188
|
+
reads/populates the cache. With the flag OFF (default / safe fallback), the
|
|
2189
|
+
full window is fetched every tick and the output is byte-identical to the
|
|
2190
|
+
cached path (same restructured assembly, cache bypassed).
|
|
2001
2191
|
"""
|
|
2002
2192
|
crk = _cache_report_load_kernel()
|
|
2003
2193
|
cctally_ns = sys.modules["cctally"]
|
|
2194
|
+
_sc = sys.modules["_lib_snapshot_cache"]
|
|
2004
2195
|
|
|
2005
2196
|
window_days = CACHE_REPORT_WINDOW_DAYS # v1: hardcoded per spec §6.1.
|
|
2006
2197
|
since = now_utc - dt.timedelta(days=window_days)
|
|
2198
|
+
pricing = cctally_ns.CLAUDE_MODEL_PRICING
|
|
2007
2199
|
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
|
|
2200
|
+
# Cache mechanics key on the BUCKETING tz (``_resolve_bucket_tz`` — the same
|
|
2201
|
+
# tz ``_aggregate_cache_by_day`` buckets by), so ``today_key`` / the closed-
|
|
2202
|
+
# day keys line up with ``CacheRow.date``. The spotlight's ``today_iso``
|
|
2203
|
+
# keeps its own UTC-fallback derivation unchanged (byte-identity); the two
|
|
2204
|
+
# agree whenever ``display_tz`` is set — always, on the dashboard.
|
|
2205
|
+
bucket_tz = crk._resolve_bucket_tz(display_tz)
|
|
2206
|
+
today_key = now_utc.astimezone(bucket_tz).strftime("%Y-%m-%d")
|
|
2012
2207
|
today_iso = now_utc.astimezone(
|
|
2013
2208
|
display_tz if display_tz is not None else dt.timezone.utc
|
|
2014
2209
|
).strftime("%Y-%m-%d")
|
|
2015
2210
|
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2211
|
+
def _wrap_day_entries(raw):
|
|
2212
|
+
# Day-mode kernel expects entries with a ``usage`` dict (matches
|
|
2213
|
+
# ``UsageEntry``); ``get_claude_session_entries`` returns flat
|
|
2214
|
+
# ``_JoinedClaudeEntry`` objects. SimpleNamespace keeps the wrapper
|
|
2215
|
+
# pure-Python and avoids a new dataclass type just for the bridge.
|
|
2216
|
+
from types import SimpleNamespace as _NS
|
|
2217
|
+
return [
|
|
2218
|
+
_NS(
|
|
2219
|
+
timestamp=e.timestamp,
|
|
2220
|
+
model=e.model,
|
|
2221
|
+
cost_usd=e.cost_usd,
|
|
2222
|
+
usage={
|
|
2223
|
+
"input_tokens": e.input_tokens,
|
|
2224
|
+
"output_tokens": e.output_tokens,
|
|
2225
|
+
"cache_creation_input_tokens": e.cache_creation_tokens,
|
|
2226
|
+
"cache_read_input_tokens": e.cache_read_tokens,
|
|
2227
|
+
},
|
|
2228
|
+
)
|
|
2229
|
+
for e in raw
|
|
2230
|
+
]
|
|
2231
|
+
|
|
2232
|
+
# Which CLOSED display-tz dates the window needs (every date overlapping
|
|
2233
|
+
# [since, now] except the current open day). If the cache holds them ALL,
|
|
2234
|
+
# fetch only the current day (the warm win); else one full-window fetch
|
|
2235
|
+
# (re)populates the closed days + recomputes today.
|
|
2236
|
+
needed_closed = _cache_report_needed_closed_dates(since, now_utc, bucket_tz)
|
|
2237
|
+
have_all = use_cache_report_cache and all(
|
|
2238
|
+
_sc.cache_report_day_get(d) is not None for d in needed_closed
|
|
2239
|
+
)
|
|
2240
|
+
|
|
2241
|
+
cached_days: dict = {} # date_key -> CachedCacheReportDay
|
|
2242
|
+
if have_all:
|
|
2243
|
+
# Steady-state warm: recompute ONLY the open bucket; reuse cached closed.
|
|
2244
|
+
raw_today = list(get_claude_session_entries(
|
|
2245
|
+
_day_start(today_key, bucket_tz), now_utc,
|
|
2246
|
+
project=None, skip_sync=skip_sync,
|
|
2247
|
+
))
|
|
2248
|
+
built = crk.build_cached_days(
|
|
2249
|
+
_wrap_day_entries(raw_today), raw_today,
|
|
2250
|
+
display_tz=display_tz, pricing=pricing,
|
|
2251
|
+
cost_calculator=_calculate_entry_cost,
|
|
2252
|
+
)
|
|
2253
|
+
for d in needed_closed:
|
|
2254
|
+
cached_days[d] = _sc.cache_report_day_get(d)
|
|
2255
|
+
cached_days.update(built) # the fresh current day (today only)
|
|
2256
|
+
else:
|
|
2257
|
+
# Cold / partial-miss / post-eviction: pay one full-window fetch, then
|
|
2258
|
+
# (re)compute + store every closed day. Runs on the rare stale tick.
|
|
2259
|
+
raw = list(get_claude_session_entries(
|
|
2260
|
+
since, now_utc, project=None, skip_sync=skip_sync,
|
|
2261
|
+
))
|
|
2262
|
+
if not raw:
|
|
2263
|
+
return _cache_report_empty(
|
|
2264
|
+
today_iso=today_iso, window_days=window_days,
|
|
2265
|
+
anomaly_threshold_pp=anomaly_threshold_pp,
|
|
2266
|
+
anomaly_window_days=anomaly_window_days,
|
|
2267
|
+
)
|
|
2268
|
+
built = crk.build_cached_days(
|
|
2269
|
+
_wrap_day_entries(raw), raw,
|
|
2270
|
+
display_tz=display_tz, pricing=pricing,
|
|
2271
|
+
cost_calculator=_calculate_entry_cost,
|
|
2026
2272
|
)
|
|
2027
|
-
|
|
2028
|
-
|
|
2273
|
+
cached_days = dict(built)
|
|
2274
|
+
if use_cache_report_cache:
|
|
2275
|
+
for d, unit in built.items():
|
|
2276
|
+
if d != today_key: # never store the OPEN day as a closed unit
|
|
2277
|
+
_sc.cache_report_day_store(d, unit)
|
|
2278
|
+
# Empty/gap-day sentinel: a genuinely activity-free CLOSED day never
|
|
2279
|
+
# appears in ``built`` (``build_cached_days`` only emits days with
|
|
2280
|
+
# entries), so register an ``is_empty`` sentinel for every needed
|
|
2281
|
+
# closed date the fold produced nothing for. Without this,
|
|
2282
|
+
# ``have_all`` would see that quiet day as a perpetual miss and the
|
|
2283
|
+
# builder would full-window-refetch on EVERY warm tick — permanently
|
|
2284
|
+
# defeating the cache for any user with an off day. Mirrors
|
|
2285
|
+
# ``projects_env_week_put`` registering a computed-empty week as a
|
|
2286
|
+
# hit. ``needed_closed`` already excludes ``today_key``; the sentinel
|
|
2287
|
+
# is skipped in the restitch below, so this is byte-identical.
|
|
2288
|
+
for d in needed_closed:
|
|
2289
|
+
if d not in built:
|
|
2290
|
+
_sc.cache_report_day_store(d, crk.empty_cached_day(d))
|
|
2291
|
+
# Window-rolloff eviction (#275): the reconcile's seq-gated pass only
|
|
2292
|
+
# evicts CHANGED days (>= the change watermark); days that roll off the
|
|
2293
|
+
# trailing edge of the [since, now] window are never pruned there and
|
|
2294
|
+
# would accrete one frozen unit/day on a long-uptime dashboard. Drop
|
|
2295
|
+
# them here on this rare cold/rollover store tick (the warm path never
|
|
2296
|
+
# reaches this branch). ``needed_closed`` is sorted ascending, so its
|
|
2297
|
+
# first element is the window's oldest still-needed closed day; every
|
|
2298
|
+
# needed day (real or ``is_empty`` sentinel) is >= it and survives.
|
|
2299
|
+
if needed_closed:
|
|
2300
|
+
_sc.cache_report_day_evict_before(needed_closed[0])
|
|
2301
|
+
|
|
2302
|
+
if not cached_days:
|
|
2303
|
+
return _cache_report_empty(
|
|
2304
|
+
today_iso=today_iso, window_days=window_days,
|
|
2029
2305
|
anomaly_threshold_pp=anomaly_threshold_pp,
|
|
2030
2306
|
anomaly_window_days=anomaly_window_days,
|
|
2031
|
-
today=empty_today,
|
|
2032
|
-
days=(), by_project=(), by_model=(),
|
|
2033
|
-
seven_day_net_usd=0.0,
|
|
2034
|
-
seven_day_anomaly_count=0,
|
|
2035
|
-
fourteen_day_counterfactual_usd=0.0,
|
|
2036
|
-
fourteen_day_efficiency_ratio=0.0,
|
|
2037
|
-
is_empty=True,
|
|
2038
2307
|
)
|
|
2039
2308
|
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
#
|
|
2043
|
-
#
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
day_entries = [
|
|
2049
|
-
_NS(
|
|
2050
|
-
timestamp=e.timestamp,
|
|
2051
|
-
model=e.model,
|
|
2052
|
-
cost_usd=e.cost_usd,
|
|
2053
|
-
usage={
|
|
2054
|
-
"input_tokens": e.input_tokens,
|
|
2055
|
-
"output_tokens": e.output_tokens,
|
|
2056
|
-
"cache_creation_input_tokens": e.cache_creation_tokens,
|
|
2057
|
-
"cache_read_input_tokens": e.cache_read_tokens,
|
|
2058
|
-
},
|
|
2059
|
-
)
|
|
2060
|
-
for e in entries
|
|
2309
|
+
# Reconstruct fresh MUTABLE rows from the frozen cached days (F7 — cached
|
|
2310
|
+
# units are never touched), then run the cross-row classify + summarize
|
|
2311
|
+
# pass over the FULL assembled set every tick (the cached day aggregates
|
|
2312
|
+
# carry no anomaly state).
|
|
2313
|
+
rows = [
|
|
2314
|
+
crk.reconstruct_cache_row(cached_days[d])
|
|
2315
|
+
for d in sorted(cached_days)
|
|
2316
|
+
if not cached_days[d].is_empty # sentinel gap days emit no CacheRow
|
|
2061
2317
|
]
|
|
2062
|
-
|
|
2063
|
-
|
|
2064
|
-
day_entries,
|
|
2318
|
+
result = crk.classify_and_summarize(
|
|
2319
|
+
rows,
|
|
2065
2320
|
now_utc=now_utc,
|
|
2066
2321
|
window_days=window_days,
|
|
2067
2322
|
anomaly_threshold_pp=anomaly_threshold_pp,
|
|
2068
2323
|
anomaly_window_days=anomaly_window_days,
|
|
2069
2324
|
display_tz=display_tz,
|
|
2070
|
-
pricing=pricing,
|
|
2071
2325
|
mode="day",
|
|
2072
|
-
cost_calculator=_calculate_entry_cost,
|
|
2073
2326
|
)
|
|
2074
2327
|
|
|
2075
2328
|
# Pick out today's row (if any) and the baseline-daily-row count for
|
|
@@ -2206,17 +2459,22 @@ def build_cache_report_snapshot(
|
|
|
2206
2459
|
# cache entry and its corresponding day row always agree on which
|
|
2207
2460
|
# bucket they belong to.
|
|
2208
2461
|
kept_dates = frozenset(r.date for r in days if r.date)
|
|
2209
|
-
bucket_tz = display_tz if display_tz is not None else dt.timezone.utc
|
|
2210
|
-
entries_in_window = [
|
|
2211
|
-
e for e in entries
|
|
2212
|
-
if e.timestamp.astimezone(bucket_tz).strftime("%Y-%m-%d") in kept_dates
|
|
2213
|
-
]
|
|
2214
2462
|
rows_in_window = [r for r in result.rows if r.date in kept_dates]
|
|
2215
|
-
|
|
2216
|
-
|
|
2217
|
-
|
|
2218
|
-
|
|
2219
|
-
|
|
2463
|
+
# by_project via the #272 §4 two-level ``stable_sum`` fold
|
|
2464
|
+
# (grouping-invariant). The per-(day, project) partials are carried on each
|
|
2465
|
+
# ``CachedCacheReportDay`` (§5) — served from cache for closed days and
|
|
2466
|
+
# freshly folded for today — so restitching them here memoizes the fold
|
|
2467
|
+
# byte-identically. Their day keys use ``_resolve_bucket_tz(display_tz)``
|
|
2468
|
+
# (matching ``CacheRow.date`` / ``kept_dates``), so combining over
|
|
2469
|
+
# ``kept_dates`` is exactly the old full-window slice, modulo the one-time
|
|
2470
|
+
# ULP fold move. Every ``kept_date`` is in ``cached_days`` (it came from a
|
|
2471
|
+
# day row); the ``if d in cached_days`` guard is defensive.
|
|
2472
|
+
by_project_rows = crk.combine_day_project_partials(
|
|
2473
|
+
{
|
|
2474
|
+
d: dict(cached_days[d].project_partials)
|
|
2475
|
+
for d in kept_dates
|
|
2476
|
+
if d in cached_days and not cached_days[d].is_empty
|
|
2477
|
+
}
|
|
2220
2478
|
)
|
|
2221
2479
|
by_model_rows = crk._aggregate_cache_breakdown_from_rows(
|
|
2222
2480
|
rows_in_window,
|
|
@@ -2464,10 +2722,130 @@ def _compute_intensity_buckets(rows: "list[DailyPanelRow]") -> list[float]:
|
|
|
2464
2722
|
return thresholds
|
|
2465
2723
|
|
|
2466
2724
|
|
|
2725
|
+
def _group_a_monthly_buckets(now_utc, *, n, range_start, display_tz):
|
|
2726
|
+
"""Assemble the monthly panel's per-month ``BucketUsage`` list via the
|
|
2727
|
+
#268 Group A cache, or ``None`` to fall back to the wide fetch (spec §5.1).
|
|
2728
|
+
|
|
2729
|
+
``all_bucket_labels`` is the contiguous set of display-tz ``"%Y-%m"``
|
|
2730
|
+
months spanned by the wide window ``[range_start, now_utc]`` — from
|
|
2731
|
+
``range_start``'s display-tz month (the oldest an entry in the window can
|
|
2732
|
+
bucket into, including the west-of-UTC boundary-spillover month) up to
|
|
2733
|
+
the current display-tz month — in ascending order. ``build_monthly_view``
|
|
2734
|
+
then reverses + caps to ``n``, dropping any spillover month exactly as
|
|
2735
|
+
the from-scratch path does. Each recompute fetches a ±2-day-padded window
|
|
2736
|
+
(clamped to the wide window) and filters ``_aggregate_monthly`` to the
|
|
2737
|
+
target month, so the boundary follows ``display_tz`` and per-bucket
|
|
2738
|
+
first-seen model order matches the wide pass byte-for-byte.
|
|
2739
|
+
"""
|
|
2740
|
+
if not _GROUP_A_CACHE_ENABLED:
|
|
2741
|
+
return None
|
|
2742
|
+
try:
|
|
2743
|
+
cache_conn = open_cache_db()
|
|
2744
|
+
except Exception:
|
|
2745
|
+
return None
|
|
2746
|
+
try:
|
|
2747
|
+
def _tz_month(instant):
|
|
2748
|
+
local = (
|
|
2749
|
+
instant.astimezone(display_tz) if display_tz is not None
|
|
2750
|
+
# internal fallback: host-local intentional
|
|
2751
|
+
else instant.astimezone()
|
|
2752
|
+
)
|
|
2753
|
+
return local.year, local.month
|
|
2754
|
+
|
|
2755
|
+
oldest_y, oldest_m = _tz_month(range_start)
|
|
2756
|
+
newest_y, newest_m = _tz_month(now_utc)
|
|
2757
|
+
# Contiguous months oldest → newest (ascending).
|
|
2758
|
+
labels: list[str] = []
|
|
2759
|
+
y, m = oldest_y, oldest_m
|
|
2760
|
+
while (y, m) <= (newest_y, newest_m):
|
|
2761
|
+
labels.append(f"{y:04d}-{m:02d}")
|
|
2762
|
+
m += 1
|
|
2763
|
+
if m == 13:
|
|
2764
|
+
m = 1
|
|
2765
|
+
y += 1
|
|
2766
|
+
current_label = f"{newest_y:04d}-{newest_m:02d}"
|
|
2767
|
+
tz_sig = display_tz.key if display_tz is not None else "local"
|
|
2768
|
+
|
|
2769
|
+
def _month_bounds(label):
|
|
2770
|
+
yy, mm = (int(x) for x in label.split("-"))
|
|
2771
|
+
start = dt.datetime(yy, mm, 1, tzinfo=dt.timezone.utc)
|
|
2772
|
+
ny, nm = (yy + 1, 1) if mm == 12 else (yy, mm + 1)
|
|
2773
|
+
nxt = dt.datetime(ny, nm, 1, tzinfo=dt.timezone.utc)
|
|
2774
|
+
return start, nxt
|
|
2775
|
+
|
|
2776
|
+
def _fetch(label):
|
|
2777
|
+
start, nxt = _month_bounds(label)
|
|
2778
|
+
# ±2 days of slack covers the display-tz month boundary for any
|
|
2779
|
+
# offset; clamp to the wide window so nothing after now_utc /
|
|
2780
|
+
# before range_start leaks in.
|
|
2781
|
+
lo = max(start - dt.timedelta(days=2), range_start)
|
|
2782
|
+
hi = min(nxt + dt.timedelta(days=2), now_utc)
|
|
2783
|
+
if hi <= lo:
|
|
2784
|
+
return []
|
|
2785
|
+
return iter_entries(cache_conn, lo, hi)
|
|
2786
|
+
|
|
2787
|
+
def _agg_one(label, entries):
|
|
2788
|
+
for b in _aggregate_monthly(entries, tz=display_tz):
|
|
2789
|
+
if b.bucket == label:
|
|
2790
|
+
return b
|
|
2791
|
+
return None
|
|
2792
|
+
|
|
2793
|
+
def _end_of(label):
|
|
2794
|
+
_start, nxt = _month_bounds(label)
|
|
2795
|
+
return nxt + dt.timedelta(days=2)
|
|
2796
|
+
|
|
2797
|
+
# #271: current-bucket accumulator inputs. `_membership` IS
|
|
2798
|
+
# `_aggregate_monthly`'s own key (display-tz `%Y-%m`);
|
|
2799
|
+
# `_all_fetch`/`_delta_fetch` reuse `_fetch`'s ±2-day-padded month
|
|
2800
|
+
# window (clamped to now_utc) via the `(id, UsageEntry)` delta sibling.
|
|
2801
|
+
def _membership(e):
|
|
2802
|
+
local = (
|
|
2803
|
+
e.timestamp.astimezone(display_tz) if display_tz is not None
|
|
2804
|
+
# internal fallback: host-local intentional
|
|
2805
|
+
else e.timestamp.astimezone()
|
|
2806
|
+
)
|
|
2807
|
+
return local.strftime("%Y-%m") == current_label
|
|
2808
|
+
|
|
2809
|
+
def _current_window(label):
|
|
2810
|
+
start, nxt = _month_bounds(label)
|
|
2811
|
+
lo = max(start - dt.timedelta(days=2), range_start)
|
|
2812
|
+
hi = min(nxt + dt.timedelta(days=2), now_utc)
|
|
2813
|
+
return lo, hi
|
|
2814
|
+
|
|
2815
|
+
def _all_fetch(label):
|
|
2816
|
+
lo, hi = _current_window(label)
|
|
2817
|
+
return [] if hi <= lo else iter_entries_with_id(cache_conn, lo, hi)
|
|
2818
|
+
|
|
2819
|
+
def _delta_fetch(label, after_seq, after_ts):
|
|
2820
|
+
lo, hi = _current_window(label)
|
|
2821
|
+
return [] if hi <= lo else iter_entries_with_id(
|
|
2822
|
+
cache_conn, lo, hi, after_seq=after_seq, after_ts=after_ts)
|
|
2823
|
+
|
|
2824
|
+
buckets = build_cached_group_a(
|
|
2825
|
+
"monthly",
|
|
2826
|
+
cache_conn=cache_conn,
|
|
2827
|
+
all_bucket_labels=labels,
|
|
2828
|
+
current_label=current_label,
|
|
2829
|
+
bucket_end_of=_end_of,
|
|
2830
|
+
fetch_bucket_entries=_fetch,
|
|
2831
|
+
aggregate_one=_agg_one,
|
|
2832
|
+
extra_signature=("monthly", tz_sig),
|
|
2833
|
+
use_current_accumulator=True,
|
|
2834
|
+
now_utc=now_utc,
|
|
2835
|
+
current_all_fetch=_all_fetch,
|
|
2836
|
+
current_delta_fetch=_delta_fetch,
|
|
2837
|
+
membership_of=_membership,
|
|
2838
|
+
)
|
|
2839
|
+
return tuple(buckets)
|
|
2840
|
+
finally:
|
|
2841
|
+
cache_conn.close()
|
|
2842
|
+
|
|
2843
|
+
|
|
2467
2844
|
def _dashboard_build_monthly_periods(conn: "sqlite3.Connection",
|
|
2468
2845
|
now_utc: "dt.datetime",
|
|
2469
2846
|
*, n: int = 12,
|
|
2470
2847
|
skip_sync: bool = False,
|
|
2848
|
+
use_group_a_cache: bool = False,
|
|
2471
2849
|
display_tz: "ZoneInfo | None" = None) -> "list[MonthlyPeriodRow]":
|
|
2472
2850
|
"""Latest n calendar months as MonthlyPeriodRow, newest-first.
|
|
2473
2851
|
|
|
@@ -2496,17 +2874,188 @@ def _dashboard_build_monthly_periods(conn: "sqlite3.Connection",
|
|
|
2496
2874
|
range_start = dt.datetime(sy, sm, 1, tzinfo=dt.timezone.utc)
|
|
2497
2875
|
range_end = now_utc
|
|
2498
2876
|
|
|
2499
|
-
|
|
2877
|
+
# #268 Group A cache: assemble the per-month BucketUsage list through the
|
|
2878
|
+
# module cache (recompute only the current + watermark-dirty months;
|
|
2879
|
+
# serve immutable past months from memory) and hand it to
|
|
2880
|
+
# build_monthly_view as aggregated_override. Gated on ``use_group_a_cache``
|
|
2881
|
+
# — set ONLY by the sync-thread rebuild, never by ``skip_sync`` (the
|
|
2882
|
+
# share-period-override path also runs ``skip_sync=True`` but on an HTTP
|
|
2883
|
+
# thread with a shifted PAST now, which would pollute the shared cache with
|
|
2884
|
+
# a partial past-month bucket — see ``_dashboard_build_daily_panel``). Every
|
|
2885
|
+
# non-sync-thread caller falls through to the from-scratch wide fetch. Also
|
|
2886
|
+
# falls back when disabled / cache DB unavailable.
|
|
2887
|
+
aggregated_override = (
|
|
2888
|
+
_group_a_monthly_buckets(
|
|
2889
|
+
now_utc, n=n, range_start=range_start, display_tz=display_tz,
|
|
2890
|
+
)
|
|
2891
|
+
if use_group_a_cache else None
|
|
2892
|
+
)
|
|
2500
2893
|
c = _cctally()
|
|
2501
|
-
|
|
2502
|
-
|
|
2894
|
+
if aggregated_override is None:
|
|
2895
|
+
entries = get_entries(range_start, range_end, skip_sync=skip_sync)
|
|
2896
|
+
view = c.build_monthly_view(entries, now_utc=now_utc, n=n,
|
|
2897
|
+
display_tz=display_tz)
|
|
2898
|
+
else:
|
|
2899
|
+
view = c.build_monthly_view((), now_utc=now_utc, n=n,
|
|
2900
|
+
display_tz=display_tz,
|
|
2901
|
+
aggregated_override=aggregated_override)
|
|
2503
2902
|
return list(view.rows)
|
|
2504
2903
|
|
|
2505
2904
|
|
|
2905
|
+
def _group_a_weekly_buckets(stats_conn, now_utc, *, weeks):
|
|
2906
|
+
"""Assemble the weekly panel's per-week ``BucketUsage`` list via the #268
|
|
2907
|
+
Group A cache, or ``None`` to fall back to the wide fetch (spec §5.1).
|
|
2908
|
+
|
|
2909
|
+
``all_bucket_labels`` = each SubWeek's ``start_date.isoformat()`` in
|
|
2910
|
+
ascending order (matching ``_aggregate_weekly``'s sorted-key output);
|
|
2911
|
+
``current_label`` = the SubWeek containing ``now_utc`` (always recomputed
|
|
2912
|
+
as the open week). Each recompute fetches ``[week.start_ts, min(week.end_ts,
|
|
2913
|
+
now_utc)]`` (clamped to now, matching the wide fetch's ``range_end`` bound)
|
|
2914
|
+
and runs ``_aggregate_weekly`` over the FULL ``weeks`` list — extracting
|
|
2915
|
+
the target week's bucket — so overlapping SubWeeks (reset-day drift) keep
|
|
2916
|
+
their first-match-wins assignment byte-for-byte with the wide pass.
|
|
2917
|
+
|
|
2918
|
+
Weekly is the multi-table case (spec §5.1): the SubWeek boundaries derive
|
|
2919
|
+
from ``weekly_usage_snapshots`` + the reset-event tables, so the cached raw
|
|
2920
|
+
aggregate could shift when any of those move. ``extra_signature`` folds in
|
|
2921
|
+
``MAX(weekly_usage_snapshots.id)`` / ``MAX(weekly_cost_snapshots.id)`` /
|
|
2922
|
+
the reset-event change-signal — a change to any full-invalidates the weekly
|
|
2923
|
+
namespace (the scoped M2.4 fallback: recompute-all on a weekly-relevant
|
|
2924
|
+
change, cache-hit only when nothing weekly-relevant moved — still the idle
|
|
2925
|
+
win, and trivially byte-identical). Pure session-entry adds stay per-week
|
|
2926
|
+
via the watermark. The overlay / Bug-K presentation reruns fresh each tick,
|
|
2927
|
+
so a snapshot change is reflected even on a cache-served bucket.
|
|
2928
|
+
"""
|
|
2929
|
+
if not _GROUP_A_CACHE_ENABLED:
|
|
2930
|
+
return None
|
|
2931
|
+
try:
|
|
2932
|
+
cache_conn = open_cache_db()
|
|
2933
|
+
except Exception:
|
|
2934
|
+
return None
|
|
2935
|
+
try:
|
|
2936
|
+
sw_by_label = {w.start_date.isoformat(): w for w in weeks}
|
|
2937
|
+
labels = [w.start_date.isoformat() for w in weeks] # ascending
|
|
2938
|
+
# current_label = the SubWeek that contains now_utc (the open week);
|
|
2939
|
+
# fall back to the newest week if none contains it.
|
|
2940
|
+
current_label = None
|
|
2941
|
+
for w in weeks:
|
|
2942
|
+
try:
|
|
2943
|
+
s = parse_iso_datetime(w.start_ts, "week.start_ts")
|
|
2944
|
+
e = parse_iso_datetime(w.end_ts, "week.end_ts")
|
|
2945
|
+
except ValueError:
|
|
2946
|
+
continue
|
|
2947
|
+
if s <= now_utc < e:
|
|
2948
|
+
current_label = w.start_date.isoformat()
|
|
2949
|
+
break
|
|
2950
|
+
if current_label is None:
|
|
2951
|
+
current_label = max(
|
|
2952
|
+
weeks, key=lambda w: w.start_date
|
|
2953
|
+
).start_date.isoformat()
|
|
2954
|
+
|
|
2955
|
+
# Weekly-relevant signature legs: any change full-invalidates.
|
|
2956
|
+
extra_signature = (
|
|
2957
|
+
_snapshot_max_id(stats_conn, "weekly_usage_snapshots"),
|
|
2958
|
+
_snapshot_max_id(stats_conn, "weekly_cost_snapshots"),
|
|
2959
|
+
_snapshot_reset_sig(stats_conn),
|
|
2960
|
+
)
|
|
2961
|
+
|
|
2962
|
+
def _fetch(label):
|
|
2963
|
+
w = sw_by_label[label]
|
|
2964
|
+
s = parse_iso_datetime(w.start_ts, "week.start_ts")
|
|
2965
|
+
e = parse_iso_datetime(w.end_ts, "week.end_ts")
|
|
2966
|
+
hi = min(e, now_utc) # cap the open week at now, like the wide fetch
|
|
2967
|
+
if hi <= s:
|
|
2968
|
+
return []
|
|
2969
|
+
return iter_entries(cache_conn, s, hi)
|
|
2970
|
+
|
|
2971
|
+
def _agg_one(label, entries):
|
|
2972
|
+
# Aggregate against the FULL weeks list so overlapping SubWeeks
|
|
2973
|
+
# resolve first-match-wins exactly as the wide pass does; extract
|
|
2974
|
+
# the target week's bucket.
|
|
2975
|
+
for b in _aggregate_weekly(entries, weeks):
|
|
2976
|
+
if b.bucket == label:
|
|
2977
|
+
return b
|
|
2978
|
+
return None
|
|
2979
|
+
|
|
2980
|
+
def _end_of(label):
|
|
2981
|
+
w = sw_by_label[label]
|
|
2982
|
+
try:
|
|
2983
|
+
return parse_iso_datetime(w.end_ts, "week.end_ts")
|
|
2984
|
+
except ValueError:
|
|
2985
|
+
return None
|
|
2986
|
+
|
|
2987
|
+
# #271: current-bucket accumulator inputs. Membership REUSES
|
|
2988
|
+
# _aggregate_weekly's exact bisect + first-match-wins assignment
|
|
2989
|
+
# (bin/_lib_aggregators.py::_week_key_or_none) — the parsed-bounds list
|
|
2990
|
+
# is built identically (same order, same parse) so an entry in an
|
|
2991
|
+
# overlap region (reset-day drift) keys to the SAME week the full pass
|
|
2992
|
+
# assigns it. _all_fetch/_delta_fetch reuse _fetch's
|
|
2993
|
+
# [week.start, min(week.end, now_utc)] window via the (id, UsageEntry)
|
|
2994
|
+
# delta sibling.
|
|
2995
|
+
_parsed_bounds = [
|
|
2996
|
+
(parse_iso_datetime(w.start_ts, "week.start_ts"),
|
|
2997
|
+
parse_iso_datetime(w.end_ts, "week.end_ts"),
|
|
2998
|
+
w.start_date.isoformat())
|
|
2999
|
+
for w in weeks
|
|
3000
|
+
]
|
|
3001
|
+
_bound_starts = [b[0] for b in _parsed_bounds]
|
|
3002
|
+
|
|
3003
|
+
def _week_key(e):
|
|
3004
|
+
ts = e.timestamp
|
|
3005
|
+
idx = bisect.bisect_right(_bound_starts, ts) - 1
|
|
3006
|
+
if idx < 0:
|
|
3007
|
+
return None
|
|
3008
|
+
while idx > 0 and (
|
|
3009
|
+
_parsed_bounds[idx - 1][0] <= ts < _parsed_bounds[idx - 1][1]
|
|
3010
|
+
):
|
|
3011
|
+
idx -= 1
|
|
3012
|
+
s_dt, e_dt, key = _parsed_bounds[idx]
|
|
3013
|
+
return key if s_dt <= ts < e_dt else None
|
|
3014
|
+
|
|
3015
|
+
def _membership(e):
|
|
3016
|
+
return _week_key(e) == current_label
|
|
3017
|
+
|
|
3018
|
+
def _current_window(label):
|
|
3019
|
+
w = sw_by_label[label]
|
|
3020
|
+
s = parse_iso_datetime(w.start_ts, "week.start_ts")
|
|
3021
|
+
e = parse_iso_datetime(w.end_ts, "week.end_ts")
|
|
3022
|
+
return s, min(e, now_utc)
|
|
3023
|
+
|
|
3024
|
+
def _all_fetch(label):
|
|
3025
|
+
s, hi = _current_window(label)
|
|
3026
|
+
return [] if hi <= s else iter_entries_with_id(cache_conn, s, hi)
|
|
3027
|
+
|
|
3028
|
+
def _delta_fetch(label, after_seq, after_ts):
|
|
3029
|
+
s, hi = _current_window(label)
|
|
3030
|
+
return [] if hi <= s else iter_entries_with_id(
|
|
3031
|
+
cache_conn, s, hi, after_seq=after_seq, after_ts=after_ts)
|
|
3032
|
+
|
|
3033
|
+
buckets = build_cached_group_a(
|
|
3034
|
+
"weekly",
|
|
3035
|
+
cache_conn=cache_conn,
|
|
3036
|
+
all_bucket_labels=labels,
|
|
3037
|
+
current_label=current_label,
|
|
3038
|
+
bucket_end_of=_end_of,
|
|
3039
|
+
fetch_bucket_entries=_fetch,
|
|
3040
|
+
aggregate_one=_agg_one,
|
|
3041
|
+
extra_signature=extra_signature,
|
|
3042
|
+
use_current_accumulator=True,
|
|
3043
|
+
now_utc=now_utc,
|
|
3044
|
+
current_all_fetch=_all_fetch,
|
|
3045
|
+
current_delta_fetch=_delta_fetch,
|
|
3046
|
+
membership_of=_membership,
|
|
3047
|
+
)
|
|
3048
|
+
return tuple(buckets)
|
|
3049
|
+
finally:
|
|
3050
|
+
cache_conn.close()
|
|
3051
|
+
|
|
3052
|
+
|
|
2506
3053
|
def _dashboard_build_weekly_periods(conn: "sqlite3.Connection",
|
|
2507
3054
|
now_utc: "dt.datetime",
|
|
2508
3055
|
*, n: int = 12,
|
|
2509
|
-
skip_sync: bool = False
|
|
3056
|
+
skip_sync: bool = False,
|
|
3057
|
+
use_group_a_cache: bool = False,
|
|
3058
|
+
use_bugk_segment_cache: bool = False) -> "list[WeeklyPeriodRow]":
|
|
2510
3059
|
"""Latest n subscription weeks as WeeklyPeriodRow, newest-first.
|
|
2511
3060
|
|
|
2512
3061
|
Thin builder-using prelude + Bug-K pre-credit synthesis on top of
|
|
@@ -2534,7 +3083,6 @@ def _dashboard_build_weekly_periods(conn: "sqlite3.Connection",
|
|
|
2534
3083
|
range_start,
|
|
2535
3084
|
parse_iso_datetime(weeks[0].start_ts, "week_start_at"),
|
|
2536
3085
|
)
|
|
2537
|
-
entries = get_entries(fetch_start, range_end, skip_sync=skip_sync)
|
|
2538
3086
|
as_of_utc = (
|
|
2539
3087
|
range_end.astimezone(dt.timezone.utc)
|
|
2540
3088
|
.replace(microsecond=0)
|
|
@@ -2542,12 +3090,37 @@ def _dashboard_build_weekly_periods(conn: "sqlite3.Connection",
|
|
|
2542
3090
|
.replace("+00:00", "Z")
|
|
2543
3091
|
)
|
|
2544
3092
|
|
|
2545
|
-
#
|
|
2546
|
-
|
|
2547
|
-
|
|
2548
|
-
|
|
2549
|
-
|
|
3093
|
+
# #268 Group A cache: assemble the per-week BucketUsage list through the
|
|
3094
|
+
# module cache (recompute only the current + watermark-dirty weeks; serve
|
|
3095
|
+
# immutable past weeks from memory) and hand it to build_weekly_view as
|
|
3096
|
+
# aggregated_override. Gated on ``use_group_a_cache`` — set ONLY by the
|
|
3097
|
+
# sync-thread rebuild, never by ``skip_sync``. The share-period-override
|
|
3098
|
+
# path also runs ``skip_sync=True`` but on an HTTP thread with a shifted
|
|
3099
|
+
# PAST now; routing it through the shared cache would clamp the
|
|
3100
|
+
# now_override-current week to a partial aggregate and cache it under a real
|
|
3101
|
+
# PAST-week label (weekly is the worst case — its extra_signature is
|
|
3102
|
+
# stats-only, so nothing invalidates the polluted week until a data change
|
|
3103
|
+
# lands). Every non-sync-thread caller falls through to the from-scratch
|
|
3104
|
+
# wide fetch, as does the cache-disabled / cache-unavailable case. The
|
|
3105
|
+
# overlay + Bug-K + delta + is_current presentation always reruns fresh over
|
|
3106
|
+
# the assembled list.
|
|
3107
|
+
aggregated_override = (
|
|
3108
|
+
_group_a_weekly_buckets(conn, now_utc, weeks=weeks)
|
|
3109
|
+
if use_group_a_cache else None
|
|
2550
3110
|
)
|
|
3111
|
+
c = _cctally()
|
|
3112
|
+
if aggregated_override is None:
|
|
3113
|
+
entries = get_entries(fetch_start, range_end, skip_sync=skip_sync)
|
|
3114
|
+
view = c.build_weekly_view(
|
|
3115
|
+
conn, entries, weeks=weeks, now_utc=now_utc,
|
|
3116
|
+
display_tz=None, as_of_utc=as_of_utc,
|
|
3117
|
+
)
|
|
3118
|
+
else:
|
|
3119
|
+
view = c.build_weekly_view(
|
|
3120
|
+
conn, (), weeks=weeks, now_utc=now_utc,
|
|
3121
|
+
display_tz=None, as_of_utc=as_of_utc,
|
|
3122
|
+
aggregated_override=aggregated_override,
|
|
3123
|
+
)
|
|
2551
3124
|
if not view.rows:
|
|
2552
3125
|
return []
|
|
2553
3126
|
|
|
@@ -2615,9 +3188,10 @@ def _dashboard_build_weekly_periods(conn: "sqlite3.Connection",
|
|
|
2615
3188
|
# _apply_reset_events_to_subweeks shifts the credited SubWeek's
|
|
2616
3189
|
# start_ts to ``effective_reset_at_utc``, so _aggregate_weekly's
|
|
2617
3190
|
# bucket for that SubWeek already covers ONLY the post-credit
|
|
2618
|
-
# interval. We rebuild the pre-credit bucket here by
|
|
2619
|
-
#
|
|
2620
|
-
# re-aggregating cost /
|
|
3191
|
+
# interval. We rebuild the pre-credit bucket here by fetching the
|
|
3192
|
+
# ``[original_start, effective)`` window directly (#268: no longer a
|
|
3193
|
+
# wide ``entries`` list on the cached path) and re-aggregating cost /
|
|
3194
|
+
# tokens / per-model.
|
|
2621
3195
|
#
|
|
2622
3196
|
# The pre-credit row's ``used_pct`` comes from the
|
|
2623
3197
|
# weekly_usage_snapshots row captured at-or-before the credit
|
|
@@ -2689,24 +3263,64 @@ def _dashboard_build_weekly_periods(conn: "sqlite3.Connection",
|
|
|
2689
3263
|
# No pre-credit interval to aggregate.
|
|
2690
3264
|
continue
|
|
2691
3265
|
|
|
2692
|
-
# Aggregate entries in [original_start, effective).
|
|
2693
|
-
|
|
2694
|
-
|
|
2695
|
-
|
|
2696
|
-
|
|
2697
|
-
|
|
2698
|
-
|
|
2699
|
-
|
|
2700
|
-
|
|
2701
|
-
|
|
2702
|
-
|
|
2703
|
-
|
|
2704
|
-
|
|
2705
|
-
|
|
2706
|
-
|
|
2707
|
-
|
|
2708
|
-
|
|
2709
|
-
|
|
3266
|
+
# Aggregate entries in [original_start, effective). Fetch this
|
|
3267
|
+
# window directly (skip_sync=True — the rebuild already ingested,
|
|
3268
|
+
# or the fallback's wide fetch above did) rather than filtering a
|
|
3269
|
+
# wide `entries` list, so Bug-K works identically on the cached
|
|
3270
|
+
# path (where no wide entry list exists) and the fallback path.
|
|
3271
|
+
#
|
|
3272
|
+
# #271 §18: the [original_start, effective) window is a CLOSED past
|
|
3273
|
+
# interval (effective is a historical credit moment), so this folded
|
|
3274
|
+
# aggregate is IMMUTABLE — cache it byte-identically (the same
|
|
3275
|
+
# "re-aggregate immutable history every tick" pattern the #269
|
|
3276
|
+
# weekref cost cache fixed). `_compute_pre_segment` is the exact
|
|
3277
|
+
# from-scratch fetch+fold closure; on the sync thread
|
|
3278
|
+
# (`use_bugk_segment_cache=True`) it routes through
|
|
3279
|
+
# `cached_bugk_segment` (get-or-compute over the canonical window
|
|
3280
|
+
# key), on EVERY other caller (CLI / share / tests / reconcile-
|
|
3281
|
+
# failed) it computes directly — byte-unchanged. The `used_pct`
|
|
3282
|
+
# snapshot query + WeeklyPeriodRow build BELOW always rerun fresh
|
|
3283
|
+
# from the segment (do NOT cache the row — Codex-BK-5).
|
|
3284
|
+
def _compute_pre_segment(_orig=original_start_dt, _eff=eff_dt):
|
|
3285
|
+
pi = po = pcc = pcr = 0
|
|
3286
|
+
pcost = 0.0
|
|
3287
|
+
pmodels: dict[str, float] = {}
|
|
3288
|
+
pcount = 0
|
|
3289
|
+
for e in get_entries(_orig, _eff, skip_sync=True):
|
|
3290
|
+
if _orig <= e.timestamp < _eff:
|
|
3291
|
+
usage = e.usage
|
|
3292
|
+
pi += usage.get("input_tokens", 0)
|
|
3293
|
+
po += usage.get("output_tokens", 0)
|
|
3294
|
+
pcc += usage.get("cache_creation_input_tokens", 0)
|
|
3295
|
+
pcr += usage.get("cache_read_input_tokens", 0)
|
|
3296
|
+
c = _calc(
|
|
3297
|
+
e.model, usage, mode="auto", cost_usd=e.cost_usd,
|
|
3298
|
+
)
|
|
3299
|
+
pcost += c
|
|
3300
|
+
pmodels[e.model] = pmodels.get(e.model, 0.0) + c
|
|
3301
|
+
pcount += 1
|
|
3302
|
+
# Codex-BK-4: freeze `models` as a tuple of (model, cost) in
|
|
3303
|
+
# first-seen (dict-insertion) order so the row's stable cost-desc
|
|
3304
|
+
# sort tie-order can't be mutated after caching.
|
|
3305
|
+
return BugKSegment(
|
|
3306
|
+
input=pi, output=po, cache_create=pcc, cache_read=pcr,
|
|
3307
|
+
cost=pcost, models=tuple(pmodels.items()), entry_count=pcount,
|
|
3308
|
+
)
|
|
3309
|
+
|
|
3310
|
+
if use_bugk_segment_cache:
|
|
3311
|
+
seg = cached_bugk_segment(
|
|
3312
|
+
key=_bugk_key(original_start_dt, eff_dt),
|
|
3313
|
+
compute=_compute_pre_segment,
|
|
3314
|
+
)
|
|
3315
|
+
else:
|
|
3316
|
+
seg = _compute_pre_segment()
|
|
3317
|
+
pre_input = seg.input
|
|
3318
|
+
pre_output = seg.output
|
|
3319
|
+
pre_cc = seg.cache_create
|
|
3320
|
+
pre_cr = seg.cache_read
|
|
3321
|
+
pre_cost = seg.cost
|
|
3322
|
+
pre_models = seg.models # ((model, cost), ...) in first-seen order
|
|
3323
|
+
pre_entry_count = seg.entry_count
|
|
2710
3324
|
if pre_entry_count == 0 and pre_cost <= 0:
|
|
2711
3325
|
# No measurable pre-credit activity — skip insertion.
|
|
2712
3326
|
continue
|
|
@@ -2729,9 +3343,12 @@ def _dashboard_build_weekly_periods(conn: "sqlite3.Connection",
|
|
|
2729
3343
|
)
|
|
2730
3344
|
|
|
2731
3345
|
pre_total = pre_input + pre_output + pre_cc + pre_cr
|
|
3346
|
+
# `pre_models` is a first-seen-order tuple of (model, cost); the
|
|
3347
|
+
# stable cost-desc sort preserves that tie-order byte-for-byte,
|
|
3348
|
+
# identical to the pre-#271 `sorted(dict.items(), ...)`.
|
|
2732
3349
|
pre_model_breakdowns = [
|
|
2733
3350
|
{"modelName": m, "cost": c}
|
|
2734
|
-
for m, c in sorted(pre_models
|
|
3351
|
+
for m, c in sorted(pre_models, key=lambda kv: -kv[1])
|
|
2735
3352
|
]
|
|
2736
3353
|
pre_label = original_start_dt.strftime("%m-%d")
|
|
2737
3354
|
pre_row = WeeklyPeriodRow(
|
|
@@ -2965,11 +3582,130 @@ def _dashboard_build_blocks_panel(conn: "sqlite3.Connection",
|
|
|
2965
3582
|
return list(view.rows)
|
|
2966
3583
|
|
|
2967
3584
|
|
|
3585
|
+
def _group_a_daily_buckets(now_utc, *, n, display_tz):
|
|
3586
|
+
"""Assemble the daily panel's per-day ``BucketUsage`` list via the #268
|
|
3587
|
+
Group A cache, or ``None`` to signal the caller should take the
|
|
3588
|
+
from-scratch wide-fetch path (spec §5.1).
|
|
3589
|
+
|
|
3590
|
+
Returns a tuple of ``BucketUsage`` in ascending date order (matching
|
|
3591
|
+
``_aggregate_daily``'s sorted output), so ``build_daily_view`` consumes
|
|
3592
|
+
it as ``aggregated_override`` with no reorder. Each recompute fetches a
|
|
3593
|
+
±1-day-padded window and filters ``_aggregate_daily`` to the target date,
|
|
3594
|
+
so the day boundary follows ``display_tz`` exactly (DST-safe) and the
|
|
3595
|
+
per-bucket first-seen model order reproduces the wide pass byte-for-byte.
|
|
3596
|
+
``None`` is returned when the cache path is disabled or the cache DB is
|
|
3597
|
+
unavailable — the builder then does the original wide aggregation.
|
|
3598
|
+
"""
|
|
3599
|
+
if not _GROUP_A_CACHE_ENABLED:
|
|
3600
|
+
return None
|
|
3601
|
+
try:
|
|
3602
|
+
cache_conn = open_cache_db()
|
|
3603
|
+
except Exception:
|
|
3604
|
+
return None
|
|
3605
|
+
try:
|
|
3606
|
+
# The from-scratch path fetches exactly [range_start, now_utc]; the
|
|
3607
|
+
# per-day fetches below clamp to the SAME window so the current day's
|
|
3608
|
+
# bucket never picks up an entry after now_utc (and nothing outside
|
|
3609
|
+
# the wide window leaks in) — byte-identical to the wide fetch.
|
|
3610
|
+
range_start = now_utc - dt.timedelta(days=n + 1)
|
|
3611
|
+
today_local = (
|
|
3612
|
+
now_utc.astimezone(display_tz) if display_tz is not None
|
|
3613
|
+
# internal fallback: host-local intentional
|
|
3614
|
+
else now_utc.astimezone()
|
|
3615
|
+
).date()
|
|
3616
|
+
# Contiguous n-day window, oldest → newest (ascending, so the
|
|
3617
|
+
# assembled list matches _aggregate_daily's sorted-key order).
|
|
3618
|
+
labels = [
|
|
3619
|
+
(today_local - dt.timedelta(days=i)).isoformat()
|
|
3620
|
+
for i in range(n - 1, -1, -1)
|
|
3621
|
+
]
|
|
3622
|
+
current_label = today_local.isoformat()
|
|
3623
|
+
tz_sig = display_tz.key if display_tz is not None else "local"
|
|
3624
|
+
|
|
3625
|
+
def _fetch(label):
|
|
3626
|
+
d = dt.date.fromisoformat(label)
|
|
3627
|
+
base = dt.datetime(d.year, d.month, d.day, tzinfo=dt.timezone.utc)
|
|
3628
|
+
# ±1 day of slack fully covers the display-tz day for any tz
|
|
3629
|
+
# offset; the aggregate filter below selects exactly the label's
|
|
3630
|
+
# entries, so over-fetch is harmless and DST-robust. Clamp to the
|
|
3631
|
+
# wide [range_start, now_utc] window for exact parity.
|
|
3632
|
+
lo = max(base - dt.timedelta(days=1), range_start)
|
|
3633
|
+
hi = min(base + dt.timedelta(days=2), now_utc)
|
|
3634
|
+
if hi <= lo:
|
|
3635
|
+
return []
|
|
3636
|
+
return iter_entries(cache_conn, lo, hi)
|
|
3637
|
+
|
|
3638
|
+
def _agg_one(label, entries):
|
|
3639
|
+
for b in _aggregate_daily(entries, tz=display_tz):
|
|
3640
|
+
if b.bucket == label:
|
|
3641
|
+
return b
|
|
3642
|
+
return None
|
|
3643
|
+
|
|
3644
|
+
def _end_of(label):
|
|
3645
|
+
d = dt.date.fromisoformat(label)
|
|
3646
|
+
# Over-estimate the day's UTC end (true end ≤ (d+1) 12:00 UTC for
|
|
3647
|
+
# any tz); over-marking dirty is safe, under-marking is not.
|
|
3648
|
+
return (
|
|
3649
|
+
dt.datetime(d.year, d.month, d.day, tzinfo=dt.timezone.utc)
|
|
3650
|
+
+ dt.timedelta(days=2)
|
|
3651
|
+
)
|
|
3652
|
+
|
|
3653
|
+
# #271: current-bucket accumulator inputs. `_membership` IS
|
|
3654
|
+
# `_aggregate_daily`'s own key (display-tz `%Y-%m-%d`), so the id-bounded
|
|
3655
|
+
# fetch keeps exactly the entries the full pass assigns to the current
|
|
3656
|
+
# day, byte-identically. `_all_fetch`/`_delta_fetch` reuse `_fetch`'s
|
|
3657
|
+
# ±1-day-padded window (clamped to now_utc) via the `(id, UsageEntry)`
|
|
3658
|
+
# delta sibling.
|
|
3659
|
+
def _membership(e):
|
|
3660
|
+
local = (
|
|
3661
|
+
e.timestamp.astimezone(display_tz) if display_tz is not None
|
|
3662
|
+
# internal fallback: host-local intentional
|
|
3663
|
+
else e.timestamp.astimezone()
|
|
3664
|
+
)
|
|
3665
|
+
return local.strftime("%Y-%m-%d") == current_label
|
|
3666
|
+
|
|
3667
|
+
def _current_window(label):
|
|
3668
|
+
d = dt.date.fromisoformat(label)
|
|
3669
|
+
base = dt.datetime(d.year, d.month, d.day, tzinfo=dt.timezone.utc)
|
|
3670
|
+
lo = max(base - dt.timedelta(days=1), range_start)
|
|
3671
|
+
hi = min(base + dt.timedelta(days=2), now_utc)
|
|
3672
|
+
return lo, hi
|
|
3673
|
+
|
|
3674
|
+
def _all_fetch(label):
|
|
3675
|
+
lo, hi = _current_window(label)
|
|
3676
|
+
return [] if hi <= lo else iter_entries_with_id(cache_conn, lo, hi)
|
|
3677
|
+
|
|
3678
|
+
def _delta_fetch(label, after_seq, after_ts):
|
|
3679
|
+
lo, hi = _current_window(label)
|
|
3680
|
+
return [] if hi <= lo else iter_entries_with_id(
|
|
3681
|
+
cache_conn, lo, hi, after_seq=after_seq, after_ts=after_ts)
|
|
3682
|
+
|
|
3683
|
+
buckets = build_cached_group_a(
|
|
3684
|
+
"daily",
|
|
3685
|
+
cache_conn=cache_conn,
|
|
3686
|
+
all_bucket_labels=labels,
|
|
3687
|
+
current_label=current_label,
|
|
3688
|
+
bucket_end_of=_end_of,
|
|
3689
|
+
fetch_bucket_entries=_fetch,
|
|
3690
|
+
aggregate_one=_agg_one,
|
|
3691
|
+
extra_signature=("daily", tz_sig),
|
|
3692
|
+
use_current_accumulator=True,
|
|
3693
|
+
now_utc=now_utc,
|
|
3694
|
+
current_all_fetch=_all_fetch,
|
|
3695
|
+
current_delta_fetch=_delta_fetch,
|
|
3696
|
+
membership_of=_membership,
|
|
3697
|
+
)
|
|
3698
|
+
return tuple(buckets)
|
|
3699
|
+
finally:
|
|
3700
|
+
cache_conn.close()
|
|
3701
|
+
|
|
3702
|
+
|
|
2968
3703
|
def _dashboard_build_daily_panel(conn: "sqlite3.Connection",
|
|
2969
3704
|
now_utc: "dt.datetime",
|
|
2970
3705
|
*,
|
|
2971
3706
|
n: int = 30,
|
|
2972
3707
|
skip_sync: bool = False,
|
|
3708
|
+
use_group_a_cache: bool = False,
|
|
2973
3709
|
display_tz: "ZoneInfo | None" = None) -> "list[DailyPanelRow]":
|
|
2974
3710
|
"""Latest n display-tz dates as DailyPanelRow, newest-first.
|
|
2975
3711
|
|
|
@@ -3004,11 +3740,37 @@ def _dashboard_build_daily_panel(conn: "sqlite3.Connection",
|
|
|
3004
3740
|
# forgiving of tz boundary issues.
|
|
3005
3741
|
range_start = now_utc - dt.timedelta(days=n + 1)
|
|
3006
3742
|
range_end = now_utc
|
|
3007
|
-
|
|
3743
|
+
|
|
3744
|
+
# #268 Group A cache: assemble the per-day BucketUsage list through the
|
|
3745
|
+
# module-level cache (recompute only the current + watermark-dirty days;
|
|
3746
|
+
# serve immutable past days from memory) and hand it to build_daily_view
|
|
3747
|
+
# as ``aggregated_override`` so every downstream derivation stays
|
|
3748
|
+
# single-sourced. Gated on ``use_group_a_cache`` — set ONLY by the
|
|
3749
|
+
# sync-thread rebuild (``_tui_build_snapshot``), which runs on a
|
|
3750
|
+
# process-consistent ``now``. NOT gated on ``skip_sync``: the
|
|
3751
|
+
# share-period-override path (``_share_apply_period_override``) also runs
|
|
3752
|
+
# ``skip_sync=True`` but on an HTTP handler thread with a shifted (PAST)
|
|
3753
|
+
# ``now_override`` — routing it through the shared module cache would clamp
|
|
3754
|
+
# the now_override-current bucket to a partial aggregate and cache it under
|
|
3755
|
+
# a real PAST-period label, truncating the live dashboard's past day (and
|
|
3756
|
+
# would mutate the module cache unlocked, off the sync thread). So every
|
|
3757
|
+
# non-sync-thread caller (default ``use_group_a_cache=False``) takes the
|
|
3758
|
+
# original from-scratch wide fetch. Also falls back when the cache is
|
|
3759
|
+
# disabled (parity tests) or the cache DB can't be opened.
|
|
3760
|
+
aggregated_override = (
|
|
3761
|
+
_group_a_daily_buckets(now_utc, n=n, display_tz=display_tz)
|
|
3762
|
+
if use_group_a_cache else None
|
|
3763
|
+
)
|
|
3008
3764
|
|
|
3009
3765
|
c = _cctally()
|
|
3010
|
-
|
|
3011
|
-
|
|
3766
|
+
if aggregated_override is None:
|
|
3767
|
+
entries = get_entries(range_start, range_end, skip_sync=skip_sync)
|
|
3768
|
+
view = c.build_daily_view(entries, now_utc=now_utc,
|
|
3769
|
+
display_tz=display_tz)
|
|
3770
|
+
else:
|
|
3771
|
+
view = c.build_daily_view((), now_utc=now_utc,
|
|
3772
|
+
display_tz=display_tz,
|
|
3773
|
+
aggregated_override=aggregated_override)
|
|
3012
3774
|
if not view.rows:
|
|
3013
3775
|
return []
|
|
3014
3776
|
|
|
@@ -3106,13 +3868,41 @@ def _projects_week_label(week_start: "dt.datetime") -> str:
|
|
|
3106
3868
|
def _projects_iter_session_entries(conn: "sqlite3.Connection",
|
|
3107
3869
|
*,
|
|
3108
3870
|
since: "dt.datetime",
|
|
3109
|
-
until: "dt.datetime"
|
|
3871
|
+
until: "dt.datetime",
|
|
3872
|
+
after_seq: "int | None" = None):
|
|
3110
3873
|
"""Read ``session_entries`` joined with ``session_files`` over
|
|
3111
3874
|
[since, until]. Yields rows directly off the passed conn — no
|
|
3112
3875
|
cache.db monkeypatch, no production ``get_claude_session_entries``
|
|
3113
3876
|
pipeline. The fixture DBs co-locate both schemas in one file; the
|
|
3114
3877
|
production wiring opens both DBs and ATTACHes cache.db as a schema
|
|
3115
3878
|
on the stats conn (see ``_run_dashboard_sync_tick``).
|
|
3879
|
+
|
|
3880
|
+
``after_seq`` (#270 §8, re-key of Codex-P2b — #271 §20): when set, the
|
|
3881
|
+
current-week accumulator's warm delta seeks by the **mutation_seq range**
|
|
3882
|
+
(``WHERE e.mutation_seq > ?``) with the ``[since, until]`` timestamp bounds
|
|
3883
|
+
applied as a residual filter. Two hints keep the planner on the
|
|
3884
|
+
``idx_entries_mutation_seq`` seek (``SEARCH e USING INDEX
|
|
3885
|
+
idx_entries_mutation_seq (mutation_seq>?)``) instead of a full ``SCAN`` /
|
|
3886
|
+
``idx_entries_timestamp`` window scan (which would re-read the whole ~12K-entry
|
|
3887
|
+
current week and let the ~63ms floor creep back): (1) the unary-plus no-op
|
|
3888
|
+
``+e.timestamp_utc`` deprioritizes ``idx_entries_timestamp`` (a TRUE no-op —
|
|
3889
|
+
it preserves the TEXT value AND the string comparison, unlike ``+ 0``); and
|
|
3890
|
+
(2) ``ORDER BY e.mutation_seq`` matches the index's leading column so the
|
|
3891
|
+
planner satisfies the order from the seek itself. ``INDEXED BY`` is NOT usable
|
|
3892
|
+
here — production runs this against a TEMP VIEW (``session_entries`` over the
|
|
3893
|
+
ATTACHed ``cache_db.session_entries``), and a view has no indexes; the hint
|
|
3894
|
+
pair drives the same seek on both the view and a base-table fixture. Unlike
|
|
3895
|
+
the old ``e.id > ?`` leg (a free INTEGER PRIMARY KEY rowid range), a bare
|
|
3896
|
+
``e.mutation_seq > ?`` lets the planner fall back to a scan on a small table,
|
|
3897
|
+
so the hints are load-bearing. Re-keying from ``e.id`` to ``e.mutation_seq``
|
|
3898
|
+
catches an id-stable in-place finalization (it advances the seq while
|
|
3899
|
+
``MAX(id)`` stays flat, so the pre-#270 ``e.id > ?`` leg missed it); on a
|
|
3900
|
+
pure-insert tick ``{mutation_seq > ?}`` == ``{id > ?}`` (seq monotone with
|
|
3901
|
+
id), so the delta row SET is byte-identical. The caller (``_fetch_delta_rows``)
|
|
3902
|
+
re-sorts the small result by ``(timestamp_utc, id)`` to reproduce the full
|
|
3903
|
+
pass's fold order, so the ``mutation_seq``-ASC SQL order is irrelevant
|
|
3904
|
+
downstream. An ``EXPLAIN QUERY PLAN`` regression asserts the mutation_seq
|
|
3905
|
+
index seek (``tests/test_projects_envelope.py``).
|
|
3116
3906
|
"""
|
|
3117
3907
|
since_iso = since.astimezone(dt.timezone.utc).strftime(
|
|
3118
3908
|
"%Y-%m-%dT%H:%M:%SZ"
|
|
@@ -3120,27 +3910,338 @@ def _projects_iter_session_entries(conn: "sqlite3.Connection",
|
|
|
3120
3910
|
until_iso = until.astimezone(dt.timezone.utc).strftime(
|
|
3121
3911
|
"%Y-%m-%dT%H:%M:%SZ"
|
|
3122
3912
|
)
|
|
3123
|
-
|
|
3124
|
-
|
|
3125
|
-
|
|
3126
|
-
|
|
3127
|
-
|
|
3128
|
-
|
|
3129
|
-
|
|
3130
|
-
|
|
3131
|
-
|
|
3132
|
-
|
|
3133
|
-
|
|
3913
|
+
if after_seq is not None:
|
|
3914
|
+
cur = conn.execute(
|
|
3915
|
+
"SELECT e.id, e.timestamp_utc, e.model, e.input_tokens, "
|
|
3916
|
+
" e.output_tokens, e.cache_create_tokens, e.cache_read_tokens, "
|
|
3917
|
+
" e.cost_usd_raw, e.source_path, "
|
|
3918
|
+
" sf.session_id, sf.project_path "
|
|
3919
|
+
"FROM session_entries e "
|
|
3920
|
+
"LEFT JOIN session_files sf ON sf.path = e.source_path "
|
|
3921
|
+
"WHERE e.mutation_seq > ? AND +e.timestamp_utc >= ? AND +e.timestamp_utc <= ? "
|
|
3922
|
+
"ORDER BY e.mutation_seq ASC",
|
|
3923
|
+
(after_seq, since_iso, until_iso),
|
|
3924
|
+
)
|
|
3925
|
+
else:
|
|
3926
|
+
cur = conn.execute(
|
|
3927
|
+
"SELECT e.id, e.timestamp_utc, e.model, e.input_tokens, "
|
|
3928
|
+
" e.output_tokens, e.cache_create_tokens, e.cache_read_tokens, "
|
|
3929
|
+
" e.cost_usd_raw, e.source_path, "
|
|
3930
|
+
" sf.session_id, sf.project_path "
|
|
3931
|
+
"FROM session_entries e "
|
|
3932
|
+
"LEFT JOIN session_files sf ON sf.path = e.source_path "
|
|
3933
|
+
"WHERE e.timestamp_utc >= ? AND e.timestamp_utc <= ? "
|
|
3934
|
+
"ORDER BY e.timestamp_utc ASC, e.id ASC",
|
|
3935
|
+
(since_iso, until_iso),
|
|
3936
|
+
)
|
|
3134
3937
|
for row in cur:
|
|
3135
3938
|
yield row
|
|
3136
3939
|
|
|
3137
3940
|
|
|
3941
|
+
class _ProjWeekBucket(NamedTuple):
|
|
3942
|
+
"""One (bucket_path, week) immutable aggregate for the projects-envelope
|
|
3943
|
+
per-week cache (#269 §14 Win 2).
|
|
3944
|
+
|
|
3945
|
+
All fields are entry-local — each entry belongs to exactly one bucket and
|
|
3946
|
+
one week — so a CLOSED week's values are stable and reproduce the
|
|
3947
|
+
full-window walk's contribution for that week byte-for-byte.
|
|
3948
|
+
|
|
3949
|
+
``first_seen`` / ``last_seen`` are the min / max PARSED event datetimes
|
|
3950
|
+
(emitted via ``_iso_z``). ``first_order`` / ``first_id`` / ``first_key`` are
|
|
3951
|
+
the SQL-FIRST entry for this bucket in this week (first-encountered under
|
|
3952
|
+
``ORDER BY timestamp_utc ASC, id ASC``): ``first_order`` is the RAW
|
|
3953
|
+
``timestamp_utc`` string, so the ``key_by_bucket`` reconstruction argmin
|
|
3954
|
+
reproduces the from-scratch global first-seen exactly (Codex-M4 P1 — the
|
|
3955
|
+
no-git ``display_key`` makes ``key_by_bucket[bp]`` non-deterministic per
|
|
3956
|
+
bucket_path; the envelope picks by global first-seen order).
|
|
3957
|
+
"""
|
|
3958
|
+
cost_usd: float
|
|
3959
|
+
sessions_count: int
|
|
3960
|
+
first_seen: "dt.datetime"
|
|
3961
|
+
last_seen: "dt.datetime"
|
|
3962
|
+
first_order: str
|
|
3963
|
+
first_id: int
|
|
3964
|
+
first_key: "Any"
|
|
3965
|
+
|
|
3966
|
+
|
|
3967
|
+
def _fold_projects_entry(
|
|
3968
|
+
mut: dict,
|
|
3969
|
+
row: tuple,
|
|
3970
|
+
*,
|
|
3971
|
+
resolver_cache: dict,
|
|
3972
|
+
week_start: "dt.datetime",
|
|
3973
|
+
) -> "float | None":
|
|
3974
|
+
"""Fold ONE ``_projects_iter_session_entries`` row onto ``mut`` (the shared
|
|
3975
|
+
per-row body, #271 §20 Codex-P1a).
|
|
3976
|
+
|
|
3977
|
+
Used by BOTH the full-window cold fold (``_aggregate_projects_week_raw``)
|
|
3978
|
+
and the current-week accumulator's warm incremental append, so their
|
|
3979
|
+
per-bucket cost sums, session sets, and first/last-seen captures are
|
|
3980
|
+
byte-identical BY CONSTRUCTION. Returns the entry cost, or ``None`` when the
|
|
3981
|
+
row is filtered out (``<synthetic>`` model, or its Monday-anchored week ≠
|
|
3982
|
+
``week_start``) — the caller then skips ``week_total`` / ``tail`` advance.
|
|
3983
|
+
|
|
3984
|
+
``mut[bp]`` is the running mutable dict ``{"cost_usd": float,
|
|
3985
|
+
"sessions": set, "first_seen": dt, "last_seen": dt, "first_order": ts_iso,
|
|
3986
|
+
"first_id": int, "first_key": ProjectKey}``. The first row seen for a
|
|
3987
|
+
``bucket_path`` captures ``first_order`` / ``first_id`` / ``first_key`` —
|
|
3988
|
+
order-safe for the warm append because every delta row sorts strictly after
|
|
3989
|
+
``tail`` (#271 §20), so a delta row is never the first-seen of a bucket that
|
|
3990
|
+
already exists in ``mut``.
|
|
3991
|
+
"""
|
|
3992
|
+
c = _cctally()
|
|
3993
|
+
(entry_id, ts_iso, model, input_tok, output_tok,
|
|
3994
|
+
cache_create, cache_read, cost_raw, source_path,
|
|
3995
|
+
session_id, project_path) = row
|
|
3996
|
+
if model == "<synthetic>":
|
|
3997
|
+
return None
|
|
3998
|
+
ts = parse_iso_datetime(ts_iso, "session_entries.timestamp_utc")
|
|
3999
|
+
if _projects_week_start_monday_utc(ts) != week_start:
|
|
4000
|
+
return None
|
|
4001
|
+
entry_cost = _calculate_entry_cost(
|
|
4002
|
+
model,
|
|
4003
|
+
{
|
|
4004
|
+
"input_tokens": input_tok or 0,
|
|
4005
|
+
"output_tokens": output_tok or 0,
|
|
4006
|
+
"cache_creation_input_tokens": cache_create or 0,
|
|
4007
|
+
"cache_read_input_tokens": cache_read or 0,
|
|
4008
|
+
},
|
|
4009
|
+
mode="auto",
|
|
4010
|
+
cost_usd=cost_raw,
|
|
4011
|
+
)
|
|
4012
|
+
pkey = c._resolve_project_key(project_path, "git-root", resolver_cache)
|
|
4013
|
+
bp = pkey.bucket_path
|
|
4014
|
+
a = mut.get(bp)
|
|
4015
|
+
if a is None:
|
|
4016
|
+
a = {
|
|
4017
|
+
"cost_usd": 0.0,
|
|
4018
|
+
"sessions": set(),
|
|
4019
|
+
"first_seen": ts,
|
|
4020
|
+
"last_seen": ts,
|
|
4021
|
+
"first_order": ts_iso,
|
|
4022
|
+
"first_id": entry_id,
|
|
4023
|
+
"first_key": pkey,
|
|
4024
|
+
}
|
|
4025
|
+
mut[bp] = a
|
|
4026
|
+
a["cost_usd"] += entry_cost
|
|
4027
|
+
if session_id:
|
|
4028
|
+
a["sessions"].add(session_id)
|
|
4029
|
+
elif source_path:
|
|
4030
|
+
a["sessions"].add(source_path)
|
|
4031
|
+
if ts < a["first_seen"]:
|
|
4032
|
+
a["first_seen"] = ts
|
|
4033
|
+
if ts > a["last_seen"]:
|
|
4034
|
+
a["last_seen"] = ts
|
|
4035
|
+
return entry_cost
|
|
4036
|
+
|
|
4037
|
+
|
|
4038
|
+
def _aggregate_projects_week_raw(
|
|
4039
|
+
conn: "sqlite3.Connection",
|
|
4040
|
+
*,
|
|
4041
|
+
week_start: "dt.datetime",
|
|
4042
|
+
week_end: "dt.datetime",
|
|
4043
|
+
resolver_cache: dict,
|
|
4044
|
+
) -> "tuple[dict, float, tuple | None]":
|
|
4045
|
+
"""Full-window fold for one Monday-anchored subscription week, returning the
|
|
4046
|
+
RAW mutable ``mut`` (sessions kept as SETS, so the current-week accumulator
|
|
4047
|
+
can dedup across ticks), the entry-order ``week_total`` float, and the
|
|
4048
|
+
``tail`` ``(ts_iso, id)`` of the last folded CURRENT-WEEK entry (``None``
|
|
4049
|
+
when no row folded).
|
|
4050
|
+
|
|
4051
|
+
Shared by the finalizing ``_aggregate_projects_week`` wrapper (closed-week
|
|
4052
|
+
cache path) and the current-week accumulator's cold seed (#271 §20
|
|
4053
|
+
Codex-P1a — the finalized public shape discards the sessions SET into a
|
|
4054
|
+
count, so cold seeding needs the raw ``mut``). Byte-identical to the
|
|
4055
|
+
original single-loop aggregate: same ``timestamp_utc ASC, id ASC`` order,
|
|
4056
|
+
same per-row ``_fold_projects_entry`` arithmetic, same entry-order
|
|
4057
|
+
``week_total`` left-fold.
|
|
4058
|
+
"""
|
|
4059
|
+
mut: "dict[str, dict]" = {}
|
|
4060
|
+
week_total = 0.0
|
|
4061
|
+
tail: "tuple | None" = None
|
|
4062
|
+
for row in _projects_iter_session_entries(
|
|
4063
|
+
conn, since=week_start, until=week_end,
|
|
4064
|
+
):
|
|
4065
|
+
entry_cost = _fold_projects_entry(
|
|
4066
|
+
mut, row, resolver_cache=resolver_cache, week_start=week_start,
|
|
4067
|
+
)
|
|
4068
|
+
if entry_cost is None:
|
|
4069
|
+
continue
|
|
4070
|
+
week_total += entry_cost
|
|
4071
|
+
tail = (row[1], row[0]) # (ts_iso, id)
|
|
4072
|
+
return mut, week_total, tail
|
|
4073
|
+
|
|
4074
|
+
|
|
4075
|
+
def _finalize_projects_mut(mut: dict) -> "dict[str, _ProjWeekBucket]":
|
|
4076
|
+
"""Finalize a raw ``mut`` (sessions SETS) into ``{bucket_path:
|
|
4077
|
+
_ProjWeekBucket}`` — ``sessions_count = len(sessions)``, the rest copied
|
|
4078
|
+
through. The public shape both the closed-week cache and the current-week
|
|
4079
|
+
accumulator return (#271 §20).
|
|
4080
|
+
"""
|
|
4081
|
+
return {
|
|
4082
|
+
bp: _ProjWeekBucket(
|
|
4083
|
+
cost_usd=a["cost_usd"],
|
|
4084
|
+
sessions_count=len(a["sessions"]),
|
|
4085
|
+
first_seen=a["first_seen"],
|
|
4086
|
+
last_seen=a["last_seen"],
|
|
4087
|
+
first_order=a["first_order"],
|
|
4088
|
+
first_id=a["first_id"],
|
|
4089
|
+
first_key=a["first_key"],
|
|
4090
|
+
)
|
|
4091
|
+
for bp, a in mut.items()
|
|
4092
|
+
}
|
|
4093
|
+
|
|
4094
|
+
|
|
4095
|
+
def _aggregate_projects_week(
|
|
4096
|
+
conn: "sqlite3.Connection",
|
|
4097
|
+
*,
|
|
4098
|
+
week_start: "dt.datetime",
|
|
4099
|
+
week_end: "dt.datetime",
|
|
4100
|
+
resolver_cache: dict,
|
|
4101
|
+
) -> "tuple[dict[str, _ProjWeekBucket], float]":
|
|
4102
|
+
"""Aggregate one Monday-anchored subscription week's entry slice.
|
|
4103
|
+
|
|
4104
|
+
Returns ``({bucket_path: _ProjWeekBucket}, week_total_cost)``. Byte-identical
|
|
4105
|
+
to the contribution the full-window walk in ``_build_projects_envelope``
|
|
4106
|
+
makes for this week: the per-week query returns entries in the same
|
|
4107
|
+
``timestamp_utc ASC, id ASC`` order, filtered to those whose
|
|
4108
|
+
Monday-anchored week equals ``week_start``, so the per-bucket cost sum, the
|
|
4109
|
+
entry-order ``week_total`` sum, the session set, and the first/last-seen +
|
|
4110
|
+
SQL-first-entry captures all reproduce the full pass. ``week_end`` is the
|
|
4111
|
+
inclusive query bound (``<= ?``); the exact next-Monday boundary entry is
|
|
4112
|
+
fetched but filtered out (it belongs to the next week) — the next week's
|
|
4113
|
+
slice keeps it, so no entry is lost or double-counted.
|
|
4114
|
+
|
|
4115
|
+
Thin wrapper over ``_aggregate_projects_week_raw`` + ``_finalize_projects_mut``
|
|
4116
|
+
(#271 §20 Codex-P1a); the public shape is unchanged for the closed-week
|
|
4117
|
+
cache path.
|
|
4118
|
+
"""
|
|
4119
|
+
mut, week_total, _tail = _aggregate_projects_week_raw(
|
|
4120
|
+
conn, week_start=week_start, week_end=week_end,
|
|
4121
|
+
resolver_cache=resolver_cache,
|
|
4122
|
+
)
|
|
4123
|
+
return _finalize_projects_mut(mut), week_total
|
|
4124
|
+
|
|
4125
|
+
|
|
4126
|
+
def _assemble_projects_via_cache(
|
|
4127
|
+
conn: "sqlite3.Connection",
|
|
4128
|
+
*,
|
|
4129
|
+
weeks_full: "list[dt.datetime]",
|
|
4130
|
+
cw_start: "dt.datetime",
|
|
4131
|
+
cw_end: "dt.datetime",
|
|
4132
|
+
cur_max_id: int,
|
|
4133
|
+
cur_max_seq: int,
|
|
4134
|
+
) -> "tuple[dict, dict, dict]":
|
|
4135
|
+
"""Flag-ON assembly (#269 §14 Win 2): recompute only the CURRENT week each
|
|
4136
|
+
warm tick; serve CLOSED weeks from the per-(project, week) cache on a hit and
|
|
4137
|
+
RECOMPUTE-AND-POPULATE on a miss (cold / Monday rollover / window slide,
|
|
4138
|
+
Codex-M4 P2). Returns ``(buckets, total_cost_by_week, key_by_bucket)`` in the
|
|
4139
|
+
exact shape the from-scratch walk produces, so the downstream disambiguation
|
|
4140
|
+
/ attribution / trend assembly runs unchanged and byte-identically.
|
|
4141
|
+
|
|
4142
|
+
#271 M4 (spec §20): the CURRENT week is no longer re-folded from scratch each
|
|
4143
|
+
warm tick — it goes through the single-slot ``accumulate_projects_current_week``
|
|
4144
|
+
accumulator, which folds only the changed-row delta (or finalizes the cached
|
|
4145
|
+
running ``mut`` unchanged on an empty-delta tick), byte-identically.
|
|
4146
|
+
``cur_max_id`` is the tick's ``MAX(session_entries.id)`` (the regression
|
|
4147
|
+
backstop + pre-existing-row cold-refold trigger); ``cur_max_seq`` is the
|
|
4148
|
+
tick's ``MAX(mutation_seq)`` (the #270 §8 delta watermark, so an id-stable
|
|
4149
|
+
in-place finalization is caught). The accumulator is single-writer here (this
|
|
4150
|
+
runs only on the sync-thread ``use_projects_env_cache=True`` path).
|
|
4151
|
+
"""
|
|
4152
|
+
c = _cctally()
|
|
4153
|
+
sc = c._load_sibling("_lib_snapshot_cache")
|
|
4154
|
+
resolver_cache: dict = {}
|
|
4155
|
+
buckets: dict = {}
|
|
4156
|
+
total_cost_by_week: dict = {}
|
|
4157
|
+
key_by_bucket: dict = {}
|
|
4158
|
+
# Reconstruct key_by_bucket by the GLOBAL first-seen SQL order
|
|
4159
|
+
# (timestamp_utc ASC, id ASC): per bucket_path, the argmin of
|
|
4160
|
+
# (first_order, first_id) across its assembled weeks (Codex-M4 P1).
|
|
4161
|
+
best_order: dict = {}
|
|
4162
|
+
|
|
4163
|
+
def _merge_week(w, week_buckets, week_total):
|
|
4164
|
+
# `total_cost_by_week` mirrors the from-scratch dict, which is only
|
|
4165
|
+
# populated for weeks that had entries (an empty week stays absent and
|
|
4166
|
+
# falls back to `.get(w, 0.0)` downstream).
|
|
4167
|
+
if week_buckets:
|
|
4168
|
+
total_cost_by_week[w] = week_total
|
|
4169
|
+
for bp, wb in week_buckets.items():
|
|
4170
|
+
buckets[(bp, w)] = {
|
|
4171
|
+
"cost_usd": wb.cost_usd,
|
|
4172
|
+
# `sessions` is only ever `len()`-d downstream; a `range` of the
|
|
4173
|
+
# cached count reproduces that without storing the id set.
|
|
4174
|
+
"sessions": range(wb.sessions_count),
|
|
4175
|
+
"first_seen": wb.first_seen,
|
|
4176
|
+
"last_seen": wb.last_seen,
|
|
4177
|
+
}
|
|
4178
|
+
cand = (wb.first_order, wb.first_id)
|
|
4179
|
+
if bp not in best_order or cand < best_order[bp]:
|
|
4180
|
+
best_order[bp] = cand
|
|
4181
|
+
key_by_bucket[bp] = wb.first_key
|
|
4182
|
+
|
|
4183
|
+
def _fetch_all_raw():
|
|
4184
|
+
return _aggregate_projects_week_raw(
|
|
4185
|
+
conn, week_start=cw_start, week_end=cw_end,
|
|
4186
|
+
resolver_cache=resolver_cache,
|
|
4187
|
+
)
|
|
4188
|
+
|
|
4189
|
+
def _fetch_delta_rows(after_seq):
|
|
4190
|
+
# Index-seek the changed-row delta (mutation_seq > after_seq, #270 §8)
|
|
4191
|
+
# over the current-week window, then PRE-FILTER to genuine current-week
|
|
4192
|
+
# non-synthetic rows (mirrors _fold_projects_entry's membership filter) so
|
|
4193
|
+
# the accumulator's fold-order gate compares a REAL current-week entry as
|
|
4194
|
+
# rows[0]. Sort by (ts_iso, id) to reproduce the full pass's fold order
|
|
4195
|
+
# exactly (SQLite BINARY collation on the Z-normalized ISO string ==
|
|
4196
|
+
# Python str compare).
|
|
4197
|
+
out = []
|
|
4198
|
+
for r in _projects_iter_session_entries(
|
|
4199
|
+
conn, since=cw_start, until=cw_end, after_seq=after_seq,
|
|
4200
|
+
):
|
|
4201
|
+
if r[2] == "<synthetic>": # r[2] = model
|
|
4202
|
+
continue
|
|
4203
|
+
ts = parse_iso_datetime(r[1], "session_entries.timestamp_utc")
|
|
4204
|
+
if _projects_week_start_monday_utc(ts) != cw_start:
|
|
4205
|
+
continue
|
|
4206
|
+
out.append(r)
|
|
4207
|
+
out.sort(key=lambda r: (r[1], r[0])) # (ts_iso, id)
|
|
4208
|
+
return out
|
|
4209
|
+
|
|
4210
|
+
for w in weeks_full:
|
|
4211
|
+
if w == cw_start:
|
|
4212
|
+
week_buckets, week_total = sc.accumulate_projects_current_week(
|
|
4213
|
+
week_key=sc.projects_env_week_key(cw_start),
|
|
4214
|
+
cur_max_id=cur_max_id,
|
|
4215
|
+
cur_max_seq=cur_max_seq,
|
|
4216
|
+
fetch_all_raw=_fetch_all_raw,
|
|
4217
|
+
fetch_delta_rows=_fetch_delta_rows,
|
|
4218
|
+
finalize=_finalize_projects_mut,
|
|
4219
|
+
fold=lambda mut, row: _fold_projects_entry(
|
|
4220
|
+
mut, row, resolver_cache=resolver_cache, week_start=cw_start,
|
|
4221
|
+
),
|
|
4222
|
+
)
|
|
4223
|
+
else:
|
|
4224
|
+
week_iso = sc.projects_env_week_key(w)
|
|
4225
|
+
hit = sc.projects_env_week_get(week_iso)
|
|
4226
|
+
if hit is not None:
|
|
4227
|
+
week_buckets, week_total = hit
|
|
4228
|
+
else:
|
|
4229
|
+
week_buckets, week_total = _aggregate_projects_week(
|
|
4230
|
+
conn, week_start=w, week_end=w + dt.timedelta(days=7),
|
|
4231
|
+
resolver_cache=resolver_cache,
|
|
4232
|
+
)
|
|
4233
|
+
sc.projects_env_week_put(week_iso, week_buckets, week_total)
|
|
4234
|
+
_merge_week(w, week_buckets, week_total)
|
|
4235
|
+
return buckets, total_cost_by_week, key_by_bucket
|
|
4236
|
+
|
|
4237
|
+
|
|
3138
4238
|
def _build_projects_envelope(
|
|
3139
4239
|
conn: "sqlite3.Connection",
|
|
3140
4240
|
*,
|
|
3141
4241
|
now_utc: "dt.datetime",
|
|
3142
4242
|
current_week: "Any | None" = None,
|
|
3143
4243
|
weeks_back: int = 12,
|
|
4244
|
+
use_projects_env_cache: bool = False,
|
|
3144
4245
|
) -> dict:
|
|
3145
4246
|
"""Build the ``projects.{current_week, trend}`` envelope block.
|
|
3146
4247
|
|
|
@@ -3161,7 +4262,9 @@ def _build_projects_envelope(
|
|
|
3161
4262
|
|
|
3162
4263
|
Determinism: same conn + same ``now_utc`` ⇒ byte-identical JSON
|
|
3163
4264
|
(R-PROJ5 invariant). Per-tick memoized on
|
|
3164
|
-
``(max(session_entries.id),
|
|
4265
|
+
``(max(session_entries.id), max_wus_id, max(mutation_seq), cw_week_start,
|
|
4266
|
+
weeks_back)`` — the ``mutation_seq`` leg (#270 §7d) busts the memo on an
|
|
4267
|
+
id-stable in-place finalization.
|
|
3165
4268
|
"""
|
|
3166
4269
|
c = _cctally()
|
|
3167
4270
|
|
|
@@ -3181,10 +4284,30 @@ def _build_projects_envelope(
|
|
|
3181
4284
|
max_wus_id = cur.fetchone()[0]
|
|
3182
4285
|
except sqlite3.OperationalError:
|
|
3183
4286
|
max_wus_id = 0
|
|
4287
|
+
# #270 (§7d, Codex-2b): this OUTER whole-envelope memo returns BEFORE the
|
|
4288
|
+
# inner per-(project, week) reconcile cache, so an id-stable in-place
|
|
4289
|
+
# finalization — which leaves MAX(id)/MAX(wus id) flat — would serve the
|
|
4290
|
+
# stale envelope regardless of the inner seq fix. Fold the mutation signal
|
|
4291
|
+
# into the memo key so it busts on any changed row. `MAX(mutation_seq)` is
|
|
4292
|
+
# read off the SAME `session_entries` surface as `max_id` (an id-stable
|
|
4293
|
+
# UPSERT advances it), index-backed by `idx_entries_mutation_seq`, and
|
|
4294
|
+
# degrades to 0 where the column is absent (old fixtures) — byte-identical.
|
|
4295
|
+
# Deliberately `MAX(mutation_seq)`, NOT the `cache_meta` counter the reconciles
|
|
4296
|
+
# read via `_entry_mutation_seq` (this conn is a project-detail view without a
|
|
4297
|
+
# `cache_meta` surface). The two are consistent — the counter is always
|
|
4298
|
+
# >= MAX(mutation_seq), and no row exists strictly between them at read time —
|
|
4299
|
+
# so do not "unify" them here (the divergence is intentional; see #270 §M3-1).
|
|
4300
|
+
try:
|
|
4301
|
+
cur = conn.execute(
|
|
4302
|
+
"SELECT COALESCE(MAX(mutation_seq), 0) FROM session_entries"
|
|
4303
|
+
)
|
|
4304
|
+
entry_mutation_seq = cur.fetchone()[0]
|
|
4305
|
+
except sqlite3.OperationalError:
|
|
4306
|
+
entry_mutation_seq = 0
|
|
3184
4307
|
cw_key: "dt.datetime | None" = None
|
|
3185
4308
|
if current_week is not None:
|
|
3186
4309
|
cw_key = getattr(current_week, "week_start_at", None)
|
|
3187
|
-
memo_key = (max_id, max_wus_id, cw_key, weeks_back)
|
|
4310
|
+
memo_key = (max_id, max_wus_id, entry_mutation_seq, cw_key, weeks_back)
|
|
3188
4311
|
cached = _PROJECTS_ENV_MEMO.get("value")
|
|
3189
4312
|
if cached is not None and _PROJECTS_ENV_MEMO.get("key") == memo_key:
|
|
3190
4313
|
return cached
|
|
@@ -3220,95 +4343,106 @@ def _build_projects_envelope(
|
|
|
3220
4343
|
until_dt = cw_end # exclusive end; SQL is `>= since AND <= until`
|
|
3221
4344
|
|
|
3222
4345
|
# ---- Bucket entries per (ProjectKey, week_start) --------------------
|
|
3223
|
-
#
|
|
3224
|
-
#
|
|
3225
|
-
#
|
|
3226
|
-
|
|
3227
|
-
|
|
3228
|
-
|
|
3229
|
-
#
|
|
3230
|
-
|
|
3231
|
-
#
|
|
3232
|
-
|
|
3233
|
-
|
|
3234
|
-
|
|
3235
|
-
|
|
3236
|
-
key_by_bucket: dict[str, Any] = {}
|
|
3237
|
-
|
|
3238
|
-
def _week_for(ts: dt.datetime) -> "dt.datetime | None":
|
|
3239
|
-
wstart = _projects_week_start_monday_utc(ts)
|
|
3240
|
-
if wstart < weeks_full[0] or wstart > weeks_full[-1]:
|
|
3241
|
-
return None
|
|
3242
|
-
return wstart
|
|
3243
|
-
|
|
3244
|
-
# Orphan handling: `_projects_iter_session_entries` LEFT JOINs
|
|
3245
|
-
# `session_files` so entries whose source_path has no
|
|
3246
|
-
# `session_files` row return `project_path = NULL`. Below,
|
|
3247
|
-
# `_resolve_project_key(None, ...)` maps that to the
|
|
3248
|
-
# `(unknown)` bucket — same identity the drill-down's explicit
|
|
3249
|
-
# orphan scan in `_project_detail_for_window` (see the
|
|
3250
|
-
# ``if unknown_bucket:`` branch around the
|
|
3251
|
-
# `orphan_cur` SELECT) collects via a NULL-side LEFT JOIN. The
|
|
3252
|
-
# two paths converge on the same `(unknown)` source_path set.
|
|
3253
|
-
for row in _projects_iter_session_entries(
|
|
3254
|
-
conn, since=since_dt, until=until_dt,
|
|
3255
|
-
):
|
|
3256
|
-
(entry_id, ts_iso, model, input_tok, output_tok,
|
|
3257
|
-
cache_create, cache_read, cost_raw, source_path,
|
|
3258
|
-
session_id, project_path) = row
|
|
3259
|
-
if model == "<synthetic>":
|
|
3260
|
-
continue
|
|
3261
|
-
# Parse timestamp; assume Z / +00:00 — production iterators do
|
|
3262
|
-
# the same via `parse_iso_datetime`.
|
|
3263
|
-
ts = parse_iso_datetime(ts_iso, "session_entries.timestamp_utc")
|
|
3264
|
-
wstart = _week_for(ts)
|
|
3265
|
-
if wstart is None:
|
|
3266
|
-
continue
|
|
3267
|
-
|
|
3268
|
-
# Entry cost via the shared pricing chokepoint.
|
|
3269
|
-
entry_cost = _calculate_entry_cost(
|
|
3270
|
-
model,
|
|
3271
|
-
{
|
|
3272
|
-
"input_tokens": input_tok or 0,
|
|
3273
|
-
"output_tokens": output_tok or 0,
|
|
3274
|
-
"cache_creation_input_tokens": cache_create or 0,
|
|
3275
|
-
"cache_read_input_tokens": cache_read or 0,
|
|
3276
|
-
},
|
|
3277
|
-
mode="auto",
|
|
3278
|
-
cost_usd=cost_raw,
|
|
4346
|
+
# The three structures the downstream disambiguation / attribution /
|
|
4347
|
+
# trend assembly consume:
|
|
4348
|
+
# buckets[(bucket_path, week_start)] -> {cost_usd, sessions, first/last_seen}
|
|
4349
|
+
# total_cost_by_week[week_start] -> attribution denominator
|
|
4350
|
+
# key_by_bucket[bucket_path] -> global first-seen ProjectKey
|
|
4351
|
+
# Flag ON (#269 §14 Win 2, sync-thread only): the CURRENT week is recomputed
|
|
4352
|
+
# fresh each warm tick; CLOSED weeks are served from the per-(project, week)
|
|
4353
|
+
# cache (hit) or recomputed-and-populated (miss). Flag OFF (CLI / tests /
|
|
4354
|
+
# HTTP-drill): the original single full-window walk, byte-unchanged.
|
|
4355
|
+
if use_projects_env_cache:
|
|
4356
|
+
buckets, total_cost_by_week, key_by_bucket = _assemble_projects_via_cache(
|
|
4357
|
+
conn, weeks_full=weeks_full, cw_start=cw_start, cw_end=cw_end,
|
|
4358
|
+
cur_max_id=max_id, cur_max_seq=entry_mutation_seq,
|
|
3279
4359
|
)
|
|
4360
|
+
else:
|
|
4361
|
+
# `_resolve_project_key` is the production resolver; we use git-root
|
|
4362
|
+
# mode (default for `cmd_project --group` absent) — matches the
|
|
4363
|
+
# CLI's default.
|
|
4364
|
+
_resolve_project_key = c._resolve_project_key
|
|
4365
|
+
resolver_cache: dict = {}
|
|
4366
|
+
|
|
4367
|
+
buckets = {}
|
|
4368
|
+
total_cost_by_week = {}
|
|
4369
|
+
# First-seen ProjectKey per bucket_path (feeds `_project_disambiguate_labels`).
|
|
4370
|
+
key_by_bucket = {}
|
|
4371
|
+
|
|
4372
|
+
def _week_for(ts: dt.datetime) -> "dt.datetime | None":
|
|
4373
|
+
wstart = _projects_week_start_monday_utc(ts)
|
|
4374
|
+
if wstart < weeks_full[0] or wstart > weeks_full[-1]:
|
|
4375
|
+
return None
|
|
4376
|
+
return wstart
|
|
4377
|
+
|
|
4378
|
+
# Orphan handling: `_projects_iter_session_entries` LEFT JOINs
|
|
4379
|
+
# `session_files` so entries whose source_path has no
|
|
4380
|
+
# `session_files` row return `project_path = NULL`. Below,
|
|
4381
|
+
# `_resolve_project_key(None, ...)` maps that to the
|
|
4382
|
+
# `(unknown)` bucket — same identity the drill-down's explicit
|
|
4383
|
+
# orphan scan in `_project_detail_for_window` (see the
|
|
4384
|
+
# ``if unknown_bucket:`` branch around the
|
|
4385
|
+
# `orphan_cur` SELECT) collects via a NULL-side LEFT JOIN. The
|
|
4386
|
+
# two paths converge on the same `(unknown)` source_path set.
|
|
4387
|
+
for row in _projects_iter_session_entries(
|
|
4388
|
+
conn, since=since_dt, until=until_dt,
|
|
4389
|
+
):
|
|
4390
|
+
(entry_id, ts_iso, model, input_tok, output_tok,
|
|
4391
|
+
cache_create, cache_read, cost_raw, source_path,
|
|
4392
|
+
session_id, project_path) = row
|
|
4393
|
+
if model == "<synthetic>":
|
|
4394
|
+
continue
|
|
4395
|
+
# Parse timestamp; assume Z / +00:00 — production iterators do
|
|
4396
|
+
# the same via `parse_iso_datetime`.
|
|
4397
|
+
ts = parse_iso_datetime(ts_iso, "session_entries.timestamp_utc")
|
|
4398
|
+
wstart = _week_for(ts)
|
|
4399
|
+
if wstart is None:
|
|
4400
|
+
continue
|
|
3280
4401
|
|
|
3281
|
-
|
|
3282
|
-
|
|
3283
|
-
|
|
3284
|
-
|
|
3285
|
-
|
|
3286
|
-
|
|
3287
|
-
|
|
3288
|
-
|
|
3289
|
-
|
|
3290
|
-
"
|
|
3291
|
-
|
|
3292
|
-
|
|
3293
|
-
|
|
3294
|
-
|
|
3295
|
-
|
|
3296
|
-
|
|
3297
|
-
|
|
3298
|
-
|
|
3299
|
-
|
|
3300
|
-
|
|
3301
|
-
|
|
3302
|
-
|
|
3303
|
-
|
|
3304
|
-
|
|
3305
|
-
|
|
3306
|
-
|
|
3307
|
-
|
|
3308
|
-
|
|
3309
|
-
|
|
3310
|
-
|
|
3311
|
-
|
|
4402
|
+
# Entry cost via the shared pricing chokepoint.
|
|
4403
|
+
entry_cost = _calculate_entry_cost(
|
|
4404
|
+
model,
|
|
4405
|
+
{
|
|
4406
|
+
"input_tokens": input_tok or 0,
|
|
4407
|
+
"output_tokens": output_tok or 0,
|
|
4408
|
+
"cache_creation_input_tokens": cache_create or 0,
|
|
4409
|
+
"cache_read_input_tokens": cache_read or 0,
|
|
4410
|
+
},
|
|
4411
|
+
mode="auto",
|
|
4412
|
+
cost_usd=cost_raw,
|
|
4413
|
+
)
|
|
4414
|
+
|
|
4415
|
+
# Project-key identity (`git_root` mode = production default).
|
|
4416
|
+
pkey = _resolve_project_key(project_path, "git-root", resolver_cache)
|
|
4417
|
+
bkey = (pkey.bucket_path, wstart)
|
|
4418
|
+
b = buckets.get(bkey)
|
|
4419
|
+
if b is None:
|
|
4420
|
+
b = {
|
|
4421
|
+
"key": pkey,
|
|
4422
|
+
"cost_usd": 0.0,
|
|
4423
|
+
"sessions": set(),
|
|
4424
|
+
"first_seen": ts,
|
|
4425
|
+
"last_seen": ts,
|
|
4426
|
+
}
|
|
4427
|
+
buckets[bkey] = b
|
|
4428
|
+
b["cost_usd"] += entry_cost
|
|
4429
|
+
if session_id:
|
|
4430
|
+
b["sessions"].add(session_id)
|
|
4431
|
+
elif source_path:
|
|
4432
|
+
# Fallback: treat one source_path as one session when
|
|
4433
|
+
# session_files.session_id is NULL (lazy population).
|
|
4434
|
+
b["sessions"].add(source_path)
|
|
4435
|
+
if ts < b["first_seen"]:
|
|
4436
|
+
b["first_seen"] = ts
|
|
4437
|
+
if ts > b["last_seen"]:
|
|
4438
|
+
b["last_seen"] = ts
|
|
4439
|
+
total_cost_by_week[wstart] = (
|
|
4440
|
+
total_cost_by_week.get(wstart, 0.0) + entry_cost
|
|
4441
|
+
)
|
|
4442
|
+
# Remember first-seen ProjectKey for each bucket_path so the
|
|
4443
|
+
# disambiguator pass below sees consistent ProjectKey instances.
|
|
4444
|
+
if pkey.bucket_path not in key_by_bucket:
|
|
4445
|
+
key_by_bucket[pkey.bucket_path] = pkey
|
|
3312
4446
|
|
|
3313
4447
|
# ---- Load weekly_usage_snapshots for attribution --------------------
|
|
3314
4448
|
# weekly_percent keyed by week_start (UTC datetime, Monday). We
|
|
@@ -4495,6 +5629,28 @@ def _build_alerts_envelope_array(
|
|
|
4495
5629
|
return out[:limit]
|
|
4496
5630
|
|
|
4497
5631
|
|
|
5632
|
+
def _resolve_dashboard_port(port_arg):
|
|
5633
|
+
"""Effective dashboard port: an explicit --port always wins; otherwise
|
|
5634
|
+
8790 under the preview channel (CCTALLY_CHANNEL=preview), else 8789.
|
|
5635
|
+
Byte-stable when the marker is unset because any explicit port (incl. 0)
|
|
5636
|
+
short-circuits before the env check — the golden harness passes --port 0,
|
|
5637
|
+
so the port default is never exercised there."""
|
|
5638
|
+
if port_arg is not None:
|
|
5639
|
+
return port_arg
|
|
5640
|
+
if _cctally_core.is_preview_channel():
|
|
5641
|
+
return 8790
|
|
5642
|
+
return 8789
|
|
5643
|
+
|
|
5644
|
+
|
|
5645
|
+
def _channel_env_fragment() -> dict:
|
|
5646
|
+
"""`{'channel': 'preview'}` under the preview channel, else `{}` — so the
|
|
5647
|
+
envelope key is omitted when the marker is unset (additive-optional, keeps
|
|
5648
|
+
the dashboard goldens byte-identical). Spliced into the envelope via `**`."""
|
|
5649
|
+
if _cctally_core.is_preview_channel():
|
|
5650
|
+
return {"channel": "preview"}
|
|
5651
|
+
return {}
|
|
5652
|
+
|
|
5653
|
+
|
|
4498
5654
|
def snapshot_to_envelope(snap: "DataSnapshot", *,
|
|
4499
5655
|
now_utc: "dt.datetime",
|
|
4500
5656
|
monotonic_now: "float | None" = None,
|
|
@@ -4590,8 +5746,16 @@ def snapshot_to_envelope(snap: "DataSnapshot", *,
|
|
|
4590
5746
|
# process. The override flows in as a canonical tz token; we layer it
|
|
4591
5747
|
# onto the config dict via _apply_display_tz_override before resolving
|
|
4592
5748
|
# so every reader downstream sees one zone.
|
|
5749
|
+
# #268 M4: read the config + update-state + doctor from the sync-thread
|
|
5750
|
+
# precompute (spec §6) so this stays a PURE renderer — no config.json read,
|
|
5751
|
+
# no `security` fork, no update-state file read per SSE client per tick.
|
|
5752
|
+
# When the fields are absent (fixtures / the initial empty snapshot /
|
|
5753
|
+
# positionally-constructed DataSnapshots) fall back to the inline reads so
|
|
5754
|
+
# behavior is byte-identical for those callers.
|
|
5755
|
+
_precomp = getattr(snap, "envelope_precompute", None)
|
|
5756
|
+
_raw_config = _precomp["config"] if _precomp is not None else load_config()
|
|
4593
5757
|
config = _apply_display_tz_override(
|
|
4594
|
-
|
|
5758
|
+
_raw_config, display_tz_pref_override
|
|
4595
5759
|
)
|
|
4596
5760
|
display_block = _compute_display_block(config, snap.generated_at)
|
|
4597
5761
|
resolved_tz_obj = ZoneInfo(display_block["resolved_tz"])
|
|
@@ -4866,7 +6030,11 @@ def snapshot_to_envelope(snap: "DataSnapshot", *,
|
|
|
4866
6030
|
# the entire snapshot — fall back to safe defaults and rely on
|
|
4867
6031
|
# `_warn_alerts_bad_config_once` for the user-visible signal.
|
|
4868
6032
|
alerts_array = list(getattr(snap, "alerts", []) or [])
|
|
4869
|
-
|
|
6033
|
+
# #268 M4: reuse the precompute's config (or the fallback load_config()
|
|
6034
|
+
# resolved above) — the envelope used to call load_config() a SECOND time
|
|
6035
|
+
# here. Within one tick both reads returned the same file, so this is
|
|
6036
|
+
# byte-identical.
|
|
6037
|
+
_cfg_for_alerts = _raw_config
|
|
4870
6038
|
try:
|
|
4871
6039
|
_alerts_cfg = _get_alerts_config(_cfg_for_alerts)
|
|
4872
6040
|
except _AlertsConfigError as exc:
|
|
@@ -4955,19 +6123,30 @@ def snapshot_to_envelope(snap: "DataSnapshot", *,
|
|
|
4955
6123
|
# produce a `null` block so a missing/corrupt state file doesn't
|
|
4956
6124
|
# 500 the entire envelope; the client falls back to the defensive
|
|
4957
6125
|
# null-state shape `coerceUpdateState` already understands.
|
|
4958
|
-
|
|
4959
|
-
|
|
4960
|
-
|
|
4961
|
-
|
|
4962
|
-
|
|
4963
|
-
|
|
4964
|
-
_update_state_envelope =
|
|
4965
|
-
|
|
4966
|
-
|
|
4967
|
-
|
|
4968
|
-
|
|
4969
|
-
|
|
4970
|
-
|
|
6126
|
+
#
|
|
6127
|
+
# #268 M4: read from the sync-thread precompute (spec §6) when present so
|
|
6128
|
+
# this stays pure — no update-state file reads per SSE client. The inline
|
|
6129
|
+
# try/except is the fallback for fixtures / the initial snapshot, and keeps
|
|
6130
|
+
# the SAME error sentinels so the derived block is byte-identical.
|
|
6131
|
+
if _precomp is not None:
|
|
6132
|
+
_update_state_envelope = _precomp["update_state"]
|
|
6133
|
+
_update_suppress_envelope = _precomp["update_suppress"]
|
|
6134
|
+
else:
|
|
6135
|
+
try:
|
|
6136
|
+
_update_state_envelope = _load_update_state()
|
|
6137
|
+
except sys.modules["cctally"].UpdateError:
|
|
6138
|
+
# _load_update_state() raises on truly malformed JSON. Surface
|
|
6139
|
+
# an _error sentinel so the client renders "no update info" the
|
|
6140
|
+
# same way it does for unreachable /api/update/status.
|
|
6141
|
+
_update_state_envelope = {"_error": "update-state.json invalid"}
|
|
6142
|
+
except Exception:
|
|
6143
|
+
_update_state_envelope = {"_error": "update-state.json read failed"}
|
|
6144
|
+
try:
|
|
6145
|
+
_update_suppress_envelope = _load_update_suppress()
|
|
6146
|
+
except Exception:
|
|
6147
|
+
_update_suppress_envelope = {
|
|
6148
|
+
"skipped_versions": [], "remind_after": None,
|
|
6149
|
+
}
|
|
4971
6150
|
update_envelope = {
|
|
4972
6151
|
"state": _update_state_envelope,
|
|
4973
6152
|
"suppress": _update_suppress_envelope,
|
|
@@ -4980,24 +6159,37 @@ def snapshot_to_envelope(snap: "DataSnapshot", *,
|
|
|
4980
6159
|
# `safety.dashboard_bind` reflects the running state, not just
|
|
4981
6160
|
# `config.json`. Defensive: never crash the snapshot pipeline on
|
|
4982
6161
|
# a doctor failure — surface a synthetic FAIL block with `_error`.
|
|
4983
|
-
|
|
4984
|
-
|
|
4985
|
-
|
|
4986
|
-
|
|
4987
|
-
|
|
4988
|
-
|
|
4989
|
-
|
|
4990
|
-
|
|
4991
|
-
|
|
4992
|
-
|
|
4993
|
-
|
|
4994
|
-
doctor_envelope =
|
|
4995
|
-
|
|
4996
|
-
|
|
4997
|
-
"
|
|
4998
|
-
|
|
4999
|
-
|
|
5000
|
-
|
|
6162
|
+
#
|
|
6163
|
+
# #268 M4: read the precomputed doctor block from the snapshot (spec §6)
|
|
6164
|
+
# so this render path never forks the `security` keychain subprocess. The
|
|
6165
|
+
# sync thread computes it ONCE per rebuild via `doctor_payload_memo`
|
|
6166
|
+
# (`_tui_precompute_doctor_payload`). The inline try/except below is the
|
|
6167
|
+
# fallback for fixtures / the initial snapshot / positionally-constructed
|
|
6168
|
+
# DataSnapshots (`doctor_payload=None`) and produces the SAME block, so
|
|
6169
|
+
# moving the computation is byte-identical. The lazy `GET /api/doctor`
|
|
6170
|
+
# endpoint keeps computing LIVE (an explicit user refresh must be fresh).
|
|
6171
|
+
_doctor_payload = getattr(snap, "doctor_payload", None)
|
|
6172
|
+
if _doctor_payload is not None:
|
|
6173
|
+
doctor_envelope: "dict" = _doctor_payload
|
|
6174
|
+
else:
|
|
6175
|
+
try:
|
|
6176
|
+
_ld = sys.modules["cctally"]._load_sibling("_lib_doctor")
|
|
6177
|
+
_doc_state = doctor_gather_state(now_utc=now_utc, runtime_bind=runtime_bind)
|
|
6178
|
+
_doc_report = _ld.run_checks(_doc_state)
|
|
6179
|
+
doctor_envelope = {
|
|
6180
|
+
"severity": _doc_report.overall_severity,
|
|
6181
|
+
"counts": dict(_doc_report.counts),
|
|
6182
|
+
"generated_at": _ld._iso_z(_doc_report.generated_at),
|
|
6183
|
+
"fingerprint": _ld.fingerprint(_doc_report),
|
|
6184
|
+
}
|
|
6185
|
+
except Exception as exc: # noqa: BLE001 — never crash SSE on doctor failure
|
|
6186
|
+
doctor_envelope = {
|
|
6187
|
+
"severity": "fail",
|
|
6188
|
+
"counts": {"ok": 0, "warn": 0, "fail": 1},
|
|
6189
|
+
"generated_at": _iso_z(now_utc),
|
|
6190
|
+
"fingerprint": "sha1:" + ("0" * 40),
|
|
6191
|
+
"_error": f"{type(exc).__name__}: {exc}",
|
|
6192
|
+
}
|
|
5001
6193
|
|
|
5002
6194
|
# B1 (#207): the "vs last week" header delta reuses the is_current trend
|
|
5003
6195
|
# row's delta_dpp ($/1% vs the previous trend row — normally the prior
|
|
@@ -5235,6 +6427,12 @@ def snapshot_to_envelope(snap: "DataSnapshot", *,
|
|
|
5235
6427
|
# Doctor aggregate-only block (spec §5.5). Full per-check
|
|
5236
6428
|
# report fetched lazily via GET /api/doctor.
|
|
5237
6429
|
"doctor": doctor_envelope,
|
|
6430
|
+
|
|
6431
|
+
# Preview channel (CCTALLY_CHANNEL=preview): additive-optional
|
|
6432
|
+
# `channel` key. Omitted when the marker is unset so prod payloads
|
|
6433
|
+
# (and the dashboard goldens) stay byte-identical; feeds the header
|
|
6434
|
+
# PREVIEW badge when set.
|
|
6435
|
+
**_channel_env_fragment(),
|
|
5238
6436
|
}
|
|
5239
6437
|
|
|
5240
6438
|
|
|
@@ -8637,6 +9835,13 @@ def _dashboard_initial_snapshot(args, *, pinned_now, display_tz_pref_override):
|
|
|
8637
9835
|
return sys.modules["cctally"]._tui_build_snapshot(
|
|
8638
9836
|
now_utc=pinned_now, skip_sync=True,
|
|
8639
9837
|
display_tz_pref_override=display_tz_pref_override,
|
|
9838
|
+
# #268 M4: precompute doctor / config / update-state on the initial
|
|
9839
|
+
# snapshot too, so the envelope is pure from the very first paint AND
|
|
9840
|
+
# under --no-sync (where no later sync tick would set them). One
|
|
9841
|
+
# `security` fork here is negligible next to the heavy sync #179 moved
|
|
9842
|
+
# to the background thread. ``getattr`` so minimal test ``args``
|
|
9843
|
+
# namespaces (no ``host``) still build an initial snapshot.
|
|
9844
|
+
precompute_envelope=True, runtime_bind=getattr(args, "host", None),
|
|
8640
9845
|
)
|
|
8641
9846
|
|
|
8642
9847
|
|
|
@@ -8647,6 +9852,11 @@ def cmd_dashboard(args: argparse.Namespace) -> int:
|
|
|
8647
9852
|
import time as _time
|
|
8648
9853
|
import webbrowser as _wb
|
|
8649
9854
|
|
|
9855
|
+
# #268 M6.1: arm the SIGUSR1 all-thread traceback dump so any future spin is
|
|
9856
|
+
# self-diagnosing (`kill -USR1 <pid>`) without root py-spy. No-op where
|
|
9857
|
+
# SIGUSR1 is unavailable (Windows).
|
|
9858
|
+
_register_faulthandler_sigusr1()
|
|
9859
|
+
|
|
8650
9860
|
# Spec §5.7: capture the un-mutated argv + PATH-resolved entrypoint
|
|
8651
9861
|
# at boot so the in-place ``execvp`` after a successful update
|
|
8652
9862
|
# re-enters the user-facing wrapper (npm Node shim → CCTALLY_PYTHON
|
|
@@ -8723,6 +9933,15 @@ def cmd_dashboard(args: argparse.Namespace) -> int:
|
|
|
8723
9933
|
# _DashboardSyncThread (started below, before the bind) owns the first full
|
|
8724
9934
|
# sync_cache — including any pending conversation-enrichment reingest — and
|
|
8725
9935
|
# SSE-pushes the populated snapshot on completion.
|
|
9936
|
+
# Self-heal removed-worktree orphans BEFORE building the first snapshot so
|
|
9937
|
+
# the initial render already excludes stale sessions from a deleted
|
|
9938
|
+
# worktree (rather than showing them until the first periodic tick).
|
|
9939
|
+
# Gated off under --no-sync (a frozen dashboard mutates nothing).
|
|
9940
|
+
_heal = _dashboard_self_heal_orphans(skip_sync=bool(args.no_sync))
|
|
9941
|
+
if _heal is not None and _heal.pruned_files:
|
|
9942
|
+
print(f"dashboard: pruned {_heal.pruned_files} orphaned cache file(s) "
|
|
9943
|
+
f"from removed sessions on startup", flush=True)
|
|
9944
|
+
|
|
8726
9945
|
initial = _dashboard_initial_snapshot(
|
|
8727
9946
|
args, pinned_now=pinned_now,
|
|
8728
9947
|
display_tz_pref_override=display_tz_pref_override,
|
|
@@ -8755,6 +9974,10 @@ def cmd_dashboard(args: argparse.Namespace) -> int:
|
|
|
8755
9974
|
_run_sync_now_locked = _make_run_sync_now_locked(
|
|
8756
9975
|
ref=ref, hub=hub, pinned_now=pinned_now,
|
|
8757
9976
|
display_tz_pref_override=display_tz_pref_override,
|
|
9977
|
+
# #268 M4: the dashboard's bound host, threaded into the sync-thread
|
|
9978
|
+
# doctor precompute so `safety.dashboard_bind` reflects the running
|
|
9979
|
+
# bind and the envelope reads the precomputed doctor block.
|
|
9980
|
+
runtime_bind=args.host,
|
|
8758
9981
|
)
|
|
8759
9982
|
_run_sync_now = _make_run_sync_now(
|
|
8760
9983
|
sync_lock=sync_lock, ref=ref, hub=hub, pinned_now=pinned_now,
|
|
@@ -8797,8 +10020,17 @@ def cmd_dashboard(args: argparse.Namespace) -> int:
|
|
|
8797
10020
|
|
|
8798
10021
|
class _DashboardSyncThread(_c_for_subclass._TuiSyncThread):
|
|
8799
10022
|
def _run(self) -> None:
|
|
10023
|
+
last_heal = _time.monotonic()
|
|
8800
10024
|
while not self._stop.is_set():
|
|
8801
10025
|
_run_sync_now(skip_sync=self._skip_sync)
|
|
10026
|
+
# Self-heal removed-worktree orphans on a ~60s cadence (far
|
|
10027
|
+
# rarer than the sync tick — a deleted worktree is not urgent).
|
|
10028
|
+
# Non-blocking on the flock, so a contended tick just retries
|
|
10029
|
+
# next cadence; gated off under --no-sync.
|
|
10030
|
+
if (not self._skip_sync
|
|
10031
|
+
and _time.monotonic() - last_heal >= 60.0):
|
|
10032
|
+
last_heal = _time.monotonic()
|
|
10033
|
+
_dashboard_self_heal_orphans(skip_sync=self._skip_sync)
|
|
8802
10034
|
for _ in range(int(max(1, self._interval * 10))):
|
|
8803
10035
|
if self._stop.is_set():
|
|
8804
10036
|
return
|
|
@@ -8828,7 +10060,8 @@ def cmd_dashboard(args: argparse.Namespace) -> int:
|
|
|
8828
10060
|
# HTTP server on its own thread so the main thread can block on signal.
|
|
8829
10061
|
# `_QuietThreadingHTTPServer` folds in `daemon_threads = True` and silences
|
|
8830
10062
|
# client-disconnect tracebacks (spec §5).
|
|
8831
|
-
|
|
10063
|
+
resolved_port = _resolve_dashboard_port(args.port)
|
|
10064
|
+
srv = _QuietThreadingHTTPServer((args.host, resolved_port), DashboardHTTPHandler)
|
|
8832
10065
|
|
|
8833
10066
|
bind_host = args.host
|
|
8834
10067
|
bind_port = srv.server_address[1]
|